[turbofan] Reduce overhead of megamorphic property accesses.

We had an optimization in Crankshaft where we would call into the
megamorphic handler stub directly if an inline cache was already
found to be megamorphic when it hit the optimizing compiler. This
way we could avoid the dispatch overhead when we know that there's
no point in checking for the other states anyways. However we somehow
missed to port this optimization to TurboFan.

Now this change introduces support to call into LoadIC_Megamorphic and
KeyedLoadIC_Megamorphic directly (plus the trampoline versions), which
saves quite a lot of overhead for the cases where the map/name pair is
found in the megamorphic stub cache, and it's quite a simple change. We
can later extend this to also handle the StoreIC and KeyedStoreIC cases
if that turns out to be beneficial.

This improves the score on the Octane/TypeScript test by around ~2%
and the TypeScript test in the web-tooling-benchmark by around ~4%. On
the ARES-6 Air test the steady state mean improves by 2-4%, and on the
ARES-6 ML test the steady state mean seems to also improve by 1-2%, but
that might be within noise.

On a micro-benchmark that just runs `o.x` in a hot loop on a set of 9
different objects, which all have `x` as the first property and are
all in fast mode, we improve by around ~30%, and are now almost on par
with JavaScriptCore.

Bug: v8:6344, v8:6936
Change-Id: Iaa4c6e34c37e78da217ee75f32f6acc95a834250
Reviewed-on: https://chromium-review.googlesource.com/1215623Reviewed-by: Jaroslav Sevcik <jarin@chromium.org>
Reviewed-by: Jakob Kummerow <jkummerow@chromium.org>
Commit-Queue: Benedikt Meurer <bmeurer@chromium.org>
Cr-Commit-Position: refs/heads/master@{#55803}

src/builtins/builtins-definitions.h

@@ -206,7 +206,6 @@ namespace internal {
   TFC(ToBooleanLazyDeoptContinuation, TypeConversionStackParameter, 1)         \
                                                                                \
   /* Handlers */                                                               \
-  TFH(KeyedLoadIC_Megamorphic, LoadWithVector)                                 \
   TFH(KeyedLoadIC_PolymorphicName, LoadWithVector)                             \
   TFH(KeyedLoadIC_Slow, LoadWithVector)                                        \
   TFH(KeyedStoreIC_Megamorphic, StoreWithVector)                               \
@@ -629,10 +628,14 @@ namespace internal {
                                                                                \
   /* ICs */                                                                    \
   TFH(LoadIC, LoadWithVector)                                                  \
+  TFH(LoadIC_Megamorphic, LoadWithVector)                                      \
   TFH(LoadIC_Noninlined, LoadWithVector)                                       \
   TFH(LoadICTrampoline, Load)                                                  \
+  TFH(LoadICTrampoline_Megamorphic, Load)                                      \
   TFH(KeyedLoadIC, LoadWithVector)                                             \
+  TFH(KeyedLoadIC_Megamorphic, LoadWithVector)                                 \
   TFH(KeyedLoadICTrampoline, Load)                                             \
+  TFH(KeyedLoadICTrampoline_Megamorphic, Load)                                 \
   TFH(StoreGlobalIC, StoreGlobalWithVector)                                    \
   TFH(StoreGlobalICTrampoline, StoreGlobal)                                    \
   TFH(StoreIC, StoreWithVector)                                                \

src/builtins/builtins-ic-gen.cc

@@ -21,13 +21,16 @@ namespace internal {
   }
 
 IC_BUILTIN(LoadIC)
+IC_BUILTIN(LoadIC_Megamorphic)
 IC_BUILTIN(LoadIC_Noninlined)
 IC_BUILTIN(LoadIC_Uninitialized)
-IC_BUILTIN(KeyedLoadIC)
 IC_BUILTIN(LoadICTrampoline)
-IC_BUILTIN(KeyedLoadICTrampoline)
+IC_BUILTIN(LoadICTrampoline_Megamorphic)
+IC_BUILTIN(KeyedLoadIC)
 IC_BUILTIN(KeyedLoadIC_Megamorphic)
 IC_BUILTIN(KeyedLoadIC_PolymorphicName)
+IC_BUILTIN(KeyedLoadICTrampoline)
+IC_BUILTIN(KeyedLoadICTrampoline_Megamorphic)
 IC_BUILTIN(StoreGlobalIC)
 IC_BUILTIN(StoreGlobalICTrampoline)
 IC_BUILTIN(StoreIC)

src/compiler/bytecode-graph-builder.cc

@@ -571,7 +571,9 @@ Node* BytecodeGraphBuilder::BuildLoadNativeContextField(int index) {
 }
 
 VectorSlotPair BytecodeGraphBuilder::CreateVectorSlotPair(int slot_id) {
-  return VectorSlotPair(feedback_vector(), FeedbackVector::ToSlot(slot_id));
+  FeedbackSlot slot = FeedbackVector::ToSlot(slot_id);
+  FeedbackNexus nexus(feedback_vector(), slot);
+  return VectorSlotPair(feedback_vector(), slot, nexus.ic_state());
 }
 
 void BytecodeGraphBuilder::CreateGraph() {

src/compiler/js-generic-lowering.cc

@@ -148,12 +148,16 @@ void JSGenericLowering::LowerJSLoadProperty(Node* node) {
   Node* outer_state = frame_state->InputAt(kFrameStateOuterStateInput);
   node->InsertInput(zone(), 2, jsgraph()->SmiConstant(p.feedback().index()));
   if (outer_state->opcode() != IrOpcode::kFrameState) {
-    Callable callable =
-        Builtins::CallableFor(isolate(), Builtins::kKeyedLoadICTrampoline);
+    Callable callable = Builtins::CallableFor(
+        isolate(), p.feedback().ic_state() == MEGAMORPHIC
+                       ? Builtins::kKeyedLoadICTrampoline_Megamorphic
+                       : Builtins::kKeyedLoadICTrampoline);
     ReplaceWithStubCall(node, callable, flags);
   } else {
-    Callable callable =
-        Builtins::CallableFor(isolate(), Builtins::kKeyedLoadIC);
+    Callable callable = Builtins::CallableFor(
+        isolate(), p.feedback().ic_state() == MEGAMORPHIC
+                       ? Builtins::kKeyedLoadIC_Megamorphic
+                       : Builtins::kKeyedLoadIC);
     Node* vector = jsgraph()->HeapConstant(p.feedback().vector());
     node->InsertInput(zone(), 3, vector);
     ReplaceWithStubCall(node, callable, flags);
@@ -168,18 +172,22 @@ void JSGenericLowering::LowerJSLoadNamed(Node* node) {
   node->InsertInput(zone(), 1, jsgraph()->HeapConstant(p.name()));
   node->InsertInput(zone(), 2, jsgraph()->SmiConstant(p.feedback().index()));
   if (outer_state->opcode() != IrOpcode::kFrameState) {
-    Callable callable =
-        Builtins::CallableFor(isolate(), Builtins::kLoadICTrampoline);
+    Callable callable = Builtins::CallableFor(
+        isolate(), p.feedback().ic_state() == MEGAMORPHIC
+                       ? Builtins::kLoadICTrampoline_Megamorphic
+                       : Builtins::kLoadICTrampoline);
     ReplaceWithStubCall(node, callable, flags);
   } else {
-    Callable callable = Builtins::CallableFor(isolate(), Builtins::kLoadIC);
+    Callable callable =
+        Builtins::CallableFor(isolate(), p.feedback().ic_state() == MEGAMORPHIC
+                                             ? Builtins::kLoadIC_Megamorphic
+                                             : Builtins::kLoadIC);
     Node* vector = jsgraph()->HeapConstant(p.feedback().vector());
     node->InsertInput(zone(), 3, vector);
     ReplaceWithStubCall(node, callable, flags);
   }
 }
 
-
 void JSGenericLowering::LowerJSLoadGlobal(Node* node) {
   CallDescriptor::Flags flags = FrameStateFlagForCall(node);
   const LoadGlobalParameters& p = LoadGlobalParametersOf(node->op());

src/compiler/js-operator.cc

@@ -266,7 +266,7 @@ NamedAccess const& NamedAccessOf(const Operator* op) {
 
 
 std::ostream& operator<<(std::ostream& os, PropertyAccess const& p) {
-  return os << p.language_mode();
+  return os << p.language_mode() << ", " << p.feedback();
 }
 
 

src/feedback-vector.cc

@@ -546,10 +546,11 @@ InlineCacheState FeedbackNexus::StateFromFeedback() const {
 
   switch (kind()) {
     case FeedbackSlotKind::kCreateClosure:
+      return MONOMORPHIC;
+
     case FeedbackSlotKind::kLiteral:
-      // CreateClosure and literal slots don't have a notion of state.
-      UNREACHABLE();
-      break;
+      if (feedback->IsSmi()) return UNINITIALIZED;
+      return MONOMORPHIC;
 
     case FeedbackSlotKind::kStoreGlobalSloppy:
     case FeedbackSlotKind::kStoreGlobalStrict:

src/globals.h

@@ -709,6 +709,27 @@ enum InlineCacheState {
   GENERIC,
 };
 
+// Printing support.
+inline const char* InlineCacheState2String(InlineCacheState state) {
+  switch (state) {
+    case UNINITIALIZED:
+      return "UNINITIALIZED";
+    case PREMONOMORPHIC:
+      return "PREMONOMORPHIC";
+    case MONOMORPHIC:
+      return "MONOMORPHIC";
+    case RECOMPUTE_HANDLER:
+      return "RECOMPUTE_HANDLER";
+    case POLYMORPHIC:
+      return "POLYMORPHIC";
+    case MEGAMORPHIC:
+      return "MEGAMORPHIC";
+    case GENERIC:
+      return "GENERIC";
+  }
+  UNREACHABLE();
+}
+
 enum WhereToStart { kStartAtReceiver, kStartAtPrototype };
 
 enum ResultSentinel { kNotFound = -1, kUnsupported = -2 };

src/ic/accessor-assembler.cc

@@ -3197,6 +3197,31 @@ void AccessorAssembler::GenerateLoadIC() {
   LoadIC(&p);
 }
 
+void AccessorAssembler::GenerateLoadIC_Megamorphic() {
+  typedef LoadWithVectorDescriptor Descriptor;
+
+  Node* receiver = Parameter(Descriptor::kReceiver);
+  Node* name = Parameter(Descriptor::kName);
+  Node* slot = Parameter(Descriptor::kSlot);
+  Node* vector = Parameter(Descriptor::kVector);
+  Node* context = Parameter(Descriptor::kContext);
+
+  ExitPoint direct_exit(this);
+  TVARIABLE(MaybeObject, var_handler);
+  Label if_handler(this, &var_handler), miss(this, Label::kDeferred);
+
+  TryProbeStubCache(isolate()->load_stub_cache(), receiver, name, &if_handler,
+                    &var_handler, &miss);
+
+  BIND(&if_handler);
+  LoadICParameters p(context, receiver, name, slot, vector);
+  HandleLoadICHandlerCase(&p, CAST(var_handler.value()), &miss, &direct_exit);
+
+  BIND(&miss);
+  direct_exit.ReturnCallRuntime(Runtime::kLoadIC_Miss, context, receiver, name,
+                                slot, vector);
+}
+
 void AccessorAssembler::GenerateLoadIC_Noninlined() {
   typedef LoadWithVectorDescriptor Descriptor;
 
@@ -3252,6 +3277,19 @@ void AccessorAssembler::GenerateLoadICTrampoline() {
   TailCallBuiltin(Builtins::kLoadIC, context, receiver, name, slot, vector);
 }
 
+void AccessorAssembler::GenerateLoadICTrampoline_Megamorphic() {
+  typedef LoadDescriptor Descriptor;
+
+  Node* receiver = Parameter(Descriptor::kReceiver);
+  Node* name = Parameter(Descriptor::kName);
+  Node* slot = Parameter(Descriptor::kSlot);
+  Node* context = Parameter(Descriptor::kContext);
+  Node* vector = LoadFeedbackVectorForStub();
+
+  TailCallBuiltin(Builtins::kLoadIC_Megamorphic, context, receiver, name, slot,
+                  vector);
+}
+
 void AccessorAssembler::GenerateLoadGlobalIC(TypeofMode typeof_mode) {
   typedef LoadGlobalWithVectorDescriptor Descriptor;
 
@@ -3294,6 +3332,19 @@ void AccessorAssembler::GenerateKeyedLoadIC() {
   KeyedLoadIC(&p);
 }
 
+void AccessorAssembler::GenerateKeyedLoadIC_Megamorphic() {
+  typedef LoadWithVectorDescriptor Descriptor;
+
+  Node* receiver = Parameter(Descriptor::kReceiver);
+  Node* name = Parameter(Descriptor::kName);
+  Node* slot = Parameter(Descriptor::kSlot);
+  Node* vector = Parameter(Descriptor::kVector);
+  Node* context = Parameter(Descriptor::kContext);
+
+  LoadICParameters p(context, receiver, name, slot, vector);
+  KeyedLoadICGeneric(&p);
+}
+
 void AccessorAssembler::GenerateKeyedLoadICTrampoline() {
   typedef LoadDescriptor Descriptor;
 
@@ -3307,17 +3358,17 @@ void AccessorAssembler::GenerateKeyedLoadICTrampoline() {
                   vector);
 }
 
-void AccessorAssembler::GenerateKeyedLoadIC_Megamorphic() {
-  typedef LoadWithVectorDescriptor Descriptor;
+void AccessorAssembler::GenerateKeyedLoadICTrampoline_Megamorphic() {
+  typedef LoadDescriptor Descriptor;
 
   Node* receiver = Parameter(Descriptor::kReceiver);
   Node* name = Parameter(Descriptor::kName);
   Node* slot = Parameter(Descriptor::kSlot);
-  Node* vector = Parameter(Descriptor::kVector);
   Node* context = Parameter(Descriptor::kContext);
+  Node* vector = LoadFeedbackVectorForStub();
 
-  LoadICParameters p(context, receiver, name, slot, vector);
-  KeyedLoadICGeneric(&p);
+  TailCallBuiltin(Builtins::kKeyedLoadIC_Megamorphic, context, receiver, name,
+                  slot, vector);
 }
 
 void AccessorAssembler::GenerateKeyedLoadIC_PolymorphicName() {

src/ic/accessor-assembler.h

@@ -28,13 +28,16 @@ class AccessorAssembler : public CodeStubAssembler {
       : CodeStubAssembler(state) {}
 
   void GenerateLoadIC();
+  void GenerateLoadIC_Megamorphic();
   void GenerateLoadIC_Noninlined();
   void GenerateLoadIC_Uninitialized();
   void GenerateLoadICTrampoline();
+  void GenerateLoadICTrampoline_Megamorphic();
   void GenerateKeyedLoadIC();
-  void GenerateKeyedLoadICTrampoline();
   void GenerateKeyedLoadIC_Megamorphic();
   void GenerateKeyedLoadIC_PolymorphicName();
+  void GenerateKeyedLoadICTrampoline();
+  void GenerateKeyedLoadICTrampoline_Megamorphic();
   void GenerateStoreIC();
   void GenerateStoreICTrampoline();
   void GenerateStoreGlobalIC();

src/objects-printer.cc

@@ -1088,29 +1088,6 @@ void FeedbackVector::FeedbackSlotPrint(std::ostream& os,
   nexus.Print(os);
 }
 
-namespace {
-
-const char* ICState2String(InlineCacheState state) {
-  switch (state) {
-    case UNINITIALIZED:
-      return "UNINITIALIZED";
-    case PREMONOMORPHIC:
-      return "PREMONOMORPHIC";
-    case MONOMORPHIC:
-      return "MONOMORPHIC";
-    case RECOMPUTE_HANDLER:
-      return "RECOMPUTE_HANDLER";
-    case POLYMORPHIC:
-      return "POLYMORPHIC";
-    case MEGAMORPHIC:
-      return "MEGAMORPHIC";
-    case GENERIC:
-      return "GENERIC";
-  }
-  UNREACHABLE();
-}
-}  // anonymous namespace
-
 void FeedbackNexus::Print(std::ostream& os) {  // NOLINT
   switch (kind()) {
     case FeedbackSlotKind::kCall:
@@ -1129,7 +1106,7 @@ void FeedbackNexus::Print(std::ostream& os) {  // NOLINT
     case FeedbackSlotKind::kStoreKeyedStrict:
     case FeedbackSlotKind::kStoreInArrayLiteral:
     case FeedbackSlotKind::kCloneObject: {
-      os << ICState2String(StateFromFeedback());
+      os << InlineCacheState2String(StateFromFeedback());
       break;
     }
     case FeedbackSlotKind::kBinaryOp: {

src/vector-slot-pair.cc

@@ -17,22 +17,24 @@ int VectorSlotPair::index() const {
 
 bool operator==(VectorSlotPair const& lhs, VectorSlotPair const& rhs) {
   return lhs.slot() == rhs.slot() &&
-         lhs.vector().location() == rhs.vector().location();
+         lhs.vector().location() == rhs.vector().location() &&
+         lhs.ic_state() == rhs.ic_state();
 }
 
 bool operator!=(VectorSlotPair const& lhs, VectorSlotPair const& rhs) {
   return !(lhs == rhs);
 }
 
-std::ostream& operator<<(std::ostream& os, const VectorSlotPair& pair) {
-  if (pair.IsValid()) {
-    return os << "VectorSlotPair(" << pair.slot() << ")";
+std::ostream& operator<<(std::ostream& os, const VectorSlotPair& p) {
+  if (p.IsValid()) {
+    return os << "VectorSlotPair(" << p.slot() << ", "
+              << InlineCacheState2String(p.ic_state()) << ")";
   }
   return os << "VectorSlotPair(INVALID)";
 }
 
 size_t hash_value(VectorSlotPair const& p) {
-  return base::hash_combine(p.slot(), p.vector().location());
+  return base::hash_combine(p.slot(), p.vector().location(), p.ic_state());
 }
 
 }  // namespace internal

src/vector-slot-pair.h

@@ -19,25 +19,28 @@ class FeedbackVector;
 class V8_EXPORT_PRIVATE VectorSlotPair {
  public:
   VectorSlotPair();
-  VectorSlotPair(Handle<FeedbackVector> vector, FeedbackSlot slot)
-      : vector_(vector), slot_(slot) {}
+  VectorSlotPair(Handle<FeedbackVector> vector, FeedbackSlot slot,
+                 InlineCacheState ic_state)
+      : vector_(vector), slot_(slot), ic_state_(ic_state) {}
 
   bool IsValid() const { return !vector_.is_null() && !slot_.IsInvalid(); }
 
   Handle<FeedbackVector> vector() const { return vector_; }
   FeedbackSlot slot() const { return slot_; }
+  InlineCacheState ic_state() const { return ic_state_; }
 
   int index() const;
 
  private:
   Handle<FeedbackVector> vector_;
   FeedbackSlot slot_;
+  InlineCacheState ic_state_ = UNINITIALIZED;
 };
 
 bool operator==(VectorSlotPair const&, VectorSlotPair const&);
 bool operator!=(VectorSlotPair const&, VectorSlotPair const&);
 
-std::ostream& operator<<(std::ostream& os, const VectorSlotPair& pair);
+std::ostream& operator<<(std::ostream& os, VectorSlotPair const&);
 
 size_t hash_value(VectorSlotPair const&);
 

test/unittests/compiler/js-call-reducer-unittest.cc

@@ -124,7 +124,7 @@ class JSCallReducerTest : public TypedGraphTest {
     // overwriting existing metadata.
     shared->set_raw_outer_scope_info_or_feedback_metadata(*metadata);
     Handle<FeedbackVector> vector = FeedbackVector::New(isolate(), shared);
-    VectorSlotPair feedback(vector, FeedbackSlot(0));
+    VectorSlotPair feedback(vector, FeedbackSlot(0), UNINITIALIZED);
     return javascript()->Call(arity, CallFrequency(), feedback,
                               ConvertReceiverMode::kAny,
                               SpeculationMode::kAllowSpeculation);