[turboprop] Unroll loop in the dynamic map checks operator

The dynamic map checks operator loads the feedback vector and performs
map checks against each map and handler entry in the feedback against
the incoming map and handler.

Instead of emitting code to iterate over this feedback vector at
runtime, we unroll this loop at compile time.

The generated code is similar to this pseudocode:

  length = feedback_slot.length
  if length >= 4: goto labels[3]
  if length == 3: goto labels[2]
  if length == 2: goto labels[1]
  if length == 1: goto labels[0]

  labels[3]:
    map = load(feedback_slot, 6)
    if incoming_map == map goto handler_check(7)
    goto labels[2]
  labels[2]:
    map = load(feedback_slot, 4)
    if incoming_map == map goto handler_check(5)
    goto labels[1]
  labels[1]:
    map = load(feedback_slot, 2)
    if incoming_map == map goto handler_check(3)
    goto labels[0]
  labels[0]:
    map = load(feedback_slot, 0)
    if incoming_map == map goto handler_check(1)
    bailout

  handler_check (index):
    handler = load(feedback_slot, index)
    if incoming_handler ==  handler goto done
    deoptimize

Bug: v8:10582, v8:9684
Change-Id: I64d64ff8eda664e4d476bf1b2612e26a344e98a6
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2339960
Commit-Queue: Sathya Gunasekaran  <gsathya@chromium.org>
Reviewed-by: Mythri Alle <mythria@chromium.org>
Reviewed-by: Jakob Gruber <jgruber@chromium.org>
Cr-Commit-Position: refs/heads/master@{#69797}

src/compiler/access-builder.cc

@@ -870,6 +870,15 @@ FieldAccess AccessBuilder::ForFeedbackVectorSlot(int index) {
   return access;
 }
 
+// static
+FieldAccess AccessBuilder::ForWeakFixedArraySlot(int index) {
+  int offset = WeakFixedArray::OffsetOfElementAt(index);
+  FieldAccess access = {kTaggedBase,      offset,
+                        Handle<Name>(),   MaybeHandle<Map>(),
+                        Type::Any(),      MachineType::AnyTagged(),
+                        kFullWriteBarrier};
+  return access;
+}
 // static
 FieldAccess AccessBuilder::ForCellValue() {
   FieldAccess access = {kTaggedBase,       Cell::kValueOffset,

src/compiler/access-builder.h

@@ -293,6 +293,7 @@ class V8_EXPORT_PRIVATE AccessBuilder final
 
   // Provides access to WeakFixedArray elements.
   static ElementAccess ForWeakFixedArrayElement();
+  static FieldAccess ForWeakFixedArraySlot(int index);
 
   // Provides access to FixedArray elements.
   static ElementAccess ForFixedArrayElement();

src/compiler/effect-control-linearizer.cc

@@ -265,6 +265,7 @@ class EffectControlLinearizer {
   Node* ObjectIsSmi(Node* value);
   Node* LoadFromSeqString(Node* receiver, Node* position, Node* is_one_byte);
   Node* TruncateWordToInt32(Node* value);
+  Node* MakeWeakForComparison(Node* heap_object);
   Node* BuildIsWeakReferenceTo(Node* maybe_object, Node* value);
   Node* BuildIsClearedWeakReference(Node* maybe_object);
   Node* BuildIsStrongReference(Node* value);
@@ -282,8 +283,9 @@ class EffectControlLinearizer {
                               DeoptimizeReason reason);
 
   // Helper functions used in LowerDynamicCheckMaps
-  void CheckPolymorphic(Node* feedback, Node* value_map, Node* handler,
-                        GraphAssemblerLabel<0>* done, Node* frame_state);
+  void CheckPolymorphic(Node* expected_polymorphic_array, Node* actual_map,
+                        Node* actual_handler, GraphAssemblerLabel<0>* done,
+                        Node* frame_state);
   void ProcessMonomorphic(Node* handler, GraphAssemblerLabel<0>* done,
                           Node* frame_state, int slot, Node* vector);
   void BranchOnICState(int slot_index, Node* vector, Node* value_map,
@@ -1881,51 +1883,107 @@ void EffectControlLinearizer::LowerCheckMaps(Node* node, Node* frame_state) {
   }
 }
 
-void EffectControlLinearizer::CheckPolymorphic(Node* feedback_slot,
-                                               Node* value_map, Node* handler,
+void EffectControlLinearizer::CheckPolymorphic(Node* expected_polymorphic_array,
+                                               Node* actual_map,
+                                               Node* actual_handler,
                                                GraphAssemblerLabel<0>* done,
                                                Node* frame_state) {
-  Node* feedback_slot_map =
-      __ LoadField(AccessBuilder::ForMap(), feedback_slot);
-  Node* is_weak_fixed_array_check =
-      __ TaggedEqual(feedback_slot_map, __ WeakFixedArrayMapConstant());
+  Node* expected_polymorphic_array_map =
+      __ LoadField(AccessBuilder::ForMap(), expected_polymorphic_array);
+  Node* is_weak_fixed_array = __ TaggedEqual(expected_polymorphic_array_map,
+                                             __ WeakFixedArrayMapConstant());
   __ DeoptimizeIfNot(DeoptimizeReason::kTransitionedToMegamorphicIC,
-                     FeedbackSource(), is_weak_fixed_array_check, frame_state,
+                     FeedbackSource(), is_weak_fixed_array, frame_state,
                      IsSafetyCheck::kCriticalSafetyCheck);
 
-  Node* length = ChangeSmiToInt32(
-      __ LoadField(AccessBuilder::ForWeakFixedArrayLength(), feedback_slot));
-  auto loop = __ MakeLoopLabel(MachineRepresentation::kWord32);
-  __ Goto(&loop, __ Int32Constant(0));
-  __ Bind(&loop);
-  {
-    Node* index = loop.PhiAt(0);
-    Node* check = __ Int32LessThan(index, length);
-    __ DeoptimizeIfNot(DeoptimizeKind::kBailout, DeoptimizeReason::kMissingMap,
-                       FeedbackSource(), check, frame_state,
-                       IsSafetyCheck::kCriticalSafetyCheck);
+  Node* polymorphic_array = expected_polymorphic_array;
 
-    Node* maybe_map = __ LoadElement(AccessBuilder::ForWeakFixedArrayElement(),
-                                     feedback_slot, index);
-    auto continue_loop = __ MakeLabel();
-
-    __ GotoIfNot(BuildIsWeakReferenceTo(maybe_map, value_map), &continue_loop);
-    constexpr int kHandlerOffsetInEntry = 1;
-    Node* maybe_handler = __ LoadElement(
-        AccessBuilder::ForWeakFixedArrayElement(), feedback_slot,
-        __ Int32Add(index, __ Int32Constant(kHandlerOffsetInEntry)));
-    Node* handler_check = __ TaggedEqual(maybe_handler, handler);
-    __ DeoptimizeIfNot(DeoptimizeReason::kWrongHandler, FeedbackSource(),
-                       handler_check, frame_state,
-                       IsSafetyCheck::kCriticalSafetyCheck);
+  // This is now a weak pointer that we're holding in the register, we
+  // need to be careful about spilling and reloading it (as it could
+  // get cleared in between). There's no runtime call here that could
+  // cause a spill so we should be safe.
+  Node* weak_actual_map = MakeWeakForComparison(actual_map);
+  Node* length = ChangeSmiToInt32(__ LoadField(
+      AccessBuilder::ForWeakFixedArrayLength(), polymorphic_array));
+  auto do_handler_check = __ MakeLabel(MachineRepresentation::kWord32);
 
-    __ Goto(done);
+  GraphAssemblerLabel<0> labels[] = {__ MakeLabel(), __ MakeLabel(),
+                                     __ MakeLabel(), __ MakeLabel()};
 
-    __ Bind(&continue_loop);
-    constexpr int kEntrySize = 2;
-    index = __ Int32Add(index, __ Int32Constant(kEntrySize));
-    __ Goto(&loop, index);
+  STATIC_ASSERT(FLAG_max_minimorphic_map_checks == arraysize(labels));
+  DCHECK_GE(FLAG_max_minimorphic_map_checks,
+            FLAG_max_valid_polymorphic_map_count);
+
+  // The following generates a switch based on the length of the
+  // array:
+  //
+  // if length >= 4: goto labels[3]
+  // if length == 3: goto labels[2]
+  // if length == 2: goto labels[1]
+  // if length == 1: goto labels[0]
+  __ GotoIf(__ Int32LessThanOrEqual(
+                __ Int32Constant(FeedbackIterator::SizeFor(4)), length),
+            &labels[3]);
+  __ GotoIf(
+      __ Word32Equal(length, __ Int32Constant(FeedbackIterator::SizeFor(3))),
+      &labels[2]);
+  __ GotoIf(
+      __ Word32Equal(length, __ Int32Constant(FeedbackIterator::SizeFor(2))),
+      &labels[1]);
+  __ GotoIf(
+      __ Word32Equal(length, __ Int32Constant(FeedbackIterator::SizeFor(1))),
+      &labels[0]);
+
+  // We should never have an polymorphic feedback array of size 0.
+  __ Unreachable(done);
+
+  // This loop generates code like this to do the dynamic map check:
+  //
+  // labels[3]:
+  //   maybe_map = load(polymorphic_array, i)
+  //   if weak_actual_map == maybe_map goto handler_check
+  //   goto labels[2]
+  // labels[2]:
+  //   maybe_map = load(polymorphic_array, i - 1)
+  //   if weak_actual_map == maybe_map goto handler_check
+  //   goto labels[1]
+  // labels[1]:
+  //   maybe_map = load(polymorphic_array, i - 2)
+  //   if weak_actual_map == maybe_map goto handler_check
+  //   goto labels[0]
+  // labels[0]:
+  //   maybe_map = load(polymorphic_array, i - 3)
+  //   if weak_actual_map == maybe_map goto handler_check
+  //   bailout
+  for (int i = arraysize(labels) - 1; i >= 0; i--) {
+    __ Bind(&labels[i]);
+    Node* maybe_map = __ LoadField(AccessBuilder::ForWeakFixedArraySlot(
+                                       FeedbackIterator::MapIndexForEntry(i)),
+                                   polymorphic_array);
+    Node* map_check = __ TaggedEqual(maybe_map, weak_actual_map);
+
+    int handler_index = FeedbackIterator::HandlerIndexForEntry(i);
+    __ GotoIf(map_check, &do_handler_check, __ Int32Constant(handler_index));
+    if (i > 0) {
+      __ Goto(&labels[i - 1]);
+    } else {
+      // TODO(turbofan): Add support for gasm->Deoptimize.
+      __ DeoptimizeIf(DeoptimizeKind::kBailout, DeoptimizeReason::kMissingMap,
+                      FeedbackSource(), __ IntPtrConstant(1),
+                      FrameState(frame_state));
+      __ Unreachable(done);
+    }
   }
+
+  __ Bind(&do_handler_check);
+  Node* handler_index = do_handler_check.PhiAt(0);
+  Node* maybe_handler =
+      __ LoadElement(AccessBuilder::ForWeakFixedArrayElement(),
+                     polymorphic_array, handler_index);
+  __ DeoptimizeIfNot(DeoptimizeReason::kWrongHandler, FeedbackSource(),
+                     __ TaggedEqual(maybe_handler, actual_handler), frame_state,
+                     IsSafetyCheck::kCriticalSafetyCheck);
+  __ Goto(done);
 }
 
 void EffectControlLinearizer::ProcessMonomorphic(Node* handler,
@@ -6438,6 +6496,13 @@ Node* EffectControlLinearizer::BuildIsStrongReference(Node* value) {
       __ Int32Constant(kHeapObjectTag));
 }
 
+Node* EffectControlLinearizer::MakeWeakForComparison(Node* heap_object) {
+  // TODO(gsathya): Specialize this for pointer compression.
+  return __ BitcastWordToTagged(
+      __ WordOr(__ BitcastTaggedToWord(heap_object),
+                __ IntPtrConstant(kWeakHeapObjectTag)));
+}
+
 Node* EffectControlLinearizer::BuildStrongReferenceFromWeakReference(
     Node* maybe_object) {
   return __ BitcastWordToTagged(

src/compiler/graph-assembler.h

@@ -91,6 +91,7 @@ class BasicBlock;
   V(Word64Or)                             \
   V(WordAnd)                              \
   V(WordEqual)                            \
+  V(WordOr)                               \
   V(WordSar)                              \
   V(WordSarShiftOutZeros)                 \
   V(WordShl)                              \

src/flags/flag-definitions.h

@@ -522,6 +522,8 @@ DEFINE_IMPLICATION(turboprop, concurrent_inlining)
 DEFINE_VALUE_IMPLICATION(turboprop, interrupt_budget, 15 * KB)
 DEFINE_VALUE_IMPLICATION(turboprop, reuse_opt_code_count, 2)
 DEFINE_IMPLICATION(turboprop, dynamic_map_checks)
+DEFINE_UINT_READONLY(max_minimorphic_map_checks, 4,
+                     "max number of map checks to perform in minimorphic state")
 
 // Flags for concurrent recompilation.
 DEFINE_BOOL(concurrent_recompilation, true,

src/objects/feedback-vector.h

@@ -757,12 +757,27 @@ class V8_EXPORT_PRIVATE FeedbackIterator final {
   Map map() { return map_; }
   MaybeObject handler() { return handler_; }
 
+  static int SizeFor(int number_of_entries) {
+    CHECK_GT(number_of_entries, 0);
+    return number_of_entries * kEntrySize;
+  }
+
+  static int MapIndexForEntry(int entry) {
+    CHECK_GE(entry, 0);
+    return entry * kEntrySize;
+  }
+
+  static int HandlerIndexForEntry(int entry) {
+    CHECK_GE(entry, 0);
+    return (entry * kEntrySize) + kHandlerOffset;
+  }
+
  private:
   void AdvancePolymorphic();
   enum State { kMonomorphic, kPolymorphic, kOther };
+
   static constexpr int kEntrySize = 2;
   static constexpr int kHandlerOffset = 1;
-
   Handle<WeakFixedArray> polymorphic_feedback_;
   Map map_;
   MaybeObject handler_;

src/objects/fixed-array.h

@@ -296,7 +296,6 @@ class WeakFixedArray
 
   int AllocatedSize();
 
- protected:
   static int OffsetOfElementAt(int index) {
     STATIC_ASSERT(kObjectsOffset == SizeFor(0));
     return SizeFor(index);