[turboprop] Introduce a builtin to perform dynamic map checks

Instead of always inlining the polymorphic map checks, this CL
introduces a builtin to perform these polymorphic map checks
when the IC is monomorphic at compile time.

This reduces the time we spend compiling and code bloat while trading it
for performance.

Bug: v8:10582, v8:9684
Change-Id: I7aea698988f8ead3cbf3f4a836218f53223f0f98
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2398525
Commit-Queue: Sathya Gunasekaran  <gsathya@chromium.org>
Reviewed-by: Jakob Gruber <jgruber@chromium.org>
Cr-Commit-Position: refs/heads/master@{#70200}

BUILD.gn

@@ -1151,6 +1151,7 @@ torque_files = [
   "src/builtins/function.tq",
   "src/builtins/growable-fixed-array.tq",
   "src/builtins/ic-callable.tq",
+  "src/builtins/ic-dynamic-map-checks.tq",
   "src/builtins/ic.tq",
   "src/builtins/internal-coverage.tq",
   "src/builtins/internal.tq",

src/builtins/base.tq

@@ -166,6 +166,9 @@ type LayoutDescriptor extends ByteArray
     generates 'TNode<LayoutDescriptor>';
 extern class TransitionArray extends WeakFixedArray;
 
+extern operator '.length_intptr' macro LoadAndUntagWeakFixedArrayLength(
+    WeakFixedArray): intptr;
+
 type InstanceType extends uint16 constexpr 'InstanceType';
 
 type NoSharedNameSentinel extends Smi;
@@ -1703,3 +1706,9 @@ struct ConstantIterator<T: type> {
 macro ConstantIterator<T: type>(value: T): ConstantIterator<T> {
   return ConstantIterator{value};
 }
+
+extern macro FeedbackIteratorSizeFor(constexpr int32): intptr;
+extern macro FeedbackIteratorMapIndexForEntry(constexpr int32): intptr;
+extern macro FeedbackIteratorHandlerIndexForEntry(constexpr int32): intptr;
+extern operator '[]' macro LoadWeakFixedArrayElement(
+    WeakFixedArray, intptr): MaybeObject;

src/builtins/cast.tq

@@ -158,6 +158,7 @@ macro Cast<A : type extends Object>(implicit context: Context)(o: Object): A
       otherwise CastError;
 }
 
+// This is required for casting MaybeObject to Object.
 Cast<Smi>(o: Object): Smi
     labels CastError {
   return TaggedToSmi(o) otherwise CastError;
@@ -671,3 +672,8 @@ macro CastOrDefault<T: type, Arg: type, Default: type>(
     implicit context: Context)(x: Arg, default: Default): T|Default {
   return Cast<T>(x) otherwise return default;
 }
+
+Cast<Object>(o: Object): Object
+labels _CastError {
+  return o;
+}

src/builtins/ic-dynamic-map-checks.tq

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be:
+// Context found in the LICENSE file.
+
+namespace ic {
+
+const kSuccess: constexpr int32
+    generates 'static_cast<int>(DynamicMapChecksStatus::kSuccess)';
+const kBailout: constexpr int32
+    generates 'static_cast<int>(DynamicMapChecksStatus::kBailout)';
+const kDeopt: constexpr int32
+    generates 'static_cast<int>(DynamicMapChecksStatus::kDeopt)';
+
+macro PerformMapAndHandlerCheck(
+    entry: constexpr int32, polymorphicArray: WeakFixedArray,
+    weakActualMap: WeakHeapObject, actualHandler: Object): void labels Next,
+    Deopt {
+  const mapIndex = FeedbackIteratorMapIndexForEntry(entry);
+  assert(mapIndex < polymorphicArray.length_intptr);
+  const maybeCachedMap =
+      Cast<WeakHeapObject>(polymorphicArray[mapIndex]) otherwise unreachable;
+  assert(IsWeakOrCleared(maybeCachedMap));
+  if (maybeCachedMap != weakActualMap) {
+    goto Next;
+  }
+
+  const handlerIndex = FeedbackIteratorHandlerIndexForEntry(entry);
+  assert(handlerIndex < polymorphicArray.length_intptr);
+  const maybeHandler =
+      Cast<Object>(polymorphicArray[handlerIndex]) otherwise unreachable;
+  if (TaggedNotEqual(maybeHandler, actualHandler)) {
+    goto Deopt;
+  }
+}
+
+builtin DynamicMapChecks(implicit context: Context)(
+    expectedPolymorphicArray: HeapObject, actualMap: Map,
+    actualHandler: Smi|DataHandler): int32 {
+  if (!Is<WeakFixedArray>(expectedPolymorphicArray)) {
+    return kDeopt;
+  }
+  try {
+    const polymorphicArray =
+        Cast<WeakFixedArray>(expectedPolymorphicArray) otherwise unreachable;
+    const weakActualMap = MakeWeak(actualMap);
+    const length = polymorphicArray.length_intptr;
+    assert(length > 0);
+
+    try {
+      if (length >= FeedbackIteratorSizeFor(4)) goto Len4;
+      if (length == FeedbackIteratorSizeFor(3)) goto Len3;
+      if (length == FeedbackIteratorSizeFor(2)) goto Len2;
+      if (length == FeedbackIteratorSizeFor(1)) goto Len1;
+
+      unreachable;
+    } label Len4 {
+      PerformMapAndHandlerCheck(
+          3, polymorphicArray, weakActualMap, actualHandler) otherwise Len3,
+          Deopt;
+      return kSuccess;
+    } label Len3 {
+      PerformMapAndHandlerCheck(
+          2, polymorphicArray, weakActualMap, actualHandler) otherwise Len2,
+          Deopt;
+      return kSuccess;
+    } label Len2 {
+      PerformMapAndHandlerCheck(
+          1, polymorphicArray, weakActualMap, actualHandler) otherwise Len1,
+          Deopt;
+      return kSuccess;
+    } label Len1 {
+      PerformMapAndHandlerCheck(
+          0, polymorphicArray, weakActualMap, actualHandler)
+          otherwise Bailout, Deopt;
+      return kSuccess;
+    }
+  } label Bailout {
+    return kBailout;
+  } label Deopt {
+    return kDeopt;
+  }
+}
+}

src/codegen/code-stub-assembler.h

@@ -15,6 +15,7 @@
 #include "src/compiler/code-assembler.h"
 #include "src/objects/arguments.h"
 #include "src/objects/bigint.h"
+#include "src/objects/feedback-vector.h"
 #include "src/objects/js-function.h"
 #include "src/objects/objects.h"
 #include "src/objects/promise.h"
@@ -3570,6 +3571,18 @@ class V8_EXPORT_PRIVATE CodeStubAssembler
       TNode<JSFinalizationRegistry> finalization_registry,
       TNode<WeakCell> weak_cell);
 
+  TNode<IntPtrT> FeedbackIteratorSizeFor(int number_of_entries) {
+    return IntPtrConstant(FeedbackIterator::SizeFor(number_of_entries));
+  }
+
+  TNode<IntPtrT> FeedbackIteratorMapIndexForEntry(int entry) {
+    return IntPtrConstant(FeedbackIterator::MapIndexForEntry(entry));
+  }
+
+  TNode<IntPtrT> FeedbackIteratorHandlerIndexForEntry(int entry) {
+    return IntPtrConstant(FeedbackIterator::HandlerIndexForEntry(entry));
+  }
+
  private:
   friend class CodeStubArguments;
 

src/common/globals.h

@@ -1695,6 +1695,12 @@ enum class TraceRetainingPathMode { kEnabled, kDisabled };
 // can be used in Torque.
 enum class VariableAllocationInfo { NONE, STACK, CONTEXT, UNUSED };
 
+enum class DynamicMapChecksStatus : uint8_t {
+  kSuccess = 0,
+  kBailout = 1,
+  kDeopt = 2
+};
+
 }  // namespace internal
 }  // namespace v8
 

src/compiler/effect-control-linearizer.cc

@@ -284,9 +284,14 @@ class EffectControlLinearizer {
                               DeoptimizeReason reason);
 
   // Helper functions used in LowerDynamicCheckMaps
-  void CheckPolymorphic(Node* expected_polymorphic_array, Node* actual_map,
-                        Node* actual_handler, GraphAssemblerLabel<0>* done,
-                        Node* frame_state);
+  void PerformPolymorphicCheckInline(Node* expected_polymorphic_array,
+                                     Node* actual_map, Node* actual_handler,
+                                     GraphAssemblerLabel<0>* done,
+                                     Node* frame_state);
+  void PerformPolymorphicCheckInBuiltin(Node* expected_polymorphic_array,
+                                        Node* actual_map, Node* actual_handler,
+                                        GraphAssemblerLabel<0>* done,
+                                        Node* frame_state);
   void ProcessMonomorphic(Node* handler, GraphAssemblerLabel<0>* done,
                           Node* frame_state, int slot, Node* vector);
   void BranchOnICState(int slot_index, Node* vector, Node* value_map,
@@ -1887,11 +1892,33 @@ void EffectControlLinearizer::LowerCheckMaps(Node* node, Node* frame_state) {
   }
 }
 
-void EffectControlLinearizer::CheckPolymorphic(Node* expected_polymorphic_array,
-                                               Node* actual_map,
-                                               Node* actual_handler,
-                                               GraphAssemblerLabel<0>* done,
-                                               Node* frame_state) {
+void EffectControlLinearizer::PerformPolymorphicCheckInBuiltin(
+    Node* expected_polymorphic_array, Node* actual_map, Node* actual_handler,
+    GraphAssemblerLabel<0>* done, Node* frame_state) {
+  Operator::Properties properties = Operator::kNoDeopt | Operator::kNoThrow;
+  Node* result =
+      CallBuiltin(Builtins::kDynamicMapChecks, properties,
+                  expected_polymorphic_array, actual_map, actual_handler);
+  __ GotoIf(__ WordEqual(result, __ IntPtrConstant(static_cast<int>(
+                                     DynamicMapChecksStatus::kSuccess))),
+            done);
+  __ DeoptimizeIf(DeoptimizeKind::kBailout, DeoptimizeReason::kMissingMap,
+                  FeedbackSource(),
+                  __ WordEqual(result, __ IntPtrConstant(static_cast<int>(
+                                           DynamicMapChecksStatus::kBailout))),
+                  frame_state, IsSafetyCheck::kCriticalSafetyCheck);
+  __ DeoptimizeIf(DeoptimizeReason::kWrongHandler, FeedbackSource(),
+                  __ WordEqual(result, __ IntPtrConstant(static_cast<int>(
+                                           DynamicMapChecksStatus::kDeopt))),
+                  frame_state, IsSafetyCheck::kCriticalSafetyCheck);
+  __ Unreachable(done);
+}
+
+void EffectControlLinearizer::PerformPolymorphicCheckInline(
+    Node* expected_polymorphic_array, Node* actual_map, Node* actual_handler,
+    GraphAssemblerLabel<0>* done, Node* frame_state) {
+  // Note: This function must stay in sync with
+  // Builtin:kDynamicMapChecks.
   Node* expected_polymorphic_array_map =
       __ LoadField(AccessBuilder::ForMap(), expected_polymorphic_array);
   Node* is_weak_fixed_array = __ TaggedEqual(expected_polymorphic_array_map,
@@ -2057,7 +2084,7 @@ void EffectControlLinearizer::LowerDynamicCheckMaps(Node* node,
   // monomorphic start, we will deopt and reoptimize the code.
   if (p.state() == DynamicCheckMapsParameters::kMonomorphic) {
     auto monomorphic_map_match = __ MakeLabel();
-    auto maybe_poly = __ MakeLabel();
+    auto maybe_poly = __ MakeDeferredLabel();
     Node* strong_feedback;
     Node* poly_array;
 
@@ -2093,10 +2120,11 @@ void EffectControlLinearizer::LowerDynamicCheckMaps(Node* node,
 
     __ Bind(&maybe_poly);
     // TODO(mythria): ICs don't drop deprecated maps from feedback vector.
-    // So it is not equired to migrate the instance for polymorphic case.
+    // So it is not required to migrate the instance for polymorphic case.
     // When we change dynamic map checks to check only four maps re-evaluate
     // if this is required.
-    CheckPolymorphic(poly_array, value_map, handler, &done, frame_state);
+    PerformPolymorphicCheckInBuiltin(poly_array, value_map, handler, &done,
+                                     frame_state);
   } else {
     DCHECK_EQ(p.state(), DynamicCheckMapsParameters::kPolymorphic);
     Node* feedback_slot = __ LoadField(
@@ -2109,7 +2137,8 @@ void EffectControlLinearizer::LowerDynamicCheckMaps(Node* node,
     __ DeoptimizeIfNot(DeoptimizeReason::kTransitionedToMonomorphicIC,
                        FeedbackSource(), is_poly_or_megamorphic, frame_state,
                        IsSafetyCheck::kCriticalSafetyCheck);
-    CheckPolymorphic(feedback_slot, value_map, handler, &done, frame_state);
+    PerformPolymorphicCheckInline(feedback_slot, value_map, handler, &done,
+                                  frame_state);
   }
   __ Bind(&done);
 }