[ic] Port CollectCallableFeedback to Torque

Collecting feedback for {Call,InstanceOf,Construct} is similar
but distressingly different. In preparation for adding a
CollectConstructFeedback helper, this CL ports {Call,InstanceOf}
feedback collection to Torque.

Bug: v8:8888
Change-Id: Iaacc137ef46a77a4fe2857ec41c5cc30614dfdf0
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2187497Reviewed-by: Leszek Swirski <leszeks@chromium.org>
Reviewed-by: Igor Sheludko <ishell@chromium.org>
Commit-Queue: Jakob Gruber <jgruber@chromium.org>
Cr-Commit-Position: refs/heads/master@{#67703}

BUILD.gn

@@ -1039,6 +1039,8 @@ torque_files = [
   "src/builtins/frames.tq",
   "src/builtins/frame-arguments.tq",
   "src/builtins/growable-fixed-array.tq",
+  "src/builtins/ic-callable.tq",
+  "src/builtins/ic.tq",
   "src/builtins/internal-coverage.tq",
   "src/builtins/iterator.tq",
   "src/builtins/math.tq",

src/builtins/base.tq

@@ -52,6 +52,7 @@ extern macro MakeWeak(HeapObject): WeakHeapObject;
 extern macro GetHeapObjectAssumeWeak(WeakHeapObject):
     HeapObject labels ClearedWeakPointer;
 extern macro IsWeakOrCleared(MaybeObject): bool;
+extern macro IsWeakReferenceToObject(MaybeObject, Object): bool;
 
 macro StrongToWeak<T: type>(x: T): Weak<T> {
   return %RawDownCast<Weak<T>>(MakeWeak(x));
@@ -382,20 +383,20 @@ type Boolean = True|False;
 
 type NumberOrUndefined = Number|Undefined;
 
-extern macro TheHoleConstant(): TheHole;
-extern macro NullConstant(): Null;
-extern macro UndefinedConstant(): Undefined;
-extern macro TrueConstant(): True;
+extern macro EmptyStringConstant(): EmptyString;
 extern macro FalseConstant(): False;
-extern macro Int32TrueConstant(): bool;
 extern macro Int32FalseConstant(): bool;
-extern macro EmptyStringConstant(): EmptyString;
+extern macro Int32TrueConstant(): bool;
+extern macro IteratorSymbolConstant(): PublicSymbol;
 extern macro LengthStringConstant(): String;
+extern macro MatchSymbolConstant(): Symbol;
 extern macro MessageStringConstant(): String;
 extern macro NanConstant(): NaN;
-extern macro IteratorSymbolConstant(): PublicSymbol;
-extern macro MatchSymbolConstant(): Symbol;
+extern macro NullConstant(): Null;
 extern macro ReturnStringConstant(): String;
+extern macro TheHoleConstant(): TheHole;
+extern macro TrueConstant(): True;
+extern macro UndefinedConstant(): Undefined;
 
 const TheHole: TheHole = TheHoleConstant();
 const Null: Null = NullConstant();
@@ -523,8 +524,6 @@ extern transitioning macro HasProperty_Inline(implicit context: Context)(
     JSReceiver, JSAny): Boolean;
 extern builtin LoadIC(
     Context, JSAny, JSAny, TaggedIndex, FeedbackVector): JSAny;
-extern macro CollectCallFeedback(
-    JSAny, Context, Undefined | FeedbackVector, uintptr);
 
 extern macro ThrowRangeError(implicit context: Context)(
     constexpr MessageTemplate): never;
@@ -724,7 +723,7 @@ macro Float64IsNaN(n: float64): bool {
 
 // The type of all tagged values that can safely be compared with TaggedEqual.
 type TaggedWithIdentity =
-    JSReceiver|FixedArrayBase|Oddball|Map|WeakCell|EmptyString;
+    JSReceiver|FixedArrayBase|Oddball|Map|WeakCell|Context|EmptyString;
 
 extern operator '==' macro TaggedEqual(TaggedWithIdentity, Object): bool;
 extern operator '==' macro TaggedEqual(Object, TaggedWithIdentity): bool;

src/builtins/cast.tq

@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 extern macro IsConstructor(HeapObject): bool;
+extern macro IsFeedbackCell(HeapObject): bool;
 extern macro IsFeedbackVector(HeapObject): bool;
 extern macro IsJSAggregateError(HeapObject): bool;
 extern macro IsJSArray(HeapObject): bool;
@@ -489,6 +490,12 @@ Cast<Map>(implicit context: Context)(o: HeapObject): Map
   goto CastError;
 }
 
+Cast<FeedbackCell>(implicit context: Context)(o: HeapObject): FeedbackCell
+    labels CastError {
+  if (IsFeedbackCell(o)) return %RawDownCast<FeedbackCell>(o);
+  goto CastError;
+}
+
 Cast<FeedbackVector>(implicit context: Context)(o: HeapObject): FeedbackVector
     labels CastError {
   if (IsFeedbackVector(o)) return %RawDownCast<FeedbackVector>(o);

src/builtins/ic-callable.tq

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+namespace ic_callable {
+
+  extern macro IncrementCallCount(FeedbackVector, uintptr): void;
+
+  macro IsMonomorphic(feedback: MaybeObject, target: JSAny): bool {
+    return IsWeakReferenceToObject(feedback, target);
+  }
+
+  macro InSameNativeContext(lhs: Context, rhs: Context): bool {
+    return LoadNativeContext(lhs) == LoadNativeContext(rhs);
+  }
+
+  macro MaybeObjectToStrong(maybeObject: MaybeObject):
+      HeapObject labels IfCleared {
+    assert(IsWeakOrCleared(maybeObject));
+    const weakObject = %RawDownCast<Weak<HeapObject>>(maybeObject);
+    return WeakToStrong(weakObject) otherwise IfCleared;
+  }
+
+  macro TryInitializeAsMonomorphic(implicit context: Context)(
+      maybeTarget: JSAny, feedbackVector: FeedbackVector,
+      slotId: uintptr): void labels TransitionToMegamorphic {
+    const targetHeapObject =
+        Cast<HeapObject>(maybeTarget) otherwise TransitionToMegamorphic;
+
+    let unwrappedTarget = targetHeapObject;
+    while (Is<JSBoundFunction>(unwrappedTarget)) {
+      unwrappedTarget =
+          UnsafeCast<JSBoundFunction>(unwrappedTarget).bound_target_function;
+    }
+
+    const unwrappedTargetJSFunction =
+        Cast<JSFunction>(unwrappedTarget) otherwise TransitionToMegamorphic;
+    if (!InSameNativeContext(unwrappedTargetJSFunction.context, context)) {
+      goto TransitionToMegamorphic;
+    }
+
+    ic::StoreWeakReferenceInFeedbackVector(
+        feedbackVector, slotId, targetHeapObject);
+    ic::ReportFeedbackUpdate(feedbackVector, slotId, 'Call:Initialize');
+  }
+
+  macro TransitionToMegamorphic(implicit context: Context)(
+      feedbackVector: FeedbackVector, slotId: uintptr): void {
+    ic::StoreFeedbackVectorSlot(feedbackVector, slotId, ic::kMegamorphicSymbol);
+    ic::ReportFeedbackUpdate(
+        feedbackVector, slotId, 'Call:TransitionMegamorphic');
+  }
+
+  macro CollectCallFeedback(
+      maybeTarget: JSAny, context: Context,
+      maybeFeedbackVector: Undefined|FeedbackVector, slotId: uintptr): void {
+    const feedbackVector =
+        Cast<FeedbackVector>(maybeFeedbackVector) otherwise return;
+    IncrementCallCount(feedbackVector, slotId);
+
+    try {
+      const feedback: MaybeObject =
+          ic::LoadFeedbackVectorSlot(feedbackVector, slotId);
+      if (IsMonomorphic(feedback, maybeTarget)) return;
+      if (ic::IsMegamorphic(feedback)) return;
+      if (ic::IsUninitialized(feedback)) goto TryInitializeAsMonomorphic;
+
+      // If cleared, we have a new chance to become monomorphic.
+      const feedbackValue: HeapObject =
+          MaybeObjectToStrong(feedback) otherwise TryInitializeAsMonomorphic;
+
+      // Try transitioning to a feedback cell.
+      // Check if {target}s feedback cell matches the {feedbackValue}.
+      const target =
+          Cast<JSFunction>(maybeTarget) otherwise TransitionToMegamorphic;
+      const targetFeedbackCell: FeedbackCell = target.feedback_cell;
+      if (TaggedEqual(feedbackValue, targetFeedbackCell)) return;
+
+      // Check if {target} and {feedbackValue} are both JSFunctions with
+      // the same feedback vector cell, and that those functions were
+      // actually compiled already.
+      const feedbackValueJSFunction =
+          Cast<JSFunction>(feedbackValue) otherwise TransitionToMegamorphic;
+      const feedbackCell: FeedbackCell = feedbackValueJSFunction.feedback_cell;
+      if (!TaggedEqual(feedbackCell, targetFeedbackCell))
+        goto TransitionToMegamorphic;
+
+      ic::StoreWeakReferenceInFeedbackVector(
+          feedbackVector, slotId, feedbackCell);
+      ic::ReportFeedbackUpdate(
+          feedbackVector, slotId, 'Call:FeedbackVectorCell');
+    } label TryInitializeAsMonomorphic {
+      TryInitializeAsMonomorphic(maybeTarget, feedbackVector, slotId)
+          otherwise TransitionToMegamorphic;
+    } label TransitionToMegamorphic {
+      TransitionToMegamorphic(feedbackVector, slotId);
+    }
+  }
+
+  macro CollectInstanceOfFeedback(
+      maybeTarget: JSAny, context: Context,
+      maybeFeedbackVector: Undefined|FeedbackVector, slotId: uintptr): void {
+    const feedbackVector =
+        Cast<FeedbackVector>(maybeFeedbackVector) otherwise return;
+    // Note: The call count is not incremented.
+
+    try {
+      const feedback: MaybeObject =
+          ic::LoadFeedbackVectorSlot(feedbackVector, slotId);
+      if (IsMonomorphic(feedback, maybeTarget)) return;
+      if (ic::IsMegamorphic(feedback)) return;
+      if (ic::IsUninitialized(feedback)) goto TryInitializeAsMonomorphic;
+
+      // If cleared, we have a new chance to become monomorphic.
+      const _feedbackValue: HeapObject =
+          MaybeObjectToStrong(feedback) otherwise TryInitializeAsMonomorphic;
+
+      goto TransitionToMegamorphic;
+    } label TryInitializeAsMonomorphic {
+      TryInitializeAsMonomorphic(maybeTarget, feedbackVector, slotId)
+          otherwise TransitionToMegamorphic;
+    } label TransitionToMegamorphic {
+      TransitionToMegamorphic(feedbackVector, slotId);
+    }
+  }
+
+}  // namespace ic_callable

src/builtins/ic.tq

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+namespace ic {
+
+  // --- The public interface (forwards to the actual implementation).
+
+  @export
+  macro CollectCallFeedback(
+      maybeTarget: JSAny, context: Context,
+      maybeFeedbackVector: Undefined|FeedbackVector, slotId: uintptr): void {
+    ic_callable::CollectCallFeedback(
+        maybeTarget, context, maybeFeedbackVector, slotId);
+  }
+
+  @export
+  macro CollectInstanceOfFeedback(
+      maybeTarget: JSAny, context: Context,
+      maybeFeedbackVector: Undefined|FeedbackVector, slotId: uintptr): void {
+    ic_callable::CollectInstanceOfFeedback(
+        maybeTarget, context, maybeFeedbackVector, slotId);
+  }
+
+  // --- Common functionality.
+
+  extern macro MegamorphicSymbolConstant(): Symbol;
+  extern macro UninitializedSymbolConstant(): Symbol;
+
+  const kMegamorphicSymbol: Symbol = MegamorphicSymbolConstant();
+  const kUninitializedSymbol: Symbol = UninitializedSymbolConstant();
+
+  macro IsMegamorphic(feedback: MaybeObject): bool {
+    return TaggedEqual(feedback, kMegamorphicSymbol);
+  }
+
+  macro IsUninitialized(feedback: MaybeObject): bool {
+    return TaggedEqual(feedback, kUninitializedSymbol);
+  }
+
+  extern macro LoadFeedbackVectorSlot(FeedbackVector, uintptr): MaybeObject;
+  extern macro StoreFeedbackVectorSlot(FeedbackVector, uintptr, MaybeObject):
+      void;
+  extern macro StoreWeakReferenceInFeedbackVector(
+      FeedbackVector, uintptr, HeapObject): MaybeObject;
+  extern macro ReportFeedbackUpdate(FeedbackVector, uintptr, constexpr string);
+
+}

src/builtins/iterator.tq

@@ -71,7 +71,8 @@ namespace iterator {
       context: Context, receiver: JSAny, iteratorMethod: JSAny, callSlot: Smi,
       feedback: Undefined|FeedbackVector): JSAny {
     const callSlotUnTagged: uintptr = Unsigned(SmiUntag(callSlot));
-    CollectCallFeedback(iteratorMethod, context, feedback, callSlotUnTagged);
+    ic::CollectCallFeedback(
+        iteratorMethod, context, feedback, callSlotUnTagged);
     const iteratorCallable: Callable = Cast<Callable>(iteratorMethod)
         otherwise ThrowCalledNonCallable(iteratorMethod);
     return Call(context, iteratorCallable, receiver);

src/codegen/code-stub-assembler.cc

@@ -146,164 +146,6 @@ TNode<IntPtrT> CodeStubAssembler::IntPtrToParameter<IntPtrT>(
   return value;
 }
 
-void CodeStubAssembler::CollectCallableFeedback(
-    TNode<Object> maybe_target, TNode<Context> context,
-    TNode<FeedbackVector> feedback_vector, TNode<UintPtrT> slot_id,
-    CallableFeedbackMode mode) {
-  Label extra_checks(this, Label::kDeferred), done(this);
-
-  // Check if we have monomorphic {target} feedback already.
-  TNode<MaybeObject> feedback =
-      LoadFeedbackVectorSlot(feedback_vector, slot_id);
-  Comment("check if monomorphic");
-  TNode<BoolT> is_monomorphic = IsWeakReferenceToObject(feedback, maybe_target);
-  GotoIf(is_monomorphic, &done);
-
-  // Check if it is a megamorphic {target}.
-  Comment("check if megamorphic");
-  TNode<BoolT> is_megamorphic = TaggedEqual(
-      feedback, HeapConstant(FeedbackVector::MegamorphicSentinel(isolate())));
-  Branch(is_megamorphic, &done, &extra_checks);
-
-  BIND(&extra_checks);
-  {
-    Label initialize(this), mark_megamorphic(this);
-
-    Comment("check if weak reference");
-    TNode<BoolT> is_uninitialized = TaggedEqual(
-        feedback,
-        HeapConstant(FeedbackVector::UninitializedSentinel(isolate())));
-    GotoIf(is_uninitialized, &initialize);
-    CSA_ASSERT(this, IsWeakOrCleared(feedback));
-
-    // If the weak reference is cleared, we have a new chance to become
-    // monomorphic.
-    Comment("check if weak reference is cleared");
-    GotoIf(IsCleared(feedback), &initialize);
-    GotoIf(TaggedIsSmi(maybe_target), &mark_megamorphic);
-
-    if (mode == CallableFeedbackMode::kDontCollectFeedbackCell) {
-      Goto(&mark_megamorphic);
-    } else {
-      Label try_transition_to_feedback_cell(this);
-
-      // Check if {target} is a JSFunction.
-      Comment("check if target is a JSFunction");
-      TNode<HeapObject> target = CAST(maybe_target);
-      GotoIfNot(IsJSFunction(target), &mark_megamorphic);
-
-      // Check if {target}s feedback vector cell matches the {feedback_value}.
-      TNode<HeapObject> feedback_value = GetHeapObjectAssumeWeak(feedback);
-      TNode<Object> target_feedback_cell =
-          LoadObjectField(target, JSFunction::kFeedbackCellOffset);
-      Branch(TaggedEqual(feedback_value, target_feedback_cell), &done,
-             &try_transition_to_feedback_cell);
-
-      BIND(&try_transition_to_feedback_cell);
-      {
-        // Check if {target} and {feedback_value} are both JSFunctions with
-        // the same feedback vector cell, and that those functions were
-        // actually compiled already.
-        GotoIfNot(IsJSFunction(feedback_value), &mark_megamorphic);
-        TNode<HeapObject> feedback_cell = CAST(
-            LoadObjectField(feedback_value, JSFunction::kFeedbackCellOffset));
-        GotoIfNot(TaggedEqual(feedback_cell, target_feedback_cell),
-                  &mark_megamorphic);
-        GotoIfNot(IsFeedbackCell(feedback_cell), &mark_megamorphic);
-
-        // Record the feedback vector cell.
-        Comment("transition to polymorphic");
-        StoreWeakReferenceInFeedbackVector(feedback_vector, slot_id,
-                                           feedback_cell);
-        ReportFeedbackUpdate(feedback_vector, slot_id,
-                             "Call:FeedbackVectorCell");
-        Goto(&done);
-      }
-    }
-
-    BIND(&initialize);
-    {
-      Comment("check if function in same native context");
-      GotoIf(TaggedIsSmi(maybe_target), &mark_megamorphic);
-      TNode<HeapObject> target = CAST(maybe_target);
-      // Check if the {target} is a JSFunction or JSBoundFunction
-      // in the current native context.
-      TVARIABLE(HeapObject, var_current, target);
-      Label loop(this, &var_current), done_loop(this);
-      Goto(&loop);
-      BIND(&loop);
-      {
-        Label if_boundfunction(this), if_function(this);
-        TNode<HeapObject> current = var_current.value();
-        TNode<Uint16T> current_instance_type = LoadInstanceType(current);
-        GotoIf(InstanceTypeEqual(current_instance_type, JS_BOUND_FUNCTION_TYPE),
-               &if_boundfunction);
-        Branch(InstanceTypeEqual(current_instance_type, JS_FUNCTION_TYPE),
-               &if_function, &mark_megamorphic);
-
-        BIND(&if_function);
-        {
-          // Check that the JSFunction {current} is in the current native
-          // context.
-          TNode<Context> current_context =
-              CAST(LoadObjectField(current, JSFunction::kContextOffset));
-          TNode<NativeContext> current_native_context =
-              LoadNativeContext(current_context);
-          Branch(
-              TaggedEqual(LoadNativeContext(context), current_native_context),
-              &done_loop, &mark_megamorphic);
-        }
-        BIND(&if_boundfunction);
-        {
-          // Continue with the [[BoundTargetFunction]] of {target}.
-          var_current = LoadObjectField<HeapObject>(
-              current, JSBoundFunction::kBoundTargetFunctionOffset);
-          Goto(&loop);
-        }
-      }
-      BIND(&done_loop);
-      StoreWeakReferenceInFeedbackVector(feedback_vector, slot_id, target);
-      ReportFeedbackUpdate(feedback_vector, slot_id, "Call:Initialize");
-      Goto(&done);
-    }
-
-    BIND(&mark_megamorphic);
-    {
-      // MegamorphicSentinel is an immortal immovable object so
-      // write-barrier is not needed.
-      Comment("transition to megamorphic");
-      DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kmegamorphic_symbol));
-      StoreFeedbackVectorSlot(
-          feedback_vector, slot_id,
-          HeapConstant(FeedbackVector::MegamorphicSentinel(isolate())),
-          SKIP_WRITE_BARRIER);
-      ReportFeedbackUpdate(feedback_vector, slot_id,
-                           "Call:TransitionMegamorphic");
-      Goto(&done);
-    }
-  }
-
-  BIND(&done);
-}
-
-void CodeStubAssembler::CollectCallFeedback(
-    TNode<Object> maybe_target, TNode<Context> context,
-    TNode<HeapObject> maybe_feedback_vector, TNode<UintPtrT> slot_id) {
-  Label feedback_done(this);
-  // If feedback_vector is not valid, then nothing to do.
-  GotoIf(IsUndefined(maybe_feedback_vector), &feedback_done);
-
-  // Increment the call count.
-  TNode<FeedbackVector> feedback_vector = CAST(maybe_feedback_vector);
-  IncrementCallCount(feedback_vector, slot_id);
-
-  // Collect the callable {target} feedback.
-  CollectCallableFeedback(maybe_target, context, feedback_vector, slot_id,
-                          CallableFeedbackMode::kCollectFeedbackCell);
-  Goto(&feedback_done);
-
-  BIND(&feedback_done);
-}
 
 void CodeStubAssembler::IncrementCallCount(
     TNode<FeedbackVector> feedback_vector, TNode<UintPtrT> slot_id) {

src/codegen/code-stub-assembler.h

@@ -2109,35 +2109,6 @@ class V8_EXPORT_PRIVATE CodeStubAssembler
 
   enum class DestroySource { kNo, kYes };
 
-  // Collect the callable |maybe_target| feedback for either a CALL_IC or
-  // an INSTANCEOF_IC in the |feedback_vector| at |slot_id|. There are
-  // two modes for feedback collection:
-  //
-  //   kCollectFeedbackCell -     collect JSFunctions, but devolve to the
-  //                              FeedbackCell as long as all JSFunctions
-  //                              seen share the same one.
-  //   kDontCollectFeedbackCell - collect JSFunctions without devolving
-  //                              to the FeedbackCell in case a
-  //                              different JSFunction appears. Go directly
-  //                              to the Megamorphic sentinel value in this
-  //                              case.
-  enum class CallableFeedbackMode {
-    kCollectFeedbackCell,
-    kDontCollectFeedbackCell
-  };
-  void CollectCallableFeedback(TNode<Object> maybe_target,
-                               TNode<Context> context,
-                               TNode<FeedbackVector> feedback_vector,
-                               TNode<UintPtrT> slot_id,
-                               CallableFeedbackMode mode);
-
-  // Collect CALL_IC feedback for |maybe_target| function in the
-  // |feedback_vector| at |slot_id|, and the call counts in
-  // the |feedback_vector| at |slot_id+1|.
-  void CollectCallFeedback(TNode<Object> maybe_target, TNode<Context> context,
-                           TNode<HeapObject> maybe_feedback_vector,
-                           TNode<UintPtrT> slot_id);
-
   // Increment the call count for a CALL_IC or construct call.
   // The call count is located at feedback_vector[slot_id + 1].
   void IncrementCallCount(TNode<FeedbackVector> feedback_vector,

src/interpreter/interpreter-generator.cc

@@ -1902,17 +1902,7 @@ IGNITION_HANDLER(TestInstanceOf, InterpreterAssembler) {
   TNode<HeapObject> maybe_feedback_vector = LoadFeedbackVector();
   TNode<Context> context = GetContext();
 
-  Label feedback_done(this);
-  GotoIf(IsUndefined(maybe_feedback_vector), &feedback_done);
-
-  // Record feedback for the {callable} in the {feedback_vector}.
-  CollectCallableFeedback(callable, context, CAST(maybe_feedback_vector),
-                          slot_id,
-                          CallableFeedbackMode::kDontCollectFeedbackCell);
-  Goto(&feedback_done);
-
-  BIND(&feedback_done);
-  // Perform the actual instanceof operation.
+  CollectInstanceOfFeedback(callable, context, maybe_feedback_vector, slot_id);
   SetAccumulator(InstanceOf(object, callable, context));
   Dispatch();
 }

src/objects/feedback-vector.tq

@@ -6,13 +6,15 @@ extern class FeedbackVector extends HeapObject {
   shared_function_info: SharedFunctionInfo;
   optimized_code_weak_or_smi: Weak<Code>|Smi;
   closure_feedback_cell_array: FixedArray;
-  length: int32;
+  const length: int32;
   invocation_count: int32;
   profiler_ticks: int32;
   // TODO(v8:9287) The padding is not necessary on platforms with 4 bytes
   // tagged pointers, we should make it conditional; however, platform-specific
   // interacts badly with GCMole, so we need to address that first.
   padding: uint32;
+  // TODO(tebbi): The variable-length feedback_slots field should be declared
+  // here once it is possible to declare tagged slots after untagged slots.
 }
 
 extern class FeedbackMetadata extends HeapObject;