[turbofan] Call frequencies for JSCallFunction and JSCallConstruct.

Extract the call counts from the type feedback vector during graph
building (either via the AstGraphBuilder or the BytecodeGraphBuilder),
and put them onto the JSCallFunction and JSCallConstruct operators,
so that they work even across inlinine through .apply and .call (which
was previously hacked by creating a temporary type feedback vector
for those).

The next logic step will be to make those call counts into real
relative call frequencies (also during graph building), so that we
can make inlining decisions that make sense for the function being
optimized (where absolute values are misleading).

R=jarin@chromium.org
BUG=v8:5267,v8:5372

Review-Url: https://codereview.chromium.org/2330883002
Cr-Commit-Position: refs/heads/master@{#39400}

src/compiler/ast-graph-builder.cc

@@ -2430,9 +2430,11 @@ void AstGraphBuilder::VisitCall(Call* expr) {
   }
 
   // Create node to perform the function call.
+  float const frequency = ComputeCallFrequency(expr->CallFeedbackICSlot());
   VectorSlotPair feedback = CreateVectorSlotPair(expr->CallFeedbackICSlot());
-  const Operator* call = javascript()->CallFunction(
-      args->length() + 2, feedback, receiver_hint, expr->tail_call_mode());
+  const Operator* call =
+      javascript()->CallFunction(args->length() + 2, frequency, feedback,
+                                 receiver_hint, expr->tail_call_mode());
   PrepareEagerCheckpoint(possibly_eval ? expr->EvalId() : expr->CallId());
   Node* value = ProcessArguments(call, args->length() + 2);
   // The callee passed to the call, we just need to push something here to
@@ -2466,7 +2468,7 @@ void AstGraphBuilder::VisitCallSuper(Call* expr) {
 
   // Create node to perform the super call.
   const Operator* call =
-      javascript()->CallConstruct(args->length() + 2, VectorSlotPair());
+      javascript()->CallConstruct(args->length() + 2, 0.0f, VectorSlotPair());
   Node* value = ProcessArguments(call, args->length() + 2);
   PrepareFrameState(value, expr->ReturnId(), OutputFrameStateCombine::Push());
   ast_context()->ProduceValue(expr, value);
@@ -2484,9 +2486,10 @@ void AstGraphBuilder::VisitCallNew(CallNew* expr) {
   environment()->Push(environment()->Peek(args->length()));
 
   // Create node to perform the construct call.
+  float const frequency = ComputeCallFrequency(expr->CallNewFeedbackSlot());
   VectorSlotPair feedback = CreateVectorSlotPair(expr->CallNewFeedbackSlot());
   const Operator* call =
-      javascript()->CallConstruct(args->length() + 2, feedback);
+      javascript()->CallConstruct(args->length() + 2, frequency, feedback);
   Node* value = ProcessArguments(call, args->length() + 2);
   PrepareFrameState(value, expr->ReturnId(), OutputFrameStateCombine::Push());
   ast_context()->ProduceValue(expr, value);
@@ -3096,6 +3099,13 @@ uint32_t AstGraphBuilder::ComputeBitsetForDynamicContext(Variable* variable) {
   return check_depths;
 }
 
+float AstGraphBuilder::ComputeCallFrequency(FeedbackVectorSlot slot) const {
+  if (slot.IsInvalid()) return 0.0f;
+  Handle<TypeFeedbackVector> feedback_vector(
+      info()->closure()->feedback_vector(), isolate());
+  CallICNexus nexus(feedback_vector, slot);
+  return nexus.ExtractCallCount();
+}
 
 Node* AstGraphBuilder::ProcessArguments(const Operator* op, int arity) {
   DCHECK(environment()->stack_height() >= arity);

src/compiler/ast-graph-builder.h

@@ -264,6 +264,9 @@ class AstGraphBuilder : public AstVisitor<AstGraphBuilder> {
   uint32_t ComputeBitsetForDynamicGlobal(Variable* variable);
   uint32_t ComputeBitsetForDynamicContext(Variable* variable);
 
+  // Computes the frequency for JSCallFunction and JSCallConstruct nodes.
+  float ComputeCallFrequency(FeedbackVectorSlot slot) const;
+
   // ===========================================================================
   // The following build methods all generate graph fragments and return one
   // resulting node. The operand stack height remains the same, variables and

src/compiler/bytecode-graph-builder.cc

@@ -1164,11 +1164,12 @@ void BytecodeGraphBuilder::BuildCall(TailCallMode tail_call_mode) {
   // Slot index of 0 is used indicate no feedback slot is available. Assert
   // the assumption that slot index 0 is never a valid feedback slot.
   STATIC_ASSERT(TypeFeedbackVector::kReservedIndexCount > 0);
-  VectorSlotPair feedback =
-      CreateVectorSlotPair(bytecode_iterator().GetIndexOperand(3));
+  int const slot_id = bytecode_iterator().GetIndexOperand(3);
+  VectorSlotPair feedback = CreateVectorSlotPair(slot_id);
 
+  float const frequency = ComputeCallFrequency(slot_id);
   const Operator* call = javascript()->CallFunction(
-      arg_count + 1, feedback, receiver_hint, tail_call_mode);
+      arg_count + 1, frequency, feedback, receiver_hint, tail_call_mode);
   Node* value = ProcessCallArguments(call, callee, receiver, arg_count + 1);
   environment()->BindAccumulator(value, &states);
 }
@@ -1271,14 +1272,15 @@ void BytecodeGraphBuilder::VisitNew() {
   // Slot index of 0 is used indicate no feedback slot is available. Assert
   // the assumption that slot index 0 is never a valid feedback slot.
   STATIC_ASSERT(TypeFeedbackVector::kReservedIndexCount > 0);
-  VectorSlotPair feedback =
-      CreateVectorSlotPair(bytecode_iterator().GetIndexOperand(3));
+  int const slot_id = bytecode_iterator().GetIndexOperand(3);
+  VectorSlotPair feedback = CreateVectorSlotPair(slot_id);
 
   Node* new_target = environment()->LookupAccumulator();
   Node* callee = environment()->LookupRegister(callee_reg);
 
-  const Operator* call =
-      javascript()->CallConstruct(static_cast<int>(arg_count) + 2, feedback);
+  float const frequency = ComputeCallFrequency(slot_id);
+  const Operator* call = javascript()->CallConstruct(
+      static_cast<int>(arg_count) + 2, frequency, feedback);
   Node* value = ProcessCallNewArguments(call, callee, new_target, first_arg,
                                         arg_count + 2);
   environment()->BindAccumulator(value, &states);
@@ -1349,6 +1351,14 @@ CompareOperationHint BytecodeGraphBuilder::GetCompareOperationHint() {
   return hint;
 }
 
+float BytecodeGraphBuilder::ComputeCallFrequency(int slot_id) const {
+  if (slot_id >= TypeFeedbackVector::kReservedIndexCount) {
+    CallICNexus nexus(feedback_vector(), feedback_vector()->ToSlot(slot_id));
+    return nexus.ExtractCallCount();
+  }
+  return 0.0f;
+}
+
 void BytecodeGraphBuilder::VisitAdd() {
   BuildBinaryOp(
       javascript()->Add(GetBinaryOperationHint(kBinaryOperationHintIndex)));

src/compiler/bytecode-graph-builder.h

@@ -151,6 +151,10 @@ class BytecodeGraphBuilder {
   // type feedback.
   CompareOperationHint GetCompareOperationHint();
 
+  // Helper function to compute call frequency from the recorded type
+  // feedback.
+  float ComputeCallFrequency(int slot_id) const;
+
   // Control flow plumbing.
   void BuildJump();
   void BuildJumpIf(Node* condition);

src/compiler/js-call-reducer.cc

@@ -14,30 +14,6 @@ namespace v8 {
 namespace internal {
 namespace compiler {
 
-namespace {
-
-VectorSlotPair CallCountFeedback(VectorSlotPair p) {
-  // Extract call count from {p}.
-  if (!p.IsValid()) return VectorSlotPair();
-  CallICNexus n(p.vector(), p.slot());
-  int const call_count = n.ExtractCallCount();
-  if (call_count <= 0) return VectorSlotPair();
-
-  // Create megamorphic CallIC feedback with the given {call_count}.
-  StaticFeedbackVectorSpec spec;
-  FeedbackVectorSlot slot = spec.AddCallICSlot();
-  Handle<TypeFeedbackMetadata> metadata =
-      TypeFeedbackMetadata::New(n.GetIsolate(), &spec);
-  Handle<TypeFeedbackVector> vector =
-      TypeFeedbackVector::New(n.GetIsolate(), metadata);
-  CallICNexus nexus(vector, slot);
-  nexus.ConfigureMegamorphic(call_count);
-  return VectorSlotPair(vector, slot);
-}
-
-}  // namespace
-
-
 Reduction JSCallReducer::Reduce(Node* node) {
   switch (node->opcode()) {
     case IrOpcode::kJSCallConstruct:
@@ -166,7 +142,7 @@ Reduction JSCallReducer::ReduceFunctionPrototypeApply(Node* node) {
   }
   // Change {node} to the new {JSCallFunction} operator.
   NodeProperties::ChangeOp(
-      node, javascript()->CallFunction(arity, CallCountFeedback(p.feedback()),
+      node, javascript()->CallFunction(arity, p.frequency(), VectorSlotPair(),
                                        convert_mode, p.tail_call_mode()));
   // Change context of {node} to the Function.prototype.apply context,
   // to ensure any exception is thrown in the correct context.
@@ -206,7 +182,7 @@ Reduction JSCallReducer::ReduceFunctionPrototypeCall(Node* node) {
     --arity;
   }
   NodeProperties::ChangeOp(
-      node, javascript()->CallFunction(arity, CallCountFeedback(p.feedback()),
+      node, javascript()->CallFunction(arity, p.frequency(), VectorSlotPair(),
                                        convert_mode, p.tail_call_mode()));
   // Try to further reduce the JSCallFunction {node}.
   Reduction const reduction = ReduceJSCallFunction(node);
@@ -287,7 +263,7 @@ Reduction JSCallReducer::ReduceJSCallFunction(Node* node) {
         arity++;
       }
       NodeProperties::ChangeOp(node, javascript()->CallFunction(
-                                         arity, CallCountFeedback(p.feedback()),
+                                         arity, p.frequency(), VectorSlotPair(),
                                          convert_mode, p.tail_call_mode()));
       // Try to further reduce the JSCallFunction {node}.
       Reduction const reduction = ReduceJSCallFunction(node);

src/compiler/js-inlining-heuristic.cc

@@ -123,18 +123,11 @@ Reduction JSInliningHeuristic::Reduce(Node* node) {
 
   // Gather feedback on how often this call site has been hit before.
   if (node->opcode() == IrOpcode::kJSCallFunction) {
-    CallFunctionParameters p = CallFunctionParametersOf(node->op());
-    if (p.feedback().IsValid()) {
-      CallICNexus nexus(p.feedback().vector(), p.feedback().slot());
-      candidate.calls = nexus.ExtractCallCount();
-    }
+    CallFunctionParameters const p = CallFunctionParametersOf(node->op());
+    candidate.frequency = p.frequency();
   } else {
-    DCHECK_EQ(IrOpcode::kJSCallConstruct, node->opcode());
-    CallConstructParameters p = CallConstructParametersOf(node->op());
-    if (p.feedback().IsValid()) {
-      CallICNexus nexus(p.feedback().vector(), p.feedback().slot());
-      candidate.calls = nexus.ExtractCallCount();
-    }
+    CallConstructParameters const p = CallConstructParametersOf(node->op());
+    candidate.frequency = p.frequency();
   }
 
   // Handling of special inlining modes right away:
@@ -278,17 +271,20 @@ Reduction JSInliningHeuristic::InlineCandidate(Candidate const& candidate) {
 
 bool JSInliningHeuristic::CandidateCompare::operator()(
     const Candidate& left, const Candidate& right) const {
-  if (left.calls != right.calls) {
-    return left.calls > right.calls;
+  if (left.frequency > right.frequency) {
+    return true;
+  } else if (left.frequency < right.frequency) {
+    return false;
+  } else {
+    return left.node->id() > right.node->id();
   }
-  return left.node->id() > right.node->id();
 }
 
 void JSInliningHeuristic::PrintCandidates() {
   PrintF("Candidates for inlining (size=%zu):\n", candidates_.size());
   for (const Candidate& candidate : candidates_) {
-    PrintF("  #%d:%s, calls:%d\n", candidate.node->id(),
-           candidate.node->op()->mnemonic(), candidate.calls);
+    PrintF("  #%d:%s, frequency:%g\n", candidate.node->id(),
+           candidate.node->op()->mnemonic(), candidate.frequency);
     for (int i = 0; i < candidate.num_functions; ++i) {
       Handle<JSFunction> function = candidate.functions[i];
       PrintF("  - size[source]:%d, size[ast]:%d, name: %s\n",

src/compiler/js-inlining-heuristic.h

@@ -37,8 +37,8 @@ class JSInliningHeuristic final : public AdvancedReducer {
   struct Candidate {
     Handle<JSFunction> functions[kMaxCallPolymorphism];
     int num_functions;
-    Node* node = nullptr;  // The call site at which to inline.
-    int calls = -1;        // Number of times the call site was hit.
+    Node* node = nullptr;    // The call site at which to inline.
+    float frequency = 0.0f;  // Relative frequency of this call site.
   };
 
   // Comparator for candidates.

src/compiler/js-intrinsic-lowering.cc

@@ -302,10 +302,10 @@ Reduction JSIntrinsicLowering::ReduceToString(Node* node) {
 
 Reduction JSIntrinsicLowering::ReduceCall(Node* node) {
   size_t const arity = CallRuntimeParametersOf(node->op()).arity();
-  NodeProperties::ChangeOp(node,
-                           javascript()->CallFunction(arity, VectorSlotPair(),
-                                                      ConvertReceiverMode::kAny,
-                                                      TailCallMode::kDisallow));
+  NodeProperties::ChangeOp(
+      node, javascript()->CallFunction(arity, 0.0f, VectorSlotPair(),
+                                       ConvertReceiverMode::kAny,
+                                       TailCallMode::kDisallow));
   return Changed(node);
 }
 

src/compiler/js-native-context-specialization.cc

@@ -849,7 +849,8 @@ JSNativeContextSpecialization::BuildPropertyAccess(
         // Introduce the call to the getter function.
         value = effect = graph()->NewNode(
             javascript()->CallFunction(
-                2, VectorSlotPair(), ConvertReceiverMode::kNotNullOrUndefined),
+                2, 0.0f, VectorSlotPair(),
+                ConvertReceiverMode::kNotNullOrUndefined),
             target, receiver, context, frame_state0, effect, control);
         control = graph()->NewNode(common()->IfSuccess(), value);
         break;
@@ -869,10 +870,11 @@ JSNativeContextSpecialization::BuildPropertyAccess(
             context, target, frame_state);
 
         // Introduce the call to the setter function.
-        effect = graph()->NewNode(
-            javascript()->CallFunction(
-                3, VectorSlotPair(), ConvertReceiverMode::kNotNullOrUndefined),
-            target, receiver, value, context, frame_state0, effect, control);
+        effect = graph()->NewNode(javascript()->CallFunction(
+                                      3, 0.0f, VectorSlotPair(),
+                                      ConvertReceiverMode::kNotNullOrUndefined),
+                                  target, receiver, value, context,
+                                  frame_state0, effect, control);
         control = graph()->NewNode(common()->IfSuccess(), effect);
         break;
       }

src/compiler/js-operator.cc

@@ -54,7 +54,8 @@ ToBooleanHints ToBooleanHintsOf(Operator const* op) {
 
 bool operator==(CallConstructParameters const& lhs,
                 CallConstructParameters const& rhs) {
-  return lhs.arity() == rhs.arity() && lhs.feedback() == rhs.feedback();
+  return lhs.arity() == rhs.arity() && lhs.frequency() == rhs.frequency() &&
+         lhs.feedback() == rhs.feedback();
 }
 
 
@@ -65,12 +66,12 @@ bool operator!=(CallConstructParameters const& lhs,
 
 
 size_t hash_value(CallConstructParameters const& p) {
-  return base::hash_combine(p.arity(), p.feedback());
+  return base::hash_combine(p.arity(), p.frequency(), p.feedback());
 }
 
 
 std::ostream& operator<<(std::ostream& os, CallConstructParameters const& p) {
-  return os << p.arity();
+  return os << p.arity() << ", " << p.frequency();
 }
 
 
@@ -81,7 +82,8 @@ CallConstructParameters const& CallConstructParametersOf(Operator const* op) {
 
 
 std::ostream& operator<<(std::ostream& os, CallFunctionParameters const& p) {
-  os << p.arity() << ", " << p.convert_mode() << ", " << p.tail_call_mode();
+  os << p.arity() << ", " << p.frequency() << ", " << p.convert_mode() << ", "
+     << p.tail_call_mode();
   return os;
 }
 
@@ -590,9 +592,9 @@ const Operator* JSOperatorBuilder::ToBoolean(ToBooleanHints hints) {
 }
 
 const Operator* JSOperatorBuilder::CallFunction(
-    size_t arity, VectorSlotPair const& feedback,
+    size_t arity, float frequency, VectorSlotPair const& feedback,
     ConvertReceiverMode convert_mode, TailCallMode tail_call_mode) {
-  CallFunctionParameters parameters(arity, feedback, tail_call_mode,
+  CallFunctionParameters parameters(arity, frequency, feedback, tail_call_mode,
                                     convert_mode);
   return new (zone()) Operator1<CallFunctionParameters>(   // --
       IrOpcode::kJSCallFunction, Operator::kNoProperties,  // opcode
@@ -626,10 +628,9 @@ const Operator* JSOperatorBuilder::CallRuntime(const Runtime::Function* f,
       parameters);                                        // parameter
 }
 
-
 const Operator* JSOperatorBuilder::CallConstruct(
-    size_t arity, VectorSlotPair const& feedback) {
-  CallConstructParameters parameters(arity, feedback);
+    uint32_t arity, float frequency, VectorSlotPair const& feedback) {
+  CallConstructParameters parameters(arity, frequency, feedback);
   return new (zone()) Operator1<CallConstructParameters>(   // --
       IrOpcode::kJSCallConstruct, Operator::kNoProperties,  // opcode
       "JSCallConstruct",                                    // name

src/compiler/js-operator.h

@@ -55,14 +55,17 @@ ToBooleanHints ToBooleanHintsOf(Operator const* op);
 // used as a parameter by JSCallConstruct operators.
 class CallConstructParameters final {
  public:
-  CallConstructParameters(size_t arity, VectorSlotPair const& feedback)
-      : arity_(arity), feedback_(feedback) {}
+  CallConstructParameters(uint32_t arity, float frequency,
+                          VectorSlotPair const& feedback)
+      : arity_(arity), frequency_(frequency), feedback_(feedback) {}
 
-  size_t arity() const { return arity_; }
+  uint32_t arity() const { return arity_; }
+  float frequency() const { return frequency_; }
   VectorSlotPair const& feedback() const { return feedback_; }
 
  private:
-  size_t const arity_;
+  uint32_t const arity_;
+  float const frequency_;
   VectorSlotPair const feedback_;
 };
 
@@ -80,15 +83,18 @@ CallConstructParameters const& CallConstructParametersOf(Operator const*);
 // used as a parameter by JSCallFunction operators.
 class CallFunctionParameters final {
  public:
-  CallFunctionParameters(size_t arity, VectorSlotPair const& feedback,
+  CallFunctionParameters(size_t arity, float frequency,
+                         VectorSlotPair const& feedback,
                          TailCallMode tail_call_mode,
                          ConvertReceiverMode convert_mode)
       : bit_field_(ArityField::encode(arity) |
                    ConvertReceiverModeField::encode(convert_mode) |
                    TailCallModeField::encode(tail_call_mode)),
+        frequency_(frequency),
         feedback_(feedback) {}
 
   size_t arity() const { return ArityField::decode(bit_field_); }
+  float frequency() const { return frequency_; }
   ConvertReceiverMode convert_mode() const {
     return ConvertReceiverModeField::decode(bit_field_);
   }
@@ -99,6 +105,7 @@ class CallFunctionParameters final {
 
   bool operator==(CallFunctionParameters const& that) const {
     return this->bit_field_ == that.bit_field_ &&
+           this->frequency_ == that.frequency_ &&
            this->feedback_ == that.feedback_;
   }
   bool operator!=(CallFunctionParameters const& that) const {
@@ -107,15 +114,16 @@ class CallFunctionParameters final {
 
  private:
   friend size_t hash_value(CallFunctionParameters const& p) {
-    return base::hash_combine(p.bit_field_, p.feedback_);
+    return base::hash_combine(p.bit_field_, p.frequency_, p.feedback_);
   }
 
   typedef BitField<size_t, 0, 29> ArityField;
   typedef BitField<ConvertReceiverMode, 29, 2> ConvertReceiverModeField;
   typedef BitField<TailCallMode, 31, 1> TailCallModeField;
 
-  const uint32_t bit_field_;
-  const VectorSlotPair feedback_;
+  uint32_t const bit_field_;
+  float const frequency_;
+  VectorSlotPair const feedback_;
 };
 
 size_t hash_value(CallFunctionParameters const&);
@@ -457,13 +465,15 @@ class JSOperatorBuilder final : public ZoneObject {
                                       int literal_flags, int literal_index);
 
   const Operator* CallFunction(
-      size_t arity, VectorSlotPair const& feedback = VectorSlotPair(),
+      size_t arity, float frequency = 0.0f,
+      VectorSlotPair const& feedback = VectorSlotPair(),
       ConvertReceiverMode convert_mode = ConvertReceiverMode::kAny,
       TailCallMode tail_call_mode = TailCallMode::kDisallow);
   const Operator* CallRuntime(Runtime::FunctionId id);
   const Operator* CallRuntime(Runtime::FunctionId id, size_t arity);
   const Operator* CallRuntime(const Runtime::Function* function, size_t arity);
-  const Operator* CallConstruct(size_t arity, VectorSlotPair const& feedback);
+  const Operator* CallConstruct(uint32_t arity, float frequency,
+                                VectorSlotPair const& feedback);
 
   const Operator* ConvertReceiver(ConvertReceiverMode convert_mode);
 

src/compiler/js-typed-lowering.cc

@@ -1754,8 +1754,8 @@ Reduction JSTypedLowering::ReduceJSCallFunction(Node* node) {
   // Maybe we did at least learn something about the {receiver}.
   if (p.convert_mode() != convert_mode) {
     NodeProperties::ChangeOp(
-        node, javascript()->CallFunction(p.arity(), p.feedback(), convert_mode,
-                                         p.tail_call_mode()));
+        node, javascript()->CallFunction(p.arity(), p.frequency(), p.feedback(),
+                                         convert_mode, p.tail_call_mode()));
     return Changed(node);
   }