[turbofan] Various cleanups/fixes in inlining heuristics

- Always account for small functions.
- Always check against the hard limit.
- Rename some things for clarity.

Change-Id: Iad98ee625d4385dfab02fb7d5e0cb2c25eb5d67a
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1686664Reviewed-by: Mythri Alle <mythria@chromium.org>
Reviewed-by: Maya Lekova <mslekova@chromium.org>
Commit-Queue: Georg Neis <neis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#62537}

src/compiler/js-inlining-heuristic.cc

@@ -21,15 +21,9 @@ namespace compiler {
   } while (false)
 
 namespace {
-
-bool IsSmallInlineFunction(BytecodeArrayRef bytecode) {
-  // Forcibly inline small functions.
-  if (bytecode.length() <= FLAG_max_inlined_bytecode_size_small) {
-    return true;
-  }
-  return false;
+bool IsSmall(BytecodeArrayRef bytecode) {
+  return bytecode.length() <= FLAG_max_inlined_bytecode_size_small;
 }
-
 }  // namespace
 
 JSInliningHeuristic::Candidate JSInliningHeuristic::CollectFunctions(
@@ -91,6 +85,11 @@ Reduction JSInliningHeuristic::Reduce(Node* node) {
 
   if (!IrOpcode::IsInlineeOpcode(node->opcode())) return NoChange();
 
+  if (total_inlined_bytecode_size_ >= FLAG_max_inlined_bytecode_size_absolute &&
+      mode_ != kStressInlining) {
+    return NoChange();
+  }
+
   // Check if we already saw that {node} before, and if so, just skip it.
   if (seen_.find(node->id()) != seen_.end()) return NoChange();
   seen_.insert(node->id());
@@ -107,7 +106,7 @@ Reduction JSInliningHeuristic::Reduce(Node* node) {
     return NoChange();
   }
 
-  bool can_inline = false, force_inline_small = true;
+  bool can_inline_candidate = false, candidate_is_small = true;
   candidate.total_size = 0;
   Node* frame_state = NodeProperties::GetFrameStateInput(node);
   FrameStateInfo const& frame_info = FrameStateInfoOf(frame_state->op());
@@ -155,15 +154,12 @@ Reduction JSInliningHeuristic::Reduce(Node* node) {
     // serialized.
     BytecodeArrayRef bytecode = candidate.bytecode[i].value();
     if (candidate.can_inline_function[i]) {
-      can_inline = true;
+      can_inline_candidate = true;
       candidate.total_size += bytecode.length();
     }
-    // We don't force inline small functions if any of them is not inlineable.
-    if (!IsSmallInlineFunction(bytecode)) {
-      force_inline_small = false;
-    }
+    candidate_is_small = candidate_is_small && IsSmall(bytecode);
   }
-  if (!can_inline) return NoChange();
+  if (!can_inline_candidate) return NoChange();
 
   // Gather feedback on how often this call site has been hit before.
   if (node->opcode() == IrOpcode::kJSCall) {
@@ -195,9 +191,8 @@ Reduction JSInliningHeuristic::Reduce(Node* node) {
   }
 
   // Forcibly inline small functions here. In the case of polymorphic inlining
-  // force_inline_small is set only when all functions are small.
-  if (force_inline_small &&
-      cumulative_count_ < FLAG_max_inlined_bytecode_size_absolute) {
+  // candidate_is_small is set only when all functions are small.
+  if (candidate_is_small) {
     TRACE("Inlining small function(s) at call site #%d:%s\n", node->id(),
           node->op()->mnemonic());
     return InlineCandidate(candidate, true);
@@ -230,7 +225,8 @@ void JSInliningHeuristic::Finalize() {
     // exposed by this function would be given a chance to inline.
     double size_of_candidate =
         candidate.total_size * FLAG_reserve_inline_budget_scale_factor;
-    int total_size = cumulative_count_ + static_cast<int>(size_of_candidate);
+    int total_size =
+        total_inlined_bytecode_size_ + static_cast<int>(size_of_candidate);
     if (total_size > FLAG_max_inlined_bytecode_size_cumulative) {
       // Try if any smaller functions are available to inline.
       continue;
@@ -632,7 +628,7 @@ Reduction JSInliningHeuristic::InlineCandidate(Candidate const& candidate,
   if (num_calls == 1) {
     Reduction const reduction = inliner_.ReduceJSCall(node);
     if (reduction.Changed()) {
-      cumulative_count_ += candidate.bytecode[0].value().length();
+      total_inlined_bytecode_size_ += candidate.bytecode[0].value().length();
     }
     return reduction;
   }
@@ -690,20 +686,19 @@ Reduction JSInliningHeuristic::InlineCandidate(Candidate const& candidate,
   ReplaceWithValue(node, value, effect, control);
 
   // Inline the individual, cloned call sites.
-  for (int i = 0; i < num_calls; ++i) {
-    Node* node = calls[i];
+  for (int i = 0; i < num_calls && total_inlined_bytecode_size_ <
+                                       FLAG_max_inlined_bytecode_size_absolute;
+       ++i) {
     if (candidate.can_inline_function[i] &&
-        (small_function ||
-         cumulative_count_ < FLAG_max_inlined_bytecode_size_cumulative)) {
+        (small_function || total_inlined_bytecode_size_ <
+                               FLAG_max_inlined_bytecode_size_cumulative)) {
+      Node* node = calls[i];
       Reduction const reduction = inliner_.ReduceJSCall(node);
       if (reduction.Changed()) {
+        total_inlined_bytecode_size_ += candidate.bytecode[i]->length();
         // Killing the call node is not strictly necessary, but it is safer to
         // make sure we do not resurrect the node.
         node->Kill();
-        // Small functions don't count towards the budget.
-        if (!small_function) {
-          cumulative_count_ += candidate.bytecode[i]->length();
-        }
       }
     }
   }

src/compiler/js-inlining-heuristic.h

@@ -97,7 +97,7 @@ class JSInliningHeuristic final : public AdvancedReducer {
   SourcePositionTable* source_positions_;
   JSGraph* const jsgraph_;
   JSHeapBroker* const broker_;
-  int cumulative_count_ = 0;
+  int total_inlined_bytecode_size_ = 0;
 };
 
 }  // namespace compiler