[maglev] Start implenting inlining

Add a --maglev-inlining flag, and add some half-baked support for
inlining functions when there is call feedback.

When the flag is enabled and there is call feedback, we create a nested
MaglevGraphBuilder for the current graph, and pause building the graph
of the outer function. We manually set up its prologue to set up its
frame with the arguments pass into the call, build the body with the
nested graph builder. This inner builder knows that it is building an
inlined function, and all Return bytecodes will instead emit a Jump to a
single merge block at the end of the function, where execution of the
outer function can resume.

These inner function basic blocks are wired into the outer graph with
new JumpToInline and JumpFromInline control nodes. The idea is that
subsequent passes will know what the inline function is, and will use
these to manage the function stack (particularly for codegen and
especially deopts).

Bug: v8:7700
Change-Id: I4e9b153f8cf4d06c56e7be6365e7a18b86a773c0
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3585958
Commit-Queue: Leszek Swirski <leszeks@chromium.org>
Reviewed-by: Jakob Linke <jgruber@chromium.org>
Cr-Commit-Position: refs/heads/main@{#80077}

src/flags/flag-definitions.h

@@ -511,6 +511,8 @@ DEFINE_BOOL(future, FUTURE_BOOL,
 #ifdef V8_ENABLE_MAGLEV
 #define V8_ENABLE_MAGLEV_BOOL true
 DEFINE_BOOL(maglev, false, "enable the maglev optimizing compiler")
+DEFINE_BOOL(maglev_inlining, false,
+            "enable inlining in the maglev optimizing compiler")
 #else
 #define V8_ENABLE_MAGLEV_BOOL false
 DEFINE_BOOL_READONLY(maglev, false, "enable the maglev optimizing compiler")

src/maglev/maglev-code-generator.cc

@@ -4,6 +4,7 @@
 
 #include "src/maglev/maglev-code-generator.h"
 
+#include "src/base/hashmap.h"
 #include "src/codegen/code-desc.h"
 #include "src/codegen/register.h"
 #include "src/codegen/safepoint-table.h"
@@ -19,6 +20,7 @@
 #include "src/maglev/maglev-ir.h"
 #include "src/maglev/maglev-regalloc-data.h"
 #include "src/objects/code-inl.h"
+#include "src/utils/identity-map.h"
 
 namespace v8 {
 namespace internal {
@@ -318,15 +320,15 @@ class MaglevCodeGeneratorImpl final {
   }
 
  private:
-  static constexpr int kFunctionLiteralIndex = 0;
-  static constexpr int kOptimizedOutConstantIndex = 1;
+  static constexpr int kOptimizedOutConstantIndex = 0;
 
   MaglevCodeGeneratorImpl(MaglevCompilationInfo* compilation_info, Graph* graph)
       : safepoint_table_builder_(compilation_info->zone()),
         translation_array_builder_(compilation_info->zone()),
         code_gen_state_(compilation_info, safepoint_table_builder()),
         processor_(compilation_info, &code_gen_state_),
-        graph_(graph) {}
+        graph_(graph),
+        deopt_literals_(compilation_info->isolate()->heap()) {}
 
   MaybeHandle<Code> Generate() {
     EmitCode();
@@ -353,6 +355,14 @@ class MaglevCodeGeneratorImpl final {
   void EmitDeopts() {
     deopt_exit_start_offset_ = __ pc_offset();
 
+    // We'll emit the optimized out constant a bunch of times, so to avoid
+    // looking it up in the literal map every time, add it now with the fixed
+    // offset 0.
+    int optimized_out_constant_index =
+        GetDeoptLiteral(ReadOnlyRoots(isolate()).optimized_out());
+    USE(optimized_out_constant_index);
+    DCHECK_EQ(kOptimizedOutConstantIndex, optimized_out_constant_index);
+
     __ RecordComment("-- Non-lazy deopts");
     for (EagerDeoptInfo* deopt_info : code_gen_state_.eager_deopts()) {
       EmitEagerDeopt(deopt_info);
@@ -381,39 +391,51 @@ class MaglevCodeGeneratorImpl final {
     }
   }
 
-  void EmitEagerDeopt(EagerDeoptInfo* deopt_info) {
-    int frame_count = 1;
-    int jsframe_count = 1;
-    int update_feedback_count = 0;
-    deopt_info->deopt_index = translation_array_builder_.BeginTranslation(
-        frame_count, jsframe_count, update_feedback_count);
-
-    const MaglevCompilationUnit& compilation_unit =
-        *code_gen_state_.compilation_info()->toplevel_compilation_unit();
+  const InputLocation* EmitDeoptFrame(const MaglevCompilationUnit& unit,
+                                      const CheckpointedInterpreterState& state,
+                                      const InputLocation* input_locations) {
+    if (state.parent) {
+      // Deopt input locations are in the order of deopt frame emission, so
+      // update the pointer after emitting the parent frame.
+      input_locations =
+          EmitDeoptFrame(*unit.caller(), *state.parent, input_locations);
+    }
 
     // Returns are used for updating an accumulator or register after a lazy
     // deopt.
     const int return_offset = 0;
     const int return_count = 0;
     translation_array_builder_.BeginInterpretedFrame(
-        deopt_info->state.bytecode_position, kFunctionLiteralIndex,
-        compilation_unit.register_count(), return_offset, return_count);
+        state.bytecode_position,
+        GetDeoptLiteral(*unit.shared_function_info().object()),
+        unit.register_count(), return_offset, return_count);
 
-    EmitDeoptFrameValues(compilation_unit, deopt_info->state.register_frame,
-                         deopt_info->input_locations,
-                         interpreter::Register::invalid_value());
+    return EmitDeoptFrameValues(unit, state.register_frame, input_locations,
+                                interpreter::Register::invalid_value());
+  }
+
+  void EmitEagerDeopt(EagerDeoptInfo* deopt_info) {
+    int frame_count = 1 + deopt_info->unit.inlining_depth();
+    int jsframe_count = frame_count;
+    int update_feedback_count = 0;
+    deopt_info->deopt_index = translation_array_builder_.BeginTranslation(
+        frame_count, jsframe_count, update_feedback_count);
+
+    EmitDeoptFrame(deopt_info->unit, deopt_info->state,
+                   deopt_info->input_locations);
   }
 
   void EmitLazyDeopt(LazyDeoptInfo* deopt_info) {
+    const MaglevCompilationUnit& unit = deopt_info->unit;
+    DCHECK_NULL(unit.caller());
+    DCHECK_EQ(unit.inlining_depth(), 0);
+
     int frame_count = 1;
     int jsframe_count = 1;
     int update_feedback_count = 0;
     deopt_info->deopt_index = translation_array_builder_.BeginTranslation(
         frame_count, jsframe_count, update_feedback_count);
 
-    const MaglevCompilationUnit& compilation_unit =
-        *code_gen_state_.compilation_info()->toplevel_compilation_unit();
-
     // Return offsets are counted from the end of the translation frame, which
     // is the array [parameters..., locals..., accumulator].
     int return_offset;
@@ -429,20 +451,20 @@ class MaglevCodeGeneratorImpl final {
       //                  ^
       // and this calculation gives, correctly:
       //   2 + 2 - 1 = 3
-      return_offset = compilation_unit.register_count() +
-                      compilation_unit.parameter_count() -
+      return_offset = unit.register_count() + unit.parameter_count() -
                       deopt_info->result_location.ToParameterIndex();
     } else {
-      return_offset = compilation_unit.register_count() -
-                      deopt_info->result_location.index();
+      return_offset =
+          unit.register_count() - deopt_info->result_location.index();
     }
     // TODO(leszeks): Support lazy deopts with multiple return values.
     int return_count = 1;
     translation_array_builder_.BeginInterpretedFrame(
-        deopt_info->state.bytecode_position, kFunctionLiteralIndex,
-        compilation_unit.register_count(), return_offset, return_count);
+        deopt_info->state.bytecode_position,
+        GetDeoptLiteral(*unit.shared_function_info().object()),
+        unit.register_count(), return_offset, return_count);
 
-    EmitDeoptFrameValues(compilation_unit, deopt_info->state.register_frame,
+    EmitDeoptFrameValues(unit, deopt_info->state.register_frame,
                          deopt_info->input_locations,
                          deopt_info->result_location);
   }
@@ -500,15 +522,20 @@ class MaglevCodeGeneratorImpl final {
         code_gen_state_.GetFramePointerOffsetForStackSlot(operand));
   }
 
-  void EmitDeoptFrameValues(
+  const InputLocation* EmitDeoptFrameValues(
       const MaglevCompilationUnit& compilation_unit,
       const CompactInterpreterFrameState* checkpoint_state,
       const InputLocation* input_locations,
       interpreter::Register result_location) {
     // Closure
-    int closure_index = DeoptStackSlotIndexFromFPOffset(
-        StandardFrameConstants::kFunctionOffset);
-    translation_array_builder_.StoreStackSlot(closure_index);
+    if (compilation_unit.inlining_depth() == 0) {
+      int closure_index = DeoptStackSlotIndexFromFPOffset(
+          StandardFrameConstants::kFunctionOffset);
+      translation_array_builder_.StoreStackSlot(closure_index);
+    } else {
+      translation_array_builder_.StoreLiteral(
+          GetDeoptLiteral(*compilation_unit.function().object()));
+    }
 
     // TODO(leszeks): The input locations array happens to be in the same order
     // as parameters+locals+accumulator are accessed here. We should make this
@@ -573,6 +600,8 @@ class MaglevCodeGeneratorImpl final {
         translation_array_builder_.StoreLiteral(kOptimizedOutConstantIndex);
       }
     }
+
+    return input_location;
   }
 
   void EmitMetadata() {
@@ -610,6 +639,7 @@ class MaglevCodeGeneratorImpl final {
         translation_array_builder_.ToTranslationArray(isolate()->factory());
 
     data->SetTranslationByteArray(*translation_array);
+    // TODO(leszeks): Fix with the real inlined function count.
     data->SetInlinedFunctionCount(Smi::zero());
     // TODO(leszeks): Support optimization IDs
     data->SetOptimizationId(Smi::zero());
@@ -624,18 +654,17 @@ class MaglevCodeGeneratorImpl final {
                                      ->shared_function_info()
                                      .object());
 
-    // TODO(leszeks): Proper literals array.
     Handle<DeoptimizationLiteralArray> literals =
-        isolate()->factory()->NewDeoptimizationLiteralArray(2);
-    literals->set(kFunctionLiteralIndex, *code_gen_state_.compilation_info()
-                                              ->toplevel_compilation_unit()
-                                              ->shared_function_info()
-                                              .object());
-    literals->set(kOptimizedOutConstantIndex,
-                  ReadOnlyRoots(isolate()).optimized_out());
+        isolate()->factory()->NewDeoptimizationLiteralArray(
+            deopt_literals_.size());
+    IdentityMap<int, base::DefaultAllocationPolicy>::IteratableScope iterate(
+        &deopt_literals_);
+    for (auto it = iterate.begin(); it != iterate.end(); ++it) {
+      literals->set(*it.entry(), it.key());
+    }
     data->SetLiteralArray(*literals);
 
-    // TODO(leszeks): Fix once we have inlining.
+    // TODO(leszeks): Fix with the real inlining positions.
     Handle<PodArray<InliningPosition>> inlining_positions =
         PodArray<InliningPosition>::New(isolate(), 0);
     data->SetInliningPositions(*inlining_positions);
@@ -687,11 +716,21 @@ class MaglevCodeGeneratorImpl final {
     return &translation_array_builder_;
   }
 
+  int GetDeoptLiteral(Object obj) {
+    IdentityMapFindResult<int> res = deopt_literals_.FindOrInsert(obj);
+    if (!res.already_exists) {
+      DCHECK_EQ(0, *res.entry);
+      *res.entry = deopt_literals_.size() - 1;
+    }
+    return *res.entry;
+  }
+
   SafepointTableBuilder safepoint_table_builder_;
   TranslationArrayBuilder translation_array_builder_;
   MaglevCodeGenState code_gen_state_;
   GraphProcessor<MaglevCodeGeneratingNodeProcessor> processor_;
   Graph* const graph_;
+  IdentityMap<int, base::DefaultAllocationPolicy> deopt_literals_;
 
   int deopt_exit_start_offset_ = -1;
 };

src/maglev/maglev-compilation-unit.cc

@@ -13,9 +13,11 @@ namespace v8 {
 namespace internal {
 namespace maglev {
 
-MaglevCompilationUnit::MaglevCompilationUnit(MaglevCompilationInfo* info,
-                                             Handle<JSFunction> function)
+MaglevCompilationUnit::MaglevCompilationUnit(
+    MaglevCompilationInfo* info, const MaglevCompilationUnit* caller,
+    Handle<JSFunction> function)
     : info_(info),
+      caller_(caller),
       function_(MakeRef(broker(), function)),
       shared_function_info_(function_.shared()),
       bytecode_(shared_function_info_.GetBytecodeArray()),
@@ -24,7 +26,8 @@ MaglevCompilationUnit::MaglevCompilationUnit(MaglevCompilationInfo* info,
       bytecode_analysis_(bytecode_.object(), zone(), BytecodeOffset::None(),
                          true),
       register_count_(bytecode_.register_count()),
-      parameter_count_(bytecode_.parameter_count()) {}
+      parameter_count_(bytecode_.parameter_count()),
+      inlining_depth_(caller == nullptr ? 0 : caller->inlining_depth_ + 1) {}
 
 compiler::JSHeapBroker* MaglevCompilationUnit::broker() const {
   return info_->broker();

src/maglev/maglev-compilation-unit.h

@@ -24,18 +24,27 @@ class MaglevCompilationUnit : public ZoneObject {
  public:
   static MaglevCompilationUnit* New(Zone* zone, MaglevCompilationInfo* info,
                                     Handle<JSFunction> function) {
-    return zone->New<MaglevCompilationUnit>(info, function);
+    return zone->New<MaglevCompilationUnit>(info, nullptr, function);
   }
+  static MaglevCompilationUnit* NewInner(Zone* zone,
+                                         const MaglevCompilationUnit* caller,
+                                         Handle<JSFunction> function) {
+    return zone->New<MaglevCompilationUnit>(caller->info(), caller, function);
+  }
+
   MaglevCompilationUnit(MaglevCompilationInfo* info,
+                        const MaglevCompilationUnit* caller,
                         Handle<JSFunction> function);
 
   MaglevCompilationInfo* info() const { return info_; }
+  const MaglevCompilationUnit* caller() const { return caller_; }
   compiler::JSHeapBroker* broker() const;
   Isolate* isolate() const;
   LocalIsolate* local_isolate() const;
   Zone* zone() const;
   int register_count() const { return register_count_; }
   int parameter_count() const { return parameter_count_; }
+  int inlining_depth() const { return inlining_depth_; }
   bool has_graph_labeller() const;
   MaglevGraphLabeller* graph_labeller() const;
   const compiler::SharedFunctionInfoRef& shared_function_info() const {
@@ -52,6 +61,7 @@ class MaglevCompilationUnit : public ZoneObject {
 
  private:
   MaglevCompilationInfo* const info_;
+  const MaglevCompilationUnit* const caller_;
   const compiler::JSFunctionRef function_;
   const compiler::SharedFunctionInfoRef shared_function_info_;
   const compiler::BytecodeArrayRef bytecode_;
@@ -59,6 +69,7 @@ class MaglevCompilationUnit : public ZoneObject {
   const compiler::BytecodeAnalysis bytecode_analysis_;
   const int register_count_;
   const int parameter_count_;
+  const int inlining_depth_;
 };
 
 }  // namespace maglev

src/maglev/maglev-compiler.cc

@@ -108,31 +108,39 @@ class UseMarkingProcessor {
   }
 
  private:
-  void MarkCheckpointNodes(NodeBase* node, const EagerDeoptInfo* deopt_info,
-                           const ProcessingState& state) {
-    const MaglevCompilationUnit& compilation_unit =
-        *state.compilation_info()->toplevel_compilation_unit();
+  void MarkCheckpointNodes(NodeBase* node, const MaglevCompilationUnit& unit,
+                           const CheckpointedInterpreterState* checkpoint_state,
+                           InputLocation* input_locations,
+                           const ProcessingState& state, int& index) {
+    if (checkpoint_state->parent) {
+      MarkCheckpointNodes(node, *unit.caller(), checkpoint_state->parent,
+                          input_locations, state, index);
+    }
+
     const CompactInterpreterFrameState* register_frame =
-        deopt_info->state.register_frame;
+        checkpoint_state->register_frame;
     int use_id = node->id();
-    int index = 0;
 
     register_frame->ForEachValue(
-        compilation_unit, [&](ValueNode* node, interpreter::Register reg) {
-          node->mark_use(use_id, &deopt_info->input_locations[index++]);
+        unit, [&](ValueNode* node, interpreter::Register reg) {
+          node->mark_use(use_id, &input_locations[index++]);
         });
   }
+  void MarkCheckpointNodes(NodeBase* node, const EagerDeoptInfo* deopt_info,
+                           const ProcessingState& state) {
+    int index = 0;
+    MarkCheckpointNodes(node, deopt_info->unit, &deopt_info->state,
+                        deopt_info->input_locations, state, index);
+  }
   void MarkCheckpointNodes(NodeBase* node, const LazyDeoptInfo* deopt_info,
                            const ProcessingState& state) {
-    const MaglevCompilationUnit& compilation_unit =
-        *state.compilation_info()->toplevel_compilation_unit();
     const CompactInterpreterFrameState* register_frame =
         deopt_info->state.register_frame;
     int use_id = node->id();
     int index = 0;
 
     register_frame->ForEachValue(
-        compilation_unit, [&](ValueNode* node, interpreter::Register reg) {
+        deopt_info->unit, [&](ValueNode* node, interpreter::Register reg) {
           // Skip over the result location.
           if (reg == deopt_info->result_location) return;
           node->mark_use(use_id, &deopt_info->input_locations[index++]);

src/maglev/maglev-graph-builder.h

@@ -27,9 +27,28 @@ class MaglevGraphBuilder {
  public:
   explicit MaglevGraphBuilder(LocalIsolate* local_isolate,
                               MaglevCompilationUnit* compilation_unit,
-                              Graph* graph);
+                              Graph* graph,
+                              MaglevGraphBuilder* parent = nullptr);
 
   void Build() {
+    DCHECK(!is_inline());
+
+    StartPrologue();
+    for (int i = 0; i < parameter_count(); i++) {
+      SetArgument(i, AddNewNode<InitialValue>(
+                         {}, interpreter::Register::FromParameterIndex(i)));
+    }
+    BuildRegisterFrameInitialization();
+    EndPrologue();
+    BuildBody();
+  }
+
+  void StartPrologue();
+  void SetArgument(int i, ValueNode* value);
+  void BuildRegisterFrameInitialization();
+  BasicBlock* EndPrologue();
+
+  void BuildBody() {
     for (iterator_.Reset(); !iterator_.done(); iterator_.Advance()) {
       VisitSingleBytecode();
       // TODO(v8:7700): Clean up after all bytecodes are supported.
@@ -343,7 +362,13 @@ class MaglevGraphBuilder {
       latest_checkpointed_state_.emplace(
           BytecodeOffset(iterator_.current_offset()),
           zone()->New<CompactInterpreterFrameState>(
-              *compilation_unit_, GetInLiveness(), current_interpreter_frame_));
+              *compilation_unit_, GetInLiveness(), current_interpreter_frame_),
+          parent_ == nullptr
+              ? nullptr
+              // TODO(leszeks): Don't always allocate for the parent state,
+              // maybe cache it on the graph builder?
+              : zone()->New<CheckpointedInterpreterState>(
+                    parent_->GetLatestCheckpointedState()));
     }
     return *latest_checkpointed_state_;
   }
@@ -352,7 +377,9 @@ class MaglevGraphBuilder {
     return CheckpointedInterpreterState(
         BytecodeOffset(iterator_.current_offset()),
         zone()->New<CompactInterpreterFrameState>(
-            *compilation_unit_, GetOutLiveness(), current_interpreter_frame_));
+            *compilation_unit_, GetOutLiveness(), current_interpreter_frame_),
+        // TODO(leszeks): Support lazy deopts in inlined functions.
+        nullptr);
   }
 
   template <typename NodeT>
@@ -444,6 +471,10 @@ class MaglevGraphBuilder {
     return block;
   }
 
+  void InlineCallFromRegisters(int argc_count,
+                               ConvertReceiverMode receiver_mode,
+                               compiler::JSFunctionRef function);
+
   void BuildCallFromRegisterList(ConvertReceiverMode receiver_mode);
   void BuildCallFromRegisters(int argc_count,
                               ConvertReceiverMode receiver_mode);
@@ -465,6 +496,7 @@ class MaglevGraphBuilder {
   void VisitBinarySmiOperation();
 
   void MergeIntoFrameState(BasicBlock* block, int target);
+  void MergeIntoInlinedReturnFrameState(BasicBlock* block);
   void BuildBranchIfTrue(ValueNode* node, int true_target, int false_target);
   void BuildBranchIfToBooleanTrue(ValueNode* node, int true_target,
                                   int false_target);
@@ -491,11 +523,17 @@ class MaglevGraphBuilder {
       } else if (interpreter::Bytecodes::Returns(bytecode) ||
                  interpreter::Bytecodes::UnconditionallyThrows(bytecode)) {
         predecessors_[iterator.next_offset()]--;
+        // Collect inline return jumps in the slot after the last bytecode.
+        if (is_inline() && interpreter::Bytecodes::Returns(bytecode)) {
+          predecessors_[array_length - 1]++;
+        }
       }
       // TODO(leszeks): Also consider handler entries (the bytecode analysis)
       // will do this automatically I guess if we merge this into that.
     }
-    DCHECK_EQ(0, predecessors_[bytecode().length()]);
+    if (!is_inline()) {
+      DCHECK_EQ(0, predecessors_[bytecode().length()]);
+    }
   }
 
   int NumPredecessors(int offset) { return predecessors_[offset]; }
@@ -530,8 +568,19 @@ class MaglevGraphBuilder {
     return compilation_unit_->graph_labeller();
   }
 
+  // True when this graph builder is building the subgraph of an inlined
+  // function.
+  bool is_inline() const { return parent_ != nullptr; }
+
+  // The fake offset used as a target for all exits of an inlined function.
+  int inline_exit_offset() const {
+    DCHECK(is_inline());
+    return bytecode().length();
+  }
+
   LocalIsolate* const local_isolate_;
   MaglevCompilationUnit* const compilation_unit_;
+  MaglevGraphBuilder* const parent_;
   Graph* const graph_;
   interpreter::BytecodeArrayIterator iterator_;
   uint32_t* predecessors_;

src/maglev/maglev-graph-printer.cc

@@ -314,10 +314,8 @@ void PrintEagerDeopt(std::ostream& os, std::vector<BasicBlock*> targets,
   os << "  ↱ eager @" << deopt_info->state.bytecode_position << " : {";
   bool first = true;
   int index = 0;
-  const MaglevCompilationUnit& compilation_unit =
-      *state.compilation_info()->toplevel_compilation_unit();
   deopt_info->state.register_frame->ForEachValue(
-      compilation_unit, [&](ValueNode* node, interpreter::Register reg) {
+      deopt_info->unit, [&](ValueNode* node, interpreter::Register reg) {
         if (first) {
           first = false;
         } else {
@@ -355,10 +353,8 @@ void PrintLazyDeopt(std::ostream& os, std::vector<BasicBlock*> targets,
   os << "  ↳ lazy @" << deopt_info->state.bytecode_position << " : {";
   bool first = true;
   int index = 0;
-  const MaglevCompilationUnit& compilation_unit =
-      *state.compilation_info()->toplevel_compilation_unit();
   deopt_info->state.register_frame->ForEachValue(
-      compilation_unit, [&](ValueNode* node, interpreter::Register reg) {
+      deopt_info->unit, [&](ValueNode* node, interpreter::Register reg) {
         if (first) {
           first = false;
         } else {

src/maglev/maglev-graph-verifier.h

@@ -70,6 +70,8 @@ class MaglevGraphVerifier {
       case Opcode::kDeopt:
       case Opcode::kJump:
       case Opcode::kJumpLoop:
+      case Opcode::kJumpToInlined:
+      case Opcode::kJumpFromInlined:
         // No input.
         DCHECK_EQ(node->input_count(), 0);
         break;

src/maglev/maglev-ir.cc

@@ -336,15 +336,32 @@ void NodeBase::Print(std::ostream& os,
   UNREACHABLE();
 }
 
+namespace {
+size_t GetInputLocationsArraySize(const MaglevCompilationUnit& compilation_unit,
+                                  const CheckpointedInterpreterState& state) {
+  size_t size = state.register_frame->size(compilation_unit);
+  const CheckpointedInterpreterState* parent = state.parent;
+  const MaglevCompilationUnit* parent_unit = compilation_unit.caller();
+  while (parent != nullptr) {
+    size += parent->register_frame->size(*parent_unit);
+    parent = parent->parent;
+    parent_unit = parent_unit->caller();
+  }
+  return size;
+}
+}  // namespace
+
 DeoptInfo::DeoptInfo(Zone* zone, const MaglevCompilationUnit& compilation_unit,
                      CheckpointedInterpreterState state)
-    : state(state),
+    : unit(compilation_unit),
+      state(state),
       input_locations(zone->NewArray<InputLocation>(
-          state.register_frame->size(compilation_unit))) {
+          GetInputLocationsArraySize(compilation_unit, state))) {
   // Default initialise if we're printing the graph, to avoid printing junk
   // values.
   if (FLAG_print_maglev_graph) {
-    for (size_t i = 0; i < state.register_frame->size(compilation_unit); ++i) {
+    for (size_t i = 0; i < GetInputLocationsArraySize(compilation_unit, state);
+         ++i) {
       new (&input_locations[i]) InputLocation();
     }
   }
@@ -894,6 +911,30 @@ void Jump::GenerateCode(MaglevCodeGenState* code_gen_state,
   }
 }
 
+void JumpToInlined::AllocateVreg(MaglevVregAllocationState* vreg_state,
+                                 const ProcessingState& state) {}
+void JumpToInlined::GenerateCode(MaglevCodeGenState* code_gen_state,
+                                 const ProcessingState& state) {
+  // Avoid emitting a jump to the next block.
+  if (target() != state.next_block()) {
+    __ jmp(target()->label());
+  }
+}
+void JumpToInlined::PrintParams(std::ostream& os,
+                                MaglevGraphLabeller* graph_labeller) const {
+  os << "(" << Brief(*unit()->shared_function_info().object()) << ")";
+}
+
+void JumpFromInlined::AllocateVreg(MaglevVregAllocationState* vreg_state,
+                                   const ProcessingState& state) {}
+void JumpFromInlined::GenerateCode(MaglevCodeGenState* code_gen_state,
+                                   const ProcessingState& state) {
+  // Avoid emitting a jump to the next block.
+  if (target() != state.next_block()) {
+    __ jmp(target()->label());
+  }
+}
+
 void JumpLoop::AllocateVreg(MaglevVregAllocationState* vreg_state,
                             const ProcessingState& state) {}
 void JumpLoop::GenerateCode(MaglevCodeGenState* code_gen_state,

src/maglev/maglev-ir.h

@@ -94,7 +94,9 @@ class CompactInterpreterFrameState;
 
 #define UNCONDITIONAL_CONTROL_NODE_LIST(V) \
   V(Jump)                                  \
-  V(JumpLoop)
+  V(JumpLoop)                              \
+  V(JumpToInlined)                         \
+  V(JumpFromInlined)
 
 #define CONTROL_NODE_LIST(V)       \
   V(Return)                        \
@@ -328,11 +330,15 @@ class CheckpointedInterpreterState {
  public:
   CheckpointedInterpreterState() = default;
   CheckpointedInterpreterState(BytecodeOffset bytecode_position,
-                               const CompactInterpreterFrameState* state)
-      : bytecode_position(bytecode_position), register_frame(state) {}
+                               const CompactInterpreterFrameState* state,
+                               const CheckpointedInterpreterState* parent)
+      : bytecode_position(bytecode_position),
+        register_frame(state),
+        parent(parent) {}
 
   BytecodeOffset bytecode_position = BytecodeOffset::None();
   const CompactInterpreterFrameState* register_frame = nullptr;
+  const CheckpointedInterpreterState* parent = nullptr;
 };
 
 class DeoptInfo {
@@ -341,6 +347,7 @@ class DeoptInfo {
             CheckpointedInterpreterState checkpoint);
 
  public:
+  const MaglevCompilationUnit& unit;
   CheckpointedInterpreterState state;
   InputLocation* input_locations = nullptr;
   Label deopt_entry_label;
@@ -1464,9 +1471,9 @@ class ControlNode : public NodeBase {
     return next_post_dominating_hole_;
   }
   void set_next_post_dominating_hole(ControlNode* node) {
-    DCHECK_IMPLIES(node != nullptr, node->Is<Jump>() || node->Is<Return>() ||
-                                        node->Is<Deopt>() ||
-                                        node->Is<JumpLoop>());
+    DCHECK_IMPLIES(node != nullptr, node->Is<UnconditionalControlNode>() ||
+                                        node->Is<Return>() ||
+                                        node->Is<Deopt>());
     next_post_dominating_hole_ = node;
   }
 
@@ -1590,6 +1597,36 @@ class JumpLoop : public UnconditionalControlNodeT<JumpLoop> {
   void PrintParams(std::ostream&, MaglevGraphLabeller*) const {}
 };
 
+class JumpToInlined : public UnconditionalControlNodeT<JumpToInlined> {
+  using Base = UnconditionalControlNodeT<JumpToInlined>;
+
+ public:
+  explicit JumpToInlined(uint32_t bitfield, BasicBlockRef* target_refs,
+                         MaglevCompilationUnit* unit)
+      : Base(bitfield, target_refs), unit_(unit) {}
+
+  void AllocateVreg(MaglevVregAllocationState*, const ProcessingState&);
+  void GenerateCode(MaglevCodeGenState*, const ProcessingState&);
+  void PrintParams(std::ostream&, MaglevGraphLabeller*) const;
+
+  const MaglevCompilationUnit* unit() const { return unit_; }
+
+ private:
+  MaglevCompilationUnit* unit_;
+};
+
+class JumpFromInlined : public UnconditionalControlNodeT<JumpFromInlined> {
+  using Base = UnconditionalControlNodeT<JumpFromInlined>;
+
+ public:
+  explicit JumpFromInlined(uint32_t bitfield, BasicBlockRef* target_refs)
+      : Base(bitfield, target_refs) {}
+
+  void AllocateVreg(MaglevVregAllocationState*, const ProcessingState&);
+  void GenerateCode(MaglevCodeGenState*, const ProcessingState&);
+  void PrintParams(std::ostream&, MaglevGraphLabeller*) const {}
+};
+
 class Return : public ControlNode {
  public:
   explicit Return(uint32_t bitfield) : ControlNode(bitfield) {

src/maglev/maglev-regalloc.cc

@@ -339,28 +339,18 @@ void StraightForwardRegisterAllocator::UpdateUse(
 
 void StraightForwardRegisterAllocator::UpdateUse(
     const EagerDeoptInfo& deopt_info) {
-  const CompactInterpreterFrameState* checkpoint_state =
-      deopt_info.state.register_frame;
-  const MaglevCompilationUnit& compilation_unit =
-      *compilation_info_->toplevel_compilation_unit();
   int index = 0;
-  checkpoint_state->ForEachValue(
-      compilation_unit, [&](ValueNode* node, interpreter::Register reg) {
-        InputLocation* input = &deopt_info.input_locations[index++];
-        input->InjectAllocated(node->allocation());
-        UpdateUse(node, input);
-      });
+  UpdateUse(deopt_info.unit, &deopt_info.state, deopt_info.input_locations,
+            index);
 }
 
 void StraightForwardRegisterAllocator::UpdateUse(
     const LazyDeoptInfo& deopt_info) {
   const CompactInterpreterFrameState* checkpoint_state =
       deopt_info.state.register_frame;
-  const MaglevCompilationUnit& compilation_unit =
-      *compilation_info_->toplevel_compilation_unit();
   int index = 0;
   checkpoint_state->ForEachValue(
-      compilation_unit, [&](ValueNode* node, interpreter::Register reg) {
+      deopt_info.unit, [&](ValueNode* node, interpreter::Register reg) {
         // Skip over the result location.
         if (reg == deopt_info.result_location) return;
         InputLocation* input = &deopt_info.input_locations[index++];
@@ -369,6 +359,22 @@ void StraightForwardRegisterAllocator::UpdateUse(
       });
 }
 
+void StraightForwardRegisterAllocator::UpdateUse(
+    const MaglevCompilationUnit& unit,
+    const CheckpointedInterpreterState* state, InputLocation* input_locations,
+    int& index) {
+  if (state->parent) {
+    UpdateUse(*unit.caller(), state->parent, input_locations, index);
+  }
+  const CompactInterpreterFrameState* checkpoint_state = state->register_frame;
+  checkpoint_state->ForEachValue(
+      unit, [&](ValueNode* node, interpreter::Register reg) {
+        InputLocation* input = &input_locations[index++];
+        input->InjectAllocated(node->allocation());
+        UpdateUse(node, input);
+      });
+}
+
 void StraightForwardRegisterAllocator::AllocateNode(Node* node) {
   for (Input& input : *node) AssignInput(input);
   AssignTemporaries(node);
@@ -546,7 +552,10 @@ void StraightForwardRegisterAllocator::AllocateControlNode(ControlNode* node,
 
   // Merge register values. Values only flowing into phis and not being
   // independently live will be killed as part of the merge.
-  if (auto unconditional = node->TryCast<UnconditionalControlNode>()) {
+  if (node->Is<JumpToInlined>()) {
+    // Do nothing.
+    // TODO(leszeks): DCHECK any useful invariants here.
+  } else if (auto unconditional = node->TryCast<UnconditionalControlNode>()) {
     // Empty blocks are immediately merged at the control of their predecessor.
     if (!block->is_empty_block()) {
       MergeRegisterValues(unconditional, unconditional->target(),

src/maglev/maglev-regalloc.h

@@ -62,6 +62,9 @@ class StraightForwardRegisterAllocator {
   void UpdateUse(ValueNode* node, InputLocation* input_location);
   void UpdateUse(const EagerDeoptInfo& deopt_info);
   void UpdateUse(const LazyDeoptInfo& deopt_info);
+  void UpdateUse(const MaglevCompilationUnit& unit,
+                 const CheckpointedInterpreterState* state,
+                 InputLocation* input_locations, int& index);
 
   void AllocateControlNode(ControlNode* node, BasicBlock* block);
   void AllocateNode(Node* node);

test/mjsunit/maglev/eager-deopt-in-inline.js

+// Copyright 2022 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --allow-natives-syntax --maglev --maglev-inlining --no-stress-opt
+
+function inner(o) {
+  "use strict"
+  return 10 + o.x + 100;
+}
+
+function foo(o) {
+  return 1000 + inner(o) + 10000;
+}
+
+%PrepareFunctionForOptimization(inner);
+%PrepareFunctionForOptimization(foo);
+assertEquals(11111, foo({x:1}));
+assertEquals(11111, foo({x:1}));
+
+%OptimizeMaglevOnNextCall(foo);
+// The inlined inner function will deopt -- this deopt should succeed.
+assertEquals(11111, foo({y:2,x:1}));

test/mjsunit/maglev/simple-inlining.js

+// Copyright 2022 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --allow-natives-syntax --maglev --maglev-inlining --no-stress-opt
+
+function global_func(x) {
+  return x;
+}
+
+function foo(x) {
+  return global_func(x);
+}
+
+%PrepareFunctionForOptimization(foo);
+%PrepareFunctionForOptimization(global_func);
+print(foo(1));
+print(foo(1));
+%OptimizeMaglevOnNextCall(foo);
+print(foo(1));