[maglev] Make regalloc aware of lazy deopts

Port the eager deopt handling in the use marker and register allocator
to do the same thing with lazy deopts. This requires moving the lazy
deopt info to be a pseudo-input before the node, same as eager deopt
info, so that the regalloc can read it without needing the Node's
opcode.

For now, this means that a node cannot both eager- and lazy-deopt; if we
need this in the future we can rethink it.

Bug: v8:7700
Change-Id: I96292af9c483f285b1e45bfb374c8dc600fa6347
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3568452Reviewed-by: Victor Gomes <victorgomes@chromium.org>
Commit-Queue: Leszek Swirski <leszeks@chromium.org>
Cr-Commit-Position: refs/heads/main@{#79757}

src/maglev/maglev-compiler.cc

@@ -72,6 +72,9 @@ class UseMarkingProcessor {
     for (Input& input : *node) {
       input.node()->mark_use(node->id(), &input);
     }
+    if constexpr (NodeT::kProperties.can_lazy_deopt()) {
+      MarkCheckpointNodes(node, node->lazy_deopt_info(), state);
+    }
   }
 
   void Process(Phi* node, const ProcessingState& state) {
@@ -104,18 +107,32 @@ class UseMarkingProcessor {
   }
 
  private:
-  void MarkCheckpointNodes(NodeBase* node,
-                           const EagerDeoptInfo* eager_deopt_info,
+  void MarkCheckpointNodes(NodeBase* node, const EagerDeoptInfo* deopt_info,
+                           const ProcessingState& state) {
+    const CompactInterpreterFrameState* register_frame =
+        deopt_info->state.register_frame;
+    int use_id = node->id();
+    int index = 0;
+
+    register_frame->ForEachValue(
+        *state.compilation_unit(),
+        [&](ValueNode* node, interpreter::Register reg) {
+          node->mark_use(use_id, &deopt_info->input_locations[index++]);
+        });
+  }
+  void MarkCheckpointNodes(NodeBase* node, const LazyDeoptInfo* deopt_info,
                            const ProcessingState& state) {
     const CompactInterpreterFrameState* register_frame =
-        eager_deopt_info->state.register_frame;
+        deopt_info->state.register_frame;
     int use_id = node->id();
     int index = 0;
 
     register_frame->ForEachValue(
         *state.compilation_unit(),
         [&](ValueNode* node, interpreter::Register reg) {
-          node->mark_use(use_id, &eager_deopt_info->input_locations[index++]);
+          // Skip over the result location.
+          if (reg == deopt_info->result_location) return;
+          node->mark_use(use_id, &deopt_info->input_locations[index++]);
         });
   }
 };

src/maglev/maglev-graph-builder.h

@@ -152,6 +152,10 @@ class MaglevGraphBuilder {
       return Node::New<NodeT>(zone(), *compilation_unit_,
                               GetLatestCheckpointedState(),
                               std::forward<Args>(args)...);
+    } else if constexpr (NodeT::kProperties.can_lazy_deopt()) {
+      return Node::New<NodeT>(zone(), *compilation_unit_,
+                              GetCheckpointedStateForLazyDeopt(),
+                              std::forward<Args>(args)...);
     } else {
       return Node::New<NodeT>(zone(), std::forward<Args>(args)...);
     }
@@ -186,8 +190,9 @@ class MaglevGraphBuilder {
     DCHECK_EQ(NodeT::kProperties.can_lazy_deopt(),
               node->properties().can_lazy_deopt());
     if constexpr (NodeT::kProperties.can_lazy_deopt()) {
-      node->AttachLazyDeopt(
-          GetLazyDeoptInfo(interpreter::Register::virtual_accumulator()));
+      DCHECK(!node->lazy_deopt_info()->result_location.is_valid());
+      node->lazy_deopt_info()->result_location =
+          interpreter::Register::virtual_accumulator();
     }
     SetAccumulatorToExistingNode(node);
   }
@@ -235,14 +240,11 @@ class MaglevGraphBuilder {
     return *latest_checkpointed_state_;
   }
 
-  LazyDeoptInfo GetLazyDeoptInfo(interpreter::Register result_location) {
-    return LazyDeoptInfo(
-        zone(), *compilation_unit_,
-        CheckpointedInterpreterState(BytecodeOffset(iterator_.current_offset()),
-                                     zone()->New<CompactInterpreterFrameState>(
-                                         *compilation_unit_, GetOutLiveness(),
-                                         current_interpreter_frame_)),
-        result_location);
+  CheckpointedInterpreterState GetCheckpointedStateForLazyDeopt() {
+    return CheckpointedInterpreterState(
+        BytecodeOffset(iterator_.current_offset()),
+        zone()->New<CompactInterpreterFrameState>(
+            *compilation_unit_, GetOutLiveness(), current_interpreter_frame_));
   }
 
   void MarkPossibleSideEffect() {

src/maglev/maglev-graph-printer.cc

@@ -348,7 +348,7 @@ void PrintLazyDeopt(std::ostream& os, std::vector<BasicBlock*> targets,
   PrintVerticalArrows(os, targets);
   PrintPadding(os, graph_labeller, 0);
 
-  LazyDeoptInfo* deopt_info = node->lazy_deopt();
+  LazyDeoptInfo* deopt_info = node->lazy_deopt_info();
   os << "  ↳ lazy @" << deopt_info->state.bytecode_position << " : {";
   bool first = true;
   deopt_info->state.register_frame->ForEachValue(

src/maglev/maglev-ir.cc

@@ -766,8 +766,8 @@ void Call::GenerateCode(MaglevCodeGenState* code_gen_state,
       break;
   }
 
-  lazy_deopt()->deopting_call_return_pc = __ pc_offset_for_safepoint();
-  code_gen_state->PushLazyDeopt(lazy_deopt());
+  lazy_deopt_info()->deopting_call_return_pc = __ pc_offset_for_safepoint();
+  code_gen_state->PushLazyDeopt(lazy_deopt_info());
 
   SafepointTableBuilder::Safepoint safepoint =
       code_gen_state->safepoint_table_builder()->DefineSafepoint(

src/maglev/maglev-ir.h

@@ -315,7 +315,6 @@ class CheckpointedInterpreterState {
 
 class DeoptInfo {
  protected:
-  DeoptInfo() = default;
   DeoptInfo(Zone* zone, const MaglevCompilationUnit& compilation_unit,
             CheckpointedInterpreterState checkpoint);
 
@@ -331,21 +330,17 @@ class EagerDeoptInfo : public DeoptInfo {
   EagerDeoptInfo(Zone* zone, const MaglevCompilationUnit& compilation_unit,
                  CheckpointedInterpreterState checkpoint)
       : DeoptInfo(zone, compilation_unit, checkpoint) {}
-
-  EagerDeoptInfo() = delete;
 };
 
 class LazyDeoptInfo : public DeoptInfo {
  public:
-  LazyDeoptInfo() = default;
   LazyDeoptInfo(Zone* zone, const MaglevCompilationUnit& compilation_unit,
-                CheckpointedInterpreterState checkpoint,
-                interpreter::Register result_location)
-      : DeoptInfo(zone, compilation_unit, checkpoint),
-        result_location(result_location) {}
+                CheckpointedInterpreterState checkpoint)
+      : DeoptInfo(zone, compilation_unit, checkpoint) {}
 
   int deopting_call_return_pc = -1;
-  interpreter::Register result_location;
+  interpreter::Register result_location =
+      interpreter::Register::invalid_value();
 };
 
 // Dummy type for the initial raw allocation.
@@ -366,21 +361,6 @@ struct opcode_of_helper;
 NODE_BASE_LIST(DEF_OPCODE_OF)
 #undef DEF_OPCODE_OF
 
-class LazyDeoptMixin {
- public:
-  LazyDeoptInfo* lazy_deopt() {
-    DCHECK_NOT_NULL(lazy_deopt_.state.register_frame);
-    return &lazy_deopt_;
-  }
-  void AttachLazyDeopt(LazyDeoptInfo&& lazy_deopt) {
-    DCHECK_NULL(lazy_deopt_.state.register_frame);
-    lazy_deopt_ = std::move(lazy_deopt);
-  }
-
- private:
-  LazyDeoptInfo lazy_deopt_;
-};
-
 }  // namespace detail
 
 class NodeBase : public ZoneObject {
@@ -420,8 +400,14 @@ class NodeBase : public ZoneObject {
   static Derived* New(Zone* zone, const MaglevCompilationUnit& compilation_unit,
                       CheckpointedInterpreterState checkpoint, Args&&... args) {
     Derived* node = New<Derived>(zone, std::forward<Args>(args)...);
-    new (node->eager_deopt_info_address())
-        EagerDeoptInfo(zone, compilation_unit, checkpoint);
+    if constexpr (Derived::kProperties.can_eager_deopt()) {
+      new (node->eager_deopt_info_address())
+          EagerDeoptInfo(zone, compilation_unit, checkpoint);
+    } else {
+      STATIC_ASSERT(Derived::kProperties.can_lazy_deopt());
+      new (node->lazy_deopt_info_address())
+          LazyDeoptInfo(zone, compilation_unit, checkpoint);
+    }
     return node;
   }
 
@@ -517,6 +503,7 @@ class NodeBase : public ZoneObject {
 
   EagerDeoptInfo* eager_deopt_info() {
     DCHECK(properties().can_eager_deopt());
+    DCHECK(!properties().can_lazy_deopt());
     return (
         reinterpret_cast<EagerDeoptInfo*>(input_address(input_count() - 1)) -
         1);
@@ -524,11 +511,27 @@ class NodeBase : public ZoneObject {
 
   const EagerDeoptInfo* eager_deopt_info() const {
     DCHECK(properties().can_eager_deopt());
+    DCHECK(!properties().can_lazy_deopt());
     return (reinterpret_cast<const EagerDeoptInfo*>(
                 input_address(input_count() - 1)) -
             1);
   }
 
+  LazyDeoptInfo* lazy_deopt_info() {
+    DCHECK(properties().can_lazy_deopt());
+    DCHECK(!properties().can_eager_deopt());
+    return (reinterpret_cast<LazyDeoptInfo*>(input_address(input_count() - 1)) -
+            1);
+  }
+
+  const LazyDeoptInfo* lazy_deopt_info() const {
+    DCHECK(properties().can_lazy_deopt());
+    DCHECK(!properties().can_eager_deopt());
+    return (reinterpret_cast<const LazyDeoptInfo*>(
+                input_address(input_count() - 1)) -
+            1);
+  }
+
  protected:
   explicit NodeBase(uint32_t bitfield) : bit_field_(bitfield) {}
 
@@ -556,16 +559,32 @@ class NodeBase : public ZoneObject {
 
   EagerDeoptInfo* eager_deopt_info_address() {
     DCHECK(properties().can_eager_deopt());
+    DCHECK(!properties().can_lazy_deopt());
     return reinterpret_cast<EagerDeoptInfo*>(input_address(input_count() - 1)) -
            1;
   }
 
+  LazyDeoptInfo* lazy_deopt_info_address() {
+    DCHECK(!properties().can_eager_deopt());
+    DCHECK(properties().can_lazy_deopt());
+    return reinterpret_cast<LazyDeoptInfo*>(input_address(input_count() - 1)) -
+           1;
+  }
+
  private:
   template <class Derived, typename... Args>
   static Derived* Allocate(Zone* zone, size_t input_count, Args&&... args) {
+    static_assert(
+        !Derived::kProperties.can_eager_deopt() ||
+            !Derived::kProperties.can_lazy_deopt(),
+        "The current deopt info representation, at the end of inputs, requires "
+        "that we cannot have both lazy and eager deopts on a node. If we ever "
+        "need this, we have to update accessors to check node->properties() "
+        "for which deopts are active.");
     const size_t size_before_node =
         input_count * sizeof(Input) +
-        (Derived::kProperties.can_eager_deopt() ? sizeof(EagerDeoptInfo) : 0);
+        (Derived::kProperties.can_eager_deopt() ? sizeof(EagerDeoptInfo) : 0) +
+        (Derived::kProperties.can_lazy_deopt() ? sizeof(LazyDeoptInfo) : 0);
     const size_t size = size_before_node + sizeof(Derived);
     intptr_t raw_buffer =
         reinterpret_cast<intptr_t>(zone->Allocate<NodeWithInlineInputs>(size));
@@ -828,8 +847,7 @@ class FixedInputValueNodeT : public ValueNodeT<Derived> {
 };
 
 template <class Derived, Operation kOperation>
-class UnaryWithFeedbackNode : public FixedInputValueNodeT<1, Derived>,
-                              public detail::LazyDeoptMixin {
+class UnaryWithFeedbackNode : public FixedInputValueNodeT<1, Derived> {
   using Base = FixedInputValueNodeT<1, Derived>;
 
  public:
@@ -853,8 +871,7 @@ class UnaryWithFeedbackNode : public FixedInputValueNodeT<1, Derived>,
 };
 
 template <class Derived, Operation kOperation>
-class BinaryWithFeedbackNode : public FixedInputValueNodeT<2, Derived>,
-                               public detail::LazyDeoptMixin {
+class BinaryWithFeedbackNode : public FixedInputValueNodeT<2, Derived> {
   using Base = FixedInputValueNodeT<2, Derived>;
 
  public:
@@ -1069,8 +1086,7 @@ class StoreField : public FixedInputNodeT<2, StoreField> {
   const int handler_;
 };
 
-class LoadGlobal : public FixedInputValueNodeT<1, LoadGlobal>,
-                   public detail::LazyDeoptMixin {
+class LoadGlobal : public FixedInputValueNodeT<1, LoadGlobal> {
   using Base = FixedInputValueNodeT<1, LoadGlobal>;
 
  public:
@@ -1091,8 +1107,7 @@ class LoadGlobal : public FixedInputValueNodeT<1, LoadGlobal>,
   const compiler::NameRef name_;
 };
 
-class LoadNamedGeneric : public FixedInputValueNodeT<2, LoadNamedGeneric>,
-                         public detail::LazyDeoptMixin {
+class LoadNamedGeneric : public FixedInputValueNodeT<2, LoadNamedGeneric> {
   using Base = FixedInputValueNodeT<2, LoadNamedGeneric>;
 
  public:
@@ -1170,7 +1185,7 @@ class Phi : public ValueNodeT<Phi> {
   friend base::ThreadedListTraits<Phi>;
 };
 
-class Call : public ValueNodeT<Call>, public detail::LazyDeoptMixin {
+class Call : public ValueNodeT<Call> {
   using Base = ValueNodeT<Call>;
 
  public:

src/maglev/maglev-regalloc.cc

@@ -317,13 +317,28 @@ void StraightForwardRegisterAllocator::UpdateUse(
 }
 
 void StraightForwardRegisterAllocator::UpdateUse(
-    const EagerDeoptInfo& eager_deopt_info) {
+    const EagerDeoptInfo& deopt_info) {
   const CompactInterpreterFrameState* checkpoint_state =
-      eager_deopt_info.state.register_frame;
+      deopt_info.state.register_frame;
   int index = 0;
   checkpoint_state->ForEachValue(
       *compilation_unit_, [&](ValueNode* node, interpreter::Register reg) {
-        InputLocation* input = &eager_deopt_info.input_locations[index++];
+        InputLocation* input = &deopt_info.input_locations[index++];
+        input->InjectAllocated(node->allocation());
+        UpdateUse(node, input);
+      });
+}
+
+void StraightForwardRegisterAllocator::UpdateUse(
+    const LazyDeoptInfo& deopt_info) {
+  const CompactInterpreterFrameState* checkpoint_state =
+      deopt_info.state.register_frame;
+  int index = 0;
+  checkpoint_state->ForEachValue(
+      *compilation_unit_, [&](ValueNode* node, interpreter::Register reg) {
+        // Skip over the result location.
+        if (reg == deopt_info.result_location) return;
+        InputLocation* input = &deopt_info.input_locations[index++];
         input->InjectAllocated(node->allocation());
         UpdateUse(node, input);
       });
@@ -339,11 +354,18 @@ void StraightForwardRegisterAllocator::AllocateNode(Node* node) {
 
   if (node->properties().is_call()) SpillAndClearRegisters();
   // TODO(verwaest): This isn't a good idea :)
-  if (node->properties().can_eager_deopt()) SpillRegisters();
+  if (node->properties().can_eager_deopt() ||
+      node->properties().can_lazy_deopt())
+    SpillRegisters();
 
   // Allocate node output.
   if (node->Is<ValueNode>()) AllocateNodeResult(node->Cast<ValueNode>());
 
+  // Lazy deopts are semantically after the node, so update them last.
+  if (node->properties().can_lazy_deopt()) {
+    UpdateUse(*node->lazy_deopt_info());
+  }
+
   if (FLAG_trace_maglev_regalloc) {
     printing_visitor_->Process(node,
                                ProcessingState(compilation_unit_, block_it_));

src/maglev/maglev-regalloc.h

@@ -47,7 +47,8 @@ class StraightForwardRegisterAllocator {
 
   void UpdateUse(Input* input) { return UpdateUse(input->node(), input); }
   void UpdateUse(ValueNode* node, InputLocation* input_location);
-  void UpdateUse(const EagerDeoptInfo& eager_deopt_info);
+  void UpdateUse(const EagerDeoptInfo& deopt_info);
+  void UpdateUse(const LazyDeoptInfo& deopt_info);
 
   void AllocateControlNode(ControlNode* node, BasicBlock* block);
   void AllocateNode(Node* node);