[maglev] Support double temporary registers

Bug: v8:7700
Change-Id: I1f607b929068e249d26df3529f676da301c79a91
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3900011
Auto-Submit: Leszek Swirski <leszeks@chromium.org>
Reviewed-by: Toon Verwaest <verwaest@chromium.org>
Commit-Queue: Toon Verwaest <verwaest@chromium.org>
Cr-Commit-Position: refs/heads/main@{#83290}

src/maglev/maglev-ir.cc

@@ -682,7 +682,7 @@ void GeneratorRestoreRegister::GenerateCode(MaglevAssembler* masm,
                                             const ProcessingState& state) {
   Register array = ToRegister(array_input());
   Register result_reg = ToRegister(result());
-  Register temp = temporaries().PopFirst();
+  Register temp = general_temporaries().PopFirst();
 
   // The input and the output can alias, if that happen we use a temporary
   // register and a move at the end.
@@ -1434,7 +1434,7 @@ void CheckedInternalizedString::AllocateVreg(
 void CheckedInternalizedString::GenerateCode(MaglevAssembler* masm,
                                              const ProcessingState& state) {
   Register object = ToRegister(object_input());
-  RegList temps = temporaries();
+  RegList temps = general_temporaries();
   Register map_tmp = temps.PopFirst();
 
   if (check_type_ == CheckType::kOmitHeapObjectCheck) {
@@ -1501,7 +1501,7 @@ void LoadDoubleField::AllocateVreg(MaglevVregAllocationState* vreg_state) {
 }
 void LoadDoubleField::GenerateCode(MaglevAssembler* masm,
                                    const ProcessingState& state) {
-  Register tmp = temporaries().PopFirst();
+  Register tmp = general_temporaries().PopFirst();
   Register object = ToRegister(object_input());
   __ AssertNotSmi(object);
   __ DecompressAnyTagged(tmp, FieldOperand(object, offset()));
@@ -2049,7 +2049,7 @@ void Int32MultiplyWithOverflow::GenerateCode(MaglevAssembler* masm,
   Register right = ToRegister(right_input());
   DCHECK_EQ(result, ToRegister(left_input()));
 
-  Register saved_left = temporaries().first();
+  Register saved_left = general_temporaries().first();
   __ movl(saved_left, result);
   // TODO(leszeks): peephole optimise multiplication by a constant.
   __ imull(result, right);
@@ -2083,8 +2083,8 @@ void Int32DivideWithOverflow::AllocateVreg(
 
 void Int32DivideWithOverflow::GenerateCode(MaglevAssembler* masm,
                                            const ProcessingState& state) {
-  DCHECK(temporaries().has(rax));
-  DCHECK(temporaries().has(rdx));
+  DCHECK(general_temporaries().has(rax));
+  DCHECK(general_temporaries().has(rdx));
   Register left = ToRegister(left_input());
   Register right = ToRegister(right_input());
   __ movl(rax, left);
@@ -2542,7 +2542,7 @@ void SetPendingMessage::GenerateCode(MaglevAssembler* masm,
     __ Move(return_value, pending_message_operand);
     __ movq(pending_message_operand, new_message);
   } else {
-    Register scratch = temporaries().PopFirst();
+    Register scratch = general_temporaries().PopFirst();
     __ Move(scratch, pending_message_operand);
     __ movq(pending_message_operand, new_message);
     __ Move(return_value, scratch);
@@ -2631,7 +2631,7 @@ void TestUndetectable::GenerateCode(MaglevAssembler* masm,
                                     const ProcessingState& state) {
   Register object = ToRegister(value());
   Register return_value = ToRegister(result());
-  Register scratch = temporaries().PopFirst();
+  Register scratch = general_temporaries().PopFirst();
 
   Label return_false, done;
   __ JumpIfSmi(object, &return_false, Label::kNear);
@@ -3161,7 +3161,7 @@ void IncreaseInterruptBudget::AllocateVreg(
 }
 void IncreaseInterruptBudget::GenerateCode(MaglevAssembler* masm,
                                            const ProcessingState& state) {
-  Register scratch = temporaries().first();
+  Register scratch = general_temporaries().first();
   __ movq(scratch, MemOperand(rbp, StandardFrameConstants::kFunctionOffset));
   __ LoadTaggedPointerField(
       scratch, FieldOperand(scratch, JSFunction::kFeedbackCellOffset));
@@ -3179,7 +3179,7 @@ void ReduceInterruptBudget::AllocateVreg(
 }
 void ReduceInterruptBudget::GenerateCode(MaglevAssembler* masm,
                                          const ProcessingState& state) {
-  Register scratch = temporaries().first();
+  Register scratch = general_temporaries().first();
   __ movq(scratch, MemOperand(rbp, StandardFrameConstants::kFunctionOffset));
   __ LoadTaggedPointerField(
       scratch, FieldOperand(scratch, JSFunction::kFeedbackCellOffset));
@@ -3497,8 +3497,8 @@ void JumpLoopPrologue::AllocateVreg(MaglevVregAllocationState* vreg_state) {
 }
 void JumpLoopPrologue::GenerateCode(MaglevAssembler* masm,
                                     const ProcessingState& state) {
-  Register scratch0 = temporaries().PopFirst();
-  Register scratch1 = temporaries().PopFirst();
+  Register scratch0 = general_temporaries().PopFirst();
+  Register scratch1 = general_temporaries().PopFirst();
 
   const Register osr_state = scratch1;
   __ Move(scratch0, unit_->feedback().object());

src/maglev/maglev-ir.h

@@ -762,8 +762,10 @@ class NodeBase : public ZoneObject {
   using OpPropertiesField =
       OpcodeField::Next<OpProperties, OpProperties::kSize>;
   using NumTemporariesNeededField = OpPropertiesField::Next<uint8_t, 2>;
+  using NumDoubleTemporariesNeededField =
+      NumTemporariesNeededField::Next<uint8_t, 1>;
   // Align input count to 32-bit.
-  using UnusedField = NumTemporariesNeededField::Next<uint8_t, 3>;
+  using UnusedField = NumDoubleTemporariesNeededField::Next<uint8_t, 2>;
   using InputCountField = UnusedField::Next<size_t, 17>;
   static_assert(InputCountField::kShift == 32);
 
@@ -875,13 +877,35 @@ class NodeBase : public ZoneObject {
     id_ = id;
   }
 
+  template <typename RegisterT>
   uint8_t num_temporaries_needed() const {
+    if constexpr (std::is_same_v<RegisterT, Register>) {
       return NumTemporariesNeededField::decode(bitfield_);
+    } else {
+      return NumDoubleTemporariesNeededField::decode(bitfield_);
+    }
   }
 
-  RegList& temporaries() { return temporaries_; }
+  template <typename RegisterT>
+  RegListBase<RegisterT>& temporaries() {
+    if constexpr (std::is_same_v<RegisterT, Register>) {
+      return temporaries_;
+    } else {
+      return double_temporaries_;
+    }
+  }
 
-  void assign_temporaries(RegList list) { temporaries_ = list; }
+  RegList& general_temporaries() { return temporaries_; }
+  DoubleRegList& double_temporaries() { return double_temporaries_; }
+
+  template <typename RegisterT>
+  void assign_temporaries(RegListBase<RegisterT> list) {
+    if constexpr (std::is_same_v<RegisterT, Register>) {
+      temporaries_ = list;
+    } else {
+      double_temporaries_ = list;
+    }
+  }
 
   void Print(std::ostream& os, MaglevGraphLabeller*,
              bool skip_targets = false) const;
@@ -950,14 +974,23 @@ class NodeBase : public ZoneObject {
   //
   // Does not include any registers requested by RequireSpecificTemporary.
   void set_temporaries_needed(uint8_t value) {
-    DCHECK_EQ(num_temporaries_needed(), 0);
+    DCHECK_EQ(num_temporaries_needed<Register>(), 0);
     bitfield_ = NumTemporariesNeededField::update(bitfield_, value);
   }
 
+  void set_double_temporaries_needed(uint8_t value) {
+    DCHECK_EQ(num_temporaries_needed<DoubleRegister>(), 0);
+    bitfield_ = NumDoubleTemporariesNeededField::update(bitfield_, value);
+  }
+
   // Require that a specific register is free (and therefore clobberable) by the
   // entry into this node.
   void RequireSpecificTemporary(Register reg) { temporaries_.set(reg); }
 
+  void RequireSpecificDoubleTemporary(DoubleRegister reg) {
+    double_temporaries_.set(reg);
+  }
+
  private:
   template <class Derived, typename... Args>
   static Derived* Allocate(Zone* zone, size_t input_count, Args&&... args) {
@@ -1022,6 +1055,7 @@ class NodeBase : public ZoneObject {
   uint64_t bitfield_;
   NodeIdT id_ = kInvalidNodeId;
   RegList temporaries_;
+  DoubleRegList double_temporaries_;
 
   NodeBase() = delete;
   NodeBase(const NodeBase&) = delete;

src/maglev/maglev-regalloc.cc

@@ -615,8 +615,10 @@ void StraightForwardRegisterAllocator::AllocateNode(Node* node) {
   // result, which could be written into a register that was previously
   // considered a temporary.
   DCHECK_EQ(general_registers_.free() |
-                (node->temporaries() - GetNodeResultRegister(node)),
+                (node->general_temporaries() - GetNodeResultRegister(node)),
             general_registers_.free());
+  DCHECK_EQ(double_registers_.free() | node->double_temporaries(),
+            double_registers_.free());
   general_registers_.clear_blocked();
   double_registers_.clear_blocked();
   VerifyRegisterState();
@@ -872,8 +874,10 @@ void StraightForwardRegisterAllocator::AllocateControlNode(ControlNode* node,
 
   if (node->Is<JumpToInlined>() || node->Is<Abort>()) {
     // Do nothing.
-    DCHECK(node->temporaries().is_empty());
-    DCHECK_EQ(node->num_temporaries_needed(), 0);
+    DCHECK(node->general_temporaries().is_empty());
+    DCHECK(node->double_temporaries().is_empty());
+    DCHECK_EQ(node->num_temporaries_needed<Register>(), 0);
+    DCHECK_EQ(node->num_temporaries_needed<DoubleRegister>(), 0);
     DCHECK_EQ(node->input_count(), 0);
     DCHECK_EQ(node->properties(), OpProperties(0));
 
@@ -882,8 +886,10 @@ void StraightForwardRegisterAllocator::AllocateControlNode(ControlNode* node,
     }
   } else if (node->Is<Deopt>()) {
     // No fixed temporaries.
-    DCHECK(node->temporaries().is_empty());
-    DCHECK_EQ(node->num_temporaries_needed(), 0);
+    DCHECK(node->general_temporaries().is_empty());
+    DCHECK(node->double_temporaries().is_empty());
+    DCHECK_EQ(node->num_temporaries_needed<Register>(), 0);
+    DCHECK_EQ(node->num_temporaries_needed<DoubleRegister>(), 0);
     DCHECK_EQ(node->input_count(), 0);
     DCHECK_EQ(node->properties(), OpProperties::EagerDeopt());
 
@@ -894,8 +900,10 @@ void StraightForwardRegisterAllocator::AllocateControlNode(ControlNode* node,
     }
   } else if (auto unconditional = node->TryCast<UnconditionalControlNode>()) {
     // No fixed temporaries.
-    DCHECK(node->temporaries().is_empty());
-    DCHECK_EQ(node->num_temporaries_needed(), 0);
+    DCHECK(node->general_temporaries().is_empty());
+    DCHECK(node->double_temporaries().is_empty());
+    DCHECK_EQ(node->num_temporaries_needed<Register>(), 0);
+    DCHECK_EQ(node->num_temporaries_needed<DoubleRegister>(), 0);
     DCHECK_EQ(node->input_count(), 0);
     DCHECK(!node->properties().can_eager_deopt());
     DCHECK(!node->properties().can_lazy_deopt());
@@ -943,8 +951,10 @@ void StraightForwardRegisterAllocator::AllocateControlNode(ControlNode* node,
 
     DCHECK(!node->properties().needs_register_snapshot());
 
-    DCHECK_EQ(general_registers_.free() | node->temporaries(),
+    DCHECK_EQ(general_registers_.free() | node->general_temporaries(),
               general_registers_.free());
+    DCHECK_EQ(double_registers_.free() | node->double_temporaries(),
+              double_registers_.free());
 
     general_registers_.clear_blocked();
     double_registers_.clear_blocked();
@@ -1533,57 +1543,81 @@ compiler::AllocatedOperand RegisterFrameState<RegisterT>::AllocateRegister(
                                     reg.code());
 }
 
-void StraightForwardRegisterAllocator::AssignFixedTemporaries(NodeBase* node) {
-  // TODO(victorgomes): Support double registers as temporaries.
-  RegList fixed_temporaries = node->temporaries();
+template <typename RegisterT>
+void StraightForwardRegisterAllocator::AssignFixedTemporaries(
+    RegisterFrameState<RegisterT>& registers, NodeBase* node) {
+  RegListBase<RegisterT> fixed_temporaries = node->temporaries<RegisterT>();
 
   // Make sure that any initially set temporaries are definitely free.
-  for (Register reg : fixed_temporaries) {
-    DCHECK(!general_registers_.is_blocked(reg));
-    if (!general_registers_.free().has(reg)) {
-      DropRegisterValue(general_registers_, reg);
-      general_registers_.AddToFree(reg);
+  for (RegisterT reg : fixed_temporaries) {
+    DCHECK(!registers.is_blocked(reg));
+    if (!registers.free().has(reg)) {
+      DropRegisterValue(registers, reg);
+      registers.AddToFree(reg);
     }
-    general_registers_.block(reg);
+    registers.block(reg);
   }
 
-  if (v8_flags.trace_maglev_regalloc) {
+  if (v8_flags.trace_maglev_regalloc && !fixed_temporaries.is_empty()) {
+    if constexpr (std::is_same_v<RegisterT, Register>) {
+      printing_visitor_->os()
+          << "Fixed Temporaries: " << fixed_temporaries << "\n";
+    } else {
       printing_visitor_->os()
-        << "Fixed temporaries: " << fixed_temporaries << "\n";
+          << "Fixed Double Temporaries: " << fixed_temporaries << "\n";
     }
+  }
+}
+
+void StraightForwardRegisterAllocator::AssignFixedTemporaries(NodeBase* node) {
+  AssignFixedTemporaries(general_registers_, node);
+  AssignFixedTemporaries(double_registers_, node);
 }
 
+template <typename RegisterT>
 void StraightForwardRegisterAllocator::AssignArbitraryTemporaries(
-    NodeBase* node) {
-  int num_temporaries_needed = node->num_temporaries_needed();
+    RegisterFrameState<RegisterT>& registers, NodeBase* node) {
+  int num_temporaries_needed = node->num_temporaries_needed<RegisterT>();
   if (num_temporaries_needed == 0) return;
 
-  RegList temporaries = node->temporaries();
+  DCHECK_GT(num_temporaries_needed, 0);
+  RegListBase<RegisterT> temporaries = node->temporaries<RegisterT>();
+  int remaining_temporaries_needed = num_temporaries_needed;
 
-  // TODO(victorgomes): Support double registers as temporaries.
-  for (Register reg : general_registers_.unblocked_free()) {
-    general_registers_.block(reg);
+  for (RegisterT reg : registers.unblocked_free()) {
+    registers.block(reg);
     DCHECK(!temporaries.has(reg));
     temporaries.set(reg);
-    if (--num_temporaries_needed == 0) break;
+    if (--remaining_temporaries_needed == 0) break;
   }
 
   // Free extra registers if necessary.
-  for (int i = 0; i < num_temporaries_needed; ++i) {
-    DCHECK(general_registers_.UnblockedFreeIsEmpty());
-    Register reg = FreeUnblockedRegister<Register>();
-    general_registers_.block(reg);
+  for (int i = 0; i < remaining_temporaries_needed; ++i) {
+    DCHECK(registers.UnblockedFreeIsEmpty());
+    RegisterT reg = FreeUnblockedRegister<RegisterT>();
+    registers.block(reg);
     DCHECK(!temporaries.has(reg));
     temporaries.set(reg);
   }
 
-  DCHECK_GE(temporaries.Count(), node->num_temporaries_needed());
+  DCHECK_GE(temporaries.Count(), num_temporaries_needed);
+
   node->assign_temporaries(temporaries);
   if (v8_flags.trace_maglev_regalloc) {
+    if constexpr (std::is_same_v<RegisterT, Register>) {
       printing_visitor_->os() << "Temporaries: " << temporaries << "\n";
+    } else {
+      printing_visitor_->os() << "Double Temporaries: " << temporaries << "\n";
+    }
   }
 }
 
+void StraightForwardRegisterAllocator::AssignArbitraryTemporaries(
+    NodeBase* node) {
+  AssignArbitraryTemporaries(general_registers_, node);
+  AssignArbitraryTemporaries(double_registers_, node);
+}
+
 namespace {
 template <typename RegisterT>
 void ClearRegisterState(RegisterFrameState<RegisterT>& registers) {

src/maglev/maglev-regalloc.h

@@ -142,7 +142,13 @@ class StraightForwardRegisterAllocator {
   void AssignArbitraryRegisterInput(Input& input);
   void AssignAnyInput(Input& input);
   void AssignInputs(NodeBase* node);
+  template <typename RegisterT>
+  void AssignFixedTemporaries(RegisterFrameState<RegisterT>& registers,
+                              NodeBase* node);
   void AssignFixedTemporaries(NodeBase* node);
+  template <typename RegisterT>
+  void AssignArbitraryTemporaries(RegisterFrameState<RegisterT>& registers,
+                                  NodeBase* node);
   void AssignArbitraryTemporaries(NodeBase* node);
   void TryAllocateToInput(Phi* phi);