[compiler] Inline 64bit immediates and RPO numbers in InstructionOperand

AddImmediate ends up pushing repeated immediates very often
unecessarily. Add support for Int64 immediates being inlined into
InstructionOperand if they fit into the payload (which is almost always
the case). Also add a seperate rpo_immediate vector for RPO numbers to
avoid having to add them to the immediates_ vector multiple times.
Ideally the RPO values would also be inlined, however JumpThreading
needs to patch RPO targets throughout the instruction stream, so we
need an indirection.

Change-Id: I75b1cdb05917f85d4f740a34c3720dd9cf0ee29c
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2782282
Commit-Queue: Ross McIlroy <rmcilroy@chromium.org>
Reviewed-by: Santiago Aboy Solanes <solanes@chromium.org>
Cr-Commit-Position: refs/heads/master@{#73687}

src/compiler/backend/ia32/code-generator-ia32.cc

@@ -633,7 +633,7 @@ void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
         LocationOperand source_location(LocationOperand::cast(source));
         __ push(source_location.GetRegister());
       } else if (source.IsImmediate()) {
-        __ Push(Immediate(ImmediateOperand::cast(source).inline_value()));
+        __ Push(Immediate(ImmediateOperand::cast(source).inline_int32_value()));
       } else {
         // Pushes of non-scalar data types is not supported.
         UNIMPLEMENTED();

src/compiler/backend/instruction.cc

@@ -169,9 +169,13 @@ std::ostream& operator<<(std::ostream& os, const InstructionOperand& op) {
     case InstructionOperand::IMMEDIATE: {
       ImmediateOperand imm = ImmediateOperand::cast(op);
       switch (imm.type()) {
-        case ImmediateOperand::INLINE:
-          return os << "#" << imm.inline_value();
-        case ImmediateOperand::INDEXED:
+        case ImmediateOperand::INLINE_INT32:
+          return os << "#" << imm.inline_int32_value();
+        case ImmediateOperand::INLINE_INT64:
+          return os << "#" << imm.inline_int64_value();
+        case ImmediateOperand::INDEXED_RPO:
+          return os << "[rpo_immediate:" << imm.indexed_value() << "]";
+        case ImmediateOperand::INDEXED_IMM:
           return os << "[immediate:" << imm.indexed_value() << "]";
       }
     }
@@ -834,6 +838,7 @@ InstructionSequence::InstructionSequence(Isolate* isolate,
       constants_(ConstantMap::key_compare(),
                  ConstantMap::allocator_type(zone())),
       immediates_(zone()),
+      rpo_immediates_(instruction_blocks->size(), zone()),
       instructions_(zone()),
       next_virtual_register_(0),
       reference_maps_(zone()),

src/compiler/backend/instruction.h

@@ -395,7 +395,7 @@ class ConstantOperand : public InstructionOperand {
 
 class ImmediateOperand : public InstructionOperand {
  public:
-  enum ImmediateType { INLINE, INDEXED };
+  enum ImmediateType { INLINE_INT32, INLINE_INT64, INDEXED_RPO, INDEXED_IMM };
 
   explicit ImmediateOperand(ImmediateType type, int32_t value)
       : InstructionOperand(IMMEDIATE) {
@@ -406,13 +406,18 @@ class ImmediateOperand : public InstructionOperand {
 
   ImmediateType type() const { return TypeField::decode(value_); }
 
-  int32_t inline_value() const {
-    DCHECK_EQ(INLINE, type());
+  int32_t inline_int32_value() const {
+    DCHECK_EQ(INLINE_INT32, type());
+    return static_cast<int64_t>(value_) >> ValueField::kShift;
+  }
+
+  int64_t inline_int64_value() const {
+    DCHECK_EQ(INLINE_INT64, type());
     return static_cast<int64_t>(value_) >> ValueField::kShift;
   }
 
   int32_t indexed_value() const {
-    DCHECK_EQ(INDEXED, type());
+    DCHECK(type() == INDEXED_IMM || type() == INDEXED_RPO);
     return static_cast<int64_t>(value_) >> ValueField::kShift;
   }
 
@@ -423,7 +428,7 @@ class ImmediateOperand : public InstructionOperand {
   INSTRUCTION_OPERAND_CASTS(ImmediateOperand, IMMEDIATE)
 
   STATIC_ASSERT(KindField::kSize == 3);
-  using TypeField = base::BitField64<ImmediateType, 3, 1>;
+  using TypeField = base::BitField64<ImmediateType, 3, 2>;
   using ValueField = base::BitField64<int32_t, 32, 32>;
 };
 
@@ -1022,6 +1027,8 @@ std::ostream& operator<<(std::ostream&, const Instruction&);
 class RpoNumber final {
  public:
   static const int kInvalidRpoNumber = -1;
+  RpoNumber() : index_(kInvalidRpoNumber) {}
+
   int ToInt() const {
     DCHECK(IsValid());
     return index_;
@@ -1091,8 +1098,15 @@ class V8_EXPORT_PRIVATE Constant final {
 
   RelocInfo::Mode rmode() const { return rmode_; }
 
+  bool FitsInInt32() const {
+    if (type() == kInt32) return true;
+    DCHECK(type() == kInt64);
+    return value_ >= std::numeric_limits<int32_t>::min() &&
+           value_ <= std::numeric_limits<int32_t>::max();
+  }
+
   int32_t ToInt32() const {
-    DCHECK(type() == kInt32 || type() == kInt64);
+    DCHECK(FitsInInt32());
     const int32_t value = static_cast<int32_t>(value_);
     DCHECK_EQ(value_, static_cast<int64_t>(value));
     return value;
@@ -1685,21 +1699,50 @@ class V8_EXPORT_PRIVATE InstructionSequence final
   using Immediates = ZoneVector<Constant>;
   Immediates& immediates() { return immediates_; }
 
+  using RpoImmediates = ZoneVector<RpoNumber>;
+  RpoImmediates& rpo_immediates() { return rpo_immediates_; }
+
   ImmediateOperand AddImmediate(const Constant& constant) {
-    if (constant.type() == Constant::kInt32 &&
-        RelocInfo::IsNone(constant.rmode())) {
-      return ImmediateOperand(ImmediateOperand::INLINE, constant.ToInt32());
+    if (RelocInfo::IsNone(constant.rmode())) {
+      if (constant.type() == Constant::kRpoNumber) {
+        // Ideally we would inline RPO numbers into the operand, however jump-
+        // threading modifies RPO values and so we indirect through a vector
+        // of rpo_immediates to enable rewriting. We keep this seperate from the
+        // immediates vector so that we don't repeatedly push the same rpo
+        // number.
+        RpoNumber rpo_number = constant.ToRpoNumber();
+        DCHECK(!rpo_immediates().at(rpo_number.ToSize()).IsValid() ||
+               rpo_immediates().at(rpo_number.ToSize()) == rpo_number);
+        rpo_immediates()[rpo_number.ToSize()] = rpo_number;
+        return ImmediateOperand(ImmediateOperand::INDEXED_RPO,
+                                rpo_number.ToInt());
+      } else if (constant.type() == Constant::kInt32) {
+        return ImmediateOperand(ImmediateOperand::INLINE_INT32,
+                                constant.ToInt32());
+      } else if (constant.type() == Constant::kInt64 &&
+                 constant.FitsInInt32()) {
+        return ImmediateOperand(ImmediateOperand::INLINE_INT64,
+                                constant.ToInt32());
+      }
     }
     int index = static_cast<int>(immediates_.size());
     immediates_.push_back(constant);
-    return ImmediateOperand(ImmediateOperand::INDEXED, index);
+    return ImmediateOperand(ImmediateOperand::INDEXED_IMM, index);
   }
 
   Constant GetImmediate(const ImmediateOperand* op) const {
     switch (op->type()) {
-      case ImmediateOperand::INLINE:
-        return Constant(op->inline_value());
-      case ImmediateOperand::INDEXED: {
+      case ImmediateOperand::INLINE_INT32:
+        return Constant(op->inline_int32_value());
+      case ImmediateOperand::INLINE_INT64:
+        return Constant(op->inline_int64_value());
+      case ImmediateOperand::INDEXED_RPO: {
+        int index = op->indexed_value();
+        DCHECK_LE(0, index);
+        DCHECK_GT(rpo_immediates_.size(), index);
+        return Constant(rpo_immediates_[index]);
+      }
+      case ImmediateOperand::INDEXED_IMM: {
         int index = op->indexed_value();
         DCHECK_LE(0, index);
         DCHECK_GT(immediates_.size(), index);
@@ -1746,6 +1789,11 @@ class V8_EXPORT_PRIVATE InstructionSequence final
 
   void RecomputeAssemblyOrderForTesting();
 
+  void IncreaseRpoForTesting(size_t rpo_count) {
+    DCHECK_GE(rpo_count, rpo_immediates().size());
+    rpo_immediates().resize(rpo_count);
+  }
+
  private:
   friend V8_EXPORT_PRIVATE std::ostream& operator<<(std::ostream&,
                                                     const InstructionSequence&);
@@ -1765,6 +1813,7 @@ class V8_EXPORT_PRIVATE InstructionSequence final
   SourcePositionMap source_positions_;
   ConstantMap constants_;
   Immediates immediates_;
+  RpoImmediates rpo_immediates_;
   InstructionDeque instructions_;
   int next_virtual_register_;
   ReferenceMapDeque reference_maps_;

src/compiler/backend/jump-threading.cc

@@ -131,8 +131,8 @@ bool JumpThreading::ComputeForwarding(Zone* local_zone,
               // Dynamic return values might use different registers at
               // different return sites and therefore cannot be shared.
               if (instr->InputAt(0)->IsImmediate()) {
-                int32_t return_size =
-                    ImmediateOperand::cast(instr->InputAt(0))->inline_value();
+                int32_t return_size = ImmediateOperand::cast(instr->InputAt(0))
+                                          ->inline_int32_value();
                 // Instructions can be shared only for blocks that share
                 // the same |must_deconstruct_frame| attribute.
                 if (block->must_deconstruct_frame()) {
@@ -243,13 +243,12 @@ void JumpThreading::ApplyForwarding(Zone* local_zone,
   }
 
   // Patch RPO immediates.
-  InstructionSequence::Immediates& immediates = code->immediates();
-  for (size_t i = 0; i < immediates.size(); i++) {
-    Constant constant = immediates[i];
-    if (constant.type() == Constant::kRpoNumber) {
-      RpoNumber rpo = constant.ToRpoNumber();
+  InstructionSequence::RpoImmediates& rpo_immediates = code->rpo_immediates();
+  for (size_t i = 0; i < rpo_immediates.size(); i++) {
+    RpoNumber rpo = rpo_immediates[i];
+    if (rpo.IsValid()) {
       RpoNumber fw = result[rpo.ToInt()];
-      if (!(fw == rpo)) immediates[i] = Constant(fw);
+      if (fw != rpo) rpo_immediates[i] = fw;
     }
   }
 

src/compiler/backend/mips/instruction-scheduler-mips.cc

@@ -1531,7 +1531,7 @@ int InstructionScheduler::GetInstructionLatency(const Instruction* instr) {
         return ShrPairLatency();
       } else {
         // auto immediate_operand = ImmediateOperand::cast(instr->InputAt(2));
-        // return ShrPairLatency(false, immediate_operand->inline_value());
+        // return ShrPairLatency(false, immediate_operand->inline_32_value());
         return 1;
       }
     }

src/compiler/backend/register-allocator-verifier.cc

@@ -44,6 +44,18 @@ void VerifyAllocatedGaps(const Instruction* instr, const char* caller_info) {
   }
 }
 
+int GetValue(const ImmediateOperand* imm) {
+  switch (imm->type()) {
+    case ImmediateOperand::INLINE_INT32:
+      return imm->inline_int32_value();
+    case ImmediateOperand::INLINE_INT64:
+      return static_cast<int>(imm->inline_int64_value());
+    case ImmediateOperand::INDEXED_RPO:
+    case ImmediateOperand::INDEXED_IMM:
+      return imm->indexed_value();
+  }
+}
+
 }  // namespace
 
 RegisterAllocatorVerifier::RegisterAllocatorVerifier(
@@ -151,10 +163,8 @@ void RegisterAllocatorVerifier::BuildConstraint(const InstructionOperand* op,
     constraint->virtual_register_ = constraint->value_;
   } else if (op->IsImmediate()) {
     const ImmediateOperand* imm = ImmediateOperand::cast(op);
-    int value = imm->type() == ImmediateOperand::INLINE ? imm->inline_value()
-                                                        : imm->indexed_value();
     constraint->type_ = kImmediate;
-    constraint->value_ = value;
+    constraint->value_ = GetValue(imm);
   } else {
     CHECK(op->IsUnallocated());
     const UnallocatedOperand* unallocated = UnallocatedOperand::cast(op);
@@ -221,9 +231,7 @@ void RegisterAllocatorVerifier::CheckConstraint(
     case kImmediate: {
       CHECK_WITH_MSG(op->IsImmediate(), caller_info_);
       const ImmediateOperand* imm = ImmediateOperand::cast(op);
-      int value = imm->type() == ImmediateOperand::INLINE
-                      ? imm->inline_value()
-                      : imm->indexed_value();
+      int value = GetValue(imm);
       CHECK_EQ(value, constraint->value_);
       return;
     }

src/compiler/backend/x64/code-generator-x64.cc

@@ -808,7 +808,7 @@ void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
         LocationOperand source_location(LocationOperand::cast(source));
         __ Push(source_location.GetRegister());
       } else if (source.IsImmediate()) {
-        __ Push(Immediate(ImmediateOperand::cast(source).inline_value()));
+        __ Push(Immediate(ImmediateOperand::cast(source).inline_int32_value()));
       } else {
         // Pushes of non-scalar data types is not supported.
         UNIMPLEMENTED();

src/compiler/backend/x64/instruction-selector-x64.cc

@@ -967,7 +967,8 @@ bool TryMatchLoadWord64AndShiftRight(InstructionSelector* selector, Node* node,
         // immediate displacement. It seems that we never use M1 and M2, but we
         // handle them here anyways.
         mode = AddDisplacementToAddressingMode(mode);
-        inputs[input_count++] = ImmediateOperand(ImmediateOperand::INLINE, 4);
+        inputs[input_count++] =
+            ImmediateOperand(ImmediateOperand::INLINE_INT32, 4);
       } else {
         // In the case that the base address was zero, the displacement will be
         // in a register and replacing it with an immediate is not allowed. This
@@ -975,7 +976,7 @@ bool TryMatchLoadWord64AndShiftRight(InstructionSelector* selector, Node* node,
         if (!inputs[input_count - 1].IsImmediate()) return false;
         int32_t displacement = g.GetImmediateIntegerValue(mleft.displacement());
         inputs[input_count - 1] =
-            ImmediateOperand(ImmediateOperand::INLINE, displacement + 4);
+            ImmediateOperand(ImmediateOperand::INLINE_INT32, displacement + 4);
       }
       InstructionOperand outputs[] = {g.DefineAsRegister(node)};
       InstructionCode code = opcode | AddressingModeField::encode(mode);

src/compiler/graph-visualizer.cc

@@ -1164,11 +1164,16 @@ std::ostream& operator<<(std::ostream& os, const InstructionOperandAsJSON& o) {
       os << "\"type\": \"immediate\", ";
       const ImmediateOperand* imm = ImmediateOperand::cast(op);
       switch (imm->type()) {
-        case ImmediateOperand::INLINE: {
-          os << "\"text\": \"#" << imm->inline_value() << "\"";
+        case ImmediateOperand::INLINE_INT32: {
+          os << "\"text\": \"#" << imm->inline_int32_value() << "\"";
           break;
         }
-        case ImmediateOperand::INDEXED: {
+        case ImmediateOperand::INLINE_INT64: {
+          os << "\"text\": \"#" << imm->inline_int64_value() << "\"";
+          break;
+        }
+        case ImmediateOperand::INDEXED_RPO:
+        case ImmediateOperand::INDEXED_IMM: {
           int index = imm->indexed_value();
           os << "\"text\": \"imm:" << index << "\",";
           os << "\"tooltip\": \"";

test/cctest/compiler/test-code-generator.cc

@@ -1062,9 +1062,9 @@ class CodeGeneratorTester {
         AllocatedOperand(LocationOperand::REGISTER,
                          MachineRepresentation::kTagged,
                          kReturnRegister0.code()),
-        ImmediateOperand(ImmediateOperand::INLINE, -1),  // poison index.
-        ImmediateOperand(ImmediateOperand::INLINE, optional_padding_slot),
-        ImmediateOperand(ImmediateOperand::INLINE, stack_slot_delta)};
+        ImmediateOperand(ImmediateOperand::INLINE_INT32, -1),  // poison index.
+        ImmediateOperand(ImmediateOperand::INLINE_INT32, optional_padding_slot),
+        ImmediateOperand(ImmediateOperand::INLINE_INT32, stack_slot_delta)};
     Instruction* tail_call =
         Instruction::New(zone_, kArchTailCallCodeObject, 0, nullptr,
                          arraysize(callee), callee, 0, nullptr);
@@ -1151,9 +1151,10 @@ class CodeGeneratorTester {
         AllocatedOperand(LocationOperand::REGISTER,
                          MachineRepresentation::kTagged,
                          kReturnRegister0.code()),
-        ImmediateOperand(ImmediateOperand::INLINE, -1),  // poison index.
-        ImmediateOperand(ImmediateOperand::INLINE, optional_padding_slot),
-        ImmediateOperand(ImmediateOperand::INLINE, first_unused_stack_slot)};
+        ImmediateOperand(ImmediateOperand::INLINE_INT32, -1),  // poison index.
+        ImmediateOperand(ImmediateOperand::INLINE_INT32, optional_padding_slot),
+        ImmediateOperand(ImmediateOperand::INLINE_INT32,
+                         first_unused_stack_slot)};
     Instruction* tail_call =
         Instruction::New(zone_, kArchTailCallCodeObject, 0, nullptr,
                          arraysize(callee), callee, 0, nullptr);

test/unittests/compiler/backend/instruction-selector-unittest.cc

@@ -90,7 +90,7 @@ InstructionSelectorTest::Stream InstructionSelectorTest::StreamBuilder::Build(
       EXPECT_NE(InstructionOperand::CONSTANT, input->kind());
       if (input->IsImmediate()) {
         auto imm = ImmediateOperand::cast(input);
-        if (imm->type() == ImmediateOperand::INDEXED) {
+        if (imm->type() == ImmediateOperand::INDEXED_IMM) {
           int index = imm->indexed_value();
           s.immediates_.insert(
               std::make_pair(index, sequence.GetImmediate(imm)));

test/unittests/compiler/backend/instruction-selector-unittest.h

@@ -267,8 +267,10 @@ class InstructionSelectorTest : public TestWithNativeContextAndZone {
       } else {
         EXPECT_EQ(InstructionOperand::IMMEDIATE, operand->kind());
         auto imm = ImmediateOperand::cast(operand);
-        if (imm->type() == ImmediateOperand::INLINE) {
-          return Constant(imm->inline_value());
+        if (imm->type() == ImmediateOperand::INLINE_INT32) {
+          return Constant(imm->inline_int32_value());
+        } else if (imm->type() == ImmediateOperand::INLINE_INT64) {
+          return Constant(imm->inline_int64_value());
         }
         i = immediates_.find(imm->indexed_value());
         EXPECT_EQ(imm->indexed_value(), i->first);

test/unittests/compiler/backend/instruction-sequence-unittest.cc

@@ -344,7 +344,7 @@ InstructionOperand* InstructionSequenceTest::ConvertInputs(
 InstructionOperand InstructionSequenceTest::ConvertInputOp(TestOperand op) {
   if (op.type_ == kImmediate) {
     CHECK_EQ(op.vreg_.value_, kNoValue);
-    return ImmediateOperand(ImmediateOperand::INLINE, op.value_);
+    return ImmediateOperand(ImmediateOperand::INLINE_INT32, op.value_);
   }
   CHECK_NE(op.vreg_.value_, kNoValue);
   switch (op.type_) {