[x64] Implement partial constant pool for x64.

Partial constant pool aims at reducing code size and only takes effect for shareable constants.
Different from ARM’s constant pool, partial constant pool does not emit constant pools at the end of each code object. Instead, it keeps the first shareable constant inlined in the instructions and uses rip-relative memory loadings for the same constants in subsequent instructions. These rip-relative memory loadings will target at the position of the first inlined constant. For example:

  REX.W movq r10,0x7f9f75a32c20	  ; 10 bytes
  …
  REX.W movq r10,0x7f9f75a32c20	  ; 10 bytes
  …
turns into

  REX.W movq r10,0x7f9f75a32c20	  ; 10 bytes
  …
  REX.W movq r10,[rip+0xffffff96] ; 7 bytes
  …


Change-Id: I25a417f6d82da96024989bddf0451d7df9340c00
Reviewed-on: https://chromium-review.googlesource.com/1082231Reviewed-by: Michael Starzinger <mstarzinger@chromium.org>
Reviewed-by: Jakob Gruber <jgruber@chromium.org>
Reviewed-by: Sigurd Schneider <sigurds@chromium.org>
Commit-Queue: Shiyu Zhang <shiyu.zhang@intel.com>
Cr-Commit-Position: refs/heads/master@{#54963}

src/compiler/x64/code-generator-x64.cc

@@ -3228,7 +3228,7 @@ void CodeGenerator::AssembleReturn(InstructionOperand* pop) {
   }
 }
 
-void CodeGenerator::FinishCode() {}
+void CodeGenerator::FinishCode() { tasm()->PatchConstPool(); }
 
 void CodeGenerator::AssembleMove(InstructionOperand* source,
                                  InstructionOperand* destination) {

src/flag-definitions.h

@@ -836,6 +836,8 @@ DEFINE_BOOL(enable_vldr_imm, false,
 DEFINE_BOOL(force_long_branches, false,
             "force all emitted branches to be in long mode (MIPS/PPC only)")
 DEFINE_STRING(mcpu, "auto", "enable optimization for specific cpu")
+DEFINE_BOOL(partial_constant_pool, true,
+            "enable use of partial constant pools (X64 only)")
 
 // Deprecated ARM flags (replaced by arm_arch).
 DEFINE_MAYBE_BOOL(enable_armv7, "deprecated (use --arm_arch instead)")

src/wasm/jump-table-assembler.cc

@@ -36,6 +36,8 @@ void JumpTableAssembler::EmitLazyCompileJumpSlot(uint32_t func_index,
   pushq(Immediate(func_index));                           // max 5 bytes
   movq(kScratchRegister, uint64_t{lazy_compile_target});  // max 10 bytes
   jmp(kScratchRegister);                                  // 3 bytes
+
+  PatchConstPool();  // force patching entries for partial const pool
 }
 
 void JumpTableAssembler::EmitJumpSlot(Address target) {

src/x64/assembler-x64.cc

@@ -353,12 +353,93 @@ void Assembler::AllocateAndInstallRequestedHeapObjects(Isolate* isolate) {
   }
 }
 
+// Partial Constant Pool.
+bool ConstPool::AddSharedEntry(uint64_t data, int offset) {
+  auto existing = entries_.find(data);
+  if (existing == entries_.end()) {
+    entries_.insert(std::make_pair(data, offset + kMoveImm64Offset));
+    return false;
+  }
+
+  // Make sure this is called with strictly ascending offsets.
+  DCHECK_GT(offset + kMoveImm64Offset, existing->second);
+
+  entries_.insert(std::make_pair(data, offset + kMoveRipRelativeDispOffset));
+  return true;
+}
+
+bool ConstPool::TryRecordEntry(intptr_t data, RelocInfo::Mode mode) {
+  if (!FLAG_partial_constant_pool) return false;
+  if (!RelocInfo::IsShareableRelocMode(mode)) return false;
+
+  // Currently, partial constant pool only handles the following kinds of
+  // RelocInfo.
+  if (mode != RelocInfo::NONE && mode != RelocInfo::EXTERNAL_REFERENCE &&
+      mode != RelocInfo::OFF_HEAP_TARGET)
+    return false;
+
+  uint64_t raw_data = static_cast<uint64_t>(data);
+  int offset = assm_->pc_offset();
+  return AddSharedEntry(raw_data, offset);
+}
+
+bool ConstPool::IsMoveRipRelative(byte* instr) {
+  if ((*reinterpret_cast<uint32_t*>(instr) & kMoveRipRelativeMask) ==
+      kMoveRipRelativeInstr)
+    return true;
+  return false;
+}
+
+void ConstPool::Clear() { entries_.clear(); }
+
+void ConstPool::PatchEntries() {
+  for (EntryMap::iterator iter = entries_.begin(); iter != entries_.end();
+       iter = entries_.upper_bound(iter->first)) {
+    std::pair<EntryMap::iterator, EntryMap::iterator> range =
+        entries_.equal_range(iter->first);
+    int constant_entry_offset = 0;
+    for (EntryMap::iterator it = range.first; it != range.second; it++) {
+      if (it == range.first) {
+        constant_entry_offset = it->second;
+        continue;
+      }
+
+      DCHECK_GT(constant_entry_offset, 0);
+      DCHECK_LT(constant_entry_offset, it->second);
+      int32_t disp32 =
+          constant_entry_offset - (it->second + kRipRelativeDispSize);
+      byte* disp_addr = assm_->addr_at(it->second);
+
+      // Check if the instruction is actually a rip-relative move.
+      DCHECK(IsMoveRipRelative(disp_addr - kMoveRipRelativeDispOffset));
+      // The displacement of the rip-relative move should be 0 before patching.
+      DCHECK(*reinterpret_cast<uint32_t*>(disp_addr) == 0);
+      *reinterpret_cast<int32_t*>(disp_addr) = disp32;
+    }
+  }
+  Clear();
+}
+
+void Assembler::PatchConstPool() {
+  // There is nothing to do if there are no pending entries.
+  if (constpool_.IsEmpty()) {
+    return;
+  }
+  constpool_.PatchEntries();
+}
+
+bool Assembler::UseConstPoolFor(RelocInfo::Mode rmode) {
+  if (!FLAG_partial_constant_pool) return false;
+  return (rmode == RelocInfo::NONE || rmode == RelocInfo::EXTERNAL_REFERENCE ||
+          rmode == RelocInfo::OFF_HEAP_TARGET);
+}
+
 // -----------------------------------------------------------------------------
 // Implementation of Assembler.
 
 Assembler::Assembler(const AssemblerOptions& options, void* buffer,
                      int buffer_size)
-    : AssemblerBase(options, buffer, buffer_size) {
+    : AssemblerBase(options, buffer, buffer_size), constpool_(this) {
 // Clear the buffer in debug mode unless it was provided by the
 // caller in which case we can't be sure it's okay to overwrite
 // existing code in it.
@@ -373,6 +454,9 @@ Assembler::Assembler(const AssemblerOptions& options, void* buffer,
 }
 
 void Assembler::GetCode(Isolate* isolate, CodeDesc* desc) {
+  PatchConstPool();
+  DCHECK(constpool_.IsEmpty());
+
   // At this point overflow() may be true, but the gap ensures
   // that we are still not overlapping instructions and relocation info.
   DCHECK(pc_ <= reloc_info_writer.pos());  // No overlap.
@@ -1681,10 +1765,17 @@ void Assembler::emit_mov(Operand dst, Immediate value, int size) {
 }
 
 void Assembler::movp(Register dst, Address value, RelocInfo::Mode rmode) {
-  EnsureSpace ensure_space(this);
-  emit_rex(dst, kPointerSize);
-  emit(0xB8 | dst.low_bits());
-  emitp(value, rmode);
+  if (constpool_.TryRecordEntry(value, rmode)) {
+    // Emit rip-relative move with offset = 0
+    Label label;
+    emit_mov(dst, Operand(&label, 0), kPointerSize);
+    bind(&label);
+  } else {
+    EnsureSpace ensure_space(this);
+    emit_rex(dst, kPointerSize);
+    emit(0xB8 | dst.low_bits());
+    emitp(value, rmode);
+  }
 }
 
 void Assembler::movp_heap_number(Register dst, double value) {
@@ -1696,13 +1787,20 @@ void Assembler::movp_heap_number(Register dst, double value) {
 }
 
 void Assembler::movq(Register dst, int64_t value, RelocInfo::Mode rmode) {
-  EnsureSpace ensure_space(this);
-  emit_rex_64(dst);
-  emit(0xB8 | dst.low_bits());
-  if (!RelocInfo::IsNone(rmode)) {
-    RecordRelocInfo(rmode, value);
+  if (constpool_.TryRecordEntry(value, rmode)) {
+    // Emit rip-relative move with offset = 0
+    Label label;
+    emit_mov(dst, Operand(&label, 0), kPointerSize);
+    bind(&label);
+  } else {
+    EnsureSpace ensure_space(this);
+    emit_rex_64(dst);
+    emit(0xB8 | dst.low_bits());
+    if (!RelocInfo::IsNone(rmode)) {
+      RecordRelocInfo(rmode, value);
+    }
+    emitq(value);
   }
-  emitq(value);
 }
 
 void Assembler::movq(Register dst, uint64_t value, RelocInfo::Mode rmode) {

src/x64/assembler-x64.h

@@ -422,6 +422,67 @@ static_assert(sizeof(Operand) <= 2 * kPointerSize,
   V(shr, 0x5)                     \
   V(sar, 0x7)
 
+// Partial Constant Pool
+// Different from complete constant pool (like arm does), partial constant pool
+// only takes effects for shareable constants in order to reduce code size.
+// Partial constant pool does not emit constant pool entries at the end of each
+// code object. Instead, it keeps the first shareable constant inlined in the
+// instructions and uses rip-relative memory loadings for the same constants in
+// subsequent instructions. These rip-relative memory loadings will target at
+// the position of the first inlined constant. For example:
+//
+//  REX.W movq r10,0x7f9f75a32c20   ; 10 bytes
+//  …
+//  REX.W movq r10,0x7f9f75a32c20   ; 10 bytes
+//  …
+//
+// turns into
+//
+//  REX.W movq r10,0x7f9f75a32c20   ; 10 bytes
+//  …
+//  REX.W movq r10,[rip+0xffffff96] ; 7 bytes
+//  …
+
+class ConstPool {
+ public:
+  explicit ConstPool(Assembler* assm) : assm_(assm) {}
+  // Returns true when partial constant pool is valid for this entry.
+  bool TryRecordEntry(intptr_t data, RelocInfo::Mode mode);
+  bool IsEmpty() const { return entries_.empty(); }
+
+  void PatchEntries();
+  // Discard any pending pool entries.
+  void Clear();
+
+ private:
+  // Adds a shared entry to entries_. Returns true if this is not the first time
+  // we add this entry, false otherwise.
+  bool AddSharedEntry(uint64_t data, int offset);
+
+  // Check if the instruction is a rip-relative move.
+  bool IsMoveRipRelative(byte* instr);
+
+  Assembler* assm_;
+
+  // Values, pc offsets of entries.
+  typedef std::multimap<uint64_t, int> EntryMap;
+  EntryMap entries_;
+
+  // Number of bytes taken up by the displacement of rip-relative addressing.
+  static constexpr int kRipRelativeDispSize = 4;  // 32-bit displacement.
+  // Distance between the address of the displacement in the rip-relative move
+  // instruction and the head address of the instruction.
+  static constexpr int kMoveRipRelativeDispOffset =
+      3;  // REX Opcode ModRM Displacement
+  // Distance between the address of the imm64 in the 'movq reg, imm64'
+  // instruction and the head address of the instruction.
+  static constexpr int kMoveImm64Offset = 2;  // REX Opcode imm64
+  // A mask for rip-relative move instruction.
+  static constexpr uint32_t kMoveRipRelativeMask = 0x00C7FFFB;
+  // The bits for a rip-relative move instruction after mask.
+  static constexpr uint32_t kMoveRipRelativeInstr = 0x00058B48;
+};
+
 class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
  private:
   // We check before assembling an instruction that there is sufficient
@@ -1894,6 +1955,12 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
   void dp(uintptr_t data) { dq(data); }
   void dq(Label* label);
 
+  // Patch entries for partial constant pool.
+  void PatchConstPool();
+
+  // Check if use partial constant pool for this rmode.
+  static bool UseConstPoolFor(RelocInfo::Mode rmode);
+
   // Check if there is less than kGap bytes available in the buffer.
   // If this is the case, we need to grow the buffer before emitting
   // an instruction or relocation information.
@@ -2374,6 +2441,10 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
   int farjmp_num_ = 0;
   std::deque<int> farjmp_positions_;
   std::map<Label*, std::vector<int>> label_farjmp_maps_;
+
+  ConstPool constpool_;
+
+  friend class ConstPool;
 };