Add near calls (32-bit displacement) to Code objects on X64 platform.

Review URL: http://codereview.chromium.org/200095

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@3021 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/arm/assembler-arm-inl.h

@@ -81,7 +81,13 @@ void RelocInfo::set_target_address(Address target) {
 
 Object* RelocInfo::target_object() {
   ASSERT(IsCodeTarget(rmode_) || rmode_ == EMBEDDED_OBJECT);
-  return reinterpret_cast<Object*>(Assembler::target_address_at(pc_));
+  return Memory::Object_at(Assembler::target_address_address_at(pc_));
+}
+
+
+Handle<Object> RelocInfo::target_object_handle(Assembler *origin) {
+  ASSERT(IsCodeTarget(rmode_) || rmode_ == EMBEDDED_OBJECT);
+  return Memory::Object_Handle_at(Assembler::target_address_address_at(pc_));
 }
 
 

src/assembler.h

@@ -191,6 +191,7 @@ class RelocInfo BASE_EMBEDDED {
   INLINE(Address target_address());
   INLINE(void set_target_address(Address target));
   INLINE(Object* target_object());
+  INLINE(Handle<Object> target_object_handle(Assembler* origin));
   INLINE(Object** target_object_address());
   INLINE(void set_target_object(Object* target));
 

src/ia32/assembler-ia32-inl.h

@@ -85,19 +85,25 @@ void RelocInfo::set_target_address(Address target) {
 
 Object* RelocInfo::target_object() {
   ASSERT(IsCodeTarget(rmode_) || rmode_ == EMBEDDED_OBJECT);
-  return *reinterpret_cast<Object**>(pc_);
+  return Memory::Object_at(pc_);
+}
+
+
+Handle<Object> RelocInfo::target_object_handle(Assembler *origin) {
+  ASSERT(IsCodeTarget(rmode_) || rmode_ == EMBEDDED_OBJECT);
+  return Memory::Object_Handle_at(pc_);
 }
 
 
 Object** RelocInfo::target_object_address() {
   ASSERT(IsCodeTarget(rmode_) || rmode_ == EMBEDDED_OBJECT);
-  return reinterpret_cast<Object**>(pc_);
+  return &Memory::Object_at(pc_);
 }
 
 
 void RelocInfo::set_target_object(Object* target) {
   ASSERT(IsCodeTarget(rmode_) || rmode_ == EMBEDDED_OBJECT);
-  *reinterpret_cast<Object**>(pc_) = target;
+  Memory::Object_at(pc_) = target;
 }
 
 

src/mark-compact.cc

@@ -282,8 +282,6 @@ class MarkingVisitor : public ObjectVisitor {
            rinfo->IsCallInstruction());
     HeapObject* code = Code::GetCodeFromTargetAddress(rinfo->call_address());
     MarkCompactCollector::MarkObject(code);
-    // When compacting we convert the call to a real object pointer.
-    if (IsCompacting()) rinfo->set_call_object(code);
   }
 
  private:
@@ -1383,6 +1381,14 @@ class UpdatingVisitor: public ObjectVisitor {
         reinterpret_cast<Code*>(target)->instruction_start());
   }
 
+  void VisitDebugTarget(RelocInfo* rinfo) {
+    ASSERT(RelocInfo::IsJSReturn(rinfo->rmode()) && rinfo->IsCallInstruction());
+    Object* target = Code::GetCodeFromTargetAddress(rinfo->call_address());
+    VisitPointer(&target);
+    rinfo->set_call_address(
+        reinterpret_cast<Code*>(target)->instruction_start());
+  }
+
  private:
   void UpdatePointer(Object** p) {
     if (!(*p)->IsHeapObject()) return;

src/memory.h

@@ -63,6 +63,10 @@ class Memory {
   static Object*& Object_at(Address addr)  {
     return *reinterpret_cast<Object**>(addr);
   }
+
+  static Handle<Object>& Object_Handle_at(Address addr)  {
+    return *reinterpret_cast<Handle<Object>*>(addr);
+  }
 };
 
 } }  // namespace v8::internal

src/objects.cc

@@ -5051,15 +5051,16 @@ void Code::CopyFrom(const CodeDesc& desc) {
   int mode_mask = RelocInfo::kCodeTargetMask |
                   RelocInfo::ModeMask(RelocInfo::EMBEDDED_OBJECT) |
                   RelocInfo::kApplyMask;
+  Assembler* origin = desc.origin;  // Needed to find target_object on X64.
   for (RelocIterator it(this, mode_mask); !it.done(); it.next()) {
     RelocInfo::Mode mode = it.rinfo()->rmode();
     if (mode == RelocInfo::EMBEDDED_OBJECT) {
-      Object** p = reinterpret_cast<Object**>(it.rinfo()->target_object());
+      Handle<Object> p = it.rinfo()->target_object_handle(origin);
       it.rinfo()->set_target_object(*p);
     } else if (RelocInfo::IsCodeTarget(mode)) {
       // rewrite code handles in inline cache targets to direct
       // pointers to the first instruction in the code object
-      Object** p = reinterpret_cast<Object**>(it.rinfo()->target_object());
+      Handle<Object> p = it.rinfo()->target_object_handle(origin);
       Code* code = Code::cast(*p);
       it.rinfo()->set_target_address(code->instruction_start());
     } else {

src/serialize.cc

@@ -922,7 +922,9 @@ class ReferenceUpdater: public ObjectVisitor {
       serializer_(serializer),
       reference_encoder_(serializer->reference_encoder_),
       offsets_(8),
-      addresses_(8) {
+      addresses_(8),
+      offsets_32_bit_(0),
+      data_32_bit_(0) {
   }
 
   virtual void VisitPointers(Object** start, Object** end) {
@@ -939,9 +941,13 @@ class ReferenceUpdater: public ObjectVisitor {
     ASSERT(RelocInfo::IsCodeTarget(rinfo->rmode()));
     Code* target = Code::GetCodeFromTargetAddress(rinfo->target_address());
     Address encoded_target = serializer_->GetSavedAddress(target);
-    offsets_.Add(rinfo->target_address_address() - obj_address_);
-    addresses_.Add(encoded_target);
-  }
+    // All calls and jumps are to code objects that encode into 32 bits.
+    offsets_32_bit_.Add(rinfo->target_address_address() - obj_address_);
+    uint32_t small_target =
+        static_cast<uint32_t>(reinterpret_cast<uintptr_t>(encoded_target));
+    ASSERT(reinterpret_cast<uintptr_t>(encoded_target) == small_target);
+    data_32_bit_.Add(small_target);
+ }
 
 
   virtual void VisitExternalReferences(Address* start, Address* end) {
@@ -965,6 +971,10 @@ class ReferenceUpdater: public ObjectVisitor {
     for (int i = 0; i < offsets_.length(); i++) {
       memcpy(start_address + offsets_[i], &addresses_[i], sizeof(Address));
     }
+    for (int i = 0; i < offsets_32_bit_.length(); i++) {
+      memcpy(start_address + offsets_32_bit_[i], &data_32_bit_[i],
+             sizeof(uint32_t));
+    }
   }
 
  private:
@@ -973,6 +983,10 @@ class ReferenceUpdater: public ObjectVisitor {
   ExternalReferenceEncoder* reference_encoder_;
   List<int> offsets_;
   List<Address> addresses_;
+  // Some updates are 32-bit even on a 64-bit platform.
+  // We keep a separate list of them on 64-bit platforms.
+  List<int> offsets_32_bit_;
+  List<uint32_t> data_32_bit_;
 };
 
 
@@ -1432,7 +1446,9 @@ void Deserializer::VisitPointers(Object** start, Object** end) {
 
 void Deserializer::VisitCodeTarget(RelocInfo* rinfo) {
   ASSERT(RelocInfo::IsCodeTarget(rinfo->rmode()));
-  Address encoded_address = reinterpret_cast<Address>(rinfo->target_object());
+  // On all platforms, the encoded code object address is only 32 bits.
+  Address encoded_address = reinterpret_cast<Address>(Memory::uint32_at(
+      reinterpret_cast<Address>(rinfo->target_object_address())));
   Code* target_object = reinterpret_cast<Code*>(Resolve(encoded_address));
   rinfo->set_target_address(target_object->instruction_start());
 }
@@ -1663,7 +1679,6 @@ Object* Deserializer::Resolve(Address encoded) {
 
   // Encoded addresses of HeapObjects always have 'HeapObject' tags.
   ASSERT(o->IsHeapObject());
-
   switch (GetSpace(encoded)) {
     // For Map space and Old space, we cache the known Pages in map_pages,
     // old_pointer_pages and old_data_pages. Even though MapSpace keeps a list

src/x64/assembler-x64-inl.h

@@ -70,6 +70,20 @@ void Assembler::emitw(uint16_t x) {
 }
 
 
+void Assembler::emit_code_target(Handle<Code> target, RelocInfo::Mode rmode) {
+  ASSERT(RelocInfo::IsCodeTarget(rmode));
+  RecordRelocInfo(rmode);
+  int current = code_targets_.length();
+  if (current > 0 && code_targets_.last().is_identical_to(target)) {
+    // Optimization if we keep jumping to the same code target.
+    emitl(current - 1);
+  } else {
+    code_targets_.Add(target);
+    emitl(current);
+  }
+}
+
+
 void Assembler::emit_rex_64(Register reg, Register rm_reg) {
   emit(0x48 | reg.high_bit() << 2 | rm_reg.high_bit());
 }
@@ -162,15 +176,18 @@ void Assembler::emit_optional_rex_32(const Operand& op) {
 
 
 Address Assembler::target_address_at(Address pc) {
-  return Memory::Address_at(pc);
+  return Memory::int32_at(pc) + pc + 4;
 }
 
 
 void Assembler::set_target_address_at(Address pc, Address target) {
-  Memory::Address_at(pc) = target;
-  CPU::FlushICache(pc, sizeof(intptr_t));
+  Memory::int32_at(pc) = target - pc - 4;
+  CPU::FlushICache(pc, sizeof(int32_t));
 }
 
+Handle<Object> Assembler::code_target_object_handle_at(Address pc) {
+  return code_targets_[Memory::int32_at(pc)];
+}
 
 // -----------------------------------------------------------------------------
 // Implementation of RelocInfo
@@ -179,15 +196,24 @@ void Assembler::set_target_address_at(Address pc, Address target) {
 void RelocInfo::apply(intptr_t delta) {
   if (IsInternalReference(rmode_)) {
     // absolute code pointer inside code object moves with the code object.
-    intptr_t* p = reinterpret_cast<intptr_t*>(pc_);
-    *p += delta;  // relocate entry
+    Memory::Address_at(pc_) += delta;
+  } else if (IsCodeTarget(rmode_)) {
+    Memory::int32_at(pc_) -= delta;
+  } else if (rmode_ == JS_RETURN && IsCallInstruction()) {
+    // Special handling of js_return when a break point is set (call
+    // instruction has been inserted).
+    Memory::int32_at(pc_ + 1) -= delta;  // relocate entry
   }
 }
 
 
 Address RelocInfo::target_address() {
   ASSERT(IsCodeTarget(rmode_) || rmode_ == RUNTIME_ENTRY);
-  return Assembler::target_address_at(pc_);
+  if (IsCodeTarget(rmode_)) {
+    return Assembler::target_address_at(pc_);
+  } else {
+    return Memory::Address_at(pc_);
+  }
 }
 
 
@@ -199,13 +225,27 @@ Address RelocInfo::target_address_address() {
 
 void RelocInfo::set_target_address(Address target) {
   ASSERT(IsCodeTarget(rmode_) || rmode_ == RUNTIME_ENTRY);
-  Assembler::set_target_address_at(pc_, target);
+  if (IsCodeTarget(rmode_)) {
+    Assembler::set_target_address_at(pc_, target);
+  } else {
+    Memory::Address_at(pc_) = target;
+  }
 }
 
 
 Object* RelocInfo::target_object() {
   ASSERT(IsCodeTarget(rmode_) || rmode_ == EMBEDDED_OBJECT);
-  return *reinterpret_cast<Object**>(pc_);
+  return Memory::Object_at(pc_);
+}
+
+
+Handle<Object> RelocInfo::target_object_handle(Assembler *origin) {
+  ASSERT(IsCodeTarget(rmode_) || rmode_ == EMBEDDED_OBJECT);
+  if (rmode_ == EMBEDDED_OBJECT) {
+    return Memory::Object_Handle_at(pc_);
+  } else {
+    return origin->code_target_object_handle_at(pc_);
+  }
 }
 
 
@@ -240,16 +280,15 @@ bool RelocInfo::IsCallInstruction() {
 
 Address RelocInfo::call_address() {
   ASSERT(IsCallInstruction());
-  return Assembler::target_address_at(
-      pc_ + Assembler::kPatchReturnSequenceAddressOffset);
+  return Memory::Address_at(
+      pc_ + Assembler::kRealPatchReturnSequenceAddressOffset);
 }
 
 
 void RelocInfo::set_call_address(Address target) {
   ASSERT(IsCallInstruction());
-  Assembler::set_target_address_at(
-      pc_ + Assembler::kPatchReturnSequenceAddressOffset,
-      target);
+  Memory::Address_at(pc_ + Assembler::kRealPatchReturnSequenceAddressOffset) =
+      target;
 }
 
 

src/x64/assembler-x64.cc

@@ -264,7 +264,8 @@ static void InitCoverageLog();
 
 byte* Assembler::spare_buffer_ = NULL;
 
-Assembler::Assembler(void* buffer, int buffer_size) {
+Assembler::Assembler(void* buffer, int buffer_size)
+    : code_targets_(100) {
   if (buffer == NULL) {
     // do our own buffer management
     if (buffer_size <= kMinimalBufferSize) {
@@ -762,6 +763,15 @@ void Assembler::call(Label* L) {
 }
 
 
+void Assembler::call(Handle<Code> target, RelocInfo::Mode rmode) {
+  EnsureSpace ensure_space(this);
+  last_pc_ = pc_;
+  // 1110 1000 #32-bit disp
+  emit(0xE8);
+  emit_code_target(target, rmode);
+}
+
+
 void Assembler::call(Register adr) {
   EnsureSpace ensure_space(this);
   last_pc_ = pc_;
@@ -1062,6 +1072,19 @@ void Assembler::j(Condition cc, Label* L) {
 }
 
 
+void Assembler::j(Condition cc,
+                  Handle<Code> target,
+                  RelocInfo::Mode rmode) {
+  EnsureSpace ensure_space(this);
+  last_pc_ = pc_;
+  ASSERT(is_uint4(cc));
+  // 0000 1111 1000 tttn #32-bit disp
+  emit(0x0F);
+  emit(0x80 | cc);
+  emit_code_target(target, rmode);
+}
+
+
 void Assembler::jmp(Label* L) {
   EnsureSpace ensure_space(this);
   last_pc_ = pc_;
@@ -1093,6 +1116,15 @@ void Assembler::jmp(Label* L) {
 }
 
 
+void Assembler::jmp(Handle<Code> target, RelocInfo::Mode rmode) {
+  EnsureSpace ensure_space(this);
+  last_pc_ = pc_;
+  // 1110 1001 #32-bit disp
+  emit(0xE9);
+  emit_code_target(target, rmode);
+}
+
+
 void Assembler::jmp(Register target) {
   EnsureSpace ensure_space(this);
   last_pc_ = pc_;
@@ -2387,7 +2419,8 @@ void Assembler::WriteRecordedPositions() {
 }
 
 
-const int RelocInfo::kApplyMask = 1 << RelocInfo::INTERNAL_REFERENCE;
-
+const int RelocInfo::kApplyMask = RelocInfo::kCodeTargetMask |
+                                  1 << RelocInfo::INTERNAL_REFERENCE |
+                                  1 << RelocInfo::JS_RETURN;
 
 } }  // namespace v8::internal

src/x64/assembler-x64.h

@@ -440,18 +440,26 @@ class Assembler : public Malloced {
   // Assembler functions are invoked in between GetCode() calls.
   void GetCode(CodeDesc* desc);
 
-  // Read/Modify the code target in the branch/call instruction at pc.
-  // On the x64 architecture, the address is absolute, not relative.
+  // Read/Modify the code target in the relative branch/call instruction at pc.
+  // On the x64 architecture, we use relative jumps with a 32-bit displacement
+  // to jump to other Code objects in the Code space in the heap.
+  // Jumps to C functions are done indirectly through a 64-bit register holding
+  // the absolute address of the target.
+  // These functions convert between absolute Addresses of Code objects and
+  // the relative displacements stored in the code.
   static inline Address target_address_at(Address pc);
   static inline void set_target_address_at(Address pc, Address target);
-
+  inline Handle<Object> code_target_object_handle_at(Address pc);
   // Distance between the address of the code target in the call instruction
-  // and the return address.  Checked in the debug build.
-  static const int kCallTargetAddressOffset = 3 + kPointerSize;
-  // Distance between start of patched return sequence and the emitted address
-  // to jump to (movq = REX.W 0xB8+r.).
-  static const int kPatchReturnSequenceAddressOffset = 2;
-
+  // and the return address pushed on the stack.
+  static const int kCallTargetAddressOffset = 4;  // Use 32-bit displacement.
+  // Distance between the start of the JS return sequence and where the
+  // 32-bit displacement of a near call would be, relative to the pushed
+  // return address.  TODO: Use return sequence length instead.
+  // Should equal Debug::kX64JSReturnSequenceLength - kCallTargetAddressOffset;
+  static const int kPatchReturnSequenceAddressOffset = 13 - 4;
+  // TODO(X64): Rename this, removing the "Real", after changing the above.
+  static const int kRealPatchReturnSequenceAddressOffset = 2;
   // ---------------------------------------------------------------------------
   // Code generation
   //
@@ -923,6 +931,7 @@ class Assembler : public Malloced {
   // Calls
   // Call near relative 32-bit displacement, relative to next instruction.
   void call(Label* L);
+  void call(Handle<Code> target, RelocInfo::Mode rmode);
 
   // Call near absolute indirect, address in register
   void call(Register adr);
@@ -932,7 +941,9 @@ class Assembler : public Malloced {
 
   // Jumps
   // Jump short or near relative.
+  // Use a 32-bit signed displacement.
   void jmp(Label* L);  // unconditional jump to L
+  void jmp(Handle<Code> target, RelocInfo::Mode rmode);
 
   // Jump near absolute indirect (r64)
   void jmp(Register adr);
@@ -942,6 +953,7 @@ class Assembler : public Malloced {
 
   // Conditional jumps
   void j(Condition cc, Label* L);
+  void j(Condition cc, Handle<Code> target, RelocInfo::Mode rmode);
 
   // Floating-point operations
   void fld(int i);
@@ -1047,14 +1059,6 @@ class Assembler : public Malloced {
   void RecordStatementPosition(int pos);
   void WriteRecordedPositions();
 
-  // Writes a doubleword of data in the code stream.
-  // Used for inline tables, e.g., jump-tables.
-  // void dd(uint32_t data);
-
-  // Writes a quadword of data in the code stream.
-  // Used for inline tables, e.g., jump-tables.
-  // void dd(uint64_t data, RelocInfo::Mode reloc_info);
-
   int pc_offset() const  { return pc_ - buffer_; }
   int current_statement_position() const { return current_statement_position_; }
   int current_position() const  { return current_position_; }
@@ -1096,9 +1100,9 @@ class Assembler : public Malloced {
 
   void emit(byte x) { *pc_++ = x; }
   inline void emitl(uint32_t x);
-  inline void emit(Handle<Object> handle);
   inline void emitq(uint64_t x, RelocInfo::Mode rmode);
   inline void emitw(uint16_t x);
+  inline void emit_code_target(Handle<Code> target, RelocInfo::Mode rmode);
   void emit(Immediate x) { emitl(x.value_); }
 
   // Emits a REX prefix that encodes a 64-bit operand size and
@@ -1276,6 +1280,7 @@ class Assembler : public Malloced {
   byte* pc_;  // the program counter; moves forward
   RelocInfoWriter reloc_info_writer;
 
+  List< Handle<Code> > code_targets_;
   // push-pop elimination
   byte* last_pc_;
 

src/x64/macro-assembler-x64.cc

@@ -348,8 +348,7 @@ void MacroAssembler::JumpToRuntime(const ExternalReference& ext,
   // Set the entry point and jump to the C entry runtime stub.
   movq(rbx, ext);
   CEntryStub ces(result_size);
-  movq(kScratchRegister, ces.GetCode(), RelocInfo::CODE_TARGET);
-  jmp(kScratchRegister);
+  jmp(ces.GetCode(), RelocInfo::CODE_TARGET);
 }
 
 
@@ -1270,17 +1269,8 @@ void MacroAssembler::Jump(Address destination, RelocInfo::Mode rmode) {
 
 
 void MacroAssembler::Jump(Handle<Code> code_object, RelocInfo::Mode rmode) {
-  ASSERT(RelocInfo::IsCodeTarget(rmode));
-  movq(kScratchRegister, code_object, rmode);
-#ifdef DEBUG
-  Label target;
-  bind(&target);
-#endif
-  jmp(kScratchRegister);
-#ifdef DEBUG
-  ASSERT_EQ(kCallTargetAddressOffset,
-            SizeOfCodeGeneratedSince(&target) + kPointerSize);
-#endif
+  // TODO(X64): Inline this
+  jmp(code_object, rmode);
 }
 
 
@@ -1299,17 +1289,7 @@ void MacroAssembler::Call(Address destination, RelocInfo::Mode rmode) {
 void MacroAssembler::Call(Handle<Code> code_object, RelocInfo::Mode rmode) {
   ASSERT(RelocInfo::IsCodeTarget(rmode));
   WriteRecordedPositions();
-  movq(kScratchRegister, code_object, rmode);
-#ifdef DEBUG
-  // Patch target is kPointer size bytes *before* target label.
-  Label target;
-  bind(&target);
-#endif
-  call(kScratchRegister);
-#ifdef DEBUG
-  ASSERT_EQ(kCallTargetAddressOffset,
-            SizeOfCodeGeneratedSince(&target) + kPointerSize);
-#endif
+  call(code_object, rmode);
 }
 
 
@@ -1576,7 +1556,7 @@ void MacroAssembler::InvokeBuiltin(Builtins::JavaScript id, InvokeFlag flag) {
   if (!resolved) {
     uint32_t flags =
         Bootstrapper::FixupFlagsArgumentsCount::encode(argc) |
-        Bootstrapper::FixupFlagsIsPCRelative::encode(false) |
+        Bootstrapper::FixupFlagsIsPCRelative::encode(true) |
         Bootstrapper::FixupFlagsUseCodeObject::encode(false);
     Unresolved entry =
         { pc_offset() - kCallTargetAddressOffset, flags, name };