Use direct jump and call instruction for X64 when the deoptimization entries are in the code range.

Review URL: https://codereview.chromium.org/11574027
Patch from Haitao Feng <haitao.feng@intel.com>.

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@13903 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/arm/assembler-arm-inl.h

@@ -178,6 +178,19 @@ Address* RelocInfo::target_reference_address() {
 }
 
 
+Address RelocInfo::target_runtime_entry(Assembler* origin) {
+  ASSERT(IsRuntimeEntry(rmode_));
+  return target_address();
+}
+
+
+void RelocInfo::set_target_runtime_entry(Address target,
+                                         WriteBarrierMode mode) {
+  ASSERT(IsRuntimeEntry(rmode_));
+  if (target_address() != target) set_target_address(target, mode);
+}
+
+
 Handle<JSGlobalPropertyCell> RelocInfo::target_cell_handle() {
   ASSERT(rmode_ == RelocInfo::GLOBAL_PROPERTY_CELL);
   Address address = Memory::Address_at(pc_);

src/assembler.h

@@ -316,6 +316,9 @@ class RelocInfo BASE_EMBEDDED {
   static inline bool IsEmbeddedObject(Mode mode) {
     return mode == EMBEDDED_OBJECT;
   }
+  static inline bool IsRuntimeEntry(Mode mode) {
+    return mode == RUNTIME_ENTRY;
+  }
   // Is the relocation mode affected by GC?
   static inline bool IsGCRelocMode(Mode mode) {
     return mode <= LAST_GCED_ENUM;
@@ -379,6 +382,10 @@ class RelocInfo BASE_EMBEDDED {
   INLINE(Object** target_object_address());
   INLINE(void set_target_object(Object* target,
                                 WriteBarrierMode mode = UPDATE_WRITE_BARRIER));
+  INLINE(Address target_runtime_entry(Assembler* origin));
+  INLINE(void set_target_runtime_entry(Address target,
+                                       WriteBarrierMode mode =
+                                           UPDATE_WRITE_BARRIER));
   INLINE(JSGlobalPropertyCell* target_cell());
   INLINE(Handle<JSGlobalPropertyCell> target_cell_handle());
   INLINE(void set_target_cell(JSGlobalPropertyCell* cell,

src/ia32/assembler-ia32-inl.h

@@ -51,7 +51,7 @@ static const byte kCallOpcode = 0xE8;
 
 // The modes possibly affected by apply must be in kApplyMask.
 void RelocInfo::apply(intptr_t delta) {
-  if (rmode_ == RUNTIME_ENTRY || IsCodeTarget(rmode_)) {
+  if (IsRuntimeEntry(rmode_) || IsCodeTarget(rmode_)) {
     int32_t* p = reinterpret_cast<int32_t*>(pc_);
     *p -= delta;  // Relocate entry.
     CPU::FlushICache(p, sizeof(uint32_t));
@@ -83,13 +83,13 @@ void RelocInfo::apply(intptr_t delta) {
 
 
 Address RelocInfo::target_address() {
-  ASSERT(IsCodeTarget(rmode_) || rmode_ == RUNTIME_ENTRY);
+  ASSERT(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_));
   return Assembler::target_address_at(pc_);
 }
 
 
 Address RelocInfo::target_address_address() {
-  ASSERT(IsCodeTarget(rmode_) || rmode_ == RUNTIME_ENTRY
+  ASSERT(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_)
                               || rmode_ == EMBEDDED_OBJECT
                               || rmode_ == EXTERNAL_REFERENCE);
   return reinterpret_cast<Address>(pc_);
@@ -103,7 +103,7 @@ int RelocInfo::target_address_size() {
 
 void RelocInfo::set_target_address(Address target, WriteBarrierMode mode) {
   Assembler::set_target_address_at(pc_, target);
-  ASSERT(IsCodeTarget(rmode_) || rmode_ == RUNTIME_ENTRY);
+  ASSERT(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_));
   if (mode == UPDATE_WRITE_BARRIER && host() != NULL && IsCodeTarget(rmode_)) {
     Object* target_code = Code::GetCodeFromTargetAddress(target);
     host()->GetHeap()->incremental_marking()->RecordWriteIntoCode(
@@ -149,6 +149,19 @@ Address* RelocInfo::target_reference_address() {
 }
 
 
+Address RelocInfo::target_runtime_entry(Assembler* origin) {
+  ASSERT(IsRuntimeEntry(rmode_));
+  return reinterpret_cast<Address>(*reinterpret_cast<int32_t*>(pc_));
+}
+
+
+void RelocInfo::set_target_runtime_entry(Address target,
+                                         WriteBarrierMode mode) {
+  ASSERT(IsRuntimeEntry(rmode_));
+  if (target_address() != target) set_target_address(target, mode);
+}
+
+
 Handle<JSGlobalPropertyCell> RelocInfo::target_cell_handle() {
   ASSERT(rmode_ == RelocInfo::GLOBAL_PROPERTY_CELL);
   Address address = Memory::Address_at(pc_);
@@ -262,7 +275,7 @@ void RelocInfo::Visit(ObjectVisitor* visitor) {
              Isolate::Current()->debug()->has_break_points()) {
     visitor->VisitDebugTarget(this);
 #endif
-  } else if (mode == RelocInfo::RUNTIME_ENTRY) {
+  } else if (IsRuntimeEntry(mode)) {
     visitor->VisitRuntimeEntry(this);
   }
 }
@@ -291,7 +304,7 @@ void RelocInfo::Visit(Heap* heap) {
               IsPatchedDebugBreakSlotSequence()))) {
     StaticVisitor::VisitDebugTarget(heap, this);
 #endif
-  } else if (mode == RelocInfo::RUNTIME_ENTRY) {
+  } else if (IsRuntimeEntry(mode)) {
     StaticVisitor::VisitRuntimeEntry(this);
   }
 }

src/ia32/assembler-ia32.cc

@@ -1425,7 +1425,11 @@ void Assembler::call(byte* entry, RelocInfo::Mode rmode) {
   EnsureSpace ensure_space(this);
   ASSERT(!RelocInfo::IsCodeTarget(rmode));
   EMIT(0xE8);
-  emit(entry - (pc_ + sizeof(int32_t)), rmode);
+  if (RelocInfo::IsRuntimeEntry(rmode)) {
+    emit(reinterpret_cast<uint32_t>(entry), rmode);
+  } else {
+    emit(entry - (pc_ + sizeof(int32_t)), rmode);
+  }
 }
 
 
@@ -1490,7 +1494,11 @@ void Assembler::jmp(byte* entry, RelocInfo::Mode rmode) {
   EnsureSpace ensure_space(this);
   ASSERT(!RelocInfo::IsCodeTarget(rmode));
   EMIT(0xE9);
-  emit(entry - (pc_ + sizeof(int32_t)), rmode);
+  if (RelocInfo::IsRuntimeEntry(rmode)) {
+    emit(reinterpret_cast<uint32_t>(entry), rmode);
+  } else {
+    emit(entry - (pc_ + sizeof(int32_t)), rmode);
+  }
 }
 
 
@@ -1547,7 +1555,11 @@ void Assembler::j(Condition cc, byte* entry, RelocInfo::Mode rmode) {
   // 0000 1111 1000 tttn #32-bit disp.
   EMIT(0x0F);
   EMIT(0x80 | cc);
-  emit(entry - (pc_ + sizeof(int32_t)), rmode);
+  if (RelocInfo::IsRuntimeEntry(rmode)) {
+    emit(reinterpret_cast<uint32_t>(entry), rmode);
+  } else {
+    emit(entry - (pc_ + sizeof(int32_t)), rmode);
+  }
 }
 
 
@@ -2564,10 +2576,7 @@ void Assembler::GrowBuffer() {
   // Relocate runtime entries.
   for (RelocIterator it(desc); !it.done(); it.next()) {
     RelocInfo::Mode rmode = it.rinfo()->rmode();
-    if (rmode == RelocInfo::RUNTIME_ENTRY) {
-      int32_t* p = reinterpret_cast<int32_t*>(it.rinfo()->pc());
-      *p -= pc_delta;  // relocate entry
-    } else if (rmode == RelocInfo::INTERNAL_REFERENCE) {
+    if (rmode == RelocInfo::INTERNAL_REFERENCE) {
       int32_t* p = reinterpret_cast<int32_t*>(it.rinfo()->pc());
       if (*p != 0) {  // 0 means uninitialized.
         *p += pc_delta;

src/mips/assembler-mips-inl.h

@@ -231,6 +231,19 @@ Address* RelocInfo::target_reference_address() {
 }
 
 
+Address RelocInfo::target_runtime_entry(Assembler* origin) {
+  ASSERT(IsRuntimeEntry(rmode_));
+  return target_address();
+}
+
+
+void RelocInfo::set_target_runtime_entry(Address target,
+                                         WriteBarrierMode mode) {
+  ASSERT(IsRuntimeEntry(rmode_));
+  if (target_address() != target) set_target_address(target, mode);
+}
+
+
 Handle<JSGlobalPropertyCell> RelocInfo::target_cell_handle() {
   ASSERT(rmode_ == RelocInfo::GLOBAL_PROPERTY_CELL);
   Address address = Memory::Address_at(pc_);

src/objects.cc

@@ -8793,8 +8793,10 @@ void Code::CopyFrom(const CodeDesc& desc) {
   int mode_mask = RelocInfo::kCodeTargetMask |
                   RelocInfo::ModeMask(RelocInfo::EMBEDDED_OBJECT) |
                   RelocInfo::ModeMask(RelocInfo::GLOBAL_PROPERTY_CELL) |
+                  RelocInfo::ModeMask(RelocInfo::RUNTIME_ENTRY) |
                   RelocInfo::kApplyMask;
-  Assembler* origin = desc.origin;  // Needed to find target_object on X64.
+  // Needed to find target_object and runtime_entry on X64
+  Assembler* origin = desc.origin;
   for (RelocIterator it(this, mode_mask); !it.done(); it.next()) {
     RelocInfo::Mode mode = it.rinfo()->rmode();
     if (mode == RelocInfo::EMBEDDED_OBJECT) {
@@ -8810,6 +8812,9 @@ void Code::CopyFrom(const CodeDesc& desc) {
       Code* code = Code::cast(*p);
       it.rinfo()->set_target_address(code->instruction_start(),
                                      SKIP_WRITE_BARRIER);
+    } else if (RelocInfo::IsRuntimeEntry(mode)) {
+      Address p = it.rinfo()->target_runtime_entry(origin);
+      it.rinfo()->set_target_runtime_entry(p, SKIP_WRITE_BARRIER);
     } else {
       it.rinfo()->apply(delta);
     }

src/spaces.h

@@ -894,6 +894,10 @@ class CodeRange {
   void TearDown();
 
   bool exists() { return this != NULL && code_range_ != NULL; }
+  Address start() {
+    if (this == NULL || code_range_ == NULL) return NULL;
+    return static_cast<Address>(code_range_->address());
+  }
   bool contains(Address address) {
     if (this == NULL || code_range_ == NULL) return false;
     Address start = static_cast<Address>(code_range_->address());

src/x64/assembler-x64-inl.h

@@ -86,6 +86,14 @@ void Assembler::emit_code_target(Handle<Code> target,
 }
 
 
+void Assembler::emit_runtime_entry(Address entry, RelocInfo::Mode rmode) {
+  ASSERT(RelocInfo::IsRuntimeEntry(rmode));
+  ASSERT(isolate()->code_range()->exists());
+  RecordRelocInfo(rmode);
+  emitl(static_cast<uint32_t>(entry - isolate()->code_range()->start()));
+}
+
+
 void Assembler::emit_rex_64(Register reg, Register rm_reg) {
   emit(0x48 | reg.high_bit() << 2 | rm_reg.high_bit());
 }
@@ -208,6 +216,12 @@ Handle<Object> Assembler::code_target_object_handle_at(Address pc) {
   return code_targets_[Memory::int32_at(pc)];
 }
 
+
+Address Assembler::runtime_entry_at(Address pc) {
+  ASSERT(isolate()->code_range()->exists());
+  return Memory::int32_at(pc) + isolate()->code_range()->start();
+}
+
 // -----------------------------------------------------------------------------
 // Implementation of RelocInfo
 
@@ -217,7 +231,7 @@ void RelocInfo::apply(intptr_t delta) {
     // absolute code pointer inside code object moves with the code object.
     Memory::Address_at(pc_) += static_cast<int32_t>(delta);
     CPU::FlushICache(pc_, sizeof(Address));
-  } else if (IsCodeTarget(rmode_)) {
+  } else if (IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_)) {
     Memory::int32_at(pc_) -= static_cast<int32_t>(delta);
     CPU::FlushICache(pc_, sizeof(int32_t));
   } else if (rmode_ == CODE_AGE_SEQUENCE) {
@@ -231,17 +245,13 @@ void RelocInfo::apply(intptr_t delta) {
 
 
 Address RelocInfo::target_address() {
-  ASSERT(IsCodeTarget(rmode_) || rmode_ == RUNTIME_ENTRY);
-  if (IsCodeTarget(rmode_)) {
-    return Assembler::target_address_at(pc_);
-  } else {
-    return Memory::Address_at(pc_);
-  }
+  ASSERT(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_));
+  return Assembler::target_address_at(pc_);
 }
 
 
 Address RelocInfo::target_address_address() {
-  ASSERT(IsCodeTarget(rmode_) || rmode_ == RUNTIME_ENTRY
+  ASSERT(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_)
                               || rmode_ == EMBEDDED_OBJECT
                               || rmode_ == EXTERNAL_REFERENCE);
   return reinterpret_cast<Address>(pc_);
@@ -258,17 +268,12 @@ int RelocInfo::target_address_size() {
 
 
 void RelocInfo::set_target_address(Address target, WriteBarrierMode mode) {
-  ASSERT(IsCodeTarget(rmode_) || rmode_ == RUNTIME_ENTRY);
-  if (IsCodeTarget(rmode_)) {
-    Assembler::set_target_address_at(pc_, target);
+  ASSERT(IsCodeTarget(rmode_) || IsRuntimeEntry(rmode_));
+  Assembler::set_target_address_at(pc_, target);
+  if (mode == UPDATE_WRITE_BARRIER && host() != NULL && IsCodeTarget(rmode_)) {
     Object* target_code = Code::GetCodeFromTargetAddress(target);
-    if (mode == UPDATE_WRITE_BARRIER && host() != NULL) {
-      host()->GetHeap()->incremental_marking()->RecordWriteIntoCode(
-          host(), this, HeapObject::cast(target_code));
-    }
-  } else {
-    Memory::Address_at(pc_) = target;
-    CPU::FlushICache(pc_, sizeof(Address));
+    host()->GetHeap()->incremental_marking()->RecordWriteIntoCode(
+        host(), this, HeapObject::cast(target_code));
   }
 }
 
@@ -314,6 +319,19 @@ void RelocInfo::set_target_object(Object* target, WriteBarrierMode mode) {
 }
 
 
+Address RelocInfo::target_runtime_entry(Assembler* origin) {
+  ASSERT(IsRuntimeEntry(rmode_));
+  return origin->runtime_entry_at(pc_);
+}
+
+
+void RelocInfo::set_target_runtime_entry(Address target,
+                                         WriteBarrierMode mode) {
+  ASSERT(IsRuntimeEntry(rmode_));
+  if (target_address() != target) set_target_address(target, mode);
+}
+
+
 Handle<JSGlobalPropertyCell> RelocInfo::target_cell_handle() {
   ASSERT(rmode_ == RelocInfo::GLOBAL_PROPERTY_CELL);
   Address address = Memory::Address_at(pc_);
@@ -443,7 +461,7 @@ void RelocInfo::Visit(ObjectVisitor* visitor) {
              Isolate::Current()->debug()->has_break_points()) {
     visitor->VisitDebugTarget(this);
 #endif
-  } else if (mode == RelocInfo::RUNTIME_ENTRY) {
+  } else if (RelocInfo::IsRuntimeEntry(mode)) {
     visitor->VisitRuntimeEntry(this);
   }
 }
@@ -472,7 +490,7 @@ void RelocInfo::Visit(Heap* heap) {
               IsPatchedDebugBreakSlotSequence()))) {
     StaticVisitor::VisitDebugTarget(heap, this);
 #endif
-  } else if (mode == RelocInfo::RUNTIME_ENTRY) {
+  } else if (RelocInfo::IsRuntimeEntry(mode)) {
     StaticVisitor::VisitRuntimeEntry(this);
   }
 }

src/x64/assembler-x64.cc

@@ -841,6 +841,16 @@ void Assembler::call(Label* L) {
 }
 
 
+void Assembler::call(Address entry, RelocInfo::Mode rmode) {
+  ASSERT(RelocInfo::IsRuntimeEntry(rmode));
+  positions_recorder()->WriteRecordedPositions();
+  EnsureSpace ensure_space(this);
+  // 1110 1000 #32-bit disp.
+  emit(0xE8);
+  emit_runtime_entry(entry, rmode);
+}
+
+
 void Assembler::call(Handle<Code> target,
                      RelocInfo::Mode rmode,
                      TypeFeedbackId ast_id) {
@@ -1247,6 +1257,16 @@ void Assembler::j(Condition cc, Label* L, Label::Distance distance) {
 }
 
 
+void Assembler::j(Condition cc, Address entry, RelocInfo::Mode rmode) {
+  ASSERT(RelocInfo::IsRuntimeEntry(rmode));
+  EnsureSpace ensure_space(this);
+  ASSERT(is_uint4(cc));
+  emit(0x0F);
+  emit(0x80 | cc);
+  emit_runtime_entry(entry, rmode);
+}
+
+
 void Assembler::j(Condition cc,
                   Handle<Code> target,
                   RelocInfo::Mode rmode) {
@@ -1309,6 +1329,15 @@ void Assembler::jmp(Handle<Code> target, RelocInfo::Mode rmode) {
 }
 
 
+void Assembler::jmp(Address entry, RelocInfo::Mode rmode) {
+  ASSERT(RelocInfo::IsRuntimeEntry(rmode));
+  EnsureSpace ensure_space(this);
+  ASSERT(RelocInfo::IsRuntimeEntry(rmode));
+  emit(0xE9);
+  emit_runtime_entry(entry, rmode);
+}
+
+
 void Assembler::jmp(Register target) {
   EnsureSpace ensure_space(this);
   // Opcode FF/4 r64.
@@ -3049,6 +3078,7 @@ void Assembler::RecordComment(const char* msg, bool force) {
 
 
 const int RelocInfo::kApplyMask = RelocInfo::kCodeTargetMask |
+    1 << RelocInfo::RUNTIME_ENTRY |
     1 << RelocInfo::INTERNAL_REFERENCE |
     1 << RelocInfo::CODE_AGE_SEQUENCE;
 

src/x64/assembler-x64.h

@@ -561,6 +561,7 @@ class Assembler : public AssemblerBase {
   }
 
   inline Handle<Object> code_target_object_handle_at(Address pc);
+  inline Address runtime_entry_at(Address pc);
   // Number of bytes taken up by the branch target in the code.
   static const int kSpecialTargetSize = 4;  // Use 32-bit displacement.
   // Distance between the address of the code target in the call instruction
@@ -1180,6 +1181,7 @@ class Assembler : public AssemblerBase {
   // Calls
   // Call near relative 32-bit displacement, relative to next instruction.
   void call(Label* L);
+  void call(Address entry, RelocInfo::Mode rmode);
   void call(Handle<Code> target,
             RelocInfo::Mode rmode = RelocInfo::CODE_TARGET,
             TypeFeedbackId ast_id = TypeFeedbackId::None());
@@ -1201,6 +1203,7 @@ class Assembler : public AssemblerBase {
   // Use a 32-bit signed displacement.
   // Unconditional jump to L
   void jmp(Label* L, Label::Distance distance = Label::kFar);
+  void jmp(Address entry, RelocInfo::Mode rmode);
   void jmp(Handle<Code> target, RelocInfo::Mode rmode);
 
   // Jump near absolute indirect (r64)
@@ -1213,6 +1216,7 @@ class Assembler : public AssemblerBase {
   void j(Condition cc,
          Label* L,
          Label::Distance distance = Label::kFar);
+  void j(Condition cc, Address entry, RelocInfo::Mode rmode);
   void j(Condition cc, Handle<Code> target, RelocInfo::Mode rmode);
 
   // Floating-point operations
@@ -1430,6 +1434,7 @@ class Assembler : public AssemblerBase {
   inline void emit_code_target(Handle<Code> target,
                                RelocInfo::Mode rmode,
                                TypeFeedbackId ast_id = TypeFeedbackId::None());
+  inline void emit_runtime_entry(Address entry, RelocInfo::Mode rmode);
   void emit(Immediate x) { emitl(x.value_); }
 
   // Emits a REX prefix that encodes a 64-bit operand size and

src/x64/lithium-codegen-x64.cc

@@ -353,9 +353,9 @@ bool LCodeGen::GenerateJumpTable() {
       }
     } else {
       if (is_lazy_deopt) {
-        __ Call(entry, RelocInfo::RUNTIME_ENTRY);
+        __ call(entry, RelocInfo::RUNTIME_ENTRY);
       } else {
-        __ Jump(entry, RelocInfo::RUNTIME_ENTRY);
+        __ jmp(entry, RelocInfo::RUNTIME_ENTRY);
       }
     }
   }
@@ -754,9 +754,9 @@ void LCodeGen::DeoptimizeIf(Condition cc, LEnvironment* environment) {
   bool needs_lazy_deopt = info()->IsStub();
   if (cc == no_condition && frame_is_built_) {
     if (needs_lazy_deopt) {
-      __ Call(entry, RelocInfo::RUNTIME_ENTRY);
+      __ call(entry, RelocInfo::RUNTIME_ENTRY);
     } else {
-      __ Jump(entry, RelocInfo::RUNTIME_ENTRY);
+      __ jmp(entry, RelocInfo::RUNTIME_ENTRY);
     }
   } else {
     // We often have several deopts to the same entry, reuse the last