[builtins] Inline the off-heap trampoline at callsites

At runtime, calls to embedded builtins do not need to take the
indirection through the off-heap trampoline. We can simply inline the
trampoline instead.

Bug: v8:6666
Change-Id: Idb7d504fdfee173a0b134fbc74bd5dc6d09629cb
Reviewed-on: https://chromium-review.googlesource.com/1068742Reviewed-by: Michael Starzinger <mstarzinger@chromium.org>
Reviewed-by: Georg Neis <neis@chromium.org>
Commit-Queue: Jakob Gruber <jgruber@chromium.org>
Cr-Commit-Position: refs/heads/master@{#53354}

src/arm/macro-assembler-arm.cc

@@ -25,6 +25,7 @@
 #include "src/register-configuration.h"
 #include "src/runtime/runtime.h"
 #include "src/snapshot/serializer-common.h"
+#include "src/snapshot/snapshot.h"
 
 #include "src/arm/macro-assembler-arm.h"
 
@@ -228,6 +229,19 @@ void TurboAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,
     add(scratch, scratch, Operand(Code::kHeaderSize - kHeapObjectTag));
     Jump(scratch, cond);
     return;
+  } else if (!isolate()->serializer_enabled()) {
+    int builtin_index = Builtins::kNoBuiltinId;
+    if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index) &&
+        Builtins::IsIsolateIndependent(builtin_index)) {
+      // Inline the trampoline.
+      EmbeddedData d = EmbeddedData::FromBlob();
+      Address entry = d.InstructionStartOfBuiltin(builtin_index);
+      // Use ip directly instead of using UseScratchRegisterScope, as we do not
+      // preserve scratch registers across calls.
+      mov(ip, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+      Jump(ip, cond);
+      return;
+    }
   }
 #endif  // V8_EMBEDDED_BUILTINS
   // 'code' is always generated ARM code, never THUMB code
@@ -320,6 +334,20 @@ void TurboAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode,
     add(ip, ip, Operand(Code::kHeaderSize - kHeapObjectTag));
     Call(ip, cond);
     return;
+  } else if (!isolate()->serializer_enabled()) {
+    int builtin_index = Builtins::kNoBuiltinId;
+    if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index) &&
+        Builtins::IsIsolateIndependent(builtin_index)) {
+      // Inline the trampoline.
+      DCHECK(Builtins::IsBuiltinId(builtin_index));
+      EmbeddedData d = EmbeddedData::FromBlob();
+      Address entry = d.InstructionStartOfBuiltin(builtin_index);
+      // Use ip directly instead of using UseScratchRegisterScope, as we do not
+      // preserve scratch registers across calls.
+      mov(ip, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+      Call(ip, cond);
+      return;
+    }
   }
 #endif  // V8_EMBEDDED_BUILTINS
   // 'code' is always generated ARM code, never THUMB code

src/arm64/macro-assembler-arm64.cc

@@ -21,6 +21,7 @@
 #include "src/register-configuration.h"
 #include "src/runtime/runtime.h"
 #include "src/snapshot/serializer-common.h"
+#include "src/snapshot/snapshot.h"
 
 #include "src/arm64/macro-assembler-arm64-inl.h"
 #include "src/arm64/macro-assembler-arm64.h"  // Cannot be the first include
@@ -1994,6 +1995,20 @@ void TurboAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,
     Add(scratch, scratch, Operand(Code::kHeaderSize - kHeapObjectTag));
     Jump(scratch, cond);
     return;
+  } else if (!isolate()->serializer_enabled()) {
+    int builtin_index = Builtins::kNoBuiltinId;
+    if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index) &&
+        Builtins::IsIsolateIndependent(builtin_index)) {
+      // Inline the trampoline.
+      CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+      UseScratchRegisterScope temps(this);
+      Register scratch = temps.AcquireX();
+      EmbeddedData d = EmbeddedData::FromBlob();
+      Address entry = d.InstructionStartOfBuiltin(builtin_index);
+      Mov(scratch, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+      Jump(scratch, cond);
+      return;
+    }
   }
 #endif  // V8_EMBEDDED_BUILTINS
   if (CanUseNearCallOrJump(rmode)) {
@@ -2053,6 +2068,20 @@ void TurboAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode) {
     Add(scratch, scratch, Operand(Code::kHeaderSize - kHeapObjectTag));
     Call(scratch);
     return;
+  } else if (!isolate()->serializer_enabled()) {
+    int builtin_index = Builtins::kNoBuiltinId;
+    if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index) &&
+        Builtins::IsIsolateIndependent(builtin_index)) {
+      // Inline the trampoline.
+      CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+      UseScratchRegisterScope temps(this);
+      Register scratch = temps.AcquireX();
+      EmbeddedData d = EmbeddedData::FromBlob();
+      Address entry = d.InstructionStartOfBuiltin(builtin_index);
+      Mov(scratch, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+      Call(scratch);
+      return;
+    }
   }
 #endif  // V8_EMBEDDED_BUILTINS
   if (CanUseNearCallOrJump(rmode)) {

src/builtins/builtins.cc

@@ -199,6 +199,17 @@ bool Builtins::IsBuiltin(const Code* code) {
   return Builtins::IsBuiltinId(code->builtin_index());
 }
 
+bool Builtins::IsBuiltinHandle(Handle<Code> code, int* index) const {
+  Object** const handle_location = bit_cast<Object**>(code.address());
+  Object* const* start = &builtins_[0];
+  Object* const* end = &builtins_[Builtins::builtin_count];
+  if (handle_location >= end) return false;
+  if (handle_location < start) return false;
+  *index = static_cast<int>(handle_location - start);
+  DCHECK(Builtins::IsBuiltinId(*index));
+  return true;
+}
+
 // static
 bool Builtins::IsEmbeddedBuiltin(const Code* code) {
 #ifdef V8_EMBEDDED_BUILTINS

src/builtins/builtins.h

@@ -113,6 +113,10 @@ class Builtins {
   // necessarily mean that its kind is Code::BUILTIN.
   static bool IsBuiltin(const Code* code);
 
+  // As above, but safe to access off the main thread since the check is done
+  // by handle location. Similar to Heap::IsRootHandle.
+  bool IsBuiltinHandle(Handle<Code> code, int* index) const;
+
   // True, iff the given code object is a builtin with off-heap embedded code.
   static bool IsEmbeddedBuiltin(const Code* code);
 

src/mips/macro-assembler-mips.cc

@@ -22,6 +22,7 @@
 #include "src/register-configuration.h"
 #include "src/runtime/runtime.h"
 #include "src/snapshot/serializer-common.h"
+#include "src/snapshot/snapshot.h"
 
 namespace v8 {
 namespace internal {
@@ -3790,6 +3791,18 @@ void TurboAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,
     LookupConstant(t9, code);
     Jump(t9, Code::kHeaderSize - kHeapObjectTag, cond, rs, rt, bd);
     return;
+  } else if (!isolate()->serializer_enabled()) {
+    int builtin_index = Builtins::kNoBuiltinId;
+    if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index) &&
+        Builtins::IsIsolateIndependent(builtin_index)) {
+      // Inline the trampoline.
+      CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+      EmbeddedData d = EmbeddedData::FromBlob();
+      Address entry = d.InstructionStartOfBuiltin(builtin_index);
+      li(t9, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+      Jump(t9, 0, cond, rs, rt, bd);
+      return;
+    }
   }
 #endif  // V8_EMBEDDED_BUILTINS
   Jump(static_cast<intptr_t>(code.address()), rmode, cond, rs, rt, bd);
@@ -3948,6 +3961,18 @@ void TurboAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode,
     LookupConstant(t9, code);
     Call(t9, Code::kHeaderSize - kHeapObjectTag, cond, rs, rt, bd);
     return;
+  } else if (!isolate()->serializer_enabled()) {
+    int builtin_index = Builtins::kNoBuiltinId;
+    if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index) &&
+        Builtins::IsIsolateIndependent(builtin_index)) {
+      // Inline the trampoline.
+      CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+      EmbeddedData d = EmbeddedData::FromBlob();
+      Address entry = d.InstructionStartOfBuiltin(builtin_index);
+      li(t9, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+      Call(t9, 0, cond, rs, rt, bd);
+      return;
+    }
   }
 #endif  // V8_EMBEDDED_BUILTINS
   Label start;

src/mips64/macro-assembler-mips64.cc

@@ -22,6 +22,7 @@
 #include "src/register-configuration.h"
 #include "src/runtime/runtime.h"
 #include "src/snapshot/serializer-common.h"
+#include "src/snapshot/snapshot.h"
 
 namespace v8 {
 namespace internal {
@@ -4221,6 +4222,18 @@ void TurboAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,
     Daddu(t9, t9, Operand(Code::kHeaderSize - kHeapObjectTag));
     Jump(t9, cond, rs, rt, bd);
     return;
+  } else if (!isolate()->serializer_enabled()) {
+    int builtin_index = Builtins::kNoBuiltinId;
+    if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index) &&
+        Builtins::IsIsolateIndependent(builtin_index)) {
+      // Inline the trampoline.
+      CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+      EmbeddedData d = EmbeddedData::FromBlob();
+      Address entry = d.InstructionStartOfBuiltin(builtin_index);
+      li(t9, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+      Jump(t9, cond, rs, rt, bd);
+      return;
+    }
   }
 #endif  // V8_EMBEDDED_BUILTINS
   Jump(static_cast<intptr_t>(code.address()), rmode, cond, rs, rt, bd);
@@ -4312,6 +4325,18 @@ void TurboAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode,
     Daddu(t9, t9, Operand(Code::kHeaderSize - kHeapObjectTag));
     Call(t9, cond, rs, rt, bd);
     return;
+  } else if (!isolate()->serializer_enabled()) {
+    int builtin_index = Builtins::kNoBuiltinId;
+    if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index) &&
+        Builtins::IsIsolateIndependent(builtin_index)) {
+      // Inline the trampoline.
+      CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+      EmbeddedData d = EmbeddedData::FromBlob();
+      Address entry = d.InstructionStartOfBuiltin(builtin_index);
+      li(t9, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+      Call(t9, cond, rs, rt, bd);
+      return;
+    }
   }
 #endif  // V8_EMBEDDED_BUILTINS
   Label start;

src/wasm/wasm-code-manager.cc

@@ -271,6 +271,7 @@ void WasmCode::Validate() const {
       case RelocInfo::WASM_CALL:
       case RelocInfo::JS_TO_WASM_CALL:
       case RelocInfo::EXTERNAL_REFERENCE:
+      case RelocInfo::OFF_HEAP_TARGET:
       case RelocInfo::COMMENT:
       case RelocInfo::CONST_POOL:
       case RelocInfo::VENEER_POOL:

src/x64/macro-assembler-x64.cc

@@ -21,6 +21,7 @@
 #include "src/objects-inl.h"
 #include "src/register-configuration.h"
 #include "src/snapshot/serializer-common.h"
+#include "src/snapshot/snapshot.h"
 #include "src/x64/assembler-x64.h"
 
 #include "src/x64/macro-assembler-x64.h"  // Cannot be the first include.
@@ -1567,6 +1568,18 @@ void TurboAssembler::Jump(Handle<Code> code_object, RelocInfo::Mode rmode,
     jmp(kScratchRegister);
     bind(&skip);
     return;
+  } else if (!isolate()->serializer_enabled()) {
+    int builtin_index = Builtins::kNoBuiltinId;
+    if (isolate()->builtins()->IsBuiltinHandle(code_object, &builtin_index) &&
+        Builtins::IsIsolateIndependent(builtin_index)) {
+      // Inline the trampoline.
+      CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+      EmbeddedData d = EmbeddedData::FromBlob();
+      Address entry = d.InstructionStartOfBuiltin(builtin_index);
+      Move(kScratchRegister, entry, RelocInfo::OFF_HEAP_TARGET);
+      jmp(kScratchRegister);
+      return;
+    }
   }
 #endif  // V8_EMBEDDED_BUILTINS
   j(cc, code_object, rmode);
@@ -1623,6 +1636,18 @@ void TurboAssembler::Call(Handle<Code> code_object, RelocInfo::Mode rmode) {
     leap(kScratchRegister, FieldOperand(kScratchRegister, Code::kHeaderSize));
     call(kScratchRegister);
     return;
+  } else if (!isolate()->serializer_enabled()) {
+    int builtin_index = Builtins::kNoBuiltinId;
+    if (isolate()->builtins()->IsBuiltinHandle(code_object, &builtin_index) &&
+        Builtins::IsIsolateIndependent(builtin_index)) {
+      // Inline the trampoline.
+      CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+      EmbeddedData d = EmbeddedData::FromBlob();
+      Address entry = d.InstructionStartOfBuiltin(builtin_index);
+      Move(kScratchRegister, entry, RelocInfo::OFF_HEAP_TARGET);
+      call(kScratchRegister);
+      return;
+    }
   }
 #endif  // V8_EMBEDDED_BUILTINS
 #ifdef DEBUG