[sparkplug][x64][arm64] Support shorter builtin calls, pt.1

This is a speed-for-memory tradeoff, which can be achieved by
re-mapping the builtins code blob into existing code range.

The feature can be enabled by v8_enable_short_builtin_calls flag and
it's off by default.

This CL adds GN flag and updates code generator to emit shorter
pc-relative calls/jumps to builtins. However, the runtime doesn't
support appearance of the off-heap builtins' PCs that point to the
embedded code blob on the stack yet.

Bug: v8:11527, v8:11421
Change-Id: Iaba384c549675852beae70739175976ee193ffef
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2727502Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Reviewed-by: Jakob Gruber <jgruber@chromium.org>
Reviewed-by: Toon Verwaest <verwaest@chromium.org>
Commit-Queue: Igor Sheludko <ishell@chromium.org>
Cr-Commit-Position: refs/heads/master@{#73458}

BUILD.gn

@@ -169,6 +169,10 @@ declare_args() {
   # Enables various testing features.
   v8_enable_test_features = ""
 
+  # Enable short builtins call instruction sequences by un-embedding builtins.
+  # Sets -dV8_SHORT_BUILTIN_CALLS
+  v8_enable_short_builtin_calls = ""
+
   # With post mortem support enabled, metadata is embedded into libv8 that
   # describes various parameters of the VM for use by debuggers. See
   # tools/gen-postmortem-metadata.py for details.
@@ -340,6 +344,11 @@ if (v8_enable_zone_compression == "") {
 if (v8_enable_heap_sandbox == "") {
   v8_enable_heap_sandbox = false
 }
+if (v8_enable_short_builtin_calls == "") {
+  v8_enable_short_builtin_calls = false
+}
+assert(!v8_enable_short_builtin_calls || v8_enable_pointer_compression,
+       "Short builtin calls feature requires pointer compression")
 if (v8_enable_single_generation == "") {
   v8_enable_single_generation = v8_disable_write_barriers
 }
@@ -768,6 +777,9 @@ config("features") {
   if (v8_fuzzilli) {
     defines += [ "V8_FUZZILLI" ]
   }
+  if (v8_enable_short_builtin_calls) {
+    defines += [ "V8_SHORT_BUILTIN_CALLS" ]
+  }
   if (v8_dict_mode_prototypes) {
     defines += [ "V8_DICT_MODE_PROTOTYPES" ]
   }

src/baseline/arm64/baseline-assembler-arm64-inl.h

@@ -111,14 +111,30 @@ void BaselineAssembler::JumpIfNotSmi(Register value, Label* target,
 }
 
 void BaselineAssembler::CallBuiltin(Builtins::Name builtin) {
+  if (FLAG_short_builtin_calls) {
+    // Generate direct or pc-relative call.
+    __ CallBuiltin(builtin);
+  } else {
     ScratchRegisterScope temps(this);
     Register temp = temps.AcquireScratch();
     __ LoadEntryFromBuiltinIndex(builtin, temp);
     __ Call(temp);
+  }
 }
 
 void BaselineAssembler::TailCallBuiltin(Builtins::Name builtin) {
-  // x17 is used to allow using "Call" (i.e. `bti c`) rather than "Jump" (i.e.]
+  if (FLAG_short_builtin_calls) {
+    // Generate direct or pc-relative call.
+    __ TailCallBuiltin(builtin);
+  } else {
+    // The control flow integrity (CFI) feature allows us to "sign" code entry
+    // points as a target for calls, jumps or both. Arm64 has special
+    // instructions for this purpose, so-called "landing pads" (see
+    // TurboAssembler::CallTarget(), TurboAssembler::JumpTarget() and
+    // TurboAssembler::JumpOrCallTarget()). Currently, we generate "Call"
+    // landing pads for CPP builtins. In order to allow tail calling to those
+    // builtins we have to use a workaround.
+    // x17 is used to allow using "Call" (i.e. `bti c`) rather than "Jump" (i.e.
     // `bti j`) landing pads for the tail-called code.
     Register temp = x17;
 
@@ -128,6 +144,7 @@ void BaselineAssembler::TailCallBuiltin(Builtins::Name builtin) {
 
     __ LoadEntryFromBuiltinIndex(builtin, temp);
     __ Jump(temp);
+  }
 }
 
 void BaselineAssembler::Test(Register value, int mask) {

src/baseline/x64/baseline-assembler-x64-inl.h

@@ -117,15 +117,25 @@ void BaselineAssembler::JumpIfNotSmi(Register value, Label* target,
 }
 
 void BaselineAssembler::CallBuiltin(Builtins::Name builtin) {
+  if (FLAG_short_builtin_calls) {
+    // Generate direct or pc-relative call.
+    __ CallBuiltin(builtin);
+  } else {
     __ RecordCommentForOffHeapTrampoline(builtin);
     __ Call(__ EntryFromBuiltinIndexAsOperand(builtin));
     if (FLAG_code_comments) __ RecordComment("]");
+  }
 }
 
 void BaselineAssembler::TailCallBuiltin(Builtins::Name builtin) {
+  if (FLAG_short_builtin_calls) {
+    // Generate direct or pc-relative jump.
+    __ TailCallBuiltin(builtin);
+  } else {
     __ RecordCommentForOffHeapTrampoline(builtin);
     __ Jump(__ EntryFromBuiltinIndexAsOperand(builtin));
     if (FLAG_code_comments) __ RecordComment("]");
+  }
 }
 
 void BaselineAssembler::Test(Register value, int mask) {

src/builtins/builtins.cc

@@ -287,7 +287,7 @@ bool Builtins::IsIsolateIndependentBuiltin(const Code code) {
 
 // static
 void Builtins::InitializeBuiltinEntryTable(Isolate* isolate) {
-  EmbeddedData d = EmbeddedData::FromBlob();
+  EmbeddedData d = EmbeddedData::FromBlob(isolate);
   Address* builtin_entry_table = isolate->builtin_entry_table();
   for (int i = 0; i < builtin_count; i++) {
     // TODO(jgruber,chromium:1020986): Remove the CHECK once the linked issue is

src/codegen/arm64/macro-assembler-arm64.cc

@@ -12,6 +12,7 @@
 #include "src/codegen/external-reference-table.h"
 #include "src/codegen/macro-assembler-inl.h"
 #include "src/codegen/register-configuration.h"
+#include "src/codegen/reloc-info.h"
 #include "src/debug/debug.h"
 #include "src/deoptimizer/deoptimizer.h"
 #include "src/execution/frame-constants.h"
@@ -1769,25 +1770,36 @@ void TurboAssembler::JumpHelper(int64_t offset, RelocInfo::Mode rmode,
   Bind(&done);
 }
 
-namespace {
-
 // The calculated offset is either:
 // * the 'target' input unmodified if this is a Wasm call, or
+// * the offset of the target from the code range start, if this is a call to
+//   un-embedded builtin, or
 // * the offset of the target from the current PC, in instructions, for any
 //   other type of call.
-static int64_t CalculateTargetOffset(Address target, RelocInfo::Mode rmode,
-                                     byte* pc) {
+int64_t TurboAssembler::CalculateTargetOffset(Address target,
+                                              RelocInfo::Mode rmode, byte* pc) {
   int64_t offset = static_cast<int64_t>(target);
+  if (rmode == RelocInfo::WASM_CALL || rmode == RelocInfo::WASM_STUB_CALL) {
     // The target of WebAssembly calls is still an index instead of an actual
     // address at this point, and needs to be encoded as-is.
-  if (rmode != RelocInfo::WASM_CALL && rmode != RelocInfo::WASM_STUB_CALL) {
+    return offset;
+  }
+  if (RelocInfo::IsRuntimeEntry(rmode)) {
+    // The runtime entry targets are used for generating short builtin calls
+    // from JIT-compiled code (it's not used during snapshot creation).
+    // The value is encoded as an offset from the code range (see
+    // Assembler::runtime_entry_at()).
+    // Note, that builtin-to-builitin calls use different OFF_HEAP_TARGET mode
+    // and therefore are encoded differently.
+    DCHECK_NE(options().code_range_start, 0);
+    offset -= static_cast<int64_t>(options().code_range_start);
+  } else {
     offset -= reinterpret_cast<int64_t>(pc);
+  }
   DCHECK_EQ(offset % kInstrSize, 0);
   offset = offset / static_cast<int>(kInstrSize);
-  }
   return offset;
 }
-}  // namespace
 
 void TurboAssembler::Jump(Address target, RelocInfo::Mode rmode,
                           Condition cond) {
@@ -1804,14 +1816,8 @@ void TurboAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,
     int builtin_index = Builtins::kNoBuiltinId;
     if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index)) {
       // Inline the trampoline.
-      RecordCommentForOffHeapTrampoline(builtin_index);
-      CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
-      UseScratchRegisterScope temps(this);
-      Register scratch = temps.AcquireX();
-      EmbeddedData d = EmbeddedData::FromBlob();
-      Address entry = d.InstructionStartOfBuiltin(builtin_index);
-      Ldr(scratch, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
-      Jump(scratch, cond);
+      CHECK_EQ(cond, Condition::al);  // Implement if necessary.
+      TailCallBuiltin(builtin_index);
       return;
     }
   }
@@ -1923,12 +1929,45 @@ void TurboAssembler::CallBuiltin(int builtin_index) {
   DCHECK(Builtins::IsBuiltinId(builtin_index));
   RecordCommentForOffHeapTrampoline(builtin_index);
   CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+  EmbeddedData d = EmbeddedData::FromBlob(isolate());
+  Address entry = d.InstructionStartOfBuiltin(builtin_index);
+  if (options().short_builtin_calls) {
+    Call(entry, RelocInfo::RUNTIME_ENTRY);
+
+  } else {
     UseScratchRegisterScope temps(this);
     Register scratch = temps.AcquireX();
-  EmbeddedData d = EmbeddedData::FromBlob();
-  Address entry = d.InstructionStartOfBuiltin(builtin_index);
     Ldr(scratch, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
     Call(scratch);
+  }
+  if (FLAG_code_comments) RecordComment("]");
+}
+
+void TurboAssembler::TailCallBuiltin(int builtin_index) {
+  DCHECK(Builtins::IsBuiltinId(builtin_index));
+  RecordCommentForOffHeapTrampoline(builtin_index);
+  CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
+  EmbeddedData d = EmbeddedData::FromBlob(isolate());
+  Address entry = d.InstructionStartOfBuiltin(builtin_index);
+  if (options().short_builtin_calls) {
+    Jump(entry, RelocInfo::RUNTIME_ENTRY);
+
+  } else {
+    // The control flow integrity (CFI) feature allows us to "sign" code entry
+    // points as a target for calls, jumps or both. Arm64 has special
+    // instructions for this purpose, so-called "landing pads" (see
+    // TurboAssembler::CallTarget(), TurboAssembler::JumpTarget() and
+    // TurboAssembler::JumpOrCallTarget()). Currently, we generate "Call"
+    // landing pads for CPP builtins. In order to allow tail calling to those
+    // builtins we have to use a workaround.
+    // x17 is used to allow using "Call" (i.e. `bti c`) rather than "Jump"
+    // (i.e. `bti j`) landing pads for the tail-called code.
+    Register temp = x17;
+
+    Ldr(temp, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
+    Jump(temp);
+  }
+  if (FLAG_code_comments) RecordComment("]");
 }
 
 void TurboAssembler::LoadCodeObjectEntry(Register destination,

src/codegen/arm64/macro-assembler-arm64.h

@@ -975,7 +975,16 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
                                  Register destination);
   MemOperand EntryFromBuiltinIndexAsOperand(Builtins::Name builtin_index);
   void CallBuiltinByIndex(Register builtin_index) override;
+  void CallBuiltin(Builtins::Name builtin) {
+    // TODO(11527): drop the int overload in favour of the Builtins::Name one.
+    return CallBuiltin(static_cast<int>(builtin));
+  }
   void CallBuiltin(int builtin_index);
+  void TailCallBuiltin(Builtins::Name builtin) {
+    // TODO(11527): drop the int overload in favour of the Builtins::Name one.
+    return TailCallBuiltin(static_cast<int>(builtin));
+  }
+  void TailCallBuiltin(int builtin_index);
 
   void LoadCodeObjectEntry(Register destination, Register code_object) override;
   void CallCodeObject(Register code_object) override;
@@ -1449,6 +1458,9 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
   void LoadStorePairMacro(const CPURegister& rt, const CPURegister& rt2,
                           const MemOperand& addr, LoadStorePairOp op);
 
+  int64_t CalculateTargetOffset(Address target, RelocInfo::Mode rmode,
+                                byte* pc);
+
   void JumpHelper(int64_t offset, RelocInfo::Mode rmode, Condition cond = al);
 
   void CallRecordWriteStub(Register object, Operand offset,

src/codegen/assembler.cc

@@ -73,6 +73,9 @@ AssemblerOptions AssemblerOptions::Default(Isolate* isolate) {
   DCHECK_IMPLIES(code_range.begin() != kNullAddress, !code_range.is_empty());
   options.code_range_start = code_range.begin();
 #endif
+  options.short_builtin_calls = FLAG_short_builtin_calls &&
+                                !generating_embedded_builtin &&
+                                (options.code_range_start != kNullAddress);
   return options;
 }
 

src/codegen/assembler.h

@@ -169,6 +169,11 @@ struct V8_EXPORT_PRIVATE AssemblerOptions {
   // Enables the use of isolate-independent builtins through an off-heap
   // trampoline. (macro assembler feature).
   bool inline_offheap_trampolines = true;
+  // Enables generation of pc-relative calls to builtins if the off-heap
+  // builtins are guaranteed to be within the reach of pc-relative call or jump
+  // instructions. For example, when the bultins code is re-embedded into the
+  // code range.
+  bool short_builtin_calls = false;
   // On some platforms, all code is within a given range in the process,
   // and the start of this range is configured here.
   Address code_range_start = 0;

src/codegen/reloc-info.h

@@ -65,6 +65,9 @@ class RelocInfo {
     WASM_CALL,  // FIRST_SHAREABLE_RELOC_MODE
     WASM_STUB_CALL,
 
+    // TODO(ishell): rename to UNEMBEDDED_BUILTIN_ENTRY.
+    // An un-embedded off-heap instruction stream target.
+    // See http://crbug.com/v8/11527 for details.
     RUNTIME_ENTRY,
 
     EXTERNAL_REFERENCE,  // The address of an external C++ function.
@@ -148,6 +151,7 @@ class RelocInfo {
     return base::IsInRange(mode, FIRST_EMBEDDED_OBJECT_RELOC_MODE,
                            LAST_EMBEDDED_OBJECT_RELOC_MODE);
   }
+  // TODO(ishell): rename to IsUnembeddedBuiltinEntry().
   static constexpr bool IsRuntimeEntry(Mode mode) {
     return mode == RUNTIME_ENTRY;
   }

src/codegen/x64/assembler-x64-inl.h

@@ -35,8 +35,10 @@ void Assembler::emitw(uint16_t x) {
   pc_ += sizeof(uint16_t);
 }
 
+// TODO(ishell): Rename accordingly once RUNTIME_ENTRY is renamed.
 void Assembler::emit_runtime_entry(Address entry, RelocInfo::Mode rmode) {
   DCHECK(RelocInfo::IsRuntimeEntry(rmode));
+  DCHECK_NE(options().code_range_start, 0);
   RecordRelocInfo(rmode);
   emitl(static_cast<uint32_t>(entry - options().code_range_start));
 }

src/codegen/x64/assembler-x64.cc

@@ -1447,6 +1447,14 @@ void Assembler::j(Condition cc, Handle<Code> target, RelocInfo::Mode rmode) {
   emitl(code_target_index);
 }
 
+void Assembler::jmp(Address entry, RelocInfo::Mode rmode) {
+  DCHECK(RelocInfo::IsRuntimeEntry(rmode));
+  EnsureSpace ensure_space(this);
+  // 1110 1001 #32-bit disp.
+  emit(0xE9);
+  emit_runtime_entry(entry, rmode);
+}
+
 void Assembler::jmp_rel(int32_t offset) {
   EnsureSpace ensure_space(this);
   // The offset is encoded relative to the next instruction.

src/codegen/x64/assembler-x64.h

@@ -833,6 +833,7 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
   // Unconditional jump to L
   void jmp(Label* L, Label::Distance distance = Label::kFar);
   void jmp(Handle<Code> target, RelocInfo::Mode rmode);
+  void jmp(Address entry, RelocInfo::Mode rmode);
 
   // Jump near absolute indirect (r64)
   void jmp(Register adr);

src/codegen/x64/macro-assembler-x64.cc

@@ -1620,14 +1620,7 @@ void TurboAssembler::Jump(Handle<Code> code_object, RelocInfo::Mode rmode,
         if (cc == never) return;
         j(NegateCondition(cc), &skip, Label::kNear);
       }
-      // Inline the trampoline.
-      RecordCommentForOffHeapTrampoline(builtin_index);
-      CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
-      EmbeddedData d = EmbeddedData::FromBlob();
-      Address entry = d.InstructionStartOfBuiltin(builtin_index);
-      Move(kScratchRegister, entry, RelocInfo::OFF_HEAP_TARGET);
-      jmp(kScratchRegister);
-      if (FLAG_code_comments) RecordComment("]");
+      TailCallBuiltin(builtin_index);
       bind(&skip);
       return;
     }
@@ -1706,10 +1699,15 @@ void TurboAssembler::CallBuiltin(int builtin_index) {
   DCHECK(Builtins::IsBuiltinId(builtin_index));
   RecordCommentForOffHeapTrampoline(builtin_index);
   CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
-  EmbeddedData d = EmbeddedData::FromBlob();
+  EmbeddedData d = EmbeddedData::FromBlob(isolate());
   Address entry = d.InstructionStartOfBuiltin(builtin_index);
+  if (options().short_builtin_calls) {
+    call(entry, RelocInfo::RUNTIME_ENTRY);
+
+  } else {
     Move(kScratchRegister, entry, RelocInfo::OFF_HEAP_TARGET);
     call(kScratchRegister);
+  }
   if (FLAG_code_comments) RecordComment("]");
 }
 
@@ -1717,10 +1715,14 @@ void TurboAssembler::TailCallBuiltin(int builtin_index) {
   DCHECK(Builtins::IsBuiltinId(builtin_index));
   RecordCommentForOffHeapTrampoline(builtin_index);
   CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
-  EmbeddedData d = EmbeddedData::FromBlob();
+  EmbeddedData d = EmbeddedData::FromBlob(isolate());
   Address entry = d.InstructionStartOfBuiltin(builtin_index);
-  Move(kScratchRegister, entry, RelocInfo::OFF_HEAP_TARGET);
-  jmp(kScratchRegister);
+  if (options().short_builtin_calls) {
+    jmp(entry, RelocInfo::RUNTIME_ENTRY);
+
+  } else {
+    Jump(entry, RelocInfo::OFF_HEAP_TARGET);
+  }
   if (FLAG_code_comments) RecordComment("]");
 }
 

src/codegen/x64/macro-assembler-x64.h

@@ -524,7 +524,15 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
   Operand EntryFromBuiltinIndexAsOperand(Builtins::Name builtin_index);
   Operand EntryFromBuiltinIndexAsOperand(Register builtin_index);
   void CallBuiltinByIndex(Register builtin_index) override;
+  void CallBuiltin(Builtins::Name builtin) {
+    // TODO(11527): drop the int overload in favour of the Builtins::Name one.
+    return CallBuiltin(static_cast<int>(builtin));
+  }
   void CallBuiltin(int builtin_index);
+  void TailCallBuiltin(Builtins::Name builtin) {
+    // TODO(11527): drop the int overload in favour of the Builtins::Name one.
+    return TailCallBuiltin(static_cast<int>(builtin));
+  }
   void TailCallBuiltin(int builtin_index);
 
   void LoadCodeObjectEntry(Register destination, Register code_object) override;

src/common/globals.h

@@ -99,6 +99,18 @@ STATIC_ASSERT(V8_DEFAULT_STACK_SIZE_KB* KB +
                   kStackLimitSlackForDeoptimizationInBytes <=
               MB);
 
+// Determine whether the short builtin calls optimization is enabled.
+#ifdef V8_SHORT_BUILTIN_CALLS
+#ifndef V8_COMPRESS_POINTERS
+// TODO(11527): Fix this by passing Isolate* to Code::OffHeapInstructionStart()
+// and friends.
+#error Short builtin calls feature requires pointer compression
+#endif
+#define V8_SHORT_BUILTIN_CALLS_BOOL true
+#else
+#define V8_SHORT_BUILTIN_CALLS_BOOL false
+#endif
+
 // Determine whether dict mode prototypes feature is enabled.
 #ifdef V8_DICT_MODE_PROTOTYPES
 #define V8_DICT_MODE_PROTOTYPES_BOOL true

src/execution/frames.cc

@@ -267,7 +267,7 @@ bool SafeStackFrameIterator::IsNoFrameBytecodeHandlerPc(Isolate* isolate,
   // Return false for builds with non-embedded bytecode handlers.
   if (Isolate::CurrentEmbeddedBlobCode() == nullptr) return false;
 
-  EmbeddedData d = EmbeddedData::FromBlob();
+  EmbeddedData d = EmbeddedData::FromBlob(isolate);
   if (pc < d.InstructionStartOfBytecodeHandlers() ||
       pc >= d.InstructionEndOfBytecodeHandlers()) {
     // Not a bytecode handler pc address.

src/execution/isolate.cc

@@ -3283,7 +3283,7 @@ void CreateOffHeapTrampolines(Isolate* isolate) {
   HandleScope scope(isolate);
   Builtins* builtins = isolate->builtins();
 
-  EmbeddedData d = EmbeddedData::FromBlob();
+  EmbeddedData d = EmbeddedData::FromBlob(isolate);
 
   STATIC_ASSERT(Builtins::kAllBuiltinsAreIsolateIndependent);
   for (int i = 0; i < Builtins::builtin_count; i++) {
@@ -3371,15 +3371,32 @@ void Isolate::CreateAndSetEmbeddedBlob() {
     SetStickyEmbeddedBlob(code, code_size, data, data_size);
   }
 
+  MaybeRemapEmbeddedBuiltinsIntoCodeRange();
+
   CreateOffHeapTrampolines(this);
 }
 
+void Isolate::MaybeRemapEmbeddedBuiltinsIntoCodeRange() {
+  if (!FLAG_short_builtin_calls || !RequiresCodeRange()) return;
+
+  CHECK_NOT_NULL(embedded_blob_code_);
+  CHECK_NE(embedded_blob_code_size_, 0);
+
+  embedded_blob_code_ = heap_.RemapEmbeddedBuiltinsIntoCodeRange(
+      embedded_blob_code_, embedded_blob_code_size_);
+  CHECK_NOT_NULL(embedded_blob_code_);
+  // The un-embedded code blob is already a part of the registered code range
+  // so it's not necessary to register it again.
+}
+
 void Isolate::TearDownEmbeddedBlob() {
   // Nothing to do in case the blob is embedded into the binary or unset.
   if (StickyEmbeddedBlobCode() == nullptr) return;
 
+  if (!FLAG_short_builtin_calls) {
     CHECK_EQ(embedded_blob_code(), StickyEmbeddedBlobCode());
     CHECK_EQ(embedded_blob_data(), StickyEmbeddedBlobData());
+  }
   CHECK_EQ(CurrentEmbeddedBlobCode(), StickyEmbeddedBlobCode());
   CHECK_EQ(CurrentEmbeddedBlobData(), StickyEmbeddedBlobData());
 
@@ -3388,8 +3405,10 @@ void Isolate::TearDownEmbeddedBlob() {
   if (current_embedded_blob_refs_ == 0 && enable_embedded_blob_refcounting_) {
     // We own the embedded blob and are the last holder. Free it.
     InstructionStream::FreeOffHeapInstructionStream(
-        const_cast<uint8_t*>(embedded_blob_code()), embedded_blob_code_size(),
-        const_cast<uint8_t*>(embedded_blob_data()), embedded_blob_data_size());
+        const_cast<uint8_t*>(CurrentEmbeddedBlobCode()),
+        embedded_blob_code_size(),
+        const_cast<uint8_t*>(CurrentEmbeddedBlobData()),
+        embedded_blob_data_size());
     ClearEmbeddedBlob();
   }
 }
@@ -3582,6 +3601,7 @@ bool Isolate::Init(SnapshotData* startup_snapshot_data,
     CreateAndSetEmbeddedBlob();
   } else {
     setup_delegate_->SetupBuiltins(this);
+    MaybeRemapEmbeddedBuiltinsIntoCodeRange();
   }
 
   // Initialize custom memcopy and memmove functions (must happen after

src/execution/isolate.h

@@ -2002,8 +2002,8 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory {
 
   void InitializeDefaultEmbeddedBlob();
   void CreateAndSetEmbeddedBlob();
+  void MaybeRemapEmbeddedBuiltinsIntoCodeRange();
   void TearDownEmbeddedBlob();
-
   void SetEmbeddedBlob(const uint8_t* code, uint32_t code_size,
                        const uint8_t* data, uint32_t data_size);
   void ClearEmbeddedBlob();

src/flags/flag-definitions.h

@@ -1534,6 +1534,10 @@ DEFINE_BOOL(adjust_os_scheduling_parameters, true,
             "adjust OS specific scheduling params for the isolate")
 DEFINE_BOOL(experimental_flush_embedded_blob_icache, false,
             "Used in an experiment to evaluate icache flushing on certain CPUs")
+DEFINE_BOOL_READONLY(
+    short_builtin_calls, V8_SHORT_BUILTIN_CALLS_BOOL,
+    "Put embedded builtins code into the code range for shorter "
+    "builtin calls/jumps")
 
 // runtime.cc
 DEFINE_BOOL(runtime_call_stats, false, "report runtime call counts and times")

src/heap/heap.cc

@@ -5196,6 +5196,49 @@ void Heap::ReplaceReadOnlySpace(SharedReadOnlySpace* space) {
   read_only_space_ = space;
 }
 
+uint8_t* Heap::RemapEmbeddedBuiltinsIntoCodeRange(
+    const uint8_t* embedded_blob_code, size_t embedded_blob_code_size) {
+  const base::AddressRegion& code_range = memory_allocator()->code_range();
+
+  CHECK_NE(code_range.begin(), kNullAddress);
+  CHECK(!code_range.is_empty());
+
+  v8::PageAllocator* code_page_allocator =
+      memory_allocator()->code_page_allocator();
+
+  const size_t kAllocatePageSize = code_page_allocator->AllocatePageSize();
+  size_t allocate_code_size =
+      RoundUp(embedded_blob_code_size, kAllocatePageSize);
+
+  // Allocate the re-embedded code blob in the end.
+  void* hint = reinterpret_cast<void*>(code_range.end() - allocate_code_size);
+
+  void* embedded_blob_copy = code_page_allocator->AllocatePages(
+      hint, allocate_code_size, kAllocatePageSize, PageAllocator::kNoAccess);
+
+  if (!embedded_blob_copy) {
+    V8::FatalProcessOutOfMemory(
+        isolate(), "Can't allocate space for re-embedded builtins");
+  }
+
+  size_t code_size =
+      RoundUp(embedded_blob_code_size, code_page_allocator->CommitPageSize());
+
+  if (!code_page_allocator->SetPermissions(embedded_blob_copy, code_size,
+                                           PageAllocator::kReadWrite)) {
+    V8::FatalProcessOutOfMemory(isolate(),
+                                "Re-embedded builtins: set permissions");
+  }
+  memcpy(embedded_blob_copy, embedded_blob_code, embedded_blob_code_size);
+
+  if (!code_page_allocator->SetPermissions(embedded_blob_copy, code_size,
+                                           PageAllocator::kReadExecute)) {
+    V8::FatalProcessOutOfMemory(isolate(),
+                                "Re-embedded builtins: set permissions");
+  }
+  return reinterpret_cast<uint8_t*>(embedded_blob_copy);
+}
+
 class StressConcurrentAllocationObserver : public AllocationObserver {
  public:
   explicit StressConcurrentAllocationObserver(Heap* heap)

src/heap/heap.h

@@ -812,6 +812,12 @@ class Heap {
   // Create ObjectStats if live_object_stats_ or dead_object_stats_ are nullptr.
   void CreateObjectStats();
 
+  // If the code range exists, allocates executable pages in the code range and
+  // copies the embedded builtins code blob there. Returns address of the copy.
+  // The builtins code region will be freed with the code range at tear down.
+  uint8_t* RemapEmbeddedBuiltinsIntoCodeRange(const uint8_t* embedded_blob_code,
+                                              size_t embedded_blob_code_size);
+
   // Sets the TearDown state, so no new GC tasks get posted.
   void StartTearDown();
 

src/objects/code.cc

@@ -150,7 +150,14 @@ Address Code::OffHeapInstructionStart() const {
   if (Isolate::CurrentEmbeddedBlobCode() == nullptr) {
     return raw_instruction_size();
   }
-  EmbeddedData d = EmbeddedData::FromBlob();
+  // TODO(11527): pass Isolate as an argument.
+  // GetIsolateFromWritableObject(*this) works for both read-only and writable
+  // objects here because short builtin calls feature requires pointer
+  // compression.
+  EmbeddedData d =
+      FLAG_short_builtin_calls
+          ? EmbeddedData::FromBlob(GetIsolateFromWritableObject(*this))
+          : EmbeddedData::FromBlob();
   return d.InstructionStartOfBuiltin(builtin_index());
 }
 
@@ -159,7 +166,14 @@ Address Code::OffHeapInstructionEnd() const {
   if (Isolate::CurrentEmbeddedBlobCode() == nullptr) {
     return raw_instruction_size();
   }
-  EmbeddedData d = EmbeddedData::FromBlob();
+  // TODO(11527): pass Isolate as an argument.
+  // GetIsolateFromWritableObject(*this) works for both read-only and writable
+  // objects here because short builtin calls feature requires pointer
+  // compression.
+  EmbeddedData d =
+      FLAG_short_builtin_calls
+          ? EmbeddedData::FromBlob(GetIsolateFromWritableObject(*this))
+          : EmbeddedData::FromBlob();
   return d.InstructionStartOfBuiltin(builtin_index()) +
          d.InstructionSizeOfBuiltin(builtin_index());
 }
@@ -889,6 +903,7 @@ void DependentCode::DeoptimizeDependentCodeGroup(
   bool marked = MarkCodeForDeoptimization(group);
   if (marked) {
     DCHECK(AllowCodeDependencyChange::IsAllowed());
+    // TODO(11527): pass Isolate as an argument.
     Deoptimizer::DeoptimizeMarkedCode(GetIsolateFromWritableObject(*this));
   }
 }

src/regexp/arm64/regexp-macro-assembler-arm64.cc

@@ -1407,16 +1407,7 @@ void RegExpMacroAssemblerARM64::CallCheckStackGuardState(Register scratch) {
       ExternalReference::re_check_stack_guard_state(isolate());
   __ Mov(scratch, check_stack_guard_state);
 
-  {
-    UseScratchRegisterScope temps(masm_);
-    Register scratch = temps.AcquireX();
-
-    EmbeddedData d = EmbeddedData::FromBlob();
-    Address entry = d.InstructionStartOfBuiltin(Builtins::kDirectCEntry);
-
-    __ Ldr(scratch, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
-    __ Call(scratch);
-  }
+  __ CallBuiltin(Builtins::kDirectCEntry);
 
   // The input string may have been moved in memory, we need to reload it.
   __ Peek(input_start(), kSystemPointerSize);

src/runtime/runtime-debug.cc

@@ -941,7 +941,7 @@ RUNTIME_FUNCTION(Runtime_ProfileCreateSnapshotDataBlob) {
 
   // Track the embedded blob size as well.
   {
-    i::EmbeddedData d = i::EmbeddedData::FromBlob();
+    i::EmbeddedData d = i::EmbeddedData::FromBlob(isolate);
     PrintF("Embedded blob is %d bytes\n",
            static_cast<int>(d.code_size() + d.data_size()));
   }

src/snapshot/deserializer.cc

@@ -690,7 +690,7 @@ void Deserializer::RelocInfoVisitor::VisitOffHeapTarget(Code host,
   DCHECK(Builtins::IsBuiltinId(builtin_index));
 
   CHECK_NOT_NULL(isolate()->embedded_blob_code());
-  EmbeddedData d = EmbeddedData::FromBlob();
+  EmbeddedData d = EmbeddedData::FromBlob(isolate());
   Address address = d.InstructionStartOfBuiltin(builtin_index);
   CHECK_NE(kNullAddress, address);
 

src/snapshot/embedded/embedded-data.cc

@@ -25,7 +25,7 @@ Builtins::Name InstructionStream::TryLookupCode(Isolate* isolate,
                                                 Address address) {
   if (!PcIsOffHeap(isolate, address)) return Builtins::kNoBuiltinId;
 
-  EmbeddedData d = EmbeddedData::FromBlob();
+  EmbeddedData d = EmbeddedData::FromBlob(isolate);
   if (address < d.InstructionStartOfBuiltin(0)) return Builtins::kNoBuiltinId;
 
   // Note: Addresses within the padding section between builtins (i.e. within

test/cctest/test-code-pages.cc

@@ -70,18 +70,23 @@ bool PagesHasExactPage(std::vector<MemoryRange>* pages, Address search_page,
   return it != pages->end();
 }
 
-bool PagesContainsAddress(std::vector<MemoryRange>* pages,
-                          Address search_address) {
+bool PagesContainsRange(std::vector<MemoryRange>* pages, Address search_address,
+                        size_t size) {
   byte* addr = reinterpret_cast<byte*>(search_address);
   auto it =
-      std::find_if(pages->begin(), pages->end(), [addr](const MemoryRange& r) {
+      std::find_if(pages->begin(), pages->end(), [=](const MemoryRange& r) {
         const byte* page_start = reinterpret_cast<const byte*>(r.start);
         const byte* page_end = page_start + r.length_in_bytes;
-        return addr >= page_start && addr < page_end;
+        return addr >= page_start && (addr + size) <= page_end;
       });
   return it != pages->end();
 }
 
+bool PagesContainsAddress(std::vector<MemoryRange>* pages,
+                          Address search_address) {
+  return PagesContainsRange(pages, search_address, 0);
+}
+
 }  // namespace
 
 TEST(CodeRangeCorrectContents) {
@@ -98,9 +103,19 @@ TEST(CodeRangeCorrectContents) {
   // We should only have the code range and the embedded code range.
   CHECK_EQ(2, pages->size());
   CHECK(PagesHasExactPage(pages, code_range.begin(), code_range.size()));
+  CHECK(PagesHasExactPage(
+      pages, reinterpret_cast<Address>(i_isolate->CurrentEmbeddedBlobCode()),
+      i_isolate->CurrentEmbeddedBlobCodeSize()));
+  if (FLAG_short_builtin_calls) {
+    // In this case embedded blob code must be included via code_range.
+    CHECK(PagesContainsRange(
+        pages, reinterpret_cast<Address>(i_isolate->embedded_blob_code()),
+        i_isolate->embedded_blob_code_size()));
+  } else {
     CHECK(PagesHasExactPage(
         pages, reinterpret_cast<Address>(i_isolate->embedded_blob_code()),
         i_isolate->embedded_blob_code_size()));
+  }
 }
 
 TEST(CodePagesCorrectContents) {

test/cctest/test-code-stub-assembler.cc

@@ -2453,6 +2453,63 @@ TEST(IsDebugActive) {
   *debug_is_active = false;
 }
 
+TEST(CallBuiltin) {
+  Isolate* isolate(CcTest::InitIsolateOnce());
+
+  const int kNumParams = 2;
+  CodeAssemblerTester asm_tester(isolate,
+                                 kNumParams + 1);  // Include receiver.
+  PromiseBuiltinsAssembler m(asm_tester.state());
+
+  {
+    auto receiver = m.Parameter<Object>(1);
+    auto name = m.Parameter<Name>(2);
+    auto context = m.Parameter<Context>(kNumParams + 3);
+
+    auto value = m.CallBuiltin(Builtins::kGetProperty, context, receiver, name);
+    m.Return(value);
+  }
+
+  FunctionTester ft(asm_tester.GenerateCode(), kNumParams);
+
+  Factory* factory = isolate->factory();
+  Handle<Name> name = factory->InternalizeUtf8String("a");
+  Handle<Object> value(Smi::FromInt(153), isolate);
+  Handle<JSObject> object = factory->NewJSObjectWithNullProto();
+  JSObject::AddProperty(isolate, object, name, value, NONE);
+
+  Handle<Object> result = ft.Call(object, name).ToHandleChecked();
+  CHECK_EQ(*value, *result);
+}
+
+TEST(TailCallBuiltin) {
+  Isolate* isolate(CcTest::InitIsolateOnce());
+
+  const int kNumParams = 2;
+  CodeAssemblerTester asm_tester(isolate,
+                                 kNumParams + 1);  // Include receiver.
+  PromiseBuiltinsAssembler m(asm_tester.state());
+
+  {
+    auto receiver = m.Parameter<Object>(1);
+    auto name = m.Parameter<Name>(2);
+    auto context = m.Parameter<Context>(kNumParams + 3);
+
+    m.TailCallBuiltin(Builtins::kGetProperty, context, receiver, name);
+  }
+
+  FunctionTester ft(asm_tester.GenerateCode(), kNumParams);
+
+  Factory* factory = isolate->factory();
+  Handle<Name> name = factory->InternalizeUtf8String("a");
+  Handle<Object> value(Smi::FromInt(153), isolate);
+  Handle<JSObject> object = factory->NewJSObjectWithNullProto();
+  JSObject::AddProperty(isolate, object, name, value, NONE);
+
+  Handle<Object> result = ft.Call(object, name).ToHandleChecked();
+  CHECK_EQ(*value, *result);
+}
+
 class AppendJSArrayCodeStubAssembler : public CodeStubAssembler {
  public:
   AppendJSArrayCodeStubAssembler(compiler::CodeAssemblerState* state,