Commit 83f8464f authored by Dan Elphick's avatar Dan Elphick Committed by Commit Bot

[builtins] Move non-JS linkage builtins code objects into RO_SPACE

Creates an allow-list of builtins that can still go in code_space
including all TFJ builtins and a small manual list that should be pared
down in the future.

For builtins that go in RO_SPACE a Code object is created that contains
no code at all (shrinking its size from 96 bytes to 64 bytes on x64),
but is there to allow the runtime to continue to work since it expects
a Code object.

This reduces code_space from ~152k to ~40k (-112k) and increases
read_only_space from 33k to 108k (+75k) in the snapshot.

Bug: v8:7464, v8:9821, v8:9338, v8:8127
Change-Id: Icc8bfc722bb267a2bcc17e2f1e27bef7f02f2376
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1795358
Commit-Queue: Dan Elphick <delphick@chromium.org>
Reviewed-by: 's avatarJakob Gruber <jgruber@chromium.org>
Reviewed-by: 's avatarMichael Starzinger <mstarzinger@chromium.org>
Cr-Commit-Position: refs/heads/master@{#64377}
parent 348c0f08
......@@ -314,6 +314,7 @@ void Builtins::EmitCodeCreateEvents(Isolate* isolate) {
}
namespace {
enum TrampolineType { kAbort, kJump };
class OffHeapTrampolineGenerator {
public:
......@@ -322,12 +323,16 @@ class OffHeapTrampolineGenerator {
masm_(isolate, CodeObjectRequired::kYes,
ExternalAssemblerBuffer(buffer_, kBufferSize)) {}
CodeDesc Generate(Address off_heap_entry) {
CodeDesc Generate(Address off_heap_entry, TrampolineType type) {
// Generate replacement code that simply tail-calls the off-heap code.
DCHECK(!masm_.has_frame());
{
FrameScope scope(&masm_, StackFrame::NONE);
masm_.JumpToInstructionStream(off_heap_entry);
if (type == TrampolineType::kJump) {
masm_.JumpToInstructionStream(off_heap_entry);
} else {
masm_.Trap();
}
}
CodeDesc desc;
......@@ -351,16 +356,22 @@ constexpr int OffHeapTrampolineGenerator::kBufferSize;
// static
Handle<Code> Builtins::GenerateOffHeapTrampolineFor(
Isolate* isolate, Address off_heap_entry, int32_t kind_specfic_flags) {
Isolate* isolate, Address off_heap_entry, int32_t kind_specfic_flags,
bool generate_jump_to_instruction_stream) {
DCHECK_NOT_NULL(isolate->embedded_blob());
DCHECK_NE(0, isolate->embedded_blob_size());
OffHeapTrampolineGenerator generator(isolate);
CodeDesc desc = generator.Generate(off_heap_entry);
CodeDesc desc =
generator.Generate(off_heap_entry, generate_jump_to_instruction_stream
? TrampolineType::kJump
: TrampolineType::kAbort);
return Factory::CodeBuilder(isolate, desc, Code::BUILTIN)
.set_self_reference(generator.CodeObject())
.set_read_only_data_container(kind_specfic_flags)
.set_self_reference(generator.CodeObject())
.set_is_executable(generate_jump_to_instruction_stream)
.Build();
}
......@@ -370,7 +381,7 @@ Handle<ByteArray> Builtins::GenerateOffHeapTrampolineRelocInfo(
OffHeapTrampolineGenerator generator(isolate);
// Generate a jump to a dummy address as we're not actually interested in the
// generated instruction stream.
CodeDesc desc = generator.Generate(kNullAddress);
CodeDesc desc = generator.Generate(kNullAddress, TrampolineType::kJump);
Handle<ByteArray> reloc_info = isolate->factory()->NewByteArray(
desc.reloc_size, AllocationType::kReadOnly);
......@@ -419,6 +430,64 @@ bool Builtins::AllowDynamicFunction(Isolate* isolate, Handle<JSFunction> target,
return isolate->MayAccess(responsible_context, target_global_proxy);
}
// static
bool Builtins::CodeObjectIsExecutable(int builtin_index) {
// If the runtime/optimized code always knows when executing a given builtin
// that it is a builtin, then that builtin does not need an executable Code
// object. Such Code objects can go in read_only_space (and can even be
// smaller with no branch instruction), thus saving memory.
// Builtins with JS linkage will always have executable Code objects since
// they can be called directly from jitted code with no way of determining
// that they are builtins at generation time. E.g.
// f = Array.of;
// f(1, 2, 3);
if (Builtins::KindOf(builtin_index) == Builtins::TFJ) return true;
// There are some other non-TF builtins that also have JS linkage like
// InterpreterEntryTrampoline which are explicitly allow-listed below.
// TODO(delphick): Some of these builtins do not fit with the above, but
// currently cause problems if they're not executable. This list should be
// pared down as much as possible.
switch (builtin_index) {
case Builtins::kInterpreterEntryTrampoline:
case Builtins::kToNumber:
case Builtins::kI64ToBigInt:
case Builtins::kBigIntToI64:
case Builtins::kCompileLazy:
case Builtins::kCompileLazyDeoptimizedCode:
case Builtins::kAllocateHeapNumber:
case Builtins::kCEntry_Return1_DontSaveFPRegs_ArgvOnStack_NoBuiltinExit:
case Builtins::kCEntry_Return1_DontSaveFPRegs_ArgvOnStack_BuiltinExit:
case Builtins::kCEntry_Return1_DontSaveFPRegs_ArgvInRegister_NoBuiltinExit:
case Builtins::kCEntry_Return1_SaveFPRegs_ArgvOnStack_NoBuiltinExit:
case Builtins::kCEntry_Return1_SaveFPRegs_ArgvOnStack_BuiltinExit:
case Builtins::kCEntry_Return2_DontSaveFPRegs_ArgvOnStack_NoBuiltinExit:
case Builtins::kCEntry_Return2_DontSaveFPRegs_ArgvOnStack_BuiltinExit:
case Builtins::kCEntry_Return2_DontSaveFPRegs_ArgvInRegister_NoBuiltinExit:
case Builtins::kCEntry_Return2_SaveFPRegs_ArgvOnStack_NoBuiltinExit:
case Builtins::kCEntry_Return2_SaveFPRegs_ArgvOnStack_BuiltinExit:
case Builtins::kCallFunction_ReceiverIsNullOrUndefined:
case Builtins::kCallFunction_ReceiverIsNotNullOrUndefined:
case Builtins::kCallFunction_ReceiverIsAny:
case Builtins::kCallBoundFunction:
case Builtins::kCall_ReceiverIsNullOrUndefined:
case Builtins::kCall_ReceiverIsNotNullOrUndefined:
case Builtins::kCall_ReceiverIsAny:
case Builtins::kArgumentsAdaptorTrampoline:
case Builtins::kHandleApiCall:
case Builtins::kInstantiateAsmJs:
case Builtins::kIterableToFixedArrayForWasm:
// required for ia32
case Builtins::kI32PairToBigInt:
case Builtins::kBigIntToI32Pair:
return true;
default:
return false;
}
}
Builtins::Name ExampleBuiltinForTorqueFunctionPointerType(
size_t function_pointer_type_id) {
switch (function_pointer_type_id) {
......
......@@ -172,14 +172,21 @@ class Builtins {
// Creates a trampoline code object that jumps to the given off-heap entry.
// The result should not be used directly, but only from the related Factory
// function.
static Handle<Code> GenerateOffHeapTrampolineFor(Isolate* isolate,
Address off_heap_entry,
int32_t kind_specific_flags);
// TODO(delphick): Come up with a better name since it may not generate an
// executable trampoline.
static Handle<Code> GenerateOffHeapTrampolineFor(
Isolate* isolate, Address off_heap_entry, int32_t kind_specific_flags,
bool generate_jump_to_instruction_stream);
// Generate the RelocInfo ByteArray that would be generated for an offheap
// trampoline.
static Handle<ByteArray> GenerateOffHeapTrampolineRelocInfo(Isolate* isolate);
// Only builtins with JS linkage should ever need to be called via their
// trampoline Code object. The remaining builtins have non-executable Code
// objects.
static bool CodeObjectIsExecutable(int builtin_index);
static bool IsJSEntryVariant(int builtin_index) {
switch (builtin_index) {
case kJSEntry:
......
......@@ -301,6 +301,7 @@ void TurboAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode,
}
// 'code' is always generated ARM code, never THUMB code
DCHECK(code->IsExecutable());
Call(code.address(), rmode, cond, mode);
}
......@@ -2549,6 +2550,8 @@ void TurboAssembler::CallForDeoptimization(Address target, int deopt_id) {
CheckConstPool(false, false);
}
void TurboAssembler::Trap() { stop(); }
} // namespace internal
} // namespace v8
......
......@@ -253,6 +253,8 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
void MovFromFloatParameter(DwVfpRegister dst);
void MovFromFloatResult(DwVfpRegister dst);
void Trap() override;
// Calls Abort(msg) if the condition cond is not satisfied.
// Use --debug-code to enable.
void Assert(Condition cond, AbortReason reason);
......
......@@ -1904,6 +1904,7 @@ void TurboAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode) {
}
}
DCHECK(code->IsExecutable());
if (CanUseNearCallOrJump(rmode)) {
EmbeddedObjectIndex index = AddEmbeddedObject(code);
DCHECK(is_int32(index));
......@@ -3036,6 +3037,8 @@ void TurboAssembler::Check(Condition cond, AbortReason reason) {
Bind(&ok);
}
void TurboAssembler::Trap() { Brk(0); }
void TurboAssembler::Abort(AbortReason reason) {
#ifdef DEBUG
RecordComment("Abort message: ");
......
......@@ -599,6 +599,8 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
inline void Debug(const char* message, uint32_t code, Instr params = BREAK);
void Trap() override;
// Print a message to stderr and abort execution.
void Abort(AbortReason reason);
......
......@@ -64,8 +64,7 @@ AssemblerOptions AssemblerOptions::Default(
// might be run on real hardware.
options.enable_simulator_code = !serializer;
#endif
options.inline_offheap_trampolines &=
!serializer && !generating_embedded_builtin;
options.inline_offheap_trampolines &= !generating_embedded_builtin;
#if V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_ARM64
const base::AddressRegion& code_range =
isolate->heap()->memory_allocator()->code_range();
......
......@@ -1577,6 +1577,7 @@ void Assembler::call(Operand adr) {
void Assembler::call(Handle<Code> code, RelocInfo::Mode rmode) {
EnsureSpace ensure_space(this);
DCHECK(RelocInfo::IsCodeTarget(rmode));
DCHECK(code->IsExecutable());
EMIT(0xE8);
emit(code, rmode);
}
......
......@@ -2093,6 +2093,8 @@ void TurboAssembler::CallForDeoptimization(Address target, int deopt_id) {
call(target, RelocInfo::RUNTIME_ENTRY);
}
void TurboAssembler::Trap() { int3(); }
} // namespace internal
} // namespace v8
......
......@@ -107,6 +107,8 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
void RetpolineJump(Register reg);
void Trap() override;
void CallForDeoptimization(Address target, int deopt_id);
// Call a runtime routine. This expects {centry} to contain a fitting CEntry
......
......@@ -85,6 +85,8 @@ class V8_EXPORT_PRIVATE TurboAssemblerBase : public Assembler {
virtual void LoadRoot(Register destination, RootIndex index) = 0;
virtual void Trap() = 0;
static int32_t RootRegisterOffsetForRootIndex(RootIndex root_index);
static int32_t RootRegisterOffsetForBuiltinIndex(int builtin_index);
......
......@@ -1090,6 +1090,7 @@ void Assembler::call(Address entry, RelocInfo::Mode rmode) {
void Assembler::call(Handle<Code> target, RelocInfo::Mode rmode) {
DCHECK(RelocInfo::IsCodeTarget(rmode));
DCHECK(target->IsExecutable());
EnsureSpace ensure_space(this);
// 1110 1000 #32-bit disp.
emit(0xE8);
......
......@@ -2857,6 +2857,8 @@ void TurboAssembler::CallForDeoptimization(Address target, int deopt_id) {
call(target, RelocInfo::RUNTIME_ENTRY);
}
void TurboAssembler::Trap() { int3(); }
} // namespace internal
} // namespace v8
......
......@@ -385,6 +385,8 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
void CallForDeoptimization(Address target, int deopt_id);
void Trap() override;
// Non-SSE2 instructions.
void Pextrd(Register dst, XMMRegister src, int8_t imm8);
void Pextrw(Register dst, XMMRegister src, int8_t imm8);
......
......@@ -10,6 +10,7 @@
#include "src/diagnostics/disassembler.h"
#include "src/heap/combined-heap.h"
#include "src/heap/heap-write-barrier-inl.h"
#include "src/heap/read-only-heap.h"
#include "src/ic/handler-configuration-inl.h"
#include "src/init/bootstrapper.h"
#include "src/logging/counters.h"
......@@ -1152,7 +1153,13 @@ void Code::CodeVerify(Isolate* isolate) {
CHECK_LE(handler_table_offset(), constant_pool_offset());
CHECK_LE(constant_pool_offset(), code_comments_offset());
CHECK_LE(code_comments_offset(), InstructionSize());
CHECK(IsAligned(raw_instruction_start(), kCodeAlignment));
CHECK_IMPLIES(!ReadOnlyHeap::Contains(*this),
IsAligned(raw_instruction_start(), kCodeAlignment));
// TODO(delphick): Refactor Factory::CodeBuilder::BuildInternal, so that the
// following CHECK works builtin trampolines. It currently fails because
// CodeVerify is called halfway through constructing the trampoline and so not
// everything is set up.
// CHECK_EQ(ReadOnlyHeap::Contains(*this), !IsExecutable());
relocation_info().ObjectVerify(isolate);
CHECK(Code::SizeFor(body_size()) <= kMaxRegularHeapObjectSize ||
isolate->heap()->InSpace(*this, CODE_LO_SPACE));
......
......@@ -2814,10 +2814,11 @@ V8_EXPORT_PRIVATE extern void _v8_internal_Print_Code(void* object) {
if (!isolate->heap()->InSpaceSlow(address, i::CODE_SPACE) &&
!isolate->heap()->InSpaceSlow(address, i::LO_SPACE) &&
!i::InstructionStream::PcIsOffHeap(isolate, address)) {
!i::InstructionStream::PcIsOffHeap(isolate, address) &&
!i::ReadOnlyHeap::Contains(address)) {
i::PrintF(
"%p is not within the current isolate's large object, code or embedded "
"spaces\n",
"%p is not within the current isolate's large object, code, read_only "
"or embedded spaces\n",
object);
return;
}
......
......@@ -489,7 +489,8 @@ Code StackFrame::LookupCode() const {
void StackFrame::IteratePc(RootVisitor* v, Address* pc_address,
Address* constant_pool_address, Code holder) {
Address pc = *pc_address;
DCHECK(holder.GetHeap()->GcSafeCodeContains(holder, pc));
DCHECK(ReadOnlyHeap::Contains(holder) ||
holder.GetHeap()->GcSafeCodeContains(holder, pc));
unsigned pc_offset = static_cast<unsigned>(pc - holder.InstructionStart());
Object code = holder;
v->VisitRootPointer(Root::kTop, nullptr, FullObjectSlot(&code));
......
......@@ -10,6 +10,7 @@
#include "src/builtins/accessors.h"
#include "src/builtins/constants-table-builder.h"
#include "src/codegen/compiler.h"
#include "src/common/globals.h"
#include "src/execution/isolate-inl.h"
#include "src/execution/protectors-inl.h"
#include "src/heap/heap-inl.h"
......@@ -117,12 +118,14 @@ MaybeHandle<Code> Factory::CodeBuilder::BuildInternal(
CodePageCollectionMemoryModificationScope code_allocation(heap);
HeapObject result;
AllocationType allocation_type =
is_executable_ ? AllocationType::kCode : AllocationType::kReadOnly;
if (retry_allocation_or_fail) {
result = heap->AllocateRawWith<Heap::kRetryOrFail>(object_size,
AllocationType::kCode);
allocation_type);
} else {
result = heap->AllocateRawWith<Heap::kLightRetry>(object_size,
AllocationType::kCode);
allocation_type);
// Return an empty handle if we cannot allocate the code object.
if (result.is_null()) return MaybeHandle<Code>();
}
......@@ -137,10 +140,12 @@ MaybeHandle<Code> Factory::CodeBuilder::BuildInternal(
result.set_map_after_allocation(*factory->code_map(), SKIP_WRITE_BARRIER);
code = handle(Code::cast(result), isolate_);
DCHECK(IsAligned(code->address(), kCodeAlignment));
DCHECK_IMPLIES(
!heap->memory_allocator()->code_range().is_empty(),
heap->memory_allocator()->code_range().contains(code->address()));
if (is_executable_) {
DCHECK(IsAligned(code->address(), kCodeAlignment));
DCHECK_IMPLIES(
!heap->memory_allocator()->code_range().is_empty(),
heap->memory_allocator()->code_range().contains(code->address()));
}
constexpr bool kIsNotOffHeapTrampoline = false;
const bool has_unwinding_info = code_desc_.unwinding_info != nullptr;
......@@ -2614,9 +2619,12 @@ Handle<Code> Factory::NewOffHeapTrampolineFor(Handle<Code> code,
CHECK_NE(0, isolate()->embedded_blob_size());
CHECK(Builtins::IsIsolateIndependentBuiltin(*code));
bool generate_jump_to_instruction_stream =
Builtins::CodeObjectIsExecutable(code->builtin_index());
Handle<Code> result = Builtins::GenerateOffHeapTrampolineFor(
isolate(), off_heap_entry,
code->code_data_container().kind_specific_flags());
code->code_data_container().kind_specific_flags(),
generate_jump_to_instruction_stream);
// The CodeDataContainer should not be modified beyond this point since it's
// now possibly canonicalized.
......@@ -2642,7 +2650,9 @@ Handle<Code> Factory::NewOffHeapTrampolineFor(Handle<Code> code,
// canonical one stored in the roots to avoid duplicating it for every
// single builtin.
ByteArray canonical_reloc_info =
ReadOnlyRoots(isolate()).off_heap_trampoline_relocation_info();
generate_jump_to_instruction_stream
? ReadOnlyRoots(isolate()).off_heap_trampoline_relocation_info()
: ReadOnlyRoots(isolate()).empty_byte_array();
#ifdef DEBUG
// Verify that the contents are the same.
ByteArray reloc_info = result->relocation_info();
......
......@@ -961,6 +961,11 @@ class V8_EXPORT_PRIVATE Factory {
return *this;
}
CodeBuilder& set_is_executable(bool executable) {
is_executable_ = executable;
return *this;
}
// Indicates the CodeDataContainer should be allocated in read-only space.
// As an optimization, if the kind-specific flags match that of a canonical
// container, it will be used instead.
......@@ -988,6 +993,7 @@ class V8_EXPORT_PRIVATE Factory {
Handle<ByteArray> source_position_table_;
Handle<DeoptimizationData> deoptimization_data_ =
DeoptimizationData::Empty(isolate_);
bool is_executable_ = true;
bool read_only_data_container_ = false;
bool is_movable_ = true;
bool is_turbofanned_ = false;
......
......@@ -130,6 +130,11 @@ void ReadOnlyHeap::ClearSharedHeapForTest() {
#endif
}
// static
bool ReadOnlyHeap::Contains(Address address) {
return MemoryChunk::FromAddress(address)->InReadOnlySpace();
}
// static
bool ReadOnlyHeap::Contains(HeapObject object) {
return MemoryChunk::FromHeapObject(object)->InReadOnlySpace();
......
......@@ -42,6 +42,8 @@ class ReadOnlyHeap final {
// and it may be safely disposed of.
void OnHeapTearDown();
// Returns whether the address is within the read-only space.
V8_EXPORT_PRIVATE static bool Contains(Address address);
// Returns whether the object resides in the read-only space.
V8_EXPORT_PRIVATE static bool Contains(HeapObject object);
// Gets read-only roots from an appropriate root list: shared read-only root
......
......@@ -597,6 +597,11 @@ bool Code::IsWeakObjectInOptimizedCode(HeapObject object) {
InstanceTypeChecker::IsContext(instance_type);
}
bool Code::IsExecutable() {
return !Builtins::IsBuiltinId(builtin_index()) || !is_off_heap_trampoline() ||
Builtins::CodeObjectIsExecutable(builtin_index());
}
// This field has to have relaxed atomic accessors because it is accessed in the
// concurrent marker.
RELAXED_INT32_ACCESSORS(CodeDataContainer, kind_specific_flags,
......
......@@ -370,7 +370,11 @@ class Code : public HeapObject {
static inline bool IsWeakObjectInOptimizedCode(HeapObject object);
// Return true if the function is inlined in the code.
// Returns false if this is an embedded builtin Code object that's in
// read_only_space and hence doesn't have execute permissions.
inline bool IsExecutable();
// Returns true if the function is inlined in the code.
bool Inlines(SharedFunctionInfo sfi);
class OptimizedCodeIterator;
......
......@@ -405,7 +405,8 @@ HeapObject Deserializer::ReadObject(SnapshotSpace space) {
#ifdef DEBUG
if (obj.IsCode()) {
DCHECK_EQ(space, SnapshotSpace::kCode);
DCHECK(space == SnapshotSpace::kCode ||
space == SnapshotSpace::kReadOnlyHeap);
} else {
DCHECK_NE(space, SnapshotSpace::kCode);
}
......
......@@ -3594,7 +3594,9 @@ TEST(TestCallBuiltinInlineTrampoline) {
Handle<String>::cast(result.ToHandleChecked())));
}
TEST(TestCallBuiltinIndirectLoad) {
// TODO(v8:9821): Remove the option to disable inlining off-heap trampolines
// along with this test.
DISABLED_TEST(TestCallBuiltinIndirectLoad) {
if (!i::FLAG_embedded_builtins) return;
Isolate* isolate(CcTest::InitIsolateOnce());
const int kNumParams = 1;
......
......@@ -291,7 +291,7 @@ TEST(DisasmIa320) {
__ bind(&L2);
__ call(Operand(ebx, ecx, times_4, 10000));
__ nop();
Handle<Code> ic = BUILTIN_CODE(isolate, LoadIC);
Handle<Code> ic = BUILTIN_CODE(isolate, ArrayFrom);
__ call(ic, RelocInfo::CODE_TARGET);
__ nop();
__ call(FUNCTION_ADDR(DummyStaticFunction), RelocInfo::RUNTIME_ENTRY);
......
......@@ -286,7 +286,7 @@ TEST(DisasmX64) {
// TODO(mstarzinger): The following is protected.
// __ call(Operand(rbx, rcx, times_4, 10000));
__ nop();
Handle<Code> ic = BUILTIN_CODE(isolate, LoadIC);
Handle<Code> ic = BUILTIN_CODE(isolate, ArrayFrom);
__ call(ic, RelocInfo::CODE_TARGET);
__ nop();
__ nop();
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment