[code] Separate instruction and metadata areas

In this CL, Code object layout changes s.t. the instruction
area is distinct / non-overlapping from the metadata area.

On-heap Code objects now have a variable-size `body` area,
containing distinct-but-adjacent `instruction` and `metadata`
areas.

Off-heap code (= embedded builtins) currently have the same,
but in the future the metadata area will move elsewhere and
no longer be adjacent to instructions.

To implement this, the main changes are:

 - The Code object header now contains instruction and metadata
   sizes, and no longer contains the safepoint table offset
   (it's implicitly the first table of the metadata section).
 - The embedded metadata table contains information about both
   instruction and metadata areas.

I've also added assertions in spots that currently rely on a
contiguous body area.

Bug: v8:11036
Change-Id: I940f0c70c07ad511dafd2d2c3e337de8c92cd4b9
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2491025Reviewed-by: Leszek Swirski <leszeks@chromium.org>
Reviewed-by: Clemens Backes <clemensb@chromium.org>
Reviewed-by: Dominik Inführ <dinfuehr@chromium.org>
Commit-Queue: Jakob Gruber <jgruber@chromium.org>
Cr-Commit-Position: refs/heads/master@{#70743}

src/builtins/setup-builtins-internal.cc

@@ -257,8 +257,8 @@ void SetupIsolateDelegate::ReplacePlaceholders(Isolate* isolate) {
       flush_icache = true;
     }
     if (flush_icache) {
-      FlushInstructionCache(code.raw_instruction_start_future(),
-                            code.raw_instruction_size_future());
+      FlushInstructionCache(code.raw_instruction_start(),
+                            code.raw_instruction_size());
     }
   }
 }

src/codegen/code-desc.h

@@ -62,6 +62,12 @@ class CodeDesc {
   int code_comments_offset = 0;
   int code_comments_size = 0;
 
+  // TODO(jgruber,v8:11036): Remove these functions once CodeDesc fields have
+  // been made consistent with Code layout.
+  int body_size() const { return instr_size + unwinding_info_size; }
+  int instruction_size() const { return safepoint_table_offset; }
+  int metadata_size() const { return body_size() - instruction_size(); }
+
   // Relocation info is located at the end of the buffer and not part of the
   // instructions area.
 

src/execution/isolate.cc

@@ -386,10 +386,12 @@ size_t Isolate::HashIsolateForEmbeddedBlob() {
         reinterpret_cast<uint8_t*>(code.ptr() - kHeapObjectTag);
 
     // These static asserts ensure we don't miss relevant fields. We don't hash
-    // instruction size and flags since they change when creating the off-heap
-    // trampolines. Other data fields must remain the same.
+    // instruction/metadata size and flags since they change when creating the
+    // off-heap trampolines. Other data fields must remain the same.
     STATIC_ASSERT(Code::kInstructionSizeOffset == Code::kDataStart);
-    STATIC_ASSERT(Code::kFlagsOffset == Code::kInstructionSizeOffsetEnd + 1);
+    STATIC_ASSERT(Code::kMetadataSizeOffset ==
+                  Code::kInstructionSizeOffsetEnd + 1);
+    STATIC_ASSERT(Code::kFlagsOffset == Code::kMetadataSizeOffsetEnd + 1);
     STATIC_ASSERT(Code::kBuiltinIndexOffset == Code::kFlagsOffsetEnd + 1);
     static constexpr int kStartOffset = Code::kBuiltinIndexOffset;
 

src/heap/code-stats.cc

@@ -212,9 +212,7 @@ void CodeStatistics::CollectCodeCommentStatistics(HeapObject obj,
     cit.Next();
   }
 
-  // TODO(jgruber,v8:11036): Revisit this when separating instruction- and
-  // metadata areas. The logic will become a bit more complex since these areas
-  // will no longer be adjacent in some cases.
+  STATIC_ASSERT(Code::kBodyIsContiguous);
   DCHECK(0 <= prev_pc_offset && prev_pc_offset <= code.raw_body_size());
   delta += static_cast<int>(code.raw_body_size() - prev_pc_offset);
   EnterComment(isolate, "NoComment", delta);

src/heap/factory.cc

@@ -120,8 +120,8 @@ MaybeHandle<Code> Factory::CodeBuilder::BuildInternal(
   }
 
   // TODO(jgruber,v8:11036): Distinguish instruction and metadata areas.
-  const int body_size = code_desc_.instr_size + code_desc_.unwinding_info_size;
-  const int object_size = Code::SizeFor(body_size);
+  STATIC_ASSERT(Code::kOnHeapBodyIsContiguous);
+  const int object_size = Code::SizeFor(code_desc_.body_size());
 
   Handle<Code> code;
   {
@@ -157,8 +157,8 @@ MaybeHandle<Code> Factory::CodeBuilder::BuildInternal(
 
     constexpr bool kIsNotOffHeapTrampoline = false;
 
-    // TODO(jgruber,v8:11036): Distinguish instruction and metadata areas.
-    code->set_raw_instruction_size(body_size);
+    code->set_raw_instruction_size(code_desc_.instruction_size());
+    code->set_raw_metadata_size(code_desc_.metadata_size());
     code->set_relocation_info(*reloc_info);
     code->initialize_flags(kind_, is_turbofanned_, stack_slots_,
                            kIsNotOffHeapTrampoline);
@@ -167,7 +167,6 @@ MaybeHandle<Code> Factory::CodeBuilder::BuildInternal(
     code->set_code_data_container(*data_container, kReleaseStore);
     code->set_deoptimization_data(*deoptimization_data_);
     code->set_source_position_table(*source_position_table_);
-    code->set_safepoint_table_offset(code_desc_.safepoint_table_offset);
     code->set_handler_table_offset(code_desc_.handler_table_offset);
     code->set_constant_pool_offset(code_desc_.constant_pool_offset);
     code->set_code_comments_offset(code_desc_.code_comments_offset);
@@ -2057,6 +2056,11 @@ Handle<Code> Factory::NewOffHeapTrampolineFor(Handle<Code> code,
       isolate(), off_heap_entry,
       code->code_data_container(kAcquireLoad).kind_specific_flags(),
       generate_jump_to_instruction_stream);
+
+  // Trampolines may not contain any metadata since all metadata offsets,
+  // stored on the Code object, refer to the off-heap metadata area.
+  CHECK_EQ(result->raw_metadata_size(), 0);
+
   // The CodeDataContainer should not be modified beyond this point since it's
   // now possibly canonicalized.
 
@@ -2071,7 +2075,6 @@ Handle<Code> Factory::NewOffHeapTrampolineFor(Handle<Code> code,
     result->initialize_flags(code->kind(), code->is_turbofanned(), stack_slots,
                              set_is_off_heap_trampoline);
     result->set_builtin_index(code->builtin_index());
-    result->set_safepoint_table_offset(code->safepoint_table_offset());
     result->set_handler_table_offset(code->handler_table_offset());
     result->set_constant_pool_offset(code->constant_pool_offset());
     result->set_code_comments_offset(code->code_comments_offset());

src/objects/code-inl.h

@@ -164,7 +164,7 @@ OBJECT_CONSTRUCTORS_IMPL(Code, HeapObject)
 NEVER_READ_ONLY_SPACE_IMPL(Code)
 
 INT_ACCESSORS(Code, raw_instruction_size, kInstructionSizeOffset)
-INT_ACCESSORS(Code, safepoint_table_offset, kSafepointTableOffsetOffset)
+INT_ACCESSORS(Code, raw_metadata_size, kMetadataSizeOffset)
 INT_ACCESSORS(Code, handler_table_offset, kHandlerTableOffsetOffset)
 INT_ACCESSORS(Code, code_comments_offset, kCodeCommentsOffsetOffset)
 INT32_ACCESSORS(Code, unwinding_info_offset, kUnwindingInfoOffsetOffset)
@@ -222,18 +222,30 @@ void Code::set_next_code_link(Object value) {
 
 Address Code::raw_body_start() const { return raw_instruction_start(); }
 
-Address Code::raw_body_end() const { return raw_instruction_end(); }
+Address Code::BodyStart() const {
+  STATIC_ASSERT(kBodyIsContiguous);
+  return InstructionStart();
+}
+
+Address Code::raw_body_end() const {
+  return raw_body_start() + raw_body_size();
+}
 
 int Code::raw_body_size() const {
-  // TODO(jgruber,v8:11036): Distinguish instruction and metadata areas.
-  DCHECK_EQ(unwinding_info_offset() + unwinding_info_size(), InstructionSize());
-  return raw_instruction_size();
+  return raw_instruction_size() + raw_metadata_size();
+}
+
+Address Code::BodyEnd() const {
+  STATIC_ASSERT(kBodyIsContiguous);
+  return MetadataEnd();
 }
 
 int Code::BodySize() const {
-  // TODO(jgruber,v8:11036): Distinguish instruction and metadata areas.
-  DCHECK_EQ(unwinding_info_offset() + unwinding_info_size(), InstructionSize());
-  return InstructionSize();
+  // TODO(jgruber,v8:11036): Update once embedded instructions and metadata are
+  // separate.
+  return V8_UNLIKELY(is_off_heap_trampoline())
+             ? OffHeapInstructionSize() + OffHeapMetadataSize()
+             : raw_body_size();
 }
 
 int Code::InstructionSize() const {
@@ -259,27 +271,37 @@ Address Code::InstructionEnd() const {
                                                : raw_instruction_end();
 }
 
-Address Code::raw_instruction_start_future() const {
-  return raw_instruction_start();
+Address Code::raw_metadata_start() const {
+  return raw_instruction_start() + raw_instruction_size();
 }
 
-Address Code::raw_instruction_end_future() const {
-  return raw_metadata_start();
+Address Code::MetadataStart() const {
+  STATIC_ASSERT(kBodyIsContiguous);
+  return V8_UNLIKELY(is_off_heap_trampoline()) ? OffHeapMetadataStart()
+                                               : raw_metadata_start();
 }
 
-int Code::raw_instruction_size_future() const {
-  return raw_instruction_size() - raw_metadata_size();
+Address Code::raw_metadata_end() const {
+  return raw_metadata_start() + raw_metadata_size();
 }
 
-Address Code::raw_metadata_start() const {
-  return raw_instruction_start() + safepoint_table_offset();
+Address Code::MetadataEnd() const {
+  STATIC_ASSERT(kBodyIsContiguous);
+  return V8_UNLIKELY(is_off_heap_trampoline()) ? OffHeapMetadataEnd()
+                                               : raw_metadata_end();
 }
 
-Address Code::raw_metadata_end() const { return raw_instruction_end(); }
+int Code::MetadataSize() const {
+  STATIC_ASSERT(kBodyIsContiguous);
+  return V8_UNLIKELY(is_off_heap_trampoline()) ? OffHeapMetadataSize()
+                                               : raw_metadata_size();
+}
 
-int Code::raw_metadata_size() const {
-  DCHECK_LE(raw_metadata_start(), raw_metadata_end());
-  return static_cast<int>(raw_metadata_end() - raw_metadata_start());
+int Code::safepoint_table_offset() const {
+  // TODO(jgruber,v8:11036): Update once embedded instructions and metadata are
+  // separate.
+  STATIC_ASSERT(kBodyIsContiguous);
+  return InstructionSize();
 }
 
 int Code::SizeIncludingMetadata() const {
@@ -555,7 +577,7 @@ Address Code::unwinding_info_start() const {
   return InstructionStart() + unwinding_info_offset();
 }
 
-Address Code::unwinding_info_end() const { return InstructionEnd(); }
+Address Code::unwinding_info_end() const { return MetadataEnd(); }
 
 int Code::unwinding_info_size() const {
   DCHECK_GE(unwinding_info_end(), unwinding_info_start());

src/objects/code.cc

@@ -88,14 +88,13 @@ void Code::Relocate(intptr_t delta) {
 }
 
 void Code::FlushICache() const {
-  // TODO(jgruber,v8:11036): This should likely flush only actual instructions,
-  // not metadata.
   FlushInstructionCache(raw_instruction_start(), raw_instruction_size());
 }
 
 void Code::CopyFromNoFlush(Heap* heap, const CodeDesc& desc) {
   // Copy code.
   // TODO(jgruber,v8:11036): Distinguish instruction and metadata areas.
+  STATIC_ASSERT(kOnHeapBodyIsContiguous);
   CopyBytes(reinterpret_cast<byte*>(raw_instruction_start()), desc.buffer,
             static_cast<size_t>(desc.instr_size));
   // TODO(jgruber,v8:11036): Merge with the above.
@@ -140,29 +139,60 @@ SafepointEntry Code::GetSafepointEntry(Address pc) {
 
 int Code::OffHeapInstructionSize() const {
   DCHECK(is_off_heap_trampoline());
-  if (Isolate::CurrentEmbeddedBlobCode() == nullptr)
+  if (Isolate::CurrentEmbeddedBlobCode() == nullptr) {
     return raw_instruction_size();
+  }
   EmbeddedData d = EmbeddedData::FromBlob();
   return d.InstructionSizeOfBuiltin(builtin_index());
 }
 
 Address Code::OffHeapInstructionStart() const {
   DCHECK(is_off_heap_trampoline());
-  if (Isolate::CurrentEmbeddedBlobCode() == nullptr)
-    return raw_instruction_start();
+  if (Isolate::CurrentEmbeddedBlobCode() == nullptr) {
+    return raw_instruction_size();
+  }
   EmbeddedData d = EmbeddedData::FromBlob();
   return d.InstructionStartOfBuiltin(builtin_index());
 }
 
 Address Code::OffHeapInstructionEnd() const {
   DCHECK(is_off_heap_trampoline());
-  if (Isolate::CurrentEmbeddedBlobCode() == nullptr)
-    return raw_instruction_end();
+  if (Isolate::CurrentEmbeddedBlobCode() == nullptr) {
+    return raw_instruction_size();
+  }
   EmbeddedData d = EmbeddedData::FromBlob();
   return d.InstructionStartOfBuiltin(builtin_index()) +
          d.InstructionSizeOfBuiltin(builtin_index());
 }
 
+int Code::OffHeapMetadataSize() const {
+  DCHECK(is_off_heap_trampoline());
+  if (Isolate::CurrentEmbeddedBlobCode() == nullptr) {
+    return raw_instruction_size();
+  }
+  EmbeddedData d = EmbeddedData::FromBlob();
+  return d.MetadataSizeOfBuiltin(builtin_index());
+}
+
+Address Code::OffHeapMetadataStart() const {
+  DCHECK(is_off_heap_trampoline());
+  if (Isolate::CurrentEmbeddedBlobCode() == nullptr) {
+    return raw_instruction_size();
+  }
+  EmbeddedData d = EmbeddedData::FromBlob();
+  return d.MetadataStartOfBuiltin(builtin_index());
+}
+
+Address Code::OffHeapMetadataEnd() const {
+  DCHECK(is_off_heap_trampoline());
+  if (Isolate::CurrentEmbeddedBlobCode() == nullptr) {
+    return raw_instruction_size();
+  }
+  EmbeddedData d = EmbeddedData::FromBlob();
+  return d.MetadataStartOfBuiltin(builtin_index()) +
+         d.MetadataSizeOfBuiltin(builtin_index());
+}
+
 int AbstractCode::SourcePosition(int offset) {
   Object maybe_table = source_position_table();
   if (maybe_table.IsException()) return kNoSourcePosition;

src/objects/code.h

@@ -83,45 +83,45 @@ class Code : public HeapObject {
   //  +--------------------------+
   //  |          header          |
   //  | padded to code alignment |
-  //  +--------------------------+  <-- raw_instruction_start() (= IS)
-  //  |       instructions       |
+  //  +--------------------------+  <-- raw_body_start()
+  //  |       instructions       |   == raw_instruction_start() (IS)
   //  |           ...            |
-  //  +--------------------------+  <-- raw_metadata_start()
-  //  |         metadata         |
+  //  +--------------------------+  <-- raw_instruction_end()
+  //  |         metadata         |   == raw_metadata_start()
   //  |           ...            |   == IS + safepoint_table_offset()
+  //  |                          |
   //  |                          |  <-- IS + handler_table_offset()
   //  |                          |  <-- IS + constant_pool_offset()
   //  |                          |  <-- IS + code_comments_offset()
   //  |                          |  <-- IS + unwinding_info_offset()
   //  | padded to obj alignment  |
-  //  +--------------------------+  <-- raw_metadata_end()
-  //  |                          |   == raw_instruction_end()
+  //  +--------------------------+  <-- raw_metadata_end() == raw_body_end()
   //  | padded to code alignment |
   //  +--------------------------+
   //
   // In other words, the variable-size 'body' consists of 'instructions' and
-  // 'metadata'. Both are currently inside [InstructionStart,InstructionEnd[.
+  // 'metadata'.
   //
   // Note the accessor functions below may be prefixed with 'raw'. In this case,
   // raw accessors (e.g. raw_instruction_start) always refer to the on-heap
   // Code object, while camel-case accessors (e.g. InstructionStart) may refer
   // to an off-heap area in the case of embedded builtins.
 
+  // TODO(jgruber,v8:11036): Update once no longer true for embedded builtins.
+  static constexpr bool kOnHeapBodyIsContiguous = true;
+  static constexpr bool kOffHeapBodyIsContiguous = true;
+  static constexpr bool kBodyIsContiguous =
+      kOnHeapBodyIsContiguous && kOffHeapBodyIsContiguous;
+
   inline Address raw_body_start() const;
+  inline Address BodyStart() const;
   inline Address raw_body_end() const;
+  inline Address BodyEnd() const;
   inline int raw_body_size() const;
-
   // TODO(jgruber,v8:11036): Replace this once the off-heap instruction and
   // metadata areas are separate.
   inline int BodySize() const;
 
-  // TODO(jgruber,v8:11036): Shrink the instructions area to contain only
-  // instructions. Until that happens, the state of the world is:
-  //
-  // raw_body_start == raw_instruction_start
-  // raw_metadata_start == raw_instruction_start + safepoint_table_offset
-  // raw_body_end == raw_instruction_end == raw_metadata_end
-
   inline Address raw_instruction_start() const;
   inline Address InstructionStart() const;
   V8_EXPORT_PRIVATE Address OffHeapInstructionStart() const;
@@ -135,21 +135,23 @@ class Code : public HeapObject {
   inline int InstructionSize() const;
   V8_EXPORT_PRIVATE int OffHeapInstructionSize() const;
 
-  // TODO(jgruber,v8:11036): Replace legacy accessors with these _future
-  // accessors. The _future accessors only refer to the range of executable
-  // instructions, *without* metadata tables.
-  inline Address raw_instruction_start_future() const;
-  inline Address raw_instruction_end_future() const;
-  inline int raw_instruction_size_future() const;
-
   inline Address raw_metadata_start() const;
+  inline Address MetadataStart() const;
+  Address OffHeapMetadataStart() const;
   inline Address raw_metadata_end() const;
+  inline Address MetadataEnd() const;
+  V8_EXPORT_PRIVATE Address OffHeapMetadataEnd() const;
   inline int raw_metadata_size() const;
+  inline void set_raw_metadata_size(int value);
+  inline int MetadataSize() const;
+  int OffHeapMetadataSize() const;
+
+  // TODO(jgruber,v8:11036): Change all these offsets to be relative to
+  // MetadataStart instead of InstructionStart.
 
   // [safepoint_table_offset]: If {has_safepoint_info()}, the offset in the
   // instruction stream where the safepoint table starts.
   inline int safepoint_table_offset() const;
-  inline void set_safepoint_table_offset(int offset);
   Address SafepointTableAddress() const;
   int safepoint_table_size() const;
   bool has_safepoint_table() const;
@@ -413,11 +415,12 @@ class Code : public HeapObject {
   /* Objects embedded into code is visited via reloc info. */             \
   V(kDataStart, 0)                                                        \
   V(kInstructionSizeOffset, kIntSize)                                     \
+  V(kMetadataSizeOffset, kIntSize)                                        \
   V(kFlagsOffset, kInt32Size)                                             \
   V(kBuiltinIndexOffset, kIntSize)                                        \
   V(kInlinedBytecodeSizeOffset, kIntSize)                                 \
-  /* Offsets describing inline metadata tables. */                        \
-  V(kSafepointTableOffsetOffset, kIntSize)                                \
+  /* Offsets describing inline metadata tables, relative to */            \
+  /* InstructionStart. */                                                 \
   V(kHandlerTableOffsetOffset, kIntSize)                                  \
   V(kConstantPoolOffsetOffset,                                            \
     FLAG_enable_embedded_constant_pool ? kIntSize : 0)                    \

src/regexp/regexp-macro-assembler.cc

@@ -172,7 +172,7 @@ int NativeRegExpMacroAssembler::CheckStackGuardState(
   DisallowHeapAllocation no_gc;
   Address old_pc = PointerAuthentication::AuthenticatePC(return_address, 0);
   DCHECK_LE(re_code.raw_instruction_start(), old_pc);
-  DCHECK_LE(old_pc, re_code.raw_instruction_end_future());
+  DCHECK_LE(old_pc, re_code.raw_instruction_end());
 
   StackLimitCheck check(isolate);
   bool js_has_overflowed = check.JsHasOverflowed();

src/snapshot/embedded/embedded-data.cc

@@ -224,11 +224,16 @@ EmbeddedData EmbeddedData::FromIsolate(Isolate* isolate) {
               Builtins::name(i));
     }
 
-    uint32_t length = static_cast<uint32_t>(code.raw_body_size());
+    uint32_t instruction_size =
+        static_cast<uint32_t>(code.raw_instruction_size());
+    uint32_t metadata_size = static_cast<uint32_t>(code.raw_metadata_size());
+    uint32_t length = instruction_size + metadata_size;
 
     DCHECK_EQ(0, raw_code_size % kCodeAlignment);
-    metadata[i].instructions_offset = raw_code_size;
-    metadata[i].instructions_length = length;
+    metadata[i].instruction_offset = raw_code_size;
+    metadata[i].instruction_length = instruction_size;
+    metadata[i].metadata_offset = raw_code_size + instruction_size;
+    metadata[i].metadata_length = metadata_size;
 
     // Align the start of each instruction stream.
     raw_code_size += PadAndAlign(length);
@@ -265,8 +270,9 @@ EmbeddedData EmbeddedData::FromIsolate(Isolate* isolate) {
   // Write the raw data section.
   STATIC_ASSERT(Builtins::kAllBuiltinsAreIsolateIndependent);
   for (int i = 0; i < Builtins::builtin_count; i++) {
+    STATIC_ASSERT(Code::kBodyIsContiguous);
     Code code = builtins->builtin(i);
-    uint32_t offset = metadata[i].instructions_offset;
+    uint32_t offset = metadata[i].instruction_offset;
     uint8_t* dst = raw_code_start + offset;
     DCHECK_LE(RawCodeOffset() + offset + code.raw_body_size(), blob_code_size);
     std::memcpy(dst, reinterpret_cast<uint8_t*>(code.raw_body_start()),
@@ -297,17 +303,30 @@ EmbeddedData EmbeddedData::FromIsolate(Isolate* isolate) {
 Address EmbeddedData::InstructionStartOfBuiltin(int i) const {
   DCHECK(Builtins::IsBuiltinId(i));
   const struct Metadata* metadata = Metadata();
-  const uint8_t* result = RawCode() + metadata[i].instructions_offset;
-  DCHECK_LE(result, code_ + code_size_);
-  DCHECK_IMPLIES(result == code_ + code_size_,
-                 InstructionSizeOfBuiltin(i) == 0);
+  const uint8_t* result = RawCode() + metadata[i].instruction_offset;
+  DCHECK_LT(result, code_ + code_size_);
   return reinterpret_cast<Address>(result);
 }
 
 uint32_t EmbeddedData::InstructionSizeOfBuiltin(int i) const {
   DCHECK(Builtins::IsBuiltinId(i));
   const struct Metadata* metadata = Metadata();
-  return metadata[i].instructions_length;
+  return metadata[i].instruction_length;
+}
+
+Address EmbeddedData::MetadataStartOfBuiltin(int i) const {
+  DCHECK(Builtins::IsBuiltinId(i));
+  STATIC_ASSERT(Code::kOffHeapBodyIsContiguous);
+  const struct Metadata* metadata = Metadata();
+  const uint8_t* result = RawCode() + metadata[i].metadata_offset;
+  DCHECK_LT(result, code_ + code_size_);
+  return reinterpret_cast<Address>(result);
+}
+
+uint32_t EmbeddedData::MetadataSizeOfBuiltin(int i) const {
+  DCHECK(Builtins::IsBuiltinId(i));
+  const struct Metadata* metadata = Metadata();
+  return metadata[i].metadata_length;
 }
 
 Address EmbeddedData::InstructionStartOfBytecodeHandlers() const {

src/snapshot/embedded/embedded-data.h

@@ -75,7 +75,8 @@ class EmbeddedData final {
   Address InstructionStartOfBytecodeHandlers() const;
   Address InstructionEndOfBytecodeHandlers() const;
 
-  bool ContainsBuiltin(int i) const { return InstructionSizeOfBuiltin(i) > 0; }
+  Address MetadataStartOfBuiltin(int i) const;
+  uint32_t MetadataSizeOfBuiltin(int i) const;
 
   uint32_t AddressForHashing(Address addr) {
     Address start = reinterpret_cast<Address>(code_);
@@ -84,9 +85,12 @@ class EmbeddedData final {
   }
 
   // Padded with kCodeAlignment.
+  // TODO(v8:11045): Consider removing code alignment.
   uint32_t PaddedInstructionSizeOfBuiltin(int i) const {
-    uint32_t size = InstructionSizeOfBuiltin(i);
-    return (size == 0) ? 0 : PadAndAlign(size);
+    STATIC_ASSERT(Code::kOffHeapBodyIsContiguous);
+    uint32_t size = InstructionSizeOfBuiltin(i) + MetadataSizeOfBuiltin(i);
+    CHECK_NE(size, 0);
+    return PadAndAlign(size);
   }
 
   size_t CreateEmbeddedBlobHash() const;
@@ -99,14 +103,28 @@ class EmbeddedData final {
     return *reinterpret_cast<const size_t*>(metadata_ + IsolateHashOffset());
   }
 
+  // Blob layout information for a single instruction stream. Corresponds
+  // roughly to Code object layout (see the instruction and metadata area).
+  // TODO(jgruber): With the addition of metadata sections in Code objects,
+  // naming here has become confusing. Metadata refers to both this struct
+  // and the Code section, and the embedded instruction area currently
+  // contains both Code's instruction and metadata areas. Fix it.
   struct Metadata {
-    // Blob layout information.
-    uint32_t instructions_offset;
-    uint32_t instructions_length;
+    // The offset and (unpadded) length of this builtin's instruction area
+    // from the start of the embedded code section.
+    uint32_t instruction_offset;
+    uint32_t instruction_length;
+    // The offset and (unpadded) length of this builtin's metadata area
+    // from the start of the embedded code section.
+    // TODO(jgruber,v8:11036): Move this to the embedded metadata area.
+    uint32_t metadata_offset;
+    uint32_t metadata_length;
   };
-  STATIC_ASSERT(offsetof(Metadata, instructions_offset) == 0);
-  STATIC_ASSERT(offsetof(Metadata, instructions_length) == kUInt32Size);
-  STATIC_ASSERT(sizeof(Metadata) == kUInt32Size + kUInt32Size);
+  STATIC_ASSERT(offsetof(Metadata, instruction_offset) == 0 * kUInt32Size);
+  STATIC_ASSERT(offsetof(Metadata, instruction_length) == 1 * kUInt32Size);
+  STATIC_ASSERT(offsetof(Metadata, metadata_offset) == 2 * kUInt32Size);
+  STATIC_ASSERT(offsetof(Metadata, metadata_length) == 3 * kUInt32Size);
+  STATIC_ASSERT(sizeof(Metadata) == 4 * kUInt32Size);
 
   // The layout of the blob is as follows:
   //

src/snapshot/embedded/embedded-file-writer.cc

@@ -156,8 +156,8 @@ void EmbeddedFileWriter::WriteInstructionStreams(
   w->AlignToCodeAlignment();
   w->DeclareLabel(EmbeddedBlobCodeDataSymbol().c_str());
 
+  STATIC_ASSERT(Builtins::kAllBuiltinsAreIsolateIndependent);
   for (int i = 0; i < i::Builtins::builtin_count; i++) {
-    if (!blob->ContainsBuiltin(i)) continue;
     WriteBuiltin(w, blob, i);
   }
   w->Newline();

src/snapshot/embedded/platform-embedded-file-writer-win.cc

@@ -109,12 +109,12 @@ void EmitUnwindData(PlatformEmbeddedFileWriterWin* w,
   w->Comment("    UnwindInfoAddress");
   w->StartPdataSection();
   {
+    STATIC_ASSERT(Builtins::kAllBuiltinsAreIsolateIndependent);
     Address prev_builtin_end_offset = 0;
     for (int i = 0; i < Builtins::builtin_count; i++) {
       // Some builtins are leaf functions from the point of view of Win64 stack
       // walking: they do not move the stack pointer and do not require a PDATA
       // entry because the return address can be retrieved from [rsp].
-      if (!blob->ContainsBuiltin(i)) continue;
       if (unwind_infos[i].is_leaf_function()) continue;
 
       uint64_t builtin_start_offset = blob->InstructionStartOfBuiltin(i) -
@@ -193,8 +193,8 @@ void EmitUnwindData(PlatformEmbeddedFileWriterWin* w,
   std::vector<int> code_chunks;
   std::vector<win64_unwindinfo::FrameOffsets> fp_adjustments;
 
+  STATIC_ASSERT(Builtins::kAllBuiltinsAreIsolateIndependent);
   for (int i = 0; i < Builtins::builtin_count; i++) {
-    if (!blob->ContainsBuiltin(i)) continue;
     if (unwind_infos[i].is_leaf_function()) continue;
 
     uint64_t builtin_start_offset = blob->InstructionStartOfBuiltin(i) -

src/wasm/wasm-code-manager.cc

@@ -896,9 +896,9 @@ WasmCode* NativeModule::AddCodeForTesting(Handle<Code> code) {
     source_pos_table->copy_out(0, source_pos.start(),
                                source_pos_table->length());
   }
-  Vector<const byte> instructions(
-      reinterpret_cast<byte*>(code->InstructionStart()),
-      static_cast<size_t>(code->InstructionSize()));
+  STATIC_ASSERT(Code::kBodyIsContiguous);
+  Vector<const byte> instructions(reinterpret_cast<byte*>(code->BodyStart()),
+                                  static_cast<size_t>(code->BodySize()));
   const int stack_slots = code->has_safepoint_info() ? code->stack_slots() : 0;
 
   // TODO(jgruber,v8:8758): Remove this translation. It exists only because
@@ -1370,10 +1370,10 @@ void NativeModule::AddCodeSpace(
         WASM_RUNTIME_STUB_LIST(RUNTIME_STUB, RUNTIME_STUB_TRAP)};
 #undef RUNTIME_STUB
 #undef RUNTIME_STUB_TRAP
+    STATIC_ASSERT(Builtins::kAllBuiltinsAreIsolateIndependent);
     Address builtin_addresses[WasmCode::kRuntimeStubCount];
     for (int i = 0; i < WasmCode::kRuntimeStubCount; ++i) {
       Builtins::Name builtin = stub_names[i];
-      CHECK(embedded_data.ContainsBuiltin(builtin));
       builtin_addresses[i] = embedded_data.InstructionStartOfBuiltin(builtin);
     }
     JumpTableAssembler::GenerateFarJumpTable(

test/cctest/test-code-layout.cc

@@ -91,7 +91,7 @@ TEST(CodeLayoutWithUnwindingInfo) {
                           .Build();
 
   CHECK(code->has_unwinding_info());
-  CHECK_EQ(code->raw_instruction_size(), buffer_size + unwinding_info_size);
+  CHECK_EQ(code->raw_body_size(), buffer_size + unwinding_info_size);
   CHECK_EQ(0, memcmp(reinterpret_cast<void*>(code->raw_instruction_start()),
                      buffer, buffer_size));
   CHECK_EQ(code->unwinding_info_size(), unwinding_info_size);