[builtins] Embed builtins into the binary

This embeds code for off-heap-safe builtins into the binary. Actual
execution of embedded code is not implemented yet.

The embedded file has the following format:

namespace v8 {
namespace internal {

namespace {

V8_EMBEDDED_TEXT_HEADER(v8_embedded_blob_)
__asm__( /* builtin offsets and lengths */ );
__asm__(V8_ASM_LABEL("Builtins_RecordWrite"));
__asm__( /* binary instruction stream */ );
/* Repeat for other builtins. */

extern "C" const uint8_t v8_embedded_blob_[];
static const uint32_t v8_embedded_blob_size_ = /* size in bytes */;

}  // namespace

const uint8_t* DefaultEmbeddedBlob() { return v8_embedded_blob_; }
uint32_t DefaultEmbeddedBlobSize() { return v8_embedded_blob_size_; }

}  // namespace internal
}  // namespace v8

Bug: v8:6666
Cq-Include-Trybots: luci.chromium.try:linux_chromium_rel_ng;luci.v8.try:v8_linux64_fyi_rel_ng
Change-Id: Ic989f01da69ebe2863f31d934bfbe2c5d6e80864
Reviewed-on: https://chromium-review.googlesource.com/946011
Commit-Queue: Jakob Gruber <jgruber@chromium.org>
Reviewed-by: Yang Guo <yangguo@chromium.org>
Cr-Commit-Position: refs/heads/master@{#51759}

BUILD.gn

@@ -817,6 +817,7 @@ if (v8_use_snapshot) {
     sources = []
 
     outputs = [
+      "$target_gen_dir/embedded.cc",
       "$target_gen_dir/snapshot.cc",
     ]
 
@@ -827,6 +828,8 @@ if (v8_use_snapshot) {
       "--turbo_instruction_scheduling",
       "--startup_src",
       rebase_path("$target_gen_dir/snapshot.cc", root_build_dir),
+      "--embedded_src",
+      rebase_path("$target_gen_dir/embedded.cc", root_build_dir),
     ]
 
     if (v8_random_seed != "0") {
@@ -940,6 +943,7 @@ v8_source_set("v8_nosnapshot") {
     "$target_gen_dir/experimental-extras-libraries.cc",
     "$target_gen_dir/extras-libraries.cc",
     "$target_gen_dir/libraries.cc",
+    "src/snapshot/embedded-empty.cc",
     "src/snapshot/snapshot-empty.cc",
   ]
 
@@ -984,6 +988,10 @@ if (v8_use_snapshot) {
       "src/setup-isolate-deserialize.cc",
     ]
 
+    if (v8_enable_embedded_builtins) {
+      sources += [ "$target_gen_dir/embedded.cc" ]
+    }
+
     if (use_jumbo_build == true) {
       jumbo_excluded_sources = [
         # TODO(mostynb@opera.com): don't exclude these http://crbug.com/752428
@@ -1018,6 +1026,10 @@ if (v8_use_external_startup_data) {
       "src/snapshot/snapshot-external.cc",
     ]
 
+    if (v8_enable_embedded_builtins) {
+      sources += [ "$target_gen_dir/embedded.cc" ]
+    }
+
     configs = [ ":internal_config" ]
   }
 }
@@ -2053,6 +2065,7 @@ v8_source_set("v8_base") {
     "src/snapshot/default-serializer-allocator.h",
     "src/snapshot/deserializer.cc",
     "src/snapshot/deserializer.h",
+    "src/snapshot/macros.h",
     "src/snapshot/natives-common.cc",
     "src/snapshot/natives.h",
     "src/snapshot/object-deserializer.cc",

src/builtins/builtins.cc

@@ -289,55 +289,6 @@ bool Builtins::IsLazy(int index) {
 bool Builtins::IsIsolateIndependent(int index) {
   DCHECK(IsBuiltinId(index));
   switch (index) {
-#ifdef DEBUG
-    case kContinueToCodeStubBuiltin:
-    case kContinueToCodeStubBuiltinWithResult:
-    case kContinueToJavaScriptBuiltin:
-    case kContinueToJavaScriptBuiltinWithResult:
-#else
-    case kAsyncFunctionAwaitFulfill:
-    case kAsyncFunctionAwaitReject:
-    case kAsyncGeneratorAwaitFulfill:
-    case kAsyncGeneratorAwaitReject:
-    case kAsyncGeneratorReturnClosedFulfill:
-    case kAsyncGeneratorReturnClosedReject:
-    case kAsyncGeneratorReturnFulfill:
-    case kAsyncGeneratorYieldFulfill:
-    case kConstructFunction:
-    case kContinueToCodeStubBuiltin:
-    case kContinueToCodeStubBuiltinWithResult:
-    case kContinueToJavaScriptBuiltin:
-    case kContinueToJavaScriptBuiltinWithResult:
-    case kKeyedLoadICTrampoline:
-    case kKeyedStoreICTrampoline:
-    case kLoadGlobalICInsideTypeofTrampoline:
-    case kLoadGlobalICTrampoline:
-    case kLoadIC_StringLength:
-    case kLoadIC_StringWrapperLength:
-    case kLoadICTrampoline:
-    case kOrderedHashTableHealIndex:
-    case kPromiseFulfillReactionJob:
-    case kStoreGlobalICTrampoline:
-    case kStoreICTrampoline:
-    case kStringRepeat:
-    case kTypeof:
-    case kWeakMapLookupHashIndex:
-#endif
-      return true;
-    default:
-      return false;
-  }
-  UNREACHABLE();
-}
-
-// static
-bool Builtins::IsOffHeapSafe(int index) {
-#ifndef V8_EMBEDDED_BUILTINS
-  return false;
-#else
-  DCHECK(IsBuiltinId(index));
-  if (IsTooShortForOffHeapTrampoline(index)) return false;
-  switch (index) {
 #ifdef DEBUG
     case kAbortJS:
     case kContinueToCodeStubBuiltin:
@@ -349,15 +300,15 @@ bool Builtins::IsOffHeapSafe(int index) {
     case kLoadGlobalIC_Slow:
     case kLoadIC_Slow:
     case kStoreGlobalIC_Slow:
+    case kWasmStackGuard:
+    case kThrowWasmTrapUnreachable:
+    case kThrowWasmTrapMemOutOfBounds:
     case kThrowWasmTrapDivByZero:
     case kThrowWasmTrapDivUnrepresentable:
+    case kThrowWasmTrapRemByZero:
     case kThrowWasmTrapFloatUnrepresentable:
     case kThrowWasmTrapFuncInvalid:
     case kThrowWasmTrapFuncSigMismatch:
-    case kThrowWasmTrapMemOutOfBounds:
-    case kThrowWasmTrapRemByZero:
-    case kThrowWasmTrapUnreachable:
-    case kWasmStackGuard:
 #else
     case kAbortJS:
     case kAdd:
@@ -496,6 +447,8 @@ bool Builtins::IsOffHeapSafe(int index) {
     case kLoadIC_FunctionPrototype:
     case kLoadIC_Noninlined:
     case kLoadIC_Slow:
+    case kLoadIC_StringLength:
+    case kLoadIC_StringWrapperLength:
     case kLoadICTrampoline:
     case kLoadIC_Uninitialized:
     case kMapPrototypeEntries:
@@ -505,34 +458,14 @@ bool Builtins::IsOffHeapSafe(int index) {
     case kMapPrototypeHas:
     case kMapPrototypeKeys:
     case kMapPrototypeValues:
-    case kMathAcos:
-    case kMathAcosh:
-    case kMathAsin:
-    case kMathAsinh:
-    case kMathAtan:
-    case kMathAtan2:
-    case kMathAtanh:
-    case kMathCbrt:
     case kMathCeil:
-    case kMathCos:
-    case kMathCosh:
-    case kMathExp:
-    case kMathExpm1:
     case kMathFloor:
     case kMathFround:
-    case kMathLog:
-    case kMathLog10:
-    case kMathLog1p:
-    case kMathLog2:
     case kMathMax:
     case kMathMin:
     case kMathRound:
     case kMathSign:
-    case kMathSin:
-    case kMathSinh:
     case kMathSqrt:
-    case kMathTan:
-    case kMathTanh:
     case kMathTrunc:
     case kModulus:
     case kMultiply:
@@ -711,6 +644,16 @@ bool Builtins::IsOffHeapSafe(int index) {
       return false;
   }
   UNREACHABLE();
+}
+
+// static
+bool Builtins::IsOffHeapSafe(int index) {
+#ifndef V8_EMBEDDED_BUILTINS
+  return false;
+#else
+  DCHECK(IsBuiltinId(index));
+  if (IsTooShortForOffHeapTrampoline(index)) return false;
+  return IsIsolateIndependent(index);
 #endif  // V8_EMBEDDED_BUILTINS
 }
 

src/flag-definitions.h

@@ -1067,6 +1067,8 @@ DEFINE_STRING(testing_string_flag, "Hello, world!", "string-flag")
 DEFINE_INT(testing_prng_seed, 42, "Seed used for threading test randomness")
 
 // mksnapshot.cc
+DEFINE_STRING(embedded_src, nullptr,
+              "Path for the generated embedded data file. (mksnapshot only)")
 DEFINE_STRING(startup_src, nullptr,
               "Write V8 startup as C++ src. (mksnapshot only)")
 DEFINE_STRING(startup_blob, nullptr,

src/isolate.cc

@@ -68,6 +68,17 @@ namespace internal {
 
 base::Atomic32 ThreadId::highest_thread_id_ = 0;
 
+#ifdef V8_EMBEDDED_BUILTINS
+extern const uint8_t* DefaultEmbeddedBlob();
+extern uint32_t DefaultEmbeddedBlobSize();
+
+const uint8_t* Isolate::embedded_blob() const { return DefaultEmbeddedBlob(); }
+
+uint32_t Isolate::embedded_blob_size() const {
+  return DefaultEmbeddedBlobSize();
+}
+#endif
+
 int ThreadId::AllocateThreadId() {
   int new_id = base::Relaxed_AtomicIncrement(&highest_thread_id_, 1);
   return new_id;
@@ -2821,45 +2832,11 @@ void PrintBuiltinSizes(Isolate* isolate) {
     const char* name = builtins->name(i);
     const char* kind = Builtins::KindNameOf(i);
     Code* code = builtins->builtin(i);
-    PrintF(stdout, "%s Builtin, %s, %d\n", kind, name,
-           code->instruction_size());
+    PrintF(stdout, "%s Builtin, %s, %d\n", kind, name, code->InstructionSize());
   }
 }
 
 #ifdef V8_EMBEDDED_BUILTINS
-#ifdef DEBUG
-bool BuiltinAliasesOffHeapTrampolineRegister(Isolate* isolate,
-                                             int builtin_index) {
-  switch (Builtins::KindOf(builtin_index)) {
-    case Builtins::CPP:
-    case Builtins::TFC:
-    case Builtins::TFH:
-    case Builtins::TFJ:
-    case Builtins::TFS:
-      break;
-    case Builtins::API:
-    case Builtins::ASM:
-      // TODO(jgruber): Extend checks to remaining kinds.
-      return false;
-  }
-
-  Callable callable = Builtins::CallableFor(
-      isolate, static_cast<Builtins::Name>(builtin_index));
-  CallInterfaceDescriptor descriptor = callable.descriptor();
-
-  if (descriptor.ContextRegister() == kOffHeapTrampolineRegister) {
-    return true;
-  }
-
-  for (int i = 0; i < descriptor.GetRegisterParameterCount(); i++) {
-    Register reg = descriptor.GetRegisterParameter(i);
-    if (reg == kOffHeapTrampolineRegister) return true;
-  }
-
-  return false;
-}
-#endif
-
 void ChangeToOffHeapTrampoline(Isolate* isolate, Handle<Code> code,
                                InstructionStream* stream) {
   DCHECK(Builtins::IsOffHeapSafe(code->builtin_index()));
@@ -2870,8 +2847,6 @@ void ChangeToOffHeapTrampoline(Isolate* isolate, Handle<Code> code,
 
   // Generate replacement code that simply tail-calls the off-heap code.
   MacroAssembler masm(isolate, buffer, buffer_size, CodeObjectRequired::kYes);
-  DCHECK(
-      !BuiltinAliasesOffHeapTrampolineRegister(isolate, code->builtin_index()));
   DCHECK(!masm.has_frame());
   {
     FrameScope scope(&masm, StackFrame::NONE);

src/isolate.h

@@ -1249,6 +1249,9 @@ class Isolate {
   BuiltinsConstantsTableBuilder* builtins_constants_table_builder() const {
     return builtins_constants_table_builder_;
   }
+
+  const uint8_t* embedded_blob() const;
+  uint32_t embedded_blob_size() const;
 #endif
 
   void set_array_buffer_allocator(v8::ArrayBuffer::Allocator* allocator) {

src/objects.cc

@@ -13980,6 +13980,7 @@ SafepointEntry Code::GetSafepointEntry(Address pc) {
   return table.FindEntry(pc);
 }
 
+#ifdef V8_EMBEDDED_BUILTINS
 int Code::OffHeapInstructionSize() {
   DCHECK(Builtins::IsOffHeapBuiltin(this));
   InstructionStream* stream =
@@ -14000,6 +14001,7 @@ Address Code::OffHeapInstructionEnd() {
       InstructionStream::TryLookupInstructionStream(GetIsolate(), this);
   return stream->bytes() + stream->byte_length();
 }
+#endif
 
 namespace {
 template <typename Code>
@@ -14136,6 +14138,36 @@ const char* AbstractCode::Kind2String(Kind kind) {
   UNREACHABLE();
 }
 
+#ifdef V8_EMBEDDED_BUILTINS
+bool Code::IsProcessIndependent() {
+  constexpr int all_real_modes_mask =
+      (1 << (RelocInfo::LAST_REAL_RELOC_MODE + 1)) - 1;
+  constexpr int mode_mask =
+      all_real_modes_mask & ~RelocInfo::ModeMask(RelocInfo::COMMENT) &
+      ~RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) &
+      ~RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE_ENCODED) &
+      ~RelocInfo::ModeMask(RelocInfo::CONST_POOL) &
+      ~RelocInfo::ModeMask(RelocInfo::VENEER_POOL);
+  STATIC_ASSERT(RelocInfo::LAST_REAL_RELOC_MODE == RelocInfo::VENEER_POOL);
+  STATIC_ASSERT(RelocInfo::ModeMask(RelocInfo::COMMENT) ==
+                (1 << RelocInfo::COMMENT));
+  STATIC_ASSERT(
+      mode_mask ==
+      (RelocInfo::ModeMask(RelocInfo::CODE_TARGET) |
+       RelocInfo::ModeMask(RelocInfo::EMBEDDED_OBJECT) |
+       RelocInfo::ModeMask(RelocInfo::WASM_CONTEXT_REFERENCE) |
+       RelocInfo::ModeMask(RelocInfo::WASM_FUNCTION_TABLE_SIZE_REFERENCE) |
+       RelocInfo::ModeMask(RelocInfo::WASM_GLOBAL_HANDLE) |
+       RelocInfo::ModeMask(RelocInfo::WASM_CALL) |
+       RelocInfo::ModeMask(RelocInfo::JS_TO_WASM_CALL) |
+       RelocInfo::ModeMask(RelocInfo::RUNTIME_ENTRY) |
+       RelocInfo::ModeMask(RelocInfo::EXTERNAL_REFERENCE)));
+
+  RelocIterator it(this, mode_mask);
+  return it.done();
+}
+#endif
+
 Handle<WeakCell> Code::WeakCellFor(Handle<Code> code) {
   DCHECK(code->kind() == OPTIMIZED_FUNCTION);
   WeakCell* raw_cell = code->CachedWeakCell();

src/objects/code.h

@@ -64,7 +64,9 @@ class Code : public HeapObject {
   // off-heap instruction stream rather than the on-heap trampoline located
   // at instruction_start.
   inline int InstructionSize();
+#ifdef V8_EMBEDDED_BUILTINS
   int OffHeapInstructionSize();
+#endif
 
   // [relocation_info]: Code relocation information
   DECL_ACCESSORS(relocation_info, ByteArray)
@@ -227,7 +229,9 @@ class Code : public HeapObject {
   // this differs from instruction_start (which would point to the off-heap
   // trampoline instead).
   inline Address InstructionStart();
+#ifdef V8_EMBEDDED_BUILTINS
   Address OffHeapInstructionStart();
+#endif
 
   // Returns the address right after the last instruction.
   inline byte* instruction_end() const;
@@ -236,7 +240,9 @@ class Code : public HeapObject {
   // objects this differs from instruction_end (which would point to the
   // off-heap trampoline instead).
   inline Address InstructionEnd();
+#ifdef V8_EMBEDDED_BUILTINS
   Address OffHeapInstructionEnd();
+#endif
 
   // Returns the size of the instructions, padding, relocation and unwinding
   // information.
@@ -336,6 +342,10 @@ class Code : public HeapObject {
   void VerifyEmbeddedObjects(VerifyMode mode = kNoContextRetainingPointers);
 #endif  // DEBUG
 
+#ifdef V8_EMBEDDED_BUILTINS
+  bool IsProcessIndependent();
+#endif
+
   inline bool CanContainWeakObjects();
 
   inline bool IsWeakObject(Object* object);

src/snapshot/embedded-empty.cc

+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Used for building without embedded data.
+
+#include <cstdint>
+
+namespace v8 {
+namespace internal {
+
+#ifdef V8_EMBEDDED_BUILTINS
+const uint8_t* DefaultEmbeddedBlob() { return nullptr; }
+uint32_t DefaultEmbeddedBlobSize() { return 0; }
+#endif
+
+}  // namespace internal
+}  // namespace v8

src/snapshot/macros.h

+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_SNAPSHOT_MACROS_H_
+#define V8_SNAPSHOT_MACROS_H_
+
+// .byte portability macros.
+
+#if defined(V8_OS_MACOSX)  // MACOSX
+#define V8_ASM_MANGLE_LABEL "_"
+#define V8_ASM_RODATA_SECTION ".const_data\n"
+#define V8_ASM_TEXT_SECTION ".text\n"
+#define V8_ASM_GLOBAL(NAME) ".globl " V8_ASM_MANGLE_LABEL NAME "\n"
+#elif defined(V8_OS_WIN)  // WIN
+#if defined(V8_TARGET_ARCH_X64)
+#define V8_ASM_MANGLE_LABEL ""
+#else
+#define V8_ASM_MANGLE_LABEL "_"
+#endif
+#define V8_ASM_RODATA_SECTION ".section .rodata\n"
+#define V8_ASM_TEXT_SECTION ".section .text\n"
+#define V8_ASM_GLOBAL(NAME) ".global " V8_ASM_MANGLE_LABEL NAME "\n"
+#else  // !MACOSX && !WIN
+#define V8_ASM_MANGLE_LABEL ""
+#define V8_ASM_RODATA_SECTION ".section .rodata\n"
+#define V8_ASM_TEXT_SECTION ".section .text\n"
+#define V8_ASM_GLOBAL(NAME) ".global " V8_ASM_MANGLE_LABEL NAME "\n"
+#endif
+
+#define V8_ASM_BALIGN16 ".balign 16\n"
+#define V8_ASM_LABEL(NAME) V8_ASM_MANGLE_LABEL NAME ":\n"
+
+#define V8_EMBEDDED_TEXT_HEADER(LABEL)              \
+  __asm__(V8_ASM_TEXT_SECTION V8_ASM_GLOBAL(#LABEL) \
+              V8_ASM_BALIGN16 V8_ASM_LABEL(#LABEL));
+
+#define V8_EMBEDDED_RODATA_HEADER(LABEL)              \
+  __asm__(V8_ASM_RODATA_SECTION V8_ASM_GLOBAL(#LABEL) \
+              V8_ASM_BALIGN16 V8_ASM_LABEL(#LABEL));
+
+#endif  // V8_SNAPSHOT_MACROS_H_

src/snapshot/mksnapshot.cc

@@ -13,6 +13,7 @@
 #include "src/msan.h"
 #include "src/snapshot/natives.h"
 #include "src/snapshot/partial-serializer.h"
+#include "src/snapshot/snapshot.h"
 #include "src/snapshot/startup-serializer.h"
 
 class SnapshotWriter {
@@ -20,6 +21,12 @@ class SnapshotWriter {
   SnapshotWriter()
       : snapshot_cpp_path_(nullptr), snapshot_blob_path_(nullptr) {}
 
+#ifdef V8_EMBEDDED_BUILTINS
+  void SetEmbeddedFile(const char* embedded_cpp_file) {
+    embedded_cpp_path_ = embedded_cpp_file;
+  }
+#endif
+
   void SetSnapshotFile(const char* snapshot_cpp_file) {
     snapshot_cpp_path_ = snapshot_cpp_file;
   }
@@ -39,6 +46,12 @@ class SnapshotWriter {
     MaybeWriteStartupBlob(blob_vector);
   }
 
+#ifdef V8_EMBEDDED_BUILTINS
+  void WriteEmbedded(const i::EmbeddedData* blob) const {
+    MaybeWriteEmbeddedFile(blob);
+  }
+#endif
+
  private:
   void MaybeWriteStartupBlob(const i::Vector<const i::byte>& blob) const {
     if (!snapshot_blob_path_) return;
@@ -58,14 +71,14 @@ class SnapshotWriter {
 
     FILE* fp = GetFileDescriptorOrDie(snapshot_cpp_path_);
 
-    WriteFilePrefix(fp);
-    WriteData(fp, blob);
-    WriteFileSuffix(fp);
+    WriteSnapshotFilePrefix(fp);
+    WriteSnapshotFileData(fp, blob);
+    WriteSnapshotFileSuffix(fp);
 
     fclose(fp);
   }
 
-  static void WriteFilePrefix(FILE* fp) {
+  static void WriteSnapshotFilePrefix(FILE* fp) {
     fprintf(fp, "// Autogenerated snapshot file. Do not edit.\n\n");
     fprintf(fp, "#include \"src/v8.h\"\n");
     fprintf(fp, "#include \"src/base/platform/platform.h\"\n\n");
@@ -74,7 +87,7 @@ class SnapshotWriter {
     fprintf(fp, "namespace internal {\n\n");
   }
 
-  static void WriteFileSuffix(FILE* fp) {
+  static void WriteSnapshotFileSuffix(FILE* fp) {
     fprintf(fp, "const v8::StartupData* Snapshot::DefaultSnapshotBlob() {\n");
     fprintf(fp, "  return &blob;\n");
     fprintf(fp, "}\n\n");
@@ -82,17 +95,18 @@ class SnapshotWriter {
     fprintf(fp, "}  // namespace v8\n");
   }
 
-  static void WriteData(FILE* fp, const i::Vector<const i::byte>& blob) {
+  static void WriteSnapshotFileData(FILE* fp,
+                                    const i::Vector<const i::byte>& blob) {
     fprintf(fp, "static const byte blob_data[] = {\n");
-    WriteSnapshotData(fp, blob);
+    WriteBinaryContentsAsCArray(fp, blob);
     fprintf(fp, "};\n");
     fprintf(fp, "static const int blob_size = %d;\n", blob.length());
     fprintf(fp, "static const v8::StartupData blob =\n");
     fprintf(fp, "{ (const char*) blob_data, blob_size };\n");
   }
 
-  static void WriteSnapshotData(FILE* fp,
-                                const i::Vector<const i::byte>& blob) {
+  static void WriteBinaryContentsAsCArray(
+      FILE* fp, const i::Vector<const i::byte>& blob) {
     for (int i = 0; i < blob.length(); i++) {
       if ((i & 0x1F) == 0x1F) fprintf(fp, "\n");
       if (i > 0) fprintf(fp, ",");
@@ -101,6 +115,96 @@ class SnapshotWriter {
     fprintf(fp, "\n");
   }
 
+#ifdef V8_EMBEDDED_BUILTINS
+  void MaybeWriteEmbeddedFile(const i::EmbeddedData* blob) const {
+    if (embedded_cpp_path_ == nullptr) return;
+
+    FILE* fp = GetFileDescriptorOrDie(embedded_cpp_path_);
+
+    WriteEmbeddedFilePrefix(fp);
+    WriteEmbeddedFileData(fp, blob);
+    WriteEmbeddedFileSuffix(fp);
+
+    fclose(fp);
+  }
+
+  static void WriteEmbeddedFilePrefix(FILE* fp) {
+    fprintf(fp, "// Autogenerated file. Do not edit.\n\n");
+    fprintf(fp, "#include <cstdint>\n\n");
+    fprintf(fp, "#include \"src/snapshot/macros.h\"\n\n");
+    fprintf(fp, "namespace v8 {\n");
+    fprintf(fp, "namespace internal {\n\n");
+    fprintf(fp, "namespace {\n\n");
+  }
+
+  static void WriteEmbeddedFileSuffix(FILE* fp) {
+    fprintf(fp, "}  // namespace\n\n");
+    fprintf(
+        fp,
+        "const uint8_t* DefaultEmbeddedBlob() { return v8_embedded_blob_; }\n");
+    fprintf(fp,
+            "uint32_t DefaultEmbeddedBlobSize() { return "
+            "v8_embedded_blob_size_; }\n\n");
+    fprintf(fp, "}  // namespace internal\n");
+    fprintf(fp, "}  // namespace v8\n");
+  }
+
+  static void WriteEmbeddedFileData(FILE* fp, const i::EmbeddedData* blob) {
+    fprintf(fp, "V8_EMBEDDED_TEXT_HEADER(v8_embedded_blob_)\n");
+    WriteBinaryContentsAsByteDirective(fp, blob->data(),
+                                       i::EmbeddedData::RawDataOffset());
+    WriteBuiltins(fp, blob);
+    fprintf(fp, "extern \"C\" const uint8_t v8_embedded_blob_[];\n");
+    fprintf(fp, "static const uint32_t v8_embedded_blob_size_ = %d;\n\n",
+            blob->size());
+  }
+
+  static void WriteBuiltins(FILE* fp, const i::EmbeddedData* blob) {
+    for (int i = 0; i < i::Builtins::builtin_count; i++) {
+      if (!blob->ContainsBuiltin(i)) continue;
+
+      fprintf(fp, "__asm__(V8_ASM_LABEL(\"Builtins_%s\"));\n",
+              i::Builtins::name(i));
+      WriteBinaryContentsAsByteDirective(
+          fp, blob->InstructionStartOfBuiltin(i),
+          blob->PaddedInstructionSizeOfBuiltin(i));
+    }
+    fprintf(fp, "\n");
+  }
+
+  static void WriteBinaryContentsAsByteDirective(FILE* fp, const uint8_t* data,
+                                                 uint32_t size) {
+    static const int kTextWidth = 80;
+    int current_line_length = 0;
+    int printed_chars;
+
+    fprintf(fp, "__asm__(\n");
+    for (uint32_t i = 0; i < size; i++) {
+      if (current_line_length == 0) {
+        printed_chars = fprintf(fp, "%s", "  \".byte ");
+        DCHECK_LT(0, printed_chars);
+        current_line_length += printed_chars;
+      } else {
+        printed_chars = fprintf(fp, ",");
+        DCHECK_EQ(1, printed_chars);
+        current_line_length += printed_chars;
+      }
+
+      printed_chars = fprintf(fp, "0x%02x", data[i]);
+      DCHECK_LT(0, printed_chars);
+      current_line_length += printed_chars;
+
+      if (current_line_length + strlen(",0xFF\\n\"") > kTextWidth) {
+        fprintf(fp, "\\n\"\n");
+        current_line_length = 0;
+      }
+    }
+
+    if (current_line_length != 0) fprintf(fp, "\\n\"\n");
+    fprintf(fp, ");\n");
+  }
+#endif
+
   static FILE* GetFileDescriptorOrDie(const char* filename) {
     FILE* fp = v8::base::OS::FOpen(filename, "wb");
     if (fp == nullptr) {
@@ -110,6 +214,9 @@ class SnapshotWriter {
     return fp;
   }
 
+#ifdef V8_EMBEDDED_BUILTINS
+  const char* embedded_cpp_path_ = nullptr;
+#endif
   const char* snapshot_cpp_path_;
   const char* snapshot_blob_path_;
 };
@@ -139,6 +246,64 @@ char* GetExtraCode(char* filename, const char* description) {
   return chars;
 }
 
+bool RunExtraCode(v8::Isolate* isolate, v8::Local<v8::Context> context,
+                  const char* utf8_source, const char* name) {
+  v8::base::ElapsedTimer timer;
+  timer.Start();
+  v8::Context::Scope context_scope(context);
+  v8::TryCatch try_catch(isolate);
+  v8::Local<v8::String> source_string;
+  if (!v8::String::NewFromUtf8(isolate, utf8_source, v8::NewStringType::kNormal)
+           .ToLocal(&source_string)) {
+    return false;
+  }
+  v8::Local<v8::String> resource_name =
+      v8::String::NewFromUtf8(isolate, name, v8::NewStringType::kNormal)
+          .ToLocalChecked();
+  v8::ScriptOrigin origin(resource_name);
+  v8::ScriptCompiler::Source source(source_string, origin);
+  v8::Local<v8::Script> script;
+  if (!v8::ScriptCompiler::Compile(context, &source).ToLocal(&script))
+    return false;
+  if (script->Run(context).IsEmpty()) return false;
+  if (i::FLAG_profile_deserialization) {
+    i::PrintF("Executing custom snapshot script %s took %0.3f ms\n", name,
+              timer.Elapsed().InMillisecondsF());
+  }
+  timer.Stop();
+  CHECK(!try_catch.HasCaught());
+  return true;
+}
+
+v8::StartupData CreateSnapshotDataBlob(v8::SnapshotCreator* snapshot_creator,
+                                       const char* script_source = NULL) {
+  // Create a new isolate and a new context from scratch, optionally run
+  // a script to embed, and serialize to create a snapshot blob.
+  v8::StartupData result = {nullptr, 0};
+  v8::base::ElapsedTimer timer;
+  timer.Start();
+  {
+    v8::Isolate* isolate = snapshot_creator->GetIsolate();
+    {
+      v8::HandleScope scope(isolate);
+      v8::Local<v8::Context> context = v8::Context::New(isolate);
+      if (script_source != nullptr &&
+          !RunExtraCode(isolate, context, script_source, "<embedded>")) {
+        return result;
+      }
+      snapshot_creator->SetDefaultContext(context);
+    }
+    result = snapshot_creator->CreateBlob(
+        v8::SnapshotCreator::FunctionCodeHandling::kClear);
+  }
+
+  if (i::FLAG_profile_deserialization) {
+    i::PrintF("Creating snapshot took %0.3f ms\n",
+              timer.Elapsed().InMillisecondsF());
+  }
+  timer.Stop();
+  return result;
+}
 
 int main(int argc, char** argv) {
   // Make mksnapshot runs predictable to create reproducible snapshots.
@@ -164,11 +329,27 @@ int main(int argc, char** argv) {
     SnapshotWriter writer;
     if (i::FLAG_startup_src) writer.SetSnapshotFile(i::FLAG_startup_src);
     if (i::FLAG_startup_blob) writer.SetStartupBlobFile(i::FLAG_startup_blob);
+#ifdef V8_EMBEDDED_BUILTINS
+    if (i::FLAG_embedded_src) writer.SetEmbeddedFile(i::FLAG_embedded_src);
+#endif
+
+    v8::StartupData blob;
+    {
+      char* embed_script =
+          GetExtraCode(argc >= 2 ? argv[1] : nullptr, "embedding");
+      v8::SnapshotCreator snapshot_creator;
+      blob = CreateSnapshotDataBlob(&snapshot_creator, embed_script);
 
-    char* embed_script =
-        GetExtraCode(argc >= 2 ? argv[1] : nullptr, "embedding");
-    v8::StartupData blob = v8::V8::CreateSnapshotDataBlob(embed_script);
-    delete[] embed_script;
+#ifdef V8_EMBEDDED_BUILTINS
+      i::Isolate* isolate =
+          reinterpret_cast<i::Isolate*>(snapshot_creator.GetIsolate());
+      i::EmbeddedData embedded_blob = i::Snapshot::CreateEmbeddedBlob(isolate);
+      writer.WriteEmbedded(&embedded_blob);
+      delete[] embedded_blob.data();
+#endif
+
+      delete[] embed_script;
+    }
 
     char* warmup_script =
         GetExtraCode(argc >= 3 ? argv[2] : nullptr, "warm up");

src/snapshot/snapshot-common.cc

@@ -8,6 +8,8 @@
 
 #include "src/api.h"
 #include "src/base/platform/platform.h"
+#include "src/callable.h"
+#include "src/interface-descriptors.h"
 #include "src/objects-inl.h"
 #include "src/snapshot/builtin-deserializer.h"
 #include "src/snapshot/builtin-serializer.h"
@@ -264,6 +266,138 @@ v8::StartupData Snapshot::CreateSnapshotBlob(
   return result;
 }
 
+#ifdef V8_EMBEDDED_BUILTINS
+#ifdef DEBUG
+namespace {
+bool BuiltinAliasesOffHeapTrampolineRegister(Isolate* isolate, Code* code) {
+  DCHECK(Builtins::IsOffHeapSafe(code->builtin_index()));
+  switch (Builtins::KindOf(code->builtin_index())) {
+    case Builtins::CPP:
+    case Builtins::TFC:
+    case Builtins::TFH:
+    case Builtins::TFJ:
+    case Builtins::TFS:
+      break;
+    case Builtins::API:
+    case Builtins::ASM:
+      // TODO(jgruber): Extend checks to remaining kinds.
+      return false;
+  }
+
+  Callable callable = Builtins::CallableFor(
+      isolate, static_cast<Builtins::Name>(code->builtin_index()));
+  CallInterfaceDescriptor descriptor = callable.descriptor();
+
+  if (descriptor.ContextRegister() == kOffHeapTrampolineRegister) {
+    return true;
+  }
+
+  for (int i = 0; i < descriptor.GetRegisterParameterCount(); i++) {
+    Register reg = descriptor.GetRegisterParameter(i);
+    if (reg == kOffHeapTrampolineRegister) return true;
+  }
+
+  return false;
+}
+}  // namespace
+#endif  // DEBUG
+
+// static
+EmbeddedData EmbeddedData::FromIsolate(Isolate* isolate) {
+  Builtins* builtins = isolate->builtins();
+
+  // Builtins must be deserialized to be copied off-heap.
+  Snapshot::EnsureAllBuiltinsAreDeserialized(isolate);
+
+  // Store instruction stream lengths and offsets.
+  std::vector<uint32_t> lengths(kTableSize);
+  std::vector<uint32_t> offsets(kTableSize);
+
+  bool saw_unsafe_builtin = false;
+  uint32_t raw_data_size = 0;
+  for (int i = 0; i < Builtins::builtin_count; i++) {
+    Code* code = builtins->builtin(i);
+
+    if (Builtins::IsOffHeapSafe(i)) {
+#ifdef DEBUG
+      // Sanity-check that the given builtin is process-independent and does not
+      // use the trampoline register in its calling convention.
+      if (!code->IsProcessIndependent()) {
+        saw_unsafe_builtin = true;
+        fprintf(stderr, "%s is not process-independent.\n", Builtins::name(i));
+      }
+      if (BuiltinAliasesOffHeapTrampolineRegister(isolate, code)) {
+        saw_unsafe_builtin = true;
+        fprintf(stderr, "%s aliases the off-heap trampoline register.\n",
+                Builtins::name(i));
+      }
+#endif
+
+      uint32_t length = static_cast<uint32_t>(code->instruction_size());
+
+      DCHECK_EQ(0, raw_data_size % kCodeAlignment);
+      offsets[i] = raw_data_size;
+      lengths[i] = length;
+
+      // Align the start of each instruction stream.
+      raw_data_size += RoundUp<kCodeAlignment>(length);
+    } else {
+      offsets[i] = raw_data_size;
+      lengths[i] = 0;
+    }
+  }
+  CHECK(!saw_unsafe_builtin);
+
+  const uint32_t blob_size = RawDataOffset() + raw_data_size;
+  uint8_t* blob = new uint8_t[blob_size];
+  std::memset(blob, 0, blob_size);
+
+  // Write the offsets and length tables.
+  DCHECK_EQ(OffsetsSize(), sizeof(offsets[0]) * offsets.size());
+  std::memcpy(blob + OffsetsOffset(), offsets.data(), OffsetsSize());
+
+  DCHECK_EQ(LengthsSize(), sizeof(lengths[0]) * lengths.size());
+  std::memcpy(blob + LengthsOffset(), lengths.data(), LengthsSize());
+
+  // Write the raw data section.
+  for (int i = 0; i < Builtins::builtin_count; i++) {
+    if (!Builtins::IsOffHeapSafe(i)) continue;
+    Code* code = builtins->builtin(i);
+    uint32_t offset = offsets[i];
+    uint8_t* dst = blob + RawDataOffset() + offset;
+    DCHECK_LE(RawDataOffset() + offset + code->instruction_size(), blob_size);
+    std::memcpy(dst, code->instruction_start(), code->instruction_size());
+  }
+
+  return {blob, blob_size};
+}
+
+EmbeddedData EmbeddedData::FromBlob(const uint8_t* data, uint32_t size) {
+  return {data, size};
+}
+
+const uint8_t* EmbeddedData::InstructionStartOfBuiltin(int i) const {
+  DCHECK(Builtins::IsBuiltinId(i));
+
+  const uint32_t* offsets = Offsets();
+  const uint8_t* result = RawData() + offsets[i];
+  DCHECK_LT(result, data_ + size_);
+  return result;
+}
+
+uint32_t EmbeddedData::InstructionSizeOfBuiltin(int i) const {
+  DCHECK(Builtins::IsBuiltinId(i));
+  const uint32_t* lengths = Lengths();
+  return lengths[i];
+}
+
+// static
+EmbeddedData Snapshot::CreateEmbeddedBlob(Isolate* isolate) {
+  DisallowHeapAllocation no_gc;
+  return EmbeddedData::FromIsolate(isolate);
+}
+#endif
+
 uint32_t Snapshot::ExtractNumContexts(const v8::StartupData* data) {
   CHECK_LT(kNumberOfContextsOffset, data->raw_size);
   uint32_t num_contexts = GetHeaderValue(data, kNumberOfContextsOffset);

src/snapshot/snapshot.h

@@ -79,6 +79,60 @@ class BuiltinSnapshotData final : public SnapshotData {
   // ... list of builtins offsets
 };
 
+#ifdef V8_EMBEDDED_BUILTINS
+class EmbeddedData final {
+ public:
+  static EmbeddedData FromIsolate(Isolate* isolate);
+  static EmbeddedData FromBlob(const uint8_t* data, uint32_t size);
+
+  const uint8_t* data() const { return data_; }
+  uint32_t size() const { return size_; }
+
+  const uint8_t* InstructionStartOfBuiltin(int i) const;
+  uint32_t InstructionSizeOfBuiltin(int i) const;
+
+  bool ContainsBuiltin(int i) const { return InstructionSizeOfBuiltin(i) > 0; }
+
+  // Padded with kCodeAlignment.
+  uint32_t PaddedInstructionSizeOfBuiltin(int i) const {
+    return RoundUp<kCodeAlignment>(InstructionSizeOfBuiltin(i));
+  }
+
+  // The layout of the blob is as follows:
+  //
+  // [0] offset of instruction stream 0
+  // ... offsets
+  // [N] length of instruction stream 0
+  // ... lengths
+  // ... instruction streams
+
+  static constexpr uint32_t kTableSize = Builtins::builtin_count;
+  static constexpr uint32_t OffsetsOffset() { return 0; }
+  static constexpr uint32_t OffsetsSize() { return kUInt32Size * kTableSize; }
+  static constexpr uint32_t LengthsOffset() {
+    return OffsetsOffset() + OffsetsSize();
+  }
+  static constexpr uint32_t LengthsSize() { return kUInt32Size * kTableSize; }
+  static constexpr uint32_t RawDataOffset() {
+    return RoundUp<kCodeAlignment>(LengthsOffset() + LengthsSize());
+  }
+
+ private:
+  EmbeddedData(const uint8_t* data, uint32_t size) : data_(data), size_(size) {}
+
+  const uint32_t* Offsets() const {
+    return reinterpret_cast<const uint32_t*>(data_ + OffsetsOffset());
+  }
+  const uint32_t* Lengths() const {
+    return reinterpret_cast<const uint32_t*>(data_ + LengthsOffset());
+  }
+  const uint8_t* RawData() const { return data_ + RawDataOffset(); }
+
+  const uint8_t* data_;
+  uint32_t size_;
+};
+#endif
+
 class Snapshot : public AllStatic {
  public:
   // ---------------- Deserialization ----------------
@@ -121,6 +175,10 @@ class Snapshot : public AllStatic {
       const std::vector<SnapshotData*>& context_snapshots,
       bool can_be_rehashed);
 
+#ifdef V8_EMBEDDED_BUILTINS
+  static EmbeddedData CreateEmbeddedBlob(Isolate* isolate);
+#endif
+
 #ifdef DEBUG
   static bool SnapshotIsValid(const v8::StartupData* snapshot_blob);
 #endif  // DEBUG

test/cctest/test-isolate-independent-builtins.cc

@@ -9,6 +9,7 @@
 #include "src/isolate.h"
 #include "src/macro-assembler-inl.h"
 #include "src/simulator.h"
+#include "src/snapshot/macros.h"
 #include "src/snapshot/snapshot.h"
 
 // To generate the binary files for the test function, enable this section and
@@ -108,87 +109,6 @@ UNINITIALIZED_TEST(VerifyBuiltinsIsolateIndependence) {
 
   v8_isolate->Dispose();
 }
-
-UNINITIALIZED_TEST(VerifyBuiltinsOffHeapSafety) {
-  FLAG_stress_off_heap_code = false;  // Disable off-heap trampolines.
-
-  v8::Isolate::CreateParams create_params;
-  create_params.array_buffer_allocator = CcTest::array_buffer_allocator();
-  v8::Isolate* v8_isolate = v8::Isolate::New(create_params);
-
-  {
-    v8::Isolate::Scope isolate_scope(v8_isolate);
-    v8::internal::Isolate* isolate =
-        reinterpret_cast<v8::internal::Isolate*>(v8_isolate);
-    HandleScope handle_scope(isolate);
-
-    Snapshot::EnsureAllBuiltinsAreDeserialized(isolate);
-
-    constexpr int all_real_modes_mask =
-        (1 << (RelocInfo::LAST_REAL_RELOC_MODE + 1)) - 1;
-    constexpr int mode_mask =
-        all_real_modes_mask & ~RelocInfo::ModeMask(RelocInfo::COMMENT) &
-        ~RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE) &
-        ~RelocInfo::ModeMask(RelocInfo::INTERNAL_REFERENCE_ENCODED) &
-        ~RelocInfo::ModeMask(RelocInfo::CONST_POOL) &
-        ~RelocInfo::ModeMask(RelocInfo::VENEER_POOL) &
-        ~RelocInfo::ModeMask(RelocInfo::EXTERNAL_REFERENCE);
-    STATIC_ASSERT(RelocInfo::LAST_REAL_RELOC_MODE == RelocInfo::VENEER_POOL);
-    STATIC_ASSERT(RelocInfo::ModeMask(RelocInfo::COMMENT) ==
-                  (1 << RelocInfo::COMMENT));
-    STATIC_ASSERT(
-        mode_mask ==
-        (RelocInfo::ModeMask(RelocInfo::CODE_TARGET) |
-         RelocInfo::ModeMask(RelocInfo::EMBEDDED_OBJECT) |
-         RelocInfo::ModeMask(RelocInfo::WASM_CONTEXT_REFERENCE) |
-         RelocInfo::ModeMask(RelocInfo::WASM_FUNCTION_TABLE_SIZE_REFERENCE) |
-         RelocInfo::ModeMask(RelocInfo::WASM_GLOBAL_HANDLE) |
-         RelocInfo::ModeMask(RelocInfo::WASM_CALL) |
-         RelocInfo::ModeMask(RelocInfo::JS_TO_WASM_CALL) |
-         RelocInfo::ModeMask(RelocInfo::RUNTIME_ENTRY)));
-
-    constexpr bool kVerbose = false;
-    bool found_mismatch = false;
-    for (int i = 0; i < Builtins::builtin_count; i++) {
-      Code* code = isolate->builtins()->builtin(i);
-
-      if (kVerbose) {
-        printf("%s %s\n", Builtins::KindNameOf(i),
-               isolate->builtins()->name(i));
-      }
-
-      bool is_off_heap_safe = true;
-      for (RelocIterator it(code, mode_mask); !it.done(); it.next()) {
-        is_off_heap_safe = false;
-#ifdef ENABLE_DISASSEMBLER
-        if (kVerbose) {
-          RelocInfo::Mode mode = it.rinfo()->rmode();
-          printf("  %s\n", RelocInfo::RelocModeName(mode));
-        }
-#endif
-      }
-
-      // TODO(jgruber): Remove once we properly set up the on-heap code
-      // trampoline.
-      if (Builtins::IsTooShortForOffHeapTrampoline(i)) is_off_heap_safe = false;
-
-      // Relaxed condition only checks whether the off-heap-safe list is
-      // valid, not whether it is complete. This is to avoid constant work
-      // updating the list.
-      bool should_be_off_heap_safe = Builtins::IsOffHeapSafe(i);
-      if (should_be_off_heap_safe && !is_off_heap_safe) {
-        found_mismatch = true;
-        printf("%s %s expected: %d, is: %d\n", Builtins::KindNameOf(i),
-               isolate->builtins()->name(i), should_be_off_heap_safe,
-               is_off_heap_safe);
-      }
-    }
-
-    CHECK(!found_mismatch);
-  }
-
-  v8_isolate->Dispose();
-}
 #endif  // V8_EMBEDDED_BUILTINS
 
 // V8_CC_MSVC is true for both MSVC and clang on windows. clang can handle
@@ -301,6 +221,7 @@ TEST(GenerateTestFunctionData) {
   std::ofstream of(TEST_FUNCTION_FILE, std::ios::out | std::ios::binary);
   of.write(reinterpret_cast<char*>(desc.buffer), desc.instr_size);
 }
+#undef __
 #endif  // GENERATE_TEST_FUNCTION_DATA
 
 #if V8_TARGET_ARCH_IA32
@@ -331,48 +252,12 @@ TEST(GenerateTestFunctionData) {
 #error "Unknown architecture."
 #endif
 
-// .byte macros to handle small differences across operating systems.
-
-#if defined(V8_OS_MACOSX)
-#define ASM_RODATA_SECTION ".const_data\n"
-#define ASM_TEXT_SECTION ".text\n"
-#define ASM_MANGLE_LABEL "_"
-#define ASM_GLOBAL(NAME) ".globl " ASM_MANGLE_LABEL NAME "\n"
-#elif defined(V8_OS_WIN)
-#define ASM_RODATA_SECTION ".section .rodata\n"
-#define ASM_TEXT_SECTION ".section .text\n"
-#if defined(V8_TARGET_ARCH_X64)
-#define ASM_MANGLE_LABEL ""
-#else
-#define ASM_MANGLE_LABEL "_"
-#endif
-#define ASM_GLOBAL(NAME) ".global " ASM_MANGLE_LABEL NAME "\n"
-#else
-#define ASM_RODATA_SECTION ".section .rodata\n"
-#define ASM_TEXT_SECTION ".section .text\n"
-#define ASM_MANGLE_LABEL ""
-#define ASM_GLOBAL(NAME) ".global " ASM_MANGLE_LABEL NAME "\n"
-#endif
-
-// clang-format off
-#define EMBED_IN_RODATA_HEADER(LABEL)    \
-  __asm__(ASM_RODATA_SECTION             \
-          ASM_GLOBAL(#LABEL)             \
-          ".balign 16\n"                 \
-          ASM_MANGLE_LABEL #LABEL ":\n");
-
-#define EMBED_IN_TEXT_HEADER(LABEL)        \
-    __asm__(ASM_TEXT_SECTION               \
-            ASM_GLOBAL(#LABEL)             \
-            ".balign 16\n"                 \
-            ASM_MANGLE_LABEL #LABEL ":\n");
-
-EMBED_IN_RODATA_HEADER(test_string0_bytes)
+V8_EMBEDDED_RODATA_HEADER(test_string0_bytes)
 __asm__(".byte 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37\n"
         ".byte 0x38, 0x39, 0x0a, 0x00\n");
 extern "C" V8_ALIGNED(16) const char test_string0_bytes[];
 
-EMBED_IN_TEXT_HEADER(test_function0_bytes)
+V8_EMBEDDED_TEXT_HEADER(test_function0_bytes)
 __asm__(FUNCTION_BYTES);
 extern "C" V8_ALIGNED(16) const char test_function0_bytes[];
 // clang-format on
@@ -396,13 +281,6 @@ TEST(ByteInText) {
 #endif  // #ifndef V8_COMPILER_IS_MSVC
 #undef V8_COMPILER_IS_MSVC
 
-#undef __
-#undef ASM_GLOBAL
-#undef ASM_MANGLE_LABEL
-#undef ASM_RODATA_SECTION
-#undef ASM_TEXT_SECTION
-#undef EMBED_IN_RODATA_HEADER
-#undef EMBED_IN_TEXT_HEADER
 #undef FUNCTION_BYTES
 #undef GENERATE_TEST_FUNCTION_DATA
 #undef TEST_FUNCTION_FILE