[builtins] Execute binary-embedded builtin code

This CL creates trampolines into binary-embedded builtins on
isolate-creation, if --stress-off-heap-code is passed.

Note that this still overwrites existing code objects with the
off-heap trampoline, and that off-heap builtins still exist both in
the snapshot and the binary. Addressing these points are the next
steps.

Drive-by-change: More efficient off-heap code lookups now that the
off-heap memory area has a contiguous and static layout.

Cq-Include-Trybots: luci.v8.try:v8_linux64_fyi_rel_ng
Bug: v8:6666
Change-Id: I7e7ef0aa2cd7b8184ae3a13fa02bdcbb4f2c9f86
Reviewed-on: https://chromium-review.googlesource.com/947969
Commit-Queue: Jakob Gruber <jgruber@chromium.org>
Reviewed-by: Michael Starzinger <mstarzinger@chromium.org>
Cr-Commit-Position: refs/heads/master@{#51809}

src/instruction-stream.cc

@@ -8,58 +8,57 @@
 #include "src/heap/heap.h"
 #include "src/objects-inl.h"
 #include "src/objects/code-inl.h"
+#include "src/snapshot/snapshot.h"
 
 namespace v8 {
 namespace internal {
 
-InstructionStream::InstructionStream(Code* code)
-    : builtin_index_(code->builtin_index()) {
-  DCHECK(Builtins::IsOffHeapBuiltin(code));
-  const size_t page_size = AllocatePageSize();
-  byte_length_ =
-      RoundUp(static_cast<size_t>(code->instruction_size()), page_size);
-
-  bytes_ = static_cast<uint8_t*>(AllocatePages(
-      GetRandomMmapAddr(), byte_length_, page_size, PageAllocator::kReadWrite));
-  CHECK_NOT_NULL(bytes_);
-
-  std::memcpy(bytes_, code->instruction_start(), code->instruction_size());
-  CHECK(SetPermissions(bytes_, byte_length_, PageAllocator::kReadExecute));
+InstructionStream::InstructionStream(uint8_t* bytes, size_t byte_length,
+                                     int builtin_index)
+    : byte_length_(byte_length), bytes_(bytes), builtin_index_(builtin_index) {
+  DCHECK(Builtins::IsBuiltinId(builtin_index_));
+  DCHECK_NOT_NULL(bytes_);
 }
 
-InstructionStream::~InstructionStream() {
-  CHECK(FreePages(bytes_, byte_length_));
+// static
+bool InstructionStream::PcIsOffHeap(Isolate* isolate, Address pc) {
+#ifdef V8_EMBEDDED_BUILTINS
+  const uint8_t* start = isolate->embedded_blob();
+  return start <= pc && pc < start + isolate->embedded_blob_size();
+#else
+  return false;
+#endif
 }
 
 // static
 Code* InstructionStream::TryLookupCode(Isolate* isolate, Address address) {
+#ifdef V8_EMBEDDED_BUILTINS
   DCHECK(FLAG_stress_off_heap_code);
-  // TODO(jgruber,v8:6666): Replace with binary search through range checks
-  // once off-heap code is mapped into a contiguous memory space.
-  for (const InstructionStream* stream : isolate->off_heap_code_) {
-    if (stream->Contains(address)) {
-      return isolate->builtins()->builtin(stream->builtin_index());
+
+  if (!PcIsOffHeap(isolate, address)) return nullptr;
+
+  EmbeddedData d = EmbeddedData::FromBlob(isolate->embedded_blob(),
+                                          isolate->embedded_blob_size());
+
+  int l = 0, r = Builtins::builtin_count;
+  while (l < r) {
+    const int mid = (l + r) / 2;
+    const uint8_t* start = d.InstructionStartOfBuiltin(mid);
+    const uint8_t* end = start + d.InstructionSizeOfBuiltin(mid);
+
+    if (address < start) {
+      r = mid;
+    } else if (address >= end) {
+      l = mid + 1;
+    } else {
+      return isolate->builtins()->builtin(mid);
     }
   }
-  return nullptr;
-}
 
-// static
-InstructionStream* InstructionStream::TryLookupInstructionStream(
-    Isolate* isolate, Code* code) {
-  DCHECK(FLAG_stress_off_heap_code);
-  // TODO(jgruber,v8:6666): Replace with binary search through range checks
-  // once off-heap code is mapped into a contiguous memory space.
-  const int builtin_index = code->builtin_index();
-  DCHECK(Builtins::IsBuiltinId(builtin_index));
-  for (InstructionStream* stream : isolate->off_heap_code_) {
-    if (stream->builtin_index() == builtin_index) return stream;
-  }
+  UNREACHABLE();
+#else
   return nullptr;
-}
-
-bool InstructionStream::Contains(Address address) const {
-  return bytes_ <= address && address < bytes_ + byte_length_;
+#endif
 }
 
 }  // namespace internal

src/instruction-stream.h

@@ -14,22 +14,18 @@ namespace internal {
 class Code;
 class Isolate;
 
-// Wraps an mmap'ed off-heap instruction stream. This class will likely become
-// unneeded once --stress-off-heap-code is removed.
+// Wraps an off-heap instruction stream.
+// TODO(jgruber,v8:6666): Remove this class.
 class InstructionStream final {
  public:
-  explicit InstructionStream(Code* code);
-  ~InstructionStream();
+  InstructionStream(uint8_t* bytes, size_t byte_length, int builtin_index);
+
+  // Returns true, iff the given pc points into an off-heap instruction stream.
+  static bool PcIsOffHeap(Isolate* isolate, Address pc);
 
   // Returns the corresponding Code object if it exists, and nullptr otherwise.
   static Code* TryLookupCode(Isolate* isolate, Address address);
 
-  // Returns the corresponding stream if it exists, and nullptr otherwise.
-  static InstructionStream* TryLookupInstructionStream(Isolate* isolate,
-                                                       Code* code);
-
-  bool Contains(Address address) const;
-
   int builtin_index() const { return builtin_index_; }
   size_t byte_length() const { return byte_length_; }
   uint8_t* bytes() const { return bytes_; }

src/isolate.cc

@@ -21,7 +21,6 @@
 #include "src/basic-block-profiler.h"
 #include "src/bootstrapper.h"
 #include "src/builtins/constants-table-builder.h"
-#include "src/callable.h"
 #include "src/cancelable-task.h"
 #include "src/code-stubs.h"
 #include "src/compilation-cache.h"
@@ -35,7 +34,6 @@
 #include "src/frames-inl.h"
 #include "src/ic/stub-cache.h"
 #include "src/instruction-stream.h"
-#include "src/interface-descriptors.h"
 #include "src/interpreter/interpreter.h"
 #include "src/isolate-inl.h"
 #include "src/libsampler/sampler.h"
@@ -2680,11 +2678,6 @@ void Isolate::Deinit() {
   root_index_map_ = nullptr;
 
   ClearSerializerData();
-
-  for (InstructionStream* stream : off_heap_code_) {
-    delete stream;
-  }
-  off_heap_code_.clear();
 }
 
 
@@ -2894,8 +2887,11 @@ void LogInstructionStream(Isolate* isolate, Code* code,
   }
 }
 
-void MoveBuiltinsOffHeap(Isolate* isolate) {
+void CreateOnHeapTrampolines(Isolate* isolate) {
   DCHECK(FLAG_stress_off_heap_code);
+  DCHECK(!isolate->serializer_enabled());
+  DCHECK_NOT_NULL(isolate->embedded_blob());
+
   HandleScope scope(isolate);
   Builtins* builtins = isolate->builtins();
 
@@ -2904,14 +2900,26 @@ void MoveBuiltinsOffHeap(Isolate* isolate) {
   // deserialization.
   Snapshot::EnsureAllBuiltinsAreDeserialized(isolate);
 
+  EmbeddedData d = EmbeddedData::FromBlob(isolate->embedded_blob(),
+                                          isolate->embedded_blob_size());
+
   CodeSpaceMemoryModificationScope code_allocation(isolate->heap());
   for (int i = 0; i < Builtins::builtin_count; i++) {
     if (!Builtins::IsOffHeapSafe(i)) continue;
+
+    const uint8_t* instruction_start = d.InstructionStartOfBuiltin(i);
+    size_t instruction_size = d.InstructionSizeOfBuiltin(i);
+
+    InstructionStream stream(const_cast<uint8_t*>(instruction_start),
+                             instruction_size, i);
+
     Handle<Code> code(builtins->builtin(i));
-    InstructionStream* stream = new InstructionStream(*code);
-    LogInstructionStream(isolate, *code, stream);
-    ChangeToOffHeapTrampoline(isolate, code, stream);
-    isolate->PushOffHeapCode(stream);
+    LogInstructionStream(isolate, *code, &stream);
+
+    // TODO(jgruber,v8:6666): Create fresh trampolines instead of rewriting
+    // existing ones. This could happen prior to serialization or
+    // post-deserialization.
+    ChangeToOffHeapTrampoline(isolate, code, &stream);
   }
 }
 #endif  // V8_EMBEDDED_BUILTINS
@@ -3078,11 +3086,10 @@ bool Isolate::Init(StartupDeserializer* des) {
   if (FLAG_print_builtin_size) PrintBuiltinSizes(this);
 
 #ifdef V8_EMBEDDED_BUILTINS
-  if (FLAG_stress_off_heap_code && !serializer_enabled()) {
-    // Artificially move code off-heap to help find & verify related code
-    // paths. Lazy deserialization should be off to avoid confusion around
-    // replacing just the kDeserializeLazy code object.
-    MoveBuiltinsOffHeap(this);
+  if (FLAG_stress_off_heap_code && !serializer_enabled() &&
+      embedded_blob() != nullptr) {
+    // Create the on-heap trampolines that jump into embedded code.
+    CreateOnHeapTrampolines(this);
   }
 #endif
 

src/isolate.h

@@ -1241,10 +1241,6 @@ class Isolate {
     return &partial_snapshot_cache_;
   }
 
-  void PushOffHeapCode(InstructionStream* stream) {
-    off_heap_code_.emplace_back(stream);
-  }
-
 #ifdef V8_EMBEDDED_BUILTINS
   BuiltinsConstantsTableBuilder* builtins_constants_table_builder() const {
     return builtins_constants_table_builder_;
@@ -1623,12 +1619,6 @@ class Isolate {
 
   std::vector<Object*> partial_snapshot_cache_;
 
-  // Stores off-heap instruction streams. Only used if --stress-off-heap-code
-  // is enabled.
-  // TODO(jgruber,v8:6666): Remove once isolate-independent builtins are
-  // implemented. Also remove friend class below.
-  std::vector<InstructionStream*> off_heap_code_;
-
 #ifdef V8_EMBEDDED_BUILTINS
   // Used during builtins compilation to build the builtins constants table,
   // which is stored on the root list prior to serialization.
@@ -1663,7 +1653,6 @@ class Isolate {
   friend class ExecutionAccess;
   friend class HandleScopeImplementer;
   friend class heap::HeapTester;
-  friend class InstructionStream;
   friend class OptimizingCompileDispatcher;
   friend class Simulator;
   friend class StackGuard;

src/objects.cc

@@ -68,6 +68,7 @@
 #include "src/regexp/jsregexp.h"
 #include "src/safepoint-table.h"
 #include "src/snapshot/code-serializer.h"
+#include "src/snapshot/snapshot.h"
 #include "src/source-position-table.h"
 #include "src/string-builder.h"
 #include "src/string-search.h"
@@ -13962,23 +13963,29 @@ SafepointEntry Code::GetSafepointEntry(Address pc) {
 #ifdef V8_EMBEDDED_BUILTINS
 int Code::OffHeapInstructionSize() {
   DCHECK(Builtins::IsOffHeapBuiltin(this));
-  InstructionStream* stream =
-      InstructionStream::TryLookupInstructionStream(GetIsolate(), this);
-  return static_cast<int>(stream->byte_length());
+  Isolate* isolate = GetIsolate();
+  EmbeddedData d = EmbeddedData::FromBlob(isolate->embedded_blob(),
+                                          isolate->embedded_blob_size());
+  return d.InstructionSizeOfBuiltin(builtin_index());
 }
 
 Address Code::OffHeapInstructionStart() {
   DCHECK(Builtins::IsOffHeapBuiltin(this));
-  InstructionStream* stream =
-      InstructionStream::TryLookupInstructionStream(GetIsolate(), this);
-  return stream->bytes();
+  Isolate* isolate = GetIsolate();
+  EmbeddedData d = EmbeddedData::FromBlob(isolate->embedded_blob(),
+                                          isolate->embedded_blob_size());
+  return reinterpret_cast<Address>(
+      const_cast<uint8_t*>(d.InstructionStartOfBuiltin(builtin_index())));
 }
 
 Address Code::OffHeapInstructionEnd() {
   DCHECK(Builtins::IsOffHeapBuiltin(this));
-  InstructionStream* stream =
-      InstructionStream::TryLookupInstructionStream(GetIsolate(), this);
-  return stream->bytes() + stream->byte_length();
+  Isolate* isolate = GetIsolate();
+  EmbeddedData d = EmbeddedData::FromBlob(isolate->embedded_blob(),
+                                          isolate->embedded_blob_size());
+  return reinterpret_cast<Address>(
+      const_cast<uint8_t*>(d.InstructionStartOfBuiltin(builtin_index()) +
+                           d.InstructionSizeOfBuiltin(builtin_index())));
 }
 #endif