[web-snapshots] Add runtime function for WebSnapshot creation Part I

This CL prepares WebSnapshot for skipping and re-injecting external
references in the web snapshot. External references are encoded as
separate object type and allows us to create partial snapshots at
runtime and reconnect a deserialised snapshot to an existing
object graph.

Part II will also collect all objects which cannot be serialized by the
web-snapshot serializer.

Usage:
  snapshot = %WebSnapshotSerialize(root, skip_externals);
  object = %eWebSnapshotDeserializ(snapshot, replaced_externals);

Drive-by-changes:
- Reduce JSObject Map size in serializer (we ended up with 4 embedder
  fields)
- Avoid adding non-HeapObject to the discovery_queue_
- Split off ReadXXX handlers into separate functions

Bug: v8:11525
Change-Id: Ia6a9914259614c6c288667621b38daa0202d4d72
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3461936Reviewed-by: Marja Hölttä <marja@chromium.org>
Reviewed-by: Leszek Swirski <leszeks@chromium.org>
Commit-Queue: Camillo Bruni <cbruni@chromium.org>
Cr-Commit-Position: refs/heads/main@{#79211}

src/runtime/runtime-test.cc

@@ -29,11 +29,13 @@
 #include "src/objects/js-array-inl.h"
 #include "src/objects/js-function-inl.h"
 #include "src/objects/js-regexp-inl.h"
+#include "src/objects/managed-inl.h"
 #include "src/objects/smi.h"
 #include "src/profiler/heap-snapshot-generator.h"
 #include "src/regexp/regexp.h"
 #include "src/runtime/runtime-utils.h"
 #include "src/snapshot/snapshot.h"
+#include "src/web-snapshot/web-snapshot.h"
 
 #if V8_ENABLE_WEBASSEMBLY
 #include "src/wasm/wasm-engine.h"
@@ -1480,6 +1482,82 @@ RUNTIME_FUNCTION(Runtime_IsSharedString) {
                                     Handle<String>::cast(obj)->IsShared());
 }
 
+RUNTIME_FUNCTION(Runtime_WebSnapshotSerialize) {
+  if (!FLAG_allow_natives_syntax) {
+    return ReadOnlyRoots(isolate).undefined_value();
+  }
+  HandleScope scope(isolate);
+  if (args.length() < 1 || args.length() > 2) {
+    THROW_NEW_ERROR_RETURN_FAILURE(
+        isolate, NewTypeError(MessageTemplate::kRuntimeWrongNumArgs));
+  }
+  Handle<Object> object = args.at<Object>(0);
+  Handle<FixedArray> block_list = isolate->factory()->empty_fixed_array();
+  if (args.length() == 2) {
+    if (!args[1].IsJSArray()) {
+      THROW_NEW_ERROR_RETURN_FAILURE(
+          isolate, NewTypeError(MessageTemplate::kInvalidArgument));
+    }
+    auto raw_blocklist = args.at<JSArray>(1);
+    ASSIGN_RETURN_FAILURE_ON_EXCEPTION(
+        isolate, block_list,
+        JSReceiver::GetOwnValues(raw_blocklist,
+                                 PropertyFilter::ENUMERABLE_STRINGS));
+  }
+
+  auto snapshot_data = std::make_shared<WebSnapshotData>();
+  WebSnapshotSerializer serializer(isolate);
+  if (!serializer.TakeSnapshot(object, block_list, *snapshot_data)) {
+    return ReadOnlyRoots(isolate).exception();
+  }
+  i::Handle<i::Object> managed_object = Managed<WebSnapshotData>::FromSharedPtr(
+      isolate, snapshot_data->buffer_size, snapshot_data);
+  return *managed_object;
+}
+
+RUNTIME_FUNCTION(Runtime_WebSnapshotDeserialize) {
+  if (!FLAG_allow_natives_syntax) {
+    return ReadOnlyRoots(isolate).undefined_value();
+  }
+  HandleScope scope(isolate);
+  if (args.length() == 0 || args.length() > 2) {
+    THROW_NEW_ERROR_RETURN_FAILURE(
+        isolate, NewTypeError(MessageTemplate::kRuntimeWrongNumArgs));
+  }
+  if (!args[0].IsForeign()) {
+    THROW_NEW_ERROR_RETURN_FAILURE(
+        isolate, NewTypeError(MessageTemplate::kInvalidArgument));
+  }
+  Handle<Foreign> foreign_data = args.at<Foreign>(0);
+  Handle<FixedArray> injected_references =
+      isolate->factory()->empty_fixed_array();
+  if (args.length() == 2) {
+    if (!args[1].IsJSArray()) {
+      THROW_NEW_ERROR_RETURN_FAILURE(
+          isolate, NewTypeError(MessageTemplate::kInvalidArgument));
+    }
+    auto js_array = args.at<JSArray>(1);
+    ASSIGN_RETURN_FAILURE_ON_EXCEPTION(
+        isolate, injected_references,
+        JSReceiver::GetOwnValues(js_array, PropertyFilter::ENUMERABLE_STRINGS));
+  }
+
+  auto data = Managed<WebSnapshotData>::cast(*foreign_data).get();
+  v8::Isolate* v8_isolate = reinterpret_cast<v8::Isolate*>(isolate);
+  WebSnapshotDeserializer deserializer(v8_isolate, data->buffer,
+                                       data->buffer_size);
+  if (!deserializer.Deserialize(injected_references)) {
+    THROW_NEW_ERROR_RETURN_FAILURE(
+        isolate, NewTypeError(MessageTemplate::kWebSnapshotError));
+  }
+  Handle<Object> object;
+  if (!deserializer.value().ToHandle(&object)) {
+    THROW_NEW_ERROR_RETURN_FAILURE(
+        isolate, NewTypeError(MessageTemplate::kWebSnapshotError));
+  }
+  return *object;
+}
+
 RUNTIME_FUNCTION(Runtime_SharedGC) {
   SealHandleScope scope(isolate);
   isolate->heap()->CollectSharedGarbage(GarbageCollectionReason::kTesting);

src/runtime/runtime.h

@@ -140,6 +140,8 @@ namespace internal {
   F(HandleDebuggerStatement, 0, 1)              \
   F(IsBreakOnException, 1, 1)                   \
   F(LiveEditPatchScript, 2, 1)                  \
+  F(WebSnapshotDeserialize, -1, 1)              \
+  F(WebSnapshotSerialize, -1, 1)                \
   F(ProfileCreateSnapshotDataBlob, 0, 1)        \
   F(ScheduleBreak, 0, 1)                        \
   F(ScriptLocationFromLine2, 4, 1)              \

src/snapshot/serializer.h

@@ -141,7 +141,7 @@ class ObjectCacheIndexMap {
   // If |obj| is in the map, immediately return true.  Otherwise add it to the
   // map and return false. In either case set |*index_out| to the index
   // associated with the map.
-  bool LookupOrInsert(Handle<HeapObject> obj, int* index_out) {
+  bool LookupOrInsert(HeapObject obj, int* index_out) {
     auto find_result = map_.FindOrInsert(obj);
     if (!find_result.already_exists) {
       *find_result.entry = next_index_++;
@@ -149,6 +149,9 @@ class ObjectCacheIndexMap {
     *index_out = *find_result.entry;
     return find_result.already_exists;
   }
+  bool LookupOrInsert(Handle<HeapObject> obj, int* index_out) {
+    return LookupOrInsert(*obj, index_out);
+  }
 
   bool Lookup(HeapObject obj, int* index_out) const {
     int* index = map_.Find(obj);

src/snapshot/startup-serializer.cc

@@ -132,7 +132,6 @@ bool IsUnexpectedCodeObject(Isolate* isolate, HeapObject obj) {
 
 }  // namespace
 #endif  // DEBUG
-
 void StartupSerializer::SerializeObjectImpl(Handle<HeapObject> obj) {
   PtrComprCageBase cage_base(isolate());
 #ifdef DEBUG

src/web-snapshot/web-snapshot.h

@@ -52,7 +52,8 @@ class WebSnapshotSerializerDeserializer {
     OBJECT_ID,
     FUNCTION_ID,
     CLASS_ID,
-    REGEXP
+    REGEXP,
+    EXTERNAL_ID
   };
 
   static constexpr uint8_t kMagicNumber[4] = {'+', '+', '+', ';'};
@@ -82,6 +83,9 @@ class WebSnapshotSerializerDeserializer {
       : isolate_(isolate) {}
   // Not virtual, on purpose (because it doesn't need to be).
   void Throw(const char* message);
+
+  inline Factory* factory() const { return isolate_->factory(); }
+
   Isolate* isolate_;
   const char* error_message_ = nullptr;
 
@@ -111,11 +115,15 @@ class V8_EXPORT WebSnapshotSerializer
     : public WebSnapshotSerializerDeserializer {
  public:
   explicit WebSnapshotSerializer(v8::Isolate* isolate);
+  explicit WebSnapshotSerializer(Isolate* isolate);
+
   ~WebSnapshotSerializer();
 
   bool TakeSnapshot(v8::Local<v8::Context> context,
                     v8::Local<v8::PrimitiveArray> exports,
                     WebSnapshotData& data_out);
+  bool TakeSnapshot(Handle<Object> object, MaybeHandle<FixedArray> block_list,
+                    WebSnapshotData& data_out);
 
   // For inspecting the state after taking a snapshot.
   uint32_t string_count() const {
@@ -150,12 +158,15 @@ class V8_EXPORT WebSnapshotSerializer
 
   void SerializePendingItems();
   void WriteSnapshot(uint8_t*& buffer, size_t& buffer_size);
+  void WriteObjects(ValueSerializer& destination, size_t count,
+                    ValueSerializer& source, const char* name);
 
   // Returns true if the object was already in the map, false if it was added.
-  bool InsertIntoIndexMap(ObjectCacheIndexMap& map, Handle<HeapObject> object,
+  bool InsertIntoIndexMap(ObjectCacheIndexMap& map, HeapObject heap_object,
                           uint32_t& id);
 
-  void Discovery(Handle<Object> object);
+  void ShallowDiscoverExternals(FixedArray externals);
+  void Discover(Handle<HeapObject> object);
   void DiscoverFunction(Handle<JSFunction> function);
   void DiscoverClass(Handle<JSFunction> function);
   void DiscoverContextAndPrototype(Handle<JSFunction> function);
@@ -177,7 +188,7 @@ class V8_EXPORT WebSnapshotSerializer
   void SerializeArray(Handle<JSArray> array);
   void SerializeObject(Handle<JSObject> object);
 
-  void SerializeExport(Handle<JSObject> object, Handle<String> export_name);
+  void SerializeExport(Handle<Object> object, Handle<String> export_name);
   void WriteValue(Handle<Object> object, ValueSerializer& serializer);
 
   uint32_t GetFunctionId(JSFunction function);
@@ -185,6 +196,7 @@ class V8_EXPORT WebSnapshotSerializer
   uint32_t GetContextId(Context context);
   uint32_t GetArrayId(JSArray array);
   uint32_t GetObjectId(JSObject object);
+  uint32_t GetExternalId(HeapObject object);
 
   ValueSerializer string_serializer_;
   ValueSerializer map_serializer_;
@@ -202,6 +214,10 @@ class V8_EXPORT WebSnapshotSerializer
   Handle<ArrayList> arrays_;
   Handle<ArrayList> objects_;
 
+  // IndexMap to keep track of explicitly blocked external objects and
+  // non-serializable/not-supporte objects (e.g. API Objects).
+  ObjectCacheIndexMap external_objects_ids_;
+
   // ObjectCacheIndexMap implements fast lookup item -> id.
   ObjectCacheIndexMap string_ids_;
   ObjectCacheIndexMap map_ids_;
@@ -212,7 +228,7 @@ class V8_EXPORT WebSnapshotSerializer
   ObjectCacheIndexMap object_ids_;
   uint32_t export_count_ = 0;
 
-  std::queue<Handle<Object>> discovery_queue_;
+  std::queue<Handle<HeapObject>> discovery_queue_;
 
   // For constructing the minimal, "compacted", source string to cover all
   // function bodies.
@@ -232,7 +248,7 @@ class V8_EXPORT WebSnapshotDeserializer
                           size_t buffer_size);
   WebSnapshotDeserializer(Isolate* isolate, Handle<Script> snapshot_as_script);
   ~WebSnapshotDeserializer();
-  bool Deserialize();
+  bool Deserialize(MaybeHandle<FixedArray> external_references = {});
 
   // For inspecting the state after deserializing a snapshot.
   uint32_t string_count() const { return string_count_; }
@@ -251,6 +267,8 @@ class V8_EXPORT WebSnapshotDeserializer
 
   void UpdatePointers();
 
+  MaybeHandle<Object> value() const { return return_value_; }
+
  private:
   WebSnapshotDeserializer(Isolate* isolate, Handle<Object> script_name,
                           base::Vector<const uint8_t> buffer);
@@ -263,7 +281,6 @@ class V8_EXPORT WebSnapshotDeserializer
   WebSnapshotDeserializer& operator=(const WebSnapshotDeserializer&) = delete;
 
   void DeserializeStrings();
-  String ReadString(bool internalize = false);
   void DeserializeMaps();
   void DeserializeContexts();
   Handle<ScopeInfo> CreateScopeInfo(uint32_t variable_count, bool has_parent,
@@ -277,9 +294,21 @@ class V8_EXPORT WebSnapshotDeserializer
   void DeserializeArrays();
   void DeserializeObjects();
   void DeserializeExports();
+
   Object ReadValue(
       Handle<HeapObject> object_for_deferred_reference = Handle<HeapObject>(),
       uint32_t index_for_deferred_reference = 0);
+
+  Object ReadInteger();
+  Object ReadNumber();
+  String ReadString(bool internalize = false);
+  Object ReadArray(Handle<HeapObject> container, uint32_t container_index);
+  Object ReadObject(Handle<HeapObject> container, uint32_t container_index);
+  Object ReadFunction(Handle<HeapObject> container, uint32_t container_index);
+  Object ReadClass(Handle<HeapObject> container, uint32_t container_index);
+  Object ReadRegexp();
+  Object ReadExternalReference();
+
   void ReadFunctionPrototype(Handle<JSFunction> function);
   bool SetFunctionPrototype(JSFunction function, JSReceiver prototype);
 
@@ -311,6 +340,9 @@ class V8_EXPORT WebSnapshotDeserializer
   Handle<FixedArray> objects_handle_;
   FixedArray objects_;
 
+  Handle<FixedArray> external_references_handle_;
+  FixedArray external_references_;
+
   Handle<ArrayList> deferred_references_;
 
   Handle<WeakFixedArray> shared_function_infos_handle_;
@@ -321,6 +353,8 @@ class V8_EXPORT WebSnapshotDeserializer
   Handle<Script> script_;
   Handle<Object> script_name_;
 
+  Handle<Object> return_value_;
+
   uint32_t string_count_ = 0;
   uint32_t map_count_ = 0;
   uint32_t context_count_ = 0;
@@ -334,6 +368,7 @@ class V8_EXPORT WebSnapshotDeserializer
   uint32_t current_object_count_ = 0;
 
   ValueDeserializer deserializer_;
+  ReadOnlyRoots roots_;
 
   bool deserialized_ = false;
 };

test/mjsunit/web-snapshot/web-snapshot-externals.js

+// Copyright 2022 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --experimental-d8-web-snapshot-api --allow-natives-syntax
+
+
+
+const external_1 = {external: 1};
+const external_2 = {external: 2};
+const object = {
+  a: [1,2],
+  b: external_1,
+  c: [external_1, external_2],
+  d: { d_a: external_2 }
+};
+
+
+(function testNoExternals() {
+  const snapshot = %WebSnapshotSerialize(object);
+  const deserialized = %WebSnapshotDeserialize(snapshot);
+  %HeapObjectVerify(deserialized);
+  assertEquals(deserialized, object);
+  assertEquals(deserialized.b, external_1);
+  assertNotSame(deserialized.b, external_1);
+  assertEquals(deserialized.d.d_a, external_2);
+  assertNotSame(deserialized.d.d_a, external_2);
+})();
+
+(function testOneExternals() {
+  const externals = [ external_1];
+  const snapshot = %WebSnapshotSerialize(object, externals);
+  const replaced_externals = [{replacement:1}]
+  const deserialized = %WebSnapshotDeserialize(snapshot, replaced_externals);
+  %HeapObjectVerify(deserialized);
+  assertEquals(deserialized.a, object.a);
+  assertSame(deserialized.b, replaced_externals[0]);
+  assertArrayEquals(deserialized.c, [replaced_externals[0], external_2]);
+  assertSame(deserialized.c[0], replaced_externals[0]);
+  assertNotSame(deserialized.c[1], external_2);
+  assertEquals(deserialized.d.d_a, external_2);
+  assertNotSame(deserialized.d.d_a, external_2);
+})();
+
+(function testTwoExternals() {
+  const externals = [external_1, external_2];
+  const snapshot = %WebSnapshotSerialize(object, externals);
+  const replaced_externals = [{replacement:1}, {replacement:2}]
+  const deserialized = %WebSnapshotDeserialize(snapshot, replaced_externals);
+  %HeapObjectVerify(deserialized);
+  assertEquals(deserialized.a, object.a);
+  assertSame(deserialized.b, replaced_externals[0]);
+  assertArrayEquals(deserialized.c, replaced_externals);
+  assertSame(deserialized.c[0], replaced_externals[0]);
+  assertSame(deserialized.c[1], replaced_externals[1]);
+  assertSame(deserialized.d.d_a, replaced_externals[1]);
+})();