[web snapshots] Web Snapshots Version 0.01

The minimal implementation which does something useful. Initial
machinery for serializing / deserializing objects and functions (only
the very simple cases are supported).

For more info, see https://docs.google.com/document/d/1Qierkg3b3klIwCQt-oZCHqhcc1_9DXNIErBwvdpD4wU/edit?usp=sharing

Bug: v8:11525

Change-Id: I73c4de11285c7912bf9870868d203d4b3d2b4e5f
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2716288Reviewed-by: Hannes Payer <hpayer@chromium.org>
Reviewed-by: Shu-yu Guo <syg@chromium.org>
Reviewed-by: Leszek Swirski <leszeks@chromium.org>
Commit-Queue: Marja Hölttä <marja@chromium.org>
Cr-Commit-Position: refs/heads/master@{#73371}

BUILD.gn

@@ -3926,6 +3926,8 @@ v8_source_set("v8_base_without_compiler") {
     "src/utils/ostreams.cc",
     "src/utils/utils.cc",
     "src/utils/version.cc",
+    "src/web-snapshot/web-snapshot.cc",
+    "src/web-snapshot/web-snapshot.h",
     "src/zone/accounting-allocator.cc",
     "src/zone/type-stats.cc",
     "src/zone/zone-segment.cc",

src/d8/d8.cc

@@ -59,6 +59,7 @@
 #include "src/trap-handler/trap-handler.h"
 #include "src/utils/ostreams.h"
 #include "src/utils/utils.h"
+#include "src/web-snapshot/web-snapshot.h"
 
 #ifdef V8_FUZZILLI
 #include "src/d8/cov.h"
@@ -723,6 +724,27 @@ bool Shell::ExecuteString(Isolate* isolate, Local<String> source,
       if (!HandleUnhandledPromiseRejections(isolate)) success = false;
     }
     data->realm_current_ = data->realm_switch_;
+
+    if (options.web_snapshot_config) {
+      std::vector<std::string> exports;
+      if (!ReadLines(options.web_snapshot_config, exports)) {
+        Throw(isolate, "Web snapshots: unable to read config");
+        CHECK(try_catch.HasCaught());
+        ReportException(isolate, &try_catch);
+        return false;
+      }
+
+      i::WebSnapshotSerializer serializer(isolate);
+      i::WebSnapshotData snapshot_data;
+      if (serializer.TakeSnapshot(context, exports, snapshot_data)) {
+        DCHECK_NOT_NULL(snapshot_data.buffer);
+        WriteChars("web.snap", snapshot_data.buffer, snapshot_data.buffer_size);
+      } else {
+        CHECK(try_catch.HasCaught());
+        ReportException(isolate, &try_catch);
+        return false;
+      }
+    }
   }
   Local<Value> result;
   if (!maybe_result.ToLocal(&result)) {
@@ -1325,6 +1347,37 @@ bool Shell::ExecuteModule(Isolate* isolate, const char* file_name) {
   return true;
 }
 
+bool Shell::ExecuteWebSnapshot(Isolate* isolate, const char* file_name) {
+  HandleScope handle_scope(isolate);
+
+  PerIsolateData* data = PerIsolateData::Get(isolate);
+  Local<Context> realm = data->realms_[data->realm_current_].Get(isolate);
+  Context::Scope context_scope(realm);
+
+  std::string absolute_path = NormalizePath(file_name, GetWorkingDirectory());
+
+  TryCatch try_catch(isolate);
+  try_catch.SetVerbose(true);
+  int length = 0;
+  std::unique_ptr<uint8_t[]> snapshot_data(
+      reinterpret_cast<uint8_t*>(ReadChars(absolute_path.c_str(), &length)));
+  if (length == 0) {
+    Throw(isolate, "Error reading the web snapshot");
+    DCHECK(try_catch.HasCaught());
+    ReportException(isolate, &try_catch);
+    return false;
+  }
+  i::WebSnapshotDeserializer deserializer(isolate);
+  if (!deserializer.UseWebSnapshot(snapshot_data.get(),
+                                   static_cast<size_t>(length))) {
+    DCHECK(try_catch.HasCaught());
+    ReportException(isolate, &try_catch);
+    return false;
+  }
+  DCHECK(!try_catch.HasCaught());
+  return true;
+}
+
 PerIsolateData::PerIsolateData(Isolate* isolate)
     : isolate_(isolate), realms_(nullptr) {
   isolate->SetData(0, this);
@@ -3059,9 +3112,9 @@ static FILE* FOpen(const char* path, const char* mode) {
 #endif
 }
 
-static char* ReadChars(const char* name, int* size_out) {
-  if (Shell::options.read_from_tcp_port >= 0) {
-    return Shell::ReadCharsFromTcpPort(name, size_out);
+char* Shell::ReadChars(const char* name, int* size_out) {
+  if (options.read_from_tcp_port >= 0) {
+    return ReadCharsFromTcpPort(name, size_out);
   }
 
   FILE* file = FOpen(name, "rb");
@@ -3086,6 +3139,20 @@ static char* ReadChars(const char* name, int* size_out) {
   return chars;
 }
 
+bool Shell::ReadLines(const char* name, std::vector<std::string>& lines) {
+  int length;
+  const char* data = reinterpret_cast<const char*>(ReadChars(name, &length));
+  if (data == nullptr) {
+    return false;
+  }
+  std::stringstream stream(data);
+  std::string line;
+  while (std::getline(stream, line, '\n')) {
+    lines.emplace_back(line);
+  }
+  return true;
+}
+
 void Shell::ReadBuffer(const v8::FunctionCallbackInfo<v8::Value>& args) {
   static_assert(sizeof(char) == sizeof(uint8_t),
                 "char and uint8_t should both have 1 byte");
@@ -3136,6 +3203,13 @@ Local<String> Shell::ReadFile(Isolate* isolate, const char* name) {
   return result;
 }
 
+void Shell::WriteChars(const char* name, uint8_t* buffer, size_t buffer_size) {
+  FILE* file = base::Fopen(name, "w");
+  if (file == nullptr) return;
+  fwrite(buffer, 1, buffer_size, file);
+  base::Fclose(file);
+}
+
 void Shell::RunShell(Isolate* isolate) {
   HandleScope outer_scope(isolate);
   v8::Local<v8::Context> context =
@@ -3393,6 +3467,15 @@ bool SourceGroup::Execute(Isolate* isolate) {
         break;
       }
       continue;
+    } else if (strcmp(arg, "--web-snapshot") == 0 && i + 1 < end_offset_) {
+      // Treat the next file as a web snapshot.
+      arg = argv_[++i];
+      Shell::set_script_executed();
+      if (!Shell::ExecuteWebSnapshot(isolate, arg)) {
+        success = false;
+        break;
+      }
+      continue;
     } else if (arg[0] == '-') {
       // Ignore other options. They have been parsed already.
       continue;
@@ -3945,6 +4028,9 @@ bool Shell::SetOptions(int argc, char* argv[]) {
       options.cpu_profiler = true;
       options.cpu_profiler_print = true;
       argv[i] = nullptr;
+    } else if (strncmp(argv[i], "--web-snapshot-config=", 22) == 0) {
+      options.web_snapshot_config = argv[i] + 22;
+      argv[i] = nullptr;
 #ifdef V8_FUZZILLI
     } else if (strcmp(argv[i], "--no-fuzzilli-enable-builtins-coverage") == 0) {
       options.fuzzilli_enable_builtins_coverage = false;
@@ -3972,10 +4058,12 @@ bool Shell::SetOptions(int argc, char* argv[]) {
   const char* usage =
       "Synopsis:\n"
       "  shell [options] [--shell] [<file>...]\n"
-      "  d8 [options] [-e <string>] [--shell] [[--module] <file>...]\n\n"
+      "  d8 [options] [-e <string>] [--shell] [[--module|--web-snapshot]"
+      " <file>...]\n\n"
       "  -e        execute a string in V8\n"
       "  --shell   run an interactive JavaScript shell\n"
-      "  --module  execute a file as a JavaScript module\n\n";
+      "  --module  execute a file as a JavaScript module\n"
+      "  --web-snapshot  execute a file as a web snapshot\n\n";
   using HelpOptions = i::FlagList::HelpOptions;
   i::FLAG_abort_on_contradictory_flags = true;
   i::FlagList::SetFlagsFromCommandLine(&argc, argv, true,
@@ -3997,8 +4085,9 @@ bool Shell::SetOptions(int argc, char* argv[]) {
       current->End(i);
       current++;
       current->Begin(argv, i + 1);
-    } else if (strcmp(str, "--module") == 0) {
-      // Pass on to SourceGroup, which understands this option.
+    } else if (strcmp(str, "--module") == 0 ||
+               strcmp(str, "--web-snapshot") == 0) {
+      // Pass on to SourceGroup, which understands these options.
     } else if (strncmp(str, "--", 2) == 0) {
       if (!i::FLAG_correctness_fuzzer_suppressions) {
         printf("Warning: unknown flag %s.\nTry --help for options\n", str);

src/d8/d8.h

@@ -401,6 +401,8 @@ class ShellOptions {
       "fuzzy-module-file-extensions", true};
   DisallowReassignment<bool> enable_system_instrumentation = {
       "enable-system-instrumentation", false};
+  DisallowReassignment<const char*> web_snapshot_config = {
+      "web-snapshot-config", nullptr};
 };
 
 class Shell : public i::AllStatic {
@@ -420,6 +422,7 @@ class Shell : public i::AllStatic {
                             ReportExceptions report_exceptions,
                             ProcessMessageQueue process_message_queue);
   static bool ExecuteModule(Isolate* isolate, const char* file_name);
+  static bool ExecuteWebSnapshot(Isolate* isolate, const char* file_name);
   static void ReportException(Isolate* isolate, Local<Message> message,
                               Local<Value> exception);
   static void ReportException(Isolate* isolate, TryCatch* try_catch);
@@ -489,11 +492,14 @@ class Shell : public i::AllStatic {
   static void Quit(const v8::FunctionCallbackInfo<v8::Value>& args);
   static void Version(const v8::FunctionCallbackInfo<v8::Value>& args);
   static void Read(const v8::FunctionCallbackInfo<v8::Value>& args);
+  static char* ReadChars(const char* name, int* size_out);
+  static bool ReadLines(const char* name, std::vector<std::string>& lines);
   static void ReadBuffer(const v8::FunctionCallbackInfo<v8::Value>& args);
   static Local<String> ReadFromStdin(Isolate* isolate);
   static void ReadLine(const v8::FunctionCallbackInfo<v8::Value>& args) {
     args.GetReturnValue().Set(ReadFromStdin(args.GetIsolate()));
   }
+  static void WriteChars(const char* name, uint8_t* buffer, size_t buffer_size);
   static void Load(const v8::FunctionCallbackInfo<v8::Value>& args);
   static void SetTimeout(const v8::FunctionCallbackInfo<v8::Value>& args);
   static void WorkerNew(const v8::FunctionCallbackInfo<v8::Value>& args);

src/flags/flag-definitions.h

@@ -490,6 +490,8 @@ DEFINE_BOOL(trace_block_coverage, false,
             "trace collected block coverage information")
 DEFINE_BOOL(trace_protector_invalidation, false,
             "trace protector cell invalidations")
+DEFINE_BOOL(trace_web_snapshot, false, "trace web snapshot deserialization")
+
 DEFINE_BOOL(feedback_normalization, false,
             "feed back normalization to constructors")
 // TODO(jkummerow): This currently adds too much load on the stub cache.

src/objects/descriptor-array.h

@@ -217,6 +217,7 @@ class DescriptorArray
   using EntryValueField = TaggedField<MaybeObject, kEntryValueOffset>;
 
  private:
+  friend class WebSnapshotDeserializer;
   DECL_INT16_ACCESSORS(filler16bits)
 
   inline void SetKey(InternalIndex descriptor_number, Name key);

src/objects/value-serializer.cc

@@ -1103,6 +1103,15 @@ ValueDeserializer::ValueDeserializer(Isolate* isolate,
       id_map_(isolate->global_handles()->Create(
           ReadOnlyRoots(isolate_).empty_fixed_array())) {}
 
+ValueDeserializer::ValueDeserializer(Isolate* isolate, const uint8_t* data,
+                                     size_t size)
+    : isolate_(isolate),
+      delegate_(nullptr),
+      position_(data),
+      end_(data + size),
+      id_map_(isolate->global_handles()->Create(
+          ReadOnlyRoots(isolate_).empty_fixed_array())) {}
+
 ValueDeserializer::~ValueDeserializer() {
   GlobalHandles::Destroy(id_map_.location());
 

src/objects/value-serializer.h

@@ -94,6 +94,8 @@ class ValueSerializer {
   void SetTreatArrayBufferViewsAsHostObjects(bool mode);
 
  private:
+  friend class WebSnapshotSerializer;
+
   // Managing allocations of the internal buffer.
   Maybe<bool> ExpandBuffer(size_t required_capacity);
 
@@ -182,6 +184,7 @@ class ValueDeserializer {
  public:
   ValueDeserializer(Isolate* isolate, Vector<const uint8_t> data,
                     v8::ValueDeserializer::Delegate* delegate);
+  ValueDeserializer(Isolate* isolate, const uint8_t* data, size_t size);
   ~ValueDeserializer();
   ValueDeserializer(const ValueDeserializer&) = delete;
   ValueDeserializer& operator=(const ValueDeserializer&) = delete;
@@ -230,6 +233,8 @@ class ValueDeserializer {
   bool ReadRawBytes(size_t length, const void** data) V8_WARN_UNUSED_RESULT;
 
  private:
+  friend class WebSnapshotDeserializer;
+
   // Reading the wire format.
   Maybe<SerializationTag> PeekTag() const V8_WARN_UNUSED_RESULT;
   void ConsumeTag(SerializationTag peeked_tag);

src/snapshot/serializer.h

@@ -152,6 +152,8 @@ class ObjectCacheIndexMap {
     return find_result.already_exists;
   }
 
+  int size() const { return next_index_; }
+
  private:
   IdentityMap<int, base::DefaultAllocationPolicy> map_;
   int next_index_;

src/web-snapshot/OWNERS

+marja@chromium.org
+leszeks@chromium.org
+syg@chromium.org
+verwaest@chromium.org

src/web-snapshot/web-snapshot.h

+// Copyright 2021 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_WEB_SNAPSHOT_WEB_SNAPSHOT_H_
+#define V8_WEB_SNAPSHOT_WEB_SNAPSHOT_H_
+
+#include <queue>
+#include <vector>
+
+#include "src/handles/handles.h"
+#include "src/objects/value-serializer.h"
+#include "src/snapshot/serializer.h"  // For ObjectCacheIndexMap
+
+namespace v8 {
+
+class Context;
+class Isolate;
+
+template <typename T>
+class Local;
+
+namespace internal {
+
+class Context;
+class Map;
+class Object;
+class String;
+
+struct WebSnapshotData {
+  uint8_t* buffer = nullptr;
+  size_t buffer_size = 0;
+  ~WebSnapshotData() { free(buffer); }
+};
+
+class WebSnapshotSerializerDeserializer {
+ public:
+  bool has_error() const { return error_message_ != nullptr; }
+  const char* error_message() const { return error_message_; }
+
+  enum ValueType : uint8_t { STRING_ID, OBJECT_ID, FUNCTION_ID };
+
+ protected:
+  explicit WebSnapshotSerializerDeserializer(Isolate* isolate)
+      : isolate_(isolate) {}
+  void Throw(const char* message);
+  Isolate* isolate_;
+  const char* error_message_ = nullptr;
+
+ private:
+  WebSnapshotSerializerDeserializer(const WebSnapshotSerializerDeserializer&) =
+      delete;
+  WebSnapshotSerializerDeserializer& operator=(
+      const WebSnapshotSerializerDeserializer&) = delete;
+};
+
+class V8_EXPORT WebSnapshotSerializer
+    : public WebSnapshotSerializerDeserializer {
+ public:
+  explicit WebSnapshotSerializer(v8::Isolate* isolate);
+  ~WebSnapshotSerializer();
+
+  bool TakeSnapshot(v8::Local<v8::Context> context,
+                    const std::vector<std::string>& exports,
+                    WebSnapshotData& data_out);
+
+  // For inspecting the state after taking a snapshot.
+  uint32_t string_count() const;
+  uint32_t map_count() const;
+  uint32_t function_count() const;
+  uint32_t object_count() const;
+
+ private:
+  WebSnapshotSerializer(const WebSnapshotSerializer&) = delete;
+  WebSnapshotSerializer& operator=(const WebSnapshotSerializer&) = delete;
+
+  void WriteSnapshot(uint8_t*& buffer, size_t& buffer_size);
+
+  // Returns true if the object was already in the map, false if it was added.
+  bool InsertIntoIndexMap(ObjectCacheIndexMap& map, Handle<HeapObject> object,
+                          uint32_t& id);
+
+  void SerializeString(Handle<String> string, uint32_t& id);
+  void SerializeMap(Handle<Map> map, uint32_t& id);
+  void SerializeJSFunction(Handle<JSFunction> function, uint32_t& id);
+  void SerializeJSObject(Handle<JSObject> object, uint32_t& id);
+  void SerializePendingJSObject(Handle<JSObject> object);
+  void SerializeExport(Handle<JSObject> object, const std::string& export_name);
+  void WriteValue(Handle<Object> object, ValueSerializer& serializer);
+
+  ValueSerializer string_serializer_;
+  ValueSerializer map_serializer_;
+  ValueSerializer function_serializer_;
+  ValueSerializer object_serializer_;
+  ValueSerializer export_serializer_;
+
+  ObjectCacheIndexMap string_ids_;
+  ObjectCacheIndexMap map_ids_;
+  ObjectCacheIndexMap function_ids_;
+  ObjectCacheIndexMap object_ids_;
+  uint32_t export_count_ = 0;
+
+  std::queue<Handle<JSObject>> pending_objects_;
+};
+
+class V8_EXPORT WebSnapshotDeserializer
+    : public WebSnapshotSerializerDeserializer {
+ public:
+  explicit WebSnapshotDeserializer(v8::Isolate* v8_isolate);
+  bool UseWebSnapshot(const uint8_t* data, size_t buffer_size);
+
+  // For inspecting the state after taking a snapshot.
+  size_t string_count() const { return strings_.size(); }
+  size_t map_count() const { return maps_.size(); }
+  size_t function_count() const { return functions_.size(); }
+  size_t object_count() const { return objects_.size(); }
+
+ private:
+  WebSnapshotDeserializer(const WebSnapshotDeserializer&) = delete;
+  WebSnapshotDeserializer& operator=(const WebSnapshotDeserializer&) = delete;
+
+  void DeserializeStrings(const uint8_t* data, size_t& ix, size_t size);
+  void DeserializeMaps(const uint8_t* data, size_t& ix, size_t size);
+  void DeserializeFunctions(const uint8_t* data, size_t& ix, size_t size);
+  void DeserializeObjects(const uint8_t* data, size_t& ix, size_t size);
+  void DeserializeExports(const uint8_t* data, size_t& ix, size_t size);
+
+  std::vector<Handle<String>> strings_;
+  std::vector<Handle<Map>> maps_;
+  std::vector<Handle<JSFunction>> functions_;
+  std::vector<Handle<JSObject>> objects_;
+};
+
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_WEB_SNAPSHOT_WEB_SNAPSHOT_H_

test/cctest/BUILD.gn

@@ -287,6 +287,7 @@ v8_source_set("cctest_sources") {
     "test-version.cc",
     "test-weakmaps.cc",
     "test-weaksets.cc",
+    "test-web-snapshots.cc",
     "torque/test-torque.cc",
     "trace-extension.cc",
     "trace-extension.h",

test/cctest/test-web-snapshots.cc

+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/web-snapshot/web-snapshot.h"
+#include "test/cctest/cctest-utils.h"
+#include "test/cctest/cctest.h"
+
+namespace v8 {
+namespace internal {
+
+TEST(Minimal) {
+  CcTest::InitializeVM();
+  v8::Isolate* isolate = CcTest::isolate();
+  v8::HandleScope scope(isolate);
+  v8::Local<v8::Context> context = isolate->GetCurrentContext();
+
+  CompileRun("var foo = {'key': 'lol'}");
+  WebSnapshotData snapshot_data;
+  {
+    std::vector<std::string> exports;
+    exports.push_back("foo");
+    WebSnapshotSerializer serializer(isolate);
+    CHECK(serializer.TakeSnapshot(context, exports, snapshot_data));
+    CHECK(!serializer.has_error());
+    CHECK_NOT_NULL(snapshot_data.buffer);
+    // Strings: 'foo', 'key', 'lol'
+    CHECK_EQ(3, serializer.string_count());
+    CHECK_EQ(1, serializer.map_count());
+    CHECK_EQ(1, serializer.object_count());
+    CHECK_EQ(0, serializer.function_count());
+  }
+
+  {
+    v8::Local<v8::Context> new_context = CcTest::NewContext();
+    v8::Context::Scope context_scope(new_context);
+    WebSnapshotDeserializer deserializer(isolate);
+    CHECK(deserializer.UseWebSnapshot(snapshot_data.buffer,
+                                      snapshot_data.buffer_size));
+    CHECK(!deserializer.has_error());
+    v8::Local<v8::String> result = CompileRun("foo.key").As<v8::String>();
+    CHECK(result->Equals(new_context, v8_str("lol")).FromJust());
+    CHECK_EQ(3, deserializer.string_count());
+    CHECK_EQ(1, deserializer.map_count());
+    CHECK_EQ(1, deserializer.object_count());
+    CHECK_EQ(0, deserializer.function_count());
+  }
+}
+
+TEST(Function) {
+  CcTest::InitializeVM();
+  v8::Isolate* isolate = CcTest::isolate();
+  v8::HandleScope scope(isolate);
+  v8::Local<v8::Context> context = isolate->GetCurrentContext();
+
+  CompileRun("var foo = {'key': function() { return '11525'; }}");
+  WebSnapshotData snapshot_data;
+  {
+    std::vector<std::string> exports;
+    exports.push_back("foo");
+    WebSnapshotSerializer serializer(isolate);
+    CHECK(serializer.TakeSnapshot(context, exports, snapshot_data));
+    CHECK(!serializer.has_error());
+    CHECK_NOT_NULL(snapshot_data.buffer);
+    // Strings: 'foo', 'key', function source code
+    CHECK_EQ(3, serializer.string_count());
+    CHECK_EQ(1, serializer.map_count());
+    CHECK_EQ(1, serializer.object_count());
+    CHECK_EQ(1, serializer.function_count());
+  }
+
+  {
+    v8::Local<v8::Context> new_context = CcTest::NewContext();
+    v8::Context::Scope context_scope(new_context);
+    WebSnapshotDeserializer deserializer(isolate);
+    CHECK(deserializer.UseWebSnapshot(snapshot_data.buffer,
+                                      snapshot_data.buffer_size));
+    CHECK(!deserializer.has_error());
+    v8::Local<v8::Function> function = CompileRun("foo.key").As<v8::Function>();
+    v8::Local<v8::Value> result =
+        function->Call(new_context, new_context->Global(), 0, nullptr)
+            .ToLocalChecked();
+    CHECK(result->Equals(new_context, v8_str("11525")).FromJust());
+    CHECK_EQ(3, deserializer.string_count());
+    CHECK_EQ(1, deserializer.map_count());
+    CHECK_EQ(1, deserializer.object_count());
+    CHECK_EQ(1, deserializer.function_count());
+  }
+}
+
+}  // namespace internal
+}  // namespace v8