Support external startup data in V8.

[Retry of crrev.com/293993021, which caused problems with 'ninja all' in Chromium. First patch set if a clean apply
of crrev.com/293993021. Subsequent sets are the actual fix
for that issue.]

If the embedder chooses, the 'natives' (library sources) and the
precompiled startup blob can be written to files during the build
process and handed over to V8 at startup. The main purpose would be
to reduce the size of the compiled binary for space constrained
platforms.

The build-time option is off by default. Nothing should change if
it's not enabled.

BUG=
R=jochen@chromium.org

Review URL: https://codereview.chromium.org/315033002

git-svn-id: https://v8.googlecode.com/svn/branches/bleeding_edge@21696 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

BUILD.gn

@@ -16,6 +16,7 @@ v8_object_print = false
 v8_postmortem_support = false
 v8_use_default_platform = true
 v8_use_snapshot = true
+v8_use_external_startup_data = false
 v8_enable_extra_checks = is_debug
 v8_target_arch = cpu_arch
 v8_random_seed = "314159265"
@@ -109,6 +110,11 @@ config("features") {
       "ENABLE_HANDLE_ZAPPING",
     ]
   }
+  if (v8_use_external_startup_data == true) {
+    defines += [
+      "V8_USE_EXTERNAL_STARTUP_DATA",
+    ]
+  }
 }
 
 config("toolchain") {
@@ -315,6 +321,7 @@ source_set("v8_nosnapshot") {
     "$target_gen_dir/experimental-libraries.cc",
     "$target_gen_dir/trig-table.cc",
     "src/snapshot-empty.cc",
+    "src/snapshot-common.cc",
   ]
 
   configs -= [ "//build/config/compiler:chromium_code" ]
@@ -643,7 +650,8 @@ source_set("v8_base") {
     "src/serialize.h",
     "src/small-pointer-list.h",
     "src/smart-pointers.h",
-    "src/snapshot-common.cc",
+    "src/snapshot-source-sink.cc",
+    "src/snapshot-source-sink.h",
     "src/snapshot.h",
     "src/spaces-inl.h",
     "src/spaces.cc",

build/features.gypi

@@ -59,6 +59,10 @@
 
     # Use the v8 provided v8::Platform implementation.
     'v8_use_default_platform%': 1,
+
+    # Use external files for startup data blobs:
+    # the JS builtins sources and the start snapshot.
+    'v8_use_external_startup_data%': 0,
   },
   'target_defaults': {
     'conditions': [
@@ -87,9 +91,10 @@
         'defines': ['V8_USE_DEFAULT_PLATFORM',],
       }],
       ['v8_compress_startup_data=="bz2"', {
-        'defines': [
-          'COMPRESS_STARTUP_DATA_BZ2',
-        ],
+        'defines': ['COMPRESS_STARTUP_DATA_BZ2',],
+      }],
+      ['v8_use_external_startup_data==1', {
+        'defines': ['V8_USE_EXTERNAL_STARTUP_DATA',],
       }],
     ],  # conditions
     'configurations': {

include/v8.h

@@ -4642,6 +4642,24 @@ class V8_EXPORT V8 {
   static void GetCompressedStartupData(StartupData* compressed_data);
   static void SetDecompressedStartupData(StartupData* decompressed_data);
 
+  /**
+   * Hand startup data to V8, in case the embedder has chosen to build
+   * V8 with external startup data.
+   *
+   * Note:
+   * - By default the startup data is linked into the V8 library, in which
+   *   case this function is not meaningful.
+   * - If this needs to be called, it needs to be called before V8
+   *   tries to make use of its built-ins.
+   * - To avoid unnecessary copies of data, V8 will point directly into the
+   *   given data blob, so pretty please keep it around until V8 exit.
+   * - Compression of the startup blob might be useful, but needs to
+   *   handled entirely on the embedders' side.
+   * - The call will abort if the data is invalid.
+   */
+  static void SetNativesDataBlob(StartupData* startup_blob);
+  static void SetSnapshotDataBlob(StartupData* startup_blob);
+
   /**
    * Adds a message listener.
    *

src/api.cc

@@ -28,9 +28,7 @@
 #include "src/icu_util.h"
 #include "src/json-parser.h"
 #include "src/messages.h"
-#ifdef COMPRESS_STARTUP_DATA_BZ2
 #include "src/natives.h"
-#endif
 #include "src/parser.h"
 #include "src/platform.h"
 #include "src/platform/time.h"
@@ -353,6 +351,24 @@ void V8::SetDecompressedStartupData(StartupData* decompressed_data) {
 }
 
 
+void V8::SetNativesDataBlob(StartupData* natives_blob) {
+#ifdef V8_USE_EXTERNAL_STARTUP_DATA
+  i::SetNativesFromFile(natives_blob);
+#else
+  CHECK(false);
+#endif
+}
+
+
+void V8::SetSnapshotDataBlob(StartupData* snapshot_blob) {
+#ifdef V8_USE_EXTERNAL_STARTUP_DATA
+  i::SetSnapshotFromFile(snapshot_blob);
+#else
+  CHECK(false);
+#endif
+}
+
+
 void V8::SetFatalErrorHandler(FatalErrorCallback that) {
   i::Isolate* isolate = i::Isolate::UncheckedCurrent();
   isolate->set_exception_behavior(that);

src/d8.cc

@@ -1320,6 +1320,15 @@ bool Shell::SetOptions(int argc, char* argv[]) {
       return false;
     }
 #endif  // V8_SHARED
+#ifdef V8_USE_EXTERNAL_STARTUP_DATA
+    else if (strncmp(argv[i], "--natives_blob=", 15) == 0) {
+      options.natives_blob = argv[i] + 15;
+      argv[i] = NULL;
+    } else if (strncmp(argv[i], "--snapshot_blob=", 16) == 0) {
+      options.snapshot_blob = argv[i] + 16;
+      argv[i] = NULL;
+    }
+#endif  // V8_USE_EXTERNAL_STARTUP_DATA
   }
 
   v8::V8::SetFlagsFromCommandLine(&argc, argv, true);
@@ -1477,9 +1486,65 @@ class MockArrayBufferAllocator : public v8::ArrayBuffer::Allocator {
 };
 
 
+#ifdef V8_USE_EXTERNAL_STARTUP_DATA
+class StartupDataHandler {
+ public:
+  StartupDataHandler(const char* natives_blob,
+                     const char* snapshot_blob) {
+    Load(natives_blob, &natives_, v8::V8::SetNativesDataBlob);
+    Load(snapshot_blob, &snapshot_, v8::V8::SetSnapshotDataBlob);
+  }
+
+  ~StartupDataHandler() {
+    delete[] natives_.data;
+    delete[] snapshot_.data;
+  }
+
+ private:
+  void Load(const char* blob_file,
+            v8::StartupData* startup_data,
+            void (*setter_fn)(v8::StartupData*)) {
+    startup_data->data = NULL;
+    startup_data->compressed_size = 0;
+    startup_data->raw_size = 0;
+
+    if (!blob_file)
+      return;
+
+    FILE* file = fopen(blob_file, "rb");
+    if (!file)
+      return;
+
+    fseek(file, 0, SEEK_END);
+    startup_data->raw_size = ftell(file);
+    rewind(file);
+
+    startup_data->data = new char[startup_data->raw_size];
+    startup_data->compressed_size = fread(
+        const_cast<char*>(startup_data->data), 1, startup_data->raw_size,
+        file);
+    fclose(file);
+
+    if (startup_data->raw_size == startup_data->compressed_size)
+      (*setter_fn)(startup_data);
+  }
+
+  v8::StartupData natives_;
+  v8::StartupData snapshot_;
+
+  // Disallow copy & assign.
+  StartupDataHandler(const StartupDataHandler& other);
+  void operator=(const StartupDataHandler& other);
+};
+#endif  // V8_USE_EXTERNAL_STARTUP_DATA
+
+
 int Shell::Main(int argc, char* argv[]) {
   if (!SetOptions(argc, argv)) return 1;
   v8::V8::InitializeICU(options.icu_data_file);
+#ifdef V8_USE_EXTERNAL_STARTUP_DATA
+  StartupDataHandler startup_data(options.natives_blob, options.snapshot_blob);
+#endif
   SetFlagsFromString("--trace-hydrogen-file=hydrogen.cfg");
   SetFlagsFromString("--redirect-code-traces-to=code.asm");
   ShellArrayBufferAllocator array_buffer_allocator;

src/d8.h

@@ -208,7 +208,9 @@ class ShellOptions {
      mock_arraybuffer_allocator(false),
      num_isolates(1),
      isolate_sources(NULL),
-     icu_data_file(NULL) { }
+     icu_data_file(NULL),
+     natives_blob(NULL),
+     snapshot_blob(NULL) { }
 
   ~ShellOptions() {
     delete[] isolate_sources;
@@ -232,6 +234,8 @@ class ShellOptions {
   int num_isolates;
   SourceGroup* isolate_sources;
   const char* icu_data_file;
+  const char* natives_blob;
+  const char* snapshot_blob;
 };
 
 #ifdef V8_SHARED

src/flag-definitions.h

@@ -634,8 +634,8 @@ DEFINE_string(raw_file, NULL, "A file to write the raw snapshot bytes to. "
                               "(mksnapshot only)")
 DEFINE_string(raw_context_file, NULL, "A file to write the raw context "
                                       "snapshot bytes to. (mksnapshot only)")
-DEFINE_bool(omit, false, "Omit raw snapshot bytes in generated code. "
-                         "(mksnapshot only)")
+DEFINE_string(startup_blob, NULL, "Write V8 startup blob file. "
+                                  "(mksnapshot only)")
 
 // code-stubs-hydrogen.cc
 DEFINE_bool(profile_hydrogen_code_stub_compilation, false,

src/heap.cc

@@ -4964,7 +4964,7 @@ bool Heap::ConfigureHeap(int max_semi_space_size,
     max_semi_space_size_ = Page::kPageSize;
   }
 
-  if (Snapshot::IsEnabled()) {
+  if (Snapshot::HaveASnapshotToStartFrom()) {
     // If we are using a snapshot we always reserve the default amount
     // of memory for each semispace because code in the snapshot has
     // write-barrier code that relies on the size and alignment of new

src/mksnapshot.cc

@@ -48,33 +48,81 @@ class SnapshotWriter {
       : fp_(GetFileDescriptorOrDie(snapshot_file))
       , raw_file_(NULL)
       , raw_context_file_(NULL)
-      , compressor_(NULL)
-      , omit_(false) {
+      , startup_blob_file_(NULL)
+      , compressor_(NULL) {
   }
 
   ~SnapshotWriter() {
     fclose(fp_);
     if (raw_file_) fclose(raw_file_);
     if (raw_context_file_) fclose(raw_context_file_);
+    if (startup_blob_file_) fclose(startup_blob_file_);
   }
 
   void SetCompressor(Compressor* compressor) {
     compressor_ = compressor;
   }
 
-  void SetOmit(bool omit) {
-    omit_ = omit;
-  }
-
   void SetRawFiles(const char* raw_file, const char* raw_context_file) {
     raw_file_ = GetFileDescriptorOrDie(raw_file);
     raw_context_file_ = GetFileDescriptorOrDie(raw_context_file);
   }
 
+  void SetStartupBlobFile(const char* startup_blob_file) {
+    if (startup_blob_file != NULL)
+      startup_blob_file_ = GetFileDescriptorOrDie(startup_blob_file);
+  }
+
   void WriteSnapshot(const i::List<char>& snapshot_data,
                      const i::Serializer& serializer,
                      const i::List<char>& context_snapshot_data,
                      const i::Serializer& context_serializer) const {
+    WriteSnapshotFile(snapshot_data, serializer,
+                      context_snapshot_data, context_serializer);
+    MaybeWriteStartupBlob(snapshot_data, serializer,
+                          context_snapshot_data, context_serializer);
+  }
+
+ private:
+  void MaybeWriteStartupBlob(const i::List<char>& snapshot_data,
+                             const i::Serializer& serializer,
+                             const i::List<char>& context_snapshot_data,
+                             const i::Serializer& context_serializer) const {
+    if (!startup_blob_file_)
+      return;
+
+    i::List<char> startup_blob;
+    ListSnapshotSink sink(&startup_blob);
+
+    int spaces[] = {
+        i::NEW_SPACE, i::OLD_POINTER_SPACE, i::OLD_DATA_SPACE, i::CODE_SPACE,
+        i::MAP_SPACE, i::CELL_SPACE,  i::PROPERTY_CELL_SPACE
+    };
+
+    i::byte* snapshot_bytes = reinterpret_cast<i::byte*>(snapshot_data.begin());
+    sink.PutBlob(snapshot_bytes, snapshot_data.length(), "snapshot");
+    for (size_t i = 0; i < ARRAY_SIZE(spaces); ++i)
+      sink.PutInt(serializer.CurrentAllocationAddress(spaces[i]), "spaces");
+
+    i::byte* context_bytes =
+        reinterpret_cast<i::byte*>(context_snapshot_data.begin());
+    sink.PutBlob(context_bytes, context_snapshot_data.length(), "context");
+    for (size_t i = 0; i < ARRAY_SIZE(spaces); ++i)
+      sink.PutInt(context_serializer.CurrentAllocationAddress(spaces[i]),
+                  "spaces");
+
+    size_t written = fwrite(startup_blob.begin(), 1, startup_blob.length(),
+                            startup_blob_file_);
+    if (written != (size_t)startup_blob.length()) {
+      i::PrintF("Writing snapshot file failed.. Aborting.\n");
+      exit(1);
+    }
+  }
+
+  void WriteSnapshotFile(const i::List<char>& snapshot_data,
+                         const i::Serializer& serializer,
+                         const i::List<char>& context_snapshot_data,
+                         const i::Serializer& context_serializer) const {
     WriteFilePrefix();
     WriteData("", snapshot_data, raw_file_);
     WriteData("context_", context_snapshot_data, raw_context_file_);
@@ -83,7 +131,6 @@ class SnapshotWriter {
     WriteFileSuffix();
   }
 
- private:
   void WriteFilePrefix() const {
     fprintf(fp_, "// Autogenerated snapshot file. Do not edit.\n\n");
     fprintf(fp_, "#include \"src/v8.h\"\n");
@@ -137,13 +184,12 @@ class SnapshotWriter {
                  const i::List<char>& source_data,
                  const i::List<char>* data_to_be_written) const {
     fprintf(fp_, "const byte Snapshot::%sdata_[] = {\n", prefix);
-    if (!omit_)
-      WriteSnapshotData(data_to_be_written);
+    WriteSnapshotData(data_to_be_written);
     fprintf(fp_, "};\n");
     fprintf(fp_, "const int Snapshot::%ssize_ = %d;\n", prefix,
             data_to_be_written->length());
 
-    if (data_to_be_written == &source_data && !omit_) {
+    if (data_to_be_written == &source_data) {
       fprintf(fp_, "const byte* Snapshot::%sraw_data_ = Snapshot::%sdata_;\n",
               prefix, prefix);
       fprintf(fp_, "const int Snapshot::%sraw_size_ = Snapshot::%ssize_;\n",
@@ -196,8 +242,8 @@ class SnapshotWriter {
   FILE* fp_;
   FILE* raw_file_;
   FILE* raw_context_file_;
+  FILE* startup_blob_file_;
   Compressor* compressor_;
-  bool omit_;
 };
 
 
@@ -381,9 +427,10 @@ int main(int argc, char** argv) {
 
     {
       SnapshotWriter writer(argv[1]);
-      writer.SetOmit(i::FLAG_omit);
       if (i::FLAG_raw_file && i::FLAG_raw_context_file)
         writer.SetRawFiles(i::FLAG_raw_file, i::FLAG_raw_context_file);
+      if (i::FLAG_startup_blob)
+        writer.SetStartupBlobFile(i::FLAG_startup_blob);
   #ifdef COMPRESS_STARTUP_DATA_BZ2
       BZip2Compressor bzip2;
       writer.SetCompressor(&bzip2);

src/natives-external.cc

+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/natives.h"
+
+#include "src/checks.h"
+#include "src/list.h"
+#include "src/list-inl.h"
+#include "src/snapshot-source-sink.h"
+#include "src/vector.h"
+
+namespace v8 {
+namespace internal {
+
+
+/**
+ * NativesStore stores the 'native' (builtin) JS libraries.
+ *
+ * NativesStore needs to be initialized before using V8, usually by the
+ * embedder calling v8::SetNativesDataBlob, which calls SetNativesFromFile
+ * below.
+ */
+class NativesStore {
+ public:
+  ~NativesStore() {}
+
+  int GetBuiltinsCount() { return native_names_.length(); }
+  int GetDebuggerCount() { return debugger_count_; }
+  Vector<const char> GetScriptName(int index) { return native_names_[index]; }
+  Vector<const char> GetRawScriptSource(int index) {
+    return native_source_[index];
+  }
+
+  int GetIndex(const char* name) {
+    for (int i = 0; i < native_names_.length(); ++i) {
+      if (strcmp(name, native_names_[i].start()) == 0) {
+        return i;
+      }
+    }
+    ASSERT(false);
+    return -1;
+  }
+
+  int GetRawScriptsSize() {
+    ASSERT(false);  // Used for compression. Doesn't really make sense here.
+    return 0;
+  }
+
+  Vector<const byte> GetScriptsSource() {
+    ASSERT(false);  // Used for compression. Doesn't really make sense here.
+    return Vector<const byte>();
+  }
+
+  static NativesStore* MakeFromScriptsSource(SnapshotByteSource* source) {
+    NativesStore* store = new NativesStore;
+
+    // We expect the libraries in the following format:
+    //   int: # of debugger sources.
+    //   2N blobs: N pairs of source name + actual source.
+    //   then, repeat for non-debugger sources.
+    int debugger_count = source->GetInt();
+    for (int i = 0; i < debugger_count; ++i)
+      store->ReadNameAndContentPair(source);
+    int library_count = source->GetInt();
+    for (int i = 0; i < library_count; ++i)
+      store->ReadNameAndContentPair(source);
+
+    store->debugger_count_ = debugger_count;
+    return store;
+  }
+
+ private:
+  NativesStore() : debugger_count_(0) {}
+
+  bool ReadNameAndContentPair(SnapshotByteSource* bytes) {
+    const byte* name;
+    int name_length;
+    const byte* source;
+    int source_length;
+    bool success = bytes->GetBlob(&name, &name_length) &&
+                   bytes->GetBlob(&source, &source_length);
+    if (success) {
+      Vector<const char> name_vector(
+          reinterpret_cast<const char*>(name), name_length);
+      Vector<const char> source_vector(
+          reinterpret_cast<const char*>(source), source_length);
+      native_names_.Add(name_vector);
+      native_source_.Add(source_vector);
+    }
+    return success;
+  }
+
+  List<Vector<const char> > native_names_;
+  List<Vector<const char> > native_source_;
+  int debugger_count_;
+
+  DISALLOW_COPY_AND_ASSIGN(NativesStore);
+};
+
+
+template<NativeType type>
+class NativesHolder {
+ public:
+  static NativesStore* get() {
+    ASSERT(holder_);
+    return holder_;
+  }
+  static void set(NativesStore* store) {
+    ASSERT(store);
+    holder_ = store;
+  }
+
+ private:
+  static NativesStore* holder_;
+};
+
+template<NativeType type>
+NativesStore* NativesHolder<type>::holder_ = NULL;
+
+
+/**
+ * Read the Natives (library sources) blob, as generated by js2c + the build
+ * system.
+ */
+void SetNativesFromFile(StartupData* natives_blob) {
+  ASSERT(natives_blob);
+  ASSERT(natives_blob->data);
+  ASSERT(natives_blob->raw_size > 0);
+
+  SnapshotByteSource bytes(
+      reinterpret_cast<const byte*>(natives_blob->data),
+      natives_blob->raw_size);
+  NativesHolder<CORE>::set(NativesStore::MakeFromScriptsSource(&bytes));
+  NativesHolder<EXPERIMENTAL>::set(NativesStore::MakeFromScriptsSource(&bytes));
+  ASSERT(!bytes.HasMore());
+}
+
+
+// Implement NativesCollection<T> bsaed on NativesHolder + NativesStore.
+//
+// (The callers expect a purely static interface, since this is how the
+//  natives are usually compiled in. Since we implement them based on
+//  runtime content, we have to implement this indirection to offer
+//  a static interface.)
+template<NativeType type>
+int NativesCollection<type>::GetBuiltinsCount() {
+  return NativesHolder<type>::get()->GetBuiltinsCount();
+}
+
+template<NativeType type>
+int NativesCollection<type>::GetDebuggerCount() {
+  return NativesHolder<type>::get()->GetDebuggerCount();
+}
+
+template<NativeType type>
+int NativesCollection<type>::GetIndex(const char* name) {
+  return NativesHolder<type>::get()->GetIndex(name);
+}
+
+template<NativeType type>
+int NativesCollection<type>::GetRawScriptsSize() {
+  return NativesHolder<type>::get()->GetRawScriptsSize();
+}
+
+template<NativeType type>
+Vector<const char> NativesCollection<type>::GetRawScriptSource(int index) {
+  return NativesHolder<type>::get()->GetRawScriptSource(index);
+}
+
+template<NativeType type>
+Vector<const char> NativesCollection<type>::GetScriptName(int index) {
+  return NativesHolder<type>::get()->GetScriptName(index);
+}
+
+template<NativeType type>
+Vector<const byte> NativesCollection<type>::GetScriptsSource() {
+  return NativesHolder<type>::get()->GetScriptsSource();
+}
+
+
+// The compiler can't 'see' all uses of the static methods and hence
+// my chose to elide them. This we'll explicitly instantiate these.
+template class NativesCollection<CORE>;
+template class NativesCollection<EXPERIMENTAL>;
+template class NativesCollection<D8>;
+template class NativesCollection<TEST>;
+
+}  // namespace v8::internal
+}  // namespace v8

src/natives.h

@@ -5,6 +5,10 @@
 #ifndef V8_NATIVES_H_
 #define V8_NATIVES_H_
 
+#include "src/vector.h"
+
+namespace v8 { class StartupData; }  // Forward declaration.
+
 namespace v8 {
 namespace internal {
 
@@ -39,6 +43,11 @@ class NativesCollection {
 typedef NativesCollection<CORE> Natives;
 typedef NativesCollection<EXPERIMENTAL> ExperimentalNatives;
 
+#ifdef V8_USE_EXTERNAL_STARTUP_DATA
+// Used for reading the natives at runtime. Implementation in natives-empty.cc
+void SetNativesFromFile(StartupData* natives_blob);
+#endif
+
 } }  // namespace v8::internal
 
 #endif  // V8_NATIVES_H_

src/serialize.cc

@@ -16,6 +16,7 @@
 #include "src/runtime.h"
 #include "src/serialize.h"
 #include "src/snapshot.h"
+#include "src/snapshot-source-sink.h"
 #include "src/stub-cache.h"
 #include "src/v8threads.h"
 
@@ -1203,19 +1204,6 @@ void Deserializer::ReadChunk(Object** current,
 }
 
 
-void SnapshotByteSink::PutInt(uintptr_t integer, const char* description) {
-  ASSERT(integer < 1 << 22);
-  integer <<= 2;
-  int bytes = 1;
-  if (integer > 0xff) bytes = 2;
-  if (integer > 0xffff) bytes = 3;
-  integer |= bytes;
-  Put(static_cast<int>(integer & 0xff), "IntPart1");
-  if (bytes > 1) Put(static_cast<int>((integer >> 8) & 0xff), "IntPart2");
-  if (bytes > 2) Put(static_cast<int>((integer >> 16) & 0xff), "IntPart3");
-}
-
-
 Serializer::Serializer(Isolate* isolate, SnapshotByteSink* sink)
     : isolate_(isolate),
       sink_(sink),
@@ -1831,12 +1819,4 @@ void Serializer::InitializeCodeAddressMap() {
 }
 
 
-bool SnapshotByteSource::AtEOF() {
-  if (0u + length_ - position_ > 2 * sizeof(uint32_t)) return false;
-  for (int x = position_; x < length_; x++) {
-    if (data_[x] != SerializerDeserializer::nop()) return false;
-  }
-  return true;
-}
-
 } }  // namespace v8::internal

src/serialize.h

@@ -6,6 +6,9 @@
 #define V8_SERIALIZE_H_
 
 #include "src/hashmap.h"
+#include "src/isolate.h"
+#include "src/snapshot-source-sink.h"
+#include "src/heap-profiler.h"
 
 namespace v8 {
 namespace internal {
@@ -132,49 +135,6 @@ class ExternalReferenceDecoder {
 };
 
 
-class SnapshotByteSource {
- public:
-  SnapshotByteSource(const byte* array, int length)
-    : data_(array), length_(length), position_(0) { }
-
-  bool HasMore() { return position_ < length_; }
-
-  int Get() {
-    ASSERT(position_ < length_);
-    return data_[position_++];
-  }
-
-  int32_t GetUnalignedInt() {
-#if defined(V8_HOST_CAN_READ_UNALIGNED) &&  __BYTE_ORDER == __LITTLE_ENDIAN
-    int32_t answer;
-    ASSERT(position_ + sizeof(answer) <= length_ + 0u);
-    answer = *reinterpret_cast<const int32_t*>(data_ + position_);
-#else
-    int32_t answer = data_[position_];
-    answer |= data_[position_ + 1] << 8;
-    answer |= data_[position_ + 2] << 16;
-    answer |= data_[position_ + 3] << 24;
-#endif
-    return answer;
-  }
-
-  void Advance(int by) { position_ += by; }
-
-  inline void CopyRaw(byte* to, int number_of_bytes);
-
-  inline int GetInt();
-
-  bool AtEOF();
-
-  int position() { return position_; }
-
- private:
-  const byte* data_;
-  int length_;
-  int position_;
-};
-
-
 // The Serializer/Deserializer class is a common superclass for Serializer and
 // Deserializer which is used to store common constants and methods used by
 // both.
@@ -268,26 +228,6 @@ class SerializerDeserializer: public ObjectVisitor {
 };
 
 
-int SnapshotByteSource::GetInt() {
-  // This way of variable-length encoding integers does not suffer from branch
-  // mispredictions.
-  uint32_t answer = GetUnalignedInt();
-  int bytes = answer & 3;
-  Advance(bytes);
-  uint32_t mask = 0xffffffffu;
-  mask >>= 32 - (bytes << 3);
-  answer &= mask;
-  answer >>= 2;
-  return answer;
-}
-
-
-void SnapshotByteSource::CopyRaw(byte* to, int number_of_bytes) {
-  MemCopy(to, data_ + position_, number_of_bytes);
-  position_ += number_of_bytes;
-}
-
-
 // A Deserializer reads a snapshot and reconstructs the Object graph it defines.
 class Deserializer: public SerializerDeserializer {
  public:
@@ -368,18 +308,6 @@ class Deserializer: public SerializerDeserializer {
 };
 
 
-class SnapshotByteSink {
- public:
-  virtual ~SnapshotByteSink() { }
-  virtual void Put(int byte, const char* description) = 0;
-  virtual void PutSection(int byte, const char* description) {
-    Put(byte, description);
-  }
-  void PutInt(uintptr_t integer, const char* description);
-  virtual int Position() = 0;
-};
-
-
 // Mapping objects to their location after deserialization.
 // This is used during building, but not at runtime by V8.
 class SerializationAddressMapper {

src/snapshot-common.cc

@@ -15,43 +15,6 @@ namespace v8 {
 namespace internal {
 
 
-static void ReserveSpaceForSnapshot(Deserializer* deserializer,
-                                    const char* file_name) {
-  int file_name_length = StrLength(file_name) + 10;
-  Vector<char> name = Vector<char>::New(file_name_length + 1);
-  OS::SNPrintF(name, "%s.size", file_name);
-  FILE* fp = OS::FOpen(name.start(), "r");
-  CHECK_NE(NULL, fp);
-  int new_size, pointer_size, data_size, code_size, map_size, cell_size,
-      property_cell_size;
-#ifdef _MSC_VER
-  // Avoid warning about unsafe fscanf from MSVC.
-  // Please note that this is only fine if %c and %s are not being used.
-#define fscanf fscanf_s
-#endif
-  CHECK_EQ(1, fscanf(fp, "new %d\n", &new_size));
-  CHECK_EQ(1, fscanf(fp, "pointer %d\n", &pointer_size));
-  CHECK_EQ(1, fscanf(fp, "data %d\n", &data_size));
-  CHECK_EQ(1, fscanf(fp, "code %d\n", &code_size));
-  CHECK_EQ(1, fscanf(fp, "map %d\n", &map_size));
-  CHECK_EQ(1, fscanf(fp, "cell %d\n", &cell_size));
-  CHECK_EQ(1, fscanf(fp, "property cell %d\n", &property_cell_size));
-#ifdef _MSC_VER
-#undef fscanf
-#endif
-  fclose(fp);
-  deserializer->set_reservation(NEW_SPACE, new_size);
-  deserializer->set_reservation(OLD_POINTER_SPACE, pointer_size);
-  deserializer->set_reservation(OLD_DATA_SPACE, data_size);
-  deserializer->set_reservation(CODE_SPACE, code_size);
-  deserializer->set_reservation(MAP_SPACE, map_size);
-  deserializer->set_reservation(CELL_SPACE, cell_size);
-  deserializer->set_reservation(PROPERTY_CELL_SPACE,
-                                property_cell_size);
-  name.Dispose();
-}
-
-
 void Snapshot::ReserveSpaceForLinkedInSnapshot(Deserializer* deserializer) {
   deserializer->set_reservation(NEW_SPACE, new_space_used_);
   deserializer->set_reservation(OLD_POINTER_SPACE, pointer_space_used_);
@@ -64,21 +27,8 @@ void Snapshot::ReserveSpaceForLinkedInSnapshot(Deserializer* deserializer) {
 }
 
 
-bool Snapshot::Initialize(const char* snapshot_file) {
-  if (snapshot_file) {
-    int len;
-    byte* str = ReadBytes(snapshot_file, &len);
-    if (!str) return false;
-    bool success;
-    {
-      SnapshotByteSource source(str, len);
-      Deserializer deserializer(&source);
-      ReserveSpaceForSnapshot(&deserializer, snapshot_file);
-      success = V8::Initialize(&deserializer);
-    }
-    DeleteArray(str);
-    return success;
-  } else if (size_ > 0) {
+bool Snapshot::Initialize() {
+  if (size_ > 0) {
     ElapsedTimer timer;
     if (FLAG_profile_deserialization) {
       timer.Start();
@@ -123,4 +73,15 @@ Handle<Context> Snapshot::NewContextFromSnapshot(Isolate* isolate) {
   return Handle<Context>(Context::cast(root));
 }
 
+
+#ifdef V8_USE_EXTERNAL_STARTUP_DATA
+// Dummy implementations of Set*FromFile(..) APIs.
+//
+// These are meant for use with snapshot-external.cc. Should this file
+// be compiled with those options we just supply these dummy implementations
+// below. This happens when compiling the mksnapshot utility.
+void SetNativesFromFile(StartupData* data) { CHECK(false); }
+void SetSnapshotFromFile(StartupData* data) { CHECK(false); }
+#endif  // V8_USE_EXTERNAL_STARTUP_DATA
+
 } }  // namespace v8::internal

src/snapshot-external.cc

+// Copyright 2006-2008 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Used for building with external snapshots.
+
+#include "src/snapshot.h"
+
+#include "src/v8.h"  // for V8::Initialize
+#include "src/serialize.h"
+#include "src/snapshot-source-sink.h"
+
+namespace v8 {
+namespace internal {
+
+
+struct SnapshotImpl {
+ public:
+  const byte* data;
+  int size;
+  int new_space_used;
+  int pointer_space_used;
+  int data_space_used;
+  int code_space_used;
+  int map_space_used;
+  int cell_space_used;
+  int property_cell_space_used;
+
+  const byte* context_data;
+  int context_size;
+  int context_new_space_used;
+  int context_pointer_space_used;
+  int context_data_space_used;
+  int context_code_space_used;
+  int context_map_space_used;
+  int context_cell_space_used;
+  int context_property_cell_space_used;
+};
+
+
+static SnapshotImpl* snapshot_impl_ = NULL;
+
+
+bool Snapshot::HaveASnapshotToStartFrom() {
+  return snapshot_impl_ != NULL;
+}
+
+
+bool Snapshot::Initialize() {
+  if (!HaveASnapshotToStartFrom())
+    return false;
+
+  ElapsedTimer timer;
+  if (FLAG_profile_deserialization) {
+    timer.Start();
+  }
+  SnapshotByteSource source(snapshot_impl_->data, snapshot_impl_->size);
+  Deserializer deserializer(&source);
+  deserializer.set_reservation(NEW_SPACE, snapshot_impl_->new_space_used);
+  deserializer.set_reservation(OLD_POINTER_SPACE,
+                               snapshot_impl_->pointer_space_used);
+  deserializer.set_reservation(OLD_DATA_SPACE,
+                               snapshot_impl_->data_space_used);
+  deserializer.set_reservation(CODE_SPACE, snapshot_impl_->code_space_used);
+  deserializer.set_reservation(MAP_SPACE, snapshot_impl_->map_space_used);
+  deserializer.set_reservation(CELL_SPACE, snapshot_impl_->cell_space_used);
+  deserializer.set_reservation(PROPERTY_CELL_SPACE,
+                               snapshot_impl_->property_cell_space_used);
+  bool success = V8::Initialize(&deserializer);
+  if (FLAG_profile_deserialization) {
+    double ms = timer.Elapsed().InMillisecondsF();
+    PrintF("[Snapshot loading and deserialization took %0.3f ms]\n", ms);
+  }
+  return success;
+}
+
+
+Handle<Context> Snapshot::NewContextFromSnapshot(Isolate* isolate) {
+  if (!HaveASnapshotToStartFrom())
+    return Handle<Context>();
+
+  SnapshotByteSource source(snapshot_impl_->context_data,
+                            snapshot_impl_->context_size);
+  Deserializer deserializer(&source);
+  deserializer.set_reservation(NEW_SPACE,
+                               snapshot_impl_->context_new_space_used);
+  deserializer.set_reservation(OLD_POINTER_SPACE,
+                               snapshot_impl_->context_pointer_space_used);
+  deserializer.set_reservation(OLD_DATA_SPACE,
+                               snapshot_impl_->context_data_space_used);
+  deserializer.set_reservation(CODE_SPACE,
+                               snapshot_impl_->context_code_space_used);
+  deserializer.set_reservation(MAP_SPACE,
+                               snapshot_impl_->context_map_space_used);
+  deserializer.set_reservation(CELL_SPACE,
+                               snapshot_impl_->context_cell_space_used);
+  deserializer.set_reservation(PROPERTY_CELL_SPACE,
+                               snapshot_impl_->
+                                   context_property_cell_space_used);
+  Object* root;
+  deserializer.DeserializePartial(isolate, &root);
+  CHECK(root->IsContext());
+  return Handle<Context>(Context::cast(root));
+}
+
+
+void SetSnapshotFromFile(StartupData* snapshot_blob) {
+  ASSERT(snapshot_blob);
+  ASSERT(snapshot_blob->data);
+  ASSERT(snapshot_blob->raw_size > 0);
+  ASSERT(!snapshot_impl_);
+
+  snapshot_impl_ = new SnapshotImpl;
+  SnapshotByteSource source(reinterpret_cast<const byte*>(snapshot_blob->data),
+                            snapshot_blob->raw_size);
+
+  bool success = source.GetBlob(&snapshot_impl_->data,
+                                &snapshot_impl_->size);
+  snapshot_impl_->new_space_used = source.GetInt();
+  snapshot_impl_->pointer_space_used = source.GetInt();
+  snapshot_impl_->data_space_used = source.GetInt();
+  snapshot_impl_->code_space_used = source.GetInt();
+  snapshot_impl_->map_space_used = source.GetInt();
+  snapshot_impl_->cell_space_used = source.GetInt();
+  snapshot_impl_->property_cell_space_used = source.GetInt();
+
+  success &= source.GetBlob(&snapshot_impl_->context_data,
+                            &snapshot_impl_->context_size);
+  snapshot_impl_->context_new_space_used = source.GetInt();
+  snapshot_impl_->context_pointer_space_used = source.GetInt();
+  snapshot_impl_->context_data_space_used = source.GetInt();
+  snapshot_impl_->context_code_space_used = source.GetInt();
+  snapshot_impl_->context_map_space_used = source.GetInt();
+  snapshot_impl_->context_cell_space_used = source.GetInt();
+  snapshot_impl_->context_property_cell_space_used = source.GetInt();
+
+  ASSERT(success);
+}
+
+} }  // namespace v8::internal

src/snapshot-source-sink.cc

+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+
+#include "src/snapshot-source-sink.h"
+#include "src/checks.h"
+
+#include "src/handles-inl.h"
+#include "src/serialize.h"  // for SerializerDeserializer::nop() in AtEOF()
+
+
+namespace v8 {
+namespace internal {
+
+
+SnapshotByteSource::SnapshotByteSource(const byte* array, int length)
+    : data_(array), length_(length), position_(0) {
+}
+
+
+SnapshotByteSource::~SnapshotByteSource() { }
+
+
+int32_t SnapshotByteSource::GetUnalignedInt() {
+  ASSERT(position_ < length_);  // Require at least one byte left.
+#if defined(V8_HOST_CAN_READ_UNALIGNED) &&  __BYTE_ORDER == __LITTLE_ENDIAN
+  int32_t answer = *reinterpret_cast<const int32_t*>(data_ + position_);
+#else
+  int32_t answer = data_[position_];
+  answer |= data_[position_ + 1] << 8;
+  answer |= data_[position_ + 2] << 16;
+  answer |= data_[position_ + 3] << 24;
+#endif
+  return answer;
+}
+
+
+void SnapshotByteSource::CopyRaw(byte* to, int number_of_bytes) {
+  MemCopy(to, data_ + position_, number_of_bytes);
+  position_ += number_of_bytes;
+}
+
+
+void SnapshotByteSink::PutInt(uintptr_t integer, const char* description) {
+  ASSERT(integer < 1 << 22);
+  integer <<= 2;
+  int bytes = 1;
+  if (integer > 0xff) bytes = 2;
+  if (integer > 0xffff) bytes = 3;
+  integer |= bytes;
+  Put(static_cast<int>(integer & 0xff), "IntPart1");
+  if (bytes > 1) Put(static_cast<int>((integer >> 8) & 0xff), "IntPart2");
+  if (bytes > 2) Put(static_cast<int>((integer >> 16) & 0xff), "IntPart3");
+}
+
+void SnapshotByteSink::PutRaw(byte* data, int number_of_bytes,
+                              const char* description) {
+  for (int i = 0; i < number_of_bytes; ++i) {
+    Put(data[i], description);
+  }
+}
+
+void SnapshotByteSink::PutBlob(byte* data, int number_of_bytes,
+                               const char* description) {
+  PutInt(number_of_bytes, description);
+  PutRaw(data, number_of_bytes, description);
+}
+
+
+bool SnapshotByteSource::AtEOF() {
+  if (0u + length_ - position_ > 2 * sizeof(uint32_t)) return false;
+  for (int x = position_; x < length_; x++) {
+    if (data_[x] != SerializerDeserializer::nop()) return false;
+  }
+  return true;
+}
+
+
+bool SnapshotByteSource::GetBlob(const byte** data, int* number_of_bytes) {
+  int size = GetInt();
+  *number_of_bytes = size;
+
+  if (position_ + size < length_) {
+    *data = &data_[position_];
+    Advance(size);
+    return true;
+  } else {
+    Advance(length_ - position_);  // proceed until end.
+    return false;
+  }
+}
+
+}  // namespace v8::internal
+}  // namespace v8

src/snapshot-source-sink.h

+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_SNAPSHOT_SOURCE_SINK_H_
+#define V8_SNAPSHOT_SOURCE_SINK_H_
+
+#include "src/checks.h"
+#include "src/utils.h"
+
+namespace v8 {
+namespace internal {
+
+
+/**
+ * Source to read snapshot and builtins files from.
+ *
+ * Note: Memory ownership remains with callee.
+ */
+class SnapshotByteSource V8_FINAL {
+ public:
+  SnapshotByteSource(const byte* array, int length);
+  ~SnapshotByteSource();
+
+  bool HasMore() { return position_ < length_; }
+
+  int Get() {
+    ASSERT(position_ < length_);
+    return data_[position_++];
+  }
+
+  int32_t GetUnalignedInt();
+
+  void Advance(int by) { position_ += by; }
+
+  void CopyRaw(byte* to, int number_of_bytes);
+
+  inline int GetInt() {
+    // This way of variable-length encoding integers does not suffer from branch
+    // mispredictions.
+    uint32_t answer = GetUnalignedInt();
+    int bytes = answer & 3;
+    Advance(bytes);
+    uint32_t mask = 0xffffffffu;
+    mask >>= 32 - (bytes << 3);
+    answer &= mask;
+    answer >>= 2;
+    return answer;
+  }
+
+  bool GetBlob(const byte** data, int* number_of_bytes);
+
+  bool AtEOF();
+
+  int position() { return position_; }
+
+ private:
+  const byte* data_;
+  int length_;
+  int position_;
+
+  DISALLOW_COPY_AND_ASSIGN(SnapshotByteSource);
+};
+
+
+/**
+ * Sink to write snapshot files to.
+ *
+ * Subclasses must implement actual storage or i/o.
+ */
+class SnapshotByteSink {
+ public:
+  virtual ~SnapshotByteSink() { }
+  virtual void Put(int byte, const char* description) = 0;
+  virtual void PutSection(int byte, const char* description) {
+    Put(byte, description);
+  }
+  void PutInt(uintptr_t integer, const char* description);
+  void PutRaw(byte* data, int number_of_bytes, const char* description);
+  void PutBlob(byte* data, int number_of_bytes, const char* description);
+  virtual int Position() = 0;
+};
+
+
+}  // namespace v8::internal
+}  // namespace v8
+
+#endif  // V8_SNAPSHOT_SOURCE_SINK_H_

src/snapshot.h

@@ -12,23 +12,16 @@ namespace internal {
 
 class Snapshot {
  public:
-  // Initialize the VM from the given snapshot file. If snapshot_file is
-  // NULL, use the internal snapshot instead. Returns false if no snapshot
+  // Initialize the VM from the internal snapshot. Returns false if no snapshot
   // could be found.
-  static bool Initialize(const char* snapshot_file = NULL);
+  static bool Initialize();
 
   static bool HaveASnapshotToStartFrom();
 
   // Create a new context using the internal partial snapshot.
   static Handle<Context> NewContextFromSnapshot(Isolate* isolate);
 
-  // Returns whether or not the snapshot is enabled.
-  static bool IsEnabled() { return size_ != 0; }
-
-  // Write snapshot to the given file. Returns true if snapshot was written
-  // successfully.
-  static bool WriteToFile(const char* snapshot_file);
-
+  // These methods support COMPRESS_STARTUP_DATA_BZ2.
   static const byte* data() { return data_; }
   static int size() { return size_; }
   static int raw_size() { return raw_size_; }
@@ -72,6 +65,10 @@ class Snapshot {
   DISALLOW_IMPLICIT_CONSTRUCTORS(Snapshot);
 };
 
+#ifdef V8_USE_EXTERNAL_STARTUP_DATA
+void SetSnapshotFromFile(StartupData* snapshot_blob);
+#endif
+
 } }  // namespace v8::internal
 
 #endif  // V8_SNAPSHOT_H_

test/cctest/test-serialize.cc

@@ -283,8 +283,60 @@ TEST(SerializeTwice) {
 //----------------------------------------------------------------------------
 // Tests that the heap can be deserialized.
 
+
+static void ReserveSpaceForSnapshot(Deserializer* deserializer,
+                                    const char* file_name) {
+  int file_name_length = StrLength(file_name) + 10;
+  Vector<char> name = Vector<char>::New(file_name_length + 1);
+  OS::SNPrintF(name, "%s.size", file_name);
+  FILE* fp = OS::FOpen(name.start(), "r");
+  name.Dispose();
+  int new_size, pointer_size, data_size, code_size, map_size, cell_size,
+      property_cell_size;
+#ifdef _MSC_VER
+  // Avoid warning about unsafe fscanf from MSVC.
+  // Please note that this is only fine if %c and %s are not being used.
+#define fscanf fscanf_s
+#endif
+  CHECK_EQ(1, fscanf(fp, "new %d\n", &new_size));
+  CHECK_EQ(1, fscanf(fp, "pointer %d\n", &pointer_size));
+  CHECK_EQ(1, fscanf(fp, "data %d\n", &data_size));
+  CHECK_EQ(1, fscanf(fp, "code %d\n", &code_size));
+  CHECK_EQ(1, fscanf(fp, "map %d\n", &map_size));
+  CHECK_EQ(1, fscanf(fp, "cell %d\n", &cell_size));
+  CHECK_EQ(1, fscanf(fp, "property cell %d\n", &property_cell_size));
+#ifdef _MSC_VER
+#undef fscanf
+#endif
+  fclose(fp);
+  deserializer->set_reservation(NEW_SPACE, new_size);
+  deserializer->set_reservation(OLD_POINTER_SPACE, pointer_size);
+  deserializer->set_reservation(OLD_DATA_SPACE, data_size);
+  deserializer->set_reservation(CODE_SPACE, code_size);
+  deserializer->set_reservation(MAP_SPACE, map_size);
+  deserializer->set_reservation(CELL_SPACE, cell_size);
+  deserializer->set_reservation(PROPERTY_CELL_SPACE, property_cell_size);
+}
+
+
+bool InitializeFromFile(const char* snapshot_file) {
+  int len;
+  byte* str = ReadBytes(snapshot_file, &len);
+  if (!str) return false;
+  bool success;
+  {
+    SnapshotByteSource source(str, len);
+    Deserializer deserializer(&source);
+    ReserveSpaceForSnapshot(&deserializer, snapshot_file);
+    success = V8::Initialize(&deserializer);
+  }
+  DeleteArray(str);
+  return success;
+}
+
+
 static void Deserialize() {
-  CHECK(Snapshot::Initialize(FLAG_testing_serialization_file));
+  CHECK(InitializeFromFile(FLAG_testing_serialization_file));
 }
 
 
@@ -443,48 +495,13 @@ TEST(PartialSerialization) {
 }
 
 
-static void ReserveSpaceForSnapshot(Deserializer* deserializer,
-                                    const char* file_name) {
-  int file_name_length = StrLength(file_name) + 10;
-  Vector<char> name = Vector<char>::New(file_name_length + 1);
-  OS::SNPrintF(name, "%s.size", file_name);
-  FILE* fp = OS::FOpen(name.start(), "r");
-  name.Dispose();
-  int new_size, pointer_size, data_size, code_size, map_size, cell_size,
-      property_cell_size;
-#ifdef _MSC_VER
-  // Avoid warning about unsafe fscanf from MSVC.
-  // Please note that this is only fine if %c and %s are not being used.
-#define fscanf fscanf_s
-#endif
-  CHECK_EQ(1, fscanf(fp, "new %d\n", &new_size));
-  CHECK_EQ(1, fscanf(fp, "pointer %d\n", &pointer_size));
-  CHECK_EQ(1, fscanf(fp, "data %d\n", &data_size));
-  CHECK_EQ(1, fscanf(fp, "code %d\n", &code_size));
-  CHECK_EQ(1, fscanf(fp, "map %d\n", &map_size));
-  CHECK_EQ(1, fscanf(fp, "cell %d\n", &cell_size));
-  CHECK_EQ(1, fscanf(fp, "property cell %d\n", &property_cell_size));
-#ifdef _MSC_VER
-#undef fscanf
-#endif
-  fclose(fp);
-  deserializer->set_reservation(NEW_SPACE, new_size);
-  deserializer->set_reservation(OLD_POINTER_SPACE, pointer_size);
-  deserializer->set_reservation(OLD_DATA_SPACE, data_size);
-  deserializer->set_reservation(CODE_SPACE, code_size);
-  deserializer->set_reservation(MAP_SPACE, map_size);
-  deserializer->set_reservation(CELL_SPACE, cell_size);
-  deserializer->set_reservation(PROPERTY_CELL_SPACE, property_cell_size);
-}
-
-
 DEPENDENT_TEST(PartialDeserialization, PartialSerialization) {
-  if (!Snapshot::IsEnabled()) {
+  if (!Snapshot::HaveASnapshotToStartFrom()) {
     int file_name_length = StrLength(FLAG_testing_serialization_file) + 10;
     Vector<char> startup_name = Vector<char>::New(file_name_length + 1);
     OS::SNPrintF(startup_name, "%s.startup", FLAG_testing_serialization_file);
 
-    CHECK(Snapshot::Initialize(startup_name.start()));
+    CHECK(InitializeFromFile(startup_name.start()));
     startup_name.Dispose();
 
     const char* file_name = FLAG_testing_serialization_file;
@@ -596,7 +613,7 @@ DEPENDENT_TEST(ContextDeserialization, ContextSerialization) {
     Vector<char> startup_name = Vector<char>::New(file_name_length + 1);
     OS::SNPrintF(startup_name, "%s.startup", FLAG_testing_serialization_file);
 
-    CHECK(Snapshot::Initialize(startup_name.start()));
+    CHECK(InitializeFromFile(startup_name.start()));
     startup_name.Dispose();
 
     const char* file_name = FLAG_testing_serialization_file;

tools/concatenate-files.py

+#!/usr/bin/env python
+#
+# Copyright 2014 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+#     * Neither the name of Google Inc. nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# This utility concatenates several files into one. On Unix-like systems
+# it is equivalent to:
+#   cat file1 file2 file3 ...files... > target
+#
+# The reason for writing a seperate utility is that 'cat' is not available
+# on all supported build platforms, but Python is, and hence this provides
+# us with an easy and uniform way of doing this on all platforms.
+
+import optparse
+
+
+def Concatenate(filenames):
+  """Concatenate files.
+
+  Args:
+    files: Array of file names.
+           The last name is the target; all earlier ones are sources.
+
+  Returns:
+    True, if the operation was successful.
+  """
+  if len(filenames) < 2:
+    print "An error occured generating %s:\nNothing to do." % filenames[-1]
+    return False
+
+  try:
+    with open(filenames[-1], "wb") as target:
+      for filename in filenames[:-1]:
+        with open(filename, "rb") as current:
+          target.write(current.read())
+    return True
+  except IOError as e:
+    print "An error occured when writing %s:\n%s" % (filenames[-1], e)
+    return False
+
+
+def main():
+  parser = optparse.OptionParser()
+  parser.set_usage("""Concatenate several files into one.
+      Equivalent to: cat file1 ... > target.""")
+  (options, args) = parser.parse_args()
+  exit(0 if Concatenate(args) else 1)
+
+
+if __name__ == "__main__":
+  main()

tools/gyp/v8.gyp

@@ -42,18 +42,22 @@
         }, {
           'toolsets': ['target'],
         }],
-        ['v8_use_snapshot=="true"', {
+
+        ['v8_use_snapshot=="true" and v8_use_external_startup_data==0', {
           # The dependency on v8_base should come from a transitive
           # dependency however the Android toolchain requires libv8_base.a
           # to appear before libv8_snapshot.a so it's listed explicitly.
           'dependencies': ['v8_base', 'v8_snapshot'],
-        },
-        {
+        }],
+        ['v8_use_snapshot!="true" and v8_use_external_startup_data==0', {
           # The dependency on v8_base should come from a transitive
           # dependency however the Android toolchain requires libv8_base.a
           # to appear before libv8_snapshot.a so it's listed explicitly.
           'dependencies': ['v8_base', 'v8_nosnapshot'],
         }],
+        ['v8_use_external_startup_data==1', {
+          'dependencies': ['v8_base', 'v8_external_snapshot'],
+        }],
         ['component=="shared_library"', {
           'type': '<(component)',
           'sources': [
@@ -148,6 +152,7 @@
         '<(SHARED_INTERMEDIATE_DIR)/experimental-libraries.cc',
         '<(SHARED_INTERMEDIATE_DIR)/trig-table.cc',
         '<(INTERMEDIATE_DIR)/snapshot.cc',
+        '../../src/snapshot-common.cc',
       ],
       'actions': [
         {
@@ -172,7 +177,7 @@
           'action': [
             '<@(_inputs)',
             '<@(mksnapshot_flags)',
-            '<@(_outputs)'
+            '<@(INTERMEDIATE_DIR)/snapshot.cc'
           ],
         },
       ],
@@ -190,6 +195,7 @@
         '<(SHARED_INTERMEDIATE_DIR)/libraries.cc',
         '<(SHARED_INTERMEDIATE_DIR)/experimental-libraries.cc',
         '<(SHARED_INTERMEDIATE_DIR)/trig-table.cc',
+        '../../src/snapshot-common.cc',
         '../../src/snapshot-empty.cc',
       ],
       'conditions': [
@@ -208,6 +214,80 @@
         }],
       ]
     },
+    {
+      'target_name': 'v8_external_snapshot',
+      'type': 'static_library',
+      'conditions': [
+        ['want_separate_host_toolset==1', {
+          'toolsets': ['host', 'target'],
+          'dependencies': [
+            'mksnapshot#host',
+            'js2c#host',
+            'generate_trig_table#host',
+            'natives_blob#host',
+        ]}, {
+          'toolsets': ['target'],
+          'dependencies': [
+            'mksnapshot',
+            'js2c',
+            'generate_trig_table',
+            'natives_blob',
+          ],
+        }],
+        ['component=="shared_library"', {
+          'defines': [
+            'V8_SHARED',
+            'BUILDING_V8_SHARED',
+          ],
+          'direct_dependent_settings': {
+            'defines': [
+              'V8_SHARED',
+              'USING_V8_SHARED',
+            ],
+          },
+        }],
+      ],
+      'dependencies': [
+        'v8_base',
+      ],
+      'include_dirs+': [
+        '../..',
+      ],
+      'sources': [
+        '<(SHARED_INTERMEDIATE_DIR)/trig-table.cc',
+        '../../src/natives-external.cc',
+        '../../src/snapshot-external.cc',
+      ],
+      'actions': [
+        {
+          'action_name': 'run_mksnapshot (external)',
+          'inputs': [
+            '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)mksnapshot<(EXECUTABLE_SUFFIX)',
+          ],
+          'outputs': [
+            '<(INTERMEDIATE_DIR)/snapshot.cc',
+            '<(PRODUCT_DIR)/snapshot_blob.bin',
+          ],
+          'variables': {
+            'mksnapshot_flags': [
+              '--log-snapshot-positions',
+              '--logfile', '<(INTERMEDIATE_DIR)/snapshot.log',
+            ],
+            'conditions': [
+              ['v8_random_seed!=0', {
+                'mksnapshot_flags': ['--random-seed', '<(v8_random_seed)'],
+              }],
+            ],
+          },
+          'action': [
+            '<@(_inputs)',
+            '<@(mksnapshot_flags)',
+            '<@(INTERMEDIATE_DIR)/snapshot.cc',
+            '--startup_blob', '<(PRODUCT_DIR)/snapshot_blob.bin',
+          ],
+        },
+      ],
+    },
     { 'target_name': 'generate_trig_table',
       'type': 'none',
       'conditions': [
@@ -541,8 +621,9 @@
         '../../src/serialize.h',
         '../../src/small-pointer-list.h',
         '../../src/smart-pointers.h',
-        '../../src/snapshot-common.cc',
         '../../src/snapshot.h',
+        '../../src/snapshot-source-sink.cc',
+        '../../src/snapshot-source-sink.h',
         '../../src/spaces-inl.h',
         '../../src/spaces.cc',
         '../../src/spaces.h',
@@ -1091,6 +1172,32 @@
         }],
       ],
     },
+    {
+      'target_name': 'natives_blob',
+      'type': 'none',
+      'conditions': [
+        [ 'v8_use_external_startup_data==1', {
+          'dependencies': ['js2c'],
+          'actions': [{
+            'action_name': 'concatenate_natives_blob',
+            'inputs': [
+              '../../tools/concatenate-files.py',
+              '<(SHARED_INTERMEDIATE_DIR)/libraries.bin',
+              '<(SHARED_INTERMEDIATE_DIR)/libraries-experimental.bin',
+            ],
+            'outputs': [
+              '<(PRODUCT_DIR)/natives_blob.bin',
+            ],
+            'action': ['python', '<@(_inputs)', '<@(_outputs)'],
+          }],
+        }],
+        ['want_separate_host_toolset==1', {
+          'toolsets': ['host'],
+        }, {
+          'toolsets': ['target'],
+        }],
+      ]
+    },
     {
       'target_name': 'js2c',
       'type': 'none',
@@ -1147,6 +1254,8 @@
           '../../src/harmony-array.js',
           '../../src/harmony-math.js'
         ],
+        'libraries_bin_file': '<(SHARED_INTERMEDIATE_DIR)/libraries.bin',
+        'libraries_experimental_bin_file': '<(SHARED_INTERMEDIATE_DIR)/libraries-experimental.bin',
       },
       'actions': [
         {
@@ -1162,12 +1271,20 @@
           'action': [
             'python',
             '../../tools/js2c.py',
-            '<@(_outputs)',
+            '<(SHARED_INTERMEDIATE_DIR)/libraries.cc',
             'CORE',
             '<(v8_compress_startup_data)',
             '<@(library_files)',
             '<@(i18n_library_files)',
           ],
+          'conditions': [
+            [ 'v8_use_external_startup_data==1', {
+              'outputs': ['<@(libraries_bin_file)'],
+              'action': [
+                '--startup_blob', '<@(libraries_bin_file)',
+              ],
+            }],
+          ],
         },
         {
           'action_name': 'js2c_experimental',
@@ -1181,11 +1298,19 @@
           'action': [
             'python',
             '../../tools/js2c.py',
-            '<@(_outputs)',
+            '<(SHARED_INTERMEDIATE_DIR)/experimental-libraries.cc',
             'EXPERIMENTAL',
             '<(v8_compress_startup_data)',
             '<@(experimental_library_files)'
           ],
+          'conditions': [
+            [ 'v8_use_external_startup_data==1', {
+              'outputs': ['<@(libraries_experimental_bin_file)'],
+              'action': [
+                '--startup_blob', '<@(libraries_experimental_bin_file)'
+              ],
+            }],
+          ],
         },
       ],
     },

tools/js2c.py

@@ -397,7 +397,7 @@ def PrepareSources(source_files):
   return result
 
 
-def BuildMetadata(sources, source_bytes, native_type, omit):
+def BuildMetadata(sources, source_bytes, native_type):
   """Build the meta data required to generate a libaries file.
 
   Args:
@@ -405,7 +405,6 @@ def BuildMetadata(sources, source_bytes, native_type, omit):
     source_bytes: A list of source bytes.
         (The concatenation of all sources; might be compressed.)
     native_type: The parameter for the NativesCollection template.
-    omit: bool, whether we should omit the sources in the output.
 
   Returns:
     A dictionary for use with HEADER_TEMPLATE.
@@ -438,7 +437,7 @@ def BuildMetadata(sources, source_bytes, native_type, omit):
   assert offset == len(raw_sources)
 
   # If we have the raw sources we can declare them accordingly.
-  have_raw_sources = source_bytes == raw_sources and not omit
+  have_raw_sources = source_bytes == raw_sources
   raw_sources_declaration = (RAW_SOURCES_DECLARATION
       if have_raw_sources else RAW_SOURCES_COMPRESSION_DECLARATION)
 
@@ -446,7 +445,6 @@ def BuildMetadata(sources, source_bytes, native_type, omit):
     "builtin_count": len(sources.modules),
     "debugger_count": sum(sources.is_debugger_id),
     "sources_declaration": SOURCES_DECLARATION % ToCArray(source_bytes),
-    "sources_data": ToCArray(source_bytes) if not omit else "",
     "raw_sources_declaration": raw_sources_declaration,
     "raw_total_length": sum(map(len, sources.modules)),
     "total_length": total_length,
@@ -477,10 +475,51 @@ def CompressMaybe(sources, compression_type):
     raise Error("Unknown compression type %s." % compression_type)
 
 
-def JS2C(source, target, native_type, compression_type, raw_file, omit):
+def PutInt(blob_file, value):
+  assert(value >= 0 and value < (1 << 20))
+  size = 1 if (value < 1 << 6) else (2 if (value < 1 << 14) else 3)
+  value_with_length = (value << 2) | size
+
+  byte_sequence = bytearray()
+  for i in xrange(size):
+    byte_sequence.append(value_with_length & 255)
+    value_with_length >>= 8;
+  blob_file.write(byte_sequence)
+
+
+def PutStr(blob_file, value):
+  PutInt(blob_file, len(value));
+  blob_file.write(value);
+
+
+def WriteStartupBlob(sources, startup_blob):
+  """Write a startup blob, as expected by V8 Initialize ...
+    TODO(vogelheim): Add proper method name.
+
+  Args:
+    sources: A Sources instance with the prepared sources.
+    startup_blob_file: Name of file to write the blob to.
+  """
+  output = open(startup_blob, "wb")
+
+  debug_sources = sum(sources.is_debugger_id);
+  PutInt(output, debug_sources)
+  for i in xrange(debug_sources):
+    PutStr(output, sources.names[i]);
+    PutStr(output, sources.modules[i]);
+
+  PutInt(output, len(sources.names) - debug_sources)
+  for i in xrange(debug_sources, len(sources.names)):
+    PutStr(output, sources.names[i]);
+    PutStr(output, sources.modules[i]);
+
+  output.close()
+
+
+def JS2C(source, target, native_type, compression_type, raw_file, startup_blob):
   sources = PrepareSources(source)
   sources_bytes = CompressMaybe(sources, compression_type)
-  metadata = BuildMetadata(sources, sources_bytes, native_type, omit)
+  metadata = BuildMetadata(sources, sources_bytes, native_type)
 
   # Optionally emit raw file.
   if raw_file:
@@ -488,6 +527,9 @@ def JS2C(source, target, native_type, compression_type, raw_file, omit):
     output.write(sources_bytes)
     output.close()
 
+  if startup_blob:
+    WriteStartupBlob(sources, startup_blob);
+
   # Emit resulting source file.
   output = open(target, "w")
   output.write(HEADER_TEMPLATE % metadata)
@@ -497,9 +539,9 @@ def JS2C(source, target, native_type, compression_type, raw_file, omit):
 def main():
   parser = optparse.OptionParser()
   parser.add_option("--raw", action="store",
-                      help="file to write the processed sources array to.")
-  parser.add_option("--omit", dest="omit", action="store_true",
-                    help="Omit the raw sources from the generated code.")
+                    help="file to write the processed sources array to.")
+  parser.add_option("--startup_blob", action="store",
+                    help="file to write the startup blob to.")
   parser.set_usage("""js2c out.cc type compression sources.js ...
       out.cc: C code to be generated.
       type: type parameter for NativesCollection template.
@@ -507,7 +549,7 @@ def main():
       sources.js: JS internal sources or macros.py.""")
   (options, args) = parser.parse_args()
 
-  JS2C(args[3:], args[0], args[1], args[2], options.raw, options.omit)
+  JS2C(args[3:], args[0], args[1], args[2], options.raw, options.startup_blob)
 
 
 if __name__ == "__main__":