Add support for startup data (snapshot) compression.

This is for mobile platforms where application footprint size is
important. To avoid including compression libraries into V8, we assume
that the host machine have them (true for Linux), and rely on embedder
to provide decompressed data.

Currently, only snapshot data can be comressed. It is also possible to
compress libraries sources, but it is more involved and will be
addressed in another CL.

BUG=none
TEST=none

Review URL: http://codereview.chromium.org/6901090

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@7724 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

SConstruct

@@ -213,6 +213,9 @@ LIBRARY_FLAGS = {
     },
     'gdbjit:on': {
       'CPPDEFINES':   ['ENABLE_GDB_JIT_INTERFACE']
+    },
+    'compress_startup_data:bz2': {
+      'CPPDEFINES':   ['COMPRESS_STARTUP_DATA_BZ2']
     }
   },
   'msvc': {
@@ -355,6 +358,11 @@ MKSNAPSHOT_EXTRA_FLAGS = {
     'os:win32': {
       'LIBS': ['winmm', 'ws2_32'],
     },
+    'compress_startup_data:bz2': {
+      'os:linux': {
+        'LIBS': ['bz2']
+      }
+    },
   },
   'msvc': {
     'all': {
@@ -512,6 +520,12 @@ SAMPLE_FLAGS = {
       'CCFLAGS':      ['-g', '-O0'],
       'CPPDEFINES':   ['DEBUG']
     },
+    'compress_startup_data:bz2': {
+      'CPPDEFINES':   ['COMPRESS_STARTUP_DATA_BZ2'],
+      'os:linux': {
+        'LIBS':       ['bz2']
+      }
+    },
   },
   'msvc': {
     'all': {
@@ -975,7 +989,12 @@ SIMPLE_OPTIONS = {
     'values': ['mips32r2', 'mips32r1'],
     'default': 'mips32r2',
     'help': 'mips variant'
-  }
+  },
+  'compress_startup_data': {
+    'values': ['off', 'bz2'],
+    'default': 'off',
+    'help': 'compress startup data (snapshot) [Linux only]'
+  },
 }
 
 ALL_OPTIONS = dict(PLATFORM_OPTIONS, **SIMPLE_OPTIONS)
@@ -1098,6 +1117,8 @@ def VerifyOptions(env):
     print env['arch']
     print env['simulator']
     Abort("Option unalignedaccesses only supported for the ARM architecture.")
+  if env['os'] != 'linux' and env['compress_startup_data'] != 'off':
+    Abort("Startup data compression is only available on Linux")
   for (name, option) in ALL_OPTIONS.iteritems():
     if (not name in env):
       message = ("A value for option %s must be specified (%s)." %

include/v8.h

@@ -2641,6 +2641,18 @@ class V8EXPORT Isolate {
 };
 
 
+class StartupData {
+ public:
+  enum CompressionAlgorithm {
+    kUncompressed,
+    kBZip2
+  };
+
+  const char* data;
+  int compressed_size;
+  int raw_size;
+};
+
 /**
  * Container class for static utility functions.
  */
@@ -2669,6 +2681,26 @@ class V8EXPORT V8 {
    */
   static bool IsDead();
 
+  /**
+   * The following 4 functions are to be used when V8 is built with
+   * the 'compress_startup_data' flag enabled. In this case, the
+   * embedder must decompress startup data prior to initializing V8.
+   *
+   * This is how interaction with V8 should look like:
+   *   int compressed_data_count = v8::V8::GetCompressedStartupDataCount();
+   *   v8::StartupData* compressed_data =
+   *     new v8::StartupData[compressed_data_count];
+   *   v8::V8::GetCompressedStartupData(compressed_data);
+   *   ... decompress data (compressed_data can be updated in-place) ...
+   *   v8::V8::SetDecompressedStartupData(compressed_data);
+   *   ... now V8 can be initialized
+   *   ... make sure the decompressed data stays valid until V8 shutdown
+   */
+  static StartupData::CompressionAlgorithm GetCompressedStartupDataAlgorithm();
+  static int GetCompressedStartupDataCount();
+  static void GetCompressedStartupData(StartupData* compressed_data);
+  static void SetDecompressedStartupData(StartupData* decompressed_data);
+
   /**
    * Adds a message listener.
    *

samples/process.cc

@@ -30,6 +30,10 @@
 #include <string>
 #include <map>
 
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+#error Using compressed startup data is not supported for this sample
+#endif
+
 using namespace std;
 using namespace v8;
 

samples/shell.cc

@@ -28,6 +28,9 @@
 #include <v8.h>
 #include <v8-testing.h>
 #include <assert.h>
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+#include <bzlib.h>
+#endif
 #include <fcntl.h>
 #include <string.h>
 #include <stdio.h>
@@ -299,6 +302,31 @@ int main(int argc, char* argv[]) {
     }
   }
 
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+  ASSERT_EQ(v8::StartupData::kBZip2,
+            v8::V8::GetCompressedStartupDataAlgorithm());
+  int compressed_data_count = v8::V8::GetCompressedStartupDataCount();
+  v8::StartupData* compressed_data = new v8::StartupData[compressed_data_count];
+  v8::V8::GetCompressedStartupData(compressed_data);
+  for (int i = 0; i < compressed_data_count; ++i) {
+    char* decompressed = new char[compressed_data[i].raw_size];
+    unsigned int decompressed_size = compressed_data[i].raw_size;
+    int result =
+        BZ2_bzBuffToBuffDecompress(decompressed,
+                                   &decompressed_size,
+                                   const_cast<char*>(compressed_data[i].data),
+                                   compressed_data[i].compressed_size,
+                                   0, 1);
+    if (result != BZ_OK) {
+      fprintf(stderr, "bzip error code: %d\n", result);
+      exit(1);
+    }
+    compressed_data[i].data = decompressed;
+    compressed_data[i].raw_size = decompressed_size;
+  }
+  v8::V8::SetDecompressedStartupData(compressed_data);
+#endif  // COMPRESS_STARTUP_DATA_BZ2
+
   v8::V8::SetFlagsFromCommandLine(&argc, argv, true);
   int result = 0;
   if (FLAG_stress_opt || FLAG_stress_deopt) {
@@ -319,6 +347,14 @@ int main(int argc, char* argv[]) {
     result = RunMain(argc, argv);
   }
   v8::V8::Dispose();
+
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+  for (int i = 0; i < compressed_data_count; ++i) {
+    delete[] compressed_data[i].data;
+  }
+  delete[] compressed_data;
+#endif  // COMPRESS_STARTUP_DATA_BZ2
+
   return result;
 }
 

src/api.cc

@@ -311,6 +311,61 @@ static inline i::Isolate* EnterIsolateIfNeeded() {
 }
 
 
+StartupData::CompressionAlgorithm V8::GetCompressedStartupDataAlgorithm() {
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+  return StartupData::kBZip2;
+#else
+  return StartupData::kUncompressed;
+#endif
+}
+
+
+enum CompressedStartupDataItems {
+  kSnapshot = 0,
+  kSnapshotContext,
+  kCompressedStartupDataCount
+};
+
+int V8::GetCompressedStartupDataCount() {
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+  return kCompressedStartupDataCount;
+#else
+  return 0;
+#endif
+}
+
+
+void V8::GetCompressedStartupData(StartupData* compressed_data) {
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+  compressed_data[kSnapshot].data =
+      reinterpret_cast<const char*>(i::Snapshot::data());
+  compressed_data[kSnapshot].compressed_size = i::Snapshot::size();
+  compressed_data[kSnapshot].raw_size = i::Snapshot::raw_size();
+
+  compressed_data[kSnapshotContext].data =
+      reinterpret_cast<const char*>(i::Snapshot::context_data());
+  compressed_data[kSnapshotContext].compressed_size =
+      i::Snapshot::context_size();
+  compressed_data[kSnapshotContext].raw_size = i::Snapshot::context_raw_size();
+#endif
+}
+
+
+void V8::SetDecompressedStartupData(StartupData* decompressed_data) {
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+  ASSERT_EQ(i::Snapshot::raw_size(), decompressed_data[kSnapshot].raw_size);
+  i::Snapshot::set_raw_data(
+      reinterpret_cast<const i::byte*>(decompressed_data[kSnapshot].data));
+
+  ASSERT_EQ(i::Snapshot::context_raw_size(),
+            decompressed_data[kSnapshotContext].raw_size);
+  i::Snapshot::set_context_raw_data(
+      reinterpret_cast<const i::byte*>(
+          decompressed_data[kSnapshotContext].data));
+#endif
+}
+
+
 void V8::SetFatalErrorHandler(FatalErrorCallback that) {
   i::Isolate* isolate = EnterIsolateIfNeeded();
   isolate->set_exception_behavior(that);

src/d8.h

@@ -31,6 +31,9 @@
 #include "v8.h"
 #include "hashmap.h"
 
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+#error Using compressed startup data is not supported for D8
+#endif
 
 namespace v8 {
 

src/list-inl.h

@@ -46,10 +46,16 @@ void List<T, P>::Add(const T& element) {
 
 template<typename T, class P>
 void List<T, P>::AddAll(const List<T, P>& other) {
-  int result_length = length_ + other.length_;
+  AddAll(other.ToVector());
+}
+
+
+template<typename T, class P>
+void List<T, P>::AddAll(const Vector<T>& other) {
+  int result_length = length_ + other.length();
   if (capacity_ < result_length) Resize(result_length);
-  for (int i = 0; i < other.length_; i++) {
-    data_[length_ + i] = other.data_[i];
+  for (int i = 0; i < other.length(); i++) {
+    data_[length_ + i] = other.at(i);
   }
   length_ = result_length;
 }

src/list.h

@@ -80,7 +80,7 @@ class List {
   INLINE(int length() const) { return length_; }
   INLINE(int capacity() const) { return capacity_; }
 
-  Vector<T> ToVector() { return Vector<T>(data_, length_); }
+  Vector<T> ToVector() const { return Vector<T>(data_, length_); }
 
   Vector<const T> ToConstVector() { return Vector<const T>(data_, length_); }
 
@@ -91,6 +91,9 @@ class List {
   // Add all the elements from the argument list to this list.
   void AddAll(const List<T, P>& other);
 
+  // Add all the elements from the vector to this list.
+  void AddAll(const Vector<T>& other);
+
   // Inserts the element at the specific index.
   void InsertAt(int index, const T& element);
 

src/mksnapshot.cc

@@ -25,6 +25,9 @@
 // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+#include <bzlib.h>
+#endif
 #include <signal.h>
 #include <string>
 #include <map>
@@ -95,11 +98,53 @@ typedef std::map<std::string, int*>::iterator CounterMapIterator;
 static CounterMap counter_table_;
 
 
-class CppByteSink : public i::SnapshotByteSink {
+class Compressor {
+ public:
+  virtual ~Compressor() {}
+  virtual bool Compress(i::Vector<char> input) = 0;
+  virtual i::Vector<char>* output() = 0;
+};
+
+
+class PartialSnapshotSink : public i::SnapshotByteSink {
  public:
-  explicit CppByteSink(const char* snapshot_file)
-      : bytes_written_(0),
-        partial_sink_(this) {
+  PartialSnapshotSink() : data_(), raw_size_(-1) { }
+  virtual ~PartialSnapshotSink() { data_.Free(); }
+  virtual void Put(int byte, const char* description) {
+    data_.Add(byte);
+  }
+  virtual int Position() { return data_.length(); }
+  void Print(FILE* fp) {
+    int length = Position();
+    for (int j = 0; j < length; j++) {
+      if ((j & 0x1f) == 0x1f) {
+        fprintf(fp, "\n");
+      }
+      if (j != 0) {
+        fprintf(fp, ",");
+      }
+      fprintf(fp, "%d", at(j));
+    }
+  }
+  char at(int i) { return data_[i]; }
+  bool Compress(Compressor* compressor) {
+    ASSERT_EQ(-1, raw_size_);
+    raw_size_ = data_.length();
+    if (!compressor->Compress(data_.ToVector())) return false;
+    data_.Clear();
+    data_.AddAll(*compressor->output());
+    return true;
+  }
+  int raw_size() { return raw_size_; }
+ private:
+  i::List<char> data_;
+  int raw_size_;
+};
+
+
+class CppByteSink : public PartialSnapshotSink {
+ public:
+  explicit CppByteSink(const char* snapshot_file) {
     fp_ = i::OS::FOpen(snapshot_file, "wb");
     if (fp_ == NULL) {
       i::PrintF("Unable to write to snapshot file \"%s\"\n", snapshot_file);
@@ -114,7 +159,18 @@ class CppByteSink : public i::SnapshotByteSink {
   }
 
   virtual ~CppByteSink() {
-    fprintf(fp_, "const int Snapshot::size_ = %d;\n\n", bytes_written_);
+    fprintf(fp_, "const int Snapshot::size_ = %d;\n", Position());
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+    fprintf(fp_, "const byte* Snapshot::raw_data_ = NULL;\n");
+    fprintf(fp_,
+            "const int Snapshot::raw_size_ = %d;\n\n",
+            raw_size());
+#else
+    fprintf(fp_,
+            "const byte* Snapshot::raw_data_ = Snapshot::data_;\n");
+    fprintf(fp_,
+            "const int Snapshot::raw_size_ = Snapshot::size_;\n\n");
+#endif
     fprintf(fp_, "} }  // namespace v8::internal\n");
     fclose(fp_);
   }
@@ -127,7 +183,6 @@ class CppByteSink : public i::SnapshotByteSink {
       int map_space_used,
       int cell_space_used,
       int large_space_used) {
-    fprintf(fp_, "};\n\n");
     fprintf(fp_, "const int Snapshot::new_space_used_ = %d;\n", new_space_used);
     fprintf(fp_,
             "const int Snapshot::pointer_space_used_ = %d;\n",
@@ -151,57 +206,60 @@ class CppByteSink : public i::SnapshotByteSink {
     int length = partial_sink_.Position();
     fprintf(fp_, "};\n\n");
     fprintf(fp_, "const int Snapshot::context_size_ = %d;\n",  length);
+    fprintf(fp_,
+            "const int Snapshot::context_raw_size_ = %d;\n",
+            partial_sink_.raw_size());
     fprintf(fp_, "const byte Snapshot::context_data_[] = {\n");
-    for (int j = 0; j < length; j++) {
-      if ((j & 0x1f) == 0x1f) {
-        fprintf(fp_, "\n");
-      }
-      char byte = partial_sink_.at(j);
-      if (j != 0) {
-        fprintf(fp_, ",");
-      }
-      fprintf(fp_, "%d", byte);
-    }
+    partial_sink_.Print(fp_);
+    fprintf(fp_, "};\n\n");
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+    fprintf(fp_, "const byte* Snapshot::context_raw_data_ = NULL;\n");
+#else
+    fprintf(fp_, "const byte* Snapshot::context_raw_data_ ="
+            " Snapshot::context_data_;\n");
+#endif
   }
 
-  virtual void Put(int byte, const char* description) {
-    if (bytes_written_ != 0) {
-      fprintf(fp_, ",");
-    }
-    fprintf(fp_, "%d", byte);
-    bytes_written_++;
-    if ((bytes_written_ & 0x1f) == 0) {
-      fprintf(fp_, "\n");
-    }
+  void WriteSnapshot() {
+    Print(fp_);
   }
 
-  virtual int Position() {
-    return bytes_written_;
-  }
+  PartialSnapshotSink* partial_sink() { return &partial_sink_; }
+
+ private:
+  FILE* fp_;
+  PartialSnapshotSink partial_sink_;
+};
 
-  i::SnapshotByteSink* partial_sink() { return &partial_sink_; }
 
-  class PartialSnapshotSink : public i::SnapshotByteSink {
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+class BZip2Compressor : public Compressor {
  public:
-    explicit PartialSnapshotSink(CppByteSink* parent)
-        : parent_(parent),
-          data_() { }
-    virtual ~PartialSnapshotSink() { data_.Free(); }
-    virtual void Put(int byte, const char* description) {
-      data_.Add(byte);
+  BZip2Compressor() : output_(NULL) {}
+  virtual ~BZip2Compressor() {
+    delete output_;
   }
-    virtual int Position() { return data_.length(); }
-    char at(int i) { return data_[i]; }
-   private:
-    CppByteSink* parent_;
-    i::List<char> data_;
-  };
+  virtual bool Compress(i::Vector<char> input) {
+    delete output_;
+    output_ = new i::ScopedVector<char>((input.length() * 101) / 100 + 1000);
+    unsigned int output_length_ = output_->length();
+    int result = BZ2_bzBuffToBuffCompress(output_->start(), &output_length_,
+                                          input.start(), input.length(),
+                                          9, 1, 0);
+    if (result == BZ_OK) {
+      output_->Truncate(output_length_);
+      return true;
+    } else {
+      fprintf(stderr, "bzlib error code: %d\n", result);
+      return false;
+    }
+  }
+  virtual i::Vector<char>* output() { return output_; }
 
  private:
-  FILE* fp_;
-  int bytes_written_;
-  PartialSnapshotSink partial_sink_;
+  i::ScopedVector<char>* output_;
 };
+#endif
 
 
 int main(int argc, char** argv) {
@@ -242,6 +300,14 @@ int main(int argc, char** argv) {
 
   ser.SerializeWeakReferences();
 
+#ifdef COMPRESS_STARTUP_DATA_BZ2
+  BZip2Compressor compressor;
+  if (!sink.Compress(&compressor))
+    return 1;
+  if (!sink.partial_sink()->Compress(&compressor))
+    return 1;
+#endif
+  sink.WriteSnapshot();
   sink.WritePartialSnapshot();
 
   sink.WriteSpaceUsed(

src/snapshot-common.cc

@@ -53,7 +53,7 @@ bool Snapshot::Initialize(const char* snapshot_file) {
     DeleteArray(str);
     return true;
   } else if (size_ > 0) {
-    Deserialize(data_, size_);
+    Deserialize(raw_data_, raw_size_);
     return true;
   }
   return false;
@@ -71,7 +71,8 @@ Handle<Context> Snapshot::NewContextFromSnapshot() {
                      map_space_used_,
                      cell_space_used_,
                      large_space_used_);
-  SnapshotByteSource source(context_data_, context_size_);
+  SnapshotByteSource source(context_raw_data_,
+                            context_raw_size_);
   Deserializer deserializer(&source);
   Object* root;
   deserializer.DeserializePartial(&root);

src/snapshot-empty.cc

@@ -35,9 +35,13 @@ namespace v8 {
 namespace internal {
 
 const byte Snapshot::data_[] = { 0 };
+const byte* Snapshot::raw_data_ = NULL;
 const int Snapshot::size_ = 0;
+const int Snapshot::raw_size_ = 0;
 const byte Snapshot::context_data_[] = { 0 };
+const byte* Snapshot::context_raw_data_ = NULL;
 const int Snapshot::context_size_ = 0;
+const int Snapshot::context_raw_size_ = 0;
 
 const int Snapshot::new_space_used_ = 0;
 const int Snapshot::pointer_space_used_ = 0;

src/snapshot.h

@@ -50,9 +50,25 @@ STATIC_CLASS Snapshot {
   // successfully.
   static bool WriteToFile(const char* snapshot_file);
 
+  static const byte* data() { return data_; }
+  static int size() { return size_; }
+  static int raw_size() { return raw_size_; }
+  static void set_raw_data(const byte* raw_data) {
+    raw_data_ = raw_data;
+  }
+  static const byte* context_data() { return context_data_; }
+  static int context_size() { return context_size_; }
+  static int context_raw_size() { return context_raw_size_; }
+  static void set_context_raw_data(
+      const byte* context_raw_data) {
+    context_raw_data_ = context_raw_data;
+  }
+
  private:
   static const byte data_[];
+  static const byte* raw_data_;
   static const byte context_data_[];
+  static const byte* context_raw_data_;
   static const int new_space_used_;
   static const int pointer_space_used_;
   static const int data_space_used_;
@@ -61,7 +77,9 @@ STATIC_CLASS Snapshot {
   static const int cell_space_used_;
   static const int large_space_used_;
   static const int size_;
+  static const int raw_size_;
   static const int context_size_;
+  static const int context_raw_size_;
 
   static bool Deserialize(const byte* content, int len);
 

tools/gyp/v8.gyp

@@ -30,6 +30,7 @@
     'use_system_v8%': 0,
     'msvs_use_common_release': 0,
     'gcc_version%': 'unknown',
+    'v8_compress_startup_data%': 'false',
     'v8_target_arch%': '<(target_arch)',
     'v8_use_snapshot%': 'true',
     'v8_use_liveobjectlist%': 'false',
@@ -76,6 +77,11 @@
               'LIVEOBJECTLIST',
             ],
           }],
+         ['v8_compress_startup_data=="bz2"', {
+            'defines': [
+              'COMPRESS_STARTUP_DATA_BZ2',
+            ],
+          }],
         ],
         'configurations': {
           'Debug': {
@@ -651,7 +657,14 @@
                   'libraries': [
                     # Needed for clock_gettime() used by src/platform-linux.cc.
                     '-lrt',
-                ]},
+                  ],
+                  'conditions': [
+                    ['v8_compress_startup_data=="bz2"', {
+                      'libraries': [
+                        '-lbz2',
+                    ]}],
+                  ],
+                },
                 'sources': [
                   '../../src/platform-linux.cc',
                   '../../src/platform-posix.cc'
@@ -785,7 +798,11 @@
             ['v8_target_arch=="arm" and host_arch=="x64" and _toolset=="host"', {
               'cflags': ['-m32'],
               'ldflags': ['-m32'],
-            }]
+            }],
+            ['v8_compress_startup_data=="bz2"', {
+              'libraries': [
+                '-lbz2',
+              ]}],
           ]
         },
         {
@@ -802,6 +819,10 @@
               # This could be gotten by not setting chromium_code, if that's OK.
               'defines': ['_CRT_SECURE_NO_WARNINGS'],
             }],
+            ['v8_compress_startup_data=="bz2"', {
+              'libraries': [
+                '-lbz2',
+              ]}],
           ],
         },
       ],