Implement heap snapshots serialization into JSON. API is designed

to avoid storing serialized snapshot on VM, instead it is emitted
using output stream interface.

The size of JSON emitted is roughly equal to used heap size
(when stored as an ASCII string).

Now a whole heap snapshot can be serialized and transmitted outside
VM. This makes possible:
  - implementing non-async UI for heap snapshots inspection;
  - storing heap snapshots for further inspection;
  - remote profiling (we can even implement a snapshotting mode
    where a snapshot isn't even stored in VM, only transmitted --
    good for mobile devices);
  - creating tools for outside heap snapshots processing, e.g.
    converting to HPROF.

Review URL: http://codereview.chromium.org/3311028

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@5450 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

include/v8-profiler.h

@@ -323,7 +323,10 @@ class V8EXPORT HeapSnapshot {
   enum Type {
     kFull = 0,       // Heap snapshot with all instances and references.
     kAggregated = 1  // Snapshot doesn't contain individual heap entries,
-                     //instead they are grouped by constructor name.
+                     // instead they are grouped by constructor name.
+  };
+  enum SerializationFormat {
+    kJSON = 0  // See format description near 'Serialize' method.
   };
 
   /** Returns heap snapshot type. */
@@ -343,6 +346,30 @@ class V8EXPORT HeapSnapshot {
    * of the same type can be compared.
    */
   const HeapSnapshotsDiff* CompareWith(const HeapSnapshot* snapshot) const;
+
+  /**
+   * Prepare a serialized representation of the snapshot. The result
+   * is written into the stream provided in chunks of specified size.
+   * The total length of the serialized snapshot is unknown in
+   * advance, it is can be roughly equal to JS heap size (that means,
+   * it can be really big - tens of megabytes).
+   *
+   * For the JSON format, heap contents are represented as an object
+   * with the following structure:
+   *
+   *  {
+   *    snapshot: {title: "...", uid: nnn},
+   *    nodes: [
+   *      meta-info (JSON string),
+   *      nodes themselves
+   *    ],
+   *    strings: [strings]
+   *  }
+   *
+   * Outgoing node links are stored after each node. Nodes reference strings
+   * and other nodes by their indexes in corresponding arrays.
+   */
+  void Serialize(OutputStream* stream, SerializationFormat format) const;
 };
 
 

include/v8.h

@@ -3196,6 +3196,26 @@ class V8EXPORT Locker {
 };
 
 
+/**
+ * An interface for exporting data from V8, using "push" model.
+ */
+class V8EXPORT OutputStream {
+public:
+  enum OutputEncoding {
+    kAscii = 0  // 7-bit ASCII.
+  };
+  virtual ~OutputStream() {}
+  /** Notify about the end of stream. */
+  virtual void EndOfStream() = 0;
+  /** Get preferred output chunk size. Called only once. */
+  virtual int GetChunkSize() { return 1024; }
+  /** Get preferred output encoding. Called only once. */
+  virtual OutputEncoding GetOutputEncoding() { return kAscii; }
+  /** Writes the next chunk of snapshot data into the stream. */
+  virtual void WriteAsciiChunk(char* data, int size) = 0;
+};
+
+
 
 // --- I m p l e m e n t a t i o n ---
 

src/api.cc

@@ -4739,6 +4739,23 @@ const HeapSnapshotsDiff* HeapSnapshot::CompareWith(
 }
 
 
+void HeapSnapshot::Serialize(OutputStream* stream,
+                             HeapSnapshot::SerializationFormat format) const {
+  IsDeadCheck("v8::HeapSnapshot::Serialize");
+  ApiCheck(format == kJSON,
+           "v8::HeapSnapshot::Serialize",
+           "Unknown serialization format");
+  ApiCheck(stream->GetOutputEncoding() == OutputStream::kAscii,
+           "v8::HeapSnapshot::Serialize",
+           "Unsupported output encoding");
+  ApiCheck(stream->GetChunkSize() > 0,
+           "v8::HeapSnapshot::Serialize",
+           "Invalid stream chunk size");
+  i::HeapSnapshotJSONSerializer serializer(ToInternal(this));
+  serializer.Serialize(stream);
+}
+
+
 int HeapProfiler::GetSnapshotsCount() {
   IsDeadCheck("v8::HeapProfiler::GetSnapshotsCount");
   return i::HeapProfiler::GetSnapshotsCount();

src/profile-generator.h

@@ -976,6 +976,53 @@ class HeapSnapshotGenerator {
   DISALLOW_COPY_AND_ASSIGN(HeapSnapshotGenerator);
 };
 
+class OutputStreamWriter;
+
+class HeapSnapshotJSONSerializer {
+ public:
+  explicit HeapSnapshotJSONSerializer(HeapSnapshot* snapshot)
+      : snapshot_(snapshot),
+        nodes_(ObjectsMatch),
+        strings_(ObjectsMatch),
+        next_node_id_(1),
+        next_string_id_(1),
+        writer_(NULL) {
+  }
+  void Serialize(v8::OutputStream* stream);
+
+ private:
+  INLINE(static bool ObjectsMatch(void* key1, void* key2)) {
+    return key1 == key2;
+  }
+
+  INLINE(static uint32_t ObjectHash(const void* key)) {
+    return static_cast<int32_t>(reinterpret_cast<intptr_t>(key));
+  }
+
+  void EnumerateNodes();
+  int GetNodeId(HeapEntry* entry);
+  int GetStringId(const char* s);
+  void SerializeEdge(HeapGraphEdge* edge);
+  void SerializeNode(HeapEntry* entry);
+  void SerializeNodes();
+  void SerializeSnapshot();
+  void SerializeString(const unsigned char* s);
+  void SerializeStrings();
+  void SortHashMap(HashMap* map, List<HashMap::Entry*>* sorted_entries);
+
+  HeapSnapshot* snapshot_;
+  HashMap nodes_;
+  HashMap strings_;
+  int next_node_id_;
+  int next_string_id_;
+  OutputStreamWriter* writer_;
+
+  friend class HeapSnapshotJSONSerializerEnumerator;
+  friend class HeapSnapshotJSONSerializerIterator;
+
+  DISALLOW_COPY_AND_ASSIGN(HeapSnapshotJSONSerializer);
+};
+
 } }  // namespace v8::internal
 
 #endif  // ENABLE_LOGGING_AND_PROFILING

src/unicode.h

@@ -120,6 +120,9 @@ class Utf8 {
   static inline unsigned Encode(char* out, uchar c);
   static const byte* ReadBlock(Buffer<const char*> str, byte* buffer,
       unsigned capacity, unsigned* chars_read, unsigned* offset);
+  static uchar CalculateValue(const byte* str,
+                              unsigned length,
+                              unsigned* cursor);
   static const uchar kBadChar = 0xFFFD;
   static const unsigned kMaxEncodedSize   = 4;
   static const unsigned kMaxOneByteChar   = 0x7f;
@@ -133,9 +136,6 @@ class Utf8 {
   static inline uchar ValueOf(const byte* str,
                               unsigned length,
                               unsigned* cursor);
-  static uchar CalculateValue(const byte* str,
-                              unsigned length,
-                              unsigned* cursor);
 };
 
 // --- C h a r a c t e r   S t r e a m ---

test/cctest/test-heap-profiler.cc

@@ -989,4 +989,138 @@ TEST(AggregatedHeapSnapshot) {
   CHECK(IsNodeRetainedAs(a_from_b, 1));  // B has 1 ref to A.
 }
 
+namespace {
+
+class TestJSONStream : public v8::OutputStream {
+ public:
+  TestJSONStream() : eos_signaled_(0) {}
+  virtual ~TestJSONStream() {}
+  virtual void EndOfStream() { ++eos_signaled_; }
+  virtual void WriteAsciiChunk(char* buffer, int chars_written) {
+    CHECK_GT(chars_written, 0);
+    i::Vector<char> chunk = buffer_.AddBlock(chars_written, '\0');
+    memcpy(chunk.start(), buffer, chars_written);
+  }
+  void WriteTo(i::Vector<char> dest) { buffer_.WriteTo(dest); }
+  int eos_signaled() { return eos_signaled_; }
+  int size() { return buffer_.size(); }
+ private:
+  i::Collector<char> buffer_;
+  int eos_signaled_;
+};
+
+class AsciiResource: public v8::String::ExternalAsciiStringResource {
+ public:
+  explicit AsciiResource(i::Vector<char> string): data_(string.start()) {
+    length_ = string.length();
+  }
+  virtual const char* data() const { return data_; }
+  virtual size_t length() const { return length_; }
+ private:
+  const char* data_;
+  size_t length_;
+};
+
+}  // namespace
+
+TEST(HeapSnapshotJSONSerialization) {
+  v8::HandleScope scope;
+  LocalContext env;
+
+#define STRING_LITERAL_FOR_TEST \
+  "\"String \\n\\r\\u0008\\u0081\\u0101\\u0801\\u8001\""
+  CompileAndRunScript(
+      "function A(s) { this.s = s; }\n"
+      "function B(x) { this.x = x; }\n"
+      "var a = new A(" STRING_LITERAL_FOR_TEST ");\n"
+      "var b = new B(a);");
+  const v8::HeapSnapshot* snapshot =
+      v8::HeapProfiler::TakeSnapshot(v8::String::New("json"));
+  TestJSONStream stream;
+  snapshot->Serialize(&stream, v8::HeapSnapshot::kJSON);
+  CHECK_GT(stream.size(), 0);
+  CHECK_EQ(1, stream.eos_signaled());
+  i::ScopedVector<char> json(stream.size());
+  stream.WriteTo(json);
+
+  // Verify that snapshot string is valid JSON.
+  AsciiResource json_res(json);
+  v8::Local<v8::String> json_string = v8::String::NewExternal(&json_res);
+  env->Global()->Set(v8::String::New("json_snapshot"), json_string);
+  v8::Local<v8::Value> snapshot_parse_result = CompileRun(
+      "var parsed = JSON.parse(json_snapshot); true;");
+  CHECK(!snapshot_parse_result.IsEmpty());
+
+  // Verify that snapshot object has required fields.
+  v8::Local<v8::Object> parsed_snapshot =
+      env->Global()->Get(v8::String::New("parsed"))->ToObject();
+  CHECK(parsed_snapshot->Has(v8::String::New("snapshot")));
+  CHECK(parsed_snapshot->Has(v8::String::New("nodes")));
+  CHECK(parsed_snapshot->Has(v8::String::New("strings")));
+
+  // Verify that nodes meta-info is valid JSON.
+  v8::Local<v8::Value> nodes_meta_parse_result = CompileRun(
+      "var parsed_meta = JSON.parse(parsed.nodes[0]); true;");
+  CHECK(!nodes_meta_parse_result.IsEmpty());
+
+  // Get node and edge "member" offsets.
+  v8::Local<v8::Value> meta_analysis_result = CompileRun(
+      "var children_count_offset ="
+      "    parsed_meta.fields.indexOf('children_count');\n"
+      "var children_offset ="
+      "    parsed_meta.fields.indexOf('children');\n"
+      "var children_meta ="
+      "    parsed_meta.types[children_offset];\n"
+      "var child_fields_count = children_meta.fields.length;\n"
+      "var child_type_offset ="
+      "    children_meta.fields.indexOf('type');\n"
+      "var child_name_offset ="
+      "    children_meta.fields.indexOf('name_or_index');\n"
+      "var child_to_node_offset ="
+      "    children_meta.fields.indexOf('to_node');\n"
+      "var property_type ="
+      "    children_meta.types[child_type_offset].indexOf('property');");
+  CHECK(!meta_analysis_result.IsEmpty());
+
+  // A helper function for processing encoded nodes.
+  CompileRun(
+      "function GetChildPosByProperty(pos, prop_name) {\n"
+      "  var nodes = parsed.nodes;\n"
+      "  var strings = parsed.strings;\n"
+      "  for (var i = 0,\n"
+      "      count = nodes[pos + children_count_offset] * child_fields_count;\n"
+      "      i < count; i += child_fields_count) {\n"
+      "    var child_pos = pos + children_offset + i;\n"
+      "    if (nodes[child_pos + child_type_offset] === property_type\n"
+      "       && strings[nodes[child_pos + child_name_offset]] === prop_name)\n"
+      "        return nodes[child_pos + child_to_node_offset];\n"
+      "  }\n"
+      "  return null;\n"
+      "}\n");
+  // Get the string index using the path: <root> -> <global>.b.x.s
+  v8::Local<v8::Value> string_obj_pos_val = CompileRun(
+      "GetChildPosByProperty(\n"
+      "  GetChildPosByProperty(\n"
+      "    GetChildPosByProperty("
+      "      parsed.nodes[1 + children_offset + child_to_node_offset],\"b\"),\n"
+      "    \"x\"),"
+      "  \"s\")");
+  CHECK(!string_obj_pos_val.IsEmpty());
+  int string_obj_pos =
+      static_cast<int>(string_obj_pos_val->ToNumber()->Value());
+  v8::Local<v8::Object> nodes_array =
+      parsed_snapshot->Get(v8::String::New("nodes"))->ToObject();
+  int string_index = static_cast<int>(
+      nodes_array->Get(string_obj_pos + 1)->ToNumber()->Value());
+  CHECK_GT(string_index, 0);
+  v8::Local<v8::Object> strings_array =
+      parsed_snapshot->Get(v8::String::New("strings"))->ToObject();
+  v8::Local<v8::String> string = strings_array->Get(string_index)->ToString();
+  v8::Local<v8::String> ref_string =
+      CompileRun(STRING_LITERAL_FOR_TEST)->ToString();
+#undef STRING_LITERAL_FOR_TEST
+  CHECK_EQ(*v8::String::Utf8Value(ref_string),
+           *v8::String::Utf8Value(string));
+}
+
 #endif  // ENABLE_LOGGING_AND_PROFILING