Introduce mkgrokdump to update tools/v8heapconst.py.

- Migrate make grokdump to GYP and GN
- Move code from d8 into stand-alone execution
- Add test case to ensure it's up-to-date

Review-Url: https://codereview.chromium.org/2809653003
Cr-Original-Original-Commit-Position: refs/heads/master@{#44687}
Committed: https://chromium.googlesource.com/v8/v8/+/0cc0c130fa56f129c90c2a74cb01bda85df5e42a
Review-Url: https://codereview.chromium.org/2809653003
Cr-Original-Commit-Position: refs/heads/master@{#44710}
Committed: https://chromium.googlesource.com/v8/v8/+/477f00557458d383bd9ebef281890ba485ee882a
Review-Url: https://codereview.chromium.org/2809653003
Cr-Commit-Position: refs/heads/master@{#44738}

Makefile

@@ -240,7 +240,6 @@ ifdef android_ndk_root
 endif
 
 # ----------------- available targets: --------------------
-# - "grokdump": rebuilds heap constants lists used by grokdump
 # - any arch listed in ARCHES (see below)
 # - any mode listed in MODES
 # - every combination <arch>.<mode>, e.g. "ia32.release"
@@ -467,12 +466,6 @@ $(ENVFILE).new:
 	$(eval CXX_TARGET_ARCH:=$(subst x86_64,x64,$(CXX_TARGET_ARCH)))
 	@mkdir -p $(OUTDIR); echo "GYPFLAGS=$(GYPFLAGS) -Dtarget_arch=$(CXX_TARGET_ARCH)" > $(ENVFILE).new;
 
-# Heap constants for grokdump.
-DUMP_FILE = tools/v8heapconst.py
-grokdump: ia32.release
-	@cat $(DUMP_FILE).tmpl > $(DUMP_FILE)
-	@$(OUTDIR)/ia32.release/d8 --dump-heap-constants >> $(DUMP_FILE)
-
 # Support for the GNU GLOBAL Source Code Tag System.
 gtags.files: $(GYPFILES) $(ENVFILE)
 	@find include src test -name '*.h' -o -name '*.cc' -o -name '*.c' > $@

gypfiles/all.gyp

@@ -10,6 +10,7 @@
       'dependencies': [
         '../src/d8.gyp:d8',
         '../test/inspector/inspector.gyp:*',
+        '../test/mkgrokdump/mkgrokdump.gyp:*',
       ],
       'conditions': [
         ['component!="shared_library"', {

src/d8.cc

@@ -2553,9 +2553,6 @@ bool Shell::SetOptions(int argc, char* argv[]) {
       continue;
     } else if (strcmp(argv[i], "--isolate") == 0) {
       options.num_isolates++;
-    } else if (strcmp(argv[i], "--dump-heap-constants") == 0) {
-      options.dump_heap_constants = true;
-      argv[i] = NULL;
     } else if (strcmp(argv[i], "--throws") == 0) {
       options.expected_to_throw = true;
       argv[i] = NULL;
@@ -2936,73 +2933,6 @@ void Shell::CleanupWorkers() {
   externalized_contents_.clear();
 }
 
-
-static void DumpHeapConstants(i::Isolate* isolate) {
-  i::Heap* heap = isolate->heap();
-  printf(
-      "# Copyright 2017 the V8 project authors. All rights reserved.\n"
-      "# Use of this source code is governed by a BSD-style license that can\n"
-      "# be found in the LICENSE file.\n\n");
-  // Dump the INSTANCE_TYPES table to the console.
-  printf("# List of known V8 instance types.\n");
-#define DUMP_TYPE(T) printf("  %d: \"%s\",\n", i::T, #T);
-  printf("INSTANCE_TYPES = {\n");
-  INSTANCE_TYPE_LIST(DUMP_TYPE)
-  printf("}\n");
-#undef DUMP_TYPE
-
-  // Dump the KNOWN_MAP table to the console.
-  printf("\n# List of known V8 maps.\n");
-#define ROOT_LIST_CASE(type, name, camel_name) \
-  if (n == NULL && o == heap->name()) n = #camel_name;
-#define STRUCT_LIST_CASE(upper_name, camel_name, name) \
-  if (n == NULL && o == heap->name##_map()) n = #camel_name "Map";
-  i::HeapObjectIterator it(heap->map_space());
-  printf("KNOWN_MAPS = {\n");
-  for (i::Object* o = it.Next(); o != NULL; o = it.Next()) {
-    i::Map* m = i::Map::cast(o);
-    const char* n = NULL;
-    intptr_t p = reinterpret_cast<intptr_t>(m) & 0x7ffff;
-    int t = m->instance_type();
-    ROOT_LIST(ROOT_LIST_CASE)
-    STRUCT_LIST(STRUCT_LIST_CASE)
-    if (n == NULL) continue;
-    printf("  0x%05" V8PRIxPTR ": (%d, \"%s\"),\n", p, t, n);
-  }
-  printf("}\n");
-#undef STRUCT_LIST_CASE
-#undef ROOT_LIST_CASE
-
-  // Dump the KNOWN_OBJECTS table to the console.
-  printf("\n# List of known V8 objects.\n");
-#define ROOT_LIST_CASE(type, name, camel_name) \
-  if (n == NULL && o == heap->name()) n = #camel_name;
-  i::OldSpaces spit(heap);
-  printf("KNOWN_OBJECTS = {\n");
-  for (i::PagedSpace* s = spit.next(); s != NULL; s = spit.next()) {
-    i::HeapObjectIterator it(s);
-    const char* sname = AllocationSpaceName(s->identity());
-    for (i::Object* o = it.Next(); o != NULL; o = it.Next()) {
-      const char* n = NULL;
-      intptr_t p = reinterpret_cast<intptr_t>(o) & 0x7ffff;
-      ROOT_LIST(ROOT_LIST_CASE)
-      if (n == NULL) continue;
-      printf("  (\"%s\", 0x%05" V8PRIxPTR "): \"%s\",\n", sname, p, n);
-    }
-  }
-  printf("}\n");
-#undef ROOT_LIST_CASE
-
-  // Dump frame markers
-  printf("\n# List of known V8 Frame Markers.\n");
-#define DUMP_MARKER(T, class) printf("  \"%s\",\n", #T);
-  printf("FRAME_MARKERS = (\n");
-  STACK_FRAME_TYPE_LIST(DUMP_MARKER)
-  printf(")\n");
-#undef DUMP_TYPE
-}
-
-
 int Shell::Main(int argc, char* argv[]) {
   std::ofstream trace_file;
 #if (defined(_WIN32) || defined(_WIN64))
@@ -3107,11 +3037,6 @@ int Shell::Main(int argc, char* argv[]) {
       tracing_controller->StartTracing(trace_config);
     }
 
-    if (options.dump_heap_constants) {
-      DumpHeapConstants(reinterpret_cast<i::Isolate*>(isolate));
-      return 0;
-    }
-
     if (options.stress_opt || options.stress_deopt) {
       Testing::SetStressRunType(options.stress_opt
                                 ? Testing::kStressTypeOpt

src/d8.h

@@ -292,7 +292,6 @@ class ShellOptions {
         stress_runs(1),
         interactive_shell(false),
         test_shell(false),
-        dump_heap_constants(false),
         expected_to_throw(false),
         mock_arraybuffer_allocator(false),
         enable_inspector(false),
@@ -323,7 +322,6 @@ class ShellOptions {
   int stress_runs;
   bool interactive_shell;
   bool test_shell;
-  bool dump_heap_constants;
   bool expected_to_throw;
   bool mock_arraybuffer_allocator;
   bool enable_inspector;

src/heap/heap.cc

@@ -2926,7 +2926,6 @@ bool Heap::RootCanBeWrittenAfterInitialization(Heap::RootListIndex root_index) {
     case kInstanceofCacheMapRootIndex:
     case kInstanceofCacheAnswerRootIndex:
     case kCodeStubsRootIndex:
-    case kEmptyScriptRootIndex:
     case kScriptListRootIndex:
     case kMaterializedObjectsRootIndex:
     case kMicrotaskQueueRootIndex:

test/BUILD.gn

@@ -11,6 +11,7 @@ group("gn_all") {
   deps = [
     ":default_tests",
     "inspector:inspector-test",
+    "mkgrokdump:mkgrokdump",
   ]
 
   if (host_os != "mac" || !is_android) {
@@ -48,6 +49,7 @@ group("default_tests") {
       ":intl_run",
       ":message_run",
       ":mjsunit_run",
+      ":mkgrokdump_run",
       ":preparser_run",
       ":unittests_run",
     ]
@@ -164,6 +166,14 @@ v8_isolate_run("mjsunit") {
   isolate = "mjsunit/mjsunit.isolate"
 }
 
+v8_isolate_run("mkgrokdump") {
+  deps = [
+    "mkgrokdump:mkgrokdump",
+  ]
+
+  isolate = "mkgrokdump/mkgrokdump.isolate"
+}
+
 v8_isolate_run("mozilla") {
   deps = [
     "..:d8_run",

test/bot_default.isolate

@@ -15,6 +15,7 @@
     'intl/intl.isolate',
     'message/message.isolate',
     'mjsunit/mjsunit.isolate',
+    'mkgrokdump/mkgrokdump.isolate',
     'preparser/preparser.isolate',
     'unittests/unittests.isolate',
     'webkit/webkit.isolate',

test/default.isolate

@@ -15,6 +15,7 @@
     'intl/intl.isolate',
     'message/message.isolate',
     'mjsunit/mjsunit.isolate',
+    'mkgrokdump/mkgrokdump.isolate',
     'preparser/preparser.isolate',
     'unittests/unittests.isolate',
   ],

test/mkgrokdump/BUILD.gn

+# Copyright 2017 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("../../gni/v8.gni")
+
+v8_executable("mkgrokdump") {
+  testonly = true
+
+  # mkgrokdump is used to create tools/v8heapconst.py.
+
+  sources = [
+    "mkgrokdump.cc",
+  ]
+
+  configs = [ "../..:internal_config_base" ]
+
+  defines = []
+
+  deps = [
+    "../..:v8",
+    "../..:v8_libbase",
+    "../..:v8_libplatform",
+    "//build/config/sanitizers:deps",
+    "//build/win:default_exe_manifest",
+  ]
+}

test/mkgrokdump/DEPS

+include_rules = [
+  "+src",
+]

test/mkgrokdump/mkgrokdump.cc

+// Copyright 2012 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <stdio.h>
+
+#include "include/libplatform/libplatform.h"
+#include "include/v8.h"
+
+#include "src/frames.h"
+#include "src/heap/heap.h"
+#include "src/heap/spaces.h"
+#include "src/isolate.h"
+#include "src/objects-inl.h"
+
+namespace v8 {
+
+static const char* kHeader =
+    "# Copyright 2017 the V8 project authors. All rights reserved.\n"
+    "# Use of this source code is governed by a BSD-style license that can\n"
+    "# be found in the LICENSE file.\n"
+    "\n"
+    "# This file is automatically generated by mkgrokdump and should not\n"
+    "# be modified manually.\n"
+    "\n"
+    "# List of known V8 instance types.\n";
+
+// Non-snapshot builds allocate objects to different places.
+// Debug builds emit debug code, affecting code object sizes.
+#if defined(V8_USE_SNAPSHOT) && !defined(DEBUG)
+static const char* kBuild = "shipping";
+#else
+static const char* kBuild = "non-shipping";
+#endif
+
+class MockArrayBufferAllocator : public v8::ArrayBuffer::Allocator {
+ public:
+  void* Allocate(size_t length) override { return nullptr; }
+  void* AllocateUninitialized(size_t length) override { return nullptr; }
+  void Free(void* p, size_t) override {}
+};
+
+static int DumpHeapConstants(const char* argv0) {
+  // Start up V8.
+  v8::Platform* platform = v8::platform::CreateDefaultPlatform();
+  v8::V8::InitializePlatform(platform);
+  v8::V8::Initialize();
+  v8::V8::InitializeExternalStartupData(argv0);
+  Isolate::CreateParams create_params;
+  MockArrayBufferAllocator mock_arraybuffer_allocator;
+  create_params.array_buffer_allocator = &mock_arraybuffer_allocator;
+  Isolate* isolate = Isolate::New(create_params);
+  {
+    Isolate::Scope scope(isolate);
+    i::Heap* heap = reinterpret_cast<i::Isolate*>(isolate)->heap();
+    i::PrintF("%s", kHeader);
+#define DUMP_TYPE(T) i::PrintF("  %d: \"%s\",\n", i::T, #T);
+    i::PrintF("INSTANCE_TYPES = {\n");
+    INSTANCE_TYPE_LIST(DUMP_TYPE)
+    i::PrintF("}\n");
+#undef DUMP_TYPE
+
+    // Dump the KNOWN_MAP table to the console.
+    i::PrintF("\n# List of known V8 maps.\n");
+#define ROOT_LIST_CASE(type, name, camel_name) \
+  if (n == NULL && o == heap->name()) n = #camel_name;
+#define STRUCT_LIST_CASE(upper_name, camel_name, name) \
+  if (n == NULL && o == heap->name##_map()) n = #camel_name "Map";
+    i::HeapObjectIterator it(heap->map_space());
+    i::PrintF("KNOWN_MAPS = {\n");
+    for (i::Object* o = it.Next(); o != NULL; o = it.Next()) {
+      i::Map* m = i::Map::cast(o);
+      const char* n = NULL;
+      intptr_t p = reinterpret_cast<intptr_t>(m) & 0x7ffff;
+      int t = m->instance_type();
+      ROOT_LIST(ROOT_LIST_CASE)
+      STRUCT_LIST(STRUCT_LIST_CASE)
+      if (n == NULL) continue;
+      i::PrintF("  0x%05" V8PRIxPTR ": (%d, \"%s\"),\n", p, t, n);
+    }
+    i::PrintF("}\n");
+#undef STRUCT_LIST_CASE
+#undef ROOT_LIST_CASE
+
+    // Dump the KNOWN_OBJECTS table to the console.
+    i::PrintF("\n# List of known V8 objects.\n");
+#define ROOT_LIST_CASE(type, name, camel_name) \
+  if (n == NULL && o == heap->name()) {        \
+    n = #camel_name;                           \
+    i = i::Heap::k##camel_name##RootIndex;     \
+  }
+    i::OldSpaces spit(heap);
+    i::PrintF("KNOWN_OBJECTS = {\n");
+    for (i::PagedSpace* s = spit.next(); s != NULL; s = spit.next()) {
+      i::HeapObjectIterator it(s);
+      // Code objects are generally platform-dependent.
+      if (s->identity() == i::CODE_SPACE) continue;
+      const char* sname = AllocationSpaceName(s->identity());
+      for (i::Object* o = it.Next(); o != NULL; o = it.Next()) {
+        const char* n = NULL;
+        i::Heap::RootListIndex i = i::Heap::kStrongRootListLength;
+        intptr_t p = reinterpret_cast<intptr_t>(o) & 0x7ffff;
+        ROOT_LIST(ROOT_LIST_CASE)
+        if (n == NULL) continue;
+        if (!i::Heap::RootIsImmortalImmovable(i)) continue;
+        i::PrintF("  (\"%s\", 0x%05" V8PRIxPTR "): \"%s\",\n", sname, p, n);
+      }
+    }
+    i::PrintF("}\n");
+#undef ROOT_LIST_CASE
+
+    // Dump frame markers
+    i::PrintF("\n# List of known V8 Frame Markers.\n");
+#define DUMP_MARKER(T, class) i::PrintF("  \"%s\",\n", #T);
+    i::PrintF("FRAME_MARKERS = (\n");
+    STACK_FRAME_TYPE_LIST(DUMP_MARKER)
+    i::PrintF(")\n");
+#undef DUMP_TYPE
+  }
+
+  i::PrintF("\n# This set of constants is generated from a %s build.\n",
+            kBuild);
+
+  // Teardown.
+  isolate->Dispose();
+  v8::V8::ShutdownPlatform();
+  delete platform;
+  return 0;
+}
+
+}  // namespace v8
+
+int main(int argc, char* argv[]) { return v8::DumpHeapConstants(argv[0]); }

test/mkgrokdump/mkgrokdump.gyp

+# Copyright 2017 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+{
+  'variables': {
+    'v8_code': 1,
+  },
+  'includes': ['../../gypfiles/toolchain.gypi', '../../gypfiles/features.gypi'],
+  'targets': [
+    {
+      'target_name': 'mkgrokdump',
+      'type': 'executable',
+      'dependencies': [
+        '../../src/v8.gyp:v8',
+        '../../src/v8.gyp:v8_libbase',
+        '../../src/v8.gyp:v8_libplatform',
+      ],
+      'include_dirs': [
+        '../..',
+      ],
+      'sources': [
+        'mkgrokdump.cc',
+      ],
+    },
+  ],
+  'conditions': [
+    ['test_isolation_mode != "noop"', {
+      'targets': [
+        {
+          'target_name': 'mkgrokdump_run',
+          'type': 'none',
+          'dependencies': [
+            'mkgrokdump',
+          ],
+          'includes': [
+            '../../gypfiles/isolate.gypi',
+          ],
+          'sources': [
+            'mkgrokdump.isolate',
+          ],
+        },
+      ],
+    }],
+  ],
+}

test/mkgrokdump/mkgrokdump.isolate

+# Copyright 2017 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+{
+  'variables': {
+    'files': [
+      './mkgrokdump.status',
+      './testcfg.py',
+      '../../tools/v8heapconst.py',
+      '<(PRODUCT_DIR)/mkgrokdump<(EXECUTABLE_SUFFIX)',
+    ],
+  },
+  'includes': [
+    '../../src/base.isolate',
+    '../../tools/testrunner/testrunner.isolate',
+  ],
+}

test/mkgrokdump/mkgrokdump.status

+# Copyright 2017 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+[
+# Only test for default mode x64.
+['variant != default or arch != x64', {
+  '*': [SKIP],
+}],  # variant != default or arch != x64
+]

test/mkgrokdump/testcfg.py

+# Copyright 2017 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import difflib
+
+from testrunner.local import testsuite
+from testrunner.objects import testcase
+
+
+class MkGrokdump(testsuite.TestSuite):
+
+  def __init__(self, name, root):
+    super(MkGrokdump, self).__init__(name, root)
+
+  def ListTests(self, context):
+    test = testcase.TestCase(self, self.shell())
+    return [test]
+
+  def GetFlagsForTestCase(self, testcase, context):
+    return []
+
+  def IsFailureOutput(self, testcase):
+    output = testcase.output
+    v8_path = os.path.dirname(os.path.dirname(os.path.abspath(self.root)))
+    expected_path = os.path.join(v8_path, "tools", "v8heapconst.py")
+    with open(expected_path) as f:
+      expected = f.read()
+    expected_lines = expected.splitlines()
+    actual_lines = output.stdout.splitlines()
+    diff = difflib.unified_diff(expected_lines, actual_lines, lineterm="",
+                                fromfile="expected_path")
+    diffstring = '\n'.join(diff)
+    if diffstring is not "":
+      if "generated from a non-shipping build" in output.stdout:
+        return False
+      if not "generated from a shipping build" in output.stdout:
+        output.stdout = "Unexpected output:\n\n" + output.stdout
+        return True
+      output.stdout = diffstring
+      return True
+    return False
+
+  def shell(self):
+    return "mkgrokdump"
+
+def GetSuite(name, root):
+  return MkGrokdump(name, root)

tools/run-tests.py

@@ -70,6 +70,7 @@ TEST_MAP = {
     "cctest",
     "inspector",
     "webkit",
+    "mkgrokdump",
     "fuzzer",
     "message",
     "preparser",
@@ -82,6 +83,7 @@ TEST_MAP = {
     "mjsunit",
     "cctest",
     "inspector",
+    "mkgrokdump",
     "fuzzer",
     "message",
     "preparser",

tools/v8heapconst.py.tmpl

-# Copyright 2013 the V8 project authors. All rights reserved.
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are
-# met:
-#
-#     * Redistributions of source code must retain the above copyright
-#       notice, this list of conditions and the following disclaimer.
-#     * Redistributions in binary form must reproduce the above
-#       copyright notice, this list of conditions and the following
-#       disclaimer in the documentation and/or other materials provided
-#       with the distribution.
-#     * Neither the name of Google Inc. nor the names of its
-#       contributors may be used to endorse or promote products derived
-#       from this software without specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-# This file is automatically generated from the V8 source and should not
-# be modified manually, run 'make grokdump' instead to update this file.
-

tools/verify_source_deps.py

@@ -31,6 +31,7 @@ GYP_FILES = [
   os.path.join(V8_BASE, 'test', 'fuzzer', 'fuzzer.gyp'),
   os.path.join(V8_BASE, 'test', 'unittests', 'unittests.gyp'),
   os.path.join(V8_BASE, 'test', 'inspector', 'inspector.gyp'),
+  os.path.join(V8_BASE, 'test', 'mkgrokdump', 'mkgrokdump.gyp'),
   os.path.join(V8_BASE, 'testing', 'gmock.gyp'),
   os.path.join(V8_BASE, 'testing', 'gtest.gyp'),
   os.path.join(V8_BASE, 'tools', 'parser-shell.gyp'),
@@ -49,6 +50,7 @@ ALL_GYP_PREFIXES = [
   os.path.join('test', 'fuzzer'),
   os.path.join('test', 'unittests'),
   os.path.join('test', 'inspector'),
+  os.path.join('test', 'mkgrokdump'),
 ]
 
 GYP_UNSUPPORTED_FEATURES = [
@@ -64,6 +66,7 @@ GN_FILES = [
   os.path.join(V8_BASE, 'test', 'cctest', 'BUILD.gn'),
   os.path.join(V8_BASE, 'test', 'unittests', 'BUILD.gn'),
   os.path.join(V8_BASE, 'test', 'inspector', 'BUILD.gn'),
+  os.path.join(V8_BASE, 'test', 'mkgrokdump', 'BUILD.gn'),
   os.path.join(V8_BASE, 'tools', 'BUILD.gn'),
 ]
 
@@ -88,6 +91,7 @@ ALL_GN_PREFIXES = [
   os.path.join('test', 'cctest'),
   os.path.join('test', 'unittests'),
   os.path.join('test', 'inspector'),
+  os.path.join('test', 'mkgrokdump'),
 ]
 
 def pathsplit(path):