[iwyu] Add script to check that headers can be included in isolation

The most important point of IWYU (include-what-you-use) is that each
header includes everything it is using, so that whoever includes that
header does not need to additionally include other things.
This CL adds a script which generates files to automatically check this.
It is automatically invoked during "gclient runhooks" if the
"check_v8_header_includes" variable is set. This script generates a
number of .cc files in the "check-header-includes" directory, together
with a "sources.gni" file which lists all the generated cc files. Each
file includes one header.
If additionally the gn args "v8_check_header_includes" is set, this gni
file is included, and all the generated CC files will be compiled. This
will detect violations of the aforementioned IWYU rule.

R=titzer@chromium.org, machenbach@chromium.org

Bug: v8:7754, v8:7965
Change-Id: Id1cf256507052c3a9ea82f8c80ea1c0385457e31
Reviewed-on: https://chromium-review.googlesource.com/1145199
Commit-Queue: Clemens Hammacher <clemensh@chromium.org>
Reviewed-by: Ben Titzer <titzer@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#54590}

.gitignore

@@ -37,6 +37,7 @@
 /base
 /build
 /buildtools
+/check-header-includes
 /hydrogen.cfg
 /obj
 /out

BUILD.gn

@@ -158,6 +158,11 @@ declare_args() {
 
   # Enable minor mark compact.
   v8_enable_minor_mc = true
+
+  # Check that each header can be included in isolation (requires also
+  # setting the "check_v8_header_includes" gclient variable to run a
+  # specific hook).
+  v8_check_header_includes = false
 }
 
 # Derived defaults.
@@ -2511,6 +2516,13 @@ v8_source_set("v8_base") {
     "src/zone/zone.h",
   ]
 
+  if (v8_check_header_includes) {
+    # This file will be generated by tools/generate-header-include-checks.py
+    # if the "check_v8_header_includes" gclient variable is set.
+    import("check-header-includes/sources.gni")
+    sources += check_header_includes_sources
+  }
+
   if (use_jumbo_build == true) {
     jumbo_excluded_sources = [
       # TODO(mostynb@vewd.com): don't exclude these http://crbug.com/752428

DEPS

@@ -8,6 +8,7 @@ vars = {
   'download_gcmole': False,
   'download_jsfunfuzz': False,
   'download_mips_toolchain': False,
+  'check_v8_header_includes': False,
 }
 
 deps = {
@@ -385,4 +386,13 @@ hooks = [
                 '-vpython-tool', 'install',
     ],
   },
+  {
+    'name': 'check_v8_header_includes',
+    'pattern': '.',
+    'condition': 'check_v8_header_includes',
+    'action': [
+      'python',
+      'v8/tools/generate-header-include-checks.py',
+    ],
+  },
 ]

tools/generate-header-include-checks.py

+#!/usr/bin/env python
+# vim:fenc=utf-8:shiftwidth=2
+
+# Copyright 2018 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Check that each header can be included in isolation.
+
+For each header we generate one .cc file which only includes this one header.
+All these .cc files are then added to a sources.gni file which is included in
+BUILD.gn. Just compile to check whether there are any violations to the rule
+that each header must be includable in isolation.
+"""
+
+import argparse
+import os
+import os.path
+import re
+import sys
+
+# TODO(clemensh): Extend to tests.
+DEFAULT_INPUT = ['base', 'src']
+DEFAULT_GN_FILE = 'BUILD.gn'
+MY_DIR = os.path.dirname(os.path.realpath(__file__))
+V8_DIR = os.path.dirname(MY_DIR)
+OUT_DIR = os.path.join(V8_DIR, 'check-header-includes')
+AUTO_EXCLUDE = [
+  # flag-definitions.h needs a mode set for being included.
+  'src/flag-definitions.h',
+  # blacklist of headers we need to fix (https://crbug.com/v8/7965).
+  'src/allocation-site-scopes.h',
+  'src/api-arguments.h',
+  'src/api-arguments-inl.h',
+  'src/api.h',
+  'src/arguments.h',
+  'src/ast/prettyprinter.h',
+  'src/builtins/builtins-constructor.h',
+  'src/builtins/builtins-utils.h',
+  'src/compiler/allocation-builder.h',
+  'src/compiler/graph-visualizer.h',
+  'src/compiler/js-context-specialization.h',
+  'src/compiler/raw-machine-assembler.h',
+  'src/dateparser-inl.h',
+  'src/debug/debug-frames.h',
+  'src/debug/debug.h',
+  'src/debug/debug-scope-iterator.h',
+  'src/debug/debug-scopes.h',
+  'src/debug/debug-stack-trace-iterator.h',
+  'src/deoptimizer.h',
+  'src/double.h',
+  'src/elements.h',
+  'src/field-type.h',
+  'src/heap/incremental-marking.h',
+  'src/heap/incremental-marking-inl.h',
+  'src/heap/local-allocator.h',
+  'src/heap/mark-compact.h',
+  'src/heap/objects-visiting.h',
+  'src/heap/scavenger.h',
+  'src/heap/store-buffer.h',
+  'src/ic/ic.h',
+  'src/json-stringifier.h',
+  'src/keys.h',
+  'src/layout-descriptor.h',
+  'src/lookup.h',
+  'src/map-updater.h',
+  'src/objects/arguments.h',
+  'src/objects/arguments-inl.h',
+  'src/objects/compilation-cache-inl.h',
+  'src/objects/data-handler-inl.h',
+  'src/objects/fixed-array-inl.h',
+  'src/objects/frame-array.h',
+  'src/objects/hash-table-inl.h',
+  'src/objects/intl-objects-inl.h',
+  'src/objects/js-array-inl.h',
+  'src/objects/js-collection.h',
+  'src/objects/js-collection-inl.h',
+  'src/objects/js-locale.h',
+  'src/objects/js-promise-inl.h',
+  'src/objects/js-proxy-inl.h',
+  'src/objects/js-regexp-inl.h',
+  'src/objects/js-regexp-string-iterator-inl.h',
+  'src/objects/maybe-object.h',
+  'src/objects/maybe-object-inl.h',
+  'src/objects/microtask-inl.h',
+  'src/objects/module-inl.h',
+  'src/objects/ordered-hash-table-inl.h',
+  'src/objects/promise-inl.h',
+  'src/objects/property-descriptor-object.h',
+  'src/objects/prototype-info-inl.h',
+  'src/objects/regexp-match-info.h',
+  'src/objects/shared-function-info-inl.h',
+  'src/parsing/parse-info.h',
+  'src/parsing/parser.h',
+  'src/parsing/preparsed-scope-data.h',
+  'src/parsing/preparser.h',
+  'src/profiler/heap-profiler.h',
+  'src/profiler/heap-snapshot-generator.h',
+  'src/profiler/heap-snapshot-generator-inl.h',
+  'src/property.h',
+  'src/prototype.h',
+  'src/regexp/jsregexp.h',
+  'src/regexp/jsregexp-inl.h',
+  'src/snapshot/object-deserializer.h',
+  'src/string-builder.h',
+  'src/string-hasher-inl.h',
+  'src/third_party/utf8-decoder/utf8-decoder.h',
+  'src/transitions.h',
+  'src/v8memory.h',
+  'src/wasm/function-body-decoder-impl.h',
+  'src/wasm/streaming-decoder.h',
+  'src/wasm/wasm-constants.h',
+  'src/wasm/wasm-objects.h',
+  'src/wasm/wasm-serialization.h',
+]
+AUTO_EXCLUDE_PATTERNS = [
+  'src/base/atomicops_internals_.*',
+  'src/torque/.*',
+] + [
+  # platform-specific headers
+  '\\b{}\\b'.format(p) for p in
+    ('win32', 'ia32', 'x64', 'arm', 'arm64', 'mips', 'mips64', 's390', 'ppc')]
+
+args = None
+def parse_args():
+  global args
+  parser = argparse.ArgumentParser()
+  parser.add_argument('-i', '--input', type=str, action='append',
+                      help='Headers or directories to check (directories '
+                           'are scanned for headers recursively); default: ' +
+                           ','.join(DEFAULT_INPUT))
+  parser.add_argument('-x', '--exclude', type=str, action='append',
+                      help='Add an exclude pattern (regex)')
+  parser.add_argument('-v', '--verbose', action='store_true',
+                      help='Be verbose')
+  args = parser.parse_args()
+  args.exclude = (args.exclude or []) + AUTO_EXCLUDE_PATTERNS
+  args.exclude += ['^' + re.escape(x) + '$' for x in AUTO_EXCLUDE]
+  if not args.input:
+    args.input=DEFAULT_INPUT
+
+
+def printv(line):
+  if args.verbose:
+    print line
+
+
+def find_all_headers():
+  printv('Searching for headers...')
+  header_files = []
+  exclude_patterns = [re.compile(x) for x in args.exclude]
+  def add_recursively(filename):
+    full_name = os.path.join(V8_DIR, filename)
+    if not os.path.exists(full_name):
+      sys.exit('File does not exist: {}'.format(full_name))
+    if os.path.isdir(full_name):
+      for subfile in os.listdir(full_name):
+        full_name = os.path.join(filename, subfile)
+        printv('Scanning {}'.format(full_name))
+        add_recursively(full_name)
+    elif filename.endswith('.h'):
+      printv('--> Found header file {}'.format(filename))
+      for p in exclude_patterns:
+        if p.search(filename):
+          printv('--> EXCLUDED (matches {})'.format(p.pattern))
+          return
+      header_files.append(filename)
+
+  for filename in args.input:
+    add_recursively(filename)
+
+  return header_files
+
+
+def get_cc_file_name(header):
+  split = os.path.split(header)
+  header_dir = os.path.relpath(split[0], V8_DIR)
+  # Prefix with the directory name, to avoid collisions in the object files.
+  prefix = header_dir.replace(os.path.sep, '-')
+  cc_file_name = 'test-include-' + prefix + '-' + split[1][:-1] + 'cc'
+  return os.path.join(OUT_DIR, cc_file_name)
+
+
+def create_including_cc_files(header_files):
+  for header in header_files:
+    cc_file_name = get_cc_file_name(header)
+    printv('Creating file {}'.format(os.path.relpath(cc_file_name, V8_DIR)))
+    with open(cc_file_name, 'w') as cc_file:
+      cc_file.write('#include "{}"  // check including this header in '
+                    'isolation\n'.format(header))
+
+
+def generate_gni(header_files):
+  gni_file = os.path.join(OUT_DIR, 'sources.gni')
+  printv('Generating file "{}"'.format(os.path.relpath(gni_file, V8_DIR)))
+  with open(gni_file, 'w') as gn:
+    gn.write("""\
+# Copyright 2018 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This list is filled automatically by tools/check_header_includes.py.
+check_header_includes_sources = [
+""");
+    for header in header_files:
+      cc_file_name = get_cc_file_name(header)
+      gn.write('    "{}",\n'.format(os.path.relpath(cc_file_name, V8_DIR)))
+    gn.write(']\n')
+
+
+def main():
+  parse_args()
+  header_files = find_all_headers()
+  if not os.path.exists(OUT_DIR):
+    os.mkdir(OUT_DIR)
+  create_including_cc_files(header_files)
+  generate_gni(header_files)
+
+if __name__ == '__main__':
+  main()