presubmit.py 19.8 KB
Newer Older
1 2
#!/usr/bin/env python
#
3
# Copyright 2012 the V8 project authors. All rights reserved.
4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#     * Neither the name of Google Inc. nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

30 31 32 33 34 35 36
try:
  import hashlib
  md5er = hashlib.md5
except ImportError, e:
  import md5
  md5er = md5.new

37

38
import json
39 40
import optparse
import os
41
from os.path import abspath, join, dirname, basename, exists
42
import pickle
43
import re
44 45
import sys
import subprocess
46
import multiprocessing
47
from subprocess import PIPE
48

49 50 51 52
from testrunner.local import statusfile
from testrunner.local import testsuite
from testrunner.local import utils

53 54
# Special LINT rules diverging from default and reason.
# build/header_guard: Our guards have the form "V8_FOO_H_", not "SRC_FOO_H_".
55
#   We now run our own header guard check in PRESUBMIT.py.
56
# build/include_what_you_use: Started giving false positives for variables
57 58 59 60 61
#   named "string" and "map" assuming that you needed to include STL headers.

LINT_RULES = """
-build/header_guard
-build/include_what_you_use
62
-readability/fn_size
63
-readability/multiline_comment
64
-runtime/references
65
-whitespace/comments
66
""".split()
67

68
LINT_OUTPUT_PATTERN = re.compile(r'^.+[:(]\d+[:)]|^Done processing')
69
FLAGS_LINE = re.compile("//\s*Flags:.*--([A-z0-9-])+_[A-z0-9].*\n")
70
ASSERT_OPTIMIZED_PATTERN = re.compile("assertOptimized")
71
FLAGS_ENABLE_OPT = re.compile("//\s*Flags:.*--opt[^-].*\n")
72 73
ASSERT_UNOPTIMIZED_PATTERN = re.compile("assertUnoptimized")
FLAGS_NO_ALWAYS_OPT = re.compile("//\s*Flags:.*--no-?always-opt.*\n")
74

75 76
TOOLS_PATH = dirname(abspath(__file__))

77 78 79 80 81 82 83 84 85
def CppLintWorker(command):
  try:
    process = subprocess.Popen(command, stderr=subprocess.PIPE)
    process.wait()
    out_lines = ""
    error_count = -1
    while True:
      out_line = process.stderr.readline()
      if out_line == '' and process.poll() != None:
86 87 88
        if error_count == -1:
          print "Failed to process %s" % command.pop()
          return 1
89 90 91 92 93
        break
      m = LINT_OUTPUT_PATTERN.match(out_line)
      if m:
        out_lines += out_line
        error_count += 1
94
    sys.stdout.write(out_lines)
95 96 97 98 99 100 101 102 103
    return error_count
  except KeyboardInterrupt:
    process.kill()
  except:
    print('Error running cpplint.py. Please make sure you have depot_tools' +
          ' in your $PATH. Lint check skipped.')
    process.kill()


104 105 106 107 108 109 110 111 112 113 114 115
class FileContentsCache(object):

  def __init__(self, sums_file_name):
    self.sums = {}
    self.sums_file_name = sums_file_name

  def Load(self):
    try:
      sums_file = None
      try:
        sums_file = open(self.sums_file_name, 'r')
        self.sums = pickle.load(sums_file)
116 117
      except:
        # Cannot parse pickle for any reason. Not much we can do about it.
118 119 120 121 122 123 124 125 126
        pass
    finally:
      if sums_file:
        sums_file.close()

  def Save(self):
    try:
      sums_file = open(self.sums_file_name, 'w')
      pickle.dump(self.sums, sums_file)
127 128 129 130 131 132 133 134
    except:
      # Failed to write pickle. Try to clean-up behind us.
      if sums_file:
        sums_file.close()
      try:
        os.unlink(self.sums_file_name)
      except:
        pass
135 136 137 138 139 140 141 142
    finally:
      sums_file.close()

  def FilterUnchangedFiles(self, files):
    changed_or_new = []
    for file in files:
      try:
        handle = open(file, "r")
143
        file_sum = md5er(handle.read()).digest()
144 145 146 147 148 149 150 151 152 153 154 155
        if not file in self.sums or self.sums[file] != file_sum:
          changed_or_new.append(file)
          self.sums[file] = file_sum
      finally:
        handle.close()
    return changed_or_new

  def RemoveFile(self, file):
    if file in self.sums:
      self.sums.pop(file)


156 157 158 159 160 161
class SourceFileProcessor(object):
  """
  Utility class that can run through a directory structure, find all relevant
  files and invoke a custom check on the files.
  """

162 163 164
  def RunOnPath(self, path):
    """Runs processor on all files under the given path."""

165 166 167
    all_files = []
    for file in self.GetPathsToSearch():
      all_files += self.FindFilesIn(join(path, file))
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
    return self.ProcessFiles(all_files)

  def RunOnFiles(self, files):
    """Runs processor only on affected files."""

    # Helper for getting directory pieces.
    dirs = lambda f: dirname(f).split(os.sep)

    # Path offsets where to look (to be in sync with RunOnPath).
    # Normalize '.' to check for it with str.startswith.
    search_paths = [('' if p == '.' else p) for p in self.GetPathsToSearch()]

    all_files = [
      f.AbsoluteLocalPath()
      for f in files
183
      if (not self.IgnoreFile(f.LocalPath()) and
184 185 186 187 188 189
          self.IsRelevant(f.LocalPath()) and
          all(not self.IgnoreDir(d) for d in dirs(f.LocalPath())) and
          any(map(f.LocalPath().startswith, search_paths)))
    ]

    return self.ProcessFiles(all_files)
190 191

  def IgnoreDir(self, name):
192
    return (name.startswith('.') or
193
            name in ('buildtools', 'data', 'gmock', 'gtest', 'kraken',
194
                     'octane', 'sunspider', 'traces-arm64'))
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219

  def IgnoreFile(self, name):
    return name.startswith('.')

  def FindFilesIn(self, path):
    result = []
    for (root, dirs, files) in os.walk(path):
      for ignored in [x for x in dirs if self.IgnoreDir(x)]:
        dirs.remove(ignored)
      for file in files:
        if not self.IgnoreFile(file) and self.IsRelevant(file):
          result.append(join(root, file))
    return result


class CppLintProcessor(SourceFileProcessor):
  """
  Lint files to check that they follow the google code style.
  """

  def IsRelevant(self, name):
    return name.endswith('.cc') or name.endswith('.h')

  def IgnoreDir(self, name):
    return (super(CppLintProcessor, self).IgnoreDir(name)
220
              or (name == 'third_party'))
221

222
  IGNORE_LINT = ['export-template.h', 'flag-definitions.h']
223

224 225 226 227
  def IgnoreFile(self, name):
    return (super(CppLintProcessor, self).IgnoreFile(name)
              or (name in CppLintProcessor.IGNORE_LINT))

228
  def GetPathsToSearch(self):
229 230 231
    dirs = ['include', 'samples', 'src']
    test_dirs = ['cctest', 'common', 'fuzzer', 'inspector', 'unittests']
    return dirs + [join('test', dir) for dir in test_dirs]
232

233 234 235 236 237 238 239 240 241
  def GetCpplintScript(self, prio_path):
    for path in [prio_path] + os.environ["PATH"].split(os.pathsep):
      path = path.strip('"')
      cpplint = os.path.join(path, "cpplint.py")
      if os.path.isfile(cpplint):
        return cpplint

    return None

242
  def ProcessFiles(self, files):
243 244 245 246 247 248 249
    good_files_cache = FileContentsCache('.cpplint-cache')
    good_files_cache.Load()
    files = good_files_cache.FilterUnchangedFiles(files)
    if len(files) == 0:
      print 'No changes in files detected. Skipping cpplint check.'
      return True

250
    filters = ",".join([n for n in LINT_RULES])
251
    cpplint = self.GetCpplintScript(TOOLS_PATH)
252 253 254 255 256
    if cpplint is None:
      print('Could not find cpplint.py. Make sure '
            'depot_tools is installed and in the path.')
      sys.exit(1)

257
    command = [sys.executable, cpplint, '--filter', filters]
258

259
    commands = [command + [file] for file in files]
260 261
    count = multiprocessing.cpu_count()
    pool = multiprocessing.Pool(count)
262
    try:
263 264 265 266 267 268 269 270
      results = pool.map_async(CppLintWorker, commands).get(999999)
    except KeyboardInterrupt:
      print "\nCaught KeyboardInterrupt, terminating workers."
      sys.exit(1)

    for i in range(len(files)):
      if results[i] > 0:
        good_files_cache.RemoveFile(files[i])
271

272 273
    total_errors = sum(results)
    print "Total errors found: %d" % total_errors
274
    good_files_cache.Save()
275
    return total_errors == 0
276 277


278
COPYRIGHT_HEADER_PATTERN = re.compile(
279
    r'Copyright [\d-]*20[0-1][0-9] the V8 project authors. All rights reserved.')
280 281

class SourceProcessor(SourceFileProcessor):
282
  """
283
  Check that all files include a copyright notice and no trailing whitespaces.
284 285
  """

286
  RELEVANT_EXTENSIONS = ['.js', '.cc', '.h', '.py', '.c', '.status', '.tq', '.g4']
287

288 289 290 291 292 293 294 295 296 297 298 299
  def __init__(self):
    self.runtime_function_call_pattern = self.CreateRuntimeFunctionCallMatcher()

  def CreateRuntimeFunctionCallMatcher(self):
    runtime_h_path = join(dirname(TOOLS_PATH), 'src/runtime/runtime.h')
    pattern = re.compile(r'\s+F\(([^,]*),.*\)')
    runtime_functions = []
    with open(runtime_h_path) as f:
      for line in f.readlines():
        m = pattern.match(line)
        if m:
          runtime_functions.append(m.group(1))
300
    if len(runtime_functions) < 450:
301 302 303 304 305 306
      print ("Runtime functions list is suspiciously short. "
             "Consider updating the presubmit script.")
      sys.exit(1)
    str = '(\%\s+(' + '|'.join(runtime_functions) + '))[\s\(]'
    return re.compile(str)

307 308 309 310 311 312 313
  # Overwriting the one in the parent class.
  def FindFilesIn(self, path):
    if os.path.exists(path+'/.git'):
      output = subprocess.Popen('git ls-files --full-name',
                                stdout=PIPE, cwd=path, shell=True)
      result = []
      for file in output.stdout.read().split():
314
        for dir_part in os.path.dirname(file).replace(os.sep, '/').split('/'):
315 316 317
          if self.IgnoreDir(dir_part):
            break
        else:
318 319
          if (self.IsRelevant(file) and os.path.exists(file)
              and not self.IgnoreFile(file)):
320 321 322 323 324
            result.append(join(path, file))
      if output.wait() == 0:
        return result
    return super(SourceProcessor, self).FindFilesIn(path)

325
  def IsRelevant(self, name):
326
    for ext in SourceProcessor.RELEVANT_EXTENSIONS:
327 328 329 330 331 332 333
      if name.endswith(ext):
        return True
    return False

  def GetPathsToSearch(self):
    return ['.']

334
  def IgnoreDir(self, name):
335
    return (super(SourceProcessor, self).IgnoreDir(name) or
Yang Guo's avatar
Yang Guo committed
336
            name in ('third_party', 'out', 'obj', 'DerivedSources'))
337

338 339
  IGNORE_COPYRIGHTS = ['box2d.js',
                       'cpplint.py',
340
                       'check_injected_script_source.py',
341 342 343
                       'copy.js',
                       'corrections.js',
                       'crypto.js',
344
                       'daemon.py',
345
                       'debugger-script.js',
346
                       'earley-boyer.js',
347 348
                       'fannkuch.js',
                       'fasta.js',
349
                       'generate_protocol_externs.py',
350 351 352 353 354
                       'injected-script.cc',
                       'injected-script.h',
                       'injected-script-source.js',
                       'java-script-call-frame.cc',
                       'java-script-call-frame.h',
355
                       'jsmin.py',
356 357
                       'libraries.cc',
                       'libraries-empty.cc',
358
                       'lua_binarytrees.js',
359
                       'meta-123.js',
360
                       'memops.js',
361
                       'poppler.js',
362 363
                       'primes.js',
                       'raytrace.js',
364
                       'regexp-pcre.js',
365
                       'resources-123.js',
366
                       'rjsmin.py',
367
                       'sqlite.js',
368 369 370
                       'sqlite-change-heap.js',
                       'sqlite-pointer-masking.js',
                       'sqlite-safe-heap.js',
371 372 373 374 375 376 377
                       'v8-debugger-script.h',
                       'v8-function-call.cc',
                       'v8-function-call.h',
                       'v8-inspector-impl.cc',
                       'v8-inspector-impl.h',
                       'v8-runtime-agent-impl.cc',
                       'v8-runtime-agent-impl.h',
378 379
                       'gnuplot-4.6.3-emscripten.js',
                       'zlib.js']
380
  IGNORE_TABS = IGNORE_COPYRIGHTS + ['unicode-test.js', 'html-comments.js']
381

382 383
  IGNORE_COPYRIGHTS_DIRECTORY = "test/test262/local-tests"

384 385 386 387 388 389 390 391
  def EndOfDeclaration(self, line):
    return line == "}" or line == "};"

  def StartOfDeclaration(self, line):
    return line.find("//") == 0 or \
           line.find("/*") == 0 or \
           line.find(") {") != -1

392
  def ProcessContents(self, name, contents):
393 394 395 396 397 398
    result = True
    base = basename(name)
    if not base in SourceProcessor.IGNORE_TABS:
      if '\t' in contents:
        print "%s contains tabs" % name
        result = False
399 400
    if not base in SourceProcessor.IGNORE_COPYRIGHTS and \
        not SourceProcessor.IGNORE_COPYRIGHTS_DIRECTORY in name:
401 402 403
      if not COPYRIGHT_HEADER_PATTERN.search(contents):
        print "%s is missing a correct copyright header." % name
        result = False
404 405 406 407 408 409 410 411 412 413 414 415 416 417 418
    if ' \n' in contents or contents.endswith(' '):
      line = 0
      lines = []
      parts = contents.split(' \n')
      if not contents.endswith(' '):
        parts.pop()
      for part in parts:
        line += part.count('\n') + 1
        lines.append(str(line))
      linenumbers = ', '.join(lines)
      if len(lines) > 1:
        print "%s has trailing whitespaces in lines %s." % (name, linenumbers)
      else:
        print "%s has trailing whitespaces in line %s." % (name, linenumbers)
      result = False
419 420 421
    if not contents.endswith('\n') or contents.endswith('\n\n'):
      print "%s does not end with a single new line." % name
      result = False
422
    # Sanitize flags for fuzzer.
423
    if "mjsunit" in name or "debugger" in name:
424 425 426 427
      match = FLAGS_LINE.search(contents)
      if match:
        print "%s Flags should use '-' (not '_')" % name
        result = False
428 429 430
      if not "mjsunit/mjsunit.js" in name:
        if ASSERT_OPTIMIZED_PATTERN.search(contents) and \
            not FLAGS_ENABLE_OPT.search(contents):
431 432
          print "%s Flag --opt should be set if " \
                "assertOptimized() is used" % name
433 434 435 436 437 438
          result = False
        if ASSERT_UNOPTIMIZED_PATTERN.search(contents) and \
            not FLAGS_NO_ALWAYS_OPT.search(contents):
          print "%s Flag --no-always-opt should be set if " \
                "assertUnoptimized() is used" % name
          result = False
439 440 441 442 443

      match = self.runtime_function_call_pattern.search(contents)
      if match:
        print "%s has unexpected spaces in a runtime call '%s'" % (name, match.group(1))
        result = False
444
    return result
445

446
  def ProcessFiles(self, files):
447
    success = True
448
    violations = 0
449 450 451 452
    for file in files:
      try:
        handle = open(file)
        contents = handle.read()
453 454 455
        if not self.ProcessContents(file, contents):
          success = False
          violations += 1
456 457
      finally:
        handle.close()
458
    print "Total violating files: %s" % violations
459 460
    return success

461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502
def _CheckStatusFileForDuplicateKeys(filepath):
  comma_space_bracket = re.compile(", *]")
  lines = []
  with open(filepath) as f:
    for line in f.readlines():
      # Skip all-comment lines.
      if line.lstrip().startswith("#"): continue
      # Strip away comments at the end of the line.
      comment_start = line.find("#")
      if comment_start != -1:
        line = line[:comment_start]
      line = line.strip()
      # Strip away trailing commas within the line.
      line = comma_space_bracket.sub("]", line)
      if len(line) > 0:
        lines.append(line)

  # Strip away trailing commas at line ends. Ugh.
  for i in range(len(lines) - 1):
    if (lines[i].endswith(",") and len(lines[i + 1]) > 0 and
        lines[i + 1][0] in ("}", "]")):
      lines[i] = lines[i][:-1]

  contents = "\n".join(lines)
  # JSON wants double-quotes.
  contents = contents.replace("'", '"')
  # Fill in keywords (like PASS, SKIP).
  for key in statusfile.KEYWORDS:
    contents = re.sub(r"\b%s\b" % key, "\"%s\"" % key, contents)

  status = {"success": True}
  def check_pairs(pairs):
    keys = {}
    for key, value in pairs:
      if key in keys:
        print("%s: Error: duplicate key %s" % (filepath, key))
        status["success"] = False
      keys[key] = True

  json.loads(contents, object_pairs_hook=check_pairs)
  return status["success"]

503 504 505 506 507

class StatusFilesProcessor(SourceFileProcessor):
  """Checks status files for incorrect syntax and duplicate keys."""

  def IsRelevant(self, name):
508 509 510
    # Several changes to files under the test directories could impact status
    # files.
    return True
511 512

  def GetPathsToSearch(self):
513
    return ['test', 'tools/testrunner']
514 515

  def ProcessFiles(self, files):
516 517 518 519 520 521 522
    success = True
    for status_file_path in sorted(self._GetStatusFiles(files)):
      success &= statusfile.PresubmitCheck(status_file_path)
      success &= _CheckStatusFileForDuplicateKeys(status_file_path)
    return success

  def _GetStatusFiles(self, files):
523
    test_path = join(dirname(TOOLS_PATH), 'test')
524 525 526 527 528 529 530 531 532 533 534 535 536
    testrunner_path = join(TOOLS_PATH, 'testrunner')
    status_files = set()

    for file_path in files:
      if file_path.startswith(testrunner_path):
        for suitepath in os.listdir(test_path):
          suitename = os.path.basename(suitepath)
          status_file = os.path.join(
              test_path, suitename, suitename + ".status")
          if os.path.exists(status_file):
            status_files.add(status_file)
        return status_files

537 538 539 540 541 542 543 544 545 546 547 548 549
    for file_path in files:
      if file_path.startswith(test_path):
        # Strip off absolute path prefix pointing to test suites.
        pieces = file_path[len(test_path):].lstrip(os.sep).split(os.sep)
        if pieces:
          # Infer affected status file name. Only care for existing status
          # files. Some directories under "test" don't have any.
          if not os.path.isdir(join(test_path, pieces[0])):
            continue
          status_file = join(test_path, pieces[0], pieces[0] + ".status")
          if not os.path.exists(status_file):
            continue
          status_files.add(status_file)
550
    return status_files
551

552

553 554 555 556 557
def CheckDeps(workspace):
  checkdeps_py = join(workspace, 'buildtools', 'checkdeps', 'checkdeps.py')
  return subprocess.call([sys.executable, checkdeps_py, workspace]) == 0


558
def PyTests(workspace):
559 560 561 562 563 564 565 566 567
  result = True
  for script in [
      join(workspace, 'tools', 'release', 'test_scripts.py'),
      join(workspace, 'tools', 'unittests', 'run_tests_test.py'),
    ]:
    print 'Running ' + script
    result &= subprocess.call(
        [sys.executable, script], stdout=subprocess.PIPE) == 0
  return result
568 569


570 571 572 573 574 575 576 577 578 579 580 581
def GetOptions():
  result = optparse.OptionParser()
  result.add_option('--no-lint', help="Do not run cpplint", default=False,
                    action="store_true")
  return result


def Main():
  workspace = abspath(join(dirname(sys.argv[0]), '..'))
  parser = GetOptions()
  (options, args) = parser.parse_args()
  success = True
582 583
  print "Running checkdeps..."
  success &= CheckDeps(workspace)
584
  if not options.no_lint:
585
    print "Running C++ lint check..."
586
    success &= CppLintProcessor().RunOnPath(workspace)
587 588
  print "Running copyright header, trailing whitespaces and " \
        "two empty lines between declarations check..."
589
  success &= SourceProcessor().RunOnPath(workspace)
590 591
  print "Running status-files check..."
  success &= StatusFilesProcessor().RunOnPath(workspace)
592 593
  print "Running python tests..."
  success &= PyTests(workspace)
594 595 596 597 598 599 600 601
  if success:
    return 0
  else:
    return 1


if __name__ == '__main__':
  sys.exit(Main())