run-clang-tidy.py 12.6 KB
Newer Older
1 2 3 4 5 6
#!/usr/bin/env python
#
# Copyright 2018 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

7 8 9
# for py2/py3 compatibility
from __future__ import print_function

10 11 12 13 14 15 16 17 18 19
import json
import multiprocessing
import optparse
import os
import re
import subprocess
import sys

CLANG_TIDY_WARNING = re.compile(r'(\/.*?)\ .*\[(.*)\]$')
CLANG_TIDY_CMDLINE_OUT = re.compile(r'^clang-tidy.*\ .*|^\./\.\*')
20
FILE_REGEXS = ['../src/*', '../test/*']
21
HEADER_REGEX = ['\.\.\/src\/.*|\.\.\/include\/.*|\.\.\/test\/.*']
22 23 24 25 26 27 28 29 30 31 32 33 34 35

THREADS = multiprocessing.cpu_count()


class ClangTidyWarning(object):
  """
  Wraps up a clang-tidy warning to present aggregated information.
  """

  def __init__(self, warning_type):
    self.warning_type = warning_type
    self.occurrences = set()

  def add_occurrence(self, file_path):
36
    self.occurrences.add(file_path.lstrip())
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79

  def __hash__(self):
    return hash(self.warning_type)

  def to_string(self, file_loc):
    s = '[%s] #%d\n' % (self.warning_type, len(self.occurrences))
    if file_loc:
      s += ' ' + '\n  '.join(self.occurrences)
      s += '\n'
    return s

  def __str__(self):
    return self.to_string(False)

  def __lt__(self, other):
    return len(self.occurrences) < len(other.occurrences)


def GenerateCompileCommands(build_folder):
  """
  Generate a compilation database.

  Currently clang-tidy-4 does not understand all flags that are passed
  by the build system, therefore, we remove them from the generated file.
  """
  ninja_ps = subprocess.Popen(
    ['ninja', '-t', 'compdb', 'cxx', 'cc'],
    stdout=subprocess.PIPE,
    cwd=build_folder)

  out_filepath = os.path.join(build_folder, 'compile_commands.json')
  with open(out_filepath, 'w') as cc_file:
    while True:
        line = ninja_ps.stdout.readline()

        if line == '':
            break

        line = line.replace('-fcomplete-member-pointers', '')
        line = line.replace('-Wno-enum-compare-switch', '')
        line = line.replace('-Wno-ignored-pragma-optimize', '')
        line = line.replace('-Wno-null-pointer-arithmetic', '')
        line = line.replace('-Wno-unused-lambda-capture', '')
80
        line = line.replace('-Wno-defaulted-function-deleted', '')
81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
        cc_file.write(line)


def skip_line(line):
  """
  Check if a clang-tidy output line should be skipped.
  """
  return bool(CLANG_TIDY_CMDLINE_OUT.search(line))


def ClangTidyRunFull(build_folder, skip_output_filter, checks, auto_fix):
  """
  Run clang-tidy on the full codebase and print warnings.
  """
  extra_args = []
  if auto_fix:
    extra_args.append('-fix')

  if checks is not None:
    extra_args.append('-checks')
    extra_args.append('-*, ' + checks)

  with open(os.devnull, 'w') as DEVNULL:
    ct_process = subprocess.Popen(
105 106
      ['run-clang-tidy', '-j' + str(THREADS), '-p', '.']
       + ['-header-filter'] + HEADER_REGEX + extra_args
107
       + FILE_REGEXS,
108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
      cwd=build_folder,
      stdout=subprocess.PIPE,
      stderr=DEVNULL)
  removing_check_header = False
  empty_lines = 0

  while True:
    line = ct_process.stdout.readline()
    if line == '':
      break

    # Skip all lines after Enbale checks and before two newlines,
    # i.e., skip clang-tidy check list.
    if line.startswith('Enabled checks'):
      removing_check_header = True
    if removing_check_header and not skip_output_filter:
      if line == '\n':
        empty_lines += 1
      if empty_lines == 2:
        removing_check_header = False
      continue

    # Different lines get removed to ease output reading.
    if not skip_output_filter and skip_line(line):
      continue

    # Print line, because no filter was matched.
    if line != '\n':
        sys.stdout.write(line)


139
def ClangTidyRunAggregate(build_folder, print_files):
140 141 142 143 144
  """
  Run clang-tidy on the full codebase and aggregate warnings into categories.
  """
  with open(os.devnull, 'w') as DEVNULL:
    ct_process = subprocess.Popen(
145 146 147
      ['run-clang-tidy', '-j' + str(THREADS), '-p', '.'] +
        ['-header-filter'] + HEADER_REGEX +
        FILE_REGEXS,
148 149 150 151 152 153 154 155 156 157 158
      cwd=build_folder,
      stdout=subprocess.PIPE,
      stderr=DEVNULL)
  warnings = dict()
  while True:
    line = ct_process.stdout.readline()
    if line == '':
      break

    res = CLANG_TIDY_WARNING.search(line)
    if res is not None:
159 160 161
      warnings.setdefault(
          res.group(2),
          ClangTidyWarning(res.group(2))).add_occurrence(res.group(1))
162 163 164 165 166

  for warning in sorted(warnings.values(), reverse=True):
    sys.stdout.write(warning.to_string(print_files))


167
def ClangTidyRunDiff(build_folder, diff_branch, auto_fix):
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182
  """
  Run clang-tidy on the diff between current and the diff_branch.
  """
  if diff_branch is None:
    diff_branch = subprocess.check_output(['git', 'merge-base',
                                           'HEAD', 'origin/master']).strip()

  git_ps = subprocess.Popen(
    ['git', 'diff', '-U0', diff_branch], stdout=subprocess.PIPE)

  extra_args = []
  if auto_fix:
    extra_args.append('-fix')

  with open(os.devnull, 'w') as DEVNULL:
183 184 185
    """
    The script `clang-tidy-diff` does not provide support to add header-
    filters. To still analyze headers we use the build path option `-path` to
186
    inject our header-filter option. This works because the script just adds
187 188 189 190 191 192
    the passed path string to the commandline of clang-tidy.
    """
    modified_build_folder = build_folder
    modified_build_folder += ' -header-filter='
    modified_build_folder += '\'' + ''.join(HEADER_REGEX) + '\''

193
    ct_ps = subprocess.Popen(
194 195
      ['clang-tidy-diff.py', '-path', modified_build_folder, '-p1'] +
        extra_args,
196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
      stdin=git_ps.stdout,
      stdout=subprocess.PIPE,
      stderr=DEVNULL)
  git_ps.wait()
  while True:
    line = ct_ps.stdout.readline()
    if line == '':
      break

    if skip_line(line):
      continue

    sys.stdout.write(line)


def rm_prefix(string, prefix):
  """
  Removes prefix from a string until the new string
  no longer starts with the prefix.
  """
  while string.startswith(prefix):
    string = string[len(prefix):]
  return string


def ClangTidyRunSingleFile(build_folder, filename_to_check, auto_fix,
                           line_ranges=[]):
  """
  Run clang-tidy on a single file.
  """
  files_with_relative_path = []

  compdb_filepath = os.path.join(build_folder, 'compile_commands.json')
  with open(compdb_filepath) as raw_json_file:
    compdb = json.load(raw_json_file)

  for db_entry in compdb:
    if db_entry['file'].endswith(filename_to_check):
      files_with_relative_path.append(db_entry['file'])

  with open(os.devnull, 'w') as DEVNULL:
    for file_with_relative_path in files_with_relative_path:
      line_filter = None
      if len(line_ranges) != 0:
        line_filter = '['
        line_filter += '{ \"lines\":[' + ', '.join(line_ranges)
        line_filter += '], \"name\":\"'
        line_filter += rm_prefix(file_with_relative_path,
                                 '../') + '\"}'
        line_filter += ']'

      extra_args = ['-line-filter=' + line_filter] if line_filter else []

      if auto_fix:
        extra_args.append('-fix')

      subprocess.call(['clang-tidy', '-p', '.'] +
                      extra_args +
                      [file_with_relative_path],
                      cwd=build_folder,
                      stderr=DEVNULL)


def CheckClangTidy():
  """
  Checks if a clang-tidy binary exists.
  """
  with open(os.devnull, 'w') as DEVNULL:
    return subprocess.call(['which', 'clang-tidy'], stdout=DEVNULL) == 0


def CheckCompDB(build_folder):
  """
  Checks if a compilation database exists in the build_folder.
  """
  return os.path.isfile(os.path.join(build_folder, 'compile_commands.json'))


274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291
def DetectBuildFolder():
    """
    Tries to auto detect the last used build folder in out/
    """
    outdirs_folder = 'out/'
    last_used = None
    last_timestamp = -1
    for outdir in [outdirs_folder + folder_name
                   for folder_name in os.listdir(outdirs_folder)
                   if os.path.isdir(outdirs_folder + folder_name)]:
        outdir_modified_timestamp = os.path.getmtime(outdir)
        if  outdir_modified_timestamp > last_timestamp:
            last_timestamp = outdir_modified_timestamp
            last_used = outdir

    return last_used


292 293 294 295 296 297 298 299 300 301
def GetOptions():
  """
  Generate the option parser for this script.
  """
  result = optparse.OptionParser()
  result.add_option(
    '-b',
    '--build-folder',
    help='Set V8 build folder',
    dest='build_folder',
302
    default=None)
303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377
  result.add_option(
    '-j',
    help='Set the amount of threads that should be used',
    dest='threads',
    default=None)
  result.add_option(
    '--gen-compdb',
    help='Generate a compilation database for clang-tidy',
    default=False,
    action='store_true')
  result.add_option(
    '--no-output-filter',
    help='Done use any output filterning',
    default=False,
    action='store_true')
  result.add_option(
    '--fix',
    help='Fix auto fixable issues',
    default=False,
    dest='auto_fix',
    action='store_true'
  )

  # Full clang-tidy.
  full_run_g = optparse.OptionGroup(result, 'Clang-tidy full', '')
  full_run_g.add_option(
    '--full',
    help='Run clang-tidy on the whole codebase',
    default=False,
    action='store_true')
  full_run_g.add_option('--checks',
                        help='Clang-tidy checks to use.',
                        default=None)
  result.add_option_group(full_run_g)

  # Aggregate clang-tidy.
  agg_run_g = optparse.OptionGroup(result, 'Clang-tidy aggregate', '')
  agg_run_g.add_option('--aggregate', help='Run clang-tidy on the whole '\
             'codebase and aggregate the warnings',
             default=False, action='store_true')
  agg_run_g.add_option('--show-loc', help='Show file locations when running '\
             'in aggregate mode', default=False,
             action='store_true')
  result.add_option_group(agg_run_g)

  # Diff clang-tidy.
  diff_run_g = optparse.OptionGroup(result, 'Clang-tidy diff', '')
  diff_run_g.add_option('--branch', help='Run clang-tidy on the diff '\
             'between HEAD and the merge-base between HEAD '\
             'and DIFF_BRANCH (origin/master by default).',
             default=None, dest='diff_branch')
  result.add_option_group(diff_run_g)

  # Single clang-tidy.
  single_run_g = optparse.OptionGroup(result, 'Clang-tidy single', '')
  single_run_g.add_option(
    '--single', help='', default=False, action='store_true')
  single_run_g.add_option(
    '--file', help='File name to check', default=None, dest='file_name')
  single_run_g.add_option('--lines', help='Limit checks to a line range. '\
              'For example: --lines="[2,4], [5,6]"',
              default=[], dest='line_ranges')

  result.add_option_group(single_run_g)
  return result


def main():
  parser = GetOptions()
  (options, _) = parser.parse_args()

  if options.threads is not None:
    global THREADS
    THREADS = options.threads

378 379 380
  if options.build_folder is None:
    options.build_folder = DetectBuildFolder()

381
  if not CheckClangTidy():
382
    print('Could not find clang-tidy')
383
  elif options.build_folder is None or not os.path.isdir(options.build_folder):
384
    print('Please provide a build folder with -b')
385 386 387
  elif options.gen_compdb:
    GenerateCompileCommands(options.build_folder)
  elif not CheckCompDB(options.build_folder):
388 389
    print('Could not find compilation database, ' \
      'please generate it with --gen-compdb')
390
  else:
391
    print('Using build folder:', options.build_folder)
392
    if options.full:
393
      print('Running clang-tidy - full')
394 395 396 397 398
      ClangTidyRunFull(options.build_folder,
                       options.no_output_filter,
                       options.checks,
                       options.auto_fix)
    elif options.aggregate:
399
      print('Running clang-tidy - aggregating warnings')
400
      if options.auto_fix:
401
        print('Auto fix not working in aggregate mode, running without.')
402
      ClangTidyRunAggregate(options.build_folder, options.show_loc)
403
    elif options.single:
404
      print('Running clang-tidy - single on ' + options.file_name)
405 406 407 408 409 410 411 412 413 414
      if options.file_name is not None:
        line_ranges = []
        for match in re.findall(r'(\[.*?\])', options.line_ranges):
          if match is not []:
            line_ranges.append(match)
        ClangTidyRunSingleFile(options.build_folder,
                               options.file_name,
                               options.auto_fix,
                               line_ranges)
      else:
415
        print('Filename provided, please specify a filename with --file')
416
    else:
417
      print('Running clang-tidy')
418 419 420
      ClangTidyRunDiff(options.build_folder,
                       options.diff_branch,
                       options.auto_fix)
421 422 423 424


if __name__ == '__main__':
  main()