presubmit_support.py 41.9 KB
Newer Older
1 2
#!/usr/bin/env python
# Copyright (c) 2011 The Chromium Authors. All rights reserved.
3 4 5 6 7 8
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Enables directory-specific presubmit checks to run at upload and/or commit.
"""

9
__version__ = '1.6.1'
10 11 12 13 14 15 16 17 18

# TODO(joi) Add caching where appropriate/needed. The API is designed to allow
# caching (between all different invocations of presubmit scripts for a given
# change). We should add it as our presubmit scripts start feeling slow.

import cPickle  # Exposed through the API.
import cStringIO  # Exposed through the API.
import fnmatch
import glob
19
import logging
20 21 22 23
import marshal  # Exposed through the API.
import optparse
import os  # Somewhat exposed through the API.
import pickle  # Exposed through the API.
24
import random
25 26 27
import re  # Exposed through the API.
import sys  # Parts exposed through API.
import tempfile  # Exposed through the API.
28
import time
29
import traceback  # Exposed through the API.
30
import types
31
import unittest  # Exposed through the API.
32
import urllib2  # Exposed through the API.
33
from warnings import warn
34

35
try:
36
  import simplejson as json  # pylint: disable=F0401
37 38
except ImportError:
  try:
39 40
    import json  # pylint: disable=F0401
  except ImportError:
41
    # Import the one included in depot_tools.
42
    sys.path.append(os.path.join(os.path.dirname(__file__), 'third_party'))
43
    import simplejson as json  # pylint: disable=F0401
44

45
# Local imports.
46
import fix_encoding
47
import gclient_utils
48
import owners
49
import presubmit_canned_checks
50
import rietveld
51
import scm
52
import subprocess2 as subprocess  # Exposed through the API.
53 54


55 56 57 58
# Ask for feedback only once in program lifetime.
_ASKED_FOR_FEEDBACK = False


59
class PresubmitFailure(Exception):
60 61 62 63 64 65 66 67 68 69 70 71
  pass


def normpath(path):
  '''Version of os.path.normpath that also changes backward slashes to
  forward slashes when not running on Windows.
  '''
  # This is safe to always do because the Windows version of os.path.normpath
  # will replace forward slashes with backward slashes.
  path = path.replace(os.sep, '/')
  return os.path.normpath(path)

72 73 74 75

def _RightHandSideLinesImpl(affected_files):
  """Implements RightHandSideLines for InputApi and GclChange."""
  for af in affected_files:
76
    lines = af.ChangedContents()
77
    for line in lines:
78
      yield (af, line[0], line[1])
79 80


81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112
class PresubmitOutput(object):
  def __init__(self, input_stream=None, output_stream=None):
    self.input_stream = input_stream
    self.output_stream = output_stream
    self.reviewers = []
    self.written_output = []
    self.error_count = 0

  def prompt_yes_no(self, prompt_string):
    self.write(prompt_string)
    if self.input_stream:
      response = self.input_stream.readline().strip().lower()
      if response not in ('y', 'yes'):
        self.fail()
    else:
      self.fail()

  def fail(self):
    self.error_count += 1

  def should_continue(self):
    return not self.error_count

  def write(self, s):
    self.written_output.append(s)
    if self.output_stream:
      self.output_stream.write(s)

  def getvalue(self):
    return ''.join(self.written_output)


113 114 115 116 117 118
class OutputApi(object):
  """This class (more like a module) gets passed to presubmit scripts so that
  they can specify various types of results.
  """
  class PresubmitResult(object):
    """Base class for result objects."""
119 120
    fatal = False
    should_prompt = False
121 122 123 124 125 126 127 128 129 130 131 132 133

    def __init__(self, message, items=None, long_text=''):
      """
      message: A short one-line message to indicate errors.
      items: A list of short strings to indicate where errors occurred.
      long_text: multi-line text output, e.g. from another tool
      """
      self._message = message
      self._items = []
      if items:
        self._items = items
      self._long_text = long_text.rstrip()

134 135 136
    def handle(self, output):
      output.write(self._message)
      output.write('\n')
137 138 139
      for index, item in enumerate(self._items):
        output.write('  ')
        # Write separately in case it's unicode.
140
        output.write(str(item))
141 142 143
        if index < len(self._items) - 1:
          output.write(' \\')
        output.write('\n')
144
      if self._long_text:
145 146 147 148
        output.write('\n***************\n')
        # Write separately in case it's unicode.
        output.write(self._long_text)
        output.write('\n***************\n')
149 150
      if self.fatal:
        output.fail()
151

152 153 154 155 156
  class PresubmitAddReviewers(PresubmitResult):
    """Add some suggested reviewers to the change."""
    def __init__(self, reviewers):
      super(OutputApi.PresubmitAddReviewers, self).__init__('')
      self.reviewers = reviewers
157

158 159
    def handle(self, output):
      output.reviewers.extend(self.reviewers)
160

161 162
  class PresubmitError(PresubmitResult):
    """A hard presubmit error."""
163
    fatal = True
164 165 166

  class PresubmitPromptWarning(PresubmitResult):
    """An warning that prompts the user if they want to continue."""
167
    should_prompt = True
168 169 170 171 172 173 174 175

  class PresubmitNotifyResult(PresubmitResult):
    """Just print something to the screen -- but it's not even a warning."""
    pass

  class MailTextResult(PresubmitResult):
    """A warning that should be included in the review request email."""
    def __init__(self, *args, **kwargs):
176
      super(OutputApi.MailTextResult, self).__init__()
177
      raise NotImplementedError()
178 179 180 181 182 183


class InputApi(object):
  """An instance of this object is passed to presubmit scripts so they can
  know stuff about the change they're looking at.
  """
184 185
  # Method could be a function
  # pylint: disable=R0201
186

187 188
  # File extensions that are considered source files from a style guide
  # perspective. Don't modify this list from a presubmit script!
189 190 191 192
  #
  # Files without an extension aren't included in the list. If you want to
  # filter them as source files, add r"(^|.*?[\\\/])[^.]+$" to the white list.
  # Note that ALL CAPS files are black listed in DEFAULT_BLACK_LIST below.
193 194
  DEFAULT_WHITE_LIST = (
      # C++ and friends
195 196
      r".+\.c$", r".+\.cc$", r".+\.cpp$", r".+\.h$", r".+\.m$", r".+\.mm$",
      r".+\.inl$", r".+\.asm$", r".+\.hxx$", r".+\.hpp$", r".+\.s$", r".+\.S$",
197
      # Scripts
198
      r".+\.js$", r".+\.py$", r".+\.sh$", r".+\.rb$", r".+\.pl$", r".+\.pm$",
199
      # Other
200
      r".+\.java$", r".+\.mk$", r".+\.am$",
201 202 203 204 205 206 207 208 209 210 211 212 213
  )

  # Path regexp that should be excluded from being considered containing source
  # files. Don't modify this list from a presubmit script!
  DEFAULT_BLACK_LIST = (
      r".*\bexperimental[\\\/].*",
      r".*\bthird_party[\\\/].*",
      # Output directories (just in case)
      r".*\bDebug[\\\/].*",
      r".*\bRelease[\\\/].*",
      r".*\bxcodebuild[\\\/].*",
      r".*\bsconsbuild[\\\/].*",
      # All caps files like README and LICENCE.
214
      r".*\b[A-Z0-9_]{2,}$",
215
      # SCM (can happen in dual SCM configuration). (Slightly over aggressive)
216 217
      r"(|.*[\\\/])\.git[\\\/].*",
      r"(|.*[\\\/])\.svn[\\\/].*",
218 219
  )

220
  def __init__(self, change, presubmit_path, is_committing,
221
      rietveld_obj, verbose):
222 223 224
    """Builds an InputApi object.

    Args:
225
      change: A presubmit.Change object.
226
      presubmit_path: The path to the presubmit script being processed.
227
      is_committing: True if the change is about to be committed.
228
      rietveld_obj: rietveld.Rietveld client object
229
    """
230 231
    # Version number of the presubmit_support script.
    self.version = [int(x) for x in __version__.split('.')]
232
    self.change = change
233
    self.is_committing = is_committing
234
    self.rietveld = rietveld_obj
235 236 237
    # TBD
    self.host_url = 'http://codereview.chromium.org'
    if self.rietveld:
238
      self.host_url = self.rietveld.url
239 240 241 242 243 244

    # We expose various modules and functions as attributes of the input_api
    # so that presubmit scripts don't have to import them.
    self.basename = os.path.basename
    self.cPickle = cPickle
    self.cStringIO = cStringIO
245
    self.json = json
246
    self.logging = logging.getLogger('PRESUBMIT')
247 248
    self.os_listdir = os.listdir
    self.os_walk = os.walk
249 250 251 252 253 254
    self.os_path = os.path
    self.pickle = pickle
    self.marshal = marshal
    self.re = re
    self.subprocess = subprocess
    self.tempfile = tempfile
255
    self.time = time
256
    self.traceback = traceback
257
    self.unittest = unittest
258 259
    self.urllib2 = urllib2

260 261 262 263
    # To easily fork python.
    self.python_executable = sys.executable
    self.environ = os.environ

264 265 266 267
    # InputApi.platform is the platform you're currently running on.
    self.platform = sys.platform

    # The local path of the currently-being-processed presubmit script.
268
    self._current_presubmit_path = os.path.dirname(presubmit_path)
269 270 271 272

    # We carry the canned checks so presubmit scripts can easily use them.
    self.canned_checks = presubmit_canned_checks

273 274 275 276
    # TODO(dpranke): figure out a list of all approved owners for a repo
    # in order to be able to handle wildcard OWNERS files?
    self.owners_db = owners.Database(change.RepositoryRoot(),
        fopen=file, os_path=self.os_path)
277
    self.verbose = verbose
278

279 280 281 282 283 284 285 286
  def PresubmitLocalPath(self):
    """Returns the local path of the presubmit script currently being run.

    This is useful if you don't want to hard-code absolute paths in the
    presubmit script.  For example, It can be used to find another file
    relative to the PRESUBMIT.py script, so the whole tree can be branched and
    the presubmit script still works, without editing its content.
    """
287
    return self._current_presubmit_path
288

289
  def DepotToLocalPath(self, depot_path):
290 291 292 293 294 295 296 297 298 299 300
    """Translate a depot path to a local path (relative to client root).

    Args:
      Depot path as a string.

    Returns:
      The local path of the depot path under the user's current client, or None
      if the file is not mapped.

      Remember to check for the None case and show an appropriate error!
    """
301
    local_path = scm.SVN.CaptureInfo(depot_path).get('Path')
302
    if local_path:
303 304
      return local_path

305
  def LocalToDepotPath(self, local_path):
306 307 308 309 310 311 312 313
    """Translate a local path to a depot path.

    Args:
      Local path (relative to current directory, or absolute) as a string.

    Returns:
      The depot path (SVN URL) of the file if mapped, otherwise None.
    """
314
    depot_path = scm.SVN.CaptureInfo(local_path).get('URL')
315
    if depot_path:
316 317
      return depot_path

318 319
  def AffectedFiles(self, include_dirs=False, include_deletes=True,
                    file_filter=None):
320 321 322 323
    """Same as input_api.change.AffectedFiles() except only lists files
    (and optionally directories) in the same directory as the current presubmit
    script, or subdirectories thereof.
    """
324
    dir_with_slash = normpath("%s/" % self.PresubmitLocalPath())
325 326
    if len(dir_with_slash) == 1:
      dir_with_slash = ''
327

328 329
    return filter(
        lambda x: normpath(x.AbsoluteLocalPath()).startswith(dir_with_slash),
330
        self.change.AffectedFiles(include_dirs, include_deletes, file_filter))
331 332 333 334 335 336 337 338 339 340 341 342 343

  def LocalPaths(self, include_dirs=False):
    """Returns local paths of input_api.AffectedFiles()."""
    return [af.LocalPath() for af in self.AffectedFiles(include_dirs)]

  def AbsoluteLocalPaths(self, include_dirs=False):
    """Returns absolute local paths of input_api.AffectedFiles()."""
    return [af.AbsoluteLocalPath() for af in self.AffectedFiles(include_dirs)]

  def ServerPaths(self, include_dirs=False):
    """Returns server paths of input_api.AffectedFiles()."""
    return [af.ServerPath() for af in self.AffectedFiles(include_dirs)]

344
  def AffectedTextFiles(self, include_deletes=None):
345 346 347 348
    """Same as input_api.change.AffectedTextFiles() except only lists files
    in the same directory as the current presubmit script, or subdirectories
    thereof.
    """
349
    if include_deletes is not None:
350 351 352 353
      warn("AffectedTextFiles(include_deletes=%s)"
               " is deprecated and ignored" % str(include_deletes),
           category=DeprecationWarning,
           stacklevel=2)
354 355
    return filter(lambda x: x.IsTextFile(),
                  self.AffectedFiles(include_dirs=False, include_deletes=False))
356

357 358 359 360 361 362 363 364 365 366 367
  def FilterSourceFile(self, affected_file, white_list=None, black_list=None):
    """Filters out files that aren't considered "source file".

    If white_list or black_list is None, InputApi.DEFAULT_WHITE_LIST
    and InputApi.DEFAULT_BLACK_LIST is used respectively.

    The lists will be compiled as regular expression and
    AffectedFile.LocalPath() needs to pass both list.

    Note: Copy-paste this function to suit your needs or use a lambda function.
    """
368
    def Find(affected_file, items):
369
      local_path = affected_file.LocalPath()
370
      for item in items:
371 372
        if self.re.match(item, local_path):
          logging.debug("%s matched %s" % (item, local_path))
373 374 375 376 377 378 379 380 381 382 383 384 385 386 387
          return True
      return False
    return (Find(affected_file, white_list or self.DEFAULT_WHITE_LIST) and
            not Find(affected_file, black_list or self.DEFAULT_BLACK_LIST))

  def AffectedSourceFiles(self, source_file):
    """Filter the list of AffectedTextFiles by the function source_file.

    If source_file is None, InputApi.FilterSourceFile() is used.
    """
    if not source_file:
      source_file = self.FilterSourceFile
    return filter(source_file, self.AffectedTextFiles())

  def RightHandSideLines(self, source_file_filter=None):
388 389 390 391 392 393 394 395 396 397 398 399 400
    """An iterator over all text lines in "new" version of changed files.

    Only lists lines from new or modified text files in the change that are
    contained by the directory of the currently executing presubmit script.

    This is useful for doing line-by-line regex checks, like checking for
    trailing whitespace.

    Yields:
      a 3 tuple:
        the AffectedFile instance of the current file;
        integer line number (1-based); and
        the contents of the line as a string.
401

402
    Note: The carriage return (LF or CR) is stripped off.
403
    """
404
    files = self.AffectedSourceFiles(source_file_filter)
405
    return _RightHandSideLinesImpl(files)
406

407
  def ReadFile(self, file_item, mode='r'):
408
    """Reads an arbitrary file.
409

410 411
    Deny reading anything outside the repository.
    """
412 413 414
    if isinstance(file_item, AffectedFile):
      file_item = file_item.AbsoluteLocalPath()
    if not file_item.startswith(self.change.RepositoryRoot()):
415
      raise IOError('Access outside the repository root is denied.')
416
    return gclient_utils.FileRead(file_item, mode)
417

418 419 420 421 422
  @property
  def tbr(self):
    """Returns if a change is TBR'ed."""
    return 'TBR' in self.change.tags

423 424 425

class AffectedFile(object):
  """Representation of a file in a change."""
426 427
  # Method could be a function
  # pylint: disable=R0201
428
  def __init__(self, path, action, repository_root=''):
429 430
    self._path = path
    self._action = action
431
    self._local_root = repository_root
432 433
    self._is_directory = None
    self._properties = {}
434 435
    self._cached_changed_contents = None
    self._cached_new_contents = None
436
    logging.debug('%s(%s)' % (self.__class__.__name__, self._path))
437 438 439 440 441 442

  def ServerPath(self):
    """Returns a path string that identifies the file in the SCM system.

    Returns the empty string if the file does not exist in SCM.
    """
443
    return ""
444 445 446 447

  def LocalPath(self):
    """Returns the path of this file on the local disk relative to client root.
    """
448
    return normpath(self._path)
449 450 451 452

  def AbsoluteLocalPath(self):
    """Returns the absolute path of this file on the local disk.
    """
453
    return os.path.abspath(os.path.join(self._local_root, self.LocalPath()))
454 455 456

  def IsDirectory(self):
    """Returns true if this object is a directory."""
457 458 459 460 461
    if self._is_directory is None:
      path = self.AbsoluteLocalPath()
      self._is_directory = (os.path.exists(path) and
                            os.path.isdir(path))
    return self._is_directory
462 463 464

  def Action(self):
    """Returns the action on this opened file, e.g. A, M, D, etc."""
465 466
    # TODO(maruel): Somewhat crappy, Could be "A" or "A  +" for svn but
    # different for other SCM.
467
    return self._action
468

469 470 471 472
  def Property(self, property_name):
    """Returns the specified SCM property of this file, or None if no such
    property.
    """
473
    return self._properties.get(property_name, None)
474

475
  def IsTextFile(self):
476
    """Returns True if the file is a text file and not a binary file.
477

478
    Deleted files are not text file."""
479 480
    raise NotImplementedError()  # Implement when needed

481 482 483 484 485 486 487
  def NewContents(self):
    """Returns an iterator over the lines in the new version of file.

    The new version is the file in the user's workspace, i.e. the "right hand
    side".

    Contents will be empty if the file is a directory or does not exist.
488
    Note: The carriage returns (LF or CR) are stripped off.
489
    """
490 491 492 493 494 495 496 497 498
    if self._cached_new_contents is None:
      self._cached_new_contents = []
      if not self.IsDirectory():
        try:
          self._cached_new_contents = gclient_utils.FileRead(
              self.AbsoluteLocalPath(), 'rU').splitlines()
        except IOError:
          pass  # File not found?  That's fine; maybe it was deleted.
    return self._cached_new_contents[:]
499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515

  def OldContents(self):
    """Returns an iterator over the lines in the old version of file.

    The old version is the file in depot, i.e. the "left hand side".
    """
    raise NotImplementedError()  # Implement when needed

  def OldFileTempPath(self):
    """Returns the path on local disk where the old contents resides.

    The old version is the file in depot, i.e. the "left hand side".
    This is a read-only cached copy of the old contents. *DO NOT* try to
    modify this file.
    """
    raise NotImplementedError()  # Implement if/when needed.

516 517 518 519 520 521 522 523
  def ChangedContents(self):
    """Returns a list of tuples (line number, line text) of all new lines.

     This relies on the scm diff output describing each changed code section
     with a line of the form

     ^@@ <old line num>,<old size> <new line num>,<new size> @@$
    """
524 525 526
    if self._cached_changed_contents is not None:
      return self._cached_changed_contents[:]
    self._cached_changed_contents = []
527 528 529 530 531 532 533 534 535 536 537
    line_num = 0

    if self.IsDirectory():
      return []

    for line in self.GenerateScmDiff().splitlines():
      m = re.match(r'^@@ [0-9\,\+\-]+ \+([0-9]+)\,[0-9]+ @@', line)
      if m:
        line_num = int(m.groups(1)[0])
        continue
      if line.startswith('+') and not line.startswith('++'):
538
        self._cached_changed_contents.append((line_num, line[1:]))
539 540
      if not line.startswith('-'):
        line_num += 1
541
    return self._cached_changed_contents[:]
542

543 544 545
  def __str__(self):
    return self.LocalPath()

546 547
  def GenerateScmDiff(self):
    raise NotImplementedError()  # Implemented in derived classes.
548

549

550 551
class SvnAffectedFile(AffectedFile):
  """Representation of a file in a change out of a Subversion checkout."""
552 553
  # Method 'NNN' is abstract in class 'NNN' but is not overridden
  # pylint: disable=W0223
554

555 556 557 558 559
  def __init__(self, *args, **kwargs):
    AffectedFile.__init__(self, *args, **kwargs)
    self._server_path = None
    self._is_text_file = None

560
  def ServerPath(self):
561
    if self._server_path is None:
562
      self._server_path = scm.SVN.CaptureInfo(
563
          self.AbsoluteLocalPath()).get('URL', '')
564
    return self._server_path
565 566

  def IsDirectory(self):
567 568
    if self._is_directory is None:
      path = self.AbsoluteLocalPath()
569 570 571
      if os.path.exists(path):
        # Retrieve directly from the file system; it is much faster than
        # querying subversion, especially on Windows.
572
        self._is_directory = os.path.isdir(path)
573
      else:
574
        self._is_directory = scm.SVN.CaptureInfo(
575
            path).get('Node Kind') in ('dir', 'directory')
576
    return self._is_directory
577 578

  def Property(self, property_name):
579
    if not property_name in self._properties:
580
      self._properties[property_name] = scm.SVN.GetFileProperty(
581
          self.AbsoluteLocalPath(), property_name).rstrip()
582
    return self._properties[property_name]
583

584
  def IsTextFile(self):
585 586 587 588 589 590 591
    if self._is_text_file is None:
      if self.Action() == 'D':
        # A deleted file is not a text file.
        self._is_text_file = False
      elif self.IsDirectory():
        self._is_text_file = False
      else:
592 593
        mime_type = scm.SVN.GetFileProperty(self.AbsoluteLocalPath(),
                                            'svn:mime-type')
594 595
        self._is_text_file = (not mime_type or mime_type.startswith('text/'))
    return self._is_text_file
596

597
  def GenerateScmDiff(self):
598 599
    return scm.SVN.GenerateDiff([self.AbsoluteLocalPath()])

600

601 602
class GitAffectedFile(AffectedFile):
  """Representation of a file in a change out of a git checkout."""
603 604
  # Method 'NNN' is abstract in class 'NNN' but is not overridden
  # pylint: disable=W0223
605 606 607 608 609 610 611 612

  def __init__(self, *args, **kwargs):
    AffectedFile.__init__(self, *args, **kwargs)
    self._server_path = None
    self._is_text_file = None

  def ServerPath(self):
    if self._server_path is None:
613
      raise NotImplementedError('TODO(maruel) Implement.')
614 615 616 617 618 619 620 621 622 623 624 625 626 627 628
    return self._server_path

  def IsDirectory(self):
    if self._is_directory is None:
      path = self.AbsoluteLocalPath()
      if os.path.exists(path):
        # Retrieve directly from the file system; it is much faster than
        # querying subversion, especially on Windows.
        self._is_directory = os.path.isdir(path)
      else:
        self._is_directory = False
    return self._is_directory

  def Property(self, property_name):
    if not property_name in self._properties:
629
      raise NotImplementedError('TODO(maruel) Implement.')
630 631 632 633 634 635 636 637 638 639 640 641 642
    return self._properties[property_name]

  def IsTextFile(self):
    if self._is_text_file is None:
      if self.Action() == 'D':
        # A deleted file is not a text file.
        self._is_text_file = False
      elif self.IsDirectory():
        self._is_text_file = False
      else:
        self._is_text_file = os.path.isfile(self.AbsoluteLocalPath())
    return self._is_text_file

643 644
  def GenerateScmDiff(self):
    return scm.GIT.GenerateDiff(self._local_root, files=[self.LocalPath(),])
645

646

647
class Change(object):
648 649 650 651
  """Describe a change.

  Used directly by the presubmit scripts to query the current change being
  tested.
652

653 654 655 656 657
  Instance members:
    tags: Dictionnary of KEY=VALUE pairs found in the change description.
    self.KEY: equivalent to tags['KEY']
  """

658 659
  _AFFECTED_FILES = AffectedFile

660
  # Matches key/value (or "tag") lines in changelist descriptions.
661
  _TAG_LINE_RE = re.compile(
662
      '^\s*(?P<key>[A-Z][A-Z_0-9]*)\s*=\s*(?P<value>.*?)\s*$')
663
  scm = ''
664

665 666
  def __init__(
      self, name, description, local_root, files, issue, patchset, author):
667 668 669 670
    if files is None:
      files = []
    self._name = name
    self._full_description = description
671 672
    # Convert root into an absolute path.
    self._local_root = os.path.abspath(local_root)
673 674
    self.issue = issue
    self.patchset = patchset
675
    self.author_email = author
676 677 678

    # From the description text, build up a dictionary of key/value pairs
    # plus the description minus all key/value or "tag" lines.
679
    description_without_tags = []
680
    self.tags = {}
maruel@chromium.org's avatar
maruel@chromium.org committed
681
    for line in self._full_description.splitlines():
682
      m = self._TAG_LINE_RE.match(line)
683 684 685
      if m:
        self.tags[m.group('key')] = m.group('value')
      else:
686
        description_without_tags.append(line)
687 688

    # Change back to text and remove whitespace at end.
689 690
    self._description_without_tags = (
        '\n'.join(description_without_tags).rstrip())
691

692
    self._affected_files = [
693 694
        self._AFFECTED_FILES(info[1], info[0].strip(), self._local_root)
        for info in files
695
    ]
696

697
  def Name(self):
698
    """Returns the change name."""
699
    return self._name
700 701 702 703 704 705 706 707

  def DescriptionText(self):
    """Returns the user-entered changelist description, minus tags.

    Any line in the user-provided description starting with e.g. "FOO="
    (whitespace permitted before and around) is considered a tag line.  Such
    lines are stripped out of the description this function returns.
    """
708
    return self._description_without_tags
709 710 711

  def FullDescriptionText(self):
    """Returns the complete changelist description including tags."""
712
    return self._full_description
713 714

  def RepositoryRoot(self):
715 716 717
    """Returns the repository (checkout) root directory for this change,
    as an absolute path.
    """
718
    return self._local_root
719 720

  def __getattr__(self, attr):
721 722 723
    """Return tags directly as attributes on the object."""
    if not re.match(r"^[A-Z_]*$", attr):
      raise AttributeError(self, attr)
724
    return self.tags.get(attr)
725

726 727
  def AffectedFiles(self, include_dirs=False, include_deletes=True,
                    file_filter=None):
728 729 730 731 732
    """Returns a list of AffectedFile instances for all files in the change.

    Args:
      include_deletes: If false, deleted files will be filtered out.
      include_dirs: True to include directories in the list
733
      file_filter: An additional filter to apply.
734 735 736 737 738

    Returns:
      [AffectedFile(path, action), AffectedFile(path, action)]
    """
    if include_dirs:
739
      affected = self._affected_files
740
    else:
741
      affected = filter(lambda x: not x.IsDirectory(), self._affected_files)
742

743 744
    affected = filter(file_filter, affected)

745 746 747 748 749
    if include_deletes:
      return affected
    else:
      return filter(lambda x: x.Action() != 'D', affected)

750 751 752
  def AffectedTextFiles(self, include_deletes=None):
    """Return a list of the existing text files in a change."""
    if include_deletes is not None:
753 754 755 756
      warn("AffectedTextFiles(include_deletes=%s)"
               " is deprecated and ignored" % str(include_deletes),
           category=DeprecationWarning,
           stacklevel=2)
757 758
    return filter(lambda x: x.IsTextFile(),
                  self.AffectedFiles(include_dirs=False, include_deletes=False))
759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785

  def LocalPaths(self, include_dirs=False):
    """Convenience function."""
    return [af.LocalPath() for af in self.AffectedFiles(include_dirs)]

  def AbsoluteLocalPaths(self, include_dirs=False):
    """Convenience function."""
    return [af.AbsoluteLocalPath() for af in self.AffectedFiles(include_dirs)]

  def ServerPaths(self, include_dirs=False):
    """Convenience function."""
    return [af.ServerPath() for af in self.AffectedFiles(include_dirs)]

  def RightHandSideLines(self):
    """An iterator over all text lines in "new" version of changed files.

    Lists lines from new or modified text files in the change.

    This is useful for doing line-by-line regex checks, like checking for
    trailing whitespace.

    Yields:
      a 3 tuple:
        the AffectedFile instance of the current file;
        integer line number (1-based); and
        the contents of the line as a string.
    """
786 787 788
    return _RightHandSideLinesImpl(
        x for x in self.AffectedFiles(include_deletes=False)
        if x.IsTextFile())
789 790


791 792
class SvnChange(Change):
  _AFFECTED_FILES = SvnAffectedFile
793 794
  scm = 'svn'
  _changelists = None
795 796 797 798 799 800

  def _GetChangeLists(self):
    """Get all change lists."""
    if self._changelists == None:
      previous_cwd = os.getcwd()
      os.chdir(self.RepositoryRoot())
801 802
      # Need to import here to avoid circular dependency.
      import gcl
803 804 805
      self._changelists = gcl.GetModifiedFiles()
      os.chdir(previous_cwd)
    return self._changelists
806 807 808

  def GetAllModifiedFiles(self):
    """Get all modified files."""
809
    changelists = self._GetChangeLists()
810 811
    all_modified_files = []
    for cl in changelists.values():
812 813
      all_modified_files.extend(
          [os.path.join(self.RepositoryRoot(), f[1]) for f in cl])
814 815 816 817
    return all_modified_files

  def GetModifiedFiles(self):
    """Get modified files in the current CL."""
818 819 820
    changelists = self._GetChangeLists()
    return [os.path.join(self.RepositoryRoot(), f[1])
            for f in changelists[self.Name()]]
821

822

823 824
class GitChange(Change):
  _AFFECTED_FILES = GitAffectedFile
825
  scm = 'git'
826

827

828
def ListRelevantPresubmitFiles(files, root):
829 830
  """Finds all presubmit files that apply to a given set of source files.

831 832 833
  If inherit-review-settings-ok is present right under root, looks for
  PRESUBMIT.py in directories enclosing root.

834 835
  Args:
    files: An iterable container containing file paths.
836
    root: Path where to stop searching.
837 838

  Return:
839
    List of absolute paths of the existing PRESUBMIT.py scripts.
840
  """
841 842 843 844 845 846 847 848 849 850 851 852 853 854
  files = [normpath(os.path.join(root, f)) for f in files]

  # List all the individual directories containing files.
  directories = set([os.path.dirname(f) for f in files])

  # Ignore root if inherit-review-settings-ok is present.
  if os.path.isfile(os.path.join(root, 'inherit-review-settings-ok')):
    root = None

  # Collect all unique directories that may contain PRESUBMIT.py.
  candidates = set()
  for directory in directories:
    while True:
      if directory in candidates:
855
        break
856 857
      candidates.add(directory)
      if directory == root:
858
        break
859 860 861 862 863 864 865 866 867 868 869 870 871
      parent_dir = os.path.dirname(directory)
      if parent_dir == directory:
        # We hit the system root directory.
        break
      directory = parent_dir

  # Look for PRESUBMIT.py in all candidate directories.
  results = []
  for directory in sorted(list(candidates)):
    p = os.path.join(directory, 'PRESUBMIT.py')
    if os.path.isfile(p):
      results.append(p)

872
  logging.debug('Presubmit files: %s' % ','.join(results))
873
  return results
874 875


876
class GetTrySlavesExecuter(object):
877
  @staticmethod
878
  def ExecPresubmitScript(script_text, presubmit_path, project):
879 880 881 882
    """Executes GetPreferredTrySlaves() from a single presubmit script.

    Args:
      script_text: The text of the presubmit script.
883 884
      presubmit_path: Project script to run.
      project: Project name to pass to presubmit script for bot selection.
885 886 887 888 889

    Return:
      A list of try slaves.
    """
    context = {}
890 891 892 893
    try:
      exec script_text in context
    except Exception, e:
      raise PresubmitFailure('"%s" had an exception.\n%s' % (presubmit_path, e))
894 895 896

    function_name = 'GetPreferredTrySlaves'
    if function_name in context:
897 898 899 900
      try:
        result = eval(function_name + '(' + repr(project) + ')', context)
      except TypeError:
        result = eval(function_name + '()', context)
901
      if not isinstance(result, types.ListType):
902
        raise PresubmitFailure(
903 904 905 906
            'Presubmit functions must return a list, got a %s instead: %s' %
            (type(result), str(result)))
      for item in result:
        if not isinstance(item, basestring):
907
          raise PresubmitFailure('All try slaves names must be strings.')
908
        if item != item.strip():
909 910
          raise PresubmitFailure(
              'Try slave names cannot start/end with whitespace')
911 912 913 914 915 916 917 918
    else:
      result = []
    return result


def DoGetTrySlaves(changed_files,
                   repository_root,
                   default_presubmit,
919
                   project,
920 921 922 923 924 925 926 927
                   verbose,
                   output_stream):
  """Get the list of try servers from the presubmit scripts.

  Args:
    changed_files: List of modified files.
    repository_root: The repository root.
    default_presubmit: A default presubmit script to execute in any case.
928
    project: Optional name of a project used in selecting trybots.
929 930 931 932 933 934 935 936 937 938 939 940 941 942
    verbose: Prints debug info.
    output_stream: A stream to write debug output to.

  Return:
    List of try slaves
  """
  presubmit_files = ListRelevantPresubmitFiles(changed_files, repository_root)
  if not presubmit_files and verbose:
    output_stream.write("Warning, no presubmit.py found.\n")
  results = []
  executer = GetTrySlavesExecuter()
  if default_presubmit:
    if verbose:
      output_stream.write("Running default presubmit script.\n")
943
    fake_path = os.path.join(repository_root, 'PRESUBMIT.py')
944 945
    results += executer.ExecPresubmitScript(
        default_presubmit, fake_path, project)
946 947 948 949 950
  for filename in presubmit_files:
    filename = os.path.abspath(filename)
    if verbose:
      output_stream.write("Running %s\n" % filename)
    # Accept CRLF presubmit script.
951
    presubmit_script = gclient_utils.FileRead(filename, 'rU')
952 953
    results += executer.ExecPresubmitScript(
        presubmit_script, filename, project)
954 955 956 957 958 959 960 961

  slaves = list(set(results))
  if slaves and verbose:
    output_stream.write(', '.join(slaves))
    output_stream.write('\n')
  return slaves


962
class PresubmitExecuter(object):
963
  def __init__(self, change, committing, rietveld_obj, verbose):
964 965
    """
    Args:
966
      change: The Change object.
967
      committing: True if 'gcl commit' is running, False if 'gcl upload' is.
968
      rietveld_obj: rietveld.Rietveld client object.
969
    """
970
    self.change = change
971
    self.committing = committing
972
    self.rietveld = rietveld_obj
973
    self.verbose = verbose
974 975 976 977 978 979 980 981 982 983 984 985

  def ExecPresubmitScript(self, script_text, presubmit_path):
    """Executes a single presubmit script.

    Args:
      script_text: The text of the presubmit script.
      presubmit_path: The path to the presubmit file (this will be reported via
        input_api.PresubmitLocalPath()).

    Return:
      A list of result objects, empty if no problems.
    """
986 987 988 989 990 991

    # Change to the presubmit file's directory to support local imports.
    main_path = os.getcwd()
    os.chdir(os.path.dirname(presubmit_path))

    # Load the presubmit script into context.
992
    input_api = InputApi(self.change, presubmit_path, self.committing,
993
                         self.rietveld, self.verbose)
994
    context = {}
995 996 997 998
    try:
      exec script_text in context
    except Exception, e:
      raise PresubmitFailure('"%s" had an exception.\n%s' % (presubmit_path, e))
999 1000 1001 1002 1003 1004 1005 1006 1007

    # These function names must change if we make substantial changes to
    # the presubmit API that are not backwards compatible.
    if self.committing:
      function_name = 'CheckChangeOnCommit'
    else:
      function_name = 'CheckChangeOnUpload'
    if function_name in context:
      context['__args'] = (input_api, OutputApi())
1008
      logging.debug('Running %s in %s' % (function_name, presubmit_path))
1009
      result = eval(function_name + '(*__args)', context)
1010
      logging.debug('Running %s done.' % function_name)
1011 1012
      if not (isinstance(result, types.TupleType) or
              isinstance(result, types.ListType)):
1013
        raise PresubmitFailure(
1014 1015 1016
          'Presubmit functions must return a tuple or list')
      for item in result:
        if not isinstance(item, OutputApi.PresubmitResult):
1017
          raise PresubmitFailure(
1018 1019 1020 1021 1022
            'All presubmit results must be of types derived from '
            'output_api.PresubmitResult')
    else:
      result = ()  # no error since the script doesn't care about current event.

1023 1024
    # Return the process to the original working directory.
    os.chdir(main_path)
1025 1026
    return result

1027

1028
def DoPresubmitChecks(change,
1029 1030 1031
                      committing,
                      verbose,
                      output_stream,
1032
                      input_stream,
1033
                      default_presubmit,
1034
                      may_prompt,
1035
                      rietveld_obj):
1036 1037 1038 1039 1040 1041 1042 1043 1044 1045
  """Runs all presubmit checks that apply to the files in the change.

  This finds all PRESUBMIT.py files in directories enclosing the files in the
  change (up to the repository root) and calls the relevant entrypoint function
  depending on whether the change is being committed or uploaded.

  Prints errors, warnings and notifications.  Prompts the user for warnings
  when needed.

  Args:
1046
    change: The Change object.
1047 1048 1049 1050
    committing: True if 'gcl commit' is running, False if 'gcl upload' is.
    verbose: Prints debug info.
    output_stream: A stream to write output from presubmit tests to.
    input_stream: A stream to read input from the user.
1051
    default_presubmit: A default presubmit script to execute in any case.
1052
    may_prompt: Enable (y/n) questions on warning or error.
1053
    rietveld_obj: rietveld.Rietveld object.
1054

1055 1056 1057 1058
  Warning:
    If may_prompt is true, output_stream SHOULD be sys.stdout and input_stream
    SHOULD be sys.stdin.

1059
  Return:
1060 1061
    A PresubmitOutput object. Use output.should_continue() to figure out
    if there were errors or warnings and the caller should abort.
1062
  """
1063 1064 1065 1066 1067
  old_environ = os.environ
  try:
    # Make sure python subprocesses won't generate .pyc files.
    os.environ = os.environ.copy()
    os.environ['PYTHONDONTWRITEBYTECODE'] = '1'
1068

1069 1070 1071
    output = PresubmitOutput(input_stream, output_stream)
    if committing:
      output.write("Running presubmit commit checks ...\n")
1072
    else:
1073 1074 1075 1076 1077 1078 1079
      output.write("Running presubmit upload checks ...\n")
    start_time = time.time()
    presubmit_files = ListRelevantPresubmitFiles(
        change.AbsoluteLocalPaths(True), change.RepositoryRoot())
    if not presubmit_files and verbose:
      output.write("Warning, no presubmit.py found.\n")
    results = []
1080
    executer = PresubmitExecuter(change, committing, rietveld_obj, verbose)
1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103
    if default_presubmit:
      if verbose:
        output.write("Running default presubmit script.\n")
      fake_path = os.path.join(change.RepositoryRoot(), 'PRESUBMIT.py')
      results += executer.ExecPresubmitScript(default_presubmit, fake_path)
    for filename in presubmit_files:
      filename = os.path.abspath(filename)
      if verbose:
        output.write("Running %s\n" % filename)
      # Accept CRLF presubmit script.
      presubmit_script = gclient_utils.FileRead(filename, 'rU')
      results += executer.ExecPresubmitScript(presubmit_script, filename)

    errors = []
    notifications = []
    warnings = []
    for result in results:
      if result.fatal:
        errors.append(result)
      elif result.should_prompt:
        warnings.append(result)
      else:
        notifications.append(result)
1104

1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126
    output.write('\n')
    for name, items in (('Messages', notifications),
                        ('Warnings', warnings),
                        ('ERRORS', errors)):
      if items:
        output.write('** Presubmit %s **\n' % name)
        for item in items:
          item.handle(output)
          output.write('\n')

    total_time = time.time() - start_time
    if total_time > 1.0:
      output.write("Presubmit checks took %.1fs to calculate.\n\n" % total_time)

    if not errors:
      if not warnings:
        output.write('Presubmit checks passed.\n')
      elif may_prompt:
        output.prompt_yes_no('There were presubmit warnings. '
                            'Are you sure you wish to continue? (y/N): ')
      else:
        output.fail()
1127

1128 1129 1130 1131 1132 1133 1134 1135 1136
    global _ASKED_FOR_FEEDBACK
    # Ask for feedback one time out of 5.
    if (len(results) and random.randint(0, 4) == 0 and not _ASKED_FOR_FEEDBACK):
      output.write("Was the presubmit check useful? Please send feedback "
                  "& hate mail to maruel@chromium.org!\n")
      _ASKED_FOR_FEEDBACK = True
    return output
  finally:
    os.environ = old_environ
1137 1138 1139 1140


def ScanSubDirs(mask, recursive):
  if not recursive:
1141
    return [x for x in glob.glob(mask) if '.svn' not in x and '.git' not in x]
1142 1143 1144 1145 1146
  else:
    results = []
    for root, dirs, files in os.walk('.'):
      if '.svn' in dirs:
        dirs.remove('.svn')
1147 1148
      if '.git' in dirs:
        dirs.remove('.git')
1149 1150 1151 1152 1153 1154 1155
      for name in files:
        if fnmatch.fnmatch(name, mask):
          results.append(os.path.join(root, name))
    return results


def ParseFiles(args, recursive):
1156
  logging.debug('Searching for %s' % args)
1157 1158
  files = []
  for arg in args:
1159
    files.extend([('M', f) for f in ScanSubDirs(arg, recursive)])
1160 1161 1162
  return files


1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185
def load_files(options, args):
  """Tries to determine the SCM."""
  change_scm = scm.determine_scm(options.root)
  files = []
  if change_scm == 'svn':
    change_class = SvnChange
    status_fn = scm.SVN.CaptureStatus
  elif change_scm == 'git':
    change_class = GitChange
    status_fn = scm.GIT.CaptureStatus
  else:
    logging.info('Doesn\'t seem under source control. Got %d files' % len(args))
    if not args:
      return None, None
    change_class = Change
  if args:
    files = ParseFiles(args, options.recursive)
  else:
    # Grab modified files.
    files = status_fn([options.root])
  return change_class, files


1186
def Main(argv):
1187
  parser = optparse.OptionParser(usage="%prog [options] <files...>",
1188
                                 version="%prog " + str(__version__))
1189
  parser.add_option("-c", "--commit", action="store_true", default=False,
1190
                   help="Use commit instead of upload checks")
1191 1192
  parser.add_option("-u", "--upload", action="store_false", dest='commit',
                   help="Use upload instead of commit checks")
1193 1194
  parser.add_option("-r", "--recursive", action="store_true",
                   help="Act recursively")
1195 1196
  parser.add_option("-v", "--verbose", action="count", default=0,
                   help="Use 2 times for more debug info")
1197
  parser.add_option("--name", default='no name')
1198
  parser.add_option("--author")
1199 1200 1201
  parser.add_option("--description", default='')
  parser.add_option("--issue", type='int', default=0)
  parser.add_option("--patchset", type='int', default=0)
1202 1203 1204 1205 1206
  parser.add_option("--root", default=os.getcwd(),
                    help="Search for PRESUBMIT.py up to this directory. "
                    "If inherit-review-settings-ok is present in this "
                    "directory, parent directories up to the root file "
                    "system directories will also be searched.")
1207 1208
  parser.add_option("--default_presubmit")
  parser.add_option("--may_prompt", action='store_true', default=False)
1209 1210 1211
  parser.add_option("--rietveld_url", help=optparse.SUPPRESS_HELP)
  parser.add_option("--rietveld_email", help=optparse.SUPPRESS_HELP)
  parser.add_option("--rietveld_password", help=optparse.SUPPRESS_HELP)
1212
  options, args = parser.parse_args(argv)
1213
  if options.verbose >= 2:
1214
    logging.basicConfig(level=logging.DEBUG)
1215 1216 1217 1218
  elif options.verbose:
    logging.basicConfig(level=logging.INFO)
  else:
    logging.basicConfig(level=logging.ERROR)
1219 1220 1221
  change_class, files = load_files(options, args)
  if not change_class:
    parser.error('For unversioned directory, <files> is not optional.')
1222
  logging.info('Found %d file(s).' % len(files))
1223 1224 1225 1226 1227 1228
  rietveld_obj = None
  if options.rietveld_url:
    rietveld_obj = rietveld.Rietveld(
        options.rietveld_url,
        options.rietveld_email,
        options.rietveld_password)
1229 1230 1231 1232 1233 1234 1235
  try:
    results = DoPresubmitChecks(
        change_class(options.name,
                    options.description,
                    options.root,
                    files,
                    options.issue,
1236 1237
                    options.patchset,
                    options.author),
1238 1239 1240 1241 1242
        options.commit,
        options.verbose,
        sys.stdout,
        sys.stdin,
        options.default_presubmit,
1243
        options.may_prompt,
1244
        rietveld_obj)
1245 1246 1247 1248 1249 1250
    return not results.should_continue()
  except PresubmitFailure, e:
    print >> sys.stderr, e
    print >> sys.stderr, 'Maybe your depot_tools is out of date?'
    print >> sys.stderr, 'If all fails, contact maruel@'
    return 2
1251 1252 1253


if __name__ == '__main__':
1254
  fix_encoding.fix_encoding()
1255
  sys.exit(Main(None))