git_cl.py 229 KB
Newer Older
1
#!/usr/bin/env python
2
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
3 4 5
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

6 7
# Copyright (C) 2008 Evan Martin <martine@danga.com>

8
"""A git-command for integrating reviews on Rietveld and Gerrit."""
9

10 11
from __future__ import print_function

12
from distutils.version import LooseVersion
13
from multiprocessing.pool import ThreadPool
14
import base64
15
import collections
16
import contextlib
17
import datetime
18
import fnmatch
19
import httplib
20
import itertools
21
import json
22
import logging
23
import multiprocessing
24 25 26
import optparse
import os
import re
27
import shutil
28
import stat
29
import sys
30
import tempfile
31
import textwrap
32
import urllib
33
import urllib2
34
import urlparse
35
import uuid
36
import webbrowser
37
import zlib
38 39

try:
40
  import readline  # pylint: disable=import-error,W0611
41 42 43
except ImportError:
  pass

44
from third_party import colorama
45
from third_party import httplib2
46
from third_party import upload
47
import auth
48
import checkout
49
import clang_format
50
import dart_format
51
import setup_color
52
import fix_encoding
53
import gclient_utils
54
import gerrit_util
55
import git_cache
56
import git_common
57
import git_footers
58
import owners
59
import owners_finder
60
import presubmit_support
61
import rietveld
62
import scm
Francois Doray's avatar
Francois Doray committed
63
import split_cl
64
import subcommand
65
import subprocess2
66 67
import watchlists

68
__version__ = '2.0'
69

70
COMMIT_BOT_EMAIL = 'commit-bot@chromium.org'
71
DEFAULT_SERVER = 'https://codereview.chromium.org'
72
POSTUPSTREAM_HOOK = '.git/hooks/post-cl-land'
73
DESCRIPTION_BACKUP_FILE = '~/.git_cl_description_backup'
74 75 76 77
REFS_THAT_ALIAS_TO_OTHER_REFS = {
    'refs/remotes/origin/lkgr': 'refs/remotes/origin/master',
    'refs/remotes/origin/lkcr': 'refs/remotes/origin/master',
}
78

79 80 81 82
# Valid extensions for files we want to lint.
DEFAULT_LINT_REGEX = r"(.*\.cpp|.*\.cc|.*\.h)"
DEFAULT_LINT_IGNORE_REGEX = r"$^"

83 84 85
# Buildbucket master name prefix.
MASTER_PREFIX = 'master.'

86 87
# Shortcut since it quickly becomes redundant.
Fore = colorama.Fore
88

89 90 91
# Initialized in main()
settings = None

92 93 94
# Used by tests/git_cl_test.py to add extra logging.
# Inside the weirdly failing test, add this:
# >>> self.mock(git_cl, '_IS_BEING_TESTED', True)
95
# And scroll up to see the stack trace printed.
96 97
_IS_BEING_TESTED = False

98

99 100 101 102
def DieWithError(message, change_desc=None):
  if change_desc:
    SaveDescriptionBackup(change_desc)

103
  print(message, file=sys.stderr)
104 105 106
  sys.exit(1)


107 108 109 110 111 112 113 114 115
def SaveDescriptionBackup(change_desc):
  backup_path = os.path.expanduser(DESCRIPTION_BACKUP_FILE)
  print('\nError after CL description prompt -- saving description to %s\n' %
        backup_path)
  backup_file = open(backup_path, 'w')
  backup_file.write(change_desc.description)
  backup_file.close()


116 117 118 119 120 121
def GetNoGitPagerEnv():
  env = os.environ.copy()
  # 'cat' is a magical git string that disables pagers on all platforms.
  env['GIT_PAGER'] = 'cat'
  return env

122

123
def RunCommand(args, error_ok=False, error_message=None, shell=False, **kwargs):
124
  try:
125
    return subprocess2.check_output(args, shell=shell, **kwargs)
126 127
  except subprocess2.CalledProcessError as e:
    logging.debug('Failed running %s', args)
128
    if not error_ok:
129
      DieWithError(
130 131 132
          'Command "%s" failed.\n%s' % (
            ' '.join(args), error_message or e.stdout or ''))
    return e.stdout
133 134 135


def RunGit(args, **kwargs):
136
  """Returns stdout."""
137
  return RunCommand(['git'] + args, **kwargs)
138 139


140
def RunGitWithCode(args, suppress_stderr=False):
141
  """Returns return code and stdout."""
142 143 144 145
  if suppress_stderr:
    stderr = subprocess2.VOID
  else:
    stderr = sys.stderr
146
  try:
147 148 149 150 151 152
    (out, _), code = subprocess2.communicate(['git'] + args,
                                             env=GetNoGitPagerEnv(),
                                             stdout=subprocess2.PIPE,
                                             stderr=stderr)
    return code, out
  except subprocess2.CalledProcessError as e:
153
    logging.debug('Failed running %s', ['git'] + args)
154
    return e.returncode, e.stdout
155 156


157
def RunGitSilent(args):
158
  """Returns stdout, suppresses stderr and ignores the return code."""
159 160 161
  return RunGitWithCode(args, suppress_stderr=True)[1]


162
def IsGitVersionAtLeast(min_version):
163
  prefix = 'git version '
164
  version = RunGit(['--version']).strip()
165 166
  return (version.startswith(prefix) and
      LooseVersion(version[len(prefix):]) >= LooseVersion(min_version))
167 168


169 170 171 172 173 174 175
def BranchExists(branch):
  """Return True if specified branch exists."""
  code, _ = RunGitWithCode(['rev-parse', '--verify', branch],
                           suppress_stderr=True)
  return not code


176 177 178 179 180 181 182
def time_sleep(seconds):
  # Use this so that it can be mocked in tests without interfering with python
  # system machinery.
  import time  # Local import to discourage others from importing time globally.
  return time.sleep(seconds)


183 184 185 186 187 188 189 190
def ask_for_data(prompt):
  try:
    return raw_input(prompt)
  except KeyboardInterrupt:
    # Hide the exception.
    sys.exit(1)


191 192 193 194
def confirm_or_exit(prefix='', action='confirm'):
  """Asks user to press enter to continue or press Ctrl+C to abort."""
  if not prefix or prefix.endswith('\n'):
    mid = 'Press'
195
  elif prefix.endswith('.') or prefix.endswith('?'):
196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214
    mid = ' Press'
  elif prefix.endswith(' '):
    mid = 'press'
  else:
    mid = ' press'
  ask_for_data('%s%s Enter to %s, or Ctrl+C to abort' % (prefix, mid, action))


def ask_for_explicit_yes(prompt):
  """Returns whether user typed 'y' or 'yes' to confirm the given prompt"""
  result = ask_for_data(prompt + ' [Yes/No]: ').lower()
  while True:
    if 'yes'.startswith(result):
      return True
    if 'no'.startswith(result):
      return False
    result = ask_for_data('Please, type yes or no: ').lower()


215 216 217 218
def _git_branch_config_key(branch, key):
  """Helper method to return Git config key for a branch."""
  assert branch, 'branch name is required to set git config for it'
  return 'branch.%s.%s' % (branch, key)
219

220

221 222 223
def _git_get_branch_config_value(key, default=None, value_type=str,
                                 branch=False):
  """Returns git config value of given or current branch if any.
224

225 226 227 228 229 230 231 232 233 234
  Returns default in all other cases.
  """
  assert value_type in (int, str, bool)
  if branch is False:  # Distinguishing default arg value from None.
    branch = GetCurrentBranch()

  if not branch:
    return default

  args = ['config']
235
  if value_type == bool:
236
    args.append('--bool')
237 238
  # git config also has --int, but apparently git config suffers from integer
  # overflows (http://crbug.com/640115), so don't use it.
239 240 241 242 243 244 245 246 247
  args.append(_git_branch_config_key(branch, key))
  code, out = RunGitWithCode(args)
  if code == 0:
    value = out.strip()
    if value_type == int:
      return int(value)
    if value_type == bool:
      return bool(value.lower() == 'true')
    return value
248 249 250
  return default


251 252 253 254 255 256 257 258 259 260
def _git_set_branch_config_value(key, value, branch=None, **kwargs):
  """Sets the value or unsets if it's None of a git branch config.

  Valid, though not necessarily existing, branch must be provided,
  otherwise currently checked out branch is used.
  """
  if not branch:
    branch = GetCurrentBranch()
    assert branch, 'a branch name OR currently checked out branch is required'
  args = ['config']
261
  # Check for boolean first, because bool is int, but int is not bool.
262 263 264 265 266 267
  if value is None:
    args.append('--unset')
  elif isinstance(value, bool):
    args.append('--bool')
    value = str(value).lower()
  else:
268 269
    # git config also has --int, but apparently git config suffers from integer
    # overflows (http://crbug.com/640115), so don't use it.
270 271 272 273 274 275 276
    value = str(value)
  args.append(_git_branch_config_key(branch, key))
  if value is not None:
    args.append(value)
  RunGit(args, **kwargs)


277
def _get_committer_timestamp(commit):
278
  """Returns Unix timestamp as integer of a committer in a commit.
279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296

  Commit can be whatever git show would recognize, such as HEAD, sha1 or ref.
  """
  # Git also stores timezone offset, but it only affects visual display,
  # actual point in time is defined by this timestamp only.
  return int(RunGit(['show', '-s', '--format=%ct', commit]).strip())


def _git_amend_head(message, committer_timestamp):
  """Amends commit with new message and desired committer_timestamp.

  Sets committer timezone to UTC.
  """
  env = os.environ.copy()
  env['GIT_COMMITTER_DATE'] = '%d+0000' % committer_timestamp
  return RunGit(['commit', '--amend', '-m', message], env=env)


297 298
def add_git_similarity(parser):
  parser.add_option(
299
      '--similarity', metavar='SIM', type=int, action='store',
300 301
      help='Sets the percentage that a pair of files need to match in order to'
           ' be considered copies (default 50)')
302 303 304 305 306 307
  parser.add_option(
      '--find-copies', action='store_true',
      help='Allows git to look for copies.')
  parser.add_option(
      '--no-find-copies', action='store_false', dest='find_copies',
      help='Disallows git from looking for copies.')
308 309

  old_parser_args = parser.parse_args
310

311 312 313 314
  def Parse(args):
    options, args = old_parser_args(args)

    if options.similarity is None:
315 316
      options.similarity = _git_get_branch_config_value(
          'git-cl-similarity', default=50, value_type=int)
317
    else:
318 319
      print('Note: Saving similarity of %d%% in git config.'
            % options.similarity)
320
      _git_set_branch_config_value('git-cl-similarity', options.similarity)
321 322

    options.similarity = max(0, min(options.similarity, 100))
323

324
    if options.find_copies is None:
325 326
      options.find_copies = _git_get_branch_config_value(
          'git-find-copies', default=True, value_type=bool)
327
    else:
328
      _git_set_branch_config_value('git-find-copies', bool(options.find_copies))
329 330

    return options, args
331

332 333 334
  parser.parse_args = Parse


335 336 337 338 339 340 341 342 343 344
def _get_properties_from_options(options):
  properties = dict(x.split('=', 1) for x in options.properties)
  for key, val in properties.iteritems():
    try:
      properties[key] = json.loads(val)
    except ValueError:
      pass  # If a value couldn't be evaluated, treat it as a string.
  return properties


345 346 347 348 349 350 351 352
def _prefix_master(master):
  """Convert user-specified master name to full master name.

  Buildbucket uses full master name(master.tryserver.chromium.linux) as bucket
  name, while the developers always use shortened master name
  (tryserver.chromium.linux) by stripping off the prefix 'master.'. This
  function does the conversion for buildbucket migration.
  """
353
  if master.startswith(MASTER_PREFIX):
354
    return master
355 356 357 358 359 360 361 362 363 364 365 366 367 368
  return '%s%s' % (MASTER_PREFIX, master)


def _unprefix_master(bucket):
  """Convert bucket name to shortened master name.

  Buildbucket uses full master name(master.tryserver.chromium.linux) as bucket
  name, while the developers always use shortened master name
  (tryserver.chromium.linux) by stripping off the prefix 'master.'. This
  function does the conversion for buildbucket migration.
  """
  if bucket.startswith(MASTER_PREFIX):
    return bucket[len(MASTER_PREFIX):]
  return bucket
369 370


371 372 373 374 375 376 377 378 379 380 381 382
def _buildbucket_retry(operation_name, http, *args, **kwargs):
  """Retries requests to buildbucket service and returns parsed json content."""
  try_count = 0
  while True:
    response, content = http.request(*args, **kwargs)
    try:
      content_json = json.loads(content)
    except ValueError:
      content_json = None

    # Buildbucket could return an error even if status==200.
    if content_json and content_json.get('error'):
383 384 385 386
      error = content_json.get('error')
      if error.get('code') == 403:
        raise BuildbucketResponseException(
            'Access denied: %s' % error.get('message', ''))
387
      msg = 'Error in response. Reason: %s. Message: %s.' % (
388
          error.get('reason', ''), error.get('message', ''))
389 390 391 392 393 394 395 396 397 398 399 400 401 402
      raise BuildbucketResponseException(msg)

    if response.status == 200:
      if not content_json:
        raise BuildbucketResponseException(
            'Buildbucket returns invalid json content: %s.\n'
            'Please file bugs at http://crbug.com, label "Infra-BuildBucket".' %
            content)
      return content_json
    if response.status < 500 or try_count >= 2:
      raise httplib2.HttpLib2Error(content)

    # status >= 500 means transient failures.
    logging.debug('Transient errors when %s. Will retry.', operation_name)
403
    time_sleep(0.5 + 1.5*try_count)
404 405 406 407
    try_count += 1
  assert False, 'unreachable'


408
def _get_bucket_map(changelist, options, option_parser):
409 410
  """Returns a dict mapping bucket names to builders and tests,
  for triggering try jobs.
411
  """
412 413
  # If no bots are listed, we try to get a set of builders and tests based
  # on GetPreferredTryMasters functions in PRESUBMIT.py files.
414 415 416
  if not options.bot:
    change = changelist.GetChange(
        changelist.GetCommonAncestorWithUpstream(), None)
417
    # Get try masters from PRESUBMIT.py files.
418
    masters = presubmit_support.DoGetTryMasters(
419 420 421 422 423 424 425
        change=change,
        changed_files=change.LocalPaths(),
        repository_root=settings.GetRoot(),
        default_presubmit=None,
        project=None,
        verbose=options.verbose,
        output_stream=sys.stdout)
426 427
    if masters is None:
      return None
428
    return {_prefix_master(m): b for m, b in masters.iteritems()}
429 430 431

  if options.bucket:
    return {options.bucket: {b: [] for b in options.bot}}
432 433
  if options.master:
    return {_prefix_master(options.master): {b: [] for b in options.bot}}
434

435 436 437 438 439 440 441 442 443
  # If bots are listed but no master or bucket, then we need to find out
  # the corresponding master for each bot.
  bucket_map, error_message = _get_bucket_map_for_builders(options.bot)
  if error_message:
    option_parser.error(
        'Tryserver master cannot be found because: %s\n'
        'Please manually specify the tryserver master, e.g. '
        '"-m tryserver.chromium.linux".' % error_message)
  return bucket_map
444 445


446 447
def _get_bucket_map_for_builders(builders):
  """Returns a map of buckets to builders for the given builders."""
448 449
  map_url = 'https://builders-map.appspot.com/'
  try:
450
    builders_map = json.load(urllib2.urlopen(map_url))
451 452 453 454 455
  except urllib2.URLError as e:
    return None, ('Failed to fetch builder-to-master map from %s. Error: %s.' %
                  (map_url, e))
  except ValueError as e:
    return None, ('Invalid json string from %s. Error: %s.' % (map_url, e))
456
  if not builders_map:
457 458
    return None, 'Failed to build master map.'

459 460
  bucket_map = {}
  for builder in builders:
461 462 463 464 465 466 467 468 469 470 471 472 473
    builder_info = builders_map.get(builder, {})
    if isinstance(builder_info, list):
      # This is a list of masters, legacy mode.
      # TODO(nodir): remove this code path.
      buckets = map(_prefix_master, builder_info)
    else:
      buckets = builder_info.get('buckets') or []
    if not buckets:
      return None, ('No matching bucket for builder %s.' % builder)
    if len(buckets) > 1:
      return None, ('The builder name %s exists in multiple buckets %s.' %
                    (builder, buckets))
    bucket_map.setdefault(buckets[0], {})[builder] = []
474 475

  return bucket_map, None
476 477


478
def _trigger_try_jobs(auth_config, changelist, buckets, options,
479
                      category='git_cl_try', patchset=None):
480 481 482 483 484 485 486 487
  """Sends a request to Buildbucket to trigger try jobs for a changelist.

  Args:
    auth_config: AuthConfig for Rietveld.
    changelist: Changelist that the try jobs are associated with.
    buckets: A nested dict mapping bucket names to builders to tests.
    options: Command-line options.
  """
488 489 490 491 492 493 494 495
  assert changelist.GetIssue(), 'CL must be uploaded first'
  codereview_url = changelist.GetCodereviewServer()
  assert codereview_url, 'CL must be uploaded first'
  patchset = patchset or changelist.GetMostRecentPatchset()
  assert patchset, 'CL must be uploaded first'

  codereview_host = urlparse.urlparse(codereview_url).hostname
  authenticator = auth.get_authenticator_for_host(codereview_host, auth_config)
496 497
  http = authenticator.authorize(httplib2.Http())
  http.force_exception_to_status_code = True
498

499 500
  buildbucket_put_url = (
      'https://{hostname}/_ah/api/buildbucket/v1/builds/batch'.format(
501
          hostname=options.buildbucket_host))
502 503 504 505
  buildset = 'patch/{codereview}/{hostname}/{issue}/{patch}'.format(
      codereview='gerrit' if changelist.IsGerrit() else 'rietveld',
      hostname=codereview_host,
      issue=changelist.GetIssue(),
506
      patch=patchset)
507

508
  shared_parameters_properties = changelist.GetTryJobProperties(patchset)
509 510 511
  shared_parameters_properties['category'] = category
  if options.clobber:
    shared_parameters_properties['clobber'] = True
512
  extra_properties = _get_properties_from_options(options)
513 514
  if extra_properties:
    shared_parameters_properties.update(extra_properties)
515 516 517 518

  batch_req_body = {'builds': []}
  print_text = []
  print_text.append('Tried jobs on:')
519 520 521 522 523
  for bucket, builders_and_tests in sorted(buckets.iteritems()):
    print_text.append('Bucket: %s' % bucket)
    master = None
    if bucket.startswith(MASTER_PREFIX):
      master = _unprefix_master(bucket)
524 525 526 527
    for builder, tests in sorted(builders_and_tests.iteritems()):
      print_text.append('  %s: %s' % (builder, tests))
      parameters = {
          'builder_name': builder,
528
          'changes': [{
529
              'author': {'email': changelist.GetIssueOwner()},
530 531
              'revision': options.revision,
          }],
532
          'properties': shared_parameters_properties.copy(),
533
      }
534 535
      if 'presubmit' in builder.lower():
        parameters['properties']['dry_run'] = 'true'
536 537
      if tests:
        parameters['properties']['testfilter'] = tests
538 539 540 541 542 543 544 545 546 547

      tags = [
          'builder:%s' % builder,
          'buildset:%s' % buildset,
          'user_agent:git_cl_try',
      ]
      if master:
        parameters['properties']['master'] = master
        tags.append('master:%s' % master)

548 549 550 551
      batch_req_body['builds'].append(
          {
              'bucket': bucket,
              'parameters_json': json.dumps(parameters),
552
              'client_operation_id': str(uuid.uuid4()),
553
              'tags': tags,
554 555 556
          }
      )

557
  _buildbucket_retry(
558
      'triggering try jobs',
559 560 561 562 563 564
      http,
      buildbucket_put_url,
      'PUT',
      body=json.dumps(batch_req_body),
      headers={'Content-Type': 'application/json'}
  )
565 566
  print_text.append('To see results here, run:        git cl try-results')
  print_text.append('To see results in browser, run:  git cl web')
567
  print('\n'.join(print_text))
568 569


570 571
def fetch_try_jobs(auth_config, changelist, buildbucket_host,
                   patchset=None):
572
  """Fetches try jobs from buildbucket.
573

574
  Returns a map from build id to build info as a dictionary.
575
  """
576 577 578 579 580 581 582 583 584
  assert buildbucket_host
  assert changelist.GetIssue(), 'CL must be uploaded first'
  assert changelist.GetCodereviewServer(), 'CL must be uploaded first'
  patchset = patchset or changelist.GetMostRecentPatchset()
  assert patchset, 'CL must be uploaded first'

  codereview_url = changelist.GetCodereviewServer()
  codereview_host = urlparse.urlparse(codereview_url).hostname
  authenticator = auth.get_authenticator_for_host(codereview_host, auth_config)
585 586 587
  if authenticator.has_cached_credentials():
    http = authenticator.authorize(httplib2.Http())
  else:
588 589
    print('Warning: Some results might be missing because %s' %
          # Get the message on how to login.
590
          (auth.LoginRequiredError(codereview_host).message,))
591 592 593 594
    http = httplib2.Http()

  http.force_exception_to_status_code = True

595 596 597
  buildset = 'patch/{codereview}/{hostname}/{issue}/{patch}'.format(
      codereview='gerrit' if changelist.IsGerrit() else 'rietveld',
      hostname=codereview_host,
598
      issue=changelist.GetIssue(),
599
      patch=patchset)
600 601 602 603 604
  params = {'tag': 'buildset:%s' % buildset}

  builds = {}
  while True:
    url = 'https://{hostname}/_ah/api/buildbucket/v1/search?{params}'.format(
605
        hostname=buildbucket_host,
606
        params=urllib.urlencode(params))
607
    content = _buildbucket_retry('fetching try jobs', http, url, 'GET')
608 609 610 611 612
    for build in content.get('builds', []):
      builds[build['id']] = build
    if 'next_cursor' in content:
      params['start_cursor'] = content['next_cursor']
    else:
613
      break
614
  return builds
615 616


617
def print_try_jobs(options, builds):
618 619
  """Prints nicely result of fetch_try_jobs."""
  if not builds:
620
    print('No try jobs scheduled.')
621 622 623 624 625 626 627 628 629 630 631 632 633 634
    return

  # Make a copy, because we'll be modifying builds dictionary.
  builds = builds.copy()
  builder_names_cache = {}

  def get_builder(b):
    try:
      return builder_names_cache[b['id']]
    except KeyError:
      try:
        parameters = json.loads(b['parameters_json'])
        name = parameters['builder_name']
      except (ValueError, KeyError) as error:
635
        print('WARNING: Failed to get builder name for build %s: %s' % (
636
              b['id'], error))
637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664
        name = None
      builder_names_cache[b['id']] = name
      return name

  def get_bucket(b):
    bucket = b['bucket']
    if bucket.startswith('master.'):
      return bucket[len('master.'):]
    return bucket

  if options.print_master:
    name_fmt = '%%-%ds %%-%ds' % (
        max(len(str(get_bucket(b))) for b in builds.itervalues()),
        max(len(str(get_builder(b))) for b in builds.itervalues()))
    def get_name(b):
      return name_fmt % (get_bucket(b), get_builder(b))
  else:
    name_fmt = '%%-%ds' % (
        max(len(str(get_builder(b))) for b in builds.itervalues()))
    def get_name(b):
      return name_fmt % get_builder(b)

  def sort_key(b):
    return b['status'], b.get('result'), get_name(b), b.get('url')

  def pop(title, f, color=None, **kwargs):
    """Pop matching builds from `builds` dict and print them."""

665
    if not options.color or color is None:
666 667 668 669 670 671 672 673 674 675
      colorize = str
    else:
      colorize = lambda x: '%s%s%s' % (color, x, Fore.RESET)

    result = []
    for b in builds.values():
      if all(b.get(k) == v for k, v in kwargs.iteritems()):
        builds.pop(b['id'])
        result.append(b)
    if result:
676
      print(colorize(title))
677
      for b in sorted(result, key=sort_key):
678
        print(' ', colorize('\t'.join(map(str, f(b)))))
679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712

  total = len(builds)
  pop(status='COMPLETED', result='SUCCESS',
      title='Successes:', color=Fore.GREEN,
      f=lambda b: (get_name(b), b.get('url')))
  pop(status='COMPLETED', result='FAILURE', failure_reason='INFRA_FAILURE',
      title='Infra Failures:', color=Fore.MAGENTA,
      f=lambda b: (get_name(b), b.get('url')))
  pop(status='COMPLETED', result='FAILURE', failure_reason='BUILD_FAILURE',
      title='Failures:', color=Fore.RED,
      f=lambda b: (get_name(b), b.get('url')))
  pop(status='COMPLETED', result='CANCELED',
      title='Canceled:', color=Fore.MAGENTA,
      f=lambda b: (get_name(b),))
  pop(status='COMPLETED', result='FAILURE',
      failure_reason='INVALID_BUILD_DEFINITION',
      title='Wrong master/builder name:', color=Fore.MAGENTA,
      f=lambda b: (get_name(b),))
  pop(status='COMPLETED', result='FAILURE',
      title='Other failures:',
      f=lambda b: (get_name(b), b.get('failure_reason'), b.get('url')))
  pop(status='COMPLETED',
      title='Other finished:',
      f=lambda b: (get_name(b), b.get('result'), b.get('url')))
  pop(status='STARTED',
      title='Started:', color=Fore.YELLOW,
      f=lambda b: (get_name(b), b.get('url')))
  pop(status='SCHEDULED',
      title='Scheduled:',
      f=lambda b: (get_name(b), 'id=%s' % b['id']))
  # The last section is just in case buildbucket API changes OR there is a bug.
  pop(title='Other:',
      f=lambda b: (get_name(b), 'id=%s' % b['id']))
  assert len(builds) == 0
713
  print('Total: %d try jobs' % total)
714

715

716 717 718 719 720 721 722 723
def write_try_results_json(output_file, builds):
  """Writes a subset of the data from fetch_try_jobs to a file as JSON.

  The input |builds| dict is assumed to be generated by Buildbucket.
  Buildbucket documentation: http://goo.gl/G0s101
  """

  def convert_build_dict(build):
724 725
    """Extracts some of the information from one build dict."""
    parameters = json.loads(build.get('parameters_json', '{}')) or {}
726 727 728
    return {
        'buildbucket_id': build.get('id'),
        'bucket': build.get('bucket'),
729 730 731
        'builder_name': parameters.get('builder_name'),
        'created_ts': build.get('created_ts'),
        'experimental': build.get('experimental'),
732
        'failure_reason': build.get('failure_reason'),
733 734 735
        'result': build.get('result'),
        'status': build.get('status'),
        'tags': build.get('tags'),
736 737 738 739 740 741 742 743 744
        'url': build.get('url'),
    }

  converted = []
  for _, build in sorted(builds.items()):
      converted.append(convert_build_dict(build))
  write_json(output_file, converted)


745
def print_stats(similarity, find_copies, args):
746 747 748 749
  """Prints statistics about the change to the user."""
  # --no-ext-diff is broken in some versions of Git, so try to work around
  # this by overriding the environment (but there is still a problem if the
  # git config key "diff.external" is used).
750
  env = GetNoGitPagerEnv()
751 752
  if 'GIT_EXTERNAL_DIFF' in env:
    del env['GIT_EXTERNAL_DIFF']
753 754

  if find_copies:
755
    similarity_options = ['-l100000', '-C%s' % similarity]
756 757 758
  else:
    similarity_options = ['-M%s' % similarity]

759 760 761 762
  try:
    stdout = sys.stdout.fileno()
  except AttributeError:
    stdout = None
763
  return subprocess2.call(
764
      ['git',
765
       'diff', '--no-ext-diff', '--stat'] + similarity_options + args,
766
      stdout=stdout, env=env)
767 768


769 770 771 772
class BuildbucketResponseException(Exception):
  pass


773 774 775 776
class Settings(object):
  def __init__(self):
    self.default_server = None
    self.cc = None
777
    self.root = None
778 779 780
    self.tree_status_url = None
    self.viewvc_url = None
    self.updated = False
781
    self.is_gerrit = None
782
    self.squash_gerrit_uploads = None
783
    self.gerrit_skip_ensure_authenticated = None
784
    self.git_editor = None
785
    self.project = None
786
    self.force_https_commit_url = None
787 788 789 790

  def LazyUpdateIfNeeded(self):
    """Updates the settings from a codereview.settings file, if available."""
    if not self.updated:
791 792
      # The only value that actually changes the behavior is
      # autoupdate = "false". Everything else means "true".
793
      autoupdate = RunGit(['config', 'rietveld.autoupdate'],
794 795 796
                          error_ok=True
                          ).strip().lower()

797
      cr_settings_file = FindCodereviewSettingsFile()
798
      if autoupdate != 'false' and cr_settings_file:
799 800 801 802 803 804
        LoadCodereviewSettingsFromFile(cr_settings_file)
      self.updated = True

  def GetDefaultServerUrl(self, error_ok=False):
    if not self.default_server:
      self.LazyUpdateIfNeeded()
805
      self.default_server = gclient_utils.UpgradeToHttps(
806
          self._GetRietveldConfig('server', error_ok=True))
807 808 809 810 811
      if error_ok:
        return self.default_server
      if not self.default_server:
        error_message = ('Could not find settings file. You must configure '
                         'your review setup by running "git cl config".')
812
        self.default_server = gclient_utils.UpgradeToHttps(
813
            self._GetRietveldConfig('server', error_message=error_message))
814 815
    return self.default_server

816 817 818
  @staticmethod
  def GetRelativeRoot():
    return RunGit(['rev-parse', '--show-cdup']).strip()
819

820
  def GetRoot(self):
821 822 823
    if self.root is None:
      self.root = os.path.abspath(self.GetRelativeRoot())
    return self.root
824

825 826
  def GetGitMirror(self, remote='origin'):
    """If this checkout is from a local git mirror, return a Mirror object."""
827
    local_url = RunGit(['config', '--get', 'remote.%s.url' % remote]).strip()
828 829 830 831
    if not os.path.isdir(local_url):
      return None
    git_cache.Mirror.SetCachePath(os.path.dirname(local_url))
    remote_url = git_cache.Mirror.CacheDirToUrl(local_url)
832
    # Use the /dev/null print_func to avoid terminal spew.
833
    mirror = git_cache.Mirror(remote_url, print_func=lambda *args: None)
834 835 836 837
    if mirror.exists():
      return mirror
    return None

838 839 840 841
  def GetTreeStatusUrl(self, error_ok=False):
    if not self.tree_status_url:
      error_message = ('You must configure your tree status URL by running '
                       '"git cl config".')
842 843
      self.tree_status_url = self._GetRietveldConfig(
          'tree-status-url', error_ok=error_ok, error_message=error_message)
844 845 846 847
    return self.tree_status_url

  def GetViewVCUrl(self):
    if not self.viewvc_url:
848
      self.viewvc_url = self._GetRietveldConfig('viewvc-url', error_ok=True)
849 850
    return self.viewvc_url

851
  def GetBugPrefix(self):
852
    return self._GetRietveldConfig('bug-prefix', error_ok=True)
853

854 855 856 857 858
  def GetIsSkipDependencyUpload(self, branch_name):
    """Returns true if specified branch should skip dep uploads."""
    return self._GetBranchConfig(branch_name, 'skip-deps-uploads',
                                 error_ok=True)

859 860 861 862 863
  def GetRunPostUploadHook(self):
    run_post_upload_hook = self._GetRietveldConfig(
        'run-post-upload-hook', error_ok=True)
    return run_post_upload_hook == "True"

864
  def GetDefaultCCList(self):
865
    return self._GetRietveldConfig('cc', error_ok=True)
866

867
  def GetDefaultPrivateFlag(self):
868
    return self._GetRietveldConfig('private', error_ok=True)
869

870
  def GetIsGerrit(self):
871
    """Return true if this repo is associated with gerrit code review system."""
872
    if self.is_gerrit is None:
873 874
      self.is_gerrit = (
          self._GetConfig('gerrit.host', error_ok=True).lower() == 'true')
875 876
    return self.is_gerrit

877 878 879
  def GetSquashGerritUploads(self):
    """Return true if uploads to Gerrit should be squashed by default."""
    if self.squash_gerrit_uploads is None:
880 881 882 883 884 885
      self.squash_gerrit_uploads = self.GetSquashGerritUploadsOverride()
      if self.squash_gerrit_uploads is None:
        # Default is squash now (http://crbug.com/611892#c23).
        self.squash_gerrit_uploads = not (
            RunGit(['config', '--bool', 'gerrit.squash-uploads'],
                   error_ok=True).strip() == 'false')
886 887
    return self.squash_gerrit_uploads

888 889 890 891 892 893 894 895 896 897 898 899 900 901
  def GetSquashGerritUploadsOverride(self):
    """Return True or False if codereview.settings should be overridden.

    Returns None if no override has been defined.
    """
    # See also http://crbug.com/611892#c23
    result = RunGit(['config', '--bool', 'gerrit.override-squash-uploads'],
                    error_ok=True).strip()
    if result == 'true':
      return True
    if result == 'false':
      return False
    return None

902 903 904 905 906
  def GetGerritSkipEnsureAuthenticated(self):
    """Return True if EnsureAuthenticated should not be done for Gerrit
    uploads."""
    if self.gerrit_skip_ensure_authenticated is None:
      self.gerrit_skip_ensure_authenticated = (
907
          RunGit(['config', '--bool', 'gerrit.skip-ensure-authenticated'],
908 909 910
                 error_ok=True).strip() == 'true')
    return self.gerrit_skip_ensure_authenticated

911 912 913 914 915 916
  def GetGitEditor(self):
    """Return the editor specified in the git config, or None if none is."""
    if self.git_editor is None:
      self.git_editor = self._GetConfig('core.editor', error_ok=True)
    return self.git_editor or None

917 918 919 920 921 922 923 924
  def GetLintRegex(self):
    return (self._GetRietveldConfig('cpplint-regex', error_ok=True) or
            DEFAULT_LINT_REGEX)

  def GetLintIgnoreRegex(self):
    return (self._GetRietveldConfig('cpplint-ignore-regex', error_ok=True) or
            DEFAULT_LINT_IGNORE_REGEX)

925 926 927 928 929
  def GetProject(self):
    if not self.project:
      self.project = self._GetRietveldConfig('project', error_ok=True)
    return self.project

930 931 932
  def _GetRietveldConfig(self, param, **kwargs):
    return self._GetConfig('rietveld.' + param, **kwargs)

933 934 935
  def _GetBranchConfig(self, branch_name, param, **kwargs):
    return self._GetConfig('branch.' + branch_name + '.' + param, **kwargs)

936 937 938 939 940
  def _GetConfig(self, param, **kwargs):
    self.LazyUpdateIfNeeded()
    return RunGit(['config', param], **kwargs).strip()


941 942 943 944 945 946 947 948 949 950 951 952
@contextlib.contextmanager
def _get_gerrit_project_config_file(remote_url):
  """Context manager to fetch and store Gerrit's project.config from
  refs/meta/config branch and store it in temp file.

  Provides a temporary filename or None if there was error.
  """
  error, _ = RunGitWithCode([
      'fetch', remote_url,
      '+refs/meta/config:refs/git_cl/meta/config'])
  if error:
    # Ref doesn't exist or isn't accessible to current user.
953
    print('WARNING: Failed to fetch project config for %s: %s' %
954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973
          (remote_url, error))
    yield None
    return

  error, project_config_data = RunGitWithCode(
      ['show', 'refs/git_cl/meta/config:project.config'])
  if error:
    print('WARNING: project.config file not found')
    yield None
    return

  with gclient_utils.temporary_directory() as tempdir:
    project_config_file = os.path.join(tempdir, 'project.config')
    gclient_utils.FileWrite(project_config_file, project_config_data)
    yield project_config_file


def _is_git_numberer_enabled(remote_url, remote_ref):
  """Returns True if Git Numberer is enabled on this ref."""
  # TODO(tandrii): this should be deleted once repos below are 100% on Gerrit.
974 975 976 977
  KNOWN_PROJECTS_WHITELIST = [
      'chromium/src',
      'external/webrtc',
      'v8/v8',
978
      'infra/experimental',
979 980
      # For webrtc.googlesource.com/src.
      'src',
981 982
  ]

983 984 985 986 987 988 989 990 991 992
  assert remote_ref and remote_ref.startswith('refs/'), remote_ref
  url_parts = urlparse.urlparse(remote_url)
  project_name = url_parts.path.lstrip('/').rstrip('git./')
  for known in KNOWN_PROJECTS_WHITELIST:
    if project_name.endswith(known):
      break
  else:
    # Early exit to avoid extra fetches for repos that aren't using Git
    # Numberer.
    return False
993

994 995 996 997
  with _get_gerrit_project_config_file(remote_url) as project_config_file:
    if project_config_file is None:
      # Failed to fetch project.config, which shouldn't happen on open source
      # repos KNOWN_PROJECTS_WHITELIST.
998
      return False
999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011
    def get_opts(x):
      code, out = RunGitWithCode(
          ['config', '-f', project_config_file, '--get-all',
           'plugin.git-numberer.validate-%s-refglob' % x])
      if code == 0:
        return out.strip().splitlines()
      return []
    enabled, disabled = map(get_opts, ['enabled', 'disabled'])

  logging.info('validator config enabled %s disabled %s refglobs for '
               '(this ref: %s)', enabled, disabled, remote_ref)

  def match_refglobs(refglobs):
1012
    for refglob in refglobs:
1013
      if remote_ref == refglob or fnmatch.fnmatch(remote_ref, refglob):
1014 1015 1016
        return True
    return False

1017 1018 1019
  if match_refglobs(disabled):
    return False
  return match_refglobs(enabled)
1020 1021


1022 1023
def ShortBranchName(branch):
  """Convert a name like 'refs/heads/foo' to just 'foo'."""
1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041
  return branch.replace('refs/heads/', '', 1)


def GetCurrentBranchRef():
  """Returns branch ref (e.g., refs/heads/master) or None."""
  return RunGit(['symbolic-ref', 'HEAD'],
                stderr=subprocess2.VOID, error_ok=True).strip() or None


def GetCurrentBranch():
  """Returns current branch or None.

  For refs/heads/* branches, returns just last part. For others, full ref.
  """
  branchref = GetCurrentBranchRef()
  if branchref:
    return ShortBranchName(branchref)
  return None
1042 1043


1044 1045 1046 1047 1048 1049 1050 1051 1052
class _CQState(object):
  """Enum for states of CL with respect to Commit Queue."""
  NONE = 'none'
  DRY_RUN = 'dry_run'
  COMMIT = 'commit'

  ALL_STATES = [NONE, DRY_RUN, COMMIT]


1053
class _ParsedIssueNumberArgument(object):
1054
  def __init__(self, issue=None, patchset=None, hostname=None, codereview=None):
1055 1056 1057
    self.issue = issue
    self.patchset = patchset
    self.hostname = hostname
1058 1059
    assert codereview in (None, 'rietveld', 'gerrit')
    self.codereview = codereview
1060 1061 1062 1063 1064 1065

  @property
  def valid(self):
    return self.issue is not None


1066
def ParseIssueNumberArgument(arg, codereview=None):
1067 1068 1069 1070
  """Parses the issue argument and returns _ParsedIssueNumberArgument."""
  fail_result = _ParsedIssueNumberArgument()

  if arg.isdigit():
1071
    return _ParsedIssueNumberArgument(issue=int(arg), codereview=codereview)
1072 1073
  if not arg.startswith('http'):
    return fail_result
1074

1075 1076 1077 1078 1079
  url = gclient_utils.UpgradeToHttps(arg)
  try:
    parsed_url = urlparse.urlparse(url)
  except ValueError:
    return fail_result
1080

1081 1082 1083 1084
  if codereview is not None:
    parsed = _CODEREVIEW_IMPLEMENTATIONS[codereview].ParseIssueURL(parsed_url)
    return parsed or fail_result

1085 1086 1087 1088 1089 1090 1091 1092 1093 1094
  results = {}
  for name, cls in _CODEREVIEW_IMPLEMENTATIONS.iteritems():
    parsed = cls.ParseIssueURL(parsed_url)
    if parsed is not None:
      results[name] = parsed

  if not results:
    return fail_result
  if len(results) == 1:
    return results.values()[0]
1095 1096 1097 1098 1099

  if parsed_url.netloc and parsed_url.netloc.split('.')[0].endswith('-review'):
    # This is likely Gerrit.
    return results['gerrit']
  # Choose Rietveld as before if URL can parsed by either.
1100
  return results['rietveld']
1101 1102


1103
class GerritChangeNotExists(Exception):
1104 1105 1106
  def __init__(self, issue, url):
    self.issue = issue
    self.url = url
1107
    super(GerritChangeNotExists, self).__init__()
1108 1109

  def __str__(self):
1110
    return 'change %s at %s does not exist or you have no access to it' % (
1111 1112 1113
        self.issue, self.url)


1114 1115 1116 1117 1118 1119
_CommentSummary = collections.namedtuple(
    '_CommentSummary', ['date', 'message', 'sender',
                        # TODO(tandrii): these two aren't known in Gerrit.
                        'approval', 'disapproval'])


1120
class Changelist(object):
1121 1122 1123 1124 1125
  """Changelist works with one changelist in local branch.

  Supports two codereview backends: Rietveld or Gerrit, selected at object
  creation.

1126 1127 1128
  Notes:
    * Not safe for concurrent multi-{thread,process} use.
    * Caches values from current branch. Therefore, re-use after branch change
1129
      with great care.
1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143
  """

  def __init__(self, branchref=None, issue=None, codereview=None, **kwargs):
    """Create a new ChangeList instance.

    If issue is given, the codereview must be given too.

    If `codereview` is given, it must be 'rietveld' or 'gerrit'.
    Otherwise, it's decided based on current configuration of the local branch,
    with default being 'rietveld' for backwards compatibility.
    See _load_codereview_impl for more details.

    **kwargs will be passed directly to codereview implementation.
    """
1144
    # Poke settings so we get the "configure your server" message if necessary.
1145 1146 1147 1148
    global settings
    if not settings:
      # Happens when git_cl.py is used as a utility library.
      settings = Settings()
1149 1150 1151 1152

    if issue:
      assert codereview, 'codereview must be known, if issue is known'

1153 1154
    self.branchref = branchref
    if self.branchref:
1155
      assert branchref.startswith('refs/heads/')
1156 1157 1158 1159
      self.branch = ShortBranchName(self.branchref)
    else:
      self.branch = None
    self.upstream_branch = None
1160 1161
    self.lookedup_issue = False
    self.issue = issue or None
1162 1163
    self.has_description = False
    self.description = None
1164
    self.lookedup_patchset = False
1165
    self.patchset = None
1166
    self.cc = None
1167
    self.more_cc = []
1168 1169
    self._remote = None

1170
    self._codereview_impl = None
1171
    self._codereview = None
1172
    self._load_codereview_impl(codereview, **kwargs)
1173 1174
    assert self._codereview_impl
    assert self._codereview in _CODEREVIEW_IMPLEMENTATIONS
1175 1176 1177

  def _load_codereview_impl(self, codereview=None, **kwargs):
    if codereview:
1178 1179 1180 1181
      assert codereview in _CODEREVIEW_IMPLEMENTATIONS
      cls = _CODEREVIEW_IMPLEMENTATIONS[codereview]
      self._codereview = codereview
      self._codereview_impl = cls(self, **kwargs)
1182 1183 1184 1185 1186 1187 1188
      return

    # Automatic selection based on issue number set for a current branch.
    # Rietveld takes precedence over Gerrit.
    assert not self.issue
    # Whether we find issue or not, we are doing the lookup.
    self.lookedup_issue = True
1189 1190 1191 1192 1193 1194 1195 1196 1197
    if self.GetBranch():
      for codereview, cls in _CODEREVIEW_IMPLEMENTATIONS.iteritems():
        issue = _git_get_branch_config_value(
            cls.IssueConfigKey(), value_type=int, branch=self.GetBranch())
        if issue:
          self._codereview = codereview
          self._codereview_impl = cls(self, **kwargs)
          self.issue = int(issue)
          return
1198 1199 1200 1201 1202 1203

    # No issue is set for this branch, so decide based on repo-wide settings.
    return self._load_codereview_impl(
        codereview='gerrit' if settings.GetIsGerrit() else 'rietveld',
        **kwargs)

1204 1205
  def IsGerrit(self):
    return self._codereview == 'gerrit'
1206 1207

  def GetCCList(self):
1208
    """Returns the users cc'd on this CL.
1209

1210 1211
    The return value is a string suitable for passing to git cl with the --cc
    flag.
1212 1213
    """
    if self.cc is None:
1214
      base_cc = settings.GetDefaultCCList()
1215
      more_cc = ','.join(self.more_cc)
1216 1217 1218
      self.cc = ','.join(filter(None, (base_cc, more_cc))) or ''
    return self.cc

1219 1220 1221
  def GetCCListWithoutDefault(self):
    """Return the users cc'd on this CL excluding default ones."""
    if self.cc is None:
1222
      self.cc = ','.join(self.more_cc)
1223 1224
    return self.cc

1225 1226 1227
  def ExtendCC(self, more_cc):
    """Extends the list of users to cc on this CL based on the changed files."""
    self.more_cc.extend(more_cc)
1228 1229 1230 1231

  def GetBranch(self):
    """Returns the short branch name, e.g. 'master'."""
    if not self.branch:
1232
      branchref = GetCurrentBranchRef()
1233 1234 1235
      if not branchref:
        return None
      self.branchref = branchref
1236 1237 1238 1239 1240 1241 1242 1243
      self.branch = ShortBranchName(self.branchref)
    return self.branch

  def GetBranchRef(self):
    """Returns the full branch name, e.g. 'refs/heads/master'."""
    self.GetBranch()  # Poke the lazy loader.
    return self.branchref

1244 1245 1246 1247
  def ClearBranch(self):
    """Clears cached branch data of this object."""
    self.branch = self.branchref = None

1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262
  def _GitGetBranchConfigValue(self, key, default=None, **kwargs):
    assert 'branch' not in kwargs, 'this CL branch is used automatically'
    kwargs['branch'] = self.GetBranch()
    return _git_get_branch_config_value(key, default, **kwargs)

  def _GitSetBranchConfigValue(self, key, value, **kwargs):
    assert 'branch' not in kwargs, 'this CL branch is used automatically'
    assert self.GetBranch(), (
        'this CL must have an associated branch to %sset %s%s' %
          ('un' if value is None else '',
           key,
           '' if value is None else ' to %r' % value))
    kwargs['branch'] = self.GetBranch()
    return _git_set_branch_config_value(key, value, **kwargs)

1263 1264
  @staticmethod
  def FetchUpstreamTuple(branch):
1265
    """Returns a tuple containing remote and remote ref,
1266 1267 1268
       e.g. 'origin', 'refs/heads/master'
    """
    remote = '.'
1269 1270
    upstream_branch = _git_get_branch_config_value('merge', branch=branch)

1271
    if upstream_branch:
1272
      remote = _git_get_branch_config_value('remote', branch=branch)
1273
    else:
1274 1275 1276 1277
      upstream_branch = RunGit(['config', 'rietveld.upstream-branch'],
                               error_ok=True).strip()
      if upstream_branch:
        remote = RunGit(['config', 'rietveld.upstream-remote']).strip()
1278
      else:
1279 1280 1281 1282 1283 1284
        # Else, try to guess the origin remote.
        remote_branches = RunGit(['branch', '-r']).split()
        if 'origin/master' in remote_branches:
          # Fall back on origin/master if it exits.
          remote = 'origin'
          upstream_branch = 'refs/heads/master'
1285
        else:
1286 1287 1288 1289 1290 1291
          DieWithError(
             'Unable to determine default branch to diff against.\n'
             'Either pass complete "git diff"-style arguments, like\n'
             '  git cl upload origin/master\n'
             'or verify this branch is set up to track another \n'
             '(via the --track argument to "git checkout -b ...").')
1292 1293 1294

    return remote, upstream_branch

1295
  def GetCommonAncestorWithUpstream(self):
1296 1297 1298 1299
    upstream_branch = self.GetUpstreamBranch()
    if not BranchExists(upstream_branch):
      DieWithError('The upstream for the current branch (%s) does not exist '
                   'anymore.\nPlease fix it and try again.' % self.GetBranch())
1300
    return git_common.get_or_create_merge_base(self.GetBranch(),
1301
                                               upstream_branch)
1302

1303 1304
  def GetUpstreamBranch(self):
    if self.upstream_branch is None:
1305
      remote, upstream_branch = self.FetchUpstreamTuple(self.GetBranch())
1306
      if remote is not '.':
1307 1308 1309 1310
        upstream_branch = upstream_branch.replace('refs/heads/',
                                                  'refs/remotes/%s/' % remote)
        upstream_branch = upstream_branch.replace('refs/branch-heads/',
                                                  'refs/remotes/branch-heads/')
1311 1312 1313
      self.upstream_branch = upstream_branch
    return self.upstream_branch

1314
  def GetRemoteBranch(self):
1315
    if not self._remote:
1316 1317 1318 1319 1320 1321 1322 1323 1324
      remote, branch = None, self.GetBranch()
      seen_branches = set()
      while branch not in seen_branches:
        seen_branches.add(branch)
        remote, branch = self.FetchUpstreamTuple(branch)
        branch = ShortBranchName(branch)
        if remote != '.' or branch.startswith('refs/remotes'):
          break
      else:
1325 1326
        remotes = RunGit(['remote'], error_ok=True).split()
        if len(remotes) == 1:
1327
          remote, = remotes
1328
        elif 'origin' in remotes:
1329
          remote = 'origin'
1330 1331
          logging.warn('Could not determine which remote this change is '
                       'associated with, so defaulting to "%s".' % self._remote)
1332 1333
        else:
          logging.warn('Could not determine which remote this change is '
1334
                       'associated with.')
1335 1336 1337
        branch = 'HEAD'
      if branch.startswith('refs/remotes'):
        self._remote = (remote, branch)
1338 1339
      elif branch.startswith('refs/branch-heads/'):
        self._remote = (remote, branch.replace('refs/', 'refs/remotes/'))
1340 1341
      else:
        self._remote = (remote, 'refs/remotes/%s/%s' % (remote, branch))
1342 1343
    return self._remote

1344 1345 1346
  def GitSanityChecks(self, upstream_git_obj):
    """Checks git repo status and ensures diff is from local commits."""

1347 1348
    if upstream_git_obj is None:
      if self.GetBranch() is None:
1349
        print('ERROR: Unable to determine current branch (detached HEAD?)',
1350
              file=sys.stderr)
1351
      else:
1352
        print('ERROR: No upstream branch.', file=sys.stderr)
1353 1354
      return False

1355 1356 1357 1358
    # Verify the commit we're diffing against is in our current branch.
    upstream_sha = RunGit(['rev-parse', '--verify', upstream_git_obj]).strip()
    common_ancestor = RunGit(['merge-base', upstream_sha, 'HEAD']).strip()
    if upstream_sha != common_ancestor:
1359 1360
      print('ERROR: %s is not in the current branch.  You may need to rebase '
            'your tracking branch' % upstream_sha, file=sys.stderr)
1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375
      return False

    # List the commits inside the diff, and verify they are all local.
    commits_in_diff = RunGit(
        ['rev-list', '^%s' % upstream_sha, 'HEAD']).splitlines()
    code, remote_branch = RunGitWithCode(['config', 'gitcl.remotebranch'])
    remote_branch = remote_branch.strip()
    if code != 0:
      _, remote_branch = self.GetRemoteBranch()

    commits_in_remote = RunGit(
        ['rev-list', '^%s' % upstream_sha, remote_branch]).splitlines()

    common_commits = set(commits_in_diff) & set(commits_in_remote)
    if common_commits:
1376 1377 1378 1379 1380 1381 1382
      print('ERROR: Your diff contains %d commits already in %s.\n'
            'Run "git log --oneline %s..HEAD" to get a list of commits in '
            'the diff.  If you are using a custom git flow, you can override'
            ' the reference used for this check with "git config '
            'gitcl.remotebranch <git-ref>".' % (
                len(common_commits), remote_branch, upstream_git_obj),
            file=sys.stderr)
1383 1384 1385
      return False
    return True

1386
  def GetGitBaseUrlFromConfig(self):
1387
    """Return the configured base URL from branch.<branchname>.baseurl.
1388 1389 1390

    Returns None if it is not set.
    """
1391
    return self._GitGetBranchConfigValue('base-url')
1392

1393 1394 1395 1396 1397
  def GetRemoteUrl(self):
    """Return the configured remote URL, e.g. 'git://example.org/foo.git/'.

    Returns None if there is no remote.
    """
1398
    remote, _ = self.GetRemoteBranch()
1399 1400 1401 1402 1403 1404 1405 1406
    url = RunGit(['config', 'remote.%s.url' % remote], error_ok=True).strip()

    # If URL is pointing to a local directory, it is probably a git cache.
    if os.path.isdir(url):
      url = RunGit(['config', 'remote.%s.url' % remote],
                   error_ok=True,
                   cwd=url).strip()
    return url
1407

1408
  def GetIssue(self):
1409
    """Returns the issue number as a int or None if not set."""
1410
    if self.issue is None and not self.lookedup_issue:
1411 1412
      self.issue = self._GitGetBranchConfigValue(
          self._codereview_impl.IssueConfigKey(), value_type=int)
1413
      self.lookedup_issue = True
1414 1415 1416 1417
    return self.issue

  def GetIssueURL(self):
    """Get the URL for a particular issue."""
1418 1419
    issue = self.GetIssue()
    if not issue:
1420
      return None
1421
    return '%s/%s' % (self._codereview_impl.GetCodereviewServer(), issue)
1422

1423 1424
  def GetDescription(self, pretty=False, force=False):
    if not self.has_description or force:
1425
      if self.GetIssue():
1426
        self.description = self._codereview_impl.FetchDescription(force=force)
1427 1428
      self.has_description = True
    if pretty:
1429 1430
      # Set width to 72 columns + 2 space indent.
      wrapper = textwrap.TextWrapper(width=74, replace_whitespace=True)
1431
      wrapper.initial_indent = wrapper.subsequent_indent = '  '
1432 1433
      lines = self.description.splitlines()
      return '\n'.join([wrapper.fill(line) for line in lines])
1434 1435
    return self.description

1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451
  def GetDescriptionFooters(self):
    """Returns (non_footer_lines, footers) for the commit message.

    Returns:
      non_footer_lines (list(str)) - Simple list of description lines without
        any footer. The lines do not contain newlines, nor does the list contain
        the empty line between the message and the footers.
      footers (list(tuple(KEY, VALUE))) - List of parsed footers, e.g.
        [("Change-Id", "Ideadbeef...."), ...]
    """
    raw_description = self.GetDescription()
    msg_lines, _, footers = git_footers.split_footers(raw_description)
    if footers:
      msg_lines = msg_lines[:len(msg_lines)-1]
    return msg_lines, footers

1452
  def GetPatchset(self):
1453
    """Returns the patchset number as a int or None if not set."""
1454
    if self.patchset is None and not self.lookedup_patchset:
1455 1456
      self.patchset = self._GitGetBranchConfigValue(
          self._codereview_impl.PatchsetConfigKey(), value_type=int)
1457
      self.lookedup_patchset = True
1458 1459 1460
    return self.patchset

  def SetPatchset(self, patchset):
1461 1462 1463
    """Set this branch's patchset. If patchset=0, clears the patchset."""
    assert self.GetBranch()
    if not patchset:
1464
      self.patchset = None
1465 1466 1467 1468
    else:
      self.patchset = int(patchset)
    self._GitSetBranchConfigValue(
        self._codereview_impl.PatchsetConfigKey(), self.patchset)
1469

1470
  def SetIssue(self, issue=None):
1471 1472
    """Set this branch's issue. If issue isn't given, clears the issue."""
    assert self.GetBranch()
1473
    if issue:
1474 1475 1476
      issue = int(issue)
      self._GitSetBranchConfigValue(
          self._codereview_impl.IssueConfigKey(), issue)
1477
      self.issue = issue
1478 1479
      codereview_server = self._codereview_impl.GetCodereviewServer()
      if codereview_server:
1480 1481 1482
        self._GitSetBranchConfigValue(
            self._codereview_impl.CodereviewServerConfigKey(),
            codereview_server)
1483
    else:
1484 1485 1486 1487 1488 1489 1490 1491 1492
      # Reset all of these just to be clean.
      reset_suffixes = [
          'last-upload-hash',
          self._codereview_impl.IssueConfigKey(),
          self._codereview_impl.PatchsetConfigKey(),
          self._codereview_impl.CodereviewServerConfigKey(),
      ] + self._PostUnsetIssueProperties()
      for prop in reset_suffixes:
        self._GitSetBranchConfigValue(prop, None, error_ok=True)
1493 1494 1495 1496 1497 1498
      msg = RunGit(['log', '-1', '--format=%B']).strip()
      if msg and git_footers.get_footer_change_id(msg):
        print('WARNING: The change patched into this branch has a Change-Id. '
              'Removing it.')
        RunGit(['commit', '--amend', '-m',
                git_footers.remove_footer(msg, 'Change-Id')])
1499
      self.issue = None
1500
      self.patchset = None
1501

1502
  def GetChange(self, upstream_branch, author, local_description=False):
1503 1504 1505
    if not self.GitSanityChecks(upstream_branch):
      DieWithError('\nGit sanity check failure')

1506
    root = settings.GetRelativeRoot()
1507 1508
    if not root:
      root = '.'
1509
    absroot = os.path.abspath(root)
1510 1511

    # We use the sha1 of HEAD as a name of this change.
1512
    name = RunGitWithCode(['rev-parse', 'HEAD'])[1].strip()
1513
    # Need to pass a relative path for msysgit.
1514
    try:
1515
      files = scm.GIT.CaptureStatus([root], '.', upstream_branch)
1516 1517
    except subprocess2.CalledProcessError:
      DieWithError(
1518
          ('\nFailed to diff against upstream branch %s\n\n'
1519 1520
           'This branch probably doesn\'t exist anymore. To reset the\n'
           'tracking branch, please run\n'
1521 1522
           '    git branch --set-upstream-to origin/master %s\n'
           'or replace origin/master with the relevant branch') %
1523
          (upstream_branch, self.GetBranch()))
1524

1525 1526
    issue = self.GetIssue()
    patchset = self.GetPatchset()
1527
    if issue and not local_description:
1528 1529 1530 1531 1532
      description = self.GetDescription()
    else:
      # If the change was never uploaded, use the log messages of all commits
      # up to the branch point, as git cl upload will prefill the description
      # with these log messages.
1533 1534
      args = ['log', '--pretty=format:%s%n%n%b', '%s...' % (upstream_branch)]
      description = RunGitWithCode(args)[1].strip()
1535 1536

    if not author:
1537
      author = RunGit(['config', 'user.email']).strip() or None
1538
    return presubmit_support.GitChange(
1539 1540 1541 1542 1543 1544
        name,
        description,
        absroot,
        files,
        issue,
        patchset,
1545 1546
        author,
        upstream=upstream_branch)
1547

1548
  def UpdateDescription(self, description, force=False):
1549
    self._codereview_impl.UpdateDescriptionRemote(description, force=force)
1550
    self.description = description
1551
    self.has_description = True
1552

1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575
  def UpdateDescriptionFooters(self, description_lines, footers, force=False):
    """Sets the description for this CL remotely.

    You can get description_lines and footers with GetDescriptionFooters.

    Args:
      description_lines (list(str)) - List of CL description lines without
        newline characters.
      footers (list(tuple(KEY, VALUE))) - List of footers, as returned by
        GetDescriptionFooters. Key must conform to the git footers format (i.e.
        `List-Of-Tokens`). It will be case-normalized so that each token is
        title-cased.
    """
    new_description = '\n'.join(description_lines)
    if footers:
      new_description += '\n'
      for k, v in footers:
        foot = '%s: %s' % (git_footers.normalize_name(k), v)
        if not git_footers.FOOTER_PATTERN.match(foot):
          raise ValueError('Invalid footer %r' % foot)
        new_description += foot + '\n'
    self.UpdateDescription(new_description, force)

1576 1577 1578 1579 1580 1581
  def RunHook(self, committing, may_prompt, verbose, change):
    """Calls sys.exit() if the hook fails; returns a HookResults otherwise."""
    try:
      return presubmit_support.DoPresubmitChecks(change, committing,
          verbose=verbose, output_stream=sys.stdout, input_stream=sys.stdin,
          default_presubmit=None, may_prompt=may_prompt,
1582
          rietveld_obj=self._codereview_impl.GetRietveldObjForPresubmit(),
1583
          gerrit_obj=self._codereview_impl.GetGerritObjForPresubmit())
1584
    except presubmit_support.PresubmitFailure as e:
1585
      DieWithError('%s\nMaybe your depot_tools is out of date?' % e)
1586

1587 1588
  def CMDPatchIssue(self, issue_arg, reject, nocommit, directory):
    """Fetches and applies the issue patch from codereview to local branch."""
1589 1590
    if isinstance(issue_arg, (int, long)) or issue_arg.isdigit():
      parsed_issue_arg = _ParsedIssueNumberArgument(int(issue_arg))
1591 1592 1593 1594 1595 1596 1597 1598
    else:
      # Assume url.
      parsed_issue_arg = self._codereview_impl.ParseIssueURL(
          urlparse.urlparse(issue_arg))
    if not parsed_issue_arg or not parsed_issue_arg.valid:
      DieWithError('Failed to parse issue argument "%s". '
                   'Must be an issue number or a valid URL.' % issue_arg)
    return self._codereview_impl.CMDPatchWithParsedIssue(
1599
        parsed_issue_arg, reject, nocommit, directory, False)
1600

1601 1602
  def CMDUpload(self, options, git_diff_args, orig_args):
    """Uploads a change to codereview."""
1603
    custom_cl_base = None
1604
    if git_diff_args:
1605
      custom_cl_base = base_branch = git_diff_args[0]
1606 1607 1608 1609 1610 1611 1612 1613
    else:
      if self.GetBranch() is None:
        DieWithError('Can\'t upload from detached HEAD state. Get on a branch!')

      # Default to diffing against common ancestor of upstream branch
      base_branch = self.GetCommonAncestorWithUpstream()
      git_diff_args = [base_branch, 'HEAD']

1614 1615 1616 1617 1618 1619 1620 1621 1622 1623
    # Warn about Rietveld deprecation for initial uploads to Rietveld.
    if not self.IsGerrit() and not self.GetIssue():
      print('=====================================')
      print('NOTICE: Rietveld is being deprecated. '
            'You can upload changes to Gerrit with')
      print('  git cl upload --gerrit')
      print('or set Gerrit to be your default code review tool with')
      print('  git config gerrit.host true')
      print('=====================================')

1624 1625 1626
    # Fast best-effort checks to abort before running potentially
    # expensive hooks if uploading is likely to fail anyway. Passing these
    # checks does not guarantee that uploading will not fail.
1627
    self._codereview_impl.EnsureAuthenticated(force=options.force)
1628
    self._codereview_impl.EnsureCanUploadPatchset(force=options.force)
1629 1630 1631 1632 1633 1634

    # Apply watchlists on upload.
    change = self.GetChange(base_branch, None)
    watchlist = watchlists.Watchlists(change.RepositoryRoot())
    files = [f.LocalPath() for f in change.AffectedFiles()]
    if not options.bypass_watchlists:
1635
      self.ExtendCC(watchlist.GetWatchersForPaths(files))
1636 1637

    if not options.bypass_hooks:
1638
      if options.reviewers or options.tbrs or options.add_owners_to:
1639 1640 1641
        # Set the reviewer list now so that presubmit checks can access it.
        change_description = ChangeDescription(change.FullDescriptionText())
        change_description.update_reviewers(options.reviewers,
1642
                                            options.tbrs,
1643
                                            options.add_owners_to,
1644 1645 1646 1647 1648 1649 1650 1651 1652 1653
                                            change)
        change.SetDescriptionText(change_description.description)
      hook_results = self.RunHook(committing=False,
                                may_prompt=not options.force,
                                verbose=options.verbose,
                                change=change)
      if not hook_results.should_continue():
        return 1
      if not options.reviewers and hook_results.reviewers:
        options.reviewers = hook_results.reviewers.split(',')
1654
      self.ExtendCC(hook_results.more_cc)
1655

1656 1657 1658
    # TODO(tandrii): Checking local patchset against remote patchset is only
    # supported for Rietveld. Extend it to Gerrit or remove it completely.
    if self.GetIssue() and not self.IsGerrit():
1659 1660 1661 1662
      latest_patchset = self.GetMostRecentPatchset()
      local_patchset = self.GetPatchset()
      if (latest_patchset and local_patchset and
          local_patchset != latest_patchset):
1663 1664 1665 1666 1667 1668
        print('The last upload made from this repository was patchset #%d but '
              'the most recent patchset on the server is #%d.'
              % (local_patchset, latest_patchset))
        print('Uploading will still work, but if you\'ve uploaded to this '
              'issue from another machine or branch the patch you\'re '
              'uploading now might not include those changes.')
1669
        confirm_or_exit(action='upload')
1670 1671

    print_stats(options.similarity, options.find_copies, git_diff_args)
1672
    ret = self.CMDUploadChange(options, git_diff_args, custom_cl_base, change)
1673
    if not ret:
1674 1675 1676 1677 1678
      if options.use_commit_queue:
        self.SetCQState(_CQState.COMMIT)
      elif options.cq_dry_run:
        self.SetCQState(_CQState.DRY_RUN)

1679 1680
      _git_set_branch_config_value('last-upload-hash',
                                   RunGit(['rev-parse', 'HEAD']).strip())
1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691
      # Run post upload hooks, if specified.
      if settings.GetRunPostUploadHook():
        presubmit_support.DoPostUploadExecuter(
            change,
            self,
            settings.GetRoot(),
            options.verbose,
            sys.stdout)

      # Upload all dependencies if specified.
      if options.dependencies:
1692 1693 1694 1695
        print()
        print('--dependencies has been specified.')
        print('All dependent local branches will be re-uploaded.')
        print()
1696 1697 1698 1699 1700 1701
        # Remove the dependencies flag from args so that we do not end up in a
        # loop.
        orig_args.remove('--dependencies')
        ret = upload_branch_deps(self, orig_args)
    return ret

1702
  def SetCQState(self, new_state):
1703
    """Updates the CQ state for the latest patchset.
1704 1705 1706 1707 1708

    Issue must have been already uploaded and known.
    """
    assert new_state in _CQState.ALL_STATES
    assert self.GetIssue()
1709
    try:
1710
      self._codereview_impl.SetCQState(new_state)
1711 1712 1713 1714
      return 0
    except KeyboardInterrupt:
      raise
    except:
1715
      print('WARNING: Failed to %s.\n'
1716
            'Either:\n'
1717 1718 1719 1720 1721 1722 1723
            ' * Your project has no CQ,\n'
            ' * You don\'t have permission to change the CQ state,\n'
            ' * There\'s a bug in this code (see stack trace below).\n'
            'Consider specifying which bots to trigger manually or asking your '
            'project owners for permissions or contacting Chrome Infra at:\n'
            'https://www.chromium.org/infra\n\n' %
            ('cancel CQ' if new_state == _CQState.NONE else 'trigger CQ'))
1724 1725 1726
      # Still raise exception so that stack trace is printed.
      raise

1727 1728
  # Forward methods to codereview specific implementation.

1729 1730
  def AddComment(self, message, publish=None):
    return self._codereview_impl.AddComment(message, publish=publish)
1731

1732
  def GetCommentsSummary(self, readable=True):
1733 1734
    """Returns list of _CommentSummary for each comment.

1735 1736
    args:
    readable: determines whether the output is designed for a human or a machine
1737
    """
1738
    return self._codereview_impl.GetCommentsSummary(readable)
1739

1740 1741 1742 1743 1744 1745 1746 1747 1748
  def CloseIssue(self):
    return self._codereview_impl.CloseIssue()

  def GetStatus(self):
    return self._codereview_impl.GetStatus()

  def GetCodereviewServer(self):
    return self._codereview_impl.GetCodereviewServer()

1749 1750 1751 1752
  def GetIssueOwner(self):
    """Get owner from codereview, which may differ from this checkout."""
    return self._codereview_impl.GetIssueOwner()

1753 1754 1755
  def GetMostRecentPatchset(self):
    return self._codereview_impl.GetMostRecentPatchset()

1756
  def CannotTriggerTryJobReason(self):
1757
    """Returns reason (str) if unable trigger try jobs on this CL or None."""
1758 1759
    return self._codereview_impl.CannotTriggerTryJobReason()

1760 1761 1762
  def GetTryJobProperties(self, patchset=None):
    """Returns dictionary of properties to launch try job."""
    return self._codereview_impl.GetTryJobProperties(patchset=patchset)
1763

1764 1765 1766
  def __getattr__(self, attr):
    # This is because lots of untested code accesses Rietveld-specific stuff
    # directly, and it's hard to fix for sure. So, just let it work, and fix
1767
    # on a case by case basis.
1768 1769 1770 1771 1772 1773
    # Note that child method defines __getattr__ as well, and forwards it here,
    # because _RietveldChangelistImpl is not cleaned up yet, and given
    # deprecation of Rietveld, it should probably be just removed.
    # Until that time, avoid infinite recursion by bypassing __getattr__
    # of implementation class.
    return self._codereview_impl.__getattribute__(attr)
1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798


class _ChangelistCodereviewBase(object):
  """Abstract base class encapsulating codereview specifics of a changelist."""
  def __init__(self, changelist):
    self._changelist = changelist  # instance of Changelist

  def __getattr__(self, attr):
    # Forward methods to changelist.
    # TODO(tandrii): maybe clean up _GerritChangelistImpl and
    # _RietveldChangelistImpl to avoid this hack?
    return getattr(self._changelist, attr)

  def GetStatus(self):
    """Apply a rough heuristic to give a simple summary of an issue's review
    or CQ status, assuming adherence to a common workflow.

    Returns None if no issue for this branch, or specific string keywords.
    """
    raise NotImplementedError()

  def GetCodereviewServer(self):
    """Returns server URL without end slash, like "https://codereview.com"."""
    raise NotImplementedError()

1799
  def FetchDescription(self, force=False):
1800 1801 1802
    """Fetches and returns description from the codereview server."""
    raise NotImplementedError()

1803
  @classmethod
1804 1805 1806
  def IssueConfigKey(cls):
    """Returns branch setting storing issue number."""
    raise NotImplementedError()
1807 1808

  @classmethod
1809 1810
  def PatchsetConfigKey(cls):
    """Returns branch setting storing patchset number."""
1811 1812
    raise NotImplementedError()

1813 1814 1815
  @classmethod
  def CodereviewServerConfigKey(cls):
    """Returns branch setting storing codereview server."""
1816 1817
    raise NotImplementedError()

1818
  def _PostUnsetIssueProperties(self):
1819
    """Which branch-specific properties to erase when unsetting issue."""
1820
    return []
1821

1822
  def GetRietveldObjForPresubmit(self):
1823
    # This is an unfortunate Rietveld-embeddedness in presubmit.
1824
    # For non-Rietveld code reviews, this probably should return a dummy object.
1825 1826
    raise NotImplementedError()

1827 1828 1829 1830
  def GetGerritObjForPresubmit(self):
    # None is valid return value, otherwise presubmit_support.GerritAccessor.
    return None

1831
  def UpdateDescriptionRemote(self, description, force=False):
1832 1833 1834
    """Update the description on codereview site."""
    raise NotImplementedError()

1835
  def AddComment(self, message, publish=None):
1836 1837 1838
    """Posts a comment to the codereview site."""
    raise NotImplementedError()

1839
  def GetCommentsSummary(self, readable=True):
1840 1841
    raise NotImplementedError()

1842 1843 1844 1845 1846 1847 1848 1849
  def CloseIssue(self):
    """Closes the issue."""
    raise NotImplementedError()

  def GetMostRecentPatchset(self):
    """Returns the most recent patchset number from the codereview site."""
    raise NotImplementedError()

1850
  def CMDPatchWithParsedIssue(self, parsed_issue_arg, reject, nocommit,
1851
                              directory, force):
1852 1853 1854 1855 1856 1857 1858 1859 1860
    """Fetches and applies the issue.

    Arguments:
      parsed_issue_arg: instance of _ParsedIssueNumberArgument.
      reject: if True, reject the failed patch instead of switching to 3-way
        merge. Rietveld only.
      nocommit: do not commit the patch, thus leave the tree dirty. Rietveld
        only.
      directory: switch to directory before applying the patch. Rietveld only.
1861
      force: if true, overwrites existing local state.
1862 1863 1864 1865 1866 1867 1868 1869 1870
    """
    raise NotImplementedError()

  @staticmethod
  def ParseIssueURL(parsed_url):
    """Parses url and returns instance of _ParsedIssueNumberArgument or None if
    failed."""
    raise NotImplementedError()

1871
  def EnsureAuthenticated(self, force, refresh=False):
1872 1873 1874 1875
    """Best effort check that user is authenticated with codereview server.

    Arguments:
      force: whether to skip confirmation questions.
1876 1877
      refresh: whether to attempt to refresh credentials. Ignored if not
        applicable.
1878
    """
1879 1880
    raise NotImplementedError()

1881
  def EnsureCanUploadPatchset(self, force):
1882 1883 1884 1885 1886
    """Best effort check that uploading isn't supposed to fail for predictable
    reasons.

    This method should raise informative exception if uploading shouldn't
    proceed.
1887 1888 1889

    Arguments:
      force: whether to skip confirmation questions.
1890
    """
1891
    raise NotImplementedError()
1892

1893
  def CMDUploadChange(self, options, git_diff_args, custom_cl_base, change):
1894 1895 1896
    """Uploads a change to codereview."""
    raise NotImplementedError()

1897
  def SetCQState(self, new_state):
1898
    """Updates the CQ state for the latest patchset.
1899 1900 1901 1902 1903

    Issue must have been already uploaded and known.
    """
    raise NotImplementedError()

1904
  def CannotTriggerTryJobReason(self):
1905
    """Returns reason (str) if unable trigger try jobs on this CL or None."""
1906 1907
    raise NotImplementedError()

1908 1909 1910
  def GetIssueOwner(self):
    raise NotImplementedError()

1911
  def GetTryJobProperties(self, patchset=None):
1912 1913
    raise NotImplementedError()

1914 1915

class _RietveldChangelistImpl(_ChangelistCodereviewBase):
1916

1917
  def __init__(self, changelist, auth_config=None, codereview_host=None):
1918 1919
    super(_RietveldChangelistImpl, self).__init__(changelist)
    assert settings, 'must be initialized in _ChangelistCodereviewBase'
1920
    if not codereview_host:
1921
      settings.GetDefaultServerUrl()
1922

1923
    self._rietveld_server = codereview_host
1924
    self._auth_config = auth_config or auth.make_auth_config()
1925 1926 1927 1928 1929 1930 1931 1932
    self._props = None
    self._rpc_server = None

  def GetCodereviewServer(self):
    if not self._rietveld_server:
      # If we're on a branch then get the server potentially associated
      # with that branch.
      if self.GetIssue():
1933 1934
        self._rietveld_server = gclient_utils.UpgradeToHttps(
            self._GitGetBranchConfigValue(self.CodereviewServerConfigKey()))
1935 1936 1937 1938
      if not self._rietveld_server:
        self._rietveld_server = settings.GetDefaultServerUrl()
    return self._rietveld_server

1939
  def EnsureAuthenticated(self, force, refresh=False):
1940 1941 1942 1943 1944 1945
    """Best effort check that user is authenticated with Rietveld server."""
    if self._auth_config.use_oauth2:
      authenticator = auth.get_authenticator_for_host(
          self.GetCodereviewServer(), self._auth_config)
      if not authenticator.has_cached_credentials():
        raise auth.LoginRequiredError(self.GetCodereviewServer())
1946 1947
      if refresh:
        authenticator.get_access_token()
1948

1949 1950 1951 1952
  def EnsureCanUploadPatchset(self, force):
    # No checks for Rietveld because we are deprecating Rietveld.
    pass

1953
  def FetchDescription(self, force=False):
1954 1955 1956
    issue = self.GetIssue()
    assert issue
    try:
1957
      return self.RpcServer().get_description(issue, force=force).strip()
1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972
    except urllib2.HTTPError as e:
      if e.code == 404:
        DieWithError(
            ('\nWhile fetching the description for issue %d, received a '
             '404 (not found)\n'
             'error. It is likely that you deleted this '
             'issue on the server. If this is the\n'
             'case, please run\n\n'
             '    git cl issue 0\n\n'
             'to clear the association with the deleted issue. Then run '
             'this command again.') % issue)
      else:
        DieWithError(
            '\nFailed to fetch issue description. HTTP error %d' % e.code)
    except urllib2.URLError as e:
1973 1974
      print('Warning: Failed to retrieve CL description due to network '
            'failure.', file=sys.stderr)
1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988
      return ''

  def GetMostRecentPatchset(self):
    return self.GetIssueProperties()['patchsets'][-1]

  def GetIssueProperties(self):
    if self._props is None:
      issue = self.GetIssue()
      if not issue:
        self._props = {}
      else:
        self._props = self.RpcServer().get_issue_properties(issue, True)
    return self._props

1989 1990 1991 1992 1993 1994 1995 1996 1997 1998
  def CannotTriggerTryJobReason(self):
    props = self.GetIssueProperties()
    if not props:
      return 'Rietveld doesn\'t know about your issue %s' % self.GetIssue()
    if props.get('closed'):
      return 'CL %s is closed' % self.GetIssue()
    if props.get('private'):
      return 'CL %s is private' % self.GetIssue()
    return None

1999 2000
  def GetTryJobProperties(self, patchset=None):
    """Returns dictionary of properties to launch try job."""
2001 2002 2003 2004 2005 2006 2007 2008 2009
    project = (self.GetIssueProperties() or {}).get('project')
    return {
      'issue': self.GetIssue(),
      'patch_project': project,
      'patch_storage': 'rietveld',
      'patchset': patchset or self.GetPatchset(),
      'rietveld': self.GetCodereviewServer(),
    }

2010 2011 2012
  def GetIssueOwner(self):
    return (self.GetIssueProperties() or {}).get('owner_email')

2013
  def AddComment(self, message, publish=None):
2014 2015
    return self.RpcServer().add_comment(self.GetIssue(), message)

2016
  def GetCommentsSummary(self, _readable=True):
2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028
    summary = []
    for message in self.GetIssueProperties().get('messages', []):
      date = datetime.datetime.strptime(message['date'], '%Y-%m-%d %H:%M:%S.%f')
      summary.append(_CommentSummary(
        date=date,
        disapproval=bool(message['disapproval']),
        approval=bool(message['approval']),
        sender=message['sender'],
        message=message['text'],
      ))
    return summary

2029
  def GetStatus(self):
2030
    """Applies a rough heuristic to give a simple summary of an issue's review
2031 2032 2033
    or CQ status, assuming adherence to a common workflow.

    Returns None if no issue for this branch, or one of the following keywords:
2034 2035 2036 2037 2038 2039 2040 2041
      * 'error'    - error from review tool (including deleted issues)
      * 'unsent'   - not sent for review
      * 'waiting'  - waiting for review
      * 'reply'    - waiting for owner to reply to review
      * 'not lgtm' - Code-Review label has been set negatively
      * 'lgtm'     - LGTM from at least one approved reviewer
      * 'commit'   - in the commit queue
      * 'closed'   - closed
2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053
    """
    if not self.GetIssue():
      return None

    try:
      props = self.GetIssueProperties()
    except urllib2.HTTPError:
      return 'error'

    if props.get('closed'):
      # Issue is closed.
      return 'closed'
2054
    if props.get('commit') and not props.get('cq_dry_run', False):
2055 2056 2057
      # Issue is in the commit queue.
      return 'commit'

2058 2059 2060 2061
    messages = props.get('messages') or []
    if not messages:
      # No message was sent.
      return 'unsent'
2062

2063
    if get_approving_reviewers(props):
2064
      return 'lgtm'
2065 2066
    elif get_approving_reviewers(props, disapproval=True):
      return 'not lgtm'
2067

2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079
    # Skip CQ messages that don't require owner's action.
    while messages and messages[-1]['sender'] == COMMIT_BOT_EMAIL:
      if 'Dry run:' in messages[-1]['text']:
        messages.pop()
      elif 'The CQ bit was unchecked' in messages[-1]['text']:
        # This message always follows prior messages from CQ,
        # so skip this too.
        messages.pop()
      else:
        # This is probably a CQ messages warranting user attention.
        break

2080
    if messages[-1]['sender'] != props.get('owner_email'):
2081
      # Non-LGTM reply from non-owner and not CQ bot.
2082 2083 2084
      return 'reply'
    return 'waiting'

2085
  def UpdateDescriptionRemote(self, description, force=False):
2086
    self.RpcServer().update_description(self.GetIssue(), description)
2087

2088
  def CloseIssue(self):
2089
    return self.RpcServer().close_issue(self.GetIssue())
2090 2091

  def SetFlag(self, flag, value):
2092 2093 2094 2095 2096
    return self.SetFlags({flag: value})

  def SetFlags(self, flags):
    """Sets flags on this CL/patchset in Rietveld.
    """
2097
    patchset = self.GetPatchset() or self.GetMostRecentPatchset()
2098
    try:
2099
      return self.RpcServer().set_flags(
2100
          self.GetIssue(), patchset, flags)
2101
    except urllib2.HTTPError as e:
2102 2103 2104 2105 2106
      if e.code == 404:
        DieWithError('The issue %s doesn\'t exist.' % self.GetIssue())
      if e.code == 403:
        DieWithError(
            ('Access denied to issue %s. Maybe the patchset %s doesn\'t '
2107
             'match?') % (self.GetIssue(), patchset))
2108
      raise
2109

2110
  def RpcServer(self):
2111 2112
    """Returns an upload.RpcServer() to access this review's rietveld instance.
    """
2113
    if not self._rpc_server:
2114
      self._rpc_server = rietveld.CachingRietveld(
2115
          self.GetCodereviewServer(),
2116
          self._auth_config)
2117
    return self._rpc_server
2118

2119
  @classmethod
2120
  def IssueConfigKey(cls):
2121
    return 'rietveldissue'
2122

2123 2124 2125
  @classmethod
  def PatchsetConfigKey(cls):
    return 'rietveldpatchset'
2126

2127 2128 2129
  @classmethod
  def CodereviewServerConfigKey(cls):
    return 'rietveldserver'
2130

2131
  def GetRietveldObjForPresubmit(self):
2132 2133
    return self.RpcServer()

2134 2135 2136 2137 2138 2139
  def SetCQState(self, new_state):
    props = self.GetIssueProperties()
    if props.get('private'):
      DieWithError('Cannot set-commit on private issue')

    if new_state == _CQState.COMMIT:
2140
      self.SetFlags({'commit': '1', 'cq_dry_run': '0'})
2141
    elif new_state == _CQState.NONE:
2142
      self.SetFlags({'commit': '0', 'cq_dry_run': '0'})
2143
    else:
2144 2145
      assert new_state == _CQState.DRY_RUN
      self.SetFlags({'commit': '1', 'cq_dry_run': '1'})
2146

2147
  def CMDPatchWithParsedIssue(self, parsed_issue_arg, reject, nocommit,
2148
                              directory, force):
2149 2150 2151 2152 2153 2154 2155 2156 2157
    # PatchIssue should never be called with a dirty tree.  It is up to the
    # caller to check this, but just in case we assert here since the
    # consequences of the caller not checking this could be dire.
    assert(not git_common.is_dirty_git_tree('apply'))
    assert(parsed_issue_arg.valid)
    self._changelist.issue = parsed_issue_arg.issue
    if parsed_issue_arg.hostname:
      self._rietveld_server = 'https://%s' % parsed_issue_arg.hostname

2158 2159 2160
    patchset = parsed_issue_arg.patchset or self.GetMostRecentPatchset()
    patchset_object = self.RpcServer().get_patch(self.GetIssue(), patchset)
    scm_obj = checkout.GitCheckout(settings.GetRoot(), None, None, None, None)
2161
    try:
2162 2163 2164
      scm_obj.apply_patch(patchset_object)
    except Exception as e:
      print(str(e))
2165 2166 2167 2168
      return 1

    # If we had an issue, commit the current state and register the issue.
    if not nocommit:
2169 2170
      self.SetIssue(self.GetIssue())
      self.SetPatchset(patchset)
2171 2172 2173 2174
      RunGit(['commit', '-m', (self.GetDescription() + '\n\n' +
                               'patch from issue %(i)s at patchset '
                               '%(p)s (http://crrev.com/%(i)s#ps%(p)s)'
                               % {'i': self.GetIssue(), 'p': patchset})])
2175
      print('Committed patch locally.')
2176
    else:
2177
      print('Patch applied to index.')
2178 2179 2180 2181 2182 2183
    return 0

  @staticmethod
  def ParseIssueURL(parsed_url):
    if not parsed_url.scheme or not parsed_url.scheme.startswith('http'):
      return None
2184 2185 2186 2187
    # Rietveld patch: https://domain/<number>/#ps<patchset>
    match = re.match(r'/(\d+)/$', parsed_url.path)
    match2 = re.match(r'ps(\d+)$', parsed_url.fragment)
    if match and match2:
2188
      return _ParsedIssueNumberArgument(
2189 2190
          issue=int(match.group(1)),
          patchset=int(match2.group(1)),
2191 2192
          hostname=parsed_url.netloc,
          codereview='rietveld')
2193 2194 2195
    # Typical url: https://domain/<issue_number>[/[other]]
    match = re.match('/(\d+)(/.*)?$', parsed_url.path)
    if match:
2196
      return _ParsedIssueNumberArgument(
2197
          issue=int(match.group(1)),
2198 2199
          hostname=parsed_url.netloc,
          codereview='rietveld')
2200 2201 2202
    # Rietveld patch: https://domain/download/issue<number>_<patchset>.diff
    match = re.match(r'/download/issue(\d+)_(\d+).diff$', parsed_url.path)
    if match:
2203
      return _ParsedIssueNumberArgument(
2204 2205
          issue=int(match.group(1)),
          patchset=int(match.group(2)),
2206 2207
          hostname=parsed_url.netloc,
          codereview='rietveld')
2208 2209
    return None

2210
  def CMDUploadChange(self, options, args, custom_cl_base, change):
2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223
    """Upload the patch to Rietveld."""
    upload_args = ['--assume_yes']  # Don't ask about untracked files.
    upload_args.extend(['--server', self.GetCodereviewServer()])
    upload_args.extend(auth.auth_config_to_command_options(self._auth_config))
    if options.emulate_svn_auto_props:
      upload_args.append('--emulate_svn_auto_props')

    change_desc = None

    if options.email is not None:
      upload_args.extend(['--email', options.email])

    if self.GetIssue():
2224
      if options.title is not None:
2225 2226 2227 2228
        upload_args.extend(['--title', options.title])
      if options.message:
        upload_args.extend(['--message', options.message])
      upload_args.extend(['--issue', str(self.GetIssue())])
2229 2230
      print('This branch is associated with issue %s. '
            'Adding patch to that issue.' % self.GetIssue())
2231
    else:
2232
      if options.title is not None:
2233
        upload_args.extend(['--title', options.title])
2234 2235 2236 2237 2238 2239
      if options.message:
        message = options.message
      else:
        message = CreateDescriptionFromLog(args)
        if options.title:
          message = options.title + '\n\n' + message
2240
      change_desc = ChangeDescription(message)
2241
      if options.reviewers or options.add_owners_to:
2242 2243
        change_desc.update_reviewers(options.reviewers, options.tbrs,
                                     options.add_owners_to, change)
2244
      if not options.force:
2245
        change_desc.prompt(bug=options.bug, git_footer=False)
2246 2247

      if not change_desc.description:
2248
        print('Description is empty; aborting.')
2249 2250 2251 2252 2253 2254 2255 2256
        return 1

      upload_args.extend(['--message', change_desc.description])
      if change_desc.get_reviewers():
        upload_args.append('--reviewers=%s' % ','.join(
            change_desc.get_reviewers()))
      if options.send_mail:
        if not change_desc.get_reviewers():
2257
          DieWithError("Must specify reviewers to send email.", change_desc)
2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270
        upload_args.append('--send_mail')

      # We check this before applying rietveld.private assuming that in
      # rietveld.cc only addresses which we can send private CLs to are listed
      # if rietveld.private is set, and so we should ignore rietveld.cc only
      # when --private is specified explicitly on the command line.
      if options.private:
        logging.warn('rietveld.cc is ignored since private flag is specified.  '
                     'You need to review and add them manually if necessary.')
        cc = self.GetCCListWithoutDefault()
      else:
        cc = self.GetCCList()
      cc = ','.join(filter(None, (cc, ','.join(options.cc))))
2271 2272
      if change_desc.get_cced():
        cc = ','.join(filter(None, (cc, ','.join(change_desc.get_cced()))))
2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286
      if cc:
        upload_args.extend(['--cc', cc])

    if options.private or settings.GetDefaultPrivateFlag() == "True":
      upload_args.append('--private')

    upload_args.extend(['--git_similarity', str(options.similarity)])
    if not options.find_copies:
      upload_args.extend(['--git_no_find_copies'])

    # Include the upstream repo's URL in the change -- this is useful for
    # projects that have their source spread across multiple repos.
    remote_url = self.GetGitBaseUrlFromConfig()
    if not remote_url:
2287 2288 2289
      if self.GetRemoteUrl() and '/' in self.GetUpstreamBranch():
        remote_url = '%s@%s' % (self.GetRemoteUrl(),
                                self.GetUpstreamBranch().split('/')[-1])
2290 2291
    if remote_url:
      remote, remote_branch = self.GetRemoteBranch()
2292
      target_ref = GetTargetRef(remote, remote_branch, options.target_branch)
2293 2294 2295 2296 2297 2298 2299 2300
      if target_ref:
        upload_args.extend(['--target_ref', target_ref])

      # Look for dependent patchsets. See crbug.com/480453 for more details.
      remote, upstream_branch = self.FetchUpstreamTuple(self.GetBranch())
      upstream_branch = ShortBranchName(upstream_branch)
      if remote is '.':
        # A local branch is being tracked.
2301
        local_branch = upstream_branch
2302
        if settings.GetIsSkipDependencyUpload(local_branch):
2303 2304 2305 2306
          print()
          print('Skipping dependency patchset upload because git config '
                'branch.%s.skip-deps-uploads is set to True.' % local_branch)
          print()
2307 2308
        else:
          auth_config = auth.extract_auth_config_from_options(options)
2309
          branch_cl = Changelist(branchref='refs/heads/'+local_branch,
2310 2311 2312 2313 2314 2315 2316 2317
                                 auth_config=auth_config)
          branch_cl_issue_url = branch_cl.GetIssueURL()
          branch_cl_issue = branch_cl.GetIssue()
          branch_cl_patchset = branch_cl.GetPatchset()
          if branch_cl_issue_url and branch_cl_issue and branch_cl_patchset:
            upload_args.extend(
                ['--depends_on_patchset', '%s:%s' % (
                     branch_cl_issue, branch_cl_patchset)])
2318
            print(
2319 2320 2321 2322 2323 2324 2325 2326 2327 2328
                '\n'
                'The current branch (%s) is tracking a local branch (%s) with '
                'an associated CL.\n'
                'Adding %s/#ps%s as a dependency patchset.\n'
                '\n' % (self.GetBranch(), local_branch, branch_cl_issue_url,
                        branch_cl_patchset))

    project = settings.GetProject()
    if project:
      upload_args.extend(['--project', project])
2329 2330 2331 2332 2333
    else:
      print()
      print('WARNING: Uploading without a project specified. Please ensure '
            'your repo\'s codereview.settings has a "PROJECT: foo" line.')
      print()
2334 2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346

    try:
      upload_args = ['upload'] + upload_args + args
      logging.info('upload.RealMain(%s)', upload_args)
      issue, patchset = upload.RealMain(upload_args)
      issue = int(issue)
      patchset = int(patchset)
    except KeyboardInterrupt:
      sys.exit(1)
    except:
      # If we got an exception after the user typed a description for their
      # change, back up the description before re-raising.
      if change_desc:
2347
        SaveDescriptionBackup(change_desc)
2348 2349 2350 2351 2352 2353 2354
      raise

    if not self.GetIssue():
      self.SetIssue(issue)
    self.SetPatchset(patchset)
    return 0

2355

2356
class _GerritChangelistImpl(_ChangelistCodereviewBase):
2357
  def __init__(self, changelist, auth_config=None, codereview_host=None):
2358 2359 2360
    # auth_config is Rietveld thing, kept here to preserve interface only.
    super(_GerritChangelistImpl, self).__init__(changelist)
    self._change_id = None
2361 2362
    # Lazily cached values.
    self._gerrit_host = None    # e.g. chromium-review.googlesource.com
2363
    self._gerrit_server = None  # e.g. https://chromium-review.googlesource.com
2364 2365
    # Map from change number (issue) to its detail cache.
    self._detail_cache = {}
2366

2367 2368 2369 2370 2371
    if codereview_host is not None:
      assert not codereview_host.startswith('https://'), codereview_host
      self._gerrit_host = codereview_host
      self._gerrit_server = 'https://%s' % codereview_host

2372 2373 2374
  def _GetGerritHost(self):
    # Lazy load of configs.
    self.GetCodereviewServer()
2375 2376 2377 2378 2379
    if self._gerrit_host and '.' not in self._gerrit_host:
      # Abbreviated domain like "chromium" instead of chromium.googlesource.com.
      # This happens for internal stuff http://crbug.com/614312.
      parsed = urlparse.urlparse(self.GetRemoteUrl())
      if parsed.scheme == 'sso':
2380
        print('WARNING: using non-https URLs for remote is likely broken\n'
2381 2382 2383
              '  Your current remote is: %s'  % self.GetRemoteUrl())
        self._gerrit_host = '%s.googlesource.com' % self._gerrit_host
        self._gerrit_server = 'https://%s' % self._gerrit_host
2384 2385
    return self._gerrit_host

2386 2387 2388 2389
  def _GetGitHost(self):
    """Returns git host to be used when uploading change to Gerrit."""
    return urlparse.urlparse(self.GetRemoteUrl()).netloc

2390 2391 2392 2393 2394
  def GetCodereviewServer(self):
    if not self._gerrit_server:
      # If we're on a branch then get the server potentially associated
      # with that branch.
      if self.GetIssue():
2395 2396 2397 2398
        self._gerrit_server = self._GitGetBranchConfigValue(
            self.CodereviewServerConfigKey())
        if self._gerrit_server:
          self._gerrit_host = urlparse.urlparse(self._gerrit_server).netloc
2399 2400 2401
      if not self._gerrit_server:
        # We assume repo to be hosted on Gerrit, and hence Gerrit server
        # has "-review" suffix for lowest level subdomain.
2402
        parts = self._GetGitHost().split('.')
2403 2404 2405 2406 2407
        parts[0] = parts[0] + '-review'
        self._gerrit_host = '.'.join(parts)
        self._gerrit_server = 'https://%s' % self._gerrit_host
    return self._gerrit_server

2408
  @classmethod
2409
  def IssueConfigKey(cls):
2410
    return 'gerritissue'
2411

2412 2413 2414 2415 2416 2417 2418 2419
  @classmethod
  def PatchsetConfigKey(cls):
    return 'gerritpatchset'

  @classmethod
  def CodereviewServerConfigKey(cls):
    return 'gerritserver'

2420
  def EnsureAuthenticated(self, force, refresh=None):
2421
    """Best effort check that user is authenticated with Gerrit server."""
2422 2423 2424 2425
    if settings.GetGerritSkipEnsureAuthenticated():
      # For projects with unusual authentication schemes.
      # See http://crbug.com/603378.
      return
2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438
    # Lazy-loader to identify Gerrit and Git hosts.
    if gerrit_util.GceAuthenticator.is_gce():
      return
    self.GetCodereviewServer()
    git_host = self._GetGitHost()
    assert self._gerrit_server and self._gerrit_host
    cookie_auth = gerrit_util.CookiesAuthenticator()

    gerrit_auth = cookie_auth.get_auth_header(self._gerrit_host)
    git_auth = cookie_auth.get_auth_header(git_host)
    if gerrit_auth and git_auth:
      if gerrit_auth == git_auth:
        return
2439
      all_gsrc = cookie_auth.get_auth_header('d0esN0tEx1st.googlesource.com')
2440
      print((
2441
          'WARNING: You have different credentials for Gerrit and git hosts:\n'
2442 2443
          '           %s\n'
          '           %s\n'
2444 2445
          '        Consider running the following command:\n'
          '          git cl creds-check\n'
2446
          '        %s\n'
2447
          '        %s') %
2448
          (git_host, self._gerrit_host,
2449
           ('Hint: delete creds for .googlesource.com' if all_gsrc else ''),
2450 2451
           cookie_auth.get_new_password_message(git_host)))
      if not force:
2452
        confirm_or_exit('If you know what you are doing', action='continue')
2453 2454 2455
      return
    else:
      missing = (
2456 2457
          ([] if gerrit_auth else [self._gerrit_host]) +
          ([] if git_auth else [git_host]))
2458 2459 2460 2461 2462 2463 2464 2465 2466
      DieWithError('Credentials for the following hosts are required:\n'
                   '  %s\n'
                   'These are read from %s (or legacy %s)\n'
                   '%s' % (
                     '\n  '.join(missing),
                     cookie_auth.get_gitcookies_path(),
                     cookie_auth.get_netrc_path(),
                     cookie_auth.get_new_password_message(git_host)))

2467
  def EnsureCanUploadPatchset(self, force):
2468 2469 2470 2471 2472
    if not self.GetIssue():
      return

    # Warm change details cache now to avoid RPCs later, reducing latency for
    # developers.
2473 2474
    self._GetChangeDetail(
        ['DETAILED_ACCOUNTS', 'CURRENT_REVISION', 'CURRENT_COMMIT'])
2475 2476 2477 2478 2479 2480 2481

    status = self._GetChangeDetail()['status']
    if status in ('MERGED', 'ABANDONED'):
      DieWithError('Change %s has been %s, new uploads are not allowed' %
                   (self.GetIssueURL(),
                    'submitted' if status == 'MERGED' else 'abandoned'))

2482 2483 2484 2485 2486 2487 2488 2489
    if gerrit_util.GceAuthenticator.is_gce():
      return
    cookies_user = gerrit_util.CookiesAuthenticator().get_auth_email(
        self._GetGerritHost())
    if self.GetIssueOwner() == cookies_user:
      return
    logging.debug('change %s owner is %s, cookies user is %s',
                  self.GetIssue(), self.GetIssueOwner(), cookies_user)
2490
    # Maybe user has linked accounts or something like that,
2491 2492 2493 2494 2495
    # so ask what Gerrit thinks of this user.
    details = gerrit_util.GetAccountDetails(self._GetGerritHost(), 'self')
    if details['email'] == self.GetIssueOwner():
      return
    if not force:
2496
      print('WARNING: Change %s is owned by %s, but you authenticate to Gerrit '
2497 2498 2499 2500 2501 2502
            'as %s.\n'
            'Uploading may fail due to lack of permissions.' %
            (self.GetIssue(), self.GetIssueOwner(), details['email']))
      confirm_or_exit(action='upload')


2503 2504
  def _PostUnsetIssueProperties(self):
    """Which branch-specific properties to erase when unsetting issue."""
2505
    return ['gerritsquashhash']
2506

2507
  def GetRietveldObjForPresubmit(self):
2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518
    class ThisIsNotRietveldIssue(object):
      def __nonzero__(self):
        # This is a hack to make presubmit_support think that rietveld is not
        # defined, yet still ensure that calls directly result in a decent
        # exception message below.
        return False

      def __getattr__(self, attr):
        print(
            'You aren\'t using Rietveld at the moment, but Gerrit.\n'
            'Using Rietveld in your PRESUBMIT scripts won\'t work.\n'
2519
            'Please, either change your PRESUBMIT to not use rietveld_obj.%s,\n'
2520 2521 2522 2523 2524
            'or use Rietveld for codereview.\n'
            'See also http://crbug.com/579160.' % attr)
        raise NotImplementedError()
    return ThisIsNotRietveldIssue()

2525 2526 2527
  def GetGerritObjForPresubmit(self):
    return presubmit_support.GerritAccessor(self._GetGerritHost())

2528
  def GetStatus(self):
2529 2530 2531 2532
    """Apply a rough heuristic to give a simple summary of an issue's review
    or CQ status, assuming adherence to a common workflow.

    Returns None if no issue for this branch, or one of the following keywords:
2533 2534 2535 2536 2537 2538 2539
      * 'error'   - error from review tool (including deleted issues)
      * 'unsent'  - no reviewers added
      * 'waiting' - waiting for review
      * 'reply'   - waiting for uploader to reply to review
      * 'lgtm'    - Code-Review label has been set
      * 'commit'  - in the commit queue
      * 'closed'  - successfully submitted or abandoned
2540 2541 2542 2543 2544
    """
    if not self.GetIssue():
      return None

    try:
2545 2546
      data = self._GetChangeDetail([
          'DETAILED_LABELS', 'CURRENT_REVISION', 'SUBMITTABLE'])
2547
    except (httplib.HTTPException, GerritChangeNotExists):
2548 2549
      return 'error'

2550
    if data['status'] in ('ABANDONED', 'MERGED'):
2551 2552
      return 'closed'

2553 2554 2555 2556 2557 2558 2559
    if data['labels'].get('Commit-Queue', {}).get('approved'):
      # The section will have an "approved" subsection if anyone has voted
      # the maximum value on the label.
      return 'commit'

    if data['labels'].get('Code-Review', {}).get('approved'):
      return 'lgtm'
2560 2561 2562 2563

    if not data.get('reviewers', {}).get('REVIEWER', []):
      return 'unsent'

2564
    owner = data['owner'].get('_account_id')
2565 2566 2567
    messages = sorted(data.get('messages', []), key=lambda m: m.get('updated'))
    last_message_author = messages.pop().get('author', {})
    while last_message_author:
2568 2569
      if last_message_author.get('email') == COMMIT_BOT_EMAIL:
        # Ignore replies from CQ.
2570
        last_message_author = messages.pop().get('author', {})
2571
        continue
2572 2573 2574 2575
      if last_message_author.get('_account_id') == owner:
        # Most recent message was by owner.
        return 'waiting'
      else:
2576 2577
        # Some reply from non-owner.
        return 'reply'
2578 2579 2580

    # Somehow there are no messages even though there are reviewers.
    return 'unsent'
2581 2582

  def GetMostRecentPatchset(self):
2583
    data = self._GetChangeDetail(['CURRENT_REVISION'])
2584 2585 2586
    patchset = data['revisions'][data['current_revision']]['_number']
    self.SetPatchset(patchset)
    return patchset
2587

2588 2589 2590
  def FetchDescription(self, force=False):
    data = self._GetChangeDetail(['CURRENT_REVISION', 'CURRENT_COMMIT'],
                                 no_cache=force)
2591
    current_rev = data['current_revision']
2592
    return data['revisions'][current_rev]['commit']['message']
2593

2594 2595 2596
  def UpdateDescriptionRemote(self, description, force=False):
    if gerrit_util.HasPendingChangeEdit(self._GetGerritHost(), self.GetIssue()):
      if not force:
2597
        confirm_or_exit(
2598
            'The description cannot be modified while the issue has a pending '
2599 2600
            'unpublished edit. Either publish the edit in the Gerrit web UI '
            'or delete it.\n\n', action='delete the unpublished edit')
2601 2602 2603

      gerrit_util.DeletePendingChangeEdit(self._GetGerritHost(),
                                          self.GetIssue())
2604
    gerrit_util.SetCommitMessage(self._GetGerritHost(), self.GetIssue(),
2605
                                 description, notify='NONE')
2606

2607
  def AddComment(self, message, publish=None):
2608
    gerrit_util.SetReview(self._GetGerritHost(), self.GetIssue(),
2609
                          msg=message, ready=publish)
2610

2611
  def GetCommentsSummary(self, readable=True):
2612
    # DETAILED_ACCOUNTS is to get emails in accounts.
2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637
    messages = self._GetChangeDetail(
        options=['MESSAGES', 'DETAILED_ACCOUNTS']).get('messages', [])
    file_comments = gerrit_util.GetChangeComments(
        self._GetGerritHost(), self.GetIssue())

    # Build dictionary of file comments for easy access and sorting later.
    # {author+date: {path: {patchset: {line: url+message}}}}
    comments = collections.defaultdict(
        lambda: collections.defaultdict(lambda: collections.defaultdict(dict)))
    for path, line_comments in file_comments.iteritems():
      for comment in line_comments:
        if comment.get('tag', '').startswith('autogenerated'):
          continue
        key = (comment['author']['email'], comment['updated'])
        if comment.get('side', 'REVISION') == 'PARENT':
          patchset = 'Base'
        else:
          patchset = 'PS%d' % comment['patch_set']
        line = comment.get('line', 0)
        url = ('https://%s/c/%s/%s/%s#%s%s' %
            (self._GetGerritHost(), self.GetIssue(), comment['patch_set'], path,
             'b' if comment.get('side') == 'PARENT' else '',
             str(line) if line else ''))
        comments[key][path][patchset][line] = (url, comment['message'])

2638
    summary = []
2639 2640 2641 2642
    for msg in messages:
      # Don't bother showing autogenerated messages.
      if msg.get('tag') and msg.get('tag').startswith('autogenerated'):
        continue
2643 2644 2645 2646
      # Gerrit spits out nanoseconds.
      assert len(msg['date'].split('.')[-1]) == 9
      date = datetime.datetime.strptime(msg['date'][:-3],
                                        '%Y-%m-%d %H:%M:%S.%f')
2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668
      message = msg['message']
      key = (msg['author']['email'], msg['date'])
      if key in comments:
        message += '\n'
      for path, patchsets in sorted(comments.get(key, {}).items()):
        if readable:
          message += '\n%s' % path
        for patchset, lines in sorted(patchsets.items()):
          for line, (url, content) in sorted(lines.items()):
            if line:
              line_str = 'Line %d' % line
              path_str = '%s:%d:' % (path, line)
            else:
              line_str = 'File comment'
              path_str = '%s:0:' % path
            if readable:
              message += '\n  %s, %s: %s' % (patchset, line_str, url)
              message += '\n  %s\n' % content
            else:
              message += '\n%s ' % path_str
              message += '\n%s\n' % content

2669 2670
      summary.append(_CommentSummary(
        date=date,
2671
        message=message,
2672 2673 2674 2675 2676 2677 2678 2679
        sender=msg['author']['email'],
        # These could be inferred from the text messages and correlated with
        # Code-Review label maximum, however this is not reliable.
        # Leaving as is until the need arises.
        approval=False,
        disapproval=False,
      ))
    return summary
2680

2681 2682 2683
  def CloseIssue(self):
    gerrit_util.AbandonChange(self._GetGerritHost(), self.GetIssue(), msg='')

2684 2685 2686
  def SubmitIssue(self, wait_for_merge=True):
    gerrit_util.SubmitChange(self._GetGerritHost(), self.GetIssue(),
                             wait_for_merge=wait_for_merge)
2687

2688 2689 2690 2691 2692 2693 2694
  def _GetChangeDetail(self, options=None, issue=None,
                       no_cache=False):
    """Returns details of the issue by querying Gerrit and caching results.

    If fresh data is needed, set no_cache=True which will clear cache and
    thus new data will be fetched from Gerrit.
    """
2695 2696
    options = options or []
    issue = issue or self.GetIssue()
2697
    assert issue, 'issue is required to query Gerrit'
2698

2699 2700 2701 2702 2703
    # Optimization to avoid multiple RPCs:
    if (('CURRENT_REVISION' in options or 'ALL_REVISIONS' in options) and
        'CURRENT_COMMIT' not in options):
      options.append('CURRENT_COMMIT')

2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722
    # Normalize issue and options for consistent keys in cache.
    issue = str(issue)
    options = [o.upper() for o in options]

    # Check in cache first unless no_cache is True.
    if no_cache:
      self._detail_cache.pop(issue, None)
    else:
      options_set = frozenset(options)
      for cached_options_set, data in self._detail_cache.get(issue, []):
        # Assumption: data fetched before with extra options is suitable
        # for return for a smaller set of options.
        # For example, if we cached data for
        #     options=[CURRENT_REVISION, DETAILED_FOOTERS]
        #   and request is for options=[CURRENT_REVISION],
        # THEN we can return prior cached data.
        if options_set.issubset(cached_options_set):
          return data

2723
    try:
2724
      data = gerrit_util.GetChangeDetail(
2725
          self._GetGerritHost(), str(issue), options)
2726 2727
    except gerrit_util.GerritError as e:
      if e.http_status == 404:
2728
        raise GerritChangeNotExists(issue, self.GetCodereviewServer())
2729
      raise
2730 2731

    self._detail_cache.setdefault(issue, []).append((frozenset(options), data))
2732
    return data
2733

2734 2735 2736
  def _GetChangeCommit(self, issue=None):
    issue = issue or self.GetIssue()
    assert issue, 'issue is required to query Gerrit'
2737 2738 2739 2740 2741 2742
    try:
      data = gerrit_util.GetChangeCommit(self._GetGerritHost(), str(issue))
    except gerrit_util.GerritError as e:
      if e.http_status == 404:
        raise GerritChangeNotExists(issue, self.GetCodereviewServer())
      raise
2743 2744
    return data

2745 2746 2747
  def CMDLand(self, force, bypass_hooks, verbose):
    if git_common.is_dirty_git_tree('land'):
      return 1
2748 2749 2750
    detail = self._GetChangeDetail(['CURRENT_REVISION', 'LABELS'])
    if u'Commit-Queue' in detail.get('labels', {}):
      if not force:
2751 2752 2753 2754
        confirm_or_exit('\nIt seems this repository has a Commit Queue, '
                        'which can test and land changes for you. '
                        'Are you sure you wish to bypass it?\n',
                        action='bypass CQ')
2755

2756
    differs = True
2757
    last_upload = self._GitGetBranchConfigValue('gerritsquashhash')
2758 2759
    # Note: git diff outputs nothing if there is no diff.
    if not last_upload or RunGit(['diff', last_upload]).strip():
2760
      print('WARNING: Some changes from local branch haven\'t been uploaded.')
2761 2762 2763 2764
    else:
      if detail['current_revision'] == last_upload:
        differs = False
      else:
2765 2766
        print('WARNING: Local branch contents differ from latest uploaded '
              'patchset.')
2767 2768
    if differs:
      if not force:
2769 2770 2771
        confirm_or_exit(
            'Do you want to submit latest Gerrit patchset and bypass hooks?\n',
            action='submit')
2772
      print('WARNING: Bypassing hooks and submitting latest uploaded patchset.')
2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783
    elif not bypass_hooks:
      hook_results = self.RunHook(
          committing=True,
          may_prompt=not force,
          verbose=verbose,
          change=self.GetChange(self.GetCommonAncestorWithUpstream(), None))
      if not hook_results.should_continue():
        return 1

    self.SubmitIssue(wait_for_merge=True)
    print('Issue %s has been submitted.' % self.GetIssueURL())
2784 2785
    links = self._GetChangeCommit().get('web_links', [])
    for link in links:
2786
      if link.get('name') == 'gitiles' and link.get('url'):
2787
        print('Landed as: %s' % link.get('url'))
2788
        break
2789 2790
    return 0

2791
  def CMDPatchWithParsedIssue(self, parsed_issue_arg, reject, nocommit,
2792
                              directory, force):
2793 2794 2795 2796 2797 2798 2799 2800 2801 2802
    assert not reject
    assert not directory
    assert parsed_issue_arg.valid

    self._changelist.issue = parsed_issue_arg.issue

    if parsed_issue_arg.hostname:
      self._gerrit_host = parsed_issue_arg.hostname
      self._gerrit_server = 'https://%s' % self._gerrit_host

2803 2804
    try:
      detail = self._GetChangeDetail(['ALL_REVISIONS'])
2805
    except GerritChangeNotExists as e:
2806
      DieWithError(str(e))
2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817

    if not parsed_issue_arg.patchset:
      # Use current revision by default.
      revision_info = detail['revisions'][detail['current_revision']]
      patchset = int(revision_info['_number'])
    else:
      patchset = parsed_issue_arg.patchset
      for revision_info in detail['revisions'].itervalues():
        if int(revision_info['_number']) == parsed_issue_arg.patchset:
          break
      else:
2818
        DieWithError('Couldn\'t find patchset %i in change %i' %
2819 2820
                     (parsed_issue_arg.patchset, self.GetIssue()))

2821 2822 2823
    remote_url = self._changelist.GetRemoteUrl()
    if remote_url.endswith('.git'):
      remote_url = remote_url[:-len('.git')]
2824
    fetch_info = revision_info['fetch']['http']
2825 2826 2827 2828 2829

    if remote_url != fetch_info['url']:
      DieWithError('Trying to patch a change from %s but this repo appears '
                   'to be %s.' % (fetch_info['url'], remote_url))

2830
    RunGit(['fetch', fetch_info['url'], fetch_info['ref']])
2831

2832
    if force:
2833 2834
      RunGit(['reset', '--hard', 'FETCH_HEAD'])
      print('Checked out commit for change %i patchset %i locally' %
2835
            (parsed_issue_arg.issue, patchset))
2836 2837 2838
    elif nocommit:
      RunGit(['cherry-pick', '--no-commit', 'FETCH_HEAD'])
      print('Patch applied to index.')
2839 2840 2841 2842 2843 2844 2845 2846 2847
    else:
      RunGit(['cherry-pick', 'FETCH_HEAD'])
      print('Committed patch for change %i patchset %i locally.' %
            (parsed_issue_arg.issue, patchset))
      print('Note: this created a local commit which does not have '
            'the same hash as the one uploaded for review. This will make '
            'uploading changes based on top of this branch difficult.\n'
            'If you want to do that, use "git cl patch --force" instead.')

2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860
    if self.GetBranch():
      self.SetIssue(parsed_issue_arg.issue)
      self.SetPatchset(patchset)
      fetched_hash = RunGit(['rev-parse', 'FETCH_HEAD']).strip()
      self._GitSetBranchConfigValue('last-upload-hash', fetched_hash)
      self._GitSetBranchConfigValue('gerritsquashhash', fetched_hash)
    else:
      print('WARNING: You are in detached HEAD state.\n'
            'The patch has been applied to your checkout, but you will not be '
            'able to upload a new patch set to the gerrit issue.\n'
            'Try using the \'-b\' option if you would like to work on a '
            'branch and/or upload a new patch set.')

2861 2862 2863 2864 2865 2866
    return 0

  @staticmethod
  def ParseIssueURL(parsed_url):
    if not parsed_url.scheme or not parsed_url.scheme.startswith('http'):
      return None
2867 2868
    # Gerrit's new UI is https://domain/c/project/+/<issue_number>[/[patchset]]
    # But old GWT UI is https://domain/#/c/project/+/<issue_number>[/[patchset]]
2869 2870 2871 2872 2873 2874
    # Short urls like https://domain/<issue_number> can be used, but don't allow
    # specifying the patchset (you'd 404), but we allow that here.
    if parsed_url.path == '/':
      part = parsed_url.fragment
    else:
      part = parsed_url.path
2875
    match = re.match('(/c(/.*/\+)?)?/(\d+)(/(\d+)?/?)?$', part)
2876 2877
    if match:
      return _ParsedIssueNumberArgument(
2878 2879
          issue=int(match.group(3)),
          patchset=int(match.group(5)) if match.group(5) else None,
2880 2881
          hostname=parsed_url.netloc,
          codereview='gerrit')
2882 2883
    return None

2884 2885 2886 2887 2888 2889 2890 2891 2892
  def _GerritCommitMsgHookCheck(self, offer_removal):
    hook = os.path.join(settings.GetRoot(), '.git', 'hooks', 'commit-msg')
    if not os.path.exists(hook):
      return
    # Crude attempt to distinguish Gerrit Codereview hook from potentially
    # custom developer made one.
    data = gclient_utils.FileRead(hook)
    if not('From Gerrit Code Review' in data and 'add_ChangeId()' in data):
      return
2893
    print('WARNING: You have Gerrit commit-msg hook installed.\n'
2894
          'It is not necessary for uploading with git cl in squash mode, '
2895 2896 2897
          'and may interfere with it in subtle ways.\n'
          'We recommend you remove the commit-msg hook.')
    if offer_removal:
2898
      if ask_for_explicit_yes('Do you want to remove it now?'):
2899 2900 2901 2902 2903
        gclient_utils.rm_file_or_tree(hook)
        print('Gerrit commit-msg hook removed.')
      else:
        print('OK, will keep Gerrit commit-msg hook in place.')

2904
  def CMDUploadChange(self, options, git_diff_args, custom_cl_base, change):
2905
    """Upload the current branch to Gerrit."""
2906 2907
    if options.squash and options.no_squash:
      DieWithError('Can only use one of --squash or --no-squash')
2908 2909 2910 2911 2912 2913

    if not options.squash and not options.no_squash:
      # Load default for user, repo, squash=true, in this order.
      options.squash = settings.GetSquashGerritUploads()
    elif options.no_squash:
      options.squash = False
2914

2915
    remote, remote_branch = self.GetRemoteBranch()
2916
    branch = GetTargetRef(remote, remote_branch, options.target_branch)
2917

2918 2919 2920
    # This may be None; default fallback value is determined in logic below.
    title = options.title

2921 2922 2923 2924 2925 2926
    # Extract bug number from branch name.
    bug = options.bug
    match = re.match(r'(?:bug|fix)[_-]?(\d+)', self.GetBranch())
    if not bug and match:
      bug = match.group(1)

2927
    if options.squash:
2928
      self._GerritCommitMsgHookCheck(offer_removal=not options.force)
2929 2930 2931 2932 2933
      if self.GetIssue():
        # Try to get the message from a previous upload.
        message = self.GetDescription()
        if not message:
          DieWithError(
2934
              'failed to fetch description from current Gerrit change %d\n'
2935
              '%s' % (self.GetIssue(), self.GetIssueURL()))
2936
        if not title:
2937
          if options.message:
2938 2939 2940
            # When uploading a subsequent patchset, -m|--message is taken
            # as the patchset title if --title was not provided.
            title = options.message.strip()
2941 2942 2943
          else:
            default_title = RunGit(
                ['show', '-s', '--format=%s', 'HEAD']).strip()
2944 2945 2946 2947 2948
            if options.force:
              title = default_title
            else:
              title = ask_for_data(
                  'Title for patchset [%s]: ' % default_title) or default_title
2949 2950 2951 2952 2953 2954 2955
        change_id = self._GetChangeDetail()['change_id']
        while True:
          footer_change_ids = git_footers.get_footer_change_id(message)
          if footer_change_ids == [change_id]:
            break
          if not footer_change_ids:
            message = git_footers.add_footer_change_id(message, change_id)
2956
            print('WARNING: appended missing Change-Id to change description.')
2957 2958 2959 2960 2961 2962 2963 2964
            continue
          # There is already a valid footer but with different or several ids.
          # Doing this automatically is non-trivial as we don't want to lose
          # existing other footers, yet we want to append just 1 desired
          # Change-Id. Thus, just create a new footer, but let user verify the
          # new description.
          message = '%s\n\nChange-Id: %s' % (message, change_id)
          print(
2965
              'WARNING: change %s has Change-Id footer(s):\n'
2966
              '  %s\n'
2967
              'but change has Change-Id %s, according to Gerrit.\n'
2968 2969 2970 2971
              'Please, check the proposed correction to the description, '
              'and edit it if necessary but keep the "Change-Id: %s" footer\n'
              % (self.GetIssue(), '\n  '.join(footer_change_ids), change_id,
                 change_id))
2972
          confirm_or_exit(action='edit')
2973 2974
          if not options.force:
            change_desc = ChangeDescription(message)
2975
            change_desc.prompt(bug=bug)
2976 2977 2978 2979 2980 2981 2982 2983
            message = change_desc.description
            if not message:
              DieWithError("Description is empty. Aborting...")
          # Continue the while loop.
        # Sanity check of this code - we should end up with proper message
        # footer.
        assert [change_id] == git_footers.get_footer_change_id(message)
        change_desc = ChangeDescription(message)
2984 2985 2986 2987
      else:  # if not self.GetIssue()
        if options.message:
          message = options.message
        else:
2988
          message = CreateDescriptionFromLog(git_diff_args)
2989 2990 2991
          if options.title:
            message = options.title + '\n\n' + message
        change_desc = ChangeDescription(message)
2992

2993
        if not options.force:
2994
          change_desc.prompt(bug=bug)
2995 2996 2997
        # On first upload, patchset title is always this string, while
        # --title flag gets converted to first line of message.
        title = 'Initial upload'
2998 2999
        if not change_desc.description:
          DieWithError("Description is empty. Aborting...")
3000
        change_ids = git_footers.get_footer_change_id(change_desc.description)
3001 3002 3003 3004
        if len(change_ids) > 1:
          DieWithError('too many Change-Id footers, at most 1 allowed.')
        if not change_ids:
          # Generate the Change-Id automatically.
3005 3006 3007 3008
          change_desc.set_description(git_footers.add_footer_change_id(
              change_desc.description,
              GenerateGerritChangeId(change_desc.description)))
          change_ids = git_footers.get_footer_change_id(change_desc.description)
3009 3010 3011
          assert len(change_ids) == 1
        change_id = change_ids[0]

3012 3013 3014 3015
      if options.reviewers or options.tbrs or options.add_owners_to:
        change_desc.update_reviewers(options.reviewers, options.tbrs,
                                     options.add_owners_to, change)

3016
      remote, upstream_branch = self.FetchUpstreamTuple(self.GetBranch())
3017 3018
      parent = self._ComputeParent(remote, upstream_branch, custom_cl_base,
                                   options.force, change_desc)
3019
      tree = RunGit(['rev-parse', 'HEAD:']).strip()
3020 3021 3022 3023 3024 3025
      with tempfile.NamedTemporaryFile(delete=False) as desc_tempfile:
        desc_tempfile.write(change_desc.description)
        desc_tempfile.close()
        ref_to_push = RunGit(['commit-tree', tree, '-p', parent,
                              '-F', desc_tempfile.name]).strip()
        os.remove(desc_tempfile.name)
3026 3027
    else:
      change_desc = ChangeDescription(
3028
          options.message or CreateDescriptionFromLog(git_diff_args))
3029 3030 3031 3032 3033
      if not change_desc.description:
        DieWithError("Description is empty. Aborting...")

      if not git_footers.get_footer_change_id(change_desc.description):
        DownloadGerritHook(False)
3034 3035
        change_desc.set_description(
            self._AddChangeIdToCommitMessage(options, git_diff_args))
3036 3037 3038
      if options.reviewers or options.tbrs or options.add_owners_to:
        change_desc.update_reviewers(options.reviewers, options.tbrs,
                                     options.add_owners_to, change)
3039
      ref_to_push = 'HEAD'
3040 3041 3042 3043
      # For no-squash mode, we assume the remote called "origin" is the one we
      # want. It is not worthwhile to support different workflows for
      # no-squash mode.
      parent = 'origin/%s' % branch
3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054
      change_id = git_footers.get_footer_change_id(change_desc.description)[0]

    assert change_desc
    commits = RunGitSilent(['rev-list', '%s..%s' % (parent,
                                                    ref_to_push)]).splitlines()
    if len(commits) > 1:
      print('WARNING: This will upload %d commits. Run the following command '
            'to see which commits will be uploaded: ' % len(commits))
      print('git log %s..%s' % (parent, ref_to_push))
      print('You can also use `git squash-branch` to squash these into a '
            'single commit.')
3055
      confirm_or_exit(action='upload')
3056

3057 3058 3059 3060
    if options.reviewers or options.tbrs or options.add_owners_to:
      change_desc.update_reviewers(options.reviewers, options.tbrs,
                                   options.add_owners_to, change)

3061 3062
    # Extra options that can be specified at push time. Doc:
    # https://gerrit-review.googlesource.com/Documentation/user-upload.html
3063
    refspec_opts = []
3064

3065 3066 3067
    # By default, new changes are started in WIP mode, and subsequent patchsets
    # don't send email. At any time, passing --send-mail will mark the change
    # ready and send email for that particular patch.
3068 3069
    if options.send_mail:
      refspec_opts.append('ready')
3070
      refspec_opts.append('notify=ALL')
3071 3072
    elif not self.GetIssue():
      refspec_opts.append('wip')
3073
    else:
3074
      refspec_opts.append('notify=NONE')
3075

3076
    # TODO(tandrii): options.message should be posted as a comment
3077
    # if --send-mail is set on non-initial upload as Rietveld used to do it.
3078

3079
    if title:
3080 3081
      # Punctuation and whitespace in |title| must be percent-encoded.
      refspec_opts.append('m=' + gerrit_util.PercentEncodeForGitRef(title))
3082

3083
    if options.private:
3084
      refspec_opts.append('private')
3085

3086 3087 3088
    if options.topic:
      # Documentation on Gerrit topics is here:
      # https://gerrit-review.googlesource.com/Documentation/user-upload.html#topic
3089 3090
      refspec_opts.append('topic=%s' % options.topic)

3091
    # Gerrit sorts hashtags, so order is not important.
3092
    hashtags = {change_desc.sanitize_hash_tag(t) for t in options.hashtags}
3093
    if not self.GetIssue():
3094
      hashtags.update(change_desc.get_hash_tags())
3095 3096
    refspec_opts += ['hashtag=%s' % t for t in sorted(hashtags)]

3097 3098 3099 3100 3101 3102
    refspec_suffix = ''
    if refspec_opts:
      refspec_suffix = '%' + ','.join(refspec_opts)
      assert ' ' not in refspec_suffix, (
          'spaces not allowed in refspec: "%s"' % refspec_suffix)
    refspec = '%s:refs/for/%s%s' % (ref_to_push, branch, refspec_suffix)
3103

3104 3105
    try:
      push_stdout = gclient_utils.CheckCallAndFilter(
3106 3107
          ['git', 'push', self.GetRemoteUrl(), refspec],
          print_stdout=True,
3108 3109 3110 3111 3112
          # Flush after every line: useful for seeing progress when running as
          # recipe.
          filter_fn=lambda _: sys.stdout.flush())
    except subprocess2.CalledProcessError:
      DieWithError('Failed to create a change. Please examine output above '
3113
                   'for the reason of the failure.\n'
3114
                   'Hint: run command below to diagnose common Git/Gerrit '
3115 3116 3117
                   'credential problems:\n'
                   '  git cl creds-check\n',
                   change_desc)
3118 3119

    if options.squash:
3120
      regex = re.compile(r'remote:\s+https?://[\w\-\.\+\/#]*/(\d+)\s.*')
3121 3122 3123 3124 3125 3126
      change_numbers = [m.group(1)
                        for m in map(regex.match, push_stdout.splitlines())
                        if m]
      if len(change_numbers) != 1:
        DieWithError(
          ('Created|Updated %d issues on Gerrit, but only 1 expected.\n'
3127
           'Change-Id: %s') % (len(change_numbers), change_id), change_desc)
3128
      self.SetIssue(change_numbers[0])
3129
      self._GitSetBranchConfigValue('gerritsquashhash', ref_to_push)
3130

3131 3132
    reviewers = sorted(change_desc.get_reviewers())

3133
    # Add cc's from the CC_LIST and --cc flag (if any).
3134 3135 3136 3137
    if not options.private:
      cc = self.GetCCList().split(',')
    else:
      cc = []
3138 3139 3140
    if options.cc:
      cc.extend(options.cc)
    cc = filter(None, [email.strip() for email in cc])
3141 3142
    if change_desc.get_cced():
      cc.extend(change_desc.get_cced())
3143 3144 3145 3146 3147

    gerrit_util.AddReviewers(
        self._GetGerritHost(), self.GetIssue(), reviewers, cc,
        notify=bool(options.send_mail))

3148
    if change_desc.get_reviewers(tbr_only=True):
3149 3150 3151 3152 3153
      labels = self._GetChangeDetail(['LABELS']).get('labels', {})
      score = 1
      if 'Code-Review' in labels and 'values' in labels['Code-Review']:
        score = max([int(x) for x in labels['Code-Review']['values'].keys()])
      print('Adding self-LGTM (Code-Review +%d) because of TBRs.' % score)
3154 3155
      gerrit_util.SetReview(
          self._GetGerritHost(), self.GetIssue(),
3156 3157
          msg='Self-approving for TBR',
          labels={'Code-Review': score})
3158

3159 3160
    return 0

3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173
  def _ComputeParent(self, remote, upstream_branch, custom_cl_base, force,
                     change_desc):
    """Computes parent of the generated commit to be uploaded to Gerrit.

    Returns revision or a ref name.
    """
    if custom_cl_base:
      # Try to avoid creating additional unintended CLs when uploading, unless
      # user wants to take this risk.
      local_ref_of_target_remote = self.GetRemoteBranch()[1]
      code, _ = RunGitWithCode(['merge-base', '--is-ancestor', custom_cl_base,
                                local_ref_of_target_remote])
      if code == 1:
3174
        print('\nWARNING: Manually specified base of this CL `%s` '
3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187
              'doesn\'t seem to belong to target remote branch `%s`.\n\n'
              'If you proceed with upload, more than 1 CL may be created by '
              'Gerrit as a result, in turn confusing or crashing git cl.\n\n'
              'If you are certain that specified base `%s` has already been '
              'uploaded to Gerrit as another CL, you may proceed.\n' %
              (custom_cl_base, local_ref_of_target_remote, custom_cl_base))
        if not force:
          confirm_or_exit(
              'Do you take responsibility for cleaning up potential mess '
              'resulting from proceeding with upload?',
              action='upload')
      return custom_cl_base

3188 3189 3190 3191 3192 3193 3194 3195
    if remote != '.':
      return self.GetCommonAncestorWithUpstream()

    # If our upstream branch is local, we base our squashed commit on its
    # squashed version.
    upstream_branch_name = scm.GIT.ShortBranchName(upstream_branch)

    if upstream_branch_name == 'master':
3196
      return self.GetCommonAncestorWithUpstream()
3197 3198

    # Check the squashed hash of the parent.
3199 3200 3201 3202 3203
    # TODO(tandrii): consider checking parent change in Gerrit and using its
    # hash if tree hash of latest parent revision (patchset) in Gerrit matches
    # the tree hash of the parent branch. The upside is less likely bogus
    # requests to reupload parent change just because it's uploadhash is
    # missing, yet the downside likely exists, too (albeit unknown to me yet).
3204 3205 3206 3207 3208 3209 3210 3211 3212 3213 3214 3215 3216 3217 3218 3219 3220
    parent = RunGit(['config',
                     'branch.%s.gerritsquashhash' % upstream_branch_name],
                    error_ok=True).strip()
    # Verify that the upstream branch has been uploaded too, otherwise
    # Gerrit will create additional CLs when uploading.
    if not parent or (RunGitSilent(['rev-parse', upstream_branch + ':']) !=
                      RunGitSilent(['rev-parse', parent + ':'])):
      DieWithError(
          '\nUpload upstream branch %s first.\n'
          'It is likely that this branch has been rebased since its last '
          'upload, so you just need to upload it again.\n'
          '(If you uploaded it with --no-squash, then branch dependencies '
          'are not supported, and you should reupload with --squash.)'
          % upstream_branch_name,
          change_desc)
    return parent

3221 3222 3223 3224 3225 3226 3227 3228 3229
  def _AddChangeIdToCommitMessage(self, options, args):
    """Re-commits using the current message, assumes the commit hook is in
    place.
    """
    log_desc = options.message or CreateDescriptionFromLog(args)
    git_command = ['commit', '--amend', '-m', log_desc]
    RunGit(git_command)
    new_log_desc = CreateDescriptionFromLog(args)
    if git_footers.get_footer_change_id(new_log_desc):
3230
      print('git-cl: Added Change-Id to commit message.')
3231 3232
      return new_log_desc
    else:
3233
      DieWithError('ERROR: Gerrit commit-msg hook not installed.')
3234

3235 3236 3237 3238 3239
  def SetCQState(self, new_state):
    """Sets the Commit-Queue label assuming canonical CQ config for Gerrit."""
    vote_map = {
        _CQState.NONE:    0,
        _CQState.DRY_RUN: 1,
3240
        _CQState.COMMIT:  2,
3241
    }
3242 3243 3244 3245
    labels = {'Commit-Queue': vote_map[new_state]}
    notify = False if new_state == _CQState.DRY_RUN else None
    gerrit_util.SetReview(self._GetGerritHost(), self.GetIssue(),
                          labels=labels, notify=notify)
3246

3247
  def CannotTriggerTryJobReason(self):
3248 3249
    try:
      data = self._GetChangeDetail()
3250 3251
    except GerritChangeNotExists:
      return 'Gerrit doesn\'t know about your change %s' % self.GetIssue()
3252

3253 3254
    if data['status'] in ('ABANDONED', 'MERGED'):
      return 'CL %s is closed' % self.GetIssue()
3255

3256 3257
  def GetTryJobProperties(self, patchset=None):
    """Returns dictionary of properties to launch try job."""
3258 3259 3260 3261 3262 3263 3264 3265
    data = self._GetChangeDetail(['ALL_REVISIONS'])
    patchset = int(patchset or self.GetPatchset())
    assert patchset
    revision_data = None  # Pylint wants it to be defined.
    for revision_data in data['revisions'].itervalues():
      if int(revision_data['_number']) == patchset:
        break
    else:
3266
      raise Exception('Patchset %d is not known in Gerrit change %d' %
3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279
                      (patchset, self.GetIssue()))
    return {
      'patch_issue': self.GetIssue(),
      'patch_set': patchset or self.GetPatchset(),
      'patch_project': data['project'],
      'patch_storage': 'gerrit',
      'patch_ref': revision_data['fetch']['http']['ref'],
      'patch_repository_url': revision_data['fetch']['http']['url'],
      'patch_gerrit_url': self.GetCodereviewServer(),
    }

  def GetIssueOwner(self):
    return self._GetChangeDetail(['DETAILED_ACCOUNTS'])['owner']['email']
3280

3281 3282 3283 3284 3285 3286

_CODEREVIEW_IMPLEMENTATIONS = {
  'rietveld': _RietveldChangelistImpl,
  'gerrit': _GerritChangelistImpl,
}

3287

3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303
def _add_codereview_issue_select_options(parser, extra=""):
  _add_codereview_select_options(parser)

  text = ('Operate on this issue number instead of the current branch\'s '
          'implicit issue.')
  if extra:
    text += ' '+extra
  parser.add_option('-i', '--issue', type=int, help=text)


def _process_codereview_issue_select_options(parser, options):
  _process_codereview_select_options(parser, options)
  if options.issue is not None and not options.forced_codereview:
    parser.error('--issue must be specified with either --rietveld or --gerrit')


3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326
def _add_codereview_select_options(parser):
  """Appends --gerrit and --rietveld options to force specific codereview."""
  parser.codereview_group = optparse.OptionGroup(
      parser, 'EXPERIMENTAL! Codereview override options')
  parser.add_option_group(parser.codereview_group)
  parser.codereview_group.add_option(
      '--gerrit', action='store_true',
      help='Force the use of Gerrit for codereview')
  parser.codereview_group.add_option(
      '--rietveld', action='store_true',
      help='Force the use of Rietveld for codereview')


def _process_codereview_select_options(parser, options):
  if options.gerrit and options.rietveld:
    parser.error('Options --gerrit and --rietveld are mutually exclusive')
  options.forced_codereview = None
  if options.gerrit:
    options.forced_codereview = 'gerrit'
  elif options.rietveld:
    options.forced_codereview = 'rietveld'


3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356 3357 3358 3359 3360 3361
def _get_bug_line_values(default_project, bugs):
  """Given default_project and comma separated list of bugs, yields bug line
  values.

  Each bug can be either:
    * a number, which is combined with default_project
    * string, which is left as is.

  This function may produce more than one line, because bugdroid expects one
  project per line.

  >>> list(_get_bug_line_values('v8', '123,chromium:789'))
      ['v8:123', 'chromium:789']
  """
  default_bugs = []
  others = []
  for bug in bugs.split(','):
    bug = bug.strip()
    if bug:
      try:
        default_bugs.append(int(bug))
      except ValueError:
        others.append(bug)

  if default_bugs:
    default_bugs = ','.join(map(str, default_bugs))
    if default_project:
      yield '%s:%s' % (default_project, default_bugs)
    else:
      yield default_bugs
  for other in sorted(others):
    # Don't bother finding common prefixes, CLs with >2 bugs are very very rare.
    yield other


3362 3363
class ChangeDescription(object):
  """Contains a parsed form of the change description."""
3364
  R_LINE = r'^[ \t]*(TBR|R)[ \t]*=[ \t]*(.*?)[ \t]*$'
3365
  CC_LINE = r'^[ \t]*(CC)[ \t]*=[ \t]*(.*?)[ \t]*$'
3366
  BUG_LINE = r'^[ \t]*(?:(BUG)[ \t]*=|Bug:)[ \t]*(.*?)[ \t]*$'
3367
  CHERRY_PICK_LINE = r'^\(cherry picked from commit [a-fA-F0-9]{40}\)$'
3368 3369 3370 3371
  STRIP_HASH_TAG_PREFIX = r'^(\s*(revert|reland)( "|:)?\s*)*'
  BRACKET_HASH_TAG = r'\s*\[([^\[\]]+)\]'
  COLON_SEPARATED_HASH_TAG = r'^([a-zA-Z0-9_\- ]+):'
  BAD_HASH_TAG_CHUNK = r'[^a-zA-Z0-9]+'
3372 3373

  def __init__(self, description):
3374
    self._description_lines = (description or '').strip().splitlines()
3375

3376
  @property               # www.logilab.org/ticket/89786
3377
  def description(self):  # pylint: disable=method-hidden
3378 3379 3380 3381 3382 3383 3384 3385 3386 3387 3388 3389
    return '\n'.join(self._description_lines)

  def set_description(self, desc):
    if isinstance(desc, basestring):
      lines = desc.splitlines()
    else:
      lines = [line.rstrip() for line in desc]
    while lines and not lines[0]:
      lines.pop(0)
    while lines and not lines[-1]:
      lines.pop(-1)
    self._description_lines = lines
3390

3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401
  def update_reviewers(self, reviewers, tbrs, add_owners_to=None, change=None):
    """Rewrites the R=/TBR= line(s) as a single line each.

    Args:
      reviewers (list(str)) - list of additional emails to use for reviewers.
      tbrs (list(str)) - list of additional emails to use for TBRs.
      add_owners_to (None|'R'|'TBR') - Pass to do an OWNERS lookup for files in
        the change that are missing OWNER coverage. If this is not None, you
        must also pass a value for `change`.
      change (Change) - The Change that should be used for OWNERS lookups.
    """
3402
    assert isinstance(reviewers, list), reviewers
3403 3404
    assert isinstance(tbrs, list), tbrs

3405
    assert add_owners_to in (None, 'TBR', 'R'), add_owners_to
3406
    assert not add_owners_to or change, add_owners_to
3407 3408

    if not reviewers and not tbrs and not add_owners_to:
3409
      return
3410 3411 3412 3413 3414 3415 3416

    reviewers = set(reviewers)
    tbrs = set(tbrs)
    LOOKUP = {
      'TBR': tbrs,
      'R': reviewers,
    }
3417

3418
    # Get the set of R= and TBR= lines and remove them from the description.
3419 3420 3421 3422 3423 3424 3425
    regexp = re.compile(self.R_LINE)
    matches = [regexp.match(line) for line in self._description_lines]
    new_desc = [l for i, l in enumerate(self._description_lines)
                if not matches[i]]
    self.set_description(new_desc)

    # Construct new unified R= and TBR= lines.
3426 3427

    # First, update tbrs/reviewers with names from the R=/TBR= lines (if any).
3428 3429 3430
    for match in matches:
      if not match:
        continue
3431 3432 3433
      LOOKUP[match.group(1)].update(cleanup_list([match.group(2).strip()]))

    # Next, maybe fill in OWNERS coverage gaps to either tbrs/reviewers.
3434
    if add_owners_to:
3435
      owners_db = owners.Database(change.RepositoryRoot(),
3436
                                  fopen=file, os_path=os.path)
3437
      missing_files = owners_db.files_not_covered_by(change.LocalPaths(),
3438
                                                     (tbrs | reviewers))
3439 3440
      LOOKUP[add_owners_to].update(
        owners_db.reviewers_for(missing_files, change.author_email))
3441

3442 3443
    # If any folks ended up in both groups, remove them from tbrs.
    tbrs -= reviewers
3444

3445 3446
    new_r_line = 'R=' + ', '.join(sorted(reviewers)) if reviewers else None
    new_tbr_line = 'TBR=' + ', '.join(sorted(tbrs)) if tbrs else None
3447 3448 3449 3450 3451 3452 3453 3454

    # Put the new lines in the description where the old first R= line was.
    line_loc = next((i for i, match in enumerate(matches) if match), -1)
    if 0 <= line_loc < len(self._description_lines):
      if new_tbr_line:
        self._description_lines.insert(line_loc, new_tbr_line)
      if new_r_line:
        self._description_lines.insert(line_loc, new_r_line)
3455
    else:
3456 3457 3458 3459
      if new_r_line:
        self.append_footer(new_r_line)
      if new_tbr_line:
        self.append_footer(new_tbr_line)
3460

3461
  def prompt(self, bug=None, git_footer=True):
3462
    """Asks the user to update the description."""
3463 3464 3465 3466
    self.set_description([
      '# Enter a description of the change.',
      '# This will be displayed on the codereview site.',
      '# The first line will also be used as the subject of the review.',
3467
      '#--------------------This line is 72 characters long'
3468 3469
      '--------------------',
    ] + self._description_lines)
3470

3471 3472
    regexp = re.compile(self.BUG_LINE)
    if not any((regexp.match(line) for line in self._description_lines)):
3473 3474
      prefix = settings.GetBugPrefix()
      values = list(_get_bug_line_values(prefix, bug or '')) or [prefix]
3475 3476 3477 3478 3479
      if git_footer:
        self.append_footer('Bug: %s' % ', '.join(values))
      else:
        for value in values:
          self.append_footer('BUG=%s' % value)
3480

3481
    content = gclient_utils.RunEditor(self.description, True,
3482
                                      git_editor=settings.GetGitEditor())
3483 3484
    if not content:
      DieWithError('Running editor failed')
3485
    lines = content.splitlines()
3486

3487 3488 3489
    # Strip off comments and default inserted "Bug:" line.
    clean_lines = [line.rstrip() for line in lines if not
                   (line.startswith('#') or line.rstrip() == "Bug:")]
3490
    if not clean_lines:
3491
      DieWithError('No CL description, aborting')
3492
    self.set_description(clean_lines)
3493

3494
  def append_footer(self, line):
3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529
    """Adds a footer line to the description.

    Differentiates legacy "KEY=xxx" footers (used to be called tags) and
    Gerrit's footers in the form of "Footer-Key: footer any value" and ensures
    that Gerrit footers are always at the end.
    """
    parsed_footer_line = git_footers.parse_footer(line)
    if parsed_footer_line:
      # Line is a gerrit footer in the form: Footer-Key: any value.
      # Thus, must be appended observing Gerrit footer rules.
      self.set_description(
          git_footers.add_footer(self.description,
                                 key=parsed_footer_line[0],
                                 value=parsed_footer_line[1]))
      return

    if not self._description_lines:
      self._description_lines.append(line)
      return

    top_lines, gerrit_footers, _ = git_footers.split_footers(self.description)
    if gerrit_footers:
      # git_footers.split_footers ensures that there is an empty line before
      # actual (gerrit) footers, if any. We have to keep it that way.
      assert top_lines and top_lines[-1] == ''
      top_lines, separator = top_lines[:-1], top_lines[-1:]
    else:
      separator = []  # No need for separator if there are no gerrit_footers.

    prev_line = top_lines[-1] if top_lines else ''
    if (not presubmit_support.Change.TAG_LINE_RE.match(prev_line) or
        not presubmit_support.Change.TAG_LINE_RE.match(line)):
      top_lines.append('')
    top_lines.append(line)
    self._description_lines = top_lines + separator + gerrit_footers
3530

3531
  def get_reviewers(self, tbr_only=False):
3532
    """Retrieves the list of reviewers."""
3533
    matches = [re.match(self.R_LINE, line) for line in self._description_lines]
3534 3535 3536
    reviewers = [match.group(2).strip()
                 for match in matches
                 if match and (not tbr_only or match.group(1).upper() == 'TBR')]
3537
    return cleanup_list(reviewers)
3538

3539 3540 3541 3542 3543 3544
  def get_cced(self):
    """Retrieves the list of reviewers."""
    matches = [re.match(self.CC_LINE, line) for line in self._description_lines]
    cced = [match.group(2).strip() for match in matches if match]
    return cleanup_list(cced)

3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575
  def get_hash_tags(self):
    """Extracts and sanitizes a list of Gerrit hashtags."""
    subject = (self._description_lines or ('',))[0]
    subject = re.sub(
        self.STRIP_HASH_TAG_PREFIX, '', subject, flags=re.IGNORECASE)

    tags = []
    start = 0
    bracket_exp = re.compile(self.BRACKET_HASH_TAG)
    while True:
      m = bracket_exp.match(subject, start)
      if not m:
        break
      tags.append(self.sanitize_hash_tag(m.group(1)))
      start = m.end()

    if not tags:
      # Try "Tag: " prefix.
      m = re.match(self.COLON_SEPARATED_HASH_TAG, subject)
      if m:
        tags.append(self.sanitize_hash_tag(m.group(1)))
    return tags

  @classmethod
  def sanitize_hash_tag(cls, tag):
    """Returns a sanitized Gerrit hash tag.

    A sanitized hashtag can be used as a git push refspec parameter value.
    """
    return re.sub(cls.BAD_HASH_TAG_CHUNK, '-', tag).strip('-').lower()

3576 3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597
  def update_with_git_number_footers(self, parent_hash, parent_msg, dest_ref):
    """Updates this commit description given the parent.

    This is essentially what Gnumbd used to do.
    Consult https://goo.gl/WMmpDe for more details.
    """
    assert parent_msg  # No, orphan branch creation isn't supported.
    assert parent_hash
    assert dest_ref
    parent_footer_map = git_footers.parse_footers(parent_msg)
    # This will also happily parse svn-position, which GnumbD is no longer
    # supporting. While we'd generate correct footers, the verifier plugin
    # installed in Gerrit will block such commit (ie git push below will fail).
    parent_position = git_footers.get_position(parent_footer_map)

    # Cherry-picks may have last line obscuring their prior footers,
    # from git_footers perspective. This is also what Gnumbd did.
    cp_line = None
    if (self._description_lines and
        re.match(self.CHERRY_PICK_LINE, self._description_lines[-1])):
      cp_line = self._description_lines.pop()

3598
    top_lines, footer_lines, _ = git_footers.split_footers(self.description)
3599 3600 3601

    # Original-ify all Cr- footers, to avoid re-lands, cherry-picks, or just
    # user interference with actual footers we'd insert below.
3602 3603 3604 3605
    for i, line in enumerate(footer_lines):
      k, v = git_footers.parse_footer(line) or (None, None)
      if k and k.startswith('Cr-'):
        footer_lines[i] = '%s: %s' % ('Cr-Original-' + k[len('Cr-'):], v)
3606 3607

    # Add Position and Lineage footers based on the parent.
3608
    lineage = list(reversed(parent_footer_map.get('Cr-Branched-From', [])))
3609 3610 3611 3612 3613 3614 3615 3616
    if parent_position[0] == dest_ref:
      # Same branch as parent.
      number = int(parent_position[1]) + 1
    else:
      number = 1  # New branch, and extra lineage.
      lineage.insert(0, '%s-%s@{#%d}' % (parent_hash, parent_position[0],
                                         int(parent_position[1])))

3617 3618
    footer_lines.append('Cr-Commit-Position: %s@{#%d}' % (dest_ref, number))
    footer_lines.extend('Cr-Branched-From: %s' % v for v in lineage)
3619 3620 3621 3622 3623 3624

    self._description_lines = top_lines
    if cp_line:
      self._description_lines.append(cp_line)
    if self._description_lines[-1] != '':
      self._description_lines.append('')  # Ensure footer separator.
3625
    self._description_lines.extend(footer_lines)
3626

3627

3628
def get_approving_reviewers(props, disapproval=False):
3629 3630 3631 3632 3633
  """Retrieves the reviewers that approved a CL from the issue properties with
  messages.

  Note that the list may contain reviewers that are not committer, thus are not
  considered by the CQ.
3634 3635

  If disapproval is True, instead returns reviewers who 'not lgtm'd the CL.
3636
  """
3637
  approval_type = 'disapproval' if disapproval else 'approval'
3638 3639 3640 3641
  return sorted(
      set(
        message['sender']
        for message in props['messages']
3642
        if message[approval_type] and message['sender'] in props['reviewers']
3643 3644 3645 3646
      )
  )


3647 3648 3649 3650 3651 3652 3653 3654
def FindCodereviewSettingsFile(filename='codereview.settings'):
  """Finds the given file starting in the cwd and going up.

  Only looks up to the top of the repository unless an
  'inherit-review-settings-ok' file exists in the root of the repository.
  """
  inherit_ok_file = 'inherit-review-settings-ok'
  cwd = os.getcwd()
3655
  root = settings.GetRoot()
3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668
  if os.path.isfile(os.path.join(root, inherit_ok_file)):
    root = '/'
  while True:
    if filename in os.listdir(cwd):
      if os.path.isfile(os.path.join(cwd, filename)):
        return open(os.path.join(cwd, filename))
    if cwd == root:
      break
    cwd = os.path.dirname(cwd)


def LoadCodereviewSettingsFromFile(fileobj):
  """Parse a codereview.settings file and updates hooks."""
3669
  keyvals = gclient_utils.ParseCodereviewSettingsContent(fileobj.read())
3670 3671 3672 3673 3674 3675 3676 3677

  def SetProperty(name, setting, unset_error_ok=False):
    fullname = 'rietveld.' + name
    if setting in keyvals:
      RunGit(['config', fullname, keyvals[setting]])
    else:
      RunGit(['config', '--unset-all', fullname], error_ok=unset_error_ok)

3678 3679
  if not keyvals.get('GERRIT_HOST', False):
    SetProperty('server', 'CODE_REVIEW_SERVER')
3680 3681 3682
  # Only server setting is required. Other settings can be absent.
  # In that case, we ignore errors raised during option deletion attempt.
  SetProperty('cc', 'CC_LIST', unset_error_ok=True)
3683
  SetProperty('private', 'PRIVATE', unset_error_ok=True)
3684 3685
  SetProperty('tree-status-url', 'STATUS', unset_error_ok=True)
  SetProperty('viewvc-url', 'VIEW_VC', unset_error_ok=True)
3686
  SetProperty('bug-prefix', 'BUG_PREFIX', unset_error_ok=True)
3687 3688
  SetProperty('cpplint-regex', 'LINT_REGEX', unset_error_ok=True)
  SetProperty('cpplint-ignore-regex', 'LINT_IGNORE_REGEX', unset_error_ok=True)
3689
  SetProperty('project', 'PROJECT', unset_error_ok=True)
3690 3691
  SetProperty('run-post-upload-hook', 'RUN_POST_UPLOAD_HOOK',
              unset_error_ok=True)
3692

3693
  if 'GERRIT_HOST' in keyvals:
3694 3695
    RunGit(['config', 'gerrit.host', keyvals['GERRIT_HOST']])

3696
  if 'GERRIT_SQUASH_UPLOADS' in keyvals:
3697 3698
    RunGit(['config', 'gerrit.squash-uploads',
            keyvals['GERRIT_SQUASH_UPLOADS']])
3699

3700
  if 'GERRIT_SKIP_ENSURE_AUTHENTICATED' in keyvals:
3701
    RunGit(['config', 'gerrit.skip-ensure-authenticated',
3702 3703
            keyvals['GERRIT_SKIP_ENSURE_AUTHENTICATED']])

3704
  if 'PUSH_URL_CONFIG' in keyvals and 'ORIGIN_URL_CONFIG' in keyvals:
3705 3706 3707
    # should be of the form
    # PUSH_URL_CONFIG: url.ssh://gitrw.chromium.org.pushinsteadof
    # ORIGIN_URL_CONFIG: http://src.chromium.org/git
3708 3709 3710 3711
    RunGit(['config', keyvals['PUSH_URL_CONFIG'],
            keyvals['ORIGIN_URL_CONFIG']])


3712 3713 3714 3715 3716 3717 3718
def urlretrieve(source, destination):
  """urllib is broken for SSL connections via a proxy therefore we
  can't use urllib.urlretrieve()."""
  with open(destination, 'w') as f:
    f.write(urllib2.urlopen(source).read())


3719 3720 3721 3722 3723 3724
def hasSheBang(fname):
  """Checks fname is a #! script."""
  with open(fname) as f:
    return f.read(2).startswith('#!')


3725 3726 3727 3728 3729
# TODO(bpastene) Remove once a cleaner fix to crbug.com/600473 presents itself.
def DownloadHooks(*args, **kwargs):
  pass


3730 3731
def DownloadGerritHook(force):
  """Download and install Gerrit commit-msg hook.
3732 3733 3734 3735 3736 3737

  Args:
    force: True to update hooks. False to install hooks if not present.
  """
  if not settings.GetIsGerrit():
    return
3738
  src = 'https://gerrit-review.googlesource.com/tools/hooks/commit-msg'
3739 3740 3741 3742 3743 3744
  dst = os.path.join(settings.GetRoot(), '.git', 'hooks', 'commit-msg')
  if not os.access(dst, os.X_OK):
    if os.path.exists(dst):
      if not force:
        return
    try:
3745
      urlretrieve(src, dst)
3746 3747 3748 3749 3750
      if not hasSheBang(dst):
        DieWithError('Not a script: %s\n'
                     'You need to download from\n%s\n'
                     'into .git/hooks/commit-msg and '
                     'chmod +x .git/hooks/commit-msg' % (dst, src))
3751 3752 3753 3754
      os.chmod(dst, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
    except Exception:
      if os.path.exists(dst):
        os.remove(dst)
3755 3756 3757 3758
      DieWithError('\nFailed to download hooks.\n'
                   'You need to download from\n%s\n'
                   'into .git/hooks/commit-msg and '
                   'chmod +x .git/hooks/commit-msg' % src)
3759 3760


3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796
def GetRietveldCodereviewSettingsInteractively():
  """Prompt the user for settings."""
  server = settings.GetDefaultServerUrl(error_ok=True)
  prompt = 'Rietveld server (host[:port])'
  prompt += ' [%s]' % (server or DEFAULT_SERVER)
  newserver = ask_for_data(prompt + ':')
  if not server and not newserver:
    newserver = DEFAULT_SERVER
  if newserver:
    newserver = gclient_utils.UpgradeToHttps(newserver)
    if newserver != server:
      RunGit(['config', 'rietveld.server', newserver])

  def SetProperty(initial, caption, name, is_url):
    prompt = caption
    if initial:
      prompt += ' ("x" to clear) [%s]' % initial
    new_val = ask_for_data(prompt + ':')
    if new_val == 'x':
      RunGit(['config', '--unset-all', 'rietveld.' + name], error_ok=True)
    elif new_val:
      if is_url:
        new_val = gclient_utils.UpgradeToHttps(new_val)
      if new_val != initial:
        RunGit(['config', 'rietveld.' + name, new_val])

  SetProperty(settings.GetDefaultCCList(), 'CC list', 'cc', False)
  SetProperty(settings.GetDefaultPrivateFlag(),
              'Private flag (rietveld only)', 'private', False)
  SetProperty(settings.GetTreeStatusUrl(error_ok=True), 'Tree status URL',
              'tree-status-url', False)
  SetProperty(settings.GetViewVCUrl(), 'ViewVC URL', 'viewvc-url', True)
  SetProperty(settings.GetBugPrefix(), 'Bug Prefix', 'bug-prefix', False)
  SetProperty(settings.GetRunPostUploadHook(), 'Run Post Upload Hook',
              'run-post-upload-hook', False)

3797

3798
class _GitCookiesChecker(object):
3799
  """Provides facilities for validating and suggesting fixes to .gitcookies."""
3800

3801 3802 3803 3804 3805 3806 3807
  _GOOGLESOURCE = 'googlesource.com'

  def __init__(self):
    # Cached list of [host, identity, source], where source is either
    # .gitcookies or .netrc.
    self._all_hosts = None

3808 3809 3810 3811 3812 3813
  def ensure_configured_gitcookies(self):
    """Runs checks and suggests fixes to make git use .gitcookies from default
    path."""
    default = gerrit_util.CookiesAuthenticator.get_gitcookies_path()
    configured_path = RunGitSilent(
        ['config', '--global', 'http.cookiefile']).strip()
3814
    configured_path = os.path.expanduser(configured_path)
3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827
    if configured_path:
      self._ensure_default_gitcookies_path(configured_path, default)
    else:
      self._configure_gitcookies_path(default)

  @staticmethod
  def _ensure_default_gitcookies_path(configured_path, default_path):
    assert configured_path
    if configured_path == default_path:
      print('git is already configured to use your .gitcookies from %s' %
            configured_path)
      return

3828
    print('WARNING: You have configured custom path to .gitcookies: %s\n'
3829 3830
          'Gerrit and other depot_tools expect .gitcookies at %s\n' %
          (configured_path, default_path))
3831

3832 3833 3834 3835 3836 3837 3838 3839 3840 3841 3842 3843
    if not os.path.exists(configured_path):
      print('However, your configured .gitcookies file is missing.')
      confirm_or_exit('Reconfigure git to use default .gitcookies?',
                      action='reconfigure')
      RunGit(['config', '--global', 'http.cookiefile', default_path])
      return

    if os.path.exists(default_path):
      print('WARNING: default .gitcookies file already exists %s' %
            default_path)
      DieWithError('Please delete %s manually and re-run git cl creds-check' %
                   default_path)
3844

3845 3846 3847
    confirm_or_exit('Move existing .gitcookies to default location?',
                    action='move')
    shutil.move(configured_path, default_path)
3848
    RunGit(['config', '--global', 'http.cookiefile', default_path])
3849 3850
    print('Moved and reconfigured git to use .gitcookies from %s' %
          default_path)
3851

3852 3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866
  @staticmethod
  def _configure_gitcookies_path(default_path):
    netrc_path = gerrit_util.CookiesAuthenticator.get_netrc_path()
    if os.path.exists(netrc_path):
      print('You seem to be using outdated .netrc for git credentials: %s' %
            netrc_path)
    print('This tool will guide you through setting up recommended '
          '.gitcookies store for git credentials.\n'
          '\n'
          'IMPORTANT: If something goes wrong and you decide to go back, do:\n'
          '  git config --global --unset http.cookiefile\n'
          '  mv %s %s.backup\n\n' % (default_path, default_path))
    confirm_or_exit(action='setup .gitcookies')
    RunGit(['config', '--global', 'http.cookiefile', default_path])
    print('Configured git to use .gitcookies from %s' % default_path)
3867

3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895
  def get_hosts_with_creds(self, include_netrc=False):
    if self._all_hosts is None:
      a = gerrit_util.CookiesAuthenticator()
      self._all_hosts = [
          (h, u, s)
          for h, u, s in itertools.chain(
              ((h, u, '.netrc') for h, (u, _, _) in a.netrc.hosts.iteritems()),
              ((h, u, '.gitcookies') for h, (u, _) in a.gitcookies.iteritems())
          )
          if h.endswith(self._GOOGLESOURCE)
      ]

    if include_netrc:
      return self._all_hosts
    return [(h, u, s) for h, u, s in self._all_hosts if s != '.netrc']

  def print_current_creds(self, include_netrc=False):
    hosts = sorted(self.get_hosts_with_creds(include_netrc=include_netrc))
    if not hosts:
      print('No Git/Gerrit credentials found')
      return
    lengths = [max(map(len, (row[i] for row in hosts))) for i in xrange(3)]
    header = [('Host', 'User', 'Which file'),
              ['=' * l for l in lengths]]
    for row in (header + hosts):
      print('\t'.join((('%%+%ds' % l) % s)
                       for l, s in zip(lengths, row)))

3896 3897
  @staticmethod
  def _parse_identity(identity):
3898 3899
    """Parses identity "git-<username>.domain" into <username> and domain."""
    # Special case: usernames that contain ".", which are generally not
3900 3901 3902 3903 3904 3905
    # distinguishable from sub-domains. But we do know typical domains:
    if identity.endswith('.chromium.org'):
      domain = 'chromium.org'
      username = identity[:-len('.chromium.org')]
    else:
      username, domain = identity.split('.', 1)
3906 3907 3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920 3921 3922 3923 3924 3925 3926
    if username.startswith('git-'):
      username = username[len('git-'):]
    return username, domain

  def _get_usernames_of_domain(self, domain):
    """Returns list of usernames referenced by .gitcookies in a given domain."""
    identities_by_domain = {}
    for _, identity, _ in self.get_hosts_with_creds():
      username, domain = self._parse_identity(identity)
      identities_by_domain.setdefault(domain, []).append(username)
    return identities_by_domain.get(domain)

  def _canonical_git_googlesource_host(self, host):
    """Normalizes Gerrit hosts (with '-review') to Git host."""
    assert host.endswith(self._GOOGLESOURCE)
    # Prefix doesn't include '.' at the end.
    prefix = host[:-(1 + len(self._GOOGLESOURCE))]
    if prefix.endswith('-review'):
      prefix = prefix[:-len('-review')]
    return prefix + '.' + self._GOOGLESOURCE

3927 3928 3929 3930 3931
  def _canonical_gerrit_googlesource_host(self, host):
    git_host = self._canonical_git_googlesource_host(host)
    prefix = git_host.split('.', 1)[0]
    return prefix + '-review.' + self._GOOGLESOURCE

3932 3933 3934 3935 3936 3937
  def _get_counterpart_host(self, host):
    assert host.endswith(self._GOOGLESOURCE)
    git = self._canonical_git_googlesource_host(host)
    gerrit = self._canonical_gerrit_googlesource_host(git)
    return git if gerrit == host else gerrit

3938 3939 3940 3941 3942 3943 3944 3945 3946 3947 3948 3949 3950 3951 3952 3953 3954 3955 3956 3957 3958 3959 3960 3961 3962
  def has_generic_host(self):
    """Returns whether generic .googlesource.com has been configured.

    Chrome Infra recommends to use explicit ${host}.googlesource.com instead.
    """
    for host, _, _ in self.get_hosts_with_creds(include_netrc=False):
      if host == '.' + self._GOOGLESOURCE:
        return True
    return False

  def _get_git_gerrit_identity_pairs(self):
    """Returns map from canonic host to pair of identities (Git, Gerrit).

    One of identities might be None, meaning not configured.
    """
    host_to_identity_pairs = {}
    for host, identity, _ in self.get_hosts_with_creds():
      canonical = self._canonical_git_googlesource_host(host)
      pair = host_to_identity_pairs.setdefault(canonical, [None, None])
      idx = 0 if canonical == host else 1
      pair[idx] = identity
    return host_to_identity_pairs

  def get_partially_configured_hosts(self):
    return set(
3963 3964 3965
        (host if i1 else self._canonical_gerrit_googlesource_host(host))
        for host, (i1, i2) in self._get_git_gerrit_identity_pairs().iteritems()
        if None in (i1, i2) and host != '.' + self._GOOGLESOURCE)
3966 3967 3968

  def get_conflicting_hosts(self):
    return set(
3969 3970
        host
        for host, (i1, i2) in self._get_git_gerrit_identity_pairs().iteritems()
3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995
        if None not in (i1, i2) and i1 != i2)

  def get_duplicated_hosts(self):
    counters = collections.Counter(h for h, _, _ in self.get_hosts_with_creds())
    return set(host for host, count in counters.iteritems() if count > 1)

  _EXPECTED_HOST_IDENTITY_DOMAINS = {
    'chromium.googlesource.com': 'chromium.org',
    'chrome-internal.googlesource.com': 'google.com',
  }

  def get_hosts_with_wrong_identities(self):
    """Finds hosts which **likely** reference wrong identities.

    Note: skips hosts which have conflicting identities for Git and Gerrit.
    """
    hosts = set()
    for host, expected in self._EXPECTED_HOST_IDENTITY_DOMAINS.iteritems():
      pair = self._get_git_gerrit_identity_pairs().get(host)
      if pair and pair[0] == pair[1]:
        _, domain = self._parse_identity(pair[0])
        if domain != expected:
          hosts.add(host)
    return hosts

3996
  @staticmethod
3997
  def _format_hosts(hosts, extra_column_func=None):
3998 3999 4000 4001 4002 4003
    hosts = sorted(hosts)
    assert hosts
    if extra_column_func is None:
      extras = [''] * len(hosts)
    else:
      extras = [extra_column_func(host) for host in hosts]
4004 4005
    tmpl = '%%-%ds    %%-%ds' % (max(map(len, hosts)), max(map(len, extras)))
    lines = []
4006
    for he in zip(hosts, extras):
4007 4008
      lines.append(tmpl % he)
    return lines
4009

4010
  def _find_problems(self):
4011
    if self.has_generic_host():
4012 4013 4014 4015
      yield ('.googlesource.com wildcard record detected',
             ['Chrome Infrastructure team recommends to list full host names '
              'explicitly.'],
             None)
4016 4017 4018

    dups = self.get_duplicated_hosts()
    if dups:
4019 4020 4021
      yield ('The following hosts were defined twice',
             self._format_hosts(dups),
             None)
4022 4023 4024

    partial = self.get_partially_configured_hosts()
    if partial:
4025 4026 4027 4028 4029
      yield ('Credentials should come in pairs for Git and Gerrit hosts. '
             'These hosts are missing',
             self._format_hosts(partial, lambda host: 'but %s defined' %
                self._get_counterpart_host(host)),
             partial)
4030 4031 4032

    conflicting = self.get_conflicting_hosts()
    if conflicting:
4033 4034 4035 4036 4037
      yield ('The following Git hosts have differing credentials from their '
             'Gerrit counterparts',
             self._format_hosts(conflicting, lambda host: '%s vs %s' %
                 tuple(self._get_git_gerrit_identity_pairs()[host])),
             conflicting)
4038 4039 4040

    wrong = self.get_hosts_with_wrong_identities()
    if wrong:
4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073
      yield ('These hosts likely use wrong identity',
             self._format_hosts(wrong, lambda host: '%s but %s recommended' %
                (self._get_git_gerrit_identity_pairs()[host][0],
                 self._EXPECTED_HOST_IDENTITY_DOMAINS[host])),
             wrong)

  def find_and_report_problems(self):
    """Returns True if there was at least one problem, else False."""
    found = False
    bad_hosts = set()
    for title, sublines, hosts in self._find_problems():
      if not found:
        found = True
        print('\n\n.gitcookies problem report:\n')
      bad_hosts.update(hosts or [])
      print('  %s%s' % (title , (':' if sublines else '')))
      if sublines:
        print()
        print('    %s' % '\n    '.join(sublines))
      print()

    if bad_hosts:
      assert found
      print('  You can manually remove corresponding lines in your %s file and '
            'visit the following URLs with correct account to generate '
            'correct credential lines:\n' %
            gerrit_util.CookiesAuthenticator.get_gitcookies_path())
      print('    %s' % '\n    '.join(sorted(set(
          gerrit_util.CookiesAuthenticator().get_new_password_url(
              self._canonical_git_googlesource_host(host))
          for host in bad_hosts
      ))))
    return found
4074

4075 4076 4077 4078 4079 4080

def CMDcreds_check(parser, args):
  """Checks credentials and suggests changes."""
  _, _ = parser.parse_args(args)

  if gerrit_util.GceAuthenticator.is_gce():
4081 4082 4083
    DieWithError(
        'This command is not designed for GCE, are you on a bot?\n'
        'If you need to run this, export SKIP_GCE_AUTH_FOR_GIT=1 in your env.')
4084

4085 4086
  checker = _GitCookiesChecker()
  checker.ensure_configured_gitcookies()
4087

4088
  print('Your .netrc and .gitcookies have credentials for these hosts:')
4089 4090
  checker.print_current_creds(include_netrc=True)

4091
  if not checker.find_and_report_problems():
4092
    print('\nNo problems detected in your .gitcookies file.')
4093 4094
    return 0
  return 1
4095 4096


4097
@subcommand.usage('[repo root containing codereview.settings]')
4098
def CMDconfig(parser, args):
4099
  """Edits configuration for this tree."""
4100

4101
  print('WARNING: git cl config works for Rietveld only.')
4102 4103
  # TODO(tandrii): remove this once we switch to Gerrit.
  # See bugs http://crbug.com/637561 and http://crbug.com/600469.
4104 4105 4106 4107 4108 4109 4110 4111 4112 4113 4114 4115 4116 4117 4118 4119
  parser.add_option('--activate-update', action='store_true',
                    help='activate auto-updating [rietveld] section in '
                         '.git/config')
  parser.add_option('--deactivate-update', action='store_true',
                    help='deactivate auto-updating [rietveld] section in '
                         '.git/config')
  options, args = parser.parse_args(args)

  if options.deactivate_update:
    RunGit(['config', 'rietveld.autoupdate', 'false'])
    return

  if options.activate_update:
    RunGit(['config', '--unset', 'rietveld.autoupdate'])
    return

4120
  if len(args) == 0:
4121
    GetRietveldCodereviewSettingsInteractively()
4122 4123 4124 4125 4126 4127 4128 4129 4130 4131 4132
    return 0

  url = args[0]
  if not url.endswith('codereview.settings'):
    url = os.path.join(url, 'codereview.settings')

  # Load code review settings and download hooks (if available).
  LoadCodereviewSettingsFromFile(urllib2.urlopen(url))
  return 0


4133
def CMDbaseurl(parser, args):
4134
  """Gets or sets base-url for this branch."""
4135 4136 4137 4138
  branchref = RunGit(['symbolic-ref', 'HEAD']).strip()
  branch = ShortBranchName(branchref)
  _, args = parser.parse_args(args)
  if not args:
4139
    print('Current base-url:')
4140 4141 4142
    return RunGit(['config', 'branch.%s.base-url' % branch],
                  error_ok=False).strip()
  else:
4143
    print('Setting base-url to %s' % args[0])
4144 4145 4146 4147
    return RunGit(['config', 'branch.%s.base-url' % branch, args[0]],
                  error_ok=False).strip()


4148 4149 4150
def color_for_status(status):
  """Maps a Changelist status to color, for CMDstatus and other tools."""
  return {
4151
    'unsent': Fore.YELLOW,
4152 4153
    'waiting': Fore.BLUE,
    'reply': Fore.YELLOW,
4154
    'not lgtm': Fore.RED,
4155 4156 4157 4158 4159 4160
    'lgtm': Fore.GREEN,
    'commit': Fore.MAGENTA,
    'closed': Fore.CYAN,
    'error': Fore.WHITE,
  }.get(status, Fore.WHITE)

4161

4162 4163
def get_cl_statuses(changes, fine_grained, max_processes=None):
  """Returns a blocking iterable of (cl, status) for given branches.
4164 4165 4166 4167 4168 4169 4170

  If fine_grained is true, this will fetch CL statuses from the server.
  Otherwise, simply indicate if there's a matching url for the given branches.

  If max_processes is specified, it is used as the maximum number of processes
  to spawn to fetch CL status from the server. Otherwise 1 process per branch is
  spawned.
4171 4172

  See GetStatus() for a list of possible statuses.
4173
  """
4174
  # Silence upload.py otherwise it becomes unwieldy.
4175 4176
  upload.verbosity = 0

4177 4178
  if not changes:
    raise StopIteration()
4179

4180 4181
  if not fine_grained:
    # Fast path which doesn't involve querying codereview servers.
4182
    # Do not use get_approving_reviewers(), since it requires an HTTP request.
4183 4184
    for cl in changes:
      yield (cl, 'waiting' if cl.GetIssueURL() else 'error')
4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213 4214 4215 4216 4217 4218 4219 4220 4221
    return

  # First, sort out authentication issues.
  logging.debug('ensuring credentials exist')
  for cl in changes:
    cl.EnsureAuthenticated(force=False, refresh=True)

  def fetch(cl):
    try:
      return (cl, cl.GetStatus())
    except:
      # See http://crbug.com/629863.
      logging.exception('failed to fetch status for %s:', cl)
      raise

  threads_count = len(changes)
  if max_processes:
    threads_count = max(1, min(threads_count, max_processes))
  logging.debug('querying %d CLs using %d threads', len(changes), threads_count)

  pool = ThreadPool(threads_count)
  fetched_cls = set()
  try:
    it = pool.imap_unordered(fetch, changes).__iter__()
    while True:
      try:
        cl, status = it.next(timeout=5)
      except multiprocessing.TimeoutError:
        break
      fetched_cls.add(cl)
      yield cl, status
  finally:
    pool.close()

  # Add any branches that failed to fetch.
  for cl in set(changes) - fetched_cls:
    yield (cl, 'error')
4222

4223 4224 4225 4226 4227 4228 4229 4230 4231 4232 4233 4234 4235 4236 4237 4238 4239 4240 4241 4242 4243 4244 4245 4246 4247

def upload_branch_deps(cl, args):
  """Uploads CLs of local branches that are dependents of the current branch.

  If the local branch dependency tree looks like:
  test1 -> test2.1 -> test3.1
                   -> test3.2
        -> test2.2 -> test3.3

  and you run "git cl upload --dependencies" from test1 then "git cl upload" is
  run on the dependent branches in this order:
  test2.1, test3.1, test3.2, test2.2, test3.3

  Note: This function does not rebase your local dependent branches. Use it when
        you make a change to the parent branch that will not conflict with its
        dependent branches, and you would like their dependencies updated in
        Rietveld.
  """
  if git_common.is_dirty_git_tree('upload-branch-deps'):
    return 1

  root_branch = cl.GetBranch()
  if root_branch is None:
    DieWithError('Can\'t find dependent branches from detached HEAD state. '
                 'Get on a branch!')
4248
  if not cl.GetIssue() or (not cl.IsGerrit() and not cl.GetPatchset()):
4249 4250 4251 4252 4253 4254 4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265 4266 4267
    DieWithError('Current branch does not have an uploaded CL. We cannot set '
                 'patchset dependencies without an uploaded CL.')

  branches = RunGit(['for-each-ref',
                     '--format=%(refname:short) %(upstream:short)',
                     'refs/heads'])
  if not branches:
    print('No local branches found.')
    return 0

  # Create a dictionary of all local branches to the branches that are dependent
  # on it.
  tracked_to_dependents = collections.defaultdict(list)
  for b in branches.splitlines():
    tokens = b.split()
    if len(tokens) == 2:
      branch_name, tracked = tokens
      tracked_to_dependents[tracked].append(branch_name)

4268 4269
  print()
  print('The dependent local branches of %s are:' % root_branch)
4270 4271 4272 4273 4274
  dependents = []
  def traverse_dependents_preorder(branch, padding=''):
    dependents_to_process = tracked_to_dependents.get(branch, [])
    padding += '  '
    for dependent in dependents_to_process:
4275
      print('%s%s' % (padding, dependent))
4276 4277 4278
      dependents.append(dependent)
      traverse_dependents_preorder(dependent, padding)
  traverse_dependents_preorder(root_branch)
4279
  print()
4280 4281

  if not dependents:
4282
    print('There are no dependent local branches for %s' % root_branch)
4283 4284
    return 0

4285 4286
  confirm_or_exit('This command will checkout all dependent branches and run '
                  '"git cl upload".', action='continue')
4287

4288
  # Add a default patchset title to all upload calls in Rietveld.
4289
  if not cl.IsGerrit():
4290 4291
    args.extend(['-t', 'Updated patchset dependency'])

4292 4293 4294 4295 4296
  # Record all dependents that failed to upload.
  failures = {}
  # Go through all dependents, checkout the branch and upload.
  try:
    for dependent_branch in dependents:
4297 4298 4299
      print()
      print('--------------------------------------')
      print('Running "git cl upload" from %s:' % dependent_branch)
4300
      RunGit(['checkout', '-q', dependent_branch])
4301
      print()
4302 4303
      try:
        if CMDupload(OptionParser(), args) != 0:
4304
          print('Upload failed for %s!' % dependent_branch)
4305
          failures[dependent_branch] = 1
4306
      except:  # pylint: disable=bare-except
4307
        failures[dependent_branch] = 1
4308
      print()
4309 4310 4311 4312
  finally:
    # Swap back to the original root branch.
    RunGit(['checkout', '-q', root_branch])

4313 4314
  print()
  print('Upload complete for dependent branches!')
4315 4316
  for dependent_branch in dependents:
    upload_status = 'failed' if failures.get(dependent_branch) else 'succeeded'
4317 4318
    print('  %s : %s' % (dependent_branch, upload_status))
  print()
4319 4320 4321 4322

  return 0


4323 4324 4325 4326
def CMDarchive(parser, args):
  """Archives and deletes branches associated with closed changelists."""
  parser.add_option(
      '-j', '--maxjobs', action='store', type=int,
4327
      help='The maximum number of jobs to use when retrieving review status.')
4328 4329 4330
  parser.add_option(
      '-f', '--force', action='store_true',
      help='Bypasses the confirmation prompt.')
4331 4332 4333 4334 4335 4336 4337
  parser.add_option(
      '-d', '--dry-run', action='store_true',
      help='Skip the branch tagging and removal steps.')
  parser.add_option(
      '-t', '--notags', action='store_true',
      help='Do not tag archived branches. '
           'Note: local commit history may be lost.')
4338 4339 4340 4341 4342 4343 4344 4345 4346 4347 4348

  auth.add_auth_options(parser)
  options, args = parser.parse_args(args)
  if args:
    parser.error('Unsupported args: %s' % ' '.join(args))
  auth_config = auth.extract_auth_config_from_options(options)

  branches = RunGit(['for-each-ref', '--format=%(refname)', 'refs/heads'])
  if not branches:
    return 0

4349
  print('Finding all branches associated with closed issues...')
4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362
  changes = [Changelist(branchref=b, auth_config=auth_config)
              for b in branches.splitlines()]
  alignment = max(5, max(len(c.GetBranch()) for c in changes))
  statuses = get_cl_statuses(changes,
                             fine_grained=True,
                             max_processes=options.maxjobs)
  proposal = [(cl.GetBranch(),
               'git-cl-archived-%s-%s' % (cl.GetIssue(), cl.GetBranch()))
              for cl, status in statuses
              if status == 'closed']
  proposal.sort()

  if not proposal:
4363
    print('No branches with closed codereview issues found.')
4364 4365 4366 4367
    return 0

  current_branch = GetCurrentBranch()

4368
  print('\nBranches with closed issues that will be archived:\n')
4369 4370 4371 4372 4373 4374 4375 4376 4377 4378 4379 4380 4381 4382
  if options.notags:
    for next_item in proposal:
      print('  ' + next_item[0])
  else:
    print('%*s | %s' % (alignment, 'Branch name', 'Archival tag name'))
    for next_item in proposal:
      print('%*s   %s' % (alignment, next_item[0], next_item[1]))

  # Quit now on precondition failure or if instructed by the user, either
  # via an interactive prompt or by command line flags.
  if options.dry_run:
    print('\nNo changes were made (dry run).\n')
    return 0
  elif any(branch == current_branch for branch, _ in proposal):
4383 4384 4385 4386 4387
    print('You are currently on a branch \'%s\' which is associated with a '
          'closed codereview issue, so archive cannot proceed. Please '
          'checkout another branch and run this command again.' %
          current_branch)
    return 1
4388
  elif not options.force:
4389 4390
    answer = ask_for_data('\nProceed with deletion (Y/n)? ').lower()
    if answer not in ('y', ''):
4391
      print('Aborted.')
4392 4393 4394
      return 1

  for branch, tagname in proposal:
4395 4396
    if not options.notags:
      RunGit(['tag', tagname, branch])
4397
    RunGit(['branch', '-D', branch])
4398

4399
  print('\nJob\'s done!')
4400 4401 4402 4403

  return 0


4404
def CMDstatus(parser, args):
4405 4406 4407
  """Show status of changelists.

  Colors are used to tell the state of the CL unless --fast is used:
4408
    - Blue     waiting for review
4409
    - Yellow   waiting for you to reply to review, or not yet sent
4410
    - Green    LGTM'ed
4411
    - Red      'not LGTM'ed
4412 4413
    - Magenta  in the commit queue
    - Cyan     was committed, branch can be deleted
4414
    - White    error, or unknown status
4415 4416 4417

  Also see 'git cl comments'.
  """
4418
  parser.add_option('--field',
4419
                    help='print only specific field (desc|id|patch|status|url)')
4420 4421
  parser.add_option('-f', '--fast', action='store_true',
                    help='Do not retrieve review status')
4422 4423 4424
  parser.add_option(
      '-j', '--maxjobs', action='store', type=int,
      help='The maximum number of jobs to use when retrieving review status')
4425 4426

  auth.add_auth_options(parser)
4427 4428
  _add_codereview_issue_select_options(
    parser, 'Must be in conjunction with --field.')
4429
  options, args = parser.parse_args(args)
4430
  _process_codereview_issue_select_options(parser, options)
4431 4432
  if args:
    parser.error('Unsupported args: %s' % args)
4433
  auth_config = auth.extract_auth_config_from_options(options)
4434

4435 4436
  if options.issue is not None and not options.field:
    parser.error('--field must be specified with --issue')
4437

4438
  if options.field:
4439 4440
    cl = Changelist(auth_config=auth_config, issue=options.issue,
                    codereview=options.forced_codereview)
4441
    if options.field.startswith('desc'):
4442
      print(cl.GetDescription())
4443 4444 4445
    elif options.field == 'id':
      issueid = cl.GetIssue()
      if issueid:
4446
        print(issueid)
4447
    elif options.field == 'patch':
4448
      patchset = cl.GetMostRecentPatchset()
4449
      if patchset:
4450
        print(patchset)
4451 4452
    elif options.field == 'status':
      print(cl.GetStatus())
4453 4454 4455
    elif options.field == 'url':
      url = cl.GetIssueURL()
      if url:
4456
        print(url)
4457 4458 4459 4460 4461 4462 4463
    return 0

  branches = RunGit(['for-each-ref', '--format=%(refname)', 'refs/heads'])
  if not branches:
    print('No local branch found.')
    return 0

4464
  changes = [
4465
      Changelist(branchref=b, auth_config=auth_config)
4466
      for b in branches.splitlines()]
4467
  print('Branches associated with reviews:')
4468
  output = get_cl_statuses(changes,
4469
                           fine_grained=not options.fast,
4470
                           max_processes=options.maxjobs)
4471

4472
  branch_statuses = {}
4473 4474 4475
  alignment = max(5, max(len(ShortBranchName(c.GetBranch())) for c in changes))
  for cl in sorted(changes, key=lambda c: c.GetBranch()):
    branch = cl.GetBranch()
4476
    while branch not in branch_statuses:
4477 4478 4479 4480 4481 4482 4483 4484
      c, status = output.next()
      branch_statuses[c.GetBranch()] = status
    status = branch_statuses.pop(branch)
    url = cl.GetIssueURL()
    if url and (not status or status == 'error'):
      # The issue probably doesn't exist anymore.
      url += ' (broken)'

4485
    color = color_for_status(status)
4486
    reset = Fore.RESET
4487
    if not setup_color.IS_TTY:
4488 4489
      color = ''
      reset = ''
4490
    status_str = '(%s)' % status if status else ''
4491
    print('  %*s : %s%s %s%s' % (
4492
          alignment, ShortBranchName(branch), color, url,
4493
          status_str, reset))
4494

4495 4496

  branch = GetCurrentBranch()
4497
  print()
4498 4499 4500 4501
  print('Current branch: %s' % branch)
  for cl in changes:
    if cl.GetBranch() == branch:
      break
4502
  if not cl.GetIssue():
4503
    print('No issue assigned.')
4504
    return 0
4505
  print('Issue number: %s (%s)' % (cl.GetIssue(), cl.GetIssueURL()))
4506
  if not options.fast:
4507 4508
    print('Issue description:')
    print(cl.GetDescription(pretty=True))
4509 4510 4511
  return 0


4512 4513 4514 4515 4516 4517 4518
def colorize_CMDstatus_doc():
  """To be called once in main() to add colors to git cl status help."""
  colors = [i for i in dir(Fore) if i[0].isupper()]

  def colorize_line(line):
    for color in colors:
      if color in line.upper():
4519
        # Extract whitespace first and the leading '-'.
4520 4521 4522 4523 4524 4525 4526 4527
        indent = len(line) - len(line.lstrip(' ')) + 1
        return line[:indent] + getattr(Fore, color) + line[indent:] + Fore.RESET
    return line

  lines = CMDstatus.__doc__.splitlines()
  CMDstatus.__doc__ = '\n'.join(colorize_line(l) for l in lines)


4528
def write_json(path, contents):
4529 4530 4531 4532 4533
  if path == '-':
    json.dump(contents, sys.stdout)
  else:
    with open(path, 'w') as f:
      json.dump(contents, f)
4534 4535


4536
@subcommand.usage('[issue_number]')
4537
def CMDissue(parser, args):
4538
  """Sets or displays the current code review issue number.
4539 4540

  Pass issue number 0 to clear the current issue.
4541
  """
4542 4543 4544 4545
  parser.add_option('-r', '--reverse', action='store_true',
                    help='Lookup the branch(es) for the specified issues. If '
                         'no issues are specified, all branches with mapped '
                         'issues will be listed.')
4546 4547
  parser.add_option('--json',
                    help='Path to JSON output file, or "-" for stdout.')
4548
  _add_codereview_select_options(parser)
4549
  options, args = parser.parse_args(args)
4550
  _process_codereview_select_options(parser, options)
4551

4552 4553
  if options.reverse:
    branches = RunGit(['for-each-ref', 'refs/heads',
4554
                       '--format=%(refname)']).splitlines()
4555 4556 4557 4558 4559 4560 4561
    # Reverse issue lookup.
    issue_branch_map = {}
    for branch in branches:
      cl = Changelist(branchref=branch)
      issue_branch_map.setdefault(cl.GetIssue(), []).append(branch)
    if not args:
      args = sorted(issue_branch_map.iterkeys())
4562
    result = {}
4563 4564 4565
    for issue in args:
      if not issue:
        continue
4566
      result[int(issue)] = issue_branch_map.get(int(issue))
4567 4568
      print('Branch for issue number %s: %s' % (
          issue, ', '.join(issue_branch_map.get(int(issue)) or ('None',))))
4569 4570
    if options.json:
      write_json(options.json, result)
4571 4572 4573 4574 4575 4576 4577 4578 4579 4580
    return 0

  if len(args) > 0:
    issue = ParseIssueNumberArgument(args[0], options.forced_codereview)
    if not issue.valid:
      DieWithError('Pass a url or number to set the issue, 0 to unset it, '
                   'or no argument to list it.\n'
                   'Maybe you want to run git cl status?')
    cl = Changelist(codereview=issue.codereview)
    cl.SetIssue(issue.issue)
4581
  else:
4582
    cl = Changelist(codereview=options.forced_codereview)
4583 4584 4585 4586 4587 4588
  print('Issue number: %s (%s)' % (cl.GetIssue(), cl.GetIssueURL()))
  if options.json:
    write_json(options.json, {
      'issue': cl.GetIssue(),
      'issue_url': cl.GetIssueURL(),
    })
4589 4590 4591
  return 0


4592
def CMDcomments(parser, args):
4593 4594 4595
  """Shows or posts review comments for any changelist."""
  parser.add_option('-a', '--add-comment', dest='comment',
                    help='comment to add to an issue')
4596 4597 4598
  parser.add_option('-i', '--issue', dest='issue',
                    help='review issue id (defaults to current issue). '
                         'If given, requires --rietveld or --gerrit')
4599 4600 4601 4602
  parser.add_option('-m', '--machine-readable', dest='readable',
                    action='store_false', default=True,
                    help='output comments in a format compatible with '
                         'editor parsing')
4603
  parser.add_option('-j', '--json-file',
4604
                    help='File to write JSON summary to, or "-" for stdout')
4605
  auth.add_auth_options(parser)
4606
  _add_codereview_select_options(parser)
4607
  options, args = parser.parse_args(args)
4608
  _process_codereview_select_options(parser, options)
4609
  auth_config = auth.extract_auth_config_from_options(options)
4610

4611 4612 4613 4614 4615 4616
  issue = None
  if options.issue:
    try:
      issue = int(options.issue)
    except ValueError:
      DieWithError('A review issue id is expected to be a number')
4617 4618
    if not options.forced_codereview:
      parser.error('--gerrit or --rietveld is required if --issue is specified')
4619

4620
  cl = Changelist(issue=issue,
4621
                  codereview=options.forced_codereview,
4622
                  auth_config=auth_config)
4623 4624 4625 4626 4627

  if options.comment:
    cl.AddComment(options.comment)
    return 0

4628 4629
  summary = sorted(cl.GetCommentsSummary(readable=options.readable),
                   key=lambda c: c.date)
4630 4631
  for comment in summary:
    if comment.disapproval:
4632
      color = Fore.RED
4633
    elif comment.approval:
4634
      color = Fore.GREEN
4635
    elif comment.sender == cl.GetIssueOwner():
4636 4637 4638
      color = Fore.MAGENTA
    else:
      color = Fore.BLUE
4639 4640 4641 4642 4643 4644 4645
    print('\n%s%s   %s%s\n%s' % (
      color,
      comment.date.strftime('%Y-%m-%d %H:%M:%S UTC'),
      comment.sender,
      Fore.RESET,
      '\n'.join('  ' + l for l in comment.message.strip().splitlines())))

4646
  if options.json_file:
4647 4648 4649 4650
    def pre_serialize(c):
      dct = c.__dict__.copy()
      dct['date'] = dct['date'].strftime('%Y-%m-%d %H:%M:%S.%f')
      return dct
4651
    with open(options.json_file, 'wb') as f:
4652
      json.dump(map(pre_serialize, summary), f)
4653 4654 4655
  return 0


4656
@subcommand.usage('[codereview url or issue id]')
4657
def CMDdescription(parser, args):
4658
  """Brings up the editor for the current CL's description."""
4659 4660
  parser.add_option('-d', '--display', action='store_true',
                    help='Display the description instead of opening an editor')
4661
  parser.add_option('-n', '--new-description',
4662 4663
                    help='New description to set for this issue (- for stdin, '
                         '+ to load from local commit HEAD)')
4664 4665 4666
  parser.add_option('-f', '--force', action='store_true',
                    help='Delete any unpublished Gerrit edits for this issue '
                         'without prompting')
4667 4668

  _add_codereview_select_options(parser)
4669
  auth.add_auth_options(parser)
4670 4671 4672
  options, args = parser.parse_args(args)
  _process_codereview_select_options(parser, options)

4673
  target_issue_arg = None
4674
  if len(args) > 0:
4675 4676
    target_issue_arg = ParseIssueNumberArgument(args[0],
                                                options.forced_codereview)
4677
    if not target_issue_arg.valid:
4678
      parser.error('invalid codereview url or CL id')
4679

4680
  auth_config = auth.extract_auth_config_from_options(options)
4681

4682 4683 4684 4685
  kwargs = {
      'auth_config': auth_config,
      'codereview': options.forced_codereview,
  }
4686
  detected_codereview_from_url = False
4687 4688 4689
  if target_issue_arg:
    kwargs['issue'] = target_issue_arg.issue
    kwargs['codereview_host'] = target_issue_arg.hostname
4690 4691 4692
    if target_issue_arg.codereview and not options.forced_codereview:
      detected_codereview_from_url = True
      kwargs['codereview'] = target_issue_arg.codereview
4693 4694

  cl = Changelist(**kwargs)
4695
  if not cl.GetIssue():
4696
    assert not detected_codereview_from_url
4697
    DieWithError('This branch has no associated changelist.')
4698 4699 4700 4701 4702

  if detected_codereview_from_url:
    logging.info('canonical issue/change URL: %s (type: %s)\n',
                 cl.GetIssueURL(), target_issue_arg.codereview)

4703
  description = ChangeDescription(cl.GetDescription())
4704

4705
  if options.display:
4706
    print(description.description)
4707
    return 0
4708 4709 4710 4711 4712

  if options.new_description:
    text = options.new_description
    if text == '-':
      text = '\n'.join(l.rstrip() for l in sys.stdin)
4713 4714 4715 4716
    elif text == '+':
      base_branch = cl.GetCommonAncestorWithUpstream()
      change = cl.GetChange(base_branch, None, local_description=True)
      text = change.FullDescriptionText()
4717 4718 4719

    description.set_description(text)
  else:
4720
    description.prompt(git_footer=cl.IsGerrit())
4721

4722
  if cl.GetDescription().strip() != description.description:
4723
    cl.UpdateDescription(description.description, force=options.force)
4724 4725 4726
  return 0


4727 4728 4729 4730 4731 4732 4733 4734 4735 4736 4737
def CreateDescriptionFromLog(args):
  """Pulls out the commit log to use as a base for the CL description."""
  log_args = []
  if len(args) == 1 and not args[0].endswith('.'):
    log_args = [args[0] + '..']
  elif len(args) == 1 and args[0].endswith('...'):
    log_args = [args[0][:-1]]
  elif len(args) == 2:
    log_args = [args[0] + '..' + args[1]]
  else:
    log_args = args[:]  # Hope for the best!
4738
  return RunGit(['log', '--pretty=format:%s\n\n%b'] + log_args)
4739 4740


4741 4742
def CMDlint(parser, args):
  """Runs cpplint on the current changelist."""
4743 4744
  parser.add_option('--filter', action='append', metavar='-x,+y',
                    help='Comma-separated list of cpplint\'s category-filters')
4745 4746 4747
  auth.add_auth_options(parser)
  options, args = parser.parse_args(args)
  auth_config = auth.extract_auth_config_from_options(options)
4748 4749

  # Access to a protected member _XX of a client class
4750
  # pylint: disable=protected-access
4751 4752 4753 4754
  try:
    import cpplint
    import cpplint_chromium
  except ImportError:
4755
    print('Your depot_tools is missing cpplint.py and/or cpplint_chromium.py.')
4756 4757 4758 4759 4760 4761 4762
    return 1

  # Change the current working directory before calling lint so that it
  # shows the correct base.
  previous_cwd = os.getcwd()
  os.chdir(settings.GetRoot())
  try:
4763
    cl = Changelist(auth_config=auth_config)
4764 4765
    change = cl.GetChange(cl.GetCommonAncestorWithUpstream(), None)
    files = [f.LocalPath() for f in change.AffectedFiles()]
4766
    if not files:
4767
      print('Cannot lint an empty CL')
4768
      return 1
4769 4770

    # Process cpplints arguments if any.
4771 4772 4773 4774
    command = args + files
    if options.filter:
      command = ['--filter=' + ','.join(options.filter)] + command
    filenames = cpplint.ParseArguments(command)
4775 4776 4777 4778 4779 4780 4781

    white_regex = re.compile(settings.GetLintRegex())
    black_regex = re.compile(settings.GetLintIgnoreRegex())
    extra_check_functions = [cpplint_chromium.CheckPointerDeclarationWhitespace]
    for filename in filenames:
      if white_regex.match(filename):
        if black_regex.match(filename):
4782
          print('Ignoring file %s' % filename)
4783 4784 4785 4786
        else:
          cpplint.ProcessFile(filename, cpplint._cpplint_state.verbose_level,
                              extra_check_functions)
      else:
4787
        print('Skipping file %s' % filename)
4788 4789
  finally:
    os.chdir(previous_cwd)
4790
  print('Total errors found: %d\n' % cpplint._cpplint_state.error_count)
4791 4792 4793 4794 4795
  if cpplint._cpplint_state.error_count != 0:
    return 1
  return 0


4796
def CMDpresubmit(parser, args):
4797
  """Runs presubmit tests on the current changelist."""
4798
  parser.add_option('-u', '--upload', action='store_true',
4799
                    help='Run upload hook instead of the push hook')
4800
  parser.add_option('-f', '--force', action='store_true',
4801
                    help='Run checks even if tree is dirty')
4802 4803
  parser.add_option('--all', action='store_true',
                    help='Run checks against all files, not just modified ones')
4804 4805 4806
  auth.add_auth_options(parser)
  options, args = parser.parse_args(args)
  auth_config = auth.extract_auth_config_from_options(options)
4807

4808
  if not options.force and git_common.is_dirty_git_tree('presubmit'):
4809
    print('use --force to check even if tree is dirty.')
4810 4811
    return 1

4812
  cl = Changelist(auth_config=auth_config)
4813 4814 4815
  if args:
    base_branch = args[0]
  else:
4816
    # Default to diffing against the common ancestor of the upstream branch.
4817
    base_branch = cl.GetCommonAncestorWithUpstream()
4818

4819 4820 4821 4822 4823 4824 4825 4826 4827 4828 4829 4830 4831 4832 4833
  if options.all:
    base_change = cl.GetChange(base_branch, None)
    files = [('M', f) for f in base_change.AllFiles()]
    change = presubmit_support.GitChange(
        base_change.Name(),
        base_change.FullDescriptionText(),
        base_change.RepositoryRoot(),
        files,
        base_change.issue,
        base_change.patchset,
        base_change.author_email,
        base_change._upstream)
  else:
    change = cl.GetChange(base_branch, None)

4834 4835 4836 4837
  cl.RunHook(
      committing=not options.upload,
      may_prompt=False,
      verbose=options.verbose,
4838
      change=change)
4839
  return 0
4840 4841


4842 4843 4844 4845 4846 4847 4848 4849 4850 4851 4852 4853 4854 4855 4856 4857 4858 4859 4860 4861 4862 4863 4864 4865 4866 4867 4868 4869 4870 4871
def GenerateGerritChangeId(message):
  """Returns Ixxxxxx...xxx change id.

  Works the same way as
  https://gerrit-review.googlesource.com/tools/hooks/commit-msg
  but can be called on demand on all platforms.

  The basic idea is to generate git hash of a state of the tree, original commit
  message, author/committer info and timestamps.
  """
  lines = []
  tree_hash = RunGitSilent(['write-tree'])
  lines.append('tree %s' % tree_hash.strip())
  code, parent = RunGitWithCode(['rev-parse', 'HEAD~0'], suppress_stderr=False)
  if code == 0:
    lines.append('parent %s' % parent.strip())
  author = RunGitSilent(['var', 'GIT_AUTHOR_IDENT'])
  lines.append('author %s' % author.strip())
  committer = RunGitSilent(['var', 'GIT_COMMITTER_IDENT'])
  lines.append('committer %s' % committer.strip())
  lines.append('')
  # Note: Gerrit's commit-hook actually cleans message of some lines and
  # whitespace. This code is not doing this, but it clearly won't decrease
  # entropy.
  lines.append(message)
  change_hash = RunCommand(['git', 'hash-object', '-t', 'commit', '--stdin'],
                           stdin='\n'.join(lines))
  return 'I%s' % change_hash.strip()


4872
def GetTargetRef(remote, remote_branch, target_branch):
4873 4874 4875 4876 4877 4878 4879 4880 4881
  """Computes the remote branch ref to use for the CL.

  Args:
    remote (str): The git remote for the CL.
    remote_branch (str): The git remote branch for the CL.
    target_branch (str): The target branch specified by the user.
  """
  if not (remote and remote_branch):
    return None
4882

4883
  if target_branch:
4884
    # Canonicalize branch references to the equivalent local full symbolic
4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903
    # refs, which are then translated into the remote full symbolic refs
    # below.
    if '/' not in target_branch:
      remote_branch = 'refs/remotes/%s/%s' % (remote, target_branch)
    else:
      prefix_replacements = (
        ('^((refs/)?remotes/)?branch-heads/', 'refs/remotes/branch-heads/'),
        ('^((refs/)?remotes/)?%s/' % remote,  'refs/remotes/%s/' % remote),
        ('^(refs/)?heads/',                   'refs/remotes/%s/' % remote),
      )
      match = None
      for regex, replacement in prefix_replacements:
        match = re.search(regex, target_branch)
        if match:
          remote_branch = target_branch.replace(match.group(0), replacement)
          break
      if not match:
        # This is a branch path but not one we recognize; use as-is.
        remote_branch = target_branch
4904 4905 4906
  elif remote_branch in REFS_THAT_ALIAS_TO_OTHER_REFS:
    # Handle the refs that need to land in different refs.
    remote_branch = REFS_THAT_ALIAS_TO_OTHER_REFS[remote_branch]
4907

4908 4909 4910 4911 4912 4913 4914 4915 4916 4917 4918 4919
  # Create the true path to the remote branch.
  # Does the following translation:
  # * refs/remotes/origin/refs/diff/test -> refs/diff/test
  # * refs/remotes/origin/master -> refs/heads/master
  # * refs/remotes/branch-heads/test -> refs/branch-heads/test
  if remote_branch.startswith('refs/remotes/%s/refs/' % remote):
    remote_branch = remote_branch.replace('refs/remotes/%s/' % remote, '')
  elif remote_branch.startswith('refs/remotes/%s/' % remote):
    remote_branch = remote_branch.replace('refs/remotes/%s/' % remote,
                                          'refs/heads/')
  elif remote_branch.startswith('refs/remotes/branch-heads'):
    remote_branch = remote_branch.replace('refs/remotes/', 'refs/')
4920

4921 4922 4923
  return remote_branch


4924 4925 4926 4927 4928 4929 4930 4931 4932 4933 4934
def cleanup_list(l):
  """Fixes a list so that comma separated items are put as individual items.

  So that "--reviewers joe@c,john@c --reviewers joa@c" results in
  options.reviewers == sorted(['joe@c', 'john@c', 'joa@c']).
  """
  items = sum((i.split(',') for i in l), [])
  stripped_items = (i.strip() for i in items)
  return sorted(filter(None, stripped_items))


4935
@subcommand.usage('[flags]')
4936
def CMDupload(parser, args):
4937 4938 4939 4940 4941 4942 4943
  """Uploads the current changelist to codereview.

  Can skip dependency patchset uploads for a branch by running:
    git config branch.branch_name.skip-deps-uploads True
  To unset run:
    git config --unset branch.branch_name.skip-deps-uploads
  Can also set the above globally by using the --global flag.
4944 4945 4946 4947

  If the name of the checked out branch starts with "bug-" or "fix-" followed by
  a bug number, this bug number is automatically populated in the CL
  description.
4948 4949 4950 4951 4952 4953

  If subject contains text in square brackets or has "<text>: " prefix, such
  text(s) is treated as Gerrit hashtags. For example, CLs with subjects
    [git-cl] add support for hashtags
    Foo bar: implement foo
  will be hashtagged with "git-cl" and "foo-bar" respectively.
4954
  """
4955 4956
  parser.add_option('--bypass-hooks', action='store_true', dest='bypass_hooks',
                    help='bypass upload presubmit hook')
4957 4958 4959
  parser.add_option('--bypass-watchlists', action='store_true',
                    dest='bypass_watchlists',
                    help='bypass watchlists auto CC-ing reviewers')
4960
  parser.add_option('-f', '--force', action='store_true', dest='force',
4961
                    help="force yes to questions (don't prompt)")
4962 4963
  parser.add_option('--message', '-m', dest='message',
                    help='message for patchset')
4964 4965 4966
  parser.add_option('-b', '--bug',
                    help='pre-populate the bug number(s) for this issue. '
                         'If several, separate with commas')
4967 4968
  parser.add_option('--message-file', dest='message_file',
                    help='file which contains message for patchset')
4969 4970
  parser.add_option('--title', '-t', dest='title',
                    help='title for patchset')
4971
  parser.add_option('-r', '--reviewers',
4972
                    action='append', default=[],
4973
                    help='reviewer email addresses')
4974 4975 4976
  parser.add_option('--tbrs',
                    action='append', default=[],
                    help='TBR email addresses')
4977
  parser.add_option('--cc',
4978
                    action='append', default=[],
4979
                    help='cc email addresses')
4980 4981 4982 4983
  parser.add_option('--hashtag', dest='hashtags',
                    action='append', default=[],
                    help=('Gerrit hashtag for new CL; '
                          'can be applied multiple times'))
4984
  parser.add_option('-s', '--send-mail', action='store_true',
4985
                    help='send email to reviewer(s) and cc(s) immediately')
4986 4987 4988
  parser.add_option('--emulate_svn_auto_props',
                    '--emulate-svn-auto-props',
                    action="store_true",
4989 4990 4991
                    dest="emulate_svn_auto_props",
                    help="Emulate Subversion's auto properties feature.")
  parser.add_option('-c', '--use-commit-queue', action='store_true',
4992 4993
                    help='tell the commit queue to commit this patchset; '
                          'implies --send-mail')
4994
  parser.add_option('--target_branch',
4995
                    '--target-branch',
4996 4997 4998
                    metavar='TARGET',
                    help='Apply CL to remote ref TARGET.  ' +
                         'Default: remote branch head, or master')
4999
  parser.add_option('--squash', action='store_true',
5000
                    help='Squash multiple commits into one')
5001
  parser.add_option('--no-squash', action='store_true',
5002
                    help='Don\'t squash multiple commits into one')
5003
  parser.add_option('--topic', default=None,
5004
                    help='Topic to specify when uploading')
5005 5006 5007 5008
  parser.add_option('--tbr-owners', dest='add_owners_to', action='store_const',
                    const='TBR', help='add a set of OWNERS to TBR')
  parser.add_option('--r-owners', dest='add_owners_to', action='store_const',
                    const='R', help='add a set of OWNERS to R')
5009 5010
  parser.add_option('-d', '--cq-dry-run', dest='cq_dry_run',
                    action='store_true',
5011 5012
                    help='Send the patchset to do a CQ dry run right after '
                         'upload.')
5013 5014 5015
  parser.add_option('--dependencies', action='store_true',
                    help='Uploads CLs of all the local branches that depend on '
                         'the current branch')
5016

5017 5018 5019 5020 5021 5022
  # TODO: remove Rietveld flags
  parser.add_option('--private', action='store_true',
                    help='set the review private (rietveld only)')
  parser.add_option('--email', default=None,
                    help='email address to use to connect to Rietveld')

5023
  orig_args = args
5024
  add_git_similarity(parser)
5025
  auth.add_auth_options(parser)
5026
  _add_codereview_select_options(parser)
5027
  (options, args) = parser.parse_args(args)
5028
  _process_codereview_select_options(parser, options)
5029
  auth_config = auth.extract_auth_config_from_options(options)
5030

5031
  if git_common.is_dirty_git_tree('upload'):
5032 5033
    return 1

5034
  options.reviewers = cleanup_list(options.reviewers)
5035
  options.tbrs = cleanup_list(options.tbrs)
5036 5037
  options.cc = cleanup_list(options.cc)

5038 5039 5040 5041 5042 5043
  if options.message_file:
    if options.message:
      parser.error('only one of --message and --message-file allowed.')
    options.message = gclient_utils.FileRead(options.message_file)
    options.message_file = None

5044 5045 5046
  if options.cq_dry_run and options.use_commit_queue:
    parser.error('only one of --use-commit-queue and --cq-dry-run allowed.')

5047 5048 5049
  if options.use_commit_queue:
    options.send_mail = True

5050 5051 5052
  # For sanity of test expectations, do this otherwise lazy-loading *now*.
  settings.GetIsGerrit()

5053
  cl = Changelist(auth_config=auth_config, codereview=options.forced_codereview)
5054
  return cl.CMDUpload(options, args, orig_args)
5055 5056


Francois Doray's avatar
Francois Doray committed
5057 5058 5059 5060 5061 5062
@subcommand.usage('--description=<description file>')
def CMDsplit(parser, args):
  """Splits a branch into smaller branches and uploads CLs.

  Creates a branch and uploads a CL for each group of files modified in the
  current branch that share a common OWNERS file. In the CL description and
5063
  comment, the string '$directory', is replaced with the directory containing
Francois Doray's avatar
Francois Doray committed
5064 5065 5066
  the shared OWNERS file.
  """
  parser.add_option("-d", "--description", dest="description_file",
5067 5068
                    help="A text file containing a CL description in which "
                         "$directory will be replaced by each CL's directory.")
Francois Doray's avatar
Francois Doray committed
5069 5070
  parser.add_option("-c", "--comment", dest="comment_file",
                    help="A text file containing a CL comment.")
5071 5072 5073 5074
  parser.add_option("-n", "--dry-run", dest="dry_run", action='store_true',
                    default=False,
                    help="List the files and reviewers for each CL that would "
                         "be created, but don't create branches or CLs.")
Francois Doray's avatar
Francois Doray committed
5075 5076 5077 5078 5079 5080 5081 5082 5083
  options, _ = parser.parse_args(args)

  if not options.description_file:
    parser.error('No --description flag specified.')

  def WrappedCMDupload(args):
    return CMDupload(OptionParser(), args)

  return split_cl.SplitCl(options.description_file, options.comment_file,
5084
                          Changelist, WrappedCMDupload, options.dry_run)
Francois Doray's avatar
Francois Doray committed
5085 5086


5087 5088 5089 5090 5091 5092 5093
@subcommand.usage('DEPRECATED')
def CMDdcommit(parser, args):
  """DEPRECATED: Used to commit the current changelist via git-svn."""
  message = ('git-cl no longer supports committing to SVN repositories via '
             'git-svn. You probably want to use `git cl land` instead.')
  print(message)
  return 1
5094 5095


5096 5097 5098 5099 5100
# Two special branches used by git cl land.
MERGE_BRANCH = 'git-cl-commit'
CHERRY_PICK_BRANCH = 'git-cl-cherry-pick'


5101 5102 5103
@subcommand.usage('[upstream branch to apply against]')
def CMDland(parser, args):
  """Commits the current changelist via git.
5104

5105 5106 5107 5108 5109
  In case of Gerrit, uses Gerrit REST api to "submit" the issue, which pushes
  upstream and closes the issue automatically and atomically.

  Otherwise (in case of Rietveld):
    Squashes branch into a single commit.
5110 5111
    Updates commit message with metadata (e.g. pointer to review).
    Pushes the code upstream.
5112
    Updates review and closes.
5113 5114 5115 5116 5117
  """
  parser.add_option('--bypass-hooks', action='store_true', dest='bypass_hooks',
                    help='bypass upload presubmit hook')
  parser.add_option('-m', dest='message',
                    help="override review description")
5118
  parser.add_option('-f', '--force', action='store_true', dest='force',
5119 5120 5121 5122 5123
                    help="force yes to questions (don't prompt)")
  parser.add_option('-c', dest='contributor',
                    help="external contributor for patch (appended to " +
                         "description and used as author for git). Should be " +
                         "formatted as 'First Last <email@example.com>'")
5124
  add_git_similarity(parser)
5125
  auth.add_auth_options(parser)
5126
  (options, args) = parser.parse_args(args)
5127 5128 5129
  auth_config = auth.extract_auth_config_from_options(options)

  cl = Changelist(auth_config=auth_config)
5130

5131 5132 5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145
  # TODO(tandrii): refactor this into _RietveldChangelistImpl method.
  if cl.IsGerrit():
    if options.message:
      # This could be implemented, but it requires sending a new patch to
      # Gerrit, as Gerrit unlike Rietveld versions messages with patchsets.
      # Besides, Gerrit has the ability to change the commit message on submit
      # automatically, thus there is no need to support this option (so far?).
      parser.error('-m MESSAGE option is not supported for Gerrit.')
    if options.contributor:
      parser.error(
          '-c CONTRIBUTOR option is not supported for Gerrit.\n'
          'Before uploading a commit to Gerrit, ensure it\'s author field is '
          'the contributor\'s "name <email>". If you can\'t upload such a '
          'commit for review, contact your repository admin and request'
          '"Forge-Author" permission.')
5146
    if not cl.GetIssue():
5147
      DieWithError('You must upload the change first to Gerrit.\n'
5148 5149
                   '  If you would rather have `git cl land` upload '
                   'automatically for you, see http://crbug.com/642759')
5150 5151 5152
    return cl._codereview_impl.CMDLand(options.force, options.bypass_hooks,
                                       options.verbose)

5153 5154
  current = cl.GetBranch()
  remote, upstream_branch = cl.FetchUpstreamTuple(cl.GetBranch())
5155
  if remote == '.':
5156 5157 5158 5159 5160 5161 5162 5163 5164 5165
    print()
    print('Attempting to push branch %r into another local branch!' % current)
    print()
    print('Either reparent this branch on top of origin/master:')
    print('  git reparent-branch --root')
    print()
    print('OR run `git rebase-update` if you think the parent branch is ')
    print('already committed.')
    print()
    print('  Current parent: %r' % upstream_branch)
5166 5167
    return 1

5168
  if not args:
5169 5170 5171
    # Default to merging against our best guess of the upstream branch.
    args = [cl.GetUpstreamBranch()]

5172 5173
  if options.contributor:
    if not re.match('^.*\s<\S+@\S+>$', options.contributor):
5174
      print("Please provide contributor as 'First Last <email@example.com>'")
5175 5176
      return 1

5177 5178
  base_branch = args[0]

5179
  if git_common.is_dirty_git_tree('land'):
5180 5181 5182 5183 5184 5185 5186
    return 1

  # This rev-list syntax means "show all commits not in my branch that
  # are in base_branch".
  upstream_commits = RunGit(['rev-list', '^' + cl.GetBranchRef(),
                             base_branch]).splitlines()
  if upstream_commits:
5187 5188
    print('Base branch "%s" has %d commits '
          'not in this branch.' % (base_branch, len(upstream_commits)))
5189
    print('Run "git merge %s" before attempting to land.' % base_branch)
5190 5191
    return 1

5192
  merge_base = RunGit(['merge-base', base_branch, 'HEAD']).strip()
5193
  if not options.bypass_hooks:
5194 5195 5196
    author = None
    if options.contributor:
      author = re.search(r'\<(.*)\>', options.contributor).group(1)
5197 5198 5199 5200
    hook_results = cl.RunHook(
        committing=True,
        may_prompt=not options.force,
        verbose=options.verbose,
5201
        change=cl.GetChange(merge_base, author))
5202 5203
    if not hook_results.should_continue():
      return 1
5204

5205 5206 5207 5208
    # Check the tree status if the tree status URL is set.
    status = GetTreeStatus()
    if 'closed' == status:
      print('The tree is closed.  Please wait for it to reopen. Use '
5209
            '"git cl land --bypass-hooks" to commit on a closed tree.')
5210 5211 5212
      return 1
    elif 'unknown' == status:
      print('Unable to determine tree status.  Please verify manually and '
5213
            'use "git cl land --bypass-hooks" to commit on a closed tree.')
5214
      return 1
5215

5216 5217 5218
  change_desc = ChangeDescription(options.message)
  if not change_desc.description and cl.GetIssue():
    change_desc = ChangeDescription(cl.GetDescription())
5219

5220
  if not change_desc.description:
5221
    if not cl.GetIssue() and options.bypass_hooks:
5222
      change_desc = ChangeDescription(CreateDescriptionFromLog([merge_base]))
5223
    else:
5224 5225
      print('No description set.')
      print('Visit %s/edit to set it.' % (cl.GetIssueURL()))
5226
      return 1
5227

5228 5229 5230
  # Keep a separate copy for the commit message, because the commit message
  # contains the link to the Rietveld issue, while the Rietveld message contains
  # the commit viewvc url.
5231
  if cl.GetIssue():
5232
    change_desc.update_reviewers(
5233
        get_approving_reviewers(cl.GetIssueProperties()), [])
5234

5235
  commit_desc = ChangeDescription(change_desc.description)
5236
  if cl.GetIssue():
5237
    # Xcode won't linkify this URL unless there is a non-whitespace character
5238 5239 5240
    # after it. Add a period on a new line to circumvent this. Also add a space
    # before the period to make sure that Gitiles continues to correctly resolve
    # the URL.
5241
    commit_desc.append_footer('Review-Url: %s .' % cl.GetIssueURL())
5242
  if options.contributor:
5243 5244
    commit_desc.append_footer('Patch from %s.' % options.contributor)

5245 5246
  print('Description:')
  print(commit_desc.description)
5247

5248
  branches = [merge_base, cl.GetBranchRef()]
5249
  if not options.force:
5250
    print_stats(options.similarity, options.find_copies, branches)
5251

5252 5253
  # We want to squash all this branch's commits into one commit with the proper
  # description. We do this by doing a "reset --soft" to the base branch (which
5254
  # keeps the working copy the same), then landing that.
5255
  # Delete the special branches if they exist.
5256 5257 5258 5259 5260
  for branch in [MERGE_BRANCH, CHERRY_PICK_BRANCH]:
    showref_cmd = ['show-ref', '--quiet', '--verify', 'refs/heads/%s' % branch]
    result = RunGitWithCode(showref_cmd)
    if result[0] == 0:
      RunGit(['branch', '-D', branch])
5261 5262 5263 5264

  # We might be in a directory that's present in this branch but not in the
  # trunk.  Move up to the top of the tree so that git commands that expect a
  # valid CWD won't fail after we check out the merge branch.
5265
  rel_base_path = settings.GetRelativeRoot()
5266 5267 5268 5269 5270 5271
  if rel_base_path:
    os.chdir(rel_base_path)

  # Stuff our change into the merge branch.
  # We wrap in a try...finally block so if anything goes wrong,
  # we clean up the branches.
5272
  retcode = -1
5273
  revision = None
5274
  try:
5275
    RunGit(['checkout', '-q', '-b', MERGE_BRANCH])
5276
    RunGit(['reset', '--soft', merge_base])
5277
    if options.contributor:
5278 5279 5280 5281 5282
      RunGit(
          [
            'commit', '--author', options.contributor,
            '-m', commit_desc.description,
          ])
5283
    else:
5284
      RunGit(['commit', '-m', commit_desc.description])
5285 5286 5287 5288 5289

    remote, branch = cl.FetchUpstreamTuple(cl.GetBranch())
    mirror = settings.GetGitMirror(remote)
    if mirror:
      pushurl = mirror.url
5290
      git_numberer_enabled = _is_git_numberer_enabled(pushurl, branch)
5291
    else:
5292
      pushurl = remote  # Usually, this is 'origin'.
5293
      git_numberer_enabled = _is_git_numberer_enabled(
5294 5295
          RunGit(['config', 'remote.%s.url' % remote]).strip(), branch)

5296 5297
    retcode = PushToGitWithAutoRebase(
        pushurl, branch, commit_desc.description, git_numberer_enabled)
5298 5299
    if retcode == 0:
      revision = RunGit(['rev-parse', 'HEAD']).strip()
5300 5301 5302
      if git_numberer_enabled:
        change_desc = ChangeDescription(
            RunGit(['show', '-s', '--format=%B', 'HEAD']).strip())
5303
  except:  # pylint: disable=bare-except
5304 5305 5306 5307 5308
    if _IS_BEING_TESTED:
      logging.exception('this is likely your ACTUAL cause of test failure.\n'
                        + '-' * 30 + '8<' + '-' * 30)
      logging.error('\n' + '-' * 30 + '8<' + '-' * 30 + '\n\n\n')
    raise
5309 5310 5311 5312
  finally:
    # And then swap back to the original branch and clean up.
    RunGit(['checkout', '-q', cl.GetBranch()])
    RunGit(['branch', '-D', MERGE_BRANCH])
5313
    RunGit(['branch', '-D', CHERRY_PICK_BRANCH], error_ok=True)
5314

5315
  if not revision:
5316
    print('Failed to push. If this persists, please file a bug.')
5317
    return 1
5318

5319 5320
  if cl.GetIssue():
    viewvc_url = settings.GetViewVCUrl()
5321 5322 5323 5324 5325
    if viewvc_url and revision:
      change_desc.append_footer(
          'Committed: %s%s' % (viewvc_url, revision))
    elif revision:
      change_desc.append_footer('Committed: %s' % (revision,))
5326 5327
    print('Closing issue '
          '(you may be prompted for your codereview password)...')
5328
    cl.UpdateDescription(change_desc.description)
5329
    cl.CloseIssue()
5330
    props = cl.GetIssueProperties()
5331
    patch_num = len(props['patchsets'])
5332 5333
    comment = "Committed patchset #%d (id:%d) manually as %s" % (
        patch_num, props['patchsets'][-1], revision)
5334 5335 5336 5337
    if options.bypass_hooks:
      comment += ' (tree was closed).' if GetTreeStatus() == 'closed' else '.'
    else:
      comment += ' (presubmit successful).'
5338
    cl.RpcServer().add_comment(cl.GetIssue(), comment)
5339

5340 5341
  if os.path.isfile(POSTUPSTREAM_HOOK):
    RunCommand([POSTUPSTREAM_HOOK, merge_base], error_ok=True)
5342

5343
  return 0
5344 5345


5346 5347 5348 5349 5350 5351 5352 5353 5354 5355 5356 5357 5358 5359 5360 5361 5362 5363 5364 5365 5366 5367 5368 5369 5370 5371 5372 5373 5374 5375 5376 5377 5378 5379 5380 5381
def PushToGitWithAutoRebase(remote, branch, original_description,
                            git_numberer_enabled, max_attempts=3):
  """Pushes current HEAD commit on top of remote's branch.

  Attempts to fetch and autorebase on push failures.
  Adds git number footers on the fly.

  Returns integer code from last command.
  """
  cherry = RunGit(['rev-parse', 'HEAD']).strip()
  code = 0
  attempts_left = max_attempts
  while attempts_left:
    attempts_left -= 1
    print('Attempt %d of %d' % (max_attempts - attempts_left, max_attempts))

    # Fetch remote/branch into local cherry_pick_branch, overriding the latter.
    # If fetch fails, retry.
    print('Fetching %s/%s...' % (remote, branch))
    code, out = RunGitWithCode(
        ['retry', 'fetch', remote,
         '+%s:refs/heads/%s' % (branch, CHERRY_PICK_BRANCH)])
    if code:
      print('Fetch failed with exit code %d.' % code)
      print(out.strip())
      continue

    print('Cherry-picking commit on top of latest %s' % branch)
    RunGitWithCode(['checkout', 'refs/heads/%s' % CHERRY_PICK_BRANCH],
                   suppress_stderr=True)
    parent_hash = RunGit(['rev-parse', 'HEAD']).strip()
    code, out = RunGitWithCode(['cherry-pick', cherry])
    if code:
      print('Your patch doesn\'t apply cleanly to \'%s\' HEAD @ %s, '
            'the following files have merge conflicts:' %
            (branch, parent_hash))
5382 5383
      print(RunGit(['-c', 'core.quotePath=false', 'diff',
                    '--name-status', '--diff-filter=U']).strip())
5384 5385 5386 5387 5388 5389 5390 5391 5392 5393 5394 5395 5396 5397 5398 5399 5400 5401 5402 5403 5404 5405
      print('Please rebase your patch and try again.')
      RunGitWithCode(['cherry-pick', '--abort'])
      break

    commit_desc = ChangeDescription(original_description)
    if git_numberer_enabled:
      logging.debug('Adding git number footers')
      parent_msg = RunGit(['show', '-s', '--format=%B', parent_hash]).strip()
      commit_desc.update_with_git_number_footers(parent_hash, parent_msg,
                                                 branch)
      # Ensure timestamps are monotonically increasing.
      timestamp = max(1 + _get_committer_timestamp(parent_hash),
                      _get_committer_timestamp('HEAD'))
      _git_amend_head(commit_desc.description, timestamp)

    code, out = RunGitWithCode(
        ['push', '--porcelain', remote, 'HEAD:%s' % branch])
    print(out)
    if code == 0:
      break
    if IsFatalPushFailure(out):
      print('Fatal push error. Make sure your .netrc credentials and git '
5406 5407 5408 5409
            'user.email are correct and you have push access to the repo.\n'
            'Hint: run command below to diangose common Git/Gerrit credential '
            'problems:\n'
            '  git cl creds-check\n')
5410 5411 5412 5413 5414 5415 5416 5417 5418
      break
  return code


def IsFatalPushFailure(push_stdout):
  """True if retrying push won't help."""
  return '(prohibited by Gerrit)' in push_stdout


5419
@subcommand.usage('<patch url or issue id or issue url>')
5420
def CMDpatch(parser, args):
5421
  """Patches in a code review."""
5422 5423
  parser.add_option('-b', dest='newbranch',
                    help='create a new branch off trunk for the patch')
5424
  parser.add_option('-f', '--force', action='store_true',
5425
                    help='overwrite state on the current or chosen branch')
5426
  parser.add_option('-d', '--directory', action='store', metavar='DIR',
5427
                    help='change to the directory DIR immediately, '
5428
                         'before doing anything else. Rietveld only.')
5429
  parser.add_option('--reject', action='store_true',
5430
                    help='failed patches spew .rej files rather than '
5431
                        'attempting a 3-way merge. Rietveld only.')
5432
  parser.add_option('-n', '--no-commit', action='store_true', dest='nocommit',
5433 5434 5435 5436 5437 5438 5439 5440 5441 5442 5443 5444 5445
                    help='don\'t commit after patch applies. Rietveld only.')


  group = optparse.OptionGroup(
      parser,
      'Options for continuing work on the current issue uploaded from a '
      'different clone (e.g. different machine). Must be used independently '
      'from the other options. No issue number should be specified, and the '
      'branch must have an issue number associated with it')
  group.add_option('--reapply', action='store_true', dest='reapply',
                   help='Reset the branch and reapply the issue.\n'
                        'CAUTION: This will undo any local changes in this '
                        'branch')
5446 5447

  group.add_option('--pull', action='store_true', dest='pull',
5448
                    help='Performs a pull before reapplying.')
5449 5450
  parser.add_option_group(group)

5451
  auth.add_auth_options(parser)
5452
  _add_codereview_select_options(parser)
5453
  (options, args) = parser.parse_args(args)
5454
  _process_codereview_select_options(parser, options)
5455 5456
  auth_config = auth.extract_auth_config_from_options(options)

5457
  if options.reapply:
5458 5459
    if options.newbranch:
      parser.error('--reapply works on the current branch only')
5460
    if len(args) > 0:
5461 5462 5463 5464 5465 5466
      parser.error('--reapply implies no additional arguments')

    cl = Changelist(auth_config=auth_config,
                    codereview=options.forced_codereview)
    if not cl.GetIssue():
      parser.error('current branch must have an associated issue')
5467 5468

    upstream = cl.GetUpstreamBranch()
5469
    if upstream is None:
5470
      parser.error('No upstream branch specified. Cannot reset branch')
5471 5472 5473 5474

    RunGit(['reset', '--hard', upstream])
    if options.pull:
      RunGit(['pull'])
5475

5476 5477
    return cl.CMDPatchIssue(cl.GetIssue(), options.reject, options.nocommit,
                            options.directory)
5478

5479 5480
  if len(args) != 1 or not args[0]:
    parser.error('Must specify issue number or url')
5481

5482 5483 5484 5485 5486 5487 5488 5489 5490 5491 5492 5493 5494 5495 5496 5497
  target_issue_arg = ParseIssueNumberArgument(args[0],
                                              options.forced_codereview)
  if not target_issue_arg.valid:
    parser.error('invalid codereview url or CL id')

  cl_kwargs = {
      'auth_config': auth_config,
      'codereview_host': target_issue_arg.hostname,
      'codereview': options.forced_codereview,
  }
  detected_codereview_from_url = False
  if target_issue_arg.codereview and not options.forced_codereview:
    detected_codereview_from_url = True
    cl_kwargs['codereview'] = target_issue_arg.codereview
    cl_kwargs['issue'] = target_issue_arg.issue

5498
  # We don't want uncommitted changes mixed up with the patch.
5499
  if git_common.is_dirty_git_tree('patch'):
5500 5501
    return 1

5502 5503 5504
  if options.newbranch:
    if options.force:
      RunGit(['branch', '-D', options.newbranch],
5505 5506 5507
             stderr=subprocess2.PIPE, error_ok=True)
    RunGit(['new-branch', options.newbranch])

5508
  cl = Changelist(**cl_kwargs)
5509 5510 5511 5512 5513 5514

  if cl.IsGerrit():
    if options.reject:
      parser.error('--reject is not supported with Gerrit codereview.')
    if options.directory:
      parser.error('--directory is not supported with Gerrit codereview.')
5515

5516 5517 5518 5519 5520
  if detected_codereview_from_url:
    print('canonical issue/change URL: %s (type: %s)\n' %
          (cl.GetIssueURL(), target_issue_arg.codereview))

  return cl.CMDPatchWithParsedIssue(target_issue_arg, options.reject,
5521 5522
                                    options.nocommit, options.directory,
                                    options.force)
5523 5524


5525
def GetTreeStatus(url=None):
5526 5527
  """Fetches the tree status and returns either 'open', 'closed',
  'unknown' or 'unset'."""
5528
  url = url or settings.GetTreeStatusUrl(error_ok=True)
5529 5530 5531 5532 5533 5534 5535 5536 5537
  if url:
    status = urllib2.urlopen(url).read().lower()
    if status.find('closed') != -1 or status == '0':
      return 'closed'
    elif status.find('open') != -1 or status == '1':
      return 'open'
    return 'unknown'
  return 'unset'

5538

5539 5540 5541
def GetTreeStatusReason():
  """Fetches the tree status from a json url and returns the message
  with the reason for the tree to be opened or closed."""
5542 5543
  url = settings.GetTreeStatusUrl()
  json_url = urlparse.urljoin(url, '/current?format=json')
5544 5545 5546 5547 5548
  connection = urllib2.urlopen(json_url)
  status = json.loads(connection.read())
  connection.close()
  return status['message']

5549

5550
def CMDtree(parser, args):
5551
  """Shows the status of the tree."""
5552
  _, args = parser.parse_args(args)
5553 5554
  status = GetTreeStatus()
  if 'unset' == status:
5555
    print('You must configure your tree status URL by running "git cl config".')
5556 5557
    return 2

5558 5559 5560
  print('The tree is %s' % status)
  print()
  print(GetTreeStatusReason())
5561 5562 5563 5564 5565
  if status != 'open':
    return 1
  return 0


5566
def CMDtry(parser, args):
5567
  """Triggers try jobs using either BuildBucket or CQ dry run."""
tandrii's avatar
tandrii committed
5568
  group = optparse.OptionGroup(parser, 'Try job options')
5569
  group.add_option(
tandrii's avatar
tandrii committed
5570 5571 5572 5573 5574 5575
      '-b', '--bot', action='append',
      help=('IMPORTANT: specify ONE builder per --bot flag. Use it multiple '
            'times to specify multiple builders. ex: '
            '"-b win_rel -b win_layout". See '
            'the try server waterfall for the builders name and the tests '
            'available.'))
5576 5577 5578
  group.add_option(
      '-B', '--bucket', default='',
      help=('Buildbucket bucket to send the try requests.'))
5579
  group.add_option(
tandrii's avatar
tandrii committed
5580
      '-m', '--master', default='',
5581
      help=('DEPRECATED, use -B. The try master where to run the builds.'))
5582
  group.add_option(
tandrii's avatar
tandrii committed
5583
      '-r', '--revision',
tandrii's avatar
tandrii committed
5584 5585 5586
      help='Revision to use for the try job; default: the revision will '
           'be determined by the try recipe that builder runs, which usually '
           'defaults to HEAD of origin/master')
5587
  group.add_option(
tandrii's avatar
tandrii committed
5588
      '-c', '--clobber', action='store_true', default=False,
tandrii's avatar
tandrii committed
5589
      help='Force a clobber before building; that is don\'t do an '
tandrii's avatar
tandrii committed
5590
           'incremental build')
5591
  group.add_option(
tandrii's avatar
tandrii committed
5592 5593
      '--project',
      help='Override which project to use. Projects are defined '
tandrii's avatar
tandrii committed
5594 5595
           'in recipe to determine to which repository or directory to '
           'apply the patch')
5596
  group.add_option(
tandrii's avatar
tandrii committed
5597 5598
      '-p', '--property', dest='properties', action='append', default=[],
      help='Specify generic properties in the form -p key1=value1 -p '
tandrii's avatar
tandrii committed
5599 5600 5601 5602
           'key2=value2 etc. The value will be treated as '
           'json if decodable, or as string otherwise. '
           'NOTE: using this may make your try job not usable for CQ, '
           'which will then schedule another try job with default properties')
5603
  group.add_option(
tandrii's avatar
tandrii committed
5604 5605
      '--buildbucket-host', default='cr-buildbucket.appspot.com',
      help='Host of buildbucket. The default host is %default.')
5606
  parser.add_option_group(group)
5607
  auth.add_auth_options(parser)
5608
  _add_codereview_issue_select_options(parser)
5609
  options, args = parser.parse_args(args)
5610
  _process_codereview_issue_select_options(parser, options)
5611
  auth_config = auth.extract_auth_config_from_options(options)
5612

5613 5614 5615
  if options.master and options.master.startswith('luci.'):
    parser.error(
        '-m option does not support LUCI. Please pass -B %s' % options.master)
5616 5617 5618 5619 5620
  # Make sure that all properties are prop=value pairs.
  bad_params = [x for x in options.properties if '=' not in x]
  if bad_params:
    parser.error('Got properties with missing "=": %s' % bad_params)

5621 5622 5623
  if args:
    parser.error('Unknown arguments: %s' % args)

5624 5625
  cl = Changelist(auth_config=auth_config, issue=options.issue,
                  codereview=options.forced_codereview)
5626 5627 5628
  if not cl.GetIssue():
    parser.error('Need to upload first')

5629 5630 5631 5632
  if cl.IsGerrit():
    # HACK: warm up Gerrit change detail cache to save on RPCs.
    cl._codereview_impl._GetChangeDetail(['DETAILED_ACCOUNTS', 'ALL_REVISIONS'])

5633 5634
  error_message = cl.CannotTriggerTryJobReason()
  if error_message:
5635
    parser.error('Can\'t trigger try jobs: %s' % error_message)
5636

5637 5638 5639
  if options.bucket and options.master:
    parser.error('Only one of --bucket and --master may be used.')

5640
  buckets = _get_bucket_map(cl, options, parser)
5641

5642 5643
  # If no bots are listed and we couldn't get a list based on PRESUBMIT files,
  # then we default to triggering a CQ dry run (see http://crbug.com/625697).
5644 5645
  if not buckets:
    if options.verbose:
5646 5647 5648
      print('git cl try with no bots now defaults to CQ dry run.')
    print('Scheduling CQ dry run on: %s' % cl.GetIssueURL())
    return cl.SetCQState(_CQState.DRY_RUN)
5649 5650

  for builders in buckets.itervalues():
5651
    if any('triggered' in b for b in builders):
5652
      print('ERROR You are trying to send a job to a triggered bot. This type '
5653 5654
            'of bot requires an initial job from a parent (usually a builder). '
            'Instead send your job to the parent.\n'
5655
            'Bot list: %s' % builders, file=sys.stderr)
5656
      return 1
5657

5658
  patchset = cl.GetMostRecentPatchset()
5659 5660 5661
  # TODO(tandrii): Checking local patchset against remote patchset is only
  # supported for Rietveld. Extend it to Gerrit or remove it completely.
  if not cl.IsGerrit() and patchset != cl.GetPatchset():
5662 5663 5664 5665 5666 5667
    print('Warning: Codereview server has newer patchsets (%s) than most '
          'recent upload from local checkout (%s). Did a previous upload '
          'fail?\n'
          'By default, git cl try uses the latest patchset from '
          'codereview, continuing to use patchset %s.\n' %
          (patchset, cl.GetPatchset(), patchset))
5668

5669
  try:
5670 5671
    _trigger_try_jobs(auth_config, cl, buckets, options, 'git_cl_try',
                      patchset)
5672 5673 5674
  except BuildbucketResponseException as ex:
    print('ERROR: %s' % ex)
    return 1
5675 5676 5677
  return 0


5678
def CMDtry_results(parser, args):
tandrii's avatar
tandrii committed
5679 5680
  """Prints info about try jobs associated with current CL."""
  group = optparse.OptionGroup(parser, 'Try job results options')
5681
  group.add_option(
tandrii's avatar
tandrii committed
5682
      '-p', '--patchset', type=int, help='patchset number if not current.')
5683
  group.add_option(
tandrii's avatar
tandrii committed
5684
      '--print-master', action='store_true', help='print master name as well.')
5685
  group.add_option(
tandrii's avatar
tandrii committed
5686 5687
      '--color', action='store_true', default=setup_color.IS_TTY,
      help='force color output, useful when piping output.')
5688
  group.add_option(
tandrii's avatar
tandrii committed
5689 5690
      '--buildbucket-host', default='cr-buildbucket.appspot.com',
      help='Host of buildbucket. The default host is %default.')
5691
  group.add_option(
5692 5693
      '--json', help=('Path of JSON output file to write try job results to,'
                      'or "-" for stdout.'))
5694 5695
  parser.add_option_group(group)
  auth.add_auth_options(parser)
5696
  _add_codereview_issue_select_options(parser)
5697
  options, args = parser.parse_args(args)
5698
  _process_codereview_issue_select_options(parser, options)
5699 5700 5701 5702
  if args:
    parser.error('Unrecognized args: %s' % ' '.join(args))

  auth_config = auth.extract_auth_config_from_options(options)
5703 5704 5705
  cl = Changelist(
      issue=options.issue, codereview=options.forced_codereview,
      auth_config=auth_config)
5706 5707 5708
  if not cl.GetIssue():
    parser.error('Need to upload first')

5709 5710 5711 5712 5713 5714
  patchset = options.patchset
  if not patchset:
    patchset = cl.GetMostRecentPatchset()
    if not patchset:
      parser.error('Codereview doesn\'t know about issue %s. '
                   'No access to issue or wrong issue number?\n'
5715
                   'Either upload first, or pass --patchset explicitly' %
5716 5717
                   cl.GetIssue())

5718 5719 5720
    # TODO(tandrii): Checking local patchset against remote patchset is only
    # supported for Rietveld. Extend it to Gerrit or remove it completely.
    if not cl.IsGerrit() and patchset != cl.GetPatchset():
5721 5722 5723
      print('Warning: Codereview server has newer patchsets (%s) than most '
            'recent upload from local checkout (%s). Did a previous upload '
            'fail?\n'
5724 5725
            'By default, git cl try-results uses the latest patchset from '
            'codereview, continuing to use patchset %s.\n' %
5726
            (patchset, cl.GetPatchset(), patchset))
5727
  try:
5728
    jobs = fetch_try_jobs(auth_config, cl, options.buildbucket_host, patchset)
5729
  except BuildbucketResponseException as ex:
5730
    print('Buildbucket error: %s' % ex)
5731
    return 1
5732 5733 5734 5735
  if options.json:
    write_try_results_json(options.json, jobs)
  else:
    print_try_jobs(options, jobs)
5736 5737 5738
  return 0


5739
@subcommand.usage('[new upstream branch]')
5740
def CMDupstream(parser, args):
5741
  """Prints or sets the name of the upstream branch, if any."""
5742
  _, args = parser.parse_args(args)
5743
  if len(args) > 1:
5744
    parser.error('Unrecognized args: %s' % ' '.join(args))
5745

5746
  cl = Changelist()
5747 5748
  if args:
    # One arg means set upstream branch.
5749
    branch = cl.GetBranch()
5750
    RunGit(['branch', '--set-upstream-to', args[0], branch])
5751
    cl = Changelist()
5752
    print('Upstream branch set to %s' % (cl.GetUpstreamBranch(),))
5753 5754 5755

    # Clear configured merge-base, if there is one.
    git_common.remove_merge_base(branch)
5756
  else:
5757
    print(cl.GetUpstreamBranch())
5758 5759 5760
  return 0


5761 5762 5763 5764 5765 5766 5767 5768
def CMDweb(parser, args):
  """Opens the current CL in the web browser."""
  _, args = parser.parse_args(args)
  if args:
    parser.error('Unrecognized args: %s' % ' '.join(args))

  issue_url = Changelist().GetIssueURL()
  if not issue_url:
5769
    print('ERROR No issue to open', file=sys.stderr)
5770 5771 5772 5773 5774 5775
    return 1

  webbrowser.open(issue_url)
  return 0


5776
def CMDset_commit(parser, args):
5777
  """Sets the commit bit to trigger the Commit Queue."""
5778 5779 5780 5781
  parser.add_option('-d', '--dry-run', action='store_true',
                    help='trigger in dry run mode')
  parser.add_option('-c', '--clear', action='store_true',
                    help='stop CQ run, if any')
5782
  auth.add_auth_options(parser)
5783
  _add_codereview_issue_select_options(parser)
5784
  options, args = parser.parse_args(args)
5785
  _process_codereview_issue_select_options(parser, options)
5786
  auth_config = auth.extract_auth_config_from_options(options)
5787 5788
  if args:
    parser.error('Unrecognized args: %s' % ' '.join(args))
5789 5790 5791
  if options.dry_run and options.clear:
    parser.error('Make up your mind: both --dry-run and --clear not allowed')

5792 5793
  cl = Changelist(auth_config=auth_config, issue=options.issue,
                  codereview=options.forced_codereview)
5794
  if options.clear:
5795
    state = _CQState.NONE
5796 5797 5798 5799 5800 5801
  elif options.dry_run:
    state = _CQState.DRY_RUN
  else:
    state = _CQState.COMMIT
  if not cl.GetIssue():
    parser.error('Must upload the issue first')
5802
  cl.SetCQState(state)
5803 5804 5805
  return 0


5806
def CMDset_close(parser, args):
5807
  """Closes the issue."""
5808
  _add_codereview_issue_select_options(parser)
5809 5810
  auth.add_auth_options(parser)
  options, args = parser.parse_args(args)
5811
  _process_codereview_issue_select_options(parser, options)
5812
  auth_config = auth.extract_auth_config_from_options(options)
5813 5814
  if args:
    parser.error('Unrecognized args: %s' % ' '.join(args))
5815 5816
  cl = Changelist(auth_config=auth_config, issue=options.issue,
                  codereview=options.forced_codereview)
5817
  # Ensure there actually is an issue to close.
5818 5819
  if not cl.GetIssue():
    DieWithError('ERROR No issue to close')
5820 5821 5822 5823
  cl.CloseIssue()
  return 0


5824
def CMDdiff(parser, args):
5825
  """Shows differences between local tree and last upload."""
5826 5827 5828 5829 5830
  parser.add_option(
      '--stat',
      action='store_true',
      dest='stat',
      help='Generate a diffstat')
5831 5832 5833 5834 5835
  auth.add_auth_options(parser)
  options, args = parser.parse_args(args)
  auth_config = auth.extract_auth_config_from_options(options)
  if args:
    parser.error('Unrecognized args: %s' % ' '.join(args))
5836

5837
  cl = Changelist(auth_config=auth_config)
5838
  issue = cl.GetIssue()
5839
  branch = cl.GetBranch()
5840 5841
  if not issue:
    DieWithError('No issue found for current branch (%s)' % branch)
5842 5843 5844 5845 5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857

  base = cl._GitGetBranchConfigValue('last-upload-hash')
  if not base:
    base = cl._GitGetBranchConfigValue('gerritsquashhash')
  if not base:
    detail = cl._GetChangeDetail(['CURRENT_REVISION', 'CURRENT_COMMIT'])
    revision_info = detail['revisions'][detail['current_revision']]
    fetch_info = revision_info['fetch']['http']
    RunGit(['fetch', fetch_info['url'], fetch_info['ref']])
    base = 'FETCH_HEAD'

  cmd = ['git', 'diff']
  if options.stat:
    cmd.append('--stat')
  cmd.append(base)
  subprocess2.check_call(cmd)
5858 5859 5860 5861

  return 0


5862
def CMDowners(parser, args):
5863
  """Finds potential owners for reviewing."""
5864 5865 5866 5867
  parser.add_option(
      '--no-color',
      action='store_true',
      help='Use this option to disable color output')
5868 5869 5870 5871
  parser.add_option(
      '--batch',
      action='store_true',
      help='Do not run interactively, just suggest some')
5872
  auth.add_auth_options(parser)
5873
  options, args = parser.parse_args(args)
5874
  auth_config = auth.extract_auth_config_from_options(options)
5875 5876 5877

  author = RunGit(['config', 'user.email']).strip() or None

5878
  cl = Changelist(auth_config=auth_config)
5879 5880 5881 5882 5883 5884 5885

  if args:
    if len(args) > 1:
      parser.error('Unknown args')
    base_branch = args[0]
  else:
    # Default to diffing against the common ancestor of the upstream branch.
5886
    base_branch = cl.GetCommonAncestorWithUpstream()
5887 5888

  change = cl.GetChange(base_branch, None)
5889 5890 5891 5892 5893 5894 5895
  affected_files = [f.LocalPath() for f in change.AffectedFiles()]

  if options.batch:
    db = owners.Database(change.RepositoryRoot(), file, os.path)
    print('\n'.join(db.reviewers_for(affected_files, author)))
    return 0

5896
  return owners_finder.OwnersFinder(
5897
      affected_files,
5898 5899
      change.RepositoryRoot(),
      author, fopen=file, os_path=os.path,
5900 5901
      disable_color=options.no_color,
      override_files=change.OriginalOwnersFiles()).run()
5902 5903


5904
def BuildGitDiffCmd(diff_type, upstream_commit, args):
5905 5906
  """Generates a diff command."""
  # Generate diff for the current branch's changes.
5907 5908
  diff_cmd = ['-c', 'core.quotePath=false', 'diff',
              '--no-ext-diff', '--no-prefix', diff_type,
5909
              upstream_commit, '--']
5910 5911 5912

  if args:
    for arg in args:
5913
      if os.path.isdir(arg) or os.path.isfile(arg):
5914 5915 5916 5917 5918 5919
        diff_cmd.append(arg)
      else:
        DieWithError('Argument "%s" is not a file or a directory' % arg)

  return diff_cmd

5920

5921 5922 5923
def MatchingFileType(file_name, extensions):
  """Returns true if the file name ends with one of the given extensions."""
  return bool([ext for ext in extensions if file_name.lower().endswith(ext)])
5924

5925

5926
@subcommand.usage('[files or directories to diff]')
5927
def CMDformat(parser, args):
5928
  """Runs auto-formatting tools (clang-format etc.) on the diff."""
5929
  CLANG_EXTS = ['.cc', '.cpp', '.h', '.m', '.mm', '.proto', '.java']
5930
  GN_EXTS = ['.gn', '.gni', '.typemap']
5931 5932 5933 5934
  parser.add_option('--full', action='store_true',
                    help='Reformat the full content of all touched files')
  parser.add_option('--dry-run', action='store_true',
                    help='Don\'t modify any file on disk.')
5935 5936
  parser.add_option('--python', action='store_true',
                    help='Format python code with yapf (experimental).')
5937 5938
  parser.add_option('--js', action='store_true',
                    help='Format javascript code with clang-format.')
5939 5940
  parser.add_option('--diff', action='store_true',
                    help='Print diff to stdout rather than modifying files.')
5941 5942
  parser.add_option('--presubmit', action='store_true',
                    help='Used when running the script from a presubmit.')
5943 5944
  opts, args = parser.parse_args(args)

5945 5946 5947 5948
  # Normalize any remaining args against the current path, so paths relative to
  # the current directory are still resolved as expected.
  args = [os.path.join(os.getcwd(), arg) for arg in args]

5949 5950
  # git diff generates paths against the root of the repository.  Change
  # to that directory so clang-format can find files even within subdirs.
5951
  rel_base_path = settings.GetRelativeRoot()
5952 5953 5954
  if rel_base_path:
    os.chdir(rel_base_path)

digit@chromium.org's avatar
digit@chromium.org committed
5955 5956 5957 5958 5959 5960 5961 5962 5963 5964 5965 5966 5967 5968 5969
  # Grab the merge-base commit, i.e. the upstream commit of the current
  # branch when it was created or the last time it was rebased. This is
  # to cover the case where the user may have called "git fetch origin",
  # moving the origin branch to a newer commit, but hasn't rebased yet.
  upstream_commit = None
  cl = Changelist()
  upstream_branch = cl.GetUpstreamBranch()
  if upstream_branch:
    upstream_commit = RunGit(['merge-base', 'HEAD', upstream_branch])
    upstream_commit = upstream_commit.strip()

  if not upstream_commit:
    DieWithError('Could not find base commit for this branch. '
                 'Are you in detached state?')

5970 5971 5972
  changed_files_cmd = BuildGitDiffCmd('--name-only', upstream_commit, args)
  diff_output = RunGit(changed_files_cmd)
  diff_files = diff_output.splitlines()
5973 5974
  # Filter out files deleted by this CL
  diff_files = [x for x in diff_files if os.path.isfile(x)]
5975

5976 5977 5978
  if opts.js:
    CLANG_EXTS.append('.js')

5979 5980 5981
  clang_diff_files = [x for x in diff_files if MatchingFileType(x, CLANG_EXTS)]
  python_diff_files = [x for x in diff_files if MatchingFileType(x, ['.py'])]
  dart_diff_files = [x for x in diff_files if MatchingFileType(x, ['.dart'])]
5982
  gn_diff_files = [x for x in diff_files if MatchingFileType(x, GN_EXTS)]
digit@chromium.org's avatar
digit@chromium.org committed
5983

5984 5985 5986
  top_dir = os.path.normpath(
      RunGit(["rev-parse", "--show-toplevel"]).rstrip('\n'))

5987 5988 5989 5990
  # Set to 2 to signal to CheckPatchFormatted() that this patch isn't
  # formatted. This is used to block during the presubmit.
  return_value = 0

5991
  if clang_diff_files:
5992 5993 5994
    # Locate the clang-format binary in the checkout
    try:
      clang_format_tool = clang_format.FindClangFormatToolInChromiumTree()
5995
    except clang_format.NotFoundError as e:
5996 5997
      DieWithError(e)

5998
    if opts.full:
5999 6000 6001
      cmd = [clang_format_tool]
      if not opts.dry_run and not opts.diff:
        cmd.append('-i')
6002
      stdout = RunCommand(cmd + clang_diff_files, cwd=top_dir)
6003 6004
      if opts.diff:
        sys.stdout.write(stdout)
6005 6006 6007 6008 6009 6010
    else:
      env = os.environ.copy()
      env['PATH'] = str(os.path.dirname(clang_format_tool))
      try:
        script = clang_format.FindClangFormatScriptInChromiumTree(
            'clang-format-diff.py')
6011
      except clang_format.NotFoundError as e:
6012 6013 6014 6015 6016 6017 6018 6019 6020 6021 6022 6023 6024 6025
        DieWithError(e)

      cmd = [sys.executable, script, '-p0']
      if not opts.dry_run and not opts.diff:
        cmd.append('-i')

      diff_cmd = BuildGitDiffCmd('-U0', upstream_commit, clang_diff_files)
      diff_output = RunGit(diff_cmd)

      stdout = RunCommand(cmd, stdin=diff_output, cwd=top_dir, env=env)
      if opts.diff:
        sys.stdout.write(stdout)
      if opts.dry_run and len(stdout) > 0:
        return_value = 2
6026

6027 6028 6029 6030 6031 6032 6033 6034
  # Similar code to above, but using yapf on .py files rather than clang-format
  # on C/C++ files
  if opts.python:
    yapf_tool = gclient_utils.FindExecutable('yapf')
    if yapf_tool is None:
      DieWithError('yapf not found in PATH')

    if opts.full:
6035
      if python_diff_files:
6036 6037 6038
        cmd = [yapf_tool]
        if not opts.dry_run and not opts.diff:
          cmd.append('-i')
6039
        stdout = RunCommand(cmd + python_diff_files, cwd=top_dir)
6040 6041 6042 6043 6044 6045 6046
        if opts.diff:
          sys.stdout.write(stdout)
    else:
      # TODO(sbc): yapf --lines mode still has some issues.
      # https://github.com/google/yapf/issues/154
      DieWithError('--python currently only works with --full')

6047 6048 6049
  # Dart's formatter does not have the nice property of only operating on
  # modified chunks, so hard code full.
  if dart_diff_files:
6050 6051 6052 6053
    try:
      command = [dart_format.FindDartFmtToolInChromiumTree()]
      if not opts.dry_run and not opts.diff:
        command.append('-w')
6054
      command.extend(dart_diff_files)
6055

6056
      stdout = RunCommand(command, cwd=top_dir)
6057 6058 6059
      if opts.dry_run and stdout:
        return_value = 2
    except dart_format.NotFoundError as e:
6060 6061 6062
      print('Warning: Unable to check Dart code formatting. Dart SDK not '
            'found in this checkout. Files in other languages are still '
            'formatted.')
6063

6064 6065
  # Format GN build files. Always run on full build files for canonical form.
  if gn_diff_files:
6066
    cmd = ['gn', 'format']
6067 6068
    if opts.dry_run or opts.diff:
      cmd.append('--dry-run')
6069
    for gn_diff_file in gn_diff_files:
6070 6071 6072 6073 6074 6075 6076 6077 6078 6079 6080 6081 6082 6083
      gn_ret = subprocess2.call(cmd + [gn_diff_file],
                                shell=sys.platform == 'win32',
                                cwd=top_dir)
      if opts.dry_run and gn_ret == 2:
        return_value = 2  # Not formatted.
      elif opts.diff and gn_ret == 2:
        # TODO this should compute and print the actual diff.
        print("This change has GN build file diff for " + gn_diff_file)
      elif gn_ret != 0:
        # For non-dry run cases (and non-2 return values for dry-run), a
        # nonzero error code indicates a failure, probably because the file
        # doesn't parse.
        DieWithError("gn format failed on " + gn_diff_file +
                     "\nTry running 'gn format' on this file manually.")
6084

6085 6086 6087 6088 6089 6090 6091 6092 6093 6094
  # Skip the metrics formatting from the global presubmit hook. These files have
  # a separate presubmit hook that issues an error if the files need formatting,
  # whereas the top-level presubmit script merely issues a warning. Formatting
  # these files is somewhat slow, so it's important not to duplicate the work.
  if not opts.presubmit:
    for xml_dir in GetDirtyMetricsDirs(diff_files):
      tool_dir = os.path.join(top_dir, xml_dir)
      cmd = [os.path.join(tool_dir, 'pretty_print.py'), '--non-interactive']
      if opts.dry_run or opts.diff:
        cmd.append('--diff')
6095
      stdout = RunCommand(cmd, cwd=top_dir)
6096 6097 6098 6099
      if opts.diff:
        sys.stdout.write(stdout)
      if opts.dry_run and stdout:
        return_value = 2  # Not formatted.
6100

6101
  return return_value
6102

6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113
def GetDirtyMetricsDirs(diff_files):
  xml_diff_files = [x for x in diff_files if MatchingFileType(x, ['.xml'])]
  metrics_xml_dirs = [
    os.path.join('tools', 'metrics', 'actions'),
    os.path.join('tools', 'metrics', 'histograms'),
    os.path.join('tools', 'metrics', 'rappor'),
    os.path.join('tools', 'metrics', 'ukm')]
  for xml_dir in metrics_xml_dirs:
    if any(file.startswith(xml_dir) for file in xml_diff_files):
      yield xml_dir

6114

6115 6116
@subcommand.usage('<codereview url or issue id>')
def CMDcheckout(parser, args):
6117
  """Checks out a branch associated with a given Rietveld or Gerrit issue."""
6118 6119 6120 6121 6122 6123
  _, args = parser.parse_args(args)

  if len(args) != 1:
    parser.print_help()
    return 1

6124
  issue_arg = ParseIssueNumberArgument(args[0])
6125
  if not issue_arg.valid:
6126
    parser.error('invalid codereview url or CL id')
6127

6128
  target_issue = str(issue_arg.issue)
6129

6130
  def find_issues(issueprefix):
6131 6132 6133 6134
    output = RunGit(['config', '--local', '--get-regexp',
                     r'branch\..*\.%s' % issueprefix],
                     error_ok=True)
    for key, issue in [x.split() for x in output.splitlines()]:
6135 6136
      if issue == target_issue:
        yield re.sub(r'branch\.(.*)\.%s' % issueprefix, r'\1', key)
6137

6138 6139
  branches = []
  for cls in _CODEREVIEW_IMPLEMENTATIONS.values():
6140
    branches.extend(find_issues(cls.IssueConfigKey()))
6141
  if len(branches) == 0:
6142
    print('No branch found for issue %s.' % target_issue)
6143 6144 6145 6146
    return 1
  if len(branches) == 1:
    RunGit(['checkout', branches[0]])
  else:
6147
    print('Multiple branches match issue %s:' % target_issue)
6148
    for i in range(len(branches)):
6149
      print('%d: %s' % (i, branches[i]))
6150 6151 6152 6153
    which = raw_input('Choose by index: ')
    try:
      RunGit(['checkout', branches[int(which)]])
    except (IndexError, ValueError):
6154
      print('Invalid selection, not checking out any branch.')
6155 6156 6157 6158 6159
      return 1

  return 0


maruel@chromium.org's avatar
maruel@chromium.org committed
6160 6161
def CMDlol(parser, args):
  # This command is intentionally undocumented.
6162
  print(zlib.decompress(base64.b64decode(
6163 6164 6165
      'eNptkLEOwyAMRHe+wupCIqW57v0Vq84WqWtXyrcXnCBsmgMJ+/SSAxMZgRB6NzE'
      'E2ObgCKJooYdu4uAQVffUEoE1sRQLxAcqzd7uK2gmStrll1ucV3uZyaY5sXyDd9'
      'JAnN+lAXsOMJ90GANAi43mq5/VeeacylKVgi8o6F1SC63FxnagHfJUTfUYdCR/W'
6166
      'Ofe+0dHL7PicpytKP750Fh1q2qnLVof4w8OZWNY')))
maruel@chromium.org's avatar
maruel@chromium.org committed
6167 6168 6169
  return 0


6170 6171 6172
class OptionParser(optparse.OptionParser):
  """Creates the option parse and add --verbose support."""
  def __init__(self, *args, **kwargs):
6173 6174
    optparse.OptionParser.__init__(
        self, *args, prog='git cl', version=__version__, **kwargs)
6175 6176 6177 6178 6179 6180 6181
    self.add_option(
        '-v', '--verbose', action='count', default=0,
        help='Use 2 times for more debugging info')

  def parse_args(self, args=None, values=None):
    options, args = optparse.OptionParser.parse_args(self, args, values)
    levels = [logging.WARNING, logging.INFO, logging.DEBUG]
6182 6183 6184 6185
    logging.basicConfig(
        level=levels[min(options.verbose, len(levels) - 1)],
        format='[%(levelname).1s%(asctime)s %(process)d %(thread)d '
               '%(filename)s] %(message)s')
6186 6187 6188
    return options, args


6189
def main(argv):
6190
  if sys.hexversion < 0x02060000:
6191 6192
    print('\nYour python version %s is unsupported, please upgrade.\n' %
          (sys.version.split(' ', 1)[0],), file=sys.stderr)
6193
    return 2
6194

6195 6196 6197 6198
  # Reload settings.
  global settings
  settings = Settings()

6199
  colorize_CMDstatus_doc()
6200 6201 6202
  dispatcher = subcommand.CommandDispatcher(__name__)
  try:
    return dispatcher.execute(OptionParser(), argv)
6203 6204
  except auth.AuthenticationError as e:
    DieWithError(str(e))
6205
  except urllib2.HTTPError as e:
6206 6207 6208 6209 6210
    if e.code != 500:
      raise
    DieWithError(
        ('AppEngine is misbehaving and returned HTTP %d, again. Keep faith '
          'and retry or visit go/isgaeup.\n%s') % (e.code, str(e)))
6211
  return 0
6212 6213 6214


if __name__ == '__main__':
6215 6216
  # These affect sys.stdout so do it outside of main() to simplify mocks in
  # unit testing.
6217
  fix_encoding.fix_encoding()
6218
  setup_color.init()
6219 6220 6221 6222 6223
  try:
    sys.exit(main(sys.argv[1:]))
  except KeyboardInterrupt:
    sys.stderr.write('interrupted\n')
    sys.exit(1)