checkout.py 30.6 KB
Newer Older
1
# coding=utf8
2
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 4 5 6 7 8 9 10 11 12 13
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Manages a project checkout.

Includes support for svn, git-svn and git.
"""

import fnmatch
import logging
import os
import re
14
import shutil
15 16 17 18
import subprocess
import sys
import tempfile

19 20 21 22 23 24
# The configparser module was renamed in Python 3.
try:
  import configparser
except ImportError:
  import ConfigParser as configparser

25 26 27 28 29
import patch
import scm
import subprocess2


30 31 32 33 34 35 36 37 38 39 40 41
if sys.platform in ('cygwin', 'win32'):
  # Disable timeouts on Windows since we can't have shells with timeouts.
  GLOBAL_TIMEOUT = None
  FETCH_TIMEOUT = None
else:
  # Default timeout of 15 minutes.
  GLOBAL_TIMEOUT = 15*60
  # Use a larger timeout for checkout since it can be a genuinely slower
  # operation.
  FETCH_TIMEOUT = 30*60


42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61
def get_code_review_setting(path, key,
    codereview_settings_file='codereview.settings'):
  """Parses codereview.settings and return the value for the key if present.

  Don't cache the values in case the file is changed."""
  # TODO(maruel): Do not duplicate code.
  settings = {}
  try:
    settings_file = open(os.path.join(path, codereview_settings_file), 'r')
    try:
      for line in settings_file.readlines():
        if not line or line.startswith('#'):
          continue
        if not ':' in line:
          # Invalid file.
          return None
        k, v = line.split(':', 1)
        settings[k.strip()] = v.strip()
    finally:
      settings_file.close()
62
  except IOError:
63 64 65 66
    return None
  return settings.get(key, None)


67 68 69 70 71 72 73 74 75 76 77
def align_stdout(stdout):
  """Returns the aligned output of multiple stdouts."""
  output = ''
  for item in stdout:
    item = item.strip()
    if not item:
      continue
    output += ''.join('  %s\n' % line for line in item.splitlines())
  return output


78 79
class PatchApplicationFailed(Exception):
  """Patch failed to be applied."""
80 81 82
  def __init__(self, p, status):
    super(PatchApplicationFailed, self).__init__(p, status)
    self.patch = p
83 84
    self.status = status

85 86 87 88 89 90 91 92 93 94 95
  @property
  def filename(self):
    if self.patch:
      return self.patch.filename

  def __str__(self):
    out = []
    if self.filename:
      out.append('Failed to apply patch for %s:' % self.filename)
    if self.status:
      out.append(self.status)
96 97
    if self.patch:
      out.append('Patch: %s' % self.patch.dump())
98 99
    return '\n'.join(out)

100 101 102 103 104

class CheckoutBase(object):
  # Set to None to have verbose output.
  VOID = subprocess2.VOID

105 106 107 108 109 110
  def __init__(self, root_dir, project_name, post_processors):
    """
    Args:
      post_processor: list of lambda(checkout, patches) to call on each of the
                      modified files.
    """
111
    super(CheckoutBase, self).__init__()
112 113
    self.root_dir = root_dir
    self.project_name = project_name
114 115 116 117
    if self.project_name is None:
      self.project_path = self.root_dir
    else:
      self.project_path = os.path.join(self.root_dir, self.project_name)
118 119
    # Only used for logging purposes.
    self._last_seen_revision = None
120
    self.post_processors = post_processors
121 122
    assert self.root_dir
    assert self.project_path
123
    assert os.path.isabs(self.project_path)
124 125 126 127

  def get_settings(self, key):
    return get_code_review_setting(self.project_path, key)

128
  def prepare(self, revision):
129 130 131 132
    """Checks out a clean copy of the tree and removes any local modification.

    This function shouldn't throw unless the remote repository is inaccessible,
    there is no free disk space or hard issues like that.
133 134 135

    Args:
      revision: The revision it should sync to, SCM specific.
136 137 138
    """
    raise NotImplementedError()

139
  def apply_patch(self, patches, post_processors=None, verbose=False):
140 141 142 143
    """Applies a patch and returns the list of modified files.

    This function should throw patch.UnsupportedPatchFormat or
    PatchApplicationFailed when relevant.
144 145 146

    Args:
      patches: patch.PatchSet object.
147 148 149 150 151 152 153
    """
    raise NotImplementedError()

  def commit(self, commit_message, user):
    """Commits the patch upstream, while impersonating 'user'."""
    raise NotImplementedError()

154 155 156 157 158 159 160 161 162
  def revisions(self, rev1, rev2):
    """Returns the count of revisions from rev1 to rev2, e.g. len(]rev1, rev2]).

    If rev2 is None, it means 'HEAD'.

    Returns None if there is no link between the two.
    """
    raise NotImplementedError()

163 164 165 166 167 168

class RawCheckout(CheckoutBase):
  """Used to apply a patch locally without any intent to commit it.

  To be used by the try server.
  """
169
  def prepare(self, revision):
170 171 172
    """Stubbed out."""
    pass

173
  def apply_patch(self, patches, post_processors=None, verbose=False):
174
    """Ignores svn properties."""
175
    post_processors = post_processors or self.post_processors or []
176
    for p in patches:
177
      stdout = []
178
      try:
179
        filepath = os.path.join(self.project_path, p.filename)
180
        if p.is_delete:
181
          os.remove(filepath)
182
          assert(not os.path.exists(filepath))
183
          stdout.append('Deleted.')
184 185 186 187 188
        else:
          dirname = os.path.dirname(p.filename)
          full_dir = os.path.join(self.project_path, dirname)
          if dirname and not os.path.isdir(full_dir):
            os.makedirs(full_dir)
189
            stdout.append('Created missing directory %s.' % dirname)
190

191
          if p.is_binary:
192
            content = p.get()
193
            with open(filepath, 'wb') as f:
194 195
              f.write(content)
            stdout.append('Added binary file %d bytes.' % len(content))
196
          else:
197 198 199
            if p.source_filename:
              if not p.is_new:
                raise PatchApplicationFailed(
200
                    p,
201 202 203 204
                    'File has a source filename specified but is not new')
              # Copy the file first.
              if os.path.isfile(filepath):
                raise PatchApplicationFailed(
205
                    p, 'File exist but was about to be overwriten')
206 207
              shutil.copy2(
                  os.path.join(self.project_path, p.source_filename), filepath)
208
              stdout.append('Copied %s -> %s' % (p.source_filename, p.filename))
209
            if p.diff_hunks:
210 211 212
              cmd = ['patch', '-u', '--binary', '-p%s' % p.patchlevel]
              if verbose:
                cmd.append('--verbose')
213 214 215 216 217 218 219 220 221 222 223 224 225
              env = os.environ.copy()
              env['TMPDIR'] = tempfile.mkdtemp(prefix='crpatch')
              try:
                stdout.append(
                    subprocess2.check_output(
                        cmd,
                        stdin=p.get(False),
                        stderr=subprocess2.STDOUT,
                        cwd=self.project_path,
                        timeout=GLOBAL_TIMEOUT,
                        env=env))
              finally:
                shutil.rmtree(env['TMPDIR'])
226
            elif p.is_new and not os.path.exists(filepath):
227
              # There is only a header. Just create the file.
228
              open(filepath, 'w').close()
229
              stdout.append('Created an empty file.')
230
        for post in post_processors:
231
          post(self, p)
232 233 234
        if verbose:
          print p.filename
          print align_stdout(stdout)
235
      except OSError, e:
236
        raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e))
237 238
      except subprocess.CalledProcessError, e:
        raise PatchApplicationFailed(
239 240 241 242 243
            p,
            'While running %s;\n%s%s' % (
              ' '.join(e.cmd),
              align_stdout(stdout),
              align_stdout([getattr(e, 'stdout', '')])))
244 245 246 247 248

  def commit(self, commit_message, user):
    """Stubbed out."""
    raise NotImplementedError('RawCheckout can\'t commit')

249 250 251
  def revisions(self, _rev1, _rev2):
    return None

252 253 254 255

class SvnConfig(object):
  """Parses a svn configuration file."""
  def __init__(self, svn_config_dir=None):
256
    super(SvnConfig, self).__init__()
257 258 259 260 261 262
    self.svn_config_dir = svn_config_dir
    self.default = not bool(self.svn_config_dir)
    if not self.svn_config_dir:
      if sys.platform == 'win32':
        self.svn_config_dir = os.path.join(os.environ['APPDATA'], 'Subversion')
      else:
263 264
        self.svn_config_dir = os.path.expanduser(
            os.path.join('~', '.subversion'))
265
    svn_config_file = os.path.join(self.svn_config_dir, 'config')
266
    parser = configparser.SafeConfigParser()
267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287
    if os.path.isfile(svn_config_file):
      parser.read(svn_config_file)
    else:
      parser.add_section('auto-props')
    self.auto_props = dict(parser.items('auto-props'))


class SvnMixIn(object):
  """MixIn class to add svn commands common to both svn and git-svn clients."""
  # These members need to be set by the subclass.
  commit_user = None
  commit_pwd = None
  svn_url = None
  project_path = None
  # Override at class level when necessary. If used, --non-interactive is
  # implied.
  svn_config = SvnConfig()
  # Set to True when non-interactivity is necessary but a custom subversion
  # configuration directory is not necessary.
  non_interactive = False

288
  def _add_svn_flags(self, args, non_interactive, credentials=True):
289 290 291 292 293
    args = ['svn'] + args
    if not self.svn_config.default:
      args.extend(['--config-dir', self.svn_config.svn_config_dir])
    if not self.svn_config.default or self.non_interactive or non_interactive:
      args.append('--non-interactive')
294 295 296 297 298
    if credentials:
      if self.commit_user:
        args.extend(['--username', self.commit_user])
      if self.commit_pwd:
        args.extend(['--password', self.commit_pwd])
299 300 301 302 303 304
    return args

  def _check_call_svn(self, args, **kwargs):
    """Runs svn and throws an exception if the command failed."""
    kwargs.setdefault('cwd', self.project_path)
    kwargs.setdefault('stdout', self.VOID)
305
    kwargs.setdefault('timeout', GLOBAL_TIMEOUT)
306
    return subprocess2.check_call_out(
307
        self._add_svn_flags(args, False), **kwargs)
308

309
  def _check_output_svn(self, args, credentials=True, **kwargs):
310 311 312 313 314
    """Runs svn and throws an exception if the command failed.

     Returns the output.
    """
    kwargs.setdefault('cwd', self.project_path)
315
    return subprocess2.check_output(
316 317
        self._add_svn_flags(args, True, credentials),
        stderr=subprocess2.STDOUT,
318
        timeout=GLOBAL_TIMEOUT,
319
        **kwargs)
320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341

  @staticmethod
  def _parse_svn_info(output, key):
    """Returns value for key from svn info output.

    Case insensitive.
    """
    values = {}
    key = key.lower()
    for line in output.splitlines(False):
      if not line:
        continue
      k, v = line.split(':', 1)
      k = k.strip().lower()
      v = v.strip()
      assert not k in values
      values[k] = v
    return values.get(key, None)


class SvnCheckout(CheckoutBase, SvnMixIn):
  """Manages a subversion checkout."""
342 343
  def __init__(self, root_dir, project_name, commit_user, commit_pwd, svn_url,
      post_processors=None):
344 345
    CheckoutBase.__init__(self, root_dir, project_name, post_processors)
    SvnMixIn.__init__(self)
346 347 348 349 350
    self.commit_user = commit_user
    self.commit_pwd = commit_pwd
    self.svn_url = svn_url
    assert bool(self.commit_user) >= bool(self.commit_pwd)

351
  def prepare(self, revision):
352
    # Will checkout if the directory is not present.
353
    assert self.svn_url
354 355 356
    if not os.path.isdir(self.project_path):
      logging.info('Checking out %s in %s' %
          (self.project_name, self.project_path))
357
    return self._revert(revision)
358

359
  def apply_patch(self, patches, post_processors=None, verbose=False):
360
    post_processors = post_processors or self.post_processors or []
361
    for p in patches:
362
      stdout = []
363
      try:
364
        filepath = os.path.join(self.project_path, p.filename)
365 366 367
        # It is important to use credentials=False otherwise credentials could
        # leak in the error message. Credentials are not necessary here for the
        # following commands anyway.
368
        if p.is_delete:
369 370
          stdout.append(self._check_output_svn(
              ['delete', p.filename, '--force'], credentials=False))
371
          assert(not os.path.exists(filepath))
372
          stdout.append('Deleted.')
373 374 375 376 377 378 379 380 381 382 383 384
        else:
          # svn add while creating directories otherwise svn add on the
          # contained files will silently fail.
          # First, find the root directory that exists.
          dirname = os.path.dirname(p.filename)
          dirs_to_create = []
          while (dirname and
              not os.path.isdir(os.path.join(self.project_path, dirname))):
            dirs_to_create.append(dirname)
            dirname = os.path.dirname(dirname)
          for dir_to_create in reversed(dirs_to_create):
            os.mkdir(os.path.join(self.project_path, dir_to_create))
385 386 387 388
            stdout.append(
                self._check_output_svn(
                  ['add', dir_to_create, '--force'], credentials=False))
            stdout.append('Created missing directory %s.' % dir_to_create)
389 390

          if p.is_binary:
391
            content = p.get()
392
            with open(filepath, 'wb') as f:
393 394
              f.write(content)
            stdout.append('Added binary file %d bytes.' % len(content))
395
          else:
396 397 398
            if p.source_filename:
              if not p.is_new:
                raise PatchApplicationFailed(
399
                    p,
400 401 402 403
                    'File has a source filename specified but is not new')
              # Copy the file first.
              if os.path.isfile(filepath):
                raise PatchApplicationFailed(
404
                    p, 'File exist but was about to be overwriten')
405 406 407 408
              stdout.append(
                  self._check_output_svn(
                    ['copy', p.source_filename, p.filename]))
              stdout.append('Copied %s -> %s' % (p.source_filename, p.filename))
409
            if p.diff_hunks:
410 411 412 413 414 415 416
              cmd = [
                'patch',
                '-p%s' % p.patchlevel,
                '--forward',
                '--force',
                '--no-backup-if-mismatch',
              ]
417 418 419 420 421 422 423 424 425 426 427 428 429
              env = os.environ.copy()
              env['TMPDIR'] = tempfile.mkdtemp(prefix='crpatch')
              try:
                stdout.append(
                    subprocess2.check_output(
                      cmd,
                      stdin=p.get(False),
                      cwd=self.project_path,
                      timeout=GLOBAL_TIMEOUT,
                      env=env))
              finally:
                shutil.rmtree(env['TMPDIR'])

430 431 432 433
            elif p.is_new and not os.path.exists(filepath):
              # There is only a header. Just create the file if it doesn't
              # exist.
              open(filepath, 'w').close()
434
              stdout.append('Created an empty file.')
435 436 437
          if p.is_new and not p.source_filename:
            # Do not run it if p.source_filename is defined, since svn copy was
            # using above.
438 439 440
            stdout.append(
                self._check_output_svn(
                  ['add', p.filename, '--force'], credentials=False))
441 442
          for name, value in p.svn_properties:
            if value is None:
443 444 445 446 447
              stdout.append(
                  self._check_output_svn(
                    ['propdel', '--quiet', name, p.filename],
                    credentials=False))
              stdout.append('Property %s deleted.' % name)
448
            else:
449 450 451 452
              stdout.append(
                  self._check_output_svn(
                    ['propset', name, value, p.filename], credentials=False))
              stdout.append('Property %s=%s' % (name, value))
453
          for prop, values in self.svn_config.auto_props.iteritems():
454
            if fnmatch.fnmatch(p.filename, prop):
455 456
              for value in values.split(';'):
                if '=' not in value:
457
                  params = [value, '.']
458 459
                else:
                  params = value.split('=', 1)
460 461 462
                if params[1] == '*':
                  # Works around crbug.com/150960 on Windows.
                  params[1] = '.'
463 464 465 466
                stdout.append(
                    self._check_output_svn(
                      ['propset'] + params + [p.filename], credentials=False))
                stdout.append('Property (auto) %s' % '='.join(params))
467
        for post in post_processors:
468
          post(self, p)
469 470 471
        if verbose:
          print p.filename
          print align_stdout(stdout)
472
      except OSError, e:
473
        raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e))
474 475
      except subprocess.CalledProcessError, e:
        raise PatchApplicationFailed(
476
            p,
477
            'While running %s;\n%s%s' % (
478 479 480
              ' '.join(e.cmd),
              align_stdout(stdout),
              align_stdout([getattr(e, 'stdout', '')])))
481 482 483 484

  def commit(self, commit_message, user):
    logging.info('Committing patch for %s' % user)
    assert self.commit_user
485
    assert isinstance(commit_message, unicode)
486 487
    handle, commit_filename = tempfile.mkstemp(text=True)
    try:
488 489 490
      # Shouldn't assume default encoding is UTF-8. But really, if you are using
      # anything else, you are living in another world.
      os.write(handle, commit_message.encode('utf-8'))
491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513
      os.close(handle)
      # When committing, svn won't update the Revision metadata of the checkout,
      # so if svn commit returns "Committed revision 3.", svn info will still
      # return "Revision: 2". Since running svn update right after svn commit
      # creates a race condition with other committers, this code _must_ parse
      # the output of svn commit and use a regexp to grab the revision number.
      # Note that "Committed revision N." is localized but subprocess2 forces
      # LANGUAGE=en.
      args = ['commit', '--file', commit_filename]
      # realauthor is parsed by a server-side hook.
      if user and user != self.commit_user:
        args.extend(['--with-revprop', 'realauthor=%s' % user])
      out = self._check_output_svn(args)
    finally:
      os.remove(commit_filename)
    lines = filter(None, out.splitlines())
    match = re.match(r'^Committed revision (\d+).$', lines[-1])
    if not match:
      raise PatchApplicationFailed(
          None,
          'Couldn\'t make sense out of svn commit message:\n' + out)
    return int(match.group(1))

514
  def _revert(self, revision):
515 516 517 518
    """Reverts local modifications or checks out if the directory is not
    present. Use depot_tools's functionality to do this.
    """
    flags = ['--ignore-externals']
519 520
    if revision:
      flags.extend(['--revision', str(revision)])
521 522 523 524 525 526
    if os.path.isdir(self.project_path):
      # This may remove any part (or all) of the checkout.
      scm.SVN.Revert(self.project_path, no_ignore=True)

    if os.path.isdir(self.project_path):
      # Revive files that were deleted in scm.SVN.Revert().
527 528
      self._check_call_svn(['update', '--force'] + flags,
                           timeout=FETCH_TIMEOUT)
529
    else:
530 531 532
      logging.info(
          'Directory %s is not present, checking it out.' % self.project_path)
      self._check_call_svn(
533 534
          ['checkout', self.svn_url, self.project_path] + flags, cwd=None,
          timeout=FETCH_TIMEOUT)
535
    return self._get_revision()
536

537
  def _get_revision(self):
538
    out = self._check_output_svn(['info', '.'])
539 540 541 542 543
    revision = int(self._parse_svn_info(out, 'revision'))
    if revision != self._last_seen_revision:
      logging.info('Updated to revision %d' % revision)
      self._last_seen_revision = revision
    return revision
544

545 546 547 548 549 550 551 552 553 554 555 556 557 558 559
  def revisions(self, rev1, rev2):
    """Returns the number of actual commits, not just the difference between
    numbers.
    """
    rev2 = rev2 or 'HEAD'
    # Revision range is inclusive and ordering doesn't matter, they'll appear in
    # the order specified.
    try:
      out = self._check_output_svn(
          ['log', '-q', self.svn_url, '-r', '%s:%s' % (rev1, rev2)])
    except subprocess.CalledProcessError:
      return None
    # Ignore the '----' lines.
    return len([l for l in out.splitlines() if l.startswith('r')]) - 1

560

561 562 563
class GitCheckout(CheckoutBase):
  """Manages a git checkout."""
  def __init__(self, root_dir, project_name, remote_branch, git_url,
564
      commit_user, post_processors=None):
565 566 567
    super(GitCheckout, self).__init__(root_dir, project_name, post_processors)
    self.git_url = git_url
    self.commit_user = commit_user
568
    self.remote_branch = remote_branch
569 570
    # The working branch where patches will be applied. It will track the
    # remote branch.
571
    self.working_branch = 'working_branch'
572
    # There is no reason to not hardcode origin.
573 574 575
    self.remote = 'origin'
    # There is no reason to not hardcode master.
    self.master_branch = 'master'
576

577
  def prepare(self, revision):
578 579 580 581
    """Resets the git repository in a clean state.

    Checks it out if not present and deletes the working branch.
    """
582
    assert self.remote_branch
583
    assert self.git_url
584 585 586

    if not os.path.isdir(self.project_path):
      # Clone the repo if the directory is not present.
587 588
      logging.info(
          'Checking out %s in %s', self.project_name, self.project_path)
589
      self._check_call_git(
590
          ['clone', self.git_url, '-b', self.remote_branch, self.project_path],
591
          cwd=None, timeout=FETCH_TIMEOUT)
592 593 594 595 596 597
    else:
      # Throw away all uncommitted changes in the existing checkout.
      self._check_call_git(['checkout', self.remote_branch])
      self._check_call_git(
          ['reset', '--hard', '--quiet',
           '%s/%s' % (self.remote, self.remote_branch)])
598

599 600 601 602
    if revision:
      try:
        # Look if the commit hash already exist. If so, we can skip a
        # 'git fetch' call.
603
        revision = self._check_output_git(['rev-parse', revision]).rstrip()
604 605 606
      except subprocess.CalledProcessError:
        self._check_call_git(
            ['fetch', self.remote, self.remote_branch, '--quiet'])
607
        revision = self._check_output_git(['rev-parse', revision]).rstrip()
608 609 610 611 612 613 614 615 616 617
      self._check_call_git(['checkout', '--force', '--quiet', revision])
    else:
      branches, active = self._branches()
      if active != self.master_branch:
        self._check_call_git(
            ['checkout', '--force', '--quiet', self.master_branch])
      self._sync_remote_branch()

      if self.working_branch in branches:
        self._call_git(['branch', '-D', self.working_branch])
618
    return self._get_head_commit_hash()
619

620 621 622
  def _sync_remote_branch(self):
    """Syncs the remote branch."""
    # We do a 'git pull origin master:refs/remotes/origin/master' instead of
623
    # 'git pull origin master' because from the manpage for git-pull:
624 625 626 627 628 629 630 631 632 633
    #   A parameter <ref> without a colon is equivalent to <ref>: when
    #   pulling/fetching, so it merges <ref> into the current branch without
    #   storing the remote branch anywhere locally.
    remote_tracked_path = 'refs/remotes/%s/%s' % (
        self.remote, self.remote_branch)
    self._check_call_git(
        ['pull', self.remote,
         '%s:%s' % (self.remote_branch, remote_tracked_path),
         '--quiet'])

634
  def _get_head_commit_hash(self):
635 636
    """Gets the current revision (in unicode) from the local branch."""
    return unicode(self._check_output_git(['rev-parse', 'HEAD']).strip())
637

638
  def apply_patch(self, patches, post_processors=None, verbose=False):
639
    """Applies a patch on 'working_branch' and switches to it.
640

641
    The changes remain staged on the current branch.
642 643 644

    Ignores svn properties and raise an exception on unexpected ones.
    """
645
    post_processors = post_processors or self.post_processors or []
646 647
    # It this throws, the checkout is corrupted. Maybe worth deleting it and
    # trying again?
648 649
    if self.remote_branch:
      self._check_call_git(
650
          ['checkout', '-b', self.working_branch, '-t', self.remote_branch,
651 652
           '--quiet'])

653
    for index, p in enumerate(patches):
654
      stdout = []
655
      try:
656
        filepath = os.path.join(self.project_path, p.filename)
657
        if p.is_delete:
658
          if (not os.path.exists(filepath) and
659
              any(p1.source_filename == p.filename for p1 in patches[0:index])):
660 661
            # The file was already deleted if a prior patch with file rename
            # was already processed because 'git apply' did it for us.
662 663
            pass
          else:
664
            stdout.append(self._check_output_git(['rm', p.filename]))
665
            assert(not os.path.exists(filepath))
666
            stdout.append('Deleted.')
667 668 669 670 671
        else:
          dirname = os.path.dirname(p.filename)
          full_dir = os.path.join(self.project_path, dirname)
          if dirname and not os.path.isdir(full_dir):
            os.makedirs(full_dir)
672
            stdout.append('Created missing directory %s.' % dirname)
673
          if p.is_binary:
674 675 676 677 678 679 680 681
            content = p.get()
            with open(filepath, 'wb') as f:
              f.write(content)
            stdout.append('Added binary file %d bytes' % len(content))
            cmd = ['add', p.filename]
            if verbose:
              cmd.append('--verbose')
            stdout.append(self._check_output_git(cmd))
682
          else:
683 684
            # No need to do anything special with p.is_new or if not
            # p.diff_hunks. git apply manages all that already.
685
            cmd = ['apply', '--index', '-3', '-p%s' % p.patchlevel]
686 687 688
            if verbose:
              cmd.append('--verbose')
            stdout.append(self._check_output_git(cmd, stdin=p.get(True)))
689
          for key, value in p.svn_properties:
690 691 692 693 694
            # Ignore some known auto-props flags through .subversion/config,
            # bails out on the other ones.
            # TODO(maruel): Read ~/.subversion/config and detect the rules that
            # applies here to figure out if the property will be correctly
            # handled.
695 696
            stdout.append('Property %s=%s' % (key, value))
            if not key in (
697
                'svn:eol-style', 'svn:executable', 'svn:mime-type'):
698 699 700
              raise patch.UnsupportedPatchFormat(
                  p.filename,
                  'Cannot apply svn property %s to file %s.' % (
701
                        key, p.filename))
702
        for post in post_processors:
703
          post(self, p)
704 705 706
        if verbose:
          print p.filename
          print align_stdout(stdout)
707
      except OSError, e:
708
        raise PatchApplicationFailed(p, '%s%s' % (align_stdout(stdout), e))
709 710
      except subprocess.CalledProcessError, e:
        raise PatchApplicationFailed(
711 712 713 714 715
            p,
            'While running %s;\n%s%s' % (
              ' '.join(e.cmd),
              align_stdout(stdout),
              align_stdout([getattr(e, 'stdout', '')])))
716
    found_files = self._check_output_git(
717 718
        ['diff', '--ignore-submodules',
         '--name-only', '--staged']).splitlines(False)
719 720 721 722 723 724 725 726
    if sorted(patches.filenames) != sorted(found_files):
      extra_files = sorted(set(found_files) - set(patches.filenames))
      unpatched_files = sorted(set(patches.filenames) - set(found_files))
      if extra_files:
        print 'Found extra files: %r' % (extra_files,)
      if unpatched_files:
        print 'Found unpatched files: %r' % (unpatched_files,)

727 728

  def commit(self, commit_message, user):
729
    """Commits, updates the commit message and pushes."""
730 731
    # TODO(hinoka): CQ no longer uses this, I think its deprecated.
    #               Delete this.
732
    assert self.commit_user
733
    assert isinstance(commit_message, unicode)
734 735 736
    current_branch = self._check_output_git(
        ['rev-parse', '--abbrev-ref', 'HEAD']).strip()
    assert current_branch == self.working_branch
737

738
    commit_cmd = ['commit', '-m', commit_message]
739 740 741 742 743 744 745 746 747 748 749
    if user and user != self.commit_user:
      # We do not have the first or last name of the user, grab the username
      # from the email and call it the original author's name.
      # TODO(rmistry): Do not need the below if user is already in
      #                "Name <email>" format.
      name = user.split('@')[0]
      commit_cmd.extend(['--author', '%s <%s>' % (name, user)])
    self._check_call_git(commit_cmd)

    # Push to the remote repository.
    self._check_call_git(
750
        ['push', 'origin', '%s:%s' % (self.working_branch, self.remote_branch),
751
         '--quiet'])
752 753
    # Get the revision after the push.
    revision = self._get_head_commit_hash()
754 755 756
    # Switch back to the remote_branch and sync it.
    self._check_call_git(['checkout', self.remote_branch])
    self._sync_remote_branch()
757 758 759 760
    # Delete the working branch since we are done with it.
    self._check_call_git(['branch', '-D', self.working_branch])

    return revision
761 762 763 764

  def _check_call_git(self, args, **kwargs):
    kwargs.setdefault('cwd', self.project_path)
    kwargs.setdefault('stdout', self.VOID)
765
    kwargs.setdefault('timeout', GLOBAL_TIMEOUT)
766
    return subprocess2.check_call_out(['git'] + args, **kwargs)
767 768 769 770 771

  def _call_git(self, args, **kwargs):
    """Like check_call but doesn't throw on failure."""
    kwargs.setdefault('cwd', self.project_path)
    kwargs.setdefault('stdout', self.VOID)
772
    kwargs.setdefault('timeout', GLOBAL_TIMEOUT)
773
    return subprocess2.call(['git'] + args, **kwargs)
774 775 776

  def _check_output_git(self, args, **kwargs):
    kwargs.setdefault('cwd', self.project_path)
777
    kwargs.setdefault('timeout', GLOBAL_TIMEOUT)
778
    return subprocess2.check_output(
779
        ['git'] + args, stderr=subprocess2.STDOUT, **kwargs)
780 781 782 783 784 785 786 787 788 789 790 791

  def _branches(self):
    """Returns the list of branches and the active one."""
    out = self._check_output_git(['branch']).splitlines(False)
    branches = [l[2:] for l in out]
    active = None
    for l in out:
      if l.startswith('*'):
        active = l[2:]
        break
    return branches, active

792 793 794 795
  def revisions(self, rev1, rev2):
    """Returns the number of actual commits between both hash."""
    self._fetch_remote()

796
    rev2 = rev2 or '%s/%s' % (self.remote, self.remote_branch)
797 798 799 800 801 802 803 804 805 806
    # Revision range is ]rev1, rev2] and ordering matters.
    try:
      out = self._check_output_git(
          ['log', '--format="%H"' , '%s..%s' % (rev1, rev2)])
    except subprocess.CalledProcessError:
      return None
    return len(out.splitlines())

  def _fetch_remote(self):
    """Fetches the remote without rebasing."""
807
    # git fetch is always verbose even with -q, so redirect its output.
808
    self._check_output_git(['fetch', self.remote, self.remote_branch],
809
                           timeout=FETCH_TIMEOUT)
810

811 812 813

class ReadOnlyCheckout(object):
  """Converts a checkout into a read-only one."""
814
  def __init__(self, checkout, post_processors=None):
815
    super(ReadOnlyCheckout, self).__init__()
816
    self.checkout = checkout
817 818
    self.post_processors = (post_processors or []) + (
        self.checkout.post_processors or [])
819

820 821
  def prepare(self, revision):
    return self.checkout.prepare(revision)
822 823 824 825

  def get_settings(self, key):
    return self.checkout.get_settings(key)

826
  def apply_patch(self, patches, post_processors=None, verbose=False):
827
    return self.checkout.apply_patch(
828
        patches, post_processors or self.post_processors, verbose)
829 830 831 832 833 834

  def commit(self, message, user):  # pylint: disable=R0201
    logging.info('Would have committed for %s with message: %s' % (
        user, message))
    return 'FAKE'

835 836 837
  def revisions(self, rev1, rev2):
    return self.checkout.revisions(rev1, rev2)

838 839 840 841 842 843 844
  @property
  def project_name(self):
    return self.checkout.project_name

  @property
  def project_path(self):
    return self.checkout.project_path