fetch.py 11 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
Tool to perform checkouts in one easy command line!

Usage:
  fetch <recipe> [--property=value [--property2=value2 ...]]

This script is a wrapper around various version control and repository
checkout commands. It requires a |recipe| name, fetches data from that
recipe in depot_tools/recipes, and then performs all necessary inits,
checkouts, pulls, fetches, etc.

Optional arguments may be passed on the command line in key-value pairs.
These parameters will be passed through to the recipe's main method.
"""

import json
22
import optparse
23
import os
24
import pipes
25 26
import subprocess
import sys
27
import textwrap
28

29 30
from distutils import spawn

31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46

SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))

#################################################
# Checkout class definitions.
#################################################
class Checkout(object):
  """Base class for implementing different types of checkouts.

  Attributes:
    |base|: the absolute path of the directory in which this script is run.
    |spec|: the spec for this checkout as returned by the recipe. Different
        subclasses will expect different keys in this dictionary.
    |root|: the directory into which the checkout will be performed, as returned
        by the recipe. This is a relative path from |base|.
  """
47
  def __init__(self, options, spec, root):
48
    self.base = os.getcwd()
49
    self.options = options
50 51 52 53 54 55 56 57 58 59 60 61
    self.spec = spec
    self.root = root

  def exists(self):
    pass

  def init(self):
    pass

  def sync(self):
    pass

62 63
  def run(self, cmd, **kwargs):
    print 'Running: %s' % (' '.join(pipes.quote(x) for x in cmd))
64
    if self.options.dry_run:
65
      return ''
66
    return subprocess.check_output(cmd, **kwargs)
67

68 69 70

class GclientCheckout(Checkout):

71
  def run_gclient(self, *cmd, **kwargs):
72 73 74 75 76
    if not spawn.find_executable('gclient'):
      cmd_prefix = (sys.executable, os.path.join(SCRIPT_PATH, 'gclient.py'))
    else:
      cmd_prefix = ('gclient',)
    return self.run(cmd_prefix + cmd, **kwargs)
77

78 79 80 81 82 83 84 85 86
  def exists(self):
    try:
      gclient_root = self.run_gclient('root').strip()
      return (os.path.exists(os.path.join(gclient_root, '.gclient')) or
              os.path.exists(os.path.join(os.getcwd(), self.root)))
    except subprocess.CalledProcessError:
      pass
    return os.path.exists(os.path.join(os.getcwd(), self.root))

87 88 89

class GitCheckout(Checkout):

90
  def run_git(self, *cmd, **kwargs):
91
    if sys.platform == 'win32' and not spawn.find_executable('git'):
92
      git_path = os.path.join(SCRIPT_PATH, 'git.bat')
93 94 95
    else:
      git_path = 'git'
    return self.run((git_path,) + cmd, **kwargs)
96 97


98 99 100
class SvnCheckout(Checkout):

  def run_svn(self, *cmd, **kwargs):
101 102 103 104 105
    if sys.platform == 'win32' and not spawn.find_executable('svn'):
      svn_path = os.path.join(SCRIPT_PATH, 'svn_bin', 'svn.exe')
    else:
      svn_path = 'svn'
    return self.run((svn_path,) + cmd, **kwargs)
106 107


108
class GclientGitCheckout(GclientCheckout, GitCheckout):
109

110 111
  def __init__(self, options, spec, root):
    super(GclientGitCheckout, self).__init__(options, spec, root)
112
    assert 'solutions' in self.spec
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130

  def _format_spec(self):
    def _format_literal(lit):
      if isinstance(lit, basestring):
        return '"%s"' % lit
      if isinstance(lit, list):
        return '[%s]' % ', '.join(_format_literal(i) for i in lit)
      return '%r' % lit
    soln_strings = []
    for soln in self.spec['solutions']:
      soln_string= '\n'.join('    "%s": %s,' % (key, _format_literal(value))
                             for key, value in soln.iteritems())
      soln_strings.append('  {\n%s\n  },' % soln_string)
    gclient_spec = 'solutions = [\n%s\n]\n' % '\n'.join(soln_strings)
    extra_keys = ['target_os', 'target_os_only']
    gclient_spec += ''.join('%s = %s\n' % (key, _format_literal(self.spec[key]))
                             for key in extra_keys if key in self.spec)
    return gclient_spec
131 132 133

  def init(self):
    # Configure and do the gclient checkout.
134
    self.run_gclient('config', '--spec', self._format_spec())
jochen@chromium.org's avatar
jochen@chromium.org committed
135
    sync_cmd = ['sync']
136
    if self.options.nohooks:
jochen@chromium.org's avatar
jochen@chromium.org committed
137
      sync_cmd.append('--nohooks')
138 139
    if self.options.no_history:
      sync_cmd.append('--no-history')
jochen@chromium.org's avatar
jochen@chromium.org committed
140 141 142
    if self.spec.get('with_branch_heads', False):
      sync_cmd.append('--with_branch_heads')
    self.run_gclient(*sync_cmd)
143 144 145

    # Configure git.
    wd = os.path.join(self.base, self.root)
146
    if self.options.dry_run:
147
      print 'cd %s' % wd
148 149 150 151
    self.run_git(
        'submodule', 'foreach',
        'git config -f $toplevel/.git/config submodule.$name.ignore all',
        cwd=wd)
152 153 154
    self.run_git(
        'config', '--add', 'remote.origin.fetch',
        '+refs/tags/*:refs/tags/*', cwd=wd)
155 156
    self.run_git('config', 'diff.ignoreSubmodules', 'all', cwd=wd)

157 158 159

class GclientGitSvnCheckout(GclientGitCheckout, SvnCheckout):

160 161
  def __init__(self, options, spec, root):
    super(GclientGitSvnCheckout, self).__init__(options, spec, root)
162 163 164 165 166 167

  def init(self):
    # Ensure we are authenticated with subversion for all submodules.
    git_svn_dirs = json.loads(self.spec.get('submodule_git_svn_spec', '{}'))
    git_svn_dirs.update({self.root: self.spec})
    for _, svn_spec in git_svn_dirs.iteritems():
168 169 170 171 172 173
      if svn_spec.get('svn_url'):
        try:
          self.run_svn('ls', '--non-interactive', svn_spec['svn_url'])
        except subprocess.CalledProcessError:
          print 'Please run `svn ls %s`' % svn_spec['svn_url']
          return 1
174 175 176

    super(GclientGitSvnCheckout, self).init()

177
    # Configure git-svn.
178 179 180 181 182
    for path, svn_spec in git_svn_dirs.iteritems():
      real_path = os.path.join(*path.split('/'))
      if real_path != self.root:
        real_path = os.path.join(self.root, real_path)
      wd = os.path.join(self.base, real_path)
183
      if self.options.dry_run:
184
        print 'cd %s' % wd
185 186 187 188 189 190 191 192 193 194 195
      if svn_spec.get('auto'):
        self.run_git('auto-svn', cwd=wd)
        continue
      self.run_git('svn', 'init', svn_spec['svn_url'], cwd=wd)
      self.run_git('config', '--unset-all', 'svn-remote.svn.fetch', cwd=wd)
      for svn_branch, git_ref in svn_spec.get('git_svn_fetch', {}).items():
        self.run_git('config', '--add', 'svn-remote.svn.fetch',
                     '%s:%s' % (svn_branch, git_ref), cwd=wd)
      for svn_branch, git_ref in svn_spec.get('git_svn_branches', {}).items():
        self.run_git('config', '--add', 'svn-remote.svn.branches',
                     '%s:%s' % (svn_branch, git_ref), cwd=wd)
196 197 198
      self.run_git('svn', 'fetch', cwd=wd)


199

200 201
CHECKOUT_TYPE_MAP = {
    'gclient':         GclientCheckout,
202
    'gclient_git':     GclientGitCheckout,
203 204 205 206 207
    'gclient_git_svn': GclientGitSvnCheckout,
    'git':             GitCheckout,
}


208
def CheckoutFactory(type_name, options, spec, root):
209 210 211 212
  """Factory to build Checkout class instances."""
  class_ = CHECKOUT_TYPE_MAP.get(type_name)
  if not class_:
    raise KeyError('unrecognized checkout type: %s' % type_name)
213
  return class_(options, spec, root)
214 215 216 217 218 219 220 221 222 223


#################################################
# Utility function and file entry point.
#################################################
def usage(msg=None):
  """Print help and exit."""
  if msg:
    print 'Error:', msg

224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
  print textwrap.dedent("""\
    usage: %s [options] <recipe> [--property=value [--property2=value2 ...]]

    This script can be used to download the Chromium sources. See
    http://www.chromium.org/developers/how-tos/get-the-code
    for full usage instructions.

    Valid options:
       -h, --help, help   Print this message.
       --nohooks          Don't run hooks after checkout.
       -n, --dry-run      Don't run commands, only print them.
       --no-history       Perform shallow clones, don't fetch the full git history.

    Valid fetch recipes:""") % os.path.basename(sys.argv[0])
  for fname in os.listdir(os.path.join(SCRIPT_PATH, 'recipes')):
    if fname.endswith('.py'):
      print '  ' + fname[:-3]

242 243 244 245 246 247 248
  sys.exit(bool(msg))


def handle_args(argv):
  """Gets the recipe name from the command line arguments."""
  if len(argv) <= 1:
    usage('Must specify a recipe.')
249 250
  if argv[1] in ('-h', '--help', 'help'):
    usage()
251

252
  dry_run = False
253
  nohooks = False
254
  no_history = False
255 256 257 258
  while len(argv) >= 2:
    arg = argv[1]
    if not arg.startswith('-'):
      break
259
    argv.pop(1)
260
    if arg in ('-n', '--dry-run'):
261
      dry_run = True
262 263
    elif arg == '--nohooks':
      nohooks = True
264 265
    elif arg == '--no-history':
      no_history = True
266 267
    else:
      usage('Invalid option %s.' % arg)
268

269 270 271 272 273 274 275 276 277
  def looks_like_arg(arg):
    return arg.startswith('--') and arg.count('=') == 1

  bad_parms = [x for x in argv[2:] if not looks_like_arg(x)]
  if bad_parms:
    usage('Got bad arguments %s' % bad_parms)

  recipe = argv[1]
  props = argv[2:]
278 279 280 281 282
  return (
      optparse.Values(
          {'dry_run':dry_run, 'nohooks':nohooks, 'no_history': no_history }),
      recipe,
      props)
283 284 285 286 287 288


def run_recipe_fetch(recipe, props, aliased=False):
  """Invoke a recipe's fetch method with the passed-through args
  and return its json output as a python object."""
  recipe_path = os.path.abspath(os.path.join(SCRIPT_PATH, 'recipes', recipe))
289
  if not os.path.exists(recipe_path + '.py'):
290 291 292
    print "Could not find a recipe for %s" % recipe
    sys.exit(1)

293 294
  cmd = [sys.executable, recipe_path + '.py', 'fetch'] + props
  result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
295

296 297 298 299 300 301 302 303 304 305 306
  spec = json.loads(result)
  if 'alias' in spec:
    assert not aliased
    return run_recipe_fetch(
        spec['alias']['recipe'], spec['alias']['props'] + props, aliased=True)
  cmd = [sys.executable, recipe_path + '.py', 'root']
  result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
  root = json.loads(result)
  return spec, root


307
def run(options, spec, root):
308 309 310
  """Perform a checkout with the given type and configuration.

    Args:
311
      options: Options instance.
312 313 314 315 316 317 318 319
      spec: Checkout configuration returned by the the recipe's fetch_spec
          method (checkout type, repository url, etc.).
      root: The directory into which the repo expects to be checkout out.
  """
  assert 'type' in spec
  checkout_type = spec['type']
  checkout_spec = spec['%s_spec' % checkout_type]
  try:
320
    checkout = CheckoutFactory(checkout_type, options, checkout_spec, root)
321 322 323
  except KeyError:
    return 1
  if checkout.exists():
324 325 326
    print 'Your current directory appears to already contain, or be part of, '
    print 'a checkout. "fetch" is used only to get new checkouts. Use '
    print '"gclient sync" to update existing checkouts.'
327 328 329
    print
    print 'Fetch also does not yet deal with partial checkouts, so if fetch'
    print 'failed, delete the checkout and start over (crbug.com/230691).'
330
    return 1
331
  return checkout.init()
332 333 334


def main():
335
  options, recipe, props = handle_args(sys.argv)
336
  spec, root = run_recipe_fetch(recipe, props)
337
  return run(options, spec, root)
338 339 340


if __name__ == '__main__':
341 342 343 344 345
  try:
    sys.exit(main())
  except KeyboardInterrupt:
    sys.stderr.write('interrupted\n')
    sys.exit(1)