fetch.py 11.2 KB
Newer Older
1 2 3 4 5 6 7 8 9
#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
Tool to perform checkouts in one easy command line!

Usage:
10
  fetch <config> [--property=value [--property2=value2 ...]]
11 12

This script is a wrapper around various version control and repository
13 14
checkout commands. It requires a |config| name, fetches data from that
config in depot_tools/fetch_configs, and then performs all necessary inits,
15 16 17
checkouts, pulls, fetches, etc.

Optional arguments may be passed on the command line in key-value pairs.
18
These parameters will be passed through to the config's main method.
19 20 21
"""

import json
22
import optparse
23
import os
24
import pipes
25 26
import subprocess
import sys
27
import textwrap
28

29 30
from distutils import spawn

31 32 33 34 35 36 37 38 39 40 41

SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))

#################################################
# Checkout class definitions.
#################################################
class Checkout(object):
  """Base class for implementing different types of checkouts.

  Attributes:
    |base|: the absolute path of the directory in which this script is run.
42
    |spec|: the spec for this checkout as returned by the config. Different
43 44
        subclasses will expect different keys in this dictionary.
    |root|: the directory into which the checkout will be performed, as returned
45
        by the config. This is a relative path from |base|.
46
  """
47
  def __init__(self, options, spec, root):
48
    self.base = os.getcwd()
49
    self.options = options
50 51 52 53 54 55 56 57 58 59 60 61
    self.spec = spec
    self.root = root

  def exists(self):
    pass

  def init(self):
    pass

  def sync(self):
    pass

62 63
  def run(self, cmd, **kwargs):
    print 'Running: %s' % (' '.join(pipes.quote(x) for x in cmd))
64
    if self.options.dry_run:
65
      return ''
66
    return subprocess.check_output(cmd, **kwargs)
67

68 69 70

class GclientCheckout(Checkout):

71
  def run_gclient(self, *cmd, **kwargs):
72 73 74 75 76
    if not spawn.find_executable('gclient'):
      cmd_prefix = (sys.executable, os.path.join(SCRIPT_PATH, 'gclient.py'))
    else:
      cmd_prefix = ('gclient',)
    return self.run(cmd_prefix + cmd, **kwargs)
77

78 79 80 81 82 83 84 85 86
  def exists(self):
    try:
      gclient_root = self.run_gclient('root').strip()
      return (os.path.exists(os.path.join(gclient_root, '.gclient')) or
              os.path.exists(os.path.join(os.getcwd(), self.root)))
    except subprocess.CalledProcessError:
      pass
    return os.path.exists(os.path.join(os.getcwd(), self.root))

87 88 89

class GitCheckout(Checkout):

90
  def run_git(self, *cmd, **kwargs):
91
    if sys.platform == 'win32' and not spawn.find_executable('git'):
92
      git_path = os.path.join(SCRIPT_PATH, 'git.bat')
93 94 95
    else:
      git_path = 'git'
    return self.run((git_path,) + cmd, **kwargs)
96 97


98 99 100
class SvnCheckout(Checkout):

  def run_svn(self, *cmd, **kwargs):
101 102 103 104 105
    if sys.platform == 'win32' and not spawn.find_executable('svn'):
      svn_path = os.path.join(SCRIPT_PATH, 'svn_bin', 'svn.exe')
    else:
      svn_path = 'svn'
    return self.run((svn_path,) + cmd, **kwargs)
106 107


108
class GclientGitCheckout(GclientCheckout, GitCheckout):
109

110 111
  def __init__(self, options, spec, root):
    super(GclientGitCheckout, self).__init__(options, spec, root)
112
    assert 'solutions' in self.spec
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130

  def _format_spec(self):
    def _format_literal(lit):
      if isinstance(lit, basestring):
        return '"%s"' % lit
      if isinstance(lit, list):
        return '[%s]' % ', '.join(_format_literal(i) for i in lit)
      return '%r' % lit
    soln_strings = []
    for soln in self.spec['solutions']:
      soln_string= '\n'.join('    "%s": %s,' % (key, _format_literal(value))
                             for key, value in soln.iteritems())
      soln_strings.append('  {\n%s\n  },' % soln_string)
    gclient_spec = 'solutions = [\n%s\n]\n' % '\n'.join(soln_strings)
    extra_keys = ['target_os', 'target_os_only']
    gclient_spec += ''.join('%s = %s\n' % (key, _format_literal(self.spec[key]))
                             for key in extra_keys if key in self.spec)
    return gclient_spec
131 132 133

  def init(self):
    # Configure and do the gclient checkout.
134
    self.run_gclient('config', '--spec', self._format_spec())
jochen@chromium.org's avatar
jochen@chromium.org committed
135
    sync_cmd = ['sync']
136
    if self.options.nohooks:
jochen@chromium.org's avatar
jochen@chromium.org committed
137
      sync_cmd.append('--nohooks')
138 139
    if self.options.no_history:
      sync_cmd.append('--no-history')
jochen@chromium.org's avatar
jochen@chromium.org committed
140 141 142
    if self.spec.get('with_branch_heads', False):
      sync_cmd.append('--with_branch_heads')
    self.run_gclient(*sync_cmd)
143 144 145

    # Configure git.
    wd = os.path.join(self.base, self.root)
146
    if self.options.dry_run:
147
      print 'cd %s' % wd
148 149 150 151
    self.run_git(
        'submodule', 'foreach',
        'git config -f $toplevel/.git/config submodule.$name.ignore all',
        cwd=wd)
152 153 154
    self.run_git(
        'config', '--add', 'remote.origin.fetch',
        '+refs/tags/*:refs/tags/*', cwd=wd)
155 156
    self.run_git('config', 'diff.ignoreSubmodules', 'all', cwd=wd)

157 158 159

class GclientGitSvnCheckout(GclientGitCheckout, SvnCheckout):

160 161
  def __init__(self, options, spec, root):
    super(GclientGitSvnCheckout, self).__init__(options, spec, root)
162 163 164 165 166 167

  def init(self):
    # Ensure we are authenticated with subversion for all submodules.
    git_svn_dirs = json.loads(self.spec.get('submodule_git_svn_spec', '{}'))
    git_svn_dirs.update({self.root: self.spec})
    for _, svn_spec in git_svn_dirs.iteritems():
168 169 170 171 172 173
      if svn_spec.get('svn_url'):
        try:
          self.run_svn('ls', '--non-interactive', svn_spec['svn_url'])
        except subprocess.CalledProcessError:
          print 'Please run `svn ls %s`' % svn_spec['svn_url']
          return 1
174 175 176

    super(GclientGitSvnCheckout, self).init()

177
    # Configure git-svn.
178 179 180 181 182
    for path, svn_spec in git_svn_dirs.iteritems():
      real_path = os.path.join(*path.split('/'))
      if real_path != self.root:
        real_path = os.path.join(self.root, real_path)
      wd = os.path.join(self.base, real_path)
183
      if self.options.dry_run:
184
        print 'cd %s' % wd
185 186 187 188 189 190 191 192
      self.run_git('svn', 'init', svn_spec['svn_url'], cwd=wd)
      self.run_git('config', '--unset-all', 'svn-remote.svn.fetch', cwd=wd)
      for svn_branch, git_ref in svn_spec.get('git_svn_fetch', {}).items():
        self.run_git('config', '--add', 'svn-remote.svn.fetch',
                     '%s:%s' % (svn_branch, git_ref), cwd=wd)
      for svn_branch, git_ref in svn_spec.get('git_svn_branches', {}).items():
        self.run_git('config', '--add', 'svn-remote.svn.branches',
                     '%s:%s' % (svn_branch, git_ref), cwd=wd)
193 194 195
      self.run_git('svn', 'fetch', cwd=wd)


196

197 198
CHECKOUT_TYPE_MAP = {
    'gclient':         GclientCheckout,
199
    'gclient_git':     GclientGitCheckout,
200 201 202 203 204
    'gclient_git_svn': GclientGitSvnCheckout,
    'git':             GitCheckout,
}


205
def CheckoutFactory(type_name, options, spec, root):
206 207 208 209
  """Factory to build Checkout class instances."""
  class_ = CHECKOUT_TYPE_MAP.get(type_name)
  if not class_:
    raise KeyError('unrecognized checkout type: %s' % type_name)
210
  return class_(options, spec, root)
211 212 213 214 215 216 217 218 219 220


#################################################
# Utility function and file entry point.
#################################################
def usage(msg=None):
  """Print help and exit."""
  if msg:
    print 'Error:', msg

221
  print textwrap.dedent("""\
222
    usage: %s [options] <config> [--property=value [--property2=value2 ...]]
223 224 225 226 227 228 229 230

    This script can be used to download the Chromium sources. See
    http://www.chromium.org/developers/how-tos/get-the-code
    for full usage instructions.

    Valid options:
       -h, --help, help   Print this message.
       --nohooks          Don't run hooks after checkout.
231
       --force            (dangerous) Don't look for existing .gclient file.
232 233 234
       -n, --dry-run      Don't run commands, only print them.
       --no-history       Perform shallow clones, don't fetch the full git history.

235
    Valid fetch configs:""") % os.path.basename(sys.argv[0])
236

237 238 239 240
  configs_dir = os.path.join(SCRIPT_PATH, 'fetch_configs')
  configs = [f[:-3] for f in os.listdir(configs_dir) if f.endswith('.py')]
  configs.sort()
  for fname in configs:
241
    print '  ' + fname
242

243 244 245 246
  sys.exit(bool(msg))


def handle_args(argv):
247
  """Gets the config name from the command line arguments."""
248
  if len(argv) <= 1:
249
    usage('Must specify a config.')
250 251
  if argv[1] in ('-h', '--help', 'help'):
    usage()
252

253
  dry_run = False
254
  nohooks = False
255
  no_history = False
256
  force = False
257 258 259 260
  while len(argv) >= 2:
    arg = argv[1]
    if not arg.startswith('-'):
      break
261
    argv.pop(1)
262
    if arg in ('-n', '--dry-run'):
263
      dry_run = True
264 265
    elif arg == '--nohooks':
      nohooks = True
266 267
    elif arg == '--no-history':
      no_history = True
268 269
    elif arg == '--force':
      force = True
270 271
    else:
      usage('Invalid option %s.' % arg)
272

273 274 275 276 277 278 279
  def looks_like_arg(arg):
    return arg.startswith('--') and arg.count('=') == 1

  bad_parms = [x for x in argv[2:] if not looks_like_arg(x)]
  if bad_parms:
    usage('Got bad arguments %s' % bad_parms)

280
  config = argv[1]
281
  props = argv[2:]
282
  return (
283 284 285 286 287
      optparse.Values({
        'dry_run': dry_run,
        'nohooks': nohooks,
        'no_history': no_history,
        'force': force}),
288
      config,
289
      props)
290 291


292 293
def run_config_fetch(config, props, aliased=False):
  """Invoke a config's fetch method with the passed-through args
294
  and return its json output as a python object."""
295 296 297 298
  config_path = os.path.abspath(
      os.path.join(SCRIPT_PATH, 'fetch_configs', config))
  if not os.path.exists(config_path + '.py'):
    print "Could not find a config for %s" % config
299 300
    sys.exit(1)

301
  cmd = [sys.executable, config_path + '.py', 'fetch'] + props
302
  result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
303

304 305 306
  spec = json.loads(result)
  if 'alias' in spec:
    assert not aliased
307 308 309
    return run_config_fetch(
        spec['alias']['config'], spec['alias']['props'] + props, aliased=True)
  cmd = [sys.executable, config_path + '.py', 'root']
310 311 312 313 314
  result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
  root = json.loads(result)
  return spec, root


315
def run(options, spec, root):
316 317 318
  """Perform a checkout with the given type and configuration.

    Args:
319
      options: Options instance.
320
      spec: Checkout configuration returned by the the config's fetch_spec
321 322 323 324 325 326 327
          method (checkout type, repository url, etc.).
      root: The directory into which the repo expects to be checkout out.
  """
  assert 'type' in spec
  checkout_type = spec['type']
  checkout_spec = spec['%s_spec' % checkout_type]
  try:
328
    checkout = CheckoutFactory(checkout_type, options, checkout_spec, root)
329 330
  except KeyError:
    return 1
331
  if not options.force and checkout.exists():
332 333 334
    print 'Your current directory appears to already contain, or be part of, '
    print 'a checkout. "fetch" is used only to get new checkouts. Use '
    print '"gclient sync" to update existing checkouts.'
335 336 337
    print
    print 'Fetch also does not yet deal with partial checkouts, so if fetch'
    print 'failed, delete the checkout and start over (crbug.com/230691).'
338
    return 1
339
  return checkout.init()
340 341 342


def main():
343 344
  options, config, props = handle_args(sys.argv)
  spec, root = run_config_fetch(config, props)
345
  return run(options, spec, root)
346 347 348


if __name__ == '__main__':
349 350 351 352 353
  try:
    sys.exit(main())
  except KeyboardInterrupt:
    sys.stderr.write('interrupted\n')
    sys.exit(1)