fetch.py 11.1 KB
Newer Older
1 2 3 4 5 6 7 8 9
#!/usr/bin/env python
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
Tool to perform checkouts in one easy command line!

Usage:
10
  fetch <config> [--property=value [--property2=value2 ...]]
11 12

This script is a wrapper around various version control and repository
13 14
checkout commands. It requires a |config| name, fetches data from that
config in depot_tools/fetch_configs, and then performs all necessary inits,
15 16 17
checkouts, pulls, fetches, etc.

Optional arguments may be passed on the command line in key-value pairs.
18
These parameters will be passed through to the config's main method.
19 20 21
"""

import json
22
import optparse
23
import os
24
import pipes
25 26
import subprocess
import sys
27
import textwrap
28

29 30
from distutils import spawn

31 32 33 34 35 36 37 38 39 40 41

SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))

#################################################
# Checkout class definitions.
#################################################
class Checkout(object):
  """Base class for implementing different types of checkouts.

  Attributes:
    |base|: the absolute path of the directory in which this script is run.
42
    |spec|: the spec for this checkout as returned by the config. Different
43 44
        subclasses will expect different keys in this dictionary.
    |root|: the directory into which the checkout will be performed, as returned
45
        by the config. This is a relative path from |base|.
46
  """
47
  def __init__(self, options, spec, root):
48
    self.base = os.getcwd()
49
    self.options = options
50 51 52 53 54 55 56 57 58 59 60 61
    self.spec = spec
    self.root = root

  def exists(self):
    pass

  def init(self):
    pass

  def sync(self):
    pass

62 63
  def run(self, cmd, **kwargs):
    print 'Running: %s' % (' '.join(pipes.quote(x) for x in cmd))
64
    if self.options.dry_run:
65
      return ''
66
    return subprocess.check_output(cmd, **kwargs)
67

68 69 70

class GclientCheckout(Checkout):

71
  def run_gclient(self, *cmd, **kwargs):
72 73 74 75 76
    if not spawn.find_executable('gclient'):
      cmd_prefix = (sys.executable, os.path.join(SCRIPT_PATH, 'gclient.py'))
    else:
      cmd_prefix = ('gclient',)
    return self.run(cmd_prefix + cmd, **kwargs)
77

78 79 80 81 82 83 84 85 86
  def exists(self):
    try:
      gclient_root = self.run_gclient('root').strip()
      return (os.path.exists(os.path.join(gclient_root, '.gclient')) or
              os.path.exists(os.path.join(os.getcwd(), self.root)))
    except subprocess.CalledProcessError:
      pass
    return os.path.exists(os.path.join(os.getcwd(), self.root))

87 88 89

class GitCheckout(Checkout):

90
  def run_git(self, *cmd, **kwargs):
91
    if sys.platform == 'win32' and not spawn.find_executable('git'):
92
      git_path = os.path.join(SCRIPT_PATH, 'git.bat')
93 94 95
    else:
      git_path = 'git'
    return self.run((git_path,) + cmd, **kwargs)
96 97


98 99 100
class SvnCheckout(Checkout):

  def run_svn(self, *cmd, **kwargs):
101 102 103 104 105
    if sys.platform == 'win32' and not spawn.find_executable('svn'):
      svn_path = os.path.join(SCRIPT_PATH, 'svn_bin', 'svn.exe')
    else:
      svn_path = 'svn'
    return self.run((svn_path,) + cmd, **kwargs)
106 107


108
class GclientGitCheckout(GclientCheckout, GitCheckout):
109

110 111
  def __init__(self, options, spec, root):
    super(GclientGitCheckout, self).__init__(options, spec, root)
112
    assert 'solutions' in self.spec
113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130

  def _format_spec(self):
    def _format_literal(lit):
      if isinstance(lit, basestring):
        return '"%s"' % lit
      if isinstance(lit, list):
        return '[%s]' % ', '.join(_format_literal(i) for i in lit)
      return '%r' % lit
    soln_strings = []
    for soln in self.spec['solutions']:
      soln_string= '\n'.join('    "%s": %s,' % (key, _format_literal(value))
                             for key, value in soln.iteritems())
      soln_strings.append('  {\n%s\n  },' % soln_string)
    gclient_spec = 'solutions = [\n%s\n]\n' % '\n'.join(soln_strings)
    extra_keys = ['target_os', 'target_os_only']
    gclient_spec += ''.join('%s = %s\n' % (key, _format_literal(self.spec[key]))
                             for key in extra_keys if key in self.spec)
    return gclient_spec
131 132 133

  def init(self):
    # Configure and do the gclient checkout.
134
    self.run_gclient('config', '--spec', self._format_spec())
jochen@chromium.org's avatar
jochen@chromium.org committed
135
    sync_cmd = ['sync']
136
    if self.options.nohooks:
jochen@chromium.org's avatar
jochen@chromium.org committed
137
      sync_cmd.append('--nohooks')
138 139
    if self.options.no_history:
      sync_cmd.append('--no-history')
jochen@chromium.org's avatar
jochen@chromium.org committed
140 141 142
    if self.spec.get('with_branch_heads', False):
      sync_cmd.append('--with_branch_heads')
    self.run_gclient(*sync_cmd)
143 144 145

    # Configure git.
    wd = os.path.join(self.base, self.root)
146
    if self.options.dry_run:
147
      print 'cd %s' % wd
148 149 150 151
    self.run_git(
        'submodule', 'foreach',
        'git config -f $toplevel/.git/config submodule.$name.ignore all',
        cwd=wd)
152 153 154
    self.run_git(
        'config', '--add', 'remote.origin.fetch',
        '+refs/tags/*:refs/tags/*', cwd=wd)
155 156
    self.run_git('config', 'diff.ignoreSubmodules', 'all', cwd=wd)

157 158 159

class GclientGitSvnCheckout(GclientGitCheckout, SvnCheckout):

160 161
  def __init__(self, options, spec, root):
    super(GclientGitSvnCheckout, self).__init__(options, spec, root)
162 163 164 165 166 167

  def init(self):
    # Ensure we are authenticated with subversion for all submodules.
    git_svn_dirs = json.loads(self.spec.get('submodule_git_svn_spec', '{}'))
    git_svn_dirs.update({self.root: self.spec})
    for _, svn_spec in git_svn_dirs.iteritems():
168 169 170 171 172 173
      if svn_spec.get('svn_url'):
        try:
          self.run_svn('ls', '--non-interactive', svn_spec['svn_url'])
        except subprocess.CalledProcessError:
          print 'Please run `svn ls %s`' % svn_spec['svn_url']
          return 1
174 175 176

    super(GclientGitSvnCheckout, self).init()

177
    # Configure git-svn.
178 179 180 181 182
    for path, svn_spec in git_svn_dirs.iteritems():
      real_path = os.path.join(*path.split('/'))
      if real_path != self.root:
        real_path = os.path.join(self.root, real_path)
      wd = os.path.join(self.base, real_path)
183
      if self.options.dry_run:
184
        print 'cd %s' % wd
185 186 187 188 189 190 191 192 193 194 195
      if svn_spec.get('auto'):
        self.run_git('auto-svn', cwd=wd)
        continue
      self.run_git('svn', 'init', svn_spec['svn_url'], cwd=wd)
      self.run_git('config', '--unset-all', 'svn-remote.svn.fetch', cwd=wd)
      for svn_branch, git_ref in svn_spec.get('git_svn_fetch', {}).items():
        self.run_git('config', '--add', 'svn-remote.svn.fetch',
                     '%s:%s' % (svn_branch, git_ref), cwd=wd)
      for svn_branch, git_ref in svn_spec.get('git_svn_branches', {}).items():
        self.run_git('config', '--add', 'svn-remote.svn.branches',
                     '%s:%s' % (svn_branch, git_ref), cwd=wd)
196 197 198
      self.run_git('svn', 'fetch', cwd=wd)


199

200 201
CHECKOUT_TYPE_MAP = {
    'gclient':         GclientCheckout,
202
    'gclient_git':     GclientGitCheckout,
203 204 205 206 207
    'gclient_git_svn': GclientGitSvnCheckout,
    'git':             GitCheckout,
}


208
def CheckoutFactory(type_name, options, spec, root):
209 210 211 212
  """Factory to build Checkout class instances."""
  class_ = CHECKOUT_TYPE_MAP.get(type_name)
  if not class_:
    raise KeyError('unrecognized checkout type: %s' % type_name)
213
  return class_(options, spec, root)
214 215 216 217 218 219 220 221 222 223


#################################################
# Utility function and file entry point.
#################################################
def usage(msg=None):
  """Print help and exit."""
  if msg:
    print 'Error:', msg

224
  print textwrap.dedent("""\
225
    usage: %s [options] <config> [--property=value [--property2=value2 ...]]
226 227 228 229 230 231 232 233 234 235 236

    This script can be used to download the Chromium sources. See
    http://www.chromium.org/developers/how-tos/get-the-code
    for full usage instructions.

    Valid options:
       -h, --help, help   Print this message.
       --nohooks          Don't run hooks after checkout.
       -n, --dry-run      Don't run commands, only print them.
       --no-history       Perform shallow clones, don't fetch the full git history.

237
    Valid fetch configs:""") % os.path.basename(sys.argv[0])
238

239 240 241 242
  configs_dir = os.path.join(SCRIPT_PATH, 'fetch_configs')
  configs = [f[:-3] for f in os.listdir(configs_dir) if f.endswith('.py')]
  configs.sort()
  for fname in configs:
243
    print '  ' + fname
244

245 246 247 248
  sys.exit(bool(msg))


def handle_args(argv):
249
  """Gets the config name from the command line arguments."""
250
  if len(argv) <= 1:
251
    usage('Must specify a config.')
252 253
  if argv[1] in ('-h', '--help', 'help'):
    usage()
254

255
  dry_run = False
256
  nohooks = False
257
  no_history = False
258 259 260 261
  while len(argv) >= 2:
    arg = argv[1]
    if not arg.startswith('-'):
      break
262
    argv.pop(1)
263
    if arg in ('-n', '--dry-run'):
264
      dry_run = True
265 266
    elif arg == '--nohooks':
      nohooks = True
267 268
    elif arg == '--no-history':
      no_history = True
269 270
    else:
      usage('Invalid option %s.' % arg)
271

272 273 274 275 276 277 278
  def looks_like_arg(arg):
    return arg.startswith('--') and arg.count('=') == 1

  bad_parms = [x for x in argv[2:] if not looks_like_arg(x)]
  if bad_parms:
    usage('Got bad arguments %s' % bad_parms)

279
  config = argv[1]
280
  props = argv[2:]
281 282 283
  return (
      optparse.Values(
          {'dry_run':dry_run, 'nohooks':nohooks, 'no_history': no_history }),
284
      config,
285
      props)
286 287


288 289
def run_config_fetch(config, props, aliased=False):
  """Invoke a config's fetch method with the passed-through args
290
  and return its json output as a python object."""
291 292 293 294
  config_path = os.path.abspath(
      os.path.join(SCRIPT_PATH, 'fetch_configs', config))
  if not os.path.exists(config_path + '.py'):
    print "Could not find a config for %s" % config
295 296
    sys.exit(1)

297
  cmd = [sys.executable, config_path + '.py', 'fetch'] + props
298
  result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
299

300 301 302
  spec = json.loads(result)
  if 'alias' in spec:
    assert not aliased
303 304 305
    return run_config_fetch(
        spec['alias']['config'], spec['alias']['props'] + props, aliased=True)
  cmd = [sys.executable, config_path + '.py', 'root']
306 307 308 309 310
  result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
  root = json.loads(result)
  return spec, root


311
def run(options, spec, root):
312 313 314
  """Perform a checkout with the given type and configuration.

    Args:
315
      options: Options instance.
316
      spec: Checkout configuration returned by the the config's fetch_spec
317 318 319 320 321 322 323
          method (checkout type, repository url, etc.).
      root: The directory into which the repo expects to be checkout out.
  """
  assert 'type' in spec
  checkout_type = spec['type']
  checkout_spec = spec['%s_spec' % checkout_type]
  try:
324
    checkout = CheckoutFactory(checkout_type, options, checkout_spec, root)
325 326 327
  except KeyError:
    return 1
  if checkout.exists():
328 329 330
    print 'Your current directory appears to already contain, or be part of, '
    print 'a checkout. "fetch" is used only to get new checkouts. Use '
    print '"gclient sync" to update existing checkouts.'
331 332 333
    print
    print 'Fetch also does not yet deal with partial checkouts, so if fetch'
    print 'failed, delete the checkout and start over (crbug.com/230691).'
334
    return 1
335
  return checkout.init()
336 337 338


def main():
339 340
  options, config, props = handle_args(sys.argv)
  spec, root = run_config_fetch(config, props)
341
  return run(options, spec, root)
342 343 344


if __name__ == '__main__':
345 346 347 348 349
  try:
    sys.exit(main())
  except KeyboardInterrupt:
    sys.stderr.write('interrupted\n')
    sys.exit(1)