First commit of depot_tools/fetch.py

This is the bare minimum to have fetch.py working for chromium.
The DEPS file pulls the new tools/recipes repo in, which will contain
the recipe for chromium and all other checkout recipes eventually (followup
CL coming shortly). fetch and fetch.bat simply invoke fetch.py. fetch.py
takes a recipe and some arguments on the command line, invokes the recipe
to get its instructions, and performs the actual checkout.

Currently only supports Gclient-Git-Svn checkout (the kind needed for
Chromium), but other checkout types will be pulled from annotated_checkout.py
soon.

Review URL: https://codereview.chromium.org/13463006

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@192130 0039d316-1c4b-4281-b951-d872f2087c98

fetch

+#!/usr/bin/env bash
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+base_dir=$(dirname "$0")
+
+PYTHONDONTWRITEBYTECODE=1 exec python "$base_dir/fetch.py" "$@"

fetch.bat

+@echo off
+:: Copyright (c) 2013 The Chromium Authors. All rights reserved.
+:: Use of this source code is governed by a BSD-style license that can be
+:: found in the LICENSE file.
+
+:: This is required with cygwin only.
+PATH=%~dp0;%PATH%
+
+:: Defer control.
+%~dp0python "%~dp0\fetch.py" %*

fetch.py

+#!/usr/bin/env python
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Tool to perform checkouts in one easy command line!
+
+Usage:
+  fetch <recipe> [--property=value [--property2=value2 ...]]
+
+This script is a wrapper around various version control and repository
+checkout commands. It requires a |recipe| name, fetches data from that
+recipe in depot_tools/recipes, and then performs all necessary inits,
+checkouts, pulls, fetches, etc.
+
+Optional arguments may be passed on the command line in key-value pairs.
+These parameters will be passed through to the recipe's main method.
+"""
+
+import json
+import os
+import subprocess
+import sys
+import pipes
+
+
+SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
+
+
+#################################################
+# Checkout class definitions.
+#################################################
+class Checkout(object):
+  """Base class for implementing different types of checkouts.
+
+  Attributes:
+    |base|: the absolute path of the directory in which this script is run.
+    |spec|: the spec for this checkout as returned by the recipe. Different
+        subclasses will expect different keys in this dictionary.
+    |root|: the directory into which the checkout will be performed, as returned
+        by the recipe. This is a relative path from |base|.
+  """
+  def __init__(self, spec, root):
+    self.base = os.getcwd()
+    self.spec = spec
+    self.root = root
+
+  def exists(self):
+    pass
+
+  def init(self):
+    pass
+
+  def sync(self):
+    pass
+
+
+class GclientCheckout(Checkout):
+
+  @staticmethod
+  def run_gclient(*cmd, **kwargs):
+    print 'Running: gclient %s' % ' '.join(pipes.quote(x) for x in cmd)
+    return subprocess.check_call(('gclient',) + cmd, **kwargs)
+
+
+class GitCheckout(Checkout):
+
+  @staticmethod
+  def run_git(*cmd, **kwargs):
+    print 'Running: git %s' % ' '.join(pipes.quote(x) for x in cmd)
+    return subprocess.check_call(('git',) + cmd, **kwargs)
+
+
+class GclientGitSvnCheckout(GclientCheckout, GitCheckout):
+
+  def __init__(self, spec, root):
+    super(GclientGitSvnCheckout, self).__init__(spec, root)
+    assert 'solutions' in self.spec
+    keys = ['solutions', 'target_os', 'target_os_only']
+    gclient_spec = '\n'.join('%s = %s' % (key, self.spec[key])
+                             for key in self.spec if key in keys)
+    self.spec['gclient_spec'] = gclient_spec
+    assert 'svn_url' in self.spec
+    assert 'svn_branch' in self.spec
+    assert 'svn_ref' in self.spec
+
+  def exists(self):
+    return os.path.exists(os.path.join(os.getcwd(), self.root))
+
+  def init(self):
+    # Configure and do the gclient checkout.
+    self.run_gclient('config', '--spec', self.spec['gclient_spec'])
+    self.run_gclient('sync')
+
+    # Configure git.
+    wd = os.path.join(self.base, self.root)
+    self.run_git(
+        'submodule', 'foreach',
+        'git config -f $toplevel/.git/config submodule.$name.ignore all',
+        cwd=wd)
+    self.run_git('config', 'diff.ignoreSubmodules', 'all', cwd=wd)
+
+    # Configure git-svn.
+    self.run_git('svn', 'init', '--prefix=origin/', '-T',
+                 self.spec['svn_branch'], self.spec['svn_url'], cwd=wd)
+    self.run_git('config', 'svn-remote.svn.fetch', self.spec['svn_branch'] +
+                 ':refs/remotes/origin/' + self.spec['svn_ref'], cwd=wd)
+    self.run_git('svn', 'fetch', cwd=wd)
+
+    # Configure git-svn submodules, if any.
+    submodules = json.loads(self.spec.get('submodule_git_svn_spec', '{}'))
+    for path, subspec in submodules.iteritems():
+      subspec = submodules[path]
+      ospath = os.path.join(*path.split('/'))
+      wd = os.path.join(self.base, self.root, ospath)
+      self.run_git('svn', 'init', '--prefix=origin/', '-T',
+                   subspec['svn_branch'], subspec['svn_url'], cwd=wd)
+      self.run_git('config', '--replace', 'svn-remote.svn.fetch',
+                   subspec['svn_branch'] + ':refs/remotes/origin/' +
+                   subspec['svn_ref'], cwd=wd)
+      self.run_git('svn', 'fetch', cwd=wd)
+
+
+CHECKOUT_TYPE_MAP = {
+    'gclient':         GclientCheckout,
+    'gclient_git_svn': GclientGitSvnCheckout,
+    'git':             GitCheckout,
+}
+
+
+def CheckoutFactory(type_name, spec, root):
+  """Factory to build Checkout class instances."""
+  class_ = CHECKOUT_TYPE_MAP.get(type_name)
+  if not class_:
+    raise KeyError('unrecognized checkout type: %s' % type_name)
+  return class_(spec, root)
+
+
+#################################################
+# Utility function and file entry point.
+#################################################
+def usage(msg=None):
+  """Print help and exit."""
+  if msg:
+    print 'Error:', msg
+
+  print (
+"""
+usage: %s <recipe> [--property=value [--property2=value2 ...]]
+""" % os.path.basename(sys.argv[0]))
+  sys.exit(bool(msg))
+
+
+def handle_args(argv):
+  """Gets the recipe name from the command line arguments."""
+  if len(argv) <= 1:
+    usage('Must specify a recipe.')
+
+  def looks_like_arg(arg):
+    return arg.startswith('--') and arg.count('=') == 1
+
+  bad_parms = [x for x in argv[2:] if not looks_like_arg(x)]
+  if bad_parms:
+    usage('Got bad arguments %s' % bad_parms)
+
+  recipe = argv[1]
+  props = argv[2:]
+  return recipe, props
+
+
+def run_recipe_fetch(recipe, props, aliased=False):
+  """Invoke a recipe's fetch method with the passed-through args
+  and return its json output as a python object."""
+  recipe_path = os.path.abspath(os.path.join(SCRIPT_PATH, 'recipes', recipe))
+  cmd = [sys.executable, recipe_path + '.py', 'fetch'] + props
+  result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
+  spec = json.loads(result)
+  if 'alias' in spec:
+    assert not aliased
+    return run_recipe_fetch(
+        spec['alias']['recipe'], spec['alias']['props'] + props, aliased=True)
+  cmd = [sys.executable, recipe_path + '.py', 'root']
+  result = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()[0]
+  root = json.loads(result)
+  return spec, root
+
+
+def run(spec, root):
+  """Perform a checkout with the given type and configuration.
+
+    Args:
+      spec: Checkout configuration returned by the the recipe's fetch_spec
+          method (checkout type, repository url, etc.).
+      root: The directory into which the repo expects to be checkout out.
+  """
+  assert 'type' in spec
+  checkout_type = spec['type']
+  checkout_spec = spec['%s_spec' % checkout_type]
+  try:
+    checkout = CheckoutFactory(checkout_type, checkout_spec, root)
+  except KeyError:
+    return 1
+  if checkout.exists():
+    print 'You appear to already have this checkout.'
+    print 'Aborting to avoid clobbering your work.'
+    return 1
+  checkout.init()
+  return 0
+
+
+def main():
+  recipe, props = handle_args(sys.argv)
+  spec, root = run_recipe_fetch(recipe, props)
+  return run(spec, root)
+
+
+if __name__ == '__main__':
+  sys.exit(main())

recipes/chromium.py

+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# pylint: disable=F0401
+import recipe_util
+import sys
+
+# pylint: disable=W0232
+class Chromium(recipe_util.Recipe):
+  """Basic Recipe class for Chromium."""
+
+  @staticmethod
+  def fetch_spec(props):
+    url = 'https://chromium.googlesource.com/chromium/src.git'
+    solution = { 'name'   :'src',
+                 'url'    : url,
+                 'deps_file': '.DEPS.git',
+                 'managed'   : True,
+                 'custom_deps': {},
+                 'safesync_url': '',
+    }
+    if props.get('webkit_rev', '') == 'ToT':
+      solution['custom_vars'] = {'webkit_rev': ''}
+    spec = {
+      'solutions': [solution],
+      'svn_url': 'svn://svn.chromium.org/chrome',
+      'svn_branch': 'trunk/src',
+      'svn_ref': 'git-svn',
+    }
+    if props.get('submodule_git_svn_spec'):
+      spec['submodule_git_svn_spec'] = props['submodule_git_svn_spec']
+    return {
+      'type': 'gclient_git_svn',
+      'gclient_git_svn_spec': spec
+    }
+
+  @staticmethod
+  def expected_root(_props):
+    return 'src'
+
+
+def main(argv=None):
+  return Chromium().handle_args(argv)
+
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))

recipes/recipe_util.py

+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""This module holds utilities which make writing recipes easier."""
+
+import json
+
+
+class Recipe(object):
+  """Base class for all recipes.
+
+  Provides methods that are expected to be overridden by child classes. Also
+  provides an command-line parsing method that converts the unified command-line
+  interface used in depot_tools to the unified python interface defined here."""
+
+  @staticmethod
+  def fetch_spec(_props):
+    """Returns instructions to check out the project, conditioned on |props|."""
+    raise NotImplementedError
+
+  @staticmethod
+  def expected_root(_props):
+    """Returns the directory into which the checkout will be performed."""
+    raise NotImplementedError
+
+  def handle_args(self, argv):
+    """Passes the command-line arguments through to the appropriate method."""
+    methods = {'fetch': self.fetch_spec,
+               'root':  self.expected_root}
+    if len(argv) <= 1 or argv[1] not in methods:
+      print 'Must specify a a fetch/root action'
+      return 1
+
+    def looks_like_arg(arg):
+      return arg.startswith('--') and arg.count('=') == 1
+
+    bad_parms = [x for x in argv[2:] if not looks_like_arg(x)]
+    if bad_parms:
+      print 'Got bad arguments %s' % bad_parms
+      return 1
+
+    method = methods[argv[1]]
+    props = dict(x.split('=', 1) for x in (y.lstrip('-') for y in argv[2:]))
+
+    self.output(method(props))
+
+  @staticmethod
+  def output(data):
+    print(json.dumps(data))