Roll recipe dependencies (trivial).

This is an automated CL created by the recipe roller. This CL rolls recipe
changes from upstream projects (e.g. depot_tools) into downstream projects
(e.g. tools/build).


More info is at https://goo.gl/zkKdpD. Use https://goo.gl/noib3a to file a bug.
recipe_engine:
  https://crrev.com/cfeeb53051e02856a35395a69b09f822ad35e2e5 [doc/recipes.py] Improve copypasta bootstrap script. (iannucci@chromium.org)


TBR=iannucci@chromium.org

Recipe-Tryjob-Bypass-Reason: Autoroller
Bugdroid-Send-Email: False
Change-Id: Ie57e5caef2c447be3d44866c1f75b1d655effe84
Reviewed-on: https://chromium-review.googlesource.com/499768
Reviewed-by: <recipe-roller@chromium.org>
Commit-Queue: <recipe-roller@chromium.org>

infra/config/recipes.cfg

@@ -15,7 +15,7 @@
   "deps": {
     "recipe_engine": {
       "branch": "master",
-      "revision": "d6c4a597be2da8be7971b4d5f66e3d95e8fc325c",
+      "revision": "cfeeb53051e02856a35395a69b09f822ad35e2e5",
       "url": "https://chromium.googlesource.com/external/github.com/luci/recipes-py.git"
     }
   },

recipes/recipes.py

 #!/usr/bin/env python
 
-# Copyright 2016 The LUCI Authors. All rights reserved.
+# Copyright 2017 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Bootstrap script to clone and forward to the recipe engine tool.
 
-***********************************************************************
-** DO NOT MODIFY EXCEPT IN THE PER-REPO CONFIGURATION SECTION BELOW. **
-***********************************************************************
+*******************
+** DO NOT MODIFY **
+*******************
 
 This is a copy of https://github.com/luci/recipes-py/blob/master/doc/recipes.py.
-To fix bugs, fix in the github repo then copy it back to here and fix the
-PER-REPO CONFIGURATION section to look like this one.
+To fix bugs, fix in the github repo then run the autoroller.
 """
 
 import os
 
-# IMPORTANT: Do not alter the header or footer line for the
-# "PER-REPO CONFIGURATION" section below, or the autoroller will not be able
-# to automatically update this file! All lines between the header and footer
-# lines will be retained verbatim by the autoroller.
-
 #### PER-REPO CONFIGURATION (editable) ####
 # The root of the repository relative to the directory of this file.
 REPO_ROOT = os.path.join(os.pardir)
@@ -29,8 +23,6 @@ REPO_ROOT = os.path.join(os.pardir)
 RECIPES_CFG = os.path.join('infra', 'config', 'recipes.cfg')
 #### END PER-REPO CONFIGURATION ####
 
-BOOTSTRAP_VERSION = 1
-
 import argparse
 import json
 import logging
@@ -40,8 +32,24 @@ import sys
 import time
 import urlparse
 
+from collections import namedtuple
+
 from cStringIO import StringIO
 
+# The dependency entry for the recipe_engine in the client repo's recipes.cfg
+#
+# url (str) - the url to the engine repo we want to use.
+# revision (str) - the git revision for the engine to get.
+# path_override (str) - the subdirectory in the engine repo we should use to
+#   find it's recipes.py entrypoint. This is here for completeness, but will
+#   essentially always be empty. It would be used if the recipes-py repo was
+#   merged as a subdirectory of some other repo and you depended on that
+#   subdirectory.
+# branch (str) - the branch to fetch for the engine as an absolute ref (e.g.
+#   refs/heads/master)
+# repo_type ("GIT"|"GITILES") - An ignored enum which will be removed soon.
+EngineDep = namedtuple('EngineDep',
+                       'url revision path_override branch repo_type')
 
 def parse(repo_root, recipes_cfg_path):
   """Parse is transitional code which parses a recipes.cfg file as either jsonpb
@@ -53,13 +61,7 @@ def parse(repo_root, recipes_cfg_path):
     recipes_cfg_path (str) - native path to the recipes.cfg file to process.
 
   Returns (as tuple):
-    engine_url (str) - the url to the engine repo we want to use.
-    engine_revision (str) - the git revision for the engine to get.
-    engine_subpath (str) - the subdirectory in the engine repo we should use to
-      find it's recipes.py entrypoint. This is here for completeness, but will
-      essentially always be empty. It would be used if the recipes-py repo was
-      merged as a subdirectory of some other repo and you depended on that
-      subdirectory.
+    engine_dep (EngineDep): The recipe_engine dependency.
     recipes_path (str) - native path to where the recipes live inside of the
       current repo (i.e. the folder containing `recipes/` and/or
       `recipe_modules`)
@@ -76,13 +78,34 @@ def parse(repo_root, recipes_cfg_path):
                        % recipes_cfg_path)
   else:
     engine = pb['deps']['recipe_engine']
-  engine_url = engine['url']
-  engine_revision = engine.get('revision', '')
-  engine_subpath = engine.get('path_override', '')
+
+  if 'url' not in engine:
+    raise ValueError(
+      'Required field "url" in dependency "recipe_engine" not found: %r' %
+      (recipes_cfg_path,)
+    )
+
+  engine.setdefault('revision', '')
+  engine.setdefault('path_override', '')
+  engine.setdefault('branch', 'refs/heads/master')
   recipes_path = pb.get('recipes_path', '')
 
+  # TODO(iannucci): only support absolute refs
+  if not engine['branch'].startswith('refs/'):
+    engine['branch'] = 'refs/heads/' + engine['branch']
+
+  engine.setdefault('repo_type', 'GIT')
+  if engine['repo_type'] not in ('GIT', 'GITILES'):
+    raise ValueError(
+      'Unsupported "repo_type" value in dependency "recipe_engine": %r' %
+      (recipes_cfg_path,)
+    )
+
   recipes_path = os.path.join(repo_root, recipes_path.replace('/', os.path.sep))
-  return engine_url, engine_revision, engine_subpath, recipes_path
+  return EngineDep(**engine), recipes_path
+
+
+GIT = 'git.bat' if sys.platform.startswith(('win', 'cygwin')) else 'git'
 
 
 def _subprocess_call(argv, **kwargs):
@@ -90,11 +113,18 @@ def _subprocess_call(argv, **kwargs):
   return subprocess.call(argv, **kwargs)
 
 
-def _subprocess_check_call(argv, **kwargs):
+def _git_check_call(argv, **kwargs):
+  argv = [GIT]+argv
   logging.info('Running %r', argv)
   subprocess.check_call(argv, **kwargs)
 
 
+def _git_output(argv, **kwargs):
+  argv = [GIT]+argv
+  logging.info('Running %r', argv)
+  return subprocess.check_output(argv, **kwargs)
+
+
 def find_engine_override(argv):
   """Since the bootstrap process attempts to defer all logic to the recipes-py
   repo, we need to be aware if the user is overriding the recipe_engine
@@ -111,65 +141,63 @@ def find_engine_override(argv):
   return None
 
 
-def main():
-  if '--verbose' in sys.argv:
-    logging.getLogger().setLevel(logging.INFO)
-
-  if REPO_ROOT is None or RECIPES_CFG is None:
-    logging.error(
-      'In order to use this script, please copy it to your repo and '
-      'replace the REPO_ROOT and RECIPES_CFG values with approprite paths.')
-    sys.exit(1)
-
-  if sys.platform.startswith(('win', 'cygwin')):
-    git = 'git.bat'
-  else:
-    git = 'git'
+def checkout_engine(repo_root, recipes_cfg_path):
+  """Checks out"""
 
-  # Find the repository and config file to operate on.
-  repo_root = os.path.abspath(
-      os.path.join(os.path.dirname(__file__), REPO_ROOT))
-  recipes_cfg_path = os.path.join(repo_root, RECIPES_CFG)
+  dep, recipes_path = parse(repo_root, recipes_cfg_path)
 
-  engine_url, engine_revision, engine_subpath, recipes_path = parse(
-    repo_root, recipes_cfg_path)
+  url = dep.url
 
   engine_path = find_engine_override(sys.argv[1:])
-  if not engine_path and engine_url.startswith('file://'):
-    engine_path = urlparse.urlparse(engine_url).path
+  if not engine_path and url.startswith('file://'):
+    engine_path = urlparse.urlparse(url).path
 
   if not engine_path:
-    deps_path = os.path.join(recipes_path, '.recipe_deps')
+    revision = dep.revision
+    subpath = dep.path_override
+    branch = dep.branch
+
     # Ensure that we have the recipe engine cloned.
-    engine_root_path = os.path.join(deps_path, 'recipe_engine')
-    engine_path = os.path.join(engine_root_path, engine_subpath)
-    def ensure_engine():
-      if not os.path.exists(deps_path):
-        os.makedirs(deps_path)
-      if not os.path.exists(engine_root_path):
-        _subprocess_check_call([git, 'clone', engine_url, engine_root_path])
-
-      needs_fetch = _subprocess_call(
-          [git, 'rev-parse', '--verify', '%s^{commit}' % engine_revision],
-          cwd=engine_root_path, stdout=open(os.devnull, 'w'))
-      if needs_fetch:
-        _subprocess_check_call([git, 'fetch'], cwd=engine_root_path)
-      _subprocess_check_call(
-          [git, 'checkout', '--quiet', engine_revision], cwd=engine_root_path)
+    engine = os.path.join(recipes_path, '.recipe_deps', 'recipe_engine')
+    engine_path = os.path.join(engine, subpath)
+
+    with open(os.devnull, 'w') as NUL:
+      # Note: this logic mirrors the logic in recipe_engine/fetch.py
+      _git_check_call(['init', engine], stdout=NUL)
+
+      try:
+        _git_check_call(['rev-parse', '--verify', '%s^{commit}' % revision],
+                        cwd=engine, stdout=NUL, stderr=NUL)
+      except subprocess.CalledProcessError:
+        _git_check_call(['fetch', url, branch], cwd=engine, stdout=NUL,
+                        stderr=NUL)
 
     try:
-      ensure_engine()
+      _git_check_call(['diff', '--quiet', revision], cwd=engine)
     except subprocess.CalledProcessError:
-      logging.exception('ensure_engine failed')
+      _git_check_call(['reset', '-q', '--hard', revision], cwd=engine)
+
+  return engine_path
+
 
-      # Retry errors.
-      time.sleep(random.uniform(2,5))
-      ensure_engine()
+def main():
+  if '--verbose' in sys.argv:
+    logging.getLogger().setLevel(logging.INFO)
+
+  repo_root = os.path.abspath(
+    _git_output(['rev-parse', '--show-toplevel'],
+                cwd=os.path.dirname(__file__)).strip())
+
+  # TODO(iannucci): Actually make the location of recipes.cfg configurable.
+  recipes_cfg_path = os.path.join(repo_root, 'infra', 'config', 'recipes.cfg')
+
+  engine_path = checkout_engine(repo_root, recipes_cfg_path)
 
   args = ['--package', recipes_cfg_path] + sys.argv[1:]
   return _subprocess_call([
       sys.executable, '-u',
       os.path.join(engine_path, 'recipes.py')] + args)
 
+
 if __name__ == '__main__':
   sys.exit(main())