Fix depot_tools presubmit's local_rietveld script.

This was breaking presubmit because the linked gae_ts_mon subdir was absent.

R=andybons@chromium.org, dsansome@chromium.org, dnj@chromium.org
BUG=608490

Review-Url: https://codereview.chromium.org/1959193002

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@300511 0039d316-1c4b-4281-b951-d872f2087c98

.gitignore

@@ -43,6 +43,7 @@
 /tags
 
 # Ignore unittest related files.
+/testing_support/_infra
 /testing_support/_rietveld
 /tests/git-svn-submodule/
 /tests/subversion_config/README.txt

fetch.py

@@ -231,6 +231,7 @@ def usage(msg=None):
     Valid options:
        -h, --help, help   Print this message.
        --nohooks          Don't run hooks after checkout.
+       --force            (dangerous) Don't look for existing .gclient file.
        -n, --dry-run      Don't run commands, only print them.
        --no-history       Perform shallow clones, don't fetch the full git history.
 
@@ -255,6 +256,7 @@ def handle_args(argv):
   dry_run = False
   nohooks = False
   no_history = False
+  force = False
   while len(argv) >= 2:
     arg = argv[1]
     if not arg.startswith('-'):
@@ -266,6 +268,8 @@ def handle_args(argv):
       nohooks = True
     elif arg == '--no-history':
       no_history = True
+    elif arg == '--force':
+      force = True
     else:
       usage('Invalid option %s.' % arg)
 
@@ -279,8 +283,11 @@ def handle_args(argv):
   config = argv[1]
   props = argv[2:]
   return (
-      optparse.Values(
-          {'dry_run':dry_run, 'nohooks':nohooks, 'no_history': no_history }),
+      optparse.Values({
+        'dry_run': dry_run,
+        'nohooks': nohooks,
+        'no_history': no_history,
+        'force': force}),
       config,
       props)
 
@@ -324,7 +331,7 @@ def run(options, spec, root):
     checkout = CheckoutFactory(checkout_type, options, checkout_spec, root)
   except KeyError:
     return 1
-  if checkout.exists():
+  if not options.force and checkout.exists():
     print 'Your current directory appears to already contain, or be part of, '
     print 'a checkout. "fetch" is used only to get new checkouts. Use '
     print '"gclient sync" to update existing checkouts.'

fetch_configs/infra.py

@@ -13,7 +13,9 @@ class Infra(config_util.Config):
   """Basic Config class for the Infrastructure repositories."""
 
   @staticmethod
-  def fetch_spec(_props):
+  def fetch_spec(props):
+    # This is used by [depot_tools]/testing_support/local_rietveld.py
+    managed = props.get('managed', 'false').lower() == 'true'
     return {
       'type': 'gclient_git',
       'gclient_git_spec': {
@@ -22,7 +24,7 @@ class Infra(config_util.Config):
             'name'     : 'infra',
             'url'      : 'https://chromium.googlesource.com/infra/infra.git',
             'deps_file': '.DEPS.git',
-            'managed'  : False,
+            'managed'  : managed,
           }
         ],
       },

testing_support/get_appengine.py

-#!/usr/bin/env python
-# Copyright (c) 2011 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""This script is copied from
-https://chromium.googlesource.com/infra/infra.git/+/master/bootstrap
-"""
-
-import datetime
-import logging
-import optparse
-import os
-import re
-import shutil
-import sys
-import time
-import tempfile
-import urllib2
-import zipfile
-
-BASE_DIR = os.path.dirname(os.path.abspath(__file__))
-
-
-def get_gae_sdk_version(gae_path):
-  """Returns the installed GAE SDK version or None."""
-  version_path = os.path.join(gae_path, 'VERSION')
-  if os.path.isfile(version_path):
-    values = dict(
-        map(lambda x: x.strip(), l.split(':'))
-        for l in open(version_path) if ':' in l)
-    if 'release' in values:
-      return values['release'].strip('"')
-
-
-def get_latest_gae_sdk_url(name):
-  """Returns the url to get the latest GAE SDK and its version."""
-  url = 'https://cloud.google.com/appengine/downloads.html'
-  logging.debug('%s', url)
-  content = urllib2.urlopen(url).read()
-  regexp = (
-      r'(https\:\/\/storage.googleapis.com\/appengine-sdks\/featured\/'
-      + re.escape(name) + r'[0-9\.]+?\.zip)')
-  m = re.search(regexp, content)
-  url = m.group(1)
-  # Calculate the version from the url.
-  new_version = re.search(re.escape(name) + r'(.+?).zip', url).group(1)
-  # Upgrade to https
-  return url.replace('http://', 'https://'), new_version
-
-
-def extract_zip(z, root_path):
-  """Extracts files in a zipfile but keep the executable bits."""
-  count = 0
-  for f in z.infolist():
-    perm = (f.external_attr >> 16L) & 0777
-    mtime = time.mktime(datetime.datetime(*f.date_time).timetuple())
-    filepath = os.path.join(root_path, f.filename)
-    logging.debug('Extracting %s', f.filename)
-    if f.filename.endswith('/'):
-      os.mkdir(filepath, perm)
-    else:
-      z.extract(f, root_path)
-      os.chmod(filepath, perm)
-      count += 1
-    os.utime(filepath, (mtime, mtime))
-  print('Extracted %d files' % count)
-
-
-def install_latest_gae_sdk(root_path, fetch_go, dry_run):
-  if fetch_go:
-    rootdir = 'go_appengine'
-    if sys.platform == 'darwin':
-      name = 'go_appengine_sdk_darwin_amd64-'
-    else:
-      # Add other platforms as needed.
-      name = 'go_appengine_sdk_linux_amd64-'
-  else:
-    rootdir = 'google_appengine'
-    name = 'google_appengine_'
-
-  # The zip file already contains 'google_appengine' (for python) or
-  # 'go_appengine' (for go) in its path so it's a bit
-  # awkward to unzip otherwise. Hard code the path in for now.
-  gae_path = os.path.join(root_path, rootdir)
-  print('Looking up path %s' % gae_path)
-  version = get_gae_sdk_version(gae_path)
-  if version:
-    print('Found installed version %s' % version)
-  else:
-    print('Didn\'t find an SDK')
-
-  url, new_version = get_latest_gae_sdk_url(name)
-  print('New version is %s' % new_version)
-  if version == new_version:
-    return 0
-
-  if os.path.isdir(gae_path):
-    print('Removing previous version')
-    if not dry_run:
-      shutil.rmtree(gae_path)
-
-  print('Fetching %s' % url)
-  if not dry_run:
-    u = urllib2.urlopen(url)
-    with tempfile.NamedTemporaryFile() as f:
-      while True:
-        chunk = u.read(2 ** 20)
-        if not chunk:
-          break
-        f.write(chunk)
-      # Assuming we're extracting there. In fact, we have no idea.
-      print('Extracting into %s' % gae_path)
-      z = zipfile.ZipFile(f, 'r')
-      try:
-        extract_zip(z, root_path)
-      finally:
-        z.close()
-  return 0
-
-
-def main():
-  parser = optparse.OptionParser(prog='python -m %s' % __package__)
-  parser.add_option('-v', '--verbose', action='store_true')
-  parser.add_option(
-      '-g', '--go', action='store_true', help='Defaults to python SDK')
-  parser.add_option(
-      '-d', '--dest', default=os.path.dirname(BASE_DIR), help='Output')
-  parser.add_option('--dry-run', action='store_true', help='Do not download')
-  options, args = parser.parse_args()
-  if args:
-    parser.error('Unsupported args: %s' % ' '.join(args))
-  logging.basicConfig(level=logging.DEBUG if options.verbose else logging.ERROR)
-  return install_latest_gae_sdk(
-      os.path.abspath(options.dest), options.go, options.dry_run)
-
-
-if __name__ == '__main__':
-  sys.exit(main())

testing_support/local_rietveld.py

@@ -26,6 +26,7 @@ except ImportError:
       os.path.join(os.path.dirname(os.path.abspath(__file__)), '..'))
   import subprocess2
 
+DEPOT_TOOLS=os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 class Failure(Exception):
   pass
@@ -55,60 +56,54 @@ class LocalRietveld(object):
     self.base_dir = base_dir
     if not self.base_dir:
       self.base_dir = os.path.dirname(os.path.abspath(__file__))
-    # TODO(maruel): This should be in /tmp but that would mean having to fetch
-    # everytime. This test is already annoyingly slow.
     self.rietveld = os.path.join(self.base_dir, '_rietveld')
+    self.infra = os.path.join(self.base_dir, '_infra')
     self.rietveld_app = os.path.join(
-        self.rietveld, 'appengine', 'chromium_rietveld')
+        self.infra, 'infra', 'appengine', 'chromium_rietveld')
+    self.dev_app = os.path.join(
+        self.infra, 'google_appengine', 'dev_appserver.py')
     self.test_server = None
     self.port = None
     self.tempdir = None
-    self.dev_app = None
 
   def install_prerequisites(self):
-    # First, install the Google AppEngine SDK.
-    cmd = [os.path.join(self.base_dir, 'get_appengine.py'),
-           '--dest=%s' % self.base_dir]
-    try:
-      subprocess2.check_call(cmd)
-    except (OSError, subprocess2.CalledProcessError), e:
-      raise Failure('Failed to run %s\n%s' % (cmd, e))
-    sdk_path = os.path.join(self.base_dir, 'google_appengine')
-    self.dev_app = os.path.join(sdk_path, 'dev_appserver.py')
-
-    if os.path.isdir(os.path.join(self.rietveld, '.hg')):
-      # Left over from mercurial. Delete it.
-      print('Deleting deprecated mercurial rietveld files...')
+    if os.path.exists(self.rietveld):
+      print "Removing old rietveld dir"
       shutil.rmtree(self.rietveld)
 
-    # Second, checkout rietveld if not available.
-    if not os.path.isdir(self.rietveld):
-      print('Checking out rietveld...')
+    sdk_path = os.path.join(self.base_dir, 'google_appengine')
+    if os.path.exists(sdk_path):
+      print "Removing old appengine SDK dir"
+      shutil.rmtree(sdk_path)
+
+    previous = os.environ.get('DEPOT_TOOLS_UPDATE')
+    os.environ['DEPOT_TOOLS_UPDATE'] = '0'
+    try:
+      if not os.path.isfile(os.path.join(self.infra, '.gclient')):
+        print('Checking out infra...')
+        shutil.rmtree(self.infra, ignore_errors=True)
         try:
-        subprocess2.check_call(['git', 'init', self.rietveld])
-        subprocess2.check_call(
-            ['git', 'remote', 'add', '-f', 'origin',
-             'https://chromium.googlesource.com/infra/infra.git'],
-            cwd=self.rietveld)
-        subprocess2.check_call(
-            ['git', 'config', 'core.sparseCheckout', 'true'],
-            cwd=self.rietveld)
-        with file(os.path.join(self.rietveld, '.git/info/sparse-checkout'),
-                  'w') as sparse_file:
-          sparse_file.write('appengine/chromium_rietveld')
-        subprocess2.check_call(
-            ['git', 'pull', 'origin', 'master'],
-            cwd=self.rietveld)
+          os.makedirs(self.infra)
+          subprocess2.call(
+            [sys.executable, os.path.join(DEPOT_TOOLS, 'fetch.py'),
+             '--force', 'infra', '--managed=true'],
+            cwd=self.infra)
         except (OSError, subprocess2.CalledProcessError), e:
-        raise Failure('Failed to clone rietveld. \n%s' % e)
+          raise Failure('Failed to clone infra. \n%s' % e)
       else:
-      print('Syncing rietveld...')
+        print('Syncing infra...')
         try:
-        subprocess2.check_call(
-            ['git', 'pull', 'origin', 'master'],
-            cwd=self.rietveld)
+          subprocess2.call(
+            [sys.executable, os.path.join(DEPOT_TOOLS, 'gclient.py'),
+             'sync', '--force'],
+            cwd=self.infra)
         except (OSError, subprocess2.CalledProcessError), e:
-        raise Failure('Failed to sync rietveld\n%s' % e)
+          raise Failure('Failed to sync infra. \n%s' % e)
+    finally:
+      if previous is None:
+        del os.environ['DEPOT_TOOLS_UPDATE']
+      else:
+        os.environ['DEPOT_TOOLS_UPDATE'] = previous
 
   def start_server(self, verbose=False):
     self.install_prerequisites()