Revert "Remove all safesync_url functionality from gclient"

This reverts commit 99a7f805.
This reverts commit 2697cd19.
This reverts commit 8c51b6f1.
This reverts commit d4aedc81.

Reason for revert:
Breaks iOS release branch builds.

BUG=623762
TBR=agable,maruel

Review URL: https://codereview.chromium.org/2162583004 .

fetch_configs/breakpad.py

@@ -18,6 +18,7 @@ class Breakpad(config_util.Config):
       'url': url,
       'managed': False,
       'custom_deps': {},
+      'safesync_url': '',
     }
     spec = {
       'solutions': [solution],

fetch_configs/chromium.py

@@ -20,6 +20,7 @@ class Chromium(config_util.Config):
                  'deps_file': '.DEPS.git',
                  'managed'   : False,
                  'custom_deps': {},
+                 'safesync_url': '',
     }
     if props.get('webkit_revision', '') == 'ToT':
       solution['custom_vars'] = {'webkit_revision': ''}

fetch_configs/dart.py

@@ -21,6 +21,7 @@ class Dart(config_util.Config):
       'deps_file': 'DEPS',
       'managed'   : False,
       'custom_deps': {},
+      'safesync_url': '',
     }
     spec = {
       'solutions': [solution],

fetch_configs/dartino.py

@@ -20,6 +20,7 @@ class Dartino(config_util.Config):
       'deps_file': 'DEPS',
       'managed'   : False,
       'custom_deps': {},
+      'safesync_url': '',
     }
     spec = {
       'solutions': [solution],

fetch_configs/dartium.py

@@ -21,6 +21,7 @@ class Dart(config_util.Config):
       'deps_file': 'tools/deps/dartium.deps/DEPS',
       'managed'   : False,
       'custom_deps': {},
+      'safesync_url': '',
     }
     spec = {
       'solutions': [solution],

fetch_configs/gyp.py

@@ -19,6 +19,7 @@ class Chromium(config_util.Config):
                  'url'    : url,
                  'managed'   : False,
                  'custom_deps': {},
+                 'safesync_url': '',
     }
     spec = {
       'solutions': [solution],

fetch_configs/ios_internal.py

@@ -20,6 +20,7 @@ class IOSInternal(config_util.Config):
                  'deps_file': 'DEPS',
                  'managed'   : False,
                  'custom_deps': {},
+                 'safesync_url': '',
     }
     spec = {
       'solutions': [solution],

fetch_configs/mojo.py

@@ -21,6 +21,7 @@ class Mojo(config_util.Config):
       'deps_file': 'DEPS',
       'managed'   : False,
       'custom_deps': {},
+      'safesync_url': '',
     }
     spec = {
       'solutions': [solution],

fetch_configs/nacl.py

@@ -22,6 +22,7 @@ class NaCl(config_util.Config):
         'deps_file'   : 'DEPS',
         'managed'     : False,
         'custom_deps' : {},
+        'safesync_url': '',
     }
     spec = {
       'solutions': [solution],

fetch_configs/naclports.py

@@ -21,6 +21,7 @@ class Naclports(config_util.Config):
         'deps_file'   : 'DEPS',
         'managed'     : False,
         'custom_deps' : {},
+        'safesync_url': '',
     }
     spec = {
       'solutions': [solution],

fetch_configs/v8.py

@@ -21,6 +21,7 @@ class V8(config_util.Config):
         'deps_file'   : 'DEPS',
         'managed'     : False,
         'custom_deps' : {},
+        'safesync_url': '',
     }
     spec = {
       'solutions': [solution],

fetch_configs/webrtc.py

@@ -23,6 +23,7 @@ class WebRTC(config_util.Config):
           'deps_file': 'DEPS',
           'managed': False,
           'custom_deps': {},
+          'safesync_url': '',
         },
       ],
       'with_branch_heads': True,

gclient.py

@@ -213,18 +213,19 @@ class GClientKeywords(object):
 class DependencySettings(GClientKeywords):
   """Immutable configuration settings."""
   def __init__(
-      self, parent, url, managed, custom_deps, custom_vars,
+      self, parent, url, safesync_url, managed, custom_deps, custom_vars,
       custom_hooks, deps_file, should_process):
     GClientKeywords.__init__(self)
 
     # These are not mutable:
     self._parent = parent
+    self._safesync_url = safesync_url
     self._deps_file = deps_file
     self._url = url
     # 'managed' determines whether or not this dependency is synced/updated by
     # gclient after gclient checks it out initially.  The difference between
     # 'managed' and 'should_process' is that the user specifies 'managed' via
-    # the --managed command-line flag or a .gclient config, where
+    # the --unmanaged command-line flag or a .gclient config, where
     # 'should_process' is dynamically set by gclient if it goes over its
     # recursion limit and controls gclient's behavior so it does not misbehave.
     self._managed = managed
@@ -280,6 +281,10 @@ class DependencySettings(GClientKeywords):
       return self or GClient(None, None)
     return self.parent.root
 
+  @property
+  def safesync_url(self):
+    return self._safesync_url
+
   @property
   def should_process(self):
     """True if this dependency should be processed, i.e. checked out."""
@@ -319,11 +324,11 @@ class DependencySettings(GClientKeywords):
 class Dependency(gclient_utils.WorkItem, DependencySettings):
   """Object that represents a dependency checkout."""
 
-  def __init__(self, parent, name, url, managed, custom_deps,
+  def __init__(self, parent, name, url, safesync_url, managed, custom_deps,
                custom_vars, custom_hooks, deps_file, should_process):
     gclient_utils.WorkItem.__init__(self, name)
     DependencySettings.__init__(
-        self, parent, url, managed, custom_deps, custom_vars,
+        self, parent, url, safesync_url, managed, custom_deps, custom_vars,
         custom_hooks, deps_file, should_process)
 
     # This is in both .gclient and DEPS files:
@@ -722,7 +727,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
         if ent is not None:
           deps_file = ent['deps_file']
       deps_to_add.append(Dependency(
-          self, name, url, None, None, self.custom_vars, None,
+          self, name, url, None, None, None, self.custom_vars, None,
           deps_file, should_process))
     deps_to_add.sort(key=lambda x: x.name)
 
@@ -1129,7 +1134,7 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
 
   def __str__(self):
     out = []
-    for i in ('name', 'url', 'parsed_url', 'custom_deps',
+    for i in ('name', 'url', 'parsed_url', 'safesync_url', 'custom_deps',
               'custom_vars', 'deps_hooks', 'file_list', 'should_process',
               'processed', 'hooks_ran', 'deps_parsed', 'requirements',
               'allowed_hosts'):
@@ -1178,26 +1183,26 @@ class GClient(Dependency):
 
   DEFAULT_CLIENT_FILE_TEXT = ("""\
 solutions = [
-  {
-    "name"        : "%(solution_name)s",
+  { "name"        : "%(solution_name)s",
     "url"         : "%(solution_url)s",
     "deps_file"   : "%(deps_file)s",
     "managed"     : %(managed)s,
-    "custom_deps" : {},
+    "custom_deps" : {
+    },
+    "safesync_url": "%(safesync_url)s",
   },
 ]
 cache_dir = %(cache_dir)r
 """)
 
   DEFAULT_SNAPSHOT_SOLUTION_TEXT = ("""\
-  {
-    "name"        : "%(solution_name)s",
+  { "name"        : "%(solution_name)s",
     "url"         : "%(solution_url)s",
     "deps_file"   : "%(deps_file)s",
     "managed"     : %(managed)s,
     "custom_deps" : {
-%(solution_deps)s
-    },
+%(solution_deps)s    },
+    "safesync_url": "%(safesync_url)s",
   },
 """)
 
@@ -1211,7 +1216,7 @@ solutions = [
     # Do not change previous behavior. Only solution level and immediate DEPS
     # are processed.
     self._recursion_limit = 2
-    Dependency.__init__(self, None, None, None, True, None, None, None,
+    Dependency.__init__(self, None, None, None, None, True, None, None, None,
                         'unused', True)
     self._options = options
     if options.deps_os:
@@ -1252,8 +1257,8 @@ The local checkout in %(checkout_path)s reports:
 
 You should ensure that the URL listed in .gclient is correct and either change
 it or fix the checkout. If you're managing your own git checkout in
-%(checkout_path)s but the URL in .gclient is for an svn repository, you should
-set 'managed': False in .gclient.
+%(checkout_path)s but the URL in .gclient is for an svn repository, you probably
+want to set 'managed': False in .gclient.
 '''  % {'checkout_path': os.path.join(self.root_dir, dep.name),
         'expected_url': dep.url,
         'expected_scm': gclient_scm.GetScmName(dep.url),
@@ -1294,13 +1299,11 @@ set 'managed': False in .gclient.
 
     deps_to_add = []
     for s in config_dict.get('solutions', []):
-      if s.get('safesync_url'):
-        raise gclient_utils.Error('safesync_url is not supported anymore. '
-                                  'Please remove it from your .gclient file.')
       try:
         deps_to_add.append(Dependency(
             self, s['name'], s['url'],
-            s.get('managed', False),
+            s.get('safesync_url', None),
+            s.get('managed', True),
             s.get('custom_deps', {}),
             s.get('custom_vars', {}),
             s.get('custom_hooks', []),
@@ -1426,11 +1429,12 @@ been automagically updated.  The previous version is available at %s.old.
     return client
 
   def SetDefaultConfig(self, solution_name, deps_file, solution_url,
-                       managed=False, cache_dir=None):
+                       safesync_url, managed=True, cache_dir=None):
     self.SetConfig(self.DEFAULT_CLIENT_FILE_TEXT % {
       'solution_name': solution_name,
       'solution_url': solution_url,
       'deps_file': deps_file,
+      'safesync_url' : safesync_url,
       'managed': managed,
       'cache_dir': cache_dir,
     })
@@ -1473,22 +1477,48 @@ been automagically updated.  The previous version is available at %s.old.
   def _EnforceRevisions(self):
     """Checks for revision overrides."""
     revision_overrides = {}
+    if self._options.head:
+      return revision_overrides
+    # Do not check safesync_url if one or more --revision flag is specified.
     if not self._options.revisions:
       for s in self.dependencies:
-        if not s.managed and not self._options.__dict__.get('head', False):
+        if not s.managed:
           self._options.revisions.append('%s@unmanaged' % s.name)
+        elif s.safesync_url:
+          self._ApplySafeSyncRev(dep=s)
     if not self._options.revisions:
       return revision_overrides
     solutions_names = [s.name for s in self.dependencies]
-    for i, revision in enumerate(self._options.revisions):
-      if '@' in revision:
-        name, rev = revision.split('@', 1)
-      else:
+    index = 0
+    for revision in self._options.revisions:
+      if not '@' in revision:
         # Support for --revision 123
-        name, rev = solutions_names[i], revision
+        revision = '%s@%s' % (solutions_names[index], revision)
+      name, rev = revision.split('@', 1)
       revision_overrides[name] = rev
+      index += 1
     return revision_overrides
 
+  def _ApplySafeSyncRev(self, dep):
+    """Finds a valid revision from the content of the safesync_url and apply it
+    by appending revisions to the revision list. Throws if revision appears to
+    be invalid for the given |dep|."""
+    assert len(dep.safesync_url) > 0
+    handle = urllib.urlopen(dep.safesync_url)
+    rev = handle.read().strip()
+    handle.close()
+    if not rev:
+      raise gclient_utils.Error(
+          'It appears your safesync_url (%s) is not working properly\n'
+          '(as it returned an empty response). Check your config.' %
+          dep.safesync_url)
+    scm = gclient_scm.CreateSCM(
+        dep.url, dep.root.root_dir, dep.name, self.outbuf)
+    safe_rev = scm.GetUsableRev(rev, self._options)
+    if self._options.verbose:
+      print('Using safesync_url revision: %s.\n' % safe_rev)
+    self._options.revisions.append('%s@%s' % (dep.name, safe_rev))
+
   def RunOnDeps(self, command, args, ignore_requirements=False, progress=True):
     """Runs a command on each dependency in a client and its dependencies.
 
@@ -1673,6 +1703,7 @@ been automagically updated.  The previous version is available at %s.old.
             'solution_name': d.name,
             'solution_url': d.url,
             'deps_file': d.deps_file,
+            'safesync_url' : d.safesync_url or '',
             'managed': d.managed,
             'solution_deps': ''.join(custom_deps),
         }
@@ -1838,7 +1869,7 @@ def CMDroot(parser, args):
     print(os.path.abspath('.'))
 
 
-@subcommand.usage('[url]')
+@subcommand.usage('[url] [safesync url]')
 def CMDconfig(parser, args):
   """Creates a .gclient file in the current directory.
 
@@ -1861,13 +1892,11 @@ def CMDconfig(parser, args):
   parser.add_option('--deps-file', default='DEPS',
                     help='overrides the default name for the DEPS file for the'
                          'main solutions and all sub-dependencies')
-  parser.add_option('--managed', action='store_true', default=False,
+  parser.add_option('--unmanaged', action='store_true', default=False,
                     help='overrides the default behavior to make it possible '
-                         'to have the main solution managed by gclient '
-                         '(gclient will always auto-sync managed solutions '
-                         ' rather than leaving them untouched)')
-  parser.add_option('--unmanaged', action='store_true', default=True,
-                    help='This flag is a no-op; unmanaged is now the default.')
+                         'to have the main solution untouched by gclient '
+                         '(gclient will check out unmanaged dependencies but '
+                         'will never sync them)')
   parser.add_option('--cache-dir',
                     help='(git only) Cache all git repos into this dir and do '
                          'shared clones from the cache, instead of cloning '
@@ -1899,8 +1928,11 @@ def CMDconfig(parser, args):
         parser.error('Do not include relative path components in --name.')
 
     deps_file = options.deps_file
-    client.SetDefaultConfig(name, deps_file, base_url,
-                            managed=options.managed,
+    safesync_url = ''
+    if len(args) > 1:
+      safesync_url = args[1]
+    client.SetDefaultConfig(name, deps_file, base_url, safesync_url,
+                            managed=not options.unmanaged,
                             cache_dir=options.cache_dir)
   client.SaveConfig()
   return 0
@@ -1988,7 +2020,8 @@ def CMDsync(parser, args):
                          'format src@rev. The src@ part is optional and can be '
                          'skipped. -r can be used multiple times when .gclient '
                          'has multiple solutions configured and will work even '
-                         'if the src@ part is skipped.')
+                         'if the src@ part is skipped. Note that specifying '
+                         '--revision means your safesync_url gets ignored.')
   parser.add_option('--with_branch_heads', action='store_true',
                     help='Clone git "branch_heads" refspecs in addition to '
                          'the default refspecs. This adds about 1/2GB to a '
@@ -2001,9 +2034,8 @@ def CMDsync(parser, args):
                           'the dependencies to the date of the given revision. '
                           'Only supported for SVN repositories.')
   parser.add_option('-H', '--head', action='store_true',
-                    help='Begin by automatically syncing the root gclient '
-                         'solutions to HEAD of the remote repository. Similar '
-                         'to making the solution temporarily "managed".')
+                    help='skips any safesync_urls specified in '
+                         'configured solutions and sync to head instead')
   parser.add_option('-D', '--delete_unversioned_trees', action='store_true',
                     help='Deletes from the working copy any dependencies that '
                          'have been removed since the last sync, as long as '
@@ -2058,6 +2090,10 @@ def CMDsync(parser, args):
   if not client:
     raise gclient_utils.Error('client not configured; see \'gclient config\'')
 
+  if options.revisions and options.head:
+    # TODO(maruel): Make it a parser.error if it doesn't break any builder.
+    print('Warning: you cannot use both --head and --revision')
+
   if options.verbose:
     client.PrintLocationAndContents()
   ret = client.RunOnDeps('update', args)
@@ -2264,6 +2300,8 @@ class OptionParser(optparse.OptionParser):
     if not hasattr(options, 'revisions'):
       # GClient.RunOnDeps expects it even if not applicable.
       options.revisions = []
+    if not hasattr(options, 'head'):
+      options.head = None
     if not hasattr(options, 'nohooks'):
       options.nohooks = True
     if not hasattr(options, 'noprehooks'):

gclient_scm.py

@@ -30,6 +30,10 @@ GSUTIL_DEFAULT_PATH = os.path.join(
     os.path.dirname(os.path.abspath(__file__)), 'gsutil.py')
 
 
+class NoUsableRevError(gclient_utils.Error):
+  """Raised if requested revision isn't found in checkout."""
+
+
 class DiffFiltererWrapper(object):
   """Simple base class which tracks which file is being diffed and
   replaces instances of its file name in the original and
@@ -749,9 +753,15 @@ class GitWrapper(SCMWrapper):
       deps_revision = default_rev
     if deps_revision.startswith('refs/heads/'):
       deps_revision = deps_revision.replace('refs/heads/', self.remote + '/')
-    if not scm.GIT.IsValidRevision(cwd=self.checkout_path, rev=deps_revision):
-      # There's a chance we just don't have the corresponding object.
-      self._Fetch(options)
+    try:
+      deps_revision = self.GetUsableRev(deps_revision, options)
+    except NoUsableRevError as e:
+      # If the DEPS entry's url and hash changed, try to update the origin.
+      # See also http://crbug.com/520067.
+      logging.warn(
+          'Couldn\'t find usable revision, will retrying to update instead: %s',
+          e.message)
+      return self.update(options, [], file_list)
 
     if file_list is not None:
       files = self._Capture(['diff', deps_revision, '--name-only']).split()
@@ -782,6 +792,73 @@ class GitWrapper(SCMWrapper):
         files = self._Capture(['diff', '--name-only', merge_base]).split()
         file_list.extend([os.path.join(self.checkout_path, f) for f in files])
 
+  def GetUsableRev(self, rev, options):
+    """Finds a useful revision for this repository.
+
+    If SCM is git-svn and the head revision is less than |rev|, git svn fetch
+    will be called on the source."""
+    sha1 = None
+    if not os.path.isdir(self.checkout_path):
+      raise NoUsableRevError(
+          ( 'We could not find a valid hash for safesync_url response "%s".\n'
+            'Safesync URLs with a git checkout currently require the repo to\n'
+            'be cloned without a safesync_url before adding the safesync_url.\n'
+            'For more info, see: '
+            'http://code.google.com/p/chromium/wiki/UsingNewGit'
+            '#Initial_checkout' ) % rev)
+    elif rev.isdigit() and len(rev) < 7:
+      # Handles an SVN rev.  As an optimization, only verify an SVN revision as
+      # [0-9]{1,6} for now to avoid making a network request.
+      if scm.GIT.IsGitSvn(cwd=self.checkout_path):
+        local_head = scm.GIT.GetGitSvnHeadRev(cwd=self.checkout_path)
+        if not local_head or local_head < int(rev):
+          try:
+            logging.debug('Looking for git-svn configuration optimizations.')
+            if scm.GIT.Capture(['config', '--get', 'svn-remote.svn.fetch'],
+                             cwd=self.checkout_path):
+              self._Fetch(options)
+          except subprocess2.CalledProcessError:
+            logging.debug('git config --get svn-remote.svn.fetch failed, '
+                          'ignoring possible optimization.')
+          if options.verbose:
+            self.Print('Running git svn fetch. This might take a while.\n')
+          scm.GIT.Capture(['svn', 'fetch'], cwd=self.checkout_path)
+        try:
+          sha1 = scm.GIT.GetBlessedSha1ForSvnRev(
+              cwd=self.checkout_path, rev=rev)
+        except gclient_utils.Error, e:
+          sha1 = e.message
+          self.Print('Warning: Could not find a git revision with accurate\n'
+                 '.DEPS.git that maps to SVN revision %s.  Sync-ing to\n'
+                 'the closest sane git revision, which is:\n'
+                 '  %s\n' % (rev, e.message))
+        if not sha1:
+          raise NoUsableRevError(
+              ( 'It appears that either your git-svn remote is incorrectly\n'
+                'configured or the revision in your safesync_url is\n'
+                'higher than git-svn remote\'s HEAD as we couldn\'t find a\n'
+                'corresponding git hash for SVN rev %s.' ) % rev)
+    else:
+      if scm.GIT.IsValidRevision(cwd=self.checkout_path, rev=rev):
+        sha1 = rev
+      else:
+        # May exist in origin, but we don't have it yet, so fetch and look
+        # again.
+        self._Fetch(options)
+        if scm.GIT.IsValidRevision(cwd=self.checkout_path, rev=rev):
+          sha1 = rev
+
+    if not sha1:
+      raise NoUsableRevError(
+          ( 'We could not find a valid hash for safesync_url response "%s".\n'
+            'Safesync URLs with a git checkout currently require a git-svn\n'
+            'remote or a safesync_url that provides git sha1s. Please add a\n'
+            'git-svn remote or change your safesync_url. For more info, see:\n'
+            'http://code.google.com/p/chromium/wiki/UsingNewGit'
+            '#Initial_checkout' ) % rev)
+
+    return sha1
+
   def FullUrlForRelativeUrl(self, url):
     # Strip from last '/'
     # Equivalent to unix basename
@@ -1586,6 +1663,14 @@ class SVNWrapper(SCMWrapper):
     else:
       self._RunAndGetFileList(command, options, file_list)
 
+  def GetUsableRev(self, rev, _options):
+    """Verifies the validity of the revision for this repository."""
+    if not scm.SVN.IsValidRevision(url='%s@%s' % (self.url, rev)):
+      raise NoUsableRevError(
+        ( '%s isn\'t a valid revision. Please check that your safesync_url is\n'
+          'correct.') % rev)
+    return rev
+
   def FullUrlForRelativeUrl(self, url):
     # Find the forth '/' and strip from there. A bit hackish.
     return '/'.join(self.url.split('/')[:4]) + url

recipe_modules/bot_update/resources/bot_update.py

@@ -653,6 +653,12 @@ def solutions_to_git(input_solutions):
       first_solution = False
 
     solution['managed'] = False
+    # We don't want gclient to be using a safesync URL. Instead it should
+    # using the lkgr/lkcr branch/tags.
+    if 'safesync_url' in solution:
+      print 'Removing safesync url %s from %s' % (solution['safesync_url'],
+                                                  parsed_path)
+      del solution['safesync_url']
   return solutions, root, buildspec
 
 

recipe_modules/gclient/api.py

@@ -133,6 +133,8 @@ class GclientApi(recipe_api.RecipeApi):
     revisions = []
     self.set_patch_project_revision(self.m.properties.get('patch_project'), cfg)
     for i, s in enumerate(cfg.solutions):
+      if s.safesync_url:  # prefer safesync_url in gclient mode
+        continue
       if i == 0 and s.revision is None:
         s.revision = RevisionFallbackChain()
 

recipe_modules/gclient/config.py

@@ -25,6 +25,7 @@ def BaseConfig(USE_MIRROR=True, GIT_MODE=False, CACHE_DIR=None,
         managed = Single(bool, empty_val=True, required=False, hidden=False),
         custom_deps = Dict(value_type=(basestring, types.NoneType)),
         custom_vars = Dict(value_type=basestring),
+        safesync_url = Single(basestring, required=False),
 
         revision = Single(
             (basestring, gclient_api.RevisionResolver),
@@ -185,12 +186,24 @@ def chromium(c):
 
 @config_ctx(includes=['chromium'])
 def chromium_lkcr(c):
+  # TODO(phajdan.jr): Add git hashes for LKCR crbug.com/349277.
+  if c.GIT_MODE:
+    raise BadConf('Git has problems with safesync_url and LKCR, '
+                  'crbug.com/349277 crbug.com/109191')  # pragma: no cover
   s = c.solutions[0]
+  s.safesync_url = 'https://build.chromium.org/p/chromium/lkcr-status/lkgr'
+  # TODO(hinoka): Once lkcr exists and is a tag, it should just be lkcr
+  #               rather than origin/lkcr.
   s.revision = 'origin/lkcr'
 
 @config_ctx(includes=['chromium'])
 def chromium_lkgr(c):
   s = c.solutions[0]
+  safesync_url = 'https://chromium-status.appspot.com/lkgr'
+  if c.GIT_MODE:  # pragma: no cover
+    safesync_url = 'https://chromium-status.appspot.com/git-lkgr'
+    raise BadConf('Git has problems with safesync_url, crbug.com/109191.')
+  s.safesync_url = safesync_url
   s.revision = 'origin/lkgr'
 
 @config_ctx(includes=['chromium_bare'])

recipe_modules/gclient/example.expected/basic.json

@@ -85,7 +85,7 @@
       "RECIPE_PACKAGE_REPO[depot_tools]/gclient.py",
       "config",
       "--spec",
-      "cache_dir = '[GIT_CACHE]'\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'WebKit', 'url': 'svn://svn.chromium.org/blink/trunk'}]"
+      "cache_dir = '[GIT_CACHE]'\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'WebKit', 'safesync_url': 'https://blink-status.appspot.com/lkgr', 'url': 'svn://svn.chromium.org/blink/trunk'}]"
     ],
     "cwd": "[SLAVE_BUILD]/src/third_party",
     "env": {

recipe_modules/gclient/example.expected/revision.json

@@ -87,7 +87,7 @@
       "RECIPE_PACKAGE_REPO[depot_tools]/gclient.py",
       "config",
       "--spec",
-      "cache_dir = '[GIT_CACHE]'\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'WebKit', 'url': 'svn://svn.chromium.org/blink/trunk'}]"
+      "cache_dir = '[GIT_CACHE]'\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'WebKit', 'safesync_url': 'https://blink-status.appspot.com/lkgr', 'url': 'svn://svn.chromium.org/blink/trunk'}]"
     ],
     "cwd": "[SLAVE_BUILD]/src/third_party",
     "env": {
@@ -107,8 +107,6 @@
       "--delete_unversioned_trees",
       "--with_branch_heads",
       "--revision",
-      "WebKit@abc",
-      "--revision",
       "third_party/WebKit@123",
       "--output-json",
       "/path/to/tmp/json"

recipe_modules/gclient/example.expected/tryserver.json

@@ -87,7 +87,7 @@
       "RECIPE_PACKAGE_REPO[depot_tools]/gclient.py",
       "config",
       "--spec",
-      "cache_dir = '[GIT_CACHE]'\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'WebKit', 'url': 'svn://svn.chromium.org/blink/trunk'}]"
+      "cache_dir = '[GIT_CACHE]'\nsolutions = [{'deps_file': 'DEPS', 'managed': True, 'name': 'WebKit', 'safesync_url': 'https://blink-status.appspot.com/lkgr', 'url': 'svn://svn.chromium.org/blink/trunk'}]"
     ],
     "cwd": "[SLAVE_BUILD]/src/third_party",
     "env": {
@@ -118,8 +118,6 @@
       "--delete_unversioned_trees",
       "--with_branch_heads",
       "--revision",
-      "WebKit@HEAD",
-      "--revision",
       "third_party/WebKit@123",
       "--output-json",
       "/path/to/tmp/json"

recipe_modules/gclient/example.py

@@ -76,6 +76,9 @@ def RunSteps(api):
   soln.url = 'svn://svn.chromium.org/blink/trunk'
   bl_cfg.revisions['third_party/WebKit'] = '123'
 
+  # Use safesync url for lkgr.
+  soln.safesync_url = 'https://blink-status.appspot.com/lkgr'
+
   bl_cfg.got_revision_mapping['src/blatley'] = 'got_blatley_revision'
   api.gclient.checkout(
       gclient_config=bl_cfg,

tests/gclient_scm_test.py

@@ -600,6 +600,106 @@ class ManagedGitWrapperTestCaseMox(BaseTestCase):
   def tearDown(self):
     BaseTestCase.tearDown(self)
 
+  def testGetUsableRevGit(self):
+    # pylint: disable=E1101
+    options = self.Options(verbose=True)
+
+    self.mox.StubOutWithMock(gclient_scm.scm.GIT, 'IsValidRevision', True)
+    gclient_scm.scm.GIT.IsValidRevision(cwd=self.base_path, rev=self.fake_hash_1
+        ).AndReturn(True)
+
+    self.mox.StubOutWithMock(gclient_scm.scm.GIT, 'IsGitSvn', True)
+    gclient_scm.scm.GIT.IsGitSvn(cwd=self.base_path).MultipleTimes(
+        ).AndReturn(False)
+
+    gclient_scm.scm.os.path.isdir(self.base_path).AndReturn(True)
+    gclient_scm.os.path.isdir(self.base_path).AndReturn(True)
+
+    self.mox.ReplayAll()
+
+    git_scm = gclient_scm.CreateSCM(url=self.url, root_dir=self.root_dir,
+                                    relpath=self.relpath)
+    # A [fake] git sha1 with a git repo should work (this is in the case that
+    # the LKGR gets flipped to git sha1's some day).
+    self.assertEquals(git_scm.GetUsableRev(self.fake_hash_1, options),
+                      self.fake_hash_1)
+    # An SVN rev with an existing purely git repo should raise an exception.
+    self.assertRaises(gclient_scm.gclient_utils.Error,
+                      git_scm.GetUsableRev, '1', options)
+
+  def testGetUsableRevGitSvn(self):
+    # pylint: disable=E1101
+    options = self.Options()
+    too_big = str(1e7)
+
+    # Pretend like the git-svn repo's HEAD is at r2.
+    self.mox.StubOutWithMock(gclient_scm.scm.GIT, 'GetGitSvnHeadRev', True)
+    gclient_scm.scm.GIT.GetGitSvnHeadRev(cwd=self.base_path).MultipleTimes(
+        ).AndReturn(2)
+
+    self.mox.StubOutWithMock(
+        gclient_scm.scm.GIT, 'GetBlessedSha1ForSvnRev', True)
+    # r1 -> first fake hash, r3 -> second fake hash.
+    gclient_scm.scm.GIT.GetBlessedSha1ForSvnRev(cwd=self.base_path, rev='1'
+        ).AndReturn(self.fake_hash_1)
+    gclient_scm.scm.GIT.GetBlessedSha1ForSvnRev(cwd=self.base_path, rev='3'
+        ).MultipleTimes().AndReturn(self.fake_hash_2)
+
+    # Ensure that we call git svn fetch if our LKGR is > the git-svn HEAD rev.
+    self.mox.StubOutWithMock(gclient_scm.GitWrapper, '_Fetch', True)
+    self.mox.StubOutWithMock(gclient_scm.scm.GIT, 'Capture', True)
+    gclient_scm.scm.GIT.Capture(['config', '--get', 'svn-remote.svn.fetch'],
+                                cwd=self.base_path).AndReturn('blah')
+    # pylint: disable=E1120
+    gclient_scm.scm.GIT.Capture(['svn', 'fetch'], cwd=self.base_path)
+    error = subprocess2.CalledProcessError(1, 'cmd', '/cwd', 'stdout', 'stderr')
+    gclient_scm.scm.GIT.Capture(['config', '--get', 'svn-remote.svn.fetch'],
+                                cwd=self.base_path).AndRaise(error)
+    gclient_scm.GitWrapper._Fetch(options)
+    gclient_scm.scm.GIT.Capture(['svn', 'fetch'], cwd=self.base_path)
+    gclient_scm.GitWrapper._Fetch(options)
+
+    self.mox.StubOutWithMock(gclient_scm.scm.GIT, 'IsGitSvn', True)
+    gclient_scm.scm.GIT.IsGitSvn(cwd=self.base_path).MultipleTimes(
+        ).AndReturn(True)
+
+    self.mox.StubOutWithMock(gclient_scm.scm.GIT, 'IsValidRevision', True)
+    gclient_scm.scm.GIT.IsValidRevision(cwd=self.base_path, rev=self.fake_hash_1
+        ).AndReturn(True)
+    gclient_scm.scm.GIT.IsValidRevision(cwd=self.base_path, rev=too_big
+        ).MultipleTimes(2).AndReturn(False)
+
+    gclient_scm.os.path.isdir(self.base_path).AndReturn(False)
+    gclient_scm.os.path.isdir(self.base_path).MultipleTimes().AndReturn(True)
+
+    self.mox.ReplayAll()
+
+    git_svn_scm = self._scm_wrapper(url=self.url, root_dir=self.root_dir,
+                                    relpath=self.relpath)
+    # Without an existing checkout, this should fail.
+    # TODO(dbeam) Fix this. http://crbug.com/109184
+    self.assertRaises(gclient_scm.gclient_utils.Error,
+                      git_svn_scm.GetUsableRev, '1', options)
+    # Given an SVN revision with a git-svn checkout, it should be translated to
+    # a git sha1 and be usable.
+    self.assertEquals(git_svn_scm.GetUsableRev('1', options),
+                      self.fake_hash_1)
+    # Our fake HEAD rev is r2, so this should call git fetch and git svn fetch
+    # to get more revs (pymox will complain if this doesn't happen). We mock an
+    # optimized checkout the first time, so this run should call git fetch.
+    self.assertEquals(git_svn_scm.GetUsableRev('3', options),
+                      self.fake_hash_2)
+    # The time we pretend we're not optimized, so no git fetch should fire.
+    self.assertEquals(git_svn_scm.GetUsableRev('3', options),
+                      self.fake_hash_2)
+    # Given a git sha1 with a git-svn checkout, it should be used as is.
+    self.assertEquals(git_svn_scm.GetUsableRev(self.fake_hash_1, options),
+                      self.fake_hash_1)
+    # We currently check for seemingly valid SVN revisions by assuming 6 digit
+    # numbers, so assure that numeric revs >= 1000000 don't work.
+    self.assertRaises(gclient_scm.gclient_utils.Error,
+                      git_svn_scm.GetUsableRev, too_big, options)
+
   def testUpdateNoDotGit(self):
     options = self.Options()
 

tests/gclient_smoketest.py

@@ -205,36 +205,52 @@ class GClientSmoke(GClientSmokeBase):
 
     test(['config', self.svn_base + 'trunk/src/'],
          ('solutions = [\n'
-          '  {\n'
-          '    "name"        : "src",\n'
+          '  { "name"        : "src",\n'
           '    "url"         : "%strunk/src",\n'
           '    "deps_file"   : "DEPS",\n'
-          '    "managed"     : False,\n'
-          '    "custom_deps" : {},\n'
+          '    "managed"     : True,\n'
+          '    "custom_deps" : {\n'
+          '    },\n'
+          '    "safesync_url": "",\n'
           '  },\n'
           ']\n'
           'cache_dir = None\n') % self.svn_base)
 
     test(['config', self.git_base + 'repo_1', '--name', 'src'],
          ('solutions = [\n'
-          '  {\n'
-          '    "name"        : "src",\n'
+          '  { "name"        : "src",\n'
           '    "url"         : "%srepo_1",\n'
           '    "deps_file"   : "DEPS",\n'
-          '    "managed"     : False,\n'
-          '    "custom_deps" : {},\n'
+          '    "managed"     : True,\n'
+          '    "custom_deps" : {\n'
+          '    },\n'
+          '    "safesync_url": "",\n'
           '  },\n'
           ']\n'
           'cache_dir = None\n') % self.git_base)
 
+    test(['config', 'foo', 'faa'],
+         'solutions = [\n'
+         '  { "name"        : "foo",\n'
+         '    "url"         : "foo",\n'
+         '    "deps_file"   : "DEPS",\n'
+          '    "managed"     : True,\n'
+         '    "custom_deps" : {\n'
+         '    },\n'
+         '    "safesync_url": "faa",\n'
+         '  },\n'
+         ']\n'
+         'cache_dir = None\n')
+
     test(['config', 'foo', '--deps', 'blah'],
          'solutions = [\n'
-         '  {\n'
-         '    "name"        : "foo",\n'
+         '  { "name"        : "foo",\n'
          '    "url"         : "foo",\n'
          '    "deps_file"   : "blah",\n'
-         '    "managed"     : False,\n'
-         '    "custom_deps" : {},\n'
+         '    "managed"     : True,\n'
+         '    "custom_deps" : {\n'
+         '    },\n'
+          '    "safesync_url": "",\n'
          '  },\n'
          ']\n'
          'cache_dir = None\n')
@@ -243,7 +259,7 @@ class GClientSmoke(GClientSmokeBase):
 
     os.remove(p)
     results = self.gclient(['config', 'foo', 'faa', 'fuu'])
-    err = ('Usage: gclient.py config [options] [url]\n\n'
+    err = ('Usage: gclient.py config [options] [url] [safesync url]\n\n'
            'gclient.py: error: Inconsistent arguments. Use either --spec or one'
            ' or 2 args\n')
     self.check(('', err, 2), results)
@@ -287,14 +303,14 @@ class GClientSmokeGIT(GClientSmokeBase):
     super(GClientSmokeGIT, self).setUp()
     self.enabled = self.FAKE_REPOS.set_up_git()
 
-  def testSyncManaged(self):
+  def testSync(self):
     if not self.enabled:
       return
-    self.gclient([
-        'config', self.git_base + 'repo_1', '--name', 'src', '--managed'])
+    # TODO(maruel): safesync.
+    self.gclient(['config', self.git_base + 'repo_1', '--name', 'src'])
     # Test unversioned checkout.
     self.parseGclient(
-        ['sync', '--deps', 'mac', '--jobs', '8'],
+        ['sync', '--deps', 'mac', '--jobs', '1'],
         ['running', 'running'])
     # TODO(maruel): http://crosbug.com/3582 hooks run even if not matching, must
     # add sync parsing to get the list of updated files.
@@ -312,7 +328,7 @@ class GClientSmokeGIT(GClientSmokeBase):
     # Test incremental versioned sync: sync backward.
     self.parseGclient(
         ['sync', '--jobs', '1', '--revision',
-        'src@' + self.githash('repo_1', 1), '--jobs', '1',
+        'src@' + self.githash('repo_1', 1),
         '--deps', 'mac', '--delete_unversioned_trees'],
         ['deleting'])
     tree = self.mangle_git_tree(('repo_1@1', 'src'),
@@ -323,7 +339,7 @@ class GClientSmokeGIT(GClientSmokeBase):
     self.assertTree(tree)
     # Test incremental sync: delete-unversioned_trees isn't there.
     self.parseGclient(
-        ['sync', '--deps', 'mac', '--jobs', '8'],
+        ['sync', '--deps', 'mac', '--jobs', '1'],
         ['running', 'running'])
     tree = self.mangle_git_tree(('repo_1@2', 'src'),
                                 ('repo_2@1', 'src/repo2'),
@@ -367,6 +383,56 @@ class GClientSmokeGIT(GClientSmokeBase):
                                 ('repo_4@2', 'src/repo4'))
     self.assertTree(tree)
 
+  def testSyncJobs(self):
+    if not self.enabled:
+      return
+    # TODO(maruel): safesync.
+    self.gclient(['config', self.git_base + 'repo_1', '--name', 'src'])
+    # Test unversioned checkout.
+    self.parseGclient(
+        ['sync', '--deps', 'mac', '--jobs', '8'],
+        ['running', 'running'],
+        untangle=True)
+    # TODO(maruel): http://crosbug.com/3582 hooks run even if not matching, must
+    # add sync parsing to get the list of updated files.
+    tree = self.mangle_git_tree(('repo_1@2', 'src'),
+                                ('repo_2@1', 'src/repo2'),
+                                ('repo_3@2', 'src/repo2/repo_renamed'))
+    tree['src/git_hooked1'] = 'git_hooked1'
+    tree['src/git_hooked2'] = 'git_hooked2'
+    self.assertTree(tree)
+
+    # Manually remove git_hooked1 before synching to make sure it's not
+    # recreated.
+    os.remove(join(self.root_dir, 'src', 'git_hooked1'))
+
+    # Test incremental versioned sync: sync backward.
+    # Use --jobs 1 otherwise the order is not deterministic.
+    self.parseGclient(
+        ['sync', '--revision', 'src@' + self.githash('repo_1', 1),
+          '--deps', 'mac', '--delete_unversioned_trees', '--jobs', '1'],
+        ['deleting'],
+        untangle=True)
+    tree = self.mangle_git_tree(('repo_1@1', 'src'),
+                                ('repo_2@2', 'src/repo2'),
+                                ('repo_3@1', 'src/repo2/repo3'),
+                                ('repo_4@2', 'src/repo4'))
+    tree['src/git_hooked2'] = 'git_hooked2'
+    self.assertTree(tree)
+    # Test incremental sync: delete-unversioned_trees isn't there.
+    self.parseGclient(
+        ['sync', '--deps', 'mac', '--jobs', '8'],
+        ['running', 'running'],
+        untangle=True)
+    tree = self.mangle_git_tree(('repo_1@2', 'src'),
+                                ('repo_2@1', 'src/repo2'),
+                                ('repo_3@1', 'src/repo2/repo3'),
+                                ('repo_3@2', 'src/repo2/repo_renamed'),
+                                ('repo_4@2', 'src/repo4'))
+    tree['src/git_hooked1'] = 'git_hooked1'
+    tree['src/git_hooked2'] = 'git_hooked2'
+    self.assertTree(tree)
+
   def testRunHooks(self):
     if not self.enabled:
       return

tests/gclient_test.py

@@ -216,7 +216,7 @@ class GclientTest(trial_dir.TestCase):
     # Invalid urls causes pain when specifying requirements. Make sure it's
     # auto-fixed.
     d = gclient.Dependency(
-        None, 'name', 'proto://host/path/@revision', None, None, None,
+        None, 'name', 'proto://host/path/@revision', None, None, None, None,
         None, '', True)
     self.assertEquals('proto://host/path@revision', d.url)
 
@@ -227,23 +227,23 @@ class GclientTest(trial_dir.TestCase):
     obj.add_dependencies_and_close(
         [
           gclient.Dependency(
-            obj, 'foo', 'url', None, None, None, None, 'DEPS', True),
+            obj, 'foo', 'url', None, None, None, None, None, 'DEPS', True),
           gclient.Dependency(
-            obj, 'bar', 'url', None, None, None, None, 'DEPS', True),
+            obj, 'bar', 'url', None, None, None, None, None, 'DEPS', True),
         ],
         [])
     obj.dependencies[0].add_dependencies_and_close(
         [
           gclient.Dependency(
-            obj.dependencies[0], 'foo/dir1', 'url', None, None, None,
+            obj.dependencies[0], 'foo/dir1', 'url', None, None, None, None,
             None, 'DEPS', True),
           gclient.Dependency(
             obj.dependencies[0], 'foo/dir2',
-            gclient.GClientKeywords.FromImpl('bar'), None, None, None,
+            gclient.GClientKeywords.FromImpl('bar'), None, None, None, None,
             None, 'DEPS', True),
           gclient.Dependency(
             obj.dependencies[0], 'foo/dir3',
-            gclient.GClientKeywords.FileImpl('url'), None, None, None,
+            gclient.GClientKeywords.FileImpl('url'), None, None, None, None,
             None, 'DEPS', True),
         ],
         [])
@@ -564,7 +564,7 @@ class GclientTest(trial_dir.TestCase):
     """Verifies expected behavior of LateOverride."""
     url = "git@github.com:dart-lang/spark.git"
     d = gclient.Dependency(None, 'name', 'url',
-                           None, None, None, None, '', True)
+                           None, None, None, None, None, '', True)
     late_url = d.LateOverride(url)
     self.assertEquals(url, late_url)