gclient: don't update cache if needed revision is already there.

Tested end-to-end, for example
  https://ci.chromium.org/swarming/task/3a0147207378b910
which contains:
  src/media/cdm/api (Elapsed: 0:00:01)
  ----------------------------------------
  [0:00:00] Started.
  _____ src\media\cdm\api at ea5df8e78fbd0a4c24cc3a1f3faefefcd1b45237
  [0:00:00] running "git cat-file -e ea5df8e78fbd0a4c24cc3a1f3faefefcd1b45237" in "e:\b\s\w\ir\cache\git\chromium.googlesource.com-chromium-cdm"
  skipping mirror update, it has rev=ea5df8e78fbd0a4c24cc3a1f3faefefcd1b45237 already

thereby saving on needless git fetch (~40s in glcient sync on win trybots),
and reducing the rate of .pack file accumulation inside cache directories.


Risks: silently broken recipes which run gclient sync (or worse, bot_update)
as a means of fetching latest commits in all repos of a solution. I think
the benefit of faster bot_update in chromium CQ is worth the potential risk.

PSA: https://groups.google.com/a/chromium.org/d/msg/infra-dev/UYLdBwAXm1Y/OV9QB6JnBQAJ

Bug: 749709
Change-Id: I7a9e8ab82a5e2b848e450f19a798ac18a0b5e201
Reviewed-on: https://chromium-review.googlesource.com/787331
Commit-Queue: Andrii Shyshkalov <tandrii@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>

gclient_scm.py

@@ -433,7 +433,7 @@ class GitWrapper(SCMWrapper):
         (os.path.isdir(self.checkout_path) and
          not os.path.exists(os.path.join(self.checkout_path, '.git')))):
       if mirror:
-        self._UpdateMirror(mirror, options)
+        self._UpdateMirrorIfNotContains(mirror, options, rev_type, revision)
       try:
         self._Clone(revision, url, options)
       except subprocess2.CalledProcessError:
@@ -456,7 +456,7 @@ class GitWrapper(SCMWrapper):
     self._maybe_break_locks(options)
 
     if mirror:
-      self._UpdateMirror(mirror, options)
+      self._UpdateMirrorIfNotContains(mirror, options, rev_type, revision)
 
     # See if the url has changed (the unittests use git://foo for the url, let
     # that through).
@@ -842,9 +842,16 @@ class GitWrapper(SCMWrapper):
       mirror_kwargs['refs'].append('refs/tags/*')
     return git_cache.Mirror(url, **mirror_kwargs)
 
-  @staticmethod
-  def _UpdateMirror(mirror, options):
-    """Update a git mirror by fetching the latest commits from the remote."""
+  def _UpdateMirrorIfNotContains(self, mirror, options, rev_type, revision):
+    """Update a git mirror by fetching the latest commits from the remote,
+    unless mirror already contains revision whose type is sha1 hash.
+    """
+    if rev_type == 'hash' and mirror.contains_revision(revision):
+      if options.verbose:
+        self.Print('skipping mirror update, it has rev=%s already' % revision,
+                   timestamp=False)
+      return
+
     if getattr(options, 'shallow', False):
       # HACK(hinoka): These repositories should be super shallow.
       if 'flash' in mirror.url:

git_cache.py

@@ -411,6 +411,24 @@ class Mirror(object):
       return False
     return True
 
+  def contains_revision(self, revision):
+    if not self.exists():
+      return False
+
+    if sys.platform.startswith('win'):
+      # Windows .bat scripts use ^ as escape sequence, which means we have to
+      # escape it with itself for every .bat invocation.
+      needle = '%s^^^^{commit}' % revision
+    else:
+      needle = '%s^{commit}' % revision
+    try:
+      # cat-file exits with 0 on success, that is git object of given hash was
+      # found.
+      self.RunGit(['cat-file', '-e', needle])
+      return True
+    except subprocess.CalledProcessError:
+      return False
+
   def exists(self):
     return os.path.isfile(os.path.join(self.mirror_path, 'config'))
 

recipes/recipe_modules/bot_update/resources/bot_update.py

@@ -29,7 +29,6 @@ import os.path as path
 # How many bytes at a time to read from pipes.
 BUF_SIZE = 256
 
-
 # Define a bunch of directory paths.
 # Relative to this script's filesystem path.
 THIS_DIR = path.dirname(path.abspath(__file__))