Added 'git-retry' bootstrap

Added the 'git-retry' bootstrap command. This can be used to wrap other 'git'
commands around a fault-tolerant retry wrapper.

BUG=295109
TEST=localtest
R=iannucci@chromium.org, petermayo@chromium.org

Review URL: https://codereview.chromium.org/401673003

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@285939 0039d316-1c4b-4281-b951-d872f2087c98

git-retry

+#!/usr/bin/env bash
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# git_freezer.py freeze -- a git-command to suspend all existing working
+# directory modifications. This can be reversed with the 'git thaw' command.
+
+SCRIPT=git_retry.py
+set -- retry "$@"
+. $(type -P python_git_runner.sh)

git_common.py

@@ -42,6 +42,55 @@ FREEZE_SECTIONS = {
 FREEZE_MATCHER = re.compile(r'%s.(%s)' % (FREEZE, '|'.join(FREEZE_SECTIONS)))
 
 
+# Retry a git operation if git returns a error response with any of these
+# messages. It's all observed 'bad' GoB responses so far.
+#
+# This list is inspired/derived from the one in ChromiumOS's Chromite:
+# <CHROMITE>/lib/git.py::GIT_TRANSIENT_ERRORS
+#
+# It was last imported from '7add3ac29564d98ac35ce426bc295e743e7c0c02'.
+GIT_TRANSIENT_ERRORS = (
+    # crbug.com/285832
+    r'! \[remote rejected\].*\(error in hook\)',
+
+    # crbug.com/289932
+    r'! \[remote rejected\].*\(failed to lock\)',
+
+    # crbug.com/307156
+    r'! \[remote rejected\].*\(error in Gerrit backend\)',
+
+    # crbug.com/285832
+    r'remote error: Internal Server Error',
+
+    # crbug.com/294449
+    r'fatal: Couldn\'t find remote ref ',
+
+    # crbug.com/220543
+    r'git fetch_pack: expected ACK/NAK, got',
+
+    # crbug.com/189455
+    r'protocol error: bad pack header',
+
+    # crbug.com/202807
+    r'The remote end hung up unexpectedly',
+
+    # crbug.com/298189
+    r'TLS packet with unexpected length was received',
+
+    # crbug.com/187444
+    r'RPC failed; result=\d+, HTTP code = \d+',
+
+    # crbug.com/315421
+    r'The requested URL returned error: 500 while accessing',
+
+    # crbug.com/388876
+    r'Connection timed out',
+)
+
+GIT_TRANSIENT_ERRORS_RE = re.compile('|'.join(GIT_TRANSIENT_ERRORS),
+                                     re.IGNORECASE)
+
+
 class BadCommitRefException(Exception):
   def __init__(self, refs):
     msg = ('one of %s does not seem to be a valid commitref.' %

git_retry.py

+#!/usr/bin/env python
+# Copyright 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import logging
+import optparse
+import subprocess
+import sys
+import threading
+import time
+
+from git_common import GIT_EXE, GIT_TRANSIENT_ERRORS_RE
+
+
+class TeeThread(threading.Thread):
+
+  def __init__(self, fd, out_fd, name):
+    super(TeeThread, self).__init__(name='git-retry.tee.%s' % (name,))
+    self.data = None
+    self.fd = fd
+    self.out_fd = out_fd
+
+  def run(self):
+    chunks = []
+    for line in self.fd:
+      chunks.append(line)
+      self.out_fd.write(line)
+    self.data = ''.join(chunks)
+
+
+class GitRetry(object):
+
+  logger = logging.getLogger('git-retry')
+  DEFAULT_DELAY_SECS = 3.0
+  DEFAULT_RETRY_COUNT = 5
+
+  def __init__(self, retry_count=None, delay=None, delay_factor=None):
+    self.retry_count = retry_count or self.DEFAULT_RETRY_COUNT
+    self.delay = max(delay, 0) if delay else 0
+    self.delay_factor = max(delay_factor, 0) if delay_factor else 0
+
+  def shouldRetry(self, stderr):
+    m = GIT_TRANSIENT_ERRORS_RE.search(stderr)
+    if not m:
+      return False
+    self.logger.info("Encountered known transient error: [%s]",
+                     stderr[m.start(): m.end()])
+    return True
+
+  @staticmethod
+  def execute(*args):
+    args = (GIT_EXE,) + args
+    proc = subprocess.Popen(
+        args,
+        stderr=subprocess.PIPE,
+    )
+    stderr_tee = TeeThread(proc.stderr, sys.stderr, 'stderr')
+
+    # Start our process. Collect/tee 'stdout' and 'stderr'.
+    stderr_tee.start()
+    try:
+      proc.wait()
+    except KeyboardInterrupt:
+      proc.kill()
+      raise
+    finally:
+      stderr_tee.join()
+    return proc.returncode, None, stderr_tee.data
+
+  def computeDelay(self, iteration):
+    """Returns: the delay (in seconds) for a given iteration
+
+    The first iteration has a delay of '0'.
+
+    Args:
+      iteration: (int) The iteration index (starting with zero as the first
+          iteration)
+    """
+    if (not self.delay) or (iteration == 0):
+      return 0
+    if self.delay_factor == 0:
+      # Linear delay
+      return iteration * self.delay
+    # Exponential delay
+    return (self.delay_factor ** (iteration - 1)) * self.delay
+
+  def __call__(self, *args):
+    returncode = 0
+    for i in xrange(self.retry_count):
+      # If the previous run failed and a delay is configured, delay before the
+      # next run.
+      delay = self.computeDelay(i)
+      if delay > 0:
+        self.logger.info("Delaying for [%s second(s)] until next retry", delay)
+        time.sleep(delay)
+
+      self.logger.debug("Executing subprocess (%d/%d) with arguments: %s",
+                        (i+1), self.retry_count, args)
+      returncode, _, stderr = self.execute(*args)
+
+      self.logger.debug("Process terminated with return code: %d", returncode)
+      if returncode == 0:
+        break
+
+      if not self.shouldRetry(stderr):
+        self.logger.error("Process failure was not known to be transient; "
+                          "terminating with return code %d", returncode)
+        break
+    return returncode
+
+
+def main(args):
+  parser = optparse.OptionParser()
+  parser.disable_interspersed_args()
+  parser.add_option('-v', '--verbose',
+                    action='count', default=0,
+                    help="Increase verbosity; can be specified multiple times")
+  parser.add_option('-c', '--retry-count', metavar='COUNT',
+                    type=int, default=GitRetry.DEFAULT_RETRY_COUNT,
+                    help="Number of times to retry (default=%default)")
+  parser.add_option('-d', '--delay', metavar='SECONDS',
+                    type=float, default=GitRetry.DEFAULT_DELAY_SECS,
+                    help="Specifies the amount of time (in seconds) to wait "
+                         "between successive retries (default=%default). This "
+                         "can be zero.")
+  parser.add_option('-D', '--delay-factor', metavar='FACTOR',
+                    type=int, default=2,
+                    help="The exponential factor to apply to delays in between "
+                         "successive failures (default=%default). If this is "
+                         "zero, delays will increase linearly. Set this to "
+                         "one to have a constant (non-increasing) delay.")
+
+  opts, args = parser.parse_args(args)
+
+  # Configure logging verbosity
+  if opts.verbose == 0:
+    logging.getLogger().setLevel(logging.WARNING)
+  elif opts.verbose == 1:
+    logging.getLogger().setLevel(logging.INFO)
+  else:
+    logging.getLogger().setLevel(logging.DEBUG)
+
+  # Execute retries
+  retry = GitRetry(
+      retry_count=opts.retry_count,
+      delay=opts.delay,
+      delay_factor=opts.delay_factor,
+  )
+  return retry(*args)
+
+
+if __name__ == '__main__':
+  logging.basicConfig()
+  logging.getLogger().setLevel(logging.WARNING)
+  sys.exit(main(sys.argv[2:]))

man/man1/git-retry.1

+'\" t
+.\"     Title: git-retry
+.\"    Author: [FIXME: author] [see http://docbook.sf.net/el/author]
+.\" Generator: DocBook XSL Stylesheets v1.78.1 <http://docbook.sf.net/>
+.\"      Date: 07/22/2014
+.\"    Manual: Chromium depot_tools Manual
+.\"    Source: depot_tools 7242c60
+.\"  Language: English
+.\"
+.TH "GIT\-RETRY" "1" "07/22/2014" "depot_tools 7242c60" "Chromium depot_tools Manual"
+.\" -----------------------------------------------------------------
+.\" * Define some portability stuff
+.\" -----------------------------------------------------------------
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.\" http://bugs.debian.org/507673
+.\" http://lists.gnu.org/archive/html/groff/2009-02/msg00013.html
+.\" ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.ie \n(.g .ds Aq \(aq
+.el       .ds Aq '
+.\" -----------------------------------------------------------------
+.\" * set default formatting
+.\" -----------------------------------------------------------------
+.\" disable hyphenation
+.nh
+.\" disable justification (adjust text to left margin only)
+.ad l
+.\" -----------------------------------------------------------------
+.\" * MAIN CONTENT STARTS HERE *
+.\" -----------------------------------------------------------------
+.SH "NAME"
+git-retry \- Bootstrap function to retry a git command\&.
+.SH "SYNOPSIS"
+.sp
+.nf
+\fIgit retry\fR [\-v] [\-c COUNT] [\-d DELAY] [\-e] \(em \fI<git_subcommand>\fR
+.fi
+.sp
+.SH "DESCRIPTION"
+.sp
+git retry is a bootstrap that wraps a standard git command execution in a fault\-tolerant retry wrapper\&.
+.sp
+If a retry succeeds, the return code of the successful attempt is returned\&. Otherwise, the return code of the last failed attempt is returned\&.
+.sp
+The wrapper is aware of git\-specific failure conditions and will only consider retrying if a given failure can be linked to such a condition\&.
+.SH "OPTIONS"
+.PP
+<git_subcommand>
+.RS 4
+The
+git
+command to retry\&. This should omit the actual
+git
+command (e\&.g\&., to retry
+git clone, use
+git retry clone)\&.
+.RE
+.PP
+\-v, \-\-verbose
+.RS 4
+Increases logging verbosity\&. By default, no additional logging is generated by the
+git retry
+command\&. This can be specified multiple times\&.
+.RE
+.PP
+\-c, \-\-retry\-count \fIcount\fR
+.RS 4
+Specify the number of retries that should be performed before giving up\&. The default retry count is
+\fB5\fR\&.
+.RE
+.PP
+\-d, \-\-delay \fIseconds\fR
+.RS 4
+Floating\-point value that specifies the amount of time (in seconds) to wait after a failure\&. This can be zero to specify no delay\&. The default delay is
+\fB3 seconds\fR\&.
+.RE
+.PP
+\-D, \-\-delay\-factor
+.RS 4
+The exponential factor to apply to the delay\&. By default this is
+\fB2\fR\&. For a given retry round
+\fBn\fR, the delay for that round will be
+\fB(<delay\-factor>^(n\-1) * delay)\fR\&. If no delay is specified, this will have no effect\&. If the delay factor is
+\fB0\fR, the delay will increase linearly (for a given retry round
+\fBn\fR, the delay will be
+\fB(n * delay)\fR)\&.
+.sp
+.if n \{\
+.RS 4
+.\}
+.nf
+(Note that a delay factor of *1* will result in a constant delay\&.)
+.fi
+.if n \{\
+.RE
+.\}
+.RE
+.SH "EXIT STATUS"
+.sp
+Upon success, git retry will exit with the successful exit code of \fB0\fR\&. On failure, it will exit with the exit code of the last failed attempt\&.
+.SH "CHROMIUM DEPOT_TOOLS"
+.sp
+Part of the chromium \fBdepot_tools\fR(7) suite\&. These tools are meant to assist with the development of chromium and related projects\&. Download the tools from \m[blue]\fBhere\fR\m[]\&\s-2\u[1]\d\s+2\&.
+.SH "NOTES"
+.IP " 1." 4
+here
+.RS 4
+\%https://chromium.googlesource.com/chromium/tools/depot_tools.git
+.RE

man/src/_git-retry_desc.helper.txt

+Bootstrap function to retry a git command.
+

man/src/git-retry.txt

+git-retry(1)
+=============
+
+NAME
+----
+git-retry -
+include::_git-retry_desc.helper.txt[]
+
+SYNOPSIS
+--------
+[verse]
+'git retry' [-v] [-c COUNT] [-d DELAY] [-e] -- _<git_subcommand>_
+
+DESCRIPTION
+-----------
+
+`git retry` is a bootstrap that wraps a standard `git` command execution in
+a fault-tolerant retry wrapper.
+
+If a retry succeeds, the return code of the successful attempt is returned.
+Otherwise, the return code of the last failed attempt is returned.
+
+The wrapper is aware of `git`-specific failure conditions and will only consider
+retrying if a given failure can be linked to such a condition.
+
+
+OPTIONS
+-------
+
+<git_subcommand>::
+  The `git` command to retry. This should omit the actual `git` command (e.g.,
+  to retry `git clone`, use `git retry clone`).
+
+-v, --verbose::
+  Increases logging verbosity. By default, no additional logging is generated
+  by the `git retry` command. This can be specified multiple times.
+
+-c, --retry-count _count_::
+  Specify the number of retries that should be performed before giving up. The
+  default retry count is *5*.
+
+-d, --delay _seconds_::
+  Floating-point value that specifies the amount of time (in seconds) to wait
+  after a failure. This can be zero to specify no delay. The default delay is
+  *3 seconds*.
+
+-D, --delay-factor::
+  The exponential factor to apply to the delay. By default this is *2*.
+  For a given retry round *n*, the delay for that round will be
+  *(<delay-factor>^(n-1) * delay)*. If no delay is specified, this will have
+  no effect.
+  If the delay factor is *0*, the delay will increase linearly (for a given
+  retry round *n*, the delay will be *(n * delay)*).
+
+    (Note that a delay factor of *1* will result in a constant delay.)
+
+
+EXIT STATUS
+-----------
+
+Upon success, `git retry` will exit with the successful exit code of *0*. On
+failure, it will exit with the exit code of the last failed attempt.
+
+
+include::_footer.txt[]
+
+// vim: ft=asciidoc: