gclient: exponential backoff of SCM commands on known failure types.

This starts at 5 seconds (used to be 15) and goes up to ~60 seconds on the tenth round.

BUG=None
TEST=None
Review URL: http://codereview.chromium.org/3326004

git-svn-id: svn://svn.chromium.org/chrome/trunk/tools/depot_tools@58488 0039d316-1c4b-4281-b951-d872f2087c98

gclient_scm.py

@@ -256,6 +256,7 @@ class GitWrapper(SCMWrapper):
         raise gclient_utils.Error('Invalid Upstream: %s' % upstream_branch)
 
     # Update the remotes first so we have all the refs.
+    backoff_time = 5
     for _ in range(10):
       try:
         remote_output, remote_err = scm.GIT.Capture(
@@ -268,8 +269,10 @@ class GitWrapper(SCMWrapper):
         # 502 in stderr should be fine.
         if '502' in e.stderr:
           options.stdout.write(str(e) + '\n')
-          options.stdout.write('Sleeping 15 seconds and retrying...\n')
-          time.sleep(15)
+          options.stdout.write('Sleeping %.1f seconds and retrying...\n' %
+                               backoff_time)
+          time.sleep(backoff_time)
+          backoff_time *= 1.3
           continue
         raise
 

scm.py

@@ -364,6 +364,7 @@ class SVN(object):
         }[args[0]]
     compiled_pattern = re.compile(pattern)
     # Place an upper limit.
+    backoff_time = 5
     for _ in range(10):
       previous_list_len = len(file_list)
       failure = []
@@ -417,8 +418,9 @@ class SVN(object):
           if len(file_list) == previous_list_len and not IsKnownFailure():
             # No known svn error was found and no progress, bail out.
             raise
-        print "Sleeping 15 seconds and retrying...."
-        time.sleep(15)
+        print "Sleeping %.1f seconds and retrying...." % backoff_time
+        time.sleep(backoff_time)
+        backoff_time *= 1.3
         continue
       break