Add retry feature for push-to-trunk script.

Make url accesses retry. Git retry requires some more analysis of git output first (follow up CL).

BUG=
R=jkummerow@chromium.org

Review URL: https://codereview.chromium.org/91733003

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@18174 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

tools/push-to-trunk/auto_roll.py

@@ -62,7 +62,8 @@ class FetchLKGR(Step):
 
   def RunStep(self):
     lkgr_url = "https://v8-status.appspot.com/lkgr"
-    self.Persist("lkgr", self.ReadURL(lkgr_url))
+    # Retry several times since app engine might have issues.
+    self.Persist("lkgr", self.ReadURL(lkgr_url, wait_plan=[5, 20, 300, 300]))
 
 
 class PushToTrunk(Step):

tools/push-to-trunk/common_includes.py

@@ -31,6 +31,7 @@ import re
 import subprocess
 import sys
 import textwrap
+import time
 import urllib2
 
 PERSISTFILE_BASENAME = "PERSISTFILE_BASENAME"
@@ -173,6 +174,7 @@ def MakeChangeLogBugReference(body):
 # Some commands don't like the pipe, e.g. calling vi from within the script or
 # from subscripts like git cl upload.
 def Command(cmd, args="", prefix="", pipe=True):
+  # TODO(machenbach): Use timeout.
   cmd_line = "%s %s %s" % (prefix, cmd, args)
   print "Command: %s" % cmd_line
   try:
@@ -200,6 +202,9 @@ class SideEffectHandler(object):
     finally:
       url_fh.close()
 
+  def Sleep(seconds):
+    time.sleep(seconds)
+
 DEFAULT_SIDE_EFFECT_HANDLER = SideEffectHandler()
 
 
@@ -231,6 +236,35 @@ class Step(object):
   def RunStep(self):
     raise NotImplementedError
 
+  def Retry(self, cb, retry_on=None, wait_plan=None):
+    """ Retry a function.
+    Params:
+      cb: The function to retry.
+      retry_on: A callback that takes the result of the function and returns
+                True if the function should be retried. A function throwing an
+                exception is always retried.
+      wait_plan: A list of waiting delays between retries in seconds. The
+                 maximum number of retries is len(wait_plan).
+    """
+    retry_on = retry_on or (lambda x: False)
+    wait_plan = list(wait_plan or [])
+    wait_plan.reverse()
+    while True:
+      got_exception = False
+      try:
+        result = cb()
+      except Exception:
+        got_exception = True
+      if got_exception or retry_on(result):
+        if not wait_plan:
+          raise Exception("Retried too often. Giving up.")
+        wait_time = wait_plan.pop()
+        print "Waiting for %f seconds." % wait_time
+        self._side_effect_handler.Sleep(wait_time)
+        print "Retrying..."
+      else:
+        return result
+
   def ReadLine(self, default=None):
     # Don't prompt in forced mode.
     if self._options and self._options.f and default is not None:
@@ -239,15 +273,18 @@ class Step(object):
     else:
       return self._side_effect_handler.ReadLine()
 
-  def Git(self, args="", prefix="", pipe=True):
-    return self._side_effect_handler.Command("git", args, prefix, pipe)
+  def Git(self, args="", prefix="", pipe=True, retry_on=None):
+    cmd = lambda: self._side_effect_handler.Command("git", args, prefix, pipe)
+    return self.Retry(cmd, retry_on, [5, 30])
 
   def Editor(self, args):
     return self._side_effect_handler.Command(os.environ["EDITOR"], args,
                                              pipe=False)
 
-  def ReadURL(self, url):
-    return self._side_effect_handler.ReadURL(url)
+  def ReadURL(self, url, retry_on=None, wait_plan=None):
+    wait_plan = wait_plan or [3, 60, 600]
+    cmd = lambda: self._side_effect_handler.ReadURL(url)
+    return self.Retry(cmd, retry_on, wait_plan)
 
   def Die(self, msg=""):
     if msg != "":

tools/push-to-trunk/push_to_trunk.py

@@ -102,7 +102,9 @@ class PrepareChangeLog(Step):
     if match:
       cl_url = "https://codereview.chromium.org/%s/description" % match.group(1)
       try:
-        body = self.ReadURL(cl_url)
+        # Fetch from Rietveld but only retry once with one second delay since
+        # there might be many revisions.
+        body = self.ReadURL(cl_url, wait_plan=[1])
       except urllib2.URLError:
         pass
     return body

tools/push-to-trunk/test_scripts.py

@@ -216,7 +216,12 @@ class SimpleMock(object):
     # callback for checking the context at the time of the call.
     if len(expected_call) == len(args) + 2:
       expected_call[len(args) + 1]()
-    return expected_call[len(args)]
+    return_value = expected_call[len(args)]
+
+    # If the return value is an exception, raise it instead of returning.
+    if isinstance(return_value, Exception):
+      raise return_value
+    return return_value
 
   def AssertFinished(self):
     if self._index < len(self._recipe) -1:
@@ -269,6 +274,9 @@ class ScriptTest(unittest.TestCase):
   def ReadURL(self, url):
     return self._url_mock.Call(url)
 
+  def Sleep(self, seconds):
+    pass
+
   def ExpectGit(self, *args):
     """Convenience wrapper."""
     self._git_mock.Expect(*args)
@@ -674,6 +682,7 @@ class ScriptTest(unittest.TestCase):
     os.environ["EDITOR"] = "vi"
 
     self.ExpectReadURL([
+      ["https://v8-status.appspot.com/lkgr", Exception("Network problem")],
       ["https://v8-status.appspot.com/lkgr", "100"],
     ])