Refactor persisting state in push and merge scripts.

- The backed state dict is now persisted and restored in the step template as a json file
- All explicit persist/restore calls are removed
- Added testing an unexpected script failure + restart with state recovery to the merge-to-branch test
- This CL is not changing external behavior of the scripts

BUG=
R=ulan@chromium.org

Review URL: https://codereview.chromium.org/170583002

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@19478 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

tools/push-to-trunk/auto_roll.py

@@ -83,10 +83,10 @@ class CheckTreeStatus(Step):
   def RunStep(self):
     status_url = "https://v8-status.appspot.com/current?format=json"
     status_json = self.ReadURL(status_url, wait_plan=[5, 20, 300, 300])
-    message = json.loads(status_json)["message"]
-    if re.search(r"nopush|no push", message, flags=re.I):
-      self.Die("Push to trunk disabled by tree state: %s" % message)
-    self.Persist("tree_message", message)
+    self["tree_message"] = json.loads(status_json)["message"]
+    if re.search(r"nopush|no push", self["tree_message"], flags=re.I):
+      self.Die("Push to trunk disabled by tree state: %s"
+               % self["tree_message"])
 
 
 class FetchLatestRevision(Step):
@@ -97,18 +97,17 @@ class FetchLatestRevision(Step):
     match = re.match(r"^r(\d+) ", log)
     if not match:
       self.Die("Could not extract current svn revision from log.")
-    self.Persist("latest", match.group(1))
+    self["latest"] = match.group(1)
 
 
 class CheckLastPush(Step):
   MESSAGE = "Checking last V8 push to trunk."
 
   def RunStep(self):
-    self.RestoreIfUnset("latest")
     log = self.Git("svn log -1 --oneline ChangeLog").strip()
     match = re.match(r"^r(\d+) \| Prepare push to trunk", log)
     if match:
-      latest = int(self._state["latest"])
+      latest = int(self["latest"])
       last_push = int(match.group(1))
       # TODO(machebach): This metric counts all revisions. It could be
       # improved by counting only the revisions on bleeding_edge.
@@ -124,7 +123,7 @@ class FetchLKGR(Step):
   def RunStep(self):
     lkgr_url = "https://v8-status.appspot.com/lkgr"
     # Retry several times since app engine might have issues.
-    self.Persist("lkgr", self.ReadURL(lkgr_url, wait_plan=[5, 20, 300, 300]))
+    self["lkgr"] = self.ReadURL(lkgr_url, wait_plan=[5, 20, 300, 300])
 
 
 class PushToTrunk(Step):
@@ -145,11 +144,8 @@ class PushToTrunk(Step):
                  wait_plan=[5, 20])
 
   def RunStep(self):
-    self.RestoreIfUnset("latest")
-    self.RestoreIfUnset("lkgr")
-    self.RestoreIfUnset("tree_message")
-    latest = int(self._state["latest"])
-    lkgr = int(self._state["lkgr"])
+    latest = int(self["latest"])
+    lkgr = int(self["lkgr"])
     if latest == lkgr:
       print "ToT (r%d) is clean. Pushing to trunk." % latest
       self.PushTreeStatus("Tree is closed (preparing to push)")
@@ -165,7 +161,7 @@ class PushToTrunk(Step):
                                                    self._options.c),
               self._side_effect_handler)
       finally:
-        self.PushTreeStatus(self._state["tree_message"])
+        self.PushTreeStatus(self["tree_message"])
     else:
       print("ToT (r%d) is ahead of the LKGR (r%d). Skipping push to trunk."
             % (latest, lkgr))

tools/push-to-trunk/common_includes.py

@@ -27,6 +27,7 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import datetime
+import json
 import os
 import re
 import subprocess
@@ -246,17 +247,35 @@ class Step(object):
     assert self._side_effect_handler is not None
     assert isinstance(options, CommonOptions)
 
+  def __getitem__(self, key):
+    # Convenience method to allow direct [] access on step classes for
+    # manipulating the backed state dict.
+    return self._state[key]
+
+  def __setitem__(self, key, value):
+    # Convenience method to allow direct [] access on step classes for
+    # manipulating the backed state dict.
+    self._state[key] = value
+
   def Config(self, key):
     return self._config[key]
 
   def Run(self):
-    if self._requires:
-      self.RestoreIfUnset(self._requires)
-      if not self._state[self._requires]:
-        return
+    # Restore state.
+    state_file = "%s-state.json" % self._config[PERSISTFILE_BASENAME]
+    if not self._state and os.path.exists(state_file):
+      self._state.update(json.loads(FileToText(state_file)))
+
+    # Skip step if requirement is not met.
+    if self._requires and not self._state.get(self._requires):
+      return
+
     print ">>> Step %d: %s" % (self._number, self._text)
     self.RunStep()
 
+    # Persist state.
+    TextToFile(json.dumps(self._state), state_file)
+
   def RunStep(self):
     raise NotImplementedError
 
@@ -349,19 +368,6 @@ class Step(object):
           msg = "Can't continue. Please delete branch %s and try again." % name
           self.Die(msg)
 
-  def Persist(self, var, value):
-    value = value or "__EMPTY__"
-    TextToFile(value, "%s-%s" % (self._config[PERSISTFILE_BASENAME], var))
-
-  def Restore(self, var):
-    value = FileToText("%s-%s" % (self._config[PERSISTFILE_BASENAME], var))
-    value = value or self.Die("Variable '%s' could not be restored." % var)
-    return "" if value == "__EMPTY__" else value
-
-  def RestoreIfUnset(self, var_name):
-    if self._state.get(var_name) is None:
-      self._state[var_name] = self.Restore(var_name)
-
   def InitialEnvironmentChecks(self):
     # Cancel if this is not a git checkout.
     if not os.path.exists(self._config[DOT_GIT_LOCATION]):
@@ -378,14 +384,13 @@ class Step(object):
       self.Die("Workspace is not clean. Please commit or undo your changes.")
 
     # Persist current branch.
-    current_branch = ""
+    self["current_branch"] = ""
     git_result = self.Git("status -s -b -uno").strip()
     for line in git_result.splitlines():
       match = re.match(r"^## (.+)", line)
       if match:
-        current_branch = match.group(1)
+        self["current_branch"] = match.group(1)
         break
-    self.Persist("current_branch", current_branch)
 
     # Fetch unfetched revisions.
     if self.Git("svn fetch") is None:
@@ -393,8 +398,7 @@ class Step(object):
 
   def PrepareBranch(self):
     # Get ahold of a safe temporary branch and check it out.
-    self.RestoreIfUnset("current_branch")
-    if self._state["current_branch"] != self._config[TEMP_BRANCH]:
+    if self["current_branch"] != self._config[TEMP_BRANCH]:
       self.DeleteBranch(self._config[TEMP_BRANCH])
       self.Git("checkout -b %s" % self._config[TEMP_BRANCH])
 
@@ -402,11 +406,10 @@ class Step(object):
     self.DeleteBranch(self._config[BRANCHNAME])
 
   def CommonCleanup(self):
-    self.RestoreIfUnset("current_branch")
-    self.Git("checkout -f %s" % self._state["current_branch"])
-    if self._config[TEMP_BRANCH] != self._state["current_branch"]:
+    self.Git("checkout -f %s" % self["current_branch"])
+    if self._config[TEMP_BRANCH] != self["current_branch"]:
       self.Git("branch -D %s" % self._config[TEMP_BRANCH])
-    if self._config[BRANCHNAME] != self._state["current_branch"]:
+    if self._config[BRANCHNAME] != self["current_branch"]:
       self.Git("branch -D %s" % self._config[BRANCHNAME])
 
     # Clean up all temporary files.
@@ -417,8 +420,7 @@ class Step(object):
       match = re.match(r"^#define %s\s+(\d*)" % def_name, line)
       if match:
         value = match.group(1)
-        self.Persist("%s%s" % (prefix, var_name), value)
-        self._state["%s%s" % (prefix, var_name)] = value
+        self["%s%s" % (prefix, var_name)] = value
     for line in LinesInFile(self._config[VERSION_FILE]):
       for (var_name, def_name) in [("major", "MAJOR_VERSION"),
                                    ("minor", "MINOR_VERSION"),
@@ -426,10 +428,6 @@ class Step(object):
                                    ("patch", "PATCH_LEVEL")]:
         ReadAndPersist(var_name, def_name)
 
-  def RestoreVersionIfUnset(self, prefix=""):
-    for v in ["major", "minor", "build", "patch"]:
-      self.RestoreIfUnset("%s%s" % (prefix, v))
-
   def WaitForLGTM(self):
     print ("Please wait for an LGTM, then type \"LGTM<Return>\" to commit "
            "your change. (If you need to iterate on the patch or double check "
@@ -509,6 +507,9 @@ def RunScript(step_classes,
               config,
               options,
               side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER):
+  state_file = "%s-state.json" % config[PERSISTFILE_BASENAME]
+  if options.s == 0 and os.path.exists(state_file):
+    os.remove(state_file)
   state = {}
   steps = []
   for (number, step_class) in enumerate(step_classes):

tools/push-to-trunk/test_scripts.py

@@ -298,6 +298,7 @@ class ScriptTest(unittest.TestCase):
   def MakeStep(self, step_class=Step, state=None, options=None):
     """Convenience wrapper."""
     options = options or CommonOptions(MakeOptions())
+    state = state if state is not None else self._state
     return MakeStep(step_class=step_class, number=0, state=state,
                     config=TEST_CONFIG, options=options,
                     side_effect_handler=self)
@@ -356,6 +357,7 @@ class ScriptTest(unittest.TestCase):
     self._rl_mock = SimpleMock("readline")
     self._url_mock = SimpleMock("readurl")
     self._tmp_files = []
+    self._state = {}
 
   def tearDown(self):
     Command("rm", "-rf %s*" % TEST_CONFIG[PERSISTFILE_BASENAME])
@@ -369,12 +371,6 @@ class ScriptTest(unittest.TestCase):
     self._rl_mock.AssertFinished()
     self._url_mock.AssertFinished()
 
-  def testPersistRestore(self):
-    self.MakeStep().Persist("test1", "")
-    self.assertEquals("", self.MakeStep().Restore("test1"))
-    self.MakeStep().Persist("test2", "AB123")
-    self.assertEquals("AB123", self.MakeStep().Restore("test2"))
-
   def testGitOrig(self):
     self.assertTrue(Command("git", "--version").startswith("git version"))
 
@@ -396,7 +392,7 @@ class ScriptTest(unittest.TestCase):
     self.ExpectReadline(["Y"])
     self.MakeStep().CommonPrepare()
     self.MakeStep().PrepareBranch()
-    self.assertEquals("some_branch", self.MakeStep().Restore("current_branch"))
+    self.assertEquals("some_branch", self._state["current_branch"])
 
   def testCommonPrepareNoConfirm(self):
     self.ExpectGit([
@@ -408,7 +404,7 @@ class ScriptTest(unittest.TestCase):
     self.ExpectReadline(["n"])
     self.MakeStep().CommonPrepare()
     self.assertRaises(Exception, self.MakeStep().PrepareBranch)
-    self.assertEquals("some_branch", self.MakeStep().Restore("current_branch"))
+    self.assertEquals("some_branch", self._state["current_branch"])
 
   def testCommonPrepareDeleteBranchFailure(self):
     self.ExpectGit([
@@ -421,7 +417,7 @@ class ScriptTest(unittest.TestCase):
     self.ExpectReadline(["Y"])
     self.MakeStep().CommonPrepare()
     self.assertRaises(Exception, self.MakeStep().PrepareBranch)
-    self.assertEquals("some_branch", self.MakeStep().Restore("current_branch"))
+    self.assertEquals("some_branch", self._state["current_branch"])
 
   def testInitialEnvironmentChecks(self):
     TEST_CONFIG[DOT_GIT_LOCATION] = self.MakeEmptyTempFile()
@@ -432,14 +428,10 @@ class ScriptTest(unittest.TestCase):
     TEST_CONFIG[VERSION_FILE] = self.MakeTempVersionFile()
     step = self.MakeStep()
     step.ReadAndPersistVersion()
-    self.assertEquals("3", self.MakeStep().Restore("major"))
-    self.assertEquals("22", self.MakeStep().Restore("minor"))
-    self.assertEquals("5", self.MakeStep().Restore("build"))
-    self.assertEquals("0", self.MakeStep().Restore("patch"))
-    self.assertEquals("3", step._state["major"])
-    self.assertEquals("22", step._state["minor"])
-    self.assertEquals("5", step._state["build"])
-    self.assertEquals("0", step._state["patch"])
+    self.assertEquals("3", step["major"])
+    self.assertEquals("22", step["minor"])
+    self.assertEquals("5", step["build"])
+    self.assertEquals("0", step["patch"])
 
   def testRegex(self):
     self.assertEqual("(issue 321)",
@@ -490,7 +482,7 @@ class ScriptTest(unittest.TestCase):
        "Title\n\nBUG=456\nLOG=N\n\n"],
     ])
 
-    self.MakeStep().Persist("last_push", "1234")
+    self._state["last_push"] = "1234"
     self.MakeStep(PrepareChangeLog).Run()
 
     actual_cl = FileToText(TEST_CONFIG[CHANGELOG_ENTRY_FILE])
@@ -522,10 +514,10 @@ class ScriptTest(unittest.TestCase):
 #"""
 
     self.assertEquals(expected_cl, actual_cl)
-    self.assertEquals("3", self.MakeStep().Restore("major"))
-    self.assertEquals("22", self.MakeStep().Restore("minor"))
-    self.assertEquals("5", self.MakeStep().Restore("build"))
-    self.assertEquals("0", self.MakeStep().Restore("patch"))
+    self.assertEquals("3", self._state["major"])
+    self.assertEquals("22", self._state["minor"])
+    self.assertEquals("5", self._state["build"])
+    self.assertEquals("0", self._state["patch"])
 
   def testEditChangeLog(self):
     TEST_CONFIG[CHANGELOG_ENTRY_FILE] = self.MakeEmptyTempFile()
@@ -545,7 +537,7 @@ class ScriptTest(unittest.TestCase):
 
   def testIncrementVersion(self):
     TEST_CONFIG[VERSION_FILE] = self.MakeTempVersionFile()
-    self.MakeStep().Persist("build", "5")
+    self._state["build"] = "5"
 
     self.ExpectReadline([
       "Y",  # Increment build number.
@@ -553,10 +545,10 @@ class ScriptTest(unittest.TestCase):
 
     self.MakeStep(IncrementVersion).Run()
 
-    self.assertEquals("3", self.MakeStep().Restore("new_major"))
-    self.assertEquals("22", self.MakeStep().Restore("new_minor"))
-    self.assertEquals("6", self.MakeStep().Restore("new_build"))
-    self.assertEquals("0", self.MakeStep().Restore("new_patch"))
+    self.assertEquals("3", self._state["new_major"])
+    self.assertEquals("22", self._state["new_minor"])
+    self.assertEquals("6", self._state["new_build"])
+    self.assertEquals("0", self._state["new_patch"])
 
   def testLastChangeLogEntries(self):
     TEST_CONFIG[CHANGELOG_FILE] = self.MakeEmptyTempFile()
@@ -584,8 +576,8 @@ class ScriptTest(unittest.TestCase):
       ["svn find-rev hash1", "123455\n"],
     ])
 
-    self.MakeStep().Persist("prepare_commit_hash", "hash1")
-    self.MakeStep().Persist("date", "1999-11-11")
+    self._state["prepare_commit_hash"] = "hash1"
+    self._state["date"] = "1999-11-11"
 
     self.MakeStep(SquashCommits).Run()
     self.assertEquals(FileToText(TEST_CONFIG[COMMITMSG_FILE]), expected_msg)
@@ -810,8 +802,11 @@ Performance and stability improvements on all platforms.""", commit)
     auto_roll.RunAutoRoll(TEST_CONFIG, AutoRollOptions(
         MakeOptions(status_password=status_password)), self)
 
-    self.assertEquals("100", self.MakeStep().Restore("lkgr"))
-    self.assertEquals("100", self.MakeStep().Restore("latest"))
+    state = json.loads(FileToText("%s-state.json"
+                                  % TEST_CONFIG[PERSISTFILE_BASENAME]))
+
+    self.assertEquals("100", state["lkgr"])
+    self.assertEquals("100", state["latest"])
 
   def testAutoRollStoppedBySettings(self):
     TEST_CONFIG[DOT_GIT_LOCATION] = self.MakeEmptyTempFile()
@@ -906,6 +901,10 @@ LOG=N
       ["svn find-rev hash3 svn/bleeding_edge", "56789"],
       ["log -1 --format=%s hash3", "Title3"],
       ["svn find-rev \"r12345\" svn/bleeding_edge", "hash4"],
+      # Simulate svn being down which stops the script.
+      ["svn find-rev \"r23456\" svn/bleeding_edge", None],
+      # Restart script in the failing step.
+      ["svn find-rev \"r12345\" svn/bleeding_edge", "hash4"],
       ["svn find-rev \"r23456\" svn/bleeding_edge", "hash2"],
       ["svn find-rev \"r34567\" svn/bleeding_edge", "hash3"],
       ["svn find-rev \"r45678\" svn/bleeding_edge", "hash1"],
@@ -964,6 +963,15 @@ LOG=N
     # ports of r12345. r56789 is the MIPS port of r34567.
     args = ["trunk", "12345", "23456", "34567"]
     self.assertTrue(merge_to_branch.ProcessOptions(options, args))
+
+    # The first run of the script stops because of the svn being down.
+    self.assertRaises(Exception,
+        lambda: RunMergeToBranch(TEST_CONFIG,
+                                 MergeToBranchOptions(options, args),
+                                 self))
+
+    # Test that state recovery after restarting the script works.
+    options.s = 3
     RunMergeToBranch(TEST_CONFIG, MergeToBranchOptions(options, args), self)