Refactor persisting state in push and merge scripts.

- The backed state dict is now persisted and restored in the step template as a json file
- All explicit persist/restore calls are removed
- Added testing an unexpected script failure + restart with state recovery to the merge-to-branch test
- This CL is not changing external behavior of the scripts

BUG=
R=ulan@chromium.org

Review URL: https://codereview.chromium.org/170583002

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@19478 ce2b1a6d-e550-0410-aec6-3dcde31c8c00
parent ae76ce8f
......@@ -83,10 +83,10 @@ class CheckTreeStatus(Step):
def RunStep(self):
status_url = "https://v8-status.appspot.com/current?format=json"
status_json = self.ReadURL(status_url, wait_plan=[5, 20, 300, 300])
message = json.loads(status_json)["message"]
if re.search(r"nopush|no push", message, flags=re.I):
self.Die("Push to trunk disabled by tree state: %s" % message)
self.Persist("tree_message", message)
self["tree_message"] = json.loads(status_json)["message"]
if re.search(r"nopush|no push", self["tree_message"], flags=re.I):
self.Die("Push to trunk disabled by tree state: %s"
% self["tree_message"])
class FetchLatestRevision(Step):
......@@ -97,18 +97,17 @@ class FetchLatestRevision(Step):
match = re.match(r"^r(\d+) ", log)
if not match:
self.Die("Could not extract current svn revision from log.")
self.Persist("latest", match.group(1))
self["latest"] = match.group(1)
class CheckLastPush(Step):
MESSAGE = "Checking last V8 push to trunk."
def RunStep(self):
self.RestoreIfUnset("latest")
log = self.Git("svn log -1 --oneline ChangeLog").strip()
match = re.match(r"^r(\d+) \| Prepare push to trunk", log)
if match:
latest = int(self._state["latest"])
latest = int(self["latest"])
last_push = int(match.group(1))
# TODO(machebach): This metric counts all revisions. It could be
# improved by counting only the revisions on bleeding_edge.
......@@ -124,7 +123,7 @@ class FetchLKGR(Step):
def RunStep(self):
lkgr_url = "https://v8-status.appspot.com/lkgr"
# Retry several times since app engine might have issues.
self.Persist("lkgr", self.ReadURL(lkgr_url, wait_plan=[5, 20, 300, 300]))
self["lkgr"] = self.ReadURL(lkgr_url, wait_plan=[5, 20, 300, 300])
class PushToTrunk(Step):
......@@ -145,11 +144,8 @@ class PushToTrunk(Step):
wait_plan=[5, 20])
def RunStep(self):
self.RestoreIfUnset("latest")
self.RestoreIfUnset("lkgr")
self.RestoreIfUnset("tree_message")
latest = int(self._state["latest"])
lkgr = int(self._state["lkgr"])
latest = int(self["latest"])
lkgr = int(self["lkgr"])
if latest == lkgr:
print "ToT (r%d) is clean. Pushing to trunk." % latest
self.PushTreeStatus("Tree is closed (preparing to push)")
......@@ -165,7 +161,7 @@ class PushToTrunk(Step):
self._options.c),
self._side_effect_handler)
finally:
self.PushTreeStatus(self._state["tree_message"])
self.PushTreeStatus(self["tree_message"])
else:
print("ToT (r%d) is ahead of the LKGR (r%d). Skipping push to trunk."
% (latest, lkgr))
......
......@@ -27,6 +27,7 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import datetime
import json
import os
import re
import subprocess
......@@ -246,17 +247,35 @@ class Step(object):
assert self._side_effect_handler is not None
assert isinstance(options, CommonOptions)
def __getitem__(self, key):
# Convenience method to allow direct [] access on step classes for
# manipulating the backed state dict.
return self._state[key]
def __setitem__(self, key, value):
# Convenience method to allow direct [] access on step classes for
# manipulating the backed state dict.
self._state[key] = value
def Config(self, key):
return self._config[key]
def Run(self):
if self._requires:
self.RestoreIfUnset(self._requires)
if not self._state[self._requires]:
# Restore state.
state_file = "%s-state.json" % self._config[PERSISTFILE_BASENAME]
if not self._state and os.path.exists(state_file):
self._state.update(json.loads(FileToText(state_file)))
# Skip step if requirement is not met.
if self._requires and not self._state.get(self._requires):
return
print ">>> Step %d: %s" % (self._number, self._text)
self.RunStep()
# Persist state.
TextToFile(json.dumps(self._state), state_file)
def RunStep(self):
raise NotImplementedError
......@@ -349,19 +368,6 @@ class Step(object):
msg = "Can't continue. Please delete branch %s and try again." % name
self.Die(msg)
def Persist(self, var, value):
value = value or "__EMPTY__"
TextToFile(value, "%s-%s" % (self._config[PERSISTFILE_BASENAME], var))
def Restore(self, var):
value = FileToText("%s-%s" % (self._config[PERSISTFILE_BASENAME], var))
value = value or self.Die("Variable '%s' could not be restored." % var)
return "" if value == "__EMPTY__" else value
def RestoreIfUnset(self, var_name):
if self._state.get(var_name) is None:
self._state[var_name] = self.Restore(var_name)
def InitialEnvironmentChecks(self):
# Cancel if this is not a git checkout.
if not os.path.exists(self._config[DOT_GIT_LOCATION]):
......@@ -378,14 +384,13 @@ class Step(object):
self.Die("Workspace is not clean. Please commit or undo your changes.")
# Persist current branch.
current_branch = ""
self["current_branch"] = ""
git_result = self.Git("status -s -b -uno").strip()
for line in git_result.splitlines():
match = re.match(r"^## (.+)", line)
if match:
current_branch = match.group(1)
self["current_branch"] = match.group(1)
break
self.Persist("current_branch", current_branch)
# Fetch unfetched revisions.
if self.Git("svn fetch") is None:
......@@ -393,8 +398,7 @@ class Step(object):
def PrepareBranch(self):
# Get ahold of a safe temporary branch and check it out.
self.RestoreIfUnset("current_branch")
if self._state["current_branch"] != self._config[TEMP_BRANCH]:
if self["current_branch"] != self._config[TEMP_BRANCH]:
self.DeleteBranch(self._config[TEMP_BRANCH])
self.Git("checkout -b %s" % self._config[TEMP_BRANCH])
......@@ -402,11 +406,10 @@ class Step(object):
self.DeleteBranch(self._config[BRANCHNAME])
def CommonCleanup(self):
self.RestoreIfUnset("current_branch")
self.Git("checkout -f %s" % self._state["current_branch"])
if self._config[TEMP_BRANCH] != self._state["current_branch"]:
self.Git("checkout -f %s" % self["current_branch"])
if self._config[TEMP_BRANCH] != self["current_branch"]:
self.Git("branch -D %s" % self._config[TEMP_BRANCH])
if self._config[BRANCHNAME] != self._state["current_branch"]:
if self._config[BRANCHNAME] != self["current_branch"]:
self.Git("branch -D %s" % self._config[BRANCHNAME])
# Clean up all temporary files.
......@@ -417,8 +420,7 @@ class Step(object):
match = re.match(r"^#define %s\s+(\d*)" % def_name, line)
if match:
value = match.group(1)
self.Persist("%s%s" % (prefix, var_name), value)
self._state["%s%s" % (prefix, var_name)] = value
self["%s%s" % (prefix, var_name)] = value
for line in LinesInFile(self._config[VERSION_FILE]):
for (var_name, def_name) in [("major", "MAJOR_VERSION"),
("minor", "MINOR_VERSION"),
......@@ -426,10 +428,6 @@ class Step(object):
("patch", "PATCH_LEVEL")]:
ReadAndPersist(var_name, def_name)
def RestoreVersionIfUnset(self, prefix=""):
for v in ["major", "minor", "build", "patch"]:
self.RestoreIfUnset("%s%s" % (prefix, v))
def WaitForLGTM(self):
print ("Please wait for an LGTM, then type \"LGTM<Return>\" to commit "
"your change. (If you need to iterate on the patch or double check "
......@@ -509,6 +507,9 @@ def RunScript(step_classes,
config,
options,
side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER):
state_file = "%s-state.json" % config[PERSISTFILE_BASENAME]
if options.s == 0 and os.path.exists(state_file):
os.remove(state_file)
state = {}
steps = []
for (number, step_class) in enumerate(step_classes):
......
This diff is collapsed.
This diff is collapsed.
......@@ -298,6 +298,7 @@ class ScriptTest(unittest.TestCase):
def MakeStep(self, step_class=Step, state=None, options=None):
"""Convenience wrapper."""
options = options or CommonOptions(MakeOptions())
state = state if state is not None else self._state
return MakeStep(step_class=step_class, number=0, state=state,
config=TEST_CONFIG, options=options,
side_effect_handler=self)
......@@ -356,6 +357,7 @@ class ScriptTest(unittest.TestCase):
self._rl_mock = SimpleMock("readline")
self._url_mock = SimpleMock("readurl")
self._tmp_files = []
self._state = {}
def tearDown(self):
Command("rm", "-rf %s*" % TEST_CONFIG[PERSISTFILE_BASENAME])
......@@ -369,12 +371,6 @@ class ScriptTest(unittest.TestCase):
self._rl_mock.AssertFinished()
self._url_mock.AssertFinished()
def testPersistRestore(self):
self.MakeStep().Persist("test1", "")
self.assertEquals("", self.MakeStep().Restore("test1"))
self.MakeStep().Persist("test2", "AB123")
self.assertEquals("AB123", self.MakeStep().Restore("test2"))
def testGitOrig(self):
self.assertTrue(Command("git", "--version").startswith("git version"))
......@@ -396,7 +392,7 @@ class ScriptTest(unittest.TestCase):
self.ExpectReadline(["Y"])
self.MakeStep().CommonPrepare()
self.MakeStep().PrepareBranch()
self.assertEquals("some_branch", self.MakeStep().Restore("current_branch"))
self.assertEquals("some_branch", self._state["current_branch"])
def testCommonPrepareNoConfirm(self):
self.ExpectGit([
......@@ -408,7 +404,7 @@ class ScriptTest(unittest.TestCase):
self.ExpectReadline(["n"])
self.MakeStep().CommonPrepare()
self.assertRaises(Exception, self.MakeStep().PrepareBranch)
self.assertEquals("some_branch", self.MakeStep().Restore("current_branch"))
self.assertEquals("some_branch", self._state["current_branch"])
def testCommonPrepareDeleteBranchFailure(self):
self.ExpectGit([
......@@ -421,7 +417,7 @@ class ScriptTest(unittest.TestCase):
self.ExpectReadline(["Y"])
self.MakeStep().CommonPrepare()
self.assertRaises(Exception, self.MakeStep().PrepareBranch)
self.assertEquals("some_branch", self.MakeStep().Restore("current_branch"))
self.assertEquals("some_branch", self._state["current_branch"])
def testInitialEnvironmentChecks(self):
TEST_CONFIG[DOT_GIT_LOCATION] = self.MakeEmptyTempFile()
......@@ -432,14 +428,10 @@ class ScriptTest(unittest.TestCase):
TEST_CONFIG[VERSION_FILE] = self.MakeTempVersionFile()
step = self.MakeStep()
step.ReadAndPersistVersion()
self.assertEquals("3", self.MakeStep().Restore("major"))
self.assertEquals("22", self.MakeStep().Restore("minor"))
self.assertEquals("5", self.MakeStep().Restore("build"))
self.assertEquals("0", self.MakeStep().Restore("patch"))
self.assertEquals("3", step._state["major"])
self.assertEquals("22", step._state["minor"])
self.assertEquals("5", step._state["build"])
self.assertEquals("0", step._state["patch"])
self.assertEquals("3", step["major"])
self.assertEquals("22", step["minor"])
self.assertEquals("5", step["build"])
self.assertEquals("0", step["patch"])
def testRegex(self):
self.assertEqual("(issue 321)",
......@@ -490,7 +482,7 @@ class ScriptTest(unittest.TestCase):
"Title\n\nBUG=456\nLOG=N\n\n"],
])
self.MakeStep().Persist("last_push", "1234")
self._state["last_push"] = "1234"
self.MakeStep(PrepareChangeLog).Run()
actual_cl = FileToText(TEST_CONFIG[CHANGELOG_ENTRY_FILE])
......@@ -522,10 +514,10 @@ class ScriptTest(unittest.TestCase):
#"""
self.assertEquals(expected_cl, actual_cl)
self.assertEquals("3", self.MakeStep().Restore("major"))
self.assertEquals("22", self.MakeStep().Restore("minor"))
self.assertEquals("5", self.MakeStep().Restore("build"))
self.assertEquals("0", self.MakeStep().Restore("patch"))
self.assertEquals("3", self._state["major"])
self.assertEquals("22", self._state["minor"])
self.assertEquals("5", self._state["build"])
self.assertEquals("0", self._state["patch"])
def testEditChangeLog(self):
TEST_CONFIG[CHANGELOG_ENTRY_FILE] = self.MakeEmptyTempFile()
......@@ -545,7 +537,7 @@ class ScriptTest(unittest.TestCase):
def testIncrementVersion(self):
TEST_CONFIG[VERSION_FILE] = self.MakeTempVersionFile()
self.MakeStep().Persist("build", "5")
self._state["build"] = "5"
self.ExpectReadline([
"Y", # Increment build number.
......@@ -553,10 +545,10 @@ class ScriptTest(unittest.TestCase):
self.MakeStep(IncrementVersion).Run()
self.assertEquals("3", self.MakeStep().Restore("new_major"))
self.assertEquals("22", self.MakeStep().Restore("new_minor"))
self.assertEquals("6", self.MakeStep().Restore("new_build"))
self.assertEquals("0", self.MakeStep().Restore("new_patch"))
self.assertEquals("3", self._state["new_major"])
self.assertEquals("22", self._state["new_minor"])
self.assertEquals("6", self._state["new_build"])
self.assertEquals("0", self._state["new_patch"])
def testLastChangeLogEntries(self):
TEST_CONFIG[CHANGELOG_FILE] = self.MakeEmptyTempFile()
......@@ -584,8 +576,8 @@ class ScriptTest(unittest.TestCase):
["svn find-rev hash1", "123455\n"],
])
self.MakeStep().Persist("prepare_commit_hash", "hash1")
self.MakeStep().Persist("date", "1999-11-11")
self._state["prepare_commit_hash"] = "hash1"
self._state["date"] = "1999-11-11"
self.MakeStep(SquashCommits).Run()
self.assertEquals(FileToText(TEST_CONFIG[COMMITMSG_FILE]), expected_msg)
......@@ -810,8 +802,11 @@ Performance and stability improvements on all platforms.""", commit)
auto_roll.RunAutoRoll(TEST_CONFIG, AutoRollOptions(
MakeOptions(status_password=status_password)), self)
self.assertEquals("100", self.MakeStep().Restore("lkgr"))
self.assertEquals("100", self.MakeStep().Restore("latest"))
state = json.loads(FileToText("%s-state.json"
% TEST_CONFIG[PERSISTFILE_BASENAME]))
self.assertEquals("100", state["lkgr"])
self.assertEquals("100", state["latest"])
def testAutoRollStoppedBySettings(self):
TEST_CONFIG[DOT_GIT_LOCATION] = self.MakeEmptyTempFile()
......@@ -906,6 +901,10 @@ LOG=N
["svn find-rev hash3 svn/bleeding_edge", "56789"],
["log -1 --format=%s hash3", "Title3"],
["svn find-rev \"r12345\" svn/bleeding_edge", "hash4"],
# Simulate svn being down which stops the script.
["svn find-rev \"r23456\" svn/bleeding_edge", None],
# Restart script in the failing step.
["svn find-rev \"r12345\" svn/bleeding_edge", "hash4"],
["svn find-rev \"r23456\" svn/bleeding_edge", "hash2"],
["svn find-rev \"r34567\" svn/bleeding_edge", "hash3"],
["svn find-rev \"r45678\" svn/bleeding_edge", "hash1"],
......@@ -964,6 +963,15 @@ LOG=N
# ports of r12345. r56789 is the MIPS port of r34567.
args = ["trunk", "12345", "23456", "34567"]
self.assertTrue(merge_to_branch.ProcessOptions(options, args))
# The first run of the script stops because of the svn being down.
self.assertRaises(Exception,
lambda: RunMergeToBranch(TEST_CONFIG,
MergeToBranchOptions(options, args),
self))
# Test that state recovery after restarting the script works.
options.s = 3
RunMergeToBranch(TEST_CONFIG, MergeToBranchOptions(options, args), self)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment