Let release scripts determine version based on tags.

Also create new v8 versions based on the roll ref, which
has some more verification steps than the candidate ref.

Improve bailout regarding existing versions. Only create a
new version if the revisions is in the future.

BUG=chromium:451975
TBR=tandrii@chromium.org
NOTRY=true
LOG=n

Review URL: https://codereview.chromium.org/878913002

Cr-Commit-Position: refs/heads/master@{#26303}

tools/release/auto_push.py

@@ -36,7 +36,6 @@ import urllib
 from common_includes import *
 import push_to_candidates
 
-PUSH_MESSAGE_RE = re.compile(r".* \(based on ([a-fA-F0-9]+)\)$")
 
 class Preparation(Step):
   MESSAGE = "Preparation."
@@ -47,30 +46,24 @@ class Preparation(Step):
 
 
 class FetchCandidate(Step):
-  MESSAGE = "Fetching V8 roll candidate ref."
+  MESSAGE = "Fetching V8 roll ref."
 
   def RunStep(self):
-    self.Git("fetch origin +refs/heads/candidate:refs/heads/candidate")
-    self["candidate"] = self.Git("show-ref -s refs/heads/candidate").strip()
+    # The roll ref points to the candidate to be rolled.
+    self.Git("fetch origin +refs/heads/roll:refs/heads/roll")
+    self["candidate"] = self.Git("show-ref -s refs/heads/roll").strip()
 
 
-class CheckLastPush(Step):
-  MESSAGE = "Checking last V8 push to candidates."
+class LastReleaseBailout(Step):
+  MESSAGE = "Checking last V8 release base."
 
   def RunStep(self):
-    last_push = self.FindLastCandidatesPush()
+    last_release = self.GetLatestReleaseBase()
+    commits = self.GitLog(
+        format="%H", git_hash="%s..%s" % (last_release, self["candidate"]))
 
-    # Retrieve the master revision of the last push from the text in
-    # the push commit message.
-    last_push_title = self.GitLog(n=1, format="%s", git_hash=last_push)
-    candidate = PUSH_MESSAGE_RE.match(last_push_title).group(1)
-
-    if not candidate:  # pragma: no cover
-      self.Die("Could not retrieve master revision for candidates push %s"
-               % last_push)
-
-    if self["candidate"] == candidate:
-      print "Already pushed current candidate %s" % candidate
+    if not commits:
+      print "Already pushed current candidate %s" % self["candidate"]
       return True
 
 
@@ -118,7 +111,7 @@ class AutoPush(ScriptsBase):
     return [
       Preparation,
       FetchCandidate,
-      CheckLastPush,
+      LastReleaseBailout,
       PushToCandidates,
     ]
 

tools/release/auto_roll.py

@@ -38,29 +38,31 @@ class CheckActiveRoll(Step):
       return True
 
 
-class DetectLastPush(Step):
-  MESSAGE = "Detect commit ID of the last push to candidates."
-
-  def RunStep(self):
-    self.vc.Fetch()
-    push_hash = self.FindLastCandidatesPush(
-        branch="origin/candidates", include_patches=True)
-    self["last_push"] = self.GetCommitPositionNumber(push_hash)
-
-
 class DetectLastRoll(Step):
   MESSAGE = "Detect commit ID of the last Chromium roll."
 
   def RunStep(self):
+    # The revision that should be rolled.
+    latest_release = self.GetLatestRelease()
+
     # Interpret the DEPS file to retrieve the v8 revision.
     # TODO(machenbach): This should be part or the roll-deps api of
     # depot_tools.
     Var = lambda var: '%s'
     exec(FileToText(os.path.join(self._options.chromium, "DEPS")))
-    last_roll = self.GetCommitPositionNumber(vars['v8_revision'])
-    # FIXME(machenbach): When rolling from master and from candidates there
-    # will be different commit numbers here. Better use version?
-    if int(last_roll) >= int(self["last_push"]):
+
+    # The revision rolled last.
+    last_roll = vars['v8_revision']
+
+    # TODO(machenbach): It is possible that the auto-push script made a new
+    # fast-forward release (e.g. 4.2.3) while somebody patches the last
+    # candidate (e.g. 4.2.2.1). In this case, the auto-roller would pick
+    # the fast-forward release. Should there be a way to prioritize the
+    # patched version?
+
+    if latest_release == last_roll:
+      # We always try to roll if the latest revision is not the revision in
+      # chromium.
       print("There is no newer v8 revision than the one in Chromium (%s)."
             % last_roll)
       return True
@@ -131,7 +133,6 @@ class AutoRoll(ScriptsBase):
   def _Steps(self):
     return [
       CheckActiveRoll,
-      DetectLastPush,
       DetectLastRoll,
       CheckClusterFuzz,
       RollChromium,

tools/release/chromium_roll.py

@@ -19,11 +19,11 @@ class Preparation(Step):
 
 
 class DetectLastPush(Step):
-  MESSAGE = "Detect commit ID of last push to candidates."
+  MESSAGE = "Detect commit ID of last release."
 
   def RunStep(self):
-    self["last_push"] = self._options.last_push or self.FindLastCandidatesPush(
-        branch="origin/candidates", include_patches=True)
+    # The revision that should be rolled.
+    self["last_push"] = self._options.last_push or self.GetLatestRelease()
     self["push_title"] = self.GitLog(n=1, format="%s",
                                      git_hash=self["last_push"])
 

tools/release/common_includes.py

@@ -46,6 +46,7 @@ from git_recipes import GitRecipesMixin
 from git_recipes import GitFailedException
 
 CHANGELOG_FILE = "ChangeLog"
+PUSH_MSG_GIT_RE = re.compile(r".* \(based on (?P<git_rev>[a-fA-F0-9]+)\)$")
 VERSION_FILE = os.path.join("src", "version.cc")
 VERSION_RE = re.compile(r"^\d+\.\d+\.\d+(?:\.\d+)?$")
 
@@ -597,23 +598,44 @@ class Step(GitRecipesMixin):
 
     # Make sure tags are fetched.
     self.Git("fetch origin +refs/tags/*:refs/tags/*")
-    version_parts = sorted(filter(VERSION_RE.match, self.vc.GetTags()),
-                           key=SortingKey, reverse=True)[0].split(".")
-    if len(version_parts) == 3:
-      version_parts.append("0")
-    self["latest_version"] = ".".join(version_parts)
-    return self["latest_version"]
-
-  def FindLastCandidatesPush(
-      self, parent_hash="", branch="", include_patches=False):
-    push_pattern = "^Version [[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*"
-    if not include_patches:
-      # Non-patched versions only have three numbers followed by the "(based
-      # on...) comment."
-      push_pattern += " (based"
-    branch = "" if parent_hash else branch or self.vc.RemoteCandidateBranch()
-    return self.GitLog(n=1, format="%H", grep=push_pattern,
-                       parent_hash=parent_hash, branch=branch)
+    version = sorted(filter(VERSION_RE.match, self.vc.GetTags()),
+                     key=SortingKey, reverse=True)[0]
+    self["latest_version"] = version
+    return version
+
+  def GetLatestRelease(self):
+    """The latest release is the git hash of the latest tagged version.
+
+    This revision should be rolled into chromium.
+    """
+    latest_version = self.GetLatestVersion()
+
+    # The latest release.
+    latest_hash = self.GitLog(n=1, format="%H", branch=latest_version)
+    assert latest_hash
+    return latest_hash
+
+  def GetLatestReleaseBase(self):
+    """The latest release base is the latest revision that is covered in the
+    last change log file. It doesn't include cherry-picked patches.
+    """
+    latest_version = self.GetLatestVersion()
+
+    # Strip patch level if it exists.
+    latest_version = ".".join(latest_version.split(".")[:3])
+
+    # The latest release base.
+    latest_hash = self.GitLog(n=1, format="%H", branch=latest_version)
+    assert latest_hash
+
+    match = PUSH_MSG_GIT_RE.match(
+        self.GitLog(n=1, format="%s", git_hash=latest_hash))
+    if match:
+      # Legacy: In the old process there's one level of indirection. The
+      # version is on the candidates branch and points to the real release
+      # base on master through the commit message.
+      latest_hash = match.group("git_rev")
+    return latest_hash
 
   def ArrayToVersion(self, prefix):
     return ".".join([self[prefix + "major"],

tools/release/push_to_candidates.py

@@ -35,8 +35,7 @@ import urllib2
 from common_includes import *
 
 PUSH_MSG_GIT_SUFFIX = " (based on %s)"
-PUSH_MSG_GIT_RE = re.compile(r".* \(based on (?P<git_rev>[a-fA-F0-9]+)\)$")
-VERSION_RE = re.compile(r"^\d+\.\d+\.\d+(?:\.\d+)?$")
+
 
 class Preparation(Step):
   MESSAGE = "Preparation."
@@ -73,44 +72,6 @@ class PreparePushRevision(Step):
       self.Die("Could not determine the git hash for the push.")
 
 
-class DetectLastPush(Step):
-  MESSAGE = "Detect commit ID of last push to CANDIDATES."
-
-  def RunStep(self):
-    last_push = self._options.last_push or self.FindLastCandidatesPush()
-    while True:
-      # Print assumed commit, circumventing git's pager.
-      print self.GitLog(n=1, git_hash=last_push)
-      if self.Confirm(
-          "Is the commit printed above the last push to candidates?"):
-        break
-      last_push = self.FindLastCandidatesPush(parent_hash=last_push)
-
-    if self._options.last_master:
-      # Read the master revision of the last push from a command-line option.
-      last_push_master = self._options.last_master
-    else:
-      # Retrieve the master revision of the last push from the text in
-      # the push commit message.
-      last_push_title = self.GitLog(n=1, format="%s", git_hash=last_push)
-      last_push_master = PUSH_MSG_GIT_RE.match(
-          last_push_title).group("git_rev")
-
-      if not last_push_master:  # pragma: no cover
-        self.Die(
-            "Could not retrieve master git hash for candidates push %s"
-            % last_push)
-
-    # This points to the git hash of the last push on candidates.
-    self["last_push_candidates"] = last_push
-    # This points to the last master revision that went into the last
-    # push.
-    # TODO(machenbach): Do we need a check to make sure we're not pushing a
-    # revision older than the last push? If we do this, the output of the
-    # current change log preparation won't make much sense.
-    self["last_push_master"] = last_push_master
-
-
 class IncrementVersion(Step):
   MESSAGE = "Increment version number."
 
@@ -145,6 +106,16 @@ class IncrementVersion(Step):
     print ("Incremented version to %s" % self["version"])
 
 
+class DetectLastRelease(Step):
+  MESSAGE = "Detect commit ID of last release base."
+
+  def RunStep(self):
+    if self._options.last_master:
+      self["last_push_master"] = self._options.last_master
+    else:
+      self["last_push_master"] = self.GetLatestReleaseBase()
+
+
 class PrepareChangeLog(Step):
   MESSAGE = "Prepare raw ChangeLog entry."
 
@@ -419,8 +390,8 @@ class PushToCandidates(ScriptsBase):
       Preparation,
       FreshBranch,
       PreparePushRevision,
-      DetectLastPush,
       IncrementVersion,
+      DetectLastRelease,
       PrepareChangeLog,
       EditChangeLog,
       StragglerCommits,

tools/release/test_scripts.py

@@ -33,7 +33,7 @@ import traceback
 import unittest
 
 import auto_push
-from auto_push import CheckLastPush
+from auto_push import LastReleaseBailout
 import auto_roll
 import common_includes
 from common_includes import *
@@ -782,22 +782,14 @@ Performance and stability improvements on all platforms."""
       Cmd("git branch", "  branch1\n* branch2\n"),
       Cmd("git branch", "  branch1\n* branch2\n"),
       Cmd(("git new-branch %s --upstream origin/master" %
-           TEST_CONFIG["BRANCHNAME"]),
-          ""),
-      Cmd(("git log -1 --format=%H --grep="
-           "\"^Version [[:digit:]]*\.[[:digit:]]*\.[[:digit:]]* (based\" "
-           "origin/candidates"), "hash2\n"),
-      Cmd("git log -1 hash2", "Log message\n"),
-    ]
-    if manual:
-      expectations.append(RL("Y"))  # Confirm last push.
-    expectations += [
-      Cmd("git log -1 --format=%s hash2",
-       "Version 3.4.5 (based on abc3)\n"),
+           TEST_CONFIG["BRANCHNAME"]), ""),
       Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""),
       Cmd("git tag", self.TAGS),
       Cmd("git checkout -f origin/master -- src/version.cc",
           "", cb=self.WriteFakeVersionFile),
+      Cmd("git log -1 --format=%H 3.22.4", "release_hash\n"),
+      Cmd("git log -1 --format=%s release_hash",
+          "Version 3.22.4 (based on abc3)\n"),
       Cmd("git log --format=%H abc3..push_hash", "rev1\n"),
       Cmd("git log -1 --format=%s rev1", "Log text 1.\n"),
       Cmd("git log -1 --format=%B rev1", "Text\nLOG=YES\nBUG=v8:321\nText\n"),
@@ -910,9 +902,9 @@ def get_list():
 
     expectations = [
       Cmd("git fetch origin", ""),
-      Cmd(("git log -1 --format=%H --grep="
-           "\"^Version [[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*\" "
-           "origin/candidates"), "push_hash\n"),
+      Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""),
+      Cmd("git tag", self.TAGS),
+      Cmd("git log -1 --format=%H 3.22.4", "push_hash\n"),
       Cmd("git log -1 --format=%s push_hash",
           "Version 3.22.5 (based on abc)\n"),
       URL("https://chromium-build.appspot.com/p/chromium/sheriff_v8.js",
@@ -945,16 +937,17 @@ def get_list():
 
   def testCheckLastPushRecently(self):
     self.Expect([
-      Cmd(("git log -1 --format=%H --grep="
-           "\"^Version [[:digit:]]*\.[[:digit:]]*\.[[:digit:]]* (based\" "
-           "origin/candidates"), "hash2\n"),
-      Cmd("git log -1 --format=%s hash2",
-          "Version 3.4.5 (based on abc123)\n"),
+      Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""),
+      Cmd("git tag", self.TAGS),
+      Cmd("git log -1 --format=%H 3.22.4", "release_hash\n"),
+      Cmd("git log -1 --format=%s release_hash",
+          "Version 3.22.4 (based on abc3)\n"),
+      Cmd("git log --format=%H abc3..abc123", "\n"),
     ])
 
     self._state["candidate"] = "abc123"
     self.assertEquals(0, self.RunStep(
-        auto_push.AutoPush, CheckLastPush, AUTO_PUSH_ARGS))
+        auto_push.AutoPush, LastReleaseBailout, AUTO_PUSH_ARGS))
 
   def testAutoPush(self):
     TextToFile("", os.path.join(TEST_CONFIG["DEFAULT_CWD"], ".git"))
@@ -963,13 +956,14 @@ def get_list():
       Cmd("git status -s -uno", ""),
       Cmd("git status -s -b -uno", "## some_branch\n"),
       Cmd("git fetch", ""),
-      Cmd("git fetch origin +refs/heads/candidate:refs/heads/candidate", ""),
-      Cmd("git show-ref -s refs/heads/candidate", "abc123\n"),
-      Cmd(("git log -1 --format=%H --grep=\""
-           "^Version [[:digit:]]*\.[[:digit:]]*\.[[:digit:]]* (based\""
-           " origin/candidates"), "push_hash\n"),
-      Cmd("git log -1 --format=%s push_hash",
-          "Version 3.4.5 (based on abc101)\n"),
+      Cmd("git fetch origin +refs/heads/roll:refs/heads/roll", ""),
+      Cmd("git show-ref -s refs/heads/roll", "abc123\n"),
+      Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""),
+      Cmd("git tag", self.TAGS),
+      Cmd("git log -1 --format=%H 3.22.4", "release_hash\n"),
+      Cmd("git log -1 --format=%s release_hash",
+          "Version 3.22.4 (based on abc3)\n"),
+      Cmd("git log --format=%H abc3..abc123", "some_stuff\n"),
     ])
 
     auto_push.AutoPush(TEST_CONFIG, self).Run(AUTO_PUSH_ARGS + ["--push"])
@@ -1010,12 +1004,9 @@ deps = {
       URL("https://codereview.chromium.org/search",
           "owner=author%40chromium.org&limit=30&closed=3&format=json",
           ("{\"results\": [{\"subject\": \"different\"}]}")),
-      Cmd("git fetch", ""),
-      Cmd(("git log -1 --format=%H --grep="
-           "\"^Version [[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*\" "
-           "origin/candidates"), "push_hash\n"),
-      Cmd("git log -1 --format=%B push_hash", self.C_V8_22624_LOG),
-      Cmd("git log -1 --format=%B abcd123455", self.C_V8_123455_LOG),
+      Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""),
+      Cmd("git tag", self.TAGS),
+      Cmd("git log -1 --format=%H 3.22.4", "push_hash\n"),
     ])
 
     result = auto_roll.AutoRoll(TEST_CONFIG, self).Run(
@@ -1032,12 +1023,9 @@ deps = {
       URL("https://codereview.chromium.org/search",
           "owner=author%40chromium.org&limit=30&closed=3&format=json",
           ("{\"results\": [{\"subject\": \"different\"}]}")),
-      Cmd("git fetch", ""),
-      Cmd(("git log -1 --format=%H --grep="
-           "\"^Version [[:digit:]]*\.[[:digit:]]*\.[[:digit:]]*\" "
-           "origin/candidates"), "push_hash\n"),
-      Cmd("git log -1 --format=%B push_hash", self.C_V8_123456_LOG),
-      Cmd("git log -1 --format=%B abcd123455", self.C_V8_123455_LOG),
+      Cmd("git fetch origin +refs/tags/*:refs/tags/*", ""),
+      Cmd("git tag", self.TAGS),
+      Cmd("git log -1 --format=%H 3.22.4", "push_hash\n"),
     ])
 
     result = auto_roll.AutoRoll(TEST_CONFIG, self).Run(