Let ChangeLog get auto-generated in push-to-trunk script.

BUG=
R=jkummerow@chromium.org

Review URL: https://codereview.chromium.org/78683002

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@17951 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

tools/push-to-trunk/common_includes.py

@@ -81,17 +81,50 @@ def GetLastChangeLogEntries(change_log_file):
   return "".join(result)
 
 
-def MakeChangeLogBody(commit_generator):
+def MakeComment(text):
+  return MSub(r"^( ?)", "#", text)
+
+
+def StripComments(text):
+  # Use split not splitlines to keep terminal newlines.
+  return "\n".join(filter(lambda x: not x.startswith("#"), text.split("\n")))
+
+
+def MakeChangeLogBody(commit_messages, auto_format=False):
   result = ""
-  for (title, body, author) in commit_generator():
+  added_titles = set()
+  for (title, body, author) in commit_messages:
+    # TODO(machenbach): Reload the commit description from rietveld in order to
+    # catch late changes.
+    title = title.rstrip()
+    if auto_format:
+      # Only add commits that set the LOG flag correctly.
+      log_exp = r"^[ \t]*LOG[ \t]*=[ \t]*(?:Y(?:ES)?)|TRUE"
+      if not re.search(log_exp, body, flags=re.I | re.M):
+        continue
+      # Never include reverts.
+      if title.startswith("Revert "):
+        continue
+      # Don't include duplicates.
+      if title in added_titles:
+        continue
+
+    # TODO(machenbach): Let python do all formatting. Get raw git title, attach
+    # issue and add/move dot to the end - all in one line. Make formatting and
+    # indentation afterwards.
+
     # Add the commit's title line.
-    result += "%s\n" % title.rstrip()
+    result += "%s\n" % title
+    added_titles.add(title)
 
     # Add bug references.
     result += MakeChangeLogBugReference(body)
 
-    # Append the commit's author for reference.
-    result += "%s\n\n" % author.rstrip()
+    # Append the commit's author for reference if not in auto-format mode.
+    if not auto_format:
+      result += "%s\n" % author.rstrip()
+
+    result += "\n"
   return result
 
 

tools/push-to-trunk/push_to_trunk.py

@@ -112,20 +112,29 @@ class PrepareChangeLog(Step):
     args = "log %s..HEAD --format=%%H" % self._state["last_push"]
     commits = self.Git(args).strip()
 
-    def GetCommitMessages():
-      for commit in commits.splitlines():
-        yield [
+    # Cache raw commit messages.
+    commit_messages = [
+      [
         self.Git("log -1 %s --format=\"%%w(80,8,8)%%s\"" % commit),
         self.Git("log -1 %s --format=\"%%B\"" % commit),
         self.Git("log -1 %s --format=\"%%w(80,8,8)(%%an)\"" % commit),
+      ] for commit in commits.splitlines()
     ]
 
-    body = MakeChangeLogBody(GetCommitMessages)
+    # Auto-format commit messages.
+    body = MakeChangeLogBody(commit_messages, auto_format=True)
     AppendToFile(body, self.Config(CHANGELOG_ENTRY_FILE))
 
-    msg = "        Performance and stability improvements on all platforms.\n"
+    msg = ("        Performance and stability improvements on all platforms."
+           "\n#\n# The change log above is auto-generated. Please review if "
+           "all relevant\n# commit messages from the list below are included."
+           "\n# All lines starting with # will be stripped.\n#\n")
     AppendToFile(msg, self.Config(CHANGELOG_ENTRY_FILE))
 
+    # Include unformatted commit messages as a reference in a comment.
+    comment_body = MakeComment(MakeChangeLogBody(commit_messages))
+    AppendToFile(comment_body, self.Config(CHANGELOG_ENTRY_FILE))
+
 
 class EditChangeLog(Step):
   def __init__(self):
@@ -143,9 +152,10 @@ class EditChangeLog(Step):
     handle, new_changelog = tempfile.mkstemp()
     os.close(handle)
 
-    # (1) Eliminate tabs, (2) fix too little and (3) too much indentation, and
-    # (4) eliminate trailing whitespace.
+    # (1) Strip comments, (2) eliminate tabs, (3) fix too little and (4) too
+    # much indentation, and (5) eliminate trailing whitespace.
     changelog_entry = FileToText(self.Config(CHANGELOG_ENTRY_FILE)).rstrip()
+    changelog_entry = StripComments(changelog_entry)
     changelog_entry = MSub(r"\t", r"        ", changelog_entry)
     changelog_entry = MSub(r"^ {1,7}([^ ])", r"        \1", changelog_entry)
     changelog_entry = MSub(r"^ {9,80}([^ ])", r"        \1", changelog_entry)

tools/push-to-trunk/test_scripts.py

@@ -53,8 +53,20 @@ TEST_CONFIG = {
 
 
 class ToplevelTest(unittest.TestCase):
+  def testMakeComment(self):
+    self.assertEquals("#   Line 1\n#   Line 2\n#",
+                      MakeComment("    Line 1\n    Line 2\n"))
+    self.assertEquals("#Line 1\n#Line 2",
+                      MakeComment("Line 1\n Line 2"))
+
+  def testStripComments(self):
+    self.assertEquals("    Line 1\n    Line 3\n",
+        StripComments("    Line 1\n#   Line 2\n    Line 3\n#\n"))
+    self.assertEquals("\nLine 2 ### Test\n #",
+        StripComments("###\n# \n\n#  Line 1\nLine 2 ### Test\n #"))
+
   def testMakeChangeLogBodySimple(self):
-    commits = lambda: [
+    commits = [
           ["        Title text 1",
            "Title text 1\n\nBUG=\n",
            "        author1@chromium.org"],
@@ -70,8 +82,27 @@ class ToplevelTest(unittest.TestCase):
                       MakeChangeLogBody(commits))
 
   def testMakeChangeLogBodyEmpty(self):
-    commits = lambda: []
-    self.assertEquals("", MakeChangeLogBody(commits))
+    self.assertEquals("", MakeChangeLogBody([]))
+
+  def testMakeChangeLogBodyAutoFormat(self):
+    commits = [
+          ["        Title text 1",
+           "Title text 1\nLOG=y\nBUG=\n",
+           "        author1@chromium.org"],
+          ["        Title text 2",
+           "Title text 2\n\nBUG=1234\n",
+           "        author2@chromium.org"],
+          ["        Title text 3",
+           "Title text 3\n\nBUG=1234\nLOG = Yes\n",
+           "        author3@chromium.org"],
+          ["        Title text 3",
+           "Title text 4\n\nBUG=1234\nLOG=\n",
+           "        author4@chromium.org"],
+        ]
+    self.assertEquals("        Title text 1\n\n"
+                      "        Title text 3\n"
+                      "        (Chromium issue 1234)\n\n",
+                      MakeChangeLogBody(commits, True))
 
   def testMakeChangeLogBugReferenceEmpty(self):
     self.assertEquals("", MakeChangeLogBugReference(""))
@@ -327,28 +358,55 @@ class ScriptTest(unittest.TestCase):
     TEST_CONFIG[CHANGELOG_ENTRY_FILE] = self.MakeEmptyTempFile()
 
     self._git_recipe = [
-      ["log 1234..HEAD --format=%H", "rev1\nrev2"],
+      ["log 1234..HEAD --format=%H", "rev1\nrev2\nrev3"],
       ["log -1 rev1 --format=\"%w(80,8,8)%s\"", "        Title text 1"],
-      ["log -1 rev1 --format=\"%B\"", "Title\n\nBUG=\n"],
+      ["log -1 rev1 --format=\"%B\"", "Title\n\nBUG=\nLOG=y\n"],
       ["log -1 rev1 --format=\"%w(80,8,8)(%an)\"",
        "        author1@chromium.org"],
       ["log -1 rev2 --format=\"%w(80,8,8)%s\"", "        Title text 2"],
-      ["log -1 rev2 --format=\"%B\"", "Title\n\nBUG=321\n"],
+      ["log -1 rev2 --format=\"%B\"", "Title\n\nBUG=123\nLOG= \n"],
       ["log -1 rev2 --format=\"%w(80,8,8)(%an)\"",
        "        author2@chromium.org"],
+      ["log -1 rev3 --format=\"%w(80,8,8)%s\"", "        Title text 3"],
+      ["log -1 rev3 --format=\"%B\"", "Title\n\nBUG=321\nLOG=true\n"],
+      ["log -1 rev3 --format=\"%w(80,8,8)(%an)\"",
+       "        author3@chromium.org"],
     ]
 
     self.MakeStep().Persist("last_push", "1234")
     self.MakeStep(PrepareChangeLog).Run()
 
-    cl = FileToText(TEST_CONFIG[CHANGELOG_ENTRY_FILE])
-    self.assertTrue(re.search(r"\d+\-\d+\-\d+: Version 3\.22\.5", cl))
-    self.assertTrue(re.search(r"        Title text 1", cl))
-    self.assertTrue(re.search(r"        Title text 2", cl))
-    self.assertTrue(re.search(r"        author1@chromium.org", cl))
-    self.assertTrue(re.search(r"        author2@chromium.org", cl))
-    self.assertTrue(re.search(r"        \(Chromium issue 321\)", cl))
-    self.assertFalse(re.search(r"BUG=", cl))
+    actual_cl = FileToText(TEST_CONFIG[CHANGELOG_ENTRY_FILE])
+
+    # TODO(machenbach): Mock out call to date() in order to make a fixed
+    # comparison here instead of a regexp match.
+    expected_cl = """\\d+\\-\\d+\\-\\d+: Version 3\\.22\\.5
+
+        Title text 1
+
+        Title text 3
+        \\(Chromium issue 321\\)
+
+        Performance and stability improvements on all platforms\\.
+#
+# The change log above is auto-generated\\. Please review if all relevant
+# commit messages from the list below are included\\.
+# All lines starting with # will be stripped\\.
+#
+#       Title text 1
+#       author1@chromium\\.org
+#
+#       Title text 2
+#       \\(Chromium issue 123\\)
+#       author2@chromium\\.org
+#
+#       Title text 3
+#       \\(Chromium issue 321\\)
+#       author3@chromium\\.org
+#
+#"""
+
+    self.assertTrue(re.match(expected_cl, actual_cl))
     self.assertEquals("3", self.MakeStep().Restore("major"))
     self.assertEquals("22", self.MakeStep().Restore("minor"))
     self.assertEquals("5", self.MakeStep().Restore("build"))
@@ -447,9 +505,17 @@ class ScriptTest(unittest.TestCase):
       self.assertTrue(re.search(r"Version 3.22.5", cl))
       self.assertTrue(re.search(r"        Log text 1", cl))
       self.assertTrue(re.search(r"        \(issue 321\)", cl))
+      self.assertFalse(re.search(r"        author1@chromium\.org", cl))
+
+      # Make sure all comments got stripped.
+      self.assertFalse(re.search(r"^#", cl, flags=re.M))
+
       version = FileToText(TEST_CONFIG[VERSION_FILE])
       self.assertTrue(re.search(r"#define BUILD_NUMBER\s+6", version))
 
+    def CheckUpload():
+      cl = FileToText(TEST_CONFIG[CHANGELOG_FILE])
+
     def CheckSVNCommit():
       commit = FileToText(TEST_CONFIG[COMMITMSG_FILE])
       self.assertTrue(re.search(r"Version 3.22.5", commit))
@@ -475,7 +541,7 @@ class ScriptTest(unittest.TestCase):
       ["log -1 1234", "Last push ouput\n"],
       ["log 1234..HEAD --format=%H", "rev1\n"],
       ["log -1 rev1 --format=\"%w(80,8,8)%s\"", "        Log text 1.\n"],
-      ["log -1 rev1 --format=\"%B\"", "Text\nBUG=v8:321\nText\n"],
+      ["log -1 rev1 --format=\"%B\"", "Text\nLOG=YES\nBUG=v8:321\nText\n"],
       ["log -1 rev1 --format=\"%w(80,8,8)(%an)\"",
        "        author1@chromium.org\n"],
       [("commit -a -m \"Prepare push to trunk.  "