push_to_trunk.py 20.4 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
#!/usr/bin/env python
# Copyright 2013 the V8 project authors. All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#     * Neither the name of Google Inc. nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import optparse
import sys
import tempfile
32
import urllib2
33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54

from common_includes import *

TRUNKBRANCH = "TRUNKBRANCH"
CHROMIUM = "CHROMIUM"
DEPS_FILE = "DEPS_FILE"

CONFIG = {
  BRANCHNAME: "prepare-push",
  TRUNKBRANCH: "trunk-push",
  PERSISTFILE_BASENAME: "/tmp/v8-push-to-trunk-tempfile",
  TEMP_BRANCH: "prepare-push-temporary-branch-created-by-script",
  DOT_GIT_LOCATION: ".git",
  VERSION_FILE: "src/version.cc",
  CHANGELOG_FILE: "ChangeLog",
  CHANGELOG_ENTRY_FILE: "/tmp/v8-push-to-trunk-tempfile-changelog-entry",
  PATCH_FILE: "/tmp/v8-push-to-trunk-tempfile-patch-file",
  COMMITMSG_FILE: "/tmp/v8-push-to-trunk-tempfile-commitmsg",
  DEPS_FILE: "DEPS",
}


55 56 57 58 59 60 61 62 63 64
class PushToTrunkOptions(CommonOptions):
  def __init__(self, options):
    super(PushToTrunkOptions, self).__init__(options, options.m)
    self.requires_editor = not options.f
    self.wait_for_lgtm = not options.f
    self.tbr_commit = not options.m
    self.l = options.l
    self.r = options.r
    self.c = options.c

65
class Preparation(Step):
66
  MESSAGE = "Preparation."
67 68 69 70

  def RunStep(self):
    self.InitialEnvironmentChecks()
    self.CommonPrepare()
71
    self.PrepareBranch()
72 73 74 75
    self.DeleteBranch(self.Config(TRUNKBRANCH))


class FreshBranch(Step):
76
  MESSAGE = "Create a fresh branch."
77 78 79 80 81 82 83 84

  def RunStep(self):
    args = "checkout -b %s svn/bleeding_edge" % self.Config(BRANCHNAME)
    if self.Git(args) is None:
      self.Die("Creating branch %s failed." % self.Config(BRANCHNAME))


class DetectLastPush(Step):
85
  MESSAGE = "Detect commit ID of last push to trunk."
86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101

  def RunStep(self):
    last_push = (self._options.l or
                 self.Git("log -1 --format=%H ChangeLog").strip())
    while True:
      # Print assumed commit, circumventing git's pager.
      print self.Git("log -1 %s" % last_push)
      if self.Confirm("Is the commit printed above the last push to trunk?"):
        break
      args = "log -1 --format=%H %s^ ChangeLog" % last_push
      last_push = self.Git(args).strip()
    self.Persist("last_push", last_push)
    self._state["last_push"] = last_push


class PrepareChangeLog(Step):
102
  MESSAGE = "Prepare raw ChangeLog entry."
103

104 105 106 107 108 109 110 111 112 113
  def Reload(self, body):
    """Attempts to reload the commit message from rietveld in order to allow
    late changes to the LOG flag. Note: This is brittle to future changes of
    the web page name or structure.
    """
    match = re.search(r"^Review URL: https://codereview\.chromium\.org/(\d+)$",
                      body, flags=re.M)
    if match:
      cl_url = "https://codereview.chromium.org/%s/description" % match.group(1)
      try:
114 115 116
        # Fetch from Rietveld but only retry once with one second delay since
        # there might be many revisions.
        body = self.ReadURL(cl_url, wait_plan=[1])
117 118 119 120
      except urllib2.URLError:
        pass
    return body

121 122 123 124 125 126
  def RunStep(self):
    self.RestoreIfUnset("last_push")

    # These version numbers are used again later for the trunk commit.
    self.ReadAndPersistVersion()

127
    date = self.GetDate()
128 129 130 131 132 133 134 135 136
    self.Persist("date", date)
    output = "%s: Version %s.%s.%s\n\n" % (date,
                                           self._state["major"],
                                           self._state["minor"],
                                           self._state["build"])
    TextToFile(output, self.Config(CHANGELOG_ENTRY_FILE))

    args = "log %s..HEAD --format=%%H" % self._state["last_push"]
    commits = self.Git(args).strip()
137

138 139 140
    # Cache raw commit messages.
    commit_messages = [
      [
141
        self.Git("log -1 %s --format=\"%%s\"" % commit),
142
        self.Reload(self.Git("log -1 %s --format=\"%%B\"" % commit)),
143
        self.Git("log -1 %s --format=\"%%an\"" % commit),
144 145 146 147 148
      ] for commit in commits.splitlines()
    ]

    # Auto-format commit messages.
    body = MakeChangeLogBody(commit_messages, auto_format=True)
149
    AppendToFile(body, self.Config(CHANGELOG_ENTRY_FILE))
150

151 152 153 154
    msg = ("        Performance and stability improvements on all platforms."
           "\n#\n# The change log above is auto-generated. Please review if "
           "all relevant\n# commit messages from the list below are included."
           "\n# All lines starting with # will be stripped.\n#\n")
155 156
    AppendToFile(msg, self.Config(CHANGELOG_ENTRY_FILE))

157 158 159 160
    # Include unformatted commit messages as a reference in a comment.
    comment_body = MakeComment(MakeChangeLogBody(commit_messages))
    AppendToFile(comment_body, self.Config(CHANGELOG_ENTRY_FILE))

161

162
class EditChangeLog(Step):
163
  MESSAGE = "Edit ChangeLog entry."
164 165 166 167 168

  def RunStep(self):
    print ("Please press <Return> to have your EDITOR open the ChangeLog "
           "entry, then edit its contents to your liking. When you're done, "
           "save the file and exit your EDITOR. ")
169
    self.ReadLine(default="")
170 171 172 173
    self.Editor(self.Config(CHANGELOG_ENTRY_FILE))
    handle, new_changelog = tempfile.mkstemp()
    os.close(handle)

174
    # Strip comments and reformat with correct indentation.
175
    changelog_entry = FileToText(self.Config(CHANGELOG_ENTRY_FILE)).rstrip()
176
    changelog_entry = StripComments(changelog_entry)
177
    changelog_entry = "\n".join(map(Fill80, changelog_entry.splitlines()))
178
    changelog_entry = changelog_entry.lstrip()
179 180 181 182 183 184 185 186 187 188 189 190 191 192

    if changelog_entry == "":
      self.Die("Empty ChangeLog entry.")

    with open(new_changelog, "w") as f:
      f.write(changelog_entry)
      f.write("\n\n\n")  # Explicitly insert two empty lines.

    AppendToFile(FileToText(self.Config(CHANGELOG_FILE)), new_changelog)
    TextToFile(FileToText(new_changelog), self.Config(CHANGELOG_FILE))
    os.remove(new_changelog)


class IncrementVersion(Step):
193
  MESSAGE = "Increment version number."
194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214

  def RunStep(self):
    self.RestoreIfUnset("build")
    new_build = str(int(self._state["build"]) + 1)

    if self.Confirm(("Automatically increment BUILD_NUMBER? (Saying 'n' will "
                     "fire up your EDITOR on %s so you can make arbitrary "
                     "changes. When you're done, save the file and exit your "
                     "EDITOR.)" % self.Config(VERSION_FILE))):
      text = FileToText(self.Config(VERSION_FILE))
      text = MSub(r"(?<=#define BUILD_NUMBER)(?P<space>\s+)\d*$",
                  r"\g<space>%s" % new_build,
                  text)
      TextToFile(text, self.Config(VERSION_FILE))
    else:
      self.Editor(self.Config(VERSION_FILE))

    self.ReadAndPersistVersion("new_")


class CommitLocal(Step):
215
  MESSAGE = "Commit to local branch."
216 217 218 219 220 221 222 223

  def RunStep(self):
    self.RestoreVersionIfUnset("new_")
    prep_commit_msg = ("Prepare push to trunk.  "
        "Now working on version %s.%s.%s." % (self._state["new_major"],
                                              self._state["new_minor"],
                                              self._state["new_build"]))
    self.Persist("prep_commit_msg", prep_commit_msg)
224 225 226

    # Include optional TBR only in the git command. The persisted commit
    # message is used for finding the commit again later.
227
    review = "\n\nTBR=%s" % self._options.r if self._options.tbr_commit else ""
228
    if self.Git("commit -a -m \"%s%s\"" % (prep_commit_msg, review)) is None:
229 230 231 232
      self.Die("'git commit -a' failed.")


class CommitRepository(Step):
233
  MESSAGE = "Commit to the repository."
234 235 236 237 238

  def RunStep(self):
    self.WaitForLGTM()
    # Re-read the ChangeLog entry (to pick up possible changes).
    # FIXME(machenbach): This was hanging once with a broken pipe.
239 240 241
    TextToFile(GetLastChangeLogEntries(self.Config(CHANGELOG_FILE)),
               self.Config(CHANGELOG_ENTRY_FILE))

242
    if self.Git("cl dcommit -f", "PRESUBMIT_TREE_CHECK=\"skip\"") is None:
243 244 245 246
      self.Die("'git cl dcommit' failed, please try again.")


class StragglerCommits(Step):
247 248
  MESSAGE = ("Fetch straggler commits that sneaked in since this script was "
             "started.")
249 250 251 252 253 254 255 256 257 258 259 260

  def RunStep(self):
    if self.Git("svn fetch") is None:
      self.Die("'git svn fetch' failed.")
    self.Git("checkout svn/bleeding_edge")
    self.RestoreIfUnset("prep_commit_msg")
    args = "log -1 --format=%%H --grep=\"%s\"" % self._state["prep_commit_msg"]
    prepare_commit_hash = self.Git(args).strip()
    self.Persist("prepare_commit_hash", prepare_commit_hash)


class SquashCommits(Step):
261
  MESSAGE = "Squash commits into one."
262 263 264 265 266 267 268 269

  def RunStep(self):
    # Instead of relying on "git rebase -i", we'll just create a diff, because
    # that's easier to automate.
    self.RestoreIfUnset("prepare_commit_hash")
    args = "diff svn/trunk %s" % self._state["prepare_commit_hash"]
    TextToFile(self.Git(args), self.Config(PATCH_FILE))

270
    # Convert the ChangeLog entry to commit message format.
271
    self.RestoreIfUnset("date")
272 273 274 275 276 277 278 279 280 281 282
    text = FileToText(self.Config(CHANGELOG_ENTRY_FILE))

    # Remove date and trailing white space.
    text = re.sub(r"^%s: " % self._state["date"], "", text.rstrip())

    # Remove indentation and merge paragraphs into single long lines, keeping
    # empty lines between them.
    def SplitMapJoin(split_text, fun, join_text):
      return lambda text: join_text.join(map(fun, text.split(split_text)))
    strip = lambda line: line.strip()
    text = SplitMapJoin("\n\n", SplitMapJoin("\n", strip, " "), "\n\n")(text)
283 284 285 286 287 288 289 290

    if not text:
      self.Die("Commit message editing failed.")
    TextToFile(text, self.Config(COMMITMSG_FILE))
    os.remove(self.Config(CHANGELOG_ENTRY_FILE))


class NewBranch(Step):
291
  MESSAGE = "Create a new branch from trunk."
292 293 294 295 296 297 298 299

  def RunStep(self):
    if self.Git("checkout -b %s svn/trunk" % self.Config(TRUNKBRANCH)) is None:
      self.Die("Checking out a new branch '%s' failed." %
               self.Config(TRUNKBRANCH))


class ApplyChanges(Step):
300
  MESSAGE = "Apply squashed changes."
301 302 303 304 305 306 307

  def RunStep(self):
    self.ApplyPatch(self.Config(PATCH_FILE))
    Command("rm", "-f %s*" % self.Config(PATCH_FILE))


class SetVersion(Step):
308
  MESSAGE = "Set correct version for trunk."
309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328

  def RunStep(self):
    self.RestoreVersionIfUnset()
    output = ""
    for line in FileToText(self.Config(VERSION_FILE)).splitlines():
      if line.startswith("#define MAJOR_VERSION"):
        line = re.sub("\d+$", self._state["major"], line)
      elif line.startswith("#define MINOR_VERSION"):
        line = re.sub("\d+$", self._state["minor"], line)
      elif line.startswith("#define BUILD_NUMBER"):
        line = re.sub("\d+$", self._state["build"], line)
      elif line.startswith("#define PATCH_LEVEL"):
        line = re.sub("\d+$", "0", line)
      elif line.startswith("#define IS_CANDIDATE_VERSION"):
        line = re.sub("\d+$", "0", line)
      output += "%s\n" % line
    TextToFile(output, self.Config(VERSION_FILE))


class CommitTrunk(Step):
329
  MESSAGE = "Commit to local trunk branch."
330 331 332 333 334 335 336 337 338

  def RunStep(self):
    self.Git("add \"%s\"" % self.Config(VERSION_FILE))
    if self.Git("commit -F \"%s\"" % self.Config(COMMITMSG_FILE)) is None:
      self.Die("'git commit' failed.")
    Command("rm", "-f %s*" % self.Config(COMMITMSG_FILE))


class SanityCheck(Step):
339
  MESSAGE = "Sanity check."
340 341 342 343 344 345 346 347 348

  def RunStep(self):
    if not self.Confirm("Please check if your local checkout is sane: Inspect "
        "%s, compile, run tests. Do you want to commit this new trunk "
        "revision to the repository?" % self.Config(VERSION_FILE)):
      self.Die("Execution canceled.")


class CommitSVN(Step):
349
  MESSAGE = "Commit to SVN."
350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365

  def RunStep(self):
    result = self.Git("svn dcommit 2>&1")
    if not result:
      self.Die("'git svn dcommit' failed.")
    result = filter(lambda x: re.search(r"^Committed r[0-9]+", x),
                    result.splitlines())
    if len(result) > 0:
      trunk_revision = re.sub(r"^Committed r([0-9]+)", r"\1", result[0])

    # Sometimes grepping for the revision fails. No idea why. If you figure
    # out why it is flaky, please do fix it properly.
    if not trunk_revision:
      print("Sorry, grepping for the SVN revision failed. Please look for it "
            "in the last command's output above and provide it manually (just "
            "the number, without the leading \"r\").")
366
      self.DieNoManualMode("Can't prompt in forced mode.")
367 368 369 370 371 372 373
      while not trunk_revision:
        print "> ",
        trunk_revision = self.ReadLine()
    self.Persist("trunk_revision", trunk_revision)


class TagRevision(Step):
374
  MESSAGE = "Tag the new revision."
375 376 377 378 379 380 381 382 383 384 385

  def RunStep(self):
    self.RestoreVersionIfUnset()
    ver = "%s.%s.%s" % (self._state["major"],
                        self._state["minor"],
                        self._state["build"])
    if self.Git("svn tag %s -m \"Tagging version %s\"" % (ver, ver)) is None:
      self.Die("'git svn tag' failed.")


class CheckChromium(Step):
386
  MESSAGE = "Ask for chromium checkout."
387 388 389 390

  def Run(self):
    chrome_path = self._options.c
    if not chrome_path:
391
      self.DieNoManualMode("Please specify the path to a Chromium checkout in "
392
                          "forced mode.")
393 394 395 396 397 398 399 400 401
      print ("Do you have a \"NewGit\" Chromium checkout and want "
          "this script to automate creation of the roll CL? If yes, enter the "
          "path to (and including) the \"src\" directory here, otherwise just "
          "press <Return>: "),
      chrome_path = self.ReadLine()
    self.Persist("chrome_path", chrome_path)


class SwitchChromium(Step):
402 403
  MESSAGE = "Switch to Chromium checkout."
  REQUIRES = "chrome_path"
404 405 406 407 408 409 410 411 412 413 414 415 416 417 418

  def RunStep(self):
    v8_path = os.getcwd()
    self.Persist("v8_path", v8_path)
    os.chdir(self._state["chrome_path"])
    self.InitialEnvironmentChecks()
    # Check for a clean workdir.
    if self.Git("status -s -uno").strip() != "":
      self.Die("Workspace is not clean. Please commit or undo your changes.")
    # Assert that the DEPS file is there.
    if not os.path.exists(self.Config(DEPS_FILE)):
      self.Die("DEPS file not present.")


class UpdateChromiumCheckout(Step):
419 420
  MESSAGE = "Update the checkout and create a new branch."
  REQUIRES = "chrome_path"
421 422 423 424 425 426 427 428 429 430 431 432 433 434 435

  def RunStep(self):
    os.chdir(self._state["chrome_path"])
    if self.Git("checkout master") is None:
      self.Die("'git checkout master' failed.")
    if self.Git("pull") is None:
      self.Die("'git pull' failed, please try again.")

    self.RestoreIfUnset("trunk_revision")
    args = "checkout -b v8-roll-%s" % self._state["trunk_revision"]
    if self.Git(args) is None:
      self.Die("Failed to checkout a new branch.")


class UploadCL(Step):
436 437
  MESSAGE = "Create and upload CL."
  REQUIRES = "chrome_path"
438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453

  def RunStep(self):
    os.chdir(self._state["chrome_path"])

    # Patch DEPS file.
    self.RestoreIfUnset("trunk_revision")
    deps = FileToText(self.Config(DEPS_FILE))
    deps = re.sub("(?<=\"v8_revision\": \")([0-9]+)(?=\")",
                  self._state["trunk_revision"],
                  deps)
    TextToFile(deps, self.Config(DEPS_FILE))

    self.RestoreVersionIfUnset()
    ver = "%s.%s.%s" % (self._state["major"],
                        self._state["minor"],
                        self._state["build"])
454
    if self._options.r:
455 456 457 458
      print "Using account %s for review." % self._options.r
      rev = self._options.r
    else:
      print "Please enter the email address of a reviewer for the roll CL: ",
459
      self.DieNoManualMode("A reviewer must be specified in forced mode.")
460
      rev = self.ReadLine()
461 462 463
    args = "commit -am \"Update V8 to version %s.\n\nTBR=%s\"" % (ver, rev)
    if self.Git(args) is None:
      self.Die("'git commit' failed.")
464
    force_flag = " -f" if self._options.force_upload else ""
465
    if self.Git("cl upload --send-mail%s" % force_flag, pipe=False) is None:
466 467 468 469 470
      self.Die("'git cl upload' failed, please try again.")
    print "CL uploaded."


class SwitchV8(Step):
471 472
  MESSAGE = "Returning to V8 checkout."
  REQUIRES = "chrome_path"
473 474 475 476 477 478 479

  def RunStep(self):
    self.RestoreIfUnset("v8_path")
    os.chdir(self._state["v8_path"])


class CleanUp(Step):
480
  MESSAGE = "Done!"
481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504

  def RunStep(self):
    self.RestoreVersionIfUnset()
    ver = "%s.%s.%s" % (self._state["major"],
                        self._state["minor"],
                        self._state["build"])
    self.RestoreIfUnset("trunk_revision")
    self.RestoreIfUnset("chrome_path")

    if self._state["chrome_path"]:
      print("Congratulations, you have successfully created the trunk "
            "revision %s and rolled it into Chromium. Please don't forget to "
            "update the v8rel spreadsheet:" % ver)
    else:
      print("Congratulations, you have successfully created the trunk "
            "revision %s. Please don't forget to roll this new version into "
            "Chromium, and to update the v8rel spreadsheet:" % ver)
    print "%s\ttrunk\t%s" % (ver, self._state["trunk_revision"])

    self.CommonCleanup()
    if self.Config(TRUNKBRANCH) != self._state["current_branch"]:
      self.Git("branch -D %s" % self.Config(TRUNKBRANCH))


505 506 507
def RunPushToTrunk(config,
                   options,
                   side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER):
508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534
  step_classes = [
    Preparation,
    FreshBranch,
    DetectLastPush,
    PrepareChangeLog,
    EditChangeLog,
    IncrementVersion,
    CommitLocal,
    UploadStep,
    CommitRepository,
    StragglerCommits,
    SquashCommits,
    NewBranch,
    ApplyChanges,
    SetVersion,
    CommitTrunk,
    SanityCheck,
    CommitSVN,
    TagRevision,
    CheckChromium,
    SwitchChromium,
    UpdateChromiumCheckout,
    UploadCL,
    SwitchV8,
    CleanUp,
  ]

535
  RunScript(step_classes, config, options, side_effect_handler)
536 537 538 539 540 541 542


def BuildOptions():
  result = optparse.OptionParser()
  result.add_option("-c", "--chromium", dest="c",
                    help=("Specify the path to your Chromium src/ "
                          "directory to automate the V8 roll."))
543 544 545 546 547 548
  result.add_option("-f", "--force", dest="f",
                    help="Don't prompt the user.",
                    default=False, action="store_true")
  result.add_option("-l", "--last-push", dest="l",
                    help=("Manually specify the git commit ID "
                          "of the last push to trunk."))
549 550 551
  result.add_option("-m", "--manual", dest="m",
                    help="Prompt the user at every important step.",
                    default=False, action="store_true")
552 553 554 555 556
  result.add_option("-r", "--reviewer", dest="r",
                    help=("Specify the account name to be used for reviews."))
  result.add_option("-s", "--step", dest="s",
                    help="Specify the step where to start work. Default: 0.",
                    default=0, type="int")
557 558 559 560 561 562 563
  return result


def ProcessOptions(options):
  if options.s < 0:
    print "Bad step number %d" % options.s
    return False
564 565 566 567 568
  if not options.m and not options.r:
    print "A reviewer (-r) is required in (semi-)automatic mode."
    return False
  if options.f and options.m:
    print "Manual and forced mode cannot be combined."
569
    return False
570 571
  if not options.m and not options.c:
    print "A chromium checkout (-c) is required in (semi-)automatic mode."
572
    return False
573 574 575 576 577 578 579 580 581
  return True


def Main():
  parser = BuildOptions()
  (options, args) = parser.parse_args()
  if not ProcessOptions(options):
    parser.print_help()
    return 1
582
  RunPushToTrunk(CONFIG, PushToTrunkOptions(options))
583 584 585

if __name__ == "__main__":
  sys.exit(Main())