common_includes.py 26.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
#!/usr/bin/env python
# Copyright 2013 the V8 project authors. All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#     * Neither the name of Google Inc. nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

29 30 31
# for py2/py3 compatibility
from __future__ import print_function

32
import argparse
33
import datetime
34
import httplib
35
import glob
36
import imp
37
import json
38 39
import os
import re
40
import shutil
41 42
import subprocess
import sys
43
import textwrap
44
import time
45
import urllib
46
import urllib2
47

48
from git_recipes import GitRecipesMixin
49
from git_recipes import GitFailedException
50

51
DAY_IN_SECONDS = 24 * 60 * 60
52
PUSH_MSG_GIT_RE = re.compile(r".* \(based on (?P<git_rev>[a-fA-F0-9]+)\)$")
53
PUSH_MSG_NEW_RE = re.compile(r"^Version \d+\.\d+\.\d+$")
machenbach's avatar
machenbach committed
54
VERSION_FILE = os.path.join("include", "v8-version.h")
55
WATCHLISTS_FILE = "WATCHLISTS"
56
RELEASE_WORKDIR = "/tmp/v8-release-scripts-work-dir/"
57

58
# V8 base directory.
59
V8_BASE = os.path.dirname(
60 61
    os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

62 63 64 65 66 67 68
# Add our copy of depot_tools to the PATH as many scripts use tools from there,
# e.g. git-cl, fetch, git-new-branch etc, and we can not depend on depot_tools
# being in the PATH on the LUCI bots.
path_to_depot_tools = os.path.join(V8_BASE, 'third_party', 'depot_tools')
new_path = path_to_depot_tools + os.pathsep + os.environ.get('PATH')
os.environ['PATH'] = new_path

69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94

def TextToFile(text, file_name):
  with open(file_name, "w") as f:
    f.write(text)


def AppendToFile(text, file_name):
  with open(file_name, "a") as f:
    f.write(text)


def LinesInFile(file_name):
  with open(file_name) as f:
    for line in f:
      yield line


def FileToText(file_name):
  with open(file_name) as f:
    return f.read()


def MSub(rexp, replacement, text):
  return re.sub(rexp, replacement, text, flags=re.MULTILINE)


95 96 97 98 99 100 101 102 103 104
def SortingKey(version):
  """Key for sorting version number strings: '3.11' > '3.2.1.1'"""
  version_keys = map(int, version.split("."))
  # Fill up to full version numbers to normalize comparison.
  while len(version_keys) < 4:  # pragma: no cover
    version_keys.append(0)
  # Fill digits.
  return ".".join(map("{0:04d}".format, version_keys))


105 106
# Some commands don't like the pipe, e.g. calling vi from within the script or
# from subscripts like git cl upload.
107 108
def Command(cmd, args="", prefix="", pipe=True, cwd=None):
  cwd = cwd or os.getcwd()
109
  # TODO(machenbach): Use timeout.
110
  cmd_line = "%s %s %s" % (prefix, cmd, args)
111 112
  print("Command: %s" % cmd_line)
  print("in %s" % cwd)
113
  sys.stdout.flush()
114 115
  try:
    if pipe:
116
      return subprocess.check_output(cmd_line, shell=True, cwd=cwd)
117
    else:
118
      return subprocess.check_call(cmd_line, shell=True, cwd=cwd)
119 120
  except subprocess.CalledProcessError:
    return None
121 122 123
  finally:
    sys.stdout.flush()
    sys.stderr.flush()
124 125


126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
def SanitizeVersionTag(tag):
    version_without_prefix = re.compile(r"^\d+\.\d+\.\d+(?:\.\d+)?$")
    version_with_prefix = re.compile(r"^tags\/\d+\.\d+\.\d+(?:\.\d+)?$")

    if version_without_prefix.match(tag):
      return tag
    elif version_with_prefix.match(tag):
        return tag[len("tags/"):]
    else:
      return None


def NormalizeVersionTags(version_tags):
  normalized_version_tags = []

  # Remove tags/ prefix because of packed refs.
  for current_tag in version_tags:
    version_tag = SanitizeVersionTag(current_tag)
    if version_tag != None:
      normalized_version_tags.append(version_tag)

  return normalized_version_tags


150
# Wrapper for side effects.
151
class SideEffectHandler(object):  # pragma: no cover
152 153 154
  def Call(self, fun, *args, **kwargs):
    return fun(*args, **kwargs)

155 156
  def Command(self, cmd, args="", prefix="", pipe=True, cwd=None):
    return Command(cmd, args, prefix, pipe, cwd=cwd)
157 158 159 160

  def ReadLine(self):
    return sys.stdin.readline().strip()

161
  def ReadURL(self, url, params=None):
162
    # pylint: disable=E1121
163
    url_fh = urllib2.urlopen(url, params, 60)
164 165 166 167 168
    try:
      return url_fh.read()
    finally:
      url_fh.close()

169
  def ReadClusterFuzzAPI(self, api_key, **params):
170
    params["api_key"] = api_key.strip()
171 172 173 174 175 176 177 178 179 180 181 182 183
    params = urllib.urlencode(params)

    headers = {"Content-type": "application/x-www-form-urlencoded"}

    conn = httplib.HTTPSConnection("backend-dot-cluster-fuzz.appspot.com")
    conn.request("POST", "/_api/", params, headers)

    response = conn.getresponse()
    data = response.read()

    try:
      return json.loads(data)
    except:
184 185
      print(data)
      print("ERROR: Could not read response. Is your key valid?")
186 187
      raise

188
  def Sleep(self, seconds):
189 190
    time.sleep(seconds)

191
  def GetUTCStamp(self):
192
    return time.mktime(datetime.datetime.utcnow().timetuple())
193

194 195 196
DEFAULT_SIDE_EFFECT_HANDLER = SideEffectHandler()


197 198 199
class NoRetryException(Exception):
  pass

200

201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216
class VCInterface(object):
  def InjectStep(self, step):
    self.step=step

  def Pull(self):
    raise NotImplementedError()

  def Fetch(self):
    raise NotImplementedError()

  def GetTags(self):
    raise NotImplementedError()

  def GetBranches(self):
    raise NotImplementedError()

217 218 219 220 221 222
  def MasterBranch(self):
    raise NotImplementedError()

  def CandidateBranch(self):
    raise NotImplementedError()

223 224 225 226 227 228 229 230 231 232 233 234
  def RemoteMasterBranch(self):
    raise NotImplementedError()

  def RemoteCandidateBranch(self):
    raise NotImplementedError()

  def RemoteBranch(self, name):
    raise NotImplementedError()

  def CLLand(self):
    raise NotImplementedError()

235 236 237 238 239
  def Tag(self, tag, remote, message):
    """Sets a tag for the current commit.

    Assumptions: The commit already landed and the commit message is unique.
    """
240 241 242
    raise NotImplementedError()


243
class GitInterface(VCInterface):
244 245 246 247 248 249 250 251 252 253
  def Pull(self):
    self.step.GitPull()

  def Fetch(self):
    self.step.Git("fetch")

  def GetTags(self):
     return self.step.Git("tag").strip().splitlines()

  def GetBranches(self):
254
    # Get relevant remote branches, e.g. "branch-heads/3.25".
255
    branches = filter(
256
        lambda s: re.match(r"^branch\-heads/\d+\.\d+$", s),
257
        self.step.GitRemotes())
258 259
    # Remove 'branch-heads/' prefix.
    return map(lambda s: s[13:], branches)
260

261 262 263 264 265 266
  def MasterBranch(self):
    return "master"

  def CandidateBranch(self):
    return "candidates"

267 268 269 270 271 272 273
  def RemoteMasterBranch(self):
    return "origin/master"

  def RemoteCandidateBranch(self):
    return "origin/candidates"

  def RemoteBranch(self, name):
274 275 276 277
    # Assume that if someone "fully qualified" the ref, they know what they
    # want.
    if name.startswith('refs/'):
      return name
278
    if name in ["candidates", "master"]:
279
      return "refs/remotes/origin/%s" % name
280 281 282 283 284 285 286 287 288 289 290 291
    try:
      # Check if branch is in heads.
      if self.step.Git("show-ref refs/remotes/origin/%s" % name).strip():
        return "refs/remotes/origin/%s" % name
    except GitFailedException:
      pass
    try:
      # Check if branch is in branch-heads.
      if self.step.Git("show-ref refs/remotes/branch-heads/%s" % name).strip():
        return "refs/remotes/branch-heads/%s" % name
    except GitFailedException:
      pass
292
    self.Die("Can't find remote of %s" % name)
293

294 295 296 297 298
  def Tag(self, tag, remote, message):
    # Wait for the commit to appear. Assumes unique commit message titles (this
    # is the case for all automated merge and push commits - also no title is
    # the prefix of another title).
    commit = None
299
    for wait_interval in [10, 30, 60, 60, 60, 60, 60]:
300 301 302 303 304 305 306 307 308 309 310 311
      self.step.Git("fetch")
      commit = self.step.GitLog(n=1, format="%H", grep=message, branch=remote)
      if commit:
        break
      print("The commit has not replicated to git. Waiting for %s seconds." %
            wait_interval)
      self.step._side_effect_handler.Sleep(wait_interval)
    else:
      self.step.Die("Couldn't determine commit for setting the tag. Maybe the "
                    "git updater is lagging behind?")

    self.step.Git("tag %s %s" % (tag, commit))
312
    self.step.Git("push origin refs/tags/%s:refs/tags/%s" % (tag, tag))
313 314 315 316

  def CLLand(self):
    self.step.GitCLLand()

317

318
class Step(GitRecipesMixin):
319
  def __init__(self, text, number, config, state, options, handler):
320 321 322 323 324 325
    self._text = text
    self._number = number
    self._config = config
    self._state = state
    self._options = options
    self._side_effect_handler = handler
326
    self.vc = GitInterface()
327
    self.vc.InjectStep(self)
328 329

    # The testing configuration might set a different default cwd.
330 331
    self.default_cwd = (self._config.get("DEFAULT_CWD") or
                        os.path.join(self._options.work_dir, "v8"))
332

333 334 335 336
    assert self._number >= 0
    assert self._config is not None
    assert self._state is not None
    assert self._side_effect_handler is not None
337

338 339 340
  def __getitem__(self, key):
    # Convenience method to allow direct [] access on step classes for
    # manipulating the backed state dict.
341
    return self._state.get(key)
342 343 344 345 346 347

  def __setitem__(self, key, value):
    # Convenience method to allow direct [] access on step classes for
    # manipulating the backed state dict.
    self._state[key] = value

348 349 350 351
  def Config(self, key):
    return self._config[key]

  def Run(self):
352
    # Restore state.
353
    state_file = "%s-state.json" % self._config["PERSISTFILE_BASENAME"]
354 355 356
    if not self._state and os.path.exists(state_file):
      self._state.update(json.loads(FileToText(state_file)))

357
    print(">>> Step %d: %s" % (self._number, self._text))
358 359 360 361 362
    try:
      return self.RunStep()
    finally:
      # Persist state.
      TextToFile(json.dumps(self._state), state_file)
363

364
  def RunStep(self):  # pragma: no cover
365 366
    raise NotImplementedError

367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383
  def Retry(self, cb, retry_on=None, wait_plan=None):
    """ Retry a function.
    Params:
      cb: The function to retry.
      retry_on: A callback that takes the result of the function and returns
                True if the function should be retried. A function throwing an
                exception is always retried.
      wait_plan: A list of waiting delays between retries in seconds. The
                 maximum number of retries is len(wait_plan).
    """
    retry_on = retry_on or (lambda x: False)
    wait_plan = list(wait_plan or [])
    wait_plan.reverse()
    while True:
      got_exception = False
      try:
        result = cb()
384
      except NoRetryException as e:
385
        raise e
386 387
      except Exception as e:
        got_exception = e
388
      if got_exception or retry_on(result):
389
        if not wait_plan:  # pragma: no cover
390 391
          raise Exception("Retried too often. Giving up. Reason: %s" %
                          str(got_exception))
392
        wait_time = wait_plan.pop()
393
        print("Waiting for %f seconds." % wait_time)
394
        self._side_effect_handler.Sleep(wait_time)
395
        print("Retrying...")
396 397 398
      else:
        return result

399 400
  def ReadLine(self, default=None):
    # Don't prompt in forced mode.
401
    if self._options.force_readline_defaults and default is not None:
402
      print("%s (forced)" % default)
403 404 405
      return default
    else:
      return self._side_effect_handler.ReadLine()
406

407 408 409 410 411 412 413 414
  def Command(self, name, args, cwd=None):
    cmd = lambda: self._side_effect_handler.Command(
        name, args, "", True, cwd=cwd or self.default_cwd)
    return self.Retry(cmd, None, [5])

  def Git(self, args="", prefix="", pipe=True, retry_on=None, cwd=None):
    cmd = lambda: self._side_effect_handler.Command(
        "git", args, prefix, pipe, cwd=cwd or self.default_cwd)
415 416 417 418
    result = self.Retry(cmd, retry_on, [5, 30])
    if result is None:
      raise GitFailedException("'git %s' failed." % args)
    return result
419 420

  def Editor(self, args):
421
    if self._options.requires_editor:
422 423 424 425 426
      return self._side_effect_handler.Command(
          os.environ["EDITOR"],
          args,
          pipe=False,
          cwd=self.default_cwd)
427

428
  def ReadURL(self, url, params=None, retry_on=None, wait_plan=None):
429
    wait_plan = wait_plan or [3, 60, 600]
430
    cmd = lambda: self._side_effect_handler.ReadURL(url, params)
431
    return self.Retry(cmd, retry_on, wait_plan)
432

433 434
  def Die(self, msg=""):
    if msg != "":
435 436
      print("Error: %s" % msg)
    print("Exiting")
437 438
    raise Exception(msg)

439
  def DieNoManualMode(self, msg=""):
440
    if not self._options.manual:  # pragma: no cover
441
      msg = msg or "Only available in manual mode."
442 443
      self.Die(msg)

444
  def Confirm(self, msg):
445
    print("%s [Y/n] " % msg, end=' ')
446
    answer = self.ReadLine(default="Y")
447 448
    return answer == "" or answer == "Y" or answer == "y"

449 450
  def DeleteBranch(self, name, cwd=None):
    for line in self.GitBranch(cwd=cwd).splitlines():
451
      if re.match(r"\*?\s*%s$" % re.escape(name), line):
452 453
        msg = "Branch %s exists, do you want to delete it?" % name
        if self.Confirm(msg):
454
          self.GitDeleteBranch(name, cwd=cwd)
455
          print("Branch %s deleted." % name)
456 457 458 459
        else:
          msg = "Can't continue. Please delete branch %s and try again." % name
          self.Die(msg)

460
  def InitialEnvironmentChecks(self, cwd):
461
    # Cancel if this is not a git checkout.
462
    if not os.path.exists(os.path.join(cwd, ".git")):  # pragma: no cover
463 464
      self.Die("%s is not a git checkout. If you know what you're doing, try "
               "deleting it and rerunning this script." % cwd)
465 466

    # Cancel if EDITOR is unset or not executable.
467
    if (self._options.requires_editor and (not os.environ.get("EDITOR") or
468 469
        self.Command(
            "which", os.environ["EDITOR"]) is None)):  # pragma: no cover
470 471 472 473
      self.Die("Please set your EDITOR environment variable, you'll need it.")

  def CommonPrepare(self):
    # Check for a clean workdir.
474
    if not self.GitIsWorkdirClean():  # pragma: no cover
475 476
      self.Die("Workspace is not clean. Please commit or undo your changes.")

477 478
    # Checkout master in case the script was left on a work branch.
    self.GitCheckout('origin/master')
479 480

    # Fetch unfetched revisions.
481
    self.vc.Fetch()
482

483
  def PrepareBranch(self):
484
    # Delete the branch that will be created later if it exists already.
485
    self.DeleteBranch(self._config["BRANCHNAME"])
486 487

  def CommonCleanup(self):
488 489
    self.GitCheckout('origin/master')
    self.GitDeleteBranch(self._config["BRANCHNAME"])
490 491

    # Clean up all temporary files.
492
    for f in glob.iglob("%s*" % self._config["PERSISTFILE_BASENAME"]):
493 494 495 496
      if os.path.isfile(f):
        os.remove(f)
      if os.path.isdir(f):
        shutil.rmtree(f)
497 498 499 500 501 502

  def ReadAndPersistVersion(self, prefix=""):
    def ReadAndPersist(var_name, def_name):
      match = re.match(r"^#define %s\s+(\d*)" % def_name, line)
      if match:
        value = match.group(1)
503
        self["%s%s" % (prefix, var_name)] = value
504
    for line in LinesInFile(os.path.join(self.default_cwd, VERSION_FILE)):
machenbach's avatar
machenbach committed
505 506 507 508
      for (var_name, def_name) in [("major", "V8_MAJOR_VERSION"),
                                   ("minor", "V8_MINOR_VERSION"),
                                   ("build", "V8_BUILD_NUMBER"),
                                   ("patch", "V8_PATCH_LEVEL")]:
509 510 511 512 513 514 515 516 517
        ReadAndPersist(var_name, def_name)

  def WaitForLGTM(self):
    print ("Please wait for an LGTM, then type \"LGTM<Return>\" to commit "
           "your change. (If you need to iterate on the patch or double check "
           "that it's sane, do so in another shell, but remember to not "
           "change the headline of the uploaded CL.")
    answer = ""
    while answer != "LGTM":
518
      print("> ", end=' ')
519
      answer = self.ReadLine(None if self._options.wait_for_lgtm else "LGTM")
520
      if answer != "LGTM":
521
        print("That was not 'LGTM'.")
522 523 524 525

  def WaitForResolvingConflicts(self, patch_file):
    print("Applying the patch \"%s\" failed. Either type \"ABORT<Return>\", "
          "or resolve the conflicts, stage *all* touched files with "
526
          "'git add', and type \"RESOLVED<Return>\"" % (patch_file))
527
    self.DieNoManualMode()
528 529 530 531 532
    answer = ""
    while answer != "RESOLVED":
      if answer == "ABORT":
        self.Die("Applying the patch failed.")
      if answer != "":
533 534
        print("That was not 'RESOLVED' or 'ABORT'.")
      print("> ", end=' ')
535 536 537
      answer = self.ReadLine()

  # Takes a file containing the patch to apply as first argument.
538
  def ApplyPatch(self, patch_file, revert=False):
539
    try:
540
      self.GitApplyPatch(patch_file, revert)
541
    except GitFailedException:
542 543
      self.WaitForResolvingConflicts(patch_file)

544 545
  def GetVersionTag(self, revision):
    tag = self.Git("describe --tags %s" % revision).strip()
546
    return SanitizeVersionTag(tag)
547

548 549 550 551 552 553 554 555 556 557 558 559
  def GetRecentReleases(self, max_age):
    # Make sure tags are fetched.
    self.Git("fetch origin +refs/tags/*:refs/tags/*")

    # Current timestamp.
    time_now = int(self._side_effect_handler.GetUTCStamp())

    # List every tag from a given period.
    revisions = self.Git("rev-list --max-age=%d --tags" %
                         int(time_now - max_age)).strip()

    # Filter out revisions who's tag is off by one or more commits.
560
    return filter(lambda r: self.GetVersionTag(r), revisions.splitlines())
561

562 563 564 565 566 567 568
  def GetLatestVersion(self):
    # Use cached version if available.
    if self["latest_version"]:
      return self["latest_version"]

    # Make sure tags are fetched.
    self.Git("fetch origin +refs/tags/*:refs/tags/*")
569 570 571 572 573

    all_tags = self.vc.GetTags()
    only_version_tags = NormalizeVersionTags(all_tags)

    version = sorted(only_version_tags,
574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589
                     key=SortingKey, reverse=True)[0]
    self["latest_version"] = version
    return version

  def GetLatestRelease(self):
    """The latest release is the git hash of the latest tagged version.

    This revision should be rolled into chromium.
    """
    latest_version = self.GetLatestVersion()

    # The latest release.
    latest_hash = self.GitLog(n=1, format="%H", branch=latest_version)
    assert latest_hash
    return latest_hash

590
  def GetLatestReleaseBase(self, version=None):
591 592 593
    """The latest release base is the latest revision that is covered in the
    last change log file. It doesn't include cherry-picked patches.
    """
594
    latest_version = version or self.GetLatestVersion()
595 596 597 598 599 600 601 602

    # Strip patch level if it exists.
    latest_version = ".".join(latest_version.split(".")[:3])

    # The latest release base.
    latest_hash = self.GitLog(n=1, format="%H", branch=latest_version)
    assert latest_hash

603 604
    title = self.GitLog(n=1, format="%s", git_hash=latest_hash)
    match = PUSH_MSG_GIT_RE.match(title)
605 606 607 608
    if match:
      # Legacy: In the old process there's one level of indirection. The
      # version is on the candidates branch and points to the real release
      # base on master through the commit message.
609 610 611 612 613 614 615 616 617
      return match.group("git_rev")
    match = PUSH_MSG_NEW_RE.match(title)
    if match:
      # This is a new-style v8 version branched from master. The commit
      # "latest_hash" is the version-file change. Its parent is the release
      # base on master.
      return self.GitLog(n=1, format="%H", git_hash="%s^" % latest_hash)

    self.Die("Unknown latest release: %s" % latest_hash)
618

619 620 621 622 623 624
  def ArrayToVersion(self, prefix):
    return ".".join([self[prefix + "major"],
                     self[prefix + "minor"],
                     self[prefix + "build"],
                     self[prefix + "patch"]])

625 626 627 628 629 630 631 632 633 634
  def StoreVersion(self, version, prefix):
    version_parts = version.split(".")
    if len(version_parts) == 3:
      version_parts.append("0")
    major, minor, build, patch = version_parts
    self[prefix + "major"] = major
    self[prefix + "minor"] = minor
    self[prefix + "build"] = build
    self[prefix + "patch"] = patch

635 636 637
  def SetVersion(self, version_file, prefix):
    output = ""
    for line in FileToText(version_file).splitlines():
machenbach's avatar
machenbach committed
638
      if line.startswith("#define V8_MAJOR_VERSION"):
639
        line = re.sub("\d+$", self[prefix + "major"], line)
machenbach's avatar
machenbach committed
640
      elif line.startswith("#define V8_MINOR_VERSION"):
641
        line = re.sub("\d+$", self[prefix + "minor"], line)
machenbach's avatar
machenbach committed
642
      elif line.startswith("#define V8_BUILD_NUMBER"):
643
        line = re.sub("\d+$", self[prefix + "build"], line)
machenbach's avatar
machenbach committed
644
      elif line.startswith("#define V8_PATCH_LEVEL"):
645
        line = re.sub("\d+$", self[prefix + "patch"], line)
646
      elif (self[prefix + "candidate"] and
machenbach's avatar
machenbach committed
647
            line.startswith("#define V8_IS_CANDIDATE_VERSION")):
648
        line = re.sub("\d+$", self[prefix + "candidate"], line)
649 650
      output += "%s\n" % line
    TextToFile(output, version_file)
651

652

653
class BootstrapStep(Step):
654
  MESSAGE = "Bootstrapping checkout and state."
655 656

  def RunStep(self):
657 658 659
    # Reserve state entry for json output.
    self['json_output'] = {}

660 661 662 663 664 665 666 667 668
    if os.path.realpath(self.default_cwd) == os.path.realpath(V8_BASE):
      self.Die("Can't use v8 checkout with calling script as work checkout.")
    # Directory containing the working v8 checkout.
    if not os.path.exists(self._options.work_dir):
      os.makedirs(self._options.work_dir)
    if not os.path.exists(self.default_cwd):
      self.Command("fetch", "v8", cwd=self._options.work_dir)


669
class UploadStep(Step):
670
  MESSAGE = "Upload for code review."
671 672

  def RunStep(self):
673
    reviewer = None
674
    if self._options.reviewer:
675
      print("Using account %s for review." % self._options.reviewer)
676
      reviewer = self._options.reviewer
677 678 679

    tbr_reviewer = None
    if self._options.tbr_reviewer:
680
      print("Using account %s for TBR review." % self._options.tbr_reviewer)
681 682 683
      tbr_reviewer = self._options.tbr_reviewer

    if not reviewer and not tbr_reviewer:
684 685 686
      print(
        "Please enter the email address of a V8 reviewer for your patch: ",
        end=' ')
687
      self.DieNoManualMode("A reviewer must be specified in forced mode.")
688
      reviewer = self.ReadLine()
689

690
    self.GitUpload(reviewer, self._options.force_upload,
691
                   bypass_hooks=self._options.bypass_upload_hooks,
692
                   cc=self._options.cc, tbr_reviewer=tbr_reviewer)
693 694


695 696 697 698 699 700 701 702 703 704 705
def MakeStep(step_class=Step, number=0, state=None, config=None,
             options=None, side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER):
    # Allow to pass in empty dictionaries.
    state = state if state is not None else {}
    config = config if config is not None else {}

    try:
      message = step_class.MESSAGE
    except AttributeError:
      message = step_class.__name__

706
    return step_class(message, number=number, config=config,
707 708 709 710
                      state=state, options=options,
                      handler=side_effect_handler)


711
class ScriptsBase(object):
712 713 714
  def __init__(self,
               config=None,
               side_effect_handler=DEFAULT_SIDE_EFFECT_HANDLER,
715
               state=None):
716
    self._config = config or self._Config()
717 718 719 720 721 722 723 724 725 726 727 728
    self._side_effect_handler = side_effect_handler
    self._state = state if state is not None else {}

  def _Description(self):
    return None

  def _PrepareOptions(self, parser):
    pass

  def _ProcessOptions(self, options):
    return True

729
  def _Steps(self):  # pragma: no cover
730 731
    raise Exception("Not implemented.")

732 733 734
  def _Config(self):
    return {}

735 736 737
  def MakeOptions(self, args=None):
    parser = argparse.ArgumentParser(description=self._Description())
    parser.add_argument("-a", "--author", default="",
738
                        help="The author email used for code review.")
739 740
    parser.add_argument("--dry-run", default=False, action="store_true",
                        help="Perform only read-only actions.")
741 742
    parser.add_argument("--json-output",
                        help="File to write results summary to.")
743 744
    parser.add_argument("-r", "--reviewer", default="",
                        help="The account name to be used for reviews.")
745 746
    parser.add_argument("--tbr-reviewer", "--tbr", default="",
                        help="The account name to be used for TBR reviews.")
747
    parser.add_argument("-s", "--step",
748 749
        help="Specify the step where to start work. Default: 0.",
        default=0, type=int)
750 751 752
    parser.add_argument("--work-dir",
                        help=("Location where to bootstrap a working v8 "
                              "checkout."))
753 754
    self._PrepareOptions(parser)

755
    if args is None:  # pragma: no cover
756 757 758 759 760
      options = parser.parse_args()
    else:
      options = parser.parse_args(args)

    # Process common options.
761
    if options.step < 0:  # pragma: no cover
762
      print("Bad step number %d" % options.step)
763 764 765 766 767 768
      parser.print_help()
      return None

    # Defaults for options, common to all scripts.
    options.manual = getattr(options, "manual", True)
    options.force = getattr(options, "force", False)
769
    options.bypass_upload_hooks = False
770 771 772 773 774 775 776 777 778 779 780

    # Derived options.
    options.requires_editor = not options.force
    options.wait_for_lgtm = not options.force
    options.force_readline_defaults = not options.manual
    options.force_upload = not options.manual

    # Process script specific options.
    if not self._ProcessOptions(options):
      parser.print_help()
      return None
781

782 783
    if not options.work_dir:
      options.work_dir = "/tmp/v8-release-scripts-work-dir"
784 785 786 787 788 789 790
    return options

  def RunSteps(self, step_classes, args=None):
    options = self.MakeOptions(args)
    if not options:
      return 1

791 792 793 794 795
    # Ensure temp dir exists for state files.
    state_dir = os.path.dirname(self._config["PERSISTFILE_BASENAME"])
    if not os.path.exists(state_dir):
      os.makedirs(state_dir)

796
    state_file = "%s-state.json" % self._config["PERSISTFILE_BASENAME"]
797 798 799 800
    if options.step == 0 and os.path.exists(state_file):
      os.remove(state_file)

    steps = []
801
    for (number, step_class) in enumerate([BootstrapStep] + step_classes):
802 803
      steps.append(MakeStep(step_class, number, self._state, self._config,
                            options, self._side_effect_handler))
804

805 806
    try:
      for step in steps[options.step:]:
807
        if step.Run():
808 809 810 811
          return 0
    finally:
      if options.json_output:
        with open(options.json_output, "w") as f:
812
          json.dump(self._state['json_output'], f)
813

814 815 816 817
    return 0

  def Run(self, args=None):
    return self.RunSteps(self._Steps(), args)