drover.py 18.9 KB
Newer Older
1
#!/usr/bin/env python
2
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 4 5
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

6 7 8
import optparse
import os
import re
9
import string
10
import sys
11
import urllib2
12

13
import breakpad  # pylint: disable=W0611
14

15
import gclient_utils
16
import subprocess2
17

18 19 20 21 22
USAGE = """
WARNING: Please use this tool in an empty directory
(or at least one that you don't mind clobbering.)

REQUIRES: SVN 1.5+
23
NOTE: NO NEED TO CHECKOUT ANYTHING IN ADVANCE OF USING THIS TOOL.
24 25 26
Valid parameters:

[Merge from trunk to branch]
27 28
--merge <revision> --branch <branch_num>
Example: %(app)s --merge 12345 --branch 187
29

30 31
[Merge from trunk to milestone]
--merge <revision> --milestone <milestone_num>
32
Example: %(app)s --merge 12345 --milestone 16
33

34 35 36 37
[Merge from trunk to local copy]
--merge <revision> --local
Example: %(app)s --merge 12345 --local

38
[Merge from branch to branch]
39
--merge <revision> --sbranch <branch_num> --branch <branch_num>
40 41
Example: %(app)s --merge 12345 --sbranch 248 --branch 249

42
[Revert from trunk]
43 44
--revert <revision>
Example: %(app)s --revert 12345
45 46

[Revert from branch]
47 48
--revert <revision> --branch <branch_num>
Example: %(app)s --revert 12345 --branch 187
49 50
"""

51 52 53
export_map_ = None
files_info_ = None
delete_map_ = None
54
file_pattern_ =  r"[ ]+([MADUC])[ ]+/((?:trunk|branches/.*?)/src(.*)/(.*))"
55
depot_tools_dir_ = os.path.dirname(os.path.abspath(__file__))
56

57

58
def runGcl(subcommand):
59
  gcl_path = os.path.join(depot_tools_dir_, "gcl")
60 61 62 63 64 65 66
  if not os.path.exists(gcl_path):
    print "WARNING: gcl not found beside drover.py. Using system gcl instead..."
    gcl_path = 'gcl'

  command = "%s %s" % (gcl_path, subcommand)
  return os.system(command)

67
def gclUpload(revision, author):
68
  command = ("upload " + str(revision) +
69
             " --send_mail --no_presubmit --reviewers=" + author)
70
  return runGcl(command)
71

72
def getSVNInfo(url, revision):
73
  info = {}
74 75 76 77 78 79 80 81 82
  try:
    svn_info = subprocess2.check_output(
        ['svn', 'info', '%s@%s' % (url, revision)]).splitlines()
    for line in svn_info:
      match = re.search(r"(.*?):(.*)", line)
      if match:
        info[match.group(1).strip()] = match.group(2).strip()
  except subprocess2.CalledProcessError:
    pass
83
  return info
84

85
def isSVNDirty():
86
  svn_status = subprocess2.check_output(['svn', 'status']).splitlines()
87 88 89 90 91 92 93
  for line in svn_status:
    match = re.search(r"^[^X?]", line)
    if match:
      return True

  return False

94 95 96 97 98
def getAuthor(url, revision):
  info = getSVNInfo(url, revision)
  if (info.has_key("Last Changed Author")):
    return info["Last Changed Author"]
  return None
99

100 101 102
def isSVNFile(url, revision):
  info = getSVNInfo(url, revision)
  if (info.has_key("Node Kind")):
103 104 105
    if (info["Node Kind"] == "file"):
      return True
  return False
106 107 108 109

def isSVNDirectory(url, revision):
  info = getSVNInfo(url, revision)
  if (info.has_key("Node Kind")):
110 111 112 113
    if (info["Node Kind"] == "directory"):
      return True
  return False

114 115 116 117
def inCheckoutRoot(path):
  info = getSVNInfo(path, "HEAD")
  if (not info.has_key("Repository Root")):
    return False
118
  repo_root = info["Repository Root"]
119 120 121 122 123
  info = getSVNInfo(os.path.dirname(os.path.abspath(path)), "HEAD")
  if (info.get("Repository Root", None) != repo_root):
    return True
  return False

124
def getRevisionLog(url, revision):
125
  """Takes an svn url and gets the associated revision."""
126
  svn_log = subprocess2.check_output(
127 128 129 130
      ['svn', 'log', url, '-r', str(revision)],
      universal_newlines=True).splitlines(True)
  # Don't include the header lines and the trailing "---..." line.
  return ''.join(svn_log[3:-1])
131

132 133
def getSVNVersionInfo():
  """Extract version information from SVN"""
134
  svn_info = subprocess2.check_output(['svn', '--version']).splitlines()
135 136
  info = {}
  for line in svn_info:
137
    match = re.search(r"svn, version ((\d+)\.(\d+)\.(\d+))", line)
138
    if match:
139 140 141 142 143
      info['version'] = match.group(1)
      info['major'] = int(match.group(2))
      info['minor'] = int(match.group(3))
      info['patch'] = int(match.group(4))
      return info
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170

  return None

def isMinimumSVNVersion(major, minor, patch=0):
  """Test for minimum SVN version"""
  return _isMinimumSVNVersion(getSVNVersionInfo(), major, minor, patch)

def _isMinimumSVNVersion(version, major, minor, patch=0):
  """Test for minimum SVN version, internal method"""
  if not version:
    return False

  if (version['major'] > major):
    return True
  elif (version['major'] < major):
    return False

  if (version['minor'] > minor):
    return True
  elif (version['minor'] < minor):
    return False

  if (version['patch'] >= patch):
    return True
  else:
    return False

171
def checkoutRevision(url, revision, branch_url, revert=False):
172
  files_info = getFileInfo(url, revision)
173 174
  paths = getBestMergePaths2(files_info, revision)
  export_map = getBestExportPathsMap2(files_info, revision)
175

176 177 178
  command = 'svn checkout -N ' + branch_url
  print command
  os.system(command)
179

180
  match = re.search(r"^[a-z]+://.*/(.*)", branch_url)
181

182 183
  if match:
    os.chdir(match.group(1))
184

185 186 187
  # This line is extremely important due to the way svn behaves in the
  # set-depths action.  If parents aren't handled before children, the child
  # directories get clobbered and the merge step fails.
188
  paths.sort()
189

190
  # Checkout the directories that already exist
191
  for path in paths:
192 193 194 195 196 197 198 199 200 201 202 203 204
    if (export_map.has_key(path) and not revert):
      print "Exclude new directory " + path
      continue
    subpaths = path.split('/')
    subpaths.pop(0)
    base = ''
    for subpath in subpaths:
      base += '/' + subpath
      # This logic ensures that you don't empty out any directories
      if not os.path.exists("." + base):
        command = ('svn update --depth empty ' + "." + base)
        print command
        os.system(command)
205

206 207 208 209
  if (revert):
    files = getAllFilesInRevision(files_info)
  else:
    files = getExistingFilesInRevision(files_info)
210 211 212 213

  for f in files:
   # Prevent the tool from clobbering the src directory
    if (f == ""):
214
      continue
215
    command = ('svn up ".' + f + '"')
216 217
    print command
    os.system(command)
218

219
def mergeRevision(url, revision):
220
  paths = getBestMergePaths(url, revision)
221
  export_map = getBestExportPathsMap(url, revision)
222

223
  for path in paths:
224 225
    if export_map.has_key(path):
      continue
226
    command = ('svn merge -N -r ' + str(revision-1) + ":" + str(revision) + " ")
227
    command += " --ignore-ancestry "
228
    command += " -x --ignore-eol-style "
229
    command += url + path + "@" + str(revision) + " ." + path
230

231 232 233
    print command
    os.system(command)

234 235 236
def exportRevision(url, revision):
  paths = getBestExportPathsMap(url, revision).keys()
  paths.sort()
237

238
  for path in paths:
239 240
    command = ('svn export -N ' + url + path + "@" + str(revision) + " ."  +
               path)
241 242
    print command
    os.system(command)
243 244

    command = 'svn add .' + path
245
    print command
246
    os.system(command)
247

248 249 250 251
def deleteRevision(url, revision):
  paths = getBestDeletePathsMap(url, revision).keys()
  paths.sort()
  paths.reverse()
252

253
  for path in paths:
254
    command = "svn delete ." + path
255 256
    print command
    os.system(command)
257

258 259 260 261
def revertExportRevision(url, revision):
  paths = getBestExportPathsMap(url, revision).keys()
  paths.sort()
  paths.reverse()
262

263
  for path in paths:
264
    command = "svn delete ." + path
265 266
    print command
    os.system(command)
267

268 269 270
def revertRevision(url, revision):
  paths = getBestMergePaths(url, revision)
  for path in paths:
271
    command = ('svn merge -N -r ' + str(revision) + ":" + str(revision-1) +
272
                " " + url + path + " ." + path)
273 274
    print command
    os.system(command)
275

276
def getFileInfo(url, revision):
277
  global files_info_
278

279 280 281
  if (files_info_ != None):
    return files_info_

282 283
  svn_log = subprocess2.check_output(
      ['svn', 'log', url, '-r', str(revision), '-v']).splitlines()
284 285

  info = []
286
  for line in svn_log:
287 288
    # A workaround to dump the (from .*) stuff, regex not so friendly in the 2nd
    # pass...
289
    match = re.search(r"(.*) \(from.*\)", line)
290
    if match:
291 292 293
      line = match.group(1)
    match = re.search(file_pattern_, line)
    if match:
294 295
      info.append([match.group(1).strip(), match.group(2).strip(),
                   match.group(3).strip(),match.group(4).strip()])
296

297
  files_info_ = info
298
  return info
299

300
def getBestMergePaths(url, revision):
301
  """Takes an svn url and gets the associated revision."""
302
  return getBestMergePaths2(getFileInfo(url, revision), revision)
303

304
def getBestMergePaths2(files_info, revision):
305
  """Takes an svn url and gets the associated revision."""
306
  return list(set([f[2] for f in files_info]))
307 308 309

def getBestExportPathsMap(url, revision):
  return getBestExportPathsMap2(getFileInfo(url, revision), revision)
310

311
def getBestExportPathsMap2(files_info, revision):
312
  """Takes an svn url and gets the associated revision."""
313
  global export_map_
314

315 316 317
  if export_map_:
    return export_map_

318
  result = {}
319 320
  for file_info in files_info:
    if (file_info[0] == "A"):
321 322
      if(isSVNDirectory("svn://svn.chromium.org/chrome/" + file_info[1],
                        revision)):
323
        result[file_info[2] + "/" + file_info[3]] = ""
324

325 326
  export_map_ = result
  return result
327 328

def getBestDeletePathsMap(url, revision):
329
  return getBestDeletePathsMap2(getFileInfo(url, revision), revision)
330 331

def getBestDeletePathsMap2(files_info, revision):
332
  """Takes an svn url and gets the associated revision."""
333
  global delete_map_
334

335 336 337
  if delete_map_:
    return delete_map_

338
  result = {}
339 340
  for file_info in files_info:
    if (file_info[0] == "D"):
341 342
      if(isSVNDirectory("svn://svn.chromium.org/chrome/" + file_info[1],
                        revision)):
343 344 345 346
        result[file_info[2] + "/" + file_info[3]] = ""

  delete_map_ = result
  return result
347

348

349
def getExistingFilesInRevision(files_info):
350
  """Checks for existing files in the revision.
351

352 353
  Anything that's A will require special treatment (either a merge or an
  export + add)
354
  """
355
  return ['%s/%s' % (f[2], f[3]) for f in files_info if f[0] != 'A']
356 357 358


def getAllFilesInRevision(files_info):
359 360 361 362
  """Checks for existing files in the revision.

  Anything that's A will require special treatment (either a merge or an
  export + add)
363
  """
364
  return ['%s/%s' % (f[2], f[3]) for f in files_info]
365

366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406

def getBranchForMilestone(milestone):
  """Queries omahaproxy.appspot.com for the branch number given |milestone|.
  """
  OMAHA_PROXY_URL = "http://omahaproxy.appspot.com"
  request = urllib2.Request(OMAHA_PROXY_URL)
  try:
    response = urllib2.urlopen(request)
  except urllib2.HTTPError, e:
    print "Failed to query %s: %d" % (OMAHA_PROXY_URL, e.code)
    return None

  # Dictionary of [branch: major]. When searching for the appropriate branch
  # matching |milestone|, all major versions that match are added to the
  # dictionary. If all of the branches are the same, this branch value is
  # returned; otherwise, the user is prompted to accept the largest branch
  # value.
  branch_dict = {}

  # Slice the first line since it's column information text.
  for line in response.readlines()[1:]:
    # Version data is CSV.
    parameters = string.split(line, ',')

    # Version is the third parameter and consists of a quad of numbers separated
    # by periods.
    version = string.split(parameters[2], '.')
    major = int(version[0], 10)
    if major != milestone:
      continue

    # Branch number is the third value in the quad.
    branch_dict[version[2]] = major

  if not branch_dict:
    # |milestone| not found.
    print "Milestone provided is invalid"
    return None

  # The following returns a sorted list of the keys of |branch_dict|.
  sorted_branches = sorted(branch_dict)
407
  branch = sorted_branches[-1]
408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423

  # If all keys match, the branch is the same for all platforms given
  # |milestone|. This is the safe case, so return the branch.
  if len(sorted_branches) == 1:
    return branch

  # Not all of the platforms have the same branch. Prompt the user and return
  # the greatest (by value) branch on success.
  if prompt("Not all platforms have the same branch number, "
            "continue with branch %s?" % branch):
    return branch

  # User cancelled.
  return None


424
def prompt(question):
425
  while True:
426
    print question + " [y|n]:",
427 428
    answer = sys.stdin.readline()
    if answer.lower().startswith('n'):
429
      return False
430
    elif answer.lower().startswith('y'):
431
      return True
432

433 434 435

def text_prompt(question, default):
  print question + " [" + default + "]:"
436 437
  answer = sys.stdin.readline()
  if answer.strip() == "":
438
    return default
439 440
  return answer

441 442

def drover(options, args):
443 444 445 446
  revision = options.revert or options.merge

  # Initialize some variables used below. They can be overwritten by
  # the drover.properties file.
447
  BASE_URL = "svn://svn.chromium.org/chrome"
448
  TRUNK_URL = BASE_URL + "/trunk/src"
449 450 451
  BRANCH_URL = BASE_URL + "/branches/$branch/src"
  SKIP_CHECK_WORKING = True
  PROMPT_FOR_AUTHOR = False
452

453 454 455 456 457 458
  # Translate a given milestone to the appropriate branch number.
  if options.milestone:
    options.branch = getBranchForMilestone(options.milestone)
    if not options.branch:
      return 1

459 460 461 462
  DEFAULT_WORKING = "drover_" + str(revision)
  if options.branch:
    DEFAULT_WORKING += ("_" + options.branch)

463
  if not isMinimumSVNVersion(1, 5):
464
    print "You need to use at least SVN version 1.5.x"
465
    return 1
466

467
  # Override the default properties if there is a drover.properties file.
468 469
  global file_pattern_
  if os.path.exists("drover.properties"):
470
    FILE_PATTERN = file_pattern_
471 472 473
    f = open("drover.properties")
    exec(f)
    f.close()
474 475
    if FILE_PATTERN:
      file_pattern_ = FILE_PATTERN
476

477
  if options.revert and options.branch:
478 479 480
    url = BRANCH_URL.replace("$branch", options.branch)
  elif options.merge and options.sbranch:
    url = BRANCH_URL.replace("$branch", options.sbranch)
481 482
  else:
    url = TRUNK_URL
483

484
  working = options.workdir or DEFAULT_WORKING
485

486 487 488 489
  if options.local:
    working = os.getcwd()
    if not inCheckoutRoot(working):
      print "'%s' appears not to be the root of a working copy" % working
490
      return 1
491 492
    if (isSVNDirty() and not
        prompt("Working copy contains uncommitted files. Continue?")):
493
      return 1
494

495 496
  command = 'svn log ' + url + " -r "+str(revision) + " -v"
  os.system(command)
497

498
  if not (options.revertbot or prompt("Is this the correct revision?")):
499
    return 0
500

501
  if (os.path.exists(working)) and not options.local:
502 503
    if not (options.revertbot or SKIP_CHECK_WORKING or
        prompt("Working directory: '%s' already exists, clobber?" % working)):
504
      return 0
505
    gclient_utils.rmtree(working)
506

507 508 509
  if not options.local:
    os.makedirs(working)
    os.chdir(working)
510

511 512
  if options.merge:
    action = "Merge"
513 514 515 516
    if not options.local:
      branch_url = BRANCH_URL.replace("$branch", options.branch)
      # Checkout everything but stuff that got added into a new dir
      checkoutRevision(url, revision, branch_url)
517 518 519 520 521 522 523 524 525 526 527 528 529 530
    # Merge everything that changed
    mergeRevision(url, revision)
    # "Export" files that were added from the source and add them to branch
    exportRevision(url, revision)
    # Delete directories that were deleted (file deletes are handled in the
    # merge).
    deleteRevision(url, revision)
  elif options.revert:
    action = "Revert"
    if options.branch:
      url = BRANCH_URL.replace("$branch", options.branch)
    checkoutRevision(url, revision, url, True)
    revertRevision(url, revision)
    revertExportRevision(url, revision)
531 532

  # Check the base url so we actually find the author who made the change
533 534 535 536 537 538
  if options.auditor:
    author = options.auditor
  else:
    author = getAuthor(url, revision)
    if not author:
      author = getAuthor(TRUNK_URL, revision)
539

540 541 542 543 544
  filename = str(revision)+".txt"
  out = open(filename,"w")
  out.write(action +" " + str(revision) + " - ")
  out.write(getRevisionLog(url, revision))
  if (author):
545
    out.write("\nTBR=" + author)
546
  out.close()
547

548
  change_cmd = 'change ' + str(revision) + " " + filename
549
  if options.revertbot:
550 551 552 553
    if sys.platform == 'win32':
      os.environ['SVN_EDITOR'] = 'cmd.exe /c exit'
    else:
      os.environ['SVN_EDITOR'] = 'true'
554
  runGcl(change_cmd)
555
  os.unlink(filename)
556 557

  if options.local:
558
    return 0
559

560 561
  print author
  print revision
562
  print ("gcl upload " + str(revision) +
563
         " --send_mail --no_presubmit --reviewers=" + author)
564

565
  if options.revertbot or prompt("Would you like to upload?"):
566
    if PROMPT_FOR_AUTHOR:
567 568 569 570 571
      author = text_prompt("Enter new author or press enter to accept default",
                           author)
    if options.revertbot and options.revertbot_reviewers:
      author += ","
      author += options.revertbot_reviewers
572 573 574
    gclUpload(revision, author)
  else:
    print "Deleting the changelist."
575
    print "gcl delete " + str(revision)
576
    runGcl("delete " + str(revision))
577
    return 0
578

579 580 581 582 583
  # We commit if the reverbot is set to commit automatically, or if this is
  # not the revertbot and the user agrees.
  if options.revertbot_commit or (not options.revertbot and
                                  prompt("Would you like to commit?")):
    print "gcl commit " + str(revision) + " --no_presubmit --force"
584
    return runGcl("commit " + str(revision) + " --no_presubmit --force")
585
  else:
586
    return 0
587

588 589

def main():
590 591 592 593 594
  option_parser = optparse.OptionParser(usage=USAGE % {"app": sys.argv[0]})
  option_parser.add_option('-m', '--merge', type="int",
                           help='Revision to merge from trunk to branch')
  option_parser.add_option('-b', '--branch',
                           help='Branch to revert or merge from')
595 596
  option_parser.add_option('-M', '--milestone', type="int",
                           help='Milestone to revert or merge from')
597 598
  option_parser.add_option('-l', '--local', action='store_true',
                           help='Local working copy to merge to')
599 600
  option_parser.add_option('-s', '--sbranch',
                           help='Source branch for merge')
601 602
  option_parser.add_option('-r', '--revert', type="int",
                           help='Revision to revert')
603 604
  option_parser.add_option('-w', '--workdir',
                           help='subdir to use for the revert')
605
  option_parser.add_option('-a', '--auditor',
606
                           help='overrides the author for reviewer')
607 608 609 610 611 612 613 614 615
  option_parser.add_option('', '--revertbot', action='store_true',
                           default=False)
  option_parser.add_option('', '--revertbot-commit', action='store_true',
                           default=False)
  option_parser.add_option('', '--revertbot-reviewers')
  options, args = option_parser.parse_args()

  if not options.merge and not options.revert:
    option_parser.error("You need at least --merge or --revert")
616
    return 1
617

618 619 620 621 622 623 624 625 626
  if options.merge and not (options.branch or options.milestone or
                            options.local):
    option_parser.error("--merge requires either --branch "
                        "or --milestone or --local")
    return 1

  if options.local and (options.revert or options.branch or options.milestone):
    option_parser.error("--local cannot be used with --revert "
                        "or --branch or --milestone")
627
    return 1
628

629 630
  if options.branch and options.milestone:
    option_parser.error("--branch cannot be used with --milestone")
631 632 633
    return 1

  return drover(options, args)
634

635 636 637

if __name__ == "__main__":
  sys.exit(main())