Skip to content

C
Chromium_Depot_tools
Commit 92bec4f5 authored by agable's avatar agable Committed by Commit bot
Browse files
Options

Delete gcl, drover, and trychange 

These tools are relatively standalone, and only ever worked for SVN.
Removing these is a good start to removing other SVN support code.

R=maruel@chromium.org
BUG=475321

Review-Url: https://codereview.chromium.org/2269413002
parent 12fa6ff6
Hide whitespace changes
Inline Side-by-side
Showing with 7 additions and 3684 deletions
codereview.settings
View file @ 92bec4f5
# This file is used by gcl to get repository specific information.
# This file is used by git cl to get repository specific information.
CODE_REVIEW_SERVER: codereview.chromium.org
CC_LIST: chromium-reviews@chromium.org
VIEW_VC: https://chromium.googlesource.com/chromium/tools/depot_tools/+/
......
drover deleted 100755 → 0
View file @ 12fa6ff6
#!/usr/bin/env bash
# Copyright (c) 2009 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# This script will try to sync the bootstrap directories and then defer control.
base_dir=$(dirname "$0")
# Use the batch file as an entry point if on cygwin.
if [ "${OSTYPE}" = "cygwin" -a "${TERM}" != "xterm" ]; then
${base_dir}/drover.bat "$@"
exit
fi
# We're on POSIX (not cygwin). We can now safely look for svn checkout.
if [ "X$DEPOT_TOOLS_UPDATE" != "X0" -a -e "${base_dir}/.svn" ]
then
# Update the bootstrap directory to stay up-to-date with the latest
# depot_tools.
svn -q up "${base_dir}"
fi
PYTHONDONTWRITEBYTECODE=1 exec python "${base_dir}/drover.py" "$@"
drover.bat deleted 100644 → 0
View file @ 12fa6ff6
@echo off
:: Copyright (c) 2009 The Chromium Authors. All rights reserved.
:: Use of this source code is governed by a BSD-style license that can be
:: found in the LICENSE file.
setlocal
set PATH=%~dp0svn;%PATH%
set PYTHONDONTWRITEBYTECODE=1
call python "%~dp0drover.py" %*
drover.py deleted 100755 → 0
View file @ 12fa6ff6
#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import datetime
import optparse
import os
import re
import sys
import urlparse
import gclient_utils
import subprocess2
USAGE = """
WARNING: Please use this tool in an empty directory
(or at least one that you don't mind clobbering.)
REQUIRES: SVN 1.5+
NOTE: NO NEED TO CHECKOUT ANYTHING IN ADVANCE OF USING THIS TOOL.
Valid parameters:
[Merge from trunk to branch]
--merge <revision> --branch <branch_num>
Example: %(app)s --merge 12345 --branch 187
[Merge from trunk to local copy]
--merge <revision> --local
Example: %(app)s --merge 12345 --local
[Merge from branch to branch]
--merge <revision> --sbranch <branch_num> --branch <branch_num>
Example: %(app)s --merge 12345 --sbranch 248 --branch 249
[Revert from trunk]
--revert <revision>
Example: %(app)s --revert 12345
[Revert from branch]
--revert <revision> --branch <branch_num>
Example: %(app)s --revert 12345 --branch 187
"""
export_map_ = None
files_info_ = None
delete_map_ = None
file_pattern_ = r"[ ]+([MADUC])[ ]+/((?:trunk|branches/.*?)/src(.*)/(.*))"
depot_tools_dir_ = os.path.dirname(os.path.abspath(__file__))
def runGcl(subcommand):
gcl_path = os.path.join(depot_tools_dir_, "gcl")
if not os.path.exists(gcl_path):
print "WARNING: gcl not found beside drover.py. Using system gcl instead..."
gcl_path = 'gcl'
command = "%s %s" % (gcl_path, subcommand)
return os.system(command)
def gclUpload(revision, author):
command = ("upload " + str(revision) +
" --send_mail --no_presubmit --reviewers=" + author)
return runGcl(command)
def getSVNInfo(url, revision):
info = {}
svn_info = subprocess2.capture(
['svn', 'info', '--non-interactive', '%s@%s' % (url, revision)],
stderr=subprocess2.VOID).splitlines()
for line in svn_info:
match = re.search(r"(.*?):(.*)", line)
if match:
info[match.group(1).strip()] = match.group(2).strip()
return info
def isSVNDirty():
svn_status = subprocess2.check_output(['svn', 'status']).splitlines()
for line in svn_status:
match = re.search(r"^[^X?]", line)
if match:
return True
return False
def getAuthor(url, revision):
info = getSVNInfo(url, revision)
if (info.has_key("Last Changed Author")):
return info["Last Changed Author"]
return None
def isSVNFile(url, revision):
info = getSVNInfo(url, revision)
if (info.has_key("Node Kind")):
if (info["Node Kind"] == "file"):
return True
return False
def isSVNDirectory(url, revision):
info = getSVNInfo(url, revision)
if (info.has_key("Node Kind")):
if (info["Node Kind"] == "directory"):
return True
return False
def inCheckoutRoot(path):
info = getSVNInfo(path, "HEAD")
if (not info.has_key("Repository Root")):
return False
repo_root = info["Repository Root"]
info = getSVNInfo(os.path.dirname(os.path.abspath(path)), "HEAD")
if (info.get("Repository Root", None) != repo_root):
return True
return False
def getRevisionLog(url, revision):
"""Takes an svn url and gets the associated revision."""
svn_log = subprocess2.check_output(
['svn', 'log', url, '-r', str(revision)],
universal_newlines=True).splitlines(True)
# Don't include the header lines and the trailing "---..." line.
return ''.join(svn_log[3:-1])
def getSVNVersionInfo():
"""Extract version information from SVN"""
svn_info = subprocess2.check_output(['svn', '--version']).splitlines()
info = {}
for line in svn_info:
match = re.search(r"svn, version ((\d+)\.(\d+)\.(\d+))", line)
if match:
info['version'] = match.group(1)
info['major'] = int(match.group(2))
info['minor'] = int(match.group(3))
info['patch'] = int(match.group(4))
return info
return None
def isMinimumSVNVersion(major, minor, patch=0):
"""Test for minimum SVN version"""
return _isMinimumSVNVersion(getSVNVersionInfo(), major, minor, patch)
def _isMinimumSVNVersion(version, major, minor, patch=0):
"""Test for minimum SVN version, internal method"""
if not version:
return False
if (version['major'] > major):
return True
elif (version['major'] < major):
return False
if (version['minor'] > minor):
return True
elif (version['minor'] < minor):
return False
if (version['patch'] >= patch):
return True
else:
return False
def checkoutRevision(url, revision, branch_url, revert=False, pop=True):
files_info = getFileInfo(url, revision)
paths = getBestMergePaths2(files_info, revision)
export_map = getBestExportPathsMap2(files_info, revision)
command = 'svn checkout -N ' + branch_url
print command
os.system(command)
match = re.search(r"^[a-z]+://.*/(.*)", branch_url)
if match:
os.chdir(match.group(1))
# This line is extremely important due to the way svn behaves in the
# set-depths action. If parents aren't handled before children, the child
# directories get clobbered and the merge step fails.
paths.sort()
# Checkout the directories that already exist
for path in paths:
if (export_map.has_key(path) and not revert):
print "Exclude new directory " + path
continue
subpaths = path.split('/')
#In the normal case, where no url override is specified and it's just
# chromium source, it's necessary to remove the 'trunk' from the filepath,
# since in the checkout we include 'trunk' or 'branch/\d+'.
#
# However, when a url is specified we want to preserve that because it's
# a part of the filepath and necessary for path operations on svn (because
# frankly, we are checking out the correct top level, and not hacking it).
if pop:
subpaths.pop(0)
base = ''
for subpath in subpaths:
base += '/' + subpath
# This logic ensures that you don't empty out any directories
if not os.path.exists("." + base):
command = ('svn update --depth empty ' + "." + base)
print command
os.system(command)
if (revert):
files = getAllFilesInRevision(files_info)
else:
files = getExistingFilesInRevision(files_info)
for f in files:
# Prevent the tool from clobbering the src directory
if (f == ""):
continue
command = ('svn up ".' + f + '"')
print command
os.system(command)
def mergeRevision(url, revision):
paths = getBestMergePaths(url, revision)
export_map = getBestExportPathsMap(url, revision)
for path in paths:
if export_map.has_key(path):
continue
command = ('svn merge -N -r ' + str(revision-1) + ":" + str(revision) + " ")
command += " --ignore-ancestry "
command += " -x --ignore-eol-style "
command += url + path + "@" + str(revision) + " ." + path
print command
os.system(command)
def exportRevision(url, revision):
paths = getBestExportPathsMap(url, revision).keys()
paths.sort()
for path in paths:
command = ('svn export -N ' + url + path + "@" + str(revision) + " ." +
path)
print command
os.system(command)
command = 'svn add .' + path
print command
os.system(command)
def deleteRevision(url, revision):
paths = getBestDeletePathsMap(url, revision).keys()
paths.sort()
paths.reverse()
for path in paths:
command = "svn delete ." + path
print command
os.system(command)
def revertExportRevision(url, revision):
paths = getBestExportPathsMap(url, revision).keys()
paths.sort()
paths.reverse()
for path in paths:
command = "svn delete ." + path
print command
os.system(command)
def revertRevision(url, revision):
command = ('svn merge --ignore-ancestry -c -%d %s .' % (revision, url))
print command
os.system(command)
def getFileInfo(url, revision):
global files_info_
if (files_info_ != None):
return files_info_
svn_log = subprocess2.check_output(
['svn', 'log', url, '-r', str(revision), '-v']).splitlines()
info = []
for line in svn_log:
# A workaround to dump the (from .*) stuff, regex not so friendly in the 2nd
# pass...
match = re.search(r"(.*) \(from.*\)", line)
if match:
line = match.group(1)
match = re.search(file_pattern_, line)
if match:
info.append([match.group(1).strip(), match.group(2).strip(),
match.group(3).strip(),match.group(4).strip()])
files_info_ = info
return info
def getBestMergePaths(url, revision):
"""Takes an svn url and gets the associated revision."""
return getBestMergePaths2(getFileInfo(url, revision), revision)
def getBestMergePaths2(files_info, revision):
"""Takes an svn url and gets the associated revision."""
return list(set([f[2] for f in files_info]))
def getBestExportPathsMap(url, revision):
return getBestExportPathsMap2(getFileInfo(url, revision), revision)
def getBestExportPathsMap2(files_info, revision):
"""Takes an svn url and gets the associated revision."""
global export_map_
if export_map_:
return export_map_
result = {}
for file_info in files_info:
if (file_info[0] == "A"):
if(isSVNDirectory("svn://svn.chromium.org/chrome/" + file_info[1],
revision)):
result[file_info[2] + "/" + file_info[3]] = ""
export_map_ = result
return result
def getBestDeletePathsMap(url, revision):
return getBestDeletePathsMap2(getFileInfo(url, revision), revision)
def getBestDeletePathsMap2(files_info, revision):
"""Takes an svn url and gets the associated revision."""
global delete_map_
if delete_map_:
return delete_map_
result = {}
for file_info in files_info:
if (file_info[0] == "D"):
if(isSVNDirectory("svn://svn.chromium.org/chrome/" + file_info[1],
revision)):
result[file_info[2] + "/" + file_info[3]] = ""
delete_map_ = result
return result
def getExistingFilesInRevision(files_info):
"""Checks for existing files in the revision.
Anything that's A will require special treatment (either a merge or an
export + add)
"""
return ['%s/%s' % (f[2], f[3]) for f in files_info if f[0] != 'A']
def getAllFilesInRevision(files_info):
"""Checks for existing files in the revision.
Anything that's A will require special treatment (either a merge or an
export + add)
"""
return ['%s/%s' % (f[2], f[3]) for f in files_info]
def getSVNAuthInfo(folder=None):
"""Fetches SVN authorization information in the subversion auth folder and
returns it as a dictionary of dictionaries."""
if not folder:
if sys.platform == 'win32':
folder = '%%APPDATA%\\Subversion\\auth'
else:
folder = '~/.subversion/auth'
folder = os.path.expandvars(os.path.expanduser(folder))
svn_simple_folder = os.path.join(folder, 'svn.simple')
results = {}
try:
for auth_file in os.listdir(svn_simple_folder):
# Read the SVN auth file, convert it into a dictionary, and store it.
results[auth_file] = dict(re.findall(r'K [0-9]+\n(.*)\nV [0-9]+\n(.*)\n',
open(os.path.join(svn_simple_folder, auth_file)).read()))
except Exception as _:
pass
return results
def getCurrentSVNUsers(url):
"""Tries to fetch the current SVN in the current checkout by scanning the
SVN authorization folder for a match with the current SVN URL."""
netloc = urlparse.urlparse(url)[1]
auth_infos = getSVNAuthInfo()
results = []
for _, auth_info in auth_infos.iteritems():
if ('svn:realmstring' in auth_info
and netloc in auth_info['svn:realmstring']):
username = auth_info['username']
results.append(username)
if 'google.com' in username:
results.append(username.replace('google.com', 'chromium.org'))
return results
def prompt(question):
while True:
print question + " [y|n]:",
answer = sys.stdin.readline()
if answer.lower().startswith('n'):
return False
elif answer.lower().startswith('y'):
return True
def text_prompt(question, default):
print question + " [" + default + "]:"
answer = sys.stdin.readline()
if answer.strip() == "":
return default
return answer
def drover(options, args):
revision = options.revert or options.merge
# Initialize some variables used below. They can be overwritten by
# the drover.properties file.
BASE_URL = "svn://svn.chromium.org/chrome"
REVERT_ALT_URLS = ['svn://svn.chromium.org/blink',
'svn://svn.chromium.org/chrome-internal',
'svn://svn.chromium.org/native_client']
TRUNK_URL = BASE_URL + "/trunk/src"
BRANCH_URL = BASE_URL + "/branches/$branch/src"
SKIP_CHECK_WORKING = True
PROMPT_FOR_AUTHOR = False
NO_ALT_URLS = options.no_alt_urls
DEFAULT_WORKING = "drover_" + str(revision)
if options.branch:
DEFAULT_WORKING += ("_" + options.branch)
if not isMinimumSVNVersion(1, 5):
print "You need to use at least SVN version 1.5.x"
return 1
# Override the default properties if there is a drover.properties file.
global file_pattern_
if os.path.exists("drover.properties"):
print 'Using options from %s' % os.path.join(
os.getcwd(), 'drover.properties')
FILE_PATTERN = file_pattern_
f = open("drover.properties")
exec(f)
f.close()
if FILE_PATTERN:
file_pattern_ = FILE_PATTERN
NO_ALT_URLS = True
if options.revert and options.branch:
print 'Note: --branch is usually not needed for reverts.'
url = BRANCH_URL.replace("$branch", options.branch)
elif options.merge and options.sbranch:
url = BRANCH_URL.replace("$branch", options.sbranch)
elif options.revert:
url = options.url or BASE_URL
file_pattern_ = r"[ ]+([MADUC])[ ]+((/.*)/(.*))"
else:
url = TRUNK_URL
working = options.workdir or DEFAULT_WORKING
if options.local:
working = os.getcwd()
if not inCheckoutRoot(working):
print "'%s' appears not to be the root of a working copy" % working
return 1
if (isSVNDirty() and not
prompt("Working copy contains uncommitted files. Continue?")):
return 1
if options.revert and not NO_ALT_URLS and not options.url:
for cur_url in [url] + REVERT_ALT_URLS:
try:
commit_date_str = getSVNInfo(
cur_url, options.revert).get('Last Changed Date', 'x').split()[0]
commit_date = datetime.datetime.strptime(commit_date_str, '%Y-%m-%d')
if (datetime.datetime.now() - commit_date).days < 180:
if cur_url != url:
print 'Guessing svn repo: %s.' % cur_url,
print 'Use --no-alt-urls to disable heuristic.'
url = cur_url
break
except ValueError:
pass
command = 'svn log ' + url + " -r "+str(revision) + " -v"
os.system(command)
if not (options.revertbot or prompt("Is this the correct revision?")):
return 0
if (os.path.exists(working)) and not options.local:
if not (options.revertbot or SKIP_CHECK_WORKING or
prompt("Working directory: '%s' already exists, clobber?" % working)):
return 0
gclient_utils.rmtree(working)
if not options.local:
os.makedirs(working)
os.chdir(working)
if options.merge:
action = "Merge"
if not options.local:
branch_url = BRANCH_URL.replace("$branch", options.branch)
# Checkout everything but stuff that got added into a new dir
checkoutRevision(url, revision, branch_url)
# Merge everything that changed
mergeRevision(url, revision)
# "Export" files that were added from the source and add them to branch
exportRevision(url, revision)
# Delete directories that were deleted (file deletes are handled in the
# merge).
deleteRevision(url, revision)
elif options.revert:
action = "Revert"
pop_em = not options.url
checkoutRevision(url, revision, url, True, pop_em)
revertRevision(url, revision)
revertExportRevision(url, revision)
# Check the base url so we actually find the author who made the change
if options.auditor:
author = options.auditor
else:
author = getAuthor(url, revision)
if not author:
author = getAuthor(TRUNK_URL, revision)
# Check that the author of the CL is different than the user making
# the revert. If they're the same, then we'll want to prompt the user
# for a different reviewer to TBR.
current_users = getCurrentSVNUsers(BASE_URL)
is_self_revert = options.revert and author in current_users
filename = str(revision)+".txt"
out = open(filename,"w")
drover_title = '%s %s' % (action, revision)
revision_log = getRevisionLog(url, revision).splitlines()
if revision_log:
commit_title = revision_log[0]
# Limit title to 68 chars so git log --oneline is <80 chars.
max_commit_title = 68 - (len(drover_title) + 3)
if len(commit_title) > max_commit_title:
commit_title = commit_title[:max_commit_title-3] + '...'
drover_title += ' "%s"' % commit_title
out.write(drover_title + '\n\n')
for line in revision_log:
out.write('> %s\n' % line)
if author:
out.write("\nTBR=" + author)
out.close()
change_cmd = 'change ' + str(revision) + " " + filename
if options.revertbot:
if sys.platform == 'win32':
os.environ['SVN_EDITOR'] = 'cmd.exe /c exit'
else:
os.environ['SVN_EDITOR'] = 'true'
runGcl(change_cmd)
os.unlink(filename)
if options.local:
return 0
print author
print revision
print ("gcl upload " + str(revision) +
" --send_mail --no_presubmit --reviewers=" + author)
if options.revertbot or prompt("Would you like to upload?"):
if PROMPT_FOR_AUTHOR or is_self_revert:
author = text_prompt("Enter new author or press enter to accept default",
author)
if options.revertbot and options.revertbot_reviewers:
author += ","
author += options.revertbot_reviewers
gclUpload(revision, author)
else:
print "Deleting the changelist."
print "gcl delete " + str(revision)
runGcl("delete " + str(revision))
return 0
# We commit if the reverbot is set to commit automatically, or if this is
# not the revertbot and the user agrees.
if options.revertbot_commit or (not options.revertbot and
prompt("Would you like to commit?")):
print "gcl commit " + str(revision) + " --no_presubmit --force"
return runGcl("commit " + str(revision) + " --no_presubmit --force")
else:
return 0
def main():
option_parser = optparse.OptionParser(usage=USAGE % {"app": sys.argv[0]})
option_parser.add_option('-m', '--merge', type="int",
help='Revision to merge from trunk to branch')
option_parser.add_option('-b', '--branch',
help='Branch to revert or merge from')
option_parser.add_option('-l', '--local', action='store_true',
help='Local working copy to merge to')
option_parser.add_option('-s', '--sbranch',
help='Source branch for merge')
option_parser.add_option('-r', '--revert', type="int",
help='Revision to revert')
option_parser.add_option('-w', '--workdir',
help='subdir to use for the revert')
option_parser.add_option('-u', '--url',
help='svn url to use for the revert')
option_parser.add_option('-a', '--auditor',
help='overrides the author for reviewer')
option_parser.add_option('--revertbot', action='store_true',
default=False)
option_parser.add_option('--no-alt-urls', action='store_true',
help='Disable heuristics used to determine svn url')
option_parser.add_option('--revertbot-commit', action='store_true',
default=False)
option_parser.add_option('--revertbot-reviewers')
options, args = option_parser.parse_args()
if not options.merge and not options.revert:
option_parser.error("You need at least --merge or --revert")
return 1
if options.merge and not (options.branch or options.local):
option_parser.error("--merge requires --branch or --local")
return 1
if options.local and (options.revert or options.branch):
option_parser.error("--local cannot be used with --revert or --branch")
return 1
return drover(options, args)
if __name__ == "__main__":
try:
sys.exit(main())
except KeyboardInterrupt:
sys.stderr.write('interrupted\n')
sys.exit(1)
gcl deleted 100755 → 0
View file @ 12fa6ff6
#!/usr/bin/env bash
# Copyright (c) 2009 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
base_dir=$(dirname "$0")
PYTHONDONTWRITEBYTECODE=1 exec python "$base_dir/gcl.py" "$@"
gcl.bat deleted 100755 → 0
View file @ 12fa6ff6
@echo off
setlocal
:: This is required with cygwin only.
PATH=%~dp0;%PATH%
set PYTHONDONTWRITEBYTECODE=1
call python "%~dp0gcl.py" %*
gcl.py deleted 100755 → 0
View file @ 12fa6ff6
#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""\
Wrapper script around Rietveld's upload.py that simplifies working with groups
of files.
"""
import json
import optparse
import os
import random
import re
import ssl
import string
import sys
import tempfile
import time
import urllib2
import auth
import fix_encoding
import gclient_utils
import git_cl
import presubmit_support
import rietveld
from scm import SVN
import subprocess2
from third_party import upload
__version__ = '1.2.1'
CODEREVIEW_SETTINGS = {
# To make gcl send reviews to a server, check in a file named
# "codereview.settings" (see |CODEREVIEW_SETTINGS_FILE| below) to your
# project's base directory and add the following line to codereview.settings:
# CODE_REVIEW_SERVER: codereview.yourserver.org
}
# globals that store the root of the current repository and the directory where
# we store information about changelists.
REPOSITORY_ROOT = ""
# Printed when people upload patches using svn.
SWITCH_TO_GIT = """You're using svn to work on depot_tools.
Consider switching to git today, so that you're ready when svn stops working
and you need a functional checkout for a future fire."""
# Filename where we store repository specific information for gcl.
CODEREVIEW_SETTINGS_FILE = "codereview.settings"
CODEREVIEW_SETTINGS_FILE_NOT_FOUND = (
'No %s file found. Please add one.' % CODEREVIEW_SETTINGS_FILE)
# Warning message when the change appears to be missing tests.
MISSING_TEST_MSG = "Change contains new or modified methods, but no new tests!"
# Global cache of files cached in GetCacheDir().
FILES_CACHE = {}
# Valid extensions for files we want to lint.
DEFAULT_LINT_REGEX = r"(.*\.cpp|.*\.cc|.*\.h)"
DEFAULT_LINT_IGNORE_REGEX = r"$^"
def CheckHomeForFile(filename):
"""Checks the users home dir for the existence of the given file. Returns
the path to the file if it's there, or None if it is not.
"""
full_path = os.path.expanduser(os.path.join('~', filename))
if os.path.exists(full_path):
return full_path
return None
def UnknownFiles():
"""Runs svn status and returns unknown files."""
return [
item[1] for item in SVN.CaptureStatus([], GetRepositoryRoot())
if item[0][0] == '?'
]
def GetRepositoryRoot():
"""Returns the top level directory of the current repository.
The directory is returned as an absolute path.
"""
global REPOSITORY_ROOT
if not REPOSITORY_ROOT:
REPOSITORY_ROOT = SVN.GetCheckoutRoot(os.getcwd())
if not REPOSITORY_ROOT:
raise gclient_utils.Error("gcl run outside of repository")
return REPOSITORY_ROOT
def GetInfoDir():
"""Returns the directory where gcl info files are stored."""
return os.path.join(GetRepositoryRoot(), '.svn', 'gcl_info')
def GetChangesDir():
"""Returns the directory where gcl change files are stored."""
return os.path.join(GetInfoDir(), 'changes')
def GetCacheDir():
"""Returns the directory where gcl change files are stored."""
return os.path.join(GetInfoDir(), 'cache')
def GetCachedFile(filename, max_age=60*60*24*3, use_root=False):
"""Retrieves a file from the repository and caches it in GetCacheDir() for
max_age seconds.
use_root: If False, look up the arborescence for the first match, otherwise go
directory to the root repository.
Note: The cache will be inconsistent if the same file is retrieved with both
use_root=True and use_root=False. Don't be stupid.
"""
if filename not in FILES_CACHE:
# Don't try to look up twice.
FILES_CACHE[filename] = None
# First we check if we have a cached version.
try:
cached_file = os.path.join(GetCacheDir(), filename)
except (gclient_utils.Error, subprocess2.CalledProcessError):
return None
if (not os.path.exists(cached_file) or
(time.time() - os.stat(cached_file).st_mtime) > max_age):
dir_info = SVN.CaptureLocalInfo([], '.')
repo_root = dir_info['Repository Root']
if use_root:
url_path = repo_root
else:
url_path = dir_info['URL']
while True:
# Look in the repository at the current level for the file.
for _ in range(5):
content = None
try:
# Take advantage of the fact that svn won't output to stderr in case
# of success but will do in case of failure so don't mind putting
# stderr into content_array.
content_array = []
svn_path = url_path + '/' + filename
args = ['svn', 'cat', svn_path]
if sys.platform != 'darwin':
# MacOSX 10.5.2 has a bug with svn 1.4.4 that will trigger the
# 'Can\'t get username or password' and can be fixed easily.
# The fix doesn't work if the user upgraded to svn 1.6.x. Bleh.
# I don't have time to fix their broken stuff.
args.append('--non-interactive')
gclient_utils.CheckCallAndFilter(
args, cwd='.', filter_fn=content_array.append)
# Exit the loop if the file was found. Override content.
content = '\n'.join(content_array)
break
except (gclient_utils.Error, subprocess2.CalledProcessError):
if content_array[0].startswith(
'svn: Can\'t get username or password'):
ErrorExit('Your svn credentials expired. Please run svn update '
'to fix the cached credentials')
if content_array[0].startswith('svn: Can\'t get password'):
ErrorExit('If are using a Mac and svn --version shows 1.4.x, '
'please hack gcl.py to remove --non-interactive usage, it\'s'
'a bug on your installed copy')
if (content_array[0].startswith('svn: File not found:') or
content_array[0].endswith('path not found')):
break
# Otherwise, fall through to trying again.
if content:
break
if url_path == repo_root:
# Reached the root. Abandoning search.
break
# Go up one level to try again.
url_path = os.path.dirname(url_path)
if content is not None or filename != CODEREVIEW_SETTINGS_FILE:
# Write a cached version even if there isn't a file, so we don't try to
# fetch it each time. codereview.settings must always be present so do
# not cache negative.
gclient_utils.FileWrite(cached_file, content or '')
else:
content = gclient_utils.FileRead(cached_file, 'r')
# Keep the content cached in memory.
FILES_CACHE[filename] = content
return FILES_CACHE[filename]
def GetCodeReviewSetting(key):
"""Returns a value for the given key for this repository."""
# Use '__just_initialized' as a flag to determine if the settings were
# already initialized.
if '__just_initialized' not in CODEREVIEW_SETTINGS:
settings_file = GetCachedFile(CODEREVIEW_SETTINGS_FILE)
if settings_file:
CODEREVIEW_SETTINGS.update(
gclient_utils.ParseCodereviewSettingsContent(settings_file))
CODEREVIEW_SETTINGS.setdefault('__just_initialized', None)
return CODEREVIEW_SETTINGS.get(key, "")
def Warn(msg):
print >> sys.stderr, msg
def ErrorExit(msg):
print >> sys.stderr, msg
sys.exit(1)
def RunShellWithReturnCode(command, print_output=False):
"""Executes a command and returns the output and the return code."""
p = subprocess2.Popen(
command,
cwd=GetRepositoryRoot(),
stdout=subprocess2.PIPE,
stderr=subprocess2.STDOUT,
universal_newlines=True)
if print_output:
output_array = []
while True:
line = p.stdout.readline()
if not line:
break
if print_output:
print line.strip('\n')
output_array.append(line)
output = "".join(output_array)
else:
output = p.stdout.read()
p.wait()
p.stdout.close()
return output, p.returncode
def RunShell(command, print_output=False):
"""Executes a command and returns the output."""
return RunShellWithReturnCode(command, print_output)[0]
def FilterFlag(args, flag):
"""Returns True if the flag is present in args list.
The flag is removed from args if present.
"""
if flag in args:
args.remove(flag)
return True
return False
class ChangeInfo(object):
"""Holds information about a changelist.
name: change name.
issue: the Rietveld issue number or 0 if it hasn't been uploaded yet.
patchset: the Rietveld latest patchset number or 0.
description: the description.
files: a list of 2 tuple containing (status, filename) of changed files,
with paths being relative to the top repository directory.
local_root: Local root directory
rietveld: rietveld server for this change
"""
# Kept for unit test support. This is for the old format, it's deprecated.
SEPARATOR = "\n-----\n"
def __init__(self, name, issue, patchset, description, files, local_root,
rietveld_url, needs_upload):
# Defer the description processing to git_cl.ChangeDescription.
self._desc = git_cl.ChangeDescription(description)
self.name = name
self.issue = int(issue)
self.patchset = int(patchset)
self._files = files or []
self.patch = None
self._local_root = local_root
self.needs_upload = needs_upload
self.rietveld = gclient_utils.UpgradeToHttps(
rietveld_url or GetCodeReviewSetting('CODE_REVIEW_SERVER'))
self._rpc_server = None
@property
def description(self):
return self._desc.description
def force_description(self, new_description):
self._desc = git_cl.ChangeDescription(new_description)
self.needs_upload = True
def append_footer(self, line):
self._desc.append_footer(line)
def get_reviewers(self):
return self._desc.get_reviewers()
def update_reviewers(self, reviewers):
self._desc.update_reviewers(reviewers)
def NeedsUpload(self):
return self.needs_upload
def GetFileNames(self):
"""Returns the list of file names included in this change."""
return [f[1] for f in self._files]
def GetFiles(self):
"""Returns the list of files included in this change with their status."""
return self._files
def GetLocalRoot(self):
"""Returns the local repository checkout root directory."""
return self._local_root
def Exists(self):
"""Returns True if this change already exists (i.e., is not new)."""
return (self.issue or self.description or self._files)
def _NonDeletedFileList(self):
"""Returns a list of files in this change, not including deleted files."""
return [f[1] for f in self.GetFiles()
if not f[0].startswith("D")]
def _AddedFileList(self):
"""Returns a list of files added in this change."""
return [f[1] for f in self.GetFiles() if f[0].startswith("A")]
def Save(self):
"""Writes the changelist information to disk."""
data = json.dumps({
'issue': self.issue,
'patchset': self.patchset,
'needs_upload': self.NeedsUpload(),
'files': self.GetFiles(),
'description': self.description,
'rietveld': self.rietveld,
}, sort_keys=True, indent=2)
gclient_utils.FileWrite(GetChangelistInfoFile(self.name), data)
def Delete(self):
"""Removes the changelist information from disk."""
os.remove(GetChangelistInfoFile(self.name))
def RpcServer(self):
if not self._rpc_server:
if not self.rietveld:
ErrorExit(CODEREVIEW_SETTINGS_FILE_NOT_FOUND)
# TODO(vadimsh): glc.py should be deleted soon. Do not bother much about
# authentication options and always use defaults.
self._rpc_server = rietveld.CachingRietveld(
self.rietveld, auth.make_auth_config())
return self._rpc_server
def CloseIssue(self):
"""Closes the Rietveld issue for this changelist."""
# Newer versions of Rietveld require us to pass an XSRF token to POST, so
# we fetch it from the server.
xsrf_token = self.SendToRietveld(
'/xsrf_token',
extra_headers={'X-Requesting-XSRF-Token': '1'})
# You cannot close an issue with a GET.
# We pass an empty string for the data so it is a POST rather than a GET.
data = [("description", self.description),
("xsrf_token", xsrf_token)]
ctype, body = upload.EncodeMultipartFormData(data, [])
self.SendToRietveld('/%d/close' % self.issue, payload=body,
content_type=ctype)
def UpdateRietveldDescription(self):
"""Sets the description for an issue on Rietveld."""
data = [("description", self.description),]
ctype, body = upload.EncodeMultipartFormData(data, [])
self.SendToRietveld('/%d/description' % self.issue, payload=body,
content_type=ctype)
self.needs_upload = False
def GetIssueDescription(self):
"""Returns the issue description from Rietveld."""
return self.SendToRietveld('/%d/description' % self.issue).replace('\r\n',
'\n')
def UpdateDescriptionFromIssue(self):
"""Updates self.description with the issue description from Rietveld."""
self._desc = git_cl.ChangeDescription(self.GetIssueDescription())
def GetApprovingReviewers(self):
"""Returns the issue reviewers list from Rietveld."""
return git_cl.get_approving_reviewers(
self.RpcServer().get_issue_properties(self.issue, True))
def AddComment(self, comment):
"""Adds a comment for an issue on Rietveld.
As a side effect, this will email everyone associated with the issue."""
return self.RpcServer().add_comment(self.issue, comment)
def PrimeLint(self):
"""Do background work on Rietveld to lint the file so that the results are
ready when the issue is viewed."""
if self.issue and self.patchset:
try:
self.SendToRietveld('/lint/issue%s_%s' % (self.issue, self.patchset),
timeout=60)
except ssl.SSLError as e:
# It takes more than 60 seconds to lint some CLs. Silently ignore
# the expected timeout.
if e.message != 'The read operation timed out':
raise
def SendToRietveld(self, request_path, timeout=None, **kwargs):
"""Send a POST/GET to Rietveld. Returns the response body."""
try:
return self.RpcServer().Send(request_path, timeout=timeout, **kwargs)
except urllib2.URLError:
if timeout is None:
ErrorExit('Error accessing url %s' % request_path)
else:
return None
def MissingTests(self):
"""Returns True if the change looks like it needs unit tests but has none.
A change needs unit tests if it contains any new source files or methods.
"""
SOURCE_SUFFIXES = [".cc", ".cpp", ".c", ".m", ".mm"]
# Ignore third_party entirely.
files = [f for f in self._NonDeletedFileList()
if f.find("third_party") == -1]
added_files = [f for f in self._AddedFileList()
if f.find("third_party") == -1]
# If the change is entirely in third_party, we're done.
if len(files) == 0:
return False
# Any new or modified test files?
# A test file's name ends with "test.*" or "tests.*".
test_files = [test for test in files
if os.path.splitext(test)[0].rstrip("s").endswith("test")]
if len(test_files) > 0:
return False
# Any new source files?
source_files = [item for item in added_files
if os.path.splitext(item)[1] in SOURCE_SUFFIXES]
if len(source_files) > 0:
return True
# Do the long test, checking the files for new methods.
return self._HasNewMethod()
def _HasNewMethod(self):
"""Returns True if the changeset contains any new functions, or if a
function signature has been changed.
A function is identified by starting flush left, containing a "(" before
the next flush-left line, and either ending with "{" before the next
flush-left line or being followed by an unindented "{".
Currently this returns True for new methods, new static functions, and
methods or functions whose signatures have been changed.
Inline methods added to header files won't be detected by this. That's
acceptable for purposes of determining if a unit test is needed, since
inline methods should be trivial.
"""
# To check for methods added to source or header files, we need the diffs.
# We'll generate them all, since there aren't likely to be many files
# apart from source and headers; besides, we'll want them all if we're
# uploading anyway.
if self.patch is None:
self.patch = GenerateDiff(self.GetFileNames())
definition = ""
for line in self.patch.splitlines():
if not line.startswith("+"):
continue
line = line.strip("+").rstrip(" \t")
# Skip empty lines, comments, and preprocessor directives.
# TODO(pamg): Handle multiline comments if it turns out to be a problem.
if line == "" or line.startswith("/") or line.startswith("#"):
continue
# A possible definition ending with "{" is complete, so check it.
if definition.endswith("{"):
if definition.find("(") != -1:
return True
definition = ""
# A { or an indented line, when we're in a definition, continues it.
if (definition != "" and
(line == "{" or line.startswith(" ") or line.startswith("\t"))):
definition += line
# A flush-left line starts a new possible function definition.
elif not line.startswith(" ") and not line.startswith("\t"):
definition = line
return False
@staticmethod
def Load(changename, local_root, fail_on_not_found, update_status):
"""Gets information about a changelist.
Args:
fail_on_not_found: if True, this function will quit the program if the
changelist doesn't exist.
update_status: if True, the svn status will be updated for all the files
and unchanged files will be removed.
Returns: a ChangeInfo object.
"""
info_file = GetChangelistInfoFile(changename)
if not os.path.exists(info_file):
if fail_on_not_found:
ErrorExit("Changelist " + changename + " not found.")
return ChangeInfo(changename, 0, 0, '', None, local_root, None, False)
content = gclient_utils.FileRead(info_file)
save = False
try:
values = ChangeInfo._LoadNewFormat(content)
except ValueError:
try:
values = ChangeInfo._LoadOldFormat(content)
save = True
except ValueError:
ErrorExit(
('Changelist file %s is corrupt.\n'
'Either run "gcl delete %s" or manually edit the file') % (
info_file, changename))
files = values['files']
if update_status:
for item in files[:]:
status_result = SVN.CaptureStatus(item[1], local_root)
if not status_result or not status_result[0][0]:
# File has been reverted.
save = True
files.remove(item)
continue
status = status_result[0][0]
if status != item[0]:
save = True
files[files.index(item)] = (status, item[1])
change_info = ChangeInfo(
changename,
values['issue'],
values['patchset'],
values['description'],
files,
local_root,
values.get('rietveld'),
values['needs_upload'])
if save:
change_info.Save()
return change_info
@staticmethod
def _LoadOldFormat(content):
# The info files have the following format:
# issue_id, patchset\n (, patchset is optional)
# SEPARATOR\n
# filepath1\n
# filepath2\n
# .
# .
# filepathn\n
# SEPARATOR\n
# description
split_data = content.split(ChangeInfo.SEPARATOR, 2)
if len(split_data) != 3:
raise ValueError('Bad change format')
values = {
'issue': 0,
'patchset': 0,
'needs_upload': False,
'files': [],
}
items = split_data[0].split(', ')
if items[0]:
values['issue'] = int(items[0])
if len(items) > 1:
values['patchset'] = int(items[1])
if len(items) > 2:
values['needs_upload'] = (items[2] == "dirty")
for line in split_data[1].splitlines():
status = line[:7]
filename = line[7:]
values['files'].append((status, filename))
values['description'] = split_data[2]
return values
@staticmethod
def _LoadNewFormat(content):
return json.loads(content)
def __str__(self):
out = ['%s:' % self.__class__.__name__]
for k in dir(self):
if k.startswith('__'):
continue
v = getattr(self, k)
if v is self or callable(getattr(self, k)):
continue
out.append(' %s: %r' % (k, v))
return '\n'.join(out)
def GetChangelistInfoFile(changename):
"""Returns the file that stores information about a changelist."""
if not changename or re.search(r'[^\w-]', changename):
ErrorExit("Invalid changelist name: " + changename)
return os.path.join(GetChangesDir(), changename)
def LoadChangelistInfoForMultiple(changenames, local_root, fail_on_not_found,
update_status):
"""Loads many changes and merge their files list into one pseudo change.
This is mainly useful to concatenate many changes into one for a 'gcl try'.
"""
changes = changenames.split(',')
aggregate_change_info = ChangeInfo(
changenames, 0, 0, '', None, local_root, None, False)
for change in changes:
aggregate_change_info._files += ChangeInfo.Load(
change, local_root, fail_on_not_found, update_status).GetFiles()
return aggregate_change_info
def GetCLs():
"""Returns a list of all the changelists in this repository."""
cls = os.listdir(GetChangesDir())
if CODEREVIEW_SETTINGS_FILE in cls:
cls.remove(CODEREVIEW_SETTINGS_FILE)
return cls
def GenerateChangeName():
"""Generate a random changelist name."""
random.seed()
current_cl_names = GetCLs()
while True:
cl_name = (random.choice(string.ascii_lowercase) +
random.choice(string.digits) +
random.choice(string.ascii_lowercase) +
random.choice(string.digits))
if cl_name not in current_cl_names:
return cl_name
def GetModifiedFiles():
"""Returns a set that maps from changelist name to (status,filename) tuples.
Files not in a changelist have an empty changelist name. Filenames are in
relation to the top level directory of the current repository. Note that
only the current directory and subdirectories are scanned, in order to
improve performance while still being flexible.
"""
files = {}
# Since the files are normalized to the root folder of the repositary, figure
# out what we need to add to the paths.
dir_prefix = os.getcwd()[len(GetRepositoryRoot()):].strip(os.sep)
# Get a list of all files in changelists.
files_in_cl = {}
for cl in GetCLs():
change_info = ChangeInfo.Load(cl, GetRepositoryRoot(),
fail_on_not_found=True, update_status=False)
for status, filename in change_info.GetFiles():
files_in_cl[filename] = change_info.name
# Get all the modified files down the current directory.
for line in SVN.CaptureStatus(None, os.getcwd()):
status = line[0]
filename = line[1]
if status[0] == "?":
continue
if dir_prefix:
filename = os.path.join(dir_prefix, filename)
change_list_name = ""
if filename in files_in_cl:
change_list_name = files_in_cl[filename]
files.setdefault(change_list_name, []).append((status, filename))
return files
def GetFilesNotInCL():
"""Returns a list of tuples (status,filename) that aren't in any changelists.
See docstring of GetModifiedFiles for information about path of files and
which directories are scanned.
"""
modified_files = GetModifiedFiles()
if "" not in modified_files:
return []
return modified_files[""]
def ListFiles(show_unknown_files):
files = GetModifiedFiles()
cl_keys = files.keys()
cl_keys.sort()
for cl_name in cl_keys:
if not cl_name:
continue
note = ""
change_info = ChangeInfo.Load(cl_name, GetRepositoryRoot(),
fail_on_not_found=True, update_status=False)
if len(change_info.GetFiles()) != len(files[cl_name]):
note = " (Note: this changelist contains files outside this directory)"
print "\n--- Changelist " + cl_name + note + ":"
for filename in files[cl_name]:
print "".join(filename)
if show_unknown_files:
unknown_files = UnknownFiles()
if (files.get('') or (show_unknown_files and len(unknown_files))):
print "\n--- Not in any changelist:"
for item in files.get('', []):
print "".join(item)
if show_unknown_files:
for filename in unknown_files:
print "? %s" % filename
return 0
def GenerateDiff(files):
return SVN.GenerateDiff(
files, GetRepositoryRoot(), full_move=False, revision=None)
def GetTreeStatus():
tree_status_url = GetCodeReviewSetting('STATUS')
return git_cl.GetTreeStatus(tree_status_url) if tree_status_url else "unset"
def OptionallyDoPresubmitChecks(change_info, committing, args):
if FilterFlag(args, "--no_presubmit") or FilterFlag(args, "--force"):
return presubmit_support.PresubmitOutput()
return DoPresubmitChecks(change_info, committing, True)
def defer_attributes(a, b):
"""Copy attributes from an object (like a function) to another."""
for x in dir(a):
if not getattr(b, x, None):
setattr(b, x, getattr(a, x))
def need_change(function):
"""Converts args -> change_info."""
# pylint: disable=W0612,W0621
def hook(args):
if not len(args) == 1:
ErrorExit("You need to pass a change list name")
change_info = ChangeInfo.Load(args[0], GetRepositoryRoot(), True, True)
return function(change_info)
defer_attributes(function, hook)
hook.need_change = True
hook.no_args = True
return hook
def need_change_and_args(function):
"""Converts args -> change_info."""
# pylint: disable=W0612,W0621
def hook(args):
if not args:
ErrorExit("You need to pass a change list name")
change_info = ChangeInfo.Load(args.pop(0), GetRepositoryRoot(), True, True)
return function(change_info, args)
defer_attributes(function, hook)
hook.need_change = True
return hook
def no_args(function):
"""Make sure no args are passed."""
# pylint: disable=W0612,W0621
def hook(args):
if args:
ErrorExit("Doesn't support arguments")
return function()
defer_attributes(function, hook)
hook.no_args = True
return hook
def attrs(**kwargs):
"""Decorate a function with new attributes."""
def decorate(function):
for k in kwargs:
setattr(function, k, kwargs[k])
return function
return decorate
@no_args
def CMDopened():
"""Lists modified files in the current directory down."""
return ListFiles(False)
@no_args
def CMDstatus():
"""Lists modified and unknown files in the current directory down."""
return ListFiles(True)
@need_change_and_args
@attrs(usage='[--no_presubmit] [--no_watchlists]')
def CMDupload(change_info, args):
"""Uploads the changelist to the server for review.
This does not submit a try job; use gcl try to submit a try job.
"""
if '-s' in args or '--server' in args:
ErrorExit('Don\'t use the -s flag, fix codereview.settings instead')
if not change_info.GetFiles():
print "Nothing to upload, changelist is empty."
return 0
output = OptionallyDoPresubmitChecks(change_info, False, args)
if not output.should_continue():
return 1
no_watchlists = (FilterFlag(args, "--no_watchlists") or
FilterFlag(args, "--no-watchlists"))
# Map --send-mail to --send_mail
if FilterFlag(args, "--send-mail"):
args.append("--send_mail")
# Replace -m with -t and --message with --title, but make sure to
# preserve anything after the -m/--message.
found_deprecated_arg = [False]
def replace_message(a):
if a.startswith('-m'):
found_deprecated_arg[0] = True
return '-t' + a[2:]
elif a.startswith('--message'):
found_deprecated_arg[0] = True
return '--title' + a[9:]
return a
args = map(replace_message, args)
if found_deprecated_arg[0]:
print >> sys.stderr, (
'\nWARNING: Use -t or --title to set the title of the patchset.\n'
'In the near future, -m or --message will send a message instead.\n'
'See http://goo.gl/JGg0Z for details.\n')
upload_arg = ["upload.py", "-y"]
upload_arg.append("--server=%s" % change_info.rietveld.encode('utf-8'))
reviewers = change_info.get_reviewers() or output.reviewers
if (reviewers and
not any(arg.startswith('-r') or arg.startswith('--reviewer') for
arg in args)):
upload_arg.append('--reviewers=%s' % ','.join(reviewers))
upload_arg.extend(args)
desc_file = None
try:
if change_info.issue:
# Uploading a new patchset.
upload_arg.append("--issue=%d" % change_info.issue)
project = GetCodeReviewSetting("PROJECT")
if project:
print SWITCH_TO_GIT
upload_arg.append("--project=%s" % project)
if not any(i.startswith('--title') or i.startswith('-t') for i in args):
upload_arg.append('--title= ')
else:
# First time we upload.
handle, desc_file = tempfile.mkstemp(text=True)
os.write(handle, change_info.description)
os.close(handle)
# Watchlist processing -- CC people interested in this changeset
# http://dev.chromium.org/developers/contributing-code/watchlists
if not no_watchlists:
import watchlists
watchlist = watchlists.Watchlists(change_info.GetLocalRoot())
watchers = watchlist.GetWatchersForPaths(change_info.GetFileNames())
# We check this before applying the "PRIVATE" parameter of codereview
# settings assuming that the author of the settings file has put
# addresses which we can send private CLs to, and so we should ignore
# CC_LIST only when --private is specified explicitly on the command
# line.
if "--private" in upload_arg:
Warn("WARNING: CC_LIST and WATCHLISTS are ignored when --private is "
"specified. You need to review and add them manually if "
"necessary.")
cc_list = ""
no_watchlists = True
else:
cc_list = GetCodeReviewSetting("CC_LIST")
if not no_watchlists and watchers:
# Filter out all empty elements and join by ','
cc_list = ','.join(filter(None, [cc_list] + watchers))
if cc_list:
upload_arg.append("--cc=" + cc_list)
upload_arg.append("--file=%s" % desc_file)
if GetCodeReviewSetting("PRIVATE") == "True":
upload_arg.append("--private")
project = GetCodeReviewSetting("PROJECT")
if project:
print SWITCH_TO_GIT
upload_arg.append("--project=%s" % project)
# If we have a lot of files with long paths, then we won't be able to fit
# the command to "svn diff". Instead, we generate the diff manually for
# each file and concatenate them before passing it to upload.py.
if change_info.patch is None:
change_info.patch = GenerateDiff(change_info.GetFileNames())
# Change the current working directory before calling upload.py so that it
# shows the correct base.
previous_cwd = os.getcwd()
os.chdir(change_info.GetLocalRoot())
try:
try:
issue, patchset = upload.RealMain(upload_arg, change_info.patch)
except KeyboardInterrupt:
sys.exit(1)
if issue and patchset:
change_info.issue = int(issue)
change_info.patchset = int(patchset)
change_info.Save()
change_info.PrimeLint()
finally:
os.chdir(previous_cwd)
finally:
if desc_file:
os.remove(desc_file)
print "*** Upload does not submit a try; use gcl try to submit a try. ***"
return 0
@need_change_and_args
@attrs(usage='[--upload]')
def CMDpresubmit(change_info, args):
"""Runs presubmit checks on the change.
The actual presubmit code is implemented in presubmit_support.py and looks
for PRESUBMIT.py files."""
if not change_info.GetFiles():
print('Nothing to presubmit check, changelist is empty.')
return 0
parser = optparse.OptionParser()
parser.add_option('--upload', action='store_true')
options, args = parser.parse_args(args)
if args:
parser.error('Unrecognized args: %s' % args)
if options.upload:
print('*** Presubmit checks for UPLOAD would report: ***')
return not DoPresubmitChecks(change_info, False, False)
else:
print('*** Presubmit checks for COMMIT would report: ***')
return not DoPresubmitChecks(change_info, True, False)
def TryChange(change_info, args, swallow_exception):
"""Create a diff file of change_info and send it to the try server."""
try:
import trychange
except ImportError:
if swallow_exception:
return 1
ErrorExit("You need to install trychange.py to use the try server.")
trychange_args = []
if change_info:
trychange_args.extend(['--name', change_info.name])
if change_info.issue:
trychange_args.extend(["--issue", str(change_info.issue)])
if change_info.patchset:
trychange_args.extend(["--patchset", str(change_info.patchset)])
change = presubmit_support.SvnChange(change_info.name,
change_info.description,
change_info.GetLocalRoot(),
change_info.GetFiles(),
change_info.issue,
change_info.patchset,
None)
else:
change = None
trychange_args.extend(args)
return trychange.TryChange(
trychange_args,
change=change,
swallow_exception=swallow_exception,
prog='gcl try',
extra_epilog='\n'
'When called from gcl, use the format gcl try <change_name>.\n')
@need_change_and_args
@attrs(usage='[--no_presubmit]')
def CMDcommit(change_info, args):
"""Commits the changelist to the repository."""
if not change_info.GetFiles():
print "Nothing to commit, changelist is empty."
return 1
# OptionallyDoPresubmitChecks has a side-effect which eats these flags.
bypassed = '--no_presubmit' in args or '--force' in args
output = OptionallyDoPresubmitChecks(change_info, True, args)
if not output.should_continue():
return 1
# We face a problem with svn here: Let's say change 'bleh' modifies
# svn:ignore on dir1\. but another unrelated change 'pouet' modifies
# dir1\foo.cc. When the user `gcl commit bleh`, foo.cc is *also committed*.
# The only fix is to use --non-recursive but that has its issues too:
# Let's say if dir1 is deleted, --non-recursive must *not* be used otherwise
# you'll get "svn: Cannot non-recursively commit a directory deletion of a
# directory with child nodes". Yay...
commit_cmd = ["svn", "commit"]
if change_info.issue:
# Get the latest description from Rietveld.
change_info.UpdateDescriptionFromIssue()
change_info.update_reviewers(change_info.GetApprovingReviewers())
commit_desc = git_cl.ChangeDescription(change_info.description)
if change_info.issue:
server = change_info.rietveld
if not server.startswith("http://") and not server.startswith("https://"):
server = "http://" + server
commit_desc.append_footer('Review URL: %s/%d' % (server, change_info.issue))
handle, commit_filename = tempfile.mkstemp(text=True)
os.write(handle, commit_desc.description)
os.close(handle)
try:
handle, targets_filename = tempfile.mkstemp(text=True)
os.write(handle, "\n".join(change_info.GetFileNames()))
os.close(handle)
try:
commit_cmd += ['--file=' + commit_filename]
commit_cmd += ['--targets=' + targets_filename]
# Change the current working directory before calling commit.
output = ''
try:
output = RunShell(commit_cmd, True)
except subprocess2.CalledProcessError, e:
ErrorExit('Commit failed.\n%s' % e)
finally:
os.remove(commit_filename)
finally:
os.remove(targets_filename)
if output.find("Committed revision") != -1:
change_info.Delete()
if change_info.issue:
revision = re.compile(".*?\nCommitted revision (\d+)",
re.DOTALL).match(output).group(1)
viewvc_url = GetCodeReviewSetting('VIEW_VC')
if viewvc_url and revision:
change_info.append_footer('Committed: ' + viewvc_url + revision)
elif revision:
change_info.append_footer('Committed: ' + revision)
change_info.CloseIssue()
props = change_info.RpcServer().get_issue_properties(
change_info.issue, False)
patch_num = len(props['patchsets'])
comment = "Committed patchset #%d (id:%d) manually as r%s" % (
patch_num, props['patchsets'][-1], revision)
if bypassed:
comment += ' (tree was closed).' if GetTreeStatus() == 'closed' else '.'
else:
comment += ' (presubmit successful).'
change_info.AddComment(comment)
return 0
def CMDchange(args):
"""Creates or edits a changelist.
Only scans the current directory and subdirectories.
"""
# Verify the user is running the change command from a read-write checkout.
svn_info = SVN.CaptureLocalInfo([], '.')
if not svn_info:
ErrorExit("Current checkout is unversioned. Please retry with a versioned "
"directory.")
if len(args) == 0:
# Generate a random changelist name.
changename = GenerateChangeName()
elif args[0] == '--force':
changename = GenerateChangeName()
else:
changename = args[0]
change_info = ChangeInfo.Load(changename, GetRepositoryRoot(), False, True)
if len(args) == 2:
if not os.path.isfile(args[1]):
ErrorExit('The change "%s" doesn\'t exist.' % args[1])
f = open(args[1], 'rU')
override_description = f.read()
f.close()
else:
override_description = None
if change_info.issue and not change_info.NeedsUpload():
try:
description = change_info.GetIssueDescription()
except urllib2.HTTPError, err:
if err.code == 404:
# The user deleted the issue in Rietveld, so forget the old issue id.
description = change_info.description
change_info.issue = 0
change_info.Save()
else:
ErrorExit("Error getting the description from Rietveld: " + err)
else:
if override_description:
description = override_description
else:
description = change_info.description
other_files = GetFilesNotInCL()
# Edited files (as opposed to files with only changed properties) will have
# a letter for the first character in the status string.
file_re = re.compile(r"^[a-z].+\Z", re.IGNORECASE)
affected_files = [x for x in other_files if file_re.match(x[0])]
unaffected_files = [x for x in other_files if not file_re.match(x[0])]
description = description.rstrip() + '\n'
separator1 = ("\n---All lines above this line become the description.\n"
"---Repository Root: " + change_info.GetLocalRoot() + "\n"
"---Paths in this changelist (" + change_info.name + "):\n")
separator2 = "\n\n---Paths modified but not in any changelist:\n\n"
text = (description + separator1 + '\n' +
'\n'.join([f[0] + f[1] for f in change_info.GetFiles()]))
if change_info.Exists():
text += (separator2 +
'\n'.join([f[0] + f[1] for f in affected_files]) + '\n')
else:
text += ('\n'.join([f[0] + f[1] for f in affected_files]) + '\n' +
separator2)
text += '\n'.join([f[0] + f[1] for f in unaffected_files]) + '\n'
result = gclient_utils.RunEditor(text, False)
if not result:
ErrorExit('Running editor failed')
split_result = result.split(separator1, 1)
if len(split_result) != 2:
ErrorExit("Don't modify the text starting with ---!\n\n%r" % result)
# Update the CL description if it has changed.
new_description = split_result[0]
cl_files_text = split_result[1]
if new_description != description or override_description:
change_info.force_description(new_description)
new_cl_files = []
for line in cl_files_text.splitlines():
if not len(line):
continue
if line.startswith("---"):
break
status = line[:7]
filename = line[7:]
new_cl_files.append((status, filename))
if (not len(change_info.GetFiles()) and not change_info.issue and
not len(new_description) and not new_cl_files):
ErrorExit("Empty changelist not saved")
change_info._files = new_cl_files
change_info.Save()
if svn_info.get('URL', '').startswith('http:'):
Warn("WARNING: Creating CL in a read-only checkout. You will need to "
"commit using a commit queue!")
print change_info.name + " changelist saved."
if change_info.MissingTests():
Warn("WARNING: " + MISSING_TEST_MSG)
# Update the Rietveld issue.
if change_info.issue and change_info.NeedsUpload():
change_info.UpdateRietveldDescription()
change_info.Save()
return 0
@need_change_and_args
def CMDlint(change_info, args):
"""Runs cpplint.py on all the files in the change list.
Checks all the files in the changelist for possible style violations.
"""
# Access to a protected member _XX of a client class
# pylint: disable=W0212
try:
import cpplint
import cpplint_chromium
except ImportError:
ErrorExit("You need to install cpplint.py to lint C++ files.")
# Change the current working directory before calling lint so that it
# shows the correct base.
previous_cwd = os.getcwd()
os.chdir(change_info.GetLocalRoot())
try:
# Process cpplints arguments if any.
filenames = cpplint.ParseArguments(args + change_info.GetFileNames())
white_list = GetCodeReviewSetting("LINT_REGEX")
if not white_list:
white_list = DEFAULT_LINT_REGEX
white_regex = re.compile(white_list)
black_list = GetCodeReviewSetting("LINT_IGNORE_REGEX")
if not black_list:
black_list = DEFAULT_LINT_IGNORE_REGEX
black_regex = re.compile(black_list)
extra_check_functions = [cpplint_chromium.CheckPointerDeclarationWhitespace]
for filename in filenames:
if white_regex.match(filename):
if black_regex.match(filename):
print "Ignoring file %s" % filename
else:
cpplint.ProcessFile(filename, cpplint._cpplint_state.verbose_level,
extra_check_functions)
else:
print "Skipping file %s" % filename
finally:
os.chdir(previous_cwd)
print "Total errors found: %d\n" % cpplint._cpplint_state.error_count
return 1
def DoPresubmitChecks(change_info, committing, may_prompt):
"""Imports presubmit, then calls presubmit.DoPresubmitChecks."""
root_presubmit = GetCachedFile('PRESUBMIT.py', use_root=True)
change = presubmit_support.SvnChange(change_info.name,
change_info.description,
change_info.GetLocalRoot(),
change_info.GetFiles(),
change_info.issue,
change_info.patchset,
None)
output = presubmit_support.DoPresubmitChecks(
change=change,
committing=committing,
verbose=False,
output_stream=sys.stdout,
input_stream=sys.stdin,
default_presubmit=root_presubmit,
may_prompt=may_prompt,
rietveld_obj=change_info.RpcServer())
if not output.should_continue() and may_prompt:
# TODO(dpranke): move into DoPresubmitChecks(), unify cmd line args.
print "\nPresubmit errors, can't continue (use --no_presubmit to bypass)"
return output
@no_args
def CMDchanges():
"""Lists all the changelists and their files."""
for cl in GetCLs():
change_info = ChangeInfo.Load(cl, GetRepositoryRoot(), True, True)
print "\n--- Changelist " + change_info.name + ":"
for filename in change_info.GetFiles():
print "".join(filename)
return 0
@no_args
def CMDdeleteempties():
"""Delete all changelists that have no files."""
print "\n--- Deleting:"
for cl in GetCLs():
change_info = ChangeInfo.Load(cl, GetRepositoryRoot(), True, True)
if not len(change_info.GetFiles()):
print change_info.name
change_info.Delete()
return 0
@no_args
def CMDnothave():
"""Lists files unknown to Subversion."""
for filename in UnknownFiles():
print "? " + "".join(filename)
return 0
@attrs(usage='<svn options>')
def CMDdiff(args):
"""Diffs all files in the changelist or all files that aren't in a CL."""
files = None
if args:
change_info = ChangeInfo.Load(args.pop(0), GetRepositoryRoot(), True, True)
files = change_info.GetFileNames()
else:
files = [f[1] for f in GetFilesNotInCL()]
root = GetRepositoryRoot()
cmd = ['svn', 'diff']
cmd.extend([os.path.join(root, x) for x in files])
cmd.extend(args)
return RunShellWithReturnCode(cmd, print_output=True)[1]
@no_args
def CMDsettings():
"""Prints code review settings for this checkout."""
# Force load settings
GetCodeReviewSetting("UNKNOWN")
del CODEREVIEW_SETTINGS['__just_initialized']
print '\n'.join(("%s: %s" % (str(k), str(v))
for (k,v) in CODEREVIEW_SETTINGS.iteritems()))
return 0
@need_change
def CMDdescription(change_info):
"""Prints the description of the specified change to stdout."""
print change_info.description
return 0
def CMDdelete(args):
"""Deletes a changelist."""
if not len(args) == 1:
ErrorExit('You need to pass a change list name')
filepath = GetChangelistInfoFile(args[0])
if not os.path.isfile(filepath):
ErrorExit('You need to pass a valid change list name')
os.remove(filepath)
return 0
def CMDtry(args):
"""Sends the change to the tryserver to do a test run on your code.
To send multiple changes as one path, use a comma-separated list of
changenames. Use 'gcl help try' for more information!"""
# When the change contains no file, send the "changename" positional
# argument to trychange.py.
# When the command is 'try' and --patchset is used, the patch to try
# is on the Rietveld server.
if not args:
ErrorExit("You need to pass a change list name")
if args[0].find(',') != -1:
change_info = LoadChangelistInfoForMultiple(args[0], GetRepositoryRoot(),
True, True)
else:
change_info = ChangeInfo.Load(args[0], GetRepositoryRoot(),
True, True)
props = change_info.RpcServer().get_issue_properties(
change_info.issue, False)
if props.get('private'):
ErrorExit('Cannot use trybots on a private issue')
if change_info.GetFiles():
args = args[1:]
else:
change_info = None
return TryChange(change_info, args, swallow_exception=False)
@attrs(usage='<old-name> <new-name>')
def CMDrename(args):
"""Renames an existing change."""
if len(args) != 2:
ErrorExit("Usage: gcl rename <old-name> <new-name>.")
src, dst = args
src_file = GetChangelistInfoFile(src)
if not os.path.isfile(src_file):
ErrorExit("Change '%s' does not exist." % src)
dst_file = GetChangelistInfoFile(dst)
if os.path.isfile(dst_file):
ErrorExit("Change '%s' already exists; pick a new name." % dst)
os.rename(src_file, dst_file)
print "Change '%s' renamed '%s'." % (src, dst)
return 0
def CMDpassthru(args):
"""Everything else that is passed into gcl we redirect to svn.
It assumes a change list name is passed and is converted with the files names.
"""
if not args or len(args) < 2:
ErrorExit("You need to pass a change list name for this svn fall-through "
"command")
cl_name = args[1]
args = ["svn", args[0]]
if len(args) > 1:
root = GetRepositoryRoot()
change_info = ChangeInfo.Load(cl_name, root, True, True)
args.extend([os.path.join(root, x) for x in change_info.GetFileNames()])
return RunShellWithReturnCode(args, print_output=True)[1]
def Command(name):
return getattr(sys.modules[__name__], 'CMD' + name, None)
def GenUsage(command):
"""Modify an OptParse object with the function's documentation."""
obj = Command(command)
display = command
more = getattr(obj, 'usage', '')
if command == 'help':
display = '<command>'
need_change_val = ''
if getattr(obj, 'need_change', None):
need_change_val = ' <change_list>'
options = ' [options]'
if getattr(obj, 'no_args', None):
options = ''
res = 'Usage: gcl %s%s%s %s\n\n' % (display, need_change_val, options, more)
res += re.sub('\n ', '\n', obj.__doc__)
return res
def CMDhelp(args):
"""Prints this help or help for the given command."""
if args and 'CMD' + args[0] in dir(sys.modules[__name__]):
print GenUsage(args[0])
# These commands defer to external tools so give this info too.
if args[0] == 'try':
TryChange(None, ['--help'], swallow_exception=False)
if args[0] == 'upload':
upload.RealMain(['upload.py', '--help'])
return 0
print GenUsage('help')
print sys.modules[__name__].__doc__
print 'version ' + __version__ + '\n'
print('Commands are:\n' + '\n'.join([
' %-12s %s' % (fn[3:], Command(fn[3:]).__doc__.split('\n')[0].strip())
for fn in dir(sys.modules[__name__]) if fn.startswith('CMD')]))
return 0
def main(argv):
if sys.hexversion < 0x02060000:
print >> sys.stderr, (
'\nYour python version %s is unsupported, please upgrade.\n' %
sys.version.split(' ', 1)[0])
return 2
sys.stderr.write('Warning: gcl is going away soon. Get off subversion!\n')
sys.stderr.write('See http://crbug.com/475321 for more details.\n')
if not argv:
argv = ['help']
command = Command(argv[0])
# Help can be run from anywhere.
if command == CMDhelp:
return command(argv[1:])
try:
GetRepositoryRoot()
except (gclient_utils.Error, subprocess2.CalledProcessError):
print >> sys.stderr, 'To use gcl, you need to be in a subversion checkout.'
return 1
# Create the directories where we store information about changelists if it
# doesn't exist.
try:
if not os.path.exists(GetInfoDir()):
os.mkdir(GetInfoDir())
if not os.path.exists(GetChangesDir()):
os.mkdir(GetChangesDir())
if not os.path.exists(GetCacheDir()):
os.mkdir(GetCacheDir())
if command:
return command(argv[1:])
# Unknown command, try to pass that to svn
return CMDpassthru(argv)
except (gclient_utils.Error, subprocess2.CalledProcessError), e:
print >> sys.stderr, 'Got an exception'
print >> sys.stderr, str(e)
return 1
except upload.ClientLoginError, e:
print >> sys.stderr, 'Got an exception logging in to Rietveld'
print >> sys.stderr, str(e)
return 1
except urllib2.HTTPError, e:
if e.code != 500:
raise
print >> sys.stderr, (
'AppEngine is misbehaving and returned HTTP %d, again. Keep faith '
'and retry or visit go/isgaeup.\n%s') % (e.code, str(e))
return 1
if __name__ == "__main__":
fix_encoding.fix_encoding()
try:
sys.exit(main(sys.argv[1:]))
except KeyboardInterrupt:
sys.stderr.write('interrupted\n')
sys.exit(1)
gclient_utils.py
View file @ 92bec4f5
......@@ -1056,7 +1056,7 @@ class ExecutionQueue(object):
work_queue.ready_cond.release()
def GetEditor(git, git_editor=None):
def GetEditor(git_editor=None):
"""Returns the most plausible editor to use.
In order of preference:
......@@ -1068,14 +1068,8 @@ def GetEditor(git, git_editor=None):
In the case of git-cl, this matches git's behaviour, except that it does not
include dumb terminal detection.
In the case of gcl, this matches svn's behaviour, except that it does not
accept a command-line flag or check the editor-cmd configuration variable.
"""
if git:
editor = os.environ.get('GIT_EDITOR') or git_editor
else:
editor = os.environ.get('SVN_EDITOR')
editor = os.environ.get('GIT_EDITOR') or git_editor
if not editor:
editor = os.environ.get('VISUAL')
if not editor:
......@@ -1105,7 +1099,7 @@ def RunEditor(content, git, git_editor=None):
fileobj.close()
try:
editor = GetEditor(git, git_editor=git_editor)
editor = GetEditor(git_editor=git_editor)
if not editor:
return None
cmd = '%s %s' % (editor, filename)
......
git_cl.py
View file @ 92bec4f5
......@@ -1025,7 +1025,7 @@ class Changelist(object):
def GetCCList(self):
"""Return the users cc'd on this CL.
Return is a string suitable for passing to gcl with the --cc flag.
Return is a string suitable for passing to git cl with the --cc flag.
"""
if self.cc is None:
base_cc = settings.GetDefaultCCList()
......
presubmit_support.py
View file @ 92bec4f5
......@@ -1009,32 +1009,6 @@ class SvnChange(Change):
scm = 'svn'
_changelists = None
def _GetChangeLists(self):
"""Get all change lists."""
if self._changelists == None:
previous_cwd = os.getcwd()
os.chdir(self.RepositoryRoot())
# Need to import here to avoid circular dependency.
import gcl
self._changelists = gcl.GetModifiedFiles()
os.chdir(previous_cwd)
return self._changelists
def GetAllModifiedFiles(self):
"""Get all modified files."""
changelists = self._GetChangeLists()
all_modified_files = []
for cl in changelists.values():
all_modified_files.extend(
[os.path.join(self.RepositoryRoot(), f[1]) for f in cl])
return all_modified_files
def GetModifiedFiles(self):
"""Get modified files in the current CL."""
changelists = self._GetChangeLists()
return [os.path.join(self.RepositoryRoot(), f[1])
for f in changelists[self.Name()]]
def AllFiles(self, root=None):
"""List all files under source control in the repo."""
root = root or self.RepositoryRoot()
......@@ -1413,7 +1387,7 @@ class PresubmitExecuter(object):
"""
Args:
change: The Change object.
committing: True if 'gcl commit' is running, False if 'gcl upload' is.
committing: True if 'git cl land' is running, False if 'git cl upload' is.
rietveld_obj: rietveld.Rietveld client object.
gerrit_obj: provides basic Gerrit codereview functionality.
dry_run: if true, some Checks will be skipped.
......@@ -1500,7 +1474,7 @@ def DoPresubmitChecks(change,
Args:
change: The Change object.
committing: True if 'gcl commit' is running, False if 'gcl upload' is.
committing: True if 'git cl land' is running, False if 'git cl upload' is.
verbose: Prints debug info.
output_stream: A stream to write output from presubmit tests to.
input_stream: A stream to read input from the user.
......
tests/trychange_unittest.py deleted 100755 → 0
View file @ 12fa6ff6
#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Unit tests for trychange.py."""
import os
import sys
import unittest
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from testing_support.super_mox import SuperMoxTestBase
import subprocess2
import trychange
class TryChangeTestsBase(SuperMoxTestBase):
"""Setups and tear downs the mocks but doesn't test anything as-is."""
def setUp(self):
SuperMoxTestBase.setUp(self)
self.mox.StubOutWithMock(subprocess2, 'communicate')
self.mox.StubOutWithMock(trychange, 'RunGit')
self.mox.StubOutWithMock(trychange.scm.GIT, 'Capture')
self.mox.StubOutWithMock(trychange.scm.GIT, 'GenerateDiff')
self.mox.StubOutWithMock(trychange.scm.GIT, 'GetCheckoutRoot')
self.mox.StubOutWithMock(trychange.scm.GIT, 'GetEmail')
self.mox.StubOutWithMock(trychange.scm.GIT, 'GetPatchName')
self.mox.StubOutWithMock(trychange.scm.GIT, 'GetUpstreamBranch')
self.mox.StubOutWithMock(trychange.scm.SVN, 'GenerateDiff')
self.mox.StubOutWithMock(trychange.scm.SVN, 'GetCheckoutRoot')
self.mox.StubOutWithMock(trychange.scm.SVN, 'GetEmail')
self.fake_root = self.Dir()
self.expected_files = ['foo.txt', 'bar.txt']
self.options = trychange.optparse.Values()
self.options.files = self.expected_files
self.options.diff = None
self.options.name = None
self.options.email = None
self.options.exclude = []
class TryChangeUnittest(TryChangeTestsBase):
"""General trychange.py tests."""
def testMembersChanged(self):
members = [
'DieWithError', 'EPILOG', 'Escape', 'GIT', 'GIT_PATCH_DIR_BASENAME',
'GetMungedDiff', 'GuessVCS', 'GIT_BRANCH_FILE',
'HELP_STRING', 'Error', 'InvalidScript', 'NoTryServerAccess',
'OptionParser', 'PrintSuccess',
'RunCommand', 'RunGit', 'SCM', 'SVN', 'TryChange', 'USAGE', 'contextlib',
'datetime', 'errno', 'fix_encoding', 'gcl', 'gclient_utils',
'gerrit_util', 'gen_parser',
'getpass', 'itertools', 'json', 'logging', 'optparse', 'os', 'posixpath',
're', 'scm', 'shutil', 'subprocess2', 'sys', 'tempfile', 'urllib',
'urllib2', 'urlparse']
# If this test fails, you should add the relevant test.
self.compareMembers(trychange, members)
class TryChangeSimpleTest(unittest.TestCase):
# Doesn't require supermox to run.
def test_flags(self):
cmd = [
'--bot', 'bot1,bot2',
'--testfilter', 'test1',
'--testfilter', 'test2',
'--user', 'joe',
'--email', 'joe@example.com',
]
options, args = trychange.gen_parser(None).parse_args(cmd)
self.assertEquals([], args)
# pylint: disable=W0212
bot_spec = trychange._ParseBotList(options.bot, options.testfilter)
if options.testfilter:
bot_spec = trychange._ApplyTestFilter(options.testfilter, bot_spec)
values = trychange._ParseSendChangeOptions(bot_spec, options)
self.assertEquals(
[
('user', 'joe'),
('name', None),
('email', 'joe@example.com'),
('bot', 'bot1:test1,test2'),
('bot', 'bot2:test1,test2'),
],
values)
def test_flags_bad_combination(self):
cmd = [
'--bot', 'bot1:test1',
'--testfilter', 'test2',
]
options, args = trychange.gen_parser(None).parse_args(cmd)
self.assertEquals([], args)
try:
# pylint: disable=W0212
trychange._ParseBotList(options.bot, options.testfilter)
self.fail()
except ValueError:
pass
class SVNUnittest(TryChangeTestsBase):
"""trychange.SVN tests."""
def testMembersChanged(self):
members = [
'AutomagicalSettings', 'CaptureStatus', 'GetCodeReviewSetting',
'ReadRootFile', 'GenerateDiff', 'GetFileNames', 'files', 'file_tuples',
]
# If this test fails, you should add the relevant test.
self.compareMembers(trychange.SVN, members)
def testBasic(self):
# pylint: disable=E1103
trychange.os.path.abspath(self.fake_root).AndReturn(self.fake_root)
trychange.scm.SVN.GetCheckoutRoot(self.fake_root).AndReturn(self.fake_root)
trychange.scm.SVN.GenerateDiff(['foo.txt', 'bar.txt'],
self.fake_root,
full_move=True,
revision=None).AndReturn('A diff')
trychange.scm.SVN.GetEmail(self.fake_root).AndReturn('georges@example.com')
self.mox.ReplayAll()
svn = trychange.SVN(self.options, self.fake_root, self.options.files)
self.assertEqual(svn.GetFileNames(), self.expected_files)
self.assertEqual(svn.checkout_root, self.fake_root)
self.assertEqual(svn.GenerateDiff(), 'A diff')
class GITUnittest(TryChangeTestsBase):
"""trychange.GIT tests."""
def testMembersChanged(self):
members = [
'AutomagicalSettings', 'CaptureStatus', 'GetCodeReviewSetting',
'ReadRootFile', 'GenerateDiff', 'GetFileNames', 'files', 'file_tuples',
]
# If this test fails, you should add the relevant test.
self.compareMembers(trychange.GIT, members)
def testBasic(self):
# pylint: disable=E1103
trychange.os.path.abspath(self.fake_root).AndReturn(self.fake_root)
trychange.scm.GIT.GetCheckoutRoot(self.fake_root).AndReturn(self.fake_root)
trychange.scm.GIT.GetUpstreamBranch(self.fake_root).AndReturn('somewhere')
trychange.RunGit(['diff-index', 'HEAD'])
trychange.scm.GIT.GenerateDiff(self.fake_root,
full_move=True,
files=['foo.txt', 'bar.txt'],
branch='somewhere').AndReturn('A diff')
trychange.scm.GIT.GetPatchName(self.fake_root).AndReturn('bleh-1233')
trychange.scm.GIT.GetEmail(self.fake_root).AndReturn('georges@example.com')
self.mox.ReplayAll()
git = trychange.GIT(self.options, self.fake_root, self.options.files)
self.assertEqual(git.GetFileNames(), self.expected_files)
self.assertEqual(git.checkout_root, self.fake_root)
self.assertEqual(git.GenerateDiff(), 'A diff')
if __name__ == '__main__':
unittest.main()
trychange.py deleted 100755 → 0
View file @ 12fa6ff6
#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Client-side script to send a try job to the try server. It communicates to
the try server by either writting to a svn/git repository or by directly
connecting to the server by HTTP.
"""
import contextlib
import datetime
import errno
import getpass
import itertools
import json
import logging
import optparse
import os
import posixpath
import re
import shutil
import sys
import tempfile
import urllib
import urllib2
import urlparse
import fix_encoding
import gcl
import gclient_utils
import gerrit_util
import scm
import subprocess2
__version__ = '1.2'
# Constants
HELP_STRING = "Sorry, Tryserver is not available."
USAGE = r"""%prog [options]
Client-side script to send a try job to the try server. It communicates to
the try server by either writting to a svn repository or by directly connecting
to the server by HTTP."""
EPILOG = """
Examples:
Send a patch directly from rietveld:
%(prog)s -R codereview.chromium.org/1337
--email recipient@example.com --root src
Try a change against a particular revision:
%(prog)s -r 123
Try a change including changes to a sub repository:
%(prog)s -s third_party/WebKit
A git patch off a web site (git inserts a/ and b/) and fix the base dir:
%(prog)s --url http://url/to/patch.diff --patchlevel 1 --root src
Use svn to store the try job, specify an alternate email address and use a
premade diff file on the local drive:
%(prog)s --email user@example.com
--svn_repo svn://svn.chromium.org/chrome-try/try --diff foo.diff
Running only on a 'mac' slave with revision 123 and clobber first; specify
manually the 3 source files to use for the try job:
%(prog)s --bot mac --revision 123 --clobber -f src/a.cc -f src/a.h
-f include/b.h
"""
GIT_PATCH_DIR_BASENAME = os.path.join('git-try', 'patches-git')
GIT_BRANCH_FILE = 'ref'
_GIT_PUSH_ATTEMPTS = 3
def DieWithError(message):
print >> sys.stderr, message
sys.exit(1)
def RunCommand(args, error_ok=False, error_message=None, **kwargs):
try:
return subprocess2.check_output(args, shell=False, **kwargs)
except subprocess2.CalledProcessError, e:
if not error_ok:
DieWithError(
'Command "%s" failed.\n%s' % (
' '.join(args), error_message or e.stdout or ''))
return e.stdout
def RunGit(args, **kwargs):
"""Returns stdout."""
return RunCommand(['git'] + args, **kwargs)
class Error(Exception):
"""An error during a try job submission.
For this error, trychange.py does not display stack trace, only message
"""
class InvalidScript(Error):
def __str__(self):
return self.args[0] + '\n' + HELP_STRING
class NoTryServerAccess(Error):
def __str__(self):
return self.args[0] + '\n' + HELP_STRING
def Escape(name):
"""Escapes characters that could interfere with the file system or try job
parsing.
"""
return re.sub(r'[^\w#-]', '_', name)
class SCM(object):
"""Simplistic base class to implement one function: ProcessOptions."""
def __init__(self, options, path, file_list):
items = path.split('@')
assert len(items) <= 2
self.checkout_root = os.path.abspath(items[0])
items.append(None)
self.diff_against = items[1]
self.options = options
# Lazy-load file list from the SCM unless files were specified in options.
self._files = None
self._file_tuples = None
if file_list:
self._files = file_list
self._file_tuples = [('M', f) for f in self.files]
self.options.files = None
self.codereview_settings = None
self.codereview_settings_file = 'codereview.settings'
self.toplevel_root = None
def GetFileNames(self):
"""Return the list of files in the diff."""
return self.files
def GetCodeReviewSetting(self, key):
"""Returns a value for the given key for this repository.
Uses gcl-style settings from the repository.
"""
if gcl:
gcl_setting = gcl.GetCodeReviewSetting(key)
if gcl_setting != '':
return gcl_setting
if self.codereview_settings is None:
self.codereview_settings = {}
settings_file = self.ReadRootFile(self.codereview_settings_file)
if settings_file:
for line in settings_file.splitlines():
if not line or line.lstrip().startswith('#'):
continue
k, v = line.split(":", 1)
self.codereview_settings[k.strip()] = v.strip()
return self.codereview_settings.get(key, '')
def _GclStyleSettings(self):
"""Set default settings based on the gcl-style settings from the repository.
The settings in the self.options object will only be set if no previous
value exists (i.e. command line flags to the try command will override the
settings in codereview.settings).
"""
settings = {
'port': self.GetCodeReviewSetting('TRYSERVER_HTTP_PORT'),
'host': self.GetCodeReviewSetting('TRYSERVER_HTTP_HOST'),
'svn_repo': self.GetCodeReviewSetting('TRYSERVER_SVN_URL'),
'gerrit_url': self.GetCodeReviewSetting('TRYSERVER_GERRIT_URL'),
'git_repo': self.GetCodeReviewSetting('TRYSERVER_GIT_URL'),
'project': self.GetCodeReviewSetting('TRYSERVER_PROJECT'),
# Primarily for revision=auto
'revision': self.GetCodeReviewSetting('TRYSERVER_REVISION'),
'root': self.GetCodeReviewSetting('TRYSERVER_ROOT'),
'patchlevel': self.GetCodeReviewSetting('TRYSERVER_PATCHLEVEL'),
}
logging.info('\n'.join(['%s: %s' % (k, v)
for (k, v) in settings.iteritems() if v]))
for (k, v) in settings.iteritems():
# Avoid overwriting options already set using command line flags.
if v and getattr(self.options, k) is None:
setattr(self.options, k, v)
def AutomagicalSettings(self):
"""Determines settings based on supported code review and checkout tools.
"""
# Try to find gclient or repo root first.
if not self.options.no_search:
self.toplevel_root = gclient_utils.FindGclientRoot(self.checkout_root)
if self.toplevel_root:
logging.info('Found .gclient at %s' % self.toplevel_root)
else:
self.toplevel_root = gclient_utils.FindFileUpwards(
os.path.join('..', '.repo'), self.checkout_root)
if self.toplevel_root:
logging.info('Found .repo dir at %s'
% os.path.dirname(self.toplevel_root))
# Parse TRYSERVER_* settings from codereview.settings before falling back
# on setting self.options.root manually further down. Otherwise
# TRYSERVER_ROOT would never be used in codereview.settings.
self._GclStyleSettings()
if self.toplevel_root and not self.options.root:
assert os.path.abspath(self.toplevel_root) == self.toplevel_root
self.options.root = gclient_utils.PathDifference(self.toplevel_root,
self.checkout_root)
else:
self._GclStyleSettings()
def ReadRootFile(self, filename):
cur = self.checkout_root
root = self.toplevel_root or self.checkout_root
assert cur.startswith(root), (root, cur)
while cur.startswith(root):
filepath = os.path.join(cur, filename)
if os.path.isfile(filepath):
logging.info('Found %s at %s' % (filename, cur))
return gclient_utils.FileRead(filepath)
cur = os.path.dirname(cur)
logging.warning('Didn\'t find %s' % filename)
return None
def _SetFileTuples(self, file_tuples):
excluded = ['!', '?', 'X', ' ', '~']
def Excluded(f):
if f[0][0] in excluded:
return True
for r in self.options.exclude:
if re.search(r, f[1]):
logging.info('Ignoring "%s"' % f[1])
return True
return False
self._file_tuples = [f for f in file_tuples if not Excluded(f)]
self._files = [f[1] for f in self._file_tuples]
def CaptureStatus(self):
"""Returns the 'svn status' emulated output as an array of (status, file)
tuples."""
raise NotImplementedError(
"abstract method -- subclass %s must override" % self.__class__)
@property
def files(self):
if self._files is None:
self._SetFileTuples(self.CaptureStatus())
return self._files
@property
def file_tuples(self):
if self._file_tuples is None:
self._SetFileTuples(self.CaptureStatus())
return self._file_tuples
class SVN(SCM):
"""Gathers the options and diff for a subversion checkout."""
def __init__(self, *args, **kwargs):
SCM.__init__(self, *args, **kwargs)
self.checkout_root = scm.SVN.GetCheckoutRoot(self.checkout_root)
if not self.options.email:
# Assumes the svn credential is an email address.
self.options.email = scm.SVN.GetEmail(self.checkout_root)
logging.info("SVN(%s)" % self.checkout_root)
def ReadRootFile(self, filename):
data = SCM.ReadRootFile(self, filename)
if data:
return data
# Try to search on the subversion repository for the file.
if not gcl:
return None
data = gcl.GetCachedFile(filename)
logging.debug('%s:\n%s' % (filename, data))
return data
def CaptureStatus(self):
return scm.SVN.CaptureStatus(None, self.checkout_root)
def GenerateDiff(self):
"""Returns a string containing the diff for the given file list.
The files in the list should either be absolute paths or relative to the
given root.
"""
return scm.SVN.GenerateDiff(self.files, self.checkout_root, full_move=True,
revision=self.diff_against)
class GIT(SCM):
"""Gathers the options and diff for a git checkout."""
def __init__(self, *args, **kwargs):
SCM.__init__(self, *args, **kwargs)
self.checkout_root = scm.GIT.GetCheckoutRoot(self.checkout_root)
if not self.options.name:
self.options.name = scm.GIT.GetPatchName(self.checkout_root)
if not self.options.email:
self.options.email = scm.GIT.GetEmail(self.checkout_root)
if not self.diff_against:
self.diff_against = scm.GIT.GetUpstreamBranch(self.checkout_root)
if not self.diff_against:
raise NoTryServerAccess(
"Unable to determine default branch to diff against. "
"Verify this branch is set up to track another"
"(via the --track argument to \"git checkout -b ...\"")
logging.info("GIT(%s)" % self.checkout_root)
def CaptureStatus(self):
return scm.GIT.CaptureStatus(
[],
self.checkout_root.replace(os.sep, '/'),
self.diff_against)
def GenerateDiff(self):
if RunGit(['diff-index', 'HEAD']):
print 'Cannot try with a dirty tree. You must commit locally first.'
return None
return scm.GIT.GenerateDiff(
self.checkout_root,
files=self.files,
full_move=True,
branch=self.diff_against)
def _ParseBotList(botlist, testfilter):
"""Parses bot configurations from a list of strings."""
bots = []
if testfilter:
for bot in itertools.chain.from_iterable(botspec.split(',')
for botspec in botlist):
tests = set()
if ':' in bot:
if bot.endswith(':compile'):
tests |= set(['compile'])
else:
raise ValueError(
'Can\'t use both --testfilter and --bot builder:test formats '
'at the same time')
bots.append((bot, tests))
else:
for botspec in botlist:
botname = botspec.split(':')[0]
tests = set()
if ':' in botspec:
tests |= set(filter(None, botspec.split(':')[1].split(',')))
bots.append((botname, tests))
return bots
def _ApplyTestFilter(testfilter, bot_spec):
"""Applies testfilter from CLI.
Specifying a testfilter strips off any builder-specified tests (except for
compile).
"""
if testfilter:
return [(botname, set(testfilter) | (tests & set(['compile'])))
for botname, tests in bot_spec]
else:
return bot_spec
def _GenTSBotSpec(checkouts, change, changed_files, options):
bot_spec = []
# Get try slaves from PRESUBMIT.py files if not specified.
# Even if the diff comes from options.url, use the local checkout for bot
# selection.
try:
import presubmit_support
root_presubmit = checkouts[0].ReadRootFile('PRESUBMIT.py')
if not change:
if not changed_files:
changed_files = checkouts[0].file_tuples
change = presubmit_support.Change(options.name,
'',
checkouts[0].checkout_root,
changed_files,
options.issue,
options.patchset,
options.email)
masters = presubmit_support.DoGetTryMasters(
change,
checkouts[0].GetFileNames(),
checkouts[0].checkout_root,
root_presubmit,
options.project,
options.verbose,
sys.stdout)
# Compatibility for old checkouts and bots that were on tryserver.chromium.
try_bots = masters.get('tryserver.chromium', [])
# Compatibility for checkouts that are not using tryserver.chromium
# but are stuck with git-try or gcl-try.
if not try_bots and len(masters) == 1:
try_bots = masters.values()[0]
if try_bots:
old_style = filter(lambda x: isinstance(x, basestring), try_bots)
new_style = filter(lambda x: isinstance(x, tuple), try_bots)
# _ParseBotList's testfilter is set to None otherwise it will complain.
bot_spec = _ApplyTestFilter(options.testfilter,
_ParseBotList(old_style, None))
bot_spec.extend(_ApplyTestFilter(options.testfilter, new_style))
except ImportError:
pass
return bot_spec
def _ParseSendChangeOptions(bot_spec, options):
"""Parse common options passed to _SendChangeHTTP, _SendChangeSVN and
_SendChangeGit.
"""
values = [
('user', options.user),
('name', options.name),
]
# A list of options to copy.
optional_values = (
'email',
'revision',
'root',
'patchlevel',
'issue',
'patchset',
'target',
'project',
)
for option_name in optional_values:
value = getattr(options, option_name)
if value:
values.append((option_name, value))
# Not putting clobber to optional_names
# because it used to have lower-case 'true'.
if options.clobber:
values.append(('clobber', 'true'))
for bot, tests in bot_spec:
values.append(('bot', ('%s:%s' % (bot, ','.join(tests)))))
return values
def _SendChangeHTTP(bot_spec, options):
"""Send a change to the try server using the HTTP protocol."""
if not options.host:
raise NoTryServerAccess('Please use the --host option to specify the try '
'server host to connect to.')
if not options.port:
raise NoTryServerAccess('Please use the --port option to specify the try '
'server port to connect to.')
values = _ParseSendChangeOptions(bot_spec, options)
values.append(('patch', options.diff))
url = 'http://%s:%s/send_try_patch' % (options.host, options.port)
logging.info('Sending by HTTP')
logging.info(''.join("%s=%s\n" % (k, v) for k, v in values))
logging.info(url)
logging.info(options.diff)
if options.dry_run:
return
try:
logging.info('Opening connection...')
connection = urllib2.urlopen(url, urllib.urlencode(values))
logging.info('Done')
except IOError, e:
logging.info(str(e))
if bot_spec and len(e.args) > 2 and e.args[2] == 'got a bad status line':
raise NoTryServerAccess('%s is unaccessible. Bad --bot argument?' % url)
else:
raise NoTryServerAccess('%s is unaccessible. Reason: %s' % (url,
str(e.args)))
if not connection:
raise NoTryServerAccess('%s is unaccessible.' % url)
logging.info('Reading response...')
response = connection.read()
logging.info('Done')
if response != 'OK':
raise NoTryServerAccess('%s is unaccessible. Got:\n%s' % (url, response))
PrintSuccess(bot_spec, options)
@contextlib.contextmanager
def _TempFilename(name, contents=None):
"""Create a temporary directory, append the specified name and yield.
In contrast to NamedTemporaryFile, does not keep the file open.
Deletes the file on __exit__.
"""
temp_dir = tempfile.mkdtemp(prefix=name)
try:
path = os.path.join(temp_dir, name)
if contents is not None:
with open(path, 'wb') as f:
f.write(contents)
yield path
finally:
shutil.rmtree(temp_dir, True)
@contextlib.contextmanager
def _PrepareDescriptionAndPatchFiles(description, options):
"""Creates temporary files with description and patch.
__enter__ called on the return value returns a tuple of patch_filename and
description_filename.
Args:
description: contents of description file.
options: patchset options object. Must have attributes: user,
name (of patch) and diff (contents of patch).
"""
current_time = str(datetime.datetime.now()).replace(':', '.')
patch_basename = '%s.%s.%s.diff' % (Escape(options.user),
Escape(options.name), current_time)
with _TempFilename('description', description) as description_filename:
with _TempFilename(patch_basename, options.diff) as patch_filename:
yield patch_filename, description_filename
def _SendChangeSVN(bot_spec, options):
"""Send a change to the try server by committing a diff file on a subversion
server."""
if not options.svn_repo:
raise NoTryServerAccess('Please use the --svn_repo option to specify the'
' try server svn repository to connect to.')
values = _ParseSendChangeOptions(bot_spec, options)
description = ''.join("%s=%s\n" % (k, v) for k, v in values)
logging.info('Sending by SVN')
logging.info(description)
logging.info(options.svn_repo)
logging.info(options.diff)
if options.dry_run:
return
with _PrepareDescriptionAndPatchFiles(description, options) as (
patch_filename, description_filename):
if sys.platform == "cygwin":
# Small chromium-specific issue here:
# git-try uses /usr/bin/python on cygwin but svn.bat will be used
# instead of /usr/bin/svn by default. That causes bad things(tm) since
# Windows' svn.exe has no clue about cygwin paths. Hence force to use
# the cygwin version in this particular context.
exe = "/usr/bin/svn"
else:
exe = "svn"
patch_dir = os.path.dirname(patch_filename)
command = [exe, 'import', '-q', patch_dir, options.svn_repo, '--file',
description_filename]
if scm.SVN.AssertVersion("1.5")[0]:
command.append('--no-ignore')
try:
subprocess2.check_call(command)
except subprocess2.CalledProcessError, e:
raise NoTryServerAccess(str(e))
PrintSuccess(bot_spec, options)
def _GetPatchGitRepo(git_url):
"""Gets a path to a Git repo with patches.
Stores patches in .git/git-try/patches-git directory, a git repo. If it
doesn't exist yet or its origin URL is different, cleans up and clones it.
If it existed before, then pulls changes.
Does not support SVN repo.
Returns a path to the directory with patches.
"""
git_dir = scm.GIT.GetGitDir(os.getcwd())
patch_dir = os.path.join(git_dir, GIT_PATCH_DIR_BASENAME)
logging.info('Looking for git repo for patches')
# Is there already a repo with the expected url or should we clone?
clone = True
if os.path.exists(patch_dir) and scm.GIT.IsInsideWorkTree(patch_dir):
existing_url = scm.GIT.Capture(
['config', '--local', 'remote.origin.url'],
cwd=patch_dir)
clone = existing_url != git_url
if clone:
if os.path.exists(patch_dir):
logging.info('Cleaning up')
shutil.rmtree(patch_dir, True)
logging.info('Cloning patch repo')
scm.GIT.Capture(['clone', git_url, GIT_PATCH_DIR_BASENAME], cwd=git_dir)
email = scm.GIT.GetEmail(cwd=os.getcwd())
scm.GIT.Capture(['config', '--local', 'user.email', email], cwd=patch_dir)
else:
if scm.GIT.IsWorkTreeDirty(patch_dir):
logging.info('Work dir is dirty: hard reset!')
scm.GIT.Capture(['reset', '--hard'], cwd=patch_dir)
logging.info('Updating patch repo')
scm.GIT.Capture(['pull', 'origin', 'master'], cwd=patch_dir)
return os.path.abspath(patch_dir)
def _SendChangeGit(bot_spec, options):
"""Sends a change to the try server by committing a diff file to a GIT repo.
Creates a temp orphan branch, commits patch.diff, creates a ref pointing to
that commit, deletes the temp branch, checks master out, adds 'ref' file
containing the name of the new ref, pushes master and the ref to the origin.
TODO: instead of creating a temp branch, use git-commit-tree.
"""
if not options.git_repo:
raise NoTryServerAccess('Please use the --git_repo option to specify the '
'try server git repository to connect to.')
values = _ParseSendChangeOptions(bot_spec, options)
comment_subject = '%s.%s' % (options.user, options.name)
comment_body = ''.join("%s=%s\n" % (k, v) for k, v in values)
description = '%s\n\n%s' % (comment_subject, comment_body)
logging.info('Sending by GIT')
logging.info(description)
logging.info(options.git_repo)
logging.info(options.diff)
if options.dry_run:
return
patch_dir = _GetPatchGitRepo(options.git_repo)
def patch_git(*args):
return scm.GIT.Capture(list(args), cwd=patch_dir)
def add_and_commit(filename, comment_filename):
patch_git('add', filename)
patch_git('commit', '-F', comment_filename)
assert scm.GIT.IsInsideWorkTree(patch_dir)
assert not scm.GIT.IsWorkTreeDirty(patch_dir)
with _PrepareDescriptionAndPatchFiles(description, options) as (
patch_filename, description_filename):
logging.info('Committing patch')
temp_branch = 'tmp_patch'
target_ref = 'refs/patches/%s/%s' % (
Escape(options.user),
os.path.basename(patch_filename).replace(' ','_'))
target_filename = os.path.join(patch_dir, 'patch.diff')
branch_file = os.path.join(patch_dir, GIT_BRANCH_FILE)
patch_git('checkout', 'master')
try:
# Try deleting an existing temp branch, if any.
try:
patch_git('branch', '-D', temp_branch)
logging.debug('Deleted an existing temp branch.')
except subprocess2.CalledProcessError:
pass
# Create a new branch and put the patch there.
patch_git('checkout', '--orphan', temp_branch)
patch_git('reset')
patch_git('clean', '-f')
shutil.copyfile(patch_filename, target_filename)
add_and_commit(target_filename, description_filename)
assert not scm.GIT.IsWorkTreeDirty(patch_dir)
# Create a ref and point it to the commit referenced by temp_branch.
patch_git('update-ref', target_ref, temp_branch)
# Delete the temp ref.
patch_git('checkout', 'master')
patch_git('branch', '-D', temp_branch)
# Update the branch file in the master.
def update_branch():
with open(branch_file, 'w') as f:
f.write(target_ref)
add_and_commit(branch_file, description_filename)
update_branch()
# Push master and target_ref to origin.
logging.info('Pushing patch')
for attempt in xrange(_GIT_PUSH_ATTEMPTS):
try:
patch_git('push', 'origin', 'master', target_ref)
except subprocess2.CalledProcessError as e:
is_last = attempt == _GIT_PUSH_ATTEMPTS - 1
if is_last:
raise NoTryServerAccess(str(e))
# Fetch, reset, update branch file again.
patch_git('fetch', 'origin')
patch_git('reset', '--hard', 'origin/master')
update_branch()
except subprocess2.CalledProcessError, e:
# Restore state.
patch_git('checkout', 'master')
patch_git('reset', '--hard', 'origin/master')
raise
PrintSuccess(bot_spec, options)
def _SendChangeGerrit(bot_spec, options):
"""Posts a try job to a Gerrit change.
Reads Change-Id from the HEAD commit, resolves the current revision, checks
that local revision matches the uploaded one, posts a try job in form of a
message, sets Tryjob-Request label to 1.
Gerrit message format: starts with !tryjob, optionally followed by a try job
definition in JSON format:
buildNames: list of strings specifying build names.
build_properties: a dict of build properties.
"""
logging.info('Sending by Gerrit')
if not options.gerrit_url:
raise NoTryServerAccess('Please use --gerrit_url option to specify the '
'Gerrit instance url to connect to')
gerrit_host = urlparse.urlparse(options.gerrit_url).hostname
logging.debug('Gerrit host: %s' % gerrit_host)
def GetChangeId(commmitish):
"""Finds Change-ID of the HEAD commit."""
CHANGE_ID_RGX = '^Change-Id: (I[a-f0-9]{10,})'
comment = scm.GIT.Capture(['log', '-1', commmitish, '--format=%b'],
cwd=os.getcwd())
change_id_match = re.search(CHANGE_ID_RGX, comment, re.I | re.M)
if not change_id_match:
raise Error('Change-Id was not found in the HEAD commit. Make sure you '
'have a Git hook installed that generates and inserts a '
'Change-Id into a commit message automatically.')
change_id = change_id_match.group(1)
return change_id
def FormatMessage():
# Build job definition.
job_def = {}
build_properties = {}
if options.testfilter:
build_properties['testfilter'] = options.testfilter
builderNames = [builder for builder, _ in bot_spec]
if builderNames:
job_def['builderNames'] = builderNames
if build_properties:
job_def['build_properties'] = build_properties
# Format message.
msg = '!tryjob'
if job_def:
msg = '%s %s' % (msg, json.dumps(job_def, sort_keys=True))
return msg
def PostTryjob(message):
logging.info('Posting gerrit message: %s' % message)
if not options.dry_run:
# Post a message and set TryJob=1 label.
try:
gerrit_util.SetReview(gerrit_host, change_id, msg=message,
labels={'Tryjob-Request': 1})
except gerrit_util.GerritError, e:
if e.http_status == 400:
raise Error(e.message)
else:
raise
head_sha = scm.GIT.Capture(['log', '-1', '--format=%H'], cwd=os.getcwd())
change_id = GetChangeId(head_sha)
try:
# Check that the uploaded revision matches the local one.
changes = gerrit_util.GetChangeCurrentRevision(gerrit_host, change_id)
except gerrit_util.GerritAuthenticationError, e:
raise NoTryServerAccess(e.message)
assert len(changes) <= 1, 'Multiple changes with id %s' % change_id
if not changes:
raise Error('A change %s was not found on the server. Was it uploaded?' %
change_id)
logging.debug('Found Gerrit change: %s' % changes[0])
if changes[0]['current_revision'] != head_sha:
raise Error('Please upload your latest local changes to Gerrit.')
# Post a try job.
message = FormatMessage()
PostTryjob(message)
change_url = urlparse.urljoin(options.gerrit_url,
'/#/c/%s' % changes[0]['_number'])
print('A try job was posted on change %s' % change_url)
def PrintSuccess(bot_spec, options):
if not options.dry_run:
text = 'Patch \'%s\' sent to try server' % options.name
if bot_spec:
text += ': %s' % ', '.join(
'%s:%s' % (b[0], ','.join(b[1])) for b in bot_spec)
print(text)
def GuessVCS(options, path, file_list):
"""Helper to guess the version control system.
NOTE: Very similar to upload.GuessVCS. Doesn't look for hg since we don't
support it yet.
This examines the path directory, guesses which SCM we're using, and
returns an instance of the appropriate class. Exit with an error if we can't
figure it out.
Returns:
A SCM instance. Exits if the SCM can't be guessed.
"""
__pychecker__ = 'no-returnvalues'
real_path = path.split('@')[0]
logging.info("GuessVCS(%s)" % path)
# Subversion has a .svn in all working directories.
if os.path.isdir(os.path.join(real_path, '.svn')):
return SVN(options, path, file_list)
# Git has a command to test if you're in a git tree.
# Try running it, but don't die if we don't have git installed.
try:
subprocess2.check_output(
['git', 'rev-parse', '--is-inside-work-tree'], cwd=real_path,
stderr=subprocess2.VOID)
return GIT(options, path, file_list)
except OSError, e:
if e.errno != errno.ENOENT:
raise
except subprocess2.CalledProcessError, e:
if e.returncode != errno.ENOENT and e.returncode != 128:
# ENOENT == 2 = they don't have git installed.
# 128 = git error code when not in a repo.
logging.warning('Unexpected error code: %s' % e.returncode)
raise
raise NoTryServerAccess(
( 'Could not guess version control system for %s.\n'
'Are you in a working copy directory?') % path)
def GetMungedDiff(path_diff, diff):
# Munge paths to match svn.
changed_files = []
for i in range(len(diff)):
if diff[i].startswith('--- ') or diff[i].startswith('+++ '):
new_file = posixpath.join(path_diff, diff[i][4:]).replace('\\', '/')
if diff[i].startswith('--- '):
file_path = new_file.split('\t')[0].strip()
if file_path.startswith('a/'):
file_path = file_path[2:]
changed_files.append(('M', file_path))
diff[i] = diff[i][0:4] + new_file
return (diff, changed_files)
class OptionParser(optparse.OptionParser):
def format_epilog(self, _):
"""Removes epilog formatting."""
return self.epilog or ''
def gen_parser(prog):
# Parse argv
parser = OptionParser(usage=USAGE, version=__version__, prog=prog)
parser.add_option("-v", "--verbose", action="count", default=0,
help="Prints debugging infos")
group = optparse.OptionGroup(parser, "Result and status")
group.add_option("-u", "--user", default=getpass.getuser(),
help="Owner user name [default: %default]")
group.add_option("-e", "--email",
default=os.environ.get('TRYBOT_RESULTS_EMAIL_ADDRESS',
os.environ.get('EMAIL_ADDRESS')),
help="Email address where to send the results. Use either "
"the TRYBOT_RESULTS_EMAIL_ADDRESS environment "
"variable or EMAIL_ADDRESS to set the email address "
"the try bots report results to [default: %default]")
group.add_option("-n", "--name",
help="Descriptive name of the try job")
group.add_option("--issue", type='int',
help="Update rietveld issue try job status")
group.add_option("--patchset", type='int',
help="Update rietveld issue try job status. This is "
"optional if --issue is used, In that case, the "
"latest patchset will be used.")
group.add_option("--dry_run", action='store_true',
help="Don't send the try job. This implies --verbose, so "
"it will print the diff.")
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Try job options")
group.add_option(
"-b", "--bot", action="append",
help=("IMPORTANT: specify ONE builder per --bot flag. Use it multiple "
"times to specify multiple builders. ex: "
"'-bwin_rel:ui_tests,webkit_unit_tests -bwin_layout'. See "
"the try server waterfall for the builders name and the tests "
"available. Can also be used to specify gtest_filter, e.g. "
"-bwin_rel:base_unittests:ValuesTest.*Value"))
group.add_option("-B", "--print_bots", action="store_true",
help="Print bots we would use (e.g. from PRESUBMIT.py)"
" and exit. Do not send patch. Like --dry_run"
" but less verbose.")
group.add_option("-r", "--revision",
help="Revision to use for the try job. If 'auto' is "
"specified, it is resolved to the revision a patch is "
"generated against (Git only). Default: the "
"revision will be determined by the try server; see "
"its waterfall for more info")
group.add_option("-c", "--clobber", action="store_true",
help="Force a clobber before building; e.g. don't do an "
"incremental build")
# TODO(maruel): help="Select a specific configuration, usually 'debug' or "
# "'release'"
group.add_option("--target", help=optparse.SUPPRESS_HELP)
group.add_option("--project",
help="Override which project to use. Projects are defined "
"server-side to define what default bot set to use")
group.add_option(
"-t", "--testfilter", action="append", default=[],
help=("Apply a testfilter to all the selected builders. Unless the "
"builders configurations are similar, use multiple "
"--bot <builder>:<test> arguments."))
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Patch to run")
group.add_option("-f", "--file", default=[], dest="files",
metavar="FILE", action="append",
help="Use many times to list the files to include in the "
"try, relative to the repository root")
group.add_option("--diff",
help="File containing the diff to try")
group.add_option("--url",
help="Url where to grab a patch, e.g. "
"http://example.com/x.diff")
group.add_option("-R", "--rietveld_url", default="codereview.chromium.org",
metavar="URL",
help="Has 2 usages, both refer to the rietveld instance: "
"Specify which code review patch to use as the try job "
"or rietveld instance to update the try job results "
"Default:%default")
group.add_option("--root",
help="Root to use for the patch; base subdirectory for "
"patch created in a subdirectory")
group.add_option("-p", "--patchlevel", type='int', metavar="LEVEL",
help="Used as -pN parameter to patch")
group.add_option("-s", "--sub_rep", action="append", default=[],
help="Subcheckout to use in addition. This is mainly "
"useful for gclient-style checkouts. In git, checkout "
"the branch with changes first. Use @rev or "
"@branch to specify the "
"revision/branch to diff against. If no @branch is "
"given the diff will be against the upstream branch. "
"If @branch then the diff is branch..HEAD. "
"All edits must be checked in.")
group.add_option("--no_search", action="store_true",
help=("Disable automatic search for gclient or repo "
"checkout root."))
group.add_option("-E", "--exclude", action="append",
default=['ChangeLog'], metavar='REGEXP',
help="Regexp patterns to exclude files. Default: %default")
group.add_option("--upstream_branch", action="store",
help="Specify the upstream branch to diff against in the "
"main checkout")
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Access the try server by HTTP")
group.add_option("--use_http",
action="store_const",
const=_SendChangeHTTP,
dest="send_patch",
help="Use HTTP to talk to the try server [default]")
group.add_option("-H", "--host",
help="Host address")
group.add_option("-P", "--port", type="int",
help="HTTP port")
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Access the try server with SVN")
group.add_option("--use_svn",
action="store_const",
const=_SendChangeSVN,
dest="send_patch",
help="Use SVN to talk to the try server")
group.add_option("-S", "--svn_repo",
metavar="SVN_URL",
help="SVN url to use to write the changes in; --use_svn is "
"implied when using --svn_repo")
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Access the try server with Git")
group.add_option("--use_git",
action="store_const",
const=_SendChangeGit,
dest="send_patch",
help="Use GIT to talk to the try server")
group.add_option("-G", "--git_repo",
metavar="GIT_URL",
help="GIT url to use to write the changes in; --use_git is "
"implied when using --git_repo")
parser.add_option_group(group)
group = optparse.OptionGroup(parser, "Access the try server with Gerrit")
group.add_option("--use_gerrit",
action="store_const",
const=_SendChangeGerrit,
dest="send_patch",
help="Use Gerrit to talk to the try server")
group.add_option("--gerrit_url",
metavar="GERRIT_URL",
help="Gerrit url to post a try job to; --use_gerrit is "
"implied when using --gerrit_url")
parser.add_option_group(group)
return parser
def TryChange(argv,
change,
swallow_exception,
prog=None,
extra_epilog=None):
"""
Args:
argv: Arguments and options.
change: Change instance corresponding to the CL.
swallow_exception: Whether we raise or swallow exceptions.
"""
parser = gen_parser(prog)
epilog = EPILOG % { 'prog': prog }
if extra_epilog:
epilog += extra_epilog
parser.epilog = epilog
options, args = parser.parse_args(argv)
# If they've asked for help, give it to them
if len(args) == 1 and args[0] == 'help':
parser.print_help()
return 0
# If they've said something confusing, don't spawn a try job until you
# understand what they want.
if args:
parser.error('Extra argument(s) "%s" not understood' % ' '.join(args))
if options.dry_run:
options.verbose += 1
LOG_FORMAT = '%(levelname)s %(filename)s(%(lineno)d): %(message)s'
if not swallow_exception:
if options.verbose == 0:
logging.basicConfig(level=logging.WARNING, format=LOG_FORMAT)
elif options.verbose == 1:
logging.basicConfig(level=logging.INFO, format=LOG_FORMAT)
elif options.verbose > 1:
logging.basicConfig(level=logging.DEBUG, format=LOG_FORMAT)
logging.debug(argv)
if (options.patchlevel is not None and
(options.patchlevel < 0 or options.patchlevel > 10)):
parser.error(
'Have you tried --port instead? You probably confused -p and -P.')
# Strip off any @ in the user, otherwise svn gets confused.
options.user = options.user.split('@', 1)[0]
if options.rietveld_url:
# Try to extract the review number if possible and fix the protocol.
if not '://' in options.rietveld_url:
options.rietveld_url = 'http://' + options.rietveld_url
match = re.match(r'^(.*)/(\d+)/?$', options.rietveld_url)
if match:
if options.issue or options.patchset:
parser.error('Cannot use both --issue and use a review number url')
options.issue = int(match.group(2))
options.rietveld_url = match.group(1)
try:
changed_files = None
# Always include os.getcwd() in the checkout settings.
path = os.getcwd()
file_list = []
if options.files:
file_list = options.files
elif change:
file_list = [f.LocalPath() for f in change.AffectedFiles()]
if options.upstream_branch:
path += '@' + options.upstream_branch
# Clear file list so that the correct list will be retrieved from the
# upstream branch.
file_list = []
current_vcs = GuessVCS(options, path, file_list)
current_vcs.AutomagicalSettings()
options = current_vcs.options
vcs_is_git = type(current_vcs) is GIT
# So far, git_repo doesn't work with SVN
if options.git_repo and not vcs_is_git:
parser.error('--git_repo option is supported only for GIT repositories')
# If revision==auto, resolve it
if options.revision and options.revision.lower() == 'auto':
if not vcs_is_git:
parser.error('--revision=auto is supported only for GIT repositories')
options.revision = scm.GIT.Capture(
['rev-parse', current_vcs.diff_against],
cwd=path)
checkouts = [current_vcs]
for item in options.sub_rep:
# Pass file_list=None because we don't know the sub repo's file list.
checkout = GuessVCS(options,
os.path.join(current_vcs.checkout_root, item),
None)
if checkout.checkout_root in [c.checkout_root for c in checkouts]:
parser.error('Specified the root %s two times.' %
checkout.checkout_root)
checkouts.append(checkout)
can_http = options.port and options.host
can_svn = options.svn_repo
can_git = options.git_repo
can_gerrit = options.gerrit_url
can_something = can_http or can_svn or can_git or can_gerrit
# If there was no transport selected yet, now we must have enough data to
# select one.
if not options.send_patch and not can_something:
parser.error('Please specify an access method.')
# Convert options.diff into the content of the diff.
if options.url:
if options.files:
parser.error('You cannot specify files and --url at the same time.')
options.diff = urllib2.urlopen(options.url).read()
elif options.diff:
if options.files:
parser.error('You cannot specify files and --diff at the same time.')
options.diff = gclient_utils.FileRead(options.diff, 'rb')
elif options.issue and options.patchset is None:
# Retrieve the patch from rietveld when the diff is not specified.
# When patchset is specified, it's because it's done by gcl/git-try.
api_url = '%s/api/%d' % (options.rietveld_url, options.issue)
logging.debug(api_url)
contents = json.loads(urllib2.urlopen(api_url).read())
options.patchset = contents['patchsets'][-1]
diff_url = ('%s/download/issue%d_%d.diff' %
(options.rietveld_url, options.issue, options.patchset))
diff = GetMungedDiff('', urllib2.urlopen(diff_url).readlines())
options.diff = ''.join(diff[0])
changed_files = diff[1]
else:
# Use this as the base.
root = checkouts[0].checkout_root
diffs = []
for checkout in checkouts:
raw_diff = checkout.GenerateDiff()
if not raw_diff:
continue
diff = raw_diff.splitlines(True)
path_diff = gclient_utils.PathDifference(root, checkout.checkout_root)
# Munge it.
diffs.extend(GetMungedDiff(path_diff, diff)[0])
if not diffs:
logging.error('Empty or non-existant diff, exiting.')
return 1
options.diff = ''.join(diffs)
if not options.name:
if options.issue:
options.name = 'Issue %s' % options.issue
else:
options.name = 'Unnamed'
print('Note: use --name NAME to change the try job name.')
if not options.email:
parser.error('Using an anonymous checkout. Please use --email or set '
'the TRYBOT_RESULTS_EMAIL_ADDRESS environment variable.')
print('Results will be emailed to: ' + options.email)
if options.bot:
bot_spec = _ApplyTestFilter(
options.testfilter, _ParseBotList(options.bot, options.testfilter))
else:
bot_spec = _GenTSBotSpec(checkouts, change, changed_files, options)
if options.testfilter:
bot_spec = _ApplyTestFilter(options.testfilter, bot_spec)
if any('triggered' in b[0] for b in bot_spec):
print >> sys.stderr, (
'ERROR You are trying to send a job to a triggered bot. This type of'
' bot requires an\ninitial job from a parent (usually a builder). '
'Instead send your job to the parent.\nBot list: %s' % bot_spec)
return 1
if options.print_bots:
print 'Bots which would be used:'
for bot in bot_spec:
if bot[1]:
print ' %s:%s' % (bot[0], ','.join(bot[1]))
else:
print ' %s' % (bot[0])
return 0
# Determine sending protocol
if options.send_patch:
# If forced.
senders = [options.send_patch]
else:
# Try sending patch using avaialble protocols
all_senders = [
(_SendChangeHTTP, can_http),
(_SendChangeSVN, can_svn),
(_SendChangeGerrit, can_gerrit),
(_SendChangeGit, can_git),
]
senders = [sender for sender, can in all_senders if can]
# Send the patch.
for sender in senders:
try:
sender(bot_spec, options)
return 0
except NoTryServerAccess:
is_last = sender == senders[-1]
if is_last:
raise
assert False, "Unreachable code"
except Error, e:
if swallow_exception:
return 1
print >> sys.stderr, e
return 1
except (gclient_utils.Error, subprocess2.CalledProcessError), e:
print >> sys.stderr, e
return 1
return 0
if __name__ == "__main__":
fix_encoding.fix_encoding()
sys.exit(TryChange(None, None, False))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment