Commit 12a537f9 authored by Edward Lemur's avatar Edward Lemur Committed by Commit Bot

depot_tools: Run git_common_test in Python 3

Bug: 1009809
Change-Id: Idfcbd26de3420798f092c7fa55a6126d7c389a8c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1834317Reviewed-by: 's avatarAnthony Polito <apolito@google.com>
Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
parent 4508b422
...@@ -6,12 +6,13 @@ ...@@ -6,12 +6,13 @@
# Derived from https://gist.github.com/aljungberg/626518 # Derived from https://gist.github.com/aljungberg/626518
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals
import multiprocessing.pool import multiprocessing.pool
from multiprocessing.pool import IMapIterator from multiprocessing.pool import IMapIterator
def wrapper(func): def wrapper(func):
def wrap(self, timeout=None): def wrap(self, timeout=None):
return func(self, timeout=timeout or 1e100) return func(self, timeout=timeout or 1 << 31)
return wrap return wrap
IMapIterator.next = wrapper(IMapIterator.next) IMapIterator.next = wrapper(IMapIterator.next)
IMapIterator.__next__ = IMapIterator.next IMapIterator.__next__ = IMapIterator.next
...@@ -303,11 +304,17 @@ class ProgressPrinter(object): ...@@ -303,11 +304,17 @@ class ProgressPrinter(object):
def once(function): def once(function):
"""@Decorates |function| so that it only performs its action once, no matter """@Decorates |function| so that it only performs its action once, no matter
how many times the decorated |function| is called.""" how many times the decorated |function| is called."""
def _inner_gen(): has_run = [False]
yield function() def _wrapper(*args, **kwargs):
while True: if not has_run[0]:
yield has_run[0] = True
return _inner_gen().next function(*args, **kwargs)
return _wrapper
def unicode_repr(s):
result = repr(s)
return result[1:] if result.startswith('u') else result
## Git functions ## Git functions
...@@ -597,8 +604,8 @@ def mktree(treedict): ...@@ -597,8 +604,8 @@ def mktree(treedict):
treedict - { name: (mode, type, ref) } treedict - { name: (mode, type, ref) }
""" """
with tempfile.TemporaryFile() as f: with tempfile.TemporaryFile() as f:
for name, (mode, typ, ref) in treedict.iteritems(): for name, (mode, typ, ref) in treedict.items():
f.write('%s %s %s\t%s\0' % (mode, typ, ref, name)) f.write(('%s %s %s\t%s\0' % (mode, typ, ref, name)).encode())
f.seek(0) f.seek(0)
return run('mktree', '-z', stdin=f) return run('mktree', '-z', stdin=f)
...@@ -612,7 +619,7 @@ def parse_commitrefs(*commitrefs): ...@@ -612,7 +619,7 @@ def parse_commitrefs(*commitrefs):
* 'cool_branch~2' * 'cool_branch~2'
""" """
try: try:
return map(binascii.unhexlify, hash_multi(*commitrefs)) return [binascii.unhexlify(h) for h in hash_multi(*commitrefs)]
except subprocess2.CalledProcessError: except subprocess2.CalledProcessError:
raise BadCommitRefException(commitrefs) raise BadCommitRefException(commitrefs)
...@@ -751,6 +758,7 @@ def run_with_stderr(*cmd, **kwargs): ...@@ -751,6 +758,7 @@ def run_with_stderr(*cmd, **kwargs):
kwargs.setdefault('shell', False) kwargs.setdefault('shell', False)
autostrip = kwargs.pop('autostrip', True) autostrip = kwargs.pop('autostrip', True)
indata = kwargs.pop('indata', None) indata = kwargs.pop('indata', None)
decode = kwargs.pop('decode', True)
cmd = (GIT_EXE, '-c', 'color.ui=never') + cmd cmd = (GIT_EXE, '-c', 'color.ui=never') + cmd
proc = subprocess2.Popen(cmd, **kwargs) proc = subprocess2.Popen(cmd, **kwargs)
...@@ -760,8 +768,12 @@ def run_with_stderr(*cmd, **kwargs): ...@@ -760,8 +768,12 @@ def run_with_stderr(*cmd, **kwargs):
raise subprocess2.CalledProcessError(retcode, cmd, os.getcwd(), ret, err) raise subprocess2.CalledProcessError(retcode, cmd, os.getcwd(), ret, err)
if autostrip: if autostrip:
ret = (ret or '').strip() ret = (ret or b'').strip()
err = (err or '').strip() err = (err or b'').strip()
if decode:
ret = ret.decode('utf-8', 'replace')
err = err.decode('utf-8', 'replace')
return ret, err return ret, err
...@@ -810,9 +822,9 @@ def status(): ...@@ -810,9 +822,9 @@ def status():
def tokenizer(stream): def tokenizer(stream):
acc = BytesIO() acc = BytesIO()
c = None c = None
while c != '': while c != b'':
c = stream.read(1) c = stream.read(1)
if c in (None, '', '\0'): if c in (None, b'', b'\0'):
if len(acc.getvalue()): if len(acc.getvalue()):
yield acc.getvalue() yield acc.getvalue()
acc = BytesIO() acc = BytesIO()
...@@ -821,12 +833,14 @@ def status(): ...@@ -821,12 +833,14 @@ def status():
def parser(tokens): def parser(tokens):
while True: while True:
# Raises StopIteration if it runs out of tokens. try:
status_dest = next(tokens) status_dest = next(tokens).decode('utf-8')
except StopIteration:
return
stat, dest = status_dest[:2], status_dest[3:] stat, dest = status_dest[:2], status_dest[3:]
lstat, rstat = stat lstat, rstat = stat
if lstat == 'R': if lstat == 'R':
src = next(tokens) src = next(tokens).decode('utf-8')
else: else:
src = dest src = dest
yield (dest, stat_entry(lstat, rstat, src)) yield (dest, stat_entry(lstat, rstat, src))
...@@ -848,7 +862,7 @@ def squash_current_branch(header=None, merge_base=None): ...@@ -848,7 +862,7 @@ def squash_current_branch(header=None, merge_base=None):
# nothing to commit at this point. # nothing to commit at this point.
print('Nothing to commit; squashed branch is empty') print('Nothing to commit; squashed branch is empty')
return False return False
run('commit', '--no-verify', '-a', '-F', '-', indata=log_msg) run('commit', '--no-verify', '-a', '-F', '-', indata=log_msg.encode())
return True return True
...@@ -858,7 +872,8 @@ def tags(*args): ...@@ -858,7 +872,8 @@ def tags(*args):
def thaw(): def thaw():
took_action = False took_action = False
for sha in (s.strip() for s in run_stream('rev-list', 'HEAD').xreadlines()): for sha in run_stream('rev-list', 'HEAD').readlines():
sha = sha.strip().decode('utf-8')
msg = run('show', '--format=%f%b', '-s', 'HEAD') msg = run('show', '--format=%f%b', '-s', 'HEAD')
match = FREEZE_MATCHER.match(msg) match = FREEZE_MATCHER.match(msg)
if not match: if not match:
...@@ -899,7 +914,7 @@ def topo_iter(branch_tree, top_down=True): ...@@ -899,7 +914,7 @@ def topo_iter(branch_tree, top_down=True):
# TODO(iannucci): There is probably a more efficient way to do these. # TODO(iannucci): There is probably a more efficient way to do these.
if top_down: if top_down:
while branch_tree: while branch_tree:
this_pass = [(b, p) for b, p in branch_tree.iteritems() this_pass = [(b, p) for b, p in branch_tree.items()
if p not in branch_tree] if p not in branch_tree]
assert this_pass, "Branch tree has cycles: %r" % branch_tree assert this_pass, "Branch tree has cycles: %r" % branch_tree
for branch, parent in sorted(this_pass): for branch, parent in sorted(this_pass):
...@@ -907,11 +922,11 @@ def topo_iter(branch_tree, top_down=True): ...@@ -907,11 +922,11 @@ def topo_iter(branch_tree, top_down=True):
del branch_tree[branch] del branch_tree[branch]
else: else:
parent_to_branches = collections.defaultdict(set) parent_to_branches = collections.defaultdict(set)
for branch, parent in branch_tree.iteritems(): for branch, parent in branch_tree.items():
parent_to_branches[parent].add(branch) parent_to_branches[parent].add(branch)
while branch_tree: while branch_tree:
this_pass = [(b, p) for b, p in branch_tree.iteritems() this_pass = [(b, p) for b, p in branch_tree.items()
if not parent_to_branches[b]] if not parent_to_branches[b]]
assert this_pass, "Branch tree has cycles: %r" % branch_tree assert this_pass, "Branch tree has cycles: %r" % branch_tree
for branch, parent in sorted(this_pass): for branch, parent in sorted(this_pass):
...@@ -1014,7 +1029,9 @@ def get_branches_info(include_tracking_status): ...@@ -1014,7 +1029,9 @@ def get_branches_info(include_tracking_status):
if info.upstream not in info_map and info.upstream not in missing_upstreams: if info.upstream not in info_map and info.upstream not in missing_upstreams:
missing_upstreams[info.upstream] = None missing_upstreams[info.upstream] = None
return dict(info_map.items() + missing_upstreams.items()) result = info_map.copy()
result.update(missing_upstreams)
return result
def make_workdir_common(repository, new_workdir, files_to_symlink, def make_workdir_common(repository, new_workdir, files_to_symlink,
......
...@@ -7,6 +7,7 @@ ...@@ -7,6 +7,7 @@
""" """
from __future__ import print_function from __future__ import print_function
from __future__ import unicode_literals
import argparse import argparse
import collections import collections
...@@ -94,11 +95,6 @@ def parse_blame(blameoutput): ...@@ -94,11 +95,6 @@ def parse_blame(blameoutput):
yield BlameLine(commit, context, lineno_then, lineno_now, False) yield BlameLine(commit, context, lineno_then, lineno_now, False)
def num_codepoints(s):
"""Gets the length of a UTF-8 byte string, in Unicode codepoints."""
return len(s.decode('utf-8', errors='replace'))
def print_table(table, colsep=' ', rowsep='\n', align=None, out=sys.stdout): def print_table(table, colsep=' ', rowsep='\n', align=None, out=sys.stdout):
"""Print a 2D rectangular array, aligning columns with spaces. """Print a 2D rectangular array, aligning columns with spaces.
...@@ -112,10 +108,9 @@ def print_table(table, colsep=' ', rowsep='\n', align=None, out=sys.stdout): ...@@ -112,10 +108,9 @@ def print_table(table, colsep=' ', rowsep='\n', align=None, out=sys.stdout):
colwidths = None colwidths = None
for row in table: for row in table:
if colwidths is None: if colwidths is None:
colwidths = [num_codepoints(x) for x in row] colwidths = [len(x) for x in row]
else: else:
colwidths = [max(colwidths[i], num_codepoints(x)) colwidths = [max(colwidths[i], len(x)) for i, x in enumerate(row)]
for i, x in enumerate(row)]
if align is None: # pragma: no cover if align is None: # pragma: no cover
align = 'l' * len(colwidths) align = 'l' * len(colwidths)
...@@ -123,7 +118,7 @@ def print_table(table, colsep=' ', rowsep='\n', align=None, out=sys.stdout): ...@@ -123,7 +118,7 @@ def print_table(table, colsep=' ', rowsep='\n', align=None, out=sys.stdout):
for row in table: for row in table:
cells = [] cells = []
for i, cell in enumerate(row): for i, cell in enumerate(row):
padding = ' ' * (colwidths[i] - num_codepoints(cell)) padding = ' ' * (colwidths[i] - len(cell))
if align[i] == 'r': if align[i] == 'r':
cell = padding + cell cell = padding + cell
elif i < len(row) - 1: elif i < len(row) - 1:
...@@ -279,7 +274,7 @@ def hyper_blame(ignored, filename, revision='HEAD', out=sys.stdout, ...@@ -279,7 +274,7 @@ def hyper_blame(ignored, filename, revision='HEAD', out=sys.stdout,
try: try:
parsed = cache_blame_from(filename, git_common.hash_one(revision)) parsed = cache_blame_from(filename, git_common.hash_one(revision))
except subprocess2.CalledProcessError as e: except subprocess2.CalledProcessError as e:
err.write(e.stderr) err.write(e.stderr.decode())
return e.returncode return e.returncode
new_parsed = [] new_parsed = []
...@@ -361,7 +356,7 @@ def main(args, stdout=sys.stdout, stderr=sys.stderr): ...@@ -361,7 +356,7 @@ def main(args, stdout=sys.stdout, stderr=sys.stderr):
try: try:
repo_root = git_common.repo_root() repo_root = git_common.repo_root()
except subprocess2.CalledProcessError as e: except subprocess2.CalledProcessError as e:
stderr.write(e.stderr) stderr.write(e.stderr.decode())
return e.returncode return e.returncode
# Make filename relative to the repository root, and cd to the root dir (so # Make filename relative to the repository root, and cd to the root dir (so
......
...@@ -75,7 +75,8 @@ def get_number_tree(prefix_bytes): ...@@ -75,7 +75,8 @@ def get_number_tree(prefix_bytes):
ref = '%s:%s' % (REF, pathlify(prefix_bytes)) ref = '%s:%s' % (REF, pathlify(prefix_bytes))
try: try:
raw = buffer(git.run('cat-file', 'blob', ref, autostrip=False)) raw = buffer(
git.run('cat-file', 'blob', ref, autostrip=False, decode=False))
return dict(struct.unpack_from(CHUNK_FMT, raw, i * CHUNK_SIZE) return dict(struct.unpack_from(CHUNK_FMT, raw, i * CHUNK_SIZE)
for i in xrange(len(raw) / CHUNK_SIZE)) for i in xrange(len(raw) / CHUNK_SIZE))
except subprocess2.CalledProcessError: except subprocess2.CalledProcessError:
......
...@@ -320,8 +320,8 @@ def main(args=None): ...@@ -320,8 +320,8 @@ def main(args=None):
else: else:
root_branch = git.root() root_branch = git.root()
if return_branch != 'HEAD': if return_branch != 'HEAD':
print("%r was merged with its parent, checking out %r instead." % print("%s was merged with its parent, checking out %s instead." %
(return_branch, root_branch)) (git.unicode_repr(return_branch), git.unicode_repr(root_branch)))
git.run('checkout', root_branch) git.run('checkout', root_branch)
# return_workdir may also not be there any more. # return_workdir may also not be there any more.
......
...@@ -15,13 +15,18 @@ import tempfile ...@@ -15,13 +15,18 @@ import tempfile
import unittest import unittest
if sys.version_info.major == 3:
# pylint: disable=redefined-builtin
basestring = (str,)
def git_hash_data(data, typ='blob'): def git_hash_data(data, typ='blob'):
"""Calculate the git-style SHA1 for some data. """Calculate the git-style SHA1 for some data.
Only supports 'blob' type data at the moment. Only supports 'blob' type data at the moment.
""" """
assert typ == 'blob', 'Only support blobs for now' assert typ == 'blob', 'Only support blobs for now'
return hashlib.sha1('blob %s\0%s' % (len(data), data)).hexdigest() return hashlib.sha1(b'blob %d\0%s' % (len(data), data)).hexdigest()
class OrderedSet(collections.MutableSet): class OrderedSet(collections.MutableSet):
...@@ -120,8 +125,8 @@ class GitRepoSchema(object): ...@@ -120,8 +125,8 @@ class GitRepoSchema(object):
This is the repo This is the repo
A - B - C - D A - B - C - D
\ E / \\ E /
Whitespace doesn't matter. Each line is a declaration of which commits come Whitespace doesn't matter. Each line is a declaration of which commits come
before which other commits. before which other commits.
...@@ -166,15 +171,15 @@ class GitRepoSchema(object): ...@@ -166,15 +171,15 @@ class GitRepoSchema(object):
is_root = True is_root = True
par_map = copy.deepcopy(self.par_map) par_map = copy.deepcopy(self.par_map)
while par_map: while par_map:
empty_keys = set(k for k, v in par_map.iteritems() if not v) empty_keys = set(k for k, v in par_map.items() if not v)
assert empty_keys, 'Cycle detected! %s' % par_map assert empty_keys, 'Cycle detected! %s' % par_map
for k in sorted(empty_keys): for k in sorted(empty_keys):
yield self.COMMIT(k, self.par_map[k], yield self.COMMIT(k, self.par_map[k],
not any(k in v for v in self.par_map.itervalues()), not any(k in v for v in self.par_map.values()),
is_root) is_root)
del par_map[k] del par_map[k]
for v in par_map.itervalues(): for v in par_map.values():
v.difference_update(empty_keys) v.difference_update(empty_keys)
is_root = False is_root = False
...@@ -321,7 +326,7 @@ class GitRepo(object): ...@@ -321,7 +326,7 @@ class GitRepo(object):
env = self.get_git_commit_env(commit_data) env = self.get_git_commit_env(commit_data)
for fname, file_data in commit_data.iteritems(): for fname, file_data in commit_data.items():
# If it isn't a string, it's one of the special keys. # If it isn't a string, it's one of the special keys.
if not isinstance(fname, basestring): if not isinstance(fname, basestring):
continue continue
...@@ -367,7 +372,9 @@ class GitRepo(object): ...@@ -367,7 +372,9 @@ class GitRepo(object):
self._date += datetime.timedelta(days=1) self._date += datetime.timedelta(days=1)
else: else:
val = getattr(self, 'DEFAULT_%s' % singleton) val = getattr(self, 'DEFAULT_%s' % singleton)
env['GIT_%s' % singleton] = str(val) if not isinstance(val, str) and not isinstance(val, bytes):
val = str(val)
env['GIT_%s' % singleton] = val
return env return env
def git(self, *args, **kwargs): def git(self, *args, **kwargs):
...@@ -377,6 +384,7 @@ class GitRepo(object): ...@@ -377,6 +384,7 @@ class GitRepo(object):
with open(os.devnull, 'wb') as devnull: with open(os.devnull, 'wb') as devnull:
output = subprocess.check_output( output = subprocess.check_output(
('git',) + args, cwd=self.repo_path, stderr=devnull, **kwargs) ('git',) + args, cwd=self.repo_path, stderr=devnull, **kwargs)
output = output.decode('utf-8')
return self.COMMAND_OUTPUT(0, output) return self.COMMAND_OUTPUT(0, output)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
return self.COMMAND_OUTPUT(e.returncode, e.output) return self.COMMAND_OUTPUT(e.returncode, e.output)
...@@ -418,17 +426,17 @@ class GitRepo(object): ...@@ -418,17 +426,17 @@ class GitRepo(object):
stdout = sys.stdout stdout = sys.stdout
stderr = sys.stderr stderr = sys.stderr
try: try:
# "multiple statements on a line" pylint: disable=multiple-statements with tempfile.TemporaryFile('w+') as out:
with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err: with tempfile.TemporaryFile('w+') as err:
sys.stdout = out sys.stdout = out
sys.stderr = err sys.stderr = err
try: try:
self.run(fn, *args, **kwargs) self.run(fn, *args, **kwargs)
except SystemExit: except SystemExit:
pass pass
out.seek(0) out.seek(0)
err.seek(0) err.seek(0)
return out.read(), err.read() return out.read(), err.read()
finally: finally:
sys.stdout = stdout sys.stdout = stdout
sys.stderr = stderr sys.stderr = stderr
......
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment