depot_tools: Run git_common_test in Python 3

Bug: 1009809
Change-Id: Idfcbd26de3420798f092c7fa55a6126d7c389a8c
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1834317Reviewed-by: Anthony Polito <apolito@google.com>
Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>

git_common.py

@@ -6,12 +6,13 @@
 # Derived from https://gist.github.com/aljungberg/626518
 
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import multiprocessing.pool
 from multiprocessing.pool import IMapIterator
 def wrapper(func):
   def wrap(self, timeout=None):
-    return func(self, timeout=timeout or 1e100)
+    return func(self, timeout=timeout or 1 << 31)
   return wrap
 IMapIterator.next = wrapper(IMapIterator.next)
 IMapIterator.__next__ = IMapIterator.next
@@ -303,11 +304,17 @@ class ProgressPrinter(object):
 def once(function):
   """@Decorates |function| so that it only performs its action once, no matter
   how many times the decorated |function| is called."""
-  def _inner_gen():
-    yield function()
-    while True:
-      yield
-  return _inner_gen().next
+  has_run = [False]
+  def _wrapper(*args, **kwargs):
+    if not has_run[0]:
+      has_run[0] = True
+      function(*args, **kwargs)
+  return _wrapper
+
+
+def unicode_repr(s):
+  result = repr(s)
+  return result[1:] if result.startswith('u') else result
 
 
 ## Git functions
@@ -597,8 +604,8 @@ def mktree(treedict):
     treedict - { name: (mode, type, ref) }
   """
   with tempfile.TemporaryFile() as f:
-    for name, (mode, typ, ref) in treedict.iteritems():
-      f.write('%s %s %s\t%s\0' % (mode, typ, ref, name))
+    for name, (mode, typ, ref) in treedict.items():
+      f.write(('%s %s %s\t%s\0' % (mode, typ, ref, name)).encode())
     f.seek(0)
     return run('mktree', '-z', stdin=f)
 
@@ -612,7 +619,7 @@ def parse_commitrefs(*commitrefs):
     * 'cool_branch~2'
   """
   try:
-    return map(binascii.unhexlify, hash_multi(*commitrefs))
+    return [binascii.unhexlify(h) for h in hash_multi(*commitrefs)]
   except subprocess2.CalledProcessError:
     raise BadCommitRefException(commitrefs)
 
@@ -751,6 +758,7 @@ def run_with_stderr(*cmd, **kwargs):
   kwargs.setdefault('shell', False)
   autostrip = kwargs.pop('autostrip', True)
   indata = kwargs.pop('indata', None)
+  decode = kwargs.pop('decode', True)
 
   cmd = (GIT_EXE, '-c', 'color.ui=never') + cmd
   proc = subprocess2.Popen(cmd, **kwargs)
@@ -760,8 +768,12 @@ def run_with_stderr(*cmd, **kwargs):
     raise subprocess2.CalledProcessError(retcode, cmd, os.getcwd(), ret, err)
 
   if autostrip:
-    ret = (ret or '').strip()
-    err = (err or '').strip()
+    ret = (ret or b'').strip()
+    err = (err or b'').strip()
+
+  if decode:
+    ret = ret.decode('utf-8', 'replace')
+    err = err.decode('utf-8', 'replace')
 
   return ret, err
 
@@ -810,9 +822,9 @@ def status():
   def tokenizer(stream):
     acc = BytesIO()
     c = None
-    while c != '':
+    while c != b'':
       c = stream.read(1)
-      if c in (None, '', '\0'):
+      if c in (None, b'', b'\0'):
         if len(acc.getvalue()):
           yield acc.getvalue()
           acc = BytesIO()
@@ -821,12 +833,14 @@ def status():
 
   def parser(tokens):
     while True:
-      # Raises StopIteration if it runs out of tokens.
-      status_dest = next(tokens)
+      try:
+        status_dest = next(tokens).decode('utf-8')
+      except StopIteration:
+        return
       stat, dest = status_dest[:2], status_dest[3:]
       lstat, rstat = stat
       if lstat == 'R':
-        src = next(tokens)
+        src = next(tokens).decode('utf-8')
       else:
         src = dest
       yield (dest, stat_entry(lstat, rstat, src))
@@ -848,7 +862,7 @@ def squash_current_branch(header=None, merge_base=None):
     # nothing to commit at this point.
     print('Nothing to commit; squashed branch is empty')
     return False
-  run('commit', '--no-verify', '-a', '-F', '-', indata=log_msg)
+  run('commit', '--no-verify', '-a', '-F', '-', indata=log_msg.encode())
   return True
 
 
@@ -858,7 +872,8 @@ def tags(*args):
 
 def thaw():
   took_action = False
-  for sha in (s.strip() for s in run_stream('rev-list', 'HEAD').xreadlines()):
+  for sha in run_stream('rev-list', 'HEAD').readlines():
+    sha = sha.strip().decode('utf-8')
     msg = run('show', '--format=%f%b', '-s', 'HEAD')
     match = FREEZE_MATCHER.match(msg)
     if not match:
@@ -899,7 +914,7 @@ def topo_iter(branch_tree, top_down=True):
   # TODO(iannucci): There is probably a more efficient way to do these.
   if top_down:
     while branch_tree:
-      this_pass = [(b, p) for b, p in branch_tree.iteritems()
+      this_pass = [(b, p) for b, p in branch_tree.items()
                    if p not in branch_tree]
       assert this_pass, "Branch tree has cycles: %r" % branch_tree
       for branch, parent in sorted(this_pass):
@@ -907,11 +922,11 @@ def topo_iter(branch_tree, top_down=True):
         del branch_tree[branch]
   else:
     parent_to_branches = collections.defaultdict(set)
-    for branch, parent in branch_tree.iteritems():
+    for branch, parent in branch_tree.items():
       parent_to_branches[parent].add(branch)
 
     while branch_tree:
-      this_pass = [(b, p) for b, p in branch_tree.iteritems()
+      this_pass = [(b, p) for b, p in branch_tree.items()
                    if not parent_to_branches[b]]
       assert this_pass, "Branch tree has cycles: %r" % branch_tree
       for branch, parent in sorted(this_pass):
@@ -1014,7 +1029,9 @@ def get_branches_info(include_tracking_status):
     if info.upstream not in info_map and info.upstream not in missing_upstreams:
       missing_upstreams[info.upstream] = None
 
-  return dict(info_map.items() + missing_upstreams.items())
+  result = info_map.copy()
+  result.update(missing_upstreams)
+  return result
 
 
 def make_workdir_common(repository, new_workdir, files_to_symlink,

git_hyper_blame.py

@@ -7,6 +7,7 @@
 """
 
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import argparse
 import collections
@@ -94,11 +95,6 @@ def parse_blame(blameoutput):
     yield BlameLine(commit, context, lineno_then, lineno_now, False)
 
 
-def num_codepoints(s):
-  """Gets the length of a UTF-8 byte string, in Unicode codepoints."""
-  return len(s.decode('utf-8', errors='replace'))
-
-
 def print_table(table, colsep=' ', rowsep='\n', align=None, out=sys.stdout):
   """Print a 2D rectangular array, aligning columns with spaces.
 
@@ -112,10 +108,9 @@ def print_table(table, colsep=' ', rowsep='\n', align=None, out=sys.stdout):
   colwidths = None
   for row in table:
     if colwidths is None:
-      colwidths = [num_codepoints(x) for x in row]
+      colwidths = [len(x) for x in row]
     else:
-      colwidths = [max(colwidths[i], num_codepoints(x))
-                   for i, x in enumerate(row)]
+      colwidths = [max(colwidths[i], len(x)) for i, x in enumerate(row)]
 
   if align is None:  # pragma: no cover
     align = 'l' * len(colwidths)
@@ -123,7 +118,7 @@ def print_table(table, colsep=' ', rowsep='\n', align=None, out=sys.stdout):
   for row in table:
     cells = []
     for i, cell in enumerate(row):
-      padding = ' ' * (colwidths[i] - num_codepoints(cell))
+      padding = ' ' * (colwidths[i] - len(cell))
       if align[i] == 'r':
         cell = padding + cell
       elif i < len(row) - 1:
@@ -279,7 +274,7 @@ def hyper_blame(ignored, filename, revision='HEAD', out=sys.stdout,
   try:
     parsed = cache_blame_from(filename, git_common.hash_one(revision))
   except subprocess2.CalledProcessError as e:
-    err.write(e.stderr)
+    err.write(e.stderr.decode())
     return e.returncode
 
   new_parsed = []
@@ -361,7 +356,7 @@ def main(args, stdout=sys.stdout, stderr=sys.stderr):
   try:
     repo_root = git_common.repo_root()
   except subprocess2.CalledProcessError as e:
-    stderr.write(e.stderr)
+    stderr.write(e.stderr.decode())
     return e.returncode
 
   # Make filename relative to the repository root, and cd to the root dir (so

git_number.py

@@ -75,7 +75,8 @@ def get_number_tree(prefix_bytes):
   ref = '%s:%s' % (REF, pathlify(prefix_bytes))
 
   try:
-    raw = buffer(git.run('cat-file', 'blob', ref, autostrip=False))
+    raw = buffer(
+        git.run('cat-file', 'blob', ref, autostrip=False, decode=False))
     return dict(struct.unpack_from(CHUNK_FMT, raw, i * CHUNK_SIZE)
                 for i in xrange(len(raw) / CHUNK_SIZE))
   except subprocess2.CalledProcessError:

git_rebase_update.py

@@ -320,8 +320,8 @@ def main(args=None):
     else:
       root_branch = git.root()
       if return_branch != 'HEAD':
-        print("%r was merged with its parent, checking out %r instead." %
-              (return_branch, root_branch))
+        print("%s was merged with its parent, checking out %s instead." %
+              (git.unicode_repr(return_branch), git.unicode_repr(root_branch)))
       git.run('checkout', root_branch)
 
     # return_workdir may also not be there any more.

testing_support/git_test_utils.py

@@ -15,13 +15,18 @@ import tempfile
 import unittest
 
 
+if sys.version_info.major == 3:
+  # pylint: disable=redefined-builtin
+  basestring = (str,)
+
+
 def git_hash_data(data, typ='blob'):
   """Calculate the git-style SHA1 for some data.
 
   Only supports 'blob' type data at the moment.
   """
   assert typ == 'blob', 'Only support blobs for now'
-  return hashlib.sha1('blob %s\0%s' % (len(data), data)).hexdigest()
+  return hashlib.sha1(b'blob %d\0%s' % (len(data), data)).hexdigest()
 
 
 class OrderedSet(collections.MutableSet):
@@ -120,8 +125,8 @@ class GitRepoSchema(object):
 
   This is the repo
 
-     A - B - C - D
-           \ E /
+     A - B -  C - D
+           \\ E /
 
   Whitespace doesn't matter. Each line is a declaration of which commits come
   before which other commits.
@@ -166,15 +171,15 @@ class GitRepoSchema(object):
     is_root = True
     par_map = copy.deepcopy(self.par_map)
     while par_map:
-      empty_keys = set(k for k, v in par_map.iteritems() if not v)
+      empty_keys = set(k for k, v in par_map.items() if not v)
       assert empty_keys, 'Cycle detected! %s' % par_map
 
       for k in sorted(empty_keys):
         yield self.COMMIT(k, self.par_map[k],
-                          not any(k in v for v in self.par_map.itervalues()),
+                          not any(k in v for v in self.par_map.values()),
                           is_root)
         del par_map[k]
-      for v in par_map.itervalues():
+      for v in par_map.values():
         v.difference_update(empty_keys)
       is_root = False
 
@@ -321,7 +326,7 @@ class GitRepo(object):
 
     env = self.get_git_commit_env(commit_data)
 
-    for fname, file_data in commit_data.iteritems():
+    for fname, file_data in commit_data.items():
       # If it isn't a string, it's one of the special keys.
       if not isinstance(fname, basestring):
         continue
@@ -367,7 +372,9 @@ class GitRepo(object):
           self._date += datetime.timedelta(days=1)
         else:
           val = getattr(self, 'DEFAULT_%s' % singleton)
-        env['GIT_%s' % singleton] = str(val)
+        if not isinstance(val, str) and not isinstance(val, bytes):
+          val = str(val)
+        env['GIT_%s' % singleton] = val
     return env
 
   def git(self, *args, **kwargs):
@@ -377,6 +384,7 @@ class GitRepo(object):
       with open(os.devnull, 'wb') as devnull:
         output = subprocess.check_output(
           ('git',) + args, cwd=self.repo_path, stderr=devnull, **kwargs)
+        output = output.decode('utf-8')
       return self.COMMAND_OUTPUT(0, output)
     except subprocess.CalledProcessError as e:
       return self.COMMAND_OUTPUT(e.returncode, e.output)
@@ -418,17 +426,17 @@ class GitRepo(object):
     stdout = sys.stdout
     stderr = sys.stderr
     try:
-      # "multiple statements on a line" pylint: disable=multiple-statements
-      with tempfile.TemporaryFile() as out, tempfile.TemporaryFile() as err:
-        sys.stdout = out
-        sys.stderr = err
-        try:
-          self.run(fn, *args, **kwargs)
-        except SystemExit:
-          pass
-        out.seek(0)
-        err.seek(0)
-        return out.read(), err.read()
+      with tempfile.TemporaryFile('w+') as out:
+        with tempfile.TemporaryFile('w+') as err:
+          sys.stdout = out
+          sys.stderr = err
+          try:
+            self.run(fn, *args, **kwargs)
+          except SystemExit:
+            pass
+          out.seek(0)
+          err.seek(0)
+          return out.read(), err.read()
     finally:
       sys.stdout = stdout
       sys.stderr = stderr