presubmit_support: Fix tests on Python 3.

Bug: 1009814
Change-Id: I2c2a835fc4fa0a7fc9e68dfff680fb0beacb491e
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1869535Reviewed-by: Anthony Polito <apolito@google.com>
Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>

.vpython3

 python_version: "3.8"
+
+# Used by:
+#   auth.py
+#   gerrit_util.py
+#   git_cl.py
+#   my_activity.py
+#   TODO(crbug.com/1002153): Add ninjalog_uploader.py
+wheel: <
+  name: "infra/python/wheels/httplib2-py3"
+  version: "version:0.13.1"
+>
+
+# Used by:
+#   my_activity.py
+wheel: <
+  name: "infra/python/wheels/python-dateutil-py2_py3"
+  version: "version:2.7.3"
+>
+wheel: <
+  name: "infra/python/wheels/six-py2_py3"
+  version: "version:1.10.0"
+>

gerrit_util.py

@@ -12,8 +12,6 @@ from __future__ import print_function
 
 import base64
 import contextlib
-import cookielib
-import httplib  # Still used for its constants.
 import httplib2
 import json
 import logging
@@ -27,8 +25,6 @@ import sys
 import tempfile
 import time
 import urllib
-import urlparse
-from cStringIO import StringIO
 from multiprocessing.pool import ThreadPool
 
 import auth
@@ -37,6 +33,17 @@ import metrics
 import metrics_utils
 import subprocess2
 
+if sys.version_info.major == 2:
+  import cookielib
+  import httplib
+  import urlparse
+  from cStringIO import StringIO
+else:
+  import http.cookiejar as cookielib
+  import http.client as httplib
+  import urllib.parse as urlparse
+  from io import StringIO
+
 LOGGER = logging.getLogger()
 # With a starting sleep time of 1.5 seconds, 2^n exponential backoff, and seven
 # total tries, the sleep time between the first and last tries will be 94.5 sec.

git_cl.py

@@ -15,7 +15,6 @@ import base64
 import collections
 import datetime
 import glob
-import httplib
 import httplib2
 import itertools
 import json
@@ -30,9 +29,6 @@ import sys
 import tempfile
 import textwrap
 import time
-import urllib
-import urllib2
-import urlparse
 import uuid
 import webbrowser
 import zlib
@@ -58,6 +54,17 @@ import subcommand
 import subprocess2
 import watchlists
 
+if sys.version_info.major == 2:
+  import httplib
+  import urllib2 as urllib_request
+  import urllib2 as urllib_error
+  import urlparse
+else:
+  import http.client as httplib
+  import urllib.request as urllib_request
+  import urllib.error as urllib_error
+  import urllib.parse as urlparse
+
 __version__ = '2.0'
 
 # Traces for git push will be stored in a traces directory inside the
@@ -3182,7 +3189,7 @@ def urlretrieve(source, destination):
   This is necessary because urllib is broken for SSL connections via a proxy.
   """
   with open(destination, 'w') as f:
-    f.write(urllib2.urlopen(source).read())
+    f.write(urllib_request.urlopen(source).read())
 
 
 def hasSheBang(fname):
@@ -4655,7 +4662,7 @@ def GetTreeStatus(url=None):
   'unknown' or 'unset'."""
   url = url or settings.GetTreeStatusUrl(error_ok=True)
   if url:
-    status = urllib2.urlopen(url).read().lower()
+    status = urllib_request.urlopen(url).read().lower()
     if status.find('closed') != -1 or status == '0':
       return 'closed'
     elif status.find('open') != -1 or status == '1':
@@ -4669,7 +4676,7 @@ def GetTreeStatusReason():
   with the reason for the tree to be opened or closed."""
   url = settings.GetTreeStatusUrl()
   json_url = urlparse.urljoin(url, '/current?format=json')
-  connection = urllib2.urlopen(json_url)
+  connection = urllib_request.urlopen(json_url)
   status = json.loads(connection.read())
   connection.close()
   return status['message']
@@ -5459,7 +5466,7 @@ def main(argv):
     return dispatcher.execute(OptionParser(), argv)
   except auth.LoginRequiredError as e:
     DieWithError(str(e))
-  except urllib2.HTTPError as e:
+  except urllib_error.HTTPError as e:
     if e.code != 500:
       raise
     DieWithError(

presubmit_canned_checks.py

@@ -5,6 +5,7 @@
 """Generic presubmit checks that can be reused by other presubmit checks."""
 
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import os as _os
 _HERE = _os.path.dirname(_os.path.abspath(__file__))
@@ -245,7 +246,7 @@ def CheckGenderNeutral(input_api, output_api, source_file_filter=None):
   submitted.
   """
   gendered_re = input_api.re.compile(
-      '(^|\s|\(|\[)([Hh]e|[Hh]is|[Hh]ers?|[Hh]im|[Ss]he|[Gg]uys?)\\b')
+      r'(^|\s|\(|\[)([Hh]e|[Hh]is|[Hh]ers?|[Hh]im|[Ss]he|[Gg]uys?)\\b')
 
   errors = []
   for f in input_api.AffectedFiles(include_deletes=False,
@@ -570,7 +571,7 @@ def CheckTreeIsOpen(input_api, output_api,
     return []
   try:
     if json_url:
-      connection = input_api.urllib2.urlopen(json_url)
+      connection = input_api.urllib_request.urlopen(json_url)
       status = input_api.json.loads(connection.read())
       connection.close()
       if not status['can_commit_freely']:
@@ -579,7 +580,7 @@ def CheckTreeIsOpen(input_api, output_api,
         return [output_api.PresubmitError(short_text, long_text=long_text)]
     else:
       # TODO(bradnelson): drop this once all users are gone.
-      connection = input_api.urllib2.urlopen(url)
+      connection = input_api.urllib_request.urlopen(url)
       status = connection.read()
       connection.close()
       if input_api.re.match(closed, status):
@@ -821,7 +822,7 @@ def GetPylint(input_api, output_api, white_list=None, black_list=None,
 
   The default white_list enforces looking only at *.py files.
   """
-  white_list = tuple(white_list or ('.*\.py$',))
+  white_list = tuple(white_list or (r'.*\.py$',))
   black_list = tuple(black_list or input_api.DEFAULT_BLACK_LIST)
   extra_paths_list = extra_paths_list or []
 
@@ -867,8 +868,7 @@ def GetPylint(input_api, output_api, white_list=None, black_list=None,
   input_api.logging.info('Running pylint on %d files', len(files))
   input_api.logging.debug('Running pylint on: %s', files)
   env = input_api.environ.copy()
-  env['PYTHONPATH'] = input_api.os_path.pathsep.join(
-    extra_paths_list).encode('utf8')
+  env['PYTHONPATH'] = input_api.os_path.pathsep.join(extra_paths_list)
   env.pop('VPYTHON_CLEAR_PYTHONPATH', None)
   input_api.logging.debug('  with extra PYTHONPATH: %r', extra_paths_list)
 
@@ -936,7 +936,7 @@ def RunPylint(input_api, *args, **kwargs):
 def CheckBuildbotPendingBuilds(input_api, output_api, url, max_pendings,
     ignored):
   try:
-    connection = input_api.urllib2.urlopen(url)
+    connection = input_api.urllib_request.urlopen(url)
     raw_data = connection.read()
     connection.close()
   except IOError:
@@ -1001,7 +1001,7 @@ def CheckOwners(input_api, output_api, source_file_filter=None):
       input_api.change.RepositoryRoot(),
       owner_email,
       reviewers,
-      fopen=file,
+      fopen=open,
       os_path=input_api.os_path,
       email_postfix='',
       disable_color=True,
@@ -1126,7 +1126,7 @@ def PanProjectChecks(input_api, output_api,
   # 2006-20xx string used on the oldest files. 2006-20xx is deprecated, but
   # tolerated on old files.
   current_year = int(input_api.time.strftime('%Y'))
-  allowed_years = (str(s) for s in reversed(xrange(2006, current_year + 1)))
+  allowed_years = (str(s) for s in reversed(range(2006, current_year + 1)))
   years_re = '(' + '|'.join(allowed_years) + '|2006-2008|2006-2009|2006-2010)'
 
   # The (c) is deprecated, but tolerate it until it's removed from all files.
@@ -1155,7 +1155,10 @@ def PanProjectChecks(input_api, output_api,
   snapshot_memory = []
   def snapshot(msg):
     """Measures & prints performance warning if a rule is running slow."""
-    dt2 = input_api.time.clock()
+    if input_api.sys.version_info.major == 2:
+      dt2 = input_api.time.clock()
+    else:
+      dt2 = input_api.time.process_time()
     if snapshot_memory:
       delta_ms = int(1000*(dt2 - snapshot_memory[0]))
       if delta_ms > 500:
@@ -1388,7 +1391,6 @@ def CheckChangedLUCIConfigs(input_api, output_api):
   import base64
   import json
   import logging
-  import urllib2
 
   import auth
   import git_cl
@@ -1425,16 +1427,16 @@ def CheckChangedLUCIConfigs(input_api, output_api):
   def request(endpoint, body=None):
     api_url = ('https://%s/_ah/api/config/v1/%s'
                % (LUCI_CONFIG_HOST_NAME, endpoint))
-    req = urllib2.Request(api_url)
+    req = input_api.urllib_request.Request(api_url)
     req.add_header('Authorization', 'Bearer %s' % acc_tkn.token)
     if body is not None:
       req.add_header('Content-Type', 'application/json')
       req.add_data(json.dumps(body))
-    return json.load(urllib2.urlopen(req))
+    return json.load(input_api.urllib_request.urlopen(req))
 
   try:
     config_sets = request('config-sets').get('config_sets')
-  except urllib2.HTTPError as e:
+  except input_api.urllib_error.HTTPError as e:
     return [output_api.PresubmitError(
         'Config set request to luci-config failed', long_text=str(e))]
   if not config_sets:
@@ -1472,7 +1474,7 @@ def CheckChangedLUCIConfigs(input_api, output_api):
         cs_to_files[cs].append({
           'path': file_path[len(dr):] if dr != '/' else file_path,
           'content': base64.b64encode(
-              '\n'.join(f.NewContents()).encode('utf-8'))
+              '\n'.join(f.NewContents()).encode('utf-8')).decode('utf-8')
         })
   outputs = []
   for cs, f in cs_to_files.items():
@@ -1480,7 +1482,7 @@ def CheckChangedLUCIConfigs(input_api, output_api):
       # TODO(myjang): parallelize
       res = request(
           'validate-config', body={'config_set': cs, 'files': f})
-    except urllib2.HTTPError as e:
+    except input_api.urllib_error.HTTPError as e:
       return [output_api.PresubmitError(
           'Validation request to luci-config failed', long_text=str(e))]
     for msg in res.get('messages', []):

presubmit_support.py

@@ -7,6 +7,7 @@
 """
 
 from __future__ import print_function
+from __future__ import unicode_literals
 
 __version__ = '1.8.0'
 
@@ -33,7 +34,6 @@ import sys  # Parts exposed through API.
 import tempfile  # Exposed through the API.
 import threading
 import time
-import types
 import unittest  # Exposed through the API.
 from warnings import warn
 
@@ -542,10 +542,12 @@ class InputApi(object):
     self.os_walk = os.walk
     self.re = re
     self.subprocess = subprocess
+    self.sys = sys
     self.tempfile = tempfile
     self.time = time
     self.unittest = unittest
-    self.urllib2 = urllib2
+    if sys.version_info.major == 2:
+      self.urllib2 = urllib2
     self.urllib_request = urllib_request
     self.urllib_error = urllib_error
 
@@ -575,7 +577,7 @@ class InputApi(object):
     # TODO(dpranke): figure out a list of all approved owners for a repo
     # in order to be able to handle wildcard OWNERS files?
     self.owners_db = owners.Database(change.RepositoryRoot(),
-                                     fopen=file, os_path=self.os_path)
+                                     fopen=open, os_path=self.os_path)
     self.owners_finder = owners_finder.OwnersFinder
     self.verbose = verbose
     self.Command = CommandData
@@ -609,9 +611,9 @@ class InputApi(object):
     if len(dir_with_slash) == 1:
       dir_with_slash = ''
 
-    return filter(
+    return list(filter(
         lambda x: normpath(x.AbsoluteLocalPath()).startswith(dir_with_slash),
-        self.change.AffectedFiles(include_deletes, file_filter))
+        self.change.AffectedFiles(include_deletes, file_filter)))
 
   def LocalPaths(self):
     """Returns local paths of input_api.AffectedFiles()."""
@@ -633,8 +635,9 @@ class InputApi(object):
                " is deprecated and ignored" % str(include_deletes),
            category=DeprecationWarning,
            stacklevel=2)
-    return filter(lambda x: x.IsTestableFile(),
-                  self.AffectedFiles(include_deletes=False, **kwargs))
+    return list(filter(
+        lambda x: x.IsTestableFile(),
+        self.AffectedFiles(include_deletes=False, **kwargs)))
 
   def AffectedTextFiles(self, include_deletes=None):
     """An alias to AffectedTestableFiles for backwards compatibility."""
@@ -667,7 +670,7 @@ class InputApi(object):
     """
     if not source_file:
       source_file = self.FilterSourceFile
-    return filter(source_file, self.AffectedTestableFiles())
+    return list(filter(source_file, self.AffectedTestableFiles()))
 
   def RightHandSideLines(self, source_file_filter=None):
     """An iterator over all text lines in "new" version of changed files.
@@ -1094,11 +1097,11 @@ class Change(object):
     Returns:
       [AffectedFile(path, action), AffectedFile(path, action)]
     """
-    affected = filter(file_filter, self._affected_files)
+    affected = list(filter(file_filter, self._affected_files))
 
     if include_deletes:
       return affected
-    return filter(lambda x: x.Action() != 'D', affected)
+    return list(filter(lambda x: x.Action() != 'D', affected))
 
   def AffectedTestableFiles(self, include_deletes=None, **kwargs):
     """Return a list of the existing text files in a change."""
@@ -1107,8 +1110,9 @@ class Change(object):
                " is deprecated and ignored" % str(include_deletes),
            category=DeprecationWarning,
            stacklevel=2)
-    return filter(lambda x: x.IsTestableFile(),
-                  self.AffectedFiles(include_deletes=False, **kwargs))
+    return list(filter(
+        lambda x: x.IsTestableFile(),
+        self.AffectedFiles(include_deletes=False, **kwargs)))
 
   def AffectedTextFiles(self, include_deletes=None):
     """An alias to AffectedTestableFiles for backwards compatibility."""
@@ -1445,9 +1449,9 @@ class PresubmitExecuter(object):
         logging.debug('Running %s done.', function_name)
         self.more_cc.extend(output_api.more_cc)
       finally:
-        map(os.remove, input_api._named_temporary_files)
-      if not (isinstance(result, types.TupleType) or
-              isinstance(result, types.ListType)):
+        for f in input_api._named_temporary_files:
+          os.remove(f)
+      if not isinstance(result, (tuple, list)):
         raise PresubmitFailure(
           'Presubmit functions must return a tuple or list')
       for item in result:
@@ -1566,7 +1570,8 @@ def DoPresubmitChecks(change,
         ]
       }
 
-      gclient_utils.FileWrite(json_output, json.dumps(presubmit_results))
+      gclient_utils.FileWrite(
+          json_output, json.dumps(presubmit_results, sort_keys=True))
 
     output.write('\n')
     for name, items in (('Messages', notifications),