Commit b9830245 authored by Edward Lemur's avatar Edward Lemur Committed by Commit Bot

presubmit_support: Fix tests on Python 3.

The problems on crbug.com/1017367 should have been fixed by crrev.com/c/1880013

Bug: 1009814
Change-Id: Ia98304ca983b4e7e293fedb9df1fee589de58ba9
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/tools/depot_tools/+/1887714Reviewed-by: 's avatarAnthony Polito <apolito@google.com>
Commit-Queue: Edward Lesmes <ehmaldonado@chromium.org>
parent 14705d8c
......@@ -15,7 +15,6 @@ import base64
import collections
import datetime
import glob
import httplib
import httplib2
import itertools
import json
......@@ -30,9 +29,6 @@ import sys
import tempfile
import textwrap
import time
import urllib
import urllib2
import urlparse
import uuid
import webbrowser
import zlib
......@@ -58,6 +54,17 @@ import subcommand
import subprocess2
import watchlists
if sys.version_info.major == 2:
import httplib
import urllib2 as urllib_request
import urllib2 as urllib_error
import urlparse
else:
import http.client as httplib
import urllib.request as urllib_request
import urllib.error as urllib_error
import urllib.parse as urlparse
__version__ = '2.0'
# Traces for git push will be stored in a traces directory inside the
......@@ -3186,7 +3193,7 @@ def urlretrieve(source, destination):
This is necessary because urllib is broken for SSL connections via a proxy.
"""
with open(destination, 'w') as f:
f.write(urllib2.urlopen(source).read())
f.write(urllib_request.urlopen(source).read())
def hasSheBang(fname):
......@@ -4659,7 +4666,7 @@ def GetTreeStatus(url=None):
'unknown' or 'unset'."""
url = url or settings.GetTreeStatusUrl(error_ok=True)
if url:
status = urllib2.urlopen(url).read().lower()
status = urllib_request.urlopen(url).read().lower()
if status.find('closed') != -1 or status == '0':
return 'closed'
elif status.find('open') != -1 or status == '1':
......@@ -4673,7 +4680,7 @@ def GetTreeStatusReason():
with the reason for the tree to be opened or closed."""
url = settings.GetTreeStatusUrl()
json_url = urlparse.urljoin(url, '/current?format=json')
connection = urllib2.urlopen(json_url)
connection = urllib_request.urlopen(json_url)
status = json.loads(connection.read())
connection.close()
return status['message']
......@@ -5462,7 +5469,7 @@ def main(argv):
return dispatcher.execute(OptionParser(), argv)
except auth.LoginRequiredError as e:
DieWithError(str(e))
except urllib2.HTTPError as e:
except urllib_error.HTTPError as e:
if e.code != 500:
raise
DieWithError(
......
......@@ -251,7 +251,7 @@ def CheckGenderNeutral(input_api, output_api, source_file_filter=None):
submitted.
"""
gendered_re = input_api.re.compile(
'(^|\s|\(|\[)([Hh]e|[Hh]is|[Hh]ers?|[Hh]im|[Ss]he|[Gg]uys?)\\b')
r'(^|\s|\(|\[)([Hh]e|[Hh]is|[Hh]ers?|[Hh]im|[Ss]he|[Gg]uys?)\\b')
errors = []
for f in input_api.AffectedFiles(include_deletes=False,
......@@ -576,7 +576,7 @@ def CheckTreeIsOpen(input_api, output_api,
return []
try:
if json_url:
connection = input_api.urllib2.urlopen(json_url)
connection = input_api.urllib_request.urlopen(json_url)
status = input_api.json.loads(connection.read())
connection.close()
if not status['can_commit_freely']:
......@@ -585,7 +585,7 @@ def CheckTreeIsOpen(input_api, output_api,
return [output_api.PresubmitError(short_text, long_text=long_text)]
else:
# TODO(bradnelson): drop this once all users are gone.
connection = input_api.urllib2.urlopen(url)
connection = input_api.urllib_request.urlopen(url)
status = connection.read()
connection.close()
if input_api.re.match(closed, status):
......@@ -752,10 +752,10 @@ def GetPythonUnitTests(input_api, output_api, unit_tests):
# We convert to str, since on Windows on Python 2 only strings are allowed
# as environment variables, but literals are unicode since we're importing
# unicode_literals from __future__.
if env.get(str('PYTHONPATH')):
backpath.append(env.get(str('PYTHONPATH')))
env[str('PYTHONPATH')] = input_api.os_path.pathsep.join((backpath))
env.pop(str('VPYTHON_CLEAR_PYTHONPATH'), None)
if env.get('PYTHONPATH'):
backpath.append(env.get('PYTHONPATH'))
env['PYTHONPATH'] = input_api.os_path.pathsep.join((backpath))
env.pop('VPYTHON_CLEAR_PYTHONPATH', None)
cmd = [input_api.python_executable, '-m', '%s' % unit_test]
results.append(input_api.Command(
name=unit_test_name,
......@@ -830,7 +830,7 @@ def GetPylint(input_api, output_api, white_list=None, black_list=None,
The default white_list enforces looking only at *.py files.
"""
white_list = tuple(white_list or ('.*\.py$',))
white_list = tuple(white_list or (r'.*\.py$',))
black_list = tuple(black_list or input_api.DEFAULT_BLACK_LIST)
extra_paths_list = extra_paths_list or []
......@@ -876,8 +876,7 @@ def GetPylint(input_api, output_api, white_list=None, black_list=None,
input_api.logging.info('Running pylint on %d files', len(files))
input_api.logging.debug('Running pylint on: %s', files)
env = input_api.environ.copy()
env['PYTHONPATH'] = input_api.os_path.pathsep.join(
extra_paths_list).encode('utf8')
env['PYTHONPATH'] = input_api.os_path.pathsep.join(extra_paths_list)
env.pop('VPYTHON_CLEAR_PYTHONPATH', None)
input_api.logging.debug(' with extra PYTHONPATH: %r', extra_paths_list)
......@@ -945,7 +944,7 @@ def RunPylint(input_api, *args, **kwargs):
def CheckBuildbotPendingBuilds(input_api, output_api, url, max_pendings,
ignored):
try:
connection = input_api.urllib2.urlopen(url)
connection = input_api.urllib_request.urlopen(url)
raw_data = connection.read()
connection.close()
except IOError:
......@@ -1010,7 +1009,7 @@ def CheckOwners(input_api, output_api, source_file_filter=None):
input_api.change.RepositoryRoot(),
owner_email,
reviewers,
fopen=file,
fopen=open,
os_path=input_api.os_path,
email_postfix='',
disable_color=True,
......@@ -1135,7 +1134,7 @@ def PanProjectChecks(input_api, output_api,
# 2006-20xx string used on the oldest files. 2006-20xx is deprecated, but
# tolerated on old files.
current_year = int(input_api.time.strftime('%Y'))
allowed_years = (str(s) for s in reversed(xrange(2006, current_year + 1)))
allowed_years = (str(s) for s in reversed(range(2006, current_year + 1)))
years_re = '(' + '|'.join(allowed_years) + '|2006-2008|2006-2009|2006-2010)'
# The (c) is deprecated, but tolerate it until it's removed from all files.
......@@ -1164,7 +1163,10 @@ def PanProjectChecks(input_api, output_api,
snapshot_memory = []
def snapshot(msg):
"""Measures & prints performance warning if a rule is running slow."""
dt2 = input_api.time.clock()
if input_api.sys.version_info.major == 2:
dt2 = input_api.time.clock()
else:
dt2 = input_api.time.process_time()
if snapshot_memory:
delta_ms = int(1000*(dt2 - snapshot_memory[0]))
if delta_ms > 500:
......@@ -1397,7 +1399,6 @@ def CheckChangedLUCIConfigs(input_api, output_api):
import base64
import json
import logging
import urllib2
import auth
import git_cl
......@@ -1434,16 +1435,16 @@ def CheckChangedLUCIConfigs(input_api, output_api):
def request(endpoint, body=None):
api_url = ('https://%s/_ah/api/config/v1/%s'
% (LUCI_CONFIG_HOST_NAME, endpoint))
req = urllib2.Request(api_url)
req = input_api.urllib_request.Request(api_url)
req.add_header('Authorization', 'Bearer %s' % acc_tkn.token)
if body is not None:
req.add_header('Content-Type', 'application/json')
req.add_data(json.dumps(body))
return json.load(urllib2.urlopen(req))
return json.load(input_api.urllib_request.urlopen(req))
try:
config_sets = request('config-sets').get('config_sets')
except urllib2.HTTPError as e:
except input_api.urllib_error.HTTPError as e:
return [output_api.PresubmitError(
'Config set request to luci-config failed', long_text=str(e))]
if not config_sets:
......@@ -1481,7 +1482,7 @@ def CheckChangedLUCIConfigs(input_api, output_api):
cs_to_files[cs].append({
'path': file_path[len(dr):] if dr != '/' else file_path,
'content': base64.b64encode(
'\n'.join(f.NewContents()).encode('utf-8'))
'\n'.join(f.NewContents()).encode('utf-8')).decode('utf-8')
})
outputs = []
for cs, f in cs_to_files.items():
......@@ -1489,7 +1490,7 @@ def CheckChangedLUCIConfigs(input_api, output_api):
# TODO(myjang): parallelize
res = request(
'validate-config', body={'config_set': cs, 'files': f})
except urllib2.HTTPError as e:
except input_api.urllib_error.HTTPError as e:
return [output_api.PresubmitError(
'Validation request to luci-config failed', long_text=str(e))]
for msg in res.get('messages', []):
......
......@@ -33,7 +33,6 @@ import sys # Parts exposed through API.
import tempfile # Exposed through the API.
import threading
import time
import types
import traceback
import unittest # Exposed through the API.
from warnings import warn
......@@ -549,10 +548,12 @@ class InputApi(object):
self.os_walk = os.walk
self.re = re
self.subprocess = subprocess
self.sys = sys
self.tempfile = tempfile
self.time = time
self.unittest = unittest
self.urllib2 = urllib2
if sys.version_info.major == 2:
self.urllib2 = urllib2
self.urllib_request = urllib_request
self.urllib_error = urllib_error
......@@ -581,7 +582,7 @@ class InputApi(object):
# TODO(dpranke): figure out a list of all approved owners for a repo
# in order to be able to handle wildcard OWNERS files?
self.owners_db = owners.Database(change.RepositoryRoot(),
fopen=file, os_path=self.os_path)
fopen=open, os_path=self.os_path)
self.owners_finder = owners_finder.OwnersFinder
self.verbose = verbose
self.Command = CommandData
......@@ -615,9 +616,9 @@ class InputApi(object):
if len(dir_with_slash) == 1:
dir_with_slash = ''
return filter(
return list(filter(
lambda x: normpath(x.AbsoluteLocalPath()).startswith(dir_with_slash),
self.change.AffectedFiles(include_deletes, file_filter))
self.change.AffectedFiles(include_deletes, file_filter)))
def LocalPaths(self):
"""Returns local paths of input_api.AffectedFiles()."""
......@@ -639,8 +640,9 @@ class InputApi(object):
" is deprecated and ignored" % str(include_deletes),
category=DeprecationWarning,
stacklevel=2)
return filter(lambda x: x.IsTestableFile(),
self.AffectedFiles(include_deletes=False, **kwargs))
return list(filter(
lambda x: x.IsTestableFile(),
self.AffectedFiles(include_deletes=False, **kwargs)))
def AffectedTextFiles(self, include_deletes=None):
"""An alias to AffectedTestableFiles for backwards compatibility."""
......@@ -673,7 +675,7 @@ class InputApi(object):
"""
if not source_file:
source_file = self.FilterSourceFile
return filter(source_file, self.AffectedTestableFiles())
return list(filter(source_file, self.AffectedTestableFiles()))
def RightHandSideLines(self, source_file_filter=None):
"""An iterator over all text lines in "new" version of changed files.
......@@ -1100,11 +1102,11 @@ class Change(object):
Returns:
[AffectedFile(path, action), AffectedFile(path, action)]
"""
affected = filter(file_filter, self._affected_files)
affected = list(filter(file_filter, self._affected_files))
if include_deletes:
return affected
return filter(lambda x: x.Action() != 'D', affected)
return list(filter(lambda x: x.Action() != 'D', affected))
def AffectedTestableFiles(self, include_deletes=None, **kwargs):
"""Return a list of the existing text files in a change."""
......@@ -1113,8 +1115,9 @@ class Change(object):
" is deprecated and ignored" % str(include_deletes),
category=DeprecationWarning,
stacklevel=2)
return filter(lambda x: x.IsTestableFile(),
self.AffectedFiles(include_deletes=False, **kwargs))
return list(filter(
lambda x: x.IsTestableFile(),
self.AffectedFiles(include_deletes=False, **kwargs)))
def AffectedTextFiles(self, include_deletes=None):
"""An alias to AffectedTestableFiles for backwards compatibility."""
......@@ -1451,9 +1454,9 @@ class PresubmitExecuter(object):
logging.debug('Running %s done.', function_name)
self.more_cc.extend(output_api.more_cc)
finally:
map(os.remove, input_api._named_temporary_files)
if not (isinstance(result, types.TupleType) or
isinstance(result, types.ListType)):
for f in input_api._named_temporary_files:
os.remove(f)
if not isinstance(result, (tuple, list)):
raise PresubmitFailure(
'Presubmit functions must return a tuple or list')
for item in result:
......@@ -1513,11 +1516,8 @@ def DoPresubmitChecks(change,
old_environ = os.environ
try:
# Make sure python subprocesses won't generate .pyc files.
# We convert to str, since on Windows on Python 2 only strings are allowed
# as environment variables, but literals are unicode since we're importing
# unicode_literals from __future__.
os.environ = os.environ.copy()
os.environ[str('PYTHONDONTWRITEBYTECODE')] = str('1')
os.environ['PYTHONDONTWRITEBYTECODE'] = '1'
output = PresubmitOutput(input_stream, output_stream)
......@@ -1575,7 +1575,8 @@ def DoPresubmitChecks(change,
]
}
gclient_utils.FileWrite(json_output, json.dumps(presubmit_results))
gclient_utils.FileWrite(
json_output, json.dumps(presubmit_results, sort_keys=True))
output.write('\n')
for name, items in (('Messages', notifications),
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment