Commit 6cad46b4 authored by Vadim Gorbachev (bmsdave)'s avatar Vadim Gorbachev (bmsdave) Committed by Commit Bot

Preparing v8 to use with python3 /tools/testrunner

There are now less that 400 days until the end of life
of Python 2(aka _legacy_ Python) https://pythonclock.org/ .
The code compatibility check for python2 and python3
used the following tools: futurize, flake8
You can see the reports here: https://travis-ci.com/bmsdave/v8/builds

This CL was uploaded by git cl split.

Bug: v8:8594
Change-Id: I2a90aaecb270f03aed1c0fc92da1a0e2621b0eb2
Reviewed-on: https://chromium-review.googlesource.com/c/1470101
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: 's avatarSergiy Belozorov <sergiyb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#59678}
parent 98c94c16
......@@ -2,6 +2,9 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# for py2/py3 compatibility
from __future__ import print_function
from functools import reduce
from collections import OrderedDict
import json
......@@ -13,6 +16,7 @@ import sys
import traceback
# Add testrunner to the path.
sys.path.insert(
0,
......@@ -248,7 +252,7 @@ class BaseTestRunner(object):
if options.swarming:
# Swarming doesn't print how isolated commands are called. Lets make
# this less cryptic by printing it ourselves.
print ' '.join(sys.argv)
print(' '.join(sys.argv))
self._load_build_config(options)
command.setup(self.target_os, options.device)
......@@ -373,7 +377,7 @@ class BaseTestRunner(object):
if any(map(lambda v: v and ',' in v,
[options.arch, options.mode])): # pragma: no cover
print 'Multiple arch/mode are deprecated'
print('Multiple arch/mode are deprecated')
raise TestRunnerError()
return options, args
......@@ -386,13 +390,13 @@ class BaseTestRunner(object):
pass
if not self.build_config: # pragma: no cover
print 'Failed to load build config'
print('Failed to load build config')
raise TestRunnerError
print 'Build found: %s' % self.outdir
print('Build found: %s' % self.outdir)
if str(self.build_config):
print '>>> Autodetected:'
print self.build_config
print('>>> Autodetected:')
print(self.build_config)
# Represents the OS where tests are run on. Same as host OS except for
# Android, which is determined by build output.
......@@ -469,7 +473,7 @@ class BaseTestRunner(object):
build_config_mode = 'debug' if self.build_config.is_debug else 'release'
if options.mode:
if options.mode not in MODES: # pragma: no cover
print '%s mode is invalid' % options.mode
print('%s mode is invalid' % options.mode)
raise TestRunnerError()
if MODES[options.mode].execution_mode != build_config_mode:
print ('execution mode (%s) for %s is inconsistent with build config '
......@@ -615,7 +619,7 @@ class BaseTestRunner(object):
test_chain = testsuite.TestGenerator(0, [], [])
for name in names:
if options.verbose:
print '>>> Loading test suite: %s' % name
print('>>> Loading test suite: %s' % name)
suite = testsuite.TestSuite.Load(
os.path.join(options.test_root, name), test_config)
......@@ -710,7 +714,7 @@ class BaseTestRunner(object):
def _prepare_procs(self, procs):
procs = filter(None, procs)
for i in xrange(0, len(procs) - 1):
for i in range(0, len(procs) - 1):
procs[i].connect_to(procs[i + 1])
procs[0].setup()
......@@ -751,8 +755,8 @@ class BaseTestRunner(object):
# TODO(machenbach): Turn this into an assert. If that's wrong on the
# bots, printing will be quite useless. Or refactor this code to make
# sure we get a return code != 0 after testing if we got here.
print "shard-run not a valid number, should be in [1:shard-count]"
print "defaulting back to running all tests"
print("shard-run not a valid number, should be in [1:shard-count]")
print("defaulting back to running all tests")
return 1, 1
return shard_run, shard_count
......
......@@ -9,7 +9,7 @@ Wrapper around the Android device abstraction from src/build/android.
import logging
import os
import sys
import re
BASE_DIR = os.path.normpath(
os.path.join(os.path.dirname(__file__), '..', '..', '..'))
......
......@@ -2,6 +2,8 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# for py2/py3 compatibility
from __future__ import print_function
import os
import re
......@@ -62,7 +64,7 @@ class BaseCommand(object):
def execute(self):
if self.verbose:
print '# %s' % self
print('# %s' % self)
process = self._start_process()
......@@ -187,7 +189,7 @@ class WindowsCommand(BaseCommand):
def _kill_process(self, process):
if self.verbose:
print 'Attempting to kill process %d' % process.pid
print('Attempting to kill process %d' % process.pid)
sys.stdout.flush()
tk = subprocess.Popen(
'taskkill /T /F /PID %d' % process.pid,
......@@ -196,10 +198,10 @@ class WindowsCommand(BaseCommand):
)
stdout, stderr = tk.communicate()
if self.verbose:
print 'Taskkill results for %d' % process.pid
print stdout
print stderr
print 'Return code: %d' % tk.returncode
print('Taskkill results for %d' % process.pid)
print(stdout)
print(stderr)
print('Return code: %d' % tk.returncode)
sys.stdout.flush()
......@@ -237,7 +239,7 @@ class AndroidCommand(BaseCommand):
This pushes all required files to the device and then runs the command.
"""
if self.verbose:
print '# %s' % self
print('# %s' % self)
self.driver.push_executable(self.shell_dir, 'bin', self.shell_name)
......
......@@ -3,7 +3,9 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from Queue import Empty
# for py2/py3 compatibility
from __future__ import print_function
from contextlib import contextmanager
from multiprocessing import Process, Queue
import os
......@@ -11,6 +13,11 @@ import signal
import time
import traceback
try:
from queue import Empty # Python 3
except ImportError:
from Queue import Empty # Python 2
from . import command
......@@ -22,7 +29,11 @@ def setup_testing():
global Process
del Queue
del Process
from Queue import Queue
try:
from queue import Queue # Python 3
except ImportError:
from Queue import Queue # Python 2
from threading import Thread as Process
# Monkeypatch threading Queue to look like multiprocessing Queue.
Queue.cancel_join_thread = lambda self: None
......@@ -70,7 +81,7 @@ def Worker(fn, work_queue, done_queue,
except command.AbortException:
# SIGINT, SIGTERM or internal hard timeout.
break
except Exception, e:
except Exception as e:
traceback.print_exc()
print(">>> EXCEPTION: %s" % e)
done_queue.put(ExceptionResult(e))
......@@ -153,7 +164,7 @@ class Pool():
# Disable sigint and sigterm to prevent subprocesses from capturing the
# signals.
with without_sig():
for w in xrange(self.num_workers):
for w in range(self.num_workers):
p = Process(target=Worker, args=(fn,
self.work_queue,
self.done_queue,
......@@ -198,7 +209,7 @@ class Pool():
def _advance_more(self, gen):
while self.processing_count < self.num_workers * self.BUFFER_FACTOR:
try:
self.work_queue.put(gen.next())
self.work_queue.put(next(gen))
self.processing_count += 1
except StopIteration:
self.advance = self._advance_empty
......
......@@ -25,6 +25,9 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# for py2/py3 compatibility
from __future__ import print_function
import os
import re
......@@ -133,7 +136,7 @@ class StatusFile(object):
variant_desc = 'variant independent'
else:
variant_desc = 'variant: %s' % variant
print 'Unused rule: %s -> %s (%s)' % (rule, value, variant_desc)
print('Unused rule: %s -> %s (%s)' % (rule, value, variant_desc))
def _JoinsPassAndFail(outcomes1, outcomes2):
......@@ -329,5 +332,5 @@ def PresubmitCheck(path):
"missing file for %s test %s" % (basename, rule))
return status["success"]
except Exception as e:
print e
print(e)
return False
......@@ -3,6 +3,7 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import unittest
import statusfile
......
......@@ -25,6 +25,8 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# for py2/py3 compatibility
from __future__ import print_function
from os.path import exists
from os.path import isdir
......@@ -147,7 +149,7 @@ def URLRetrieve(source, destination):
return
except:
# If there's no curl, fall back to urlopen.
print "Curl is currently not installed. Falling back to python."
print("Curl is currently not installed. Falling back to python.")
pass
with open(destination, 'w') as f:
f.write(urllib2.urlopen(source).read())
......
......@@ -25,6 +25,8 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# for py2/py3 compatibility
from __future__ import print_function
import sys
import time
......@@ -63,7 +65,7 @@ def PrintReport(tests):
else:
assert False # Unreachable # TODO: check this in outcomes parsing phase.
print REPORT_TEMPLATE % {
print(REPORT_TEMPLATE % {
"total": total,
"skipped": skipped,
"nocrash": nocrash,
......@@ -71,17 +73,17 @@ def PrintReport(tests):
"fail_ok": fail_ok,
"fail": fail,
"crash": crash,
}
})
def PrintTestSource(tests):
for test in tests:
print "--- begin source: %s ---" % test
print("--- begin source: %s ---" % test)
if test.is_source_available():
print test.get_source()
print(test.get_source())
else:
print '(no source available)'
print "--- end source: %s ---" % test
print('(no source available)')
print("--- end source: %s ---" % test)
def FormatTime(d):
......@@ -92,11 +94,11 @@ def FormatTime(d):
def PrintTestDurations(suites, outputs, overall_time):
# Write the times to stderr to make it easy to separate from the
# test output.
print
print()
sys.stderr.write("--- Total time: %s ---\n" % FormatTime(overall_time))
timed_tests = [(t, outputs[t].duration) for s in suites for t in s.tests
if t in outputs]
timed_tests.sort(key=lambda (_, duration): duration, reverse=True)
timed_tests.sort(key=lambda test_duration: test_duration[1], reverse=True)
index = 1
for test, duration in timed_tests[:20]:
t = FormatTime(duration)
......
......@@ -4,6 +4,8 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# for py2/py3 compatibility
from __future__ import print_function
import random
import sys
......@@ -97,8 +99,8 @@ class NumFuzzer(base_runner.BaseTestRunner):
if options.combine_tests:
if options.combine_min > options.combine_max:
print ('min_group_size (%d) cannot be larger than max_group_size (%d)' %
options.min_group_size, options.max_group_size)
print(('min_group_size (%d) cannot be larger than max_group_size (%d)' %
options.min_group_size, options.max_group_size))
raise base_runner.TestRunnerError()
if options.variants != 'default':
......@@ -167,7 +169,7 @@ class NumFuzzer(base_runner.BaseTestRunner):
for indicator in indicators:
indicator.finished()
print '>>> %d tests ran' % results.total
print('>>> %d tests ran' % results.total)
if results.failed:
return utils.EXIT_CODE_FAILURES
......
......@@ -53,6 +53,12 @@ MODULE_RESOURCES_PATTERN_1 = re.compile(
MODULE_RESOURCES_PATTERN_2 = re.compile(
r"(?:import|export).*from (?:'|\")([^'\"]+)(?:'|\")")
try:
cmp # Python 2
except NameError:
def cmp(x, y): # Python 3
return (x > y) - (x < y)
class TestCase(object):
def __init__(self, suite, path, name, test_config):
......
......@@ -4,6 +4,9 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# for py2/py3 compatibility
from __future__ import print_function
from functools import reduce
import os
import re
......@@ -217,7 +220,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
def CheckTestMode(name, option): # pragma: no cover
if not option in ["run", "skip", "dontcare"]:
print "Unknown %s mode %s" % (name, option)
print("Unknown %s mode %s" % (name, option))
raise base_runner.TestRunnerError()
CheckTestMode("slow test", options.slow_tests)
CheckTestMode("pass|fail test", options.pass_fail_tests)
......@@ -240,7 +243,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
for v in user_variants:
if v not in ALL_VARIANTS:
print 'Unknown variant: %s' % v
print('Unknown variant: %s' % v)
raise base_runner.TestRunnerError()
assert False, 'Unreachable'
......@@ -280,7 +283,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
def _do_execute(self, tests, args, options):
jobs = options.j
print '>>> Running with test processors'
print('>>> Running with test processors')
loader = LoadProc(tests)
results = self._create_result_tracker(options)
indicators = self._create_progress_indicators(
......@@ -326,11 +329,11 @@ class StandardTestRunner(base_runner.BaseTestRunner):
else:
percentage = 0
print ('>>> %d base tests produced %d tests (%d%s)'
print (('>>> %d base tests produced %d tests (%d%s)'
' non-filtered tests') % (
tests.test_count_estimate, results.total, percentage, '%')
tests.test_count_estimate, results.total, percentage, '%'))
print '>>> %d tests ran' % (results.total - results.remaining)
print('>>> %d tests ran' % (results.total - results.remaining))
exit_code = utils.EXIT_CODE_PASS
if results.failed:
......
......@@ -2,6 +2,9 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# for py2/py3 compatibility
from __future__ import print_function
from collections import defaultdict
import time
......@@ -9,7 +12,6 @@ from . import base
from ..objects import testcase
from ..outproc import base as outproc
class CombinerProc(base.TestProc):
def __init__(self, rng, min_group_size, max_group_size, count):
"""
......@@ -62,7 +64,7 @@ class CombinerProc(base.TestProc):
self._send_next_test()
def generate_initial_tests(self, num=1):
for _ in xrange(0, num):
for _ in range(0, num):
self._send_next_test()
def _send_next_test(self):
......@@ -122,4 +124,4 @@ class TestGroups(object):
group_key = rng.choice(self._keys)
tests = self._groups[group_key]
return [rng.choice(tests) for _ in xrange(0, max_size)]
return [rng.choice(tests) for _ in range(0, max_size)]
......@@ -2,6 +2,9 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# for py2/py3 compatibility
from __future__ import print_function
import json
import os
import sys
......@@ -15,10 +18,10 @@ def print_failure_header(test):
negative_marker = '[negative] '
else:
negative_marker = ''
print "=== %(label)s %(negative)s===" % {
print("=== %(label)s %(negative)s===" % {
'label': test,
'negative': negative_marker,
}
})
class ResultsTracker(base.TestProcObserver):
......@@ -41,7 +44,7 @@ class ResultsTracker(base.TestProcObserver):
if result.has_unexpected_output:
self.failed += 1
if self.max_failures and self.failed >= self.max_failures:
print '>>> Too many failures, exiting...'
print('>>> Too many failures, exiting...')
self.stop()
......@@ -64,33 +67,33 @@ class SimpleProgressIndicator(ProgressIndicator):
def finished(self):
crashed = 0
print
print()
for test, result in self._failed:
print_failure_header(test)
if result.output.stderr:
print "--- stderr ---"
print result.output.stderr.strip()
print("--- stderr ---")
print(result.output.stderr.strip())
if result.output.stdout:
print "--- stdout ---"
print result.output.stdout.strip()
print "Command: %s" % result.cmd.to_string()
print("--- stdout ---")
print(result.output.stdout.strip())
print("Command: %s" % result.cmd.to_string())
if result.output.HasCrashed():
print "exit code: %d" % result.output.exit_code
print "--- CRASHED ---"
print("exit code: %d" % result.output.exit_code)
print("--- CRASHED ---")
crashed += 1
if result.output.HasTimedOut():
print "--- TIMEOUT ---"
print("--- TIMEOUT ---")
if len(self._failed) == 0:
print "==="
print "=== All tests succeeded"
print "==="
print("===")
print("=== All tests succeeded")
print("===")
else:
print
print "==="
print "=== %i tests failed" % len(self._failed)
print()
print("===")
print("=== %i tests failed" % len(self._failed))
if crashed > 0:
print "=== %i tests CRASHED" % crashed
print "==="
print("=== %i tests CRASHED" % crashed)
print("===")
class VerboseProgressIndicator(SimpleProgressIndicator):
......@@ -99,7 +102,7 @@ class VerboseProgressIndicator(SimpleProgressIndicator):
self._last_printed_time = time.time()
def _print(self, text):
print text
print(text)
sys.stdout.flush()
self._last_printed_time = time.time()
......@@ -181,19 +184,19 @@ class CompactProgressIndicator(ProgressIndicator):
self._clear_line(self._last_status_length)
print_failure_header(test)
if len(stdout):
print self._templates['stdout'] % stdout
print(self._templates['stdout'] % stdout)
if len(stderr):
print self._templates['stderr'] % stderr
print "Command: %s" % result.cmd
print(self._templates['stderr'] % stderr)
print("Command: %s" % result.cmd)
if output.HasCrashed():
print "exit code: %d" % output.exit_code
print "--- CRASHED ---"
print("exit code: %d" % output.exit_code)
print("--- CRASHED ---")
if output.HasTimedOut():
print "--- TIMEOUT ---"
print("--- TIMEOUT ---")
def finished(self):
self._print_progress('Done')
print
print()
def _print_progress(self, name):
self._clear_line(self._last_status_length)
......@@ -212,7 +215,7 @@ class CompactProgressIndicator(ProgressIndicator):
}
status = self._truncate(status, 78)
self._last_status_length = len(status)
print status,
print(status, end=' ')
sys.stdout.flush()
def _truncate(self, string, length):
......@@ -238,7 +241,7 @@ class ColorProgressIndicator(CompactProgressIndicator):
super(ColorProgressIndicator, self).__init__(templates)
def _clear_line(self, last_length):
print "\033[1K\r",
print("\033[1K\r", end=' ')
class MonochromeProgressIndicator(CompactProgressIndicator):
......@@ -252,7 +255,7 @@ class MonochromeProgressIndicator(CompactProgressIndicator):
super(MonochromeProgressIndicator, self).__init__(templates)
def _clear_line(self, last_length):
print ("\r" + (" " * last_length) + "\r"),
print(("\r" + (" " * last_length) + "\r"), end=' ')
class JsonTestProgressIndicator(ProgressIndicator):
......@@ -321,7 +324,7 @@ class JsonTestProgressIndicator(ProgressIndicator):
float(len(self.tests)))
# Sort tests by duration.
self.tests.sort(key=lambda (_, duration, cmd): duration, reverse=True)
self.tests.sort(key=lambda __duration_cmd: __duration_cmd[1], reverse=True)
slowest_tests = [
{
"name": str(test),
......
......@@ -35,7 +35,7 @@ class SeedProc(base.TestProcProducer):
def _next_test(self, test):
is_loaded = False
for _ in xrange(0, self._parallel_subtests):
for _ in range(0, self._parallel_subtests):
is_loaded |= self._try_send_next_test(test)
return is_loaded
......
......@@ -2,6 +2,9 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# for py2/py3 compatibility
from __future__ import print_function
import signal
from . import base
......@@ -21,11 +24,11 @@ class SignalProc(base.TestProcObserver):
signal.signal(signal.SIGTERM, self._on_sigterm)
def _on_ctrlc(self, _signum, _stack_frame):
print '>>> Ctrl-C detected, early abort...'
print('>>> Ctrl-C detected, early abort...')
self.exit_code = utils.EXIT_CODE_INTERRUPTED
self.stop()
def _on_sigterm(self, _signum, _stack_frame):
print '>>> SIGTERM received, early abort...'
print('>>> SIGTERM received, early abort...')
self.exit_code = utils.EXIT_CODE_TERMINATED
self.stop()
......@@ -11,6 +11,9 @@ Raw gyp values are supported - they will be tranformed into valid json.
"""
# TODO(machenbach): Remove this when gyp is deprecated.
# for py2/py3 compatibility
from __future__ import print_function
import json
import os
import sys
......@@ -47,7 +50,7 @@ def as_json(kv):
try:
return k, json.loads(v2)
except ValueError as e:
print(k, v, v2)
print((k, v, v2))
raise e
with open(sys.argv[1], 'w') as f:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment