Preparing v8 to use with python3 /tools/testrunner

There are now less that 400 days until the end of life
of Python 2(aka _legacy_ Python) https://pythonclock.org/ .
The code compatibility check for python2 and python3
used the following tools: futurize, flake8
You can see the reports here: https://travis-ci.com/bmsdave/v8/builds

This CL was uploaded by git cl split.

Bug: v8:8594
Change-Id: I2a90aaecb270f03aed1c0fc92da1a0e2621b0eb2
Reviewed-on: https://chromium-review.googlesource.com/c/1470101
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Sergiy Belozorov <sergiyb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#59678}

tools/testrunner/base_runner.py

@@ -2,6 +2,9 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+# for py2/py3 compatibility
+from __future__ import print_function
+from functools import reduce
 
 from collections import OrderedDict
 import json
@@ -13,6 +16,7 @@ import sys
 import traceback
 
 
+
 # Add testrunner to the path.
 sys.path.insert(
   0,
@@ -248,7 +252,7 @@ class BaseTestRunner(object):
       if options.swarming:
         # Swarming doesn't print how isolated commands are called. Lets make
         # this less cryptic by printing it ourselves.
-        print ' '.join(sys.argv)
+        print(' '.join(sys.argv))
 
       self._load_build_config(options)
       command.setup(self.target_os, options.device)
@@ -373,7 +377,7 @@ class BaseTestRunner(object):
 
     if any(map(lambda v: v and ',' in v,
                 [options.arch, options.mode])):  # pragma: no cover
-      print 'Multiple arch/mode are deprecated'
+      print('Multiple arch/mode are deprecated')
       raise TestRunnerError()
 
     return options, args
@@ -386,13 +390,13 @@ class BaseTestRunner(object):
         pass
 
     if not self.build_config:  # pragma: no cover
-      print 'Failed to load build config'
+      print('Failed to load build config')
       raise TestRunnerError
 
-    print 'Build found: %s' % self.outdir
+    print('Build found: %s' % self.outdir)
     if str(self.build_config):
-      print '>>> Autodetected:'
-      print self.build_config
+      print('>>> Autodetected:')
+      print(self.build_config)
 
     # Represents the OS where tests are run on. Same as host OS except for
     # Android, which is determined by build output.
@@ -469,7 +473,7 @@ class BaseTestRunner(object):
     build_config_mode = 'debug' if self.build_config.is_debug else 'release'
     if options.mode:
       if options.mode not in MODES:  # pragma: no cover
-        print '%s mode is invalid' % options.mode
+        print('%s mode is invalid' % options.mode)
         raise TestRunnerError()
       if MODES[options.mode].execution_mode != build_config_mode:
         print ('execution mode (%s) for %s is inconsistent with build config '
@@ -615,7 +619,7 @@ class BaseTestRunner(object):
     test_chain = testsuite.TestGenerator(0, [], [])
     for name in names:
       if options.verbose:
-        print '>>> Loading test suite: %s' % name
+        print('>>> Loading test suite: %s' % name)
       suite = testsuite.TestSuite.Load(
           os.path.join(options.test_root, name), test_config)
 
@@ -710,7 +714,7 @@ class BaseTestRunner(object):
 
   def _prepare_procs(self, procs):
     procs = filter(None, procs)
-    for i in xrange(0, len(procs) - 1):
+    for i in range(0, len(procs) - 1):
       procs[i].connect_to(procs[i + 1])
     procs[0].setup()
 
@@ -751,8 +755,8 @@ class BaseTestRunner(object):
       # TODO(machenbach): Turn this into an assert. If that's wrong on the
       # bots, printing will be quite useless. Or refactor this code to make
       # sure we get a return code != 0 after testing if we got here.
-      print "shard-run not a valid number, should be in [1:shard-count]"
-      print "defaulting back to running all tests"
+      print("shard-run not a valid number, should be in [1:shard-count]")
+      print("defaulting back to running all tests")
       return 1, 1
 
     return shard_run, shard_count

tools/testrunner/local/android.py

@@ -9,7 +9,7 @@ Wrapper around the Android device abstraction from src/build/android.
 import logging
 import os
 import sys
-
+import re
 
 BASE_DIR = os.path.normpath(
     os.path.join(os.path.dirname(__file__), '..', '..', '..'))

tools/testrunner/local/command.py

@@ -2,6 +2,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+# for py2/py3 compatibility
+from __future__ import print_function
 
 import os
 import re
@@ -62,7 +64,7 @@ class BaseCommand(object):
 
   def execute(self):
     if self.verbose:
-      print '# %s' % self
+      print('# %s' % self)
 
     process = self._start_process()
 
@@ -187,7 +189,7 @@ class WindowsCommand(BaseCommand):
 
   def _kill_process(self, process):
     if self.verbose:
-      print 'Attempting to kill process %d' % process.pid
+      print('Attempting to kill process %d' % process.pid)
       sys.stdout.flush()
     tk = subprocess.Popen(
         'taskkill /T /F /PID %d' % process.pid,
@@ -196,10 +198,10 @@ class WindowsCommand(BaseCommand):
     )
     stdout, stderr = tk.communicate()
     if self.verbose:
-      print 'Taskkill results for %d' % process.pid
-      print stdout
-      print stderr
-      print 'Return code: %d' % tk.returncode
+      print('Taskkill results for %d' % process.pid)
+      print(stdout)
+      print(stderr)
+      print('Return code: %d' % tk.returncode)
       sys.stdout.flush()
 
 
@@ -237,7 +239,7 @@ class AndroidCommand(BaseCommand):
     This pushes all required files to the device and then runs the command.
     """
     if self.verbose:
-      print '# %s' % self
+      print('# %s' % self)
 
     self.driver.push_executable(self.shell_dir, 'bin', self.shell_name)
 

tools/testrunner/local/pool.py

@@ -3,7 +3,9 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-from Queue import Empty
+# for py2/py3 compatibility
+from __future__ import print_function
+
 from contextlib import contextmanager
 from multiprocessing import Process, Queue
 import os
@@ -11,6 +13,11 @@ import signal
 import time
 import traceback
 
+try:
+  from queue import Empty  # Python 3
+except ImportError:
+  from Queue import Empty  # Python 2
+
 from . import command
 
 
@@ -22,7 +29,11 @@ def setup_testing():
   global Process
   del Queue
   del Process
-  from Queue import Queue
+  try:
+    from queue import Queue  # Python 3
+  except ImportError:
+    from Queue import Queue  # Python 2
+
   from threading import Thread as Process
   # Monkeypatch threading Queue to look like multiprocessing Queue.
   Queue.cancel_join_thread = lambda self: None
@@ -70,7 +81,7 @@ def Worker(fn, work_queue, done_queue,
       except command.AbortException:
         # SIGINT, SIGTERM or internal hard timeout.
         break
-      except Exception, e:
+      except Exception as e:
         traceback.print_exc()
         print(">>> EXCEPTION: %s" % e)
         done_queue.put(ExceptionResult(e))
@@ -153,7 +164,7 @@ class Pool():
       # Disable sigint and sigterm to prevent subprocesses from capturing the
       # signals.
       with without_sig():
-        for w in xrange(self.num_workers):
+        for w in range(self.num_workers):
           p = Process(target=Worker, args=(fn,
                                           self.work_queue,
                                           self.done_queue,
@@ -198,7 +209,7 @@ class Pool():
   def _advance_more(self, gen):
     while self.processing_count < self.num_workers * self.BUFFER_FACTOR:
       try:
-        self.work_queue.put(gen.next())
+        self.work_queue.put(next(gen))
         self.processing_count += 1
       except StopIteration:
         self.advance = self._advance_empty

tools/testrunner/local/statusfile.py

@@ -25,6 +25,9 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+# for py2/py3 compatibility
+from __future__ import print_function
+
 import os
 import re
 
@@ -133,7 +136,7 @@ class StatusFile(object):
             variant_desc = 'variant independent'
           else:
             variant_desc = 'variant: %s' % variant
-          print 'Unused rule: %s -> %s (%s)' % (rule, value, variant_desc)
+          print('Unused rule: %s -> %s (%s)' % (rule, value, variant_desc))
 
 
 def _JoinsPassAndFail(outcomes1, outcomes2):
@@ -329,5 +332,5 @@ def PresubmitCheck(path):
                   "missing file for %s test %s" % (basename, rule))
     return status["success"]
   except Exception as e:
-    print e
+    print(e)
     return False

tools/testrunner/local/statusfile_unittest.py

@@ -3,6 +3,7 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+
 import unittest
 
 import statusfile

tools/testrunner/local/utils.py

@@ -25,6 +25,8 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+# for py2/py3 compatibility
+from __future__ import print_function
 
 from os.path import exists
 from os.path import isdir
@@ -147,7 +149,7 @@ def URLRetrieve(source, destination):
       return
     except:
       # If there's no curl, fall back to urlopen.
-      print "Curl is currently not installed. Falling back to python."
+      print("Curl is currently not installed. Falling back to python.")
       pass
   with open(destination, 'w') as f:
     f.write(urllib2.urlopen(source).read())

tools/testrunner/local/verbose.py

@@ -25,6 +25,8 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+# for py2/py3 compatibility
+from __future__ import print_function
 
 import sys
 import time
@@ -63,7 +65,7 @@ def PrintReport(tests):
     else:
       assert False # Unreachable # TODO: check this in outcomes parsing phase.
 
-  print REPORT_TEMPLATE % {
+  print(REPORT_TEMPLATE % {
     "total": total,
     "skipped": skipped,
     "nocrash": nocrash,
@@ -71,17 +73,17 @@ def PrintReport(tests):
     "fail_ok": fail_ok,
     "fail": fail,
     "crash": crash,
-  }
+  })
 
 
 def PrintTestSource(tests):
   for test in tests:
-    print "--- begin source: %s ---" % test
+    print("--- begin source: %s ---" % test)
     if test.is_source_available():
-      print test.get_source()
+      print(test.get_source())
     else:
-      print '(no source available)'
-    print "--- end source: %s ---" % test
+      print('(no source available)')
+    print("--- end source: %s ---" % test)
 
 
 def FormatTime(d):
@@ -92,11 +94,11 @@ def FormatTime(d):
 def PrintTestDurations(suites, outputs, overall_time):
     # Write the times to stderr to make it easy to separate from the
     # test output.
-    print
+    print()
     sys.stderr.write("--- Total time: %s ---\n" % FormatTime(overall_time))
     timed_tests = [(t, outputs[t].duration) for s in suites for t in s.tests
                    if t in outputs]
-    timed_tests.sort(key=lambda (_, duration): duration, reverse=True)
+    timed_tests.sort(key=lambda test_duration: test_duration[1], reverse=True)
     index = 1
     for test, duration in timed_tests[:20]:
       t = FormatTime(duration)

tools/testrunner/num_fuzzer.py

@@ -4,6 +4,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+# for py2/py3 compatibility
+from __future__ import print_function
 
 import random
 import sys
@@ -97,8 +99,8 @@ class NumFuzzer(base_runner.BaseTestRunner):
 
     if options.combine_tests:
       if options.combine_min > options.combine_max:
-        print ('min_group_size (%d) cannot be larger than max_group_size (%d)' %
-               options.min_group_size, options.max_group_size)
+        print(('min_group_size (%d) cannot be larger than max_group_size (%d)' %
+               options.min_group_size, options.max_group_size))
         raise base_runner.TestRunnerError()
 
     if options.variants != 'default':
@@ -167,7 +169,7 @@ class NumFuzzer(base_runner.BaseTestRunner):
     for indicator in indicators:
       indicator.finished()
 
-    print '>>> %d tests ran' % results.total
+    print('>>> %d tests ran' % results.total)
     if results.failed:
       return utils.EXIT_CODE_FAILURES
 

tools/testrunner/objects/testcase.py

@@ -53,6 +53,12 @@ MODULE_RESOURCES_PATTERN_1 = re.compile(
 MODULE_RESOURCES_PATTERN_2 = re.compile(
     r"(?:import|export).*from (?:'|\")([^'\"]+)(?:'|\")")
 
+try:
+  cmp             # Python 2
+except NameError:
+  def cmp(x, y):  # Python 3
+    return (x > y) - (x < y)
+
 
 class TestCase(object):
   def __init__(self, suite, path, name, test_config):

tools/testrunner/standard_runner.py

@@ -4,6 +4,9 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+# for py2/py3 compatibility
+from __future__ import print_function
+from functools import reduce
 
 import os
 import re
@@ -217,7 +220,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
 
       def CheckTestMode(name, option):  # pragma: no cover
         if not option in ["run", "skip", "dontcare"]:
-          print "Unknown %s mode %s" % (name, option)
+          print("Unknown %s mode %s" % (name, option))
           raise base_runner.TestRunnerError()
       CheckTestMode("slow test", options.slow_tests)
       CheckTestMode("pass|fail test", options.pass_fail_tests)
@@ -240,7 +243,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
 
       for v in user_variants:
         if v not in ALL_VARIANTS:
-          print 'Unknown variant: %s' % v
+          print('Unknown variant: %s' % v)
           raise base_runner.TestRunnerError()
       assert False, 'Unreachable'
 
@@ -280,7 +283,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
     def _do_execute(self, tests, args, options):
       jobs = options.j
 
-      print '>>> Running with test processors'
+      print('>>> Running with test processors')
       loader = LoadProc(tests)
       results = self._create_result_tracker(options)
       indicators = self._create_progress_indicators(
@@ -326,11 +329,11 @@ class StandardTestRunner(base_runner.BaseTestRunner):
       else:
         percentage = 0
 
-      print ('>>> %d base tests produced %d tests (%d%s)'
+      print (('>>> %d base tests produced %d tests (%d%s)'
              ' non-filtered tests') % (
-          tests.test_count_estimate, results.total, percentage, '%')
+          tests.test_count_estimate, results.total, percentage, '%'))
 
-      print '>>> %d tests ran' % (results.total - results.remaining)
+      print('>>> %d tests ran' % (results.total - results.remaining))
 
       exit_code = utils.EXIT_CODE_PASS
       if results.failed:

tools/testrunner/testproc/combiner.py

@@ -2,6 +2,9 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+# for py2/py3 compatibility
+from __future__ import print_function
+
 from collections import defaultdict
 import time
 
@@ -9,7 +12,6 @@ from . import base
 from ..objects import testcase
 from ..outproc import base as outproc
 
-
 class CombinerProc(base.TestProc):
   def __init__(self, rng, min_group_size, max_group_size, count):
     """
@@ -62,7 +64,7 @@ class CombinerProc(base.TestProc):
     self._send_next_test()
 
   def generate_initial_tests(self, num=1):
-    for _ in xrange(0, num):
+    for _ in range(0, num):
       self._send_next_test()
 
   def _send_next_test(self):
@@ -122,4 +124,4 @@ class TestGroups(object):
 
     group_key = rng.choice(self._keys)
     tests = self._groups[group_key]
-    return [rng.choice(tests) for _ in xrange(0, max_size)]
+    return [rng.choice(tests) for _ in range(0, max_size)]

tools/testrunner/testproc/progress.py

@@ -2,6 +2,9 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+# for py2/py3 compatibility
+from __future__ import print_function
+
 import json
 import os
 import sys
@@ -15,10 +18,10 @@ def print_failure_header(test):
     negative_marker = '[negative] '
   else:
     negative_marker = ''
-  print "=== %(label)s %(negative)s===" % {
+  print("=== %(label)s %(negative)s===" % {
     'label': test,
     'negative': negative_marker,
-  }
+  })
 
 
 class ResultsTracker(base.TestProcObserver):
@@ -41,7 +44,7 @@ class ResultsTracker(base.TestProcObserver):
     if result.has_unexpected_output:
       self.failed += 1
       if self.max_failures and self.failed >= self.max_failures:
-        print '>>> Too many failures, exiting...'
+        print('>>> Too many failures, exiting...')
         self.stop()
 
 
@@ -64,33 +67,33 @@ class SimpleProgressIndicator(ProgressIndicator):
 
   def finished(self):
     crashed = 0
-    print
+    print()
     for test, result in self._failed:
       print_failure_header(test)
       if result.output.stderr:
-        print "--- stderr ---"
-        print result.output.stderr.strip()
+        print("--- stderr ---")
+        print(result.output.stderr.strip())
       if result.output.stdout:
-        print "--- stdout ---"
-        print result.output.stdout.strip()
-      print "Command: %s" % result.cmd.to_string()
+        print("--- stdout ---")
+        print(result.output.stdout.strip())
+      print("Command: %s" % result.cmd.to_string())
       if result.output.HasCrashed():
-        print "exit code: %d" % result.output.exit_code
-        print "--- CRASHED ---"
+        print("exit code: %d" % result.output.exit_code)
+        print("--- CRASHED ---")
         crashed += 1
       if result.output.HasTimedOut():
-        print "--- TIMEOUT ---"
+        print("--- TIMEOUT ---")
     if len(self._failed) == 0:
-      print "==="
-      print "=== All tests succeeded"
-      print "==="
+      print("===")
+      print("=== All tests succeeded")
+      print("===")
     else:
-      print
-      print "==="
-      print "=== %i tests failed" % len(self._failed)
+      print()
+      print("===")
+      print("=== %i tests failed" % len(self._failed))
       if crashed > 0:
-        print "=== %i tests CRASHED" % crashed
-      print "==="
+        print("=== %i tests CRASHED" % crashed)
+      print("===")
 
 
 class VerboseProgressIndicator(SimpleProgressIndicator):
@@ -99,7 +102,7 @@ class VerboseProgressIndicator(SimpleProgressIndicator):
     self._last_printed_time = time.time()
 
   def _print(self, text):
-    print text
+    print(text)
     sys.stdout.flush()
     self._last_printed_time = time.time()
 
@@ -181,19 +184,19 @@ class CompactProgressIndicator(ProgressIndicator):
       self._clear_line(self._last_status_length)
       print_failure_header(test)
       if len(stdout):
-        print self._templates['stdout'] % stdout
+        print(self._templates['stdout'] % stdout)
       if len(stderr):
-        print self._templates['stderr'] % stderr
-      print "Command: %s" % result.cmd
+        print(self._templates['stderr'] % stderr)
+      print("Command: %s" % result.cmd)
       if output.HasCrashed():
-        print "exit code: %d" % output.exit_code
-        print "--- CRASHED ---"
+        print("exit code: %d" % output.exit_code)
+        print("--- CRASHED ---")
       if output.HasTimedOut():
-        print "--- TIMEOUT ---"
+        print("--- TIMEOUT ---")
 
   def finished(self):
     self._print_progress('Done')
-    print
+    print()
 
   def _print_progress(self, name):
     self._clear_line(self._last_status_length)
@@ -212,7 +215,7 @@ class CompactProgressIndicator(ProgressIndicator):
     }
     status = self._truncate(status, 78)
     self._last_status_length = len(status)
-    print status,
+    print(status, end=' ')
     sys.stdout.flush()
 
   def _truncate(self, string, length):
@@ -238,7 +241,7 @@ class ColorProgressIndicator(CompactProgressIndicator):
     super(ColorProgressIndicator, self).__init__(templates)
 
   def _clear_line(self, last_length):
-    print "\033[1K\r",
+    print("\033[1K\r", end=' ')
 
 
 class MonochromeProgressIndicator(CompactProgressIndicator):
@@ -252,7 +255,7 @@ class MonochromeProgressIndicator(CompactProgressIndicator):
     super(MonochromeProgressIndicator, self).__init__(templates)
 
   def _clear_line(self, last_length):
-    print ("\r" + (" " * last_length) + "\r"),
+    print(("\r" + (" " * last_length) + "\r"), end=' ')
 
 
 class JsonTestProgressIndicator(ProgressIndicator):
@@ -321,7 +324,7 @@ class JsonTestProgressIndicator(ProgressIndicator):
           float(len(self.tests)))
 
     # Sort tests by duration.
-    self.tests.sort(key=lambda (_, duration, cmd): duration, reverse=True)
+    self.tests.sort(key=lambda __duration_cmd: __duration_cmd[1], reverse=True)
     slowest_tests = [
       {
         "name": str(test),

tools/testrunner/testproc/seed.py

@@ -35,7 +35,7 @@ class SeedProc(base.TestProcProducer):
 
   def _next_test(self, test):
     is_loaded = False
-    for _ in xrange(0, self._parallel_subtests):
+    for _ in range(0, self._parallel_subtests):
       is_loaded |= self._try_send_next_test(test)
 
     return is_loaded

tools/testrunner/testproc/sigproc.py

@@ -2,6 +2,9 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+# for py2/py3 compatibility
+from __future__ import print_function
+
 import signal
 
 from . import base
@@ -21,11 +24,11 @@ class SignalProc(base.TestProcObserver):
     signal.signal(signal.SIGTERM, self._on_sigterm)
 
   def _on_ctrlc(self, _signum, _stack_frame):
-    print '>>> Ctrl-C detected, early abort...'
+    print('>>> Ctrl-C detected, early abort...')
     self.exit_code = utils.EXIT_CODE_INTERRUPTED
     self.stop()
 
   def _on_sigterm(self, _signum, _stack_frame):
-    print '>>> SIGTERM received, early abort...'
+    print('>>> SIGTERM received, early abort...')
     self.exit_code = utils.EXIT_CODE_TERMINATED
     self.stop()

tools/testrunner/utils/dump_build_config_gyp.py

@@ -11,6 +11,9 @@ Raw gyp values are supported - they will be tranformed into valid json.
 """
 # TODO(machenbach): Remove this when gyp is deprecated.
 
+# for py2/py3 compatibility
+from __future__ import print_function
+
 import json
 import os
 import sys
@@ -47,7 +50,7 @@ def as_json(kv):
   try:
     return k, json.loads(v2)
   except ValueError as e:
-    print(k, v, v2)
+    print((k, v, v2))
     raise e
 
 with open(sys.argv[1], 'w') as f: