[foozzie] Fix more Python3 incompatibilities

NOTRY=true
TBR=tmrts@chromium.org

Bug: chromium:1065624
Change-Id: I6e49c48bb95e10b7fad1ff2c589a2dd459fff562
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2124326Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#66900}

tools/clusterfuzz/v8_commands.py

@@ -12,6 +12,8 @@ from threading import Event, Timer
 
 import v8_fuzz_config
 
+PYTHON3 = sys.version_info >= (3, 0)
+
 # List of default flags passed to each d8 run.
 DEFAULT_FLAGS = [
   '--correctness-fuzzer-suppressions',
@@ -107,12 +109,16 @@ class Output(object):
 
 def Execute(args, cwd, timeout=None):
   popen_args = [c for c in args if c != ""]
+  kwargs = {}
+  if PYTHON3:
+    kwargs['encoding'] = 'utf-8'
   try:
     process = subprocess.Popen(
       args=popen_args,
       stdout=subprocess.PIPE,
       stderr=subprocess.PIPE,
       cwd=cwd,
+      **kwargs
     )
   except Exception as e:
     sys.stderr.write("Error executing: %s\n" % popen_args)
@@ -135,6 +141,6 @@ def Execute(args, cwd, timeout=None):
   return Output(
       process.returncode,
       timeout_event.is_set(),
-      stdout.decode('utf-8', 'replace').encode('utf-8'),
+      stdout,
       process.pid,
   )

tools/clusterfuzz/v8_foozzie.py

@@ -25,6 +25,8 @@ from collections import namedtuple
 import v8_commands
 import v8_suppressions
 
+PYTHON3 = sys.version_info >= (3, 0)
+
 CONFIGS = dict(
   default=[],
   ignition=[
@@ -283,7 +285,15 @@ def print_difference(
   first_config_label = '%s,%s' % (options.first.arch, options.first.config)
   second_config_label = '%s,%s' % (options.second.arch, options.second.config)
   source_file_text = SOURCE_FILE_TEMPLATE % source if source else ''
-  print((FAILURE_TEMPLATE % dict(
+
+  if PYTHON3:
+    first_stdout = first_config_output.stdout
+    second_stdout = second_config_output.stdout
+  else:
+    first_stdout = first_config_output.stdout.decode('utf-8', 'replace')
+    second_stdout = second_config_output.stdout.decode('utf-8', 'replace')
+
+  text = (FAILURE_TEMPLATE % dict(
       configs='%s:%s' % (first_config_label, second_config_label),
       source_file_text=source_file_text,
       source_key=source_key,
@@ -292,13 +302,15 @@ def print_difference(
       second_config_label=second_config_label,
       first_config_flags=' '.join(first_command.flags),
       second_config_flags=' '.join(second_command.flags),
-      first_config_output=
-          first_config_output.stdout.decode('utf-8', 'replace'),
-      second_config_output=
-          second_config_output.stdout.decode('utf-8', 'replace'),
+      first_config_output=first_stdout,
+      second_config_output=second_stdout,
       source=source,
-      difference=difference.decode('utf-8', 'replace'),
-  )).encode('utf-8', 'replace'))
+      difference=difference,
+  ))
+  if PYTHON3:
+    print(text)
+  else:
+    print(text.encode('utf-8', 'replace'))
 
 
 def main():
@@ -312,7 +324,10 @@ def main():
   )
 
   # Static bailout based on test case content or metadata.
-  with open(options.testcase) as f:
+  kwargs = {}
+  if PYTHON3:
+    kwargs['encoding'] = 'utf-8'
+  with open(options.testcase, 'r', **kwargs) as f:
     content = f.read()
   if content_bailout(get_meta_data(content), suppress.ignore_by_metadata):
     return RETURN_FAIL
@@ -354,7 +369,8 @@ def main():
   difference, source = suppress.diff(first_config_output, second_config_output)
 
   if source:
-    source_key = hashlib.sha1(source).hexdigest()[:ORIGINAL_SOURCE_HASH_LENGTH]
+    long_key = hashlib.sha1(source.encode('utf-8')).hexdigest()
+    source_key = long_key[:ORIGINAL_SOURCE_HASH_LENGTH]
   else:
     source_key = ORIGINAL_SOURCE_DEFAULT
 

tools/clusterfuzz/v8_foozzie_test.py

@@ -19,6 +19,8 @@ try:
 except NameError:
   basestring = str
 
+PYTHON3 = sys.version_info >= (3, 0)
+
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 FOOZZIE = os.path.join(BASE_DIR, 'v8_foozzie.py')
 TEST_DATA = os.path.join(BASE_DIR, 'testdata')
@@ -108,12 +110,12 @@ class UnitTest(unittest.TestCase):
     one = ''
     two = ''
     diff = None, None
-    self.assertEquals(diff, diff_fun(one, two))
+    self.assertEqual(diff, diff_fun(one, two))
 
     one = 'a \n  b\nc();'
     two = 'a \n  b\nc();'
     diff = None, None
-    self.assertEquals(diff, diff_fun(one, two))
+    self.assertEqual(diff, diff_fun(one, two))
 
     # Ignore line before caret, caret position and error message.
     one = """
@@ -131,7 +133,7 @@ somefile.js: TypeError: baz is not a function
   undefined
 """
     diff = None, None
-    self.assertEquals(diff, diff_fun(one, two))
+    self.assertEqual(diff, diff_fun(one, two))
 
     one = """
 Still equal
@@ -141,7 +143,7 @@ Extra line
 Still equal
 """
     diff = '- Extra line', None
-    self.assertEquals(diff, diff_fun(one, two))
+    self.assertEqual(diff, diff_fun(one, two))
 
     one = """
 Still equal
@@ -151,7 +153,7 @@ Still equal
 Extra line
 """
     diff = '+ Extra line', None
-    self.assertEquals(diff, diff_fun(one, two))
+    self.assertEqual(diff, diff_fun(one, two))
 
     one = """
 undefined
@@ -163,7 +165,7 @@ otherfile.js: TypeError: undefined is not a constructor
 """
     diff = """- somefile.js: TypeError: undefined is not a constructor
 + otherfile.js: TypeError: undefined is not a constructor""", None
-    self.assertEquals(diff, diff_fun(one, two))
+    self.assertEqual(diff, diff_fun(one, two))
 
     # Test that skipping suppressions works.
     one = """
@@ -174,10 +176,10 @@ v8-foozzie source: foo
 v8-foozzie source: foo
 42:TypeError: baz is not a function
 """
-    self.assertEquals((None, 'foo'), diff_fun(one, two))
+    self.assertEqual((None, 'foo'), diff_fun(one, two))
     diff = """- 23:TypeError: bar is not a function
 + 42:TypeError: baz is not a function""", 'foo'
-    self.assertEquals(diff, diff_fun(one, two, skip=True))
+    self.assertEqual(diff, diff_fun(one, two, skip=True))
 
   def testOutputCapping(self):
     def output(stdout, is_crash):
@@ -189,7 +191,7 @@ v8-foozzie source: foo
               capped_lines2):
       output1 = output(stdout1, is_crash1)
       output2 = output(stdout2, is_crash2)
-      self.assertEquals(
+      self.assertEqual(
           (capped_lines1, capped_lines2),
           v8_suppressions.get_output_capped(output1, output2))
 
@@ -229,6 +231,9 @@ def run_foozzie(second_d8_dir, *extra_flags, **kwargs):
   second_config = 'ignition_turbo'
   if 'second_config' in kwargs:
     second_config = 'jitless'
+  kwargs = {}
+  if PYTHON3:
+    kwargs['text'] = True
   return subprocess.check_output([
     sys.executable, FOOZZIE,
     '--random-seed', '12345',
@@ -237,8 +242,7 @@ def run_foozzie(second_d8_dir, *extra_flags, **kwargs):
     '--first-config', 'ignition',
     '--second-config', second_config,
     os.path.join(TEST_DATA, 'fuzz-123.js'),
-  ] + list(extra_flags))
-
+  ] + list(extra_flags), **kwargs)
 
 class SystemTest(unittest.TestCase):
   """This tests the whole correctness-fuzzing harness with fake build
@@ -254,7 +258,7 @@ class SystemTest(unittest.TestCase):
   """
   def testSyntaxErrorDiffPass(self):
     stdout = run_foozzie('build1', '--skip-sanity-checks')
-    self.assertEquals('# V8 correctness - pass\n', cut_verbose_output(stdout))
+    self.assertEqual('# V8 correctness - pass\n', cut_verbose_output(stdout))
     # Default comparison includes suppressions.
     self.assertIn('v8_suppressions.js', stdout)
     # Default comparison doesn't include any specific mock files.
@@ -270,8 +274,8 @@ class SystemTest(unittest.TestCase):
                   '--first-config-extra-flags=--flag2=0',
                   '--second-config-extra-flags=--flag3')
     e = ctx.exception
-    self.assertEquals(v8_foozzie.RETURN_FAIL, e.returncode)
-    self.assertEquals(expected_output, cut_verbose_output(e.output))
+    self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
+    self.assertEqual(expected_output, cut_verbose_output(e.output))
 
   def testSanityCheck(self):
     with open(os.path.join(TEST_DATA, 'sanity_check_output.txt')) as f:
@@ -279,8 +283,8 @@ class SystemTest(unittest.TestCase):
     with self.assertRaises(subprocess.CalledProcessError) as ctx:
       run_foozzie('build2')
     e = ctx.exception
-    self.assertEquals(v8_foozzie.RETURN_FAIL, e.returncode)
-    self.assertEquals(expected_output, e.output)
+    self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
+    self.assertEqual(expected_output, e.output)
 
   def testDifferentArch(self):
     """Test that the architecture-specific mocks are passed to both runs when
@@ -319,7 +323,7 @@ class SystemTest(unittest.TestCase):
       run_foozzie(
           'build1', '--skip-sanity-checks', '--skip-suppressions')
     e = ctx.exception
-    self.assertEquals(v8_foozzie.RETURN_FAIL, e.returncode)
+    self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
     self.assertNotIn('v8_suppressions.js', e.output)
 
 

tools/clusterfuzz/v8_suppressions.py

@@ -27,6 +27,13 @@ to silence a particular class of problems.
 import itertools
 import re
 
+try:
+  # Python 3
+  from itertools import zip_longest
+except ImportError:
+  # Python 2
+  from itertools import izip_longest as zip_longest
+
 # Max line length for regular experessions checking for lines to ignore.
 MAX_LINE_LENGTH = 512
 
@@ -135,7 +142,7 @@ def get_output_capped(output1, output2):
 
 
 def line_pairs(lines):
-  return itertools.izip_longest(
+  return zip_longest(
       lines, itertools.islice(lines, 1, None), fillvalue=None)
 
 
@@ -183,14 +190,14 @@ def diff_output(output1, output2, allowed, ignore1, ignore2):
       return all(not e.match(line) for e in ignore)
     return fun
 
-  lines1 = filter(useful_line(ignore1), output1)
-  lines2 = filter(useful_line(ignore2), output2)
+  lines1 = list(filter(useful_line(ignore1), output1))
+  lines2 = list(filter(useful_line(ignore2), output2))
 
   # This keeps track where we are in the original source file of the fuzz
   # test case.
   source = None
 
-  for ((line1, lookahead1), (line2, lookahead2)) in itertools.izip_longest(
+  for ((line1, lookahead1), (line2, lookahead2)) in zip_longest(
       line_pairs(lines1), line_pairs(lines2), fillvalue=(None, None)):
 
     # Only one of the two iterators should run out.