[foozzie] Add option to skip suppressions

This will allow uploading repro test cases to clusterfuzz for
already suppressed known issues. This will allow tracking if those
issues still reproduce and that suppressions don't become stale.

No-Try: true
Bug: chromium:1044942
Change-Id: I997f11293c51836b97d143b0fea992055b39955e
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2036083
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Liviu Rau <liviurau@chromium.org>
Reviewed-by: Tamer Tas <tmrts@chromium.org>
Cr-Commit-Position: refs/heads/master@{#66114}

tools/clusterfuzz/v8_commands.py

@@ -28,10 +28,10 @@ DEFAULT_FLAGS = [
 BASE_PATH = os.path.dirname(os.path.abspath(__file__))
 
 # List of files passed to each d8 run before the testcase.
-DEFAULT_FILES = [
-  os.path.join(BASE_PATH, 'v8_mock.js'),
-  os.path.join(BASE_PATH, 'v8_suppressions.js'),
-]
+DEFAULT_MOCK = os.path.join(BASE_PATH, 'v8_mock.js')
+
+# Suppressions on JavaScript level for known issues.
+JS_SUPPRESSIONS = os.path.join(BASE_PATH, 'v8_suppressions.js')
 
 # Config-specific mock files.
 ARCH_MOCKS = os.path.join(BASE_PATH, 'v8_mock_archs.js')
@@ -43,7 +43,9 @@ TIMEOUT = 3
 
 def _startup_files(options):
   """Default files and optional config-specific mock files."""
-  files = DEFAULT_FILES[:]
+  files = [DEFAULT_MOCK]
+  if not options.skip_suppressions:
+    files.append(JS_SUPPRESSIONS)
   if options.first.arch != options.second.arch:
     files.append(ARCH_MOCKS)
   # Mock out WebAssembly when comparing with jitless mode.

tools/clusterfuzz/v8_foozzie.py

@@ -212,6 +212,9 @@ def parse_args():
   parser.add_argument(
       '--skip-sanity-checks', default=False, action='store_true',
       help='skip sanity checks for testing purposes')
+  parser.add_argument(
+      '--skip-suppressions', default=False, action='store_true',
+      help='skip suppressions to reproduce known issues')
 
   # Add arguments for each run configuration.
   first_config_arguments.add_arguments(parser, 'ignition')
@@ -309,6 +312,7 @@ def main():
   suppress = v8_suppressions.get_suppression(
       options.first.arch, options.first.config,
       options.second.arch, options.second.config,
+      options.skip_suppressions,
   )
 
   # Static bailout based on test case content or metadata.

tools/clusterfuzz/v8_foozzie_test.py

@@ -52,10 +52,9 @@ class ConfigTest(unittest.TestCase):
 
 class UnitTest(unittest.TestCase):
   def testDiff(self):
-    # TODO(machenbach): Mock out suppression configuration.
-    suppress = v8_suppressions.get_suppression(
-        'x64', 'ignition', 'x64', 'ignition_turbo')
-    def diff_fun(one, two):
+    def diff_fun(one, two, skip=False):
+      suppress = v8_suppressions.get_suppression(
+          'x64', 'ignition', 'x64', 'ignition_turbo', skip)
       return suppress.diff_lines(one.splitlines(), two.splitlines())
 
     one = ''
@@ -118,6 +117,20 @@ otherfile.js: TypeError: undefined is not a constructor
 + otherfile.js: TypeError: undefined is not a constructor""", None
     self.assertEquals(diff, diff_fun(one, two))
 
+    # Test that skipping suppressions works.
+    one = """
+v8-foozzie source: foo
+23:TypeError: bar is not a function
+"""
+    two = """
+v8-foozzie source: foo
+42:TypeError: baz is not a function
+"""
+    self.assertEquals((None, 'foo'), diff_fun(one, two))
+    diff = """- 23:TypeError: bar is not a function
++ 42:TypeError: baz is not a function""", 'foo'
+    self.assertEquals(diff, diff_fun(one, two, skip=True))
+
   def testOutputCapping(self):
     def output(stdout, is_crash):
       exit_code = -1 if is_crash else 0
@@ -186,6 +199,8 @@ class SystemTest(unittest.TestCase):
   def testSyntaxErrorDiffPass(self):
     stdout = run_foozzie('build1', '--skip-sanity-checks')
     self.assertEquals('# V8 correctness - pass\n', cut_verbose_output(stdout))
+    # Default comparison includes suppressions.
+    self.assertIn('v8_suppressions.js', stdout)
     # Default comparison doesn't include any specific mock files.
     self.assertNotIn('v8_mock_archs.js', stdout)
     self.assertNotIn('v8_mock_webassembly.js', stdout)
@@ -233,6 +248,24 @@ class SystemTest(unittest.TestCase):
     self.assertIn('v8_mock_webassembly.js', lines[1])
     self.assertIn('v8_mock_webassembly.js', lines[3])
 
+  def testSkipSuppressions(self):
+    """Test that the suppressions file is not passed when skipping
+    suppressions.
+    """
+    # Compare baseline with baseline. This passes as there is no difference.
+    stdout = run_foozzie(
+        'baseline', '--skip-sanity-checks', '--skip-suppressions')
+    self.assertNotIn('v8_suppressions.js', stdout)
+
+    # Compare with a build that usually suppresses a difference. Now we fail
+    # since we skip suppressions.
+    with self.assertRaises(subprocess.CalledProcessError) as ctx:
+      run_foozzie(
+          'build1', '--skip-sanity-checks', '--skip-suppressions')
+    e = ctx.exception
+    self.assertEquals(v8_foozzie.RETURN_FAIL, e.returncode)
+    self.assertNotIn('v8_suppressions.js', e.output)
+
 
 if __name__ == '__main__':
   unittest.main()

tools/clusterfuzz/v8_suppressions.py

@@ -259,8 +259,8 @@ def diff_output(output1, output2, allowed, ignore1, ignore2):
   return None, source
 
 
-def get_suppression(arch1, config1, arch2, config2):
-  return V8Suppression(arch1, config1, arch2, config2)
+def get_suppression(arch1, config1, arch2, config2, skip=False):
+  return V8Suppression(arch1, config1, arch2, config2, skip)
 
 
 class Suppression(object):
@@ -281,11 +281,19 @@ class Suppression(object):
 
 
 class V8Suppression(Suppression):
-  def __init__(self, arch1, config1, arch2, config2):
+  def __init__(self, arch1, config1, arch2, config2, skip):
     self.arch1 = arch1
     self.config1 = config1
     self.arch2 = arch2
     self.config2 = config2
+    if skip:
+      self.allowed_line_diffs = []
+      self.ignore_output = {}
+      self.ignore_sources = {}
+    else:
+      self.allowed_line_diffs = ALLOWED_LINE_DIFFS
+      self.ignore_output = IGNORE_OUTPUT
+      self.ignore_sources = IGNORE_SOURCES
 
   def diff(self, output1, output2):
     # Diff capped lines in the presence of crashes.
@@ -295,7 +303,7 @@ class V8Suppression(Suppression):
     return diff_output(
         output1_lines,
         output2_lines,
-        ALLOWED_LINE_DIFFS,
+        self.allowed_line_diffs,
         IGNORE_LINES,
         IGNORE_LINES,
     )
@@ -318,7 +326,7 @@ class V8Suppression(Suppression):
     return None
 
   def ignore_by_metadata(self, metadata):
-    for bug, sources in IGNORE_SOURCES.iteritems():
+    for bug, sources in self.ignore_sources.iteritems():
       for source in sources:
         if source in metadata['sources']:
           return bug
@@ -337,7 +345,7 @@ class V8Suppression(Suppression):
           return bug
       return None
     for key in ['', arch, config]:
-      bug = check(IGNORE_OUTPUT.get(key, {}))
+      bug = check(self.ignore_output.get(key, {}))
       if bug:
         return bug
     return None