[foozzie] Align running main comparisons and smoke tests

This shares the code of running the comparisons on the fuzz test
and on the smoke tests. While this change retains behavior for running
the fuzz test, the smoke test has slight changes in favor of code
readability:

1. Smoke tests can now bail out on output suppressions (There are
none now. And if we'd ever add any we'd need to write them in a way,
such that they ignore the output of the smoke tests, which don't
have much output anyways).
2. Crashes in smoke tests are now a hard failure. This is a desired
feature anyways. It's unlikely that the smoke test crashes as then
nothing would work.

No-Try: true
Bug: chromium:1100114
Change-Id: Ice3c6d18b549727c486a70756a72038c8b2029ac
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2273125
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Liviu Rau <liviurau@chromium.org>
Cr-Commit-Position: refs/heads/master@{#68586}

tools/clusterfuzz/testdata/sanity_check_output.txt

@@ -17,6 +17,9 @@
 - unknown
 + not unknown
 #
+# Source file:
+name/to/file.js
+#
 ### Start of configuration x64,ignition:
 
 1

tools/clusterfuzz/v8_foozzie.py

@@ -367,24 +367,53 @@ def cluster_failures(source, known_failures=None):
   return long_key[:ORIGINAL_SOURCE_HASH_LENGTH]
 
 
-def run_sanity_checks(suppress, first_config, second_config):
-  """Run some fixed smoke tests in all configs to ensure nothing
-  is fundamentally wrong, in order to prevent bug flooding.
+def run_comparisons(suppress, first_config, second_config, test_case, timeout,
+                    verbose=True, ignore_crashes=True, source_key=None):
+  """Runs two configurations and bails out on output difference.
+
+  Args:
+    suppress: The helper object for textual suppressions.
+    first_config: The baseline configuration to run and compare.
+    second_config: The secondary configuration to run and compare.
+    test_case: The test case to run.
+    timeout: Timeout in seconds for one run.
+    verbose: Prints the executed commands.
+    ignore_crashes: Typically we ignore crashes during fuzzing as they are
+        frequent. However, when running sanity checks we should not crash
+        and immediately flag crashes as a failure.
+    source_key: A fixed source key. If not given, it will be inferred from the
+        output.
   """
   first_config_output = first_config.command.run(
-      SANITY_CHECKS, timeout=SANITY_CHECK_TIMEOUT_SEC)
+      test_case, timeout=timeout, verbose=verbose)
 
   second_config_output = second_config.command.run(
-      SANITY_CHECKS, timeout=SANITY_CHECK_TIMEOUT_SEC)
+      test_case, timeout=timeout, verbose=verbose)
+
+  difference, source = suppress.diff(first_config_output, second_config_output)
 
-  difference, _ = suppress.diff(first_config_output, second_config_output)
   if difference:
-    # Special source key for sanity checks so that clusterfuzz dedupes all
-    # cases on this in case it's hit.
-    source_key = 'sanity check failed'
+    # Only bail out due to suppressed output if there was a difference. If a
+    # suppression doesn't show up anymore in the statistics, we might want to
+    # remove it.
+    fail_bailout(first_config_output, suppress.ignore_by_output)
+    fail_bailout(second_config_output, suppress.ignore_by_output)
+
+    source_key = source_key or cluster_failures(source)
     raise FailException(format_difference(
         source_key, first_config, second_config,
-        first_config_output, second_config_output, difference))
+        first_config_output, second_config_output, difference, source))
+
+  if first_config_output.HasCrashed() or second_config_output.HasCrashed():
+    if ignore_crashes:
+      # Show if a crash has happened in one of the runs and no difference was
+      # detected. This is only for the statistics during experiments.
+      raise PassException('# V8 correctness - C-R-A-S-H')
+    else:
+      # Subsume unexpected crashes (e.g. during sanity checks) with one failure
+      # state.
+      raise FailException(FAILURE_HEADER_TEMPLATE % dict(
+          configs='', source_key='', suppression='unexpected crash'))
 
 
 def main():
@@ -403,41 +432,33 @@ def main():
   first_config = ExecutionConfig(options, 'first')
   second_config = ExecutionConfig(options, 'second')
 
-  # Sanity checks. Run both configurations with the sanity-checks file only and
-  # bail out early if different.
+  # First, run some fixed smoke tests in all configs to ensure nothing
+  # is fundamentally wrong, in order to prevent bug flooding.
   if not options.skip_sanity_checks:
-    run_sanity_checks(suppress, first_config, second_config)
-
-  first_config_output = first_config.command.run(
-      options.testcase, timeout=TEST_TIMEOUT_SEC, verbose=True)
-
-  second_config_output = second_config.command.run(
-      options.testcase, timeout=TEST_TIMEOUT_SEC, verbose=True)
-
-  difference, source = suppress.diff(first_config_output, second_config_output)
-
-  if difference:
-    # Only bail out due to suppressed output if there was a difference. If a
-    # suppression doesn't show up anymore in the statistics, we might want to
-    # remove it.
-    fail_bailout(first_config_output, suppress.ignore_by_output)
-    fail_bailout(second_config_output, suppress.ignore_by_output)
+    run_comparisons(
+        suppress, first_config, second_config,
+        test_case=SANITY_CHECKS,
+        timeout=SANITY_CHECK_TIMEOUT_SEC,
+        verbose=False,
+        # Don't accept crashes during sanity checks. A crash would hint at
+        # a flag that might be incompatible or a broken test file.
+        ignore_crashes=False,
+        # Special source key for sanity checks so that clusterfuzz dedupes all
+        # cases on this in case it's hit.
+        source_key = 'sanity check failed',
+    )
 
-    source_key = cluster_failures(source)
-    raise FailException(format_difference(
-        source_key, first_config, second_config,
-        first_config_output, second_config_output, difference, source))
+  # Second, run all configs against the fuzz test case.
+  run_comparisons(
+      suppress, first_config, second_config,
+      test_case=options.testcase,
+      timeout=TEST_TIMEOUT_SEC,
+  )
 
-  # Show if a crash has happened in one of the runs and no difference was
-  # detected.
-  if first_config_output.HasCrashed() or second_config_output.HasCrashed():
-    print('# V8 correctness - C-R-A-S-H')
-  else:
   # TODO(machenbach): Figure out if we could also return a bug in case
   # there's no difference, but one of the line suppressions has matched -
   # and without the match there would be a difference.
   print('# V8 correctness - pass')
-
   return RETURN_PASS