[tools] Generate additional variant for FAIL_PHASE_ONLY tests

The additional variant does not use wrapper disabling phase tests and negated
outcome processor. This allows to ensure that tests marked FAIL_PHASE_ONLY, do
actually fail without it.

R=machenbach@chromium.org

Bug: v8:8467
Change-Id: I66e07bd7107520872cc013bf0f33fdc6664baf56
Reviewed-on: https://chromium-review.googlesource.com/c/1361164
Commit-Queue: Sergiy Belozorov <sergiyb@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#58211}

test/test262/testcfg.py

@@ -87,15 +87,23 @@ class VariantsGenerator(testsuite.VariantsGenerator):
   def gen(self, test):
     flags_set = self._get_flags_set(test)
     test_record = test.test_record
-    for n, variant in enumerate(self._get_variants(test)):
-      flags = flags_set[variant][0]
-      if 'noStrict' in test_record:
-        yield (variant, flags, str(n))
-      elif 'onlyStrict' in test_record:
-        yield (variant, flags + ['--use-strict'], 'strict-%d' % n)
-      else:
-        yield (variant, flags, str(n))
-        yield (variant, flags + ['--use-strict'], 'strict-%d' % n)
+
+    # Add a reverse test ensuring that FAIL_PHASE_ONLY is only used for tests
+    # that actually fail to throw an exception at wrong phase.
+    phase_variants = ['']
+    if test.fail_phase_only:
+      phase_variants.append('-fail-phase-reverse')
+
+    for phase_var in phase_variants:
+      for n, variant in enumerate(self._get_variants(test)):
+        flags = flags_set[variant][0]
+        if 'noStrict' in test_record:
+          yield (variant, flags, str(n) + phase_var)
+        elif 'onlyStrict' in test_record:
+          yield (variant, flags + ['--use-strict'], 'strict-%d' % n + phase_var)
+        else:
+          yield (variant, flags, str(n))
+          yield (variant, flags + ['--use-strict'], 'strict-%d' % n + phase_var)
 
 
 class TestSuite(testsuite.TestSuite):
@@ -169,13 +177,34 @@ class TestCase(testcase.D8TestCase):
           .get('type', None)
     )
 
+    # We disallow combining FAIL_PHASE_ONLY with any other fail outcome types.
+    # Outcome parsing logic in the base class converts all outcomes specified in
+    # the status file into either FAIL, CRASH or PASS, thus we do not need to
+    # handle FAIL_OK, FAIL_SLOPPY and various other outcomes.
+    if self.fail_phase_only:
+      assert (
+          statusfile.FAIL not in self.expected_outcomes and
+          statusfile.CRASH not in self.expected_outcomes), self.name
+
+  @property
+  def fail_phase_only(self):
+    # The FAIL_PHASE_ONLY is defined in tools/testrunner/local/statusfile.py and
+    # can be used in status files to mark tests that throw an exception at wrong
+    # phase, e.g. SyntaxError is thrown at execution phase instead of parsing
+    # phase. See https://crbug.com/v8/8467 for more details.
+    return statusfile.FAIL_PHASE_ONLY in self._statusfile_outcomes
+
+  @property
+  def _fail_phase_reverse(self):
+    return 'fail-phase-reverse' in self.procid
+
   def _get_files_params(self):
     return (
         list(self.suite.harness) +
         ([os.path.join(self.suite.root, "harness-agent.js")]
          if self.path.startswith('built-ins/Atomics') else []) +
         ([os.path.join(self.suite.root, "harness-adapt-donotevaluate.js")]
-         if self.fail_phase_only else []) +
+         if self.fail_phase_only and not self._fail_phase_reverse else []) +
         self._get_includes() +
         (["--module"] if "module" in self.test_record else []) +
         [self._get_source_path()]
@@ -213,7 +242,12 @@ class TestCase(testcase.D8TestCase):
   def output_proc(self):
     if self._expected_exception is not None:
       return test262.ExceptionOutProc(self.expected_outcomes,
-                                      self._expected_exception)
+                                      self._expected_exception,
+                                      self._fail_phase_reverse)
+    else:
+      # We only support fail phase reverse on tests that expect an exception.
+      assert not self._fail_phase_reverse
+
     if self.expected_outcomes == outproc.OUTCOMES_PASS:
       return test262.PASS_NO_EXCEPTION
     return test262.NoExceptionOutProc(self.expected_outcomes)

tools/testrunner/objects/testcase.py

@@ -112,10 +112,6 @@ class TestCase(object):
       self._parse_status_file_outcomes(self._statusfile_outcomes))
 
   def _parse_status_file_outcomes(self, outcomes):
-    # This flag does not affect the test execution or outcome parsing by
-    # default, but subclasses can implement custom logic when it is set.
-    self.fail_phase_only = statusfile.FAIL_PHASE_ONLY in outcomes
-
     if (statusfile.FAIL_SLOPPY in outcomes and
         '--use-strict' not in self.variant_flags):
       return outproc.OUTCOMES_FAIL

tools/testrunner/outproc/test262.py

@@ -7,18 +7,29 @@ import re
 from . import base
 
 
+def _is_failure_output(output):
+  return (
+    output.exit_code != 0 or
+    'FAILED!' in output.stdout
+  )
+
+
 class ExceptionOutProc(base.OutProc):
   """Output processor for tests with expected exception."""
-  def __init__(self, expected_outcomes, expected_exception=None):
+  def __init__(
+      self, expected_outcomes, expected_exception=None, negative=False):
     super(ExceptionOutProc, self).__init__(expected_outcomes)
     self._expected_exception = expected_exception
+    self._negative = negative
+
+  @property
+  def negative(self):
+    return self._negative
 
   def _is_failure_output(self, output):
-    if output.exit_code != 0:
-      return True
     if self._expected_exception != self._parse_exception(output.stdout):
       return True
-    return 'FAILED!' in output.stdout
+    return _is_failure_output(output)
 
   def _parse_exception(self, string):
     # somefile:somelinenumber: someerror[: sometext]
@@ -31,16 +42,13 @@ class ExceptionOutProc(base.OutProc):
       return None
 
 
-def _is_failure_output(self, output):
-  return (
-    output.exit_code != 0 or
-    'FAILED!' in output.stdout
-  )
-
-
 class NoExceptionOutProc(base.OutProc):
   """Output processor optimized for tests without expected exception."""
-NoExceptionOutProc._is_failure_output = _is_failure_output
+  def __init__(self, expected_outcomes):
+    super(NoExceptionOutProc, self).__init__(expected_outcomes)
+
+  def _is_failure_output(self, output):
+    return _is_failure_output(output)
 
 
 class PassNoExceptionOutProc(base.PassOutProc):
@@ -48,7 +56,8 @@ class PassNoExceptionOutProc(base.PassOutProc):
   Output processor optimized for tests expected to PASS without expected
   exception.
   """
-PassNoExceptionOutProc._is_failure_output = _is_failure_output
+  def _is_failure_output(self, output):
+    return _is_failure_output(output)
 
 
 PASS_NO_EXCEPTION = PassNoExceptionOutProc()