[test] Extract expected outcomes. Simpler FAIL_* handling.

Bug: v8:6917
Cq-Include-Trybots: master.tryserver.v8:v8_linux_noi18n_rel_ng
Change-Id: I864de452bacb9e34fa1bc70722bf4c2fa3de4204
Reviewed-on: https://chromium-review.googlesource.com/782723Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Sergiy Byelozyorov <sergiyb@chromium.org>
Commit-Queue: Michał Majewski <majeski@google.com>
Cr-Commit-Position: refs/heads/master@{#49614}

test/benchmarks/testcfg.py

@@ -39,8 +39,8 @@ class BenchmarksVariantGenerator(testsuite.VariantGenerator):
   # always opt to match the way the benchmarks are run for performance
   # testing.
   def FilterVariantsByTest(self, testcase):
-    outcomes = self.suite.GetOutcomesForTestCase(testcase)
-    if outcomes and statusfile.OnlyStandardVariant(outcomes):
+    outcomes = self.suite.GetStatusFileOutcomes(testcase)
+    if statusfile.OnlyStandardVariant(outcomes):
       return self.standard_variant
     return self.fast_variants
 

test/test262/testcfg.py

@@ -102,7 +102,7 @@ FAST_VARIANTS = {
 
 class Test262VariantGenerator(testsuite.VariantGenerator):
   def GetFlagSets(self, testcase, variant):
-    outcomes = testcase.suite.GetOutcomesForTestCase(testcase)
+    outcomes = testcase.suite.GetStatusFileOutcomes(testcase)
     if outcomes and statusfile.OnlyFastVariants(outcomes):
       variant_flags = FAST_VARIANTS
     else:
@@ -242,16 +242,12 @@ class Test262TestSuite(testsuite.TestSuite):
         return True
     return "FAILED!" in output.stdout
 
-  def HasUnexpectedOutput(self, testcase):
-    outcome = self.GetOutcome(testcase)
-    outcomes = self.GetOutcomesForTestCase(testcase)
+  def GetExpectedOutcomes(self, testcase):
+    outcomes = self.GetStatusFileOutcomes(testcase)
     if (statusfile.FAIL_SLOPPY in outcomes and
-        "--use-strict" not in testcase.flags):
-      return outcome != statusfile.FAIL
-    return not outcome in ([outcome for outcome in outcomes
-                                    if not outcome.startswith('--')
-                                       and outcome != statusfile.FAIL_SLOPPY]
-                           or [statusfile.PASS])
+        '--use-strict' not in testcase.flags):
+      return [statusfile.FAIL]
+    return super(Test262TestSuite, self).GetExpectedOutcomes(testcase)
 
   def PrepareSources(self):
     # The archive is created only on swarming. Local checkouts have the

tools/testrunner/local/execution.py

@@ -209,7 +209,7 @@ class Runner(object):
     self.suite_names = [s.name for s in suites]
 
     # Always pre-sort by status file, slowest tests first.
-    slow_key = lambda t: statusfile.IsSlow(t.suite.GetOutcomesForTestCase(t))
+    slow_key = lambda t: statusfile.IsSlow(t.suite.GetStatusFileOutcomes(t))
     self.tests.sort(key=slow_key, reverse=True)
 
     # Sort by stored duration of not opted out.

tools/testrunner/local/progress.py

@@ -337,7 +337,7 @@ class JsonTestProgressIndicator(ProgressIndicator):
         "command": self._EscapeCommand(test).replace(ABS_PATH_PREFIX, ""),
         "duration": test.duration,
         "marked_slow": statusfile.IsSlow(
-          test.suite.GetOutcomesForTestCase(test)),
+          test.suite.GetStatusFileOutcomes(test)),
       } for test in timed_tests[:20]
     ]
 
@@ -370,7 +370,7 @@ class JsonTestProgressIndicator(ProgressIndicator):
       "stderr": test.output.stderr,
       "exit_code": test.output.exit_code,
       "result": test.suite.GetOutcome(test),
-      "expected": list(test.suite.GetOutcomesForTestCase(test) or ["PASS"]),
+      "expected": test.suite.GetExpectedOutcomes(test),
       "duration": test.duration,
 
       # TODO(machenbach): This stores only the global random seed from the
@@ -415,11 +415,7 @@ class FlakinessTestProgressIndicator(ProgressIndicator):
     assert outcome in ["PASS", "FAIL", "CRASH", "TIMEOUT"]
     if test.run == 1:
       # First run of this test.
-      expected_outcomes = ([
-        expected
-        for expected in (test.suite.GetOutcomesForTestCase(test) or ["PASS"])
-        if expected in ["PASS", "FAIL", "CRASH", "TIMEOUT"]
-      ] or ["PASS"])
+      expected_outcomes = test.suite.GetExpectedOutcomes(test)
       self.results[key] = {
         "actual": outcome,
         "expected": " ".join(expected_outcomes),

tools/testrunner/local/statusfile.py

@@ -31,29 +31,29 @@ import re
 from variants import ALL_VARIANTS
 from utils import Freeze
 
-# These outcomes can occur in a TestCase's outcomes list:
-SKIP = "SKIP"
+# Possible outcomes
 FAIL = "FAIL"
 PASS = "PASS"
-OKAY = "OKAY" # TODO(majeski): unused in status files
 TIMEOUT = "TIMEOUT" # TODO(majeski): unused in status files
 CRASH = "CRASH" # TODO(majeski): unused in status files
+
+# Outcomes only for status file, need special handling
+FAIL_OK = "FAIL_OK"
+FAIL_SLOPPY = "FAIL_SLOPPY"
+
+# Modifiers
+SKIP = "SKIP"
 SLOW = "SLOW"
 FAST_VARIANTS = "FAST_VARIANTS"
 NO_VARIANTS = "NO_VARIANTS"
-# These are just for the status files and are mapped below in DEFS:
-FAIL_OK = "FAIL_OK"
-FAIL_SLOPPY = "FAIL_SLOPPY"
 
 ALWAYS = "ALWAYS"
 
 KEYWORDS = {}
-for key in [SKIP, FAIL, PASS, OKAY, CRASH, SLOW, FAIL_OK,
-            FAST_VARIANTS, NO_VARIANTS, FAIL_SLOPPY, ALWAYS]:
+for key in [SKIP, FAIL, PASS, CRASH, SLOW, FAIL_OK, FAST_VARIANTS, NO_VARIANTS,
+            FAIL_SLOPPY, ALWAYS]:
   KEYWORDS[key] = key
 
-DEFS = {FAIL_OK: [FAIL, OKAY]}
-
 # Support arches, modes to be written as keywords instead of strings.
 VARIABLES = {ALWAYS: True}
 for var in ["debug", "release", "big", "little",
@@ -85,24 +85,13 @@ def OnlyFastVariants(outcomes):
 
 
 def IsPassOrFail(outcomes):
-  return ((PASS in outcomes) and (FAIL in outcomes) and
-          (not CRASH in outcomes) and (not OKAY in outcomes))
+  return (PASS in outcomes and
+          FAIL in outcomes and
+          CRASH not in outcomes)
 
 
 def IsFailOk(outcomes):
-    return (FAIL in outcomes) and (OKAY in outcomes)
-
-
-def _AddOutcome(result, new):
-  if new in DEFS:
-    mapped = DEFS[new]
-    if type(mapped) == list:
-      for m in mapped:
-        _AddOutcome(result, m)
-    elif type(mapped) == str:
-      _AddOutcome(result, mapped)
-  else:
-    result.add(new)
+  return FAIL_OK in outcomes
 
 
 def _JoinsPassAndFail(outcomes1, outcomes2):
@@ -111,8 +100,8 @@ def _JoinsPassAndFail(outcomes1, outcomes2):
   """
   return (
       PASS in outcomes1 and
-      not FAIL in outcomes1 and
-      FAIL in outcomes2
+      not (FAIL in outcomes1 or FAIL_OK in outcomes1) and
+      (FAIL in outcomes2 or FAIL_OK in outcomes2)
   )
 
 VARIANT_EXPRESSION = object()
@@ -155,7 +144,7 @@ def _ParseOutcomeList(rule, outcomes, variables, target_dict):
     outcomes = [outcomes]
   for item in outcomes:
     if type(item) == str:
-      _AddOutcome(result, item)
+      result.add(item)
     elif type(item) == list:
       condition = item[0]
       exp = _EvalExpression(condition, variables)
@@ -170,7 +159,7 @@ def _ParseOutcomeList(rule, outcomes, variables, target_dict):
 
       for outcome in item[1:]:
         assert type(outcome) == str
-        _AddOutcome(result, outcome)
+        result.add(outcome)
     else:
       assert False
   if len(result) == 0:

tools/testrunner/local/testsuite.py

@@ -50,7 +50,7 @@ class VariantGenerator(object):
 
   def FilterVariantsByTest(self, testcase):
     result = self.all_variants
-    outcomes = testcase.suite.GetOutcomesForTestCase(testcase)
+    outcomes = testcase.suite.GetStatusFileOutcomes(testcase)
     if outcomes:
       if statusfile.OnlyStandardVariant(outcomes):
         return self.standard_variant
@@ -59,7 +59,7 @@ class VariantGenerator(object):
     return result
 
   def GetFlagSets(self, testcase, variant):
-    outcomes = testcase.suite.GetOutcomesForTestCase(testcase)
+    outcomes = testcase.suite.GetStatusFileOutcomes(testcase)
     if outcomes and statusfile.OnlyFastVariants(outcomes):
       return FAST_VARIANT_FLAGS[variant]
     else:
@@ -147,7 +147,7 @@ class TestSuite(object):
     before using this function.
     """
     flags = []
-    for outcome in self.GetOutcomesForTestCase(test):
+    for outcome in self.GetStatusFileOutcomes(test):
       if outcome.startswith('--'):
         flags.append(outcome)
     return flags
@@ -179,7 +179,7 @@ class TestSuite(object):
         (mode == 'skip' and pass_fail))
 
     def _compliant(test):
-      outcomes = self.GetOutcomesForTestCase(test)
+      outcomes = self.GetStatusFileOutcomes(test)
       if statusfile.DoSkip(outcomes):
         return False
       if _skip_slow(statusfile.IsSlow(outcomes), slow_tests_mode):
@@ -256,7 +256,29 @@ class TestSuite(object):
           break
     self.tests = filtered
 
-  def GetOutcomesForTestCase(self, testcase):
+  def GetExpectedOutcomes(self, testcase):
+    """Gets expected outcomes from status file.
+
+    It differs from GetStatusFileOutcomes by selecting only outcomes that can
+    be result of test execution.
+    Status file has to be loaded before using this function.
+    """
+    outcomes = self.GetStatusFileOutcomes(testcase)
+
+    expected = []
+    if (statusfile.FAIL in outcomes or
+        statusfile.FAIL_OK in outcomes):
+      expected.append(statusfile.FAIL)
+
+    if statusfile.CRASH in outcomes:
+      expected.append(statusfile.CRASH)
+
+    if statusfile.PASS in outcomes:
+      expected.append(statusfile.PASS)
+
+    return expected or [statusfile.PASS]
+
+  def GetStatusFileOutcomes(self, testcase):
     """Gets outcomes from status file.
 
     Merges variant dependent and independent rules. Status file has to be loaded
@@ -323,9 +345,7 @@ class TestSuite(object):
       return statusfile.PASS
 
   def HasUnexpectedOutput(self, testcase):
-    outcome = self.GetOutcome(testcase)
-    return not outcome in (self.GetOutcomesForTestCase(testcase)
-                           or [statusfile.PASS])
+    return self.GetOutcome(testcase) not in self.GetExpectedOutcomes(testcase)
 
   def StripOutputForTransmit(self, testcase):
     if not self.HasUnexpectedOutput(testcase):

tools/testrunner/local/testsuite_unittest.py

@@ -39,7 +39,7 @@ class TestSuiteTest(unittest.TestCase):
         [TestCase(suite, 'baz/bar')],
         suite.tests,
     )
-    outcomes = suite.GetOutcomesForTestCase(suite.tests[0])
+    outcomes = suite.GetStatusFileOutcomes(suite.tests[0])
     self.assertEquals(set(['PASS', 'FAIL', 'SLOW']), outcomes)
 
   def test_filter_testcases_by_status_second_pass(self):
@@ -89,13 +89,31 @@ class TestSuiteTest(unittest.TestCase):
 
     self.assertEquals(
         set(['PREV', 'PASS', 'SLOW']),
-        suite.GetOutcomesForTestCase(suite.tests[0]),
+        suite.GetStatusFileOutcomes(suite.tests[0]),
     )
     self.assertEquals(
         set(['PREV', 'PASS', 'FAIL', 'SLOW']),
-        suite.GetOutcomesForTestCase(suite.tests[1]),
+        suite.GetStatusFileOutcomes(suite.tests[1]),
     )
 
+  def test_fail_ok_outcome(self):
+    suite = TestSuite('foo', 'bar')
+    suite.tests = [
+      TestCase(suite, 'foo/bar'),
+      TestCase(suite, 'baz/bar'),
+    ]
+    suite.rules = {
+      '': {
+        'foo/bar': set(['FAIL_OK']),
+        'baz/bar': set(['FAIL']),
+      },
+    }
+    suite.prefix_rules = {}
+
+    for t in suite.tests:
+      expected_outcomes = suite.GetExpectedOutcomes(t)
+      self.assertEquals(['FAIL'], expected_outcomes)
+
 
 if __name__ == '__main__':
     unittest.main()

tools/testrunner/local/verbose.py

@@ -45,7 +45,7 @@ def PrintReport(tests):
   total = len(tests)
   skipped = nocrash = passes = fail_ok = fail = 0
   for t in tests:
-    outcomes = t.suite.GetOutcomesForTestCase(t)
+    outcomes = t.suite.GetStatusFileOutcomes(t)
     if not outcomes:
       passes += 1
       continue