[test] Store outcomes in the testcase

Bug: v8:6917
Cq-Include-Trybots: master.tryserver.v8:v8_linux_noi18n_rel_ng
Change-Id: Ia52d4bedbeff5b93915ef69a2dc78f6d92669061
Reviewed-on: https://chromium-review.googlesource.com/832467
Commit-Queue: Michał Majewski <majeski@google.com>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50200}

test/benchmarks/testcfg.py

@@ -39,8 +39,7 @@ class VariantGenerator(testsuite.VariantGenerator):
   # always opt to match the way the benchmarks are run for performance
   # testing.
   def FilterVariantsByTest(self, test):
-    outcomes = self.suite.GetStatusFileOutcomes(test.name, test.variant)
-    if statusfile.OnlyStandardVariant(outcomes):
+    if test.only_standard_variant:
       return self.standard_variant
     return self.fast_variants
 

test/test262/testcfg.py

@@ -104,8 +104,7 @@ FAST_VARIANTS = {
 
 class VariantGenerator(testsuite.VariantGenerator):
   def GetFlagSets(self, test, variant):
-    outcomes = test.suite.GetStatusFileOutcomes(test.name, test.variant)
-    if outcomes and statusfile.OnlyFastVariants(outcomes):
+    if test.only_fast_variants:
       variant_flags = FAST_VARIANTS
     else:
       variant_flags = ALL_VARIANTS

tools/testrunner/local/execution.py

@@ -128,9 +128,7 @@ class Runner(object):
     self.suite_names = [s.name for s in suites]
 
     # Always pre-sort by status file, slowest tests first.
-    slow_key = lambda t: statusfile.IsSlow(
-        t.suite.GetStatusFileOutcomes(t.name, t.variant))
-    self.tests.sort(key=slow_key, reverse=True)
+    self.tests.sort(key=lambda t: t.is_slow, reverse=True)
 
     # Sort by stored duration if not opted out.
     if not context.no_sorting:

tools/testrunner/local/progress.py

@@ -324,8 +324,7 @@ class JsonTestProgressIndicator(ProgressIndicator):
         "flags": test.cmd.args,
         "command": test.cmd.to_string(relative=True),
         "duration": duration,
-        "marked_slow": statusfile.IsSlow(
-            test.suite.GetStatusFileOutcomes(test.name, test.variant)),
+        "marked_slow": test.is_slow,
       } for (test, duration) in self.tests[:20]
     ]
 

tools/testrunner/local/statusfile.py

@@ -68,32 +68,6 @@ for var in ALL_VARIANTS:
   VARIABLES[var] = var
 
 
-def DoSkip(outcomes):
-  return SKIP in outcomes
-
-
-def IsSlow(outcomes):
-  return SLOW in outcomes
-
-
-def OnlyStandardVariant(outcomes):
-  return NO_VARIANTS in outcomes
-
-
-def OnlyFastVariants(outcomes):
-  return FAST_VARIANTS in outcomes
-
-
-def IsPassOrFail(outcomes):
-  return (PASS in outcomes and
-          FAIL in outcomes and
-          CRASH not in outcomes)
-
-
-def IsFailOk(outcomes):
-  return FAIL_OK in outcomes
-
-
 def _JoinsPassAndFail(outcomes1, outcomes2):
   """Indicates if we join PASS and FAIL from two different outcome sets and
   the first doesn't already contain both.

tools/testrunner/local/testsuite.py

@@ -49,18 +49,14 @@ class VariantGenerator(object):
     self.standard_variant = STANDARD_VARIANT & variants
 
   def FilterVariantsByTest(self, test):
-    result = self.all_variants
-    outcomes = test.suite.GetStatusFileOutcomes(test.name, test.variant)
-    if outcomes:
-      if statusfile.OnlyStandardVariant(outcomes):
-        return self.standard_variant
-      if statusfile.OnlyFastVariants(outcomes):
-        result = self.fast_variants
-    return result
+    if test.only_standard_variant:
+      return self.standard_variant
+    if test.only_fast_variants:
+      return self.fast_variants
+    return self.all_variants
 
   def GetFlagSets(self, test, variant):
-    outcomes = test.suite.GetStatusFileOutcomes(test.name, test.variant)
-    if outcomes and statusfile.OnlyFastVariants(outcomes):
+    if test.only_fast_variants:
       return FAST_VARIANT_FLAGS[variant]
     else:
       return ALL_VARIANT_FLAGS[variant]
@@ -87,8 +83,6 @@ class TestSuite(object):
     self.rules = None  # {variant: {test name: [rule]}}
     self.prefix_rules = None  # {variant: {test name prefix: [rule]}}
 
-    self._outcomes_cache = dict()
-
   def status_file(self):
     return "%s/%s.status" % (self.root, self.name)
 
@@ -145,13 +139,11 @@ class TestSuite(object):
         (mode == 'skip' and pass_fail))
 
     def _compliant(test):
-      outcomes = self.GetStatusFileOutcomes(test.name, test.variant)
-      if statusfile.DoSkip(outcomes):
+      if test.do_skip:
         return False
-      if _skip_slow(statusfile.IsSlow(outcomes), slow_tests_mode):
+      if _skip_slow(test.is_slow, slow_tests_mode):
         return False
-      if _skip_pass_fail(statusfile.IsPassOrFail(outcomes),
-                         pass_fail_tests_mode):
+      if _skip_pass_fail(test.is_pass_or_fail, pass_fail_tests_mode):
         return False
       return True
 
@@ -177,13 +169,13 @@ class TestSuite(object):
 
       if test.name in self.rules.get(variant, {}):
         used_rules.add((test.name, variant))
-        if statusfile.DoSkip(self.rules[variant][test.name]):
+        if statusfile.SKIP in self.rules[variant][test.name]:
           continue
 
       for prefix in self.prefix_rules.get(variant, {}):
         if test.name.startswith(prefix):
           used_rules.add((prefix, variant))
-          if statusfile.DoSkip(self.prefix_rules[variant][prefix]):
+          if statusfile.SKIP in self.prefix_rules[variant][prefix]:
             break
 
     for variant in variants:
@@ -229,29 +221,25 @@ class TestSuite(object):
     """
 
     variant = variant or ''
-    cache_key = '%s$%s' % (testname, variant)
-
-    if cache_key not in self._outcomes_cache:
-      # Load statusfile to get outcomes for the first time.
-      assert(self.rules is not None)
-      assert(self.prefix_rules is not None)
 
-      outcomes = frozenset()
+    # Load statusfile to get outcomes for the first time.
+    assert(self.rules is not None)
+    assert(self.prefix_rules is not None)
 
-      for key in set([variant, '']):
-        rules = self.rules.get(key, {})
-        prefix_rules = self.prefix_rules.get(key, {})
+    outcomes = frozenset()
 
-        if testname in rules:
-          outcomes |= rules[testname]
+    for key in set([variant, '']):
+      rules = self.rules.get(key, {})
+      prefix_rules = self.prefix_rules.get(key, {})
 
-        for prefix in prefix_rules:
-          if testname.startswith(prefix):
-            outcomes |= prefix_rules[prefix]
+      if testname in rules:
+        outcomes |= rules[testname]
 
-      self._outcomes_cache[cache_key] = outcomes
+      for prefix in prefix_rules:
+        if testname.startswith(prefix):
+          outcomes |= prefix_rules[prefix]
 
-    return self._outcomes_cache[cache_key]
+    return outcomes
 
   def IsFailureOutput(self, testcase, output):
     return output.exit_code != 0

tools/testrunner/local/verbose.py

@@ -45,18 +45,18 @@ def PrintReport(tests):
   total = len(tests)
   skipped = nocrash = passes = fail_ok = fail = 0
   for t in tests:
-    outcomes = t.suite.GetStatusFileOutcomes(t.name, t.variant)
+    outcomes = t.statusfile_outcomes
     if not outcomes:
       passes += 1
       continue
-    if statusfile.DoSkip(outcomes):
+    if t.do_skip:
       skipped += 1
       continue
-    if statusfile.IsPassOrFail(outcomes):
+    if t.is_pass_or_fail:
       nocrash += 1
     if list(outcomes) == [statusfile.PASS]:
       passes += 1
-    if statusfile.IsFailOk(outcomes):
+    if t.is_fail_ok:
       fail_ok += 1
     if statusfile.FAIL in outcomes and statusfile.PASS not in outcomes:
       fail += 1

tools/testrunner/objects/testcase.py

@@ -47,13 +47,14 @@ class TestCase(object):
     self.variant = None       # name of the used testing variant
     self.variant_flags = []   # list of strings, flags specific to this test
 
-    self.expected_outcomes = None
-
     self.id = None  # int, used to map result back to TestCase instance
     self.run = 1  # The nth time this test is executed.
     self.cmd = None
 
-    self._prepare_expected_outcomes()
+    self.statusfile_outcomes = None
+    self.expected_outcomes = None
+    self._statusfile_flags = None
+    self._prepare_outcomes()
 
   def create_variant(self, variant, flags):
     """Makes a shallow copy of the object and updates variant, variant_flags and
@@ -66,15 +67,22 @@ class TestCase(object):
       other.variant_flags = self.variant_flags + flags
     other.variant = variant
 
-    other._prepare_expected_outcomes()
+    other._prepare_outcomes(variant != self.variant)
 
     return other
 
-  def _prepare_expected_outcomes(self):
-    status_file_outcomes = self.suite.GetStatusFileOutcomes(self.name,
-                                                            self.variant)
+  def _prepare_outcomes(self, force_update=True):
+    if force_update or self.statusfile_outcomes is None:
+      def is_flag(outcome):
+        return outcome.startswith('--')
+      def not_flag(outcome):
+        return not is_flag(outcome)
+
+      outcomes = self.suite.GetStatusFileOutcomes(self.name, self.variant)
+      self.statusfile_outcomes = filter(not_flag, outcomes)
+      self._statusfile_flags = filter(is_flag, outcomes)
     self.expected_outcomes = (
-      self._parse_status_file_outcomes(status_file_outcomes))
+      self._parse_status_file_outcomes(self.statusfile_outcomes))
 
   def _parse_status_file_outcomes(self, outcomes):
     if (statusfile.FAIL_SLOPPY in outcomes and
@@ -91,6 +99,32 @@ class TestCase(object):
       expected_outcomes.append(statusfile.PASS)
     return expected_outcomes or [statusfile.PASS]
 
+  @property
+  def do_skip(self):
+    return statusfile.SKIP in self.statusfile_outcomes
+
+  @property
+  def is_slow(self):
+    return statusfile.SLOW in self.statusfile_outcomes
+
+  @property
+  def is_fail_ok(self):
+    return statusfile.FAIL_OK in self.statusfile_outcomes
+
+  @property
+  def is_pass_or_fail(self):
+    return (statusfile.PASS in self.statusfile_outcomes and
+            statusfile.FAIL in self.statusfile_outcomes and
+            statusfile.CRASH not in self.statusfile_outcomes)
+
+  @property
+  def only_standard_variant(self):
+    return statusfile.NO_VARIANTS in self.statusfile_outcomes
+
+  @property
+  def only_fast_variants(self):
+    return statusfile.FAST_VARIANTS in self.statusfile_outcomes
+
   def get_command(self, context):
     params = self._get_cmd_params(context)
     env = self._get_cmd_env()
@@ -135,15 +169,9 @@ class TestCase(object):
   def _get_statusfile_flags(self):
     """Gets runtime flags from a status file.
 
-    Every outcome that starts with "--" is a flag. Status file has to be loaded
-    before using this function.
+    Every outcome that starts with "--" is a flag.
     """
-
-    flags = []
-    for outcome in self.suite.GetStatusFileOutcomes(self.name, self.variant):
-      if outcome.startswith('--'):
-        flags.append(outcome)
-    return flags
+    return self._statusfile_flags
 
   def _get_mode_flags(self, ctx):
     return ctx.mode_flags