[test] Change wildcards to prefix rules.

Test suite contract changes:
- support * only at the end of the rule.
- loading status file is mandatory before filtering by status file.

Bug: v8:6917
Change-Id: Ia345ebfa7827c50f13f20e5cb7489e62c53f3357
Reviewed-on: https://chromium-review.googlesource.com/779185
Commit-Queue: Michał Majewski <majeski@google.com>
Reviewed-by: Sergiy Byelozyorov <sergiyb@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#49496}

tools/testrunner/local/statusfile.py

@@ -129,7 +129,7 @@ def _EvalExpression(exp, variables):
     return VARIANT_EXPRESSION
 
 
-def _EvalVariantExpression(section, rules, wildcards, variant, variables):
+def _EvalVariantExpression(section, rules, prefix_rules, variant, variables):
   variables_with_variant = {}
   variables_with_variant.update(variables)
   variables_with_variant["variant"] = variant
@@ -139,7 +139,7 @@ def _EvalVariantExpression(section, rules, wildcards, variant, variables):
     _ReadSection(
         section[1],
         rules[variant],
-        wildcards[variant],
+        prefix_rules[variant],
         variables_with_variant,
     )
   else:
@@ -191,12 +191,12 @@ def ReadContent(content):
 
 
 def ReadStatusFile(content, variables):
-  # Empty defaults for rules and wildcards. Variant-independent
+  # Empty defaults for rules and prefix_rules. Variant-independent
   # rules are mapped by "", others by the variant name.
   rules = {variant: {} for variant in ALL_VARIANTS}
   rules[""] = {}
-  wildcards = {variant: {} for variant in ALL_VARIANTS}
-  wildcards[""] = {}
+  prefix_rules = {variant: {} for variant in ALL_VARIANTS}
+  prefix_rules[""] = {}
 
   variables.update(VARIABLES)
   for section in ReadContent(content):
@@ -210,25 +210,30 @@ def ReadStatusFile(content, variables):
       # If the expression contains one or more "variant" keywords, we evaluate
       # it for all possible variants and create rules for those that apply.
       for variant in ALL_VARIANTS:
-        _EvalVariantExpression(section, rules, wildcards, variant, variables)
+        _EvalVariantExpression(section, rules, prefix_rules, variant, variables)
     else:
       # The expression is variant-independent and evaluates to True.
       assert exp is True, "Make sure expressions evaluate to boolean values"
       _ReadSection(
           section[1],
           rules[""],
-          wildcards[""],
+          prefix_rules[""],
           variables,
       )
-  return Freeze(rules), Freeze(wildcards)
+  return Freeze(rules), Freeze(prefix_rules)
 
 
-def _ReadSection(section, rules, wildcards, variables):
+def _ReadSection(section, rules, prefix_rules, variables):
   assert type(section) == dict
   for rule in section:
     assert type(rule) == str
+
+    # Wildcards are allowed only as the last character.
+    wildcards_count = rule.count('*')
+    assert wildcards_count == 0 or (wildcards_count == 1 and rule[-1] == '*')
+
     if rule[-1] == '*':
-      _ParseOutcomeList(rule, section[rule], wildcards, variables)
+      _ParseOutcomeList(rule[:-1], section[rule], prefix_rules, variables)
     else:
       _ParseOutcomeList(rule, section[rule], rules, variables)
 

tools/testrunner/local/statusfile_unittest.py

@@ -87,7 +87,7 @@ class StatusFileTest(unittest.TestCase):
     )
 
   def test_read_statusfile_section_true(self):
-    rules, wildcards = statusfile.ReadStatusFile(
+    rules, prefix_rules = statusfile.ReadStatusFile(
         TEST_STATUS_FILE % 'system==linux', make_variables())
 
     self.assertEquals(
@@ -99,15 +99,15 @@ class StatusFileTest(unittest.TestCase):
     )
     self.assertEquals(
         {
-          'foo/*': set(['SLOW', 'FAIL']),
+          'foo/': set(['SLOW', 'FAIL']),
         },
-        wildcards[''],
+        prefix_rules[''],
     )
     self.assertEquals({}, rules['default'])
-    self.assertEquals({}, wildcards['default'])
+    self.assertEquals({}, prefix_rules['default'])
 
   def test_read_statusfile_section_false(self):
-    rules, wildcards = statusfile.ReadStatusFile(
+    rules, prefix_rules = statusfile.ReadStatusFile(
         TEST_STATUS_FILE % 'system==windows', make_variables())
 
     self.assertEquals(
@@ -119,15 +119,15 @@ class StatusFileTest(unittest.TestCase):
     )
     self.assertEquals(
         {
-          'foo/*': set(['PASS', 'SLOW']),
+          'foo/': set(['PASS', 'SLOW']),
         },
-        wildcards[''],
+        prefix_rules[''],
     )
     self.assertEquals({}, rules['default'])
-    self.assertEquals({}, wildcards['default'])
+    self.assertEquals({}, prefix_rules['default'])
 
   def test_read_statusfile_section_variant(self):
-    rules, wildcards = statusfile.ReadStatusFile(
+    rules, prefix_rules = statusfile.ReadStatusFile(
         TEST_STATUS_FILE % 'system==linux and variant==default',
         make_variables(),
     )
@@ -141,9 +141,9 @@ class StatusFileTest(unittest.TestCase):
     )
     self.assertEquals(
         {
-          'foo/*': set(['PASS', 'SLOW']),
+          'foo/': set(['PASS', 'SLOW']),
         },
-        wildcards[''],
+        prefix_rules[''],
     )
     self.assertEquals(
         {
@@ -153,9 +153,9 @@ class StatusFileTest(unittest.TestCase):
     )
     self.assertEquals(
         {
-          'foo/*': set(['FAIL']),
+          'foo/': set(['FAIL']),
         },
-        wildcards['default'],
+        prefix_rules['default'],
     )
 
 

tools/testrunner/local/testsuite.py

@@ -86,8 +86,8 @@ class TestSuite(object):
     self.name = name  # string
     self.root = root  # string containing path
     self.tests = None  # list of TestCase objects
-    self.rules = None  # dictionary mapping test path to list of outcomes
-    self.wildcards = None  # dictionary mapping test paths to list of outcomes
+    self.rules = None  # {variant: {test name: [rule]}}
+    self.prefix_rules = None  # {variant: {test name prefix: [rule]}}
     self.total_duration = None  # float, assigned on demand
 
   def suffix(self):
@@ -130,7 +130,7 @@ class TestSuite(object):
 
   def ReadStatusFile(self, variables):
     with open(self.status_file()) as f:
-      self.rules, self.wildcards = (
+      self.rules, self.prefix_rules = (
           statusfile.ReadStatusFile(f.read(), variables))
 
   def ReadTestCases(self, context):
@@ -148,18 +148,21 @@ class TestSuite(object):
                               slow_tests="dontcare",
                               pass_fail_tests="dontcare",
                               variants=False):
+    # Load statusfile before.
+    assert(self.rules is not None)
+    assert(self.prefix_rules is not None)
 
-    # Use only variants-dependent rules and wildcards when filtering
+    # Use only variants-dependent rules and prefix_rules when filtering
     # respective test cases and generic rules when filtering generic test
     # cases.
     if not variants:
       rules = self.rules[""]
-      wildcards = self.wildcards[""]
+      prefix_rules = self.prefix_rules[""]
     else:
-      # We set rules and wildcards to a variant-specific version for each test
-      # below.
+      # We set rules and prefix_rules to a variant-specific version for each
+      # test below.
       rules = {}
-      wildcards = {}
+      prefix_rules = {}
 
     filtered = []
 
@@ -174,7 +177,7 @@ class TestSuite(object):
       variant = t.variant or ""
       if variants:
         rules = self.rules[variant]
-        wildcards = self.wildcards[variant]
+        prefix_rules = self.prefix_rules[variant]
       if testname in rules:
         used_rules.add((testname, variant))
         # Even for skipped tests, as the TestCase object stays around and
@@ -188,14 +191,13 @@ class TestSuite(object):
         slow = statusfile.IsSlow(t.outcomes)
         pass_fail = statusfile.IsPassOrFail(t.outcomes)
       skip = False
-      for rule in wildcards:
-        assert rule[-1] == '*'
-        if testname.startswith(rule[:-1]):
-          used_rules.add((rule, variant))
-          t.outcomes = t.outcomes | wildcards[rule]
+      for prefix in prefix_rules:
+        if testname.startswith(prefix):
+          used_rules.add((prefix, variant))
+          t.outcomes = t.outcomes | prefix_rules[prefix]
           if statusfile.DoSkip(t.outcomes):
             skip = True
-            break  # "for rule in wildcards"
+            break  # "for rule in prefix_rules"
           slow = slow or statusfile.IsSlow(t.outcomes)
           pass_fail = pass_fail or statusfile.IsPassOrFail(t.outcomes)
       if (skip
@@ -213,20 +215,20 @@ class TestSuite(object):
         if (rule, "") not in used_rules:
           print("Unused rule: %s -> %s (variant independent)" % (
               rule, self.rules[""][rule]))
-      for rule in self.wildcards[""]:
+      for rule in self.prefix_rules[""]:
         if (rule, "") not in used_rules:
           print("Unused rule: %s -> %s (variant independent)" % (
-              rule, self.wildcards[""][rule]))
+              rule, self.prefix_rules[""][rule]))
     else:
       for variant in ALL_VARIANTS:
         for rule in self.rules[variant]:
           if (rule, variant) not in used_rules:
             print("Unused rule: %s -> %s (variant: %s)" % (
                 rule, self.rules[variant][rule], variant))
-        for rule in self.wildcards[variant]:
+        for rule in self.prefix_rules[variant]:
           if (rule, variant) not in used_rules:
             print("Unused rule: %s -> %s (variant: %s)" % (
-                rule, self.wildcards[variant][rule], variant))
+                rule, self.prefix_rules[variant][rule], variant))
 
 
   def FilterTestCasesByArgs(self, args):

tools/testrunner/local/testsuite_unittest.py

@@ -29,9 +29,9 @@ class TestSuiteTest(unittest.TestCase):
         'baz/bar': set(['PASS', 'FAIL']),
       },
     }
-    suite.wildcards = {
+    suite.prefix_rules = {
       '': {
-        'baz/*': set(['PASS', 'SLOW']),
+        'baz/': set(['PASS', 'SLOW']),
       },
     }
     suite.FilterTestCasesByStatus(warn_unused_rules=False)
@@ -67,12 +67,12 @@ class TestSuiteTest(unittest.TestCase):
         'baz/bar': set(['SKIP']),
       },
     }
-    suite.wildcards = {
+    suite.prefix_rules = {
       'default': {
-        'baz/*': set(['PASS', 'SLOW']),
+        'baz/': set(['PASS', 'SLOW']),
       },
       'stress': {
-        'foo/*': set(['PASS', 'SLOW']),
+        'foo/': set(['PASS', 'SLOW']),
       },
     }
     suite.FilterTestCasesByStatus(warn_unused_rules=False, variants=True)