[test] Enable test status filtering by variant

This adds the possibility to address test cases in the
status file with the variant under which the test is running.
This is only allowed in top-level sections.

Example:
[{
  'test-case': [PASS, SLOW],
}]

['variant == foo', {
  'test-case': [FAIL],
}]

The test case "test-case" is marked as slow in all variants.
Additionally, in variant foo, it'll be expected to fail.

This CL also exemplifies the new feature with test cases
running under the ignition_turbofan variant. The
corresponding legacy flag is deprecated.

BUG=v8:5238

Review-Url: https://codereview.chromium.org/2203013002
Cr-Commit-Position: refs/heads/master@{#38342}

test/cctest/cctest.status

@@ -389,7 +389,7 @@
 }],  # 'arch == ppc64 and simulator_run == True'
 
 ##############################################################################
-['ignition_turbofan', {
+['variant == ignition_turbofan', {
   # TODO(5251): Inlining is currently disabled for the BytecodeGraphBuilder.
   'test-run-inlining/InlineLoopGuardedTwice': [FAIL],
   'test-run-inlining/InlineSurplusArgumentsDeopt': [FAIL],
@@ -457,6 +457,6 @@
 
   # BUG(4751). Flaky with Ignition.
   'test-cpu-profiler/JsNativeJsSample': [SKIP],
-}],  # ignition_turbofan
+}],  # variant == ignition_turbofan
 
 ]

test/mjsunit/mjsunit.status

@@ -698,7 +698,7 @@
 }],  # 'arch == ppc64'
 
 ##############################################################################
-['ignition_turbofan', {
+['variant == ignition_turbofan', {
   # TODO(mythria, 4780): Related to type feedback for calls in interpreter.
   'array-literal-feedback': [FAIL],
   'regress/regress-4121': [FAIL],
@@ -723,25 +723,25 @@
 
   # TODO(rmcilroy): Flaky OOM.
   'unicodelctest-no-optimization': [SKIP],
-}],  # ignition_turbofan
+}],  # variant == ignition_turbofan
 
-['ignition_turbofan and arch == arm64', {
+['variant == ignition_turbofan and arch == arm64', {
   # TODO(rmcilroy,4680): Arm64 specific timeouts.
   'asm/construct-double': [SKIP],
   'compiler/osr-one': [SKIP],
   'compiler/osr-two': [SKIP],
   'wasm/asm-wasm-i32': [SKIP],
   'wasm/asm-wasm-u32': [SKIP],
-}],  # ignition_turbofan and arch == arm64
+}],  # variant == ignition_turbofan and arch == arm64
 
 ['ignition_turbofan and arch == arm', {
   # TODO(rmcilroy,4680): Arm specific timeouts.
   'compiler/osr-one': [SKIP],
   'compiler/osr-two': [SKIP],
   'regress/regress-1257': [SKIP],
-}],  # ignition_turbofan and arch == arm
+}],  # variant == ignition_turbofan and arch == arm
 
-['ignition_turbofan and msan', {
+['variant == ignition_turbofan and msan', {
   # TODO(mythria,4680): All of these tests have large loops and hence slow
   # and timeout.
   'compiler/osr-big': [SKIP],
@@ -752,7 +752,7 @@
   'try': [SKIP],
   # Too slow for interpreter and msan.
   'es6/tail-call-megatest*': [SKIP],
-}],  # ignition_turbofan and msan
+}],  # variant == ignition_turbofan and msan
 
 ##############################################################################
 ['gcov_coverage', {

test/webkit/webkit.status

@@ -112,11 +112,11 @@
 }],  # msan
 
 ##############################################################################
-['ignition_turbofan and msan', {
+['variant == ignition_turbofan and msan', {
   # TODO(mythria,4680): Too slow and timeout on ignition.
   'dfg-double-vote-fuzz': [SKIP],
   'dfg-int-overflow-in-loop': [SKIP],
-}],  # ignition_turbofan and msan
+}],  # variant == ignition_turbofan and msan
 
 ##############################################################################
 ['gcov_coverage', {

tools/run-deopt-fuzzer.py

@@ -394,7 +394,6 @@ def Execute(arch, mode, args, options, suites, workspace):
     "deopt_fuzzer": True,
     "gc_stress": False,
     "gcov_coverage": False,
-    "ignition_turbofan": False,
     "isolates": options.isolates,
     "mode": mode,
     "no_i18n": False,

tools/run-tests.py

@@ -45,7 +45,7 @@ import time
 from testrunner.local import execution
 from testrunner.local import progress
 from testrunner.local import testsuite
-from testrunner.local.testsuite import ALL_VARIANTS
+from testrunner.local.variants import ALL_VARIANTS
 from testrunner.local import utils
 from testrunner.local import verbose
 from testrunner.network import network_execution
@@ -258,8 +258,9 @@ def BuildOptions():
   result.add_option("--extra-flags",
                     help="Additional flags to pass to each test command",
                     default="")
+  # TODO(machenbach): Remove this flag when not reference by infrastructure.
   result.add_option("--ignition-turbofan",
-                    help="Skip tests which don't run in ignition_turbofan",
+                    help="Deprecated",
                     default=False, action="store_true")
   result.add_option("--isolates", help="Whether to test isolates",
                     default=False, action="store_true")
@@ -752,7 +753,6 @@ def Execute(arch, mode, args, options, suites):
     "deopt_fuzzer": False,
     "gc_stress": options.gc_stress,
     "gcov_coverage": options.gcov_coverage,
-    "ignition_turbofan": options.ignition_turbofan,
     "isolates": options.isolates,
     "mode": MODES[mode]["status_mode"],
     "no_i18n": options.no_i18n,
@@ -775,8 +775,12 @@ def Execute(arch, mode, args, options, suites):
     if len(args) > 0:
       s.FilterTestCasesByArgs(args)
     all_tests += s.tests
+
+    # First filtering by status applying the generic rules (independent of
+    # variants).
     s.FilterTestCasesByStatus(options.warn_unused, options.slow_tests,
                               options.pass_fail_tests)
+
     if options.cat:
       verbose.PrintTestSource(s.tests)
       continue
@@ -804,6 +808,10 @@ def Execute(arch, mode, args, options, suites):
     else:
       s.tests = variant_tests
 
+    # Second filtering by status applying the variant-dependent rules.
+    s.FilterTestCasesByStatus(options.warn_unused, options.slow_tests,
+                              options.pass_fail_tests, variants=True)
+
     s.tests = ShardTests(s.tests, options)
     num_tests += len(s.tests)
 

tools/testrunner/local/statusfile.py

@@ -26,6 +26,10 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import os
+import re
+
+from variants import ALL_VARIANTS
+from utils import Freeze
 
 # These outcomes can occur in a TestCase's outcomes list:
 SKIP = "SKIP"
@@ -63,6 +67,10 @@ for var in ["debug", "release", "big", "little",
             "windows", "linux", "aix"]:
   VARIABLES[var] = var
 
+# Allow using variants as keywords.
+for var in ALL_VARIANTS:
+  VARIABLES[var] = var
+
 
 def DoSkip(outcomes):
   return SKIP in outcomes
@@ -116,6 +124,34 @@ def _JoinsPassAndFail(outcomes1, outcomes2):
       FAIL in outcomes2
   )
 
+VARIANT_EXPRESSION = object()
+
+def _EvalExpression(exp, variables):
+  try:
+    return eval(exp, variables)
+  except NameError as e:
+    identifier = re.match("name '(.*)' is not defined", e.message).group(1)
+    assert identifier == "variant", "Unknown identifier: %s" % identifier
+    return VARIANT_EXPRESSION
+
+
+def _EvalVariantExpression(section, rules, wildcards, variant, variables):
+  variables_with_variant = {}
+  variables_with_variant.update(variables)
+  variables_with_variant["variant"] = variant
+  result = _EvalExpression(section[0], variables_with_variant)
+  assert result != VARIANT_EXPRESSION
+  if result is True:
+    _ReadSection(
+        section[1],
+        rules[variant],
+        wildcards[variant],
+        variables_with_variant,
+    )
+  else:
+    assert result is False, "Make sure expressions evaluate to boolean values"
+
+
 def _ParseOutcomeList(rule, outcomes, target_dict, variables):
   result = set([])
   if type(outcomes) == str:
@@ -124,7 +160,16 @@ def _ParseOutcomeList(rule, outcomes, target_dict, variables):
     if type(item) == str:
       _AddOutcome(result, item)
     elif type(item) == list:
-      if not eval(item[0], variables): continue
+      exp = _EvalExpression(item[0], variables)
+      assert exp != VARIANT_EXPRESSION, (
+        "Nested variant expressions are not supported")
+      if exp is False:
+        continue
+
+      # Ensure nobody uses an identifier by mistake, like "default",
+      # which would evaluate to true here otherwise.
+      assert exp is True, "Make sure expressions evaluate to boolean values"
+
       for outcome in item[1:]:
         assert type(outcome) == str
         _AddOutcome(result, outcome)
@@ -146,35 +191,57 @@ def _ParseOutcomeList(rule, outcomes, target_dict, variables):
     target_dict[rule] = result
 
 
-def ReadContent(path):
-  with open(path) as f:
-    global KEYWORDS
-    return eval(f.read(), KEYWORDS)
+def ReadContent(content):
+  global KEYWORDS
+  return eval(content, KEYWORDS)
 
 
-def ReadStatusFile(path, variables):
-  contents = ReadContent(path)
+def ReadStatusFile(content, variables):
+  # Empty defaults for rules and wildcards. Variant-independent
+  # rules are mapped by "", others by the variant name.
+  rules = {variant: {} for variant in ALL_VARIANTS}
+  rules[""] = {}
+  wildcards = {variant: {} for variant in ALL_VARIANTS}
+  wildcards[""] = {}
 
-  rules = {}
-  wildcards = {}
   variables.update(VARIABLES)
-  for section in contents:
+  for section in ReadContent(content):
     assert type(section) == list
     assert len(section) == 2
-    if not eval(section[0], variables): continue
-    section = section[1]
-    assert type(section) == dict
-    for rule in section:
-      assert type(rule) == str
-      if rule[-1] == '*':
-        _ParseOutcomeList(rule, section[rule], wildcards, variables)
-      else:
-        _ParseOutcomeList(rule, section[rule], rules, variables)
-  return rules, wildcards
+    exp = _EvalExpression(section[0], variables)
+    if exp is False:
+      # The expression is variant-independent and evaluates to False.
+      continue
+    elif exp == VARIANT_EXPRESSION:
+      # If the expression contains one or more "variant" keywords, we evaluate
+      # it for all possible variants and create rules for those that apply.
+      for variant in ALL_VARIANTS:
+        _EvalVariantExpression(section, rules, wildcards, variant, variables)
+    else:
+      # The expression is variant-independent and evaluates to True.
+      assert exp is True, "Make sure expressions evaluate to boolean values"
+      _ReadSection(
+          section[1],
+          rules[""],
+          wildcards[""],
+          variables,
+      )
+  return Freeze(rules), Freeze(wildcards)
+
+
+def _ReadSection(section, rules, wildcards, variables):
+  assert type(section) == dict
+  for rule in section:
+    assert type(rule) == str
+    if rule[-1] == '*':
+      _ParseOutcomeList(rule, section[rule], wildcards, variables)
+    else:
+      _ParseOutcomeList(rule, section[rule], rules, variables)
 
 
 def PresubmitCheck(path):
-  contents = ReadContent(path)
+  with open(path) as f:
+    contents = ReadContent(f.read())
   root_prefix = os.path.basename(os.path.dirname(path)) + "/"
   status = {"success": True}
   def _assert(check, message):  # Like "assert", but doesn't throw.

tools/testrunner/local/statusfile_unittest.py

+#!/usr/bin/env python
+# Copyright 2016 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import unittest
+
+import statusfile
+from utils import Freeze
+
+
+TEST_VARIABLES = {
+  'system': 'linux',
+  'mode': 'release',
+}
+
+
+TEST_STATUS_FILE = """
+[
+[ALWAYS, {
+  'foo/bar': [PASS, SKIP],
+  'baz/bar': [PASS, FAIL],
+  'foo/*': [PASS, SLOW],
+}],  # ALWAYS
+
+['%s', {
+  'baz/bar': [PASS, SLOW],
+  'foo/*': [FAIL],
+}],
+]
+"""
+
+
+def make_variables():
+  variables = {}
+  variables.update(TEST_VARIABLES)
+  return variables
+
+
+class UtilsTest(unittest.TestCase):
+  def test_freeze(self):
+    self.assertEqual(2, Freeze({1: [2]})[1][0])
+    self.assertEqual(set([3]), Freeze({1: [2], 2: set([3])})[2])
+
+    with self.assertRaises(Exception):
+      Freeze({1: [], 2: set([3])})[2] = 4
+    with self.assertRaises(Exception):
+      Freeze({1: [], 2: set([3])}).update({3: 4})
+    with self.assertRaises(Exception):
+      Freeze({1: [], 2: set([3])})[1].append(2)
+    with self.assertRaises(Exception):
+      Freeze({1: [], 2: set([3])})[2] |= set([3])
+
+    # Sanity check that we can do the same calls on a non-frozen object.
+    {1: [], 2: set([3])}[2] = 4
+    {1: [], 2: set([3])}.update({3: 4})
+    {1: [], 2: set([3])}[1].append(2)
+    {1: [], 2: set([3])}[2] |= set([3])
+
+
+class StatusFileTest(unittest.TestCase):
+  def test_eval_expression(self):
+    variables = make_variables()
+    variables.update(statusfile.VARIABLES)
+
+    self.assertTrue(
+        statusfile._EvalExpression(
+            'system==linux and mode==release', variables))
+    self.assertTrue(
+        statusfile._EvalExpression(
+            'system==linux or variant==default', variables))
+    self.assertFalse(
+        statusfile._EvalExpression(
+            'system==linux and mode==debug', variables))
+    self.assertRaises(
+        AssertionError,
+        lambda: statusfile._EvalExpression(
+            'system==linux and mode==foo', variables))
+    self.assertRaises(
+        SyntaxError,
+        lambda: statusfile._EvalExpression(
+            'system==linux and mode=release', variables))
+    self.assertEquals(
+        statusfile.VARIANT_EXPRESSION,
+        statusfile._EvalExpression(
+            'system==linux and variant==default', variables)
+    )
+
+  def test_read_statusfile_section_true(self):
+    rules, wildcards = statusfile.ReadStatusFile(
+        TEST_STATUS_FILE % 'system==linux', make_variables())
+
+    self.assertEquals(
+        {
+          'foo/bar': set(['PASS', 'SKIP']),
+          'baz/bar': set(['PASS', 'FAIL', 'SLOW']),
+        },
+        rules[''],
+    )
+    self.assertEquals(
+        {
+          'foo/*': set(['SLOW', 'FAIL']),
+        },
+        wildcards[''],
+    )
+    self.assertEquals({}, rules['default'])
+    self.assertEquals({}, wildcards['default'])
+
+  def test_read_statusfile_section_false(self):
+    rules, wildcards = statusfile.ReadStatusFile(
+        TEST_STATUS_FILE % 'system==windows', make_variables())
+
+    self.assertEquals(
+        {
+          'foo/bar': set(['PASS', 'SKIP']),
+          'baz/bar': set(['PASS', 'FAIL']),
+        },
+        rules[''],
+    )
+    self.assertEquals(
+        {
+          'foo/*': set(['PASS', 'SLOW']),
+        },
+        wildcards[''],
+    )
+    self.assertEquals({}, rules['default'])
+    self.assertEquals({}, wildcards['default'])
+
+  def test_read_statusfile_section_variant(self):
+    rules, wildcards = statusfile.ReadStatusFile(
+        TEST_STATUS_FILE % 'system==linux and variant==default',
+        make_variables(),
+    )
+
+    self.assertEquals(
+        {
+          'foo/bar': set(['PASS', 'SKIP']),
+          'baz/bar': set(['PASS', 'FAIL']),
+        },
+        rules[''],
+    )
+    self.assertEquals(
+        {
+          'foo/*': set(['PASS', 'SLOW']),
+        },
+        wildcards[''],
+    )
+    self.assertEquals(
+        {
+          'baz/bar': set(['PASS', 'SLOW']),
+        },
+        rules['default'],
+    )
+    self.assertEquals(
+        {
+          'foo/*': set(['FAIL']),
+        },
+        wildcards['default'],
+    )
+
+
+if __name__ == '__main__':
+    unittest.main()

tools/testrunner/local/testsuite.py

@@ -33,33 +33,9 @@ from . import commands
 from . import statusfile
 from . import utils
 from ..objects import testcase
+from variants import ALL_VARIANTS, ALL_VARIANT_FLAGS, FAST_VARIANT_FLAGS
+
 
-# Use this to run several variants of the tests.
-ALL_VARIANT_FLAGS = {
-  "default": [[]],
-  "stress": [["--stress-opt", "--always-opt"]],
-  "turbofan": [["--turbo"]],
-  "turbofan_opt": [["--turbo", "--always-opt"]],
-  "nocrankshaft": [["--nocrankshaft"]],
-  "ignition": [["--ignition"]],
-  "ignition_turbofan": [["--ignition-staging", "--turbo"]],
-  "preparser": [["--min-preparse-length=0"]],
-}
-
-# FAST_VARIANTS implies no --always-opt.
-FAST_VARIANT_FLAGS = {
-  "default": [[]],
-  "stress": [["--stress-opt"]],
-  "turbofan": [["--turbo"]],
-  "nocrankshaft": [["--nocrankshaft"]],
-  "ignition": [["--ignition"]],
-  "ignition_turbofan": [["--ignition-staging", "--turbo"]],
-  "preparser": [["--min-preparse-length=0"]],
-}
-
-ALL_VARIANTS = set(["default", "stress", "turbofan", "turbofan_opt",
-                    "nocrankshaft", "ignition", "ignition_turbofan",
-                    "preparser"])
 FAST_VARIANTS = set(["default", "turbofan"])
 STANDARD_VARIANT = set(["default"])
 IGNITION_VARIANT = set(["ignition"])
@@ -153,8 +129,9 @@ class TestSuite(object):
     pass
 
   def ReadStatusFile(self, variables):
-    (self.rules, self.wildcards) = \
-        statusfile.ReadStatusFile(self.status_file(), variables)
+    with open(self.status_file()) as f:
+      self.rules, self.wildcards = (
+          statusfile.ReadStatusFile(f.read(), variables))
 
   def ReadTestCases(self, context):
     self.tests = self.ListTests(context)
@@ -169,18 +146,40 @@ class TestSuite(object):
 
   def FilterTestCasesByStatus(self, warn_unused_rules,
                               slow_tests="dontcare",
-                              pass_fail_tests="dontcare"):
+                              pass_fail_tests="dontcare",
+                              variants=False):
+
+    # Use only variants-dependent rules and wildcards when filtering
+    # respective test cases and generic rules when filtering generic test
+    # cases.
+    if not variants:
+      rules = self.rules[""]
+      wildcards = self.wildcards[""]
+    else:
+      # We set rules and wildcards to a variant-specific version for each test
+      # below.
+      rules = {}
+      wildcards = {}
+
     filtered = []
+
+    # Remember used rules as tuples of (rule, variant), where variant is "" for
+    # variant-independent rules.
     used_rules = set()
+
     for t in self.tests:
       slow = False
       pass_fail = False
       testname = self.CommonTestName(t)
-      if testname in self.rules:
-        used_rules.add(testname)
+      variant = t.variant or ""
+      if variants:
+        rules = self.rules[variant]
+        wildcards = self.wildcards[variant]
+      if testname in rules:
+        used_rules.add((testname, variant))
         # Even for skipped tests, as the TestCase object stays around and
         # PrintReport() uses it.
-        t.outcomes = self.rules[testname]
+        t.outcomes |= rules[testname]
         if statusfile.DoSkip(t.outcomes):
           continue  # Don't add skipped tests to |filtered|.
         for outcome in t.outcomes:
@@ -189,14 +188,14 @@ class TestSuite(object):
         slow = statusfile.IsSlow(t.outcomes)
         pass_fail = statusfile.IsPassOrFail(t.outcomes)
       skip = False
-      for rule in self.wildcards:
+      for rule in wildcards:
         assert rule[-1] == '*'
         if testname.startswith(rule[:-1]):
-          used_rules.add(rule)
-          t.outcomes |= self.wildcards[rule]
+          used_rules.add((rule, variant))
+          t.outcomes |= wildcards[rule]
           if statusfile.DoSkip(t.outcomes):
             skip = True
-            break  # "for rule in self.wildcards"
+            break  # "for rule in wildcards"
           slow = slow or statusfile.IsSlow(t.outcomes)
           pass_fail = pass_fail or statusfile.IsPassOrFail(t.outcomes)
       if (skip
@@ -209,12 +208,26 @@ class TestSuite(object):
     if not warn_unused_rules:
       return
 
-    for rule in self.rules:
-      if rule not in used_rules:
-        print("Unused rule: %s -> %s" % (rule, self.rules[rule]))
-    for rule in self.wildcards:
-      if rule not in used_rules:
-        print("Unused rule: %s -> %s" % (rule, self.wildcards[rule]))
+    if not variants:
+      for rule in self.rules[""]:
+        if (rule, "") not in used_rules:
+          print("Unused rule: %s -> %s (variant independent)" % (
+              rule, self.rules[""][rule]))
+      for rule in self.wildcards[""]:
+        if (rule, "") not in used_rules:
+          print("Unused rule: %s -> %s (variant independent)" % (
+              rule, self.wildcards[""][rule]))
+    else:
+      for variant in ALL_VARIANTS:
+        for rule in self.rules[variant]:
+          if (rule, variant) not in used_rules:
+            print("Unused rule: %s -> %s (variant: %s)" % (
+                rule, self.rules[variant][rule], variant))
+        for rule in self.wildcards[variant]:
+          if (rule, variant) not in used_rules:
+            print("Unused rule: %s -> %s (variant: %s)" % (
+                rule, self.wildcards[variant][rule], variant))
+
 
   def FilterTestCasesByArgs(self, args):
     """Filter test cases based on command-line arguments.

tools/testrunner/local/testsuite_unittest.py

+#!/usr/bin/env python
+# Copyright 2016 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import os
+import sys
+import unittest
+
+# Needed because the test runner contains relative imports.
+TOOLS_PATH = os.path.dirname(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__))))
+sys.path.append(TOOLS_PATH)
+
+from testrunner.local.testsuite import TestSuite
+from testrunner.objects.testcase import TestCase
+
+
+class TestSuiteTest(unittest.TestCase):
+  def test_filter_testcases_by_status_first_pass(self):
+    suite = TestSuite('foo', 'bar')
+    suite.tests = [
+      TestCase(suite, 'foo/bar'),
+      TestCase(suite, 'baz/bar'),
+    ]
+    suite.rules = {
+      '': {
+        'foo/bar': set(['PASS', 'SKIP']),
+        'baz/bar': set(['PASS', 'FAIL']),
+      },
+    }
+    suite.wildcards = {
+      '': {
+        'baz/*': set(['PASS', 'SLOW']),
+      },
+    }
+    suite.FilterTestCasesByStatus(warn_unused_rules=False)
+    self.assertEquals(
+        [TestCase(suite, 'baz/bar')],
+        suite.tests,
+    )
+    self.assertEquals(set(['PASS', 'FAIL', 'SLOW']), suite.tests[0].outcomes)
+
+  def test_filter_testcases_by_status_second_pass(self):
+    suite = TestSuite('foo', 'bar')
+    suite.tests = [
+      TestCase(suite, 'foo/bar', variant='default'),
+      TestCase(suite, 'foo/bar', variant='stress', flags=['-v']),
+      TestCase(suite, 'baz/bar', variant='default'),
+      TestCase(suite, 'baz/bar', variant='stress', flags=['-v']),
+    ]
+
+    # Contrived outcomes from filtering by variant-independent rules.
+    suite.tests[0].outcomes = set(['PREV'])
+    suite.tests[1].outcomes = set(['PREV'])
+    suite.tests[2].outcomes = set(['PREV'])
+    suite.tests[3].outcomes = set(['PREV'])
+
+    suite.rules = {
+      'default': {
+        'foo/bar': set(['PASS', 'SKIP']),
+        'baz/bar': set(['PASS', 'FAIL']),
+      },
+      'stress': {
+        'baz/bar': set(['SKIP']),
+      },
+    }
+    suite.wildcards = {
+      'default': {
+        'baz/*': set(['PASS', 'SLOW']),
+      },
+      'stress': {
+        'foo/*': set(['PASS', 'SLOW']),
+      },
+    }
+    suite.FilterTestCasesByStatus(warn_unused_rules=False, variants=True)
+    self.assertEquals(
+        [
+          TestCase(suite, 'foo/bar', flags=['-v']),
+          TestCase(suite, 'baz/bar'),
+        ],
+        suite.tests,
+    )
+
+    self.assertEquals(
+        set(['PASS', 'SLOW', 'PREV']),
+        suite.tests[0].outcomes,
+    )
+    self.assertEquals(
+        set(['PASS', 'FAIL', 'SLOW', 'PREV']),
+        suite.tests[1].outcomes,
+    )
+
+
+if __name__ == '__main__':
+    unittest.main()

tools/testrunner/local/utils.py

@@ -136,3 +136,24 @@ def URLRetrieve(source, destination):
       pass
   with open(destination, 'w') as f:
     f.write(urllib2.urlopen(source).read())
+
+
+class FrozenDict(dict):
+  def __setitem__(self, *args, **kwargs):
+    raise Exception('Tried to mutate a frozen dict')
+
+  def update(self, *args, **kwargs):
+    raise Exception('Tried to mutate a frozen dict')
+
+
+def Freeze(obj):
+  if isinstance(obj, dict):
+    return FrozenDict((k, Freeze(v)) for k, v in obj.iteritems())
+  elif isinstance(obj, set):
+    return frozenset(obj)
+  elif isinstance(obj, list):
+    return tuple(Freeze(item) for item in obj)
+  else:
+    # Make sure object is hashable.
+    hash(obj)
+    return obj

tools/testrunner/local/variants.py

+# Copyright 2016 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Use this to run several variants of the tests.
+ALL_VARIANT_FLAGS = {
+  "default": [[]],
+  "stress": [["--stress-opt", "--always-opt"]],
+  "turbofan": [["--turbo"]],
+  "turbofan_opt": [["--turbo", "--always-opt"]],
+  "nocrankshaft": [["--nocrankshaft"]],
+  "ignition": [["--ignition"]],
+  "ignition_turbofan": [["--ignition-staging", "--turbo"]],
+  "preparser": [["--min-preparse-length=0"]],
+}
+
+# FAST_VARIANTS implies no --always-opt.
+FAST_VARIANT_FLAGS = {
+  "default": [[]],
+  "stress": [["--stress-opt"]],
+  "turbofan": [["--turbo"]],
+  "nocrankshaft": [["--nocrankshaft"]],
+  "ignition": [["--ignition"]],
+  "ignition_turbofan": [["--ignition-staging", "--turbo"]],
+  "preparser": [["--min-preparse-length=0"]],
+}
+
+ALL_VARIANTS = set(["default", "stress", "turbofan", "turbofan_opt",
+                    "nocrankshaft", "ignition", "ignition_turbofan",
+                    "preparser"])

tools/testrunner/objects/testcase.py

@@ -29,7 +29,7 @@
 from . import output
 
 class TestCase(object):
-  def __init__(self, suite, path, variant='default', flags=None,
+  def __init__(self, suite, path, variant=None, flags=None,
                override_shell=None):
     self.suite = suite        # TestSuite object
     self.path = path          # string, e.g. 'div-mod', 'test-api/foo'
@@ -108,3 +108,6 @@ class TestCase(object):
         (self.suite.name, self.path, self.flags),
         (other.suite.name, other.path, other.flags),
     )
+
+  def __str__(self):
+    return "[%s/%s  %s]" % (self.suite.name, self.path, self.flags)