[test] Key variant flags by variant name everywhere.

This allows variants to be named on test failures (follow
up) and then to be used in the test runner for a repro.

This also speeds up variant iteration for test262 and fixes
a bug with variants for benchmarks.

BUG=chromium:511215
NOTREECHECKS=true
LOG=n

Review URL: https://codereview.chromium.org/1245623005

Cr-Commit-Position: refs/heads/master@{#29899}

test/benchmarks/testcfg.py

@@ -31,10 +31,25 @@ import shutil
 import subprocess
 import tarfile
 
+from testrunner.local import statusfile
 from testrunner.local import testsuite
 from testrunner.objects import testcase
 
 
+class BenchmarksVariantGenerator(testsuite.VariantGenerator):
+  # Both --nocrankshaft and --stressopt are very slow. Add TF but without
+  # always opt to match the way the benchmarks are run for performance
+  # testing.
+  def FilterVariantsByTest(self, testcase):
+    if testcase.outcomes and statusfile.OnlyStandardVariant(
+        testcase.outcomes):
+      return self.standard_variant
+    return self.fast_variants
+
+  def GetFlagSets(self, testcase, variant):
+    return testsuite.FAST_VARIANT_FLAGS[variant]
+
+
 class BenchmarksTestSuite(testsuite.TestSuite):
 
   def __init__(self, name, root):
@@ -182,11 +197,8 @@ class BenchmarksTestSuite(testsuite.TestSuite):
 
     os.chdir(old_cwd)
 
-  def VariantFlags(self, testcase, default_flags):
-    # Both --nocrankshaft and --stressopt are very slow. Add TF but without
-    # always opt to match the way the benchmarks are run for performance
-    # testing.
-    return [[], ["--turbo"]]
+  def _VariantGeneratorFactory(self):
+    return BenchmarksVariantGenerator
 
 
 def GetSuite(name, root):

test/preparser/testcfg.py

@@ -126,8 +126,8 @@ class PreparserTestSuite(testsuite.TestSuite):
     with open(testcase.flags[0]) as f:
       return f.read()
 
-  def VariantFlags(self, testcase, default_flags):
-    return [[]];
+  def _VariantGeneratorFactory(self):
+    return testsuite.StandardVariantGenerator
 
 
 def GetSuite(name, root):

test/test262-es6/testcfg.py

@@ -48,6 +48,55 @@ TEST_262_SUITE_PATH = ["data", "test"]
 TEST_262_HARNESS_PATH = ["data", "harness"]
 TEST_262_TOOLS_PATH = ["data", "tools", "packaging"]
 
+ALL_VARIANT_FLAGS_STRICT = dict(
+    (v, [flags + ["--use-strict"] for flags in flag_sets])
+    for v, flag_sets in testsuite.ALL_VARIANT_FLAGS.iteritems()
+)
+
+FAST_VARIANT_FLAGS_STRICT = dict(
+    (v, [flags + ["--use-strict"] for flags in flag_sets])
+    for v, flag_sets in testsuite.FAST_VARIANT_FLAGS.iteritems()
+)
+
+ALL_VARIANT_FLAGS_BOTH = dict(
+    (v, [flags for flags in testsuite.ALL_VARIANT_FLAGS[v] +
+                            ALL_VARIANT_FLAGS_STRICT[v]])
+    for v in testsuite.ALL_VARIANT_FLAGS
+)
+
+FAST_VARIANT_FLAGS_BOTH = dict(
+    (v, [flags for flags in testsuite.FAST_VARIANT_FLAGS[v] +
+                            FAST_VARIANT_FLAGS_STRICT[v]])
+    for v in testsuite.FAST_VARIANT_FLAGS
+)
+
+ALL_VARIANTS = {
+  'nostrict': testsuite.ALL_VARIANT_FLAGS,
+  'strict': ALL_VARIANT_FLAGS_STRICT,
+  'both': ALL_VARIANT_FLAGS_BOTH,
+}
+
+FAST_VARIANTS = {
+  'nostrict': testsuite.FAST_VARIANT_FLAGS,
+  'strict': FAST_VARIANT_FLAGS_STRICT,
+  'both': FAST_VARIANT_FLAGS_BOTH,
+}
+
+class Test262VariantGenerator(testsuite.VariantGenerator):
+  def GetFlagSets(self, testcase, variant):
+    if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes):
+      variant_flags = FAST_VARIANTS
+    else:
+      variant_flags = ALL_VARIANTS
+
+    test_record = self.suite.GetTestRecord(testcase)
+    if "noStrict" in test_record:
+      return variant_flags["nostrict"][variant]
+    if "onlyStrict" in test_record:
+      return variant_flags["strict"][variant]
+    return variant_flags["both"][variant]
+
+
 class Test262TestSuite(testsuite.TestSuite):
 
   def __init__(self, name, root):
@@ -81,15 +130,8 @@ class Test262TestSuite(testsuite.TestSuite):
             self.GetIncludesForTest(testcase) + ["--harmony"] +
             [os.path.join(self.testroot, testcase.path + ".js")])
 
-  def VariantFlags(self, testcase, default_flags):
-    flags = super(Test262TestSuite, self).VariantFlags(testcase, default_flags)
-    test_record = self.GetTestRecord(testcase)
-    if "noStrict" in test_record:
-      return flags
-    strict_flags = [f + ["--use-strict"] for f in flags]
-    if "onlyStrict" in test_record:
-      return strict_flags
-    return flags + strict_flags
+  def _VariantGeneratorFactory(self):
+    return Test262VariantGenerator
 
   def LoadParseTestRecord(self):
     if not self.ParseTestRecord:

tools/run-tests.py

@@ -44,7 +44,7 @@ import time
 from testrunner.local import execution
 from testrunner.local import progress
 from testrunner.local import testsuite
-from testrunner.local.testsuite import VARIANT_FLAGS
+from testrunner.local.testsuite import ALL_VARIANTS
 from testrunner.local import utils
 from testrunner.local import verbose
 from testrunner.network import network_execution
@@ -332,7 +332,7 @@ def BuildbotToV8Mode(config):
   return mode.lower()
 
 def ProcessOptions(options):
-  global VARIANT_FLAGS
+  global ALL_VARIANTS
   global VARIANTS
 
   # Architecture and mode related stuff.
@@ -423,8 +423,8 @@ def ProcessOptions(options):
     VARIANTS = ["stress"]
   if options.variants:
     VARIANTS = options.variants.split(",")
-    if not set(VARIANTS).issubset(VARIANT_FLAGS.keys()):
-      print "All variants must be in %s" % str(VARIANT_FLAGS.keys())
+    if not set(VARIANTS).issubset(ALL_VARIANTS.keys()):
+      print "All variants must be in %s" % str(ALL_VARIANTS.keys())
       return False
   if options.predictable:
     VARIANTS = ["default"]
@@ -621,10 +621,11 @@ def Execute(arch, mode, args, options, suites, workspace):
     if options.cat:
       verbose.PrintTestSource(s.tests)
       continue
-    variant_flags = [VARIANT_FLAGS[var] for var in VARIANTS]
-    variant_tests = [ t.CopyAddingFlags(v)
+    variant_gen = s.CreateVariantGenerator(VARIANTS)
+    variant_tests = [ t.CopyAddingFlags(flags)
                       for t in s.tests
-                      for v in s.VariantFlags(t, variant_flags) ]
+                      for v in variant_gen.FilterVariantsByTest(t)
+                      for flags in variant_gen.GetFlagSets(t, v) ]
 
     if options.random_seed_stress_count > 1:
       # Duplicate test for random seed stress mode.
@@ -637,9 +638,9 @@ def Execute(arch, mode, args, options, suites, workspace):
           else:
             yield ["--random-seed=%d" % RandomSeed()]
       s.tests = [
-        t.CopyAddingFlags(v)
+        t.CopyAddingFlags(flags)
         for t in variant_tests
-        for v in iter_seed_flags()
+        for flags in iter_seed_flags()
       ]
     else:
       s.tests = variant_tests

tools/testrunner/local/testsuite.py

@@ -35,15 +35,47 @@ from . import utils
 from ..objects import testcase
 
 # Use this to run several variants of the tests.
-VARIANT_FLAGS = {
-    "default": [],
-    "stress": ["--stress-opt", "--always-opt"],
-    "turbofan": ["--turbo", "--always-opt"],
-    "nocrankshaft": ["--nocrankshaft"]}
+ALL_VARIANT_FLAGS = {
+  "default": [[]],
+  "stress": [["--stress-opt", "--always-opt"]],
+  "turbofan": [["--turbo", "--always-opt"]],
+  "nocrankshaft": [["--nocrankshaft"]],
+}
+
+# FAST_VARIANTS implies no --always-opt.
+FAST_VARIANT_FLAGS = {
+  "default": [[]],
+  "stress": [["--stress-opt"]],
+  "turbofan": [["--turbo"]],
+  "nocrankshaft": [["--nocrankshaft"]],
+}
+
+ALL_VARIANTS = set(["default", "stress", "turbofan", "nocrankshaft"])
+FAST_VARIANTS = set(["default", "turbofan"])
+STANDARD_VARIANT = set(["default"])
+
+
+class VariantGenerator(object):
+  def __init__(self, suite, variants):
+    self.suite = suite
+    self.all_variants = ALL_VARIANTS & variants
+    self.fast_variants = FAST_VARIANTS & variants
+    self.standard_variant = STANDARD_VARIANT & variants
+
+  def FilterVariantsByTest(self, testcase):
+    if testcase.outcomes and statusfile.OnlyStandardVariant(
+        testcase.outcomes):
+      return self.standard_variant
+    if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes):
+      return self.fast_variants
+    return self.all_variants
+
+  def GetFlagSets(self, testcase, variant):
+    if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes):
+      return FAST_VARIANT_FLAGS[variant]
+    else:
+      return ALL_VARIANT_FLAGS[variant]
 
-FAST_VARIANT_FLAGS = [
-    f for v, f in VARIANT_FLAGS.iteritems() if v in ["default", "turbofan"]
-]
 
 class TestSuite(object):
 
@@ -89,15 +121,19 @@ class TestSuite(object):
   def ListTests(self, context):
     raise NotImplementedError
 
-  def VariantFlags(self, testcase, default_flags):
-    if testcase.outcomes and statusfile.OnlyStandardVariant(testcase.outcomes):
-      return [[]]
-    if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes):
-      # FAST_VARIANTS implies no --always-opt.
-      return [ filter(lambda flag: flag != "--always-opt", f)
-               for f in filter(lambda flags: flags in FAST_VARIANT_FLAGS,
-                               default_flags) ]
-    return default_flags
+  def _VariantGeneratorFactory(self):
+    """The variant generator class to be used."""
+    return VariantGenerator
+
+  def CreateVariantGenerator(self, variants):
+    """Return a generator for the testing variants of this suite.
+
+    Args:
+      variants: List of variant names to be run as specified by the test
+                runner.
+    Returns: An object of type VariantGenerator.
+    """
+    return self._VariantGeneratorFactory()(self, set(variants))
 
   def DownloadData(self):
     pass
@@ -252,6 +288,11 @@ class TestSuite(object):
     return self.total_duration
 
 
+class StandardVariantGenerator(VariantGenerator):
+  def FilterVariantsByTest(self, testcase):
+    return self.standard_variant
+
+
 class GoogleTestSuite(TestSuite):
   def __init__(self, name, root):
     super(GoogleTestSuite, self).__init__(name, root)
@@ -285,8 +326,8 @@ class GoogleTestSuite(TestSuite):
             ["--gtest_print_time=0"] +
             context.mode_flags)
 
-  def VariantFlags(self, testcase, default_flags):
-    return [[]]
+  def _VariantGeneratorFactory(self):
+    return StandardVariantGenerator
 
   def shell(self):
     return self.name