Revert "[test] Keep the order of variants"

This reverts commit 8f4407fb.

Reason for revert: Failing in old test runner. See comment.

Original change's description:
> [test] Keep the order of variants
> 
> Bug: v8:6917
> Change-Id: I7e04255649f62a28f0ed07ccb4fbaf4825498bf6
> Reviewed-on: https://chromium-review.googlesource.com/886841
> Commit-Queue: Michał Majewski <majeski@google.com>
> Reviewed-by: Michael Achenbach <machenbach@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#50874}

TBR=machenbach@chromium.org,sergiyb@chromium.org,majeski@google.com

Change-Id: Ia8608053e080c70d2ac09d7921a6abb6cd0e338f
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: v8:6917
Reviewed-on: https://chromium-review.googlesource.com/888749Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50895}

tools/testrunner/local/testsuite.py

@@ -43,7 +43,7 @@ STANDARD_VARIANT = set(["default"])
 class LegacyVariantsGenerator(object):
   def __init__(self, suite, variants):
     self.suite = suite
-    self.all_variants = variants
+    self.all_variants = ALL_VARIANTS & variants
     self.standard_variant = STANDARD_VARIANT & variants
 
   def FilterVariantsByTest(self, test):

tools/testrunner/local/variants.py

@@ -25,27 +25,4 @@ ALL_VARIANT_FLAGS = {
   "wasm_traps": [["--wasm_trap_handler", "--invoke-weak-callbacks", "--wasm-jit-to-native"]],
 }
 
-SLOW_VARIANTS = set([
-  'stress',
-  'nooptimization',
-])
-
-FAST_VARIANTS = set([
-  'default'
-])
-
-
-def _variant_order_key(v):
-  if v in SLOW_VARIANTS:
-    return 0
-  if v in FAST_VARIANTS:
-    return 100
-  return 50
-
-ALL_VARIANTS = sorted(ALL_VARIANT_FLAGS.keys(),
-                      key=_variant_order_key)
-
-# Check {SLOW,FAST}_VARIANTS entries
-for variants in [SLOW_VARIANTS, FAST_VARIANTS]:
-  for v in variants:
-    assert v in ALL_VARIANT_FLAGS
+ALL_VARIANTS = set(ALL_VARIANT_FLAGS.keys())

tools/testrunner/standard_runner.py

@@ -39,13 +39,14 @@ from testrunner.testproc.variant import VariantProc
 
 TIMEOUT_DEFAULT = 60
 
+# Variants ordered by expected runtime (slowest first).
 VARIANTS = ["default"]
 
 MORE_VARIANTS = [
-  "nooptimization",
   "stress",
-  "stress_background_compile",
   "stress_incremental_marking",
+  "nooptimization",
+  "stress_background_compile",
   "wasm_traps",
 ]
 
@@ -111,7 +112,6 @@ class StandardTestRunner(base_runner.BaseTestRunner):
         super(StandardTestRunner, self).__init__(*args, **kwargs)
 
         self.sancov_dir = None
-        self._variants = None
 
     def _get_default_suite_names(self):
       return ['default']
@@ -249,6 +249,8 @@ class StandardTestRunner(base_runner.BaseTestRunner):
       return False
 
     def _process_options(self, options):
+      global VARIANTS
+
       if options.sancov_dir:
         self.sancov_dir = options.sancov_dir
         if not os.path.exists(self.sancov_dir):
@@ -313,6 +315,9 @@ class StandardTestRunner(base_runner.BaseTestRunner):
       if options.random_seed_stress_count <= 1 and options.random_seed == 0:
         options.random_seed = self._random_seed()
 
+      # Use developer defaults if no variant was specified.
+      options.variants = options.variants or "dev"
+
       if options.variants == "infra_staging":
         options.variants = "exhaustive"
         options.infra_staging = True
@@ -320,7 +325,18 @@ class StandardTestRunner(base_runner.BaseTestRunner):
       # Use staging on whitelisted masters/builders.
       options.infra_staging = self._use_staging(options)
 
-      self._variants = self._parse_variants(options.variants)
+      # Resolve variant aliases and dedupe.
+      # TODO(machenbach): Don't mutate global variable. Rather pass mutated
+      # version as local variable.
+      VARIANTS = list(set(reduce(
+          list.__add__,
+          (VARIANT_ALIASES.get(v, [v]) for v in options.variants.split(",")),
+          [],
+      )))
+
+      if not set(VARIANTS).issubset(ALL_VARIANTS):
+        print "All variants must be in %s" % str(ALL_VARIANTS)
+        raise base_runner.TestRunnerError()
 
       def CheckTestMode(name, option):  # pragma: no cover
         if not option in ["run", "skip", "dontcare"]:
@@ -334,23 +350,6 @@ class StandardTestRunner(base_runner.BaseTestRunner):
         # TODO(machenbach): uncomment after infra side lands.
         # base_runner.TEST_MAP["d8_default"].remove("intl")
 
-    def _parse_variants(self, aliases_str):
-      # Use developer defaults if no variant was specified.
-      aliases_str = aliases_str or 'dev'
-      aliases = aliases_str.split(',')
-      user_variants = set(reduce(
-          list.__add__, [VARIANT_ALIASES.get(a, [a]) for a in aliases]))
-
-      result = [v for v in ALL_VARIANTS if v in user_variants]
-      if len(result) == len(user_variants):
-        return result
-
-      for v in user_variants:
-        if v not in ALL_VARIANTS:
-          print 'Unknown variant: %s' % v
-          raise base_runner.TestRunnerError()
-      assert False, 'Unreachable'
-
     def _setup_env(self):
       super(StandardTestRunner, self)._setup_env()
 
@@ -482,7 +481,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
         if options.cat:
           verbose.PrintTestSource(s.tests)
           continue
-        variant_gen = s.CreateLegacyVariantsGenerator(self._variants)
+        variant_gen = s.CreateLegacyVariantsGenerator(VARIANTS)
         variant_tests = [ t.create_variant(v, flags)
                           for t in s.tests
                           for v in variant_gen.FilterVariantsByTest(t)
@@ -598,7 +597,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
         StatusFileFilterProc(options.slow_tests, options.pass_fail_tests),
         self._create_shard_proc(options),
         tests_counter,
-        VariantProc(self._variants),
+        VariantProc(VARIANTS),
         StatusFileFilterProc(options.slow_tests, options.pass_fail_tests),
       ] + indicators + [
         results,