[test] Implement variant generators as processors

Bug: v8:6917
Cq-Include-Trybots: luci.v8.try:v8_linux64_fyi_rel_ng;luci.v8.try:v8_linux_noi18n_rel_ng
Change-Id: Ib5bfdf4d6fee6102f62c7334a1b22146f1a1fc5b
Reviewed-on: https://chromium-review.googlesource.com/857376
Commit-Queue: Michał Majewski <majeski@google.com>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50525}

test/benchmarks/testcfg.py

@@ -34,7 +34,7 @@ from testrunner.local import testsuite
 from testrunner.objects import testcase
 
 
-class VariantGenerator(testsuite.VariantGenerator):
+class LegacyVariantsGenerator(testsuite.LegacyVariantsGenerator):
   # Both --noopt and --stressopt are very slow. Add TF but without
   # always opt to match the way the benchmarks are run for performance
   # testing.
@@ -47,6 +47,16 @@ class VariantGenerator(testsuite.VariantGenerator):
     return testsuite.FAST_VARIANT_FLAGS[variant]
 
 
+class VariantsGenerator(testsuite.VariantsGenerator):
+  def _get_flags_set(self, test):
+    return testsuite.FAST_VARIANT_FLAGS
+
+  def _get_variants(self, test):
+    if test.only_standard_variant:
+      return self._standard_variant
+    return self._fast_variants
+
+
 class TestSuite(testsuite.TestSuite):
   def __init__(self, name, root):
     super(TestSuite, self).__init__(name, root)
@@ -117,8 +127,11 @@ class TestSuite(testsuite.TestSuite):
   def _test_class(self):
     return TestCase
 
-  def _VariantGeneratorFactory(self):
-    return VariantGenerator
+  def _variants_gen_class(self):
+    return VariantsGenerator
+
+  def _LegacyVariantsGeneratorFactory(self):
+    return LegacyVariantsGenerator
 
 
 class TestCase(testcase.TestCase):

test/fuzzer/testcfg.py

@@ -8,7 +8,7 @@ from testrunner.local import testsuite
 from testrunner.objects import testcase
 
 
-class VariantGenerator(testsuite.VariantGenerator):
+class LegacyVariantsGenerator(testsuite.LegacyVariantsGenerator):
   # Only run the fuzzer with standard variant.
   def FilterVariantsByTest(self, test):
     return self.standard_variant
@@ -17,6 +17,14 @@ class VariantGenerator(testsuite.VariantGenerator):
     return testsuite.FAST_VARIANT_FLAGS[variant]
 
 
+class VariantsGenerator(testsuite.VariantsGenerator):
+  def _get_flags_set(self, test):
+    return testsuite.FAST_VARIANT_FLAGS
+
+  def _get_variants(self, test):
+    return self._standard_variant
+
+
 class TestSuite(testsuite.TestSuite):
   SUB_TESTS = ( 'json', 'parser', 'regexp', 'wasm', 'wasm_async',
           'wasm_call', 'wasm_code', 'wasm_compile', 'wasm_data_section',
@@ -38,8 +46,11 @@ class TestSuite(testsuite.TestSuite):
   def _test_class(self):
     return TestCase
 
-  def _VariantGeneratorFactory(self):
-    return VariantGenerator
+  def _variants_gen_class(self):
+    return VariantsGenerator
+
+  def _LegacyVariantsGeneratorFactory(self):
+    return LegacyVariantsGenerator
 
 
 class TestCase(testcase.TestCase):

test/message/testcfg.py

@@ -58,10 +58,13 @@ class TestSuite(testsuite.TestSuite):
   def _test_class(self):
     return TestCase
 
-  def CreateVariantGenerator(self, variants):
-    return super(TestSuite, self).CreateVariantGenerator(
+  def CreateLegacyVariantsGenerator(self, variants):
+    return super(TestSuite, self).CreateLegacyVariantsGenerator(
         variants + ["preparser"])
 
+  def create_variant_proc(self, variants):
+    return super(TestSuite, self).create_variant_proc(variants + ['preparser'])
+
 
 class TestCase(testcase.TestCase):
   def __init__(self, *args, **kwargs):

test/preparser/testcfg.py

@@ -32,6 +32,11 @@ from testrunner.local import testsuite
 from testrunner.objects import testcase
 
 
+class VariantsGenerator(testsuite.VariantsGenerator):
+  def _get_variants(self, test):
+    return self._standard_variant
+
+
 class TestSuite(testsuite.TestSuite):
   def _ParsePythonTestTemplates(self, result, filename):
     pathname = os.path.join(self.root, filename + ".pyt")
@@ -72,8 +77,11 @@ class TestSuite(testsuite.TestSuite):
   def _test_class(self):
     return TestCase
 
-  def _VariantGeneratorFactory(self):
-    return testsuite.StandardVariantGenerator
+  def _LegacyVariantsGeneratorFactory(self):
+    return testsuite.StandardLegacyVariantsGenerator
+
+  def _variants_gen_class(self):
+    return VariantsGenerator
 
 
 class TestCase(testcase.TestCase):

test/test262/testcfg.py

@@ -105,7 +105,7 @@ FAST_VARIANTS = {
   'both': FAST_VARIANT_FLAGS_BOTH,
 }
 
-class VariantGenerator(testsuite.VariantGenerator):
+class LegacyVariantsGenerator(testsuite.LegacyVariantsGenerator):
   def GetFlagSets(self, test, variant):
     if test.only_fast_variants:
       variant_flags = FAST_VARIANTS
@@ -120,6 +120,26 @@ class VariantGenerator(testsuite.VariantGenerator):
     return variant_flags["both"][variant]
 
 
+class VariantsGenerator(testsuite.VariantsGenerator):
+  def _variants_gen(self, test):
+    flags_set = self._get_flags_set(test)
+    test_record = test.test_record
+    for n, variant in enumerate(self._get_variants(test)):
+      flags = flags_set[variant][0]
+      if 'noStrict' in test_record:
+        yield (variant, flags, str(n))
+      elif 'onlyStrict' in test_record:
+        yield (variant, flags + ['--use-strict'], 'strict-%d' % n)
+      else:
+        yield (variant, flags, str(n))
+        yield (variant, flags + ['--use-strict'], 'strict-%d' % n)
+
+  def _get_flags_set(self, test):
+    if test.only_fast_variants:
+      return testsuite.FAST_VARIANTS_FLAGS
+    return testsuite.ALL_VARIANT_FLAGS
+
+
 class TestSuite(testsuite.TestSuite):
   # Match the (...) in '/path/to/v8/test/test262/subdir/test/(...).js'
   # In practice, subdir is data or local-tests
@@ -192,8 +212,11 @@ class TestSuite(testsuite.TestSuite):
   def _test_class(self):
     return TestCase
 
-  def _VariantGeneratorFactory(self):
-    return VariantGenerator
+  def _LegacyVariantsGeneratorFactory(self):
+    return LegacyVariantsGenerator
+
+  def _variants_gen_class(self):
+    return VariantsGenerator
 
 
 class TestCase(testcase.TestCase):
@@ -255,7 +278,5 @@ class TestCase(testcase.TestCase):
     return test262.NoExceptionOutProc(self.expected_outcomes)
 
 
-
-
 def GetSuite(name, root):
   return TestSuite(name, root)

test/unittests/testcfg.py

@@ -50,8 +50,8 @@ class TestSuite(testsuite.TestSuite):
   def _test_class(self):
     return TestCase
 
-  def _VariantGeneratorFactory(self):
-    return testsuite.StandardVariantGenerator
+  def _LegacyVariantsGeneratorFactory(self):
+    return testsuite.StandardLegacyVariantsGenerator
 
 
 class TestCase(testcase.TestCase):

tools/testrunner/local/testsuite.py

@@ -41,7 +41,7 @@ FAST_VARIANTS = set(["default", "turbofan"])
 STANDARD_VARIANT = set(["default"])
 
 
-class VariantGenerator(object):
+class LegacyVariantsGenerator(object):
   def __init__(self, suite, variants):
     self.suite = suite
     self.all_variants = ALL_VARIANTS & variants
@@ -62,6 +62,37 @@ class VariantGenerator(object):
       return ALL_VARIANT_FLAGS[variant]
 
 
+class StandardLegacyVariantsGenerator(LegacyVariantsGenerator):
+  def FilterVariantsByTest(self, testcase):
+    return self.standard_variant
+
+
+class VariantsGenerator(object):
+  def __init__(self, variants):
+    self._all_variants = [v for v in variants if v in ALL_VARIANTS]
+    self._fast_variants = [v for v in variants if v in FAST_VARIANTS]
+    self._standard_variant = [v for v in variants if v in STANDARD_VARIANT]
+
+  def gen(self, test):
+    """Generator producing (variant, flags, procid suffix) tuples."""
+    flags_set = self._get_flags_set(test)
+    for n, variant in enumerate(self._get_variants(test)):
+      yield (variant, flags_set[variant][0], n)
+
+  def _get_flags_set(self, test):
+    if test.only_fast_variants:
+      return FAST_VARIANT_FLAGS
+    else:
+      return ALL_VARIANT_FLAGS
+
+  def _get_variants(self, test):
+    if test.only_standard_variant:
+      return self._standard_variant
+    if test.only_fast_variants:
+      return self._fast_variants
+    return self._all_variants
+
+
 class TestSuite(object):
   @staticmethod
   def LoadTestSuite(root):
@@ -88,19 +119,25 @@ class TestSuite(object):
   def ListTests(self, context):
     raise NotImplementedError
 
-  def _VariantGeneratorFactory(self):
+  def _LegacyVariantsGeneratorFactory(self):
     """The variant generator class to be used."""
-    return VariantGenerator
+    return LegacyVariantsGenerator
 
-  def CreateVariantGenerator(self, variants):
+  def CreateLegacyVariantsGenerator(self, variants):
     """Return a generator for the testing variants of this suite.
 
     Args:
       variants: List of variant names to be run as specified by the test
                 runner.
-    Returns: An object of type VariantGenerator.
+    Returns: An object of type LegacyVariantsGenerator.
     """
-    return self._VariantGeneratorFactory()(self, set(variants))
+    return self._LegacyVariantsGeneratorFactory()(self, set(variants))
+
+  def get_variants_gen(self, variants):
+    return self._variants_gen_class()(variants)
+
+  def _variants_gen_class(self):
+    return VariantsGenerator
 
   def ReadStatusFile(self, variables):
     self.statusfile = statusfile.StatusFile(self.status_file(), variables)
@@ -182,8 +219,3 @@ class TestSuite(object):
     if utils.IsWindows():
       return path.replace("\\", "/")
     return path
-
-
-class StandardVariantGenerator(VariantGenerator):
-  def FilterVariantsByTest(self, testcase):
-    return self.standard_variant

tools/testrunner/objects/testcase.py

@@ -63,11 +63,16 @@ class TestCase(object):
     self._statusfile_flags = None
     self._prepare_outcomes()
 
-  def create_subtest(self, processor, subtest_id):
+  def create_subtest(self, processor, subtest_id, variant=None, flags=None):
     subtest = copy.copy(self)
     subtest.origin = self
     subtest.processor = processor
     subtest.procid += '.%s' % subtest_id
+    if variant is not None:
+      assert self.variant is None
+      subtest.variant = variant
+      subtest.variant_flags = flags
+      subtest._prepare_outcomes()
     return subtest
 
   def create_variant(self, variant, flags, procid_suffix=None):

tools/testrunner/standard_runner.py

@@ -32,6 +32,7 @@ from testrunner.testproc.loader import LoadProc
 from testrunner.testproc.progress import (VerboseProgressIndicator,
                                           ResultsTracker)
 from testrunner.testproc.rerun import RerunProc
+from testrunner.testproc.variant import VariantProc
 
 
 TIMEOUT_DEFAULT = 60
@@ -442,13 +443,15 @@ class StandardTestRunner(base_runner.BaseTestRunner):
         if options.cat:
           verbose.PrintTestSource(s.tests)
           continue
-        variant_gen = s.CreateVariantGenerator(VARIANTS)
-        variant_tests = [
-          t.create_variant(v, flags, n)
+        if not options.infra_staging:
+          variant_gen = s.CreateLegacyVariantsGenerator(VARIANTS)
+          variant_tests = [ t.create_variant(v, flags)
                             for t in s.tests
                             for v in variant_gen.FilterVariantsByTest(t)
-          for n, flags in enumerate(variant_gen.GetFlagSets(t, v))
-        ]
+                            for flags in variant_gen.GetFlagSets(t, v) ]
+        else:
+          # Variants will be created in the test processors pipeline
+          variant_tests = s.tests
 
         if options.random_seed_stress_count > 1:
           # Duplicate test for random seed stress mode.
@@ -590,23 +593,16 @@ class StandardTestRunner(base_runner.BaseTestRunner):
       jobs = options.j
 
       print '>>> Running with test processors'
-      procs = []
-      indicators = progress_indicator.ToProgressIndicatorProcs()
-
-      # TODO(majeski): Implement all indicators and remove this filter.
-      indicators = filter(None, indicators)
-
       loader = LoadProc()
-      procs.append(loader)
-
       results = ResultsTracker(count_subtests=False)
+      indicators = progress_indicator.ToProgressIndicatorProcs()
 
-      procs.append(StatusFileFilterProc(options.slow_tests,
-                                        options.pass_fail_tests))
-
-      procs.append(results)
-
-      procs += indicators
+      procs = [
+        loader,
+        VariantProc(VARIANTS),
+        StatusFileFilterProc(options.slow_tests, options.pass_fail_tests),
+        results,
+      ] + indicators
 
       if context.rerun_failures_count:
         procs.append(RerunProc(

tools/testrunner/testproc/variant.py

+# Copyright 2018 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from . import base
+
+from ..local.variants import ALL_VARIANTS, ALL_VARIANT_FLAGS, FAST_VARIANT_FLAGS
+
+
+FAST_VARIANTS = set(["default", "turbofan"])
+STANDARD_VARIANT = set(["default"])
+
+
+class VariantProc(base.TestProcProducer):
+  """Processor creating variants.
+
+  For each test it keeps generator that returns variant, flags and id suffix.
+  It produces variants one at a time, so it's waiting for the result of one
+  variant to create another variant of the same test.
+  It maintains the order of the variants passed to the init.
+
+  There are some cases when particular variant of the test is not valid. To
+  ignore subtests like that, StatusFileFilterProc should be placed somewhere
+  after the VariantProc.
+  """
+
+  def __init__(self, variants):
+    super(VariantProc, self).__init__('VariantProc')
+    self._next_test_iter = {}
+    self._variant_gens = {}
+    self._variants = variants
+
+  def _next_test(self, test):
+    self._next_test_iter[test.procid] = iter(self._variants_gen(test))
+    return self._try_send_new_subtest(test)
+
+  def _result_for(self, test, subtest, result, is_last):
+    if not is_last:
+      self._send_result(subtest, result, is_last=False)
+      return
+
+    has_sent = self._try_send_new_subtest(test)
+    self._send_result(subtest, result, is_last=not has_sent)
+
+  def _try_send_new_subtest(self, test):
+    # Keep trying until variant is not ignored by the next processors or there
+    # are no more variants to generate.
+    for variant, flags, suffix in self._next_test_iter[test.procid]:
+      subtest = self._create_subtest(test, '%s-%s' % (variant, suffix),
+                                     variant=variant, flags=flags)
+      if self._send_test(subtest):
+        return True
+
+    del self._next_test_iter[test.procid]
+    return False
+
+  def _variants_gen(self, test):
+    """Generator producing (variant, flags, procid suffix) tuples."""
+    return self._get_variants_gen(test).gen(test)
+
+  def _get_variants_gen(self, test):
+    key = test.suite.name
+    variants_gen = self._variant_gens.get(key)
+    if not variants_gen:
+      variants_gen = test.suite.get_variants_gen(self._variants)
+      self._variant_gens[key] = variants_gen
+    return variants_gen

tools/unittests/run_tests_test.py

@@ -210,6 +210,9 @@ class SystemTest(unittest.TestCase):
           'sweet/strawberries',
           infra_staging=infra_staging,
       )
+      if infra_staging:
+        self.assertIn('Running 1 tests', result.stdout, result)
+      else:
         self.assertIn('Running 2 tests', result.stdout, result)
       self.assertIn('Done running sweet/strawberries: FAIL', result.stdout, result)
       self.assertEqual(1, result.returncode, result)