[test] Add combine tests option to numfuzz

Bug: v8:6917
Change-Id: I3ba4ca3df8bac400c248fa16c58fcba3497da806
Reviewed-on: https://chromium-review.googlesource.com/881167
Commit-Queue: Michał Majewski <majeski@google.com>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50871}

test/mjsunit/testcfg.py

@@ -30,6 +30,7 @@ import re
 
 from testrunner.local import testsuite
 from testrunner.objects import testcase
+from testrunner.outproc import base as outproc
 
 FILES_PATTERN = re.compile(r"//\s+Files:(.*)")
 ENV_PATTERN = re.compile(r"//\s+Environment Variables:(.*)")
@@ -55,6 +56,9 @@ class TestSuite(testsuite.TestSuite):
           tests.append(test)
     return tests
 
+  def _test_combiner_class(self):
+    return TestCombiner
+
   def _test_class(self):
     return TestCase
 
@@ -128,5 +132,81 @@ class TestCase(testcase.TestCase):
     return os.path.join(self.suite.root, self.path + self._get_suffix())
 
 
+class TestCombiner(testsuite.TestCombiner):
+  def get_group_key(self, test):
+    """Combine tests with the same set of flags.
+    Ignore:
+    1. Some special cases where it's not obvious what to pass in the command.
+    2. Tests with flags that can cause failure even inside try-catch wrapper.
+    3. Tests that use async functions. Async functions can be scheduled after
+      exiting from try-catch wrapper and cause failure.
+    """
+    if (len(test._files_suffix) > 1 or
+        test._env or
+        not test._mjsunit_files or
+        test._source_files):
+      return None
+
+    source_flags = test._get_source_flags()
+    if ('--expose-trigger-failure' in source_flags or
+        '--throws' in source_flags):
+      return None
+
+    source_code = test.get_source()
+    # Maybe we could just update the tests to await all async functions they
+    # call?
+    if 'async' in source_code:
+      return None
+
+    # TODO(majeski): Investigate if we can maybe ignore the flags while
+    # grouping.
+    return str(sorted(list(set(source_flags + test._get_statusfile_flags()))))
+
+  def _combined_test_class(self):
+    return CombinedTest
+
+
+class CombinedTest(testcase.TestCase):
+  """Behaves like normal mjsunit tests except:
+    1. Expected outcome is always PASS
+    2. Instead of one file there is a try-catch wrapper with all combined tests
+      passed as arguments.
+  """
+  def __init__(self, name, tests):
+    super(CombinedTest, self).__init__(tests[0].suite, '', name)
+    self._tests = tests
+
+  def _prepare_outcomes(self, force_update=True):
+    self._statusfile_outcomes = outproc.OUTCOMES_PASS
+    self.expected_outcomes = outproc.OUTCOMES_PASS
+
+  def _get_shell_with_flags(self, ctx):
+    """In addition to standard set of shell flags it appends:
+      --disable-abortjs: %AbortJS can abort the test even inside
+        trycatch-wrapper, so we disable it.
+      --quiet-load: suppress any stdout from load() function used by
+        trycatch-wrapper.
+    """
+    shell = 'd8'
+    shell_flags = ['--test', '--disable-abortjs', '--quiet-load']
+    if ctx.random_seed:
+      shell_flags.append('--random-seed=%s' % ctx.random_seed)
+    return shell, shell_flags
+
+  def _get_cmd_params(self, ctx):
+    return (
+      super(CombinedTest, self)._get_cmd_params(ctx) +
+      self._tests[0]._mjsunit_files +
+      ['tools/testrunner/trycatch_loader.js', '--'] +
+      [t._files_suffix[0] for t in self._tests]
+    )
+
+  def _get_source_flags(self):
+    return self._tests[0]._get_source_flags()
+
+  def _get_statusfile_flags(self):
+    return self._tests[0]._get_statusfile_flags()
+
+
 def GetSuite(name, root):
   return TestSuite(name, root)

tools/testrunner/local/testsuite.py

@@ -80,6 +80,24 @@ class VariantsGenerator(object):
     return self._all_variants
 
 
+class TestCombiner(object):
+  def get_group_key(self, test):
+    """To indicate what tests can be combined with each other we define a group
+    key for each test. Tests with the same group key can be combined. Test
+    without a group key (None) is not combinable with any other test.
+    """
+    raise NotImplementedError()
+
+  def combine(self, name, tests):
+    """Returns test combined from `tests`. Since we identify tests by their
+    suite and name, `name` parameter should be unique within one suite.
+    """
+    return self._combined_test_class()(name, tests)
+
+  def _combined_test_class(self):
+    raise NotImplementedError()
+
+
 class TestSuite(object):
   @staticmethod
   def LoadTestSuite(root):
@@ -126,6 +144,21 @@ class TestSuite(object):
   def _variants_gen_class(self):
     return VariantsGenerator
 
+  def test_combiner_available(self):
+    return bool(self._test_combiner_class())
+
+  def get_test_combiner(self):
+    cls = self._test_combiner_class()
+    if cls:
+      return cls()
+    return None
+
+  def _test_combiner_class(self):
+    """Returns Combiner subclass. None if suite doesn't support combining
+    tests.
+    """
+    return None
+
   def ReadStatusFile(self, variables):
     self.statusfile = statusfile.StatusFile(self.status_file(), variables)
 

tools/testrunner/num_fuzzer.py

@@ -19,6 +19,7 @@ from testrunner.objects import context
 
 from testrunner.testproc import fuzzer
 from testrunner.testproc.base import TestProcProducer
+from testrunner.testproc.combiner import CombinerProc
 from testrunner.testproc.execution import ExecutionProc
 from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc
 from testrunner.testproc.loader import LoadProc
@@ -75,6 +76,15 @@ class NumFuzzer(base_runner.BaseTestRunner):
     parser.add_option("--swarming",
                       help="Indicates running test driver on swarming.",
                       default=False, action="store_true")
+    parser.add_option("--tests-count", default=5, type="int",
+                      help="Number of tests to generate from each base test. "
+                           "Can be combined with --total-timeout-sec with "
+                           "value 0 to provide infinite number of subtests. "
+                           "When --combine-tests is set it indicates how many "
+                           "tests to create in total")
+    parser.add_option("--total-timeout-sec", default=0, type="int",
+                      help="How long should fuzzer run. It overrides "
+                           "--tests-count")
 
     # Stress gc
     parser.add_option("--stress-marking", default=0, type="int",
@@ -98,23 +108,19 @@ class NumFuzzer(base_runner.BaseTestRunner):
                       help="extends --stress-deopt to have minimum interval "
                            "between deopt points")
 
-    parser.add_option("--tests-count", default=5, type="int",
-                      help="Number of tests to generate from each base test. "
-                           "Can be combined with --total-timeout-sec with "
-                           "value 0 to provide infinite number of subtests.")
-    parser.add_option("--total-timeout-sec", default=0, type="int",
-                      help="How long should fuzzer run")
-
     # Combine multiple tests
     parser.add_option("--combine-tests", default=False, action="store_true",
                       help="Combine multiple tests as one and run with "
                            "try-catch wrapper")
+    parser.add_option("--combine-max", default=100, type="int",
+                      help="Maximum number of tests to combine")
+    parser.add_option("--combine-min", default=2, type="int",
+                      help="Minimum number of tests to combine")
 
     return parser
 
 
   def _process_options(self, options):
-    # Special processing of other options, sorted alphabetically.
     options.command_prefix = shlex.split(options.command_prefix)
     options.extra_flags = shlex.split(options.extra_flags)
     if options.j == 0:
@@ -129,6 +135,12 @@ class NumFuzzer(base_runner.BaseTestRunner):
     if options.total_timeout_sec:
       options.tests_count = 0
 
+    if options.combine_tests:
+      if options.combine_min > options.combine_max:
+        print ('min_group_size (%d) cannot be larger than max_group_size (%d)' %
+               options.min_group_size, options.max_group_size)
+        raise base_runner.TestRunnerError()
+
     return True
 
   def _get_default_suite_names(self):
@@ -152,14 +164,8 @@ class NumFuzzer(base_runner.BaseTestRunner):
 
     loader = LoadProc()
     fuzzer_rng = random.Random(options.fuzzer_random_seed)
-    fuzzer_proc = fuzzer.FuzzerProc(
-        fuzzer_rng,
-        options.tests_count,
-        self._create_fuzzer_configs(options),
-        options.total_timeout_sec,
-        disable_analysis=options.combine_tests,
-    )
 
+    combiner = self._create_combiner(fuzzer_rng, options)
     results = ResultsTracker()
     execproc = ExecutionProc(options.j, ctx)
     indicators = progress_indicator.ToProgressIndicatorProcs()
@@ -167,8 +173,11 @@ class NumFuzzer(base_runner.BaseTestRunner):
       loader,
       NameFilterProc(args) if args else None,
       StatusFileFilterProc(None, None),
+      # TODO(majeski): Improve sharding when combiner is present. Maybe select
+      # different random seeds for shards instead of splitting tests.
       self._create_shard_proc(options),
-      fuzzer_proc,
+      combiner,
+      self._create_fuzzer(fuzzer_rng, options)
     ] + indicators + [
       results,
       self._create_timeout_proc(options),
@@ -177,6 +186,10 @@ class NumFuzzer(base_runner.BaseTestRunner):
     ]
     self._prepare_procs(procs)
     loader.load_tests(tests)
+
+    # TODO(majeski): maybe some notification from loader would be better?
+    if combiner:
+      combiner.generate_initial_tests(options.j * 4)
     execproc.start()
 
     for indicator in indicators:
@@ -221,6 +234,9 @@ class NumFuzzer(base_runner.BaseTestRunner):
     return ctx
 
   def _load_tests(self, options, suites, ctx):
+    if options.combine_tests:
+      suites = [s for s in suites if s.test_combiner_available()]
+
     # Find available test suites and read test cases from them.
     deopt_fuzzer = bool(options.stress_deopt)
     gc_stress = bool(options.stress_gc)
@@ -266,6 +282,26 @@ class NumFuzzer(base_runner.BaseTestRunner):
       procs[i].connect_to(procs[i + 1])
     procs[0].setup()
 
+  def _create_combiner(self, rng, options):
+    if not options.combine_tests:
+      return None
+    return CombinerProc(rng, options.combine_min, options.combine_max,
+                        options.tests_count)
+
+  def _create_fuzzer(self, rng, options):
+    if options.combine_tests:
+      count = 1
+      disable_analysis = True
+    else:
+      count = options.tests_count
+      disable_analysis = False
+    return fuzzer.FuzzerProc(
+        rng,
+        count,
+        self._create_fuzzer_configs(options),
+        disable_analysis,
+    )
+
   def _create_fuzzer_configs(self, options):
     fuzzers = []
     def add(name, prob, *args):

tools/testrunner/testproc/combiner.py

+# Copyright 2018 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from collections import defaultdict
+import time
+
+from . import base
+from ..objects import testcase
+from ..outproc import base as outproc
+
+
+class CombinerProc(base.TestProc):
+  def __init__(self, rng, min_group_size, max_group_size, count):
+    """
+    Args:
+      rng: random number generator
+      min_group_size: minimum number of tests to combine
+      max_group_size: maximum number of tests to combine
+      count: how many tests to generate. 0 means infinite running
+    """
+    super(CombinerProc, self).__init__()
+
+    self._rng = rng
+    self._min_size = min_group_size
+    self._max_size = max_group_size
+    self._count = count
+
+    # Index of the last generated test
+    self._current_num = 0
+
+    # {suite name: instance of TestGroups}
+    self._groups = defaultdict(TestGroups)
+
+    # {suite name: instance of TestCombiner}
+    self._combiners = {}
+
+  def setup(self, requirement=base.DROP_RESULT):
+    # Combiner is not able to pass results (even as None) to the previous
+    # processor.
+    assert requirement == base.DROP_RESULT
+    self._next_proc.setup(base.DROP_RESULT)
+
+  def next_test(self, test):
+    group_key = self._get_group_key(test)
+    if not group_key:
+      # Test not suitable for combining
+      return
+
+    self._groups[test.suite.name].add_test(group_key, test)
+
+  def _get_group_key(self, test):
+    combiner =  self._get_combiner(test.suite)
+    if not combiner:
+      print ('>>> Warning: There is no combiner for %s testsuite' %
+             test.suite.name)
+      return None
+    return combiner.get_group_key(test)
+
+  def result_for(self, test, result):
+    self._send_next_test()
+
+  def generate_initial_tests(self, num=1):
+    for _ in xrange(0, num):
+      self._send_next_test()
+
+  def _send_next_test(self):
+    if self.is_stopped:
+      return
+
+    if self._count and self._current_num >= self._count:
+      return
+
+    combined_test = self._create_new_test()
+    if not combined_test:
+      # Not enough tests
+      return
+
+    self._send_test(combined_test)
+
+  def _create_new_test(self):
+    suite, combiner = self._select_suite()
+    groups = self._groups[suite]
+
+    max_size = self._rng.randint(self._min_size, self._max_size)
+    sample = groups.sample(self._rng, max_size)
+    if not sample:
+      return None
+
+    self._current_num += 1
+    return combiner.combine('%s-%d' % (suite, self._current_num), sample)
+
+  def _select_suite(self):
+    """Returns pair (suite name, combiner)."""
+    selected = self._rng.randint(0, len(self._groups) - 1)
+    for n, suite in enumerate(self._groups):
+      if n == selected:
+        return suite, self._combiners[suite]
+
+  def _get_combiner(self, suite):
+    combiner = self._combiners.get(suite.name)
+    if not combiner:
+      combiner = suite.get_test_combiner()
+      self._combiners[suite.name] = combiner
+    return combiner
+
+
+class TestGroups(object):
+  def __init__(self):
+    self._groups = defaultdict(list)
+    self._keys = []
+
+  def add_test(self, key, test):
+    self._groups[key].append(test)
+    self._keys.append(key)
+
+  def sample(self, rng, max_size):
+    # Not enough tests
+    if not self._groups:
+      return None
+
+    group_key = rng.choice(self._keys)
+    tests = self._groups[group_key]
+    return [rng.choice(tests) for _ in xrange(0, max_size)]

tools/testrunner/testproc/execution.py

@@ -37,6 +37,8 @@ class Job(object):
     return JobResult(self.test_id, result)
 
 
+# TODO(majeski): Stop workers when processor is stopped. It will also require
+# to call stop both directions from TimeoutProc.
 class ExecutionProc(base.TestProc):
   """Last processor in the chain. Instead of passing tests further it creates
   commands and output processors, executes them in multiple worker processes and

tools/testrunner/testproc/fuzzer.py

@@ -45,14 +45,12 @@ class Fuzzer(object):
 
 # TODO(majeski): Allow multiple subtests to run at once.
 class FuzzerProc(base.TestProcProducer):
-  def __init__(self, rng, count, fuzzers, fuzz_duration_sec=0,
-               disable_analysis=False):
+  def __init__(self, rng, count, fuzzers, disable_analysis=False):
     """
     Args:
       rng: random number generator used to select flags and values for them
       count: number of tests to generate based on each base test
       fuzzers: list of FuzzerConfig instances
-      fuzz_duration_sec: how long it should run, overrides count
       disable_analysis: disable analysis phase and filtering base on it. When
         set, processor passes None as analysis result to fuzzers
     """
@@ -61,7 +59,6 @@ class FuzzerProc(base.TestProcProducer):
     self._rng = rng
     self._count = count
     self._fuzzer_configs = fuzzers
-    self._fuzz_duration_sec = fuzz_duration_sec
     self._disable_analysis = disable_analysis
     self._gens = {}
 

tools/testrunner/trycatch_loader.js

+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+
+// Wrapper loading javascript tests passed as arguments used by gc fuzzer.
+// It ignores all exceptions and run tests in a separate namespaces.
+//
+// It can't prevent %AbortJS function from aborting execution, so it should be
+// used with d8's --disable-abortjs flag to ignore all possible errors inside
+// tests.
+for (let jstest of arguments) {
+  print("Loading " + jstest);
+
+  // anonymous function to not populate global namespace.
+  (function () {
+    try {
+      load(jstest);
+    } catch (err) {
+      // ignore all errors
+    }
+  })();
+}