[test] Let numfuzz tests automatically use all available instances

All --stress-* flags are now automatically tested. This also removes
a superfluous option that was never changed. The default value is
now inlined.

No-Try: true
Bug: v8:13113
Change-Id: If7428b383ed01ff36a93f618badababfc448db26
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3899259Reviewed-by: Alexander Schulze <alexschulze@chromium.org>
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/main@{#83258}

tools/testrunner/num_fuzzer.py

@@ -77,9 +77,6 @@ class NumFuzzer(base_runner.BaseTestRunner):
     parser.add_option("--stress-deopt", default=0, type="int",
                       help="probability [0-10] of adding --deopt-every-n-times "
                            "flag to the test")
-    parser.add_option("--stress-deopt-min", default=1, type="int",
-                      help="extends --stress-deopt to have minimum interval "
-                           "between deopt points")
     parser.add_option("--stress-interrupt-budget", default=0, type="int",
                       help="probability [0-10] of adding the --interrupt-budget "
                            "flag to the test")

tools/testrunner/standard_runner_test.py

@@ -29,6 +29,7 @@ sys.path.append(TOOLS_ROOT)
 from testrunner import standard_runner
 from testrunner import num_fuzzer
 from testrunner.testproc import base
+from testrunner.testproc import fuzzer
 from testrunner.utils.test_utils import (
     temp_base,
     TestRunnerTest,
@@ -538,18 +539,12 @@ class NumFuzzerTest(TestRunnerTest):
     return num_fuzzer.NumFuzzer
 
   def testNumFuzzer(self):
-    # TODO(machenbach): Retrieve these flags automatically from the list of
-    # existing fuzzers.
-    for fuzz_flag in (
-        '--stress-compaction=1',
-        '--stress-delay-tasks=1',
-        '--stress-deopt=1',
-        '--stress-gc=1',
-        '--stress-interrupt-budget=1',
-        '--stress-marking=1',
-        '--stress-scavenge=1',
-        '--stress-stack-size=1',
-        '--stress-thread-pool-size=1'):
+    fuzz_flags = [
+      f'{flag}=1' for flag in self.get_runner_options()
+      if flag.startswith('--stress-')
+    ]
+    self.assertEqual(len(fuzz_flags), len(fuzzer.FUZZERS))
+    for fuzz_flag in fuzz_flags:
       # The fake timeout observer above will stop after proessing the 10th
       # test. This still executes an 11th. Each test causes a test- and a
       # result event internally. We test both paths here.

tools/testrunner/testproc/fuzzer.py

@@ -53,6 +53,9 @@ EXTRA_FLAGS = [
     (0.1, '--turbo-force-mid-tier-regalloc'),
 ]
 
+MIN_DEOPT = 1
+MAX_DEOPT = 10**9
+
 
 def random_extra_flags(rng):
   """Returns a random list of flags chosen from the configurations in
@@ -355,21 +358,15 @@ class ThreadPoolSizeFuzzer(Fuzzer):
 
 
 class DeoptAnalyzer(Analyzer):
-  MAX_DEOPT=1000000000
-
-  def __init__(self, min_interval):
-    super(DeoptAnalyzer, self).__init__()
-    self._min = min_interval
-
   def get_analysis_flags(self):
-    return ['--deopt-every-n-times=%d' % self.MAX_DEOPT,
+    return ['--deopt-every-n-times=%d' % MAX_DEOPT,
             '--print-deopt-stress']
 
   def do_analysis(self, result):
     for line in reversed(result.output.stdout.splitlines()):
       if line.startswith('=== Stress deopt counter: '):
-        counter = self.MAX_DEOPT - int(line.split(' ')[-1])
-        if counter < self._min:
+        counter = MAX_DEOPT - int(line.split(' ')[-1])
+        if counter < MIN_DEOPT:
           # Skip this test since we won't generate any meaningful interval with
           # given minimum.
           return None
@@ -377,17 +374,13 @@ class DeoptAnalyzer(Analyzer):
 
 
 class DeoptFuzzer(Fuzzer):
-  def __init__(self, min_interval):
-    super(DeoptFuzzer, self).__init__()
-    self._min = min_interval
-
   def create_flags_generator(self, rng, test, analysis_value):
     while True:
       if analysis_value:
         value = analysis_value // 2
       else:
         value = 10000
-      interval = rng.randint(self._min, max(value, self._min))
+      interval = rng.randint(MIN_DEOPT, max(value, MIN_DEOPT))
       yield ['--deopt-every-n-times=%d' % interval]
 
 

tools/testrunner/utils/augmented_options.py

@@ -57,9 +57,9 @@ class AugmentedOptions(optparse.Values):
 
   def fuzzer_configs(self):
     fuzzers = []
-    def add(name, prob, *args):
+    def add(name, prob):
       if prob:
-        fuzzers.append(fuzzer.create_fuzzer_config(name, prob, *args))
+        fuzzers.append(fuzzer.create_fuzzer_config(name, prob))
 
     add('compaction', self.stress_compaction)
     add('interrupt', self.stress_interrupt_budget)
@@ -69,7 +69,7 @@ class AugmentedOptions(optparse.Values):
     add('stack', self.stress_stack_size)
     add('threads', self.stress_thread_pool_size)
     add('delay', self.stress_delay_tasks)
-    add('deopt', self.stress_deopt, self.stress_deopt_min)
+    add('deopt', self.stress_deopt)
     return fuzzers
 
   def fuzzer_tests_count(self):

tools/testrunner/utils/test_utils.py

@@ -183,9 +183,16 @@ class TestRunnerTest(unittest.TestCase):
         json_out = clean_json_output(json_out_path, basedir)
         return TestResult(stdout.getvalue(), stderr.getvalue(), code, json_out, self)
 
-    def get_runner_class():
-      """Implement to return the runner class"""
-      return None
+  def get_runner_options(self, baseroot='testroot1'):
+    """Returns a list of all flags parsed by the test runner."""
+    with temp_base(baseroot=baseroot) as basedir:
+      runner = self.get_runner_class()(basedir=basedir)
+      parser = runner._create_parser()
+      return [i.get_opt_string() for i in parser.option_list]
+
+  def get_runner_class():
+    """Implement to return the runner class"""
+    return None
 
 
 class FakeOSContext(DefaultOSContext):