[foozzie] Overhaul --no-lazy-feedback-allocation comparisons

Pass --no-lazy-feedback-allocation in all second runs depending
on a probability. Also combine with --interrupt-budget=100.

This also allows adding several extra flags behind one probability.
The tests are improved to ensure valid flags and configs.

No-Try: true
Bug: v8:10215
Change-Id: I2766ef5044cd8c7096f6b76f39b60b568f550bde
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2059991
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Tamer Tas <tmrts@chromium.org>
Reviewed-by: Jakob Gruber <jgruber@chromium.org>
Reviewed-by: Mythri Alle <mythria@chromium.org>
Cr-Commit-Position: refs/heads/master@{#66289}

tools/clusterfuzz/v8_foozzie.py

@@ -60,13 +60,11 @@ CONFIGS = dict(
   ignition_turbo_opt=[
     '--always-opt',
     '--no-liftoff',
-    '--no-lazy-feedback-allocation'
   ],
   ignition_turbo_opt_eager=[
     '--always-opt',
     '--no-lazy',
     '--no-lazy-inner-functions',
-    '--no-lazy-feedback-allocation',
   ],
   jitless=[
     '--jitless',
@@ -77,7 +75,6 @@ CONFIGS = dict(
   slow_path_opt=[
     '--always-opt',
     '--force-slow-path',
-    '--no-lazy-feedback-allocation',
   ],
   trusted=[
     '--no-untrusted-code-mitigations',
@@ -85,7 +82,6 @@ CONFIGS = dict(
   trusted_opt=[
     '--always-opt',
     '--no-untrusted-code-mitigations',
-    '--no-lazy-feedback-allocation',
   ],
 )
 

tools/clusterfuzz/v8_foozzie_test.py

@@ -18,35 +18,78 @@ BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 FOOZZIE = os.path.join(BASE_DIR, 'v8_foozzie.py')
 TEST_DATA = os.path.join(BASE_DIR, 'testdata')
 
+KNOWN_BUILDS = [
+  'd8',
+  'clang_x86/d8',
+  'clang_x86_v8_arm/d8',
+  'clang_x64_v8_arm64/d8',
+  'clang_x64_pointer_compression/d8',
+]
+
 
 class ConfigTest(unittest.TestCase):
   def testExperiments(self):
-    """Test that probabilities add up to 100 and that all config names exist.
-    """
-    EXPERIMENTS = v8_fuzz_config.FOOZZIE_EXPERIMENTS
+    """Test integrity of probabilities and configs."""
     CONFIGS = v8_foozzie.CONFIGS
+    EXPERIMENTS = v8_fuzz_config.FOOZZIE_EXPERIMENTS
+    FLAGS = v8_fuzz_config.ADDITIONAL_FLAGS
+    # Probabilities add up to 100%.
+    first_is_int = lambda x: type(x[0]) == int
+    assert all(map(first_is_int, EXPERIMENTS))
     assert sum(x[0] for x in EXPERIMENTS) == 100
+    # Configs used in experiments are defined.
     assert all(map(lambda x: x[1] in CONFIGS, EXPERIMENTS))
     assert all(map(lambda x: x[2] in CONFIGS, EXPERIMENTS))
-    assert all(map(lambda x: x[3].endswith('d8'), EXPERIMENTS))
+    # The last config item points to a known build configuration.
+    assert all(map(lambda x: x[3] in KNOWN_BUILDS, EXPERIMENTS))
+    # Ensure we compare different configs and same d8, or same config
+    # to different d8.
+    is_sane_comparison = lambda x: (x[1] == x[2]) == ('d8' != x[3])
+    assert all(map(is_sane_comparison, EXPERIMENTS))
+    # All flags have a probability.
+    first_is_float = lambda x: type(x[0]) == float
+    assert all(map(first_is_float, FLAGS))
+    first_between_0_and_1 = lambda x: x[0] > 0 and x[0] < 1
+    assert all(map(first_between_0_and_1, FLAGS))
+    # Test consistent flags.
+    second_is_string = lambda x: isinstance(x[1], basestring)
+    assert all(map(second_is_string, FLAGS))
+    # We allow spaces to separate more flags. We don't allow spaces in the flag
+    # value.
+    is_flag = lambda x: x.startswith('--')
+    all_parts_are_flags = lambda x: all(map(is_flag, x[1].split()))
+    assert all(map(all_parts_are_flags, FLAGS))
 
   def testConfig(self):
-    """Smoke test how to choose experiments.
-
-    When experiment distribution changes this test might change, too.
-    """
+    """Smoke test how to choose experiments."""
+    config = v8_fuzz_config.Config('foo', random.Random(42))
+    experiments = [
+      [25, 'ignition', 'jitless', 'd8'],
+      [75, 'ignition', 'ignition', 'clang_x86/d8'],
+    ]
+    flags = [
+      [0.1, '--flag'],
+      [0.3, '--baz'],
+      [0.3, '--foo --bar'],
+    ]
+    self.assertEqual(
+        [
+          '--first-config=ignition',
+          '--second-config=jitless',
+          '--second-d8=d8',
+          '--second-config-extra-flags=--baz',
+          '--second-config-extra-flags=--foo',
+          '--second-config-extra-flags=--bar',
+        ],
+        config.choose_foozzie_flags(experiments, flags),
+    )
     self.assertEqual(
         [
           '--first-config=ignition',
-          '--second-config=ignition_turbo_opt',
+          '--second-config=jitless',
           '--second-d8=d8',
-          '--second-config-extra-flags=--stress-scavenge=100',
-          '--second-config-extra-flags=--no-regexp-tier-up',
-          '--second-config-extra-flags=--no-enable-ssse3',
-          '--second-config-extra-flags=--no-enable-bmi2',
-          '--second-config-extra-flags=--no-enable-lzcnt',
         ],
-        v8_fuzz_config.Config('foo', random.Random(42)).choose_foozzie_flags(),
+        config.choose_foozzie_flags(experiments, flags),
     )
 
 

tools/clusterfuzz/v8_fuzz_config.py

@@ -31,22 +31,29 @@ class Config(object):
     self.name = name
     self.rng = rng or random.Random()
 
-  def choose_foozzie_flags(self):
+  def choose_foozzie_flags(self, foozzie_experiments=None, additional_flags=None):
     """Randomly chooses a configuration from FOOZZIE_EXPERIMENTS.
 
+    Args:
+      foozzie_experiments: Override experiment config for testing.
+      additional_flags: Override additional flags for testing.
+
     Returns: List of flags to pass to v8_foozzie.py fuzz harness.
     """
+    foozzie_experiments = foozzie_experiments or FOOZZIE_EXPERIMENTS
+    additional_flags = additional_flags or ADDITIONAL_FLAGS
 
     # Add additional flags to second config based on experiment percentages.
     extra_flags = []
-    for p, flag in ADDITIONAL_FLAGS:
+    for p, flags in additional_flags:
       if self.rng.random() < p:
-        extra_flags.append('--second-config-extra-flags=%s' % flag)
+        for flag in flags.split():
+          extra_flags.append('--second-config-extra-flags=%s' % flag)
 
     # Calculate flags determining the experiment.
     acc = 0
     threshold = self.rng.random() * 100
-    for prob, first_config, second_config, second_d8 in FOOZZIE_EXPERIMENTS:
+    for prob, first_config, second_config, second_d8 in foozzie_experiments:
       acc += prob
       if acc > threshold:
         return [

tools/clusterfuzz/v8_fuzz_flags.json

@@ -27,5 +27,8 @@
   [0.1, "--no-enable-bmi1"],
   [0.1, "--no-enable-bmi2"],
   [0.1, "--no-enable-lzcnt"],
-  [0.1, "--no-enable-popcnt"]
+  [0.1, "--no-enable-popcnt"],
+  [0.25, "--no-lazy-feedback-allocation"],
+  [0.1, "--no-lazy-feedback-allocation --interrupt-budget=100"],
+  [0.05, "--budget-for-feedback-vector-allocation=0"]
 ]
\ No newline at end of file