[test] Don't use --testing-d8-test-runner on NumFuzz

The extra checks done with --testing-d8-test-runner might not hold
when using NumFuzz. This refactors the test runner and allows passing
implementation specific flags. The --testing-d8-test-runner flag is
now only passed in the standard-runner not in the numfuzz
implementation.

Bug: v8:10220
Change-Id: I83cac57a948c98c34f2d84f41d719e0434e25ee7
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2064217
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Mythri Alle <mythria@chromium.org>
Cr-Commit-Position: refs/heads/master@{#66341}

tools/testrunner/base_runner.py

@@ -123,9 +123,8 @@ class ModeConfig(object):
     self.execution_mode = execution_mode
 
 
-DEBUG_FLAGS = ["--nohard-abort", "--enable-slow-asserts", "--verify-heap",
-               "--testing-d8-test-runner"]
-RELEASE_FLAGS = ["--nohard-abort", "--testing-d8-test-runner"]
+DEBUG_FLAGS = ["--nohard-abort", "--enable-slow-asserts", "--verify-heap"]
+RELEASE_FLAGS = ["--nohard-abort"]
 MODES = {
   "debug": ModeConfig(
     flags=DEBUG_FLAGS,
@@ -704,13 +703,17 @@ class BaseTestRunner(object):
       "pointer_compression": self.build_config.pointer_compression,
     }
 
+  def _runner_flags(self):
+    """Extra default flags specific to the test runner implementation."""
+    return []
+
   def _create_test_config(self, options):
     timeout = options.timeout * self._timeout_scalefactor(options)
     return TestConfig(
         command_prefix=options.command_prefix,
         extra_flags=options.extra_flags,
         isolates=options.isolates,
-        mode_flags=self.mode_options.flags,
+        mode_flags=self.mode_options.flags + self._runner_flags(),
         no_harness=options.no_harness,
         noi18n=self.build_config.no_i18n,
         random_seed=options.random_seed,

tools/testrunner/standard_runner.py

@@ -50,6 +50,9 @@ VARIANT_ALIASES = {
             'instruction_scheduling'],
 }
 
+# Extra flags passed to all tests using the standard test runner.
+EXTRA_DEFAULT_FLAGS = ['--testing-d8-test-runner']
+
 GC_STRESS_FLAGS = ['--gc-interval=500', '--stress-compaction',
                    '--concurrent-recompilation-queue-length=64',
                    '--concurrent-recompilation-delay=500',
@@ -239,6 +242,9 @@ class StandardTestRunner(base_runner.BaseTestRunner):
           prefix="v8-test-runner-")
       options.json_test_results = self._temporary_json_output_file.name
 
+  def _runner_flags(self):
+    return EXTRA_DEFAULT_FLAGS
+
   def _parse_variants(self, aliases_str):
     # Use developer defaults if no variant was specified.
     aliases_str = aliases_str or 'dev'

tools/unittests/run_tests_test.py

@@ -665,6 +665,24 @@ class SystemTest(unittest.TestCase):
 
       self.assertEqual(0, result.returncode, result)
 
+  def testRunnerFlags(self):
+    """Test that runner-specific flags are passed to tests."""
+    with temp_base() as basedir:
+      result = run_tests(
+          basedir,
+          '--mode=Release',
+          '--progress=verbose',
+          '--variants=default',
+          '--random-seed=42',
+          'sweet/bananas',
+          '-v',
+      )
+
+      self.assertIn(
+          '--test bananas --random-seed=42 --nohard-abort --testing-d8-test-runner',
+          result.stdout, result)
+      self.assertEqual(0, result.returncode, result)
+
 
 if __name__ == '__main__':
   unittest.main()