[test] Expose test-framework name in test results

This will allow to distinguish between the standard runner and the num fuzzer
on the infra side when generating flako command lines. The value could be
inferred, but it'd be more confusing.

Bug: v8:8971
Change-Id: I78f5104135d1c7fd7d98bceb4b17897e79421455
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1564050
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Sergiy Belozorov <sergiyb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#60769}

tools/testrunner/base_runner.py

@@ -244,6 +244,11 @@ class BaseTestRunner(object):
     self.mode_options = None
     self.target_os = None
 
+  @property
+  def framework_name(self):
+    """String name of the base-runner subclass, used in test results."""
+    raise NotImplementedError()
+
   def execute(self, sys_args=None):
     if sys_args is None:  # pragma: no cover
       sys_args = sys.argv[1:]
@@ -772,6 +777,7 @@ class BaseTestRunner(object):
     procs = [PROGRESS_INDICATORS[options.progress]()]
     if options.json_test_results:
       procs.append(progress.JsonTestProgressIndicator(
+        self.framework_name,
         options.json_test_results,
         self.build_config.arch,
         self.mode_options.execution_mode))

tools/testrunner/num_fuzzer.py

@@ -33,6 +33,10 @@ class NumFuzzer(base_runner.BaseTestRunner):
   def __init__(self, *args, **kwargs):
     super(NumFuzzer, self).__init__(*args, **kwargs)
 
+  @property
+  def framework_name(self):
+    return 'num_fuzzer'
+
   def _add_parser_options(self, parser):
     parser.add_option("--fuzzer-random-seed", default=0,
                       help="Default seed for initializing fuzzer random "

tools/testrunner/standard_runner.py

@@ -70,6 +70,10 @@ class StandardTestRunner(base_runner.BaseTestRunner):
       self.sancov_dir = None
       self._variants = None
 
+  @property
+  def framework_name(self):
+    return 'standard_runner'
+
   def _get_default_suite_names(self):
     return ['default']
 

tools/testrunner/testproc/progress.py

@@ -259,7 +259,7 @@ class MonochromeProgressIndicator(CompactProgressIndicator):
 
 
 class JsonTestProgressIndicator(ProgressIndicator):
-  def __init__(self, json_test_results, arch, mode):
+  def __init__(self, framework_name, json_test_results, arch, mode):
     super(JsonTestProgressIndicator, self).__init__()
     # We want to drop stdout/err for all passed tests on the first try, but we
     # need to get outputs for all runs after the first one. To accommodate that,
@@ -267,6 +267,7 @@ class JsonTestProgressIndicator(ProgressIndicator):
     # keep_output set to True in the RerunProc.
     self._requirement = base.DROP_PASS_STDOUT
 
+    self.framework_name = framework_name
     self.json_test_results = json_test_results
     self.arch = arch
     self.mode = mode
@@ -308,6 +309,7 @@ class JsonTestProgressIndicator(ProgressIndicator):
         "target_name": test.get_shell(),
         "variant": test.variant,
         "variant_flags": test.variant_flags,
+        "framework_name": self.framework_name,
       })
 
   def finished(self):

tools/unittests/testdata/expected_test_results1.json

@@ -16,6 +16,7 @@
         "--random-seed=123",
         "--nohard-abort"
       ],
+      "framework_name": "standard_runner",
       "name": "sweet/strawberries",
       "random_seed": 123,
       "result": "FAIL",
@@ -39,6 +40,7 @@
         "--random-seed=123",
         "--nohard-abort"
       ],
+      "framework_name": "standard_runner",
       "name": "sweet/strawberries",
       "random_seed": 123,
       "result": "FAIL",
@@ -62,6 +64,7 @@
         "--random-seed=123",
         "--nohard-abort"
       ],
+      "framework_name": "standard_runner",
       "name": "sweet/strawberries",
       "random_seed": 123,
       "result": "FAIL",

tools/unittests/testdata/expected_test_results2.json

@@ -15,6 +15,7 @@
         "--random-seed=123",
         "--nohard-abort"
       ],
+      "framework_name": "standard_runner",
       "name": "sweet/bananaflakes",
       "random_seed": 123,
       "result": "FAIL",
@@ -37,6 +38,7 @@
         "--random-seed=123",
         "--nohard-abort"
       ],
+      "framework_name": "standard_runner",
       "name": "sweet/bananaflakes",
       "random_seed": 123,
       "result": "PASS",