[test] Add new json test results output for flakiness dashboard

BUG=chromium:689108

Change-Id: Id39d247f267131b1abf968546c7555b3657b14f3
Reviewed-on: https://chromium-review.googlesource.com/439566Reviewed-by: Jakob Kummerow <jkummerow@chromium.org>
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#43546}

tools/run-tests.py

@@ -323,6 +323,8 @@ def BuildOptions():
                     default=False, action="store_true")
   result.add_option("--json-test-results",
                     help="Path to a file for storing json results.")
+  result.add_option("--flakiness-results",
+                    help="Path to a file for storing flakiness json.")
   result.add_option("--rerun-failures-count",
                     help=("Number of times to rerun each failing test case. "
                           "Very slow tests will be rerun only once."),
@@ -878,6 +880,9 @@ def Execute(arch, mode, args, options, suites):
     progress_indicator.Register(progress.JsonTestProgressIndicator(
         options.json_test_results, arch, MODES[mode]["execution_mode"],
         ctx.random_seed))
+  if options.flakiness_results:
+    progress_indicator.Register(progress.FlakinessTestProgressIndicator(
+        options.flakiness_results))
 
   run_networked = not options.no_network
   if not run_networked:

tools/testrunner/local/progress.py

@@ -380,6 +380,58 @@ class JsonTestProgressIndicator(ProgressIndicator):
     })
 
 
+class FlakinessTestProgressIndicator(ProgressIndicator):
+
+  def __init__(self, json_test_results):
+    self.json_test_results = json_test_results
+    self.results = {}
+    self.summary = {
+      "PASS": 0,
+      "FAIL": 0,
+      "CRASH": 0,
+      "TIMEOUT": 0,
+    }
+    self.seconds_since_epoch = time.time()
+
+  def Done(self):
+    with open(self.json_test_results, "w") as f:
+      json.dump({
+        "interrupted": False,
+        "num_failures_by_type": self.summary,
+        "path_delimiter": "/",
+        "seconds_since_epoch": self.seconds_since_epoch,
+        "tests": self.results,
+        "version": 3,
+      }, f)
+
+  def HasRun(self, test, has_unexpected_output):
+    key = "/".join(
+        sorted(flag.lstrip("-")
+               for flag in self.runner.context.extra_flags + test.flags) +
+        ["test", test.GetLabel()],
+    )
+    outcome = test.suite.GetOutcome(test)
+    assert outcome in ["PASS", "FAIL", "CRASH", "TIMEOUT"]
+    if test.run == 1:
+      # First run of this test.
+      expected_outcomes = ([
+        expected
+        for expected in (test.outcomes or ["PASS"])
+        if expected in ["PASS", "FAIL", "CRASH", "TIMEOUT"]
+      ] or ["PASS"])
+      self.results[key] = {
+        "actual": outcome,
+        "expected": " ".join(expected_outcomes),
+        "times": [test.duration],
+      }
+      self.summary[outcome] = self.summary[outcome] + 1
+    else:
+      # This is a rerun and a previous result exists.
+      result = self.results[key]
+      result["actual"] = "%s %s" % (result["actual"], outcome)
+      result["times"].append(test.duration)
+
+
 PROGRESS_INDICATORS = {
   'verbose': VerboseProgressIndicator,
   'dots': DotsProgressIndicator,