[test] testrunner: Print diff if stdout does not match

This extends the ExpectedOutProc runner to print a diff between the
expected and actual output. The behavior of other runners is unchanged.

Change-Id: If2b89d39cf98b8d257b1a209b5471a79ec3868ff
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3771641Reviewed-by: Jakob Kummerow <jkummerow@chromium.org>
Reviewed-by: Liviu Rau <liviurau@chromium.org>
Commit-Queue: Clemens Backes <clemensb@chromium.org>
Auto-Submit: Matthias Liedtke <mliedtke@google.com>
Reviewed-by: Clemens Backes <clemensb@chromium.org>
Cr-Commit-Position: refs/heads/main@{#81966}

tools/testrunner/outproc/base.py

@@ -9,6 +9,7 @@ from ..testproc.base import (
 from ..local import statusfile
 from ..testproc.result import Result
 
+import difflib
 
 OUTCOMES_PASS = [statusfile.PASS]
 OUTCOMES_FAIL = [statusfile.FAIL]
@@ -38,15 +39,17 @@ class BaseOutProc(object):
     """
     if reduction == DROP_RESULT:
       return None
+    error_details = \
+      self._get_error_details(output) if has_unexpected_output else None
     if reduction == DROP_OUTPUT:
-      return Result(has_unexpected_output, None)
+      return Result(has_unexpected_output, None, error_details=error_details)
     if not has_unexpected_output:
       if reduction == DROP_PASS_OUTPUT:
         return Result(has_unexpected_output, None)
       if reduction == DROP_PASS_STDOUT:
         return Result(has_unexpected_output, output.without_text())
 
-    return Result(has_unexpected_output, output)
+    return Result(has_unexpected_output, output, error_details=error_details)
 
   def get_outcome(self, output):
     if output.HasCrashed():
@@ -67,6 +70,9 @@ class BaseOutProc(object):
   def _is_failure_output(self, output):
     return output.exit_code != 0
 
+  def _get_error_details(self, output):
+    return None
+
   @property
   def negative(self):
     return False
@@ -163,6 +169,16 @@ class ExpectedOutProc(OutProc):
       for _, line in enumerate(lines):
         f.write(line+'\n')
 
+  def _get_error_details(self, output):
+    """Return diff between expected and actual output."""
+    expected = open(self._expected_filename, 'r', encoding='utf-8').readlines()
+    actual = output.stdout.splitlines(True)
+    if expected == actual:
+      return None
+    lines = difflib.unified_diff(
+        expected, actual, fromfile=self._expected_filename, tofile='<actual>')
+    return 'Output does not match expectation:\n' + ''.join(lines)
+
   def _act_block_iterator(self, output):
     """Iterates over blocks of actual output lines."""
     lines = output.stdout.splitlines()

tools/testrunner/standard_runner_test.py

@@ -420,6 +420,18 @@ class StandardRunnerTest(TestRunnerTest):
     result.stdout_includes('sweet/bananas')
     result.has_returncode(1)
 
+  def testCompactErrorDetails(self):
+    result = self.run_tests(
+        '--progress=mono',
+        'sweet/cherries',
+        'sweet/strawberries',
+        infra_staging=False,
+    )
+    result.stdout_includes('sweet/cherries')
+    result.stdout_includes('sweet/strawberries')
+    result.stdout_includes('+Mock diff')
+    result.has_returncode(1)
+
   def testExitAfterNFailures(self):
     result = self.run_tests(
         '--progress=verbose',

tools/testrunner/testdata/expected_test_results1.json

@@ -4,6 +4,7 @@
     {
       "command": "/usr/bin/python out/build/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner", 
       "duration": 1, 
+      "error_details": "+Mock diff",
       "exit_code": 1, 
       "expected": [
         "PASS"
@@ -29,6 +30,7 @@
     {
       "command": "/usr/bin/python out/build/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner", 
       "duration": 1, 
+      "error_details": "+Mock diff",
       "exit_code": 1, 
       "expected": [
         "PASS"
@@ -54,6 +56,7 @@
     {
       "command": "/usr/bin/python out/build/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner", 
       "duration": 1, 
+      "error_details": "+Mock diff",
       "exit_code": 1, 
       "expected": [
         "PASS"

tools/testrunner/testdata/expected_test_results2.json

@@ -4,6 +4,7 @@
     {
       "command": "/usr/bin/python out/build/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner", 
       "duration": 1, 
+      "error_details": null,
       "exit_code": 1, 
       "expected": [
         "PASS"
@@ -28,6 +29,7 @@
     {
       "command": "/usr/bin/python out/build/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner", 
       "duration": 1, 
+      "error_details": null,
       "exit_code": 0, 
       "expected": [
         "PASS"

tools/testrunner/testdata/testroot1/test/sweet/testcfg.py

@@ -8,6 +8,8 @@ Dummy test suite extension with some fruity tests.
 
 from testrunner.local import testsuite
 from testrunner.objects import testcase
+from testrunner.outproc.base import OutProc
+
 
 class TestLoader(testsuite.TestLoader):
   def _list_test_filenames(self):
@@ -25,9 +27,25 @@ class TestSuite(testsuite.TestSuite):
     return TestCase
 
 
+class MockOutProc(OutProc):
+
+  def __init__(self, expected_outcomes):
+    OutProc.__init__(self, expected_outcomes)
+
+  def _get_error_details(self, output):
+    return "+Mock diff"
+
+
 class TestCase(testcase.D8TestCase):
   def get_shell(self):
     return 'd8_mocked.py'
 
   def _get_files_params(self):
     return [self.name]
+
+  @property
+  def output_proc(self):
+    if self.name == 'strawberries':
+      return MockOutProc([])
+    else:
+      return super(TestCase, self).output_proc

tools/testrunner/testproc/indicators.py

@@ -239,6 +239,8 @@ class CompactProgressIndicator(ProgressIndicator):
         self.printFormatted('stdout', stdout)
       if len(stderr):
         self.printFormatted('stderr', stderr)
+      if result.error_details:
+        self.printFormatted('failure', result.error_details)
       self.printFormatted('command',
                           "Command: %s" % result.cmd.to_string(relative=True))
       if output.HasCrashed():
@@ -374,6 +376,7 @@ class JsonTestProgressIndicator(ProgressIndicator):
           "result": test.output_proc.get_outcome(output),
           "stdout": output.stdout,
           "stderr": output.stderr,
+          "error_details": result.error_details,
       })
       self.results.append(record)
 

tools/testrunner/testproc/result.py

@@ -20,10 +20,15 @@ class ResultBase(object):
 class Result(ResultBase):
   """Result created by the output processor."""
 
-  def __init__(self, has_unexpected_output, output, cmd=None):
+  def __init__(self,
+               has_unexpected_output,
+               output,
+               cmd=None,
+               error_details=None):
     self.has_unexpected_output = has_unexpected_output
     self.output = output
     self.cmd = cmd
+    self.error_details = error_details
 
   def status(self):
     if self.has_unexpected_output: