[test] Forgive tests timing out on num-fuzzers

Timeouts on num-fuzzer only lead to false positives, as tests might
take unpredictably longer.

This CL forgives timeouts through a global override mechanism of the
expected outcomes. This allows to remove already existing scattered
code that allowed timeouts in some test suites only.

Bug: v8:6917
Change-Id: Ib131765d360e335789c1952bc6793ed051e016ea
Reviewed-on: https://chromium-review.googlesource.com/908454
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Sergiy Byelozyorov <sergiyb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#51188}

test/benchmarks/testcfg.py

@@ -104,9 +104,6 @@ class TestSuite(testsuite.TestSuite):
   def _test_class(self):
     return TestCase
 
-  def _suppressed_test_class(self):
-    return SuppressedTimeoutTestCase
-
 
 class TestCase(testcase.TestCase):
   def _get_files_params(self):
@@ -136,12 +133,5 @@ class TestCase(testcase.TestCase):
     return os.path.join(self.suite.testroot, self.path + self._get_suffix())
 
 
-class SuppressedTimeoutTestCase(TestCase):
-  """The same as a standard test case allowing timeouts."""
-  def _prepare_outcomes(self, *args, **kwargs):
-    super(SuppressedTimeoutTestCase, self)._prepare_outcomes(*args, **kwargs)
-    self.expected_outcomes = self.expected_outcomes + [statusfile.TIMEOUT]
-
-
 def GetSuite(*args, **kwargs):
   return TestSuite(*args, **kwargs)

test/mjsunit/testcfg.py

@@ -297,8 +297,6 @@ class SuppressedTestCase(TestCase):
     # can't differentiate that.
     if statusfile.FAIL in self._statusfile_outcomes:
       self._statusfile_outcomes = [statusfile.SKIP]
-    else:
-      self.expected_outcomes = self.expected_outcomes + [statusfile.TIMEOUT]
 
   def _get_extra_flags(self, *args, **kwargs):
     return (

tools/testrunner/num_fuzzer.py

@@ -17,6 +17,7 @@ from testrunner.testproc import fuzzer
 from testrunner.testproc.base import TestProcProducer
 from testrunner.testproc.combiner import CombinerProc
 from testrunner.testproc.execution import ExecutionProc
+from testrunner.testproc.expectation import ForgiveTimeoutProc
 from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc
 from testrunner.testproc.loader import LoadProc
 from testrunner.testproc.progress import ResultsTracker, TestsCounter
@@ -135,6 +136,7 @@ class NumFuzzer(base_runner.BaseTestRunner):
       # TODO(majeski): Improve sharding when combiner is present. Maybe select
       # different random seeds for shards instead of splitting tests.
       self._create_shard_proc(options),
+      ForgiveTimeoutProc(),
       combiner,
       self._create_fuzzer(fuzzer_rng, options),
       self._create_signal_proc(),

tools/testrunner/outproc/base.py

@@ -12,9 +12,8 @@ from ..testproc.result import Result
 
 OUTCOMES_PASS = [statusfile.PASS]
 OUTCOMES_FAIL = [statusfile.FAIL]
-
-# We tolerate timeout with combined tests.
 OUTCOMES_PASS_OR_TIMEOUT = [statusfile.PASS, statusfile.TIMEOUT]
+OUTCOMES_FAIL_OR_TIMEOUT = [statusfile.FAIL, statusfile.TIMEOUT]
 
 
 class BaseOutProc(object):

tools/testrunner/testproc/expectation.py

+# Copyright 2018 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+from . import base
+
+from testrunner.local import statusfile
+from testrunner.outproc import base as outproc
+
+class ForgiveTimeoutProc(base.TestProcProducer):
+  """Test processor passing tests and results through and forgiving timeouts."""
+  def __init__(self):
+    super(ForgiveTimeoutProc, self).__init__('no-timeout')
+
+  def _next_test(self, test):
+    subtest = self._create_subtest(test, 'no_timeout')
+    if subtest.expected_outcomes == outproc.OUTCOMES_PASS:
+      subtest.expected_outcomes = outproc.OUTCOMES_PASS_OR_TIMEOUT
+    elif subtest.expected_outcomes == outproc.OUTCOMES_FAIL:
+      subtest.expected_outcomes = outproc.OUTCOMES_FAIL_OR_TIMEOUT
+    elif statusfile.TIMEOUT not in subtest.expected_outcomes:
+      subtest.expected_outcomes = (
+          subtest.expected_outcomes + [statusfile.TIMEOUT])
+    self._send_test(subtest)
+
+  def _result_for(self, test, subtest, result):
+    self._send_result(test, result)