Add PASS expectations in NumFuzz behind infra-staging

Bug: v8:11826
Cq-Include-Trybots: luci.v8.try:v8_numfuzz_dbg_ng,v8_numfuzz_ng,v8_numfuzz_tsan_ng
Change-Id: I1e77562cd86aa5b3a3a8c51e7bb4d37079b450d8
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3118947
Commit-Queue: Almothana Athamneh <almuthanna@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/main@{#76534}

tools/testrunner/num_fuzzer.py

@@ -20,7 +20,7 @@ from testrunner.testproc import fuzzer
 from testrunner.testproc.base import TestProcProducer
 from testrunner.testproc.combiner import CombinerProc
 from testrunner.testproc.execution import ExecutionProc
-from testrunner.testproc.expectation import ForgiveTimeoutProc
+from testrunner.testproc.expectation import ExpectationProc
 from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc
 from testrunner.testproc.loader import LoadProc
 from testrunner.testproc.progress import ResultsTracker
@@ -163,7 +163,7 @@ class NumFuzzer(base_runner.BaseTestRunner):
       # TODO(majeski): Improve sharding when combiner is present. Maybe select
       # different random seeds for shards instead of splitting tests.
       self._create_shard_proc(options),
-      ForgiveTimeoutProc(),
+      ExpectationProc(self.infra_staging),
       combiner,
       self._create_fuzzer(fuzzer_rng, options),
       sigproc,

tools/testrunner/objects/testcase.py

@@ -166,6 +166,15 @@ class TestCase(object):
       self._expected_outcomes = (
           self.expected_outcomes + [statusfile.TIMEOUT])
 
+  def allow_pass(self):
+    if self.expected_outcomes == outproc.OUTCOMES_TIMEOUT:
+      self._expected_outcomes = outproc.OUTCOMES_PASS_OR_TIMEOUT
+    elif self.expected_outcomes == outproc.OUTCOMES_FAIL:
+      self._expected_outcomes = outproc.OUTCOMES_FAIL_OR_PASS
+    elif statusfile.PASS not in self.expected_outcomes:
+      self._expected_outcomes = (
+          self.expected_outcomes + [statusfile.PASS])
+
   @property
   def expected_outcomes(self):
     def is_flag(maybe_flag):

tools/testrunner/outproc/base.py

@@ -12,8 +12,10 @@ from ..testproc.result import Result
 
 OUTCOMES_PASS = [statusfile.PASS]
 OUTCOMES_FAIL = [statusfile.FAIL]
+OUTCOMES_TIMEOUT = [statusfile.TIMEOUT]
 OUTCOMES_PASS_OR_TIMEOUT = [statusfile.PASS, statusfile.TIMEOUT]
 OUTCOMES_FAIL_OR_TIMEOUT = [statusfile.FAIL, statusfile.TIMEOUT]
+OUTCOMES_FAIL_OR_PASS = [statusfile.FAIL, statusfile.PASS]
 
 
 class BaseOutProc(object):

tools/testrunner/testproc/expectation.py

@@ -7,14 +7,17 @@ from . import base
 from testrunner.local import statusfile
 from testrunner.outproc import base as outproc
 
-class ForgiveTimeoutProc(base.TestProcProducer):
+class ExpectationProc(base.TestProcProducer):
   """Test processor passing tests and results through and forgiving timeouts."""
-  def __init__(self):
-    super(ForgiveTimeoutProc, self).__init__('no-timeout')
+  def __init__(self, infra_staging):
+    super(ExpectationProc, self).__init__('no-timeout')
+    self.infra_staging = infra_staging
 
   def _next_test(self, test):
     subtest = self._create_subtest(test, 'no_timeout')
     subtest.allow_timeouts()
+    if self.infra_staging:
+      subtest.allow_pass()
     return self._send_test(subtest)
 
   def _result_for(self, test, subtest, result):