Commit efc4b9ba authored by Michael Achenbach's avatar Michael Achenbach Committed by Commit Bot

[test] Fix verify-predictable outcome check

This moves the error-code check for verify-predictable mode to the API
method checking for status file outcomes, overwriting the default
behavior.

This is resembling the behavior prior to:
https://chromium-review.googlesource.com/c/808971/

Otherwise, the status file outcomes will expect some negative tests
to fail in the mozilla test suite, which pass in predictable mode.

Now, negative tests are simply not supported.

Bug: v8:7166
Change-Id: I1d4bcaf66cb54c5fbb217dd9091b88ecc5b0e456
Reviewed-on: https://chromium-review.googlesource.com/817741Reviewed-by: 's avatarSergiy Byelozyorov <sergiyb@chromium.org>
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50110}
parent 78ab62e4
......@@ -282,30 +282,36 @@ class TestSuite(object):
def IsNegativeTest(self, testcase):
return False
def HasFailed(self, testcase, output, ctx=None):
if ctx and ctx.predictable:
# Only check the exit code of the predictable_wrapper in
# verify-predictable mode.
execution_failed = output.exit_code != 0
else:
execution_failed = self.IsFailureOutput(testcase, output)
def HasFailed(self, testcase, output):
execution_failed = self.IsFailureOutput(testcase, output)
if self.IsNegativeTest(testcase):
return not execution_failed
else:
return execution_failed
def GetOutcome(self, testcase, output, ctx=None):
def GetOutcome(self, testcase, output):
if output.HasCrashed():
return statusfile.CRASH
elif output.HasTimedOut():
return statusfile.TIMEOUT
elif self.HasFailed(testcase, output, ctx):
elif self.HasFailed(testcase, output):
return statusfile.FAIL
else:
return statusfile.PASS
def HasUnexpectedOutput(self, testcase, output, ctx=None):
return (self.GetOutcome(testcase, output, ctx)
if ctx and ctx.predictable:
# Only check the exit code of the predictable_wrapper in
# verify-predictable mode. Negative tests are not supported as they
# usually also don't print allocation hashes. There are two versions of
# negative tests: one specified by the test, the other specified through
# the status file (e.g. known bugs).
return (
output.exit_code != 0 and
not self.IsNegativeTest(testcase) and
statusfile.FAIL not in self.GetExpectedOutcomes(testcase)
)
return (self.GetOutcome(testcase, output)
not in self.GetExpectedOutcomes(testcase))
def _create_test(self, path, **kwargs):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment