Commit 1f604666 authored by Michal Majewski's avatar Michal Majewski Committed by Commit Bot

[test] Store outcomes in the testcase

Bug: v8:6917
Cq-Include-Trybots: master.tryserver.v8:v8_linux_noi18n_rel_ng
Change-Id: Ia52d4bedbeff5b93915ef69a2dc78f6d92669061
Reviewed-on: https://chromium-review.googlesource.com/832467
Commit-Queue: Michał Majewski <majeski@google.com>
Reviewed-by: 's avatarMichael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50200}
parent 8ae67cf1
......@@ -39,8 +39,7 @@ class VariantGenerator(testsuite.VariantGenerator):
# always opt to match the way the benchmarks are run for performance
# testing.
def FilterVariantsByTest(self, test):
outcomes = self.suite.GetStatusFileOutcomes(test.name, test.variant)
if statusfile.OnlyStandardVariant(outcomes):
if test.only_standard_variant:
return self.standard_variant
return self.fast_variants
......
......@@ -104,8 +104,7 @@ FAST_VARIANTS = {
class VariantGenerator(testsuite.VariantGenerator):
def GetFlagSets(self, test, variant):
outcomes = test.suite.GetStatusFileOutcomes(test.name, test.variant)
if outcomes and statusfile.OnlyFastVariants(outcomes):
if test.only_fast_variants:
variant_flags = FAST_VARIANTS
else:
variant_flags = ALL_VARIANTS
......
......@@ -128,9 +128,7 @@ class Runner(object):
self.suite_names = [s.name for s in suites]
# Always pre-sort by status file, slowest tests first.
slow_key = lambda t: statusfile.IsSlow(
t.suite.GetStatusFileOutcomes(t.name, t.variant))
self.tests.sort(key=slow_key, reverse=True)
self.tests.sort(key=lambda t: t.is_slow, reverse=True)
# Sort by stored duration if not opted out.
if not context.no_sorting:
......
......@@ -324,8 +324,7 @@ class JsonTestProgressIndicator(ProgressIndicator):
"flags": test.cmd.args,
"command": test.cmd.to_string(relative=True),
"duration": duration,
"marked_slow": statusfile.IsSlow(
test.suite.GetStatusFileOutcomes(test.name, test.variant)),
"marked_slow": test.is_slow,
} for (test, duration) in self.tests[:20]
]
......
......@@ -68,32 +68,6 @@ for var in ALL_VARIANTS:
VARIABLES[var] = var
def DoSkip(outcomes):
return SKIP in outcomes
def IsSlow(outcomes):
return SLOW in outcomes
def OnlyStandardVariant(outcomes):
return NO_VARIANTS in outcomes
def OnlyFastVariants(outcomes):
return FAST_VARIANTS in outcomes
def IsPassOrFail(outcomes):
return (PASS in outcomes and
FAIL in outcomes and
CRASH not in outcomes)
def IsFailOk(outcomes):
return FAIL_OK in outcomes
def _JoinsPassAndFail(outcomes1, outcomes2):
"""Indicates if we join PASS and FAIL from two different outcome sets and
the first doesn't already contain both.
......
......@@ -49,18 +49,14 @@ class VariantGenerator(object):
self.standard_variant = STANDARD_VARIANT & variants
def FilterVariantsByTest(self, test):
result = self.all_variants
outcomes = test.suite.GetStatusFileOutcomes(test.name, test.variant)
if outcomes:
if statusfile.OnlyStandardVariant(outcomes):
return self.standard_variant
if statusfile.OnlyFastVariants(outcomes):
result = self.fast_variants
return result
if test.only_standard_variant:
return self.standard_variant
if test.only_fast_variants:
return self.fast_variants
return self.all_variants
def GetFlagSets(self, test, variant):
outcomes = test.suite.GetStatusFileOutcomes(test.name, test.variant)
if outcomes and statusfile.OnlyFastVariants(outcomes):
if test.only_fast_variants:
return FAST_VARIANT_FLAGS[variant]
else:
return ALL_VARIANT_FLAGS[variant]
......@@ -87,8 +83,6 @@ class TestSuite(object):
self.rules = None # {variant: {test name: [rule]}}
self.prefix_rules = None # {variant: {test name prefix: [rule]}}
self._outcomes_cache = dict()
def status_file(self):
return "%s/%s.status" % (self.root, self.name)
......@@ -145,13 +139,11 @@ class TestSuite(object):
(mode == 'skip' and pass_fail))
def _compliant(test):
outcomes = self.GetStatusFileOutcomes(test.name, test.variant)
if statusfile.DoSkip(outcomes):
if test.do_skip:
return False
if _skip_slow(statusfile.IsSlow(outcomes), slow_tests_mode):
if _skip_slow(test.is_slow, slow_tests_mode):
return False
if _skip_pass_fail(statusfile.IsPassOrFail(outcomes),
pass_fail_tests_mode):
if _skip_pass_fail(test.is_pass_or_fail, pass_fail_tests_mode):
return False
return True
......@@ -177,13 +169,13 @@ class TestSuite(object):
if test.name in self.rules.get(variant, {}):
used_rules.add((test.name, variant))
if statusfile.DoSkip(self.rules[variant][test.name]):
if statusfile.SKIP in self.rules[variant][test.name]:
continue
for prefix in self.prefix_rules.get(variant, {}):
if test.name.startswith(prefix):
used_rules.add((prefix, variant))
if statusfile.DoSkip(self.prefix_rules[variant][prefix]):
if statusfile.SKIP in self.prefix_rules[variant][prefix]:
break
for variant in variants:
......@@ -229,29 +221,25 @@ class TestSuite(object):
"""
variant = variant or ''
cache_key = '%s$%s' % (testname, variant)
if cache_key not in self._outcomes_cache:
# Load statusfile to get outcomes for the first time.
assert(self.rules is not None)
assert(self.prefix_rules is not None)
outcomes = frozenset()
# Load statusfile to get outcomes for the first time.
assert(self.rules is not None)
assert(self.prefix_rules is not None)
for key in set([variant, '']):
rules = self.rules.get(key, {})
prefix_rules = self.prefix_rules.get(key, {})
outcomes = frozenset()
if testname in rules:
outcomes |= rules[testname]
for key in set([variant, '']):
rules = self.rules.get(key, {})
prefix_rules = self.prefix_rules.get(key, {})
for prefix in prefix_rules:
if testname.startswith(prefix):
outcomes |= prefix_rules[prefix]
if testname in rules:
outcomes |= rules[testname]
self._outcomes_cache[cache_key] = outcomes
for prefix in prefix_rules:
if testname.startswith(prefix):
outcomes |= prefix_rules[prefix]
return self._outcomes_cache[cache_key]
return outcomes
def IsFailureOutput(self, testcase, output):
return output.exit_code != 0
......
......@@ -45,18 +45,18 @@ def PrintReport(tests):
total = len(tests)
skipped = nocrash = passes = fail_ok = fail = 0
for t in tests:
outcomes = t.suite.GetStatusFileOutcomes(t.name, t.variant)
outcomes = t.statusfile_outcomes
if not outcomes:
passes += 1
continue
if statusfile.DoSkip(outcomes):
if t.do_skip:
skipped += 1
continue
if statusfile.IsPassOrFail(outcomes):
if t.is_pass_or_fail:
nocrash += 1
if list(outcomes) == [statusfile.PASS]:
passes += 1
if statusfile.IsFailOk(outcomes):
if t.is_fail_ok:
fail_ok += 1
if statusfile.FAIL in outcomes and statusfile.PASS not in outcomes:
fail += 1
......
......@@ -47,13 +47,14 @@ class TestCase(object):
self.variant = None # name of the used testing variant
self.variant_flags = [] # list of strings, flags specific to this test
self.expected_outcomes = None
self.id = None # int, used to map result back to TestCase instance
self.run = 1 # The nth time this test is executed.
self.cmd = None
self._prepare_expected_outcomes()
self.statusfile_outcomes = None
self.expected_outcomes = None
self._statusfile_flags = None
self._prepare_outcomes()
def create_variant(self, variant, flags):
"""Makes a shallow copy of the object and updates variant, variant_flags and
......@@ -66,15 +67,22 @@ class TestCase(object):
other.variant_flags = self.variant_flags + flags
other.variant = variant
other._prepare_expected_outcomes()
other._prepare_outcomes(variant != self.variant)
return other
def _prepare_expected_outcomes(self):
status_file_outcomes = self.suite.GetStatusFileOutcomes(self.name,
self.variant)
def _prepare_outcomes(self, force_update=True):
if force_update or self.statusfile_outcomes is None:
def is_flag(outcome):
return outcome.startswith('--')
def not_flag(outcome):
return not is_flag(outcome)
outcomes = self.suite.GetStatusFileOutcomes(self.name, self.variant)
self.statusfile_outcomes = filter(not_flag, outcomes)
self._statusfile_flags = filter(is_flag, outcomes)
self.expected_outcomes = (
self._parse_status_file_outcomes(status_file_outcomes))
self._parse_status_file_outcomes(self.statusfile_outcomes))
def _parse_status_file_outcomes(self, outcomes):
if (statusfile.FAIL_SLOPPY in outcomes and
......@@ -91,6 +99,32 @@ class TestCase(object):
expected_outcomes.append(statusfile.PASS)
return expected_outcomes or [statusfile.PASS]
@property
def do_skip(self):
return statusfile.SKIP in self.statusfile_outcomes
@property
def is_slow(self):
return statusfile.SLOW in self.statusfile_outcomes
@property
def is_fail_ok(self):
return statusfile.FAIL_OK in self.statusfile_outcomes
@property
def is_pass_or_fail(self):
return (statusfile.PASS in self.statusfile_outcomes and
statusfile.FAIL in self.statusfile_outcomes and
statusfile.CRASH not in self.statusfile_outcomes)
@property
def only_standard_variant(self):
return statusfile.NO_VARIANTS in self.statusfile_outcomes
@property
def only_fast_variants(self):
return statusfile.FAST_VARIANTS in self.statusfile_outcomes
def get_command(self, context):
params = self._get_cmd_params(context)
env = self._get_cmd_env()
......@@ -135,15 +169,9 @@ class TestCase(object):
def _get_statusfile_flags(self):
"""Gets runtime flags from a status file.
Every outcome that starts with "--" is a flag. Status file has to be loaded
before using this function.
Every outcome that starts with "--" is a flag.
"""
flags = []
for outcome in self.suite.GetStatusFileOutcomes(self.name, self.variant):
if outcome.startswith('--'):
flags.append(outcome)
return flags
return self._statusfile_flags
def _get_mode_flags(self, ctx):
return ctx.mode_flags
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment