[test] Fix --report output.

Fix report mode and make it use testcase properties so statusfile
outcomes can be private.

Bug: v8:6917
Change-Id: Id38e89e0ba427c3bbb7ad12ba93e02beb7e46219
Reviewed-on: https://chromium-review.googlesource.com/833909
Commit-Queue: Michał Majewski <majeski@google.com>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50209}

tools/testrunner/local/verbose.py

@@ -38,28 +38,30 @@ REPORT_TEMPLATE = (
  * %(nocrash)4d tests are expected to be flaky but not crash
  * %(pass)4d tests are expected to pass
  * %(fail_ok)4d tests are expected to fail that we won't fix
- * %(fail)4d tests are expected to fail that we should fix""")
+ * %(fail)4d tests are expected to fail that we should fix
+ * %(crash)4d tests are expected to crash
+""")
 
 
+# TODO(majeski): Turn it into an observer.
 def PrintReport(tests):
   total = len(tests)
-  skipped = nocrash = passes = fail_ok = fail = 0
+  skipped = nocrash = passes = fail_ok = fail = crash = 0
   for t in tests:
-    outcomes = t.statusfile_outcomes
-    if not outcomes:
-      passes += 1
-      continue
     if t.do_skip:
       skipped += 1
-      continue
-    if t.is_pass_or_fail:
+    elif t.is_pass_or_fail:
       nocrash += 1
-    if list(outcomes) == [statusfile.PASS]:
-      passes += 1
-    if t.is_fail_ok:
+    elif t.is_fail_ok:
       fail_ok += 1
-    if statusfile.FAIL in outcomes and statusfile.PASS not in outcomes:
+    elif t.expected_outcomes == [statusfile.PASS]:
+      passes += 1
+    elif t.expected_outcomes == [statusfile.FAIL]:
       fail += 1
+    elif t.expected_outcomes == [statusfile.CRASH]:
+      crash += 1
+    else:
+      assert False # Unreachable # TODO: check this in outcomes parsing phase.
 
   print REPORT_TEMPLATE % {
     "total": total,
@@ -67,7 +69,8 @@ def PrintReport(tests):
     "nocrash": nocrash,
     "pass": passes,
     "fail_ok": fail_ok,
-    "fail": fail
+    "fail": fail,
+    "crash": crash,
   }
 
 

tools/testrunner/objects/testcase.py

@@ -51,7 +51,7 @@ class TestCase(object):
     self.run = 1  # The nth time this test is executed.
     self.cmd = None
 
-    self.statusfile_outcomes = None
+    self._statusfile_outcomes = None
     self.expected_outcomes = None
     self._statusfile_flags = None
     self._prepare_outcomes()
@@ -72,17 +72,17 @@ class TestCase(object):
     return other
 
   def _prepare_outcomes(self, force_update=True):
-    if force_update or self.statusfile_outcomes is None:
+    if force_update or self._statusfile_outcomes is None:
       def is_flag(outcome):
         return outcome.startswith('--')
       def not_flag(outcome):
         return not is_flag(outcome)
 
       outcomes = self.suite.statusfile.get_outcomes(self.name, self.variant)
-      self.statusfile_outcomes = filter(not_flag, outcomes)
+      self._statusfile_outcomes = filter(not_flag, outcomes)
       self._statusfile_flags = filter(is_flag, outcomes)
     self.expected_outcomes = (
-      self._parse_status_file_outcomes(self.statusfile_outcomes))
+      self._parse_status_file_outcomes(self._statusfile_outcomes))
 
   def _parse_status_file_outcomes(self, outcomes):
     if (statusfile.FAIL_SLOPPY in outcomes and
@@ -101,29 +101,29 @@ class TestCase(object):
 
   @property
   def do_skip(self):
-    return statusfile.SKIP in self.statusfile_outcomes
+    return statusfile.SKIP in self._statusfile_outcomes
 
   @property
   def is_slow(self):
-    return statusfile.SLOW in self.statusfile_outcomes
+    return statusfile.SLOW in self._statusfile_outcomes
 
   @property
   def is_fail_ok(self):
-    return statusfile.FAIL_OK in self.statusfile_outcomes
+    return statusfile.FAIL_OK in self._statusfile_outcomes
 
   @property
   def is_pass_or_fail(self):
-    return (statusfile.PASS in self.statusfile_outcomes and
-            statusfile.FAIL in self.statusfile_outcomes and
-            statusfile.CRASH not in self.statusfile_outcomes)
+    return (statusfile.PASS in self._statusfile_outcomes and
+            statusfile.FAIL in self._statusfile_outcomes and
+            statusfile.CRASH not in self._statusfile_outcomes)
 
   @property
   def only_standard_variant(self):
-    return statusfile.NO_VARIANTS in self.statusfile_outcomes
+    return statusfile.NO_VARIANTS in self._statusfile_outcomes
 
   @property
   def only_fast_variants(self):
-    return statusfile.FAST_VARIANTS in self.statusfile_outcomes
+    return statusfile.FAST_VARIANTS in self._statusfile_outcomes
 
   def get_command(self, context):
     params = self._get_cmd_params(context)