[test] Remove output from testcase.

Bug: v8:6917
Cq-Include-Trybots: master.tryserver.v8:v8_linux_noi18n_rel_ng
Change-Id: I5ca840db75766413659cb96fd28b922712040cec
Reviewed-on: https://chromium-review.googlesource.com/823843
Commit-Queue: Michał Majewski <majeski@google.com>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Sergiy Byelozyorov <sergiyb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50078}

test/inspector/testcfg.py

@@ -47,7 +47,7 @@ class TestSuite(testsuite.TestSuite):
             string == "Warning: unknown flag --enable-slow-asserts." or
             string == "Try --help for options")
 
-  def IsFailureOutput(self, test):
+  def IsFailureOutput(self, test, output):
     file_name = os.path.join(self.root, test.path) + EXPECTED_SUFFIX
     with file(file_name, "r") as expected:
       expected_lines = expected.readlines()
@@ -66,7 +66,7 @@ class TestSuite(testsuite.TestSuite):
 
     def ActBlockIterator():
       """Iterates over blocks of actual output lines."""
-      lines = test.output.stdout.splitlines()
+      lines = output.stdout.splitlines()
       start_index = 0
       found_eqeq = False
       for index, line in enumerate(lines):

test/message/testcfg.py

@@ -81,11 +81,10 @@ class TestSuite(testsuite.TestSuite):
       path = head
     return False
 
-  def IsFailureOutput(self, test):
-    output = test.output
+  def IsFailureOutput(self, test, output):
     testpath = test.path
     expected_fail = self._GetExpectedFail(test)
-    fail = test.output.exit_code != 0
+    fail = output.exit_code != 0
     if expected_fail != fail:
       return True
     expected_path = os.path.join(self.root, testpath + ".out")

test/mkgrokdump/testcfg.py

@@ -19,8 +19,7 @@ class TestSuite(testsuite.TestSuite):
   def _test_class(self):
     return TestCase
 
-  def IsFailureOutput(self, test):
-    output = test.output
+  def IsFailureOutput(self, test, output):
     v8_path = os.path.dirname(os.path.dirname(os.path.abspath(self.root)))
     expected_path = os.path.join(v8_path, "tools", "v8heapconst.py")
     with open(expected_path) as f:

test/mozilla/testcfg.py

@@ -86,10 +86,10 @@ class TestSuite(testsuite.TestSuite):
   def IsNegativeTest(self, test):
     return test.path.endswith("-n")
 
-  def IsFailureOutput(self, test):
-    if test.output.exit_code != 0:
+  def IsFailureOutput(self, test, output):
+    if output.exit_code != 0:
       return True
-    return "FAILED!" in test.output.stdout
+    return "FAILED!" in output.stdout
 
 
 class TestCase(testcase.TestCase):

test/test262/testcfg.py

@@ -190,8 +190,7 @@ class TestSuite(testsuite.TestSuite):
   def _test_class(self):
     return TestCase
 
-  def IsFailureOutput(self, test):
-    output = test.output
+  def IsFailureOutput(self, test, output):
     test_record = test.test_record
     if output.exit_code != 0:
       return True

test/webkit/testcfg.py

@@ -73,8 +73,8 @@ class TestSuite(testsuite.TestSuite):
             string == "Warning: unknown flag --enable-slow-asserts." or
             string == "Try --help for options")
 
-  def IsFailureOutput(self, test):
-    if super(TestSuite, self).IsFailureOutput(test):
+  def IsFailureOutput(self, test, output):
+    if super(TestSuite, self).IsFailureOutput(test, output):
       return True
     file_name = os.path.join(self.root, test.path) + "-expected.txt"
     with file(file_name, "r") as expected:
@@ -94,7 +94,7 @@ class TestSuite(testsuite.TestSuite):
 
     def ActBlockIterator():
       """Iterates over blocks of actual output lines."""
-      lines = test.output.stdout.splitlines()
+      lines = output.stdout.splitlines()
       start_index = 0
       found_eqeq = False
       for index, line in enumerate(lines):

tools/testrunner/deopt_fuzzer.py

@@ -332,7 +332,7 @@ class DeoptFuzzer(base_runner.BaseTestRunner):
     for s in suites:
       test_results = {}
       for t in s.tests:
-        for line in t.output.stdout.splitlines():
+        for line in runner.outputs[t].stdout.splitlines():
           if line.startswith("=== Stress deopt counter: "):
             test_results[t.path] = MAX_DEOPT - int(line.split(" ")[-1])
       for t in s.tests:

tools/testrunner/gc_fuzzer.py

@@ -213,13 +213,15 @@ class GCFuzzer(base_runner.BaseTestRunner):
     for s in suites:
       for t in s.tests:
         # Skip failed tests.
-        if s.HasUnexpectedOutput(t):
+        if s.HasUnexpectedOutput(t, runner.outputs[t]):
           print '%s failed, skipping' % t.path
           continue
-        max_limit = self._get_max_limit_reached(t)
+        max_limit = self._get_max_limit_reached(runner.outputs[t])
         if max_limit:
           test_results[t.path] = max_limit
 
+    runner = None
+
     if options.dump_results_file:
       with file("%s.%d.txt" % (options.dump_results_file, time.time()),
                 "w") as f:
@@ -308,7 +310,7 @@ class GCFuzzer(base_runner.BaseTestRunner):
   # incremental marking limit (0-100).
   # Skips values >=100% since they already trigger incremental marking.
   @staticmethod
-  def _get_max_limit_reached(test):
+  def _get_max_limit_reached(output):
     def is_im_line(l):
       return 'IncrementalMarking' in l and '% of the memory limit reached' in l
 
@@ -318,10 +320,10 @@ class GCFuzzer(base_runner.BaseTestRunner):
     def percent_str_to_float(s):
       return float(s[:-1])
 
-    if not (test.output and test.output.stdout):
+    if not (output and output.stdout):
       return None
 
-    im_lines = filter(is_im_line, test.output.stdout.splitlines())
+    im_lines = filter(is_im_line, output.stdout.splitlines())
     percents_str = map(line_to_percent, im_lines)
     percents = map(percent_str_to_float, percents_str)
 

tools/testrunner/local/execution.py

@@ -37,7 +37,6 @@ from . import command
 from . import perfdata
 from . import statusfile
 from . import utils
-from ..objects import output
 from pool import Pool
 
 
@@ -119,6 +118,10 @@ class Runner(object):
     self.perf_failures = False
     self.printed_allocations = False
     self.tests = [t for s in suites for t in s.tests]
+
+    # TODO(majeski): Pass outputs dynamically instead of keeping them in the
+    # runner.
+    self.outputs = {t: None for t in self.tests}
     self.suite_names = [s.name for s in suites]
 
     # Always pre-sort by status file, slowest tests first.
@@ -174,19 +177,20 @@ class Runner(object):
 
       # Rerun this test.
       test.duration = None
-      test.output = None
       test.run += 1
       pool.add([TestJob(test.id, test.cmd, test.run)])
       self.remaining += 1
       self.total += 1
 
   def _ProcessTest(self, test, result, pool):
-    test.output = result[1]
+    output = result[1]
+    self.outputs[test] = output
     test.duration = result[2]
-    has_unexpected_output = test.suite.HasUnexpectedOutput(test, self.context)
+    has_unexpected_output = test.suite.HasUnexpectedOutput(
+        test, output, self.context)
     if has_unexpected_output:
       self.failed.append(test)
-      if test.output.HasCrashed():
+      if output.HasCrashed():
         self.crashed += 1
     else:
       self.succeeded += 1
@@ -194,7 +198,7 @@ class Runner(object):
     # For the indicator, everything that happens after the first run is treated
     # as unexpected even if it flakily passes in order to include it in the
     # output.
-    self.indicator.HasRun(test, has_unexpected_output or test.run > 1)
+    self.indicator.HasRun(test, output, has_unexpected_output or test.run > 1)
     if has_unexpected_output:
       # Rerun test failures after the indicator has processed the results.
       self._VerbosePrint("Attempting to rerun test after failure.")

tools/testrunner/local/progress.py

@@ -50,7 +50,7 @@ class ProgressIndicator(object):
   def Done(self):
     pass
 
-  def HasRun(self, test, has_unexpected_output):
+  def HasRun(self, test, output, has_unexpected_output):
     pass
 
   def Heartbeat(self):
@@ -100,18 +100,19 @@ class SimpleProgressIndicator(ProgressIndicator):
   def Done(self):
     print
     for failed in self.runner.failed:
+      output = self.runner.outputs[failed]
       self.PrintFailureHeader(failed)
-      if failed.output.stderr:
+      if output.stderr:
         print "--- stderr ---"
-        print failed.output.stderr.strip()
-      if failed.output.stdout:
+        print output.stderr.strip()
+      if output.stdout:
         print "--- stdout ---"
-        print failed.output.stdout.strip()
+        print output.stdout.strip()
       print "Command: %s" % failed.cmd.to_string()
-      if failed.output.HasCrashed():
-        print "exit code: %d" % failed.output.exit_code
+      if output.HasCrashed():
+        print "exit code: %d" % output.exit_code
         print "--- CRASHED ---"
-      if failed.output.HasTimedOut():
+      if output.HasTimedOut():
         print "--- TIMEOUT ---"
     if len(self.runner.failed) == 0:
       print "==="
@@ -128,9 +129,9 @@ class SimpleProgressIndicator(ProgressIndicator):
 
 class VerboseProgressIndicator(SimpleProgressIndicator):
 
-  def HasRun(self, test, has_unexpected_output):
+  def HasRun(self, test, output, has_unexpected_output):
     if has_unexpected_output:
-      if test.output.HasCrashed():
+      if output.HasCrashed():
         outcome = 'CRASH'
       else:
         outcome = 'FAIL'
@@ -146,15 +147,15 @@ class VerboseProgressIndicator(SimpleProgressIndicator):
 
 class DotsProgressIndicator(SimpleProgressIndicator):
 
-  def HasRun(self, test, has_unexpected_output):
+  def HasRun(self, test, output, has_unexpected_output):
     total = self.runner.succeeded + len(self.runner.failed)
     if (total > 1) and (total % 50 == 1):
       sys.stdout.write('\n')
     if has_unexpected_output:
-      if test.output.HasCrashed():
+      if output.HasCrashed():
         sys.stdout.write('C')
         sys.stdout.flush()
-      elif test.output.HasTimedOut():
+      elif output.HasTimedOut():
         sys.stdout.write('T')
         sys.stdout.flush()
       else:
@@ -178,22 +179,22 @@ class CompactProgressIndicator(ProgressIndicator):
     self.PrintProgress('Done')
     print ""  # Line break.
 
-  def HasRun(self, test, has_unexpected_output):
+  def HasRun(self, test, output, has_unexpected_output):
     self.PrintProgress(str(test))
     if has_unexpected_output:
       self.ClearLine(self.last_status_length)
       self.PrintFailureHeader(test)
-      stdout = test.output.stdout.strip()
+      stdout = output.stdout.strip()
       if len(stdout):
         print self.templates['stdout'] % stdout
-      stderr = test.output.stderr.strip()
+      stderr = output.stderr.strip()
       if len(stderr):
         print self.templates['stderr'] % stderr
       print "Command: %s" % test.cmd.to_string()
-      if test.output.HasCrashed():
-        print "exit code: %d" % test.output.exit_code
+      if output.HasCrashed():
+        print "exit code: %d" % output.exit_code
         print "--- CRASHED ---"
-      if test.output.HasTimedOut():
+      if output.HasTimedOut():
         print "--- TIMEOUT ---"
 
   def Truncate(self, string, length):
@@ -269,19 +270,19 @@ class JUnitTestProgressIndicator(ProgressIndicator):
     if self.outfile != sys.stdout:
       self.outfile.close()
 
-  def HasRun(self, test, has_unexpected_output):
+  def HasRun(self, test, output, has_unexpected_output):
     fail_text = ""
     if has_unexpected_output:
-      stdout = test.output.stdout.strip()
+      stdout = output.stdout.strip()
       if len(stdout):
         fail_text += "stdout:\n%s\n" % stdout
-      stderr = test.output.stderr.strip()
+      stderr = output.stderr.strip()
       if len(stderr):
         fail_text += "stderr:\n%s\n" % stderr
       fail_text += "Command: %s" % self.test.cmd.to_string()
-      if test.output.HasCrashed():
-        fail_text += "exit code: %d\n--- CRASHED ---" % test.output.exit_code
-      if test.output.HasTimedOut():
+      if output.HasCrashed():
+        fail_text += "exit code: %d\n--- CRASHED ---" % output.exit_code
+      if output.HasTimedOut():
         fail_text += "--- TIMEOUT ---"
     self.outputter.HasRunTest(
         test_name=str(test),
@@ -340,7 +341,7 @@ class JsonTestProgressIndicator(ProgressIndicator):
     with open(self.json_test_results, "w") as f:
       f.write(json.dumps(complete_results))
 
-  def HasRun(self, test, has_unexpected_output):
+  def HasRun(self, test, output, has_unexpected_output):
     # Buffer all tests for sorting the durations in the end.
     self.tests.append(test)
     if not has_unexpected_output:
@@ -353,9 +354,9 @@ class JsonTestProgressIndicator(ProgressIndicator):
       "flags": test.cmd.args,
       "command": test.cmd.to_string(relative=True),
       "run": test.run,
-      "stdout": test.output.stdout,
-      "stderr": test.output.stderr,
-      "exit_code": test.output.exit_code,
+      "stdout": output.stdout,
+      "stderr": output.stderr,
+      "exit_code": output.exit_code,
       "result": test.suite.GetOutcome(test),
       "expected": test.suite.GetExpectedOutcomes(test),
       "duration": test.duration,
@@ -393,7 +394,7 @@ class FlakinessTestProgressIndicator(ProgressIndicator):
         "version": 3,
       }, f)
 
-  def HasRun(self, test, has_unexpected_output):
+  def HasRun(self, test, output, has_unexpected_output):
     key = test.get_id()
     outcome = test.suite.GetOutcome(test)
     assert outcome in ["PASS", "FAIL", "CRASH", "TIMEOUT"]

tools/testrunner/local/testsuite.py

@@ -276,36 +276,36 @@ class TestSuite(object):
 
     return self._outcomes_cache[cache_key]
 
-  def IsFailureOutput(self, testcase):
-    return testcase.output.exit_code != 0
+  def IsFailureOutput(self, testcase, output):
+    return output.exit_code != 0
 
   def IsNegativeTest(self, testcase):
     return False
 
-  def HasFailed(self, testcase, ctx=None):
+  def HasFailed(self, testcase, output, ctx=None):
     if ctx and ctx.predictable:
       # Only check the exit code of the predictable_wrapper in
       # verify-predictable mode.
-      execution_failed = testcase.output.exit_code != 0
+      execution_failed = output.exit_code != 0
     else:
-      execution_failed = self.IsFailureOutput(testcase)
+      execution_failed = self.IsFailureOutput(testcase, output)
     if self.IsNegativeTest(testcase):
       return not execution_failed
     else:
       return execution_failed
 
-  def GetOutcome(self, testcase, ctx=None):
-    if testcase.output.HasCrashed():
+  def GetOutcome(self, testcase, output, ctx=None):
+    if output.HasCrashed():
       return statusfile.CRASH
-    elif testcase.output.HasTimedOut():
+    elif output.HasTimedOut():
       return statusfile.TIMEOUT
-    elif self.HasFailed(testcase, ctx):
+    elif self.HasFailed(testcase, output, ctx):
       return statusfile.FAIL
     else:
       return statusfile.PASS
 
-  def HasUnexpectedOutput(self, testcase, ctx=None):
-    return (self.GetOutcome(testcase, ctx)
+  def HasUnexpectedOutput(self, testcase, output, ctx=None):
+    return (self.GetOutcome(testcase, output, ctx)
             not in self.GetExpectedOutcomes(testcase))
 
   def _create_test(self, path, **kwargs):

tools/testrunner/objects/testcase.py

@@ -45,7 +45,6 @@ class TestCase(object):
     self.variant = None       # name of the used testing variant
     self.variant_flags = []   # list of strings, flags specific to this test
 
-    self.output = None
     self.id = None  # int, used to map result back to TestCase instance
     self.duration = None  # assigned during execution
     self.run = 1  # The nth time this test is executed.
@@ -198,6 +197,9 @@ class TestCase(object):
         (other.suite.name, other.name, other.variant_flags)
     )
 
+  def __hash__(self):
+    return hash((self.suite.name, self.name, ''.join(self.variant_flags)))
+
   def __str__(self):
     return self.suite.name + '/' + self.name