[test] Store expected outcomes in the testcase.

Bug: v8:6917
Cq-Include-Trybots: master.tryserver.v8:v8_linux_noi18n_rel_ng
Change-Id: I69d15801b79bf7aa846582367e9f3037b6612431
Reviewed-on: https://chromium-review.googlesource.com/829033
Commit-Queue: Michał Majewski <majeski@google.com>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Sergiy Byelozyorov <sergiyb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50163}

test/debugger/testcfg.py

@@ -36,21 +36,10 @@ class TestCase(testcase.TestCase):
   def __init__(self, *args, **kwargs):
     super(TestCase, self).__init__(*args, **kwargs)
 
-    # precomputed
-    self._source_files = None
-    self._source_flags = None
-
-  def precompute(self):
     source = self.get_source()
     self._source_files = self._parse_source_files(source)
     self._source_flags = self._parse_source_flags(source)
 
-  def _copy(self):
-    copy = super(TestCase, self)._copy()
-    copy._source_files = self._source_files
-    copy._source_flags = self._source_flags
-    return copy
-
   def _parse_source_files(self, source):
     files_list = []  # List of file names to append to command arguments.
     files_match = FILES_PATTERN.search(source);

test/inspector/testcfg.py

@@ -95,17 +95,8 @@ class TestCase(testcase.TestCase):
   def __init__(self, *args, **kwargs):
     super(TestCase, self).__init__(*args, **kwargs)
 
-    # precomputed
-    self._source_flags = None
-
-  def precompute(self):
     self._source_flags = self._parse_source_flags()
 
-  def _copy(self):
-    copy = super(TestCase, self)._copy()
-    copy._source_flags = self._source_flags
-    return copy
-
   def _get_files_params(self, ctx):
     return [
       os.path.join(self.suite.root, PROTOCOL_TEST_JS),

test/intl/testcfg.py

@@ -57,17 +57,8 @@ class TestCase(testcase.TestCase):
   def __init__(self, *args, **kwargs):
     super(TestCase, self).__init__(*args, **kwargs)
 
-    # precomputed
-    self._source_flags = None
-
-  def precompute(self):
     self._source_flags = self._parse_source_flags()
 
-  def _copy(self):
-    copy = super(TestCase, self)._copy()
-    copy._source_flags = self._source_flags
-    return copy
-
   def _get_files_params(self, ctx):
     files = map(lambda f: os.path.join(self.suite.root, f), [
         'assert.js',

test/message/testcfg.py

@@ -115,21 +115,10 @@ class TestCase(testcase.TestCase):
   def __init__(self, *args, **kwargs):
     super(TestCase, self).__init__(*args, **kwargs)
 
-    # precomputed
-    self._source_files = None
-    self._source_flags = None
-
-  def precompute(self):
     source = self.get_source()
     self._source_files = self._parse_source_files(source)
     self._source_flags = self._parse_source_flags(source)
 
-  def _copy(self):
-    copy = super(TestCase, self)._copy()
-    copy._source_files = self._source_files
-    copy._source_flags = self._source_flags
-    return copy
-
   def _parse_source_files(self, source):
     files = []
     if MODULE_PATTERN.search(source):

test/mjsunit/testcfg.py

@@ -63,14 +63,6 @@ class TestCase(testcase.TestCase):
   def __init__(self, *args, **kwargs):
     super(TestCase, self).__init__(*args, **kwargs)
 
-    # precomputed
-    self._source_files = None
-    self._source_flags = None
-    self._mjsunit_files = None
-    self._files_suffix = None
-    self._env = None
-
-  def precompute(self):
     source = self.get_source()
 
     files_list = []  # List of file names to append to command arguments.
@@ -84,7 +76,8 @@ class TestCase(testcase.TestCase):
         break
     files = [ os.path.normpath(os.path.join(self.suite.root, '..', '..', f))
               for f in files_list ]
-    testfilename = os.path.join(self.suite.root, self.path + self._get_suffix())
+    testfilename = os.path.join(self.suite.root,
+                                self.path + self._get_suffix())
     if SELF_SCRIPT_PATTERN.search(source):
       files = (
         ["-e", "TEST_FILE_NAME=\"%s\"" % testfilename.replace("\\", "\\\\")] +
@@ -106,15 +99,6 @@ class TestCase(testcase.TestCase):
     self._files_suffix = files_suffix
     self._env = self._parse_source_env(source)
 
-  def _copy(self):
-    copy = super(TestCase, self)._copy()
-    copy._source_files = self._source_files
-    copy._source_flags = self._source_flags
-    copy._mjsunit_files = self._mjsunit_files
-    copy._files_suffix = self._files_suffix
-    copy._env = self._env
-    return copy
-
   def _parse_source_env(self, source):
     env_match = ENV_PATTERN.search(source)
     env = {}

test/preparser/testcfg.py

@@ -79,13 +79,10 @@ class TestSuite(testsuite.TestSuite):
 class TestCase(testcase.TestCase):
   def __init__(self, suite, path, name, source, template_flags):
     super(TestCase, self).__init__(suite, path, name)
+
     self._source = source
     self._template_flags = template_flags
 
-  def _copy(self):
-    return self.__class__(
-        self.suite, self.path, self.name, self._source, self._template_flags)
-
   def _get_cmd_params(self, ctx):
     return (
         self._get_files_params(ctx) +

test/test262/testcfg.py

@@ -212,12 +212,6 @@ class TestSuite(testsuite.TestSuite):
       print "Error parsing exception for %s" % test
       return None
 
-  def GetExpectedOutcomes(self, test):
-    outcomes = self.GetStatusFileOutcomes(test.name, test.variant)
-    if (statusfile.FAIL_SLOPPY in outcomes and not test.uses_strict()):
-      return [statusfile.FAIL]
-    return super(TestSuite, self).GetExpectedOutcomes(test)
-
   def _VariantGeneratorFactory(self):
     return VariantGenerator
 
@@ -226,18 +220,9 @@ class TestCase(testcase.TestCase):
   def __init__(self, *args, **kwargs):
     super(TestCase, self).__init__(*args, **kwargs)
 
-    # precomputed
-    self.test_record = None
-
-  def precompute(self):
     source = self.get_source()
     self.test_record = self.suite.parse_test_record(source, self.path)
 
-  def _copy(self):
-    copy = super(TestCase, self)._copy()
-    copy.test_record = self.test_record
-    return copy
-
   def _get_files_params(self, ctx):
     return (
         list(self.suite.harness) +
@@ -275,9 +260,6 @@ class TestCase(testcase.TestCase):
       return path
     return os.path.join(self.suite.testroot, filename)
 
-  def uses_strict(self):
-    return '--use-strict' in self.variant_flags
-
 
 def GetSuite(name, root):
   return TestSuite(name, root)

test/webkit/testcfg.py

@@ -123,21 +123,10 @@ class TestCase(testcase.TestCase):
   def __init__(self, *args, **kwargs):
     super(TestCase, self).__init__(*args, **kwargs)
 
-    # precomputed
-    self._source_files = None
-    self._source_flags = None
-
-  def precompute(self):
     source = self.get_source()
     self._source_files = self._parse_source_files(source)
     self._source_flags = self._parse_source_flags(source)
 
-  def _copy(self):
-    copy = super(TestCase, self)._copy()
-    copy._source_files = self._source_files
-    copy._source_flags = self._source_flags
-    return copy
-
   def _parse_source_files(self, source):
     files_list = []  # List of file names to append to command arguments.
     files_match = FILES_PATTERN.search(source);

tools/testrunner/local/progress.py

@@ -358,7 +358,7 @@ class JsonTestProgressIndicator(ProgressIndicator):
       "stderr": output.stderr,
       "exit_code": output.exit_code,
       "result": test.suite.GetOutcome(test, output),
-      "expected": test.suite.GetExpectedOutcomes(test),
+      "expected": test.expected_outcomes,
       "duration": output.duration,
 
       # TODO(machenbach): This stores only the global random seed from the
@@ -400,10 +400,9 @@ class FlakinessTestProgressIndicator(ProgressIndicator):
     assert outcome in ["PASS", "FAIL", "CRASH", "TIMEOUT"]
     if test.run == 1:
       # First run of this test.
-      expected_outcomes = test.suite.GetExpectedOutcomes(test)
       self.results[key] = {
         "actual": outcome,
-        "expected": " ".join(expected_outcomes),
+        "expected": " ".join(test.expected_outcomes),
         "times": [output.duration],
       }
       self.summary[outcome] = self.summary[outcome] + 1

tools/testrunner/local/testsuite.py

@@ -221,28 +221,6 @@ class TestSuite(object):
           break
     self.tests = filtered
 
-  def GetExpectedOutcomes(self, test):
-    """Gets expected outcomes from status file.
-
-    It differs from GetStatusFileOutcomes by selecting only outcomes that can
-    be result of test execution.
-    Status file has to be loaded before using this function.
-    """
-    outcomes = self.GetStatusFileOutcomes(test.name, test.variant)
-
-    expected = []
-    if (statusfile.FAIL in outcomes or
-        statusfile.FAIL_OK in outcomes):
-      expected.append(statusfile.FAIL)
-
-    if statusfile.CRASH in outcomes:
-      expected.append(statusfile.CRASH)
-
-    if statusfile.PASS in outcomes:
-      expected.append(statusfile.PASS)
-
-    return expected or [statusfile.PASS]
-
   def GetStatusFileOutcomes(self, testname, variant=None):
     """Gets outcomes from status file.
 
@@ -308,14 +286,12 @@ class TestSuite(object):
       return (
           output.exit_code != 0 and
           not self.IsNegativeTest(testcase) and
-          statusfile.FAIL not in self.GetExpectedOutcomes(testcase)
+          statusfile.FAIL not in testcase.expected_outcomes
       )
-    return (self.GetOutcome(testcase, output)
-            not in self.GetExpectedOutcomes(testcase))
+    return self.GetOutcome(testcase, output) not in testcase.expected_outcomes
 
   def _create_test(self, path, **kwargs):
     test = self._test_class()(self, path, self._path_to_name(path), **kwargs)
-    test.precompute()
     return test
 
   def _test_class(self):

tools/testrunner/objects/testcase.py

@@ -25,11 +25,13 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+import copy
 import os
 import re
 import shlex
 
 from ..local import command
+from ..local import statusfile
 from ..local import utils
 
 FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)")
@@ -45,31 +47,49 @@ class TestCase(object):
     self.variant = None       # name of the used testing variant
     self.variant_flags = []   # list of strings, flags specific to this test
 
+    self.expected_outcomes = None
+
     self.id = None  # int, used to map result back to TestCase instance
     self.run = 1  # The nth time this test is executed.
     self.cmd = None
 
-  def precompute(self):
-    """It precomputes all things that can be shared among all variants of this
-    object (like flags from source file). Values calculated here should be
-    immutable and shared among all copies (in _copy implementation).
-    """
-    pass
+    self._prepare_expected_outcomes()
 
   def create_variant(self, variant, flags):
-    copy = self._copy()
+    """Makes a shallow copy of the object and updates variant, variant_flags and
+    all fields that depend on it, e.g. expected outcomes.
+    """
+    other = copy.copy(self)
     if not self.variant_flags:
-      copy.variant_flags = flags
+      other.variant_flags = flags
     else:
-      copy.variant_flags = self.variant_flags + flags
-    copy.variant = variant
-    return copy
-
-  def _copy(self):
-    """Makes a copy of the object. It should be overriden in case of any
-    additional constructor parameters or precomputed fields.
-    """
-    return self.__class__(self.suite, self.path, self.name)
+      other.variant_flags = self.variant_flags + flags
+    other.variant = variant
+
+    other._prepare_expected_outcomes()
+
+    return other
+
+  def _prepare_expected_outcomes(self):
+    status_file_outcomes = self.suite.GetStatusFileOutcomes(self.name,
+                                                            self.variant)
+    self.expected_outcomes = (
+      self._parse_status_file_outcomes(status_file_outcomes))
+
+  def _parse_status_file_outcomes(self, outcomes):
+    if (statusfile.FAIL_SLOPPY in outcomes and
+        '--use-strict' not in self.variant_flags):
+      return [statusfile.FAIL]
+
+    expected_outcomes = []
+    if (statusfile.FAIL in outcomes or
+        statusfile.FAIL_OK in outcomes):
+      expected_outcomes.append(statusfile.FAIL)
+    if statusfile.CRASH in outcomes:
+      expected_outcomes.append(statusfile.CRASH)
+    if statusfile.PASS in outcomes:
+      expected_outcomes.append(statusfile.PASS)
+    return expected_outcomes or [statusfile.PASS]
 
   def get_command(self, context):
     params = self._get_cmd_params(context)