testcfg.py 9.52 KB
Newer Older
1
# Copyright 2012 the V8 project authors. All rights reserved.
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#     * Neither the name of Google Inc. nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

28
import imp
29
import itertools
30
import os
31
import re
32
import sys
33

34
from testrunner.local import statusfile
35
from testrunner.local import testsuite
36
from testrunner.local import utils
37
from testrunner.objects import testcase
38 39 40
from testrunner.outproc import base as outproc
from testrunner.outproc import test262

41

Daniel Ehrenberg's avatar
Daniel Ehrenberg committed
42 43
# TODO(littledan): move the flag mapping into the status file
FEATURE_FLAGS = {
44
  'Intl.NumberFormat-v3': '--harmony_intl_number_format_v3',
Mathias Bynens's avatar
Mathias Bynens committed
45
  'Symbol.prototype.description': '--harmony-symbol-description',
46 47
  'FinalizationRegistry': '--harmony-weak-refs-with-cleanup-some',
  'WeakRef': '--harmony-weak-refs-with-cleanup-some',
48
  'host-gc-required': '--expose-gc-as=v8GC',
Shu-yu Guo's avatar
Shu-yu Guo committed
49
  'IsHTMLDDA': '--allow-natives-syntax',
50
  'top-level-await': '--harmony-top-level-await',
Joshua Litt's avatar
Joshua Litt committed
51 52
  'regexp-match-indices': '--harmony-regexp-match-indices',
  'regexp-named-groups': '--harmony-regexp-match-indices',
Shu-yu Guo's avatar
Shu-yu Guo committed
53
  'error-cause': '--harmony-error-cause',
Frank Tang's avatar
Frank Tang committed
54
  'import-assertions': '--harmony-import-assertions',
55
  'Object.hasOwn': '--harmony-object-has-own',
Shu-yu Guo's avatar
Shu-yu Guo committed
56
  'class-static-block': '--harmony-class-static-blocks',
57
  'resizable-arraybuffer': '--harmony-rab-gsab',
58
  'Temporal': '--harmony-temporal',
Frank Tang's avatar
Frank Tang committed
59
  'array-find-from-last': '--harmony_array_find_last',
60
  'ShadowRealm': '--harmony-shadow-realm',
Daniel Ehrenberg's avatar
Daniel Ehrenberg committed
61 62
}

63
SKIPPED_FEATURES = set([])
Daniel Ehrenberg's avatar
Daniel Ehrenberg committed
64

65
DATA = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")
66

67 68 69
BASE_DIR = os.path.dirname(
    os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

70
TEST_262_HARNESS_FILES = ["sta.js", "assert.js"]
71
TEST_262_NATIVE_FILES = ["detachArrayBuffer.js"]
72 73 74

TEST_262_SUITE_PATH = ["data", "test"]
TEST_262_HARNESS_PATH = ["data", "harness"]
75
TEST_262_TOOLS_ABS_PATH = [BASE_DIR, "third_party", "test262-harness", "src"]
76 77
TEST_262_LOCAL_TESTS_PATH = ["local-tests", "test"]

78
sys.path.append(os.path.join(*TEST_262_TOOLS_ABS_PATH))
79

80

81
class VariantsGenerator(testsuite.VariantsGenerator):
82
  def gen(self, test):
83 84
    flags_set = self._get_flags_set(test)
    test_record = test.test_record
85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101

    # Add a reverse test ensuring that FAIL_PHASE_ONLY is only used for tests
    # that actually fail to throw an exception at wrong phase.
    phase_variants = ['']
    if test.fail_phase_only:
      phase_variants.append('-fail-phase-reverse')

    for phase_var in phase_variants:
      for n, variant in enumerate(self._get_variants(test)):
        flags = flags_set[variant][0]
        if 'noStrict' in test_record:
          yield (variant, flags, str(n) + phase_var)
        elif 'onlyStrict' in test_record:
          yield (variant, flags + ['--use-strict'], 'strict-%d' % n + phase_var)
        else:
          yield (variant, flags, str(n))
          yield (variant, flags + ['--use-strict'], 'strict-%d' % n + phase_var)
102 103


104 105 106 107 108 109 110
class TestLoader(testsuite.JSTestLoader):
  @property
  def test_dirs(self):
    return [
      self.test_root,
      os.path.join(self.suite.root, *TEST_262_LOCAL_TESTS_PATH),
    ]
111

112 113 114 115 116 117 118 119 120 121 122 123 124 125
  @property
  def excluded_suffixes(self):
    return {"_FIXTURE.js"}

  @property
  def excluded_dirs(self):
    return {"intl402"} if self.test_config.noi18n else set()

  def _should_filter_by_test(self, test):
    features = test.test_record.get("features", [])
    return SKIPPED_FEATURES.intersection(features)


class TestSuite(testsuite.TestSuite):
126 127 128

  def __init__(self, ctx, *args, **kwargs):
    super(TestSuite, self).__init__(ctx, *args, **kwargs)
129 130 131
    self.test_root = os.path.join(self.root, *TEST_262_SUITE_PATH)
    # TODO: this makes the TestLoader mutable, refactor it.
    self._test_loader.test_root = self.test_root
132 133 134
    self.harnesspath = os.path.join(self.root, *TEST_262_HARNESS_PATH)
    self.harness = [os.path.join(self.harnesspath, f)
                    for f in TEST_262_HARNESS_FILES]
135
    self.harness += [os.path.join(self.root, "harness-adapt.js")]
136
    self.local_test_root = os.path.join(self.root, *TEST_262_LOCAL_TESTS_PATH)
137
    self.parse_test_record = self._load_parse_test_record()
138 139

  def _load_parse_test_record(self):
140
    root = os.path.join(*TEST_262_TOOLS_ABS_PATH)
141 142 143 144 145 146
    f = None
    try:
      (f, pathname, description) = imp.find_module("parseTestRecord", [root])
      module = imp.load_module("parseTestRecord", f, pathname, description)
      return module.parseTestRecord
    except:
147
      print('Cannot load parseTestRecord')
148
      raise
149 150 151
    finally:
      if f:
        f.close()
152

153 154
  def _test_loader_class(self):
    return TestLoader
155

156
  def _test_class(self):
157
    return TestCase
158

159 160
  def _variants_gen_class(self):
    return VariantsGenerator
161 162


163
class TestCase(testcase.D8TestCase):
164
  def __init__(self, *args, **kwargs):
165
    super(TestCase, self).__init__(*args, **kwargs)
166 167 168

    source = self.get_source()
    self.test_record = self.suite.parse_test_record(source, self.path)
169
    self._expected_exception = (
170 171 172 173 174
        self.test_record
          .get('negative', {})
          .get('type', None)
    )

175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
    # We disallow combining FAIL_PHASE_ONLY with any other fail outcome types.
    # Outcome parsing logic in the base class converts all outcomes specified in
    # the status file into either FAIL, CRASH or PASS, thus we do not need to
    # handle FAIL_OK, FAIL_SLOPPY and various other outcomes.
    if self.fail_phase_only:
      assert (
          statusfile.FAIL not in self.expected_outcomes and
          statusfile.CRASH not in self.expected_outcomes), self.name

  @property
  def fail_phase_only(self):
    # The FAIL_PHASE_ONLY is defined in tools/testrunner/local/statusfile.py and
    # can be used in status files to mark tests that throw an exception at wrong
    # phase, e.g. SyntaxError is thrown at execution phase instead of parsing
    # phase. See https://crbug.com/v8/8467 for more details.
    return statusfile.FAIL_PHASE_ONLY in self._statusfile_outcomes

  @property
  def _fail_phase_reverse(self):
    return 'fail-phase-reverse' in self.procid

196 197 198 199
  def __needs_harness_agent(self):
    tokens = self.path.split(os.path.sep)
    return tokens[:2] == ["built-ins", "Atomics"]

200
  def _get_files_params(self):
201 202 203
    return (
        list(self.suite.harness) +
        ([os.path.join(self.suite.root, "harness-agent.js")]
204
         if self.__needs_harness_agent() else []) +
Shu-yu Guo's avatar
Shu-yu Guo committed
205 206
        ([os.path.join(self.suite.root, "harness-ishtmldda.js")]
         if "IsHTMLDDA" in self.test_record.get("features", []) else []) +
207
        ([os.path.join(self.suite.root, "harness-adapt-donotevaluate.js")]
208
         if self.fail_phase_only and not self._fail_phase_reverse else []) +
209 210 211 212 213
        self._get_includes() +
        (["--module"] if "module" in self.test_record else []) +
        [self._get_source_path()]
    )

214
  def _get_suite_flags(self):
215
    return (
216
        ["--ignore-unhandled-promises"] +
217 218 219 220 221
        (["--throws"] if "negative" in self.test_record else []) +
        (["--allow-natives-syntax"]
         if "detachArrayBuffer.js" in self.test_record.get("includes", [])
         else []) +
        [flag for (feature, flag) in FEATURE_FLAGS.items()
222 223
          if feature in self.test_record.get("features", [])] +
        ["--no-arguments"]  # disable top-level arguments in d8
224 225 226 227 228 229 230 231 232 233 234 235 236 237
    )

  def _get_includes(self):
    return [os.path.join(self._base_path(filename), filename)
            for filename in self.test_record.get("includes", [])]

  def _base_path(self, filename):
    if filename in TEST_262_NATIVE_FILES:
      return self.suite.root
    else:
      return self.suite.harnesspath

  def _get_source_path(self):
    filename = self.path + self._get_suffix()
238
    path = os.path.join(self.suite.local_test_root, filename)
239 240
    if os.path.exists(path):
      return path
241
    return os.path.join(self.suite.test_root, filename)
242

243 244
  @property
  def output_proc(self):
245
    if self._expected_exception is not None:
246
      return test262.ExceptionOutProc(self.expected_outcomes,
247 248 249 250 251 252
                                      self._expected_exception,
                                      self._fail_phase_reverse)
    else:
      # We only support fail phase reverse on tests that expect an exception.
      assert not self._fail_phase_reverse

253
    if self.expected_outcomes == outproc.OUTCOMES_PASS:
254 255
      return test262.PASS_NO_EXCEPTION
    return test262.NoExceptionOutProc(self.expected_outcomes)