testcfg.py 9.3 KB
Newer Older
1
# Copyright 2008 the V8 project authors. All rights reserved.
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#     * Neither the name of Google Inc. nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

28 29
from collections import OrderedDict
import itertools
30 31
import os
import re
32

33
from testrunner.local import statusfile
34
from testrunner.local import testsuite
35
from testrunner.objects import testcase
36
from testrunner.outproc import base as outproc
37

38 39 40 41 42
try:
  basestring       # Python 2
except NameError:  # Python 3
  basestring = str

43
FILES_PATTERN = re.compile(r"//\s+Files:(.*)")
44
ENV_PATTERN = re.compile(r"//\s+Environment Variables:(.*)")
45
SELF_SCRIPT_PATTERN = re.compile(r"//\s+Env: TEST_FILE_NAME")
46
NO_HARNESS_PATTERN = re.compile(r"^// NO HARNESS$", flags=re.MULTILINE)
47

48

49 50
# Flags known to misbehave when combining arbitrary mjsunit tests. Can also
# be compiled regular expressions.
51
MISBEHAVING_COMBINED_TESTS_FLAGS= [
52
  '--check-handle-count',
53 54 55
  '--enable-tracing',
  re.compile('--experimental.*'),
  '--expose-trigger-failure',
56
  re.compile('--harmony.*'),
57
  '--mock-arraybuffer-allocator',
58
  '--print-ast',
59
  re.compile('--trace.*'),
60 61
  '--wasm-lazy-compilation',
]
62

63 64 65 66 67 68

class TestLoader(testsuite.JSTestLoader):
  @property
  def excluded_files(self):
    return {
      "mjsunit.js",
69
      "mjsunit_numfuzz.js",
70 71 72
    }


73
class TestSuite(testsuite.TestSuite):
74 75
  def _test_loader_class(self):
    return TestLoader
76

77 78 79
  def _test_combiner_class(self):
    return TestCombiner

80
  def _test_class(self):
81
    return TestCase
82 83


84
class TestCase(testcase.D8TestCase):
85
  def __init__(self, *args, **kwargs):
86
    super(TestCase, self).__init__(*args, **kwargs)
87

88
    source = self.get_source()
89 90 91 92 93 94 95 96 97 98

    files_list = []  # List of file names to append to command arguments.
    files_match = FILES_PATTERN.search(source);
    # Accept several lines of 'Files:'.
    while True:
      if files_match:
        files_list += files_match.group(1).strip().split()
        files_match = FILES_PATTERN.search(source, files_match.end())
      else:
        break
99
    files = [ os.path.normpath(os.path.join(self.suite.root, '..', '..', f))
100
              for f in files_list ]
101
    testfilename = self._get_source_path()
102
    if SELF_SCRIPT_PATTERN.search(source):
103 104 105
      files = (
        ["-e", "TEST_FILE_NAME=\"%s\"" % testfilename.replace("\\", "\\\\")] +
        files)
106

107 108 109 110
    if NO_HARNESS_PATTERN.search(source):
      mjsunit_files = []
    else:
      mjsunit_files = [os.path.join(self.suite.root, "mjsunit.js")]
111

112 113 114
    if self.suite.framework_name == 'num_fuzzer':
      mjsunit_files.append(os.path.join(self.suite.root, "mjsunit_numfuzz.js"))

115 116 117
    self._source_files = files
    self._source_flags = self._parse_source_flags(source)
    self._mjsunit_files = mjsunit_files
118
    self._files_suffix = [testfilename]
119 120 121
    self._env = self._parse_source_env(source)

  def _parse_source_env(self, source):
122
    env_match = ENV_PATTERN.search(source)
123
    env = {}
124 125 126
    if env_match:
      for env_pair in env_match.group(1).strip().split():
        var, value = env_pair.split('=')
127 128
        env[var] = value
    return env
129

130 131 132
  def _get_source_flags(self):
    return self._source_flags

133
  def _get_files_params(self):
134
    files = list(self._source_files)
135
    if not self._test_config.no_harness:
136 137
      files += self._mjsunit_files
    files += self._files_suffix
138
    if self._test_config.isolates:
139 140 141 142 143 144
      files += ['--isolate'] + files

    return files

  def _get_cmd_env(self):
    return self._env
145

146
  def _get_source_path(self):
147 148 149 150 151 152 153
    base_path = os.path.join(self.suite.root, self.path)
    # Try .js first, and fall back to .mjs.
    # TODO(v8:9406): clean this up by never separating the path from
    # the extension in the first place.
    if os.path.exists(base_path + self._get_suffix()):
      return base_path + self._get_suffix()
    return base_path + '.mjs'
154 155


156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181
class TestCombiner(testsuite.TestCombiner):
  def get_group_key(self, test):
    """Combine tests with the same set of flags.
    Ignore:
    1. Some special cases where it's not obvious what to pass in the command.
    2. Tests with flags that can cause failure even inside try-catch wrapper.
    3. Tests that use async functions. Async functions can be scheduled after
      exiting from try-catch wrapper and cause failure.
    """
    if (len(test._files_suffix) > 1 or
        test._env or
        not test._mjsunit_files or
        test._source_files):
      return None

    source_flags = test._get_source_flags()
    if ('--expose-trigger-failure' in source_flags or
        '--throws' in source_flags):
      return None

    source_code = test.get_source()
    # Maybe we could just update the tests to await all async functions they
    # call?
    if 'async' in source_code:
      return None

182 183 184
    # TODO(machenbach): Remove grouping if combining tests in a flag-independent
    # way works well.
    return 1
185 186 187 188 189

  def _combined_test_class(self):
    return CombinedTest


190
class CombinedTest(testcase.D8TestCase):
191 192 193 194 195 196
  """Behaves like normal mjsunit tests except:
    1. Expected outcome is always PASS
    2. Instead of one file there is a try-catch wrapper with all combined tests
      passed as arguments.
  """
  def __init__(self, name, tests):
197 198
    super(CombinedTest, self).__init__(tests[0].suite, '', name,
                                       tests[0]._test_config)
199 200 201
    self._tests = tests

  def _prepare_outcomes(self, force_update=True):
202 203
    self._statusfile_outcomes = outproc.OUTCOMES_PASS_OR_TIMEOUT
    self.expected_outcomes = outproc.OUTCOMES_PASS_OR_TIMEOUT
204

205
  def _get_shell_flags(self):
206 207 208
    """In addition to standard set of shell flags it appends:
      --disable-abortjs: %AbortJS can abort the test even inside
        trycatch-wrapper, so we disable it.
209
      --es-staging: We skip all harmony flags due to false positives,
210
          but always pass the staging flag to cover the mature features.
211
      --omit-quit: Calling quit() in JS would otherwise early terminate.
212 213 214
      --quiet-load: suppress any stdout from load() function used by
        trycatch-wrapper.
    """
215
    return [
216 217 218 219 220 221
      '--test',
      '--disable-abortjs',
      '--es-staging',
      '--omit-quit',
      '--quiet-load',
    ]
222

223
  def _get_cmd_params(self):
224
    return (
225
      super(CombinedTest, self)._get_cmd_params() +
226
      ['tools/testrunner/trycatch_loader.js', '--'] +
227 228
      self._tests[0]._mjsunit_files +
      ['--'] +
229 230 231
      [t._files_suffix[0] for t in self._tests]
    )

232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
  def _merge_flags(self, flags):
    """Merges flags from a list of flags.

    Flag values not starting with '-' are merged with the preceeding flag,
    e.g. --foo 1 will become --foo=1. All other flags remain the same.

    Returns: A generator of flags.
    """
    if not flags:
      return
    # Iterate over flag pairs. ['-'] is a sentinel value for the last iteration.
    for flag1, flag2 in itertools.izip(flags, flags[1:] + ['-']):
      if not flag2.startswith('-'):
        assert '=' not in flag1
        yield flag1 + '=' + flag2
      elif flag1.startswith('-'):
        yield flag1

250 251
  def _is_flag_blocked(self, flag):
    for item in MISBEHAVING_COMBINED_TESTS_FLAGS:
252 253 254 255 256 257
      if isinstance(item, basestring):
        if item == flag:
          return True
      elif item.match(flag):
        return True
    return False
258 259 260 261 262 263 264 265 266 267 268 269

  def _get_combined_flags(self, flags_gen):
    """Combines all flags - dedupes, keeps order and filters some flags.

    Args:
      flags_gen: Generator for flag lists.
    Returns: A list of flags.
    """
    merged_flags = self._merge_flags(list(itertools.chain(*flags_gen)))
    unique_flags = OrderedDict((flag, True) for flag in merged_flags).keys()
    return [
      flag for flag in unique_flags
270
      if not self._is_flag_blocked(flag)
271 272
    ]

273
  def _get_source_flags(self):
274 275 276
    # Combine flags from all source files.
    return self._get_combined_flags(
        test._get_source_flags() for test in self._tests)
277 278

  def _get_statusfile_flags(self):
279 280 281
    # Combine flags from all status file entries.
    return self._get_combined_flags(
        test._get_statusfile_flags() for test in self._tests)
282 283


284 285
def GetSuite(*args, **kwargs):
  return TestSuite(*args, **kwargs)