testsuite.py 12 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31
# Copyright 2012 the V8 project authors. All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
#     * Redistributions of source code must retain the above copyright
#       notice, this list of conditions and the following disclaimer.
#     * Redistributions in binary form must reproduce the above
#       copyright notice, this list of conditions and the following
#       disclaimer in the documentation and/or other materials provided
#       with the distribution.
#     * Neither the name of Google Inc. nor the names of its
#       contributors may be used to endorse or promote products derived
#       from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


import imp
import os

32
from . import commands
33
from . import statusfile
34
from . import utils
35
from ..objects import testcase
36 37
from variants import ALL_VARIANTS, ALL_VARIANT_FLAGS, FAST_VARIANT_FLAGS

38

39 40 41 42 43 44 45 46 47 48 49 50
FAST_VARIANTS = set(["default", "turbofan"])
STANDARD_VARIANT = set(["default"])


class VariantGenerator(object):
  def __init__(self, suite, variants):
    self.suite = suite
    self.all_variants = ALL_VARIANTS & variants
    self.fast_variants = FAST_VARIANTS & variants
    self.standard_variant = STANDARD_VARIANT & variants

  def FilterVariantsByTest(self, testcase):
51 52 53 54 55 56 57
    result = self.all_variants
    if testcase.outcomes:
      if statusfile.OnlyStandardVariant(testcase.outcomes):
        return self.standard_variant
      if statusfile.OnlyFastVariants(testcase.outcomes):
        result = self.fast_variants
    return result
58 59 60 61 62 63

  def GetFlagSets(self, testcase, variant):
    if testcase.outcomes and statusfile.OnlyFastVariants(testcase.outcomes):
      return FAST_VARIANT_FLAGS[variant]
    else:
      return ALL_VARIANT_FLAGS[variant]
64 65


66 67 68
class TestSuite(object):

  @staticmethod
69
  def LoadTestSuite(root, global_init=True):
70 71 72 73 74
    name = root.split(os.path.sep)[-1]
    f = None
    try:
      (f, pathname, description) = imp.find_module("testcfg", [root])
      module = imp.load_module("testcfg", f, pathname, description)
75
      return module.GetSuite(name, root)
76
    except ImportError:
77 78
      # Use default if no testcfg is present.
      return GoogleTestSuite(name, root)
79 80 81 82 83
    finally:
      if f:
        f.close()

  def __init__(self, name, root):
84
    # Note: This might be called concurrently from different processes.
85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
    self.name = name  # string
    self.root = root  # string containing path
    self.tests = None  # list of TestCase objects
    self.rules = None  # dictionary mapping test path to list of outcomes
    self.wildcards = None  # dictionary mapping test paths to list of outcomes
    self.total_duration = None  # float, assigned on demand

  def shell(self):
    return "d8"

  def suffix(self):
    return ".js"

  def status_file(self):
    return "%s/%s.status" % (self.root, self.name)

  # Used in the status file and for stdout printing.
  def CommonTestName(self, testcase):
103 104 105 106
    if utils.IsWindows():
      return testcase.path.replace("\\", "/")
    else:
      return testcase.path
107 108 109 110

  def ListTests(self, context):
    raise NotImplementedError

111 112 113 114 115 116 117 118 119 120 121 122 123
  def _VariantGeneratorFactory(self):
    """The variant generator class to be used."""
    return VariantGenerator

  def CreateVariantGenerator(self, variants):
    """Return a generator for the testing variants of this suite.

    Args:
      variants: List of variant names to be run as specified by the test
                runner.
    Returns: An object of type VariantGenerator.
    """
    return self._VariantGeneratorFactory()(self, set(variants))
124

125 126 127 128 129 130 131 132
  def PrepareSources(self):
    """Called once before multiprocessing for doing file-system operations.

    This should not access the network. For network access use the method
    below.
    """
    pass

133 134 135 136
  def DownloadData(self):
    pass

  def ReadStatusFile(self, variables):
137 138 139
    with open(self.status_file()) as f:
      self.rules, self.wildcards = (
          statusfile.ReadStatusFile(f.read(), variables))
140 141 142 143

  def ReadTestCases(self, context):
    self.tests = self.ListTests(context)

144 145 146 147 148 149 150 151 152 153
  @staticmethod
  def _FilterSlow(slow, mode):
    return (mode == "run" and not slow) or (mode == "skip" and slow)

  @staticmethod
  def _FilterPassFail(pass_fail, mode):
    return (mode == "run" and not pass_fail) or (mode == "skip" and pass_fail)

  def FilterTestCasesByStatus(self, warn_unused_rules,
                              slow_tests="dontcare",
154 155 156 157 158 159 160 161 162 163 164 165 166 167 168
                              pass_fail_tests="dontcare",
                              variants=False):

    # Use only variants-dependent rules and wildcards when filtering
    # respective test cases and generic rules when filtering generic test
    # cases.
    if not variants:
      rules = self.rules[""]
      wildcards = self.wildcards[""]
    else:
      # We set rules and wildcards to a variant-specific version for each test
      # below.
      rules = {}
      wildcards = {}

169
    filtered = []
170 171 172

    # Remember used rules as tuples of (rule, variant), where variant is "" for
    # variant-independent rules.
173
    used_rules = set()
174

175
    for t in self.tests:
176 177
      slow = False
      pass_fail = False
178
      testname = self.CommonTestName(t)
179 180 181 182 183 184
      variant = t.variant or ""
      if variants:
        rules = self.rules[variant]
        wildcards = self.wildcards[variant]
      if testname in rules:
        used_rules.add((testname, variant))
185 186
        # Even for skipped tests, as the TestCase object stays around and
        # PrintReport() uses it.
187
        t.outcomes = t.outcomes | rules[testname]
188
        if statusfile.DoSkip(t.outcomes):
189
          continue  # Don't add skipped tests to |filtered|.
190 191 192
        for outcome in t.outcomes:
          if outcome.startswith('Flags: '):
            t.flags += outcome[7:].split()
193 194
        slow = statusfile.IsSlow(t.outcomes)
        pass_fail = statusfile.IsPassOrFail(t.outcomes)
195
      skip = False
196
      for rule in wildcards:
197 198
        assert rule[-1] == '*'
        if testname.startswith(rule[:-1]):
199
          used_rules.add((rule, variant))
200
          t.outcomes = t.outcomes | wildcards[rule]
201 202
          if statusfile.DoSkip(t.outcomes):
            skip = True
203
            break  # "for rule in wildcards"
204 205
          slow = slow or statusfile.IsSlow(t.outcomes)
          pass_fail = pass_fail or statusfile.IsPassOrFail(t.outcomes)
206
      if (skip
207 208
          or self._FilterSlow(slow, slow_tests)
          or self._FilterPassFail(pass_fail, pass_fail_tests)):
209
        continue  # "for t in self.tests"
210 211 212 213 214 215
      filtered.append(t)
    self.tests = filtered

    if not warn_unused_rules:
      return

216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
    if not variants:
      for rule in self.rules[""]:
        if (rule, "") not in used_rules:
          print("Unused rule: %s -> %s (variant independent)" % (
              rule, self.rules[""][rule]))
      for rule in self.wildcards[""]:
        if (rule, "") not in used_rules:
          print("Unused rule: %s -> %s (variant independent)" % (
              rule, self.wildcards[""][rule]))
    else:
      for variant in ALL_VARIANTS:
        for rule in self.rules[variant]:
          if (rule, variant) not in used_rules:
            print("Unused rule: %s -> %s (variant: %s)" % (
                rule, self.rules[variant][rule], variant))
        for rule in self.wildcards[variant]:
          if (rule, variant) not in used_rules:
            print("Unused rule: %s -> %s (variant: %s)" % (
                rule, self.wildcards[variant][rule], variant))

236 237

  def FilterTestCasesByArgs(self, args):
238 239 240 241 242 243
    """Filter test cases based on command-line arguments.

    An argument with an asterisk in the end will match all test cases
    that have the argument as a prefix. Without asterisk, only exact matches
    will be used with the exeption of the test-suite name as argument.
    """
244
    filtered = []
245 246
    globs = []
    exact_matches = []
247
    for a in args:
248
      argpath = a.split('/')
249 250 251 252
      if argpath[0] != self.name:
        continue
      if len(argpath) == 1 or (len(argpath) == 2 and argpath[1] == '*'):
        return  # Don't filter, run all tests in this suite.
253
      path = '/'.join(argpath[1:])
254 255
      if path[-1] == '*':
        path = path[:-1]
256 257 258
        globs.append(path)
      else:
        exact_matches.append(path)
259
    for t in self.tests:
260
      for a in globs:
261 262 263
        if t.path.startswith(a):
          filtered.append(t)
          break
264 265 266 267
      for a in exact_matches:
        if t.path == a:
          filtered.append(t)
          break
268 269 270 271 272 273 274 275
    self.tests = filtered

  def GetFlagsForTestCase(self, testcase, context):
    raise NotImplementedError

  def GetSourceForTest(self, testcase):
    return "(no source available)"

276 277
  def IsFailureOutput(self, testcase):
    return testcase.output.exit_code != 0
278 279 280 281 282

  def IsNegativeTest(self, testcase):
    return False

  def HasFailed(self, testcase):
283
    execution_failed = self.IsFailureOutput(testcase)
284 285 286 287 288
    if self.IsNegativeTest(testcase):
      return not execution_failed
    else:
      return execution_failed

289
  def GetOutcome(self, testcase):
290
    if testcase.output.HasCrashed():
291
      return statusfile.CRASH
292
    elif testcase.output.HasTimedOut():
293
      return statusfile.TIMEOUT
294
    elif self.HasFailed(testcase):
295
      return statusfile.FAIL
296
    else:
297 298 299 300 301
      return statusfile.PASS

  def HasUnexpectedOutput(self, testcase):
    outcome = self.GetOutcome(testcase)
    return not outcome in (testcase.outcomes or [statusfile.PASS])
302 303 304 305 306 307 308 309 310 311 312

  def StripOutputForTransmit(self, testcase):
    if not self.HasUnexpectedOutput(testcase):
      testcase.output.stdout = ""
      testcase.output.stderr = ""

  def CalculateTotalDuration(self):
    self.total_duration = 0.0
    for t in self.tests:
      self.total_duration += t.duration
    return self.total_duration
313 314


315 316 317 318 319
class StandardVariantGenerator(VariantGenerator):
  def FilterVariantsByTest(self, testcase):
    return self.standard_variant


320 321 322 323 324 325 326 327
class GoogleTestSuite(TestSuite):
  def __init__(self, name, root):
    super(GoogleTestSuite, self).__init__(name, root)

  def ListTests(self, context):
    shell = os.path.abspath(os.path.join(context.shell_dir, self.shell()))
    if utils.IsWindows():
      shell += ".exe"
328 329 330 331 332 333 334 335 336

    output = None
    for i in xrange(3): # Try 3 times in case of errors.
      output = commands.Execute(context.command_prefix +
                                [shell, "--gtest_list_tests"] +
                                context.extra_flags)
      if output.exit_code == 0:
        break
      print "Test executable failed to list the tests (try %d).\n\nStdout:" % i
337
      print output.stdout
338
      print "\nStderr:"
339
      print output.stderr
340 341
      print "\nExit code: %d" % output.exit_code
    else:
342
      raise Exception("Test executable failed to list the tests.")
343

344 345 346 347 348 349 350
    tests = []
    test_case = ''
    for line in output.stdout.splitlines():
      test_desc = line.strip().split()[0]
      if test_desc.endswith('.'):
        test_case = test_desc
      elif test_case and test_desc:
351
        test = testcase.TestCase(self, test_case + test_desc)
352
        tests.append(test)
353
    tests.sort(key=lambda t: t.path)
354 355 356 357 358 359 360 361
    return tests

  def GetFlagsForTestCase(self, testcase, context):
    return (testcase.flags + ["--gtest_filter=" + testcase.path] +
            ["--gtest_random_seed=%s" % context.random_seed] +
            ["--gtest_print_time=0"] +
            context.mode_flags)

362 363
  def _VariantGeneratorFactory(self):
    return StandardVariantGenerator
364

365 366
  def shell(self):
    return self.name