standard_runner.py 13.1 KB
Newer Older
1 2 3 4 5 6 7 8
#!/usr/bin/env python
#
# Copyright 2017 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


import os
9
import re
10 11 12 13 14 15 16
import sys

# Adds testrunner to the path hence it has to be imported at the beggining.
import base_runner

from testrunner.local import utils
from testrunner.local.variants import ALL_VARIANTS
17
from testrunner.objects import predictable
18
from testrunner.testproc.execution import ExecutionProc
19
from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc
20
from testrunner.testproc.loader import LoadProc
21
from testrunner.testproc.progress import ResultsTracker, TestsCounter
22
from testrunner.testproc.seed import SeedProc
23
from testrunner.testproc.variant import VariantProc
24
from testrunner.utils import random_utils
25 26


27 28
ARCH_GUESS = utils.DefaultArch()

29 30 31
VARIANTS = ["default"]

MORE_VARIANTS = [
32
  "nooptimization",
33
  "stress",
34
  "stress_background_compile",
35
  "stress_incremental_marking",
36 37 38 39 40 41 42 43
  "wasm_traps",
]

VARIANT_ALIASES = {
  # The default for developer workstations.
  "dev": VARIANTS,
  # Additional variants, run on all bots.
  "more": MORE_VARIANTS,
44 45
  # Shortcut for the two above ("more" first - it has the longer running tests).
  "exhaustive": MORE_VARIANTS + VARIANTS,
46
  # Additional variants, run on a subset of bots.
47
  "extra": ["future", "liftoff", "trusted", "wasm_no_native"],
48 49 50 51 52 53 54
}

GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction",
                   "--concurrent-recompilation-queue-length=64",
                   "--concurrent-recompilation-delay=500",
                   "--concurrent-recompilation"]

55 56 57
RANDOM_GC_STRESS_FLAGS = ["--random-gc-interval=5000",
                          "--stress-compaction-random"]

58

59 60 61
PREDICTABLE_WRAPPER = os.path.join(
    base_runner.BASE_DIR, 'tools', 'predictable_wrapper.py')

62 63

class StandardTestRunner(base_runner.BaseTestRunner):
64 65
    def __init__(self, *args, **kwargs):
        super(StandardTestRunner, self).__init__(*args, **kwargs)
66

67
        self.sancov_dir = None
68
        self._variants = None
69

70 71 72
    def _get_default_suite_names(self):
      return ['default']

73 74
    def _add_parser_options(self, parser):
      parser.add_option("--novfp3",
75 76 77
                        help="Indicates that V8 was compiled without VFP3"
                        " support",
                        default=False, action="store_true")
78 79

      # Variants
80
      parser.add_option("--no-variants", "--novariants",
81 82
                        help="Deprecated. "
                             "Equivalent to passing --variants=default",
83
                        default=False, dest="no_variants", action="store_true")
84
      parser.add_option("--variants",
85 86
                        help="Comma-separated list of testing variants;"
                        " default: \"%s\"" % ",".join(VARIANTS))
87
      parser.add_option("--exhaustive-variants",
88
                        default=False, action="store_true",
89 90
                        help="Deprecated. "
                             "Equivalent to passing --variants=exhaustive")
91 92 93 94 95 96

      # Filters
      parser.add_option("--slow-tests", default="dontcare",
                        help="Regard slow tests (run|skip|dontcare)")
      parser.add_option("--pass-fail-tests", default="dontcare",
                        help="Regard pass|fail tests (run|skip|dontcare)")
97
      parser.add_option("--quickcheck", default=False, action="store_true",
98
                        help=("Quick check mode (skip slow tests)"))
99
      parser.add_option("--dont-skip-slow-simulator-tests",
100 101 102 103
                        help="Don't skip more slow tests when using a"
                        " simulator.",
                        default=False, action="store_true",
                        dest="dont_skip_simulator_slow_tests")
104 105 106 107

      # Stress modes
      parser.add_option("--gc-stress",
                        help="Switch on GC stress mode",
108
                        default=False, action="store_true")
109 110
      parser.add_option("--random-gc-stress",
                        help="Switch on random GC stress mode",
111
                        default=False, action="store_true")
112
      parser.add_option("--random-seed-stress-count", default=1, type="int",
113
                        dest="random_seed_stress_count",
114 115 116
                        help="Number of runs with different random seeds. Only "
                             "with test processors: 0 means infinite "
                             "generation.")
117

118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151
      # Noop
      parser.add_option("--cfi-vptr",
                        help="Run tests with UBSAN cfi_vptr option.",
                        default=False, action="store_true")
      parser.add_option("--infra-staging", help="Use new test runner features",
                        dest='infra_staging', default=None,
                        action="store_true")
      parser.add_option("--no-infra-staging",
                        help="Opt out of new test runner features",
                        dest='infra_staging', default=None,
                        action="store_false")
      parser.add_option("--no-sorting", "--nosorting",
                        help="Don't sort tests according to duration of last"
                        " run.",
                        default=False, dest="no_sorting", action="store_true")
      parser.add_option("--no-presubmit", "--nopresubmit",
                        help='Skip presubmit checks (deprecated)',
                        default=False, dest="no_presubmit", action="store_true")

      # Unimplemented for test processors
      parser.add_option("--sancov-dir",
                        help="Directory where to collect coverage data")
      parser.add_option("--cat", help="Print the source of the tests",
                        default=False, action="store_true")
      parser.add_option("--flakiness-results",
                        help="Path to a file for storing flakiness json.")
      parser.add_option("--time", help="Print timing information after running",
                        default=False, action="store_true")
      parser.add_option("--warn-unused", help="Report unused rules",
                        default=False, action="store_true")
      parser.add_option("--report", default=False, action="store_true",
                        help="Print a summary of the tests to be run")


152
    def _process_options(self, options):
153 154 155 156 157
      if options.sancov_dir:
        self.sancov_dir = options.sancov_dir
        if not os.path.exists(self.sancov_dir):
          print("sancov-dir %s doesn't exist" % self.sancov_dir)
          raise base_runner.TestRunnerError()
158 159 160 161

      if options.gc_stress:
        options.extra_flags += GC_STRESS_FLAGS

162 163 164
      if options.random_gc_stress:
        options.extra_flags += RANDOM_GC_STRESS_FLAGS

165
      if self.build_config.asan:
166 167 168 169 170 171
        options.extra_flags.append("--invoke-weak-callbacks")
        options.extra_flags.append("--omit-quit")

      if options.novfp3:
        options.extra_flags.append("--noenable-vfp3")

172
      if options.no_variants:  # pragma: no cover
173 174 175 176 177
        print ("Option --no-variants is deprecated. "
               "Pass --variants=default instead.")
        assert not options.variants
        options.variants = "default"

178
      if options.exhaustive_variants:  # pragma: no cover
179 180 181
        # TODO(machenbach): Switch infra to --variants=exhaustive after M65.
        print ("Option --exhaustive-variants is deprecated. "
               "Pass --variants=exhaustive instead.")
182 183 184
        # This is used on many bots. It includes a larger set of default
        # variants.
        # Other options for manipulating variants still apply afterwards.
185 186 187 188 189 190 191 192 193 194 195 196 197 198
        assert not options.variants
        options.variants = "exhaustive"

      if options.quickcheck:
        assert not options.variants
        options.variants = "stress,default"
        options.slow_tests = "skip"
        options.pass_fail_tests = "skip"

      if self.build_config.predictable:
        options.variants = "default"
        options.extra_flags.append("--predictable")
        options.extra_flags.append("--verify_predictable")
        options.extra_flags.append("--no-inline-new")
199
        # Add predictable wrapper to command prefix.
200 201
        options.command_prefix = (
            [sys.executable, PREDICTABLE_WRAPPER] + options.command_prefix)
202 203 204

      # TODO(machenbach): Figure out how to test a bigger subset of variants on
      # msan.
205
      if self.build_config.msan:
206
        options.variants = "default"
207

208 209
      if options.variants == "infra_staging":
        options.variants = "exhaustive"
210

211
      self._variants = self._parse_variants(options.variants)
212

213
      def CheckTestMode(name, option):  # pragma: no cover
214 215
        if not option in ["run", "skip", "dontcare"]:
          print "Unknown %s mode %s" % (name, option)
216 217 218
          raise base_runner.TestRunnerError()
      CheckTestMode("slow test", options.slow_tests)
      CheckTestMode("pass|fail test", options.pass_fail_tests)
219
      if self.build_config.no_i18n:
220 221
        base_runner.TEST_MAP["bot_default"].remove("intl")
        base_runner.TEST_MAP["default"].remove("intl")
222 223
        # TODO(machenbach): uncomment after infra side lands.
        # base_runner.TEST_MAP["d8_default"].remove("intl")
224

225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241
    def _parse_variants(self, aliases_str):
      # Use developer defaults if no variant was specified.
      aliases_str = aliases_str or 'dev'
      aliases = aliases_str.split(',')
      user_variants = set(reduce(
          list.__add__, [VARIANT_ALIASES.get(a, [a]) for a in aliases]))

      result = [v for v in ALL_VARIANTS if v in user_variants]
      if len(result) == len(user_variants):
        return result

      for v in user_variants:
        if v not in ALL_VARIANTS:
          print 'Unknown variant: %s' % v
          raise base_runner.TestRunnerError()
      assert False, 'Unreachable'

242 243
    def _setup_env(self):
      super(StandardTestRunner, self)._setup_env()
244

245
      symbolizer_option = self._get_external_symbolizer_option()
246

247
      if self.sancov_dir:
248 249
        os.environ['ASAN_OPTIONS'] = ":".join([
          'coverage=1',
250 251
          'coverage_dir=%s' % self.sancov_dir,
          symbolizer_option,
252 253 254
          "allow_user_segv_handler=1",
        ])

255 256 257 258
    def _get_statusfile_variables(self, options):
      variables = (
          super(StandardTestRunner, self)._get_statusfile_variables(options))

259 260 261 262 263
      simulator_run = (
        not options.dont_skip_simulator_slow_tests and
        self.build_config.arch in [
          'arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', 'ppc',
          'ppc64', 's390', 's390x'] and
264 265 266 267 268 269 270 271 272 273
        bool(ARCH_GUESS) and
        self.build_config.arch != ARCH_GUESS)

      variables.update({
        'gc_stress': options.gc_stress or options.random_gc_stress,
        'novfp3': options.novfp3,
        'simulator_run': simulator_run,
      })
      return variables

274
    def _do_execute(self, tests, args, options):
275 276 277 278
      jobs = options.j

      print '>>> Running with test processors'
      loader = LoadProc()
279 280
      tests_counter = TestsCounter()
      results = ResultsTracker()
281
      indicators = self._create_progress_indicators(options)
282 283 284 285

      outproc_factory = None
      if self.build_config.predictable:
        outproc_factory = predictable.get_outproc
286
      execproc = ExecutionProc(jobs, outproc_factory)
287
      sigproc = self._create_signal_proc()
288

289 290
      procs = [
        loader,
291
        NameFilterProc(args) if args else None,
292 293
        StatusFileFilterProc(options.slow_tests, options.pass_fail_tests),
        self._create_shard_proc(options),
294
        tests_counter,
295
        VariantProc(self._variants),
296
        StatusFileFilterProc(options.slow_tests, options.pass_fail_tests),
297
        self._create_predictable_filter(),
298
        self._create_seed_proc(options),
299
        sigproc,
300
      ] + indicators + [
301
        results,
302
        self._create_timeout_proc(options),
303
        self._create_rerun_proc(options),
304 305
        execproc,
      ]
306

307
      self._prepare_procs(procs)
308
      tests.sort(key=lambda t: t.is_slow, reverse=True)
309

310
      loader.load_tests(tests)
311

312 313 314
      print '>>> Running %d base tests' % tests_counter.total
      tests_counter.remove_from_chain()

315 316 317
      # This starts up worker processes and blocks until all tests are
      # processed.
      execproc.run()
318

319 320 321
      for indicator in indicators:
        indicator.finished()

322
      print '>>> %d tests ran' % (results.total - results.remaining)
323

324
      exit_code = utils.EXIT_CODE_PASS
325
      if results.failed:
326
        exit_code = utils.EXIT_CODE_FAILURES
327
      if not results.total:
328
        exit_code = utils.EXIT_CODE_NO_TESTS
329

330 331 332 333
      # Indicate if a SIGINT or SIGTERM happened.
      exit_code = max(exit_code, sigproc.exit_code)

      if exit_code == utils.EXIT_CODE_FAILURES and options.json_test_results:
334 335
        print("Force exit code 0 after failures. Json test results file "
              "generated with failure information.")
336
        exit_code = utils.EXIT_CODE_PASS
337 338
      return exit_code

339 340 341 342 343
    def _create_predictable_filter(self):
      if not self.build_config.predictable:
        return None
      return predictable.PredictableFilterProc()

344
    def _create_seed_proc(self, options):
345
      if options.random_seed_stress_count == 1:
346
        return None
347 348
      return SeedProc(options.random_seed_stress_count, options.random_seed,
                      options.j * 4)
349

350 351 352

if __name__ == '__main__':
  sys.exit(StandardTestRunner().execute())