standard_runner.py 13.4 KB
Newer Older
1 2 3 4 5 6 7 8
#!/usr/bin/env python
#
# Copyright 2017 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


import os
9
import re
10 11 12 13 14 15 16
import sys

# Adds testrunner to the path hence it has to be imported at the beggining.
import base_runner

from testrunner.local import utils
from testrunner.local.variants import ALL_VARIANTS
17
from testrunner.objects import predictable
18
from testrunner.testproc.execution import ExecutionProc
19
from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc
20
from testrunner.testproc.loader import LoadProc
21
from testrunner.testproc.progress import ResultsTracker, TestsCounter
22
from testrunner.testproc.seed import SeedProc
23
from testrunner.testproc.variant import VariantProc
24
from testrunner.utils import random_utils
25 26


27 28
ARCH_GUESS = utils.DefaultArch()

29 30 31
VARIANTS = ["default"]

MORE_VARIANTS = [
32
  "nooptimization",
33
  "stress",
34
  "stress_background_compile",
35
  "stress_incremental_marking",
36 37 38 39 40 41 42
]

VARIANT_ALIASES = {
  # The default for developer workstations.
  "dev": VARIANTS,
  # Additional variants, run on all bots.
  "more": MORE_VARIANTS,
43 44
  # Shortcut for the two above ("more" first - it has the longer running tests).
  "exhaustive": MORE_VARIANTS + VARIANTS,
45
  # Additional variants, run on a subset of bots.
46
  "extra": ["future", "no_liftoff", "no_wasm_traps", "trusted"],
47 48 49 50 51 52 53
}

GC_STRESS_FLAGS = ["--gc-interval=500", "--stress-compaction",
                   "--concurrent-recompilation-queue-length=64",
                   "--concurrent-recompilation-delay=500",
                   "--concurrent-recompilation"]

54 55 56
RANDOM_GC_STRESS_FLAGS = ["--random-gc-interval=5000",
                          "--stress-compaction-random"]

57

58 59 60
PREDICTABLE_WRAPPER = os.path.join(
    base_runner.BASE_DIR, 'tools', 'predictable_wrapper.py')

61 62

class StandardTestRunner(base_runner.BaseTestRunner):
63 64
    def __init__(self, *args, **kwargs):
        super(StandardTestRunner, self).__init__(*args, **kwargs)
65

66
        self.sancov_dir = None
67
        self._variants = None
68

69 70 71
    def _get_default_suite_names(self):
      return ['default']

72 73
    def _add_parser_options(self, parser):
      parser.add_option("--novfp3",
74 75 76
                        help="Indicates that V8 was compiled without VFP3"
                        " support",
                        default=False, action="store_true")
77 78

      # Variants
79
      parser.add_option("--no-variants", "--novariants",
80 81
                        help="Deprecated. "
                             "Equivalent to passing --variants=default",
82
                        default=False, dest="no_variants", action="store_true")
83
      parser.add_option("--variants",
84 85
                        help="Comma-separated list of testing variants;"
                        " default: \"%s\"" % ",".join(VARIANTS))
86
      parser.add_option("--exhaustive-variants",
87
                        default=False, action="store_true",
88 89
                        help="Deprecated. "
                             "Equivalent to passing --variants=exhaustive")
90 91 92 93 94 95

      # Filters
      parser.add_option("--slow-tests", default="dontcare",
                        help="Regard slow tests (run|skip|dontcare)")
      parser.add_option("--pass-fail-tests", default="dontcare",
                        help="Regard pass|fail tests (run|skip|dontcare)")
96
      parser.add_option("--quickcheck", default=False, action="store_true",
97
                        help=("Quick check mode (skip slow tests)"))
98
      parser.add_option("--dont-skip-slow-simulator-tests",
99 100 101 102
                        help="Don't skip more slow tests when using a"
                        " simulator.",
                        default=False, action="store_true",
                        dest="dont_skip_simulator_slow_tests")
103 104 105 106

      # Stress modes
      parser.add_option("--gc-stress",
                        help="Switch on GC stress mode",
107
                        default=False, action="store_true")
108 109
      parser.add_option("--random-gc-stress",
                        help="Switch on random GC stress mode",
110
                        default=False, action="store_true")
111
      parser.add_option("--random-seed-stress-count", default=1, type="int",
112
                        dest="random_seed_stress_count",
113 114 115
                        help="Number of runs with different random seeds. Only "
                             "with test processors: 0 means infinite "
                             "generation.")
116

117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150
      # Noop
      parser.add_option("--cfi-vptr",
                        help="Run tests with UBSAN cfi_vptr option.",
                        default=False, action="store_true")
      parser.add_option("--infra-staging", help="Use new test runner features",
                        dest='infra_staging', default=None,
                        action="store_true")
      parser.add_option("--no-infra-staging",
                        help="Opt out of new test runner features",
                        dest='infra_staging', default=None,
                        action="store_false")
      parser.add_option("--no-sorting", "--nosorting",
                        help="Don't sort tests according to duration of last"
                        " run.",
                        default=False, dest="no_sorting", action="store_true")
      parser.add_option("--no-presubmit", "--nopresubmit",
                        help='Skip presubmit checks (deprecated)',
                        default=False, dest="no_presubmit", action="store_true")

      # Unimplemented for test processors
      parser.add_option("--sancov-dir",
                        help="Directory where to collect coverage data")
      parser.add_option("--cat", help="Print the source of the tests",
                        default=False, action="store_true")
      parser.add_option("--flakiness-results",
                        help="Path to a file for storing flakiness json.")
      parser.add_option("--time", help="Print timing information after running",
                        default=False, action="store_true")
      parser.add_option("--warn-unused", help="Report unused rules",
                        default=False, action="store_true")
      parser.add_option("--report", default=False, action="store_true",
                        help="Print a summary of the tests to be run")


151
    def _process_options(self, options):
152 153 154 155 156
      if options.sancov_dir:
        self.sancov_dir = options.sancov_dir
        if not os.path.exists(self.sancov_dir):
          print("sancov-dir %s doesn't exist" % self.sancov_dir)
          raise base_runner.TestRunnerError()
157 158 159 160

      if options.gc_stress:
        options.extra_flags += GC_STRESS_FLAGS

161 162 163
      if options.random_gc_stress:
        options.extra_flags += RANDOM_GC_STRESS_FLAGS

164
      if self.build_config.asan:
165 166 167
        options.extra_flags.append("--invoke-weak-callbacks")
        options.extra_flags.append("--omit-quit")

168 169 170 171 172
      if self.build_config.no_snap:
        # Speed up slow nosnap runs. Allocation verification is covered by
        # running mksnapshot on other builders.
        options.extra_flags.append("--no-turbo-verify-allocation")

173 174 175
      if options.novfp3:
        options.extra_flags.append("--noenable-vfp3")

176
      if options.no_variants:  # pragma: no cover
177 178 179 180 181
        print ("Option --no-variants is deprecated. "
               "Pass --variants=default instead.")
        assert not options.variants
        options.variants = "default"

182
      if options.exhaustive_variants:  # pragma: no cover
183 184 185
        # TODO(machenbach): Switch infra to --variants=exhaustive after M65.
        print ("Option --exhaustive-variants is deprecated. "
               "Pass --variants=exhaustive instead.")
186 187 188
        # This is used on many bots. It includes a larger set of default
        # variants.
        # Other options for manipulating variants still apply afterwards.
189 190 191 192 193 194 195 196 197 198 199 200 201 202
        assert not options.variants
        options.variants = "exhaustive"

      if options.quickcheck:
        assert not options.variants
        options.variants = "stress,default"
        options.slow_tests = "skip"
        options.pass_fail_tests = "skip"

      if self.build_config.predictable:
        options.variants = "default"
        options.extra_flags.append("--predictable")
        options.extra_flags.append("--verify_predictable")
        options.extra_flags.append("--no-inline-new")
203
        # Add predictable wrapper to command prefix.
204 205
        options.command_prefix = (
            [sys.executable, PREDICTABLE_WRAPPER] + options.command_prefix)
206 207 208

      # TODO(machenbach): Figure out how to test a bigger subset of variants on
      # msan.
209
      if self.build_config.msan:
210
        options.variants = "default"
211

212 213
      if options.variants == "infra_staging":
        options.variants = "exhaustive"
214

215
      self._variants = self._parse_variants(options.variants)
216

217
      def CheckTestMode(name, option):  # pragma: no cover
218 219
        if not option in ["run", "skip", "dontcare"]:
          print "Unknown %s mode %s" % (name, option)
220 221 222
          raise base_runner.TestRunnerError()
      CheckTestMode("slow test", options.slow_tests)
      CheckTestMode("pass|fail test", options.pass_fail_tests)
223
      if self.build_config.no_i18n:
224 225
        base_runner.TEST_MAP["bot_default"].remove("intl")
        base_runner.TEST_MAP["default"].remove("intl")
226 227
        # TODO(machenbach): uncomment after infra side lands.
        # base_runner.TEST_MAP["d8_default"].remove("intl")
228

229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
    def _parse_variants(self, aliases_str):
      # Use developer defaults if no variant was specified.
      aliases_str = aliases_str or 'dev'
      aliases = aliases_str.split(',')
      user_variants = set(reduce(
          list.__add__, [VARIANT_ALIASES.get(a, [a]) for a in aliases]))

      result = [v for v in ALL_VARIANTS if v in user_variants]
      if len(result) == len(user_variants):
        return result

      for v in user_variants:
        if v not in ALL_VARIANTS:
          print 'Unknown variant: %s' % v
          raise base_runner.TestRunnerError()
      assert False, 'Unreachable'

246 247
    def _setup_env(self):
      super(StandardTestRunner, self)._setup_env()
248

249
      symbolizer_option = self._get_external_symbolizer_option()
250

251
      if self.sancov_dir:
252 253
        os.environ['ASAN_OPTIONS'] = ":".join([
          'coverage=1',
254 255
          'coverage_dir=%s' % self.sancov_dir,
          symbolizer_option,
256 257 258
          "allow_user_segv_handler=1",
        ])

259 260 261 262
    def _get_statusfile_variables(self, options):
      variables = (
          super(StandardTestRunner, self)._get_statusfile_variables(options))

263 264 265 266 267
      simulator_run = (
        not options.dont_skip_simulator_slow_tests and
        self.build_config.arch in [
          'arm64', 'arm', 'mipsel', 'mips', 'mips64', 'mips64el', 'ppc',
          'ppc64', 's390', 's390x'] and
268 269 270 271 272
        bool(ARCH_GUESS) and
        self.build_config.arch != ARCH_GUESS)

      variables.update({
        'gc_stress': options.gc_stress or options.random_gc_stress,
273
        'gc_fuzzer': options.random_gc_stress,
274 275 276 277 278
        'novfp3': options.novfp3,
        'simulator_run': simulator_run,
      })
      return variables

279
    def _do_execute(self, tests, args, options):
280 281 282 283
      jobs = options.j

      print '>>> Running with test processors'
      loader = LoadProc()
284 285
      tests_counter = TestsCounter()
      results = ResultsTracker()
286
      indicators = self._create_progress_indicators(options)
287 288 289 290

      outproc_factory = None
      if self.build_config.predictable:
        outproc_factory = predictable.get_outproc
291
      execproc = ExecutionProc(jobs, outproc_factory)
292
      sigproc = self._create_signal_proc()
293

294 295
      procs = [
        loader,
296
        NameFilterProc(args) if args else None,
297 298
        StatusFileFilterProc(options.slow_tests, options.pass_fail_tests),
        self._create_shard_proc(options),
299
        tests_counter,
300
        VariantProc(self._variants),
301
        StatusFileFilterProc(options.slow_tests, options.pass_fail_tests),
302
        self._create_predictable_filter(),
303
        self._create_seed_proc(options),
304
        sigproc,
305
      ] + indicators + [
306
        results,
307
        self._create_timeout_proc(options),
308
        self._create_rerun_proc(options),
309 310
        execproc,
      ]
311

312
      self._prepare_procs(procs)
313
      tests.sort(key=lambda t: t.is_slow, reverse=True)
314

315
      loader.load_tests(tests)
316

317 318 319
      print '>>> Running %d base tests' % tests_counter.total
      tests_counter.remove_from_chain()

320 321 322
      # This starts up worker processes and blocks until all tests are
      # processed.
      execproc.run()
323

324 325 326
      for indicator in indicators:
        indicator.finished()

327
      print '>>> %d tests ran' % (results.total - results.remaining)
328

329
      exit_code = utils.EXIT_CODE_PASS
330
      if results.failed:
331
        exit_code = utils.EXIT_CODE_FAILURES
332
      if not results.total:
333
        exit_code = utils.EXIT_CODE_NO_TESTS
334

335 336 337 338
      # Indicate if a SIGINT or SIGTERM happened.
      exit_code = max(exit_code, sigproc.exit_code)

      if exit_code == utils.EXIT_CODE_FAILURES and options.json_test_results:
339 340
        print("Force exit code 0 after failures. Json test results file "
              "generated with failure information.")
341
        exit_code = utils.EXIT_CODE_PASS
342 343
      return exit_code

344 345 346 347 348
    def _create_predictable_filter(self):
      if not self.build_config.predictable:
        return None
      return predictable.PredictableFilterProc()

349
    def _create_seed_proc(self, options):
350
      if options.random_seed_stress_count == 1:
351
        return None
352 353
      return SeedProc(options.random_seed_stress_count, options.random_seed,
                      options.j * 4)
354

355 356 357

if __name__ == '__main__':
  sys.exit(StandardTestRunner().execute())