num_fuzzer.py 7.75 KB
Newer Older
1
#!/usr/bin/env python3
2 3 4 5 6 7 8 9
#
# Copyright 2017 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import sys

# Adds testrunner to the path hence it has to be imported at the beggining.
10
from testrunner import base_runner
11 12 13 14

from testrunner.local import utils

from testrunner.testproc import fuzzer
15
from testrunner.testproc.combiner import CombinerProc
16
from testrunner.testproc.execution import ExecutionProc
17
from testrunner.testproc.expectation import ExpectationProc
18 19
from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc
from testrunner.testproc.loader import LoadProc
20
from testrunner.utils import random_utils
21 22
from testrunner.testproc.rerun import RerunProc
from testrunner.testproc.timeout import TimeoutProc
23
from testrunner.testproc.progress import ResultsTracker, ProgressProc
24
from testrunner.testproc.shard import ShardProc
25 26 27 28 29 30 31 32 33


DEFAULT_SUITES = ["mjsunit", "webkit", "benchmarks"]


class NumFuzzer(base_runner.BaseTestRunner):
  def __init__(self, *args, **kwargs):
    super(NumFuzzer, self).__init__(*args, **kwargs)

34 35 36 37
  @property
  def framework_name(self):
    return 'num_fuzzer'

38 39 40 41
  def _add_parser_options(self, parser):
    parser.add_option("--fuzzer-random-seed", default=0,
                      help="Default seed for initializing fuzzer random "
                      "generator")
42 43 44 45 46 47
    parser.add_option("--tests-count", default=5, type="int",
                      help="Number of tests to generate from each base test. "
                           "Can be combined with --total-timeout-sec with "
                           "value 0 to provide infinite number of subtests. "
                           "When --combine-tests is set it indicates how many "
                           "tests to create in total")
48

49
    # Stress gc
50 51 52 53 54 55 56 57 58 59 60 61
    parser.add_option("--stress-marking", default=0, type="int",
                      help="probability [0-10] of adding --stress-marking "
                           "flag to the test")
    parser.add_option("--stress-scavenge", default=0, type="int",
                      help="probability [0-10] of adding --stress-scavenge "
                           "flag to the test")
    parser.add_option("--stress-compaction", default=0, type="int",
                      help="probability [0-10] of adding --stress-compaction "
                           "flag to the test")
    parser.add_option("--stress-gc", default=0, type="int",
                      help="probability [0-10] of adding --random-gc-interval "
                           "flag to the test")
62

63 64 65 66 67
    # Stress stack size
    parser.add_option("--stress-stack-size", default=0, type="int",
                      help="probability [0-10] of adding --stack-size "
                           "flag to the test")

68 69 70 71
    # Stress tasks
    parser.add_option("--stress-delay-tasks", default=0, type="int",
                      help="probability [0-10] of adding --stress-delay-tasks "
                           "flag to the test")
72 73 74
    parser.add_option("--stress-thread-pool-size", default=0, type="int",
                      help="probability [0-10] of adding --thread-pool-size "
                           "flag to the test")
75

76
    # Stress compiler
77 78 79 80 81 82
    parser.add_option("--stress-deopt", default=0, type="int",
                      help="probability [0-10] of adding --deopt-every-n-times "
                           "flag to the test")
    parser.add_option("--stress-deopt-min", default=1, type="int",
                      help="extends --stress-deopt to have minimum interval "
                           "between deopt points")
83 84 85
    parser.add_option("--stress-interrupt-budget", default=0, type="int",
                      help="probability [0-10] of adding the --interrupt-budget "
                           "flag to the test")
86

87 88 89 90
    # Combine multiple tests
    parser.add_option("--combine-tests", default=False, action="store_true",
                      help="Combine multiple tests as one and run with "
                           "try-catch wrapper")
91 92 93 94
    parser.add_option("--combine-max", default=100, type="int",
                      help="Maximum number of tests to combine")
    parser.add_option("--combine-min", default=2, type="int",
                      help="Minimum number of tests to combine")
95

96 97 98 99
    # Miscellaneous
    parser.add_option("--variants", default='default',
                      help="Comma-separated list of testing variants")

100 101 102
    return parser


103 104 105
  def _process_options(self):
    if not self.options.fuzzer_random_seed:
      self.options.fuzzer_random_seed = random_utils.random_seed()
106

107 108
    if self.options.total_timeout_sec:
      self.options.tests_count = 0
109

110 111
    if self.options.combine_tests:
      if self.options.combine_min > self.options.combine_max:
112
        print(('min_group_size (%d) cannot be larger than max_group_size (%d)' %
113
               self.options.min_group_size, self.options.max_group_size))
114 115
        raise base_runner.TestRunnerError()

116
    if self.options.variants != 'default':
117
      print ('Only default testing variant is supported with numfuzz')
118
      raise base_runner.TestRunnerError()
119

120 121 122 123 124
    return True

  def _get_default_suite_names(self):
    return DEFAULT_SUITES

125 126
  def _runner_flags(self):
    """Extra default flags specific to the test runner implementation."""
127
    return [
128
        '--exit-on-contradictory-flags', '--testing-d8-test-runner', '--no-fail'
129
    ]
130

131
  def _get_statusfile_variables(self):
132
    variables = (
133
        super(NumFuzzer, self)._get_statusfile_variables())
134
    variables.update({
135 136 137 138 139 140 141 142 143 144
      'deopt_fuzzer': bool(self.options.stress_deopt),
      'endurance_fuzzer': bool(self.options.combine_tests),
      'gc_stress': bool(self.options.stress_gc),
      'gc_fuzzer': bool(max([self.options.stress_marking,
                             self.options.stress_scavenge,
                             self.options.stress_compaction,
                             self.options.stress_gc,
                             self.options.stress_delay_tasks,
                             self.options.stress_stack_size,
                             self.options.stress_thread_pool_size])),
145 146
    })
    return variables
147

148
  def _do_execute(self, tests, args, ctx):
149
    loader = LoadProc(tests)
150 151
    combiner = CombinerProc.create(self.options)
    results = ResultsTracker.create(self.options)
152
    execproc = ExecutionProc(ctx, self.options.j)
153
    sigproc = self._create_signal_proc()
154
    progress = ProgressProc(ctx, self.options, self.framework_name,
155
                            tests.test_count_estimate)
156
    procs = [
157 158 159 160 161 162 163 164 165 166 167 168 169 170 171
        loader,
        NameFilterProc(args) if args else None,
        StatusFileFilterProc(None, None),
        # TODO(majeski): Improve sharding when combiner is present. Maybe select
        # different random seeds for shards instead of splitting tests.
        ShardProc.create(self.options),
        ExpectationProc(),
        combiner,
        fuzzer.FuzzerProc.create(self.options),
        sigproc,
        progress,
        results,
        TimeoutProc.create(self.options),
        RerunProc.create(self.options),
        execproc,
172
    ]
173 174
    procs = [p for p in procs if p]

175
    self._prepare_procs(procs)
176
    loader.load_initial_tests()
177 178 179

    # TODO(majeski): maybe some notification from loader would be better?
    if combiner:
180
      combiner.generate_initial_tests(self.options.j * 4)
181 182 183 184

    # This starts up worker processes and blocks until all tests are
    # processed.
    execproc.run()
185

186
    progress.finished()
187

188
    print('>>> %d tests ran' % results.total)
189
    if results.failed:
190
      return utils.EXIT_CODE_FAILURES
191

192 193
    # Indicate if a SIGINT or SIGTERM happened.
    return sigproc.exit_code
194

195 196
  def _is_testsuite_supported(self, suite):
    return not self.options.combine_tests or suite.test_combiner_available()
197 198 199


if __name__ == '__main__':
200
  sys.exit(NumFuzzer().execute()) # pragma: no cover