num_fuzzer.py 8.41 KB
Newer Older
1 2 3 4 5 6
#!/usr/bin/env python
#
# Copyright 2017 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

7
# for py2/py3 compatibility
8
from __future__ import absolute_import
9
from __future__ import print_function
10 11 12 13 14

import random
import sys

# Adds testrunner to the path hence it has to be imported at the beggining.
15
from . import base_runner
16 17 18 19 20

from testrunner.local import utils

from testrunner.testproc import fuzzer
from testrunner.testproc.base import TestProcProducer
21
from testrunner.testproc.combiner import CombinerProc
22
from testrunner.testproc.execution import ExecutionProc
23
from testrunner.testproc.expectation import ForgiveTimeoutProc
24 25
from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc
from testrunner.testproc.loader import LoadProc
26
from testrunner.testproc.progress import ResultsTracker
27
from testrunner.utils import random_utils
28 29 30 31 32 33 34 35 36


DEFAULT_SUITES = ["mjsunit", "webkit", "benchmarks"]


class NumFuzzer(base_runner.BaseTestRunner):
  def __init__(self, *args, **kwargs):
    super(NumFuzzer, self).__init__(*args, **kwargs)

37 38 39 40
  @property
  def framework_name(self):
    return 'num_fuzzer'

41 42 43 44
  def _add_parser_options(self, parser):
    parser.add_option("--fuzzer-random-seed", default=0,
                      help="Default seed for initializing fuzzer random "
                      "generator")
45 46 47 48 49 50
    parser.add_option("--tests-count", default=5, type="int",
                      help="Number of tests to generate from each base test. "
                           "Can be combined with --total-timeout-sec with "
                           "value 0 to provide infinite number of subtests. "
                           "When --combine-tests is set it indicates how many "
                           "tests to create in total")
51

52
    # Stress gc
53 54 55 56 57 58 59 60 61 62 63 64
    parser.add_option("--stress-marking", default=0, type="int",
                      help="probability [0-10] of adding --stress-marking "
                           "flag to the test")
    parser.add_option("--stress-scavenge", default=0, type="int",
                      help="probability [0-10] of adding --stress-scavenge "
                           "flag to the test")
    parser.add_option("--stress-compaction", default=0, type="int",
                      help="probability [0-10] of adding --stress-compaction "
                           "flag to the test")
    parser.add_option("--stress-gc", default=0, type="int",
                      help="probability [0-10] of adding --random-gc-interval "
                           "flag to the test")
65 66 67 68 69

    # Stress tasks
    parser.add_option("--stress-delay-tasks", default=0, type="int",
                      help="probability [0-10] of adding --stress-delay-tasks "
                           "flag to the test")
70 71 72
    parser.add_option("--stress-thread-pool-size", default=0, type="int",
                      help="probability [0-10] of adding --thread-pool-size "
                           "flag to the test")
73 74

    # Stress deopt
75 76 77 78 79 80
    parser.add_option("--stress-deopt", default=0, type="int",
                      help="probability [0-10] of adding --deopt-every-n-times "
                           "flag to the test")
    parser.add_option("--stress-deopt-min", default=1, type="int",
                      help="extends --stress-deopt to have minimum interval "
                           "between deopt points")
81

82 83 84 85
    # Combine multiple tests
    parser.add_option("--combine-tests", default=False, action="store_true",
                      help="Combine multiple tests as one and run with "
                           "try-catch wrapper")
86 87 88 89
    parser.add_option("--combine-max", default=100, type="int",
                      help="Maximum number of tests to combine")
    parser.add_option("--combine-min", default=2, type="int",
                      help="Minimum number of tests to combine")
90

91 92 93 94
    # Miscellaneous
    parser.add_option("--variants", default='default',
                      help="Comma-separated list of testing variants")

95 96 97 98
    return parser


  def _process_options(self, options):
99 100
    if not options.fuzzer_random_seed:
      options.fuzzer_random_seed = random_utils.random_seed()
101 102 103 104

    if options.total_timeout_sec:
      options.tests_count = 0

105 106
    if options.combine_tests:
      if options.combine_min > options.combine_max:
107 108
        print(('min_group_size (%d) cannot be larger than max_group_size (%d)' %
               options.min_group_size, options.max_group_size))
109 110
        raise base_runner.TestRunnerError()

111 112
    if options.variants != 'default':
      print ('Only default testing variant is supported with numfuzz')
113
      raise base_runner.TestRunnerError()
114

115 116 117 118 119
    return True

  def _get_default_suite_names(self):
    return DEFAULT_SUITES

120 121 122 123
  def _runner_flags(self):
    """Extra default flags specific to the test runner implementation."""
    return ['--no-abort-on-contradictory-flags']

124 125 126 127 128
  def _get_statusfile_variables(self, options):
    variables = (
        super(NumFuzzer, self)._get_statusfile_variables(options))
    variables.update({
      'deopt_fuzzer': bool(options.stress_deopt),
129
      'endurance_fuzzer': bool(options.combine_tests),
130 131 132 133
      'gc_stress': bool(options.stress_gc),
      'gc_fuzzer': bool(max([options.stress_marking,
                             options.stress_scavenge,
                             options.stress_compaction,
134
                             options.stress_gc,
135
                             options.stress_delay_tasks,
136
                             options.stress_thread_pool_size])),
137 138
    })
    return variables
139

140
  def _do_execute(self, tests, args, options):
141
    loader = LoadProc(tests)
142 143
    fuzzer_rng = random.Random(options.fuzzer_random_seed)

144
    combiner = self._create_combiner(fuzzer_rng, options)
145
    results = self._create_result_tracker(options)
146
    execproc = ExecutionProc(options.j)
147
    sigproc = self._create_signal_proc()
148 149
    indicators = self._create_progress_indicators(
      tests.test_count_estimate, options)
150 151 152 153
    procs = [
      loader,
      NameFilterProc(args) if args else None,
      StatusFileFilterProc(None, None),
154 155
      # TODO(majeski): Improve sharding when combiner is present. Maybe select
      # different random seeds for shards instead of splitting tests.
156
      self._create_shard_proc(options),
157
      ForgiveTimeoutProc(),
158
      combiner,
159
      self._create_fuzzer(fuzzer_rng, options),
160
      sigproc,
161
    ] + indicators + [
162
      results,
163
      self._create_timeout_proc(options),
164
      self._create_rerun_proc(options),
165 166 167
      execproc,
    ]
    self._prepare_procs(procs)
Tamer Tas's avatar
Tamer Tas committed
168
    loader.load_initial_tests(initial_batch_size=float('inf'))
169 170 171 172

    # TODO(majeski): maybe some notification from loader would be better?
    if combiner:
      combiner.generate_initial_tests(options.j * 4)
173 174 175 176

    # This starts up worker processes and blocks until all tests are
    # processed.
    execproc.run()
177

178 179
    for indicator in indicators:
      indicator.finished()
180

181
    print('>>> %d tests ran' % results.total)
182
    if results.failed:
183
      return utils.EXIT_CODE_FAILURES
184

185 186
    # Indicate if a SIGINT or SIGTERM happened.
    return sigproc.exit_code
187

188 189
  def _is_testsuite_supported(self, suite, options):
    return not options.combine_tests or suite.test_combiner_available()
190

191 192 193 194 195 196 197 198 199
  def _create_combiner(self, rng, options):
    if not options.combine_tests:
      return None
    return CombinerProc(rng, options.combine_min, options.combine_max,
                        options.tests_count)

  def _create_fuzzer(self, rng, options):
    return fuzzer.FuzzerProc(
        rng,
200
        self._tests_count(options),
201
        self._create_fuzzer_configs(options),
202
        self._disable_analysis(options),
203 204
    )

205 206 207 208 209 210 211
  def _tests_count(self, options):
    if options.combine_tests:
      return 1
    return options.tests_count

  def _disable_analysis(self, options):
    """Disable analysis phase when options are used that don't support it."""
212
    return options.combine_tests
213

214 215
  def _create_fuzzer_configs(self, options):
    fuzzers = []
216 217 218 219 220 221 222 223
    def add(name, prob, *args):
      if prob:
        fuzzers.append(fuzzer.create_fuzzer_config(name, prob, *args))

    add('compaction', options.stress_compaction)
    add('marking', options.stress_marking)
    add('scavenge', options.stress_scavenge)
    add('gc_interval', options.stress_gc)
224
    add('threads', options.stress_thread_pool_size)
225
    add('delay', options.stress_delay_tasks)
226
    add('deopt', options.stress_deopt, options.stress_deopt_min)
227 228 229 230 231
    return fuzzers


if __name__ == '__main__':
  sys.exit(NumFuzzer().execute())