num_fuzzer.py 8.26 KB
Newer Older
1 2 3 4 5 6
#!/usr/bin/env python
#
# Copyright 2017 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

7
# for py2/py3 compatibility
8
from __future__ import absolute_import
9
from __future__ import print_function
10 11 12 13 14

import random
import sys

# Adds testrunner to the path hence it has to be imported at the beggining.
15
from . import base_runner
16 17 18 19 20

from testrunner.local import utils

from testrunner.testproc import fuzzer
from testrunner.testproc.base import TestProcProducer
21
from testrunner.testproc.combiner import CombinerProc
22
from testrunner.testproc.execution import ExecutionProc
23
from testrunner.testproc.expectation import ForgiveTimeoutProc
24 25
from testrunner.testproc.filter import StatusFileFilterProc, NameFilterProc
from testrunner.testproc.loader import LoadProc
26
from testrunner.testproc.progress import ResultsTracker
27
from testrunner.utils import random_utils
28 29 30 31 32 33 34 35 36


DEFAULT_SUITES = ["mjsunit", "webkit", "benchmarks"]


class NumFuzzer(base_runner.BaseTestRunner):
  def __init__(self, *args, **kwargs):
    super(NumFuzzer, self).__init__(*args, **kwargs)

37 38 39 40
  @property
  def framework_name(self):
    return 'num_fuzzer'

41 42 43 44
  def _add_parser_options(self, parser):
    parser.add_option("--fuzzer-random-seed", default=0,
                      help="Default seed for initializing fuzzer random "
                      "generator")
45 46 47 48 49 50
    parser.add_option("--tests-count", default=5, type="int",
                      help="Number of tests to generate from each base test. "
                           "Can be combined with --total-timeout-sec with "
                           "value 0 to provide infinite number of subtests. "
                           "When --combine-tests is set it indicates how many "
                           "tests to create in total")
51

52
    # Stress gc
53 54 55 56 57 58 59 60 61 62 63 64
    parser.add_option("--stress-marking", default=0, type="int",
                      help="probability [0-10] of adding --stress-marking "
                           "flag to the test")
    parser.add_option("--stress-scavenge", default=0, type="int",
                      help="probability [0-10] of adding --stress-scavenge "
                           "flag to the test")
    parser.add_option("--stress-compaction", default=0, type="int",
                      help="probability [0-10] of adding --stress-compaction "
                           "flag to the test")
    parser.add_option("--stress-gc", default=0, type="int",
                      help="probability [0-10] of adding --random-gc-interval "
                           "flag to the test")
65 66 67 68 69

    # Stress tasks
    parser.add_option("--stress-delay-tasks", default=0, type="int",
                      help="probability [0-10] of adding --stress-delay-tasks "
                           "flag to the test")
70 71 72
    parser.add_option("--stress-thread-pool-size", default=0, type="int",
                      help="probability [0-10] of adding --thread-pool-size "
                           "flag to the test")
73 74

    # Stress deopt
75 76 77 78 79 80
    parser.add_option("--stress-deopt", default=0, type="int",
                      help="probability [0-10] of adding --deopt-every-n-times "
                           "flag to the test")
    parser.add_option("--stress-deopt-min", default=1, type="int",
                      help="extends --stress-deopt to have minimum interval "
                           "between deopt points")
81

82 83 84 85
    # Combine multiple tests
    parser.add_option("--combine-tests", default=False, action="store_true",
                      help="Combine multiple tests as one and run with "
                           "try-catch wrapper")
86 87 88 89
    parser.add_option("--combine-max", default=100, type="int",
                      help="Maximum number of tests to combine")
    parser.add_option("--combine-min", default=2, type="int",
                      help="Minimum number of tests to combine")
90

91 92 93 94
    # Miscellaneous
    parser.add_option("--variants", default='default',
                      help="Comma-separated list of testing variants")

95 96 97 98
    return parser


  def _process_options(self, options):
99 100
    if not options.fuzzer_random_seed:
      options.fuzzer_random_seed = random_utils.random_seed()
101 102 103 104

    if options.total_timeout_sec:
      options.tests_count = 0

105 106
    if options.combine_tests:
      if options.combine_min > options.combine_max:
107 108
        print(('min_group_size (%d) cannot be larger than max_group_size (%d)' %
               options.min_group_size, options.max_group_size))
109 110
        raise base_runner.TestRunnerError()

111 112
    if options.variants != 'default':
      print ('Only default testing variant is supported with numfuzz')
113
      raise base_runner.TestRunnerError()
114

115 116 117 118 119
    return True

  def _get_default_suite_names(self):
    return DEFAULT_SUITES

120 121 122 123 124
  def _get_statusfile_variables(self, options):
    variables = (
        super(NumFuzzer, self)._get_statusfile_variables(options))
    variables.update({
      'deopt_fuzzer': bool(options.stress_deopt),
125
      'endurance_fuzzer': bool(options.combine_tests),
126 127 128 129
      'gc_stress': bool(options.stress_gc),
      'gc_fuzzer': bool(max([options.stress_marking,
                             options.stress_scavenge,
                             options.stress_compaction,
130
                             options.stress_gc,
131
                             options.stress_delay_tasks,
132
                             options.stress_thread_pool_size])),
133 134
    })
    return variables
135

136
  def _do_execute(self, tests, args, options):
137
    loader = LoadProc(tests)
138 139
    fuzzer_rng = random.Random(options.fuzzer_random_seed)

140
    combiner = self._create_combiner(fuzzer_rng, options)
141
    results = self._create_result_tracker(options)
142
    execproc = ExecutionProc(options.j)
143
    sigproc = self._create_signal_proc()
144 145
    indicators = self._create_progress_indicators(
      tests.test_count_estimate, options)
146 147 148 149
    procs = [
      loader,
      NameFilterProc(args) if args else None,
      StatusFileFilterProc(None, None),
150 151
      # TODO(majeski): Improve sharding when combiner is present. Maybe select
      # different random seeds for shards instead of splitting tests.
152
      self._create_shard_proc(options),
153
      ForgiveTimeoutProc(),
154
      combiner,
155
      self._create_fuzzer(fuzzer_rng, options),
156
      sigproc,
157
    ] + indicators + [
158
      results,
159
      self._create_timeout_proc(options),
160
      self._create_rerun_proc(options),
161 162 163
      execproc,
    ]
    self._prepare_procs(procs)
Tamer Tas's avatar
Tamer Tas committed
164
    loader.load_initial_tests(initial_batch_size=float('inf'))
165 166 167 168

    # TODO(majeski): maybe some notification from loader would be better?
    if combiner:
      combiner.generate_initial_tests(options.j * 4)
169 170 171 172

    # This starts up worker processes and blocks until all tests are
    # processed.
    execproc.run()
173

174 175
    for indicator in indicators:
      indicator.finished()
176

177
    print('>>> %d tests ran' % results.total)
178
    if results.failed:
179
      return utils.EXIT_CODE_FAILURES
180

181 182
    # Indicate if a SIGINT or SIGTERM happened.
    return sigproc.exit_code
183

184 185
  def _is_testsuite_supported(self, suite, options):
    return not options.combine_tests or suite.test_combiner_available()
186

187 188 189 190 191 192 193 194 195
  def _create_combiner(self, rng, options):
    if not options.combine_tests:
      return None
    return CombinerProc(rng, options.combine_min, options.combine_max,
                        options.tests_count)

  def _create_fuzzer(self, rng, options):
    return fuzzer.FuzzerProc(
        rng,
196
        self._tests_count(options),
197
        self._create_fuzzer_configs(options),
198
        self._disable_analysis(options),
199 200
    )

201 202 203 204 205 206 207
  def _tests_count(self, options):
    if options.combine_tests:
      return 1
    return options.tests_count

  def _disable_analysis(self, options):
    """Disable analysis phase when options are used that don't support it."""
208
    return options.combine_tests
209

210 211
  def _create_fuzzer_configs(self, options):
    fuzzers = []
212 213 214 215 216 217 218 219
    def add(name, prob, *args):
      if prob:
        fuzzers.append(fuzzer.create_fuzzer_config(name, prob, *args))

    add('compaction', options.stress_compaction)
    add('marking', options.stress_marking)
    add('scavenge', options.stress_scavenge)
    add('gc_interval', options.stress_gc)
220
    add('threads', options.stress_thread_pool_size)
221
    add('delay', options.stress_delay_tasks)
222
    add('deopt', options.stress_deopt, options.stress_deopt_min)
223 224 225 226 227
    return fuzzers


if __name__ == '__main__':
  sys.exit(NumFuzzer().execute())