v8_foozzie.py 11.2 KB
Newer Older
1 2 3 4 5 6 7 8 9 10
#!/usr/bin/env python
# Copyright 2016 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
V8 correctness fuzzer launcher script.
"""

import argparse
11
import hashlib
12
import itertools
13
import json
14
import os
15
import random
16 17 18 19 20 21 22 23
import re
import sys
import traceback

import v8_commands
import v8_suppressions

CONFIGS = dict(
24
  default=[],
25 26
  ignition=[
    '--turbo-filter=~',
27
    '--noopt',
28 29
    '--liftoff',
    '--no-wasm-tier-up',
30
  ],
31 32
  ignition_asm=[
    '--turbo-filter=~',
33
    '--noopt',
34 35 36
    '--validate-asm',
    '--stress-validate-asm',
  ],
37 38
  ignition_eager=[
    '--turbo-filter=~',
39
    '--noopt',
40 41 42
    '--no-lazy',
    '--no-lazy-inner-functions',
  ],
43
  ignition_turbo=[],
44 45
  ignition_turbo_opt=[
    '--always-opt',
46 47
    '--no-liftoff',
    '--no-wasm-tier-up',
48
  ],
49
  ignition_turbo_opt_eager=[
50 51 52 53
    '--always-opt',
    '--no-lazy',
    '--no-lazy-inner-functions',
  ],
54 55 56 57 58 59 60
  slow_path=[
    '--force-slow-path',
  ],
  slow_path_opt=[
    '--always-opt',
    '--force-slow-path',
  ],
61 62 63 64 65 66 67
  trusted=[
    '--no-untrusted-code-mitigations',
  ],
  trusted_opt=[
    '--always-opt',
    '--no-untrusted-code-mitigations',
  ],
68 69
)

70 71 72
# Additional flag experiments. List of tuples like
# (<likelihood to use flags in [0,1)>, <flag>).
ADDITIONAL_FLAGS = [
73 74 75
  (0.1, '--stress-marking=100'),
  (0.1, '--stress-scavenge=100'),
  (0.1, '--stress-compaction-random'),
76
  (0.1, '--random-gc-interval=2000'),
77
  (0.2, '--noanalyze-environment-liveness'),
78 79
]

80 81 82 83 84
# Timeout in seconds for one d8 run.
TIMEOUT = 3

# Return codes.
RETURN_PASS = 0
85
RETURN_FAIL = 2
86 87 88 89 90 91

BASE_PATH = os.path.dirname(os.path.abspath(__file__))
PREAMBLE = [
  os.path.join(BASE_PATH, 'v8_mock.js'),
  os.path.join(BASE_PATH, 'v8_suppressions.js'),
]
92
ARCH_MOCKS = os.path.join(BASE_PATH, 'v8_mock_archs.js')
93

94 95
FLAGS = ['--abort_on_stack_or_string_length_overflow', '--expose-gc',
         '--allow-natives-syntax', '--invoke-weak-callbacks', '--omit-quit',
96 97
         '--es-staging', '--wasm-num-compilation-tasks=0',
         '--suppress-asm-messages']
98 99 100 101

SUPPORTED_ARCHS = ['ia32', 'x64', 'arm', 'arm64']

# Output for suppressed failure case.
102
FAILURE_HEADER_TEMPLATE = """#
103 104
# V8 correctness failure
# V8 correctness configs: %(configs)s
105
# V8 correctness sources: %(source_key)s
106 107
# V8 correctness suppression: %(suppression)s
"""
108 109

# Extended output for failure case. The 'CHECK' is for the minimizer.
110
FAILURE_TEMPLATE = FAILURE_HEADER_TEMPLATE + """#
111 112 113 114 115 116 117 118 119 120 121 122
# CHECK
#
# Compared %(first_config_label)s with %(second_config_label)s
#
# Flags of %(first_config_label)s:
%(first_config_flags)s
# Flags of %(second_config_label)s:
%(second_config_flags)s
#
# Difference:
%(difference)s
#
123 124 125
# Source file:
%(source)s
#
126 127 128 129 130 131 132
### Start of configuration %(first_config_label)s:
%(first_config_output)s
### End of configuration %(first_config_label)s
#
### Start of configuration %(second_config_label)s:
%(second_config_output)s
### End of configuration %(second_config_label)s
133
"""
134

135
FUZZ_TEST_RE = re.compile(r'.*fuzz(-\d+\.js)')
136
SOURCE_RE = re.compile(r'print\("v8-foozzie source: (.*)"\);')
137

138 139 140 141 142 143 144
# The number of hex digits used from the hash of the original source file path.
# Keep the number small to avoid duplicate explosion.
ORIGINAL_SOURCE_HASH_LENGTH = 3

# Placeholder string if no original source file could be determined.
ORIGINAL_SOURCE_DEFAULT = 'none'

145 146 147 148 149 150

def infer_arch(d8):
  """Infer the V8 architecture from the build configuration next to the
  executable.
  """
  with open(os.path.join(os.path.dirname(d8), 'v8_build_config.json')) as f:
151
    arch = json.load(f)['v8_current_cpu']
152 153 154
  return 'ia32' if arch == 'x86' else arch


155 156 157 158 159 160
def parse_args():
  parser = argparse.ArgumentParser()
  parser.add_argument(
    '--random-seed', type=int, required=True,
    help='random seed passed to both runs')
  parser.add_argument(
161
      '--first-config', help='first configuration', default='ignition')
162
  parser.add_argument(
163
      '--second-config', help='second configuration', default='ignition_turbo')
164 165 166 167 168 169 170
  parser.add_argument(
      '--first-d8', default='d8',
      help='optional path to first d8 executable, '
           'default: bundled in the same directory as this script')
  parser.add_argument(
      '--second-d8',
      help='optional path to second d8 executable, default: same as first')
171
  parser.add_argument('testcase', help='path to test case')
172 173 174 175 176 177 178 179 180 181 182
  options = parser.parse_args()

  # Ensure we have a test case.
  assert (os.path.exists(options.testcase) and
          os.path.isfile(options.testcase)), (
      'Test case %s doesn\'t exist' % options.testcase)

  # Use first d8 as default for second d8.
  options.second_d8 = options.second_d8 or options.first_d8

  # Ensure absolute paths.
183 184 185 186
  if not os.path.isabs(options.first_d8):
    options.first_d8 = os.path.join(BASE_PATH, options.first_d8)
  if not os.path.isabs(options.second_d8):
    options.second_d8 = os.path.join(BASE_PATH, options.second_d8)
187 188 189 190 191

  # Ensure executables exist.
  assert os.path.exists(options.first_d8)
  assert os.path.exists(options.second_d8)

192 193 194 195 196
  # Infer architecture from build artifacts.
  options.first_arch = infer_arch(options.first_d8)
  options.second_arch = infer_arch(options.second_d8)

  # Ensure we make a sane comparison.
197 198 199
  if (options.first_arch == options.second_arch and
      options.first_config == options.second_config):
    parser.error('Need either arch or config difference.')
200 201 202 203
  assert options.first_arch in SUPPORTED_ARCHS
  assert options.second_arch in SUPPORTED_ARCHS
  assert options.first_config in CONFIGS
  assert options.second_config in CONFIGS
204 205 206 207

  return options


208 209 210 211 212 213 214 215 216 217 218 219 220
def get_meta_data(content):
  """Extracts original-source-file paths from test case content."""
  sources = []
  for line in content.splitlines():
    match = SOURCE_RE.match(line)
    if match:
      sources.append(match.group(1))
  return {'sources': sources}


def content_bailout(content, ignore_fun):
  """Print failure state and return if ignore_fun matches content."""
  bug = (ignore_fun(content) or '').strip()
221 222
  if bug:
    print FAILURE_HEADER_TEMPLATE % dict(
223
        configs='', source_key='', suppression=bug)
224 225 226 227
    return True
  return False


228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
def pass_bailout(output, step_number):
  """Print info and return if in timeout or crash pass states."""
  if output.HasTimedOut():
    # Dashed output, so that no other clusterfuzz tools can match the
    # words timeout or crash.
    print '# V8 correctness - T-I-M-E-O-U-T %d' % step_number
    return True
  if output.HasCrashed():
    print '# V8 correctness - C-R-A-S-H %d' % step_number
    return True
  return False


def fail_bailout(output, ignore_by_output_fun):
  """Print failure state and return if ignore_by_output_fun matches output."""
  bug = (ignore_by_output_fun(output.stdout) or '').strip()
  if bug:
    print FAILURE_HEADER_TEMPLATE % dict(
246
        configs='', source_key='', suppression=bug)
247 248 249 250 251 252
    return True
  return False


def main():
  options = parse_args()
253
  rng = random.Random(options.random_seed)
254 255 256 257 258 259 260

  # Suppressions are architecture and configuration specific.
  suppress = v8_suppressions.get_suppression(
      options.first_arch, options.first_config,
      options.second_arch, options.second_config,
  )

261 262 263 264
  # Static bailout based on test case content or metadata.
  with open(options.testcase) as f:
    content = f.read()
  if content_bailout(get_meta_data(content), suppress.ignore_by_metadata):
265
    return RETURN_FAIL
266
  if content_bailout(content, suppress.ignore_by_content):
267 268
    return RETURN_FAIL

269
  # Set up runtime arguments.
270 271 272 273
  common_flags = FLAGS + ['--random-seed', str(options.random_seed)]
  first_config_flags = common_flags + CONFIGS[options.first_config]
  second_config_flags = common_flags + CONFIGS[options.second_config]

274 275 276 277 278
  # Add additional flags to second config based on experiment percentages.
  for p, flag in ADDITIONAL_FLAGS:
    if rng.random() < p:
      second_config_flags.append(flag)

279
  def run_d8(d8, config_flags):
280 281 282 283
    preamble = PREAMBLE[:]
    if options.first_arch != options.second_arch:
      preamble.append(ARCH_MOCKS)
    args = [d8] + config_flags + preamble + [options.testcase]
284
    print " ".join(args)
285 286 287
    if d8.endswith('.py'):
      # Wrap with python in tests.
      args = [sys.executable] + args
288
    return v8_commands.Execute(
289
        args,
290
        cwd=os.path.dirname(os.path.abspath(options.testcase)),
291 292 293 294 295 296 297 298 299 300 301 302 303 304 305
        timeout=TIMEOUT,
    )

  first_config_output = run_d8(options.first_d8, first_config_flags)

  # Early bailout based on first run's output.
  if pass_bailout(first_config_output, 1):
    return RETURN_PASS

  second_config_output = run_d8(options.second_d8, second_config_flags)

  # Bailout based on second run's output.
  if pass_bailout(second_config_output, 2):
    return RETURN_PASS

306
  difference, source = suppress.diff(
307
      first_config_output.stdout, second_config_output.stdout)
308 309 310 311 312 313 314

  if source:
    source_key = hashlib.sha1(source).hexdigest()[:ORIGINAL_SOURCE_HASH_LENGTH]
  else:
    source = ORIGINAL_SOURCE_DEFAULT
    source_key = ORIGINAL_SOURCE_DEFAULT

315
  if difference:
316 317 318 319 320 321 322 323
    # Only bail out due to suppressed output if there was a difference. If a
    # suppression doesn't show up anymore in the statistics, we might want to
    # remove it.
    if fail_bailout(first_config_output, suppress.ignore_by_output1):
      return RETURN_FAIL
    if fail_bailout(second_config_output, suppress.ignore_by_output2):
      return RETURN_FAIL

324 325 326 327
    # The first three entries will be parsed by clusterfuzz. Format changes
    # will require changes on the clusterfuzz side.
    first_config_label = '%s,%s' % (options.first_arch, options.first_config)
    second_config_label = '%s,%s' % (options.second_arch, options.second_config)
328
    print (FAILURE_TEMPLATE % dict(
329
        configs='%s:%s' % (first_config_label, second_config_label),
330
        source_key=source_key,
331 332 333 334 335
        suppression='', # We can't tie bugs to differences.
        first_config_label=first_config_label,
        second_config_label=second_config_label,
        first_config_flags=' '.join(first_config_flags),
        second_config_flags=' '.join(second_config_flags),
336
        first_config_output=
337
            first_config_output.stdout.decode('utf-8', 'replace'),
338
        second_config_output=
339
            second_config_output.stdout.decode('utf-8', 'replace'),
340
        source=source,
341 342
        difference=difference.decode('utf-8', 'replace'),
    )).encode('utf-8', 'replace')
343
    return RETURN_FAIL
344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359

  # TODO(machenbach): Figure out if we could also return a bug in case there's
  # no difference, but one of the line suppressions has matched - and without
  # the match there would be a difference.

  print '# V8 correctness - pass'
  return RETURN_PASS


if __name__ == "__main__":
  try:
    result = main()
  except SystemExit:
    # Make sure clusterfuzz reports internal errors and wrong usage.
    # Use one label for all internal and usage errors.
    print FAILURE_HEADER_TEMPLATE % dict(
360
        configs='', source_key='', suppression='wrong_usage')
361
    result = RETURN_FAIL
362 363 364 365
  except MemoryError:
    # Running out of memory happens occasionally but is not actionable.
    print '# V8 correctness - pass'
    result = RETURN_PASS
366 367
  except Exception as e:
    print FAILURE_HEADER_TEMPLATE % dict(
368
        configs='', source_key='', suppression='internal_error')
369 370
    print '# Internal error: %s' % e
    traceback.print_exc(file=sys.stdout)
371
    result = RETURN_FAIL
372 373

  sys.exit(result)