v8_suppressions.py 9.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
# Copyright 2016 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
Suppressions for V8 correctness fuzzer failures.

We support three types of suppressions:
1. Ignore test case by pattern.
Map a regular expression to a bug entry. A new failure will be reported
when the pattern matches a JS test case.
Subsequent matches will be recoreded under the first failure.

2. Ignore test run by output pattern:
Map a regular expression to a bug entry. A new failure will be reported
when the pattern matches the output of a particular run.
Subsequent matches will be recoreded under the first failure.

3. Relax line-to-line comparisons with expressions of lines to ignore and
lines to be normalized (i.e. ignore only portions of lines).
These are not tied to bugs, be careful to not silently switch off this tool!

Alternatively, think about adding a behavior change to v8_suppressions.js
to silence a particular class of problems.
"""

import itertools
import re

# Max line length for regular experessions checking for lines to ignore.
MAX_LINE_LENGTH = 512

# For ignoring lines before carets and to ignore caret positions.
CARET_RE = re.compile(r'^\s*\^\s*$')

36 37
# Ignore by original source files. Map from bug->list of relative file paths in
# V8, e.g. '/v8/test/mjsunit/d8-performance-now.js' including /v8/. A test will
38 39 40
# be suppressed if one of the files below was used to mutate the test.
IGNORE_SOURCES = {
  # This contains a usage of f.arguments that often fires.
41 42 43 44
  'crbug.com/662424': [
    '/v8/test/mjsunit/bugs/bug-222.js',
    '/v8/test/mjsunit/bugs/bug-941049.js',
    '/v8/test/mjsunit/regress/regress-crbug-668795.js',
45
    '/v8/test/mjsunit/regress/regress-1079.js',
46 47
    '/v8/test/mjsunit/regress/regress-2989.js',
  ],
48

49 50 51
  'crbug.com/688159': [
    '/v8/test/mjsunit/es7/exponentiation-operator.js',
  ],
52 53 54 55 56 57 58 59 60 61

  # TODO(machenbach): Implement blacklisting files for particular configs only,
  # here ignition_eager.
  'crbug.com/691589': [
    '/v8/test/mjsunit/regress/regress-1200351.js',
  ],

  'crbug.com/691587': [
    '/v8/test/mjsunit/asm/regress-674089.js',
  ],
62 63 64 65

  'crbug.com/774805': [
    '/v8/test/mjsunit/console.js',
  ],
66 67
}

68 69 70 71 72
# Ignore by test case pattern. Map from config->bug->regexp. Config '' is used
# to match all configurations. Otherwise use either a compiler configuration,
# e.g. ignition or validate_asm or an architecture, e.g. x64 or ia32.
# Bug is preferred to be a crbug.com/XYZ, but can be any short distinguishable
# label.
73
# Regular expressions are assumed to be compiled. We use regexp.search.
74 75 76
IGNORE_TEST_CASES = {
}

77 78
# Ignore by output pattern. Map from config->bug->regexp. See IGNORE_TEST_CASES
# on how to specify config keys.
79 80 81 82 83 84
# Bug is preferred to be a crbug.com/XYZ, but can be any short distinguishable
# label.
# Regular expressions are assumed to be compiled. We use regexp.search.
IGNORE_OUTPUT = {
  '': {
    'crbug.com/664068':
85
        re.compile(r'RangeError(?!: byte length)', re.S),
86 87
    'crbug.com/667678':
        re.compile(r'\[native code\]', re.S),
88 89
    'crbug.com/689877':
        re.compile(r'^.*SyntaxError: .*Stack overflow$', re.M),
90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
  },
}

# Lines matching any of the following regular expressions will be ignored
# if appearing on both sides. The capturing groups need to match exactly.
# Use uncompiled regular expressions - they'll be compiled later.
ALLOWED_LINE_DIFFS = [
  # Ignore caret position in stack traces.
  r'^\s*\^\s*$',

  # Ignore some stack trace headers as messages might not match.
  r'^(.*)TypeError: .* is not a function$',
  r'^(.*)TypeError: .* is not a constructor$',
  r'^(.*)TypeError: (.*) is not .*$',
  r'^(.*)ReferenceError: .* is not defined$',
  r'^(.*):\d+: ReferenceError: .* is not defined$',

  # These are rarely needed. It includes some cases above.
  r'^\w*Error: .* is not .*$',
  r'^(.*) \w*Error: .* is not .*$',
  r'^(.*):\d+: \w*Error: .* is not .*$',

  # Some test cases just print the message.
  r'^.* is not a function(.*)$',
  r'^(.*) is not a .*$',

116 117
  # crbug.com/680064. This subsumes one of the above expressions.
  r'^(.*)TypeError: .* function$',
118

119 120
  # crbug.com/664068
  r'^(.*)(?:Array buffer allocation failed|Invalid array buffer length)(.*)$',
121 122 123 124 125 126 127 128
]

# Lines matching any of the following regular expressions will be ignored.
# Use uncompiled regular expressions - they'll be compiled later.
IGNORE_LINES = [
  r'^Warning: unknown flag .*$',
  r'^Warning: .+ is deprecated.*$',
  r'^Try --help for options$',
129

130 131
  # crbug.com/705962
  r'^\s\[0x[0-9a-f]+\]$',
132 133 134 135 136 137 138 139 140 141
]


###############################################################################
# Implementation - you should not need to change anything below this point.

# Compile regular expressions.
ALLOWED_LINE_DIFFS = [re.compile(exp) for exp in ALLOWED_LINE_DIFFS]
IGNORE_LINES = [re.compile(exp) for exp in IGNORE_LINES]

142
ORIGINAL_SOURCE_PREFIX = 'v8-foozzie source: '
143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179

def line_pairs(lines):
  return itertools.izip_longest(
      lines, itertools.islice(lines, 1, None), fillvalue=None)


def caret_match(line1, line2):
  if (not line1 or
      not line2 or
      len(line1) > MAX_LINE_LENGTH or
      len(line2) > MAX_LINE_LENGTH):
    return False
  return bool(CARET_RE.match(line1) and CARET_RE.match(line2))


def short_line_output(line):
  if len(line) <= MAX_LINE_LENGTH:
    # Avoid copying.
    return line
  return line[0:MAX_LINE_LENGTH] + '...'


def ignore_by_regexp(line1, line2, allowed):
  if len(line1) > MAX_LINE_LENGTH or len(line2) > MAX_LINE_LENGTH:
    return False
  for exp in allowed:
    match1 = exp.match(line1)
    match2 = exp.match(line2)
    if match1 and match2:
      # If there are groups in the regexp, ensure the groups matched the same
      # things.
      if match1.groups() == match2.groups():  # tuple comparison
        return True
  return False


def diff_output(output1, output2, allowed, ignore1, ignore2):
180
  """Returns a tuple (difference, source).
181 182 183 184

  The difference is None if there's no difference, otherwise a string
  with a readable diff.

185 186
  The source is the last source output within the test case, or None if no
  such output existed.
187
  """
188 189 190 191 192 193 194 195
  def useful_line(ignore):
    def fun(line):
      return all(not e.match(line) for e in ignore)
    return fun

  lines1 = filter(useful_line(ignore1), output1)
  lines2 = filter(useful_line(ignore2), output2)

196 197
  # This keeps track where we are in the original source file of the fuzz
  # test case.
198
  source = None
199

200 201 202 203 204 205 206 207
  for ((line1, lookahead1), (line2, lookahead2)) in itertools.izip_longest(
      line_pairs(lines1), line_pairs(lines2), fillvalue=(None, None)):

    # Only one of the two iterators should run out.
    assert not (line1 is None and line2 is None)

    # One iterator ends earlier.
    if line1 is None:
208
      return '+ %s' % short_line_output(line2), source
209
    if line2 is None:
210
      return '- %s' % short_line_output(line1), source
211 212 213

    # If lines are equal, no further checks are necessary.
    if line1 == line2:
214 215 216 217 218
      # Instrumented original-source-file output must be equal in both
      # versions. It only makes sense to update it here when both lines
      # are equal.
      if line1.startswith(ORIGINAL_SOURCE_PREFIX):
        source = line1[len(ORIGINAL_SOURCE_PREFIX):]
219 220 221 222 223 224 225 226 227 228 229
      continue

    # Look ahead. If next line is a caret, ignore this line.
    if caret_match(lookahead1, lookahead2):
      continue

    # Check if a regexp allows these lines to be different.
    if ignore_by_regexp(line1, line2, allowed):
      continue

    # Lines are different.
230 231
    return (
        '- %s\n+ %s' % (short_line_output(line1), short_line_output(line2)),
232
        source,
233
    )
234 235

  # No difference found.
236
  return None, source
237 238 239 240 241 242 243 244 245 246


def get_suppression(arch1, config1, arch2, config2):
  return V8Suppression(arch1, config1, arch2, config2)


class Suppression(object):
  def diff(self, output1, output2):
    return None

247
  def ignore_by_metadata(self, metadata):
248
    return None
249 250

  def ignore_by_content(self, testcase):
251
    return None
252 253

  def ignore_by_output1(self, output):
254
    return None
255 256

  def ignore_by_output2(self, output):
257
    return None
258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275


class V8Suppression(Suppression):
  def __init__(self, arch1, config1, arch2, config2):
    self.arch1 = arch1
    self.config1 = config1
    self.arch2 = arch2
    self.config2 = config2

  def diff(self, output1, output2):
    return diff_output(
        output1.splitlines(),
        output2.splitlines(),
        ALLOWED_LINE_DIFFS,
        IGNORE_LINES,
        IGNORE_LINES,
    )

276
  def ignore_by_content(self, testcase):
277
    # Strip off test case preamble.
278 279
    try:
      lines = testcase.splitlines()
280 281
      lines = lines[lines.index(
          'print("js-mutation: start generated test case");'):]
282 283 284 285 286
      content = '\n'.join(lines)
    except ValueError:
      # Search the whole test case if preamble can't be found. E.g. older
      # already minimized test cases might have dropped the delimiter line.
      content = testcase
287 288 289 290 291
    for key in ['', self.arch1, self.arch2, self.config1, self.config2]:
      for bug, exp in IGNORE_TEST_CASES.get(key, {}).iteritems():
        if exp.search(content):
          return bug
    return None
292

293
  def ignore_by_metadata(self, metadata):
294 295 296 297
    for bug, sources in IGNORE_SOURCES.iteritems():
      for source in sources:
        if source in metadata['sources']:
          return bug
298
    return None
299

300 301 302 303 304 305 306 307 308 309 310 311
  def ignore_by_output1(self, output):
    return self.ignore_by_output(output, self.arch1, self.config1)

  def ignore_by_output2(self, output):
    return self.ignore_by_output(output, self.arch2, self.config2)

  def ignore_by_output(self, output, arch, config):
    def check(mapping):
      for bug, exp in mapping.iteritems():
        if exp.search(output):
          return bug
      return None
312 313 314 315
    for key in ['', arch, config]:
      bug = check(IGNORE_OUTPUT.get(key, {}))
      if bug:
        return bug
316
    return None