v8_suppressions.py 8.21 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29
# Copyright 2016 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
Suppressions for V8 correctness fuzzer failures.

We support three types of suppressions:
1. Ignore test case by pattern.
Map a regular expression to a bug entry. A new failure will be reported
when the pattern matches a JS test case.
Subsequent matches will be recoreded under the first failure.

2. Ignore test run by output pattern:
Map a regular expression to a bug entry. A new failure will be reported
when the pattern matches the output of a particular run.
Subsequent matches will be recoreded under the first failure.

3. Relax line-to-line comparisons with expressions of lines to ignore and
lines to be normalized (i.e. ignore only portions of lines).
These are not tied to bugs, be careful to not silently switch off this tool!

Alternatively, think about adding a behavior change to v8_suppressions.js
to silence a particular class of problems.
"""

import itertools
import re

30 31 32 33 34 35 36
try:
  # Python 3
  from itertools import zip_longest
except ImportError:
  # Python 2
  from itertools import izip_longest as zip_longest

37 38 39 40 41 42
# Max line length for regular experessions checking for lines to ignore.
MAX_LINE_LENGTH = 512

# For ignoring lines before carets and to ignore caret positions.
CARET_RE = re.compile(r'^\s*\^\s*$')

43 44 45
# Ignore by original source files. Map from bug->list of relative file paths,
# e.g. 'v8/test/mjsunit/d8-performance-now.js'. A test will be suppressed if
# one of the files below was used to mutate the test.
46 47 48
IGNORE_SOURCES = {
}

49
# Ignore by test case pattern. Map from bug->regexp.
50 51
# Bug is preferred to be a crbug.com/XYZ, but can be any short distinguishable
# label.
52
# Regular expressions are assumed to be compiled. We use regexp.search.
53 54 55
IGNORE_TEST_CASES = {
}

56
# Ignore by output pattern. Map from bug->regexp like above.
57
IGNORE_OUTPUT = {
58 59
  'crbug.com/689877':
      re.compile(r'^.*SyntaxError: .*Stack overflow$', re.M),
60 61
  '_fake_difference_':
      re.compile(r'^.*___fake_difference___$', re.M),
62 63 64 65 66 67 68 69 70 71 72 73 74 75 76
}

# Lines matching any of the following regular expressions will be ignored
# if appearing on both sides. The capturing groups need to match exactly.
# Use uncompiled regular expressions - they'll be compiled later.
ALLOWED_LINE_DIFFS = [
  # Ignore caret position in stack traces.
  r'^\s*\^\s*$',
]

# Lines matching any of the following regular expressions will be ignored.
# Use uncompiled regular expressions - they'll be compiled later.
IGNORE_LINES = [
  r'^Warning: .+ is deprecated.*$',
  r'^Try --help for options$',
77

78 79
  # crbug.com/705962
  r'^\s\[0x[0-9a-f]+\]$',
80 81 82 83 84 85 86 87 88 89
]


###############################################################################
# Implementation - you should not need to change anything below this point.

# Compile regular expressions.
ALLOWED_LINE_DIFFS = [re.compile(exp) for exp in ALLOWED_LINE_DIFFS]
IGNORE_LINES = [re.compile(exp) for exp in IGNORE_LINES]

90
ORIGINAL_SOURCE_PREFIX = 'v8-foozzie source: '
91

92

93 94
def get_output_capped(output1, output2):
  """Returns a pair of stdout strings.
95

96
  The strings are safely capped if at least one run has crashed.
97 98 99
  """

  # No length difference or no crash -> no capping.
100
  if (len(output1.stdout) == len(output2.stdout) or
101
      (not output1.HasCrashed() and not output2.HasCrashed())):
102
    return output1.stdout, output2.stdout
103 104 105

  # Both runs have crashed, cap by the shorter output.
  if output1.HasCrashed() and output2.HasCrashed():
106
    cap = min(len(output1.stdout), len(output2.stdout))
107 108
  # Only the first run has crashed, cap by its output length.
  elif output1.HasCrashed():
109
    cap = len(output1.stdout)
110 111
  # Similar if only the second run has crashed.
  else:
112
    cap = len(output2.stdout)
113

114
  return output1.stdout[0:cap], output2.stdout[0:cap]
115 116


117
def line_pairs(lines):
118
  return zip_longest(
119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152
      lines, itertools.islice(lines, 1, None), fillvalue=None)


def caret_match(line1, line2):
  if (not line1 or
      not line2 or
      len(line1) > MAX_LINE_LENGTH or
      len(line2) > MAX_LINE_LENGTH):
    return False
  return bool(CARET_RE.match(line1) and CARET_RE.match(line2))


def short_line_output(line):
  if len(line) <= MAX_LINE_LENGTH:
    # Avoid copying.
    return line
  return line[0:MAX_LINE_LENGTH] + '...'


def ignore_by_regexp(line1, line2, allowed):
  if len(line1) > MAX_LINE_LENGTH or len(line2) > MAX_LINE_LENGTH:
    return False
  for exp in allowed:
    match1 = exp.match(line1)
    match2 = exp.match(line2)
    if match1 and match2:
      # If there are groups in the regexp, ensure the groups matched the same
      # things.
      if match1.groups() == match2.groups():  # tuple comparison
        return True
  return False


def diff_output(output1, output2, allowed, ignore1, ignore2):
153
  """Returns a tuple (difference, source).
154 155 156 157

  The difference is None if there's no difference, otherwise a string
  with a readable diff.

158 159
  The source is the last source output within the test case, or None if no
  such output existed.
160
  """
161 162 163 164 165
  def useful_line(ignore):
    def fun(line):
      return all(not e.match(line) for e in ignore)
    return fun

166 167
  lines1 = list(filter(useful_line(ignore1), output1))
  lines2 = list(filter(useful_line(ignore2), output2))
168

169 170
  # This keeps track where we are in the original source file of the fuzz
  # test case.
171
  source = None
172

173
  for ((line1, lookahead1), (line2, lookahead2)) in zip_longest(
174 175 176 177 178 179 180
      line_pairs(lines1), line_pairs(lines2), fillvalue=(None, None)):

    # Only one of the two iterators should run out.
    assert not (line1 is None and line2 is None)

    # One iterator ends earlier.
    if line1 is None:
181
      return '+ %s' % short_line_output(line2), source
182
    if line2 is None:
183
      return '- %s' % short_line_output(line1), source
184 185 186

    # If lines are equal, no further checks are necessary.
    if line1 == line2:
187 188 189 190 191
      # Instrumented original-source-file output must be equal in both
      # versions. It only makes sense to update it here when both lines
      # are equal.
      if line1.startswith(ORIGINAL_SOURCE_PREFIX):
        source = line1[len(ORIGINAL_SOURCE_PREFIX):]
192 193 194 195 196 197 198 199 200 201 202
      continue

    # Look ahead. If next line is a caret, ignore this line.
    if caret_match(lookahead1, lookahead2):
      continue

    # Check if a regexp allows these lines to be different.
    if ignore_by_regexp(line1, line2, allowed):
      continue

    # Lines are different.
203 204
    return (
        '- %s\n+ %s' % (short_line_output(line1), short_line_output(line2)),
205
        source,
206
    )
207 208

  # No difference found.
209
  return None, source
210 211


212 213
def get_suppression(skip=False):
  return V8Suppression(skip)
214 215


216 217
class V8Suppression(object):
  def __init__(self, skip):
218 219 220 221 222 223 224 225
    if skip:
      self.allowed_line_diffs = []
      self.ignore_output = {}
      self.ignore_sources = {}
    else:
      self.allowed_line_diffs = ALLOWED_LINE_DIFFS
      self.ignore_output = IGNORE_OUTPUT
      self.ignore_sources = IGNORE_SOURCES
226 227

  def diff(self, output1, output2):
228
    # Diff capped lines in the presence of crashes.
229 230
    return self.diff_lines(
        *map(str.splitlines, get_output_capped(output1, output2)))
231 232

  def diff_lines(self, output1_lines, output2_lines):
233
    return diff_output(
234 235
        output1_lines,
        output2_lines,
236
        self.allowed_line_diffs,
237 238 239 240
        IGNORE_LINES,
        IGNORE_LINES,
    )

241
  def ignore_by_content(self, testcase):
242
    # Strip off test case preamble.
243 244
    try:
      lines = testcase.splitlines()
245 246
      lines = lines[lines.index(
          'print("js-mutation: start generated test case");'):]
247 248 249 250 251
      content = '\n'.join(lines)
    except ValueError:
      # Search the whole test case if preamble can't be found. E.g. older
      # already minimized test cases might have dropped the delimiter line.
      content = testcase
252 253 254
    for bug, exp in IGNORE_TEST_CASES.items():
      if exp.search(content):
        return bug
255
    return None
256

257
  def ignore_by_metadata(self, metadata):
258
    for bug, sources in self.ignore_sources.items():
259 260 261
      for source in sources:
        if source in metadata['sources']:
          return bug
262
    return None
263

264
  def ignore_by_output(self, output):
265
    def check(mapping):
266
      for bug, exp in mapping.items():
267 268 269
        if exp.search(output):
          return bug
      return None
270 271 272
    bug = check(self.ignore_output)
    if bug:
      return bug
273
    return None