v8_foozzie_test.py 11.3 KB
Newer Older
1
#!/usr/bin/env python
2 3 4 5
# Copyright 2016 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

6
import os
7
import random
8 9
import subprocess
import sys
10 11
import unittest

12
import v8_commands
13
import v8_foozzie
14
import v8_fuzz_config
15 16
import v8_suppressions

Vadim's avatar
Vadim committed
17 18 19 20 21
try:
  basestring
except NameError:
  basestring = str

22 23
PYTHON3 = sys.version_info >= (3, 0)

24 25 26 27
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
FOOZZIE = os.path.join(BASE_DIR, 'v8_foozzie.py')
TEST_DATA = os.path.join(BASE_DIR, 'testdata')

28 29 30 31 32 33 34 35
KNOWN_BUILDS = [
  'd8',
  'clang_x86/d8',
  'clang_x86_v8_arm/d8',
  'clang_x64_v8_arm64/d8',
  'clang_x64_pointer_compression/d8',
]

36 37 38

class ConfigTest(unittest.TestCase):
  def testExperiments(self):
39
    """Test integrity of probabilities and configs."""
40
    CONFIGS = v8_foozzie.CONFIGS
41 42 43 44 45
    EXPERIMENTS = v8_fuzz_config.FOOZZIE_EXPERIMENTS
    FLAGS = v8_fuzz_config.ADDITIONAL_FLAGS
    # Probabilities add up to 100%.
    first_is_int = lambda x: type(x[0]) == int
    assert all(map(first_is_int, EXPERIMENTS))
46
    assert sum(x[0] for x in EXPERIMENTS) == 100
47
    # Configs used in experiments are defined.
48 49
    assert all(map(lambda x: x[1] in CONFIGS, EXPERIMENTS))
    assert all(map(lambda x: x[2] in CONFIGS, EXPERIMENTS))
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64
    # The last config item points to a known build configuration.
    assert all(map(lambda x: x[3] in KNOWN_BUILDS, EXPERIMENTS))
    # All flags have a probability.
    first_is_float = lambda x: type(x[0]) == float
    assert all(map(first_is_float, FLAGS))
    first_between_0_and_1 = lambda x: x[0] > 0 and x[0] < 1
    assert all(map(first_between_0_and_1, FLAGS))
    # Test consistent flags.
    second_is_string = lambda x: isinstance(x[1], basestring)
    assert all(map(second_is_string, FLAGS))
    # We allow spaces to separate more flags. We don't allow spaces in the flag
    # value.
    is_flag = lambda x: x.startswith('--')
    all_parts_are_flags = lambda x: all(map(is_flag, x[1].split()))
    assert all(map(all_parts_are_flags, FLAGS))
65 66

  def testConfig(self):
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
    """Smoke test how to choose experiments."""
    config = v8_fuzz_config.Config('foo', random.Random(42))
    experiments = [
      [25, 'ignition', 'jitless', 'd8'],
      [75, 'ignition', 'ignition', 'clang_x86/d8'],
    ]
    flags = [
      [0.1, '--flag'],
      [0.3, '--baz'],
      [0.3, '--foo --bar'],
    ]
    self.assertEqual(
        [
          '--first-config=ignition',
          '--second-config=jitless',
          '--second-d8=d8',
          '--second-config-extra-flags=--baz',
          '--second-config-extra-flags=--foo',
          '--second-config-extra-flags=--bar',
        ],
        config.choose_foozzie_flags(experiments, flags),
    )
89 90
    self.assertEqual(
        [
91
          '--first-config=ignition',
92
          '--second-config=jitless',
93 94
          '--second-d8=d8',
        ],
95
        config.choose_foozzie_flags(experiments, flags),
96 97 98
    )


99
class UnitTest(unittest.TestCase):
100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116
  def testCluster(self):
    crash_test_example_path = 'CrashTests/path/to/file.js'
    self.assertEqual(
        v8_foozzie.ORIGINAL_SOURCE_DEFAULT,
        v8_foozzie.cluster_failures(''))
    self.assertEqual(
        v8_foozzie.ORIGINAL_SOURCE_CRASHTESTS,
        v8_foozzie.cluster_failures(crash_test_example_path))
    self.assertEqual(
        '_o_O_',
        v8_foozzie.cluster_failures(
            crash_test_example_path,
            known_failures={crash_test_example_path: '_o_O_'}))
    self.assertEqual(
        '980',
        v8_foozzie.cluster_failures('v8/test/mjsunit/apply.js'))

117
  def testDiff(self):
118
    def diff_fun(one, two, skip=False):
119
      suppress = v8_suppressions.get_suppression(skip)
120 121
      return suppress.diff_lines(one.splitlines(), two.splitlines())

122 123
    one = ''
    two = ''
124
    diff = None, None
125
    self.assertEqual(diff, diff_fun(one, two))
126 127 128

    one = 'a \n  b\nc();'
    two = 'a \n  b\nc();'
129
    diff = None, None
130
    self.assertEqual(diff, diff_fun(one, two))
131

132
    # Ignore line before caret and caret position.
133 134 135 136
    one = """
undefined
weird stuff
      ^
137
somefile.js: TypeError: suppressed message
138 139 140 141 142 143
  undefined
"""
    two = """
undefined
other weird stuff
            ^
144
somefile.js: TypeError: suppressed message
145 146
  undefined
"""
147
    diff = None, None
148
    self.assertEqual(diff, diff_fun(one, two))
149 150 151 152 153 154 155 156

    one = """
Still equal
Extra line
"""
    two = """
Still equal
"""
157
    diff = '- Extra line', None
158
    self.assertEqual(diff, diff_fun(one, two))
159 160 161 162 163 164 165 166

    one = """
Still equal
"""
    two = """
Still equal
Extra line
"""
167
    diff = '+ Extra line', None
168
    self.assertEqual(diff, diff_fun(one, two))
169 170 171 172 173 174 175 176 177 178

    one = """
undefined
somefile.js: TypeError: undefined is not a constructor
"""
    two = """
undefined
otherfile.js: TypeError: undefined is not a constructor
"""
    diff = """- somefile.js: TypeError: undefined is not a constructor
179
+ otherfile.js: TypeError: undefined is not a constructor""", None
180
    self.assertEqual(diff, diff_fun(one, two))
181

182 183 184
    # Test that skipping suppressions works.
    one = """
v8-foozzie source: foo
185 186
weird stuff
      ^
187 188 189
"""
    two = """
v8-foozzie source: foo
190 191
other weird stuff
            ^
192
"""
193
    self.assertEqual((None, 'foo'), diff_fun(one, two))
194
    diff = ('-       ^\n+             ^', 'foo')
195
    self.assertEqual(diff, diff_fun(one, two, skip=True))
196

197 198 199
  def testOutputCapping(self):
    def output(stdout, is_crash):
      exit_code = -1 if is_crash else 0
200
      return v8_commands.Output(exit_code=exit_code, stdout=stdout, pid=0)
201 202 203 204 205

    def check(stdout1, stdout2, is_crash1, is_crash2, capped_lines1,
              capped_lines2):
      output1 = output(stdout1, is_crash1)
      output2 = output(stdout2, is_crash2)
206
      self.assertEqual(
207 208
          (capped_lines1, capped_lines2),
          v8_suppressions.get_output_capped(output1, output2))
209 210 211 212 213 214 215 216 217

    # No capping, already equal.
    check('1\n2', '1\n2', True, True, '1\n2', '1\n2')
    # No crash, no capping.
    check('1\n2', '1\n2\n3', False, False, '1\n2', '1\n2\n3')
    check('1\n2\n3', '1\n2', False, False, '1\n2\n3', '1\n2')
    # Cap smallest if all runs crash.
    check('1\n2', '1\n2\n3', True, True, '1\n2', '1\n2')
    check('1\n2\n3', '1\n2', True, True, '1\n2', '1\n2')
218 219
    check('1\n2', '1\n23', True, True, '1\n2', '1\n2')
    check('1\n23', '1\n2', True, True, '1\n2', '1\n2')
220 221 222
    # Cap the non-crashy run.
    check('1\n2\n3', '1\n2', False, True, '1\n2', '1\n2')
    check('1\n2', '1\n2\n3', True, False, '1\n2', '1\n2')
223 224
    check('1\n23', '1\n2', False, True, '1\n2', '1\n2')
    check('1\n2', '1\n23', True, False, '1\n2', '1\n2')
225 226 227
    # The crashy run has more output.
    check('1\n2\n3', '1\n2', True, False, '1\n2\n3', '1\n2')
    check('1\n2', '1\n2\n3', False, True, '1\n2', '1\n2\n3')
228 229
    check('1\n23', '1\n2', True, False, '1\n23', '1\n2')
    check('1\n2', '1\n23', False, True, '1\n2', '1\n23')
230 231 232
    # Keep output difference when capping.
    check('1\n2', '3\n4\n5', True, True, '1\n2', '3\n4')
    check('1\n2\n3', '4\n5', True, True, '1\n2', '4\n5')
233 234
    check('12', '345', True, True, '12', '34')
    check('123', '45', True, True, '12', '45')
235 236


237 238 239 240
def cut_verbose_output(stdout, n_comp):
  # This removes the first lines containing d8 commands of `n_comp` comparison
  # runs.
  return '\n'.join(stdout.split('\n')[n_comp * 2:])
machenbach's avatar
machenbach committed
241 242


243 244 245 246
def run_foozzie(second_d8_dir, *extra_flags, **kwargs):
  second_config = 'ignition_turbo'
  if 'second_config' in kwargs:
    second_config = 'jitless'
247 248 249
  kwargs = {}
  if PYTHON3:
    kwargs['text'] = True
250 251 252
  return subprocess.check_output([
    sys.executable, FOOZZIE,
    '--random-seed', '12345',
253 254
    '--first-d8', os.path.join(TEST_DATA, 'baseline', 'd8.py'),
    '--second-d8', os.path.join(TEST_DATA, second_d8_dir, 'd8.py'),
255
    '--first-config', 'ignition',
256
    '--second-config', second_config,
257
    os.path.join(TEST_DATA, 'fuzz-123.js'),
258
  ] + list(extra_flags), **kwargs)
259 260

class SystemTest(unittest.TestCase):
261 262 263 264 265 266 267 268 269 270 271
  """This tests the whole correctness-fuzzing harness with fake build
  artifacts.

  Overview of fakes:
    baseline: Example foozzie output including a syntax error.
    build1: Difference to baseline is a stack trace differece expected to
            be suppressed.
    build2: Difference to baseline is a non-suppressed output difference
            causing the script to fail.
    build3: As build1 but with an architecture difference as well.
  """
272
  def testSyntaxErrorDiffPass(self):
273
    stdout = run_foozzie('build1', '--skip-sanity-checks')
274 275
    self.assertEqual('# V8 correctness - pass\n',
                     cut_verbose_output(stdout, 3))
276 277
    # Default comparison includes suppressions.
    self.assertIn('v8_suppressions.js', stdout)
278 279 280
    # Default comparison doesn't include any specific mock files.
    self.assertNotIn('v8_mock_archs.js', stdout)
    self.assertNotIn('v8_mock_webassembly.js', stdout)
281 282 283 284 285

  def testDifferentOutputFail(self):
    with open(os.path.join(TEST_DATA, 'failure_output.txt')) as f:
      expected_output = f.read()
    with self.assertRaises(subprocess.CalledProcessError) as ctx:
286
      run_foozzie('build2', '--skip-sanity-checks',
287 288 289
                  '--first-config-extra-flags=--flag1',
                  '--first-config-extra-flags=--flag2=0',
                  '--second-config-extra-flags=--flag3')
290
    e = ctx.exception
291
    self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
292
    self.assertEqual(expected_output, cut_verbose_output(e.output, 2))
293

294 295 296 297
  def testSanityCheck(self):
    with open(os.path.join(TEST_DATA, 'sanity_check_output.txt')) as f:
      expected_output = f.read()
    with self.assertRaises(subprocess.CalledProcessError) as ctx:
298
      run_foozzie('build2')
299
    e = ctx.exception
300 301
    self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
    self.assertEqual(expected_output, e.output)
302

303 304 305 306 307 308 309 310 311 312 313 314
  def testDifferentArch(self):
    """Test that the architecture-specific mocks are passed to both runs when
    we use executables with different architectures.
    """
    # Build 3 simulates x86, while the baseline is x64.
    stdout = run_foozzie('build3', '--skip-sanity-checks')
    lines = stdout.split('\n')
    # TODO(machenbach): Don't depend on the command-lines being printed in
    # particular lines.
    self.assertIn('v8_mock_archs.js', lines[1])
    self.assertIn('v8_mock_archs.js', lines[3])

315 316 317 318 319 320 321 322 323 324
  def testJitless(self):
    """Test that webassembly is mocked out when comparing with jitless."""
    stdout = run_foozzie(
        'build1', '--skip-sanity-checks', second_config='jitless')
    lines = stdout.split('\n')
    # TODO(machenbach): Don't depend on the command-lines being printed in
    # particular lines.
    self.assertIn('v8_mock_webassembly.js', lines[1])
    self.assertIn('v8_mock_webassembly.js', lines[3])

325 326 327 328 329 330 331 332 333 334 335 336 337 338 339
  def testSkipSuppressions(self):
    """Test that the suppressions file is not passed when skipping
    suppressions.
    """
    # Compare baseline with baseline. This passes as there is no difference.
    stdout = run_foozzie(
        'baseline', '--skip-sanity-checks', '--skip-suppressions')
    self.assertNotIn('v8_suppressions.js', stdout)

    # Compare with a build that usually suppresses a difference. Now we fail
    # since we skip suppressions.
    with self.assertRaises(subprocess.CalledProcessError) as ctx:
      run_foozzie(
          'build1', '--skip-sanity-checks', '--skip-suppressions')
    e = ctx.exception
340
    self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode)
341 342
    self.assertNotIn('v8_suppressions.js', e.output)

343

344 345
if __name__ == '__main__':
  unittest.main()