run_perf_test.py 21.7 KB
Newer Older
1 2 3 4 5
#!/usr/bin/env python
# Copyright 2014 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

6 7 8
# for py2/py3 compatibility
from __future__ import print_function

9 10 11
from collections import namedtuple
import json
import os
12
import platform
13
import shutil
14
import subprocess
15
import sys
16 17 18
import tempfile
import unittest

19 20 21
import coverage
import mock

22 23 24
# Requires python-coverage and python-mock. Native python coverage
# version >= 3.7.1 should be installed to get the best speed.

25 26 27 28
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
RUN_PERF = os.path.join(BASE_DIR, 'run_perf.py')
TEST_DATA = os.path.join(BASE_DIR, 'unittests', 'testdata')

29
TEST_WORKSPACE = os.path.join(tempfile.gettempdir(), 'test-v8-run-perf')
30 31

V8_JSON = {
32 33 34
  'path': ['.'],
  'owners': ['username@chromium.org'],
  'binary': 'd7',
35
  'timeout': 60,
36 37 38 39 40 41 42
  'flags': ['--flag'],
  'main': 'run.js',
  'run_count': 1,
  'results_regexp': '^%s: (.+)$',
  'tests': [
    {'name': 'Richards'},
    {'name': 'DeltaBlue'},
43 44 45 46
  ]
}

V8_NESTED_SUITES_JSON = {
47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68
  'path': ['.'],
  'owners': ['username@chromium.org'],
  'flags': ['--flag'],
  'run_count': 1,
  'units': 'score',
  'tests': [
    {'name': 'Richards',
     'path': ['richards'],
     'binary': 'd7',
     'main': 'run.js',
     'resources': ['file1.js', 'file2.js'],
     'run_count': 2,
     'results_regexp': '^Richards: (.+)$'},
    {'name': 'Sub',
     'path': ['sub'],
     'tests': [
       {'name': 'Leaf',
        'path': ['leaf'],
        'run_count_x64': 3,
        'units': 'ms',
        'main': 'run.js',
        'results_regexp': '^Simple: (.+) ms.$'},
69 70
     ]
    },
71 72 73 74 75 76 77 78 79
    {'name': 'DeltaBlue',
     'path': ['delta_blue'],
     'main': 'run.js',
     'flags': ['--flag2'],
     'results_regexp': '^DeltaBlue: (.+)$'},
    {'name': 'ShouldntRun',
     'path': ['.'],
     'archs': ['arm'],
     'main': 'run.js'},
80 81 82 83
  ]
}

V8_GENERIC_JSON = {
84 85 86 87 88 89 90
  'path': ['.'],
  'owners': ['username@chromium.org'],
  'binary': 'cc',
  'flags': ['--flag'],
  'generic': True,
  'run_count': 1,
  'units': 'ms',
91 92 93 94 95
}

class PerfTest(unittest.TestCase):
  @classmethod
  def setUpClass(cls):
96
    sys.path.insert(0, BASE_DIR)
97
    cls._cov = coverage.coverage(
98
        include=([os.path.join(BASE_DIR, 'run_perf.py')]))
99 100
    cls._cov.start()
    import run_perf
101
    from testrunner.local import command
102 103
    from testrunner.objects.output import Output, NULL_OUTPUT
    global command, run_perf, Output, NULL_OUTPUT
104 105 106 107

  @classmethod
  def tearDownClass(cls):
    cls._cov.stop()
108
    print('')
109
    print(cls._cov.report())
110 111 112

  def setUp(self):
    self.maxDiff = None
113
    if os.path.exists(TEST_WORKSPACE):
114 115 116 117
      shutil.rmtree(TEST_WORKSPACE)
    os.makedirs(TEST_WORKSPACE)

  def tearDown(self):
118 119
    mock.patch.stopall()
    if os.path.exists(TEST_WORKSPACE):
120 121 122
      shutil.rmtree(TEST_WORKSPACE)

  def _WriteTestInput(self, json_content):
123 124
    self._test_input = os.path.join(TEST_WORKSPACE, 'test.json')
    with open(self._test_input, 'w') as f:
125 126
      f.write(json.dumps(json_content))

127
  def _MockCommand(self, *args, **kwargs):
128
    # Fake output for each test run.
129
    test_outputs = [Output(stdout=arg,
130
                           timed_out=kwargs.get('timed_out', False),
131 132
                           exit_code=kwargs.get('exit_code', 0),
                           duration=42)
133
                    for arg in args[1]]
134
    def create_cmd(*args, **kwargs):
135
      cmd = mock.MagicMock()
136 137
      def execute(*args, **kwargs):
        return test_outputs.pop()
138
      cmd.execute = mock.MagicMock(side_effect=execute)
139 140
      return cmd

141
    mock.patch.object(
142
        run_perf.command, 'PosixCommand',
143
        mock.MagicMock(side_effect=create_cmd)).start()
144 145

    # Check that d8 is called from the correct cwd for each test run.
146
    dirs = [os.path.join(TEST_WORKSPACE, arg) for arg in args[0]]
147
    def chdir(*args, **kwargs):
148
      self.assertEqual(dirs.pop(), args[0])
149
    os.chdir = mock.MagicMock(side_effect=chdir)
150

151 152
    subprocess.check_call = mock.MagicMock()
    platform.system = mock.MagicMock(return_value='Linux')
153

154
  def _CallMain(self, *args):
155
    self._test_output = os.path.join(TEST_WORKSPACE, 'results.json')
156
    all_args=[
157
      '--json-test-results',
158 159 160 161 162 163
      self._test_output,
      self._test_input,
    ]
    all_args += args
    return run_perf.Main(all_args)

164 165
  def _LoadResults(self, file_name=None):
    with open(file_name or self._test_output) as f:
166 167
      return json.load(f)

168
  def _VerifyResults(self, suite, units, traces, file_name=None):
169
    self.assertListEqual(sorted([
170 171 172
      {'units': units,
       'graphs': [suite, trace['name']],
       'results': trace['results'],
173 174
       'stddev': trace['stddev']} for trace in traces]),
      sorted(self._LoadResults(file_name)['traces']))
175

176
  def _VerifyRunnableDurations(self, runs, timeout, file_name=None):
177 178 179 180 181 182 183
    self.assertListEqual([
      {
        'graphs': ['test'],
        'durations': [42] * runs,
        'timeout': timeout,
      },
    ], self._LoadResults(file_name)['runnables'])
184

185
  def _VerifyErrors(self, errors):
186
    self.assertListEqual(errors, self._LoadResults()['errors'])
187

188
  def _VerifyMock(self, binary, *args, **kwargs):
189
    shell = os.path.join(os.path.dirname(BASE_DIR), binary)
190 191 192 193 194
    command.Command.assert_called_with(
        cmd_prefix=[],
        shell=shell,
        args=list(args),
        timeout=kwargs.get('timeout', 60))
195

196
  def _VerifyMockMultiple(self, *args, **kwargs):
197
    self.assertEqual(len(args), len(command.Command.call_args_list))
198 199 200
    for arg, actual in zip(args, command.Command.call_args_list):
      expected = {
        'cmd_prefix': [],
201
        'shell': os.path.join(os.path.dirname(BASE_DIR), arg[0]),
202 203 204
        'args': list(arg[1:]),
        'timeout': kwargs.get('timeout', 60)
      }
205
      self.assertTupleEqual((expected, ), actual)
206 207 208

  def testOneRun(self):
    self._WriteTestInput(V8_JSON)
209
    self._MockCommand(['.'], ['x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n'])
210
    self.assertEqual(0, self._CallMain())
211
    self._VerifyResults('test', 'score', [
212 213
      {'name': 'Richards', 'results': [1.234], 'stddev': ''},
      {'name': 'DeltaBlue', 'results': [10657567.0], 'stddev': ''},
214
    ])
215
    self._VerifyRunnableDurations(1, 60)
216
    self._VerifyErrors([])
217 218
    self._VerifyMock(
        os.path.join('out', 'x64.release', 'd7'), '--flag', 'run.js')
219

220 221
  def testOneRunWithTestFlags(self):
    test_input = dict(V8_JSON)
222
    test_input['test_flags'] = ['2', 'test_name']
223
    self._WriteTestInput(test_input)
224
    self._MockCommand(['.'], ['Richards: 1.234\nDeltaBlue: 10657567'])
225
    self.assertEqual(0, self._CallMain())
226
    self._VerifyResults('test', 'score', [
227 228
      {'name': 'Richards', 'results': [1.234], 'stddev': ''},
      {'name': 'DeltaBlue', 'results': [10657567.0], 'stddev': ''},
229 230
    ])
    self._VerifyErrors([])
231 232
    self._VerifyMock(os.path.join(
      'out', 'x64.release', 'd7'), '--flag', 'run.js', '--', '2', 'test_name')
233

234 235
  def testTwoRuns_Units_SuiteName(self):
    test_input = dict(V8_JSON)
236 237 238
    test_input['run_count'] = 2
    test_input['name'] = 'v8'
    test_input['units'] = 'ms'
239
    self._WriteTestInput(test_input)
240 241 242
    self._MockCommand(['.', '.'],
                      ['Richards: 100\nDeltaBlue: 200\n',
                       'Richards: 50\nDeltaBlue: 300\n'])
243
    self.assertEqual(0, self._CallMain())
244
    self._VerifyResults('v8', 'ms', [
245 246
      {'name': 'Richards', 'results': [50.0, 100.0], 'stddev': ''},
      {'name': 'DeltaBlue', 'results': [300.0, 200.0], 'stddev': ''},
247 248
    ])
    self._VerifyErrors([])
249 250
    self._VerifyMock(os.path.join(
      'out', 'x64.release', 'd7'), '--flag', 'run.js')
251 252 253

  def testTwoRuns_SubRegexp(self):
    test_input = dict(V8_JSON)
254 255 256 257
    test_input['run_count'] = 2
    del test_input['results_regexp']
    test_input['tests'][0]['results_regexp'] = '^Richards: (.+)$'
    test_input['tests'][1]['results_regexp'] = '^DeltaBlue: (.+)$'
258
    self._WriteTestInput(test_input)
259 260 261
    self._MockCommand(['.', '.'],
                      ['Richards: 100\nDeltaBlue: 200\n',
                       'Richards: 50\nDeltaBlue: 300\n'])
262
    self.assertEqual(0, self._CallMain())
263
    self._VerifyResults('test', 'score', [
264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314
      {'name': 'Richards', 'results': [50.0, 100.0], 'stddev': ''},
      {'name': 'DeltaBlue', 'results': [300.0, 200.0], 'stddev': ''},
    ])
    self._VerifyErrors([])
    self._VerifyMock(os.path.join(
      'out', 'x64.release', 'd7'), '--flag', 'run.js')

  def testPerfectConfidenceRuns(self):
    self._WriteTestInput(V8_JSON)
    self._MockCommand(
        ['.'], ['x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n'] * 10)
    self.assertEqual(0, self._CallMain('--confidence-level', '1'))
    self._VerifyResults('test', 'score', [
      {'name': 'Richards', 'results': [1.234] * 10, 'stddev': ''},
      {'name': 'DeltaBlue', 'results': [10657567.0] * 10, 'stddev': ''},
    ])
    self._VerifyErrors([])
    self._VerifyMock(os.path.join(
      'out', 'x64.release', 'd7'), '--flag', 'run.js')

  def testNoisyConfidenceRuns(self):
    self._WriteTestInput(V8_JSON)
    self._MockCommand(
        ['.'],
        reversed([
          # First 10 runs are mandatory. DeltaBlue is slightly noisy.
          'x\nRichards: 1.234\nDeltaBlue: 10757567\ny\n',
          'x\nRichards: 1.234\nDeltaBlue: 10557567\ny\n',
          'x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n',
          'x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n',
          'x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n',
          'x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n',
          'x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n',
          'x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n',
          'x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n',
          'x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n',
          # Need 4 more runs for confidence in DeltaBlue results.
          'x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n',
          'x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n',
          'x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n',
          'x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n',
        ]),
    )
    self.assertEqual(0, self._CallMain('--confidence-level', '1'))
    self._VerifyResults('test', 'score', [
      {'name': 'Richards', 'results': [1.234] * 14, 'stddev': ''},
      {
        'name': 'DeltaBlue',
        'results': [10757567.0, 10557567.0] + [10657567.0] * 12,
        'stddev': '',
      },
315 316
    ])
    self._VerifyErrors([])
317 318
    self._VerifyMock(os.path.join(
      'out', 'x64.release', 'd7'), '--flag', 'run.js')
319 320 321

  def testNestedSuite(self):
    self._WriteTestInput(V8_NESTED_SUITES_JSON)
322 323 324 325 326 327 328
    self._MockCommand(['delta_blue', 'sub/leaf', 'richards'],
                      ['DeltaBlue: 200\n',
                       'Simple: 1 ms.\n',
                       'Simple: 2 ms.\n',
                       'Simple: 3 ms.\n',
                       'Richards: 100\n',
                       'Richards: 50\n'])
329 330
    self.assertEqual(0, self._CallMain())
    self.assertListEqual(sorted([
331 332
      {'units': 'score',
       'graphs': ['test', 'Richards'],
333
       'results': [50.0, 100.0],
334 335 336
       'stddev': ''},
      {'units': 'ms',
       'graphs': ['test', 'Sub', 'Leaf'],
337
       'results': [3.0, 2.0, 1.0],
338 339 340
       'stddev': ''},
      {'units': 'score',
       'graphs': ['test', 'DeltaBlue'],
341
       'results': [200.0],
342
       'stddev': ''},
343
      ]), sorted(self._LoadResults()['traces']))
344 345
    self._VerifyErrors([])
    self._VerifyMockMultiple(
346 347 348 349 350 351 352
        (os.path.join('out', 'x64.release', 'd7'), '--flag', 'run.js'),
        (os.path.join('out', 'x64.release', 'd7'), '--flag', 'run.js'),
        (os.path.join('out', 'x64.release', 'd8'), '--flag', 'run.js'),
        (os.path.join('out', 'x64.release', 'd8'), '--flag', 'run.js'),
        (os.path.join('out', 'x64.release', 'd8'), '--flag', 'run.js'),
        (os.path.join('out', 'x64.release', 'd8'),
         '--flag', '--flag2', 'run.js'))
353 354 355

  def testOneRunStdDevRegExp(self):
    test_input = dict(V8_JSON)
356
    test_input['stddev_regexp'] = '^%s\-stddev: (.+)$'
357
    self._WriteTestInput(test_input)
358 359
    self._MockCommand(['.'], ['Richards: 1.234\nRichards-stddev: 0.23\n'
                              'DeltaBlue: 10657567\nDeltaBlue-stddev: 106\n'])
360
    self.assertEqual(0, self._CallMain())
361
    self._VerifyResults('test', 'score', [
362 363
      {'name': 'Richards', 'results': [1.234], 'stddev': '0.23'},
      {'name': 'DeltaBlue', 'results': [10657567.0], 'stddev': '106'},
364 365
    ])
    self._VerifyErrors([])
366 367
    self._VerifyMock(
        os.path.join('out', 'x64.release', 'd7'), '--flag', 'run.js')
368 369 370

  def testTwoRunsStdDevRegExp(self):
    test_input = dict(V8_JSON)
371 372
    test_input['stddev_regexp'] = '^%s\-stddev: (.+)$'
    test_input['run_count'] = 2
373
    self._WriteTestInput(test_input)
374 375 376 377
    self._MockCommand(['.'], ['Richards: 3\nRichards-stddev: 0.7\n'
                              'DeltaBlue: 6\nDeltaBlue-boom: 0.9\n',
                              'Richards: 2\nRichards-stddev: 0.5\n'
                              'DeltaBlue: 5\nDeltaBlue-stddev: 0.8\n'])
378
    self.assertEqual(1, self._CallMain())
379
    self._VerifyResults('test', 'score', [
380 381
      {'name': 'Richards', 'results': [2.0, 3.0], 'stddev': '0.7'},
      {'name': 'DeltaBlue', 'results': [5.0, 6.0], 'stddev': '0.8'},
382 383
    ])
    self._VerifyErrors(
384 385 386 387 388 389 390 391
        ['Test test/Richards should only run once since a stddev is provided '
         'by the test.',
         'Test test/DeltaBlue should only run once since a stddev is provided '
         'by the test.',
         'Regexp "^DeltaBlue\-stddev: (.+)$" did not match for test '
         'test/DeltaBlue.'])
    self._VerifyMock(
        os.path.join('out', 'x64.release', 'd7'), '--flag', 'run.js')
392 393 394

  def testBuildbot(self):
    self._WriteTestInput(V8_JSON)
395 396 397 398
    self._MockCommand(['.'], ['Richards: 1.234\nDeltaBlue: 10657567\n'])
    mock.patch.object(
        run_perf.Platform, 'ReadBuildConfig',
        mock.MagicMock(return_value={'is_android': False})).start()
399
    self.assertEqual(0, self._CallMain('--buildbot'))
400
    self._VerifyResults('test', 'score', [
401 402
      {'name': 'Richards', 'results': [1.234], 'stddev': ''},
      {'name': 'DeltaBlue', 'results': [10657567.0], 'stddev': ''},
403 404
    ])
    self._VerifyErrors([])
405
    self._VerifyMock(os.path.join('out', 'Release', 'd7'), '--flag', 'run.js')
406 407 408

  def testBuildbotWithTotal(self):
    test_input = dict(V8_JSON)
409
    test_input['total'] = True
410
    self._WriteTestInput(test_input)
411 412 413 414
    self._MockCommand(['.'], ['Richards: 1.234\nDeltaBlue: 10657567\n'])
    mock.patch.object(
        run_perf.Platform, 'ReadBuildConfig',
        mock.MagicMock(return_value={'is_android': False})).start()
415
    self.assertEqual(0, self._CallMain('--buildbot'))
416
    self._VerifyResults('test', 'score', [
417 418 419
      {'name': 'Richards', 'results': [1.234], 'stddev': ''},
      {'name': 'DeltaBlue', 'results': [10657567.0], 'stddev': ''},
      {'name': 'Total', 'results': [3626.491097190233], 'stddev': ''},
420 421
    ])
    self._VerifyErrors([])
422
    self._VerifyMock(os.path.join('out', 'Release', 'd7'), '--flag', 'run.js')
423 424 425

  def testBuildbotWithTotalAndErrors(self):
    test_input = dict(V8_JSON)
426
    test_input['total'] = True
427
    self._WriteTestInput(test_input)
428 429 430 431
    self._MockCommand(['.'], ['x\nRichards: bla\nDeltaBlue: 10657567\ny\n'])
    mock.patch.object(
        run_perf.Platform, 'ReadBuildConfig',
        mock.MagicMock(return_value={'is_android': False})).start()
432
    self.assertEqual(1, self._CallMain('--buildbot'))
433
    self._VerifyResults('test', 'score', [
434
      {'name': 'DeltaBlue', 'results': [10657567.0], 'stddev': ''},
435 436
    ])
    self._VerifyErrors(
437 438
        ['Regexp "^Richards: (.+)$" '
         'returned a non-numeric for test test/Richards.',
439 440
         'Not all traces have produced results. Can not compute total for '
         'test.'])
441
    self._VerifyMock(os.path.join('out', 'Release', 'd7'), '--flag', 'run.js')
442 443 444

  def testRegexpNoMatch(self):
    self._WriteTestInput(V8_JSON)
445
    self._MockCommand(['.'], ['x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n'])
446
    self.assertEqual(1, self._CallMain())
447
    self._VerifyResults('test', 'score', [
448
      {'name': 'DeltaBlue', 'results': [10657567.0], 'stddev': ''},
449 450
    ])
    self._VerifyErrors(
451 452 453
        ['Regexp "^Richards: (.+)$" did not match for test test/Richards.'])
    self._VerifyMock(
        os.path.join('out', 'x64.release', 'd7'), '--flag', 'run.js')
454

455
  def testOneRunCrashed(self):
456 457 458
    test_input = dict(V8_JSON)
    test_input['retry_count'] = 1
    self._WriteTestInput(test_input)
459
    self._MockCommand(
460 461
        ['.'], ['x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n', ''],
        exit_code=-1)
462
    self.assertEqual(1, self._CallMain())
463
    self._VerifyResults('test', 'score', [])
464
    self._VerifyErrors([])
465 466
    self._VerifyMock(
        os.path.join('out', 'x64.release', 'd7'), '--flag', 'run.js')
467

468 469
  def testOneRunTimingOut(self):
    test_input = dict(V8_JSON)
470
    test_input['timeout'] = 70
471
    test_input['retry_count'] = 0
472
    self._WriteTestInput(test_input)
473
    self._MockCommand(['.'], [''], timed_out=True)
474
    self.assertEqual(1, self._CallMain())
475
    self._VerifyResults('test', 'score', [])
476
    self._VerifyErrors([])
477 478
    self._VerifyMock(os.path.join('out', 'x64.release', 'd7'),
                     '--flag', 'run.js', timeout=70)
479 480 481

  def testAndroid(self):
    self._WriteTestInput(V8_JSON)
482 483 484 485 486
    mock.patch('run_perf.AndroidPlatform.PreExecution').start()
    mock.patch('run_perf.AndroidPlatform.PostExecution').start()
    mock.patch('run_perf.AndroidPlatform.PreTests').start()
    mock.patch(
        'run_perf.AndroidPlatform.Run',
487 488
        return_value=(Output(stdout='Richards: 1.234\nDeltaBlue: 10657567\n'),
                      NULL_OUTPUT)).start()
489 490 491 492
    mock.patch('testrunner.local.android._Driver', autospec=True).start()
    mock.patch(
        'run_perf.Platform.ReadBuildConfig',
        return_value={'is_android': True}).start()
493
    self.assertEqual(0, self._CallMain('--arch', 'arm'))
494
    self._VerifyResults('test', 'score', [
495 496
      {'name': 'Richards', 'results': [1.234], 'stddev': ''},
      {'name': 'DeltaBlue', 'results': [10657567.0], 'stddev': ''},
497
    ])
498 499 500

  def testTwoRuns_Trybot(self):
    test_input = dict(V8_JSON)
501
    test_input['run_count'] = 2
502
    self._WriteTestInput(test_input)
503 504 505 506 507 508 509
    self._MockCommand(['.', '.', '.', '.'],
                      ['Richards: 100\nDeltaBlue: 200\n',
                       'Richards: 200\nDeltaBlue: 20\n',
                       'Richards: 50\nDeltaBlue: 200\n',
                       'Richards: 100\nDeltaBlue: 20\n'])
    test_output_secondary = os.path.join(
        TEST_WORKSPACE, 'results_secondary.json')
510
    self.assertEqual(0, self._CallMain(
511 512
        '--outdir-secondary', 'out-secondary',
        '--json-test-results-secondary', test_output_secondary,
513
    ))
514
    self._VerifyResults('test', 'score', [
515 516
      {'name': 'Richards', 'results': [100.0, 200.0], 'stddev': ''},
      {'name': 'DeltaBlue', 'results': [20.0, 20.0], 'stddev': ''},
517
    ])
518
    self._VerifyResults('test', 'score', [
519 520
      {'name': 'Richards', 'results': [50.0, 100.0], 'stddev': ''},
      {'name': 'DeltaBlue', 'results': [200.0, 200.0], 'stddev': ''},
521
    ], test_output_secondary)
522
    self._VerifyRunnableDurations(2, 60, test_output_secondary)
523 524
    self._VerifyErrors([])
    self._VerifyMockMultiple(
525 526 527 528 529 530
        (os.path.join('out', 'x64.release', 'd7'), '--flag', 'run.js'),
        (os.path.join('out-secondary', 'x64.release', 'd7'),
         '--flag', 'run.js'),
        (os.path.join('out', 'x64.release', 'd7'), '--flag', 'run.js'),
        (os.path.join('out-secondary', 'x64.release', 'd7'),
         '--flag', 'run.js'),
531 532
    )

533 534 535
  def testWrongBinaryWithProf(self):
    test_input = dict(V8_JSON)
    self._WriteTestInput(test_input)
536
    self._MockCommand(['.'], ['x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n'])
537
    self.assertEqual(0, self._CallMain('--extra-flags=--prof'))
538
    self._VerifyResults('test', 'score', [
539 540
      {'name': 'Richards', 'results': [1.234], 'stddev': ''},
      {'name': 'DeltaBlue', 'results': [10657567.0], 'stddev': ''},
541 542
    ])
    self._VerifyErrors([])
543 544
    self._VerifyMock(os.path.join('out', 'x64.release', 'd7'),
                     '--flag', '--prof', 'run.js')
545

546 547 548 549
  #############################################################################
  ### System tests

  def _RunPerf(self, mocked_d8, test_json):
550
    output_json = os.path.join(TEST_WORKSPACE, 'output.json')
551
    args = [
552 553 554
      os.sys.executable, RUN_PERF,
      '--binary-override-path', os.path.join(TEST_DATA, mocked_d8),
      '--json-test-results', output_json,
555 556 557 558 559 560
      os.path.join(TEST_DATA, test_json),
    ]
    subprocess.check_output(args)
    return self._LoadResults(output_json)

  def testNormal(self):
561
    results = self._RunPerf('d8_mocked1.py', 'test1.json')
562 563
    self.assertListEqual([], results['errors'])
    self.assertListEqual(sorted([
564 565 566
      {
        'units': 'score',
        'graphs': ['test1', 'Richards'],
567
        'results': [1.2, 1.2],
568 569 570 571 572
        'stddev': '',
      },
      {
        'units': 'score',
        'graphs': ['test1', 'DeltaBlue'],
573
        'results': [2.1, 2.1],
574 575
        'stddev': '',
      },
576
    ]), sorted(results['traces']))
577 578

  def testResultsProcessor(self):
579
    results = self._RunPerf('d8_mocked2.py', 'test2.json')
580 581
    self.assertListEqual([], results['errors'])
    self.assertListEqual([
582 583 584
      {
        'units': 'score',
        'graphs': ['test2', 'Richards'],
585
        'results': [1.2, 1.2],
586 587 588 589 590
        'stddev': '',
      },
      {
        'units': 'score',
        'graphs': ['test2', 'DeltaBlue'],
591
        'results': [2.1, 2.1],
592 593 594 595 596
        'stddev': '',
      },
    ], results['traces'])

  def testResultsProcessorNested(self):
597
    results = self._RunPerf('d8_mocked2.py', 'test3.json')
598 599
    self.assertListEqual([], results['errors'])
    self.assertListEqual([
600 601 602
      {
        'units': 'score',
        'graphs': ['test3', 'Octane', 'Richards'],
603
        'results': [1.2],
604 605 606 607 608
        'stddev': '',
      },
      {
        'units': 'score',
        'graphs': ['test3', 'Octane', 'DeltaBlue'],
609
        'results': [2.1],
610 611 612
        'stddev': '',
      },
    ], results['traces'])
613 614 615 616


if __name__ == '__main__':
  unittest.main()