try_perf.py 4.35 KB
Newer Older
1 2 3 4 5
#!/usr/bin/env python
# Copyright 2014 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

6 7 8
# for py2/py3 compatibility
from __future__ import print_function

9
import argparse
10 11
import os
import subprocess
12 13
import sys

14
BOTS = {
15
  '--chromebook': 'v8_chromebook_perf_try',
16 17
  '--linux32': 'v8_linux32_perf_try',
  '--linux64': 'v8_linux64_perf_try',
18
  '--linux64_atom': 'v8_linux64_atom_perf_try',
19 20
  '--nexus5': 'v8_nexus5_perf_try',
  '--nexus7': 'v8_nexus7_perf_try',
21
  '--nokia1': 'v8_nokia1_perf_try',
22 23
  '--odroid32': 'v8_odroid32_perf_try',
  '--pixel2': 'v8_pixel2_perf_try',
24 25 26
}

DEFAULT_BOTS = [
27
  'v8_chromebook_perf_try',
28
  'v8_linux32_perf_try',
29
  'v8_linux64_perf_try',
30 31
]

32 33
PUBLIC_BENCHMARKS = [
  'arewefastyet',
34 35
  'ares6',
  'blazor',
36
  'compile',
37 38 39 40 41 42 43 44 45
  'embenchen',
  'emscripten',
  'jetstream',
  'jsbench',
  'jstests',
  'kraken_orig',
  'massive',
  'memory',
  'octane',
46
  'octane-noopt',
47 48 49 50
  'octane-pr',
  'octane-tf',
  'octane-tf-pr',
  'sunspider',
51
  'unity',
52
  'wasm',
53
  'web-tooling-benchmark',
54 55
]

56 57
V8_BASE = os.path.abspath(os.path.dirname(os.path.dirname(__file__)))

58 59
def main():
  parser = argparse.ArgumentParser(description='')
60 61 62
  parser.add_argument('benchmarks', nargs='+', help='The benchmarks to run.')
  parser.add_argument('--extra-flags', default='',
                      help='Extra flags to be passed to the executable.')
63 64 65 66
  parser.add_argument('-r', '--revision', type=str, default=None,
                      help='Revision (use full hash!) to use for the try job; '
                           'default: the revision will be determined by the '
                           'try server; see its waterfall for more info')
67 68
  parser.add_argument('-v', '--verbose', action='store_true',
                      help='Print debug information')
69 70 71 72 73 74 75 76 77
  parser.add_argument('-c', '--confidence-level', type=float,
                      help='Repeatedly runs each benchmark until specified '
                      'confidence level is reached. The value is interpreted '
                      'as the number of standard deviations from the mean that '
                      'all values must lie within. Typical values are 1, 2 and '
                      '3 and correspond to 68%%, 95%% and 99.7%% probability '
                      'that the measured value is within 0.1%% of the true '
                      'value. Larger values result in more retries and thus '
                      'longer runtime, but also provide more reliable results.')
78 79 80 81 82 83
  for option in sorted(BOTS):
    parser.add_argument(
        option, dest='bots', action='append_const', const=BOTS[option],
        help='Add %s trybot.' % BOTS[option])
  options = parser.parse_args()
  if not options.bots:
84
    print('No trybots specified. Using default %s.' % ','.join(DEFAULT_BOTS))
85 86 87
    options.bots = DEFAULT_BOTS

  if not options.benchmarks:
88
    print('Please specify the benchmarks to run as arguments.')
89 90
    return 1

91 92 93 94 95
  for benchmark in options.benchmarks:
    if benchmark not in PUBLIC_BENCHMARKS:
      print ('%s not found in our benchmark list. The respective trybot might '
            'fail, unless you run something this script isn\'t aware of. '
            'Available public benchmarks: %s' % (benchmark, PUBLIC_BENCHMARKS))
96
      print('Proceed anyways? [Y/n] ', end=' ')
97 98 99 100
      answer = sys.stdin.readline().strip()
      if answer != "" and answer != "Y" and answer != "y":
        return 1

101 102 103
  assert '"' not in options.extra_flags and '\'' not in options.extra_flags, (
      'Invalid flag specification.')

104 105
  # Ensure depot_tools are updated.
  subprocess.check_output(
106
      'update_depot_tools', shell=True, stderr=subprocess.STDOUT, cwd=V8_BASE)
107

108 109
  cmd = ['git cl try', '-B', 'luci.v8-internal.try']
  cmd += ['-b %s' % bot for bot in options.bots]
110 111
  if options.revision:
    cmd.append('-r %s' % options.revision)
112
  benchmarks = ['"%s"' % benchmark for benchmark in options.benchmarks]
113
  cmd.append('-p \'testfilter=[%s]\'' % ','.join(benchmarks))
114
  if options.extra_flags:
115 116 117
    cmd.append('-p \'extra_flags="%s"\'' % options.extra_flags)
  if options.confidence_level:
    cmd.append('-p confidence_level=%f' % options.confidence_level)
118 119
  if options.verbose:
    cmd.append('-vv')
120
    print('Running %s' % ' '.join(cmd))
121
  subprocess.check_call(' '.join(cmd), shell=True, cwd=V8_BASE)
122

123
if __name__ == '__main__':  # pragma: no cover
124
  sys.exit(main())