callstats.py 25.9 KB
Newer Older
1 2 3 4 5
#!/usr/bin/env python
# Copyright 2016 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
'''
6
Usage: callstats.py [-h] <command> ...
7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27

Optional arguments:
  -h, --help  show this help message and exit

Commands:
  run         run chrome with --runtime-call-stats and generate logs
  stats       process logs and print statistics
  json        process logs from several versions and generate JSON
  help        help information

For each command, you can try ./runtime-call-stats.py help command.
'''

import argparse
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
28
import operator
29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48

import numpy
import scipy
import scipy.stats
from math import sqrt


# Run benchmarks.

def print_command(cmd_args):
  def fix_for_printing(arg):
    m = re.match(r'^--([^=]+)=(.*)$', arg)
    if m and (' ' in m.group(2) or m.group(2).startswith('-')):
      arg = "--{}='{}'".format(m.group(1), m.group(2))
    elif ' ' in arg:
      arg = "'{}'".format(arg)
    return arg
  print " ".join(map(fix_for_printing, cmd_args))


49
def start_replay_server(args, sites, discard_output=True):
50 51 52
  with tempfile.NamedTemporaryFile(prefix='callstats-inject-', suffix='.js',
                                   mode='wt', delete=False) as f:
    injection = f.name
53
    generate_injection(f, sites, args.refresh)
54 55
  http_port = 4080 + args.port_offset
  https_port = 4443 + args.port_offset
56 57
  cmd_args = [
      args.replay_bin,
58 59
      "--port=%s" % http_port,
      "--ssl_port=%s" % https_port,
60 61 62
      "--no-dns_forwarding",
      "--use_closest_match",
      "--no-diff_unknown_requests",
63
      "--inject_scripts=deterministic.js,{}".format(injection),
64 65 66 67
      args.replay_wpr,
  ]
  print "=" * 80
  print_command(cmd_args)
68 69 70 71 72
  if discard_output:
    with open(os.devnull, 'w') as null:
      server = subprocess.Popen(cmd_args, stdout=null, stderr=null)
  else:
      server = subprocess.Popen(cmd_args)
73 74
  print "RUNNING REPLAY SERVER: %s with PID=%s" % (args.replay_bin, server.pid)
  print "=" * 80
75
  return {'process': server, 'injection': injection}
76 77 78


def stop_replay_server(server):
79 80 81 82 83
  print("SHUTTING DOWN REPLAY SERVER %s" % server['process'].pid)
  server['process'].terminate()
  os.remove(server['injection'])


84
def generate_injection(f, sites, refreshes=0):
85 86
  print >> f, """\
(function() {
87 88 89 90
  var s = window.sessionStorage.getItem("refreshCounter");
  var refreshTotal = """, refreshes, """;
  var refreshCounter = s ? parseInt(s) : refreshTotal;
  var refreshId = refreshTotal - refreshCounter;
91 92 93
  if (refreshCounter > 0) {
    window.sessionStorage.setItem("refreshCounter", refreshCounter-1);
  }
94
  function match(url, item) {
95 96 97
    if ('regexp' in item) { return url.match(item.regexp) !== null };
    var url_wanted = item.url;
    /* Allow automatic redirections from http to https. */
98 99 100 101
    if (url_wanted.startsWith("http://") && url.startsWith("https://")) {
      url_wanted = "https://" + url_wanted.substr(7);
    }
    return url.startsWith(url_wanted);
102
  };
103 104
  function onLoad(url) {
    for (var item of sites) {
105
      if (!match(url, item)) continue;
106
      var timeout = 'timeline' in item ? 2000 * item.timeline
107 108 109
                  : 'timeout'  in item ? 1000 * (item.timeout - 3)
                  : 10000;
      console.log("Setting time out of " + timeout + " for: " + url);
110
      window.setTimeout(function() {
111
        console.log("Time is out for: " + url);
112
        var msg = "STATS: (" + refreshId + ") " + url;
113 114
        %GetAndResetRuntimeCallStats(1, msg);
        if (refreshCounter > 0) {
115 116
          console.log(
              "Refresh counter is " + refreshCounter + ", refreshing: " + url);
117 118
          window.location.reload();
        }
119 120 121 122 123
      }, timeout);
      return;
    }
    console.log("Ignoring: " + url);
  };
124 125 126
  var sites =
    """, json.dumps(sites), """;
  onLoad(window.location.href);
127
})();"""
128

129
def get_chrome_flags(js_flags, user_data_dir, arg_delimiter=""):
130 131 132 133 134
  return [
      "--no-default-browser-check",
      "--no-sandbox",
      "--disable-translate",
      "--enable-benchmarking",
135 136
      "--enable-stats-table",
      "--js-flags={}{}{}".format(arg_delimiter, js_flags, arg_delimiter),
137
      "--no-first-run",
138 139
      "--user-data-dir={}{}{}".format(arg_delimiter, user_data_dir,
                                      arg_delimiter),
140 141
    ]

142
def get_chrome_replay_flags(args, arg_delimiter=""):
143 144 145
  http_port = 4080 + args.port_offset
  https_port = 4443 + args.port_offset
  return [
146 147 148 149 150
      "--host-resolver-rules=%sMAP *:80 localhost:%s, "  \
                              "MAP *:443 localhost:%s, " \
                              "EXCLUDE localhost%s" % (
                               arg_delimiter, http_port, https_port,
                               arg_delimiter),
151 152 153 154 155 156
      "--ignore-certificate-errors",
      "--disable-seccomp-sandbox",
      "--disable-web-security",
      "--reduce-security-for-testing",
      "--allow-insecure-localhost",
    ]
157 158 159 160 161 162 163

def run_site(site, domain, args, timeout=None):
  print "="*80
  print "RUNNING DOMAIN %s" % domain
  print "="*80
  result_template = "{domain}#{count}.txt" if args.repeat else "{domain}.txt"
  count = 0
164
  if timeout is None: timeout = args.timeout
165 166 167
  if args.replay_wpr:
    timeout *= 1 + args.refresh
    timeout += 1
168
  retries_since_good_run = 0
169 170 171 172 173 174 175
  while count == 0 or args.repeat is not None and count < args.repeat:
    count += 1
    result = result_template.format(domain=domain, count=count)
    retries = 0
    while args.retries is None or retries < args.retries:
      retries += 1
      try:
176 177 178
        if args.user_data_dir:
          user_data_dir = args.user_data_dir
        else:
179
          user_data_dir = tempfile.mkdtemp(prefix="chr_")
180
        js_flags = "--runtime-call-stats --noconcurrent-recompilation"
181
        if args.replay_wpr: js_flags += " --allow-natives-syntax"
182
        if args.js_flags: js_flags += " " + args.js_flags
183
        chrome_flags = get_chrome_flags(js_flags, user_data_dir)
184
        if args.replay_wpr:
185
          chrome_flags += get_chrome_replay_flags(args)
186
        else:
187
          chrome_flags += [ "--single-process", ]
188 189 190 191 192 193 194 195 196 197
        if args.chrome_flags:
          chrome_flags += args.chrome_flags.split()
        cmd_args = [
            "timeout", str(timeout),
            args.with_chrome
        ] + chrome_flags + [ site ]
        print "- " * 40
        print_command(cmd_args)
        print "- " * 40
        with open(result, "wt") as f:
198 199
          with open(args.log_stderr or os.devnull, 'at') as err:
            status = subprocess.call(cmd_args, stdout=f, stderr=err)
200 201 202 203 204 205 206 207 208 209 210 211
        # 124 means timeout killed chrome, 0 means the user was bored first!
        # If none of these two happened, then chrome apparently crashed, so
        # it must be called again.
        if status != 124 and status != 0:
          print("CHROME CRASHED, REPEATING RUN");
          continue
        # If the stats file is empty, chrome must be called again.
        if os.path.isfile(result) and os.path.getsize(result) > 0:
          if args.print_url:
            with open(result, "at") as f:
              print >> f
              print >> f, "URL: {}".format(site)
212
          retries_since_good_run = 0
213
          break
214 215 216 217 218
        if retries_since_good_run < 6:
          timeout += 2 ** retries_since_good_run
          retries_since_good_run += 1
        print("EMPTY RESULT, REPEATING RUN ({})".format(
            retries_since_good_run));
219
      finally:
220
        if not args.user_data_dir:
221 222 223 224 225 226 227 228 229 230 231
          shutil.rmtree(user_data_dir)


def read_sites_file(args):
  try:
    sites = []
    try:
      with open(args.sites_file, "rt") as f:
        for item in json.load(f):
          if 'timeout' not in item:
            # This is more-or-less arbitrary.
232
            item['timeout'] = int(1.5 * item['timeline'] + 7)
233 234 235 236 237 238 239 240 241 242 243 244 245 246
          if item['timeout'] > args.timeout: item['timeout'] = args.timeout
          sites.append(item)
    except ValueError:
      with open(args.sites_file, "rt") as f:
        for line in f:
          line = line.strip()
          if not line or line.startswith('#'): continue
          sites.append({'url': line, 'timeout': args.timeout})
    return sites
  except IOError as e:
    args.error("Cannot read from {}. {}.".format(args.sites_file, e.strerror))
    sys.exit(1)


247
def read_sites(args):
248 249
  # Determine the websites to benchmark.
  if args.sites_file:
250 251 252 253 254 255
    return read_sites_file(args)
  return [{'url': site, 'timeout': args.timeout} for site in args.sites]

def do_run(args):
  sites = read_sites(args)
  replay_server = start_replay_server(args, sites) if args.replay_wpr else None
256 257 258 259 260
  # Disambiguate domains, if needed.
  L = []
  domains = {}
  for item in sites:
    site = item['url']
261 262 263 264 265 266 267 268 269 270 271
    domain = None
    if args.domain:
      domain = args.domain
    elif 'domain' in item:
      domain = item['domain']
    else:
      m = re.match(r'^(https?://)?([^/]+)(/.*)?$', site)
      if not m:
        args.error("Invalid URL {}.".format(site))
        continue
      domain = m.group(2)
272 273 274 275 276 277 278 279 280 281 282 283 284 285
    entry = [site, domain, None, item['timeout']]
    if domain not in domains:
      domains[domain] = entry
    else:
      if not isinstance(domains[domain], int):
        domains[domain][2] = 1
        domains[domain] = 1
      domains[domain] += 1
      entry[2] = domains[domain]
    L.append(entry)
  try:
    # Run them.
    for site, domain, count, timeout in L:
      if count is not None: domain = "{}%{}".format(domain, count)
286
      print(site, domain, timeout)
287 288 289 290 291 292
      run_site(site, domain, args, timeout)
  finally:
    if replay_server:
      stop_replay_server(replay_server)


293 294 295 296 297 298 299 300
def do_run_replay_server(args):
  sites = read_sites(args)
  print("- " * 40)
  print("Available URLs:")
  for site in sites:
    print("    "+site['url'])
  print("- " * 40)
  print("Launch chromium with the following commands for debugging:")
301 302 303 304
  flags = get_chrome_flags("--runtime-call-stats --allow-natives-syntax",
                           "/var/tmp/`date +%s`", '"')
  flags += get_chrome_replay_flags(args, "'")
  print("    $CHROMIUM_DIR/out/Release/chrome " + (" ".join(flags)) + " <URL>")
305
  print("- " * 40)
306
  replay_server = start_replay_server(args, sites, discard_output=False)
307 308 309 310 311 312
  try:
    replay_server['process'].wait()
  finally:
   stop_replay_server(replay_server)


313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344
# Calculate statistics.

def statistics(data):
  N = len(data)
  average = numpy.average(data)
  median = numpy.median(data)
  low = numpy.min(data)
  high= numpy.max(data)
  if N > 1:
    # evaluate sample variance by setting delta degrees of freedom (ddof) to
    # 1. The degree used in calculations is N - ddof
    stddev = numpy.std(data, ddof=1)
    # Get the endpoints of the range that contains 95% of the distribution
    t_bounds = scipy.stats.t.interval(0.95, N-1)
    #assert abs(t_bounds[0] + t_bounds[1]) < 1e-6
    # sum mean to the confidence interval
    ci = {
        'abs': t_bounds[1] * stddev / sqrt(N),
        'low': average + t_bounds[0] * stddev / sqrt(N),
        'high': average + t_bounds[1] * stddev / sqrt(N)
    }
  else:
    stddev = 0
    ci = { 'abs': 0, 'low': average, 'high': average }
  if abs(stddev) > 0.0001 and abs(average) > 0.0001:
    ci['perc'] = t_bounds[1] * stddev / sqrt(N) / average * 100
  else:
    ci['perc'] = 0
  return { 'samples': N, 'average': average, 'median': median,
           'stddev': stddev, 'min': low, 'max': high, 'ci': ci }


345 346 347 348
def read_stats(path, domain, args):
  groups = [];
  if args.aggregate:
    groups = [
349
        ('Group-IC', re.compile(".*IC_.*")),
350 351
        ('Group-Optimize',
         re.compile("StackGuard|.*Optimize.*|.*Deoptimize.*|Recompile.*")),
352
        ('Group-CompileBackground', re.compile("(.*CompileBackground.*)")),
353
        ('Group-Compile', re.compile("(^Compile.*)|(.*_Compile.*)")),
354
        ('Group-ParseBackground', re.compile(".*ParseBackground.*")),
355 356 357
        ('Group-Parse', re.compile(".*Parse.*")),
        ('Group-Callback', re.compile(".*Callback.*")),
        ('Group-API', re.compile(".*API.*")),
358 359 360
        ('Group-GC', re.compile("GC|AllocateInTargetSpace")),
        ('Group-JavaScript', re.compile("JS_Execution")),
        ('Group-Runtime', re.compile(".*"))]
361 362
  with open(path, "rt") as f:
    # Process the whole file and sum repeating entries.
363 364 365
    entries = { 'Sum': {'time': 0, 'count': 0} }
    for group_name, regexp in groups:
      entries[group_name] = { 'time': 0, 'count': 0 }
366 367 368 369 370 371 372 373
    for line in f:
      line = line.strip()
      # Discard headers and footers.
      if not line: continue
      if line.startswith("Runtime Function"): continue
      if line.startswith("===="): continue
      if line.startswith("----"): continue
      if line.startswith("URL:"): continue
374
      if line.startswith("STATS:"): continue
375 376 377 378 379
      # We have a regular line.
      fields = line.split()
      key = fields[0]
      time = float(fields[1].replace("ms", ""))
      count = int(fields[3])
380 381 382
      if key not in entries: entries[key] = { 'time': 0, 'count': 0 }
      entries[key]['time'] += time
      entries[key]['count'] += count
383 384
      # We calculate the sum, if it's not the "total" line.
      if key != "Total":
385 386 387 388 389 390 391
        entries['Sum']['time'] += time
        entries['Sum']['count'] += count
        for group_name, regexp in groups:
          if not regexp.match(key): continue
          entries[group_name]['time'] += time
          entries[group_name]['count'] += count
          break
392
    # Calculate the V8-Total (all groups except Callback)
393
    group_data = { 'time': 0, 'count': 0 }
394 395
    for group_name, regexp in groups:
      if group_name == 'Group-Callback': continue
396 397 398 399 400 401
      group_data['time'] += entries[group_name]['time']
      group_data['count'] += entries[group_name]['count']
    entries['Group-Total-V8'] = group_data
    # Calculate the Parse-Total group
    group_data = { 'time': 0, 'count': 0 }
    for group_name, regexp in groups:
402
      if not group_name.startswith('Group-Parse'): continue
403 404 405
      group_data['time'] += entries[group_name]['time']
      group_data['count'] += entries[group_name]['count']
    entries['Group-Parse-Total'] = group_data
406 407 408 409 410 411 412
    # Calculate the Compile-Total group
    group_data = { 'time': 0, 'count': 0 }
    for group_name, regexp in groups:
      if not group_name.startswith('Group-Compile'): continue
      group_data['time'] += entries[group_name]['time']
      group_data['count'] += entries[group_name]['count']
    entries['Group-Compile-Total'] = group_data
413
    # Append the sums as single entries to domain.
414
    for key in entries:
415 416 417
      if key not in domain: domain[key] = { 'time_list': [], 'count_list': [] }
      domain[key]['time_list'].append(entries[key]['time'])
      domain[key]['count_list'].append(entries[key]['count'])
418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455


def print_stats(S, args):
  # Sort by ascending/descending time average, then by ascending/descending
  # count average, then by ascending name.
  def sort_asc_func(item):
    return (item[1]['time_stat']['average'],
            item[1]['count_stat']['average'],
            item[0])
  def sort_desc_func(item):
    return (-item[1]['time_stat']['average'],
            -item[1]['count_stat']['average'],
            item[0])
  # Sorting order is in the commend-line arguments.
  sort_func = sort_asc_func if args.sort == "asc" else sort_desc_func
  # Possibly limit how many elements to print.
  L = [item for item in sorted(S.items(), key=sort_func)
       if item[0] not in ["Total", "Sum"]]
  N = len(L)
  if args.limit == 0:
    low, high = 0, N
  elif args.sort == "desc":
    low, high = 0, args.limit
  else:
    low, high = N-args.limit, N
  # How to print entries.
  def print_entry(key, value):
    def stats(s, units=""):
      conf = "{:0.1f}({:0.2f}%)".format(s['ci']['abs'], s['ci']['perc'])
      return "{:8.1f}{} +/- {:15s}".format(s['average'], units, conf)
    print "{:>50s}  {}  {}".format(
      key,
      stats(value['time_stat'], units="ms"),
      stats(value['count_stat'])
    )
  # Print and calculate partial sums, if necessary.
  for i in range(low, high):
    print_entry(*L[i])
456
    if args.totals and args.limit != 0 and not args.aggregate:
457 458 459 460 461 462 463 464 465 466 467 468
      if i == low:
        partial = { 'time_list': [0] * len(L[i][1]['time_list']),
                    'count_list': [0] * len(L[i][1]['count_list']) }
      assert len(partial['time_list']) == len(L[i][1]['time_list'])
      assert len(partial['count_list']) == len(L[i][1]['count_list'])
      for j, v in enumerate(L[i][1]['time_list']):
        partial['time_list'][j] += v
      for j, v in enumerate(L[i][1]['count_list']):
        partial['count_list'][j] += v
  # Print totals, if necessary.
  if args.totals:
    print '-' * 80
469
    if args.limit != 0 and not args.aggregate:
470 471 472 473 474 475 476 477
      partial['time_stat'] = statistics(partial['time_list'])
      partial['count_stat'] = statistics(partial['count_list'])
      print_entry("Partial", partial)
    print_entry("Sum", S["Sum"])
    print_entry("Total", S["Total"])


def do_stats(args):
478
  domains = {}
479 480 481 482
  for path in args.logfiles:
    filename = os.path.basename(path)
    m = re.match(r'^([^#]+)(#.*)?$', filename)
    domain = m.group(1)
483 484 485 486 487 488
    if domain not in domains: domains[domain] = {}
    read_stats(path, domains[domain], args)
  if args.aggregate:
    create_total_page_stats(domains, args)
  for i, domain in enumerate(sorted(domains)):
    if len(domains) > 1:
489 490 491
      if i > 0: print
      print "{}:".format(domain)
      print '=' * 80
492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510
    domain_stats = domains[domain]
    for key in domain_stats:
      domain_stats[key]['time_stat'] = \
          statistics(domain_stats[key]['time_list'])
      domain_stats[key]['count_stat'] = \
          statistics(domain_stats[key]['count_list'])
    print_stats(domain_stats, args)


# Create a Total page with all entries summed up.
def create_total_page_stats(domains, args):
  total = {}
  def sum_up(parent, key, other):
    sums = parent[key]
    for i, item in enumerate(other[key]):
      if i >= len(sums):
        sums.extend([0] * (i - len(sums) + 1))
      if item is not None:
        sums[i] += item
511 512 513 514 515 516
  # Exclude adwords and speedometer pages from aggrigate total, since adwords
  # dominates execution time and speedometer is measured elsewhere.
  excluded_domains = ['adwords.google.com', 'speedometer-angular',
                      'speedometer-jquery', 'speedometer-backbone',
                      'speedometer-ember', 'speedometer-vanilla'];
  # Sum up all the entries/metrics from all non-excluded domains
517
  for domain, entries in domains.items():
518 519
    if domain in excluded_domains:
      continue;
520 521 522 523 524 525 526 527 528 529
    for key, domain_stats in entries.items():
      if key not in total:
        total[key] = {}
        total[key]['time_list'] = list(domain_stats['time_list'])
        total[key]['count_list'] = list(domain_stats['count_list'])
      else:
        sum_up(total[key], 'time_list', domain_stats)
        sum_up(total[key], 'count_list', domain_stats)
  # Add a new "Total" page containing the summed up metrics.
  domains['Total'] = total
530 531 532 533 534


# Generate JSON file.

def do_json(args):
535
  versions = {}
536 537 538 539
  for path in args.logdirs:
    if os.path.isdir(path):
      for root, dirs, files in os.walk(path):
        version = os.path.basename(root)
540
        if version not in versions: versions[version] = {}
541 542
        for filename in files:
          if filename.endswith(".txt"):
543
            m = re.match(r'^([^#]+)(#.*)?\.txt$', filename)
544
            domain = m.group(1)
545 546 547 548 549 550 551 552 553
            if domain not in versions[version]: versions[version][domain] = {}
            read_stats(os.path.join(root, filename),
                       versions[version][domain], args)
  for version, domains in versions.items():
    if args.aggregate:
      create_total_page_stats(domains, args)
    for domain, entries in domains.items():
      stats = []
      for name, value in entries.items():
554 555 556 557
        # We don't want the calculated sum in the JSON file.
        if name == "Sum": continue
        entry = [name]
        for x in ['time_list', 'count_list']:
558
          s = statistics(entries[name][x])
559 560 561
          entry.append(round(s['average'], 1))
          entry.append(round(s['ci']['abs'], 1))
          entry.append(round(s['ci']['perc'], 2))
562 563 564
        stats.append(entry)
      domains[domain] = stats
  print json.dumps(versions, separators=(',', ':'))
565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580


# Help.

def do_help(parser, subparsers, args):
  if args.help_cmd:
    if args.help_cmd in subparsers:
      subparsers[args.help_cmd].print_help()
    else:
      args.error("Unknown command '{}'".format(args.help_cmd))
  else:
    parser.print_help()


# Main program, parse command line and execute.

581 582 583 584
def coexist(*l):
  given = sum(1 for x in l if x)
  return given == 0 or given == len(l)

585 586 587 588 589 590 591
def main():
  parser = argparse.ArgumentParser()
  subparser_adder = parser.add_subparsers(title="commands", dest="command",
                                          metavar="<command>")
  subparsers = {}
  # Command: run.
  subparsers["run"] = subparser_adder.add_parser(
592
      "run", help="Replay websites and collect runtime stats data.")
593 594 595 596 597 598 599 600 601 602 603 604 605 606 607
  subparsers["run"].set_defaults(
      func=do_run, error=subparsers["run"].error)
  subparsers["run"].add_argument(
      "--chrome-flags", type=str, default="",
      help="specify additional chrome flags")
  subparsers["run"].add_argument(
      "--js-flags", type=str, default="",
      help="specify additional V8 flags")
  subparsers["run"].add_argument(
      "-u", "--user-data-dir", type=str, metavar="<path>",
      help="specify user data dir (default is temporary)")
  subparsers["run"].add_argument(
      "-c", "--with-chrome", type=str, metavar="<path>",
      default="/usr/bin/google-chrome",
      help="specify chrome executable to use")
608
  subparsers["run"].add_argument(
609 610 611 612 613 614 615 616
      "-r", "--retries", type=int, metavar="<num>",
      help="specify retries if website is down (default: forever)")
  subparsers["run"].add_argument(
      "--no-url", dest="print_url", action="store_false", default=True,
      help="do not include url in statistics file")
  subparsers["run"].add_argument(
      "--domain", type=str, default="",
      help="specify the output file domain name")
617
  subparsers["run"].add_argument(
618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655
      "-n", "--repeat", type=int, metavar="<num>",
      help="specify iterations for each website (default: once)")

  def add_replay_args(subparser):
    subparser.add_argument(
        "-k", "--refresh", type=int, metavar="<num>", default=0,
        help="specify refreshes for each iteration (default: 0)")
    subparser.add_argument(
        "--replay-wpr", type=str, metavar="<path>",
        help="use the specified web page replay (.wpr) archive")
    subparser.add_argument(
        "--replay-bin", type=str, metavar="<path>",
        help="specify the replay.py script typically located in " \
             "$CHROMIUM/src/third_party/webpagereplay/replay.py")
    subparser.add_argument(
        "-f", "--sites-file", type=str, metavar="<path>",
        help="specify file containing benchmark websites")
    subparser.add_argument(
        "-t", "--timeout", type=int, metavar="<seconds>", default=60,
        help="specify seconds before chrome is killed")
    subparser.add_argument(
        "-p", "--port-offset", type=int, metavar="<offset>", default=0,
        help="specify the offset for the replay server's default ports")
    subparser.add_argument(
        "-l", "--log-stderr", type=str, metavar="<path>",
        help="specify where chrome's stderr should go (default: /dev/null)")
    subparser.add_argument(
        "sites", type=str, metavar="<URL>", nargs="*",
        help="specify benchmark website")
  add_replay_args(subparsers["run"])

  # Command: replay-server
  subparsers["replay"] = subparser_adder.add_parser(
      "replay", help="Run the replay server for debugging purposes")
  subparsers["replay"].set_defaults(
      func=do_run_replay_server, error=subparsers["replay"].error)
  add_replay_args(subparsers["replay"])

656 657
  # Command: stats.
  subparsers["stats"] = subparser_adder.add_parser(
658
      "stats", help="Analize the results file create by the 'run' command.")
659 660 661 662 663 664 665 666 667 668 669 670 671 672
  subparsers["stats"].set_defaults(
      func=do_stats, error=subparsers["stats"].error)
  subparsers["stats"].add_argument(
      "-l", "--limit", type=int, metavar="<num>", default=0,
      help="limit how many items to print (default: none)")
  subparsers["stats"].add_argument(
      "-s", "--sort", choices=["asc", "desc"], default="asc",
      help="specify sorting order (default: ascending)")
  subparsers["stats"].add_argument(
      "-n", "--no-total", dest="totals", action="store_false", default=True,
      help="do not print totals")
  subparsers["stats"].add_argument(
      "logfiles", type=str, metavar="<logfile>", nargs="*",
      help="specify log files to parse")
673 674
  subparsers["stats"].add_argument(
      "--aggregate", dest="aggregate", action="store_true", default=False,
675
      help="Create aggregated entries. Adds Group-* entries at the toplevel. " \
676
      "Additionally creates a Total page with all entries.")
677

678 679
  # Command: json.
  subparsers["json"] = subparser_adder.add_parser(
680 681
      "json", help="Collect results file created by the 'run' command into" \
          "a single json file.")
682 683 684 685 686
  subparsers["json"].set_defaults(
      func=do_json, error=subparsers["json"].error)
  subparsers["json"].add_argument(
      "logdirs", type=str, metavar="<logdir>", nargs="*",
      help="specify directories with log files to parse")
687 688
  subparsers["json"].add_argument(
      "--aggregate", dest="aggregate", action="store_true", default=False,
689
      help="Create aggregated entries. Adds Group-* entries at the toplevel. " \
690
      "Additionally creates a Total page with all entries.")
691

692 693 694 695 696 697 698 699 700
  # Command: help.
  subparsers["help"] = subparser_adder.add_parser(
      "help", help="help information")
  subparsers["help"].set_defaults(
      func=lambda args: do_help(parser, subparsers, args),
      error=subparsers["help"].error)
  subparsers["help"].add_argument(
      "help_cmd", type=str, metavar="<command>", nargs="?",
      help="command for which to display help")
701

702 703
  # Execute the command.
  args = parser.parse_args()
704 705 706
  setattr(args, 'script_path', os.path.dirname(sys.argv[0]))
  if args.command == "run" and coexist(args.sites_file, args.sites):
    args.error("use either option --sites-file or site URLs")
707
    sys.exit(1)
708 709
  elif args.command == "run" and not coexist(args.replay_wpr, args.replay_bin):
    args.error("options --replay-wpr and --replay-bin must be used together")
710 711 712 713 714 715
    sys.exit(1)
  else:
    args.func(args)

if __name__ == "__main__":
  sys.exit(main())