generate-runtime-call-stats.py 13.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
#!/usr/bin/python3
# Copyright 2019 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# Runs chromium/src/run_benchmark for a given story and extracts the generated
# runtime call stats.

import argparse
import csv
import json
12
import glob
13 14 15 16 17
import os
import pathlib
import re
import tabulate
import shutil
18
import statistics
19 20 21 22
import subprocess
import sys
import tempfile

23 24
from callstats_groups import RUNTIME_CALL_STATS_GROUPS

25

26 27
JSON_FILE_EXTENSION=".pb_converted.json"

28 29 30 31
def parse_args():
  parser = argparse.ArgumentParser(
      description="Run story and collect runtime call stats.")
  parser.add_argument("story", metavar="story", nargs=1, help="story to run")
32 33 34 35 36
  parser.add_argument(
      "--group",
      dest="group",
      action="store_true",
      help="group common stats together into buckets")
37 38 39 40 41 42 43 44 45 46 47 48 49 50 51
  parser.add_argument(
      "-r",
      "--repeats",
      dest="repeats",
      metavar="N",
      action="store",
      type=int,
      default=1,
      help="number of times to run the story")
  parser.add_argument(
      "-v",
      "--verbose",
      dest="verbose",
      action="store_true",
      help="output benchmark runs to stdout")
52 53 54 55 56
  parser.add_argument(
      "--device",
      dest="device",
      action="store",
      help="device to run the test on. Passed directly to run_benchmark")
57 58 59 60 61 62 63 64 65 66 67 68 69
  parser.add_argument(
      "-d",
      "--dir",
      dest="dir",
      action="store",
      help=("directory to look for already generated output in. This must "
            "already exists and it won't re-run the benchmark"))
  parser.add_argument(
      "-f",
      "--format",
      dest="format",
      action="store",
      choices=["csv", "table"],
70
      help="output as CSV")
71 72 73 74 75 76
  parser.add_argument(
      "-o",
      "--output",
      metavar="FILE",
      dest="out_file",
      action="store",
77 78 79 80 81 82 83 84 85
      help="write table to FILE rather stdout")
  parser.add_argument(
      "--browser",
      dest="browser",
      metavar="BROWSER_TYPE",
      action="store",
      default="release",
      help=("Passed directly to --browser option of run_benchmark. Ignored if "
            "-executable is used"))
86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102
  parser.add_argument(
      "-e",
      "--executable",
      dest="executable",
      metavar="EXECUTABLE",
      action="store",
      help=("path to executable to run. If not given it will pass '--browser "
            "release' to run_benchmark"))
  parser.add_argument(
      "--chromium-dir",
      dest="chromium_dir",
      metavar="DIR",
      action="store",
      default=".",
      help=("path to chromium directory. If not given, the script must be run "
            "inside the chromium/src directory"))
  parser.add_argument(
103
      "--js-flags", dest="js_flags", action="store", help="flags to pass to v8")
104 105 106 107 108
  parser.add_argument(
      "--extra-browser-args",
      dest="browser_args",
      action="store",
      help="flags to pass to chrome")
109 110 111 112 113 114 115 116 117 118 119
  parser.add_argument(
      "--benchmark",
      dest="benchmark",
      action="store",
      default="v8.browsing_desktop",
      help="benchmark to run")
  parser.add_argument(
      "--stdev",
      dest="stdev",
      action="store_true",
      help="adds columns for the standard deviation")
120 121 122 123 124
  parser.add_argument(
      "--filter",
      dest="filter",
      action="append",
      help="useable with --group to only show buckets specified by filter")
125 126 127 128 129 130 131 132 133
  parser.add_argument(
      "--retain",
      dest="retain",
      action="store",
      default="json",
      choices=["none", "json", "all"],
      help=("controls artifacts to be retained after run. With none, all files "
            "are deleted; only the json.gz file is retained for each run; and "
            "all keep all files"))
134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159

  return parser.parse_args()


def process_trace(trace_file):
  text_string = pathlib.Path(trace_file).read_text()
  result = json.loads(text_string)

  output = {}
  result = result["traceEvents"]
  for o in result:
    o = o["args"]
    if "runtime-call-stats" in o:
      r = o["runtime-call-stats"]
      for name in r:
        count = r[name][0]
        duration = r[name][1]
        if name in output:
          output[name]["count"] += count
          output[name]["duration"] += duration
        else:
          output[name] = {"count": count, "duration": duration}

  return output


160 161 162 163 164 165 166 167 168 169 170
def run_benchmark(story,
                  repeats=1,
                  output_dir=".",
                  verbose=False,
                  js_flags=None,
                  browser_args=None,
                  chromium_dir=".",
                  executable=None,
                  benchmark="v8.browsing_desktop",
                  device=None,
                  browser="release"):
171 172 173 174 175 176 177 178 179 180 181 182 183 184 185

  orig_chromium_dir = chromium_dir
  xvfb = os.path.join(chromium_dir, "testing", "xvfb.py")
  if not os.path.isfile(xvfb):
    chromium_dir = os.path(chromium_dir, "src")
    xvfb = os.path.join(chromium_dir, "testing", "xvfb.py")
    if not os.path.isfile(xvfb):
      print(("chromium_dir does not point to a valid chromium checkout: " +
             orig_chromium_dir))
      sys.exit(1)

  command = [
      xvfb,
      os.path.join(chromium_dir, "tools", "perf", "run_benchmark"),
      "run",
186 187 188 189 190 191 192 193
      "--story",
      story,
      "--pageset-repeat",
      str(repeats),
      "--output-dir",
      output_dir,
      "--intermediate-dir",
      os.path.join(output_dir, "artifacts"),
194
      benchmark,
195 196 197 198 199
  ]

  if executable:
    command += ["--browser-executable", executable]
  else:
200
    command += ["--browser", browser]
201

202 203
  if device:
    command += ["--device", device]
204 205 206 207 208
  if browser_args:
    command += ["--extra-browser-args", browser_args]
  if js_flags:
    command += ["--js-flags", js_flags]

209 210 211 212 213 214 215 216 217 218
  if not benchmark.startswith("v8."):
    # Most benchmarks by default don't collect runtime call stats so enable them
    # manually.
    categories = [
        "v8",
        "disabled-by-default-v8.runtime_stats",
    ]

    command += ["--extra-chrome-categories", ",".join(categories)]

219 220 221 222 223 224 225 226 227
  print("Output directory: %s" % output_dir)
  stdout = ""
  print(f"Running: {' '.join(command)}\n")
  proc = subprocess.Popen(
      command,
      stdout=subprocess.PIPE,
      stderr=subprocess.PIPE,
      universal_newlines=True)
  proc.stderr.close()
228
  status_matcher = re.compile(r"\[ +(\w+) +\]")
229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
  for line in iter(proc.stdout.readline, ""):
    stdout += line
    match = status_matcher.match(line)
    if verbose or match:
      print(line, end="")

  proc.stdout.close()

  if proc.wait() != 0:
    print("\nrun_benchmark failed:")
    # If verbose then everything has already been printed.
    if not verbose:
      print(stdout)
    sys.exit(1)

  print("\nrun_benchmark completed")


247
def write_output(f, table, headers, run_count, format="table"):
248
  if format == "csv":
249
    # strip new lines from CSV output
250
    headers = [h.replace("\n", " ") for h in headers]
251 252 253 254
    writer = csv.writer(f)
    writer.writerow(headers)
    writer.writerows(table)
  else:
255 256 257 258
    # First column is name, and then they alternate between counts and durations
    summary_count = len(headers) - 2 * run_count - 1
    floatfmt = ("",) + (".0f", ".2f") * run_count + (".2f",) * summary_count
    f.write(tabulate.tabulate(table, headers=headers, floatfmt=floatfmt))
259 260 261
    f.write("\n")


262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369
class Row:

  def __init__(self, name, run_count):
    self.name = name
    self.durations = [0] * run_count
    self.counts = [0] * run_count
    self.mean_duration = None
    self.mean_count = None
    self.stdev_duration = None
    self.stdev_count = None

  def __repr__(self):
    data_str = ", ".join(
        str((c, d)) for (c, d) in zip(self.counts, self.durations))
    return (f"{self.name}: {data_str}, mean_count: {self.mean_count}, " +
            f"mean_duration: {self.mean_duration}")

  def add_data(self, counts, durations):
    self.counts = counts
    self.durations = durations

  def add_data_point(self, run, count, duration):
    self.counts[run] = count
    self.durations[run] = duration

  def prepare(self, stdev=False):
    if len(self.durations) > 1:
      self.mean_duration = statistics.mean(self.durations)
      self.mean_count = statistics.mean(self.counts)
      if stdev:
        self.stdev_duration = statistics.stdev(self.durations)
        self.stdev_count = statistics.stdev(self.counts)

  def as_list(self):
    l = [self.name]
    for (c, d) in zip(self.counts, self.durations):
      l += [c, d]
    if self.mean_duration is not None:
      l += [self.mean_count]
      if self.stdev_count is not None:
        l += [self.stdev_count]
      l += [self.mean_duration]
      if self.stdev_duration is not None:
        l += [self.stdev_duration]
    return l

  def key(self):
    if self.mean_duration is not None:
      return self.mean_duration
    else:
      return self.durations[0]


class Bucket:

  def __init__(self, name, run_count):
    self.name = name
    self.run_count = run_count
    self.data = {}
    self.table = None
    self.total_row = None

  def __repr__(self):
    s = "Bucket: " + self.name + " {\n"
    if self.table:
      s += "\n  ".join(str(row) for row in self.table) + "\n"
    elif self.data:
      s += "\n  ".join(str(row) for row in self.data.values()) + "\n"
    if self.total_row:
      s += "  " + str(self.total_row) + "\n"
    return s + "}"

  def add_data_point(self, name, run, count, duration):
    if name not in self.data:
      self.data[name] = Row(name, self.run_count)

    self.data[name].add_data_point(run, count, duration)

  def prepare(self, stdev=False):
    if self.data:
      for row in self.data.values():
        row.prepare(stdev)

      self.table = sorted(self.data.values(), key=Row.key)
      self.total_row = Row("Total", self.run_count)
      self.total_row.add_data([
          sum(r.counts[i]
              for r in self.data.values())
          for i in range(0, self.run_count)
      ], [
          sum(r.durations[i]
              for r in self.data.values())
          for i in range(0, self.run_count)
      ])
      self.total_row.prepare(stdev)

  def as_list(self, add_bucket_titles=True, filter=None):
    t = []
    if filter is None or self.name in filter:
      if add_bucket_titles:
        t += [["\n"], [self.name]]
      t += [r.as_list() for r in self.table]
      t += [self.total_row.as_list()]
    return t


def collect_buckets(story, group=True, repeats=1, output_dir="."):
  if group:
370 371 372 373 374 375
    groups = RUNTIME_CALL_STATS_GROUPS
  else:
    groups = []

  buckets = {}

376
  for i in range(0, repeats):
377
    story_dir = f"{story.replace(':', '_')}_{i + 1}"
378 379
    trace_dir = os.path.join(output_dir, "artifacts", story_dir, "trace",
                             "traceEvents")
380 381 382

    # run_benchmark now dumps two files: a .pb.gz file and a .pb_converted.json
    # file. We only need the latter.
383
    trace_file_glob = os.path.join(trace_dir, "*" + JSON_FILE_EXTENSION)
384 385
    trace_files = glob.glob(trace_file_glob)
    if not trace_files:
386
      print("Could not find *%s file in %s" % (JSON_FILE_EXTENSION, trace_dir))
387 388 389 390 391 392
      sys.exit(1)
    if len(trace_files) > 1:
      print("Expecting one file but got: %s" % trace_files)
      sys.exit(1)

    trace_file = trace_files[0]
393 394 395

    output = process_trace(trace_file)
    for name in output:
396 397 398 399 400 401
      bucket_name = "Other"
      for group in groups:
        if group[1].match(name):
          bucket_name = group[0]
          break

402
      value = output[name]
403
      if bucket_name not in buckets:
404
        bucket = Bucket(bucket_name, repeats)
405 406 407 408 409
        buckets[bucket_name] = bucket
      else:
        bucket = buckets[bucket_name]

      bucket.add_data_point(name, i, value["count"], value["duration"] / 1000.0)
410
  return buckets
411 412


413 414 415 416 417 418
def create_table(buckets, record_bucket_names=True, filter=None):
  table = []
  for bucket in buckets.values():
    table += bucket.as_list(
        add_bucket_titles=record_bucket_names, filter=filter)
  return table
419 420


421 422 423
def main():
  args = parse_args()
  story = args.story[0]
424

425 426 427 428 429 430
  retain = args.retain
  if args.dir is not None:
    output_dir = args.dir
    if not os.path.isdir(output_dir):
      print("Specified output directory does not exist: " % output_dir)
      sys.exit(1)
431
  else:
432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464
    output_dir = tempfile.mkdtemp(prefix="runtime_call_stats_")
    run_benchmark(
        story,
        repeats=args.repeats,
        output_dir=output_dir,
        verbose=args.verbose,
        js_flags=args.js_flags,
        browser_args=args.browser_args,
        chromium_dir=args.chromium_dir,
        benchmark=args.benchmark,
        executable=args.executable,
        browser=args.browser,
        device=args.device)

  try:
    buckets = collect_buckets(
        story, group=args.group, repeats=args.repeats, output_dir=output_dir)

    for b in buckets.values():
      b.prepare(args.stdev)

    table = create_table(
        buckets, record_bucket_names=args.group, filter=args.filter)

    headers = [""] + ["Count", "Duration\n(ms)"] * args.repeats
    if args.repeats > 1:
      if args.stdev:
        headers += [
            "Count\nMean", "Count\nStdev", "Duration\nMean (ms)",
            "Duration\nStdev (ms)"
        ]
      else:
        headers += ["Count\nMean", "Duration\nMean (ms)"]
465

466 467 468 469 470 471 472 473 474
    if args.out_file:
      with open(args.out_file, "w", newline="") as f:
        write_output(f, table, headers, args.repeats, args.format)
    else:
      write_output(sys.stdout, table, headers, args.repeats, args.format)
  finally:
    if retain == "none":
      shutil.rmtree(output_dir)
    elif retain == "json":
475 476
      # Delete all files bottom up except ones ending in JSON_FILE_EXTENSION and
      # attempt to delete subdirectories (ignoring errors).
477 478 479
      for dir_name, subdir_list, file_list in os.walk(
          output_dir, topdown=False):
        for file_name in file_list:
480
          if not file_name.endswith(JSON_FILE_EXTENSION):
481 482 483 484 485 486 487 488 489
            os.remove(os.path.join(dir_name, file_name))
        for subdir in subdir_list:
          try:
            os.rmdir(os.path.join(dir_name, subdir))
          except OSError:
            pass


if __name__ == "__main__":
490
  sys.exit(main())