Add Collections perf tests to v8 public repo.

R=machenbach@chromium.org
BUG=374708
LOG=N

Review URL: https://codereview.chromium.org/530793002

git-svn-id: https://v8.googlecode.com/svn/branches/bleeding_edge@23570 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

test/perf-test/Collections/Collections.json

+{
+  "path": ["."],
+  "main": "run.js",
+  "flags": ["--harmony-collections"],
+  "run_count": 5,
+  "units": "score",
+  "results_regexp": "^%s\\-Collections\\(Score\\): (.+)$",
+  "total": true,
+  "tests": [
+    {"name": "Map"},
+    {"name": "Set"},
+    {"name": "WeakMap"},
+    {"name": "WeakSet"}
+  ]
+}

test/perf-test/Collections/base.js

+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+
+// Performance.now is used in latency benchmarks, the fallback is Date.now.
+var performance = performance || {};
+performance.now = (function() {
+  return performance.now       ||
+         performance.mozNow    ||
+         performance.msNow     ||
+         performance.oNow      ||
+         performance.webkitNow ||
+         Date.now;
+})();
+
+// Simple framework for running the benchmark suites and
+// computing a score based on the timing measurements.
+
+
+// A benchmark has a name (string) and a function that will be run to
+// do the performance measurement. The optional setup and tearDown
+// arguments are functions that will be invoked before and after
+// running the benchmark, but the running time of these functions will
+// not be accounted for in the benchmark score.
+function Benchmark(name, doWarmup, doDeterministic, deterministicIterations,
+                   run, setup, tearDown, rmsResult, minIterations) {
+  this.name = name;
+  this.doWarmup = doWarmup;
+  this.doDeterministic = doDeterministic;
+  this.deterministicIterations = deterministicIterations;
+  this.run = run;
+  this.Setup = setup ? setup : function() { };
+  this.TearDown = tearDown ? tearDown : function() { };
+  this.rmsResult = rmsResult ? rmsResult : null;
+  this.minIterations = minIterations ? minIterations : 32;
+}
+
+
+// Benchmark results hold the benchmark and the measured time used to
+// run the benchmark. The benchmark score is computed later once a
+// full benchmark suite has run to completion. If latency is set to 0
+// then there is no latency score for this benchmark.
+function BenchmarkResult(benchmark, time, latency) {
+  this.benchmark = benchmark;
+  this.time = time;
+  this.latency = latency;
+}
+
+
+// Automatically convert results to numbers. Used by the geometric
+// mean computation.
+BenchmarkResult.prototype.valueOf = function() {
+  return this.time;
+}
+
+
+// Suites of benchmarks consist of a name and the set of benchmarks in
+// addition to the reference timing that the final score will be based
+// on. This way, all scores are relative to a reference run and higher
+// scores implies better performance.
+function BenchmarkSuite(name, reference, benchmarks) {
+  this.name = name;
+  this.reference = reference;
+  this.benchmarks = benchmarks;
+  BenchmarkSuite.suites.push(this);
+}
+
+
+// Keep track of all declared benchmark suites.
+BenchmarkSuite.suites = [];
+
+// Scores are not comparable across versions. Bump the version if
+// you're making changes that will affect that scores, e.g. if you add
+// a new benchmark or change an existing one.
+BenchmarkSuite.version = '1';
+
+
+// Defines global benchsuite running mode that overrides benchmark suite
+// behavior. Intended to be set by the benchmark driver. Undefined
+// values here allow a benchmark to define behaviour itself.
+BenchmarkSuite.config = {
+  doWarmup: undefined,
+  doDeterministic: undefined
+};
+
+
+// Override the alert function to throw an exception instead.
+alert = function(s) {
+  throw "Alert called with argument: " + s;
+};
+
+
+// To make the benchmark results predictable, we replace Math.random
+// with a 100% deterministic alternative.
+BenchmarkSuite.ResetRNG = function() {
+  Math.random = (function() {
+    var seed = 49734321;
+    return function() {
+      // Robert Jenkins' 32 bit integer hash function.
+      seed = ((seed + 0x7ed55d16) + (seed << 12))  & 0xffffffff;
+      seed = ((seed ^ 0xc761c23c) ^ (seed >>> 19)) & 0xffffffff;
+      seed = ((seed + 0x165667b1) + (seed << 5))   & 0xffffffff;
+      seed = ((seed + 0xd3a2646c) ^ (seed << 9))   & 0xffffffff;
+      seed = ((seed + 0xfd7046c5) + (seed << 3))   & 0xffffffff;
+      seed = ((seed ^ 0xb55a4f09) ^ (seed >>> 16)) & 0xffffffff;
+      return (seed & 0xfffffff) / 0x10000000;
+    };
+  })();
+}
+
+
+// Runs all registered benchmark suites and optionally yields between
+// each individual benchmark to avoid running for too long in the
+// context of browsers. Once done, the final score is reported to the
+// runner.
+BenchmarkSuite.RunSuites = function(runner, skipBenchmarks) {
+  skipBenchmarks = typeof skipBenchmarks === 'undefined' ? [] : skipBenchmarks;
+  var continuation = null;
+  var suites = BenchmarkSuite.suites;
+  var length = suites.length;
+  BenchmarkSuite.scores = [];
+  var index = 0;
+  function RunStep() {
+    while (continuation || index < length) {
+      if (continuation) {
+        continuation = continuation();
+      } else {
+        var suite = suites[index++];
+        if (runner.NotifyStart) runner.NotifyStart(suite.name);
+        if (skipBenchmarks.indexOf(suite.name) > -1) {
+          suite.NotifySkipped(runner);
+        } else {
+          continuation = suite.RunStep(runner);
+        }
+      }
+      if (continuation && typeof window != 'undefined' && window.setTimeout) {
+        window.setTimeout(RunStep, 25);
+        return;
+      }
+    }
+
+    // show final result
+    if (runner.NotifyScore) {
+      var score = BenchmarkSuite.GeometricMean(BenchmarkSuite.scores);
+      var formatted = BenchmarkSuite.FormatScore(100 * score);
+      runner.NotifyScore(formatted);
+    }
+  }
+  RunStep();
+}
+
+
+// Counts the total number of registered benchmarks. Useful for
+// showing progress as a percentage.
+BenchmarkSuite.CountBenchmarks = function() {
+  var result = 0;
+  var suites = BenchmarkSuite.suites;
+  for (var i = 0; i < suites.length; i++) {
+    result += suites[i].benchmarks.length;
+  }
+  return result;
+}
+
+
+// Computes the geometric mean of a set of numbers.
+BenchmarkSuite.GeometricMean = function(numbers) {
+  var log = 0;
+  for (var i = 0; i < numbers.length; i++) {
+    log += Math.log(numbers[i]);
+  }
+  return Math.pow(Math.E, log / numbers.length);
+}
+
+
+// Computes the geometric mean of a set of throughput time measurements.
+BenchmarkSuite.GeometricMeanTime = function(measurements) {
+  var log = 0;
+  for (var i = 0; i < measurements.length; i++) {
+    log += Math.log(measurements[i].time);
+  }
+  return Math.pow(Math.E, log / measurements.length);
+}
+
+
+// Computes the geometric mean of a set of rms measurements.
+BenchmarkSuite.GeometricMeanLatency = function(measurements) {
+  var log = 0;
+  var hasLatencyResult = false;
+  for (var i = 0; i < measurements.length; i++) {
+    if (measurements[i].latency != 0) {
+      log += Math.log(measurements[i].latency);
+      hasLatencyResult = true;
+    }
+  }
+  if (hasLatencyResult) {
+    return Math.pow(Math.E, log / measurements.length);
+  } else {
+    return 0;
+  }
+}
+
+
+// Converts a score value to a string with at least three significant
+// digits.
+BenchmarkSuite.FormatScore = function(value) {
+  if (value > 100) {
+    return value.toFixed(0);
+  } else {
+    return value.toPrecision(3);
+  }
+}
+
+// Notifies the runner that we're done running a single benchmark in
+// the benchmark suite. This can be useful to report progress.
+BenchmarkSuite.prototype.NotifyStep = function(result) {
+  this.results.push(result);
+  if (this.runner.NotifyStep) this.runner.NotifyStep(result.benchmark.name);
+}
+
+
+// Notifies the runner that we're done with running a suite and that
+// we have a result which can be reported to the user if needed.
+BenchmarkSuite.prototype.NotifyResult = function() {
+  var mean = BenchmarkSuite.GeometricMeanTime(this.results);
+  var score = this.reference[0] / mean;
+  BenchmarkSuite.scores.push(score);
+  if (this.runner.NotifyResult) {
+    var formatted = BenchmarkSuite.FormatScore(100 * score);
+    this.runner.NotifyResult(this.name, formatted);
+  }
+  if (this.reference.length == 2) {
+    var meanLatency = BenchmarkSuite.GeometricMeanLatency(this.results);
+    if (meanLatency != 0) {
+      var scoreLatency = this.reference[1] / meanLatency;
+      BenchmarkSuite.scores.push(scoreLatency);
+      if (this.runner.NotifyResult) {
+        var formattedLatency = BenchmarkSuite.FormatScore(100 * scoreLatency)
+        this.runner.NotifyResult(this.name + "Latency", formattedLatency);
+      }
+    }
+  }
+}
+
+
+BenchmarkSuite.prototype.NotifySkipped = function(runner) {
+  BenchmarkSuite.scores.push(1);  // push default reference score.
+  if (runner.NotifyResult) {
+    runner.NotifyResult(this.name, "Skipped");
+  }
+}
+
+
+// Notifies the runner that running a benchmark resulted in an error.
+BenchmarkSuite.prototype.NotifyError = function(error) {
+  if (this.runner.NotifyError) {
+    this.runner.NotifyError(this.name, error);
+  }
+  if (this.runner.NotifyStep) {
+    this.runner.NotifyStep(this.name);
+  }
+}
+
+
+// Runs a single benchmark for at least a second and computes the
+// average time it takes to run a single iteration.
+BenchmarkSuite.prototype.RunSingleBenchmark = function(benchmark, data) {
+  var config = BenchmarkSuite.config;
+  var doWarmup = config.doWarmup !== undefined
+                 ? config.doWarmup
+                 : benchmark.doWarmup;
+  var doDeterministic = config.doDeterministic !== undefined
+                        ? config.doDeterministic
+                        : benchmark.doDeterministic;
+
+  function Measure(data) {
+    var elapsed = 0;
+    var start = new Date();
+
+  // Run either for 1 second or for the number of iterations specified
+  // by minIterations, depending on the config flag doDeterministic.
+    for (var i = 0; (doDeterministic ?
+      i<benchmark.deterministicIterations : elapsed < 1000); i++) {
+      benchmark.run();
+      elapsed = new Date() - start;
+    }
+    if (data != null) {
+      data.runs += i;
+      data.elapsed += elapsed;
+    }
+  }
+
+  // Sets up data in order to skip or not the warmup phase.
+  if (!doWarmup && data == null) {
+    data = { runs: 0, elapsed: 0 };
+  }
+
+  if (data == null) {
+    Measure(null);
+    return { runs: 0, elapsed: 0 };
+  } else {
+    Measure(data);
+    // If we've run too few iterations, we continue for another second.
+    if (data.runs < benchmark.minIterations) return data;
+    var usec = (data.elapsed * 1000) / data.runs;
+    var rms = (benchmark.rmsResult != null) ? benchmark.rmsResult() : 0;
+    this.NotifyStep(new BenchmarkResult(benchmark, usec, rms));
+    return null;
+  }
+}
+
+
+// This function starts running a suite, but stops between each
+// individual benchmark in the suite and returns a continuation
+// function which can be invoked to run the next benchmark. Once the
+// last benchmark has been executed, null is returned.
+BenchmarkSuite.prototype.RunStep = function(runner) {
+  BenchmarkSuite.ResetRNG();
+  this.results = [];
+  this.runner = runner;
+  var length = this.benchmarks.length;
+  var index = 0;
+  var suite = this;
+  var data;
+
+  // Run the setup, the actual benchmark, and the tear down in three
+  // separate steps to allow the framework to yield between any of the
+  // steps.
+
+  function RunNextSetup() {
+    if (index < length) {
+      try {
+        suite.benchmarks[index].Setup();
+      } catch (e) {
+        suite.NotifyError(e);
+        return null;
+      }
+      return RunNextBenchmark;
+    }
+    suite.NotifyResult();
+    return null;
+  }
+
+  function RunNextBenchmark() {
+    try {
+      data = suite.RunSingleBenchmark(suite.benchmarks[index], data);
+    } catch (e) {
+      suite.NotifyError(e);
+      return null;
+    }
+    // If data is null, we're done with this benchmark.
+    return (data == null) ? RunNextTearDown : RunNextBenchmark();
+  }
+
+  function RunNextTearDown() {
+    try {
+      suite.benchmarks[index++].TearDown();
+    } catch (e) {
+      suite.NotifyError(e);
+      return null;
+    }
+    return RunNextSetup;
+  }
+
+  // Start out running the setup.
+  return RunNextSetup();
+}

test/perf-test/Collections/map.js

+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+
+var MapBenchmark = new BenchmarkSuite('Map', [1000], [
+  new Benchmark('Set', false, false, 0, MapSet),
+  new Benchmark('Has', false, false, 0, MapHas, MapSetup, MapTearDown),
+  new Benchmark('Get', false, false, 0, MapGet, MapSetup, MapTearDown),
+  new Benchmark('Delete', false, false, 0, MapDelete, MapSetup, MapTearDown),
+  new Benchmark('ForEach', false, false, 0, MapForEach, MapSetup, MapTearDown),
+]);
+
+
+var map;
+var N = 10;
+
+
+function MapSetup() {
+  map = new Map;
+  for (var i = 0; i < N; i++) {
+    map.set(i, i);
+  }
+}
+
+
+function MapTearDown() {
+  map = null;
+}
+
+
+function MapSet() {
+  MapSetup();
+  MapTearDown();
+}
+
+
+function MapHas() {
+  for (var i = 0; i < N; i++) {
+    if (!map.has(i)) {
+      throw new Error();
+    }
+  }
+  for (var i = N; i < 2 * N; i++) {
+    if (map.has(i)) {
+      throw new Error();
+    }
+  }
+}
+
+
+function MapGet() {
+  for (var i = 0; i < N; i++) {
+    if (map.get(i) !== i) {
+      throw new Error();
+    }
+  }
+  for (var i = N; i < 2 * N; i++) {
+    if (map.get(i) !== undefined) {
+      throw new Error();
+    }
+  }
+}
+
+
+function MapDelete() {
+  // This is run more than once per setup so we will end up deleting items
+  // more than once. Therefore, we do not the return value of delete.
+  for (var i = 0; i < N; i++) {
+    map.delete(i);
+  }
+}
+
+
+function MapForEach() {
+  map.forEach(function(v, k) {
+    if (v !== k) {
+      throw new Error();
+    }
+  });
+}

test/perf-test/Collections/run.js

+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+
+load('base.js');
+load('map.js');
+load('set.js');
+load('weakmap.js');
+load('weakset.js');
+
+
+var success = true;
+
+function PrintResult(name, result) {
+  print(name + '-Collections(Score): ' + result);
+}
+
+
+function PrintError(name, error) {
+  PrintResult(name, error);
+  success = false;
+}
+
+
+BenchmarkSuite.config.doWarmup = undefined;
+BenchmarkSuite.config.doDeterministic = undefined;
+
+BenchmarkSuite.RunSuites({ NotifyResult: PrintResult,
+                           NotifyError: PrintError });

test/perf-test/Collections/set.js

+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+
+var SetBenchmark = new BenchmarkSuite('Set', [1000], [
+  new Benchmark('Add', false, false, 0, SetAdd),
+  new Benchmark('Has', false, false, 0, SetHas, SetSetup, SetTearDown),
+  new Benchmark('Delete', false, false, 0, SetDelete, SetSetup, SetTearDown),
+  new Benchmark('ForEach', false, false, 0, SetForEach, SetSetup, SetTearDown),
+]);
+
+
+var set;
+var N = 10;
+
+
+function SetSetup() {
+  set = new Set;
+  for (var i = 0; i < N; i++) {
+    set.add(i);
+  }
+}
+
+
+function SetTearDown() {
+  map = null;
+}
+
+
+function SetAdd() {
+  SetSetup();
+  SetTearDown();
+}
+
+
+function SetHas() {
+  for (var i = 0; i < N; i++) {
+    if (!set.has(i)) {
+      throw new Error();
+    }
+  }
+  for (var i = N; i < 2 * N; i++) {
+    if (set.has(i)) {
+      throw new Error();
+    }
+  }
+}
+
+
+function SetDelete() {
+  // This is run more than once per setup so we will end up deleting items
+  // more than once. Therefore, we do not the return value of delete.
+  for (var i = 0; i < N; i++) {
+    set.delete(i);
+  }
+}
+
+
+function SetForEach() {
+  set.forEach(function(v, k) {
+    if (v !== k) {
+      throw new Error();
+    }
+  });
+}

test/perf-test/Collections/weakmap.js

+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+
+var MapBenchmark = new BenchmarkSuite('WeakMap', [1000], [
+  new Benchmark('Set', false, false, 0, WeakMapSet),
+  new Benchmark('Has', false, false, 0, WeakMapHas, WeakMapSetup,
+      WeakMapTearDown),
+  new Benchmark('Get', false, false, 0, WeakMapGet, WeakMapSetup,
+      WeakMapTearDown),
+  new Benchmark('Delete', false, false, 0, WeakMapDelete, WeakMapSetup,
+      WeakMapTearDown),
+]);
+
+
+var wm;
+var N = 10;
+var keys = [];
+
+
+for (var i = 0; i < N * 2; i++) {
+  keys[i] = {};
+}
+
+
+function WeakMapSetup() {
+  wm = new WeakMap;
+  for (var i = 0; i < N; i++) {
+    wm.set(keys[i], i);
+  }
+}
+
+
+function WeakMapTearDown() {
+  wm = null;
+}
+
+
+function WeakMapSet() {
+  WeakMapSetup();
+  WeakMapTearDown();
+}
+
+
+function WeakMapHas() {
+  for (var i = 0; i < N; i++) {
+    if (!wm.has(keys[i])) {
+      throw new Error();
+    }
+  }
+  for (var i = N; i < 2 * N; i++) {
+    if (wm.has(keys[i])) {
+      throw new Error();
+    }
+  }
+}
+
+
+function WeakMapGet() {
+  for (var i = 0; i < N; i++) {
+    if (wm.get(keys[i]) !== i) {
+      throw new Error();
+    }
+  }
+  for (var i = N; i < 2 * N; i++) {
+    if (wm.get(keys[i]) !== undefined) {
+      throw new Error();
+    }
+  }
+}
+
+
+function WeakMapDelete() {
+  // This is run more than once per setup so we will end up deleting items
+  // more than once. Therefore, we do not the return value of delete.
+  for (var i = 0; i < N; i++) {
+    wm.delete(keys[i]);
+  }
+}

test/perf-test/Collections/weakset.js

+// Copyright 2014 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+
+var SetBenchmark = new BenchmarkSuite('WeakSet', [1000], [
+  new Benchmark('Add', false, false, 0, WeakSetAdd),
+  new Benchmark('Has', false, false, 0, WeakSetHas, WeakSetSetup,
+      WeakSetTearDown),
+  new Benchmark('Delete', false, false, 0, WeakSetDelete, WeakSetSetup,
+      WeakSetTearDown),
+]);
+
+
+var ws;
+var N = 10;
+var keys = [];
+
+
+for (var i = 0; i < N * 2; i++) {
+  keys[i] = {};
+}
+
+
+function WeakSetSetup() {
+  ws = new WeakSet;
+  for (var i = 0; i < N; i++) {
+    ws.add(keys[i]);
+  }
+}
+
+
+function WeakSetTearDown() {
+  ws = null;
+}
+
+
+function WeakSetAdd() {
+  WeakSetSetup();
+  WeakSetTearDown();
+}
+
+
+function WeakSetHas() {
+  for (var i = 0; i < N; i++) {
+    if (!ws.has(keys[i])) {
+      throw new Error();
+    }
+  }
+  for (var i = N; i < 2 * N; i++) {
+    if (ws.has(keys[i])) {
+      throw new Error();
+    }
+  }
+}
+
+
+function WeakSetDelete() {
+  // This is run more than once per setup so we will end up deleting items
+  // more than once. Therefore, we do not the return value of delete.
+  for (var i = 0; i < N; i++) {
+    ws.delete(keys[i]);
+  }
+}