Update the V8 benchmark suite with the following fixes:

*) Fix a couple of typos in DeltaBlue.
*) Make sure Splay doesn't always remove the node just added.
*) Run all benchmarks for at least 32 times (and warm up).
Review URL: http://codereview.chromium.org/2836031

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@4966 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

benchmarks/README.txt

@@ -66,6 +66,12 @@ extensions enabled.
 Changes from Version 5 to Version 6
 ===================================
 
-Removed dead code from the RayTrace benchmark and changed the Splay
-benchmark to avoid converting the same numeric key to a string over
-and over again.
+Removed dead code from the RayTrace benchmark and fixed a couple of
+typos in the DeltaBlue implementation. Changed the Splay benchmark to
+avoid converting the same numeric key to a string over and over again
+and to avoid inserting and removing the same element repeatedly thus
+increasing pressure on the memory subsystem.
+
+Furthermore, the benchmark runner was changed to run the benchmarks
+for at least a few times to stabilize the reported numbers on slower
+machines.

benchmarks/base.js

@@ -198,15 +198,33 @@ BenchmarkSuite.prototype.NotifyError = function(error) {
 
 // Runs a single benchmark for at least a second and computes the
 // average time it takes to run a single iteration.
-BenchmarkSuite.prototype.RunSingleBenchmark = function(benchmark) {
+BenchmarkSuite.prototype.RunSingleBenchmark = function(benchmark, data) {
+  function Measure(data) {
     var elapsed = 0;
     var start = new Date();
     for (var n = 0; elapsed < 1000; n++) {
       benchmark.run();
       elapsed = new Date() - start;
     }
-  var usec = (elapsed * 1000) / n;
+    if (data != null) {
+      data.runs += n;
+      data.elapsed += elapsed;
+    }
+  }
+
+  if (data == null) {
+    // Measure the benchmark once for warm up and throw the result
+    // away. Return a fresh data object.
+    Measure(null);
+    return { runs: 0, elapsed: 0 };
+  } else {
+    Measure(data);
+    // If we've run too few iterations, we continue for another second.
+    if (data.runs < 32) return data;
+    var usec = (data.elapsed * 1000) / data.runs;
     this.NotifyStep(new BenchmarkResult(benchmark, usec));
+    return null;
+  }
 }
 
 
@@ -220,6 +238,7 @@ BenchmarkSuite.prototype.RunStep = function(runner) {
   var length = this.benchmarks.length;
   var index = 0;
   var suite = this;
+  var data;
 
   // Run the setup, the actual benchmark, and the tear down in three
   // separate steps to allow the framework to yield between any of the
@@ -241,12 +260,13 @@ BenchmarkSuite.prototype.RunStep = function(runner) {
 
   function RunNextBenchmark() {
     try {
-      suite.RunSingleBenchmark(suite.benchmarks[index]);
+      data = suite.RunSingleBenchmark(suite.benchmarks[index], data);
     } catch (e) {
       suite.NotifyError(e);
       return null;
     }
-    return RunNextTearDown;
+    // If data is null, we're done with this benchmark.
+    return (data == null) ? RunNextTearDown : RunNextBenchmark();
   }
 
   function RunNextTearDown() {

benchmarks/crypto.js

@@ -31,7 +31,7 @@
 
 
 // The code has been adapted for use as a benchmark by Google.
-var Crypto = new BenchmarkSuite('Crypto', 203037, [
+var Crypto = new BenchmarkSuite('Crypto', 110465, [
   new Benchmark("Encrypt", encrypt),
   new Benchmark("Decrypt", decrypt)
 ]);

benchmarks/deltablue.js

@@ -23,13 +23,13 @@
 // more like a JavaScript program.
 
 
-var DeltaBlue = new BenchmarkSuite('DeltaBlue', 71104, [
+var DeltaBlue = new BenchmarkSuite('DeltaBlue', 30282, [
   new Benchmark('DeltaBlue', deltaBlue)
 ]);
 
 
 /**
- * A JavaScript implementation of the DeltaBlue constrain-solving
+ * A JavaScript implementation of the DeltaBlue constraint-solving
  * algorithm, as described in:
  *
  * "The DeltaBlue Algorithm: An Incremental Constraint Hierarchy Solver"
@@ -349,13 +349,13 @@ function BinaryConstraint(var1, var2, strength) {
 BinaryConstraint.inheritsFrom(Constraint);
 
 /**
- * Decides if this constratint can be satisfied and which way it
+ * Decides if this constraint can be satisfied and which way it
  * should flow based on the relative strength of the variables related,
  * and record that decision.
  */
 BinaryConstraint.prototype.chooseMethod = function (mark) {
   if (this.v1.mark == mark) {
-    this.direction = (this.v1.mark != mark && Strength.stronger(this.strength, this.v2.walkStrength))
+    this.direction = (this.v2.mark != mark && Strength.stronger(this.strength, this.v2.walkStrength))
       ? Direction.FORWARD
       : Direction.NONE;
   }

benchmarks/earley-boyer.js

 // This file is automatically generated by scheme2js, except for the
 // benchmark harness code at the beginning and end of the file.
 
-var EarleyBoyer = new BenchmarkSuite('EarleyBoyer', 765819, [
+var EarleyBoyer = new BenchmarkSuite('EarleyBoyer', 280581, [
   new Benchmark("Earley", function () { BgL_earleyzd2benchmarkzd2(); }),
   new Benchmark("Boyer", function () { BgL_nboyerzd2benchmarkzd2(); })
 ]);

benchmarks/raytrace.js

@@ -8,7 +8,7 @@
 // untouched. This file also contains a copy of parts of the Prototype
 // JavaScript framework which is used by the ray tracer.
 
-var RayTrace = new BenchmarkSuite('RayTrace', 932666, [
+var RayTrace = new BenchmarkSuite('RayTrace', 533115, [
   new Benchmark('RayTrace', renderScene)
 ]);
 

benchmarks/regexp.js

@@ -35,7 +35,7 @@
 // letters in the data are encoded using ROT13 in a way that does not
 // affect how the regexps match their input.
 
-var RegRxp = new BenchmarkSuite('RegExp', 995230, [
+var RegRxp = new BenchmarkSuite('RegExp', 601250, [
   new Benchmark("RegExp", runRegExpBenchmark)
 ]);
 

benchmarks/revisions.html

@@ -22,10 +22,15 @@ the benchmark suite.
 
 <div class="subtitle"><h3>Version 6 (<a href="http://v8.googlecode.com/svn/data/benchmarks/v6/run.html">link</a>)</h3></div>
 
-<p>Removed dead code from the RayTrace benchmark and changed the Splay
-benchmark to avoid converting the same numeric key to a string over
-and over again.
-</p>
+<p>Removed dead code from the RayTrace benchmark and fixed a couple of
+typos in the DeltaBlue implementation. Changed the Splay benchmark to
+avoid converting the same numeric key to a string over and over again
+and to avoid inserting and removing the same element repeatedly thus
+increasing pressure on the memory subsystem.</p>
+
+<p>Furthermore, the benchmark runner was changed to run the benchmarks
+for at least a few times to stabilize the reported numbers on slower
+machines.</p>
 
 <div class="subtitle"><h3>Version 5 (<a href="http://v8.googlecode.com/svn/data/benchmarks/v5/run.html">link</a>)</h3></div>
 

benchmarks/richards.js

@@ -35,7 +35,7 @@
 // Martin Richards.
 
 
-var Richards = new BenchmarkSuite('Richards', 34886, [
+var Richards = new BenchmarkSuite('Richards', 20687, [
   new Benchmark("Richards", runRichards)
 ]);
 

benchmarks/run.html

@@ -116,7 +116,7 @@ higher scores means better performance: <em>Bigger is better!</em>
 <li><b>RegExp</b><br>Regular expression benchmark generated by extracting regular expression operations from 50 of the most popular web pages
 (<i>1614 lines</i>).
 </li>
-<li><b>Splay</b><br>Data manipulation benchmark that deals with splay trees and exercises the automatic memory management subsystem (<i>379 lines</i>).</li>
+<li><b>Splay</b><br>Data manipulation benchmark that deals with splay trees and exercises the automatic memory management subsystem (<i>394 lines</i>).</li>
 </ul>
 
 <p>

benchmarks/splay.js

@@ -33,7 +33,7 @@
 // also has to deal with a lot of changes to the large tree object
 // graph.
 
-var Splay = new BenchmarkSuite('Splay', 126125, [
+var Splay = new BenchmarkSuite('Splay', 21915, [
   new Benchmark("Splay", SplayRun, SplaySetup, SplayTearDown)
 ]);
 
@@ -230,9 +230,24 @@ SplayTree.prototype.find = function(key) {
 };
 
 
+/**
+ * @return {SplayTree.Node} Node having the maximum key value.
+ */
+SplayTree.prototype.findMax = function(opt_startNode) {
+  if (this.isEmpty()) {
+    return null;
+  }
+  var current = opt_startNode || this.root_;
+  while (current.right) {
+    current = current.right;
+  }
+  return current;
+};
+
+
 /**
  * @return {SplayTree.Node} Node having the maximum key value that
- *     is less or equal to the specified key value.
+ *     is less than the specified key value.
  */
 SplayTree.prototype.findGreatestLessThan = function(key) {
   if (this.isEmpty()) {
@@ -243,7 +258,7 @@ SplayTree.prototype.findGreatestLessThan = function(key) {
   this.splay_(key);
   // Now the result is either the root node or the greatest node in
   // the left subtree.
-  if (this.root_.key <= key) {
+  if (this.root_.key < key) {
     return this.root_;
   } else if (this.root_.left) {
     return this.findMax(this.root_.left);