Add "benchmarks" test suite

R=machenbach@chromium.org

Review URL: https://codereview.chromium.org/21645003

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@16056 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

.gitignore

@@ -37,6 +37,12 @@ shell_g
 /out
 /perf.data
 /perf.data.old
+/test/benchmarks/benchmarks.status2
+/test/benchmarks/CHECKED_OUT_*
+/test/benchmarks/downloaded_*
+/test/benchmarks/kraken
+/test/benchmarks/octane
+/test/benchmarks/sunspider
 /test/cctest/cctest.status2
 /test/message/message.status2
 /test/mjsunit/mjsunit.status2

test/benchmarks/benchmarks.status

+# Copyright 2013 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+#     * Neither the name of Google Inc. nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+# Too slow in Debug mode.
+octane/mandreel: PASS, SKIP if $mode == debug

test/benchmarks/testcfg.py

+# Copyright 2013 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+#       notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+#       copyright notice, this list of conditions and the following
+#       disclaimer in the documentation and/or other materials provided
+#       with the distribution.
+#     * Neither the name of Google Inc. nor the names of its
+#       contributors may be used to endorse or promote products derived
+#       from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+import os
+import shutil
+import subprocess
+import tarfile
+
+from testrunner.local import testsuite
+from testrunner.objects import testcase
+
+
+class BenchmarksTestSuite(testsuite.TestSuite):
+
+  def __init__(self, name, root):
+    super(BenchmarksTestSuite, self).__init__(name, root)
+    self.testroot = root
+
+  def ListTests(self, context):
+    tests = []
+    for test in [
+        "kraken/ai-astar",
+        "kraken/audio-beat-detection",
+        "kraken/audio-dft",
+        "kraken/audio-fft",
+        "kraken/audio-oscillator",
+        "kraken/imaging-darkroom",
+        "kraken/imaging-desaturate",
+        "kraken/imaging-gaussian-blur",
+        "kraken/json-parse-financial",
+        "kraken/json-stringify-tinderbox",
+        "kraken/stanford-crypto-aes",
+        "kraken/stanford-crypto-ccm",
+        "kraken/stanford-crypto-pbkdf2",
+        "kraken/stanford-crypto-sha256-iterative",
+
+        "octane/box2d",
+        "octane/code-load",
+        "octane/crypto",
+        "octane/deltablue",
+        "octane/earley-boyer",
+        "octane/gbemu",
+        "octane/mandreel",
+        "octane/navier-stokes",
+        "octane/pdfjs",
+        "octane/raytrace",
+        "octane/regexp",
+        "octane/richards",
+        "octane/splay",
+
+        "sunspider/3d-cube",
+        "sunspider/3d-morph",
+        "sunspider/3d-raytrace",
+        "sunspider/access-binary-trees",
+        "sunspider/access-fannkuch",
+        "sunspider/access-nbody",
+        "sunspider/access-nsieve",
+        "sunspider/bitops-3bit-bits-in-byte",
+        "sunspider/bitops-bits-in-byte",
+        "sunspider/bitops-bitwise-and",
+        "sunspider/bitops-nsieve-bits",
+        "sunspider/controlflow-recursive",
+        "sunspider/crypto-aes",
+        "sunspider/crypto-md5",
+        "sunspider/crypto-sha1",
+        "sunspider/date-format-tofte",
+        "sunspider/date-format-xparb",
+        "sunspider/math-cordic",
+        "sunspider/math-partial-sums",
+        "sunspider/math-spectral-norm",
+        "sunspider/regexp-dna",
+        "sunspider/string-base64",
+        "sunspider/string-fasta",
+        "sunspider/string-tagcloud",
+        "sunspider/string-unpack-code",
+        "sunspider/string-validate-input"]:
+      tests.append(testcase.TestCase(self, test))
+    return tests
+
+  def GetFlagsForTestCase(self, testcase, context):
+    result = []
+    result += context.mode_flags
+    if testcase.path.startswith("kraken"):
+      result.append(os.path.join(self.testroot, "%s-data.js" % testcase.path))
+      result.append(os.path.join(self.testroot, "%s.js" % testcase.path))
+    elif testcase.path.startswith("octane"):
+      result.append(os.path.join(self.testroot, "octane/base.js"))
+      result.append(os.path.join(self.testroot, "%s.js" % testcase.path))
+      result += ["-e", "BenchmarkSuite.RunSuites({});"]
+    elif testcase.path.startswith("sunspider"):
+      result.append(os.path.join(self.testroot, "%s.js" % testcase.path))
+    return testcase.flags + result
+
+  def GetSourceForTest(self, testcase):
+    filename = os.path.join(self.testroot, testcase.path + ".js")
+    with open(filename) as f:
+      return f.read()
+
+  def _DownloadIfNecessary(self, url, revision, target_dir):
+    # Maybe we're still up to date?
+    revision_file = "CHECKED_OUT_%s" % target_dir
+    checked_out_revision = None
+    if os.path.exists(revision_file):
+      with open(revision_file) as f:
+        checked_out_revision = f.read()
+    if checked_out_revision == revision:
+      return
+
+    # If we have a local archive file with the test data, extract it.
+    if os.path.exists(target_dir):
+      shutil.rmtree(target_dir)
+    archive_file = "downloaded_%s_%s.tar.gz" % (target_dir, revision)
+    if os.path.exists(archive_file):
+      with tarfile.open(archive_file, "r:gz") as tar:
+        tar.extractall()
+      with open(revision_file, "w") as f:
+        f.write(revision)
+      return
+
+    # No cached copy. Check out via SVN, and pack as .tar.gz for later use.
+    command = "svn co %s -r %s %s" % (url, revision, target_dir)
+    code = subprocess.call(command, shell=True)
+    if code != 0:
+      raise Exception("Error checking out %s benchmark" % target_dir)
+    with tarfile.open(archive_file, "w:gz") as tar:
+      tar.add("%s" % target_dir)
+    with open(revision_file, "w") as f:
+      f.write(revision)
+
+  def DownloadData(self):
+    old_cwd = os.getcwd()
+    os.chdir(os.path.abspath(self.root))
+
+    self._DownloadIfNecessary(
+        ("http://svn.webkit.org/repository/webkit/trunk/PerformanceTests/"
+         "SunSpider/tests/sunspider-1.0/"),
+        "153700", "sunspider")
+
+    self._DownloadIfNecessary(
+        ("http://kraken-mirror.googlecode.com/svn/trunk/kraken/tests/"
+         "kraken-1.1/"),
+        "8", "kraken")
+
+    self._DownloadIfNecessary(
+        "http://octane-benchmark.googlecode.com/svn/trunk/",
+        "22", "octane")
+
+    os.chdir(old_cwd)
+
+  def VariantFlags(self):
+    # Both --nocrankshaft and --stressopt are very slow.
+    return [[]]
+
+
+def GetSuite(name, root):
+  return BenchmarksTestSuite(name, root)