[test] Add basic test-runner system tests

This tests some high-level scenarios of the V8 test runner, with
fake executable, test-suite extension and build configs.

The runners are slightly modified to be easier testable. Args
are passed from the tests now and the V8 root folder can be faked
by the tests.

We support coverage if python coverage 4.0 is installed. Otherwise
we run without it.

Bug: v8:6917
Change-Id: Ib149fd88027cbdc3382bcaea2d82020582f79d2d
Reviewed-on: https://chromium-review.googlesource.com/831506
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Sergiy Byelozyorov <sergiyb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50310}

tools/testrunner/PRESUBMIT.py

+# Copyright 2017 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+def CheckChangeOnCommit(input_api, output_api):
+  tests = input_api.canned_checks.GetUnitTestsInDirectory(
+      input_api, output_api, '../unittests', whitelist=['run_tests_test.py$'])
+  return input_api.RunTests(tests)

tools/testrunner/base_runner.py

@@ -177,16 +177,19 @@ class BuildConfig(object):
 
 
 class BaseTestRunner(object):
-  def __init__(self):
+  def __init__(self, basedir=None):
+    self.basedir = basedir or BASE_DIR
     self.outdir = None
     self.build_config = None
     self.mode_name = None
     self.mode_options = None
 
-  def execute(self):
+  def execute(self, sys_args=None):
+    if sys_args is None:  # pragma: no cover
+      sys_args = sys.argv[1:]
     try:
       parser = self._create_parser()
-      options, args = self._parse_args(parser)
+      options, args = self._parse_args(parser, sys_args)
 
       self._load_build_config(options)
 
@@ -231,11 +234,11 @@ class BaseTestRunner(object):
   def _add_parser_options(self, parser):
     pass
 
-  def _parse_args(self, parser):
-    options, args = parser.parse_args()
+  def _parse_args(self, parser, sys_args):
+    options, args = parser.parse_args(sys_args)
 
     if any(map(lambda v: v and ',' in v,
-                [options.arch, options.mode])):
+                [options.arch, options.mode])):  # pragma: no cover
       print 'Multiple arch/mode are deprecated'
       raise TestRunnerError()
 
@@ -248,7 +251,7 @@ class BaseTestRunner(object):
       except TestRunnerError:
         pass
 
-    if not self.build_config:
+    if not self.build_config:  # pragma: no cover
       print 'Failed to load build config'
       raise TestRunnerError
 
@@ -274,14 +277,14 @@ class BaseTestRunner(object):
                           '%s.%s' % (options.arch, options.mode))
 
     for outdir in outdirs():
-      yield os.path.join(BASE_DIR, outdir)
+      yield os.path.join(self.basedir, outdir)
 
       # buildbot option
       if options.mode:
-        yield os.path.join(BASE_DIR, outdir, options.mode)
+        yield os.path.join(self.basedir, outdir, options.mode)
 
   def _get_gn_outdir(self):
-    gn_out_dir = os.path.join(BASE_DIR, DEFAULT_OUT_GN)
+    gn_out_dir = os.path.join(self.basedir, DEFAULT_OUT_GN)
     latest_timestamp = -1
     latest_config = None
     for gn_config in os.listdir(gn_out_dir):
@@ -305,7 +308,7 @@ class BaseTestRunner(object):
     with open(build_config_path) as f:
       try:
         build_config_json = json.load(f)
-      except Exception:
+      except Exception:  # pragma: no cover
         print("%s exists but contains invalid json. Is your build up-to-date?"
               % build_config_path)
         raise TestRunnerError()
@@ -324,7 +327,7 @@ class BaseTestRunner(object):
 
     build_config_mode = 'debug' if self.build_config.is_debug else 'release'
     if options.mode:
-      if options.mode not in MODES:
+      if options.mode not in MODES:  # pragma: no cover
         print '%s mode is invalid' % options.mode
         raise TestRunnerError()
       if MODES[options.mode].execution_mode != build_config_mode:
@@ -346,7 +349,7 @@ class BaseTestRunner(object):
         options.arch, self.build_config.arch))
       raise TestRunnerError()
 
-    if options.shell_dir:
+    if options.shell_dir:  # pragma: no cover
       print('Warning: --shell-dir is deprecated. Searching for executables in '
             'build directory (%s) instead.' % self.outdir)
 
@@ -364,7 +367,7 @@ class BaseTestRunner(object):
 
   def _setup_env(self):
     # Use the v8 root as cwd as some test cases use "load" with relative paths.
-    os.chdir(BASE_DIR)
+    os.chdir(self.basedir)
 
     # Many tests assume an English interface.
     os.environ['LANG'] = 'en_US.UTF-8'
@@ -403,7 +406,7 @@ class BaseTestRunner(object):
 
     if self.build_config.tsan:
       suppressions_file = os.path.join(
-          BASE_DIR,
+          self.basedir,
           'tools',
           'sanitizers',
           'tsan_suppressions.txt')
@@ -418,7 +421,7 @@ class BaseTestRunner(object):
 
   def _get_external_symbolizer_option(self):
     external_symbolizer_path = os.path.join(
-        BASE_DIR,
+        self.basedir,
         'third_party',
         'llvm-build',
         'Release+Asserts',

tools/testrunner/deopt_fuzzer.py

@@ -37,8 +37,8 @@ DISTRIBUTION_MODES = ["smooth", "random"]
 
 
 class DeoptFuzzer(base_runner.BaseTestRunner):
-  def __init__(self):
-    super(DeoptFuzzer, self).__init__()
+  def __init__(self, *args, **kwargs):
+    super(DeoptFuzzer, self).__init__(*args, **kwargs)
 
   class RandomDistribution:
     def __init__(self, seed=None):
@@ -200,7 +200,7 @@ class DeoptFuzzer(base_runner.BaseTestRunner):
     return shard
 
   def _do_execute(self, options, args):
-    suite_paths = utils.GetSuitePaths(join(base_runner.BASE_DIR, "test"))
+    suite_paths = utils.GetSuitePaths(join(self.basedir, "test"))
 
     if len(args) == 0:
       suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ]
@@ -215,7 +215,7 @@ class DeoptFuzzer(base_runner.BaseTestRunner):
     suites = []
     for root in suite_paths:
       suite = testsuite.TestSuite.LoadTestSuite(
-          os.path.join(base_runner.BASE_DIR, "test", root))
+          os.path.join(self.basedir, "test", root))
       if suite:
         suites.append(suite)
 

tools/testrunner/gc_fuzzer.py

@@ -36,8 +36,8 @@ SLOW_ARCHS = ["arm",
 
 
 class GCFuzzer(base_runner.BaseTestRunner):
-  def __init__(self):
-    super(GCFuzzer, self).__init__()
+  def __init__(self, *args, **kwargs):
+    super(GCFuzzer, self).__init__(*args, **kwargs)
 
     self.fuzzer_rng = None
 
@@ -118,7 +118,7 @@ class GCFuzzer(base_runner.BaseTestRunner):
     return shard
 
   def _do_execute(self, options, args):
-    suite_paths = utils.GetSuitePaths(join(base_runner.BASE_DIR, "test"))
+    suite_paths = utils.GetSuitePaths(join(self.basedir, "test"))
 
     if len(args) == 0:
       suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ]
@@ -133,7 +133,7 @@ class GCFuzzer(base_runner.BaseTestRunner):
     suites = []
     for root in suite_paths:
       suite = testsuite.TestSuite.LoadTestSuite(
-          os.path.join(base_runner.BASE_DIR, "test", root))
+          os.path.join(self.basedir, "test", root))
       if suite:
         suites.append(suite)
 

tools/testrunner/standard_runner.py

@@ -71,8 +71,8 @@ PREDICTABLE_WRAPPER = os.path.join(
 
 
 class StandardTestRunner(base_runner.BaseTestRunner):
-    def __init__(self):
-        super(StandardTestRunner, self).__init__()
+    def __init__(self, *args, **kwargs):
+        super(StandardTestRunner, self).__init__(*args, **kwargs)
 
         self.sancov_dir = None
 
@@ -92,7 +92,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
           except Exception:
             pass
 
-      suite_paths = utils.GetSuitePaths(join(base_runner.BASE_DIR, "test"))
+      suite_paths = utils.GetSuitePaths(join(self.basedir, "test"))
 
       # Use default tests if no test configuration was provided at the cmd line.
       if len(args) == 0:
@@ -119,7 +119,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
         if options.verbose:
           print '>>> Loading test suite: %s' % root
         suite = testsuite.TestSuite.LoadTestSuite(
-            os.path.join(base_runner.BASE_DIR, "test", root))
+            os.path.join(self.basedir, "test", root))
         if suite:
           suites.append(suite)
 
@@ -257,13 +257,13 @@ class StandardTestRunner(base_runner.BaseTestRunner):
       if options.novfp3:
         options.extra_flags.append("--noenable-vfp3")
 
-      if options.no_variants:
+      if options.no_variants:  # pragma: no cover
         print ("Option --no-variants is deprecated. "
                "Pass --variants=default instead.")
         assert not options.variants
         options.variants = "default"
 
-      if options.exhaustive_variants:
+      if options.exhaustive_variants:  # pragma: no cover
         # TODO(machenbach): Switch infra to --variants=exhaustive after M65.
         print ("Option --exhaustive-variants is deprecated. "
                "Pass --variants=exhaustive instead.")
@@ -319,7 +319,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
         print "All variants must be in %s" % str(ALL_VARIANTS)
         raise base_runner.TestRunnerError()
 
-      def CheckTestMode(name, option):
+      def CheckTestMode(name, option):  # pragma: no cover
         if not option in ["run", "skip", "dontcare"]:
           print "Unknown %s mode %s" % (name, option)
           raise base_runner.TestRunnerError()
@@ -482,7 +482,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
       progress_indicator = progress.IndicatorNotifier()
       progress_indicator.Register(
         progress.PROGRESS_INDICATORS[options.progress]())
-      if options.junitout:
+      if options.junitout:  # pragma: no cover
         progress_indicator.Register(progress.JUnitTestProgressIndicator(
             options.junitout, options.junittestsuite))
       if options.json_test_results:
@@ -491,7 +491,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
           self.build_config.arch,
           self.mode_options.execution_mode,
           ctx.random_seed))
-      if options.flakiness_results:
+      if options.flakiness_results:  # pragma: no cover
         progress_indicator.Register(progress.FlakinessTestProgressIndicator(
             options.flakiness_results))
 
@@ -516,8 +516,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
           print "Merging sancov files."
           subprocess.check_call([
             sys.executable,
-            join(
-              base_runner.BASE_DIR, "tools", "sanitizers", "sancov_merger.py"),
+            join(self.basedir, "tools", "sanitizers", "sancov_merger.py"),
             "--coverage-dir=%s" % self.sancov_dir])
         except:
           print >> sys.stderr, "Error: Merging sancov files failed."
@@ -540,16 +539,20 @@ class StandardTestRunner(base_runner.BaseTestRunner):
       if options.shard_count > 1:
         # Log if a value was passed on the cmd line and it differs from the
         # environment variables.
-        if options.shard_count != shard_count:
+        if options.shard_count != shard_count:  # pragma: no cover
           print("shard_count from cmd line differs from environment variable "
                 "GTEST_TOTAL_SHARDS")
-        if options.shard_run > 1 and options.shard_run != shard_run:
+        if (options.shard_run > 1 and
+            options.shard_run != shard_run):  # pragma: no cover
           print("shard_run from cmd line differs from environment variable "
                 "GTEST_SHARD_INDEX")
 
       if shard_count < 2:
         return tests
       if shard_run < 1 or shard_run > shard_count:
+        # TODO(machenbach): Turn this into an assert. If that's wrong on the
+        # bots, printing will be quite useless. Or refactor this code to make
+        # sure we get a return code != 0 after testing if we got here.
         print "shard-run not a valid number, should be in [1:shard-count]"
         print "defaulting back to running all tests"
         return tests

tools/unittests/testdata/expected_test_results1.json

+{
+  "arch": "x64", 
+  "duration_mean": 1, 
+  "mode": "release", 
+  "results": [
+    {
+      "command": "/usr/bin/python out/Release/d8_mocked.py --random-seed=123 strawberries --nohard-abort", 
+      "duration": 1, 
+      "exit_code": 1, 
+      "expected": [
+        "PASS"
+      ], 
+      "flags": [
+        "--random-seed=123", 
+        "strawberries", 
+        "--nohard-abort"
+      ], 
+      "name": "sweet/strawberries", 
+      "random_seed": 123, 
+      "result": "FAIL", 
+      "run": 1, 
+      "stderr": "", 
+      "stdout": "--random-seed=123 strawberries --nohard-abort\n", 
+      "target_name": "d8_mocked.py", 
+      "variant": "default"
+    }, 
+    {
+      "command": "/usr/bin/python out/Release/d8_mocked.py --random-seed=123 strawberries --nohard-abort", 
+      "duration": 1, 
+      "exit_code": 1, 
+      "expected": [
+        "PASS"
+      ], 
+      "flags": [
+        "--random-seed=123", 
+        "strawberries", 
+        "--nohard-abort"
+      ], 
+      "name": "sweet/strawberries", 
+      "random_seed": 123, 
+      "result": "FAIL", 
+      "run": 2, 
+      "stderr": "", 
+      "stdout": "--random-seed=123 strawberries --nohard-abort\n", 
+      "target_name": "d8_mocked.py", 
+      "variant": "default"
+    }, 
+    {
+      "command": "/usr/bin/python out/Release/d8_mocked.py --random-seed=123 strawberries --nohard-abort", 
+      "duration": 1, 
+      "exit_code": 1, 
+      "expected": [
+        "PASS"
+      ], 
+      "flags": [
+        "--random-seed=123", 
+        "strawberries", 
+        "--nohard-abort"
+      ], 
+      "name": "sweet/strawberries", 
+      "random_seed": 123, 
+      "result": "FAIL", 
+      "run": 3, 
+      "stderr": "", 
+      "stdout": "--random-seed=123 strawberries --nohard-abort\n", 
+      "target_name": "d8_mocked.py", 
+      "variant": "default"
+    }
+  ], 
+  "slowest_tests": [
+    {
+      "command": "/usr/bin/python out/Release/d8_mocked.py --random-seed=123 strawberries --nohard-abort", 
+      "duration": 1, 
+      "flags": [
+        "--random-seed=123", 
+        "strawberries", 
+        "--nohard-abort"
+      ], 
+      "marked_slow": true, 
+      "name": "sweet/strawberries"
+    }, 
+    {
+      "command": "/usr/bin/python out/Release/d8_mocked.py --random-seed=123 strawberries --nohard-abort", 
+      "duration": 1, 
+      "flags": [
+        "--random-seed=123", 
+        "strawberries", 
+        "--nohard-abort"
+      ], 
+      "marked_slow": true, 
+      "name": "sweet/strawberries"
+    }, 
+    {
+      "command": "/usr/bin/python out/Release/d8_mocked.py --random-seed=123 strawberries --nohard-abort", 
+      "duration": 1, 
+      "flags": [
+        "--random-seed=123", 
+        "strawberries", 
+        "--nohard-abort"
+      ], 
+      "marked_slow": true, 
+      "name": "sweet/strawberries"
+    }
+  ], 
+  "test_total": 3
+}
+

tools/unittests/testdata/testroot1/d8_mocked.py

+# Copyright 2017 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Dummy d8 replacement. Just passes all test, except if 'berries' is in args.
+"""
+
+import sys
+
+args = ' '.join(sys.argv[1:])
+print args
+# Let all berries fail.
+if 'berries' in args:
+  sys.exit(1)
+sys.exit(0)

tools/unittests/testdata/testroot1/test/sweet/sweet.status

+# Copyright 2017 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+[
+[ALWAYS, {
+  'raspberries': FAIL,
+  'strawberries': [PASS, ['mode == release', SLOW], ['mode == debug', NO_VARIANTS]],
+
+  # Both cherries and apples are to test how PASS an FAIL from different
+  # sections are merged.
+  'cherries': [PASS, SLOW],
+  'apples': [FAIL],
+
+  # Unused rule.
+  'carrots': [PASS, FAIL],
+}],
+
+['variant == nooptimization', {
+  'strawberries': [SKIP],
+}],
+
+['arch == x64', {
+  'cherries': [FAIL],
+  'apples': [PASS, SLOW],
+
+  # Unused rule.
+  'regress/*': [CRASH],
+}],
+
+['no_snap', {
+  'bananas': [PASS, NO_VARIANTS],
+  'raspberries': [FAIL, FAST_VARIANTS],
+}],
+]

tools/unittests/testdata/testroot1/test/sweet/testcfg.py

+# Copyright 2017 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Dummy test suite extension with some fruity tests.
+"""
+
+from testrunner.local import testsuite
+from testrunner.objects import testcase
+
+class TestSuite(testsuite.TestSuite):
+  def ListTests(self, context):
+    return map(
+        self._create_test,
+        ['bananas', 'apples', 'cherries', 'strawberries', 'raspberries'],
+    )
+
+  def _test_class(self):
+    return TestCase
+
+
+class TestCase(testcase.TestCase):
+  def get_shell(self):
+    return 'd8_mocked.py'
+
+  def _get_files_params(self, ctx):
+    return [self.name]
+
+def GetSuite(name, root):
+  return TestSuite(name, root)

tools/unittests/testdata/testroot1/v8_build_config.json

+{
+  "current_cpu": "x64",
+  "dcheck_always_on": false,
+  "is_asan": false,
+  "is_cfi": false,
+  "is_component_build": false,
+  "is_debug": false,
+  "is_gcov_coverage": false,
+  "is_ubsan_vptr": false,
+  "is_msan": false,
+  "is_tsan": false,
+  "target_cpu": "x64",
+  "v8_current_cpu": "x64",
+  "v8_enable_i18n_support": true,
+  "v8_enable_verify_predictable": false,
+  "v8_target_cpu": "x64",
+  "v8_use_snapshot": true
+}