[test] Move loading test suites to the base runner

Bug: v8:6917
Change-Id: I5ad7f9f28be5eb0d8cdc424c0ed5cb41623212e5
Reviewed-on: https://chromium-review.googlesource.com/866505
Commit-Queue: Michał Majewski <majeski@google.com>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50601}

tools/testrunner/base_runner.py

@@ -3,6 +3,7 @@
 # found in the LICENSE file.
 
 
+from collections import OrderedDict
 import json
 import optparse
 import os
@@ -16,6 +17,7 @@ sys.path.insert(
     os.path.dirname(os.path.abspath(__file__))))
 
 
+from local import testsuite
 from local import utils
 
 
@@ -210,10 +212,14 @@ class BaseTestRunner(object):
         parser.print_help()
         raise
 
+      suites = self._get_suites(args, options.verbose)
+
       self._setup_env()
-      return self._do_execute(options, args)
+      return self._do_execute(suites, args, options)
     except TestRunnerError:
       return 1
+    except KeyboardInterrupt:
+      return 2
 
   def _create_parser(self):
     parser = optparse.OptionParser()
@@ -445,7 +451,40 @@ class BaseTestRunner(object):
 
     return 'external_symbolizer_path=%s' % external_symbolizer_path
 
+  def _get_suites(self, args, verbose=False):
+    names = self._args_to_suite_names(args)
+    return self._load_suites(names, verbose)
+
+  def _args_to_suite_names(self, args):
+    # Use default tests if no test configuration was provided at the cmd line.
+    if not args:
+      args = self._get_default_suite_names()
+
+    # Expand arguments with grouped tests. The args should reflect the list
+    # of suites as otherwise filters would break.
+    def expand_test_group(name):
+      return TEST_MAP.get(name, [name])
+
+    args = reduce(list.__add__, map(expand_test_group, args), [])
+
+    all_names = set(utils.GetSuitePaths(os.path.join(self.basedir, 'test')))
+    args_names = OrderedDict([(arg.split('/')[0], None) for arg in args]) # set
+    return [name for name in args_names if name in all_names]
+
+  def _get_default_suite_names(self):
+    return []
+
+  def _expand_test_group(self, name):
+    return TEST_MAP.get(name, [name])
+
+  def _load_suites(self, names, verbose=False):
+    def load_suite(name):
+      if verbose:
+        print '>>> Loading test suite: %s' % name
+      return testsuite.TestSuite.LoadTestSuite(
+          os.path.join(self.basedir, 'test', name))
+    return map(load_suite, names)
 
   # TODO(majeski): remove options & args parameters
-  def _do_execute(self, options, args):
+  def _do_execute(self, suites, args, options):
     raise NotImplementedError()

tools/testrunner/deopt_fuzzer.py

@@ -26,7 +26,7 @@ from testrunner.local import verbose
 from testrunner.objects import context
 
 
-DEFAULT_TESTS = ["mjsunit", "webkit"]
+DEFAULT_SUITES = ["mjsunit", "webkit"]
 TIMEOUT_DEFAULT = 60
 
 # Double the timeout for these:
@@ -199,32 +199,6 @@ class DeoptFuzzer(base_runner.BaseTestRunner):
       count += 1
     return shard
 
-  def _do_execute(self, options, args):
-    suite_paths = utils.GetSuitePaths(join(self.basedir, "test"))
-
-    if len(args) == 0:
-      suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ]
-    else:
-      args_suites = set()
-      for arg in args:
-        suite = arg.split(os.path.sep)[0]
-        if not suite in args_suites:
-          args_suites.add(suite)
-      suite_paths = [ s for s in suite_paths if s in args_suites ]
-
-    suites = []
-    for root in suite_paths:
-      suite = testsuite.TestSuite.LoadTestSuite(
-          os.path.join(self.basedir, "test", root))
-      if suite:
-        suites.append(suite)
-
-    try:
-      return self._execute(args, options, suites)
-    except KeyboardInterrupt:
-      return 2
-
-
   def _calculate_n_tests(self, m, options):
     """Calculates the number of tests from m deopt points with exponential
     coverage.
@@ -235,8 +209,10 @@ class DeoptFuzzer(base_runner.BaseTestRunner):
     l = float(options.coverage_lift)
     return int(math.pow(m, (m * c + l) / (m + l)))
 
+  def _get_default_suite_names(self):
+    return DEFAULT_SUITES
 
-  def _execute(self, args, options, suites):
+  def _do_execute(self, suites, args, options):
     print(">>> Running tests for %s.%s" % (self.build_config.arch,
                                            self.mode_name))
 

tools/testrunner/gc_fuzzer.py

@@ -27,7 +27,7 @@ from testrunner.local import verbose
 from testrunner.objects import context
 
 
-DEFAULT_TESTS = ["mjsunit", "webkit"]
+DEFAULT_SUITES = ["mjsunit", "webkit"]
 TIMEOUT_DEFAULT = 60
 
 # Double the timeout for these:
@@ -117,32 +117,6 @@ class GCFuzzer(base_runner.BaseTestRunner):
       count += 1
     return shard
 
-  def _do_execute(self, options, args):
-    suite_paths = utils.GetSuitePaths(join(self.basedir, "test"))
-
-    if len(args) == 0:
-      suite_paths = [ s for s in suite_paths if s in DEFAULT_TESTS ]
-    else:
-      args_suites = set()
-      for arg in args:
-        suite = arg.split(os.path.sep)[0]
-        if not suite in args_suites:
-          args_suites.add(suite)
-      suite_paths = [ s for s in suite_paths if s in args_suites ]
-
-    suites = []
-    for root in suite_paths:
-      suite = testsuite.TestSuite.LoadTestSuite(
-          os.path.join(self.basedir, "test", root))
-      if suite:
-        suites.append(suite)
-
-    try:
-      return self._execute(args, options, suites)
-    except KeyboardInterrupt:
-      return 2
-
-
   def _calculate_n_tests(self, m, options):
     """Calculates the number of tests from m points with exponential coverage.
     The coverage is expected to be between 0.0 and 1.0.
@@ -152,8 +126,10 @@ class GCFuzzer(base_runner.BaseTestRunner):
     l = float(options.coverage_lift)
     return int(math.pow(m, (m * c + l) / (m + l)))
 
+  def _get_default_suite_names(self):
+    return DEFAULT_SUITES
 
-  def _execute(self, args, options, suites):
+  def _do_execute(self, suites, args, options):
     print(">>> Running tests for %s.%s" % (self.build_config.arch,
                                            self.mode_name))
 

tools/testrunner/standard_runner.py

@@ -84,7 +84,10 @@ class StandardTestRunner(base_runner.BaseTestRunner):
 
         self.sancov_dir = None
 
-    def _do_execute(self, options, args):
+    def _get_default_suite_names(self):
+      return ['default']
+
+    def _do_execute(self, suites, args, options):
       if options.swarming:
         # Swarming doesn't print how isolated commands are called. Lets make
         # this less cryptic by printing it ourselves.
@@ -100,41 +103,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
           except Exception:
             pass
 
-      suite_paths = utils.GetSuitePaths(join(self.basedir, "test"))
-
-      # Use default tests if no test configuration was provided at the cmd line.
-      if len(args) == 0:
-        args = ["default"]
-
-      # Expand arguments with grouped tests. The args should reflect the list
-      # of suites as otherwise filters would break.
-      def ExpandTestGroups(name):
-        if name in base_runner.TEST_MAP:
-          return [suite for suite in base_runner.TEST_MAP[name]]
-        else:
-          return [name]
-      args = reduce(lambda x, y: x + y,
-            [ExpandTestGroups(arg) for arg in args],
-            [])
-
-      args_suites = OrderedDict() # Used as set
-      for arg in args:
-        args_suites[arg.split('/')[0]] = True
-      suite_paths = [ s for s in args_suites if s in suite_paths ]
-
-      suites = []
-      for root in suite_paths:
-        if options.verbose:
-          print '>>> Loading test suite: %s' % root
-        suite = testsuite.TestSuite.LoadTestSuite(
-            os.path.join(self.basedir, "test", root))
-        if suite:
-          suites.append(suite)
-
-      try:
-        return self._execute(args, options, suites)
-      except KeyboardInterrupt:
-        return 2
+      return self._execute(args, options, suites)
 
     def _add_parser_options(self, parser):
       parser.add_option("--sancov-dir",