Reland "[test] refactor testsuite configuration"

This is a reland of 7f92ad0a

Original change's description:
> [test] refactor testsuite configuration
> 
> Every testsuite configuration consist of at least 30% code duplication.
> 
> The code age ranges from 10 years old to 5 years old. Implementing anything that
> touches the testsuite code becomes a technical fight to the death.
> 
> This CL removes all the duplication by refactoring the common functionality.
> 
> This CL contains structural changes without any logical changes % small bug
> fixes.
> 
> R=machenbach@chromium.org
> CC=yangguo@chromium.org,sergiyb@chromium.org
> 
> Bug: v8:8174, v8:8769
> Change-Id: Iee299569caa7abdc0307ecf606136669034a28a2
> Reviewed-on: https://chromium-review.googlesource.com/c/1445881
> Commit-Queue: Sergiy Belozorov <sergiyb@chromium.org>
> Reviewed-by: Michael Achenbach <machenbach@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#59361}

Bug: v8:8174, v8:8769
Change-Id: I8e7078cfb875ceb3777e57084e6f8dfac09693e7
Reviewed-on: https://chromium-review.googlesource.com/c/1454485Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Commit-Queue: Tamer Tas <tmrts@chromium.org>
Cr-Commit-Position: refs/heads/master@{#59369}

test/benchmarks/testcfg.py

@@ -34,13 +34,9 @@ from testrunner.local import testsuite
 from testrunner.objects import testcase
 
 
-class TestSuite(testsuite.TestSuite):
-  def __init__(self, *args, **kwargs):
-    super(TestSuite, self).__init__(*args, **kwargs)
-    self.testroot = os.path.join(self.root, "data")
-
-  def ListTests(self):
-    tests = map(self._create_test, [
+class TestLoader(testsuite.TestLoader):
+  def _list_test_filenames(self):
+    return [
         "kraken/ai-astar",
         "kraken/audio-beat-detection",
         "kraken/audio-dft",
@@ -98,8 +94,16 @@ class TestSuite(testsuite.TestSuite):
         "sunspider/string-tagcloud",
         "sunspider/string-unpack-code",
         "sunspider/string-validate-input",
-    ])
-    return tests
+    ]
+
+
+class TestSuite(testsuite.TestSuite):
+  def __init__(self, *args, **kwargs):
+    super(TestSuite, self).__init__(*args, **kwargs)
+    self.testroot = os.path.join(self.root, "data")
+
+  def _test_loader_class(self):
+    return TestLoader
 
   def _test_class(self):
     return TestCase

test/cctest/testcfg.py

@@ -36,8 +36,8 @@ from testrunner.objects import testcase
 SHELL = 'cctest'
 
 
-class TestSuite(testsuite.TestSuite):
-  def ListTests(self):
+class TestLoader(testsuite.TestLoader):
+  def _list_test_filenames(self):
     shell = os.path.abspath(os.path.join(self.test_config.shell_dir, SHELL))
     if utils.IsWindows():
       shell += ".exe"
@@ -46,14 +46,19 @@ class TestSuite(testsuite.TestSuite):
         shell=shell,
         args=["--list"] + self.test_config.extra_flags)
     output = cmd.execute()
+    # TODO make errors visible (see duplicated code in 'unittests')
     if output.exit_code != 0:
       print cmd
       print output.stdout
       print output.stderr
       return []
-    tests = map(self._create_test, output.stdout.strip().split())
-    tests.sort(key=lambda t: t.path)
-    return tests
+
+    return output.stdout.strip().split()
+
+
+class TestSuite(testsuite.TestSuite):
+  def _test_loader_class(self):
+    return TestLoader
 
   def _test_class(self):
     return TestCase

test/debugger/testcfg.py

@@ -11,22 +11,16 @@ from testrunner.objects import testcase
 FILES_PATTERN = re.compile(r"//\s+Files:(.*)")
 MODULE_PATTERN = re.compile(r"^// MODULE$", flags=re.MULTILINE)
 
+
+class TestLoader(testsuite.JSTestLoader):
+  @property
+  def excluded_files(self):
+    return {"test-api.js"}
+
+
 class TestSuite(testsuite.TestSuite):
-  def ListTests(self):
-    tests = []
-    for dirname, dirs, files in os.walk(self.root):
-      for dotted in [x for x in dirs if x.startswith('.')]:
-        dirs.remove(dotted)
-      dirs.sort()
-      files.sort()
-      for filename in files:
-        if (filename.endswith(".js") and filename != "test-api.js"):
-          fullpath = os.path.join(dirname, filename)
-          relpath = fullpath[len(self.root) + 1 : -3]
-          testname = relpath.replace(os.path.sep, "/")
-          test = self._create_test(testname)
-          tests.append(test)
-    return tests
+  def _test_loader_class(self):
+    return TestLoader
 
   def _test_class(self):
     return TestCase

test/fuzzer/testcfg.py

@@ -7,26 +7,35 @@ import os
 from testrunner.local import testsuite
 from testrunner.objects import testcase
 
+SUB_TESTS = [
+  'json',
+  'parser',
+  'regexp_builtins',
+  'regexp',
+  'multi_return',
+  'wasm',
+  'wasm_async',
+  'wasm_code',
+  'wasm_compile',
+]
 
 class VariantsGenerator(testsuite.VariantsGenerator):
   def _get_variants(self, test):
     return self._standard_variant
 
 
+class TestLoader(testsuite.GenericTestLoader):
+  @property
+  def test_dirs(self):
+    return SUB_TESTS
+
+  def _to_relpath(self, abspath, _):
+    return os.path.relpath(abspath, self.suite.root)
+
+
 class TestSuite(testsuite.TestSuite):
-  SUB_TESTS = ( 'json', 'parser', 'regexp_builtins', 'regexp', 'multi_return', 'wasm',
-          'wasm_async', 'wasm_code', 'wasm_compile')
-
-  def ListTests(self):
-    tests = []
-    for subtest in TestSuite.SUB_TESTS:
-      for fname in os.listdir(os.path.join(self.root, subtest)):
-        if not os.path.isfile(os.path.join(self.root, subtest, fname)):
-          continue
-        test = self._create_test('%s/%s' % (subtest, fname))
-        tests.append(test)
-    tests.sort()
-    return tests
+  def _test_loader_class(self):
+    return TestLoader
 
   def _test_class(self):
     return TestCase
@@ -37,7 +46,7 @@ class TestSuite(testsuite.TestSuite):
 
 class TestCase(testcase.TestCase):
   def _get_files_params(self):
-    suite, name = self.path.split('/')
+    suite, name = self.path.split(os.path.sep)
     return [os.path.join(self.suite.root, suite, name)]
 
   def _get_variant_flags(self):
@@ -50,7 +59,7 @@ class TestCase(testcase.TestCase):
     return []
 
   def get_shell(self):
-    group, _ = self.path.split('/', 1)
+    group, _ = self.path.split(os.path.sep, 1)
     return 'v8_simple_%s_fuzzer' % group
 
 

test/inspector/testcfg.py

@@ -13,25 +13,19 @@ PROTOCOL_TEST_JS = "protocol-test.js"
 EXPECTED_SUFFIX = "-expected.txt"
 RESOURCES_FOLDER = "resources"
 
+class TestLoader(testsuite.JSTestLoader):
+  @property
+  def excluded_files(self):
+    return {PROTOCOL_TEST_JS}
+
+  @property
+  def excluded_dirs(self):
+    return {RESOURCES_FOLDER}
+
+
 class TestSuite(testsuite.TestSuite):
-  def ListTests(self):
-    tests = []
-    for dirname, dirs, files in os.walk(
-        os.path.join(self.root), followlinks=True):
-      for dotted in [x for x in dirs if x.startswith('.')]:
-        dirs.remove(dotted)
-      if dirname.endswith(os.path.sep + RESOURCES_FOLDER):
-        continue
-      dirs.sort()
-      files.sort()
-      for filename in files:
-        if filename.endswith(".js") and filename != PROTOCOL_TEST_JS:
-          fullpath = os.path.join(dirname, filename)
-          relpath = fullpath[len(self.root) + 1 : -3]
-          testname = relpath.replace(os.path.sep, "/")
-          test = self._create_test(testname)
-          tests.append(test)
-    return tests
+  def _test_loader_class(self):
+    return TestLoader
 
   def _test_class(self):
     return TestCase

test/intl/testcfg.py

@@ -33,23 +33,16 @@ from testrunner.objects import testcase
 
 ENV_PATTERN = re.compile(r"//\s+Environment Variables:(.*)")
 
+
+class TestLoader(testsuite.JSTestLoader):
+  @property
+  def excluded_files(self):
+    return {"assert.js", "utils.js"}
+
+
 class TestSuite(testsuite.TestSuite):
-  def ListTests(self):
-    tests = []
-    for dirname, dirs, files in os.walk(self.root):
-      for dotted in [x for x in dirs if x.startswith('.')]:
-        dirs.remove(dotted)
-      dirs.sort()
-      files.sort()
-      for filename in files:
-        if (filename.endswith(".js") and filename != "assert.js" and
-            filename != "utils.js"):
-          fullpath = os.path.join(dirname, filename)
-          relpath = fullpath[len(self.root) + 1 : -3]
-          testname = relpath.replace(os.path.sep, "/")
-          test = self._create_test(testname)
-          tests.append(test)
-    return tests
+  def _test_loader_class(self):
+    return TestLoader
 
   def _test_class(self):
     return TestCase

test/message/testcfg.py

@@ -39,21 +39,8 @@ MODULE_PATTERN = re.compile(r"^// MODULE$", flags=re.MULTILINE)
 
 
 class TestSuite(testsuite.TestSuite):
-  def ListTests(self):
-    tests = []
-    for dirname, dirs, files in os.walk(self.root):
-      for dotted in [x for x in dirs if x.startswith('.')]:
-        dirs.remove(dotted)
-      dirs.sort()
-      files.sort()
-      for filename in files:
-        if filename.endswith(".js"):
-          fullpath = os.path.join(dirname, filename)
-          relpath = fullpath[len(self.root) + 1 : -3]
-          testname = relpath.replace(os.path.sep, "/")
-          test = self._create_test(testname)
-          tests.append(test)
-    return tests
+  def _test_loader_class(self):
+    return testsuite.JSTestLoader
 
   def _test_class(self):
     return TestCase

test/mjsunit/testcfg.py

@@ -56,24 +56,19 @@ COMBINE_TESTS_FLAGS_BLACKLIST = [
   '--wasm-lazy-compilation',
 ]
 
+
+class TestLoader(testsuite.JSTestLoader):
+  @property
+  def excluded_files(self):
+    return {
+      "mjsunit.js",
+      "mjsunit_suppressions.js",
+    }
+
+
 class TestSuite(testsuite.TestSuite):
-  def ListTests(self):
-    tests = []
-    for dirname, dirs, files in os.walk(self.root, followlinks=True):
-      for dotted in [x for x in dirs if x.startswith('.')]:
-        dirs.remove(dotted)
-      dirs.sort()
-      files.sort()
-      for filename in files:
-        if (filename.endswith(".js") and
-            filename != "mjsunit.js" and
-            filename != "mjsunit_suppressions.js"):
-          fullpath = os.path.join(dirname, filename)
-          relpath = fullpath[len(self.root) + 1 : -3]
-          testname = relpath.replace(os.path.sep, "/")
-          test = self._create_test(testname)
-          tests.append(test)
-    return tests
+  def _test_loader_class(self):
+    return TestLoader
 
   def _test_combiner_class(self):
     return TestCombiner

test/mkgrokdump/testcfg.py

@@ -11,6 +11,11 @@ from testrunner.outproc import mkgrokdump
 
 SHELL = 'mkgrokdump'
 
+
+class TestLoader(testsuite.TestLoader):
+  pass
+
+#TODO(tmrts): refactor the test creation logic to migrate to TestLoader
 class TestSuite(testsuite.TestSuite):
   def __init__(self, *args, **kwargs):
     super(TestSuite, self).__init__(*args, **kwargs)
@@ -19,9 +24,12 @@ class TestSuite(testsuite.TestSuite):
     self.expected_path = os.path.join(v8_path, 'tools', 'v8heapconst.py')
 
   def ListTests(self):
-    test = self._create_test(SHELL)
+    test = self._test_loader._create_test(SHELL, self)
     return [test]
 
+  def _test_loader_class(self):
+    return TestLoader
+
   def _test_class(self):
     return TestCase
 

test/mozilla/testcfg.py

@@ -32,54 +32,55 @@ from testrunner.local import testsuite
 from testrunner.objects import testcase
 from testrunner.outproc import mozilla
 
-EXCLUDED = ["CVS", ".svn"]
-
+EXCLUDED = [
+  "CVS",
+  ".svn",
+]
+
+FRAMEWORK = [
+  "browser.js",
+  "shell.js",
+  "jsref.js",
+  "template.js",
+]
+
+TEST_DIRS = [
+  "ecma",
+  "ecma_2",
+  "ecma_3",
+  "js1_1",
+  "js1_2",
+  "js1_3",
+  "js1_4",
+  "js1_5",
+]
+
+class TestLoader(testsuite.JSTestLoader):
+  @property
+  def excluded_files(self):
+    return set(FRAMEWORK)
 
-FRAMEWORK = """
-  browser.js
-  shell.js
-  jsref.js
-  template.js
-""".split()
+  @property
+  def excluded_dirs(self):
+    return set(EXCLUDED)
 
+  @property
+  def test_dirs(self):
+    return TEST_DIRS
 
-TEST_DIRS = """
-  ecma
-  ecma_2
-  ecma_3
-  js1_1
-  js1_2
-  js1_3
-  js1_4
-  js1_5
-""".split()
+  def _to_relpath(self, abspath, _):
+    # TODO: refactor this by setting the test path during the TestCase creation
+    return os.path.relpath(abspath, self.test_root)
 
 
 class TestSuite(testsuite.TestSuite):
   def __init__(self, *args, **kwargs):
     super(TestSuite, self).__init__(*args, **kwargs)
-    self.testroot = os.path.join(self.root, "data")
-
-  def ListTests(self):
-    tests = []
-    for testdir in TEST_DIRS:
-      current_root = os.path.join(self.testroot, testdir)
-      for dirname, dirs, files in os.walk(current_root):
-        for dotted in [x for x in dirs if x.startswith(".")]:
-          dirs.remove(dotted)
-        for excluded in EXCLUDED:
-          if excluded in dirs:
-            dirs.remove(excluded)
-        dirs.sort()
-        files.sort()
-        for filename in files:
-          if filename.endswith(".js") and not filename in FRAMEWORK:
-            fullpath = os.path.join(dirname, filename)
-            relpath = fullpath[len(self.testroot) + 1 : -3]
-            testname = relpath.replace(os.path.sep, "/")
-            case = self._create_test(testname)
-            tests.append(case)
-    return tests
+    self.test_root = os.path.join(self.root, "data")
+    self._test_loader.test_root = self.test_root
+
+  def _test_loader_class(self):
+    return TestLoader
 
   def _test_class(self):
     return TestCase
@@ -91,20 +92,20 @@ class TestCase(testcase.D8TestCase):
     testfilename = self.path + ".js"
     testfilepath = testfilename.split("/")
     for i in xrange(len(testfilepath)):
-      script = os.path.join(self.suite.testroot,
+      script = os.path.join(self.suite.test_root,
                             reduce(os.path.join, testfilepath[:i], ""),
                             "shell.js")
       if os.path.exists(script):
         files.append(script)
 
-    files.append(os.path.join(self.suite.testroot, testfilename))
+    files.append(os.path.join(self.suite.test_root, testfilename))
     return files
 
   def _get_suite_flags(self):
     return ['--expose-gc']
 
   def _get_source_path(self):
-    return os.path.join(self.suite.testroot, self.path + self._get_suffix())
+    return os.path.join(self.suite.test_root, self.path + self._get_suffix())
 
   @property
   def output_proc(self):
@@ -117,6 +118,5 @@ class TestCase(testcase.D8TestCase):
     return mozilla.OutProc(self.expected_outcomes)
 
 
-
 def GetSuite(*args, **kwargs):
   return TestSuite(*args, **kwargs)

test/preparser/testcfg.py

@@ -37,6 +37,11 @@ class VariantsGenerator(testsuite.VariantsGenerator):
     return self._standard_variant
 
 
+class TestLoader(testsuite.TestLoader):
+  pass
+
+
+# TODO(tmrts): refactor the python template parsing then use the TestLoader.
 class TestSuite(testsuite.TestSuite):
   def _ParsePythonTestTemplates(self, result, filename):
     pathname = os.path.join(self.root, filename + ".pyt")
@@ -71,8 +76,11 @@ class TestSuite(testsuite.TestSuite):
     return result
 
   def _create_test(self, path, source, template_flags):
-    return super(TestSuite, self)._create_test(
-        path, source=source, template_flags=template_flags)
+    return self._test_loader._create_test(
+        path, self, source=source, template_flags=template_flags)
+
+  def _test_loader_class(self):
+    return TestLoader
 
   def _test_class(self):
     return TestCase

test/test262/testcfg.py

@@ -73,9 +73,6 @@ TEST_262_HARNESS_PATH = ["data", "harness"]
 TEST_262_TOOLS_PATH = ["harness", "src"]
 TEST_262_LOCAL_TESTS_PATH = ["local-tests", "test"]
 
-TEST_262_RELPATH_REGEXP = re.compile(
-    r'.*[\\/]test[\\/]test262[\\/][^\\/]+[\\/]test[\\/](.*)\.js')
-
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),
                              *TEST_262_TOOLS_PATH))
 
@@ -103,18 +100,38 @@ class VariantsGenerator(testsuite.VariantsGenerator):
           yield (variant, flags + ['--use-strict'], 'strict-%d' % n + phase_var)
 
 
-class TestSuite(testsuite.TestSuite):
-  # Match the (...) in '/path/to/v8/test/test262/subdir/test/(...).js'
-  # In practice, subdir is data or local-tests
+class TestLoader(testsuite.JSTestLoader):
+  @property
+  def test_dirs(self):
+    return [
+      self.test_root,
+      os.path.join(self.suite.root, *TEST_262_LOCAL_TESTS_PATH),
+    ]
 
+  @property
+  def excluded_suffixes(self):
+    return {"_FIXTURE.js"}
+
+  @property
+  def excluded_dirs(self):
+    return {"intl402"} if self.test_config.noi18n else set()
+
+  def _should_filter_by_test(self, test):
+    features = test.test_record.get("features", [])
+    return SKIPPED_FEATURES.intersection(features)
+
+
+class TestSuite(testsuite.TestSuite):
   def __init__(self, *args, **kwargs):
     super(TestSuite, self).__init__(*args, **kwargs)
-    self.testroot = os.path.join(self.root, *TEST_262_SUITE_PATH)
+    self.test_root = os.path.join(self.root, *TEST_262_SUITE_PATH)
+    # TODO: this makes the TestLoader mutable, refactor it.
+    self._test_loader.test_root = self.test_root
     self.harnesspath = os.path.join(self.root, *TEST_262_HARNESS_PATH)
     self.harness = [os.path.join(self.harnesspath, f)
                     for f in TEST_262_HARNESS_FILES]
     self.harness += [os.path.join(self.root, "harness-adapt.js")]
-    self.localtestroot = os.path.join(self.root, *TEST_262_LOCAL_TESTS_PATH)
+    self.local_test_root = os.path.join(self.root, *TEST_262_LOCAL_TESTS_PATH)
     self.parse_test_record = self._load_parse_test_record()
 
   def _load_parse_test_record(self):
@@ -132,28 +149,8 @@ class TestSuite(testsuite.TestSuite):
       if f:
         f.close()
 
-  def ListTests(self):
-    testnames = set()
-    for dirname, dirs, files in itertools.chain(os.walk(self.testroot),
-                                                os.walk(self.localtestroot)):
-      for dotted in [x for x in dirs if x.startswith(".")]:
-        dirs.remove(dotted)
-      if self.test_config.noi18n and "intl402" in dirs:
-        dirs.remove("intl402")
-      dirs.sort()
-      files.sort()
-      for filename in files:
-        if not filename.endswith(".js"):
-          continue
-        if filename.endswith("_FIXTURE.js"):
-          continue
-        fullpath = os.path.join(dirname, filename)
-        relpath = re.match(TEST_262_RELPATH_REGEXP, fullpath).group(1)
-        testnames.add(relpath.replace(os.path.sep, "/"))
-    cases = map(self._create_test, testnames)
-    return [case for case in cases if len(
-                SKIPPED_FEATURES.intersection(
-                    case.test_record.get("features", []))) == 0]
+  def _test_loader_class(self):
+    return TestLoader
 
   def _test_class(self):
     return TestCase
@@ -195,11 +192,15 @@ class TestCase(testcase.D8TestCase):
   def _fail_phase_reverse(self):
     return 'fail-phase-reverse' in self.procid
 
+  def __needs_harness_agent(self):
+    tokens = self.path.split(os.path.sep)
+    return tokens[:2] == ["built-ins", "Atomics"]
+
   def _get_files_params(self):
     return (
         list(self.suite.harness) +
         ([os.path.join(self.suite.root, "harness-agent.js")]
-         if self.path.startswith('built-ins/Atomics') else []) +
+         if self.__needs_harness_agent() else []) +
         ([os.path.join(self.suite.root, "harness-adapt-donotevaluate.js")]
          if self.fail_phase_only and not self._fail_phase_reverse else []) +
         self._get_includes() +
@@ -230,10 +231,10 @@ class TestCase(testcase.D8TestCase):
 
   def _get_source_path(self):
     filename = self.path + self._get_suffix()
-    path = os.path.join(self.suite.localtestroot, filename)
+    path = os.path.join(self.suite.local_test_root, filename)
     if os.path.exists(path):
       return path
-    return os.path.join(self.suite.testroot, filename)
+    return os.path.join(self.suite.test_root, filename)
 
   @property
   def output_proc(self):

test/unittests/testcfg.py

@@ -15,9 +15,10 @@ class VariantsGenerator(testsuite.VariantsGenerator):
     return self._standard_variant
 
 
-class TestSuite(testsuite.TestSuite):
-  def ListTests(self):
-    shell = os.path.abspath(os.path.join(self.test_config.shell_dir, self.name))
+class TestLoader(testsuite.TestLoader):
+  def _list_test_filenames(self):
+    shell = os.path.abspath(
+      os.path.join(self.test_config.shell_dir, "unittests"))
     if utils.IsWindows():
       shell += ".exe"
 
@@ -30,6 +31,7 @@ class TestSuite(testsuite.TestSuite):
       output = cmd.execute()
       if output.exit_code == 0:
         break
+
       print "Test executable failed to list the tests (try %d).\n\nCmd:" % i
       print cmd
       print "\nStdout:"
@@ -40,17 +42,22 @@ class TestSuite(testsuite.TestSuite):
     else:
       raise Exception("Test executable failed to list the tests.")
 
-    tests = []
-    test_case = ''
+    # TODO create an ExecutableTestLoader for refactoring this similar to
+    # JSTestLoader.
+    test_names = []
     for line in output.stdout.splitlines():
       test_desc = line.strip().split()[0]
       if test_desc.endswith('.'):
         test_case = test_desc
       elif test_case and test_desc:
-        test_path = test_case + test_desc
-        tests.append(self._create_test(test_path))
-    tests.sort(key=lambda t: t.path)
-    return tests
+        test_names.append(test_case + test_desc)
+
+    return test_names
+
+
+class TestSuite(testsuite.TestSuite):
+  def _test_loader_class(self):
+    return TestLoader
 
   def _test_class(self):
     return TestCase

test/wasm-js/testcfg.py

@@ -12,28 +12,23 @@ ANY_JS = ".any.js"
 WPT_ROOT = "/wasm/jsapi/"
 META_SCRIPT_REGEXP = re.compile(r"META:\s*script=(.*)")
 
+
+class TestLoader(testsuite.JSTestLoader):
+  @property
+  def extension(self):
+    return ANY_JS
+
+
 class TestSuite(testsuite.TestSuite):
   def __init__(self, *args, **kwargs):
     super(TestSuite, self).__init__(*args, **kwargs)
-    self.testroot = os.path.join(self.root, "data", "test", "js-api")
     self.mjsunit_js = os.path.join(os.path.dirname(self.root), "mjsunit",
                                    "mjsunit.js")
+    self.test_root = os.path.join(self.root, "data", "test", "js-api")
+    self._test_loader.test_root = self.test_root
 
-  def ListTests(self):
-    tests = []
-    for dirname, dirs, files in os.walk(self.testroot):
-      for dotted in [x for x in dirs if x.startswith(".")]:
-        dirs.remove(dotted)
-      dirs.sort()
-      files.sort()
-      for filename in files:
-        if (filename.endswith(ANY_JS)):
-          fullpath = os.path.join(dirname, filename)
-          relpath = fullpath[len(self.testroot) + 1 : -len(ANY_JS)]
-          testname = relpath.replace(os.path.sep, "/")
-          test = self._create_test(testname)
-          tests.append(test)
-    return tests
+  def _test_loader_class(self):
+    return TestLoader
 
   def _test_class(self):
     return TestCase
@@ -49,7 +44,7 @@ class TestCase(testcase.D8TestCase):
       if script.startswith(WPT_ROOT):
         # Matched an absolute path, strip the root and replace it with our
         # local root.
-        script = os.path.join(self.suite.testroot, script[len(WPT_ROOT):])
+        script = os.path.join(self.suite.test_root, script[len(WPT_ROOT):])
       elif not script.startswith("/"):
         # Matched a relative path, prepend this test's directory.
         thisdir = os.path.dirname(self._get_source_path())
@@ -67,7 +62,7 @@ class TestCase(testcase.D8TestCase):
 
   def _get_source_path(self):
     # All tests are named `path/name.any.js`
-    return os.path.join(self.suite.testroot, self.path + ANY_JS)
+    return os.path.join(self.suite.test_root, self.path + ANY_JS)
 
 
 def GetSuite(*args, **kwargs):

test/wasm-spec-tests/testcfg.py

@@ -7,20 +7,13 @@ import os
 from testrunner.local import testsuite
 from testrunner.objects import testcase
 
+
+class TestLoader(testsuite.JSTestLoader):
+  pass
+
 class TestSuite(testsuite.TestSuite):
-  def ListTests(self):
-    tests = []
-    for dirname, dirs, files in os.walk(self.root):
-      for dotted in [x for x in dirs if x.startswith('.')]:
-        dirs.remove(dotted)
-      for filename in files:
-        if (filename.endswith(".js")):
-          fullpath = os.path.join(dirname, filename)
-          relpath = fullpath[len(self.root) + 1 : -3]
-          testname = relpath.replace(os.path.sep, "/")
-          test = self._create_test(testname)
-          tests.append(test)
-    return tests
+  def _test_loader_class(self):
+    return TestLoader
 
   def _test_class(self):
     return TestCase

test/webkit/testcfg.py

@@ -36,26 +36,15 @@ FILES_PATTERN = re.compile(r"//\s+Files:(.*)")
 SELF_SCRIPT_PATTERN = re.compile(r"//\s+Env: TEST_FILE_NAME")
 
 
-# TODO (machenbach): Share commonalities with mjstest.
+class TestLoader(testsuite.JSTestLoader):
+  @property
+  def excluded_dirs(self):
+    return {"resources"}
+
+
 class TestSuite(testsuite.TestSuite):
-  def ListTests(self):
-    tests = []
-    for dirname, dirs, files in os.walk(self.root):
-      for dotted in [x for x in dirs if x.startswith('.')]:
-        dirs.remove(dotted)
-      if 'resources' in dirs:
-        dirs.remove('resources')
-
-      dirs.sort()
-      files.sort()
-      for filename in files:
-        if filename.endswith(".js"):
-          fullpath = os.path.join(dirname, filename)
-          relpath = fullpath[len(self.root) + 1 : -3]
-          testname = relpath.replace(os.path.sep, "/")
-          test = self._create_test(testname)
-          tests.append(test)
-    return tests
+  def _test_loader_class(self):
+    return TestLoader
 
   def _test_class(self):
     return TestCase

tools/testrunner/local/fake_testsuite/testcfg.py

@@ -8,13 +8,19 @@ import sys
 from testrunner.local import testsuite, statusfile
 
 
+class TestLoader(testsuite.TestLoader):
+  pass
+
 class TestSuite(testsuite.TestSuite):
+  def _test_loader_class(self):
+    return TestLoader
+
   def _test_class(self):
     return testsuite.TestCase
 
   def ListTests(self):
-    fast = self._create_test("fast")
-    slow = self._create_test("slow")
+    fast = self._test_loader._create_test("fast", self)
+    slow = self._test_loader._create_test("slow", self)
     slow._statusfile_outcomes.append(statusfile.SLOW)
     yield fast
     yield slow

tools/testrunner/local/testsuite.py

@@ -78,6 +78,128 @@ class TestCombiner(object):
   def _combined_test_class(self):
     raise NotImplementedError()
 
+
+class TestLoader(object):
+  """Base class for loading TestSuite tests after applying test suite
+  transformations."""
+
+  def __init__(self, suite, test_class, test_config, test_root):
+    self.suite = suite
+    self.test_class = test_class
+    self.test_config = test_config
+    self.test_root = test_root
+
+  def _list_test_filenames(self):
+    """Implemented by the subclassed TestLoaders to list filenames."""
+    raise NotImplementedError
+
+  def _should_filter_by_name(self, name):
+    return False
+
+  def _should_filter_by_test(self, test):
+    return False
+
+  def _filename_to_testname(self, filename):
+    """Hook for subclasses to write their own filename transformation
+    logic before the test creation."""
+    return filename
+
+  # TODO: not needed for every TestLoader, extract it into a subclass.
+  def _path_to_name(self, path):
+    if utils.IsWindows():
+      return path.replace(os.path.sep, "/")
+
+    return path
+
+  def _create_test(self, path, suite, **kwargs):
+    """Converts paths into test objects using the given options"""
+    return self.test_class(
+      suite, path, self._path_to_name(path), self.test_config, **kwargs)
+
+  def list_tests(self):
+    """Loads and returns the test objects for a TestSuite"""
+    cases = []
+    filenames = sorted(self._list_test_filenames())
+    for filename in filenames:
+      if self._should_filter_by_name(filename):
+        continue
+
+      testname = self._filename_to_testname(filename)
+      case = self._create_test(testname, self.suite)
+      if self._should_filter_by_test(case):
+        continue
+
+      cases.append(case)
+
+    return cases
+
+
+class GenericTestLoader(TestLoader):
+  """Generic TestLoader implementing the logic for listing filenames"""
+  @property
+  def excluded_files(self):
+    return set()
+
+  @property
+  def excluded_dirs(self):
+    return set()
+
+  @property
+  def excluded_suffixes(self):
+    return set()
+
+  @property
+  def test_dirs(self):
+    return [self.test_root]
+
+  @property
+  def extension(self):
+    return ""
+
+  def _should_filter_by_name(self, filename):
+    if not filename.endswith(self.extension):
+      return True
+
+    for suffix in self.excluded_suffixes:
+      if filename.endswith(suffix):
+        return True
+
+    if os.path.basename(filename) in self.excluded_files:
+      return True
+
+    return False
+
+  def _filename_to_testname(self, filename):
+    if not self.extension:
+      return filename
+
+    return filename[:-len(self.extension)]
+
+  def _to_relpath(self, abspath, test_root):
+    return os.path.relpath(abspath, test_root)
+
+  def _list_test_filenames(self):
+    filenames = []
+    for test_dir in self.test_dirs:
+      test_root = os.path.join(self.test_root, test_dir)
+      for dirname, dirs, files in os.walk(test_root, followlinks=True):
+        for dir in dirs:
+          if dir in self.excluded_dirs or dir.startswith('.'):
+            dirs.remove(dir)
+
+        for filename in files:
+          abspath = os.path.join(dirname, filename)
+          filenames.append(self._to_relpath(abspath, test_root))
+
+    return filenames
+
+
+class JSTestLoader(GenericTestLoader):
+  @property
+  def extension(self):
+    return ".js"
+
+
 @contextmanager
 def _load_testsuite_module(name, root):
   f = None
@@ -102,12 +224,19 @@ class TestSuite(object):
     self.tests = None  # list of TestCase objects
     self.statusfile = None
 
+    self._test_loader = self._test_loader_class()(
+      self, self._test_class(), self.test_config, self.root)
+
   def status_file(self):
     return "%s/%s.status" % (self.root, self.name)
 
-  def ListTests(self):
+  @property
+  def _test_loader_class(self):
     raise NotImplementedError
 
+  def ListTests(self):
+    return self._test_loader.list_tests()
+
   def load_tests_from_disk(self, statusfile_variables):
     self.statusfile = statusfile.StatusFile(
       self.status_file(), statusfile_variables)
@@ -138,15 +267,5 @@ class TestSuite(object):
     """
     return None
 
-  def _create_test(self, path, **kwargs):
-    test_class = self._test_class()
-    return test_class(self, path, self._path_to_name(path), self.test_config,
-                      **kwargs)
-
   def _test_class(self):
     raise NotImplementedError
-
-  def _path_to_name(self, path):
-    if utils.IsWindows():
-      return path.replace("\\", "/")
-    return path

tools/unittests/testdata/testroot1/test/sweet/testcfg.py

@@ -9,14 +9,17 @@ Dummy test suite extension with some fruity tests.
 from testrunner.local import testsuite
 from testrunner.objects import testcase
 
-class TestSuite(testsuite.TestSuite):
-  def ListTests(self):
-    return map(
-        self._create_test, [
+class TestLoader(testsuite.TestLoader):
+  def _list_test_filenames(self):
+    return [
           'bananas', 'apples', 'cherries', 'mangoes', 'strawberries',
           'blackberries', 'raspberries',
-        ],
-    )
+    ]
+
+
+class TestSuite(testsuite.TestSuite):
+  def _test_loader_class(self):
+    return TestLoader
 
   def _test_class(self):
     return TestCase

tools/unittests/testdata/testroot2/test/sweet/testcfg.py

@@ -9,12 +9,15 @@ Dummy test suite extension with some flaky fruity tests.
 from testrunner.local import testsuite
 from testrunner.objects import testcase
 
+
+class TestLoader(testsuite.TestLoader):
+  def _list_test_filenames(self):
+    return ['bananaflakes']
+
+
 class TestSuite(testsuite.TestSuite):
-  def ListTests(self):
-    return map(
-        self._create_test,
-        ['bananaflakes'],
-    )
+  def _test_loader_class(self):
+    return TestLoader
 
   def _test_class(self):
     return TestCase