Revert "Reland "[test] refactor testsuite configuration""

This reverts commit 81eec150.

Reason for revert: windows mozilla test failures

Original change's description:
> Reland "[test] refactor testsuite configuration"
> 
> This is a reland of 7f92ad0a
> 
> Original change's description:
> > [test] refactor testsuite configuration
> > 
> > Every testsuite configuration consist of at least 30% code duplication.
> > 
> > The code age ranges from 10 years old to 5 years old. Implementing anything that
> > touches the testsuite code becomes a technical fight to the death.
> > 
> > This CL removes all the duplication by refactoring the common functionality.
> > 
> > This CL contains structural changes without any logical changes % small bug
> > fixes.
> > 
> > R=machenbach@chromium.org
> > CC=yangguo@chromium.org,sergiyb@chromium.org
> > 
> > Bug: v8:8174, v8:8769
> > Change-Id: Iee299569caa7abdc0307ecf606136669034a28a2
> > Reviewed-on: https://chromium-review.googlesource.com/c/1445881
> > Commit-Queue: Sergiy Belozorov <sergiyb@chromium.org>
> > Reviewed-by: Michael Achenbach <machenbach@chromium.org>
> > Cr-Commit-Position: refs/heads/master@{#59361}
> 
> Bug: v8:8174, v8:8769
> Change-Id: I8e7078cfb875ceb3777e57084e6f8dfac09693e7
> Reviewed-on: https://chromium-review.googlesource.com/c/1454485
> Reviewed-by: Michael Achenbach <machenbach@chromium.org>
> Commit-Queue: Tamer Tas <tmrts@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#59369}

TBR=machenbach@chromium.org,sergiyb@chromium.org,tmrts@chromium.org

Change-Id: I8f5650b5f46be299c004e2fa8b708fa2c17a4dc2
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: v8:8174, v8:8769
Reviewed-on: https://chromium-review.googlesource.com/c/1454607Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Commit-Queue: Tamer Tas <tmrts@chromium.org>
Cr-Commit-Position: refs/heads/master@{#59370}

test/benchmarks/testcfg.py

@@ -34,9 +34,13 @@ from testrunner.local import testsuite
 from testrunner.objects import testcase
 
 
-class TestLoader(testsuite.TestLoader):
-  def _list_test_filenames(self):
-    return [
+class TestSuite(testsuite.TestSuite):
+  def __init__(self, *args, **kwargs):
+    super(TestSuite, self).__init__(*args, **kwargs)
+    self.testroot = os.path.join(self.root, "data")
+
+  def ListTests(self):
+    tests = map(self._create_test, [
         "kraken/ai-astar",
         "kraken/audio-beat-detection",
         "kraken/audio-dft",
@@ -94,16 +98,8 @@ class TestLoader(testsuite.TestLoader):
         "sunspider/string-tagcloud",
         "sunspider/string-unpack-code",
         "sunspider/string-validate-input",
-    ]
-
-
-class TestSuite(testsuite.TestSuite):
-  def __init__(self, *args, **kwargs):
-    super(TestSuite, self).__init__(*args, **kwargs)
-    self.testroot = os.path.join(self.root, "data")
-
-  def _test_loader_class(self):
-    return TestLoader
+    ])
+    return tests
 
   def _test_class(self):
     return TestCase

test/cctest/testcfg.py

@@ -36,8 +36,8 @@ from testrunner.objects import testcase
 SHELL = 'cctest'
 
 
-class TestLoader(testsuite.TestLoader):
-  def _list_test_filenames(self):
+class TestSuite(testsuite.TestSuite):
+  def ListTests(self):
     shell = os.path.abspath(os.path.join(self.test_config.shell_dir, SHELL))
     if utils.IsWindows():
       shell += ".exe"
@@ -46,19 +46,14 @@ class TestLoader(testsuite.TestLoader):
         shell=shell,
         args=["--list"] + self.test_config.extra_flags)
     output = cmd.execute()
-    # TODO make errors visible (see duplicated code in 'unittests')
     if output.exit_code != 0:
       print cmd
       print output.stdout
       print output.stderr
       return []
-
-    return output.stdout.strip().split()
-
-
-class TestSuite(testsuite.TestSuite):
-  def _test_loader_class(self):
-    return TestLoader
+    tests = map(self._create_test, output.stdout.strip().split())
+    tests.sort(key=lambda t: t.path)
+    return tests
 
   def _test_class(self):
     return TestCase

test/debugger/testcfg.py

@@ -11,16 +11,22 @@ from testrunner.objects import testcase
 FILES_PATTERN = re.compile(r"//\s+Files:(.*)")
 MODULE_PATTERN = re.compile(r"^// MODULE$", flags=re.MULTILINE)
 
-
-class TestLoader(testsuite.JSTestLoader):
-  @property
-  def excluded_files(self):
-    return {"test-api.js"}
-
-
 class TestSuite(testsuite.TestSuite):
-  def _test_loader_class(self):
-    return TestLoader
+  def ListTests(self):
+    tests = []
+    for dirname, dirs, files in os.walk(self.root):
+      for dotted in [x for x in dirs if x.startswith('.')]:
+        dirs.remove(dotted)
+      dirs.sort()
+      files.sort()
+      for filename in files:
+        if (filename.endswith(".js") and filename != "test-api.js"):
+          fullpath = os.path.join(dirname, filename)
+          relpath = fullpath[len(self.root) + 1 : -3]
+          testname = relpath.replace(os.path.sep, "/")
+          test = self._create_test(testname)
+          tests.append(test)
+    return tests
 
   def _test_class(self):
     return TestCase

test/fuzzer/testcfg.py

@@ -7,35 +7,26 @@ import os
 from testrunner.local import testsuite
 from testrunner.objects import testcase
 
-SUB_TESTS = [
-  'json',
-  'parser',
-  'regexp_builtins',
-  'regexp',
-  'multi_return',
-  'wasm',
-  'wasm_async',
-  'wasm_code',
-  'wasm_compile',
-]
 
 class VariantsGenerator(testsuite.VariantsGenerator):
   def _get_variants(self, test):
     return self._standard_variant
 
 
-class TestLoader(testsuite.GenericTestLoader):
-  @property
-  def test_dirs(self):
-    return SUB_TESTS
-
-  def _to_relpath(self, abspath, _):
-    return os.path.relpath(abspath, self.suite.root)
-
-
 class TestSuite(testsuite.TestSuite):
-  def _test_loader_class(self):
-    return TestLoader
+  SUB_TESTS = ( 'json', 'parser', 'regexp_builtins', 'regexp', 'multi_return', 'wasm',
+          'wasm_async', 'wasm_code', 'wasm_compile')
+
+  def ListTests(self):
+    tests = []
+    for subtest in TestSuite.SUB_TESTS:
+      for fname in os.listdir(os.path.join(self.root, subtest)):
+        if not os.path.isfile(os.path.join(self.root, subtest, fname)):
+          continue
+        test = self._create_test('%s/%s' % (subtest, fname))
+        tests.append(test)
+    tests.sort()
+    return tests
 
   def _test_class(self):
     return TestCase
@@ -46,7 +37,7 @@ class TestSuite(testsuite.TestSuite):
 
 class TestCase(testcase.TestCase):
   def _get_files_params(self):
-    suite, name = self.path.split(os.path.sep)
+    suite, name = self.path.split('/')
     return [os.path.join(self.suite.root, suite, name)]
 
   def _get_variant_flags(self):
@@ -59,7 +50,7 @@ class TestCase(testcase.TestCase):
     return []
 
   def get_shell(self):
-    group, _ = self.path.split(os.path.sep, 1)
+    group, _ = self.path.split('/', 1)
     return 'v8_simple_%s_fuzzer' % group
 
 

test/inspector/testcfg.py

@@ -13,19 +13,25 @@ PROTOCOL_TEST_JS = "protocol-test.js"
 EXPECTED_SUFFIX = "-expected.txt"
 RESOURCES_FOLDER = "resources"
 
-class TestLoader(testsuite.JSTestLoader):
-  @property
-  def excluded_files(self):
-    return {PROTOCOL_TEST_JS}
-
-  @property
-  def excluded_dirs(self):
-    return {RESOURCES_FOLDER}
-
-
 class TestSuite(testsuite.TestSuite):
-  def _test_loader_class(self):
-    return TestLoader
+  def ListTests(self):
+    tests = []
+    for dirname, dirs, files in os.walk(
+        os.path.join(self.root), followlinks=True):
+      for dotted in [x for x in dirs if x.startswith('.')]:
+        dirs.remove(dotted)
+      if dirname.endswith(os.path.sep + RESOURCES_FOLDER):
+        continue
+      dirs.sort()
+      files.sort()
+      for filename in files:
+        if filename.endswith(".js") and filename != PROTOCOL_TEST_JS:
+          fullpath = os.path.join(dirname, filename)
+          relpath = fullpath[len(self.root) + 1 : -3]
+          testname = relpath.replace(os.path.sep, "/")
+          test = self._create_test(testname)
+          tests.append(test)
+    return tests
 
   def _test_class(self):
     return TestCase

test/intl/testcfg.py

@@ -33,16 +33,23 @@ from testrunner.objects import testcase
 
 ENV_PATTERN = re.compile(r"//\s+Environment Variables:(.*)")
 
-
-class TestLoader(testsuite.JSTestLoader):
-  @property
-  def excluded_files(self):
-    return {"assert.js", "utils.js"}
-
-
 class TestSuite(testsuite.TestSuite):
-  def _test_loader_class(self):
-    return TestLoader
+  def ListTests(self):
+    tests = []
+    for dirname, dirs, files in os.walk(self.root):
+      for dotted in [x for x in dirs if x.startswith('.')]:
+        dirs.remove(dotted)
+      dirs.sort()
+      files.sort()
+      for filename in files:
+        if (filename.endswith(".js") and filename != "assert.js" and
+            filename != "utils.js"):
+          fullpath = os.path.join(dirname, filename)
+          relpath = fullpath[len(self.root) + 1 : -3]
+          testname = relpath.replace(os.path.sep, "/")
+          test = self._create_test(testname)
+          tests.append(test)
+    return tests
 
   def _test_class(self):
     return TestCase

test/message/testcfg.py

@@ -39,8 +39,21 @@ MODULE_PATTERN = re.compile(r"^// MODULE$", flags=re.MULTILINE)
 
 
 class TestSuite(testsuite.TestSuite):
-  def _test_loader_class(self):
-    return testsuite.JSTestLoader
+  def ListTests(self):
+    tests = []
+    for dirname, dirs, files in os.walk(self.root):
+      for dotted in [x for x in dirs if x.startswith('.')]:
+        dirs.remove(dotted)
+      dirs.sort()
+      files.sort()
+      for filename in files:
+        if filename.endswith(".js"):
+          fullpath = os.path.join(dirname, filename)
+          relpath = fullpath[len(self.root) + 1 : -3]
+          testname = relpath.replace(os.path.sep, "/")
+          test = self._create_test(testname)
+          tests.append(test)
+    return tests
 
   def _test_class(self):
     return TestCase

test/mjsunit/testcfg.py

@@ -56,19 +56,24 @@ COMBINE_TESTS_FLAGS_BLACKLIST = [
   '--wasm-lazy-compilation',
 ]
 
-
-class TestLoader(testsuite.JSTestLoader):
-  @property
-  def excluded_files(self):
-    return {
-      "mjsunit.js",
-      "mjsunit_suppressions.js",
-    }
-
-
 class TestSuite(testsuite.TestSuite):
-  def _test_loader_class(self):
-    return TestLoader
+  def ListTests(self):
+    tests = []
+    for dirname, dirs, files in os.walk(self.root, followlinks=True):
+      for dotted in [x for x in dirs if x.startswith('.')]:
+        dirs.remove(dotted)
+      dirs.sort()
+      files.sort()
+      for filename in files:
+        if (filename.endswith(".js") and
+            filename != "mjsunit.js" and
+            filename != "mjsunit_suppressions.js"):
+          fullpath = os.path.join(dirname, filename)
+          relpath = fullpath[len(self.root) + 1 : -3]
+          testname = relpath.replace(os.path.sep, "/")
+          test = self._create_test(testname)
+          tests.append(test)
+    return tests
 
   def _test_combiner_class(self):
     return TestCombiner

test/mkgrokdump/testcfg.py

@@ -11,11 +11,6 @@ from testrunner.outproc import mkgrokdump
 
 SHELL = 'mkgrokdump'
 
-
-class TestLoader(testsuite.TestLoader):
-  pass
-
-#TODO(tmrts): refactor the test creation logic to migrate to TestLoader
 class TestSuite(testsuite.TestSuite):
   def __init__(self, *args, **kwargs):
     super(TestSuite, self).__init__(*args, **kwargs)
@@ -24,12 +19,9 @@ class TestSuite(testsuite.TestSuite):
     self.expected_path = os.path.join(v8_path, 'tools', 'v8heapconst.py')
 
   def ListTests(self):
-    test = self._test_loader._create_test(SHELL, self)
+    test = self._create_test(SHELL)
     return [test]
 
-  def _test_loader_class(self):
-    return TestLoader
-
   def _test_class(self):
     return TestCase
 

test/mozilla/testcfg.py

@@ -32,55 +32,54 @@ from testrunner.local import testsuite
 from testrunner.objects import testcase
 from testrunner.outproc import mozilla
 
-EXCLUDED = [
-  "CVS",
-  ".svn",
-]
-
-FRAMEWORK = [
-  "browser.js",
-  "shell.js",
-  "jsref.js",
-  "template.js",
-]
-
-TEST_DIRS = [
-  "ecma",
-  "ecma_2",
-  "ecma_3",
-  "js1_1",
-  "js1_2",
-  "js1_3",
-  "js1_4",
-  "js1_5",
-]
-
-class TestLoader(testsuite.JSTestLoader):
-  @property
-  def excluded_files(self):
-    return set(FRAMEWORK)
+EXCLUDED = ["CVS", ".svn"]
 
-  @property
-  def excluded_dirs(self):
-    return set(EXCLUDED)
 
-  @property
-  def test_dirs(self):
-    return TEST_DIRS
+FRAMEWORK = """
+  browser.js
+  shell.js
+  jsref.js
+  template.js
+""".split()
 
-  def _to_relpath(self, abspath, _):
-    # TODO: refactor this by setting the test path during the TestCase creation
-    return os.path.relpath(abspath, self.test_root)
+
+TEST_DIRS = """
+  ecma
+  ecma_2
+  ecma_3
+  js1_1
+  js1_2
+  js1_3
+  js1_4
+  js1_5
+""".split()
 
 
 class TestSuite(testsuite.TestSuite):
   def __init__(self, *args, **kwargs):
     super(TestSuite, self).__init__(*args, **kwargs)
-    self.test_root = os.path.join(self.root, "data")
-    self._test_loader.test_root = self.test_root
-
-  def _test_loader_class(self):
-    return TestLoader
+    self.testroot = os.path.join(self.root, "data")
+
+  def ListTests(self):
+    tests = []
+    for testdir in TEST_DIRS:
+      current_root = os.path.join(self.testroot, testdir)
+      for dirname, dirs, files in os.walk(current_root):
+        for dotted in [x for x in dirs if x.startswith(".")]:
+          dirs.remove(dotted)
+        for excluded in EXCLUDED:
+          if excluded in dirs:
+            dirs.remove(excluded)
+        dirs.sort()
+        files.sort()
+        for filename in files:
+          if filename.endswith(".js") and not filename in FRAMEWORK:
+            fullpath = os.path.join(dirname, filename)
+            relpath = fullpath[len(self.testroot) + 1 : -3]
+            testname = relpath.replace(os.path.sep, "/")
+            case = self._create_test(testname)
+            tests.append(case)
+    return tests
 
   def _test_class(self):
     return TestCase
@@ -92,20 +91,20 @@ class TestCase(testcase.D8TestCase):
     testfilename = self.path + ".js"
     testfilepath = testfilename.split("/")
     for i in xrange(len(testfilepath)):
-      script = os.path.join(self.suite.test_root,
+      script = os.path.join(self.suite.testroot,
                             reduce(os.path.join, testfilepath[:i], ""),
                             "shell.js")
       if os.path.exists(script):
         files.append(script)
 
-    files.append(os.path.join(self.suite.test_root, testfilename))
+    files.append(os.path.join(self.suite.testroot, testfilename))
     return files
 
   def _get_suite_flags(self):
     return ['--expose-gc']
 
   def _get_source_path(self):
-    return os.path.join(self.suite.test_root, self.path + self._get_suffix())
+    return os.path.join(self.suite.testroot, self.path + self._get_suffix())
 
   @property
   def output_proc(self):
@@ -118,5 +117,6 @@ class TestCase(testcase.D8TestCase):
     return mozilla.OutProc(self.expected_outcomes)
 
 
+
 def GetSuite(*args, **kwargs):
   return TestSuite(*args, **kwargs)

test/preparser/testcfg.py

@@ -37,11 +37,6 @@ class VariantsGenerator(testsuite.VariantsGenerator):
     return self._standard_variant
 
 
-class TestLoader(testsuite.TestLoader):
-  pass
-
-
-# TODO(tmrts): refactor the python template parsing then use the TestLoader.
 class TestSuite(testsuite.TestSuite):
   def _ParsePythonTestTemplates(self, result, filename):
     pathname = os.path.join(self.root, filename + ".pyt")
@@ -76,11 +71,8 @@ class TestSuite(testsuite.TestSuite):
     return result
 
   def _create_test(self, path, source, template_flags):
-    return self._test_loader._create_test(
-        path, self, source=source, template_flags=template_flags)
-
-  def _test_loader_class(self):
-    return TestLoader
+    return super(TestSuite, self)._create_test(
+        path, source=source, template_flags=template_flags)
 
   def _test_class(self):
     return TestCase

test/test262/testcfg.py

@@ -73,6 +73,9 @@ TEST_262_HARNESS_PATH = ["data", "harness"]
 TEST_262_TOOLS_PATH = ["harness", "src"]
 TEST_262_LOCAL_TESTS_PATH = ["local-tests", "test"]
 
+TEST_262_RELPATH_REGEXP = re.compile(
+    r'.*[\\/]test[\\/]test262[\\/][^\\/]+[\\/]test[\\/](.*)\.js')
+
 sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),
                              *TEST_262_TOOLS_PATH))
 
@@ -100,38 +103,18 @@ class VariantsGenerator(testsuite.VariantsGenerator):
           yield (variant, flags + ['--use-strict'], 'strict-%d' % n + phase_var)
 
 
-class TestLoader(testsuite.JSTestLoader):
-  @property
-  def test_dirs(self):
-    return [
-      self.test_root,
-      os.path.join(self.suite.root, *TEST_262_LOCAL_TESTS_PATH),
-    ]
-
-  @property
-  def excluded_suffixes(self):
-    return {"_FIXTURE.js"}
-
-  @property
-  def excluded_dirs(self):
-    return {"intl402"} if self.test_config.noi18n else set()
-
-  def _should_filter_by_test(self, test):
-    features = test.test_record.get("features", [])
-    return SKIPPED_FEATURES.intersection(features)
-
-
 class TestSuite(testsuite.TestSuite):
+  # Match the (...) in '/path/to/v8/test/test262/subdir/test/(...).js'
+  # In practice, subdir is data or local-tests
+
   def __init__(self, *args, **kwargs):
     super(TestSuite, self).__init__(*args, **kwargs)
-    self.test_root = os.path.join(self.root, *TEST_262_SUITE_PATH)
-    # TODO: this makes the TestLoader mutable, refactor it.
-    self._test_loader.test_root = self.test_root
+    self.testroot = os.path.join(self.root, *TEST_262_SUITE_PATH)
     self.harnesspath = os.path.join(self.root, *TEST_262_HARNESS_PATH)
     self.harness = [os.path.join(self.harnesspath, f)
                     for f in TEST_262_HARNESS_FILES]
     self.harness += [os.path.join(self.root, "harness-adapt.js")]
-    self.local_test_root = os.path.join(self.root, *TEST_262_LOCAL_TESTS_PATH)
+    self.localtestroot = os.path.join(self.root, *TEST_262_LOCAL_TESTS_PATH)
     self.parse_test_record = self._load_parse_test_record()
 
   def _load_parse_test_record(self):
@@ -149,8 +132,28 @@ class TestSuite(testsuite.TestSuite):
       if f:
         f.close()
 
-  def _test_loader_class(self):
-    return TestLoader
+  def ListTests(self):
+    testnames = set()
+    for dirname, dirs, files in itertools.chain(os.walk(self.testroot),
+                                                os.walk(self.localtestroot)):
+      for dotted in [x for x in dirs if x.startswith(".")]:
+        dirs.remove(dotted)
+      if self.test_config.noi18n and "intl402" in dirs:
+        dirs.remove("intl402")
+      dirs.sort()
+      files.sort()
+      for filename in files:
+        if not filename.endswith(".js"):
+          continue
+        if filename.endswith("_FIXTURE.js"):
+          continue
+        fullpath = os.path.join(dirname, filename)
+        relpath = re.match(TEST_262_RELPATH_REGEXP, fullpath).group(1)
+        testnames.add(relpath.replace(os.path.sep, "/"))
+    cases = map(self._create_test, testnames)
+    return [case for case in cases if len(
+                SKIPPED_FEATURES.intersection(
+                    case.test_record.get("features", []))) == 0]
 
   def _test_class(self):
     return TestCase
@@ -192,15 +195,11 @@ class TestCase(testcase.D8TestCase):
   def _fail_phase_reverse(self):
     return 'fail-phase-reverse' in self.procid
 
-  def __needs_harness_agent(self):
-    tokens = self.path.split(os.path.sep)
-    return tokens[:2] == ["built-ins", "Atomics"]
-
   def _get_files_params(self):
     return (
         list(self.suite.harness) +
         ([os.path.join(self.suite.root, "harness-agent.js")]
-         if self.__needs_harness_agent() else []) +
+         if self.path.startswith('built-ins/Atomics') else []) +
         ([os.path.join(self.suite.root, "harness-adapt-donotevaluate.js")]
          if self.fail_phase_only and not self._fail_phase_reverse else []) +
         self._get_includes() +
@@ -231,10 +230,10 @@ class TestCase(testcase.D8TestCase):
 
   def _get_source_path(self):
     filename = self.path + self._get_suffix()
-    path = os.path.join(self.suite.local_test_root, filename)
+    path = os.path.join(self.suite.localtestroot, filename)
     if os.path.exists(path):
       return path
-    return os.path.join(self.suite.test_root, filename)
+    return os.path.join(self.suite.testroot, filename)
 
   @property
   def output_proc(self):

test/unittests/testcfg.py

@@ -15,10 +15,9 @@ class VariantsGenerator(testsuite.VariantsGenerator):
     return self._standard_variant
 
 
-class TestLoader(testsuite.TestLoader):
-  def _list_test_filenames(self):
-    shell = os.path.abspath(
-      os.path.join(self.test_config.shell_dir, "unittests"))
+class TestSuite(testsuite.TestSuite):
+  def ListTests(self):
+    shell = os.path.abspath(os.path.join(self.test_config.shell_dir, self.name))
     if utils.IsWindows():
       shell += ".exe"
 
@@ -31,7 +30,6 @@ class TestLoader(testsuite.TestLoader):
       output = cmd.execute()
       if output.exit_code == 0:
         break
-
       print "Test executable failed to list the tests (try %d).\n\nCmd:" % i
       print cmd
       print "\nStdout:"
@@ -42,22 +40,17 @@ class TestLoader(testsuite.TestLoader):
     else:
       raise Exception("Test executable failed to list the tests.")
 
-    # TODO create an ExecutableTestLoader for refactoring this similar to
-    # JSTestLoader.
-    test_names = []
+    tests = []
+    test_case = ''
     for line in output.stdout.splitlines():
       test_desc = line.strip().split()[0]
       if test_desc.endswith('.'):
         test_case = test_desc
       elif test_case and test_desc:
-        test_names.append(test_case + test_desc)
-
-    return test_names
-
-
-class TestSuite(testsuite.TestSuite):
-  def _test_loader_class(self):
-    return TestLoader
+        test_path = test_case + test_desc
+        tests.append(self._create_test(test_path))
+    tests.sort(key=lambda t: t.path)
+    return tests
 
   def _test_class(self):
     return TestCase

test/wasm-js/testcfg.py

@@ -12,23 +12,28 @@ ANY_JS = ".any.js"
 WPT_ROOT = "/wasm/jsapi/"
 META_SCRIPT_REGEXP = re.compile(r"META:\s*script=(.*)")
 
-
-class TestLoader(testsuite.JSTestLoader):
-  @property
-  def extension(self):
-    return ANY_JS
-
-
 class TestSuite(testsuite.TestSuite):
   def __init__(self, *args, **kwargs):
     super(TestSuite, self).__init__(*args, **kwargs)
+    self.testroot = os.path.join(self.root, "data", "test", "js-api")
     self.mjsunit_js = os.path.join(os.path.dirname(self.root), "mjsunit",
                                    "mjsunit.js")
-    self.test_root = os.path.join(self.root, "data", "test", "js-api")
-    self._test_loader.test_root = self.test_root
 
-  def _test_loader_class(self):
-    return TestLoader
+  def ListTests(self):
+    tests = []
+    for dirname, dirs, files in os.walk(self.testroot):
+      for dotted in [x for x in dirs if x.startswith(".")]:
+        dirs.remove(dotted)
+      dirs.sort()
+      files.sort()
+      for filename in files:
+        if (filename.endswith(ANY_JS)):
+          fullpath = os.path.join(dirname, filename)
+          relpath = fullpath[len(self.testroot) + 1 : -len(ANY_JS)]
+          testname = relpath.replace(os.path.sep, "/")
+          test = self._create_test(testname)
+          tests.append(test)
+    return tests
 
   def _test_class(self):
     return TestCase
@@ -44,7 +49,7 @@ class TestCase(testcase.D8TestCase):
       if script.startswith(WPT_ROOT):
         # Matched an absolute path, strip the root and replace it with our
         # local root.
-        script = os.path.join(self.suite.test_root, script[len(WPT_ROOT):])
+        script = os.path.join(self.suite.testroot, script[len(WPT_ROOT):])
       elif not script.startswith("/"):
         # Matched a relative path, prepend this test's directory.
         thisdir = os.path.dirname(self._get_source_path())
@@ -62,7 +67,7 @@ class TestCase(testcase.D8TestCase):
 
   def _get_source_path(self):
     # All tests are named `path/name.any.js`
-    return os.path.join(self.suite.test_root, self.path + ANY_JS)
+    return os.path.join(self.suite.testroot, self.path + ANY_JS)
 
 
 def GetSuite(*args, **kwargs):

test/wasm-spec-tests/testcfg.py

@@ -7,13 +7,20 @@ import os
 from testrunner.local import testsuite
 from testrunner.objects import testcase
 
-
-class TestLoader(testsuite.JSTestLoader):
-  pass
-
 class TestSuite(testsuite.TestSuite):
-  def _test_loader_class(self):
-    return TestLoader
+  def ListTests(self):
+    tests = []
+    for dirname, dirs, files in os.walk(self.root):
+      for dotted in [x for x in dirs if x.startswith('.')]:
+        dirs.remove(dotted)
+      for filename in files:
+        if (filename.endswith(".js")):
+          fullpath = os.path.join(dirname, filename)
+          relpath = fullpath[len(self.root) + 1 : -3]
+          testname = relpath.replace(os.path.sep, "/")
+          test = self._create_test(testname)
+          tests.append(test)
+    return tests
 
   def _test_class(self):
     return TestCase

test/webkit/testcfg.py

@@ -36,15 +36,26 @@ FILES_PATTERN = re.compile(r"//\s+Files:(.*)")
 SELF_SCRIPT_PATTERN = re.compile(r"//\s+Env: TEST_FILE_NAME")
 
 
-class TestLoader(testsuite.JSTestLoader):
-  @property
-  def excluded_dirs(self):
-    return {"resources"}
-
-
+# TODO (machenbach): Share commonalities with mjstest.
 class TestSuite(testsuite.TestSuite):
-  def _test_loader_class(self):
-    return TestLoader
+  def ListTests(self):
+    tests = []
+    for dirname, dirs, files in os.walk(self.root):
+      for dotted in [x for x in dirs if x.startswith('.')]:
+        dirs.remove(dotted)
+      if 'resources' in dirs:
+        dirs.remove('resources')
+
+      dirs.sort()
+      files.sort()
+      for filename in files:
+        if filename.endswith(".js"):
+          fullpath = os.path.join(dirname, filename)
+          relpath = fullpath[len(self.root) + 1 : -3]
+          testname = relpath.replace(os.path.sep, "/")
+          test = self._create_test(testname)
+          tests.append(test)
+    return tests
 
   def _test_class(self):
     return TestCase

tools/testrunner/local/fake_testsuite/testcfg.py

@@ -8,19 +8,13 @@ import sys
 from testrunner.local import testsuite, statusfile
 
 
-class TestLoader(testsuite.TestLoader):
-  pass
-
 class TestSuite(testsuite.TestSuite):
-  def _test_loader_class(self):
-    return TestLoader
-
   def _test_class(self):
     return testsuite.TestCase
 
   def ListTests(self):
-    fast = self._test_loader._create_test("fast", self)
-    slow = self._test_loader._create_test("slow", self)
+    fast = self._create_test("fast")
+    slow = self._create_test("slow")
     slow._statusfile_outcomes.append(statusfile.SLOW)
     yield fast
     yield slow

tools/testrunner/local/testsuite.py

@@ -78,128 +78,6 @@ class TestCombiner(object):
   def _combined_test_class(self):
     raise NotImplementedError()
 
-
-class TestLoader(object):
-  """Base class for loading TestSuite tests after applying test suite
-  transformations."""
-
-  def __init__(self, suite, test_class, test_config, test_root):
-    self.suite = suite
-    self.test_class = test_class
-    self.test_config = test_config
-    self.test_root = test_root
-
-  def _list_test_filenames(self):
-    """Implemented by the subclassed TestLoaders to list filenames."""
-    raise NotImplementedError
-
-  def _should_filter_by_name(self, name):
-    return False
-
-  def _should_filter_by_test(self, test):
-    return False
-
-  def _filename_to_testname(self, filename):
-    """Hook for subclasses to write their own filename transformation
-    logic before the test creation."""
-    return filename
-
-  # TODO: not needed for every TestLoader, extract it into a subclass.
-  def _path_to_name(self, path):
-    if utils.IsWindows():
-      return path.replace(os.path.sep, "/")
-
-    return path
-
-  def _create_test(self, path, suite, **kwargs):
-    """Converts paths into test objects using the given options"""
-    return self.test_class(
-      suite, path, self._path_to_name(path), self.test_config, **kwargs)
-
-  def list_tests(self):
-    """Loads and returns the test objects for a TestSuite"""
-    cases = []
-    filenames = sorted(self._list_test_filenames())
-    for filename in filenames:
-      if self._should_filter_by_name(filename):
-        continue
-
-      testname = self._filename_to_testname(filename)
-      case = self._create_test(testname, self.suite)
-      if self._should_filter_by_test(case):
-        continue
-
-      cases.append(case)
-
-    return cases
-
-
-class GenericTestLoader(TestLoader):
-  """Generic TestLoader implementing the logic for listing filenames"""
-  @property
-  def excluded_files(self):
-    return set()
-
-  @property
-  def excluded_dirs(self):
-    return set()
-
-  @property
-  def excluded_suffixes(self):
-    return set()
-
-  @property
-  def test_dirs(self):
-    return [self.test_root]
-
-  @property
-  def extension(self):
-    return ""
-
-  def _should_filter_by_name(self, filename):
-    if not filename.endswith(self.extension):
-      return True
-
-    for suffix in self.excluded_suffixes:
-      if filename.endswith(suffix):
-        return True
-
-    if os.path.basename(filename) in self.excluded_files:
-      return True
-
-    return False
-
-  def _filename_to_testname(self, filename):
-    if not self.extension:
-      return filename
-
-    return filename[:-len(self.extension)]
-
-  def _to_relpath(self, abspath, test_root):
-    return os.path.relpath(abspath, test_root)
-
-  def _list_test_filenames(self):
-    filenames = []
-    for test_dir in self.test_dirs:
-      test_root = os.path.join(self.test_root, test_dir)
-      for dirname, dirs, files in os.walk(test_root, followlinks=True):
-        for dir in dirs:
-          if dir in self.excluded_dirs or dir.startswith('.'):
-            dirs.remove(dir)
-
-        for filename in files:
-          abspath = os.path.join(dirname, filename)
-          filenames.append(self._to_relpath(abspath, test_root))
-
-    return filenames
-
-
-class JSTestLoader(GenericTestLoader):
-  @property
-  def extension(self):
-    return ".js"
-
-
 @contextmanager
 def _load_testsuite_module(name, root):
   f = None
@@ -224,18 +102,11 @@ class TestSuite(object):
     self.tests = None  # list of TestCase objects
     self.statusfile = None
 
-    self._test_loader = self._test_loader_class()(
-      self, self._test_class(), self.test_config, self.root)
-
   def status_file(self):
     return "%s/%s.status" % (self.root, self.name)
 
-  @property
-  def _test_loader_class(self):
-    raise NotImplementedError
-
   def ListTests(self):
-    return self._test_loader.list_tests()
+    raise NotImplementedError
 
   def load_tests_from_disk(self, statusfile_variables):
     self.statusfile = statusfile.StatusFile(
@@ -267,5 +138,15 @@ class TestSuite(object):
     """
     return None
 
+  def _create_test(self, path, **kwargs):
+    test_class = self._test_class()
+    return test_class(self, path, self._path_to_name(path), self.test_config,
+                      **kwargs)
+
   def _test_class(self):
     raise NotImplementedError
+
+  def _path_to_name(self, path):
+    if utils.IsWindows():
+      return path.replace("\\", "/")
+    return path

tools/unittests/testdata/testroot1/test/sweet/testcfg.py

@@ -9,17 +9,14 @@ Dummy test suite extension with some fruity tests.
 from testrunner.local import testsuite
 from testrunner.objects import testcase
 
-class TestLoader(testsuite.TestLoader):
-  def _list_test_filenames(self):
-    return [
+class TestSuite(testsuite.TestSuite):
+  def ListTests(self):
+    return map(
+        self._create_test, [
           'bananas', 'apples', 'cherries', 'mangoes', 'strawberries',
           'blackberries', 'raspberries',
-    ]
-
-
-class TestSuite(testsuite.TestSuite):
-  def _test_loader_class(self):
-    return TestLoader
+        ],
+    )
 
   def _test_class(self):
     return TestCase

tools/unittests/testdata/testroot2/test/sweet/testcfg.py

@@ -9,15 +9,12 @@ Dummy test suite extension with some flaky fruity tests.
 from testrunner.local import testsuite
 from testrunner.objects import testcase
 
-
-class TestLoader(testsuite.TestLoader):
-  def _list_test_filenames(self):
-    return ['bananaflakes']
-
-
 class TestSuite(testsuite.TestSuite):
-  def _test_loader_class(self):
-    return TestLoader
+  def ListTests(self):
+    return map(
+        self._create_test,
+        ['bananaflakes'],
+    )
 
   def _test_class(self):
     return TestCase