[testrunner] use selection sort generator instead of timsort for test cases

V8 testrunner is loading every test it has to run into memory greedily in order
to sort by slowness of the test case. The memory and CPU overhead for loading
the test-suites are non-trivial.

This CL restructures it by changing the sorting method.

R=machenbach@chromium.org
CC=​​sergiyb@chromium.org,yangguo@chromium.org

Bug: v8:8174
Change-Id: I08331182147b92cf4ac54823eea0e2b472f51e84
Reviewed-on: https://chromium-review.googlesource.com/c/1406684
Commit-Queue: Tamer Tas <tmrts@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Sergiy Belozorov <sergiyb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#58821}

tools/testrunner/base_runner.py

@@ -260,11 +260,10 @@ class BaseTestRunner(object):
         raise
 
       args = self._parse_test_args(args)
-      suites = self._load_testsuites(args, options)
+      tests = self._load_testsuite_generators(args, options)
       self._setup_env()
       print(">>> Running tests for %s.%s" % (self.build_config.arch,
                                             self.mode_name))
-      tests = [t for s in suites for t in s.tests]
       return self._do_execute(tests, args, options)
     except TestRunnerError:
       return utils.EXIT_CODE_INTERNAL_ERROR
@@ -602,11 +601,11 @@ class BaseTestRunner(object):
   def _get_default_suite_names(self):
     return []
 
-  def _load_testsuites(self, args, options):
+  def _load_testsuite_generators(self, args, options):
     names = self._args_to_suite_names(args, options.test_root)
     test_config = self._create_test_config(options)
     variables = self._get_statusfile_variables(options)
-    suites = []
+    slow_chain, fast_chain = [], []
     for name in names:
       if options.verbose:
         print '>>> Loading test suite: %s' % name
@@ -614,10 +613,17 @@ class BaseTestRunner(object):
           os.path.join(options.test_root, name), test_config)
 
       if self._is_testsuite_supported(suite, options):
-        suite.load_tests_from_disk(variables)
-        suites.append(suite)
+        slow_tests, fast_tests = suite.load_tests_from_disk(variables)
+        slow_chain.append(slow_tests)
+        fast_chain.append(fast_tests)
 
-    return suites
+    for tests in slow_chain:
+      for test in tests:
+        yield test
+
+    for tests in fast_chain:
+      for test in tests:
+        yield test
 
   def _is_testsuite_supported(self, suite, options):
     """A predicate that can be overridden to filter out unsupported TestSuite

tools/testrunner/local/fake_testsuite/testcfg.py

@@ -5,12 +5,19 @@
 
 import sys
 
-from testrunner.local import testsuite
+from testrunner.local import testsuite, statusfile
 
 
 class TestSuite(testsuite.TestSuite):
+  def _test_class(self):
+    return testsuite.TestCase
+
   def ListTests(self):
-    return []
+    fast = self._create_test("fast")
+    slow = self._create_test("slow")
+    slow._statusfile_outcomes.append(statusfile.SLOW)
+    yield fast
+    yield slow
 
 def GetSuite(*args, **kwargs):
   return TestSuite(*args, **kwargs)

tools/testrunner/local/testsuite.py

@@ -111,7 +111,11 @@ class TestSuite(object):
   def load_tests_from_disk(self, statusfile_variables):
     self.statusfile = statusfile.StatusFile(
       self.status_file(), statusfile_variables)
-    self.tests = self.ListTests()
+
+    slow_tests = (test for test in self.ListTests() if test.is_slow)
+    fast_tests = (test for test in self.ListTests() if not test.is_slow)
+
+    return slow_tests, fast_tests
 
   def get_variants_gen(self, variants):
     return self._variants_gen_class()(variants)

tools/testrunner/local/testsuite_unittest.py

@@ -47,10 +47,19 @@ class TestSuiteTest(unittest.TestCase):
     self.assertIsNone(self.suite.statusfile)
 
   def testLoadingTestsFromDisk(self):
-    self.suite.load_tests_from_disk(statusfile_variables={})
+    slow_tests, fast_tests = self.suite.load_tests_from_disk(
+      statusfile_variables={})
+    def is_generator(iterator):
+      return iterator == iter(iterator)
 
+    self.assertTrue(is_generator(slow_tests))
+    self.assertTrue(is_generator(fast_tests))
+
+    slow_tests, fast_tests = list(slow_tests), list(fast_tests)
     # Verify that the components of the TestSuite are loaded.
-    self.assertIsNotNone(self.suite.tests)
+    self.assertTrue(len(slow_tests) == len(fast_tests) == 1)
+    self.assertTrue(all(test.is_slow for test in slow_tests))
+    self.assertFalse(any(test.is_slow for test in fast_tests))
     self.assertIsNotNone(self.suite.statusfile)
 
 

tools/testrunner/standard_runner.py

@@ -311,7 +311,6 @@ class StandardTestRunner(base_runner.BaseTestRunner):
       ]
 
       self._prepare_procs(procs)
-      tests.sort(key=lambda t: t.is_slow, reverse=True)
 
       loader.load_tests(tests)