[testrunner] load tests concurrently into test execution processor

loading every test up-front into the processing queue costs about 224MB for a
x64 testsuite run.

This CL eliminates that overhead by utilizing generators and threading.

LoadingProc now loads test after receiving the results of the loaded tests.

R=machenbach@chromium.org
CC=​yangguo@chromium.org,sergiyb@chromium.org

Bug: v8:8174,v8:8731
Change-Id: Ifee79c3e213da568f092de0f1623016174e9410c
Reviewed-on: https://chromium-review.googlesource.com/c/1439240
Commit-Queue: Tamer Tas <tmrts@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Reviewed-by: Sergiy Belozorov <sergiyb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#59223}

tools/testrunner/num_fuzzer.py

@@ -153,7 +153,7 @@ class NumFuzzer(base_runner.BaseTestRunner):
       execproc,
     ]
     self._prepare_procs(procs)
-    loader.load_tests(tests)
+    loader.load_tests(tests, initial_batch_size=float('inf')
 
     # TODO(majeski): maybe some notification from loader would be better?
     if combiner:

tools/testrunner/standard_runner.py

@@ -281,7 +281,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
       jobs = options.j
 
       print '>>> Running with test processors'
-      loader = LoadProc()
+      loader = LoadProc(tests)
       results = self._create_result_tracker(options)
       indicators = self._create_progress_indicators(options)
 
@@ -310,7 +310,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
 
       self._prepare_procs(procs)
 
-      loader.load_tests(tests)
+      loader.load_initial_tests(initial_batch_size=options.j*2)
 
       # This starts up worker processes and blocks until all tests are
       # processed.

tools/testrunner/testproc/loader.py

@@ -9,19 +9,34 @@ class LoadProc(base.TestProc):
   """First processor in the chain that passes all tests to the next processor.
   """
 
-  def load_tests(self, tests):
-    loaded = set()
-    for test in tests:
-      if test.procid in loaded:
-        print 'Warning: %s already obtained' % test.procid
-        continue
+  def __init__(self, tests):
+    super(LoadProc, self).__init__()
 
-      loaded.add(test.procid)
-      self._send_test(test)
+    self.tests = tests
+
+  def load_initial_tests(self, initial_batch_size):
+    """
+    Args:
+      exec_proc: execution processor that the tests are being loaded into
+      initial_batch_size: initial number of tests to load
+    """
+    loaded_tests = 0
+    while loaded_tests < initial_batch_size:
+      try:
+        t = next(self.tests)
+      except StopIteration:
+        return
+
+      if self._send_test(t):
+        loaded_tests += 1
 
   def next_test(self, test):
     assert False, 'Nothing can be connected to the LoadProc'
 
   def result_for(self, test, result):
-    # Ignore all results.
+    try:
+      while not self._send_test(next(self.tests)):
+        pass
+    except StopIteration:
+      # No more tests to load.
       pass