[testrunner] load tests concurrently into test execution processor

loading every test up-front into the processing queue costs about 224MB for a
x64 testsuite run.

This CL eliminates that overhead by utilizing generators and threading.

LoadingProc now loads test after receiving the results of the loaded tests.

R=machenbach@chromium.org
CC=​yangguo@chromium.org,sergiyb@chromium.org

Bug: v8:8174
Change-Id: I8f4e6de38430c54fe126e4504b52851866769efb
Reviewed-on: https://chromium-review.googlesource.com/c/1420678
Commit-Queue: Tamer Tas <tmrts@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#59056}

tools/testrunner/standard_runner.py

@@ -281,7 +281,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
       jobs = options.j
 
       print '>>> Running with test processors'
-      loader = LoadProc()
+      loader = LoadProc(tests)
       results = self._create_result_tracker(options)
       indicators = self._create_progress_indicators(options)
 
@@ -310,7 +310,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
 
       self._prepare_procs(procs)
 
-      loader.load_tests(tests)
+      loader.load_initial_tests(execproc, initial_batch_size=options.j*2)
 
       # This starts up worker processes and blocks until all tests are
       # processed.

tools/testrunner/testproc/execution.py

@@ -48,6 +48,7 @@ class ExecutionProc(base.TestProc):
     self._pool = pool.Pool(jobs)
     self._outproc_factory = outproc_factory or (lambda t: t.output_proc)
     self._tests = {}
+    self.loaded_tests = 0
 
   def connect_to(self, next_proc):
     assert False, 'ExecutionProc cannot be connected to anything'
@@ -73,6 +74,8 @@ class ExecutionProc(base.TestProc):
     outproc = self._outproc_factory(test)
     self._pool.add([Job(test_id, cmd, outproc, test.keep_output)])
 
+    self.loaded_tests += 1
+
   def result_for(self, test, result):
     assert False, 'ExecutionProc cannot receive results'
 

tools/testrunner/testproc/loader.py

@@ -9,19 +9,31 @@ class LoadProc(base.TestProc):
   """First processor in the chain that passes all tests to the next processor.
   """
 
-  def load_tests(self, tests):
-    loaded = set()
-    for test in tests:
-      if test.procid in loaded:
-        print 'Warning: %s already obtained' % test.procid
-        continue
+  def __init__(self, tests):
+    super(LoadProc, self).__init__()
 
-      loaded.add(test.procid)
-      self._send_test(test)
+    self.tests = tests
+
+  def load_initial_tests(self, exec_proc, initial_batch_size):
+    """
+    Args:
+      exec_proc: execution processor that the tests are being loaded into
+      initial_batch_size: initial number of tests to load
+    """
+    while exec_proc.loaded_tests < initial_batch_size:
+      try:
+        t = next(self.tests)
+      except StopIteration:
+        return
+
+      self._send_test(t)
 
   def next_test(self, test):
     assert False, 'Nothing can be connected to the LoadProc'
 
   def result_for(self, test, result):
-    # Ignore all results.
-    pass
+    try:
+      self._send_test(next(self.tests))
+    except StopIteration:
+      # No more tests to load.
+      pass