Revert "[testrunner] load tests concurrently into test execution processor"

This reverts commit 25457c60.

Reason for revert: https://crbug.com/v8/8731

Original change's description:
> [testrunner] load tests concurrently into test execution processor
> 
> loading every test up-front into the processing queue costs about 224MB for a
> x64 testsuite run.
> 
> This CL eliminates that overhead by utilizing generators and threading.
> 
> LoadingProc now loads test after receiving the results of the loaded tests.
> 
> R=​machenbach@chromium.org
> CC=​​yangguo@chromium.org,sergiyb@chromium.org
> 
> Bug: v8:8174
> Change-Id: I8f4e6de38430c54fe126e4504b52851866769efb
> Reviewed-on: https://chromium-review.googlesource.com/c/1420678
> Commit-Queue: Tamer Tas <tmrts@chromium.org>
> Reviewed-by: Michael Achenbach <machenbach@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#59056}

TBR=machenbach@chromium.org,tmrts@chromium.org

Change-Id: I1e074a031dced367a32a93827b9e863b0331340f
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: v8:8174
Reviewed-on: https://chromium-review.googlesource.com/c/1433792Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#59065}

tools/testrunner/standard_runner.py

@@ -281,7 +281,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
       jobs = options.j
 
       print '>>> Running with test processors'
-      loader = LoadProc(tests)
+      loader = LoadProc()
       results = self._create_result_tracker(options)
       indicators = self._create_progress_indicators(options)
 
@@ -310,7 +310,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
 
       self._prepare_procs(procs)
 
-      loader.load_initial_tests(execproc, initial_batch_size=options.j*2)
+      loader.load_tests(tests)
 
       # This starts up worker processes and blocks until all tests are
       # processed.

tools/testrunner/testproc/execution.py

@@ -48,7 +48,6 @@ class ExecutionProc(base.TestProc):
     self._pool = pool.Pool(jobs)
     self._outproc_factory = outproc_factory or (lambda t: t.output_proc)
     self._tests = {}
-    self.loaded_tests = 0
 
   def connect_to(self, next_proc):
     assert False, 'ExecutionProc cannot be connected to anything'
@@ -74,8 +73,6 @@ class ExecutionProc(base.TestProc):
     outproc = self._outproc_factory(test)
     self._pool.add([Job(test_id, cmd, outproc, test.keep_output)])
 
-    self.loaded_tests += 1
-
   def result_for(self, test, result):
     assert False, 'ExecutionProc cannot receive results'
 

tools/testrunner/testproc/loader.py

@@ -9,31 +9,19 @@ class LoadProc(base.TestProc):
   """First processor in the chain that passes all tests to the next processor.
   """
 
-  def __init__(self, tests):
-    super(LoadProc, self).__init__()
+  def load_tests(self, tests):
+    loaded = set()
+    for test in tests:
+      if test.procid in loaded:
+        print 'Warning: %s already obtained' % test.procid
+        continue
 
-    self.tests = tests
-
-  def load_initial_tests(self, exec_proc, initial_batch_size):
-    """
-    Args:
-      exec_proc: execution processor that the tests are being loaded into
-      initial_batch_size: initial number of tests to load
-    """
-    while exec_proc.loaded_tests < initial_batch_size:
-      try:
-        t = next(self.tests)
-      except StopIteration:
-        return
-
-      self._send_test(t)
+      loaded.add(test.procid)
+      self._send_test(test)
 
   def next_test(self, test):
     assert False, 'Nothing can be connected to the LoadProc'
 
   def result_for(self, test, result):
-    try:
-      self._send_test(next(self.tests))
-    except StopIteration:
-      # No more tests to load.
-      pass
+    # Ignore all results.
+    pass