[tools] Implement --exit-after-n-failures functionality via a flag

R=machenbach@chromium.org

Bug: v8:8239
Change-Id: I2451230f92fa6ad66ce6446f97efaa7fafd04e12
Reviewed-on: https://chromium-review.googlesource.com/1251524
Commit-Queue: Sergiy Byelozyorov <sergiyb@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#56337}

tools/testrunner/base_runner.py

@@ -311,6 +311,9 @@ class BaseTestRunner(object):
     parser.add_option("--junitout", help="File name of the JUnit output")
     parser.add_option("--junittestsuite", default="v8tests",
                       help="The testsuite name in the JUnit output file")
+    parser.add_option("--exit-after-n-failures", type="int", default=100,
+                      help="Exit after the first N failures instead of "
+                           "running all tests. Pass 0 to disable this feature.")
 
     # Rerun
     parser.add_option("--rerun-failures-count", default=0, type=int,
@@ -744,6 +747,9 @@ class BaseTestRunner(object):
         self.mode_options.execution_mode))
     return procs
 
+  def _create_result_tracker(self, options):
+    return progress.ResultsTracker(options.exit_after_n_failures)
+
   def _create_timeout_proc(self, options):
     if not options.total_timeout_sec:
       return None

tools/testrunner/local/pool.py

@@ -26,6 +26,9 @@ def setup_testing():
   from threading import Thread as Process
   # Monkeypatch threading Queue to look like multiprocessing Queue.
   Queue.cancel_join_thread = lambda self: None
+  # Monkeypatch os.kill and add fake pid property on Thread.
+  os.kill = lambda *args: None
+  Process.pid = property(lambda self: None)
 
 
 class NormalResult():

tools/testrunner/num_fuzzer.py

@@ -138,7 +138,7 @@ class NumFuzzer(base_runner.BaseTestRunner):
     fuzzer_rng = random.Random(options.fuzzer_random_seed)
 
     combiner = self._create_combiner(fuzzer_rng, options)
-    results = ResultsTracker()
+    results = self._create_result_tracker(options)
     execproc = ExecutionProc(options.j)
     sigproc = self._create_signal_proc()
     indicators = self._create_progress_indicators(options)

tools/testrunner/standard_runner.py

@@ -282,7 +282,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
       print '>>> Running with test processors'
       loader = LoadProc()
       tests_counter = TestsCounter()
-      results = ResultsTracker()
+      results = self._create_result_tracker(options)
       indicators = self._create_progress_indicators(options)
 
       outproc_factory = None

tools/testrunner/testproc/progress.py

@@ -32,13 +32,15 @@ class TestsCounter(base.TestProcObserver):
 
 
 class ResultsTracker(base.TestProcObserver):
-  def __init__(self):
+  """Tracks number of results and stops to run tests if max_failures reached."""
+  def __init__(self, max_failures):
     super(ResultsTracker, self).__init__()
     self._requirement = base.DROP_OUTPUT
 
     self.failed = 0
     self.remaining = 0
     self.total = 0
+    self.max_failures = max_failures
 
   def _on_next_test(self, test):
     self.total += 1
@@ -48,6 +50,9 @@ class ResultsTracker(base.TestProcObserver):
     self.remaining -= 1
     if result.has_unexpected_output:
       self.failed += 1
+      if self.max_failures and self.failed >= self.max_failures:
+        print '>>> Too many failures, exiting...'
+        self.stop()
 
 
 class ProgressIndicator(base.TestProcObserver):

tools/unittests/run_tests_test.py

@@ -684,5 +684,28 @@ class SystemTest(unittest.TestCase):
       self.assertIn('sweet/bananas', result.stdout)
       self.assertEqual(1, result.returncode, result)
 
+  def testExitAfterNFailures(self):
+    with temp_base() as basedir:
+      result = run_tests(
+          basedir,
+          '--mode=Release',
+          '--progress=verbose',
+          '--exit-after-n-failures=2',
+          '-j1',
+          'sweet/mangoes',       # PASS
+          'sweet/strawberries',  # FAIL
+          'sweet/blackberries',  # FAIL
+          'sweet/raspberries',   # should not run
+      )
+      self.assertIn('Running 4 base tests', result.stdout, result)
+      self.assertIn('sweet/mangoes: pass', result.stdout, result)
+      self.assertIn('sweet/strawberries: FAIL', result.stdout, result)
+      self.assertIn('Too many failures, exiting...', result.stdout, result)
+      self.assertIn('sweet/blackberries: FAIL', result.stdout, result)
+      self.assertNotIn('Done running sweet/raspberries', result.stdout, result)
+      self.assertIn('2 tests failed', result.stdout, result)
+      self.assertIn('3 tests ran', result.stdout, result)
+      self.assertEqual(1, result.returncode, result)
+
 if __name__ == '__main__':
   unittest.main()

tools/unittests/testdata/testroot1/test/sweet/sweet.status

@@ -6,6 +6,7 @@
 [ALWAYS, {
   'raspberries': FAIL,
   'strawberries': [PASS, ['mode == release', SLOW], ['mode == debug', NO_VARIANTS]],
+  'mangoes': [PASS, SLOW],
 
   # Both cherries and apples are to test how PASS an FAIL from different
   # sections are merged.

tools/unittests/testdata/testroot1/test/sweet/testcfg.py

@@ -12,8 +12,10 @@ from testrunner.objects import testcase
 class TestSuite(testsuite.TestSuite):
   def ListTests(self):
     return map(
-        self._create_test,
-        ['bananas', 'apples', 'cherries', 'strawberries', 'raspberries'],
+        self._create_test, [
+          'bananas', 'apples', 'cherries', 'mangoes', 'strawberries',
+          'blackberries', 'raspberries',
+        ],
     )
 
   def _test_class(self):