Reland "[test] Add test runner system tests for flaky tests"

This is a reland of 96f55796
Original change's description:
> [test] Add test runner system tests for flaky tests
>
> This uses a temporary file in the mocked d8 script to let d8 flip
> between FAIL and PASS on rerun.
>
> This adds a separate test root with dedicated test suite for testing
> flakes to no interfere with existing status file configs.
>
> NOTRY=true
>
> Bug: v8:6917
> Change-Id: Id43753650195fb74cceb2a3ee9014100cabad546
> Reviewed-on: https://chromium-review.googlesource.com/867917
> Commit-Queue: Michael Achenbach <machenbach@chromium.org>
> Reviewed-by: Sergiy Byelozyorov <sergiyb@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#50621}

TBR=sergiyb@chromium.org
NOTRY=true

Bug: v8:6917
Change-Id: I4e7bca272dfd8778fbb8d012fcd6fd8406158e43
Reviewed-on: https://chromium-review.googlesource.com/868433Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Commit-Queue: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50625}

tools/unittests/run_tests_test.py

@@ -217,6 +217,31 @@ class SystemTest(unittest.TestCase):
       self.assertIn('Done running sweet/strawberries: FAIL', result.stdout, result)
       self.assertEqual(1, result.returncode, result)
 
+  def check_cleaned_json_output(self, expected_results_name, actual_json):
+    # Check relevant properties of the json output.
+    with open(actual_json) as f:
+      json_output = json.load(f)[0]
+      pretty_json = json.dumps(json_output, indent=2, sort_keys=True)
+
+    # Replace duration in actual output as it's non-deterministic. Also
+    # replace the python executable prefix as it has a different absolute
+    # path dependent on where this runs.
+    def replace_variable_data(data):
+      data['duration'] = 1
+      data['command'] = ' '.join(
+          ['/usr/bin/python'] + data['command'].split()[1:])
+    for data in json_output['slowest_tests']:
+      replace_variable_data(data)
+    for data in json_output['results']:
+      replace_variable_data(data)
+    json_output['duration_mean'] = 1
+
+    with open(os.path.join(TEST_DATA_ROOT, expected_results_name)) as f:
+      expected_test_results = json.load(f)
+
+    msg = None  # Set to pretty_json for bootstrapping.
+    self.assertDictEqual(json_output, expected_test_results, msg)
+
   def testFailWithRerunAndJSONProc(self):
     self.testFailWithRerunAndJSON(infra_staging=True)
 
@@ -246,33 +271,43 @@ class SystemTest(unittest.TestCase):
         self.assertIn('1 tests failed', result.stdout, result)
       self.assertEqual(0, result.returncode, result)
 
-      # Check relevant properties of the json output.
-      with open(json_path) as f:
-        json_output = json.load(f)[0]
-        pretty_json = json.dumps(json_output, indent=2, sort_keys=True)
-
-      # Replace duration in actual output as it's non-deterministic. Also
-      # replace the python executable prefix as it has a different absolute
-      # path dependent on where this runs.
-      def replace_variable_data(data):
-        data['duration'] = 1
-        data['command'] = ' '.join(
-            ['/usr/bin/python'] + data['command'].split()[1:])
-      for data in json_output['slowest_tests']:
-        replace_variable_data(data)
-      for data in json_output['results']:
-        replace_variable_data(data)
-      json_output['duration_mean'] = 1
-
-      expected_results_name = 'expected_test_results1.json'
-      with open(os.path.join(TEST_DATA_ROOT, expected_results_name)) as f:
-        expected_test_results = json.load(f)
-
       # TODO(majeski): Previously we only reported the variant flags in the
       # flags field of the test result.
       # After recent changes we report all flags, including the file names.
       # This is redundant to the command. Needs investigation.
-      self.assertEqual(json_output, expected_test_results, pretty_json)
+      self.check_cleaned_json_output('expected_test_results1.json', json_path)
+
+  # FIXME(majeski): This runs 0 tests.
+  #def testFlakeWithRerunAndJSONProc(self):
+  #  self.testFlakeWithRerunAndJSON(infra_staging=True)
+
+  def testFlakeWithRerunAndJSON(self, infra_staging=False):
+    """Test re-running a failing test and output to json."""
+    with temp_base(baseroot='testroot2') as basedir:
+      json_path = os.path.join(basedir, 'out.json')
+      result = run_tests(
+          basedir,
+          '--mode=Release',
+          '--progress=verbose',
+          '--variants=default',
+          '--rerun-failures-count=2',
+          '--random-seed=123',
+          '--json-test-results', json_path,
+          'sweet',
+          infra_staging=infra_staging,
+      )
+      self.assertIn('Running 1 tests', result.stdout, result)
+      if not infra_staging:
+        self.assertIn(
+            'Done running sweet/bananaflakes: FAIL', result.stdout, result)
+        self.assertIn('1 tests failed', result.stdout, result)
+      else:
+        # TODO(majeski): Evaluate if this output is reasonable for a FAIL, PASS.
+        self.assertIn(
+            'Done running sweet/bananaflakes: pass', result.stdout, result)
+        self.assertIn('All tests succeeded', result.stdout, result)
+      self.assertEqual(0, result.returncode, result)
+      self.check_cleaned_json_output('expected_test_results2.json', json_path)
 
   def testAutoDetect(self):
     """Fake a build with several auto-detected options.

tools/unittests/testdata/expected_test_results2.json

+{
+  "arch": "x64", 
+  "duration_mean": 1, 
+  "mode": "release", 
+  "results": [
+    {
+      "command": "/usr/bin/python out/Release/d8_mocked.py --random-seed=123 bananaflakes --nohard-abort", 
+      "duration": 1, 
+      "exit_code": 1, 
+      "expected": [
+        "PASS"
+      ], 
+      "flags": [
+        "--random-seed=123", 
+        "bananaflakes", 
+        "--nohard-abort"
+      ], 
+      "name": "sweet/bananaflakes", 
+      "random_seed": 123, 
+      "result": "FAIL", 
+      "run": 1, 
+      "stderr": "", 
+      "stdout": "--random-seed=123 bananaflakes --nohard-abort\n", 
+      "target_name": "d8_mocked.py", 
+      "variant": "default"
+    }, 
+    {
+      "command": "/usr/bin/python out/Release/d8_mocked.py --random-seed=123 bananaflakes --nohard-abort", 
+      "duration": 1, 
+      "exit_code": 0, 
+      "expected": [
+        "PASS"
+      ], 
+      "flags": [
+        "--random-seed=123", 
+        "bananaflakes", 
+        "--nohard-abort"
+      ], 
+      "name": "sweet/bananaflakes", 
+      "random_seed": 123, 
+      "result": "PASS", 
+      "run": 2, 
+      "stderr": "", 
+      "stdout": "--random-seed=123 bananaflakes --nohard-abort\n", 
+      "target_name": "d8_mocked.py", 
+      "variant": "default"
+    }
+  ], 
+  "slowest_tests": [
+    {
+      "command": "/usr/bin/python out/Release/d8_mocked.py --random-seed=123 bananaflakes --nohard-abort", 
+      "duration": 1, 
+      "flags": [
+        "--random-seed=123", 
+        "bananaflakes", 
+        "--nohard-abort"
+      ], 
+      "marked_slow": false, 
+      "name": "sweet/bananaflakes"
+    }, 
+    {
+      "command": "/usr/bin/python out/Release/d8_mocked.py --random-seed=123 bananaflakes --nohard-abort", 
+      "duration": 1, 
+      "flags": [
+        "--random-seed=123", 
+        "bananaflakes", 
+        "--nohard-abort"
+      ], 
+      "marked_slow": false, 
+      "name": "sweet/bananaflakes"
+    }
+  ], 
+  "test_total": 2
+}

tools/unittests/testdata/testroot2/d8_mocked.py

+# Copyright 2018 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Dummy d8 replacement for flaky tests.
+"""
+
+import os
+import sys
+
+PATH = os.path.dirname(os.path.abspath(__file__))
+
+print ' '.join(sys.argv[1:])
+
+# Test files ending in 'flakes' should first fail then pass. We store state in
+# a file side by side with the executable. No clean-up required as all tests
+# run in a temp test root. Restriction: Only one variant is supported for now.
+for arg in sys.argv[1:]:
+  if arg.endswith('flakes'):
+    flake_state = os.path.join(PATH, arg)
+    if os.path.exists(flake_state):
+      sys.exit(0)
+    else:
+      with open(flake_state, 'w') as f:
+        f.write('something')
+      sys.exit(1)
+
+sys.exit(0)

tools/unittests/testdata/testroot2/test/sweet/sweet.status

+# Copyright 2018 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+[
+]

tools/unittests/testdata/testroot2/test/sweet/testcfg.py

+# Copyright 2018 the V8 project authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+Dummy test suite extension with some flaky fruity tests.
+"""
+
+from testrunner.local import testsuite
+from testrunner.objects import testcase
+
+class TestSuite(testsuite.TestSuite):
+  def ListTests(self, context):
+    return map(
+        self._create_test,
+        ['bananaflakes'],
+    )
+
+  def _test_class(self):
+    return TestCase
+
+
+class TestCase(testcase.TestCase):
+  def get_shell(self):
+    return 'd8_mocked.py'
+
+  def _get_files_params(self, ctx):
+    return [self.name]
+
+def GetSuite(name, root):
+  return TestSuite(name, root)

tools/unittests/testdata/testroot2/v8_build_config.json

+{
+  "current_cpu": "x64",
+  "dcheck_always_on": false,
+  "is_asan": false,
+  "is_cfi": false,
+  "is_component_build": false,
+  "is_debug": false,
+  "is_gcov_coverage": false,
+  "is_ubsan_vptr": false,
+  "is_msan": false,
+  "is_tsan": false,
+  "target_cpu": "x64",
+  "v8_current_cpu": "x64",
+  "v8_enable_i18n_support": true,
+  "v8_enable_verify_predictable": false,
+  "v8_target_cpu": "x64",
+  "v8_use_snapshot": true
+}