Skip to content

V
V8
Commit 5f7bc600 authored by Liviu Rau's avatar Liviu Rau Committed by Commit Bot
Browse files
Options

[testing] Collect all relevant data about slow tests 

Slow tests are now collected in a heap with a fixed size.
When the maximum size is reached we evict the fastest test
after adding a new test to the heap.

Bug: v8:10168
Change-Id: If3298df85d6e924451f55fe9350e293169cc849d
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2106205
Commit-Queue: Liviu Rau <liviurau@chromium.org>
Reviewed-by: 's avatarMichael Achenbach <machenbach@chromium.org>
Reviewed-by: 's avatarTamer Tas <tmrts@chromium.org>
Cr-Commit-Position: refs/heads/master@{#67572}
parent 37cdcdf4
Hide whitespace changes
Inline Side-by-side
Showing with 360 additions and 192 deletions
tools/testrunner/testproc/progress.py
View file @ 5f7bc600
......@@ -12,6 +12,7 @@ import platform
import subprocess
import sys
import time
import util
from . import base
......@@ -329,7 +330,14 @@ class JsonTestProgressIndicator(ProgressIndicator):
self.arch = arch
self.mode = mode
self.results = []
self.tests = []
self.duration_sum = 0
self.test_count = 0
def configure(self, options):
super(JsonTestProgressIndicator, self).configure(options)
self.tests = util.FixedSizeTopList(
self.options.slow_tests_cutoff,
key=lambda rec: rec['duration'])
def _on_result_for(self, test, result):
if result.is_rerun:
......@@ -341,9 +349,8 @@ class JsonTestProgressIndicator(ProgressIndicator):
for run, result in enumerate(results):
# TODO(majeski): Support for dummy/grouped results
output = result.output
# Buffer all tests for sorting the durations in the end.
# TODO(machenbach): Running average + buffer only slowest 20 tests.
self.tests.append((test, output.duration, result.cmd))
self._buffer_slow_tests(test, result, output, run)
# Omit tests that run as expected on the first try.
# Everything that happens after the first run is included in the output
......@@ -351,15 +358,36 @@ class JsonTestProgressIndicator(ProgressIndicator):
if not result.has_unexpected_output and run == 0:
continue
self.results.append({
record = self._test_record(test, result, output, run)
record.update({
"result": test.output_proc.get_outcome(output),
"stdout": output.stdout,
"stderr": output.stderr,
})
self.results.append(record)
def _buffer_slow_tests(self, test, result, output, run):
def result_value(test, result, output):
if not result.has_unexpected_output:
return ""
return test.output_proc.get_outcome(output)
record = self._test_record(test, result, output, run)
record.update({
"result": result_value(test, result, output),
"marked_slow": test.is_slow,
})
self.tests.add(record)
self.duration_sum += record['duration']
self.test_count += 1
def _test_record(self, test, result, output, run):
return {
"name": str(test),
"flags": result.cmd.args,
"command": result.cmd.to_string(relative=True),
"run": run + 1,
"stdout": output.stdout,
"stderr": output.stderr,
"exit_code": output.exit_code,
"result": test.output_proc.get_outcome(output),
"expected": test.expected_outcomes,
"duration": output.duration,
"random_seed": test.random_seed,
......@@ -367,7 +395,7 @@ class JsonTestProgressIndicator(ProgressIndicator):
"variant": test.variant,
"variant_flags": test.variant_flags,
"framework_name": self.framework_name,
})
}
def finished(self):
complete_results = []
......@@ -377,36 +405,17 @@ class JsonTestProgressIndicator(ProgressIndicator):
complete_results = json.loads(f.read() or "[]")
duration_mean = None
if self.tests:
# Get duration mean.
duration_mean = (
sum(duration for (_, duration, cmd) in self.tests) /
float(len(self.tests)))
# Sort tests by duration.
self.tests.sort(key=lambda __duration_cmd: __duration_cmd[1], reverse=True)
cutoff = self.options.slow_tests_cutoff
slowest_tests = self._test_records(self.tests[:cutoff])
if self.test_count:
duration_mean = self.duration_sum / self.test_count
complete_results.append({
"arch": self.arch,
"mode": self.mode,
"results": self.results,
"slowest_tests": slowest_tests,
"slowest_tests": self.tests.as_list(),
"duration_mean": duration_mean,
"test_total": len(self.tests),
"test_total": self.test_count,
})
with open(self.options.json_test_results, "w") as f:
f.write(json.dumps(complete_results))
def _test_records(self, tests):
return [
{
"name": str(test),
"flags": cmd.args,
"command": cmd.to_string(relative=True),
"duration": duration,
"marked_slow": test.is_slow,
} for (test, duration, cmd) in tests
]
tools/testrunner/testproc/util.py 0 → 100644
View file @ 5f7bc600
#!/usr/bin/env python
# Copyright 2020 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import heapq
import random
class FixedSizeTopList():
"""Utility collection for gathering a fixed number of elements with the
biggest value for the given key. It employs a heap from which we pop the
smallest element when the collection is 'full'.
If you need a reversed behaviour (collect min values) just provide an
inverse key."""
def __init__(self, size, key=None):
self.size = size
self.key = key or (lambda x: x)
self.data = []
self.discriminator = 0
def add(self, elem):
elem_k = self.key(elem)
heapq.heappush(self.data, (elem_k, self.extra_key(), elem))
if len(self.data) > self.size:
heapq.heappop(self.data)
def extra_key(self):
# Avoid key clash in tuples sent to the heap.
# We want to avoid comparisons on the last element of the tuple
# since those elements might not be comparable.
self.discriminator += 1
return self.discriminator
def as_list(self):
original_data = [rec for (_, _, rec) in self.data]
return sorted(original_data, key=self.key, reverse=True)
tools/testrunner/testproc/util_unittest.py 0 → 100644
View file @ 5f7bc600
#!/usr/bin/env python
# Copyright 2020 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
from util import FixedSizeTopList
import unittest
class TestOrderedFixedSizeList(unittest.TestCase):
def test_empty(self):
ofsl = FixedSizeTopList(3)
self.assertEqual(ofsl.as_list(), [])
def test_12(self):
ofsl = FixedSizeTopList(3)
ofsl.add(1)
ofsl.add(2)
self.assertEqual(ofsl.as_list(), [2,1])
def test_4321(self):
ofsl = FixedSizeTopList(3)
ofsl.add(4)
ofsl.add(3)
ofsl.add(2)
ofsl.add(1)
data = ofsl.as_list()
self.assertEqual(data, [4,3,2])
def test_544321(self):
ofsl = FixedSizeTopList(4)
ofsl.add(5)
ofsl.add(4)
ofsl.add(4)
ofsl.add(3)
ofsl.add(2)
ofsl.add(1)
data = ofsl.as_list()
self.assertEqual(data, [5, 4, 4, 3])
def test_withkey(self):
ofsl = FixedSizeTopList(3,key=lambda x: x['val'])
ofsl.add({'val':4, 'something': 'four'})
ofsl.add({'val':3, 'something': 'three'})
ofsl.add({'val':-1, 'something': 'minusone'})
ofsl.add({'val':5, 'something': 'five'})
ofsl.add({'val':0, 'something': 'zero'})
data = [e['something'] for e in ofsl.as_list()]
self.assertEqual(data, ['five', 'four', 'three'])
def test_withkeyclash(self):
# Test that a key clash does not throw exeption
ofsl = FixedSizeTopList(2,key=lambda x: x['val'])
ofsl.add({'val':2, 'something': 'two'})
ofsl.add({'val':2, 'something': 'two'})
ofsl.add({'val':0, 'something': 'zero'})
data = [e['something'] for e in ofsl.as_list()]
self.assertEqual(data, ['two', 'two'])
if __name__ == '__main__':
unittest.main()
tools/unittests/run_tests_test.py
View file @ 5f7bc600
......@@ -253,7 +253,6 @@ class SystemTest(unittest.TestCase):
# Check relevant properties of the json output.
with open(actual_json) as f:
json_output = json.load(f)[0]
pretty_json = json.dumps(json_output, indent=2, sort_keys=True)
# Replace duration in actual output as it's non-deterministic. Also
# replace the python executable prefix as it has a different absolute
......@@ -268,10 +267,15 @@ class SystemTest(unittest.TestCase):
for data in json_output['results']:
replace_variable_data(data)
json_output['duration_mean'] = 1
# We need lexicographic sorting here to avoid non-deterministic behaviour
# The original sorting key is duration, but in our fake test we have
# non-deterministic durations before we reset them to 1
json_output['slowest_tests'].sort(key= lambda x: str(x))
with open(os.path.join(TEST_DATA_ROOT, expected_results_name)) as f:
expected_test_results = json.load(f)
pretty_json = json.dumps(json_output, indent=2, sort_keys=True)
msg = None # Set to pretty_json for bootstrapping.
self.assertDictEqual(json_output, expected_test_results, msg)
......
tools/unittests/testdata/expected_test_results1.json
View file @ 5f7bc600
{
"arch": "x64",
"duration_mean": 1,
"mode": "release",
"arch": "x64",
"duration_mean": 1,
"mode": "release",
"results": [
{
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"exit_code": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"exit_code": 1,
"expected": [
"PASS"
],
],
"flags": [
"--test",
"strawberries",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"framework_name": "standard_runner",
"name": "sweet/strawberries",
"random_seed": 123,
"result": "FAIL",
"run": 1,
"stderr": "",
"stdout": "--test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner\n",
"target_name": "d8_mocked.py",
"variant": "default",
"--test",
"strawberries",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"framework_name": "standard_runner",
"name": "sweet/strawberries",
"random_seed": 123,
"result": "FAIL",
"run": 1,
"stderr": "",
"stdout": "--test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner\n",
"target_name": "d8_mocked.py",
"variant": "default",
"variant_flags": []
},
},
{
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"exit_code": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"exit_code": 1,
"expected": [
"PASS"
],
],
"flags": [
"--test",
"strawberries",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"framework_name": "standard_runner",
"name": "sweet/strawberries",
"random_seed": 123,
"result": "FAIL",
"run": 2,
"stderr": "",
"stdout": "--test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner\n",
"target_name": "d8_mocked.py",
"variant": "default",
"--test",
"strawberries",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"framework_name": "standard_runner",
"name": "sweet/strawberries",
"random_seed": 123,
"result": "FAIL",
"run": 2,
"stderr": "",
"stdout": "--test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner\n",
"target_name": "d8_mocked.py",
"variant": "default",
"variant_flags": []
},
},
{
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"exit_code": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"exit_code": 1,
"expected": [
"PASS"
],
],
"flags": [
"--test",
"strawberries",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"framework_name": "standard_runner",
"name": "sweet/strawberries",
"random_seed": 123,
"result": "FAIL",
"run": 3,
"stderr": "",
"stdout": "--test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner\n",
"target_name": "d8_mocked.py",
"variant": "default",
"--test",
"strawberries",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"framework_name": "standard_runner",
"name": "sweet/strawberries",
"random_seed": 123,
"result": "FAIL",
"run": 3,
"stderr": "",
"stdout": "--test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner\n",
"target_name": "d8_mocked.py",
"variant": "default",
"variant_flags": []
}
],
"slowest_tests": [
],
"slowest_tests": [
{
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"exit_code": 1,
"expected": [
"PASS"
],
"flags": [
"--test",
"strawberries",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"marked_slow": true,
"name": "sweet/strawberries"
},
"--test",
"strawberries",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"framework_name": "standard_runner",
"marked_slow": true,
"name": "sweet/strawberries",
"random_seed": 123,
"result": "FAIL",
"run": 1,
"target_name": "d8_mocked.py",
"variant": "default",
"variant_flags": []
},
{
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"exit_code": 1,
"expected": [
"PASS"
],
"flags": [
"--test",
"strawberries",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"marked_slow": true,
"name": "sweet/strawberries"
},
"--test",
"strawberries",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"framework_name": "standard_runner",
"marked_slow": true,
"name": "sweet/strawberries",
"random_seed": 123,
"result": "FAIL",
"run": 2,
"target_name": "d8_mocked.py",
"variant": "default",
"variant_flags": []
},
{
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"exit_code": 1,
"expected": [
"PASS"
],
"flags": [
"--test",
"strawberries",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"marked_slow": true,
"name": "sweet/strawberries"
"--test",
"strawberries",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"framework_name": "standard_runner",
"marked_slow": true,
"name": "sweet/strawberries",
"random_seed": 123,
"result": "FAIL",
"run": 3,
"target_name": "d8_mocked.py",
"variant": "default",
"variant_flags": []
}
],
],
"test_total": 3
}
}
\ No newline at end of file
tools/unittests/testdata/expected_test_results2.json
View file @ 5f7bc600
{
"arch": "x64",
"duration_mean": 1,
"mode": "release",
"arch": "x64",
"duration_mean": 1,
"mode": "release",
"results": [
{
"command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"exit_code": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"exit_code": 1,
"expected": [
"PASS"
],
],
"flags": [
"bananaflakes",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"framework_name": "standard_runner",
"name": "sweet/bananaflakes",
"random_seed": 123,
"result": "FAIL",
"run": 1,
"stderr": "",
"stdout": "bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner\n",
"target_name": "d8_mocked.py",
"variant": "default",
"bananaflakes",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"framework_name": "standard_runner",
"name": "sweet/bananaflakes",
"random_seed": 123,
"result": "FAIL",
"run": 1,
"stderr": "",
"stdout": "bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner\n",
"target_name": "d8_mocked.py",
"variant": "default",
"variant_flags": []
},
},
{
"command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"exit_code": 0,
"command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"exit_code": 0,
"expected": [
"PASS"
],
],
"flags": [
"bananaflakes",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"framework_name": "standard_runner",
"name": "sweet/bananaflakes",
"random_seed": 123,
"result": "PASS",
"run": 2,
"stderr": "",
"stdout": "bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner\n",
"target_name": "d8_mocked.py",
"variant": "default",
"bananaflakes",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"framework_name": "standard_runner",
"name": "sweet/bananaflakes",
"random_seed": 123,
"result": "PASS",
"run": 2,
"stderr": "",
"stdout": "bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner\n",
"target_name": "d8_mocked.py",
"variant": "default",
"variant_flags": []
}
],
],
"slowest_tests": [
{
"command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"exit_code": 0,
"expected": [
"PASS"
],
"flags": [
"bananaflakes",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"marked_slow": false,
"name": "sweet/bananaflakes"
},
"bananaflakes",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"framework_name": "standard_runner",
"marked_slow": false,
"name": "sweet/bananaflakes",
"random_seed": 123,
"result": "",
"run": 2,
"target_name": "d8_mocked.py",
"variant": "default",
"variant_flags": []
},
{
"command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort --testing-d8-test-runner",
"duration": 1,
"exit_code": 1,
"expected": [
"PASS"
],
"flags": [
"bananaflakes",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"marked_slow": false,
"name": "sweet/bananaflakes"
"bananaflakes",
"--random-seed=123",
"--nohard-abort",
"--testing-d8-test-runner"
],
"framework_name": "standard_runner",
"marked_slow": false,
"name": "sweet/bananaflakes",
"random_seed": 123,
"result": "FAIL",
"run": 1,
"target_name": "d8_mocked.py",
"variant": "default",
"variant_flags": []
}
],
],
"test_total": 2
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment