Commit 08a05c2b authored by Michal Majewski's avatar Michal Majewski Committed by Commit Bot

[test] Move data access from suite to test.

- create testcase subclass for each test
- move get_command, get_source from suite to test
- promises-aplus tests are broken
- moving expected outcomes etc. is still in progress

Bug: v8:6917
Cq-Include-Trybots: master.tryserver.v8:v8_linux_noi18n_rel_ng
Change-Id: I006e64ecf51dda95c41dff88c68b24f17a638566
Reviewed-on: https://chromium-review.googlesource.com/798331
Commit-Queue: Michał Majewski <majeski@google.com>
Reviewed-by: 's avatarMichael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50065}
parent cd99bada
......@@ -31,15 +31,15 @@ import shutil
from testrunner.local import statusfile
from testrunner.local import testsuite
from testrunner.objects import testcase
from testrunner.objects.testcase import TestCase
class BenchmarksVariantGenerator(testsuite.VariantGenerator):
# Both --noopt and --stressopt are very slow. Add TF but without
# always opt to match the way the benchmarks are run for performance
# testing.
def FilterVariantsByTest(self, testcase):
outcomes = self.suite.GetStatusFileOutcomes(testcase)
def FilterVariantsByTest(self, test):
outcomes = self.suite.GetStatusFileOutcomes(test.name, test.variant)
if statusfile.OnlyStandardVariant(outcomes):
return self.standard_variant
return self.fast_variants
......@@ -49,15 +49,12 @@ class BenchmarksVariantGenerator(testsuite.VariantGenerator):
class BenchmarksTestSuite(testsuite.TestSuite):
def __init__(self, name, root):
super(BenchmarksTestSuite, self).__init__(name, root)
self.testroot = os.path.join(root, "data")
def ListTests(self, context):
def create_test(path):
return testcase.TestCase(self, path)
tests = map(create_test, [
tests = map(self._create_test, [
"kraken/ai-astar",
"kraken/audio-beat-detection",
"kraken/audio-dft",
......@@ -114,38 +111,43 @@ class BenchmarksTestSuite(testsuite.TestSuite):
"sunspider/string-fasta",
"sunspider/string-tagcloud",
"sunspider/string-unpack-code",
"sunspider/string-validate-input"])
"sunspider/string-validate-input",
])
return tests
def GetParametersForTestCase(self, testcase, context):
def _test_class(self):
return BenchmarksTestCase
def _VariantGeneratorFactory(self):
return BenchmarksVariantGenerator
class BenchmarksTestCase(TestCase):
def _get_files_params(self, ctx):
path = self.path
testroot = self.suite.testroot
files = []
if testcase.path.startswith("kraken"):
files.append(os.path.join(self.testroot, "%s-data.js" % testcase.path))
files.append(os.path.join(self.testroot, "%s.js" % testcase.path))
elif testcase.path.startswith("octane"):
files.append(os.path.join(self.testroot, "octane/base.js"))
files.append(os.path.join(self.testroot, "%s.js" % testcase.path))
if testcase.path.startswith("octane/gbemu"):
files.append(os.path.join(self.testroot, "octane/gbemu-part2.js"))
elif testcase.path.startswith("octane/typescript"):
files.append(os.path.join(self.testroot,
if path.startswith("kraken"):
files.append(os.path.join(testroot, "%s-data.js" % path))
files.append(os.path.join(testroot, "%s.js" % path))
elif path.startswith("octane"):
files.append(os.path.join(testroot, "octane/base.js"))
files.append(os.path.join(testroot, "%s.js" % path))
if path.startswith("octane/gbemu"):
files.append(os.path.join(testroot, "octane/gbemu-part2.js"))
elif path.startswith("octane/typescript"):
files.append(os.path.join(testroot,
"octane/typescript-compiler.js"))
files.append(os.path.join(self.testroot, "octane/typescript-input.js"))
elif testcase.path.startswith("octane/zlib"):
files.append(os.path.join(self.testroot, "octane/zlib-data.js"))
files.append(os.path.join(testroot, "octane/typescript-input.js"))
elif path.startswith("octane/zlib"):
files.append(os.path.join(testroot, "octane/zlib-data.js"))
files += ["-e", "BenchmarkSuite.RunSuites({});"]
elif testcase.path.startswith("sunspider"):
files.append(os.path.join(self.testroot, "%s.js" % testcase.path))
return files, testcase.flags + context.mode_flags, {}
def GetSourceForTest(self, testcase):
filename = os.path.join(self.testroot, testcase.path + ".js")
with open(filename) as f:
return f.read()
elif path.startswith("sunspider"):
files.append(os.path.join(testroot, "%s.js" % path))
return files
def _VariantGeneratorFactory(self):
return BenchmarksVariantGenerator
def _get_source_path(self):
return os.path.join(self.suite.testroot, self.path + self._get_suffix())
def GetSuite(name, root):
......
......@@ -31,17 +31,13 @@ import shutil
from testrunner.local import command
from testrunner.local import testsuite
from testrunner.local import utils
from testrunner.objects import testcase
from testrunner.objects.testcase import TestCase
SHELL = 'cctest'
class CcTestSuite(testsuite.TestSuite):
def ListTests(self, context):
def create_test(path):
return testcase.TestCase(self, path)
shell = os.path.abspath(os.path.join(context.shell_dir, SHELL))
if utils.IsWindows():
shell += ".exe"
......@@ -55,15 +51,20 @@ class CcTestSuite(testsuite.TestSuite):
print output.stdout
print output.stderr
return []
tests = map(create_test, output.stdout.strip().split())
tests = map(self._create_test, output.stdout.strip().split())
tests.sort(key=lambda t: t.path)
return tests
def GetShellForTestCase(self, testcase):
def _test_class(self):
return CcTestCase
class CcTestCase(TestCase):
def _get_shell(self):
return SHELL
def GetParametersForTestCase(self, testcase, context):
return [testcase.path], testcase.flags + context.mode_flags, {}
def _get_files_params(self, ctx):
return [self.path]
def GetSuite(name, root):
......
......@@ -6,16 +6,12 @@ import os
import re
from testrunner.local import testsuite
from testrunner.objects import testcase
from testrunner.objects.testcase import TestCase
FILES_PATTERN = re.compile(r"//\s+Files:(.*)")
MODULE_PATTERN = re.compile(r"^// MODULE$", flags=re.MULTILINE)
class DebuggerTestSuite(testsuite.TestSuite):
def __init__(self, name, root):
super(DebuggerTestSuite, self).__init__(name, root)
def ListTests(self, context):
tests = []
for dirname, dirs, files in os.walk(self.root):
......@@ -28,19 +24,34 @@ class DebuggerTestSuite(testsuite.TestSuite):
fullpath = os.path.join(dirname, filename)
relpath = fullpath[len(self.root) + 1 : -3]
testname = relpath.replace(os.path.sep, "/")
test = testcase.TestCase(self, testname)
test = self._create_test(testname)
tests.append(test)
return tests
def GetParametersForTestCase(self, testcase, context):
flags = (
testcase.flags +
["--enable-inspector", "--allow-natives-syntax"] +
context.mode_flags
)
source = self.GetSourceForTest(testcase)
flags += self._parse_source_flags(testcase, source)
def _test_class(self):
return DebuggerTestCase
class DebuggerTestCase(TestCase):
def __init__(self, *args, **kwargs):
super(DebuggerTestCase, self).__init__(*args, **kwargs)
# precomputed
self._source_files = None
self._source_flags = None
def precompute(self):
source = self.get_source()
self._source_files = self._parse_source_files(source)
self._source_flags = self._parse_source_flags(source)
def _copy(self):
copy = super(DebuggerTestCase, self)._copy()
copy._source_files = self._source_files
copy._source_flags = self._source_flags
return copy
def _parse_source_files(self, source):
files_list = [] # List of file names to append to command arguments.
files_match = FILES_PATTERN.search(source);
# Accept several lines of 'Files:'.
......@@ -53,24 +64,30 @@ class DebuggerTestSuite(testsuite.TestSuite):
files = []
files.append(os.path.normpath(os.path.join(
self.root, "..", "mjsunit", "mjsunit.js")))
files.append(os.path.join(self.root, "test-api.js"))
files.extend([ os.path.normpath(os.path.join(self.root, '..', '..', f))
for f in files_list ])
self.suite.root, "..", "mjsunit", "mjsunit.js")))
files.append(os.path.join(self.suite.root, "test-api.js"))
files.extend([os.path.normpath(os.path.join(self.suite.root, '..', '..', f))
for f in files_list])
if MODULE_PATTERN.search(source):
files.append("--module")
files.append(os.path.join(self.root, testcase.path + self.suffix()))
files.append(os.path.join(self.suite.root, self.path + self._get_suffix()))
return files
def _get_files_params(self, ctx):
files = self._source_files
if ctx.isolates:
files = files + ['--isolate'] + files
return files
def _get_source_flags(self):
return self._source_flags
all_files = list(files)
if context.isolates:
all_files += ["--isolate"] + files
def _get_suite_flags(self, ctx):
return ['--enable-inspector', '--allow-natives-syntax']
return all_files, flags, {}
def _get_source_path(self):
return os.path.join(self.suite.root, self.path + self._get_suffix())
def GetSourceForTest(self, testcase):
filename = os.path.join(self.root, testcase.path + self.suffix())
with open(filename) as f:
return f.read()
def GetSuite(name, root):
return DebuggerTestSuite(name, root)
......@@ -5,15 +5,15 @@
import os
from testrunner.local import testsuite
from testrunner.objects import testcase
from testrunner.objects.testcase import TestCase
class FuzzerVariantGenerator(testsuite.VariantGenerator):
# Only run the fuzzer with standard variant.
def FilterVariantsByTest(self, testcase):
def FilterVariantsByTest(self, test):
return self.standard_variant
def GetFlagSets(self, testcase, variant):
def GetFlagSets(self, test, variant):
return testsuite.FAST_VARIANT_FLAGS[variant]
......@@ -24,31 +24,42 @@ class FuzzerTestSuite(testsuite.TestSuite):
'wasm_imports_section', 'wasm_memory_section', 'wasm_names_section',
'wasm_types_section' )
def __init__(self, name, root):
super(FuzzerTestSuite, self).__init__(name, root)
def ListTests(self, context):
tests = []
for subtest in FuzzerTestSuite.SUB_TESTS:
for fname in os.listdir(os.path.join(self.root, subtest)):
if not os.path.isfile(os.path.join(self.root, subtest, fname)):
continue
test = testcase.TestCase(self, '%s/%s' % (subtest, fname))
test = self._create_test('%s/%s' % (subtest, fname))
tests.append(test)
tests.sort()
return tests
def GetShellForTestCase(self, testcase):
group, _ = testcase.path.split('/', 1)
return 'v8_simple_%s_fuzzer' % group
def GetParametersForTestCase(self, testcase, context):
suite, name = testcase.path.split('/')
return [os.path.join(self.root, suite, name)], [], {}
def _test_class(self):
return FuzzerTestCase
def _VariantGeneratorFactory(self):
return FuzzerVariantGenerator
class FuzzerTestCase(TestCase):
def _get_files_params(self, ctx):
suite, name = self.path.split('/')
return [os.path.join(self.suite.root, suite, name)]
def _get_variant_flags(self):
return []
def _get_statusfile_flags(self):
return []
def _get_mode_flags(self, ctx):
return []
def _get_shell(self):
group, _ = self.path.split('/', 1)
return 'v8_simple_%s_fuzzer' % group
def GetSuite(name, root):
return FuzzerTestSuite(name, root)
......@@ -7,16 +7,13 @@ import os
from testrunner.local import testsuite
from testrunner.local import utils
from testrunner.objects import testcase
from testrunner.objects.testcase import TestCase
PROTOCOL_TEST_JS = "protocol-test.js"
EXPECTED_SUFFIX = "-expected.txt"
RESOURCES_FOLDER = "resources"
class InspectorProtocolTestSuite(testsuite.TestSuite):
def __init__(self, name, root):
super(InspectorProtocolTestSuite, self).__init__(name, root)
def ListTests(self, context):
tests = []
for dirname, dirs, files in os.walk(os.path.join(self.root), followlinks=True):
......@@ -31,26 +28,12 @@ class InspectorProtocolTestSuite(testsuite.TestSuite):
fullpath = os.path.join(dirname, filename)
relpath = fullpath[len(self.root) + 1 : -3]
testname = relpath.replace(os.path.sep, "/")
test = testcase.TestCase(self, testname)
test = self._create_test(testname)
tests.append(test)
return tests
def GetShellForTestCase(self, testcase):
return 'inspector-test'
def GetParametersForTestCase(self, testcase, context):
flags = testcase.flags + context.mode_flags
flags += self._parse_source_flags(testcase)
files = [
os.path.join(self.root, PROTOCOL_TEST_JS),
os.path.join(self.root, testcase.path + self.suffix()),
]
return files, flags, {}
def GetSourceForTest(self, testcase):
filename = os.path.join(self.root, testcase.path + self.suffix())
with open(filename) as f:
return f.read()
def _test_class(self):
return InspectorTestCase
def _IgnoreLine(self, string):
"""Ignore empty lines, valgrind output and Android output."""
......@@ -103,5 +86,37 @@ class InspectorProtocolTestSuite(testsuite.TestSuite):
return True
return False
class InspectorTestCase(TestCase):
def __init__(self, *args, **kwargs):
super(InspectorTestCase, self).__init__(*args, **kwargs)
# precomputed
self._source_flags = None
def precompute(self):
self._source_flags = self._parse_source_flags()
def _copy(self):
copy = super(InspectorTestCase, self)._copy()
copy._source_flags = self._source_flags
return copy
def _get_files_params(self, ctx):
return [
os.path.join(self.suite.root, PROTOCOL_TEST_JS),
os.path.join(self.suite.root, self.path + self._get_suffix()),
]
def _get_source_flags(self):
return self._source_flags
def _get_source_path(self):
return os.path.join(self.suite.root, self.path + self._get_suffix())
def _get_shell(self):
return 'inspector-test'
def GetSuite(name, root):
return InspectorProtocolTestSuite(name, root)
......@@ -28,13 +28,9 @@
import os
from testrunner.local import testsuite
from testrunner.objects import testcase
from testrunner.objects.testcase import TestCase
class IntlTestSuite(testsuite.TestSuite):
def __init__(self, name, root):
super(IntlTestSuite, self).__init__(name, root)
def ListTests(self, context):
tests = []
for dirname, dirs, files in os.walk(self.root):
......@@ -49,31 +45,51 @@ class IntlTestSuite(testsuite.TestSuite):
fullpath = os.path.join(dirname, filename)
relpath = fullpath[len(self.root) + 1 : -3]
testname = relpath.replace(os.path.sep, "/")
test = testcase.TestCase(self, testname)
test = self._create_test(testname)
tests.append(test)
return tests
def GetParametersForTestCase(self, testcase, context):
flags = testcase.flags + ["--allow-natives-syntax"] + context.mode_flags
flags += self._parse_source_flags(testcase)
def _test_class(self):
return IntlTestCase
class IntlTestCase(TestCase):
def __init__(self, *args, **kwargs):
super(IntlTestCase, self).__init__(*args, **kwargs)
# precomputed
self._source_flags = None
def precompute(self):
self._source_flags = self._parse_source_flags()
def _copy(self):
copy = super(IntlTestCase, self)._copy()
copy._source_flags = self._source_flags
return copy
def _get_files_params(self, ctx):
files = map(lambda f: os.path.join(self.suite.root, f), [
'assert.js',
'utils.js',
'regexp-prepare.js',
self.path + self._get_suffix(),
'regexp-assert.js',
])
if ctx.isolates:
files += ['--isolate'] + files
return files
files = []
files.append(os.path.join(self.root, "assert.js"))
files.append(os.path.join(self.root, "utils.js"))
files.append(os.path.join(self.root, "regexp-prepare.js"))
files.append(os.path.join(self.root, testcase.path + self.suffix()))
files.append(os.path.join(self.root, "regexp-assert.js"))
def _get_source_flags(self):
return self._source_flags
all_files = list(files)
if context.isolates:
all_files += ["--isolate"] + files
def _get_suite_flags(self, ctx):
return ['--allow-natives-syntax']
return all_files, flags, {}
def _get_source_path(self):
return os.path.join(self.suite.root, self.path + self._get_suffix())
def GetSourceForTest(self, testcase):
filename = os.path.join(self.root, testcase.path + self.suffix())
with open(filename) as f:
return f.read()
def GetSuite(name, root):
return IntlTestSuite(name, root)
......@@ -31,7 +31,7 @@ import re
from testrunner.local import testsuite
from testrunner.local import utils
from testrunner.objects import testcase
from testrunner.objects.testcase import TestCase
INVALID_FLAGS = ["--enable-slow-asserts"]
......@@ -39,9 +39,6 @@ MODULE_PATTERN = re.compile(r"^// MODULE$", flags=re.MULTILINE)
class MessageTestSuite(testsuite.TestSuite):
def __init__(self, name, root):
super(MessageTestSuite, self).__init__(name, root)
def ListTests(self, context):
tests = []
for dirname, dirs, files in os.walk(self.root):
......@@ -54,30 +51,17 @@ class MessageTestSuite(testsuite.TestSuite):
fullpath = os.path.join(dirname, filename)
relpath = fullpath[len(self.root) + 1 : -3]
testname = relpath.replace(os.path.sep, "/")
test = testcase.TestCase(self, testname)
test = self._create_test(testname)
tests.append(test)
return tests
def _test_class(self):
return MessageTestCase
def CreateVariantGenerator(self, variants):
return super(MessageTestSuite, self).CreateVariantGenerator(
variants + ["preparser"])
def GetParametersForTestCase(self, testcase, context):
source = self.GetSourceForTest(testcase)
files = []
if MODULE_PATTERN.search(source):
files.append("--module")
files.append(os.path.join(self.root, testcase.path + ".js"))
flags = testcase.flags + context.mode_flags
flags += self._parse_source_flags(testcase, source)
flags = [x for x in flags if x not in INVALID_FLAGS]
return files, flags, {}
def GetSourceForTest(self, testcase):
filename = os.path.join(self.root, testcase.path + self.suffix())
with open(filename) as f:
return f.read()
def _IgnoreLine(self, string):
"""Ignore empty lines, valgrind output, Android output."""
if not string: return True
......@@ -127,5 +111,44 @@ class MessageTestSuite(testsuite.TestSuite):
pass
class MessageTestCase(TestCase):
def __init__(self, *args, **kwargs):
super(MessageTestCase, self).__init__(*args, **kwargs)
# precomputed
self._source_files = None
self._source_flags = None
def precompute(self):
source = self.get_source()
self._source_files = self._parse_source_files(source)
self._source_flags = self._parse_source_flags(source)
def _copy(self):
copy = super(MessageTestCase, self)._copy()
copy._source_files = self._source_files
copy._source_flags = self._source_flags
return copy
def _parse_source_files(self, source):
files = []
if MODULE_PATTERN.search(source):
files.append("--module")
files.append(os.path.join(self.suite.root, self.path + ".js"))
return files
def _get_cmd_params(self, ctx):
params = super(MessageTestCase, self)._get_cmd_params(ctx)
return [p for p in params if p not in INVALID_FLAGS]
def _get_files_params(self, ctx):
return self._source_files
def _get_source_flags(self):
return self._source_flags
def _get_source_path(self):
return os.path.join(self.suite.root, self.path + self._get_suffix())
def GetSuite(name, root):
return MessageTestSuite(name, root)
......@@ -29,7 +29,7 @@ import os
import re
from testrunner.local import testsuite
from testrunner.objects import testcase
from testrunner.objects.testcase import TestCase
FILES_PATTERN = re.compile(r"//\s+Files:(.*)")
ENV_PATTERN = re.compile(r"//\s+Environment Variables:(.*)")
......@@ -39,10 +39,6 @@ NO_HARNESS_PATTERN = re.compile(r"^// NO HARNESS$", flags=re.MULTILINE)
class MjsunitTestSuite(testsuite.TestSuite):
def __init__(self, name, root):
super(MjsunitTestSuite, self).__init__(name, root)
def ListTests(self, context):
tests = []
for dirname, dirs, files in os.walk(self.root, followlinks=True):
......@@ -55,17 +51,27 @@ class MjsunitTestSuite(testsuite.TestSuite):
fullpath = os.path.join(dirname, filename)
relpath = fullpath[len(self.root) + 1 : -3]
testname = relpath.replace(os.path.sep, "/")
test = testcase.TestCase(self, testname)
test = self._create_test(testname)
tests.append(test)
return tests
def GetParametersForTestCase(self, testcase, context):
source = self.GetSourceForTest(testcase)
def _test_class(self):
return MjsunitTestCase
class MjsunitTestCase(TestCase):
def __init__(self, *args, **kwargs):
super(MjsunitTestCase, self).__init__(*args, **kwargs)
flags = testcase.flags + context.mode_flags
env = self._get_env(source)
# precomputed
self._source_files = None
self._source_flags = None
self._mjsunit_files = None
self._files_suffix = None
self._env = None
flags += self._parse_source_flags(testcase, source)
def precompute(self):
source = self.get_source()
files_list = [] # List of file names to append to command arguments.
files_match = FILES_PATTERN.search(source);
......@@ -76,28 +82,40 @@ class MjsunitTestSuite(testsuite.TestSuite):
files_match = FILES_PATTERN.search(source, files_match.end())
else:
break
files = [ os.path.normpath(os.path.join(self.root, '..', '..', f))
files = [ os.path.normpath(os.path.join(self.suite.root, '..', '..', f))
for f in files_list ]
testfilename = os.path.join(self.root, testcase.path + self.suffix())
testfilename = os.path.join(self.suite.root, self.path + self._get_suffix())
if SELF_SCRIPT_PATTERN.search(source):
files = (
["-e", "TEST_FILE_NAME=\"%s\"" % testfilename.replace("\\", "\\\\")] +
files)
if not context.no_harness:
files += self._get_mjsunit_files(source)
if NO_HARNESS_PATTERN.search(source):
mjsunit_files = []
else:
mjsunit_files = [os.path.join(self.suite.root, "mjsunit.js")]
files_suffix = []
if MODULE_PATTERN.search(source):
files.append("--module")
files.append(testfilename)
all_files = list(files)
if context.isolates:
all_files += ["--isolate"] + files
return all_files, flags, env
def _get_env(self, source):
files_suffix.append("--module")
files_suffix.append(testfilename)
self._source_files = files
self._source_flags = self._parse_source_flags(source)
self._mjsunit_files = mjsunit_files
self._files_suffix = files_suffix
self._env = self._parse_source_env(source)
def _copy(self):
copy = super(MjsunitTestCase, self)._copy()
copy._source_files = self._source_files
copy._source_flags = self._source_flags
copy._mjsunit_files = self._mjsunit_files
copy._files_suffix = self._files_suffix
copy._env = self._env
return copy
def _parse_source_env(self, source):
env_match = ENV_PATTERN.search(source)
env = {}
if env_match:
......@@ -106,16 +124,24 @@ class MjsunitTestSuite(testsuite.TestSuite):
env[var] = value
return env
def _get_mjsunit_files(self, source):
if NO_HARNESS_PATTERN.search(source):
return []
else:
return [os.path.join(self.root, 'mjsunit.js')]
def _get_source_flags(self):
return self._source_flags
def _get_files_params(self, ctx):
files = list(self._source_files)
if not ctx.no_harness:
files += self._mjsunit_files
files += self._files_suffix
if ctx.isolates:
files += ['--isolate'] + files
return files
def _get_cmd_env(self):
return self._env
def GetSourceForTest(self, testcase):
filename = os.path.join(self.root, testcase.path + self.suffix())
with open(filename) as f:
return f.read()
def _get_source_path(self):
return os.path.join(self.suite.root, self.path + self._get_suffix())
def GetSuite(name, root):
......
......@@ -6,25 +6,18 @@ import os
import difflib
from testrunner.local import testsuite
from testrunner.objects import testcase
from testrunner.objects.testcase import TestCase
SHELL = 'mkgrokdump'
class MkGrokdump(testsuite.TestSuite):
def __init__(self, name, root):
super(MkGrokdump, self).__init__(name, root)
def ListTests(self, context):
test = testcase.TestCase(self, SHELL)
test = self._create_test(SHELL)
return [test]
def GetShellForTestCase(self, testcase):
return SHELL
def GetParametersForTestCase(self, testcase, context):
return [], [], {}
def _test_class(self):
return MkGrokTestCase
def IsFailureOutput(self, testcase):
output = testcase.output
......@@ -47,5 +40,20 @@ class MkGrokdump(testsuite.TestSuite):
return True
return False
class MkGrokTestCase(TestCase):
def _get_variant_flags(self):
return []
def _get_statusfile_flags(self):
return []
def _get_mode_flags(self, ctx):
return []
def _get_shell(self):
return SHELL
def GetSuite(name, root):
return MkGrokdump(name, root)
......@@ -29,7 +29,7 @@
import os
from testrunner.local import testsuite
from testrunner.objects import testcase
from testrunner.objects.testcase import TestCase
EXCLUDED = ["CVS", ".svn"]
......@@ -55,7 +55,6 @@ TEST_DIRS = """
class MozillaTestSuite(testsuite.TestSuite):
def __init__(self, name, root):
super(MozillaTestSuite, self).__init__(name, root)
self.testroot = os.path.join(root, "data")
......@@ -77,36 +76,42 @@ class MozillaTestSuite(testsuite.TestSuite):
fullpath = os.path.join(dirname, filename)
relpath = fullpath[len(self.testroot) + 1 : -3]
testname = relpath.replace(os.path.sep, "/")
case = testcase.TestCase(self, testname)
case = self._create_test(testname)
tests.append(case)
return tests
def GetParametersForTestCase(self, testcase, context):
files = [os.path.join(self.root, "mozilla-shell-emulation.js")]
testfilename = testcase.path + ".js"
def _test_class(self):
return MozillaTestCase
def IsNegativeTest(self, testcase):
return testcase.path.endswith("-n")
def IsFailureOutput(self, testcase):
if testcase.output.exit_code != 0:
return True
return "FAILED!" in testcase.output.stdout
class MozillaTestCase(TestCase):
def _get_files_params(self, ctx):
files = [os.path.join(self.suite.root, "mozilla-shell-emulation.js")]
testfilename = self.path + ".js"
testfilepath = testfilename.split("/")
for i in xrange(len(testfilepath)):
script = os.path.join(self.testroot,
script = os.path.join(self.suite.testroot,
reduce(os.path.join, testfilepath[:i], ""),
"shell.js")
if os.path.exists(script):
files.append(script)
files.append(os.path.join(self.testroot, testfilename))
flags = testcase.flags + context.mode_flags + ["--expose-gc"]
return files, flags, {}
def GetSourceForTest(self, testcase):
filename = os.path.join(self.testroot, testcase.path + ".js")
with open(filename) as f:
return f.read()
files.append(os.path.join(self.suite.testroot, testfilename))
return files
def IsNegativeTest(self, testcase):
return testcase.path.endswith("-n")
def _get_suite_flags(self, ctx):
return ['--expose-gc']
def IsFailureOutput(self, testcase):
if testcase.output.exit_code != 0:
return True
return "FAILED!" in testcase.output.stdout
def _get_source_path(self):
return os.path.join(self.suite.testroot, self.path + self._get_suffix())
def GetSuite(name, root):
......
......@@ -29,23 +29,20 @@
import os
from testrunner.local import testsuite
from testrunner.objects import testcase
from testrunner.objects.testcase import TestCase
class PreparserTestSuite(testsuite.TestSuite):
def __init__(self, name, root):
super(PreparserTestSuite, self).__init__(name, root)
def _ParsePythonTestTemplates(self, result, filename):
pathname = os.path.join(self.root, filename + ".pyt")
def Test(name, source, expectation, template_flags=[]):
def Test(name, source, expectation):
source = source.replace("\n", " ")
testname = os.path.join(filename, name)
flags = ["-e", source]
path = os.path.join(filename, name)
if expectation:
flags += ["--throws"]
flags += template_flags
test = testcase.TestCase(self, testname, flags=flags)
template_flags = ["--throws"]
else:
template_flags = []
test = self._create_test(path, source, template_flags)
result.append(test)
def Template(name, source):
def MkTest(replacement, expectation):
......@@ -68,16 +65,48 @@ class PreparserTestSuite(testsuite.TestSuite):
self._ParsePythonTestTemplates(result, f)
return result
def GetParametersForTestCase(self, testcase, context):
return [], testcase.flags, {}
def _create_test(self, path, source, template_flags):
return super(PreparserTestSuite, self)._create_test(
path, source=source, template_flags=template_flags)
def GetSourceForTest(self, testcase):
assert testcase.flags[0] == "-e"
return testcase.flags[1]
def _test_class(self):
return PreparserTestCase
def _VariantGeneratorFactory(self):
return testsuite.StandardVariantGenerator
class PreparserTestCase(TestCase):
def __init__(self, suite, path, name, source, template_flags):
super(PreparserTestCase, self).__init__(suite, path, name)
self._source = source
self._template_flags = template_flags
def _copy(self):
return self.__class__(
self.suite, self.path, self.name, self._source, self._template_flags)
def _get_cmd_params(self, ctx):
return (
self._get_files_params(ctx) +
self._get_extra_flags(ctx) +
['-e', self._source] +
self._template_flags +
self._get_variant_flags() +
self._get_statusfile_flags() +
self._get_mode_flags(ctx) +
self._get_source_flags()
)
def _get_mode_flags(self, ctx):
return []
def is_source_available(self):
return True
def get_source(self):
return self._source
def GetSuite(name, root):
return PreparserTestSuite(name, root)
......@@ -63,22 +63,28 @@ class PromiseAplusTestSuite(testsuite.TestSuite):
['run-tests.js']
]
def CommonTestName(self, testcase):
return testcase.path.split(os.path.sep)[-1]
def ListTests(self, context):
return [testcase.TestCase(self, fname[:-len('.js')]) for fname in
return [self._create_test(fname[:-len('.js')]) for fname in
os.listdir(os.path.join(self.root, TEST_NAME, 'lib', 'tests'))
if fname.endswith('.js')]
def GetParametersForTestCase(self, testcase, context):
files = (
def _get_cmd_params(self, test, ctx):
return (
self._get_files_params(test, ctx) +
self._get_extra_flags(test, ctx) +
self._get_user_flags(test) +
self._get_statusfile_flags(test) +
self._get_mode_flags(test, ctx) +
self._get_source_flags(test) +
['--allow-natives-syntax']
)
def _get_files_params(self, test, ctx):
return (
self.helper_files_pre +
[os.path.join(self.test_files_root, testcase.path + '.js')] +
[os.path.join(self.test_files_root, test.path + '.js')] +
self.helper_files_post
)
flags = testcase.flags + context.mode_flags + ['--allow-natives-syntax']
return files, flags, {}
def GetSourceForTest(self, testcase):
filename = os.path.join(self.root, TEST_NAME,
......@@ -95,5 +101,9 @@ class PromiseAplusTestSuite(testsuite.TestSuite):
return not 'All tests have run.' in testcase.output.stdout or \
'FAIL:' in testcase.output.stdout
def _path_to_name(self, path):
return path.split(os.path.sep)[-1]
def GetSuite(name, root):
return PromiseAplusTestSuite(name, root)
......@@ -37,7 +37,7 @@ import tarfile
from testrunner.local import statusfile
from testrunner.local import testsuite
from testrunner.local import utils
from testrunner.objects import testcase
from testrunner.objects.testcase import TestCase
# TODO(littledan): move the flag mapping into the status file
FEATURE_FLAGS = {
......@@ -103,14 +103,14 @@ FAST_VARIANTS = {
}
class Test262VariantGenerator(testsuite.VariantGenerator):
def GetFlagSets(self, testcase, variant):
outcomes = testcase.suite.GetStatusFileOutcomes(testcase)
def GetFlagSets(self, test, variant):
outcomes = test.suite.GetStatusFileOutcomes(test.name, test.variant)
if outcomes and statusfile.OnlyFastVariants(outcomes):
variant_flags = FAST_VARIANTS
else:
variant_flags = ALL_VARIANTS
test_record = self.suite.GetTestRecord(testcase)
test_record = test.test_record
if "noStrict" in test_record:
return variant_flags["nostrict"][variant]
if "onlyStrict" in test_record:
......@@ -130,9 +130,9 @@ class Test262TestSuite(testsuite.TestSuite):
for f in TEST_262_HARNESS_FILES]
self.harness += [os.path.join(self.root, "harness-adapt.js")]
self.localtestroot = os.path.join(self.root, *TEST_262_LOCAL_TESTS_PATH)
self.ParseTestRecord = None
self._extract_sources()
self._parse_test_record = self._load_parse_test_record()
self.parse_test_record = self._load_parse_test_record()
def _extract_sources(self):
# The archive is created only on swarming. Local checkouts have the
......@@ -157,8 +157,9 @@ class Test262TestSuite(testsuite.TestSuite):
module = imp.load_module("parseTestRecord", f, pathname, description)
return module.parseTestRecord
except:
raise ImportError("Cannot load parseTestRecord; you may need to "
"gclient sync for test262")
print ('Cannot load parseTestRecord; '
'you may need to gclient sync for test262')
raise
finally:
if f:
f.close()
......@@ -181,74 +182,17 @@ class Test262TestSuite(testsuite.TestSuite):
fullpath = os.path.join(dirname, filename)
relpath = re.match(TEST_262_RELPATH_REGEXP, fullpath).group(1)
testnames.add(relpath.replace(os.path.sep, "/"))
cases = [testcase.TestCase(self, testname) for testname in testnames]
cases = map(self._create_test, testnames)
return [case for case in cases if len(
SKIPPED_FEATURES.intersection(
self.GetTestRecord(case).get("features", []))) == 0]
def GetParametersForTestCase(self, testcase, context):
files = (
list(self.harness) +
([os.path.join(self.root, "harness-agent.js")]
if testcase.path.startswith('built-ins/Atomics') else []) +
self.GetIncludesForTest(testcase) +
(["--module"] if "module" in self.GetTestRecord(testcase) else []) +
[self.GetPathForTest(testcase)]
)
flags = (
testcase.flags + context.mode_flags +
(["--throws"] if "negative" in self.GetTestRecord(testcase)
else []) +
(["--allow-natives-syntax"]
if "detachArrayBuffer.js" in
self.GetTestRecord(testcase).get("includes", [])
else []) +
([flag for (feature, flag) in FEATURE_FLAGS.items()
if feature in self.GetTestRecord(testcase).get("features", [])])
)
return files, flags, {}
case.test_record.get("features", []))) == 0]
def _VariantGeneratorFactory(self):
return Test262VariantGenerator
def GetTestRecord(self, testcase):
if not hasattr(testcase, "test_record"):
testcase.test_record = self._parse_test_record(
self.GetSourceForTest(testcase), testcase.path)
return testcase.test_record
def BasePath(self, filename):
return self.root if filename in TEST_262_NATIVE_FILES else self.harnesspath
def GetIncludesForTest(self, testcase):
test_record = self.GetTestRecord(testcase)
return [os.path.join(self.BasePath(filename), filename)
for filename in test_record.get("includes", [])]
def GetPathForTest(self, testcase):
filename = os.path.join(self.localtestroot, testcase.path + ".js")
if not os.path.exists(filename):
filename = os.path.join(self.testroot, testcase.path + ".js")
return filename
def GetSourceForTest(self, testcase):
with open(self.GetPathForTest(testcase)) as f:
return f.read()
def _ParseException(self, str, testcase):
# somefile:somelinenumber: someerror[: sometext]
# somefile might include an optional drive letter on windows e.g. "e:".
match = re.search(
'^(?:\w:)?[^:]*:[0-9]+: ([^: ]+?)($|: )', str, re.MULTILINE)
if match:
return match.group(1).strip()
else:
print "Error parsing exception for %s" % testcase.GetLabel()
return None
def _test_class(self):
return Test262TestCase
def IsFailureOutput(self, testcase):
output = testcase.output
test_record = self.GetTestRecord(testcase)
test_record = testcase.test_record
if output.exit_code != 0:
return True
if ("negative" in test_record and
......@@ -258,12 +202,80 @@ class Test262TestSuite(testsuite.TestSuite):
return True
return "FAILED!" in output.stdout
def GetExpectedOutcomes(self, testcase):
outcomes = self.GetStatusFileOutcomes(testcase)
def _ParseException(self, str, testcase):
# somefile:somelinenumber: someerror[: sometext]
# somefile might include an optional drive letter on windows e.g. "e:".
match = re.search(
'^(?:\w:)?[^:]*:[0-9]+: ([^: ]+?)($|: )', str, re.MULTILINE)
if match:
return match.group(1).strip()
else:
print "Error parsing exception for %s" % testcase
return None
def GetExpectedOutcomes(self, test):
outcomes = self.GetStatusFileOutcomes(test.name, test.variant)
if (statusfile.FAIL_SLOPPY in outcomes and
'--use-strict' not in testcase.flags):
'--use-strict' not in test.cmd.args):
return [statusfile.FAIL]
return super(Test262TestSuite, self).GetExpectedOutcomes(testcase)
return super(Test262TestSuite, self).GetExpectedOutcomes(test)
def _VariantGeneratorFactory(self):
return Test262VariantGenerator
class Test262TestCase(TestCase):
def __init__(self, *args, **kwargs):
super(Test262TestCase, self).__init__(*args, **kwargs)
# precomputed
self.test_record = None
def precompute(self):
source = self.get_source()
self.test_record = self.suite.parse_test_record(source, self.path)
def _copy(self):
copy = super(Test262TestCase, self)._copy()
copy.test_record = self.test_record
return copy
def _get_files_params(self, ctx):
return (
list(self.suite.harness) +
([os.path.join(self.suite.root, "harness-agent.js")]
if self.path.startswith('built-ins/Atomics') else []) +
self._get_includes() +
(["--module"] if "module" in self.test_record else []) +
[self._get_source_path()]
)
def _get_suite_flags(self, ctx):
return (
(["--throws"] if "negative" in self.test_record else []) +
(["--allow-natives-syntax"]
if "detachArrayBuffer.js" in self.test_record.get("includes", [])
else []) +
[flag for (feature, flag) in FEATURE_FLAGS.items()
if feature in self.test_record.get("features", [])]
)
def _get_includes(self):
return [os.path.join(self._base_path(filename), filename)
for filename in self.test_record.get("includes", [])]
def _base_path(self, filename):
if filename in TEST_262_NATIVE_FILES:
return self.suite.root
else:
return self.suite.harnesspath
def _get_source_path(self):
filename = self.path + self._get_suffix()
path = os.path.join(self.suite.localtestroot, filename)
if os.path.exists(path):
return path
return os.path.join(self.suite.testroot, filename)
def GetSuite(name, root):
......
......@@ -7,16 +7,12 @@ import os
from testrunner.local import command
from testrunner.local import utils
from testrunner.local.testsuite import TestSuite, StandardVariantGenerator
from testrunner.objects import testcase
from testrunner.objects.testcase import TestCase
class GoogleTestSuite(TestSuite):
def __init__(self, name, root):
super(GoogleTestSuite, self).__init__(name, root)
def ListTests(self, context):
shell = os.path.abspath(
os.path.join(context.shell_dir, self.GetShellForTestCase(None)))
shell = os.path.abspath(os.path.join(context.shell_dir, self.name))
if utils.IsWindows():
shell += ".exe"
......@@ -46,25 +42,28 @@ class GoogleTestSuite(TestSuite):
if test_desc.endswith('.'):
test_case = test_desc
elif test_case and test_desc:
test = testcase.TestCase(self, test_case + test_desc)
tests.append(test)
test_path = test_case + test_desc
tests.append(self._create_test(test_path))
tests.sort(key=lambda t: t.path)
return tests
def GetParametersForTestCase(self, testcase, context):
flags = (
testcase.flags +
["--gtest_filter=" + testcase.path] +
["--gtest_random_seed=%s" % context.random_seed] +
["--gtest_print_time=0"] +
context.mode_flags)
return [], flags, {}
def _test_class(self):
return GoogleTestCase
def _VariantGeneratorFactory(self):
return StandardVariantGenerator
def GetShellForTestCase(self, testcase):
return self.name
class GoogleTestCase(TestCase):
def _get_suite_flags(self, ctx):
return (
["--gtest_filter=" + self.path] +
["--gtest_random_seed=%s" % ctx.random_seed] +
["--gtest_print_time=0"]
)
def _get_shell(self):
return self.suite.name
def GetSuite(name, root):
......
......@@ -5,12 +5,9 @@
import os
from testrunner.local import testsuite
from testrunner.objects import testcase
class WasmSpecTestsTestSuite(testsuite.TestSuite):
def __init__(self, name, root):
super(WasmSpecTestsTestSuite, self).__init__(name, root)
from testrunner.objects.testcase import TestCase
class WasmSpecTestSuite(testsuite.TestSuite):
def ListTests(self, context):
tests = []
for dirname, dirs, files in os.walk(self.root):
......@@ -21,15 +18,18 @@ class WasmSpecTestsTestSuite(testsuite.TestSuite):
fullpath = os.path.join(dirname, filename)
relpath = fullpath[len(self.root) + 1 : -3]
testname = relpath.replace(os.path.sep, "/")
test = testcase.TestCase(self, testname)
test = self._create_test(testname)
tests.append(test)
return tests
def GetParametersForTestCase(self, testcase, context):
flags = testcase.flags + context.mode_flags
files = [os.path.join(self.root, testcase.path + self.suffix())]
return files, flags, {}
def _test_class(self):
return WasmSpecTestCase
class WasmSpecTestCase(TestCase):
def _get_files_params(self, ctx):
return [os.path.join(self.suite.root, self.path + self._get_suffix())]
def GetSuite(name, root):
return WasmSpecTestsTestSuite(name, root)
return WasmSpecTestSuite(name, root)
......@@ -30,7 +30,7 @@ import os
import re
from testrunner.local import testsuite
from testrunner.objects import testcase
from testrunner.objects.testcase import TestCase
FILES_PATTERN = re.compile(r"//\s+Files:(.*)")
SELF_SCRIPT_PATTERN = re.compile(r"//\s+Env: TEST_FILE_NAME")
......@@ -38,10 +38,6 @@ SELF_SCRIPT_PATTERN = re.compile(r"//\s+Env: TEST_FILE_NAME")
# TODO (machenbach): Share commonalities with mjstest.
class WebkitTestSuite(testsuite.TestSuite):
def __init__(self, name, root):
super(WebkitTestSuite, self).__init__(name, root)
def ListTests(self, context):
tests = []
for dirname, dirs, files in os.walk(self.root):
......@@ -57,44 +53,12 @@ class WebkitTestSuite(testsuite.TestSuite):
fullpath = os.path.join(dirname, filename)
relpath = fullpath[len(self.root) + 1 : -3]
testname = relpath.replace(os.path.sep, "/")
test = testcase.TestCase(self, testname)
test = self._create_test(testname)
tests.append(test)
return tests
def GetParametersForTestCase(self, testcase, context):
source = self.GetSourceForTest(testcase)
flags = testcase.flags + context.mode_flags
flags += self._parse_source_flags(testcase, source)
files_list = [] # List of file names to append to command arguments.
files_match = FILES_PATTERN.search(source);
# Accept several lines of 'Files:'.
while True:
if files_match:
files_list += files_match.group(1).strip().split()
files_match = FILES_PATTERN.search(source, files_match.end())
else:
break
files = [ os.path.normpath(os.path.join(self.root, '..', '..', f))
for f in files_list ]
testfilename = os.path.join(self.root, testcase.path + self.suffix())
if SELF_SCRIPT_PATTERN.search(source):
env = ["-e", "TEST_FILE_NAME=\"%s\"" % testfilename.replace("\\", "\\\\")]
files = env + files
files.append(os.path.join(self.root, "resources/standalone-pre.js"))
files.append(testfilename)
files.append(os.path.join(self.root, "resources/standalone-post.js"))
all_files = list(files)
if context.isolates:
all_files += ["--isolate"] + files
return all_files, flags, {}
def GetSourceForTest(self, testcase):
filename = os.path.join(self.root, testcase.path + self.suffix())
with open(filename) as f:
return f.read()
def _test_class(self):
return WebkitTestCase
# TODO(machenbach): Share with test/message/testcfg.py
def _IgnoreLine(self, string):
......@@ -152,5 +116,58 @@ class WebkitTestSuite(testsuite.TestSuite):
return False
class WebkitTestCase(TestCase):
def __init__(self, *args, **kwargs):
super(WebkitTestCase, self).__init__(*args, **kwargs)
# precomputed
self._source_files = None
self._source_flags = None
def precompute(self):
source = self.get_source()
self._source_files = self._parse_source_files(source)
self._source_flags = self._parse_source_flags(source)
def _copy(self):
copy = super(WebkitTestCase, self)._copy()
copy._source_files = self._source_files
copy._source_flags = self._source_flags
return copy
def _parse_source_files(self, source):
files_list = [] # List of file names to append to command arguments.
files_match = FILES_PATTERN.search(source);
# Accept several lines of 'Files:'.
while True:
if files_match:
files_list += files_match.group(1).strip().split()
files_match = FILES_PATTERN.search(source, files_match.end())
else:
break
files = [ os.path.normpath(os.path.join(self.suite.root, '..', '..', f))
for f in files_list ]
testfilename = os.path.join(self.suite.root, self.path + self._get_suffix())
if SELF_SCRIPT_PATTERN.search(source):
env = ["-e", "TEST_FILE_NAME=\"%s\"" % testfilename.replace("\\", "\\\\")]
files = env + files
files.append(os.path.join(self.suite.root, "resources/standalone-pre.js"))
files.append(testfilename)
files.append(os.path.join(self.suite.root, "resources/standalone-post.js"))
return files
def _get_files_params(self, ctx):
files = self._source_files
if ctx.isolates:
files = files + ['--isolate'] + files
return files
def _get_source_flags(self):
return self._source_flags
def _get_source_path(self):
return os.path.join(self.suite.root, self.path + self._get_suffix())
def GetSuite(name, root):
return WebkitTestSuite(name, root)
......@@ -305,17 +305,15 @@ class DeoptFuzzer(base_runner.BaseTestRunner):
if len(args) > 0:
s.FilterTestCasesByArgs(args)
s.FilterTestCasesByStatus(False)
for t in s.tests:
t.flags += s.GetStatusfileFlags(t)
test_backup[s] = s.tests
analysis_flags = ["--deopt-every-n-times", "%d" % MAX_DEOPT,
"--print-deopt-stress"]
s.tests = [t.CopyAddingFlags(t.variant, analysis_flags) for t in s.tests]
s.tests = [t.create_variant(t.variant, analysis_flags) for t in s.tests]
num_tests += len(s.tests)
for t in s.tests:
t.id = test_id
t.cmd = s.GetCommand(t, ctx)
t.cmd = t.get_command(ctx)
test_id += 1
if num_tests == 0:
......@@ -360,11 +358,11 @@ class DeoptFuzzer(base_runner.BaseTestRunner):
print "%s %s" % (t.path, distribution)
for i in distribution:
fuzzing_flags = ["--deopt-every-n-times", "%d" % i]
s.tests.append(t.CopyAddingFlags(t.variant, fuzzing_flags))
s.tests.append(t.create_variant(t.variant, fuzzing_flags))
num_tests += len(s.tests)
for t in s.tests:
t.id = test_id
t.cmd = s.GetCommand(t, ctx)
t.cmd = t.get_command(ctx)
test_id += 1
if num_tests == 0:
......
......@@ -199,10 +199,10 @@ class GCFuzzer(base_runner.BaseTestRunner):
'--stress_marking', '1000',
'--trace_incremental_marking',
]
s.tests = map(lambda t: t.CopyAddingFlags(t.variant, analysis_flags),
s.tests = map(lambda t: t.create_variant(t.variant, analysis_flags),
s.tests)
for t in s.tests:
t.cmd = s.GetCommand(t, ctx)
t.cmd = t.get_command(ctx)
progress_indicator = progress.PROGRESS_INDICATORS[options.progress]()
runner = execution.Runner(suites, progress_indicator, ctx)
......@@ -247,10 +247,9 @@ class GCFuzzer(base_runner.BaseTestRunner):
]
if options.stress_compaction:
fuzzing_flags.append('--stress_compaction_random')
s.tests.append(t.CopyAddingFlags(t.variant, fuzzing_flags))
s.tests.append(t.create_variant(t.variant, fuzzing_flags))
for t in s.tests:
t.cmd = s.GetCommand(t, ctx)
t.cmd = t.get_command(ctx)
num_tests += len(s.tests)
if num_tests == 0:
......@@ -297,8 +296,6 @@ class GCFuzzer(base_runner.BaseTestRunner):
if len(args) > 0:
s.FilterTestCasesByArgs(args)
s.FilterTestCasesByStatus(False)
for t in s.tests:
t.flags += s.GetStatusfileFlags(t)
num_tests += len(s.tests)
for t in s.tests:
......
......@@ -122,7 +122,8 @@ class Runner(object):
self.suite_names = [s.name for s in suites]
# Always pre-sort by status file, slowest tests first.
slow_key = lambda t: statusfile.IsSlow(t.suite.GetStatusFileOutcomes(t))
slow_key = lambda t: statusfile.IsSlow(
t.suite.GetStatusFileOutcomes(t.name, t.variant))
self.tests.sort(key=slow_key, reverse=True)
# Sort by stored duration of not opted out.
......
......@@ -34,9 +34,10 @@ class JUnitTestOutput:
self.root = xml.Element("testsuite")
self.root.attrib["name"] = test_suite_name
def HasRunTest(self, test_name, test_duration, test_failure):
def HasRunTest(self, test_name, test_cmd, test_duration, test_failure):
testCaseElement = xml.Element("testcase")
testCaseElement.attrib["name"] = " ".join(test_name)
testCaseElement.attrib["name"] = test_name
testCaseElement.attrib["cmd"] = test_cmd
testCaseElement.attrib["time"] = str(round(test_duration, 3))
if len(test_failure):
failureElement = xml.Element("failure")
......@@ -44,5 +45,5 @@ class JUnitTestOutput:
testCaseElement.append(failureElement)
self.root.append(testCaseElement)
def FinishAndWrite(self, file):
xml.ElementTree(self.root).write(file, "UTF-8")
def FinishAndWrite(self, f):
xml.ElementTree(self.root).write(f, "UTF-8")
......@@ -62,21 +62,16 @@ class PerfDataStore(object):
self.database.close()
self.closed = True
def GetKey(self, test):
"""Computes the key used to access data for the given testcase."""
flags = "".join(test.flags)
return str("%s.%s.%s" % (test.suitename(), test.path, flags))
def FetchPerfData(self, test):
"""Returns the observed duration for |test| as read from the store."""
key = self.GetKey(test)
key = test.get_id()
if key in self.database:
return self.database[key].avg
return None
def UpdatePerfData(self, test):
"""Updates the persisted value in the store with test.duration."""
testkey = self.GetKey(test)
testkey = test.get_id()
self.RawUpdatePerfData(testkey, test.duration)
def RawUpdatePerfData(self, testkey, duration):
......
......@@ -62,8 +62,8 @@ class ProgressIndicator(object):
else:
negative_marker = ''
print "=== %(label)s %(negative)s===" % {
'label': test.GetLabel(),
'negative': negative_marker
'label': test,
'negative': negative_marker,
}
......@@ -136,7 +136,7 @@ class VerboseProgressIndicator(SimpleProgressIndicator):
outcome = 'FAIL'
else:
outcome = 'pass'
print 'Done running %s: %s' % (test.GetLabel(), outcome)
print 'Done running %s: %s' % (test, outcome)
sys.stdout.flush()
def Heartbeat(self):
......@@ -179,7 +179,7 @@ class CompactProgressIndicator(ProgressIndicator):
print "" # Line break.
def HasRun(self, test, has_unexpected_output):
self.PrintProgress(test.GetLabel())
self.PrintProgress(str(test))
if has_unexpected_output:
self.ClearLine(self.last_status_length)
self.PrintFailureHeader(test)
......@@ -284,9 +284,10 @@ class JUnitTestProgressIndicator(ProgressIndicator):
if test.output.HasTimedOut():
fail_text += "--- TIMEOUT ---"
self.outputter.HasRunTest(
[test.GetLabel()] + self.runner.context.mode_flags + test.flags,
test.duration,
fail_text)
test_name=str(test),
test_cmd=test.cmd.to_string(relative=True),
test_duration=test.duration,
test_failure=fail_text)
class JsonTestProgressIndicator(ProgressIndicator):
......@@ -318,12 +319,12 @@ class JsonTestProgressIndicator(ProgressIndicator):
timed_tests.sort(lambda a, b: cmp(b.duration, a.duration))
slowest_tests = [
{
"name": test.GetLabel(),
"flags": test.flags,
"name": str(test),
"flags": test.cmd.args,
"command": test.cmd.to_string(relative=True),
"duration": test.duration,
"marked_slow": statusfile.IsSlow(
test.suite.GetStatusFileOutcomes(test)),
test.suite.GetStatusFileOutcomes(test.name, test.variant)),
} for test in timed_tests[:20]
]
......@@ -348,8 +349,8 @@ class JsonTestProgressIndicator(ProgressIndicator):
return
self.results.append({
"name": test.GetLabel(),
"flags": test.flags,
"name": str(test),
"flags": test.cmd.args,
"command": test.cmd.to_string(relative=True),
"run": test.run,
"stdout": test.output.stdout,
......@@ -362,7 +363,7 @@ class JsonTestProgressIndicator(ProgressIndicator):
# TODO(machenbach): This stores only the global random seed from the
# context and not possible overrides when using random-seed stress.
"random_seed": self.random_seed,
"target_name": test.suite.GetShellForTestCase(test),
"target_name": test.cmd.shell,
"variant": test.variant,
})
......@@ -393,11 +394,7 @@ class FlakinessTestProgressIndicator(ProgressIndicator):
}, f)
def HasRun(self, test, has_unexpected_output):
key = "/".join(
sorted(flag.lstrip("-")
for flag in self.runner.context.extra_flags + test.flags) +
["test", test.GetLabel()],
)
key = test.get_id()
outcome = test.suite.GetOutcome(test)
assert outcome in ["PASS", "FAIL", "CRASH", "TIMEOUT"]
if test.run == 1:
......
......@@ -29,20 +29,17 @@
import fnmatch
import imp
import os
import re
import shlex
from . import command
from . import statusfile
from . import utils
from ..objects.testcase import TestCase
from variants import ALL_VARIANTS, ALL_VARIANT_FLAGS, FAST_VARIANT_FLAGS
FAST_VARIANTS = set(["default", "turbofan"])
STANDARD_VARIANT = set(["default"])
FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)")
class VariantGenerator(object):
def __init__(self, suite, variants):
......@@ -51,9 +48,9 @@ class VariantGenerator(object):
self.fast_variants = FAST_VARIANTS & variants
self.standard_variant = STANDARD_VARIANT & variants
def FilterVariantsByTest(self, testcase):
def FilterVariantsByTest(self, test):
result = self.all_variants
outcomes = testcase.suite.GetStatusFileOutcomes(testcase)
outcomes = test.suite.GetStatusFileOutcomes(test.name, test.variant)
if outcomes:
if statusfile.OnlyStandardVariant(outcomes):
return self.standard_variant
......@@ -61,8 +58,8 @@ class VariantGenerator(object):
result = self.fast_variants
return result
def GetFlagSets(self, testcase, variant):
outcomes = testcase.suite.GetStatusFileOutcomes(testcase)
def GetFlagSets(self, test, variant):
outcomes = test.suite.GetStatusFileOutcomes(test.name, test.variant)
if outcomes and statusfile.OnlyFastVariants(outcomes):
return FAST_VARIANT_FLAGS[variant]
else:
......@@ -70,9 +67,8 @@ class VariantGenerator(object):
class TestSuite(object):
@staticmethod
def LoadTestSuite(root, global_init=True):
def LoadTestSuite(root):
name = root.split(os.path.sep)[-1]
f = None
try:
......@@ -100,13 +96,6 @@ class TestSuite(object):
def status_file(self):
return "%s/%s.status" % (self.root, self.name)
# Used in the status file and for stdout printing.
def CommonTestName(self, testcase):
if utils.IsWindows():
return testcase.path.replace("\\", "/")
else:
return testcase.path
def ListTests(self, context):
raise NotImplementedError
......@@ -132,17 +121,6 @@ class TestSuite(object):
def ReadTestCases(self, context):
self.tests = self.ListTests(context)
def GetStatusfileFlags(self, test):
"""Gets runtime flags from a status file.
Every outcome that starts with "--" is a flag. Status file has to be loaded
before using this function.
"""
flags = []
for outcome in self.GetStatusFileOutcomes(test):
if outcome.startswith('--'):
flags.append(outcome)
return flags
def FilterTestCasesByStatus(self,
slow_tests_mode=None,
......@@ -171,7 +149,7 @@ class TestSuite(object):
(mode == 'skip' and pass_fail))
def _compliant(test):
outcomes = self.GetStatusFileOutcomes(test)
outcomes = self.GetStatusFileOutcomes(test.name, test.variant)
if statusfile.DoSkip(outcomes):
return False
if _skip_slow(statusfile.IsSlow(outcomes), slow_tests_mode):
......@@ -198,17 +176,16 @@ class TestSuite(object):
variants = ['']
used_rules = set()
for t in self.tests:
testname = self.CommonTestName(t)
variant = t.variant or ""
for test in self.tests:
variant = test.variant or ""
if testname in self.rules.get(variant, {}):
used_rules.add((testname, variant))
if statusfile.DoSkip(self.rules[variant][testname]):
if test.name in self.rules.get(variant, {}):
used_rules.add((test.name, variant))
if statusfile.DoSkip(self.rules[variant][test.name]):
continue
for prefix in self.prefix_rules.get(variant, {}):
if testname.startswith(prefix):
if test.name.startswith(prefix):
used_rules.add((prefix, variant))
if statusfile.DoSkip(self.prefix_rules[variant][prefix]):
break
......@@ -248,14 +225,14 @@ class TestSuite(object):
break
self.tests = filtered
def GetExpectedOutcomes(self, testcase):
def GetExpectedOutcomes(self, test):
"""Gets expected outcomes from status file.
It differs from GetStatusFileOutcomes by selecting only outcomes that can
be result of test execution.
Status file has to be loaded before using this function.
"""
outcomes = self.GetStatusFileOutcomes(testcase)
outcomes = self.GetStatusFileOutcomes(test.name, test.variant)
expected = []
if (statusfile.FAIL in outcomes or
......@@ -270,14 +247,14 @@ class TestSuite(object):
return expected or [statusfile.PASS]
def GetStatusFileOutcomes(self, testcase):
def GetStatusFileOutcomes(self, testname, variant=None):
"""Gets outcomes from status file.
Merges variant dependent and independent rules. Status file has to be loaded
before using this function.
"""
variant = testcase.variant or ''
testname = self.CommonTestName(testcase)
variant = variant or ''
cache_key = '%s$%s' % (testname, variant)
if cache_key not in self._outcomes_cache:
......@@ -302,64 +279,6 @@ class TestSuite(object):
return self._outcomes_cache[cache_key]
def GetCommand(self, test, context):
shell = self.GetShellForTestCase(test)
shell_flags = []
if shell == 'd8':
shell_flags.append('--test')
if utils.IsWindows():
shell += '.exe'
if context.random_seed:
shell_flags.append('--random-seed=%s' % context.random_seed)
files, flags, env = self.GetParametersForTestCase(test, context)
return command.Command(
cmd_prefix=context.command_prefix,
shell=os.path.abspath(os.path.join(context.shell_dir, shell)),
args=(
shell_flags +
files +
context.extra_flags +
flags
),
env=env,
timeout=self.GetTimeout(test, context),
verbose=context.verbose
)
def GetTimeout(self, testcase, context):
timeout = context.timeout
if ("--stress-opt" in testcase.flags or
"--stress-opt" in context.mode_flags or
"--stress-opt" in context.extra_flags):
timeout *= 4
if "--noenable-vfp3" in context.extra_flags:
timeout *= 2
# TODO(majeski): make it slow outcome dependent.
timeout *= 2
return timeout
def GetShellForTestCase(self, testcase):
"""Returns shell to be executed for this test case."""
return 'd8'
def GetParametersForTestCase(self, testcase, context):
"""Returns a tuple of (files, flags, env) for this test case."""
raise NotImplementedError
def _parse_source_flags(self, test, source=None):
if not source:
source = self.GetSourceForTest(test)
flags = []
for match in re.findall(FLAGS_PATTERN, source):
flags += shlex.split(match.strip())
return flags
def GetSourceForTest(self, testcase):
return "(no source available)"
def IsFailureOutput(self, testcase):
return testcase.output.exit_code != 0
......@@ -403,6 +322,19 @@ class TestSuite(object):
self.total_duration += t.duration
return self.total_duration
def _create_test(self, path, **kwargs):
test = self._test_class()(self, path, self._path_to_name(path), **kwargs)
test.precompute()
return test
def _test_class(self):
raise NotImplementedError
def _path_to_name(self, path):
if utils.IsWindows():
return path.replace("\\", "/")
return path
class StandardVariantGenerator(VariantGenerator):
def FilterVariantsByTest(self, testcase):
......
......@@ -20,8 +20,8 @@ class TestSuiteTest(unittest.TestCase):
def test_filter_testcases_by_status_first_pass(self):
suite = TestSuite('foo', 'bar')
suite.tests = [
TestCase(suite, 'foo/bar'),
TestCase(suite, 'baz/bar'),
TestCase(suite, 'foo/bar', 'foo/bar'),
TestCase(suite, 'baz/bar', 'baz/bar'),
]
suite.rules = {
'': {
......@@ -36,17 +36,18 @@ class TestSuiteTest(unittest.TestCase):
}
suite.FilterTestCasesByStatus()
self.assertEquals(
[TestCase(suite, 'baz/bar')],
[TestCase(suite, 'baz/bar', 'baz/bar')],
suite.tests,
)
outcomes = suite.GetStatusFileOutcomes(suite.tests[0])
outcomes = suite.GetStatusFileOutcomes(suite.tests[0].name,
suite.tests[0].variant)
self.assertEquals(set(['PASS', 'FAIL', 'SLOW']), outcomes)
def test_filter_testcases_by_status_second_pass(self):
suite = TestSuite('foo', 'bar')
test1 = TestCase(suite, 'foo/bar')
test2 = TestCase(suite, 'baz/bar')
test1 = TestCase(suite, 'foo/bar', 'foo/bar')
test2 = TestCase(suite, 'baz/bar', 'baz/bar')
suite.tests = [
test1.CopyAddingFlags(variant='default', flags=[]),
......@@ -81,26 +82,28 @@ class TestSuiteTest(unittest.TestCase):
suite.FilterTestCasesByStatus()
self.assertEquals(
[
TestCase(suite, 'foo/bar', flags=['-v']),
TestCase(suite, 'baz/bar'),
TestCase(suite, 'foo/bar', 'foo/bar', flags=['-v']),
TestCase(suite, 'baz/bar', 'baz/bar'),
],
suite.tests,
)
self.assertEquals(
set(['PREV', 'PASS', 'SLOW']),
suite.GetStatusFileOutcomes(suite.tests[0]),
suite.GetStatusFileOutcomes(suite.tests[0].name,
suite.tests[0].variant),
)
self.assertEquals(
set(['PREV', 'PASS', 'FAIL', 'SLOW']),
suite.GetStatusFileOutcomes(suite.tests[1]),
suite.GetStatusFileOutcomes(suite.tests[1].name,
suite.tests[1].variant),
)
def test_fail_ok_outcome(self):
suite = TestSuite('foo', 'bar')
suite.tests = [
TestCase(suite, 'foo/bar'),
TestCase(suite, 'baz/bar'),
TestCase(suite, 'foo/bar', 'foo/bar'),
TestCase(suite, 'baz/bar', 'baz/bar'),
]
suite.rules = {
'': {
......
......@@ -45,7 +45,7 @@ def PrintReport(tests):
total = len(tests)
skipped = nocrash = passes = fail_ok = fail = 0
for t in tests:
outcomes = t.suite.GetStatusFileOutcomes(t)
outcomes = t.suite.GetStatusFileOutcomes(t.name, t.variant)
if not outcomes:
passes += 1
continue
......@@ -73,12 +73,12 @@ def PrintReport(tests):
def PrintTestSource(tests):
for test in tests:
suite = test.suite
source = suite.GetSourceForTest(test).strip()
if len(source) > 0:
print "--- begin source: %s/%s ---" % (suite.name, test.path)
print source
print "--- end source: %s/%s ---" % (suite.name, test.path)
print "--- begin source: %s ---" % test
if test.is_source_available():
print test.get_source()
else:
print '(no source available)'
print "--- end source: %s ---" % test
def FormatTime(d):
......@@ -95,7 +95,7 @@ def PrintTestDurations(suites, overall_time):
if t.duration is not None ]
timed_tests.sort(lambda a, b: cmp(b.duration, a.duration))
index = 1
for entry in timed_tests[:20]:
t = FormatTime(entry.duration)
sys.stderr.write("%4i (%s) %s\n" % (index, t, entry.GetLabel()))
for test in timed_tests[:20]:
t = FormatTime(test.duration)
sys.stderr.write("%4i (%s) %s\n" % (index, t, test))
index += 1
......@@ -25,43 +25,186 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import os
import re
import shlex
from ..local import command
from ..local import utils
class TestCase(object):
def __init__(self, suite, path, variant=None, flags=None):
def __init__(self, suite, path, name):
self.suite = suite # TestSuite object
self.path = path # string, e.g. 'div-mod', 'test-api/foo'
self.flags = flags or [] # list of strings, flags specific to this test
self.variant = variant # name of the used testing variant
self.name = name # string that identifies test in the status file
self.variant = None # name of the used testing variant
self.variant_flags = [] # list of strings, flags specific to this test
self.output = None
self.id = None # int, used to map result back to TestCase instance
self.duration = None # assigned during execution
self.run = 1 # The nth time this test is executed.
self.cmd = None
def CopyAddingFlags(self, variant, flags):
return TestCase(self.suite, self.path, variant, self.flags + flags)
def precompute(self):
"""It precomputes all things that can be shared among all variants of this
object (like flags from source file). Values calculated here should be
immutable and shared among all copies (in _copy implementation).
"""
pass
def create_variant(self, variant, flags):
copy = self._copy()
if not self.variant_flags:
copy.variant_flags = flags
else:
copy.variant_flags = self.variant_flags + flags
copy.variant = variant
return copy
def suitename(self):
return self.suite.name
def _copy(self):
"""Makes a copy of the object. It should be overriden in case of any
additional constructor parameters or precomputed fields.
"""
return self.__class__(self.suite, self.path, self.name)
def GetLabel(self):
return self.suitename() + "/" + self.suite.CommonTestName(self)
def get_command(self, context):
params = self._get_cmd_params(context)
env = self._get_cmd_env()
shell, shell_flags = self._get_shell_with_flags(context)
timeout = self._get_timeout(params, context.timeout)
return self._create_cmd(shell, shell_flags + params, env, timeout, context)
def __getstate__(self):
"""Representation to pickle test cases.
def _get_cmd_params(self, ctx):
"""Gets command parameters and combines them in the following order:
- files [empty by default]
- extra flags (from command line)
- user flags (variant/fuzzer flags)
- statusfile flags
- mode flags (based on chosen mode)
- source flags (from source code) [empty by default]
The original suite won't be sent beyond process boundaries. Instead
send the name only and retrieve a process-local suite later.
The best way to modify how parameters are created is to only override
methods for getting partial parameters.
"""
return dict(self.__dict__, suite=self.suite.name)
return (
self._get_files_params(ctx) +
self._get_extra_flags(ctx) +
self._get_variant_flags() +
self._get_statusfile_flags() +
self._get_mode_flags(ctx) +
self._get_source_flags() +
self._get_suite_flags(ctx)
)
def _get_cmd_env(self):
return {}
def _get_files_params(self, ctx):
return []
def _get_extra_flags(self, ctx):
return ctx.extra_flags
def _get_variant_flags(self):
return self.variant_flags
def _get_statusfile_flags(self):
"""Gets runtime flags from a status file.
Every outcome that starts with "--" is a flag. Status file has to be loaded
before using this function.
"""
flags = []
for outcome in self.suite.GetStatusFileOutcomes(self.name, self.variant):
if outcome.startswith('--'):
flags.append(outcome)
return flags
def _get_mode_flags(self, ctx):
return ctx.mode_flags
def _get_source_flags(self):
return []
def _get_suite_flags(self, ctx):
return []
def _get_shell_with_flags(self, ctx):
shell = self._get_shell()
shell_flags = []
if shell == 'd8':
shell_flags.append('--test')
if utils.IsWindows():
shell += '.exe'
if ctx.random_seed:
shell_flags.append('--random-seed=%s' % ctx.random_seed)
return shell, shell_flags
def _get_timeout(self, params, timeout):
if "--stress-opt" in params:
timeout *= 4
if "--noenable-vfp3" in params:
timeout *= 2
# TODO(majeski): make it slow outcome dependent.
timeout *= 2
return timeout
def _get_shell(self):
return 'd8'
def _get_suffix(self):
return '.js'
def _create_cmd(self, shell, params, env, timeout, ctx):
return command.Command(
cmd_prefix=ctx.command_prefix,
shell=os.path.abspath(os.path.join(ctx.shell_dir, shell)),
args=params,
env=env,
timeout=timeout,
verbose=ctx.verbose
)
def _parse_source_flags(self, source=None):
PATTERN = re.compile(r"//\s+Flags:(.*)")
if not source:
source = self.get_source()
flags = []
for match in re.findall(PATTERN, source):
flags += shlex.split(match.strip())
return flags
def is_source_available(self):
return self._get_source_path() is not None
def get_source(self):
with open(self._get_source_path()) as f:
return f.read()
def _get_source_path(self):
return None
def __cmp__(self, other):
# Make sure that test cases are sorted correctly if sorted without
# key function. But using a key function is preferred for speed.
return cmp(
(self.suite.name, self.path, self.flags),
(other.suite.name, other.path, other.flags),
(self.suite.name, self.name, self.variant_flags),
(other.suite.name, other.name, other.variant_flags)
)
def __str__(self):
return "[%s/%s %s]" % (self.suite.name, self.path, self.flags)
return self.suite.name + '/' + self.name
# TODO(majeski): Rename `id` field or `get_id` function since they're
# unrelated.
def get_id(self):
return '%s/%s %s' % (
self.suite.name, self.name, ' '.join(self.variant_flags))
......@@ -429,7 +429,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
verbose.PrintTestSource(s.tests)
continue
variant_gen = s.CreateVariantGenerator(VARIANTS)
variant_tests = [ t.CopyAddingFlags(v, flags)
variant_tests = [ t.create_variant(v, flags)
for t in s.tests
for v in variant_gen.FilterVariantsByTest(t)
for flags in variant_gen.GetFlagSets(t, v) ]
......@@ -445,7 +445,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
else:
yield ["--random-seed=%d" % self._random_seed()]
s.tests = [
t.CopyAddingFlags(t.variant, flags)
t.create_variant(t.variant, flags)
for t in variant_tests
for flags in iter_seed_flags()
]
......@@ -459,8 +459,7 @@ class StandardTestRunner(base_runner.BaseTestRunner):
s.tests = self._shard_tests(s.tests, options)
for t in s.tests:
t.flags += s.GetStatusfileFlags(t)
t.cmd = s.GetCommand(t, ctx)
t.cmd = t.get_command(ctx)
num_tests += len(s.tests)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment