Migrate to new test status file syntax

R=machenbach@chromium.org

Review URL: https://codereview.chromium.org/23498058

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@16919 ce2b1a6d-e550-0410-aec6-3dcde31c8c00
parent 9de57484
......@@ -37,25 +37,16 @@ shell_g
/out
/perf.data
/perf.data.old
/test/benchmarks/benchmarks.status2
/test/benchmarks/CHECKED_OUT_*
/test/benchmarks/downloaded_*
/test/benchmarks/kraken
/test/benchmarks/octane
/test/benchmarks/sunspider
/test/cctest/cctest.status2
/test/intl/intl.status2
/test/message/message.status2
/test/mjsunit/mjsunit.status2
/test/mozilla/CHECKED_OUT_VERSION
/test/mozilla/data
/test/mozilla/downloaded_*
/test/mozilla/mozilla.status2
/test/preparser/preparser.status2
/test/test262/data
/test/test262/test262-*
/test/test262/test262.status2
/test/webkit/webkit.status2
/third_party
/tools/jsfunfuzz
/tools/jsfunfuzz.zip
......
......@@ -25,5 +25,9 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# Too slow in Debug mode.
octane/mandreel: PASS, SKIP if $mode == debug
[
['mode == debug', {
# Too slow in Debug mode.
'octane/mandreel': [SKIP],
}], # 'mode == debug'
]
......@@ -25,112 +25,118 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
prefix cctest
[
[ALWAYS, {
# All tests prefixed with 'Bug' are expected to fail.
'test-api/Bug*': [FAIL],
# All tests prefixed with 'Bug' are expected to fail.
test-api/Bug*: FAIL
##############################################################################
# BUG(382): Weird test. Can't guarantee that it never times out.
'test-api/ApplyInterruption': [PASS, TIMEOUT],
##############################################################################
# BUG(382): Weird test. Can't guarantee that it never times out.
test-api/ApplyInterruption: PASS || TIMEOUT
# BUG(2907): Allocation while in DisallowHeapAllocation scope.
test-object-observe/NamedAccessCheck: SKIP
test-object-observe/DisallowAllForAccessKeys: SKIP
test-object-observe/AccessCheckDisallowApiModifications: SKIP
# BUG(2907): Allocation while in DisallowHeapAllocation scope.
'test-object-observe/NamedAccessCheck': [SKIP],
'test-object-observe/DisallowAllForAccessKeys': [SKIP],
'test-object-observe/AccessCheckDisallowApiModifications': [SKIP],
# TODO(mstarzinger): Fail gracefully on multiple V8::Dispose calls.
test-api/InitializeAndDisposeOnce: SKIP
test-api/InitializeAndDisposeMultiple: SKIP
# TODO(mstarzinger): Fail gracefully on multiple V8::Dispose calls.
'test-api/InitializeAndDisposeOnce': [SKIP],
'test-api/InitializeAndDisposeMultiple': [SKIP],
# These tests always fail. They are here to test test.py. If
# they don't fail then test.py has failed.
test-serialize/TestThatAlwaysFails: FAIL
test-serialize/DependentTestThatAlwaysFails: FAIL
# These tests always fail. They are here to test test.py. If
# they don't fail then test.py has failed.
'test-serialize/TestThatAlwaysFails': [FAIL],
'test-serialize/DependentTestThatAlwaysFails': [FAIL],
# This test always fails. It tests that LiveEdit causes abort when turned off.
test-debug/LiveEditDisabled: FAIL
# This test always fails. It tests that LiveEdit causes abort when turned off.
'test-debug/LiveEditDisabled': [FAIL],
# TODO(gc): Temporarily disabled in the GC branch.
test-log/EquivalenceOfLoggingAndTraversal: PASS || FAIL
# TODO(gc): Temporarily disabled in the GC branch.
'test-log/EquivalenceOfLoggingAndTraversal': [PASS, FAIL],
# We do not yet shrink weak maps after they have been emptied by the GC
test-weakmaps/Shrinking: FAIL
test-weaksets/WeakSet_Shrinking: FAIL
# We do not yet shrink weak maps after they have been emptied by the GC
'test-weakmaps/Shrinking': [FAIL],
'test-weaksets/WeakSet_Shrinking': [FAIL],
# Boot up memory use is bloated in debug mode.
test-mark-compact/BootUpMemoryUse: PASS, PASS || FAIL if $mode == debug
# Boot up memory use is bloated in debug mode.
'test-mark-compact/BootUpMemoryUse': [PASS, PASS, ['mode == debug', FAIL]],
# Some CPU profiler tests are flaky.
test-cpu-profiler/*: PASS || FLAKY
# Some CPU profiler tests are flaky.
'test-cpu-profiler/*': [PASS, FLAKY],
}], # ALWAYS
##############################################################################
[ $arch == arm ]
# We cannot assume that we can throw OutOfMemory exceptions in all situations.
# Apparently our ARM box is in such a state. Skip the test as it also runs for
# a long time.
test-api/OutOfMemory: SKIP
test-api/OutOfMemoryNested: SKIP
# BUG(355): Test crashes on ARM.
test-log/ProfLazyMode: SKIP
# BUG(1075): Unresolved crashes.
test-serialize/Deserialize: SKIP
test-serialize/DeserializeFromSecondSerializationAndRunScript2: SKIP
test-serialize/DeserializeAndRunScript2: SKIP
test-serialize/DeserializeFromSecondSerialization: SKIP
# BUG(2874): Threading problems.
test-api/*: PASS || FLAKY
['arch == arm', {
# We cannot assume that we can throw OutOfMemory exceptions in all situations.
# Apparently our ARM box is in such a state. Skip the test as it also runs for
# a long time.
'test-api/OutOfMemory': [SKIP],
'test-api/OutOfMemoryNested': [SKIP],
# BUG(355): Test crashes on ARM.
'test-log/ProfLazyMode': [SKIP],
# BUG(1075): Unresolved crashes.
'test-serialize/Deserialize': [SKIP],
'test-serialize/DeserializeFromSecondSerializationAndRunScript2': [SKIP],
'test-serialize/DeserializeAndRunScript2': [SKIP],
'test-serialize/DeserializeFromSecondSerialization': [SKIP],
# BUG(2874): Threading problems.
'test-api/*': [PASS, FLAKY],
}], # 'arch == arm'
##############################################################################
[ $arch == mipsel ]
['arch == mipsel', {
# BUG(2657): Test sometimes times out on MIPS simulator.
test-thread-termination/TerminateMultipleV8ThreadsDefaultIsolate: PASS || TIMEOUT
# BUG(2657): Test sometimes times out on MIPS simulator.
'test-thread-termination/TerminateMultipleV8ThreadsDefaultIsolate': [PASS, TIMEOUT],
# BUG(1075): Unresolved crashes on MIPS also.
test-serialize/Deserialize: SKIP
test-serialize/DeserializeFromSecondSerializationAndRunScript2: SKIP
test-serialize/DeserializeAndRunScript2: SKIP
test-serialize/DeserializeFromSecondSerialization: SKIP
# BUG(1075): Unresolved crashes on MIPS also.
'test-serialize/Deserialize': [SKIP],
'test-serialize/DeserializeFromSecondSerializationAndRunScript2': [SKIP],
'test-serialize/DeserializeAndRunScript2': [SKIP],
'test-serialize/DeserializeFromSecondSerialization': [SKIP],
}], # 'arch == mipsel'
##############################################################################
[ $arch == android_arm || $arch == android_ia32 ]
['arch == android_arm or arch == android_ia32', {
# Tests crash as there is no /tmp directory in Android.
test-log/LogAccessorCallbacks: SKIP
test-log/LogCallbacks: SKIP
test-log/ProfLazyMode: SKIP
# Tests crash as there is no /tmp directory in Android.
'test-log/LogAccessorCallbacks': [SKIP],
'test-log/LogCallbacks': [SKIP],
'test-log/ProfLazyMode': [SKIP],
# platform-tls.h does not contain an ANDROID-related header.
test-platform-tls/FastTLS: SKIP
# platform-tls.h does not contain an ANDROID-related header.
'test-platform-tls/FastTLS': [SKIP],
# This test times out.
test-threads/ThreadJoinSelf: SKIP
# This test times out.
'test-threads/ThreadJoinSelf': [SKIP],
}], # 'arch == android_arm or arch == android_ia32'
##############################################################################
[ $arch == nacl_ia32 || $arch == nacl_x64 ]
# NaCl builds have problems with threaded tests since Pepper_28.
# V8 Issue 2786
test-api/Threading1: SKIP
test-lockers/MultithreadedParallelIsolates: SKIP
test-lockers/ExtensionsRegistration: SKIP
# These tests fail as there is no /tmp directory in Native Client.
test-log/LogAccessorCallbacks: SKIP
test-log/LogCallbacks: SKIP
test-log/ProfLazyMode: SKIP
# Native Client doesn't support sockets.
test-debug/DebuggerAgent: SKIP
test-debug/DebuggerAgentProtocolOverflowHeader: SKIP
test-socket/Socket: SKIP
# Profiling doesn't work on Native Client.
test-cpu-profiler/*: SKIP
# Fails since 16322 (new test).
test-code-stubs-arm/ConvertDToI: SKIP
['arch == nacl_ia32 or arch == nacl_x64', {
# NaCl builds have problems with threaded tests since Pepper_28.
# V8 Issue 2786
'test-api/Threading1': [SKIP],
'test-lockers/MultithreadedParallelIsolates': [SKIP],
'test-lockers/ExtensionsRegistration': [SKIP],
# These tests fail as there is no /tmp directory in Native Client.
'test-log/LogAccessorCallbacks': [SKIP],
'test-log/LogCallbacks': [SKIP],
'test-log/ProfLazyMode': [SKIP],
# Native Client doesn't support sockets.
'test-debug/DebuggerAgent': [SKIP],
'test-debug/DebuggerAgentProtocolOverflowHeader': [SKIP],
'test-socket/Socket': [SKIP],
# Profiling doesn't work on Native Client.
'test-cpu-profiler/*': [SKIP],
# Fails since 16322 (new test).
'test-code-stubs-arm/ConvertDToI': [SKIP],
}], # 'arch == nacl_ia32 or arch == nacl_x64'
]
......@@ -25,15 +25,17 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
prefix intl
[
[ALWAYS, {
# The following tests use getDefaultTimeZone().
'date-format/resolved-options': [FAIL],
'date-format/timezone': [FAIL],
'general/v8Intl-exists': [FAIL],
# The following tests use getDefaultTimeZone().
date-format/resolved-options: FAIL
date-format/timezone: FAIL
general/v8Intl-exists: FAIL
# TODO(jochen): The following test is flaky.
'overrides/caching': [PASS, FAIL],
# TODO(jochen): The following test is flaky.
overrides/caching: PASS || FAIL
# BUG(2899): default locale for search fails on mac and on android.
collator/default-locale: FAIL if ($system == macos || $arch == android_arm || $arch == android_ia32)
# BUG(2899): default locale for search fails on mac and on android.
'collator/default-locale': [['system == macos or arch == android_arm or arch == android_ia32', FAIL]],
}], # ALWAYS
]
......@@ -25,7 +25,9 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
prefix message
# All tests in the bug directory are expected to fail.
bugs/*: FAIL
[
[ALWAYS, {
# All tests in the bug directory are expected to fail.
'bugs/*': [FAIL],
}], # ALWAYS
]
This diff is collapsed.
This diff is collapsed.
......@@ -25,14 +25,17 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
prefix preparser
[
[ALWAYS, {
# We don't parse RegExps at scanning time, so we can't fail on octal
# escapes (we need to parse to distinguish octal escapes from valid
# back-references).
'strict-octal-regexp': [FAIL],
}], # ALWAYS
# We don't parse RegExps at scanning time, so we can't fail on octal
# escapes (we need to parse to distinguish octal escapes from valid
# back-references).
strict-octal-regexp: FAIL
[ $arch == android_arm || $arch == android_ia32 ]
# Remove this once the issue above is fixed. Android test runner does not
# handle "FAIL" test expectation correctly.
strict-octal-regexp: SKIP
['arch == android_arm or arch == android_ia32', {
# Remove this once the issue above is fixed. Android test runner does not
# handle "FAIL" test expectation correctly.
'strict-octal-regexp': [SKIP],
}], # 'arch == android_arm or arch == android_ia32'
]
......@@ -25,95 +25,97 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
prefix test262
def FAIL_OK = FAIL, OKAY
############################### BUGS ###################################
# Sequencing of getter side effects on receiver and argument properties
# is wrong. The receiver callback should be called before any arguments
# are evaluated.
# V8 Bug: http://code.google.com/p/v8/issues/detail?id=691
11.2.3-3_3: FAIL
15.5.4.9_CE: SKIP if $no_i18n
######################## NEEDS INVESTIGATION ###########################
# These test failures are specific to the intl402 suite and need investigation
# to be either marked as bugs with issues filed for them or as deliberate
# incompatibilities if the test cases turn out to be broken or ambiguous.
6.2.3: FAIL
9.2.1_2: FAIL
9.2.5_11_g_ii_2: FAIL
9.2.6_2: FAIL
10.1.1_a: FAIL
10.1.1_19_c: PASS || FAIL
10.1.2.1_4: FAIL
10.2.3_b: PASS || FAIL
10.3_a: FAIL
11.1.1_17: PASS || FAIL
11.1.1_19: PASS || FAIL
11.1.1_20_c: FAIL
11.1.1_a: FAIL
11.1.2.1_4: FAIL
11.3.2_FN_2: PASS || FAIL
11.3.2_TRF: PASS || FAIL
11.3.2_TRP: FAIL
11.3_a: FAIL
12.1.1_a: FAIL
12.1.2.1_4: FAIL
12.3.2_FDT_7_a_iv: FAIL
12.3.3: FAIL
12.3_a: FAIL
15.5.4.9_3: PASS || FAIL
##################### DELIBERATE INCOMPATIBILITIES #####################
# This tests precision of Math functions. The implementation for those
# trigonometric functions are platform/compiler dependent. Furthermore, the
# expectation values by far deviates from the actual result given by an
# arbitrary-precision calculator, making those tests partly bogus.
S15.8.2.8_A6: PASS || FAIL_OK # Math.exp (less precise with --fast-math)
S15.8.2.16_A7: PASS || FAIL_OK # Math.sin
S15.8.2.18_A7: PASS || FAIL_OK # Math.tan
# Linux for ia32 (and therefore simulators) default to extended 80 bit floating
# point formats, so these tests checking 64-bit FP precision fail. The other
# platforms/arch's pass these tests.
# We follow the other major JS engines by keeping this default.
S8.5_A2.1: PASS || FAIL_OK
S8.5_A2.2: PASS || FAIL_OK
############################ INVALID TESTS #############################
# The reference value calculated by Test262 is incorrect if you run these tests
# in PST/PDT between first Sunday in March and first Sunday in April. The DST
# switch was moved in 2007 whereas Test262 bases the reference value on 2000.
# Test262 Bug: https://bugs.ecmascript.org/show_bug.cgi?id=293
S15.9.3.1_A5_T1: PASS || FAIL_OK
S15.9.3.1_A5_T2: PASS || FAIL_OK
S15.9.3.1_A5_T3: PASS || FAIL_OK
S15.9.3.1_A5_T4: PASS || FAIL_OK
S15.9.3.1_A5_T5: PASS || FAIL_OK
S15.9.3.1_A5_T6: PASS || FAIL_OK
############################ SKIPPED TESTS #############################
# These tests take a looong time to run in debug mode.
S15.1.3.1_A2.5_T1: PASS, SKIP if $mode == debug
S15.1.3.2_A2.5_T1: PASS, SKIP if $mode == debug
[ $arch == arm || $arch == mipsel ]
# TODO(mstarzinger): Causes stack overflow on simulators due to eager
# compilation of parenthesized function literals. Needs investigation.
S13.2.1_A1_T1: SKIP
# BUG(3251225): Tests that timeout with --nocrankshaft.
S15.1.3.1_A2.4_T1: SKIP
S15.1.3.1_A2.5_T1: SKIP
S15.1.3.2_A2.4_T1: SKIP
S15.1.3.2_A2.5_T1: SKIP
S15.1.3.3_A2.3_T1: SKIP
S15.1.3.4_A2.3_T1: SKIP
[
[ALWAYS, {
############################### BUGS ###################################
# Sequencing of getter side effects on receiver and argument properties
# is wrong. The receiver callback should be called before any arguments
# are evaluated.
# V8 Bug: http://code.google.com/p/v8/issues/detail?id=691
'11.2.3-3_3': [FAIL],
'15.5.4.9_CE': [['no_i18n', SKIP]],
######################## NEEDS INVESTIGATION ###########################
# These test failures are specific to the intl402 suite and need investigation
# to be either marked as bugs with issues filed for them or as deliberate
# incompatibilities if the test cases turn out to be broken or ambiguous.
'6.2.3': [FAIL],
'9.2.1_2': [FAIL],
'9.2.5_11_g_ii_2': [FAIL],
'9.2.6_2': [FAIL],
'10.1.1_a': [FAIL],
'10.1.1_19_c': [PASS, FAIL],
'10.1.2.1_4': [FAIL],
'10.2.3_b': [PASS, FAIL],
'10.3_a': [FAIL],
'11.1.1_17': [PASS, FAIL],
'11.1.1_19': [PASS, FAIL],
'11.1.1_20_c': [FAIL],
'11.1.1_a': [FAIL],
'11.1.2.1_4': [FAIL],
'11.3.2_FN_2': [PASS, FAIL],
'11.3.2_TRF': [PASS, FAIL],
'11.3.2_TRP': [FAIL],
'11.3_a': [FAIL],
'12.1.1_a': [FAIL],
'12.1.2.1_4': [FAIL],
'12.3.2_FDT_7_a_iv': [FAIL],
'12.3.3': [FAIL],
'12.3_a': [FAIL],
'15.5.4.9_3': [PASS, FAIL],
##################### DELIBERATE INCOMPATIBILITIES #####################
# This tests precision of Math functions. The implementation for those
# trigonometric functions are platform/compiler dependent. Furthermore, the
# expectation values by far deviates from the actual result given by an
# arbitrary-precision calculator, making those tests partly bogus.
'S15.8.2.8_A6': [PASS, FAIL_OK], # Math.exp (less precise with --fast-math)
'S15.8.2.16_A7': [PASS, FAIL_OK], # Math.sin
'S15.8.2.18_A7': [PASS, FAIL_OK], # Math.tan
# Linux for ia32 (and therefore simulators) default to extended 80 bit
# floating point formats, so these tests checking 64-bit FP precision fail.
# The other platforms/arch's pass these tests.
# We follow the other major JS engines by keeping this default.
'S8.5_A2.1': [PASS, FAIL_OK],
'S8.5_A2.2': [PASS, FAIL_OK],
############################ INVALID TESTS #############################
# The reference value calculated by Test262 is incorrect if you run these
# tests in PST/PDT between first Sunday in March and first Sunday in April.
# The DST switch was moved in 2007 whereas Test262 bases the reference value
# on 2000. Test262 Bug: https://bugs.ecmascript.org/show_bug.cgi?id=293
'S15.9.3.1_A5_T1': [PASS, FAIL_OK],
'S15.9.3.1_A5_T2': [PASS, FAIL_OK],
'S15.9.3.1_A5_T3': [PASS, FAIL_OK],
'S15.9.3.1_A5_T4': [PASS, FAIL_OK],
'S15.9.3.1_A5_T5': [PASS, FAIL_OK],
'S15.9.3.1_A5_T6': [PASS, FAIL_OK],
############################ SKIPPED TESTS #############################
# These tests take a looong time to run in debug mode.
'S15.1.3.1_A2.5_T1': [PASS, ['mode == debug', SKIP]],
'S15.1.3.2_A2.5_T1': [PASS, ['mode == debug', SKIP]],
}], # ALWAYS
['arch == arm or arch == mipsel', {
# TODO(mstarzinger): Causes stack overflow on simulators due to eager
# compilation of parenthesized function literals. Needs investigation.
'S13.2.1_A1_T1': [SKIP],
# BUG(3251225): Tests that timeout with --nocrankshaft.
'S15.1.3.1_A2.4_T1': [SKIP],
'S15.1.3.1_A2.5_T1': [SKIP],
'S15.1.3.2_A2.4_T1': [SKIP],
'S15.1.3.2_A2.5_T1': [SKIP],
'S15.1.3.3_A2.3_T1': [SKIP],
'S15.1.3.4_A2.3_T1': [SKIP],
}], # 'arch == arm or arch == mipsel'
]
......@@ -25,11 +25,12 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# Too slow in debug mode.
dfg-int-overflow-in-loop: PASS, SKIP if $mode == debug
dfg-double-vote-fuzz: PASS, SKIP if $mode == debug
reentrant-caching: PASS, SKIP if $mode == debug
sort-large-array: PASS, SKIP if $mode == debug
##############################################################################
[ $deopt_fuzzer == True ]
[
['mode == debug', {
# Too slow in debug mode.
'dfg-int-overflow-in-loop': [SKIP],
'dfg-double-vote-fuzz': [SKIP],
'reentrant-caching': [SKIP],
'sort-large-array': [SKIP],
}], # 'mode == debug'
]
This diff is collapsed.
......@@ -26,14 +26,6 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
# These imports are required for the on-demand conversion from
# old to new status file format.
from os.path import exists
from os.path import getmtime
from . import old_statusfile
# These outcomes can occur in a TestCase's outcomes list:
SKIP = "SKIP"
FAIL = "FAIL"
......@@ -116,18 +108,6 @@ def _ParseOutcomeList(rule, outcomes, target_dict, variables):
def ReadStatusFile(path, variables):
# As long as the old-format .status files are authoritative, just
# create the converted version on demand and cache it to speed up
# subsequent runs.
if path.endswith(".status"):
newpath = path + "2"
if not exists(newpath) or getmtime(newpath) < getmtime(path):
print "Converting status file."
converted = old_statusfile.ConvertNotation(path).GetOutput()
with open(newpath, 'w') as f:
f.write(converted)
path = newpath
with open(path) as f:
global KEYWORDS
contents = eval(f.read(), KEYWORDS)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment