Commit f62f846c authored by littledan's avatar littledan Committed by Commit bot

[test] Allow command-line flags to be turned on per test262 test

This patch changes the test262 infrastructure to pass individual flags,
specified in the status file, for tests for experimental features, rather
than passing --harmony for all runs. With this change, it should be
easier to run test262 tests in automation when developing new features.
The new workflow would be, when adding a flag, include the flag in the
test expectations file, and when removing the flag, remove the lines from
the test expectations file. This way, the status file does not have to
change when staging or unstaging, and you get the benefit of the automated
tests before staging starts.

R=adamk
CQ_INCLUDE_TRYBOTS=master.tryserver.v8:v8_linux_noi18n_rel_ng

Review-Url: https://codereview.chromium.org/2601393002
Cr-Commit-Position: refs/heads/master@{#42249}
parent 785cedf1
...@@ -28,13 +28,6 @@ ...@@ -28,13 +28,6 @@
[ [
[ALWAYS, { [ALWAYS, {
###################### NEEDS INVESTIGATION #######################
# Date tests that fail in CE(S)T timezone.
# https://bugs.chromium.org/p/v8/issues/detail?id=5449
'built-ins/Date/prototype/setFullYear/new-value-time-clip': [PASS, FAIL],
'built-ins/Date/prototype/setMonth/new-value-time-clip': [PASS, FAIL],
###################### MISSING ES6 FEATURES ####################### ###################### MISSING ES6 FEATURES #######################
# https://code.google.com/p/v8/issues/detail?id=4248 # https://code.google.com/p/v8/issues/detail?id=4248
...@@ -348,6 +341,58 @@ ...@@ -348,6 +341,58 @@
'built-ins/Number/S9.3.1_A3_T1_U180E': [FAIL], 'built-ins/Number/S9.3.1_A3_T1_U180E': [FAIL],
'built-ins/Number/S9.3.1_A2_U180E': [FAIL], 'built-ins/Number/S9.3.1_A2_U180E': [FAIL],
# https://bugs.chromium.org/p/v8/issues/detail?id=5051
'language/expressions/arrow-function/params-trailing-comma': ['--harmony-trailing-commas'],
'language/expressions/arrow-function/params-trailing-comma-length': ['--harmony-trailing-commas'],
'language/expressions/call/trailing-comma': ['--harmony-trailing-commas'],
'language/expressions/function/params-trailing-comma': ['--harmony-trailing-commas'],
'language/expressions/function/params-trailing-comma-arguments': ['--harmony-trailing-commas'],
'language/expressions/function/params-trailing-comma-length': ['--harmony-trailing-commas'],
'language/expressions/generators/params-trailing-comma': ['--harmony-trailing-commas'],
'language/expressions/generators/params-trailing-comma-arguments': ['--harmony-trailing-commas'],
'language/expressions/generators/params-trailing-comma-length': ['--harmony-trailing-commas'],
'language/expressions/object/method-definition/params-trailing-comma': ['--harmony-trailing-commas'],
'language/expressions/object/method-definition/params-trailing-comma-arguments': ['--harmony-trailing-commas'],
'language/expressions/object/method-definition/params-trailing-comma-length': ['--harmony-trailing-commas'],
'language/statements/class/definition/params-trailing-comma': ['--harmony-trailing-commas'],
'language/statements/class/definition/params-trailing-comma-arguments': ['--harmony-trailing-commas'],
'language/statements/class/definition/params-trailing-comma-length': ['--harmony-trailing-commas'],
'language/statements/function/params-trailing-comma': ['--harmony-trailing-commas'],
'language/statements/function/params-trailing-comma-arguments': ['--harmony-trailing-commas'],
'language/statements/function/params-trailing-comma-length': ['--harmony-trailing-commas'],
'language/statements/generators/params-trailing-comma': ['--harmony-trailing-commas'],
'language/statements/generators/params-trailing-comma-length': ['--harmony-trailing-commas'],
# https://bugs.chromium.org/p/v8/issues/detail?id=4698
'language/expressions/call/tco-call-args': ['--harmony-tailcalls'],
'language/expressions/call/tco-member-args': ['--harmony-tailcalls'],
'language/expressions/comma/tco-final': ['--harmony-tailcalls'],
'language/expressions/conditional/tco-cond': ['--harmony-tailcalls'],
'language/expressions/conditional/tco-pos': ['--harmony-tailcalls'],
'language/expressions/logical-and/tco-right': ['--harmony-tailcalls'],
'language/expressions/logical-or/tco-right': ['--harmony-tailcalls'],
'language/expressions/tagged-template/tco-call': ['--harmony-tailcalls'],
'language/expressions/tagged-template/tco-member': ['--harmony-tailcalls'],
'language/expressions/tco-pos': ['--harmony-tailcalls'],
'language/statements/block/tco-stmt': ['--harmony-tailcalls'],
'language/statements/block/tco-stmt-list': ['--harmony-tailcalls'],
'language/statements/do-while/tco-body': ['--harmony-tailcalls'],
'language/statements/for/tco-const-body': ['--harmony-tailcalls'],
'language/statements/for/tco-let-body': ['--harmony-tailcalls'],
'language/statements/for/tco-lhs-body': ['--harmony-tailcalls'],
'language/statements/for/tco-var-body': ['--harmony-tailcalls'],
'language/statements/if/tco-else-body': ['--harmony-tailcalls'],
'language/statements/if/tco-if-body': ['--harmony-tailcalls'],
'language/statements/labeled/tco': ['--harmony-tailcalls'],
'language/statements/return/tco': ['--harmony-tailcalls'],
'language/statements/switch/tco-case-body': ['--harmony-tailcalls'],
'language/statements/switch/tco-case-body-dflt': ['--harmony-tailcalls'],
'language/statements/switch/tco-dftl-body': ['--harmony-tailcalls'],
'language/statements/try/tco-catch': ['--harmony-tailcalls'],
'language/statements/try/tco-catch-finally': ['--harmony-tailcalls'],
'language/statements/try/tco-finally': ['--harmony-tailcalls'],
'language/statements/while/tco-body': ['--harmony-tailcalls'],
# https://bugs.chromium.org/p/v8/issues/detail?id=5064 # https://bugs.chromium.org/p/v8/issues/detail?id=5064
'language/expressions/arrow-function/params-dflt-duplicates': [FAIL], 'language/expressions/arrow-function/params-dflt-duplicates': [FAIL],
...@@ -422,6 +467,24 @@ ...@@ -422,6 +467,24 @@
# Possible spec bug, see comment in https://github.com/tc39/ecma262/pull/747. # Possible spec bug, see comment in https://github.com/tc39/ecma262/pull/747.
'language/module-code/namespace/internals/delete-non-exported': [SKIP], 'language/module-code/namespace/internals/delete-non-exported': [SKIP],
# https://code.google.com/p/v8/issues/detail?id=4476
'built-ins/String/prototype/toLowerCase/special_casing_conditional': ['--icu-case-mapping'],
'built-ins/String/prototype/toLowerCase/supplementary_plane': ['--icu-case-mapping'],
'built-ins/String/prototype/toUpperCase/supplementary_plane': ['--icu-case-mapping'],
'built-ins/String/prototype/toLocaleLowerCase/Final_Sigma_U180E': ['--icu-case-mapping'],
'built-ins/String/prototype/toLowerCase/Final_Sigma_U180E': ['--icu-case-mapping'],
# https://code.google.com/p/v8/issues/detail?id=4477
'built-ins/String/prototype/toLocaleUpperCase/supplementary_plane': ['--icu-case-mapping'],
'built-ins/String/prototype/toLocaleLowerCase/supplementary_plane': ['--icu-case-mapping'],
'built-ins/String/prototype/toLocaleLowerCase/special_casing_conditional': ['--icu-case-mapping'],
'intl402/String/prototype/toLocaleLowerCase/special_casing_Azeri': ['--icu-case-mapping'],
'intl402/String/prototype/toLocaleLowerCase/special_casing_Lithuanian': ['--icu-case-mapping'],
'intl402/String/prototype/toLocaleLowerCase/special_casing_Turkish': ['--icu-case-mapping'],
'intl402/String/prototype/toLocaleUpperCase/special_casing_Azeri': ['--icu-case-mapping'],
'intl402/String/prototype/toLocaleUpperCase/special_casing_Lithuanian': ['--icu-case-mapping'],
'intl402/String/prototype/toLocaleUpperCase/special_casing_Turkish': ['--icu-case-mapping'],
######################## NEEDS INVESTIGATION ########################### ######################## NEEDS INVESTIGATION ###########################
# These test failures are specific to the intl402 suite and need investigation # These test failures are specific to the intl402 suite and need investigation
...@@ -489,6 +552,11 @@ ...@@ -489,6 +552,11 @@
# https://github.com/tc39/test262/commit/c779cba592fcb39051fd1d467e472ea71aa2b1d6#commitcomment-20276482 # https://github.com/tc39/test262/commit/c779cba592fcb39051fd1d467e472ea71aa2b1d6#commitcomment-20276482
'language/statements/class/strict-mode/arguments-caller': [FAIL], 'language/statements/class/strict-mode/arguments-caller': [FAIL],
# Date tests that fail in CE(S)T timezone.
# https://bugs.chromium.org/p/v8/issues/detail?id=5449
'built-ins/Date/prototype/setFullYear/new-value-time-clip': [PASS, FAIL],
'built-ins/Date/prototype/setMonth/new-value-time-clip': [PASS, FAIL],
############################ SKIPPED TESTS ############################# ############################ SKIPPED TESTS #############################
# These tests take a looong time to run. # These tests take a looong time to run.
......
...@@ -142,7 +142,7 @@ class Test262TestSuite(testsuite.TestSuite): ...@@ -142,7 +142,7 @@ class Test262TestSuite(testsuite.TestSuite):
def GetFlagsForTestCase(self, testcase, context): def GetFlagsForTestCase(self, testcase, context):
return (testcase.flags + context.mode_flags + self.harness + return (testcase.flags + context.mode_flags + self.harness +
self.GetIncludesForTest(testcase) + ["--harmony"] + self.GetIncludesForTest(testcase) +
(["--module"] if "module" in self.GetTestRecord(testcase) else []) + (["--module"] if "module" in self.GetTestRecord(testcase) else []) +
[self.GetPathForTest(testcase)] + [self.GetPathForTest(testcase)] +
(["--throws"] if "negative" in self.GetTestRecord(testcase) (["--throws"] if "negative" in self.GetTestRecord(testcase)
...@@ -150,7 +150,8 @@ class Test262TestSuite(testsuite.TestSuite): ...@@ -150,7 +150,8 @@ class Test262TestSuite(testsuite.TestSuite):
(["--allow-natives-syntax"] (["--allow-natives-syntax"]
if "detachArrayBuffer.js" in if "detachArrayBuffer.js" in
self.GetTestRecord(testcase).get("includes", []) self.GetTestRecord(testcase).get("includes", [])
else [])) else []) +
([flag for flag in testcase.outcomes if flag.startswith("--")]))
def _VariantGeneratorFactory(self): def _VariantGeneratorFactory(self):
return Test262VariantGenerator return Test262VariantGenerator
...@@ -221,7 +222,9 @@ class Test262TestSuite(testsuite.TestSuite): ...@@ -221,7 +222,9 @@ class Test262TestSuite(testsuite.TestSuite):
if (statusfile.FAIL_SLOPPY in testcase.outcomes and if (statusfile.FAIL_SLOPPY in testcase.outcomes and
"--use-strict" not in testcase.flags): "--use-strict" not in testcase.flags):
return outcome != statusfile.FAIL return outcome != statusfile.FAIL
return not outcome in (testcase.outcomes or [statusfile.PASS]) return not outcome in ([outcome for outcome in testcase.outcomes
if not outcome.startswith('--')]
or [statusfile.PASS])
def PrepareSources(self): def PrepareSources(self):
# The archive is created only on swarming. Local checkouts have the # The archive is created only on swarming. Local checkouts have the
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment