Test262-es6 test runner should handle sloppy fail better

This adds a new FAIL_SLOPPY expected output. It then uses this to
determine if the test case has unexpected output.

BUG=v8:4164
LOG=N
R=machenbach@chromium.org

Review URL: https://codereview.chromium.org/1197913002

Cr-Commit-Position: refs/heads/master@{#29194}

test/test262-es6/test262-es6.status

@@ -272,11 +272,29 @@
 
   'language/asi/S7.9_A5.7_T1': [PASS, FAIL_OK],
   'language/class/*': [PASS, ['sloppy', FAIL]],
-  'language/computed-property-names/class/*': [PASS, ['sloppy', FAIL]],
-  'language/computed-property-names/to-name-side-effects/class': [PASS, ['sloppy', FAIL]],
-  'language/computed-property-names/to-name-side-effects/numbers-class': [PASS, ['sloppy', FAIL]],
+
+  # The order of adding the name property is wrong
+  # https://code.google.com/p/v8/issues/detail?id=4199
+  'language/computed-property-names/class/static/method-number': [FAIL, FAIL_SLOPPY],
+  'language/computed-property-names/class/static/method-symbol': [FAIL, FAIL_SLOPPY],
+  'language/computed-property-names/class/static/method-string': [FAIL, FAIL_SLOPPY],
+
+  # https://code.google.com/p/v8/issues/detail?id=3305
+  'language/computed-property-names/to-name-side-effects/class': [PASS, FAIL_SLOPPY],
+  'language/computed-property-names/to-name-side-effects/numbers-class': [PASS, FAIL_SLOPPY],
+  'language/computed-property-names/class/accessor/*': [PASS, FAIL_SLOPPY],
+  'language/computed-property-names/class/method/*': [PASS, FAIL_SLOPPY],
+  'language/computed-property-names/class/static/generator-constructor': [PASS, FAIL_SLOPPY],
+  'language/computed-property-names/class/static/generator-prototype': [PASS, FAIL_SLOPPY],
+  'language/computed-property-names/class/static/getter-constructor': [PASS, FAIL_SLOPPY],
+  'language/computed-property-names/class/static/getter-prototype': [PASS, FAIL_SLOPPY],
+  'language/computed-property-names/class/static/method-constructor': [PASS, FAIL_SLOPPY],
+  'language/computed-property-names/class/static/method-prototype': [PASS, FAIL_SLOPPY],
+  'language/computed-property-names/class/static/setter-constructor': [PASS, FAIL_SLOPPY],
+  'language/computed-property-names/class/static/setter-prototype': [PASS, FAIL_SLOPPY],
 
   # new.target
+  # https://code.google.com/p/v8/issues/detail?id=3887
   'language/expressions/arrow-function/lexical-new.target': [FAIL],
   'language/expressions/arrow-function/lexical-new.target-closure-returned': [FAIL],
 

test/test262-es6/testcfg.py

@@ -33,6 +33,7 @@ import sys
 import tarfile
 import imp
 
+from testrunner.local import statusfile
 from testrunner.local import testsuite
 from testrunner.local import utils
 from testrunner.objects import testcase
@@ -136,6 +137,13 @@ class Test262TestSuite(testsuite.TestSuite):
       return True
     return "FAILED!" in output.stdout
 
+  def HasUnexpectedOutput(self, testcase):
+    outcome = self.GetOutcome(testcase)
+    if (statusfile.FAIL_SLOPPY in testcase.outcomes and
+        "--use-strict" not in testcase.flags):
+      return outcome != statusfile.FAIL
+    return not outcome in (testcase.outcomes or [statusfile.PASS])
+
   def DownloadData(self):
     revision = TEST_262_ARCHIVE_REVISION
     archive_url = TEST_262_URL % revision

tools/testrunner/local/statusfile.py

@@ -40,12 +40,13 @@ NO_VARIANTS = "NO_VARIANTS"
 # These are just for the status files and are mapped below in DEFS:
 FAIL_OK = "FAIL_OK"
 PASS_OR_FAIL = "PASS_OR_FAIL"
+FAIL_SLOPPY = "FAIL_SLOPPY"
 
 ALWAYS = "ALWAYS"
 
 KEYWORDS = {}
 for key in [SKIP, FAIL, PASS, OKAY, TIMEOUT, CRASH, SLOW, FLAKY, FAIL_OK,
-            FAST_VARIANTS, NO_VARIANTS, PASS_OR_FAIL, ALWAYS]:
+            FAST_VARIANTS, NO_VARIANTS, PASS_OR_FAIL, FAIL_SLOPPY, ALWAYS]:
   KEYWORDS[key] = key
 
 DEFS = {FAIL_OK: [FAIL, OKAY],