Make test262 test runner check for which exception is thrown

test262 "negative" test expectations list which exception is thrown. The ES2017
draft specification is very specific about which exception class is thrown
from which path, and V8 works hard to be correct with respect to that spec.

Previously, the test262 test runner would accept any nonzero status code,
such as from a crash, or a FAIL printed out, for a negative test. This
patch makes negative tests check for the right answer using a quick-and-dirty
parsing of the exception printing from d8 to find the exception class.
It invokes d8 in a way to get a status code of 0 from thrown exceptions
so that 'negative' tests aren't actually implemented by negating the output.

Amazingly, this didn't catch any test262 failures, but I verified the extra
checking interactively by changing a negative test to expect a different type
and saw it fail.

BUG=v8:4803
R=machenbach
LOG=Y

Review URL: https://codereview.chromium.org/1766503002

Cr-Commit-Position: refs/heads/master@{#34763}

test/message/testcfg.py

@@ -94,7 +94,9 @@ class MessageTestSuite(testsuite.TestSuite):
             string.find("Native Client module will be loaded") > 0 or
             string.find("NaClHostDescOpen:") > 0)
 
-  def IsFailureOutput(self, output, testpath):
+  def IsFailureOutput(self, testcase):
+    output = testcase.output
+    testpath = testcase.path
     expected_path = os.path.join(self.root, testpath + ".out")
     expected_lines = []
     # Can't use utils.ReadLinesFrom() here because it strips whitespace.

test/mozilla/testcfg.py

@@ -107,10 +107,10 @@ class MozillaTestSuite(testsuite.TestSuite):
   def IsNegativeTest(self, testcase):
     return testcase.path.endswith("-n")
 
-  def IsFailureOutput(self, output, testpath):
-    if output.exit_code != 0:
+  def IsFailureOutput(self, testcase):
+    if testcase.output.exit_code != 0:
       return True
-    return "FAILED!" in output.stdout
+    return "FAILED!" in testcase.output.stdout
 
   def DownloadData(self):
     print "Mozilla download is deprecated. It's part of DEPS."

test/promises-aplus/testcfg.py

@@ -91,11 +91,11 @@ class PromiseAplusTestSuite(testsuite.TestSuite):
   def IsNegativeTest(self, testcase):
     return '@negative' in self.GetSourceForTest(testcase)
 
-  def IsFailureOutput(self, output, testpath):
-    if output.exit_code != 0:
+  def IsFailureOutput(self, testcase):
+    if testcase.output.exit_code != 0:
       return True
-    return not 'All tests have run.' in output.stdout or \
-           'FAIL:' in output.stdout
+    return not 'All tests have run.' in testcase.output.stdout or \
+           'FAIL:' in testcase.output.stdout
 
   def DownloadTestData(self):
     archive = os.path.join(self.root, TEST_ARCHIVE)

test/simdjs/testcfg.py

@@ -48,10 +48,10 @@ class SimdJsTestSuite(testsuite.TestSuite):
   def IsNegativeTest(self, testcase):
     return False
 
-  def IsFailureOutput(self, output, testpath):
-    if output.exit_code != 0:
+  def IsFailureOutput(self, testcase):
+    if testcase.output.exit_code != 0:
       return True
-    return "FAILED!" in output.stdout
+    return "FAILED!" in testcase.output.stdout
 
   def DownloadData(self):
     print "SimdJs download is deprecated. It's part of DEPS."

test/test262/testcfg.py

@@ -129,7 +129,8 @@ class Test262TestSuite(testsuite.TestSuite):
   def GetFlagsForTestCase(self, testcase, context):
     return (testcase.flags + context.mode_flags + self.harness +
             self.GetIncludesForTest(testcase) + ["--harmony"] +
-            [os.path.join(self.testroot, testcase.path + ".js")])
+            [os.path.join(self.testroot, testcase.path + ".js")] +
+            (["--throws"] if "negative" in self.GetTestRecord(testcase) else []))
 
   def _VariantGeneratorFactory(self):
     return Test262VariantGenerator
@@ -144,7 +145,7 @@ class Test262TestSuite(testsuite.TestSuite):
         self.ParseTestRecord = module.parseTestRecord
       except:
         raise ImportError("Cannot load parseTestRecord; you may need to "
-                          "--download-data for test262")
+                          "gclient sync for test262")
       finally:
         if f:
           f.close()
@@ -171,13 +172,20 @@ class Test262TestSuite(testsuite.TestSuite):
     with open(filename) as f:
       return f.read()
 
-  def IsNegativeTest(self, testcase):
-    test_record = self.GetTestRecord(testcase)
-    return "negative" in test_record
+  def _ParseException(self, str):
+    for line in str.split("\n")[::-1]:
+      if line and not line[0].isspace() and ":" in line:
+        return line.split(":")[0]
+
 
-  def IsFailureOutput(self, output, testpath):
+  def IsFailureOutput(self, testcase):
+    output = testcase.output
+    test_record = self.GetTestRecord(testcase)
     if output.exit_code != 0:
       return True
+    if "negative" in test_record:
+      if self._ParseException(output.stdout) != test_record["negative"]:
+        return True
     return "FAILED!" in output.stdout
 
   def HasUnexpectedOutput(self, testcase):

test/webkit/testcfg.py

@@ -117,10 +117,10 @@ class WebkitTestSuite(testsuite.TestSuite):
             string == "Warning: unknown flag --enable-slow-asserts." or
             string == "Try --help for options")
 
-  def IsFailureOutput(self, output, testpath):
-    if super(WebkitTestSuite, self).IsFailureOutput(output, testpath):
+  def IsFailureOutput(self, testcase):
+    if super(WebkitTestSuite, self).IsFailureOutput(testcase):
       return True
-    file_name = os.path.join(self.root, testpath) + "-expected.txt"
+    file_name = os.path.join(self.root, testcase.path) + "-expected.txt"
     with file(file_name, "r") as expected:
       expected_lines = expected.readlines()
 
@@ -136,7 +136,7 @@ class WebkitTestSuite(testsuite.TestSuite):
 
     def ActBlockIterator():
       """Iterates over blocks of actual output lines."""
-      lines = output.stdout.splitlines()
+      lines = testcase.output.stdout.splitlines()
       start_index = 0
       found_eqeq = False
       for index, line in enumerate(lines):
@@ -147,7 +147,7 @@ class WebkitTestSuite(testsuite.TestSuite):
             found_eqeq = True
           else:
             yield ActIterator(lines[start_index:index])
-          # The next block of ouput lines starts after the separator.
+          # The next block of output lines starts after the separator.
           start_index = index + 1
       # Iterate over complete output if no separator was found.
       if not found_eqeq:

tools/testrunner/local/testsuite.py

@@ -256,14 +256,14 @@ class TestSuite(object):
   def GetSourceForTest(self, testcase):
     return "(no source available)"
 
-  def IsFailureOutput(self, output, testpath):
-    return output.exit_code != 0
+  def IsFailureOutput(self, testcase):
+    return testcase.output.exit_code != 0
 
   def IsNegativeTest(self, testcase):
     return False
 
   def HasFailed(self, testcase):
-    execution_failed = self.IsFailureOutput(testcase.output, testcase.path)
+    execution_failed = self.IsFailureOutput(testcase)
     if self.IsNegativeTest(testcase):
       return not execution_failed
     else: