[regexp] fix assertion failure when parsing close to stack overflow.

R=jkummerow@chromium.org
BUG=chromium:600257
LOG=N

Review URL: https://codereview.chromium.org/1884143002

Cr-Commit-Position: refs/heads/master@{#35496}

src/regexp/regexp-parser.cc

@@ -130,6 +130,7 @@ bool RegExpParser::IsSyntaxCharacterOrSlash(uc32 c) {
 
 
 RegExpTree* RegExpParser::ReportError(Vector<const char> message) {
+  if (failed_) return NULL;  // Do not overwrite any existing error.
   failed_ = true;
   *error_ = isolate()->factory()->NewStringFromAscii(message).ToHandleChecked();
   // Zip to the end to make sure the no more input is read.
@@ -511,9 +512,8 @@ RegExpTree* RegExpParser::ParseDisjunction() {
         break;
       case '{': {
         int dummy;
-        if (ParseIntervalQuantifier(&dummy, &dummy)) {
-          return ReportError(CStrVector("Nothing to repeat"));
-        }
+        bool parsed = ParseIntervalQuantifier(&dummy, &dummy CHECK_FAILED);
+        if (parsed) return ReportError(CStrVector("Nothing to repeat"));
         // fallthrough
       }
       case '}':

test/mjsunit/regress/regress-crbug-600257.js

+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --stack-size=100
+
+(function rec() {
+  try {
+    rec();
+  } catch (e) {
+    /{/;
+  }
+})();