[regexp] catch stack overflow when parsing back references.

R=jkummerow@chromium.org
BUG=chromium:595657
LOG=N

Review URL: https://codereview.chromium.org/1811913006

Cr-Commit-Position: refs/heads/master@{#34894}

src/regexp/regexp-parser.cc

@@ -387,7 +387,8 @@ RegExpTree* RegExpParser::ParseDisjunction() {
           case '8':
           case '9': {
             int index = 0;
-            if (ParseBackReferenceIndex(&index)) {
+            bool is_backref = ParseBackReferenceIndex(&index CHECK_FAILED);
+            if (is_backref) {
               if (state->IsInsideCaptureGroup(index)) {
                 // The back reference is inside the capture group it refers to.
                 // Nothing can possibly have been captured yet, so we use empty

test/mjsunit/regress/regress-crbug-595657.js

+// Copyright 2016 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --stack-size=100
+
+function test() {
+  try {
+    test();
+  } catch(e) {
+    /(\2)(a)/.test("");
+  }
+}
+
+test();