[regexp] Properly handle failed RegExp compilations

Compilation can fail e.g. on stack overflow. This ensures that we exit
early from StringReplaceGlobalRegExpWithString in that case.

BUG=v8:5437,chromium:705934

Review-Url: https://codereview.chromium.org/2778953004
Cr-Commit-Position: refs/heads/master@{#44215}

src/runtime/runtime-regexp.cc

@@ -606,7 +606,10 @@ MUST_USE_RESULT static Object* StringReplaceGlobalRegExpWithString(
   JSRegExp::Type typeTag = regexp->TypeTag();
   if (typeTag == JSRegExp::IRREGEXP) {
     // Ensure the RegExp is compiled so we can access the capture-name map.
-    RegExpImpl::IrregexpPrepare(regexp, subject);
+    if (RegExpImpl::IrregexpPrepare(regexp, subject) == -1) {
+      DCHECK(isolate->has_pending_exception());
+      return isolate->heap()->exception();
+    }
   }
 
   // CompiledReplacement uses zone allocation.

test/mjsunit/regress/regress-705934.js

+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --harmony-regexp-named-captures --stack-size=100
+
+function call_replace_close_to_stack_overflow() {
+  try {
+    call_replace_close_to_stack_overflow();
+  } catch(e) {
+    "b".replace(/(b)/g);
+  }
+}
+
+call_replace_close_to_stack_overflow();