Correctly check length when allocating string.

R=jkummerow@chromium.org
BUG=
TEST=regress-regexp-overflow.js

Review URL: https://chromiumcodereview.appspot.com/10538012

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@11720 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/runtime.cc

@@ -2985,7 +2985,15 @@ MUST_USE_RESULT static MaybeObject* StringReplaceAtomRegExpWithString(
   int matches = indices.length();
   if (matches == 0) return *subject;
 
-  int result_len = (replacement_len - pattern_len) * matches + subject_len;
+  // Detect integer overflow.
+  int64_t result_len_64 =
+      (static_cast<int64_t>(replacement_len) -
+       static_cast<int64_t>(pattern_len)) *
+      static_cast<int64_t>(matches) +
+      static_cast<int64_t>(subject_len);
+  if (result_len_64 > INT_MAX) return Failure::OutOfMemoryException();
+  int result_len = static_cast<int>(result_len_64);
+
   int subject_pos = 0;
   int result_pos = 0;
 

test/cctest/test-strings.cc

@@ -672,3 +672,20 @@ TEST(RobustSubStringStub) {
   CompileRun("var slice = long.slice(1, 15);");
   CheckException("%_SubString(slice, 0, 17);");
 }
+
+
+TEST(RegExpOverflow) {
+  // Result string has the length 2^32, causing a 32-bit integer overflow.
+  InitializeVM();
+  HandleScope scope;
+  LocalContext context;
+  v8::V8::IgnoreOutOfMemoryException();
+  v8::Local<v8::Value> result = CompileRun(
+      "var a = 'a';                     "
+      "for (var i = 0; i < 16; i++) {   "
+      "  a += a;                        "
+      "}                                "
+      "a.replace(/a/g, a);              ");
+  CHECK(result.IsEmpty());
+  CHECK(context->HasOutOfMemoryException());
+}