If an allocation is so huge that we cannot code the size needed in the failure

object then we just return an out of memory failure object (instead of a retry
after GC failure object).  Not all places that checked for retry-after-GC were
able to handle an immediate out of memory failure.

This fixes http://code.google.com/p/v8/issues/detail?id=70
Review URL: http://codereview.chromium.org/6340

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@477 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/builtins.cc

@@ -674,11 +674,16 @@ void Builtins::Setup(bool create_heap_objects) {
       masm.GetCode(&desc);
       Code::Flags flags =  functions[i].flags;
       Object* code = Heap::CreateCode(desc, NULL, flags);
+      if (code->IsFailure()) {
         if (code->IsRetryAfterGC()) {
           CHECK(Heap::CollectGarbage(Failure::cast(code)->requested(),
                                      Failure::cast(code)->allocation_space()));
           code = Heap::CreateCode(desc, NULL, flags);
         }
+        if (code->IsFailure()) {
+          v8::internal::V8::FatalProcessOutOfMemory("CreateCode");
+        }
+      }
       // Add any unresolved jumps or calls to the fixup list in the
       // bootstrapper.
       Bootstrapper::AddFixup(Code::cast(code), &masm);

src/factory.cc

@@ -467,10 +467,12 @@ Handle<DescriptorArray> Factory::CopyAppendProxyDescriptor(
   GC_GREEDY_CHECK();
   CallbacksDescriptor desc(*key, *value, attributes);
   Object* obj = array->CopyInsert(&desc, REMOVE_TRANSITIONS);
+  if (obj->IsFailure()) {
     if (obj->IsRetryAfterGC()) {
       CALL_GC(obj);
       CallbacksDescriptor desc(*key, *value, attributes);
       obj = array->CopyInsert(&desc, REMOVE_TRANSITIONS);
+    }
     if (obj->IsFailure()) {
       // TODO(1181417): Fix this.
       V8::FatalProcessOutOfMemory("CopyAppendProxyDescriptor");

src/handles.cc

@@ -128,7 +128,7 @@ void TransformToFastProperties(Handle<JSObject> object,
 
 void FlattenString(Handle<String> string) {
   if (string->IsFlat()) return;
-  CALL_HEAP_FUNCTION_VOID(String::cast(*string)->Flatten());
+  CALL_HEAP_FUNCTION_VOID(string->Flatten());
   ASSERT(string->IsFlat());
 }
 

src/heap-inl.h

@@ -183,6 +183,9 @@ OldSpace* Heap::TargetSpace(HeapObject* object) {
           return Handle<TYPE>();                                             \
         }                                                                    \
       } else {                                                               \
+        if (__object__->IsOutOfMemoryFailure()) {                            \
+          v8::internal::V8::FatalProcessOutOfMemory("CALL_HEAP_FUNCTION");   \
+        }                                                                    \
         return Handle<TYPE>();                                               \
       }                                                                      \
     }                                                                        \
@@ -211,7 +214,10 @@ OldSpace* Heap::TargetSpace(HeapObject* object) {
         return;                                                     \
       }                                                             \
     } else {                                                        \
-      return;                                                       \
+      if (__object__->IsOutOfMemoryFailure()) {                     \
+         V8::FatalProcessOutOfMemory("Handles");                    \
+      }                                                             \
+      UNREACHABLE();                                                \
     }                                                               \
   }
 

src/jsregexp.cc

@@ -216,10 +216,14 @@ Handle<Object> RegExpImpl::JsreCompile(Handle<JSRegExp> re,
   unsigned number_of_captures;
   const char* error_message = NULL;
 
-  malloc_failure = Failure::Exception();
-  JscreRegExp* code;
+  JscreRegExp* code = NULL;
   FlattenString(pattern);
 
+  bool first_time = true;
+
+  while (true) {
+    first_time = false;
+    malloc_failure = Failure::Exception();
     if (pattern->IsAsciiRepresentation()) {
       Vector<const char> contents = pattern->ToAsciiVector();
       code = jsRegExpCompile(contents.start(),
@@ -241,53 +245,30 @@ Handle<Object> RegExpImpl::JsreCompile(Handle<JSRegExp> re,
                              &JSREMalloc,
                              &JSREFree);
     }
-
-  if (code == NULL && malloc_failure->IsRetryAfterGC()) {
-    // Performs a GC, then retries.
+    if (code == NULL) {
+      if (first_time && malloc_failure->IsRetryAfterGC()) {
         if (!Heap::CollectGarbage(malloc_failure->requested(),
                                   malloc_failure->allocation_space())) {
           // TODO(1181417): Fix this.
           V8::FatalProcessOutOfMemory("RegExpImpl::JsreCompile");
         }
-    malloc_failure = Failure::Exception();
-
-    if (pattern->IsAsciiRepresentation()) {
-      Vector<const char> contents = pattern->ToAsciiVector();
-      code = jsRegExpCompile(contents.start(),
-                             contents.length(),
-                             case_option,
-                             multiline_option,
-                             &number_of_captures,
-                             &error_message,
-                             &JSREMalloc,
-                             &JSREFree);
-    } else {
-      Vector<const uc16> contents = pattern->ToUC16Vector();
-      code = jsRegExpCompile(contents.start(),
-                             contents.length(),
-                             case_option,
-                             multiline_option,
-                             &number_of_captures,
-                             &error_message,
-                             &JSREMalloc,
-                             &JSREFree);
+        continue;
       }
-
-    if (code == NULL && malloc_failure->IsRetryAfterGC()) {
+      if (malloc_failure->IsRetryAfterGC() ||
+          malloc_failure->IsOutOfMemoryFailure()) {
         // TODO(1181417): Fix this.
         V8::FatalProcessOutOfMemory("RegExpImpl::JsreCompile");
-    }
-  }
-
-  if (error_message != NULL) {
+      } else {
         // Throw an exception.
         Handle<JSArray> array = Factory::NewJSArray(2);
         SetElement(array, 0, pattern);
-    SetElement(array, 1, Factory::NewStringFromUtf8(CStrVector(error_message)));
+        SetElement(array, 1, Factory::NewStringFromUtf8(CStrVector(
+            (error_message == NULL) ? "Unknown regexp error" : error_message)));
         Handle<Object> regexp_err =
             Factory::NewSyntaxError("malformed_regexp", array);
         return Handle<Object>(Top::Throw(*regexp_err));
       }
+    }
 
     ASSERT(code != NULL);
 
@@ -302,6 +283,7 @@ Handle<Object> RegExpImpl::JsreCompile(Handle<JSRegExp> re,
     re->set_data(*value);
 
     return re;
+  }
 }
 
 

src/objects-inl.h

@@ -211,6 +211,12 @@ bool Object::IsRetryAfterGC() {
 }
 
 
+bool Object::IsOutOfMemoryFailure() {
+  return HAS_FAILURE_TAG(this)
+    && Failure::cast(this)->IsOutOfMemoryException();
+}
+
+
 bool Object::IsException() {
   return this == Failure::Exception();
 }

src/objects.h

@@ -601,6 +601,7 @@ class Object BASE_EMBEDDED {
   inline bool IsByteArray();
   inline bool IsFailure();
   inline bool IsRetryAfterGC();
+  inline bool IsOutOfMemoryFailure();
   inline bool IsException();
   inline bool IsJSObject();
   inline bool IsMap();

test/cctest/test-api.cc

@@ -2341,6 +2341,33 @@ TEST(ErrorReporting) {
 }
 
 
+static const char* js_code_causing_huge_string_flattening =
+    "var str = 'X';"
+    "for (var i = 0; i < 29; i++) {"
+    "  str = str + str;"
+    "}"
+    "str.match(/X/);";
+
+
+void OOMCallback(const char* location, const char* message) {
+  exit(0);
+}
+
+
+TEST(RegexpOutOfMemory) {
+  // Execute a script that causes out of memory when flattening a string.
+  v8::HandleScope scope;
+  v8::V8::SetFatalErrorHandler(OOMCallback);
+  LocalContext context;
+  Local<Script> script =
+      Script::Compile(String::New(js_code_causing_huge_string_flattening));
+  last_location = NULL;
+  Local<Value> result = script->Run();
+
+  CHECK(false);  // Should not return.
+}
+
+
 static void MissingScriptInfoMessageListener(v8::Handle<v8::Message> message,
                                              v8::Handle<Value> data) {
   CHECK_EQ(v8::Undefined(), data);