[wasm] Throw an exception when wasm memory allocation fails

V8 crashed with a FATAL when memory allocation during instantiation
failed. With this CL, a RangeError is thrown instead.

This is not the only possible OOM that can happen during the startup of
a WebAssembly app, but since the allocation of WebAssembly memory is
among the biggest allocations, this change may already prevent several
crashes.

R=clemensb@chromium.org

Bug: chromium:1268898
Change-Id: I9376830ba2fe9df62b5595b6b19c92e35a75dfda
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3380586Reviewed-by: Clemens Backes <clemensb@chromium.org>
Commit-Queue: Andreas Haas <ahaas@chromium.org>
Cr-Commit-Position: refs/heads/main@{#78569}

src/objects/backing-store.cc

@@ -479,8 +479,10 @@ std::unique_ptr<BackingStore> BackingStore::TryAllocateAndPartiallyCommitMemory(
   if (!gc_retry(commit_memory)) {
     TRACE_BS("BSw:try   failed to set permissions (%p, %zu)\n", buffer_start,
              committed_byte_length);
+    CHECK(FreePages(page_allocator, allocation_base, reservation_size));
     // SetPermissions put us over the process memory limit.
-    V8::FatalProcessOutOfMemory(nullptr, "BackingStore::AllocateMemory()");
+    // We return an empty result so that the caller can throw an exception.
+    return {};
   }
 
   DebugCheckZero(buffer_start, byte_length);  // touch the bytes.

src/objects/value-serializer.cc

@@ -2034,7 +2034,7 @@ MaybeHandle<WasmMemoryObject> ValueDeserializer::ReadWasmMemory() {
   }
 
   Handle<WasmMemoryObject> result =
-      WasmMemoryObject::New(isolate_, buffer, maximum_pages);
+      WasmMemoryObject::New(isolate_, buffer, maximum_pages).ToHandleChecked();
 
   AddObjectWithID(id, result);
   return result;

src/wasm/module-instantiate.cc

@@ -562,7 +562,8 @@ MaybeHandle<WasmInstanceObject> InstanceBuilder::Build() {
         static_cast<int>(RoundUp(buffer->byte_length(), wasm::kWasmPageSize) /
                          wasm::kWasmPageSize);
     memory_object_ =
-        WasmMemoryObject::New(isolate_, memory_buffer_, maximum_pages);
+        WasmMemoryObject::New(isolate_, memory_buffer_, maximum_pages)
+            .ToHandleChecked();
   } else {
     // Actual wasm module must have either imported or created memory.
     CHECK(memory_buffer_.is_null());

src/wasm/wasm-objects.cc

@@ -799,13 +799,14 @@ void SetInstanceMemory(Handle<WasmInstanceObject> instance,
 }
 }  // namespace
 
-Handle<WasmMemoryObject> WasmMemoryObject::New(
+MaybeHandle<WasmMemoryObject> WasmMemoryObject::New(
     Isolate* isolate, MaybeHandle<JSArrayBuffer> maybe_buffer, int maximum) {
   Handle<JSArrayBuffer> buffer;
   if (!maybe_buffer.ToHandle(&buffer)) {
     // If no buffer was provided, create a zero-length one.
     auto backing_store =
         BackingStore::AllocateWasmMemory(isolate, 0, 0, SharedFlag::kNotShared);
+    if (!backing_store) return {};
     buffer = isolate->factory()->NewJSArrayBuffer(std::move(backing_store));
   }
 

src/wasm/wasm-objects.h

@@ -255,7 +255,7 @@ class WasmMemoryObject
                                             Handle<WasmInstanceObject> object);
   inline bool has_maximum_pages();
 
-  V8_EXPORT_PRIVATE static Handle<WasmMemoryObject> New(
+  V8_EXPORT_PRIVATE static MaybeHandle<WasmMemoryObject> New(
       Isolate* isolate, MaybeHandle<JSArrayBuffer> buffer, int maximum);
 
   V8_EXPORT_PRIVATE static MaybeHandle<WasmMemoryObject> New(Isolate* isolate,

test/unittests/objects/value-serializer-unittest.cc

@@ -2200,7 +2200,7 @@ TEST_F(ValueSerializerTestWithSharedArrayBufferClone,
     i::Isolate* i_isolate = reinterpret_cast<i::Isolate*>(isolate());
     i::Handle<i::JSArrayBuffer> obj = Utils::OpenHandle(*input_buffer());
     input = Utils::Convert<i::WasmMemoryObject, Value>(
-        i::WasmMemoryObject::New(i_isolate, obj, kMaxPages));
+        i::WasmMemoryObject::New(i_isolate, obj, kMaxPages).ToHandleChecked());
   }
   RoundTripTest(input);
   ExpectScriptTrue("result instanceof WebAssembly.Memory");