[wasm] Run GC if page allocation fails, then retry

This adds another instance of the "if allocation fails, run GC then
retry" pattern, this time for making the actual memory reservation for
wasm memory.

R=mlippautz@chromium.org

Bug: chromium:883639, v8:7872, v8:8158
Change-Id: I40ed020ed2bbc253c4bbcbe51e3e9f5a0278d7a1
Reviewed-on: https://chromium-review.googlesource.com/1227117Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Commit-Queue: Clemens Hammacher <clemensh@chromium.org>
Cr-Commit-Position: refs/heads/master@{#55936}

src/wasm/wasm-code-manager.cc

@@ -893,27 +893,37 @@ std::unique_ptr<NativeModule> WasmCodeManager::NewNativeModule(
         ->MemoryPressureNotification(MemoryPressureLevel::kCritical);
   }
 
-  VirtualMemory mem;
   // If the code must be contiguous, reserve enough address space up front.
   size_t vmem_size = kRequiresCodeRange ? kMaxWasmCodeMemory : memory_estimate;
-  TryAllocate(vmem_size, &mem);
-  if (mem.IsReserved()) {
-    Address start = mem.address();
-    size_t size = mem.size();
-    Address end = mem.end();
-    std::unique_ptr<NativeModule> ret(
-        new NativeModule(isolate, enabled, can_request_more, std::move(mem),
-                         this, std::move(module), env));
-    TRACE_HEAP("New NativeModule %p: Mem: %" PRIuPTR ",+%zu\n", this, start,
-               size);
-    base::LockGuard<base::Mutex> lock(&native_modules_mutex_);
-    AssignRanges(start, end, ret.get());
-    native_modules_.emplace(ret.get());
-    return ret;
+  // Try up to three times; getting rid of dead JSArrayBuffer allocations might
+  // require two GCs because the first GC maybe incremental and may have
+  // floating garbage.
+  static constexpr int kAllocationRetries = 2;
+  VirtualMemory mem;
+  for (int retries = 0;; ++retries) {
+    TryAllocate(vmem_size, &mem);
+    if (mem.IsReserved()) break;
+    if (retries == kAllocationRetries) {
+      V8::FatalProcessOutOfMemory(isolate, "WasmCodeManager::NewNativeModule");
+      return nullptr;
+    }
+    // Run one GC, then try the allocation again.
+    isolate->heap()->MemoryPressureNotification(MemoryPressureLevel::kCritical,
+                                                true);
   }
 
-  V8::FatalProcessOutOfMemory(isolate, "WasmCodeManager::NewNativeModule");
-  return nullptr;
+  Address start = mem.address();
+  size_t size = mem.size();
+  Address end = mem.end();
+  std::unique_ptr<NativeModule> ret(
+      new NativeModule(isolate, enabled, can_request_more, std::move(mem), this,
+                       std::move(module), env));
+  TRACE_HEAP("New NativeModule %p: Mem: %" PRIuPTR ",+%zu\n", this, start,
+             size);
+  base::LockGuard<base::Mutex> lock(&native_modules_mutex_);
+  AssignRanges(start, end, ret.get());
+  native_modules_.emplace(ret.get());
+  return ret;
 }
 
 bool NativeModule::SetExecutable(bool executable) {