[test] Better error handling for mock allocator

Not just the large virtual allocation but *all* memory operations
should handle resource exhaustion gracefully.

Bug: chromium:1042943
Change-Id: I8d36a3d7fa267b588b35b927172bfe56f64fffd2
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2007489
Auto-Submit: Jakob Kummerow <jkummerow@chromium.org>
Commit-Queue: Clemens Backes <clemensb@chromium.org>
Reviewed-by: Clemens Backes <clemensb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#65836}

src/d8/d8.cc

@@ -236,15 +236,23 @@ class MultiMappedAllocator : public ArrayBufferAllocatorBase {
     int flags = MAP_SHARED | MAP_ANONYMOUS;
     void* real_alloc = mmap(nullptr, kChunkSize, prot, flags, -1, 0);
     if (reinterpret_cast<intptr_t>(real_alloc) == -1) {
+      // If we ran into some limit (physical or virtual memory, or number
+      // of mappings, etc), return {nullptr}, which callers can handle.
+      if (errno == ENOMEM) {
+        return nullptr;
+      }
+      // Other errors may be bugs which we want to learn about.
       FATAL("mmap (real) failed with error %d: %s", errno, strerror(errno));
     }
     void* virtual_alloc =
         mmap(nullptr, rounded_length, prot, flags | MAP_NORESERVE, -1, 0);
     if (reinterpret_cast<intptr_t>(virtual_alloc) == -1) {
-      // Virtual memory allocation failed, probably because the requested
-      // size was too big. Callers can handle this.
-      munmap(real_alloc, kChunkSize);
-      return nullptr;
+      if (errno == ENOMEM) {
+        // Undo earlier, successful mappings.
+        munmap(real_alloc, kChunkSize);
+        return nullptr;
+      }
+      FATAL("mmap (virtual) failed with error %d: %s", errno, strerror(errno));
     }
     i::Address virtual_base = reinterpret_cast<i::Address>(virtual_alloc);
     i::Address virtual_end = virtual_base + rounded_length;
@@ -257,6 +265,12 @@ class MultiMappedAllocator : public ArrayBufferAllocatorBase {
           mremap(real_alloc, 0, kChunkSize, MREMAP_MAYMOVE | MREMAP_FIXED,
                  reinterpret_cast<void*>(to_map));
       if (reinterpret_cast<intptr_t>(result) == -1) {
+        if (errno == ENOMEM) {
+          // Undo earlier, successful mappings.
+          munmap(real_alloc, kChunkSize);
+          munmap(virtual_alloc, (to_map - virtual_base));
+          return nullptr;
+        }
         FATAL("mremap failed with error %d: %s", errno, strerror(errno));
       }
     }