[heap] Retry ArrayBuffer backing store allocation after GCs.

This makes allocation of backing stores more robust by perfoming GCs
on allocation failure. The GCs help if there are existing large backing
stores that are retained by dead JSArrayBuffer objects.

Bug: chromium:1008938, v8:9380
Change-Id: Ic80b29214b8843427dfcdd141df71363821afe71
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1855998
Commit-Queue: Ulan Degenbaev <ulan@chromium.org>
Reviewed-by: Jakob Kummerow <jkummerow@chromium.org>
Cr-Commit-Position: refs/heads/master@{#64241}

src/heap/heap.cc

@@ -2759,6 +2759,22 @@ HeapObject Heap::AlignWithFiller(HeapObject object, int object_size,
   return object;
 }
 
+void* Heap::AllocateExternalBackingStore(
+    const std::function<void*(size_t)>& allocate, size_t byte_length) {
+  // TODO(ulan): Perform GCs proactively based on the byte_length and
+  // the current external backing store counters.
+  void* result = allocate(byte_length);
+  if (result) return result;
+  for (int i = 0; i < 2; i++) {
+    CollectGarbage(OLD_SPACE, GarbageCollectionReason::kExternalMemoryPressure);
+    result = allocate(byte_length);
+    if (result) return result;
+  }
+  isolate()->counters()->gc_last_resort_from_handles()->Increment();
+  CollectAllAvailableGarbage(GarbageCollectionReason::kExternalMemoryPressure);
+  return allocate(byte_length);
+}
+
 void Heap::RegisterBackingStore(JSArrayBuffer buffer,
                                 std::shared_ptr<BackingStore> backing_store) {
   ArrayBufferTracker::RegisterNew(this, buffer, std::move(backing_store));

src/heap/heap.h

@@ -1230,6 +1230,17 @@ class Heap {
   AlignWithFiller(HeapObject object, int object_size, int allocation_size,
                   AllocationAlignment alignment);
 
+  // Allocate an external backing store with the given allocation callback.
+  // If the callback fails (indicated by a nullptr result) then this function
+  // will re-try the allocation after performing GCs. This is useful for
+  // external backing stores that may be retained by (unreachable) V8 objects
+  // such as ArrayBuffers, ExternalStrings, etc.
+  //
+  // The function may also proactively trigger GCs even if the allocation
+  // callback does not fail to keep the memory usage low.
+  V8_EXPORT_PRIVATE void* AllocateExternalBackingStore(
+      const std::function<void*(size_t)>& allocate, size_t byte_length);
+
   // ===========================================================================
   // ArrayBuffer tracking. =====================================================
   // ===========================================================================

src/objects/backing-store.cc

@@ -176,8 +176,11 @@ std::unique_ptr<BackingStore> BackingStore::Allocate(
     if (shared == SharedFlag::kShared) {
       counters->shared_array_allocations()->AddSample(mb_length);
     }
-    if (initialized == InitializedFlag::kZeroInitialized) {
-      buffer_start = allocator->Allocate(byte_length);
+    auto allocate_buffer = [allocator, initialized](size_t byte_length) {
+      if (initialized == InitializedFlag::kUninitialized) {
+        return allocator->AllocateUninitialized(byte_length);
+      }
+      void* buffer_start = allocator->Allocate(byte_length);
       if (buffer_start) {
         // TODO(wasm): node does not implement the zero-initialization API.
         // Reenable this debug check when node does implement it properly.
@@ -188,9 +191,12 @@ std::unique_ptr<BackingStore> BackingStore::Allocate(
           DebugCheckZero(buffer_start, byte_length);
         }
       }
-    } else {
-      buffer_start = allocator->AllocateUninitialized(byte_length);
-    }
+      return buffer_start;
+    };
+
+    buffer_start = isolate->heap()->AllocateExternalBackingStore(
+        allocate_buffer, byte_length);
+
     if (buffer_start == nullptr) {
       // Allocation failed.
       counters->array_buffer_new_size_failures()->AddSample(mb_length);

test/cctest/heap/test-heap.cc

@@ -6782,6 +6782,19 @@ HEAP_TEST(MemoryReducerActivationForSmallHeaps) {
   CHECK_EQ(heap->memory_reducer()->state_.action, MemoryReducer::Action::kWait);
 }
 
+TEST(AllocateExternalBackingStore) {
+  ManualGCScope manual_gc_scope;
+  LocalContext env;
+  Isolate* isolate = CcTest::i_isolate();
+  Heap* heap = isolate->heap();
+  int initial_ms_count = heap->ms_count();
+  void* result =
+      heap->AllocateExternalBackingStore([](size_t) { return nullptr; }, 10);
+  CHECK_NULL(result);
+  // At least two GCs should happen.
+  CHECK_LE(2, heap->ms_count() - initial_ms_count);
+}
+
 TEST(CodeObjectRegistry) {
   // We turn off compaction to ensure that code is not moving.
   FLAG_never_compact = true;