cppgc: Limit sweeping on allocation if possible

Align slow path allocation with V8 in that:
1. Try to refill from the free list.
2. Perform limited sweeping of a space if necessary and retry the free
   list.
3. Try to expand the space.
4. Perform full sweeping of a space if necessary and retry the free
   list.
5. Finish sweeping fully as we would anyways do a GC at this point.
6. Retry the free list again
7. Try expanding again as finishing sweeping may have freed up pages.

Specifically, this adresses a performance problem where we would fully
sweep the whole heap, possibly causing 100ms of jank on allocation. In
such cases the new approach maintains performance and stays fast at the
expense of using more memory.

Allocations usually find memory in 1.-3. Steps 4.-7. are slow paths
that are definitely expensive but prevent failing with OOM.

Bug: v8:13294
Change-Id: I56133fa4cbbc74f8abcdec49c7e10125c2dbc3e9
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3899260
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: Omer Katz <omerkatz@chromium.org>
Reviewed-by: Anton Bikineev <bikineev@chromium.org>
Cr-Commit-Position: refs/heads/main@{#83268}

src/heap/cppgc/object-allocator.cc

@@ -187,42 +187,64 @@ void* ObjectAllocator::OutOfLineAllocateImpl(NormalPageSpace& space,
   return result;
 }
 
+bool ObjectAllocator::TryExpandAndRefillLinearAllocationBuffer(
+    NormalPageSpace& space) {
+  auto* const new_page = NormalPage::TryCreate(page_backend_, space);
+  if (!new_page) return false;
+
+  space.AddPage(new_page);
+  // Set linear allocation buffer to new page.
+  ReplaceLinearAllocationBuffer(space, stats_collector_,
+                                new_page->PayloadStart(),
+                                new_page->PayloadSize());
+  return true;
+}
+
 bool ObjectAllocator::TryRefillLinearAllocationBuffer(NormalPageSpace& space,
                                                       size_t size) {
   // Try to allocate from the freelist.
   if (TryRefillLinearAllocationBufferFromFreeList(space, size)) return true;
 
-  // Lazily sweep pages of this heap until we find a freed area for this
-  // allocation or we finish sweeping all pages of this heap.
   Sweeper& sweeper = raw_heap_.heap()->sweeper();
-  // TODO(chromium:1056170): Investigate whether this should be a loop which
-  // would result in more aggressive re-use of memory at the expense of
-  // potentially larger allocation time.
-  if (sweeper.SweepForAllocationIfRunning(&space, size)) {
-    // Sweeper found a block of at least `size` bytes. Allocation from the
-    // free list may still fail as actual  buckets are not exhaustively
-    // searched for a suitable block. Instead, buckets are tested from larger
-    // sizes that are guaranteed to fit the block to smaller bucket sizes that
-    // may only potentially fit the block. For the bucket that may exactly fit
-    // the allocation of `size` bytes (no overallocation), only the first
-    // entry is checked.
-    if (TryRefillLinearAllocationBufferFromFreeList(space, size)) return true;
+  // Lazily sweep pages of this heap. This is not exhaustive to limit jank on
+  // allocation. Allocation from the free list may still fail as actual  buckets
+  // are not exhaustively searched for a suitable block. Instead, buckets are
+  // tested from larger sizes that are guaranteed to fit the block to smaller
+  // bucket sizes that may only potentially fit the block. For the bucket that
+  // may exactly fit the allocation of `size` bytes (no overallocation), only
+  // the first entry is checked.
+  if (sweeper.SweepForAllocationIfRunning(
+          &space, size, v8::base::TimeDelta::FromMicroseconds(500)) &&
+      TryRefillLinearAllocationBufferFromFreeList(space, size)) {
+    return true;
   }
 
-  sweeper.FinishIfRunning();
-  // TODO(chromium:1056170): Make use of the synchronously freed memory.
-
-  auto* new_page = NormalPage::TryCreate(page_backend_, space);
-  if (!new_page) {
-    return false;
+  // Sweeping was off or did not yield in any memory within limited
+  // contributing. We expand at this point as that's cheaper than possibly
+  // continuing sweeping the whole heap.
+  if (TryExpandAndRefillLinearAllocationBuffer(space)) return true;
+
+  // Expansion failed. Before finishing all sweeping, finish sweeping of a given
+  // space which is cheaper.
+  if (sweeper.SweepForAllocationIfRunning(&space, size,
+                                          v8::base::TimeDelta::Max()) &&
+      TryRefillLinearAllocationBufferFromFreeList(space, size)) {
+    return true;
   }
 
-  space.AddPage(new_page);
-  // Set linear allocation buffer to new page.
-  ReplaceLinearAllocationBuffer(space, stats_collector_,
-                                new_page->PayloadStart(),
-                                new_page->PayloadSize());
-  return true;
+  // Heap expansion and sweeping of a space failed. At this point the caller
+  // could run OOM or do a full GC which needs to finish sweeping if it's
+  // running. Hence, we may as well finish sweeping here. Note that this is
+  // possibly very expensive but not more expensive than running a full GC as
+  // the alternative is OOM.
+  if (sweeper.FinishIfRunning()) {
+    // Sweeping may have added memory to the free list.
+    if (TryRefillLinearAllocationBufferFromFreeList(space, size)) return true;
+
+    // Sweeping may have freed pages completely.
+    if (TryExpandAndRefillLinearAllocationBuffer(space)) return true;
+  }
+  return false;
 }
 
 bool ObjectAllocator::TryRefillLinearAllocationBufferFromFreeList(

src/heap/cppgc/object-allocator.h

@@ -70,6 +70,7 @@ class V8_EXPORT_PRIVATE ObjectAllocator final : public cppgc::AllocationHandle {
 
   bool TryRefillLinearAllocationBuffer(NormalPageSpace&, size_t);
   bool TryRefillLinearAllocationBufferFromFreeList(NormalPageSpace&, size_t);
+  bool TryExpandAndRefillLinearAllocationBuffer(NormalPageSpace&);
 
   RawHeap& raw_heap_;
   PageBackend& page_backend_;

src/heap/cppgc/sweeper.cc

@@ -11,6 +11,7 @@
 #include "include/cppgc/platform.h"
 #include "src/base/optional.h"
 #include "src/base/platform/mutex.h"
+#include "src/base/platform/time.h"
 #include "src/heap/cppgc/free-list.h"
 #include "src/heap/cppgc/globals.h"
 #include "src/heap/cppgc/heap-base.h"
@@ -787,7 +788,8 @@ class Sweeper::SweeperImpl final {
     }
   }
 
-  bool SweepForAllocationIfRunning(NormalPageSpace* space, size_t size) {
+  bool SweepForAllocationIfRunning(NormalPageSpace* space, size_t size,
+                                   v8::base::TimeDelta max_duration) {
     if (!is_in_progress_) return false;
 
     // Bail out for recursive sweeping calls. This can happen when finalizers
@@ -809,13 +811,31 @@ class Sweeper::SweeperImpl final {
         stats_collector_, StatsCollector::kSweepOnAllocation);
     MutatorThreadSweepingScope sweeping_in_progress(*this);
 
+    const auto deadline = v8::base::TimeTicks::Now() + max_duration;
+    size_t page_count = 0;
+    const auto ShouldYield = [&page_count, deadline]() {
+      constexpr size_t kPageInterruptInterval = 10;
+      if (page_count++ == kPageInterruptInterval) {
+        if (deadline < v8::base::TimeTicks::Now()) {
+          return true;
+        }
+        page_count = 0;
+      }
+      return false;
+    };
+
     {
       // First, process unfinalized pages as finalizing a page is faster than
       // sweeping.
       SweepFinalizer finalizer(platform_, config_.free_memory_handling);
       while (auto page = space_state.swept_unfinalized_pages.Pop()) {
         finalizer.FinalizePage(&*page);
-        if (size <= finalizer.largest_new_free_list_entry()) return true;
+        if (size <= finalizer.largest_new_free_list_entry()) {
+          return true;
+        }
+        if (ShouldYield()) {
+          return false;
+        }
       }
     }
     {
@@ -825,19 +845,24 @@ class Sweeper::SweeperImpl final {
                                    config_.free_memory_handling);
       while (auto page = space_state.unswept_pages.Pop()) {
         sweeper.SweepPage(**page);
-        if (size <= sweeper.largest_new_free_list_entry()) return true;
+        if (size <= sweeper.largest_new_free_list_entry()) {
+          return true;
+        }
+        if (ShouldYield()) {
+          return false;
+        }
       }
     }
 
     return false;
   }
 
-  void FinishIfRunning() {
-    if (!is_in_progress_) return;
+  bool FinishIfRunning() {
+    if (!is_in_progress_) return false;
 
     // Bail out for recursive sweeping calls. This can happen when finalizers
     // allocate new memory.
-    if (is_sweeping_on_mutator_thread_) return;
+    if (is_sweeping_on_mutator_thread_) return false;
 
     {
       StatsCollector::EnabledScope stats_scope(
@@ -852,6 +877,7 @@ class Sweeper::SweeperImpl final {
       Finish();
     }
     NotifyDone();
+    return true;
   }
 
   void FinishIfOutOfWork() {
@@ -1060,14 +1086,16 @@ Sweeper::~Sweeper() = default;
 void Sweeper::Start(SweepingConfig config) {
   impl_->Start(config, heap_.platform());
 }
-void Sweeper::FinishIfRunning() { impl_->FinishIfRunning(); }
+
+bool Sweeper::FinishIfRunning() { return impl_->FinishIfRunning(); }
 void Sweeper::FinishIfOutOfWork() { impl_->FinishIfOutOfWork(); }
 void Sweeper::WaitForConcurrentSweepingForTesting() {
   impl_->WaitForConcurrentSweepingForTesting();
 }
 void Sweeper::NotifyDoneIfNeeded() { impl_->NotifyDoneIfNeeded(); }
-bool Sweeper::SweepForAllocationIfRunning(NormalPageSpace* space, size_t size) {
-  return impl_->SweepForAllocationIfRunning(space, size);
+bool Sweeper::SweepForAllocationIfRunning(NormalPageSpace* space, size_t size,
+                                          v8::base::TimeDelta max_duration) {
+  return impl_->SweepForAllocationIfRunning(space, size, max_duration);
 }
 bool Sweeper::IsSweepingOnMutatorThread() const {
   return impl_->IsSweepingOnMutatorThread();

src/heap/cppgc/sweeper.h

@@ -47,13 +47,17 @@ class V8_EXPORT_PRIVATE Sweeper final {
 
   // Sweeper::Start assumes the heap holds no linear allocation buffers.
   void Start(SweepingConfig);
-  void FinishIfRunning();
+  // Returns true when sweeping was finished and false if it was not running or
+  // couldn't be finished due to being a recursive sweep call.
+  bool FinishIfRunning();
   void FinishIfOutOfWork();
   void NotifyDoneIfNeeded();
-  // SweepForAllocationIfRunning sweeps the given |space| until a slot that can
-  // fit an allocation of size |size| is found. Returns true if a slot was
-  // found.
-  bool SweepForAllocationIfRunning(NormalPageSpace* space, size_t size);
+  // SweepForAllocationIfRunning sweeps the given `space` until a slot that can
+  // fit an allocation of `min_wanted_size` bytes is found. Returns true if a
+  // slot was found. Aborts after `max_duration`.
+  bool SweepForAllocationIfRunning(NormalPageSpace* space,
+                                   size_t min_wanted_size,
+                                   v8::base::TimeDelta max_duration);
 
   bool IsSweepingOnMutatorThread() const;
   bool IsSweepingInProgress() const;