[heap] Allocation buffer parking in NewSpace.

Change-Id: If01a247f2bae3c90708f9f8355325fa5b7113913
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2560196Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Commit-Queue: Hannes Payer <hpayer@chromium.org>
Cr-Commit-Position: refs/heads/master@{#71402}

src/flags/flag-definitions.h

@@ -1100,6 +1100,7 @@ DEFINE_BOOL(memory_reducer_for_small_heaps, true,
 DEFINE_INT(heap_growing_percent, 0,
            "specifies heap growing factor as (1 + heap_growing_percent/100)")
 DEFINE_INT(v8_os_page_size, 0, "override OS page size (in KBytes)")
+DEFINE_BOOL(allocation_buffer_parking, true, "allocation buffer parking")
 DEFINE_BOOL(always_compact, false, "Perform compaction on every full GC")
 DEFINE_BOOL(never_compact, false,
             "Never perform compaction on full GC - testing only")

src/heap/heap.cc

@@ -863,6 +863,7 @@ void Heap::GarbageCollectionPrologueInSafepoint() {
 
   UpdateNewSpaceAllocationCounter();
   CheckNewSpaceExpansionCriteria();
+  new_space_->ResetParkedAllocationBuffers();
 }
 
 size_t Heap::SizeOfObjects() {

src/heap/new-spaces.cc

@@ -262,6 +262,13 @@ void SemiSpace::PrependPage(Page* page) {
   }
 }
 
+void SemiSpace::MovePageToTheEnd(Page* page) {
+  DCHECK_EQ(page->owner(), this);
+  memory_chunk_list_.Remove(page);
+  memory_chunk_list_.PushBack(page);
+  current_page_ = page;
+}
+
 void SemiSpace::Swap(SemiSpace* from, SemiSpace* to) {
   // We won't be swapping semispaces without data in them.
   DCHECK(from->first_page());
@@ -415,6 +422,10 @@ void NewSpace::TearDown() {
   from_space_.TearDown();
 }
 
+void NewSpace::ResetParkedAllocationBuffers() {
+  parked_allocation_buffers_.clear();
+}
+
 void NewSpace::Flip() { SemiSpace::Swap(&from_space_, &to_space_); }
 
 void NewSpace::Grow() {
@@ -465,10 +476,10 @@ bool NewSpace::Rebalance() {
          from_space_.EnsureCurrentCapacity();
 }
 
-void NewSpace::UpdateLinearAllocationArea() {
+void NewSpace::UpdateLinearAllocationArea(Address known_top) {
   AdvanceAllocationObservers();
 
-  Address new_top = to_space_.page_low();
+  Address new_top = known_top == 0 ? to_space_.page_low() : known_top;
   BasicMemoryChunk::UpdateHighWaterMark(allocation_info_.top());
   allocation_info_.Reset(new_top, to_space_.page_high());
   // The order of the following two stores is important.
@@ -509,15 +520,23 @@ bool NewSpace::AddFreshPage() {
   Address top = allocation_info_.top();
   DCHECK(!OldSpace::IsAtPageStart(top));
 
+  // Clear remainder of current page.
+  Address limit = Page::FromAllocationAreaAddress(top)->area_end();
+  int remaining_in_page = static_cast<int>(limit - top);
+  heap()->CreateFillerObjectAt(top, remaining_in_page, ClearRecordedSlots::kNo);
+
   if (!to_space_.AdvancePage()) {
     // No more pages left to advance.
     return false;
   }
 
-  // Clear remainder of current page.
-  Address limit = Page::FromAllocationAreaAddress(top)->area_end();
-  int remaining_in_page = static_cast<int>(limit - top);
-  heap()->CreateFillerObjectAt(top, remaining_in_page, ClearRecordedSlots::kNo);
+  // We park unused allocation buffer space of allocations happenting from the
+  // mutator.
+  if (FLAG_allocation_buffer_parking && heap()->gc_state() == Heap::NOT_IN_GC &&
+      remaining_in_page >= kAllocationBufferParkingThreshold) {
+    parked_allocation_buffers_.push_back(
+        ParkedAllocationBuffer(remaining_in_page, top));
+  }
   UpdateLinearAllocationArea();
 
   return true;
@@ -528,6 +547,30 @@ bool NewSpace::AddFreshPageSynchronized() {
   return AddFreshPage();
 }
 
+bool NewSpace::AddParkedAllocationBuffer(int size_in_bytes,
+                                         AllocationAlignment alignment) {
+  int parked_size = 0;
+  Address start = 0;
+  for (auto it = parked_allocation_buffers_.begin();
+       it != parked_allocation_buffers_.end();) {
+    parked_size = it->first;
+    start = it->second;
+    int filler_size = Heap::GetFillToAlign(start, alignment);
+    if (size_in_bytes + filler_size <= parked_size) {
+      parked_allocation_buffers_.erase(it);
+      Page* page = Page::FromAddress(start);
+      // We move a page with a parked allocaiton to the end of the pages list
+      // to maintain the invariant that the last page is the used one.
+      to_space_.MovePageToTheEnd(page);
+      UpdateLinearAllocationArea(start);
+      return true;
+    } else {
+      it++;
+    }
+  }
+  return false;
+}
+
 bool NewSpace::EnsureAllocation(int size_in_bytes,
                                 AllocationAlignment alignment) {
   AdvanceAllocationObservers();
@@ -544,7 +587,10 @@ bool NewSpace::EnsureAllocation(int size_in_bytes,
 
   // Not enough room in the page, try to allocate a new one.
   if (!AddFreshPage()) {
-    return false;
+    // When we cannot grow NewSpace anymore we query for parked allocations.
+    if (!FLAG_allocation_buffer_parking ||
+        !AddParkedAllocationBuffer(size_in_bytes, alignment))
+      return false;
   }
 
   old_top = allocation_info_.top();

src/heap/new-spaces.h

@@ -6,10 +6,12 @@
 #define V8_HEAP_NEW_SPACES_H_
 
 #include <atomic>
+#include <map>
 #include <memory>
 
 #include "src/base/macros.h"
 #include "src/base/platform/mutex.h"
+#include "src/common/globals.h"
 #include "src/heap/heap.h"
 #include "src/heap/spaces.h"
 #include "src/logging/log.h"
@@ -23,6 +25,9 @@ class MemoryChunk;
 
 enum SemiSpaceId { kFromSpace = 0, kToSpace = 1 };
 
+using ParkedAllocationBuffer = std::pair<int, Address>;
+using ParkedAllocationBuffersVector = std::vector<ParkedAllocationBuffer>;
+
 // -----------------------------------------------------------------------------
 // SemiSpace in young generation
 //
@@ -105,6 +110,7 @@ class SemiSpace : public Space {
 
   void RemovePage(Page* page);
   void PrependPage(Page* page);
+  void MovePageToTheEnd(Page* page);
 
   Page* InitializePage(MemoryChunk* chunk);
 
@@ -248,6 +254,8 @@ class V8_EXPORT_PRIVATE NewSpace
   // is not deallocated here.
   void TearDown();
 
+  void ResetParkedAllocationBuffers();
+
   // Flip the pair of spaces.
   void Flip();
 
@@ -415,6 +423,9 @@ class V8_EXPORT_PRIVATE NewSpace
   bool AddFreshPage();
   bool AddFreshPageSynchronized();
 
+  bool AddParkedAllocationBuffer(int size_in_bytes,
+                                 AllocationAlignment alignment);
+
 #ifdef VERIFY_HEAP
   // Verify the active semispace.
   virtual void Verify(Isolate* isolate);
@@ -463,8 +474,10 @@ class V8_EXPORT_PRIVATE NewSpace
   void MaybeFreeUnusedLab(LinearAllocationArea info);
 
  private:
+  static const int kAllocationBufferParkingThreshold = 4 * KB;
+
   // Update linear allocation area to match the current to-space page.
-  void UpdateLinearAllocationArea();
+  void UpdateLinearAllocationArea(Address known_top = 0);
 
   base::Mutex mutex_;
 
@@ -478,6 +491,8 @@ class V8_EXPORT_PRIVATE NewSpace
   SemiSpace from_space_;
   VirtualMemory reservation_;
 
+  ParkedAllocationBuffersVector parked_allocation_buffers_;
+
   // Internal allocation methods.
   V8_WARN_UNUSED_RESULT V8_INLINE AllocationResult
   AllocateFastAligned(int size_in_bytes, int* aligned_size_in_bytes,