[heap] Introduce LocalHeap::AllocateRaw method

LocalHeap::AllocateRaw will be similar to Heap::AllocateRaw and
handle all allocations. LocalHeap::AllocateRawOrFail will perform a GC
and afterwards retry the allocation in a loop.

Bug: v8:10315
Change-Id: I68468962cf9102697aa547b2aa05c7ec6bafd19e
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2332801Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Commit-Queue: Dominik Inführ <dinfuehr@chromium.org>
Cr-Commit-Position: refs/heads/master@{#69193}

src/DEPS

@@ -19,6 +19,7 @@ include_rules = [
   "+src/heap/heap-write-barrier-inl.h",
   "+src/heap/heap-write-barrier.h",
   "+src/heap/local-heap.h",
+  "+src/heap/local-heap-inl.h",
   # TODO(v8:10496): Don't expose memory chunk outside of heap/.
   "+src/heap/memory-chunk.h",
   "+src/heap/memory-chunk-inl.h",

src/heap/concurrent-allocator-inl.h

@@ -17,9 +17,9 @@
 namespace v8 {
 namespace internal {
 
-AllocationResult ConcurrentAllocator::Allocate(int object_size,
-                                               AllocationAlignment alignment,
-                                               AllocationOrigin origin) {
+AllocationResult ConcurrentAllocator::AllocateRaw(int object_size,
+                                                  AllocationAlignment alignment,
+                                                  AllocationOrigin origin) {
   // TODO(dinfuehr): Add support for allocation observers
   CHECK(FLAG_concurrent_allocation);
 
@@ -33,14 +33,6 @@ AllocationResult ConcurrentAllocator::Allocate(int object_size,
   return AllocateInLab(object_size, alignment, origin);
 }
 
-Address ConcurrentAllocator::AllocateOrFail(int object_size,
-                                            AllocationAlignment alignment,
-                                            AllocationOrigin origin) {
-  AllocationResult result = Allocate(object_size, alignment, origin);
-  if (!result.IsRetry()) return result.ToObject().address();
-  return PerformCollectionAndAllocateAgain(object_size, alignment, origin);
-}
-
 AllocationResult ConcurrentAllocator::AllocateInLab(
     int object_size, AllocationAlignment alignment, AllocationOrigin origin) {
   AllocationResult allocation = lab_.AllocateRawAligned(object_size, alignment);

src/heap/concurrent-allocator.cc

@@ -7,7 +7,7 @@
 #include "src/execution/isolate.h"
 #include "src/handles/persistent-handles.h"
 #include "src/heap/concurrent-allocator-inl.h"
-#include "src/heap/local-heap.h"
+#include "src/heap/local-heap-inl.h"
 #include "src/heap/marking.h"
 
 namespace v8 {
@@ -16,21 +16,20 @@ namespace internal {
 void StressConcurrentAllocatorTask::RunInternal() {
   Heap* heap = isolate_->heap();
   LocalHeap local_heap(heap);
-  ConcurrentAllocator* allocator = local_heap.old_space_allocator();
 
   const int kNumIterations = 2000;
   const int kObjectSize = 10 * kTaggedSize;
   const int kLargeObjectSize = 8 * KB;
 
   for (int i = 0; i < kNumIterations; i++) {
-    Address address = allocator->AllocateOrFail(
-        kObjectSize, AllocationAlignment::kWordAligned,
-        AllocationOrigin::kRuntime);
+    Address address = local_heap.AllocateRawOrFail(
+        kObjectSize, AllocationType::kOld, AllocationOrigin::kRuntime,
+        AllocationAlignment::kWordAligned);
     heap->CreateFillerObjectAtBackground(
         address, kObjectSize, ClearFreedMemoryMode::kDontClearFreedMemory);
-    address = allocator->AllocateOrFail(kLargeObjectSize,
-                                        AllocationAlignment::kWordAligned,
-                                        AllocationOrigin::kRuntime);
+    address = local_heap.AllocateRawOrFail(
+        kLargeObjectSize, AllocationType::kOld, AllocationOrigin::kRuntime,
+        AllocationAlignment::kWordAligned);
     heap->CreateFillerObjectAtBackground(
         address, kLargeObjectSize, ClearFreedMemoryMode::kDontClearFreedMemory);
     if (i % 10 == 0) {
@@ -61,7 +60,7 @@ Address ConcurrentAllocator::PerformCollectionAndAllocateAgain(
       heap->RequestAndWaitForCollection();
     }
 
-    AllocationResult result = Allocate(object_size, alignment, origin);
+    AllocationResult result = AllocateRaw(object_size, alignment, origin);
     if (!result.IsRetry()) {
       local_heap_->allocation_failed_ = false;
       return result.ToObjectChecked().address();

src/heap/concurrent-allocator.h

@@ -42,12 +42,9 @@ class ConcurrentAllocator {
         space_(space),
         lab_(LocalAllocationBuffer::InvalidBuffer()) {}
 
-  inline AllocationResult Allocate(int object_size,
-                                   AllocationAlignment alignment,
-                                   AllocationOrigin origin);
-
-  inline Address AllocateOrFail(int object_size, AllocationAlignment alignment,
-                                AllocationOrigin origin);
+  inline AllocationResult AllocateRaw(int object_size,
+                                      AllocationAlignment alignment,
+                                      AllocationOrigin origin);
 
   void FreeLinearAllocationArea();
   void MakeLinearAllocationAreaIterable();

src/heap/local-heap-inl.h

@@ -6,6 +6,7 @@
 #define V8_HEAP_LOCAL_HEAP_INL_H_
 
 #include "src/handles/persistent-handles.h"
+#include "src/heap/concurrent-allocator-inl.h"
 #include "src/heap/local-heap.h"
 
 namespace v8 {
@@ -24,6 +25,32 @@ Handle<T> LocalHeap::NewPersistentHandle(Handle<T> object) {
   return NewPersistentHandle(*object);
 }
 
+AllocationResult LocalHeap::AllocateRaw(int size_in_bytes, AllocationType type,
+                                        AllocationOrigin origin,
+                                        AllocationAlignment alignment) {
+  DCHECK_EQ(LocalHeap::Current(), this);
+  DCHECK(AllowHandleAllocation::IsAllowed());
+  DCHECK(AllowHeapAllocation::IsAllowed());
+  DCHECK_IMPLIES(type == AllocationType::kCode,
+                 alignment == AllocationAlignment::kCodeAligned);
+  DCHECK_EQ(heap()->gc_state(), Heap::HeapState::NOT_IN_GC);
+
+  bool large_object = size_in_bytes > kMaxRegularHeapObjectSize;
+  CHECK_EQ(type, AllocationType::kOld);
+  CHECK(!large_object);
+
+  return old_space_allocator()->AllocateRaw(size_in_bytes, alignment, origin);
+}
+
+Address LocalHeap::AllocateRawOrFail(int object_size, AllocationType type,
+                                     AllocationOrigin origin,
+                                     AllocationAlignment alignment) {
+  AllocationResult result = AllocateRaw(object_size, type, origin, alignment);
+  if (!result.IsRetry()) return result.ToObject().address();
+  return PerformCollectionAndAllocateAgain(object_size, type, origin,
+                                           alignment);
+}
+
 }  // namespace internal
 }  // namespace v8
 

src/heap/local-heap.cc

@@ -7,8 +7,10 @@
 #include <memory>
 
 #include "src/base/platform/mutex.h"
+#include "src/common/globals.h"
 #include "src/handles/local-handles.h"
 #include "src/heap/heap-inl.h"
+#include "src/heap/local-heap-inl.h"
 #include "src/heap/safepoint.h"
 
 namespace v8 {
@@ -133,5 +135,27 @@ void LocalHeap::UnmarkLinearAllocationArea() {
   old_space_allocator_.UnmarkLinearAllocationArea();
 }
 
+Address LocalHeap::PerformCollectionAndAllocateAgain(
+    int object_size, AllocationType type, AllocationOrigin origin,
+    AllocationAlignment alignment) {
+  allocation_failed_ = true;
+  static const int kMaxNumberOfRetries = 3;
+
+  for (int i = 0; i < kMaxNumberOfRetries; i++) {
+    {
+      ParkedScope scope(this);
+      heap_->RequestAndWaitForCollection();
+    }
+
+    AllocationResult result = AllocateRaw(object_size, type, origin, alignment);
+    if (!result.IsRetry()) {
+      allocation_failed_ = false;
+      return result.ToObjectChecked().address();
+    }
+  }
+
+  heap_->FatalProcessOutOfMemory("LocalHeap: allocation failed");
+}
+
 }  // namespace internal
 }  // namespace v8

src/heap/local-heap.h

@@ -78,6 +78,19 @@ class V8_EXPORT_PRIVATE LocalHeap {
   // with the current thread.
   static LocalHeap* Current();
 
+  // Allocate an uninitialized object.
+  V8_WARN_UNUSED_RESULT inline AllocationResult AllocateRaw(
+      int size_in_bytes, AllocationType allocation,
+      AllocationOrigin origin = AllocationOrigin::kRuntime,
+      AllocationAlignment alignment = kWordAligned);
+
+  // Allocates an uninitialized object and crashes when object
+  // cannot be allocated.
+  V8_WARN_UNUSED_RESULT inline Address AllocateRawOrFail(
+      int size_in_bytes, AllocationType allocation,
+      AllocationOrigin origin = AllocationOrigin::kRuntime,
+      AllocationAlignment alignment = kWordAligned);
+
  private:
   enum class ThreadState {
     // Threads in this state need to be stopped in a safepoint.
@@ -89,6 +102,13 @@ class V8_EXPORT_PRIVATE LocalHeap {
     Safepoint
   };
 
+  // Slow path of allocation that performs GC and then retries allocation in
+  // loop.
+  Address PerformCollectionAndAllocateAgain(int object_size,
+                                            AllocationType type,
+                                            AllocationOrigin origin,
+                                            AllocationAlignment alignment);
+
   void Park();
   void Unpark();
   void EnsureParkedBeforeDestruction();

src/objects/script-inl.h

@@ -5,8 +5,8 @@
 #ifndef V8_OBJECTS_SCRIPT_INL_H_
 #define V8_OBJECTS_SCRIPT_INL_H_
 
+#include "src/objects/managed.h"
 #include "src/objects/script.h"
-
 #include "src/objects/shared-function-info.h"
 #include "src/objects/smi-inl.h"
 #include "src/objects/string-inl.h"

test/cctest/heap/test-concurrent-allocation.cc

@@ -14,7 +14,7 @@
 #include "src/handles/persistent-handles.h"
 #include "src/heap/concurrent-allocator-inl.h"
 #include "src/heap/heap.h"
-#include "src/heap/local-heap.h"
+#include "src/heap/local-heap-inl.h"
 #include "src/heap/safepoint.h"
 #include "src/objects/heap-number.h"
 #include "src/objects/heap-object.h"
@@ -48,16 +48,15 @@ class ConcurrentAllocationThread final : public v8::base::Thread {
 
   void Run() override {
     LocalHeap local_heap(heap_);
-    ConcurrentAllocator* allocator = local_heap.old_space_allocator();
 
     for (int i = 0; i < kNumIterations; i++) {
-      Address address = allocator->AllocateOrFail(
-          kSmallObjectSize, AllocationAlignment::kWordAligned,
-          AllocationOrigin::kRuntime);
+      Address address = local_heap.AllocateRawOrFail(
+          kSmallObjectSize, AllocationType::kOld, AllocationOrigin::kRuntime,
+          AllocationAlignment::kWordAligned);
       CreateFixedArray(heap_, address, kSmallObjectSize);
-      address = allocator->AllocateOrFail(kMediumObjectSize,
-                                          AllocationAlignment::kWordAligned,
-                                          AllocationOrigin::kRuntime);
+      address = local_heap.AllocateRawOrFail(
+          kMediumObjectSize, AllocationType::kOld, AllocationOrigin::kRuntime,
+          AllocationAlignment::kWordAligned);
       CreateFixedArray(heap_, address, kMediumObjectSize);
       if (i % 10 == 0) {
         local_heap.Safepoint();
@@ -120,7 +119,6 @@ class ConcurrentBlackAllocationThread final : public v8::base::Thread {
 
   void Run() override {
     LocalHeap local_heap(heap_);
-    ConcurrentAllocator* allocator = local_heap.old_space_allocator();
 
     for (int i = 0; i < kNumIterations; i++) {
       if (i == kWhiteIterations) {
@@ -128,14 +126,14 @@ class ConcurrentBlackAllocationThread final : public v8::base::Thread {
         sema_white_->Signal();
         sema_marking_started_->Wait();
       }
-      Address address = allocator->AllocateOrFail(
-          kSmallObjectSize, AllocationAlignment::kWordAligned,
-          AllocationOrigin::kRuntime);
+      Address address = local_heap.AllocateRawOrFail(
+          kSmallObjectSize, AllocationType::kOld, AllocationOrigin::kRuntime,
+          AllocationAlignment::kWordAligned);
       objects_->push_back(address);
       CreateFixedArray(heap_, address, kSmallObjectSize);
-      address = allocator->AllocateOrFail(kMediumObjectSize,
-                                          AllocationAlignment::kWordAligned,
-                                          AllocationOrigin::kRuntime);
+      address = local_heap.AllocateRawOrFail(
+          kMediumObjectSize, AllocationType::kOld, AllocationOrigin::kRuntime,
+          AllocationAlignment::kWordAligned);
       objects_->push_back(address);
       CreateFixedArray(heap_, address, kMediumObjectSize);
     }