heap: Factor out raw allocation functions into HeapAllocator

This CL is mostly mechanic and provides runtime and static
dispatch for allocation of objects using HeapAllocator.

Future CLs will remove the Heap bottelenecks.

Bug: v8:12615
Change-Id: Id2becf7da4bd5273f96abc0e1a4ac6c04bddb1cb
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3474674Reviewed-by: Dominik Inführ <dinfuehr@chromium.org>
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Cr-Commit-Position: refs/heads/main@{#79229}

BUILD.bazel

@@ -1348,6 +1348,9 @@ filegroup(
         "src/heap/gc-idle-time-handler.h",
         "src/heap/gc-tracer.cc",
         "src/heap/gc-tracer.h",
+        "src/heap/heap-allocator-inl.h",
+        "src/heap/heap-allocator.cc",
+        "src/heap/heap-allocator.h",
         "src/heap/heap-controller.cc",
         "src/heap/heap-controller.h",
         "src/heap/heap-inl.h",

BUILD.gn

@@ -2979,6 +2979,8 @@ v8_header_set("v8_internal_headers") {
     "src/heap/free-list.h",
     "src/heap/gc-idle-time-handler.h",
     "src/heap/gc-tracer.h",
+    "src/heap/heap-allocator-inl.h",
+    "src/heap/heap-allocator.h",
     "src/heap/heap-controller.h",
     "src/heap/heap-inl.h",
     "src/heap/heap-layout-tracer.h",
@@ -4182,6 +4184,7 @@ v8_source_set("v8_base_without_compiler") {
     "src/heap/free-list.cc",
     "src/heap/gc-idle-time-handler.cc",
     "src/heap/gc-tracer.cc",
+    "src/heap/heap-allocator.cc",
     "src/heap/heap-controller.cc",
     "src/heap/heap-layout-tracer.cc",
     "src/heap/heap-write-barrier.cc",

src/heap/allocation-result.h

@@ -13,6 +13,15 @@
 namespace v8 {
 namespace internal {
 
+enum class AllocationOrigin {
+  kGeneratedCode = 0,
+  kRuntime = 1,
+  kGC = 2,
+  kFirstAllocationOrigin = kGeneratedCode,
+  kLastAllocationOrigin = kGC,
+  kNumberOfAllocationOrigins = kLastAllocationOrigin + 1
+};
+
 // The result of an allocation attempt. Either represents a successful
 // allocation that can be turned into an object or a failed attempt.
 class AllocationResult final {

src/heap/heap-allocator-inl.h

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_HEAP_HEAP_ALLOCATOR_INL_H_
+#define V8_HEAP_HEAP_ALLOCATOR_INL_H_
+
+#include "src/base/logging.h"
+#include "src/common/globals.h"
+#include "src/heap/concurrent-allocator-inl.h"
+#include "src/heap/heap-allocator.h"
+#include "src/heap/large-spaces.h"
+#include "src/heap/new-spaces.h"
+#include "src/heap/paged-spaces.h"
+#include "src/heap/read-only-spaces.h"
+#include "src/heap/third-party/heap-api.h"
+
+namespace v8 {
+namespace internal {
+
+PagedSpace* HeapAllocator::code_space() const {
+  return static_cast<PagedSpace*>(spaces_[CODE_SPACE]);
+}
+
+CodeLargeObjectSpace* HeapAllocator::code_lo_space() const {
+  return static_cast<CodeLargeObjectSpace*>(spaces_[CODE_LO_SPACE]);
+}
+
+OldLargeObjectSpace* HeapAllocator::lo_space() const {
+  return static_cast<OldLargeObjectSpace*>(spaces_[LO_SPACE]);
+}
+
+PagedSpace* HeapAllocator::map_space() const {
+  return static_cast<PagedSpace*>(spaces_[MAP_SPACE]);
+}
+
+NewSpace* HeapAllocator::new_space() const {
+  return static_cast<NewSpace*>(spaces_[NEW_SPACE]);
+}
+
+NewLargeObjectSpace* HeapAllocator::new_lo_space() const {
+  return static_cast<NewLargeObjectSpace*>(spaces_[NEW_LO_SPACE]);
+}
+
+PagedSpace* HeapAllocator::old_space() const {
+  return static_cast<PagedSpace*>(spaces_[OLD_SPACE]);
+}
+
+ReadOnlySpace* HeapAllocator::read_only_space() const {
+  return read_only_space_;
+}
+
+template <AllocationType type>
+V8_WARN_UNUSED_RESULT V8_INLINE AllocationResult HeapAllocator::AllocateRaw(
+    int size_in_bytes, AllocationOrigin origin, AllocationAlignment alignment) {
+  DCHECK_EQ(heap_->gc_state(), Heap::NOT_IN_GC);
+  DCHECK(AllowHandleAllocation::IsAllowed());
+  DCHECK(AllowHeapAllocation::IsAllowed());
+
+  if (FLAG_single_generation && type == AllocationType::kYoung) {
+    return AllocateRaw(size_in_bytes, AllocationType::kOld, origin, alignment);
+  }
+
+#ifdef V8_ENABLE_ALLOCATION_TIMEOUT
+  if (FLAG_random_gc_interval > 0 || FLAG_gc_interval >= 0) {
+    if (!heap_->always_allocate() && allocation_timeout_-- <= 0) {
+      return AllocationResult::Failure();
+    }
+  }
+#endif  // V8_ENABLE_ALLOCATION_TIMEOUT
+
+#ifdef DEBUG
+  IncrementObjectCounters();
+#endif  // DEBUG
+
+  if (heap_->CanSafepoint()) {
+    heap_->main_thread_local_heap()->Safepoint();
+  }
+
+  const size_t large_object_threshold = heap_->MaxRegularHeapObjectSize(type);
+  const bool large_object =
+      static_cast<size_t>(size_in_bytes) > large_object_threshold;
+
+  HeapObject object;
+  AllocationResult allocation;
+
+  if (V8_ENABLE_THIRD_PARTY_HEAP_BOOL) {
+    allocation = heap_->tp_heap_->Allocate(size_in_bytes, type, alignment);
+  } else {
+    if (V8_UNLIKELY(large_object)) {
+      allocation =
+          AllocateRawLargeInternal(size_in_bytes, type, origin, alignment);
+    } else {
+      switch (type) {
+        case AllocationType::kYoung:
+          allocation =
+              new_space()->AllocateRaw(size_in_bytes, alignment, origin);
+          break;
+        case AllocationType::kOld:
+          allocation =
+              old_space()->AllocateRaw(size_in_bytes, alignment, origin);
+          break;
+        case AllocationType::kCode:
+          DCHECK_EQ(alignment, AllocationAlignment::kTaggedAligned);
+          DCHECK(AllowCodeAllocation::IsAllowed());
+          allocation = code_space()->AllocateRawUnaligned(size_in_bytes);
+          break;
+        case AllocationType::kMap:
+          DCHECK_EQ(alignment, AllocationAlignment::kTaggedAligned);
+          allocation = map_space()->AllocateRawUnaligned(size_in_bytes);
+          break;
+        case AllocationType::kReadOnly:
+          DCHECK(read_only_space()->writable());
+          DCHECK_EQ(AllocationOrigin::kRuntime, origin);
+          allocation = read_only_space()->AllocateRaw(size_in_bytes, alignment);
+          break;
+        case AllocationType::kSharedMap:
+          allocation = shared_map_allocator_->AllocateRaw(size_in_bytes,
+                                                          alignment, origin);
+          break;
+        case AllocationType::kSharedOld:
+          allocation = shared_old_allocator_->AllocateRaw(size_in_bytes,
+                                                          alignment, origin);
+          break;
+      }
+    }
+  }
+
+  if (allocation.To(&object)) {
+    if (AllocationType::kCode == type && !V8_ENABLE_THIRD_PARTY_HEAP_BOOL) {
+      // Unprotect the memory chunk of the object if it was not unprotected
+      // already.
+      heap_->UnprotectAndRegisterMemoryChunk(
+          object, UnprotectMemoryOrigin::kMainThread);
+      heap_->ZapCodeObject(object.address(), size_in_bytes);
+      if (!large_object) {
+        MemoryChunk::FromHeapObject(object)
+            ->GetCodeObjectRegistry()
+            ->RegisterNewlyAllocatedCodeObject(object.address());
+      }
+    }
+
+#ifdef V8_ENABLE_CONSERVATIVE_STACK_SCANNING
+    if (AllocationType::kReadOnly != type) {
+      DCHECK_TAG_ALIGNED(object.address());
+      Page::FromHeapObject(object)->object_start_bitmap()->SetBit(
+          object.address());
+    }
+#endif  // V8_ENABLE_CONSERVATIVE_STACK_SCANNING
+
+    for (auto& tracker : heap_->allocation_trackers_) {
+      tracker->AllocationEvent(object.address(), size_in_bytes);
+    }
+  }
+
+  return allocation;
+}
+
+AllocationResult HeapAllocator::AllocateRaw(int size_in_bytes,
+                                            AllocationType type,
+                                            AllocationOrigin origin,
+                                            AllocationAlignment alignment) {
+  switch (type) {
+    case AllocationType::kYoung:
+      return AllocateRaw<AllocationType::kYoung>(size_in_bytes, origin,
+                                                 alignment);
+    case AllocationType::kOld:
+      return AllocateRaw<AllocationType::kOld>(size_in_bytes, origin,
+                                               alignment);
+    case AllocationType::kCode:
+      return AllocateRaw<AllocationType::kCode>(size_in_bytes, origin,
+                                                alignment);
+    case AllocationType::kMap:
+      return AllocateRaw<AllocationType::kMap>(size_in_bytes, origin,
+                                               alignment);
+    case AllocationType::kReadOnly:
+      return AllocateRaw<AllocationType::kReadOnly>(size_in_bytes, origin,
+                                                    alignment);
+    case AllocationType::kSharedMap:
+      return AllocateRaw<AllocationType::kSharedMap>(size_in_bytes, origin,
+                                                     alignment);
+    case AllocationType::kSharedOld:
+      return AllocateRaw<AllocationType::kSharedOld>(size_in_bytes, origin,
+                                                     alignment);
+  }
+  UNREACHABLE();
+}
+
+AllocationResult HeapAllocator::AllocateRawData(int size_in_bytes,
+                                                AllocationType type,
+                                                AllocationOrigin origin,
+                                                AllocationAlignment alignment) {
+  switch (type) {
+    case AllocationType::kYoung:
+      return AllocateRaw<AllocationType::kYoung>(size_in_bytes, origin,
+                                                 alignment);
+    case AllocationType::kOld:
+      return AllocateRaw<AllocationType::kOld>(size_in_bytes, origin,
+                                               alignment);
+    case AllocationType::kCode:
+    case AllocationType::kMap:
+    case AllocationType::kReadOnly:
+    case AllocationType::kSharedMap:
+    case AllocationType::kSharedOld:
+      UNREACHABLE();
+  }
+  UNREACHABLE();
+}
+
+template <HeapAllocator::AllocationRetryMode mode>
+V8_WARN_UNUSED_RESULT V8_INLINE HeapObject HeapAllocator::AllocateRawWith(
+    int size, AllocationType allocation, AllocationOrigin origin,
+    AllocationAlignment alignment) {
+  AllocationResult result;
+  HeapObject object;
+  if (allocation == AllocationType::kYoung) {
+    result = AllocateRaw<AllocationType::kYoung>(size, origin, alignment);
+    if (result.To(&object)) {
+      return object;
+    }
+  } else if (allocation == AllocationType::kOld) {
+    result = AllocateRaw<AllocationType::kOld>(size, origin, alignment);
+    if (result.To(&object)) {
+      return object;
+    }
+  }
+  switch (mode) {
+    case kLightRetry:
+      result = AllocateRawWithLightRetrySlowPath(size, allocation, origin,
+                                                 alignment);
+      break;
+    case kRetryOrFail:
+      result = AllocateRawWithRetryOrFailSlowPath(size, allocation, origin,
+                                                  alignment);
+      break;
+  }
+  if (result.To(&object)) {
+    return object;
+  }
+  return HeapObject();
+}
+
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_HEAP_HEAP_ALLOCATOR_INL_H_

src/heap/heap-allocator.cc

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/heap/heap-allocator.h"
+
+#include "src/base/logging.h"
+#include "src/common/globals.h"
+#include "src/execution/isolate.h"
+#include "src/heap/heap-allocator-inl.h"
+#include "src/heap/heap-inl.h"
+#include "src/logging/counters.h"
+
+namespace v8 {
+namespace internal {
+
+class Heap;
+
+HeapAllocator::HeapAllocator(Heap* heap) : heap_(heap) {}
+
+void HeapAllocator::Setup() {
+  for (int i = FIRST_SPACE; i <= LAST_SPACE; ++i) {
+    spaces_[i] = heap_->space(i);
+  }
+  shared_old_allocator_ = heap_->shared_old_allocator_.get();
+  shared_map_allocator_ = heap_->shared_map_allocator_.get();
+}
+
+void HeapAllocator::SetReadOnlySpace(ReadOnlySpace* read_only_space) {
+  read_only_space_ = read_only_space;
+}
+
+AllocationResult HeapAllocator::AllocateRawLargeInternal(
+    int size_in_bytes, AllocationType allocation, AllocationOrigin origin,
+    AllocationAlignment alignment) {
+  DCHECK_GT(size_in_bytes, heap_->MaxRegularHeapObjectSize(allocation));
+  switch (allocation) {
+    case AllocationType::kYoung:
+      return new_lo_space()->AllocateRaw(size_in_bytes);
+    case AllocationType::kOld:
+      return lo_space()->AllocateRaw(size_in_bytes);
+    case AllocationType::kCode:
+      return code_lo_space()->AllocateRaw(size_in_bytes);
+    case AllocationType::kMap:
+    case AllocationType::kReadOnly:
+    case AllocationType::kSharedMap:
+    case AllocationType::kSharedOld:
+      UNREACHABLE();
+  }
+}
+
+namespace {
+
+constexpr AllocationSpace AllocationTypeToGCSpace(AllocationType type) {
+  switch (type) {
+    case AllocationType::kYoung:
+      return NEW_SPACE;
+    case AllocationType::kOld:
+    case AllocationType::kCode:
+    case AllocationType::kMap:
+      // OLD_SPACE indicates full GC.
+      return OLD_SPACE;
+    case AllocationType::kReadOnly:
+    case AllocationType::kSharedMap:
+    case AllocationType::kSharedOld:
+      UNREACHABLE();
+  }
+}
+
+}  // namespace
+
+AllocationResult HeapAllocator::AllocateRawWithLightRetrySlowPath(
+    int size, AllocationType allocation, AllocationOrigin origin,
+    AllocationAlignment alignment) {
+  AllocationResult result = AllocateRaw(size, allocation, origin, alignment);
+  if (!result.IsFailure()) {
+    return result;
+  }
+
+  // Two GCs before returning failure.
+  for (int i = 0; i < 2; i++) {
+    if (IsSharedAllocationType(allocation)) {
+      heap_->CollectSharedGarbage(GarbageCollectionReason::kAllocationFailure);
+    } else {
+      heap_->CollectGarbage(AllocationTypeToGCSpace(allocation),
+                            GarbageCollectionReason::kAllocationFailure);
+    }
+    result = AllocateRaw(size, allocation, origin, alignment);
+    if (!result.IsFailure()) {
+      return result;
+    }
+  }
+  return result;
+}
+
+AllocationResult HeapAllocator::AllocateRawWithRetryOrFailSlowPath(
+    int size, AllocationType allocation, AllocationOrigin origin,
+    AllocationAlignment alignment) {
+  AllocationResult result =
+      AllocateRawWithLightRetrySlowPath(size, allocation, origin, alignment);
+  if (!result.IsFailure()) return result;
+
+  heap_->isolate()->counters()->gc_last_resort_from_handles()->Increment();
+  if (IsSharedAllocationType(allocation)) {
+    heap_->CollectSharedGarbage(GarbageCollectionReason::kLastResort);
+
+    // We need always_allocate() to be true both on the client- and
+    // server-isolate. It is used in both code paths.
+    AlwaysAllocateScope shared_scope(
+        heap_->isolate()->shared_isolate()->heap());
+    AlwaysAllocateScope client_scope(heap_);
+    result = AllocateRaw(size, allocation, origin, alignment);
+  } else {
+    heap_->CollectAllAvailableGarbage(GarbageCollectionReason::kLastResort);
+
+    AlwaysAllocateScope scope(heap_);
+    result = AllocateRaw(size, allocation, origin, alignment);
+  }
+
+  if (!result.IsFailure()) {
+    return result;
+  }
+
+  v8::internal::V8::FatalProcessOutOfMemory(heap_->isolate(),
+                                            "CALL_AND_RETRY_LAST", true);
+}
+
+#ifdef DEBUG
+
+void HeapAllocator::IncrementObjectCounters() {
+  heap_->isolate()->counters()->objs_since_last_full()->Increment();
+  heap_->isolate()->counters()->objs_since_last_young()->Increment();
+}
+
+#endif  // DEBUG
+
+#ifdef V8_ENABLE_ALLOCATION_TIMEOUT
+
+void HeapAllocator::SetAllocationTimeout(int allocation_timeout) {
+  allocation_timeout_ = allocation_timeout;
+}
+
+void HeapAllocator::UpdateAllocationTimeout() {
+  if (FLAG_random_gc_interval <= 0 && FLAG_gc_interval < 0) return;
+
+  int new_timeout;
+  if (FLAG_random_gc_interval > 0) {
+    new_timeout = allocation_timeout_ <= 0
+                      ? heap_->isolate()->fuzzer_rng()->NextInt(
+                            FLAG_random_gc_interval + 1)
+                      : allocation_timeout_;
+
+  } else {
+    DCHECK_GE(FLAG_gc_interval, 0);
+    new_timeout = FLAG_gc_interval;
+  }
+  DCHECK_GE(new_timeout, 0);
+
+  // Reset the allocation timeout, but make sure to allow at least a few
+  // allocations after a collection. The reason for this is that we have a lot
+  // of allocation sequences and we assume that a garbage collection will allow
+  // the subsequent allocation attempts to go through.
+  //
+  // TODO(v8:12615): Move `kFewAllocationsHeadroom` behind
+  // `FLAG_random_gc_interval`.
+  constexpr int kFewAllocationsHeadroom = 6;
+  allocation_timeout_ = std::max(kFewAllocationsHeadroom, new_timeout);
+}
+
+#endif  // V8_ENABLE_ALLOCATION_TIMEOUT
+
+}  // namespace internal
+}  // namespace v8

src/heap/heap-allocator.h

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_HEAP_HEAP_ALLOCATOR_H_
+#define V8_HEAP_HEAP_ALLOCATOR_H_
+
+#include "include/v8config.h"
+#include "src/base/macros.h"
+#include "src/common/globals.h"
+#include "src/heap/allocation-result.h"
+
+namespace v8 {
+namespace internal {
+
+class CodeLargeObjectSpace;
+class ConcurrentAllocator;
+class Heap;
+class NewSpace;
+class NewLargeObjectSpace;
+class OldLargeObjectSpace;
+class PagedSpace;
+class ReadOnlySpace;
+class Space;
+
+// Allocator for the main thread. All exposed functions internally call the
+// right bottleneck.
+class V8_EXPORT_PRIVATE HeapAllocator final {
+ public:
+  explicit HeapAllocator(Heap*);
+
+  void Setup();
+  void SetReadOnlySpace(ReadOnlySpace*);
+
+  // Supports all `AllocationType` types.
+  //
+  // Returns a failed result on an unsuccessful allocation attempt.
+  V8_WARN_UNUSED_RESULT V8_INLINE AllocationResult
+  AllocateRaw(int size_in_bytes, AllocationType allocation,
+              AllocationOrigin origin = AllocationOrigin::kRuntime,
+              AllocationAlignment alignment = kTaggedAligned);
+
+  // Supports all `AllocationType` types. Use when type is statically known.
+  //
+  // Returns a failed result on an unsuccessful allocation attempt.
+  template <AllocationType type>
+  V8_WARN_UNUSED_RESULT V8_INLINE AllocationResult AllocateRaw(
+      int size_in_bytes, AllocationOrigin origin = AllocationOrigin::kRuntime,
+      AllocationAlignment alignment = kTaggedAligned);
+
+  // Supports only `AllocationType::kYoung` and `AllocationType::kOld`.
+  //
+  // Returns a failed result on an unsuccessful allocation attempt.
+  V8_WARN_UNUSED_RESULT V8_INLINE AllocationResult
+  AllocateRawData(int size_in_bytes, AllocationType allocation,
+                  AllocationOrigin origin = AllocationOrigin::kRuntime,
+                  AllocationAlignment alignment = kTaggedAligned);
+
+  enum AllocationRetryMode { kLightRetry, kRetryOrFail };
+
+  // Supports all `AllocationType` types and allows specifying retry handling.
+  template <AllocationRetryMode mode>
+  V8_WARN_UNUSED_RESULT V8_INLINE HeapObject
+  AllocateRawWith(int size, AllocationType allocation,
+                  AllocationOrigin origin = AllocationOrigin::kRuntime,
+                  AllocationAlignment alignment = kTaggedAligned);
+
+#ifdef V8_ENABLE_ALLOCATION_TIMEOUT
+  void UpdateAllocationTimeout();
+  void SetAllocationTimeout(int allocation_timeout);
+#endif  // V8_ENABLE_ALLOCATION_TIMEOUT
+
+ private:
+  V8_INLINE PagedSpace* code_space() const;
+  V8_INLINE CodeLargeObjectSpace* code_lo_space() const;
+  V8_INLINE PagedSpace* map_space() const;
+  V8_INLINE NewSpace* new_space() const;
+  V8_INLINE NewLargeObjectSpace* new_lo_space() const;
+  V8_INLINE OldLargeObjectSpace* lo_space() const;
+  V8_INLINE PagedSpace* old_space() const;
+  V8_INLINE ReadOnlySpace* read_only_space() const;
+
+  V8_WARN_UNUSED_RESULT AllocationResult AllocateRawLargeInternal(
+      int size_in_bytes, AllocationType allocation, AllocationOrigin origin,
+      AllocationAlignment alignment);
+
+  V8_WARN_UNUSED_RESULT AllocationResult AllocateRawWithRetryOrFailSlowPath(
+      int size, AllocationType allocation, AllocationOrigin origin,
+      AllocationAlignment alignment);
+
+  V8_WARN_UNUSED_RESULT AllocationResult AllocateRawWithLightRetrySlowPath(
+      int size, AllocationType allocation, AllocationOrigin origin,
+      AllocationAlignment alignment);
+
+#ifdef DEBUG
+  void IncrementObjectCounters();
+#endif  // DEBUG
+
+  Heap* const heap_;
+  Space* spaces_[LAST_SPACE + 1];
+  ReadOnlySpace* read_only_space_;
+
+  ConcurrentAllocator* shared_old_allocator_;
+  ConcurrentAllocator* shared_map_allocator_;
+
+#ifdef V8_ENABLE_ALLOCATION_TIMEOUT
+  // If the --gc-interval flag is set to a positive value, this variable
+  // holds the value indicating the number of allocations remain until the
+  // next failure and garbage collection.
+  int allocation_timeout_ = 0;
+#endif  // V8_ENABLE_ALLOCATION_TIMEOUT
+};
+
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_HEAP_HEAP_ALLOCATOR_H_

src/heap/heap-inl.h

@@ -21,6 +21,7 @@
 #include "src/heap/code-object-registry.h"
 #include "src/heap/concurrent-allocator-inl.h"
 #include "src/heap/concurrent-allocator.h"
+#include "src/heap/heap-allocator-inl.h"
 #include "src/heap/heap-write-barrier.h"
 #include "src/heap/heap.h"
 #include "src/heap/large-spaces.h"
@@ -199,146 +200,26 @@ int Heap::MaxRegularHeapObjectSize(AllocationType allocation) {
 AllocationResult Heap::AllocateRaw(int size_in_bytes, AllocationType type,
                                    AllocationOrigin origin,
                                    AllocationAlignment alignment) {
-  DCHECK_EQ(gc_state(), NOT_IN_GC);
-  DCHECK(AllowHandleAllocation::IsAllowed());
-  DCHECK(AllowHeapAllocation::IsAllowed());
-#ifdef V8_ENABLE_ALLOCATION_TIMEOUT
-  if (FLAG_random_gc_interval > 0 || FLAG_gc_interval >= 0) {
-    if (!always_allocate() && Heap::allocation_timeout_-- <= 0) {
-      return AllocationResult::Failure();
-    }
-  }
-#endif  // V8_ENABLE_ALLOCATION_TIMEOUT
-#ifdef DEBUG
-  IncrementObjectCounters();
-#endif  // DEBUG
-
-  if (CanSafepoint()) {
-    main_thread_local_heap()->Safepoint();
-  }
-
-  const size_t large_object_threshold = MaxRegularHeapObjectSize(type);
-  const bool large_object =
-      static_cast<size_t>(size_in_bytes) > large_object_threshold;
-
-  HeapObject object;
-  AllocationResult allocation;
-
-  if (FLAG_single_generation && type == AllocationType::kYoung) {
-    type = AllocationType::kOld;
-  }
-
-  if (V8_ENABLE_THIRD_PARTY_HEAP_BOOL) {
-    allocation = tp_heap_->Allocate(size_in_bytes, type, alignment);
-  } else {
-    if (V8_UNLIKELY(large_object)) {
-      allocation =
-          AllocateRawLargeInternal(size_in_bytes, type, origin, alignment);
-    } else {
-      switch (type) {
-        case AllocationType::kYoung:
-          allocation =
-              new_space_->AllocateRaw(size_in_bytes, alignment, origin);
-          break;
-        case AllocationType::kOld:
-          allocation =
-              old_space_->AllocateRaw(size_in_bytes, alignment, origin);
-          break;
-        case AllocationType::kCode:
-          DCHECK_EQ(alignment, AllocationAlignment::kTaggedAligned);
-          DCHECK(AllowCodeAllocation::IsAllowed());
-          allocation = code_space_->AllocateRawUnaligned(size_in_bytes);
-          break;
-        case AllocationType::kMap: {
-          DCHECK_EQ(alignment, AllocationAlignment::kTaggedAligned);
-          PagedSpace* allocation_space =
-              V8_LIKELY(map_space_) ? static_cast<PagedSpace*>(map_space_)
-                                    : static_cast<PagedSpace*>(old_space_);
-          allocation = allocation_space->AllocateRawUnaligned(size_in_bytes);
-          break;
-        }
-        case AllocationType::kReadOnly:
-          DCHECK(CanAllocateInReadOnlySpace());
-          DCHECK_EQ(AllocationOrigin::kRuntime, origin);
-          allocation = read_only_space_->AllocateRaw(size_in_bytes, alignment);
-          break;
-        case AllocationType::kSharedMap: {
-          ConcurrentAllocator* allocator = V8_LIKELY(shared_map_allocator_)
-                                               ? shared_map_allocator_.get()
-                                               : shared_old_allocator_.get();
-          allocation = allocator->AllocateRaw(size_in_bytes, alignment, origin);
-          break;
-        }
-        case AllocationType::kSharedOld:
-          allocation = shared_old_allocator_->AllocateRaw(size_in_bytes,
-                                                          alignment, origin);
-          break;
-      }
-    }
-  }
-
-  if (allocation.To(&object)) {
-    if (AllocationType::kCode == type && !V8_ENABLE_THIRD_PARTY_HEAP_BOOL) {
-      // Unprotect the memory chunk of the object if it was not unprotected
-      // already.
-      UnprotectAndRegisterMemoryChunk(object,
-                                      UnprotectMemoryOrigin::kMainThread);
-      ZapCodeObject(object.address(), size_in_bytes);
-      if (!large_object) {
-        MemoryChunk::FromHeapObject(object)
-            ->GetCodeObjectRegistry()
-            ->RegisterNewlyAllocatedCodeObject(object.address());
-      }
-    }
-
-#ifdef V8_ENABLE_CONSERVATIVE_STACK_SCANNING
-    if (AllocationType::kReadOnly != type) {
-      DCHECK_TAG_ALIGNED(object.address());
-      Page::FromHeapObject(object)->object_start_bitmap()->SetBit(
-          object.address());
-    }
-#endif  // V8_ENABLE_CONSERVATIVE_STACK_SCANNING
-
-    for (auto& tracker : allocation_trackers_) {
-      tracker->AllocationEvent(object.address(), size_in_bytes);
-    }
-  }
-
-  return allocation;
+  return heap_allocator_.AllocateRaw(size_in_bytes, type, origin, alignment);
 }
 
 template <Heap::AllocationRetryMode mode>
 HeapObject Heap::AllocateRawWith(int size, AllocationType allocation,
                                  AllocationOrigin origin,
                                  AllocationAlignment alignment) {
-  DCHECK(AllowHandleAllocation::IsAllowed());
-  DCHECK(AllowHeapAllocation::IsAllowed());
-  DCHECK_EQ(gc_state(), NOT_IN_GC);
-  if (allocation == AllocationType::kYoung) {
-    auto result = AllocateRaw(size, AllocationType::kYoung, origin, alignment);
-    HeapObject object;
-    if (result.To(&object)) return object;
-
-  } else if (allocation == AllocationType::kOld) {
-    auto result = AllocateRaw(size, AllocationType::kOld, origin, alignment);
-    HeapObject object;
-    if (result.To(&object)) return object;
-  }
-  switch (mode) {
-    case kLightRetry:
-      return AllocateRawWithLightRetrySlowPath(size, allocation, origin,
-                                               alignment);
-    case kRetryOrFail:
-      return AllocateRawWithRetryOrFailSlowPath(size, allocation, origin,
-                                                alignment);
-  }
-  UNREACHABLE();
+  return heap_allocator_.AllocateRawWith < mode ==
+                 AllocationRetryMode::kLightRetry
+             ? HeapAllocator::kLightRetry
+             : HeapAllocator::kRetryOrFail >
+                   (size, allocation, origin, alignment);
 }
 
 Address Heap::AllocateRawOrFail(int size, AllocationType allocation,
                                 AllocationOrigin origin,
                                 AllocationAlignment alignment) {
-  return AllocateRawWith<kRetryOrFail>(size, allocation, origin, alignment)
+  return heap_allocator_
+      .AllocateRawWith<HeapAllocator::kRetryOrFail>(size, allocation, origin,
+                                                    alignment)
       .address();
 }
 

src/heap/heap.cc

@@ -204,6 +204,7 @@ class ScavengeTaskObserver : public AllocationObserver {
 
 Heap::Heap()
     : isolate_(isolate()),
+      heap_allocator_(this),
       memory_pressure_level_(MemoryPressureLevel::kNone),
       global_pretenuring_feedback_(kInitialFeedbackCapacity),
       safepoint_(std::make_unique<IsolateSafepoint>(this)),
@@ -1022,15 +1023,8 @@ void Heap::GarbageCollectionPrologue(
   if (force_gc_on_next_allocation_) force_gc_on_next_allocation_ = false;
 
 #ifdef V8_ENABLE_ALLOCATION_TIMEOUT
-  // Reset the allocation timeout, but make sure to allow at least a few
-  // allocations after a collection. The reason for this is that we have a lot
-  // of allocation sequences and we assume that a garbage collection will allow
-  // the subsequent allocation attempts to go through.
-  if (FLAG_random_gc_interval > 0 || FLAG_gc_interval >= 0) {
-    allocation_timeout_ =
-        std::max(6, NextAllocationTimeout(allocation_timeout_));
-  }
-#endif
+  heap_allocator_.UpdateAllocationTimeout();
+#endif  // V8_ENABLE_ALLOCATION_TIMEOUT
 
   // There may be an allocation memento behind objects in new space. Upon
   // evacuation of a non-full new space (or if we are on the last page) there
@@ -5637,118 +5631,13 @@ void Heap::DisableInlineAllocation() {
   }
 }
 
-namespace {
-
-constexpr AllocationSpace AllocationTypeToGCSpace(AllocationType type) {
-  switch (type) {
-    case AllocationType::kYoung:
-      return NEW_SPACE;
-    case AllocationType::kOld:
-    case AllocationType::kCode:
-    case AllocationType::kMap:
-      // OLD_SPACE indicates full GC.
-      return OLD_SPACE;
-    case AllocationType::kReadOnly:
-    case AllocationType::kSharedMap:
-    case AllocationType::kSharedOld:
-      UNREACHABLE();
-  }
-}
-
-}  // namespace
-
-HeapObject Heap::AllocateRawWithLightRetrySlowPath(
-    int size, AllocationType allocation, AllocationOrigin origin,
-    AllocationAlignment alignment) {
-  HeapObject result;
-  AllocationResult alloc = AllocateRaw(size, allocation, origin, alignment);
-  if (alloc.To(&result)) {
-    // DCHECK that the successful allocation is not "exception". The one
-    // exception to this is when allocating the "exception" object itself, in
-    // which case this must be an ROSpace allocation and the exception object
-    // in the roots has to be unset.
-    DCHECK((CanAllocateInReadOnlySpace() &&
-            allocation == AllocationType::kReadOnly &&
-            ReadOnlyRoots(this).unchecked_exception() == Smi::zero()) ||
-           result != ReadOnlyRoots(this).exception());
-    return result;
-  }
-  // Two GCs before panicking. In newspace will almost always succeed.
-  for (int i = 0; i < 2; i++) {
-    if (IsSharedAllocationType(allocation)) {
-      CollectSharedGarbage(GarbageCollectionReason::kAllocationFailure);
-    } else {
-      CollectGarbage(AllocationTypeToGCSpace(allocation),
-                     GarbageCollectionReason::kAllocationFailure);
-    }
-    alloc = AllocateRaw(size, allocation, origin, alignment);
-    if (alloc.To(&result)) {
-      DCHECK(result != ReadOnlyRoots(this).exception());
-      return result;
-    }
-  }
-  return HeapObject();
-}
-
-AllocationResult Heap::AllocateRawLargeInternal(int size_in_bytes,
-                                                AllocationType allocation,
-                                                AllocationOrigin origin,
-                                                AllocationAlignment alignment) {
-  DCHECK_GT(size_in_bytes, MaxRegularHeapObjectSize(allocation));
-  switch (allocation) {
-    case AllocationType::kYoung:
-      return new_lo_space_->AllocateRaw(size_in_bytes);
-    case AllocationType::kOld:
-      return lo_space_->AllocateRaw(size_in_bytes);
-    case AllocationType::kCode:
-      return code_lo_space_->AllocateRaw(size_in_bytes);
-    case AllocationType::kMap:
-    case AllocationType::kReadOnly:
-    case AllocationType::kSharedMap:
-    case AllocationType::kSharedOld:
-      UNREACHABLE();
-  }
-}
-
-HeapObject Heap::AllocateRawWithRetryOrFailSlowPath(
-    int size, AllocationType allocation, AllocationOrigin origin,
-    AllocationAlignment alignment) {
-  AllocationResult alloc;
-  HeapObject result =
-      AllocateRawWithLightRetrySlowPath(size, allocation, origin, alignment);
-  if (!result.is_null()) return result;
-
-  isolate()->counters()->gc_last_resort_from_handles()->Increment();
-  if (IsSharedAllocationType(allocation)) {
-    CollectSharedGarbage(GarbageCollectionReason::kLastResort);
-
-    // We need always_allocate() to be true both on the client- and
-    // server-isolate. It is used in both code paths.
-    AlwaysAllocateScope shared_scope(isolate()->shared_isolate()->heap());
-    AlwaysAllocateScope client_scope(isolate()->heap());
-    alloc = AllocateRaw(size, allocation, origin, alignment);
-  } else {
-    CollectAllAvailableGarbage(GarbageCollectionReason::kLastResort);
-
-    AlwaysAllocateScope scope(this);
-    alloc = AllocateRaw(size, allocation, origin, alignment);
-  }
-
-  if (alloc.To(&result)) {
-    DCHECK(result != ReadOnlyRoots(this).exception());
-    return result;
-  }
-  // TODO(1181417): Fix this.
-  FatalProcessOutOfMemory("CALL_AND_RETRY_LAST");
-}
-
 void Heap::SetUp(LocalHeap* main_thread_local_heap) {
   DCHECK_NULL(main_thread_local_heap_);
   main_thread_local_heap_ = main_thread_local_heap;
 
 #ifdef V8_ENABLE_ALLOCATION_TIMEOUT
-  allocation_timeout_ = NextAllocationTimeout();
-#endif
+  heap_allocator_.UpdateAllocationTimeout();
+#endif  // V8_ENABLE_ALLOCATION_TIMEOUT
 
 #ifdef V8_ENABLE_THIRD_PARTY_HEAP
   tp_heap_ = third_party_heap::Heap::New(isolate());
@@ -5846,6 +5735,7 @@ void Heap::SetUpFromReadOnlyHeap(ReadOnlyHeap* ro_heap) {
                  read_only_space_ == ro_heap->read_only_space());
   space_[RO_SPACE] = nullptr;
   read_only_space_ = ro_heap->read_only_space();
+  heap_allocator_.SetReadOnlySpace(read_only_space_);
 }
 
 void Heap::ReplaceReadOnlySpace(SharedReadOnlySpace* space) {
@@ -5856,6 +5746,7 @@ void Heap::ReplaceReadOnlySpace(SharedReadOnlySpace* space) {
   }
 
   read_only_space_ = space;
+  heap_allocator_.SetReadOnlySpace(read_only_space_);
 }
 
 class StressConcurrentAllocationObserver : public AllocationObserver {
@@ -5966,6 +5857,7 @@ void Heap::SetUpSpaces(LinearAllocationArea* new_allocation_info,
   }
 
   main_thread_local_heap()->SetUpMainThread();
+  heap_allocator_.Setup();
 }
 
 void Heap::InitializeHashSeed() {
@@ -5981,19 +5873,6 @@ void Heap::InitializeHashSeed() {
       0, reinterpret_cast<byte*>(&new_hash_seed), kInt64Size);
 }
 
-int Heap::NextAllocationTimeout(int current_timeout) {
-  if (FLAG_random_gc_interval > 0) {
-    // If current timeout hasn't reached 0 the GC was caused by something
-    // different than --stress-atomic-gc flag and we don't update the timeout.
-    if (current_timeout <= 0) {
-      return isolate()->fuzzer_rng()->NextInt(FLAG_random_gc_interval + 1);
-    } else {
-      return current_timeout;
-    }
-  }
-  return FLAG_gc_interval;
-}
-
 void Heap::PrintMaxMarkingLimitReached() {
   PrintF("\n### Maximum marking limit reached = %.02lf\n",
          max_marking_limit_reached_);
@@ -7577,5 +7456,11 @@ void StrongRootBlockAllocator::deallocate(Address* p, size_t n) noexcept {
   base::Free(block);
 }
 
+#ifdef V8_ENABLE_ALLOCATION_TIMEOUT
+void Heap::set_allocation_timeout(int allocation_timeout) {
+  heap_allocator_.SetAllocationTimeout(allocation_timeout);
+}
+#endif  // V8_ENABLE_ALLOCATION_TIMEOUT
+
 }  // namespace internal
 }  // namespace v8

src/heap/heap.h

@@ -27,6 +27,7 @@
 #include "src/common/globals.h"
 #include "src/heap/allocation-observer.h"
 #include "src/heap/allocation-result.h"
+#include "src/heap/heap-allocator.h"
 #include "src/init/heap-symbols.h"
 #include "src/objects/allocation-site.h"
 #include "src/objects/fixed-array.h"
@@ -126,15 +127,6 @@ enum ExternalBackingStoreType { kArrayBuffer, kExternalString, kNumTypes };
 
 enum class RetainingPathOption { kDefault, kTrackEphemeronPath };
 
-enum class AllocationOrigin {
-  kGeneratedCode = 0,
-  kRuntime = 1,
-  kGC = 2,
-  kFirstAllocationOrigin = kGeneratedCode,
-  kLastAllocationOrigin = kGC,
-  kNumberOfAllocationOrigins = kLastAllocationOrigin + 1
-};
-
 // These values are persisted to logs. Entries should not be renumbered and
 // numeric values should never be reused. If you add new items here, update
 // src/tools/metrics/histograms/enums.xml in chromium.
@@ -1585,8 +1577,8 @@ class Heap {
 #endif
 
 #ifdef V8_ENABLE_ALLOCATION_TIMEOUT
-  void set_allocation_timeout(int timeout) { allocation_timeout_ = timeout; }
-#endif
+  void V8_EXPORT_PRIVATE set_allocation_timeout(int allocation_timeout);
+#endif  // V8_ENABLE_ALLOCATION_TIMEOUT
 
 #ifdef DEBUG
   void VerifyCountersAfterSweeping();
@@ -1844,8 +1836,6 @@ class Heap {
                                 GCIdleTimeHeapState heap_state, double start_ms,
                                 double deadline_in_ms);
 
-  int NextAllocationTimeout(int current_timeout = 0);
-
   void PrintMaxMarkingLimitReached();
   void PrintMaxNewSpaceSizeReached();
 
@@ -2057,13 +2047,6 @@ class Heap {
               AllocationOrigin origin = AllocationOrigin::kRuntime,
               AllocationAlignment alignment = kTaggedAligned);
 
-  // Allocates an uninitialized large object. Used as dispatch by
-  // `AllocateRaw()` for large objects. Do not call this from anywhere else.
-  V8_EXPORT_PRIVATE V8_WARN_UNUSED_RESULT AllocationResult
-  AllocateRawLargeInternal(int size_in_bytes, AllocationType allocation,
-                           AllocationOrigin origin = AllocationOrigin::kRuntime,
-                           AllocationAlignment alignment = kTaggedAligned);
-
   // This method will try to allocate objects quickly (AllocationType::kYoung)
   // otherwise it falls back to a slower path indicated by the mode.
   enum AllocationRetryMode { kLightRetry, kRetryOrFail };
@@ -2079,25 +2062,6 @@ class Heap {
       AllocationOrigin origin = AllocationOrigin::kRuntime,
       AllocationAlignment alignment = kTaggedAligned);
 
-  // This method will try to perform an allocation of a given size of a given
-  // AllocationType. If the allocation fails, a regular full garbage collection
-  // is triggered and the allocation is retried. This is performed multiple
-  // times. If after that retry procedure the allocation still fails nullptr is
-  // returned.
-  V8_WARN_UNUSED_RESULT HeapObject AllocateRawWithLightRetrySlowPath(
-      int size, AllocationType allocation, AllocationOrigin origin,
-      AllocationAlignment alignment = kTaggedAligned);
-
-  // This method will try to perform an allocation of a given size of a given
-  // AllocationType. If the allocation fails, a regular full garbage collection
-  // is triggered and the allocation is retried. This is performed multiple
-  // times. If after that retry procedure the allocation still fails a "hammer"
-  // garbage collection is triggered which tries to significantly reduce memory.
-  // If the allocation still fails after that a fatal error is thrown.
-  V8_WARN_UNUSED_RESULT HeapObject AllocateRawWithRetryOrFailSlowPath(
-      int size, AllocationType allocation, AllocationOrigin origin,
-      AllocationAlignment alignment = kTaggedAligned);
-
   // Allocates a heap object based on the map.
   V8_WARN_UNUSED_RESULT AllocationResult Allocate(Handle<Map> map,
                                                   AllocationType allocation);
@@ -2149,6 +2113,8 @@ class Heap {
   // more expedient to get at the isolate directly from within Heap methods.
   Isolate* isolate_ = nullptr;
 
+  HeapAllocator heap_allocator_;
+
   // These limits are initialized in Heap::ConfigureHeap based on the resource
   // constraints and flags.
   size_t code_range_size_ = 0;
@@ -2442,13 +2408,6 @@ class Heap {
   base::Mutex unprotected_memory_chunks_mutex_;
   std::unordered_set<MemoryChunk*> unprotected_memory_chunks_;
 
-#ifdef V8_ENABLE_ALLOCATION_TIMEOUT
-  // If the --gc-interval flag is set to a positive value, this
-  // variable holds the value indicating the number of allocations
-  // remain until the next failure and garbage collection.
-  int allocation_timeout_ = 0;
-#endif  // V8_ENABLE_ALLOCATION_TIMEOUT
-
   std::unordered_map<HeapObject, HeapObject, Object::Hasher> retainer_;
   std::unordered_map<HeapObject, Root, Object::Hasher> retaining_root_;
   // If an object is retained by an ephemeron, then the retaining key of the
@@ -2473,6 +2432,7 @@ class Heap {
   friend class EvacuateVisitorBase;
   friend class GCCallbacksScope;
   friend class GCTracer;
+  friend class HeapAllocator;
   friend class HeapObjectIterator;
   friend class ScavengeTaskObserver;
   friend class IgnoreLocalGCRequests;
@@ -2562,6 +2522,7 @@ class V8_NODISCARD AlwaysAllocateScope {
   friend class AlwaysAllocateScopeForTesting;
   friend class Evacuator;
   friend class Heap;
+  friend class HeapAllocator;
   friend class Isolate;
 
   explicit inline AlwaysAllocateScope(Heap* heap);