[heap] Introduce OffThreadLargeObjectSpace

Add a new LargeObjectSpace for off-thread allocation. Similar to the
paged OffThreadSpace, it always allocates pages, doesn't participate
in mark or sweep, and can be merged into the OldLargeObjectSpace once
its objects are ready.

Bug: chromium:1011762
Change-Id: I95e2d38b10a9cc5eae4ffd35afef95272e13c731
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1881153
Commit-Queue: Leszek Swirski <leszeks@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#64727}

src/heap/spaces.cc

@@ -4311,6 +4311,18 @@ OldLargeObjectSpace::OldLargeObjectSpace(Heap* heap)
 OldLargeObjectSpace::OldLargeObjectSpace(Heap* heap, AllocationSpace id)
     : LargeObjectSpace(heap, id) {}
 
+void OldLargeObjectSpace::MergeOffThreadSpace(
+    OffThreadLargeObjectSpace* other) {
+  DCHECK(identity() == other->identity());
+
+  while (!other->memory_chunk_list().Empty()) {
+    LargePage* page = other->first_page();
+    int size = page->GetObject().Size();
+    other->RemovePage(page, size);
+    AddPage(page, size);
+  }
+}
+
 NewLargeObjectSpace::NewLargeObjectSpace(Heap* heap, size_t capacity)
     : LargeObjectSpace(heap, NEW_LO_SPACE),
       pending_object_(0),
@@ -4415,5 +4427,20 @@ void CodeLargeObjectSpace::RemovePage(LargePage* page, size_t object_size) {
   OldLargeObjectSpace::RemovePage(page, object_size);
 }
 
+OffThreadLargeObjectSpace::OffThreadLargeObjectSpace(Heap* heap)
+    : LargeObjectSpace(heap, LO_SPACE) {}
+
+AllocationResult OffThreadLargeObjectSpace::AllocateRaw(int object_size) {
+  LargePage* page = AllocateLargePage(object_size, NOT_EXECUTABLE);
+  if (page == nullptr) return AllocationResult::Retry(identity());
+
+  return page->GetObject();
+}
+
+void OffThreadLargeObjectSpace::FreeUnmarkedObjects() {
+  // We should never try to free objects in this space.
+  UNREACHABLE();
+}
+
 }  // namespace internal
 }  // namespace v8

src/heap/spaces.h

@@ -3244,7 +3244,7 @@ class LargeObjectSpace : public Space {
   int PageCount() { return page_count_; }
 
   // Frees unmarked objects.
-  void FreeUnmarkedObjects();
+  virtual void FreeUnmarkedObjects();
 
   // Checks whether a heap object is in this space; O(1).
   V8_EXPORT_PRIVATE bool Contains(HeapObject obj);
@@ -3288,6 +3288,8 @@ class LargeObjectSpace : public Space {
   friend class LargeObjectSpaceObjectIterator;
 };
 
+class OffThreadLargeObjectSpace;
+
 class OldLargeObjectSpace : public LargeObjectSpace {
  public:
   explicit OldLargeObjectSpace(Heap* heap);
@@ -3300,6 +3302,8 @@ class OldLargeObjectSpace : public LargeObjectSpace {
 
   void PromoteNewLargeObject(LargePage* page);
 
+  V8_EXPORT_PRIVATE void MergeOffThreadSpace(OffThreadLargeObjectSpace* other);
+
  protected:
   explicit OldLargeObjectSpace(Heap* heap, AllocationSpace id);
   V8_WARN_UNUSED_RESULT AllocationResult AllocateRaw(int object_size,
@@ -3359,6 +3363,22 @@ class CodeLargeObjectSpace : public OldLargeObjectSpace {
   std::unordered_map<Address, LargePage*> chunk_map_;
 };
 
+class V8_EXPORT_PRIVATE OffThreadLargeObjectSpace : public LargeObjectSpace {
+ public:
+  explicit OffThreadLargeObjectSpace(Heap* heap);
+
+  V8_WARN_UNUSED_RESULT AllocationResult AllocateRaw(int object_size);
+
+  void FreeUnmarkedObjects() override;
+
+ protected:
+  // OldLargeObjectSpace can mess with OffThreadLargeObjectSpace during merging.
+  friend class OldLargeObjectSpace;
+
+  V8_WARN_UNUSED_RESULT AllocationResult AllocateRaw(int object_size,
+                                                     Executability executable);
+};
+
 class LargeObjectSpaceObjectIterator : public ObjectIterator {
  public:
   explicit LargeObjectSpaceObjectIterator(LargeObjectSpace* space);

test/unittests/heap/spaces-unittest.cc

@@ -4,6 +4,7 @@
 
 #include "src/heap/spaces.h"
 #include <memory>
+#include "src/common/globals.h"
 #include "src/execution/isolate.h"
 #include "src/heap/heap-inl.h"
 #include "src/heap/heap-write-barrier-inl.h"
@@ -135,6 +136,82 @@ TEST_F(SpacesTest, OffThreadSpaceMerge) {
             old_space->CountTotalPages());
 }
 
+class LargeOffThreadAllocationThread final : public base::Thread {
+ public:
+  explicit LargeOffThreadAllocationThread(Heap* heap)
+      : Thread(Options("LargeOffThreadAllocationThread")), heap_(heap) {}
+  void Run() override {
+    off_thread_lo_space_ = std::make_unique<OffThreadLargeObjectSpace>(heap_);
+    EXPECT_TRUE(off_thread_lo_space_ != nullptr);
+
+    const int kNumObjects = 10;
+    const int kExpectedPages = kNumObjects;
+    for (int i = 0; i < kNumObjects; i++) {
+      HeapObject object =
+          off_thread_lo_space_
+              ->AllocateRaw(kMaxRegularHeapObjectSize + kTaggedSize)
+              .ToObjectChecked();
+      heap_->CreateFillerObjectAt(object.address(),
+                                  kMaxRegularHeapObjectSize + kTaggedSize,
+                                  ClearRecordedSlots::kNo);
+    }
+    int pages_in_off_thread_space = off_thread_lo_space_->PageCount();
+    EXPECT_EQ(kExpectedPages, pages_in_off_thread_space);
+  }
+
+  OffThreadLargeObjectSpace* space() { return off_thread_lo_space_.get(); }
+
+ private:
+  Heap* heap_;
+  std::unique_ptr<OffThreadLargeObjectSpace> off_thread_lo_space_;
+};
+
+TEST_F(SpacesTest, OffThreadLargeObjectSpaceAllocate) {
+  Heap* heap = i_isolate()->heap();
+
+  static const int kNumThreads = 10;
+  std::unique_ptr<LargeOffThreadAllocationThread> threads[10];
+  for (int i = 0; i < kNumThreads; ++i) {
+    threads[i] = std::make_unique<LargeOffThreadAllocationThread>(heap);
+  }
+  for (int i = 0; i < kNumThreads; ++i) {
+    CHECK(threads[i]->Start());
+  }
+  for (int i = 0; i < kNumThreads; ++i) {
+    threads[i]->Join();
+  }
+}
+
+TEST_F(SpacesTest, OffThreadLargeObjectSpaceMerge) {
+  Heap* heap = i_isolate()->heap();
+  OldLargeObjectSpace* lo_space = heap->lo_space();
+  EXPECT_TRUE(lo_space != nullptr);
+
+  static const int kNumThreads = 10;
+  std::unique_ptr<LargeOffThreadAllocationThread> threads[10];
+  for (int i = 0; i < kNumThreads; ++i) {
+    threads[i] = std::make_unique<LargeOffThreadAllocationThread>(heap);
+  }
+  for (int i = 0; i < kNumThreads; ++i) {
+    CHECK(threads[i]->Start());
+  }
+  for (int i = 0; i < kNumThreads; ++i) {
+    threads[i]->Join();
+  }
+
+  int pages_in_old_space = lo_space->PageCount();
+
+  int expected_merged_pages = 0;
+  for (int i = 0; i < kNumThreads; ++i) {
+    int pages_in_off_thread_space = threads[i]->space()->PageCount();
+
+    lo_space->MergeOffThreadSpace(threads[i]->space());
+    expected_merged_pages += pages_in_off_thread_space;
+  }
+
+  EXPECT_EQ(pages_in_old_space + expected_merged_pages, lo_space->PageCount());
+}
+
 TEST_F(SpacesTest, WriteBarrierFromHeapObject) {
   constexpr Address address1 = Page::kPageSize;
   HeapObject object1 = HeapObject::unchecked_cast(Object(address1));