Reland "[heap] Introduce per-thread storage for concurrent sweeping"

This is a reland of commit a1b863c1

Original change's description:
> [heap] Introduce per-thread storage for concurrent sweeping
>
> Introduce ConcurrentSweeper as indirection between SweeperJob and
> Sweeper to hold per-thread state during sweeping.
> This will be used by MinorMC sweeping to hold the pretenuring feedback
> map.
>
> Bug: v8:12612
> Change-Id: Ib363339f9109b405e4cae7f2c08cb4f0eacff8d0
> Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3829466
> Commit-Queue: Omer Katz <omerkatz@chromium.org>
> Reviewed-by: Dominik Inführ <dinfuehr@chromium.org>
> Cr-Commit-Position: refs/heads/main@{#82442}

Bug: v8:12612
Change-Id: Ic475d0a8a6e9f9e8b5945bb8d28588db1d524510
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3829480Reviewed-by: Dominik Inführ <dinfuehr@chromium.org>
Commit-Queue: Dominik Inführ <dinfuehr@chromium.org>
Auto-Submit: Omer Katz <omerkatz@chromium.org>
Cr-Commit-Position: refs/heads/main@{#82454}

src/heap/sweeper.cc

@@ -4,6 +4,9 @@
 
 #include "src/heap/sweeper.h"
 
+#include <memory>
+#include <vector>
+
 #include "src/common/globals.h"
 #include "src/execution/vm-state-inl.h"
 #include "src/heap/base/active-system-pages.h"
@@ -25,6 +28,8 @@ Sweeper::Sweeper(Heap* heap, NonAtomicMarkingState* marking_state)
       sweeping_in_progress_(false),
       should_reduce_memory_(false) {}
 
+Sweeper::~Sweeper() { DCHECK(concurrent_sweepers_.empty()); }
+
 Sweeper::PauseScope::PauseScope(Sweeper* sweeper) : sweeper_(sweeper) {
   if (!sweeper_->sweeping_in_progress()) return;
 
@@ -63,10 +68,31 @@ Sweeper::FilterSweepingPagesScope::~FilterSweepingPagesScope() {
   // old_space_sweeping_list_ does not need to be cleared as we don't use it.
 }
 
+class Sweeper::ConcurrentSweeper final {
+ public:
+  explicit ConcurrentSweeper(Sweeper* sweeper) : sweeper_(sweeper) {}
+
+  bool ConcurrentSweepSpace(AllocationSpace identity, JobDelegate* delegate) {
+    while (!delegate->ShouldYield()) {
+      Page* page = sweeper_->GetSweepingPageSafe(identity);
+      if (page == nullptr) return true;
+      sweeper_->ParallelSweepPage(page, identity,
+                                  SweepingMode::kLazyOrConcurrent);
+    }
+    return false;
+  }
+
+ private:
+  Sweeper* const sweeper_;
+};
+
 class Sweeper::SweeperJob final : public JobTask {
  public:
-  SweeperJob(Isolate* isolate, Sweeper* sweeper)
-      : sweeper_(sweeper), tracer_(isolate->heap()->tracer()) {}
+  SweeperJob(Isolate* isolate, Sweeper* sweeper,
+             std::vector<ConcurrentSweeper>* concurrent_sweepers)
+      : sweeper_(sweeper),
+        concurrent_sweepers_(concurrent_sweepers),
+        tracer_(isolate->heap()->tracer()) {}
 
   ~SweeperJob() override = default;
 
@@ -87,7 +113,7 @@ class Sweeper::SweeperJob final : public JobTask {
   size_t GetMaxConcurrency(size_t worker_count) const override {
     const size_t kPagePerTask = 2;
     return std::min<size_t>(
-        kMaxSweeperTasks,
+        concurrent_sweepers_->size(),
         worker_count +
             (sweeper_->ConcurrentSweepingPageCount() + kPagePerTask - 1) /
                 kPagePerTask);
@@ -96,15 +122,19 @@ class Sweeper::SweeperJob final : public JobTask {
  private:
   void RunImpl(JobDelegate* delegate) {
     const int offset = delegate->GetTaskId();
+    DCHECK_LT(offset, concurrent_sweepers_->size());
+    ConcurrentSweeper& sweeper = (*concurrent_sweepers_)[offset];
     for (int i = 0; i < kNumberOfSweepingSpaces; i++) {
       const AllocationSpace space_id = static_cast<AllocationSpace>(
           FIRST_GROWABLE_PAGED_SPACE +
           ((i + offset) % kNumberOfSweepingSpaces));
       DCHECK(IsValidSweepingSpace(space_id));
-      if (!sweeper_->ConcurrentSweepSpace(space_id, delegate)) return;
+      if (!sweeper.ConcurrentSweepSpace(space_id, delegate)) return;
     }
   }
+
   Sweeper* const sweeper_;
+  std::vector<ConcurrentSweeper>* const concurrent_sweepers_;
   GCTracer* const tracer_;
 };
 
@@ -134,13 +164,26 @@ void Sweeper::StartSweeping() {
   });
 }
 
+int Sweeper::NumberOfConcurrentSweepers() const {
+  DCHECK(FLAG_concurrent_sweeping);
+  return std::min(Sweeper::kMaxSweeperTasks,
+                  V8::GetCurrentPlatform()->NumberOfWorkerThreads() + 1);
+}
+
 void Sweeper::StartSweeperTasks() {
   DCHECK(!job_handle_ || !job_handle_->IsValid());
   if (FLAG_concurrent_sweeping && sweeping_in_progress_ &&
       !heap_->delay_sweeper_tasks_for_testing_) {
+    if (concurrent_sweepers_.empty()) {
+      for (int i = 0; i < NumberOfConcurrentSweepers(); ++i) {
+        concurrent_sweepers_.emplace_back(this);
+      }
+    }
+    DCHECK_EQ(NumberOfConcurrentSweepers(), concurrent_sweepers_.size());
     job_handle_ = V8::GetCurrentPlatform()->PostJob(
         TaskPriority::kUserVisible,
-        std::make_unique<SweeperJob>(heap_->isolate(), this));
+        std::make_unique<SweeperJob>(heap_->isolate(), this,
+                                     &concurrent_sweepers_));
   }
 }
 
@@ -169,6 +212,8 @@ void Sweeper::EnsureCompleted() {
   ForAllSweepingSpaces([this](AllocationSpace space) {
     CHECK(sweeping_list_[GetSweepSpaceIndex(space)].empty());
   });
+
+  concurrent_sweepers_.clear();
   sweeping_in_progress_ = false;
 }
 
@@ -414,20 +459,6 @@ size_t Sweeper::ConcurrentSweepingPageCount() {
          sweeping_list_[GetSweepSpaceIndex(MAP_SPACE)].size();
 }
 
-bool Sweeper::ConcurrentSweepSpace(AllocationSpace identity,
-                                   JobDelegate* delegate) {
-  while (!delegate->ShouldYield()) {
-    Page* page = GetSweepingPageSafe(identity);
-    if (page == nullptr) return true;
-    // Typed slot sets are only recorded on code pages. Code pages
-    // are not swept concurrently to the application to ensure W^X.
-    DCHECK_NULL((page->typed_slot_set<OLD_TO_NEW>()));
-    DCHECK_NULL((page->typed_slot_set<OLD_TO_OLD>()));
-    ParallelSweepPage(page, identity, SweepingMode::kLazyOrConcurrent);
-  }
-  return false;
-}
-
 int Sweeper::ParallelSweepSpace(AllocationSpace identity,
                                 SweepingMode sweeping_mode,
                                 int required_freed_bytes, int max_pages) {

src/heap/sweeper.h

@@ -75,6 +75,7 @@ class Sweeper {
   enum class SweepingMode { kEagerDuringGC, kLazyOrConcurrent };
 
   Sweeper(Heap* heap, NonAtomicMarkingState* marking_state);
+  ~Sweeper();
 
   bool sweeping_in_progress() const { return sweeping_in_progress_; }
 
@@ -107,11 +108,12 @@ class Sweeper {
   Page* GetSweptPageSafe(PagedSpaceBase* space);
 
  private:
+  class ConcurrentSweeper;
   class SweeperJob;
 
   static const int kNumberOfSweepingSpaces =
       LAST_GROWABLE_PAGED_SPACE - FIRST_GROWABLE_PAGED_SPACE + 1;
-  static const int kMaxSweeperTasks = 3;
+  static constexpr int kMaxSweeperTasks = 3;
 
   template <typename Callback>
   void ForAllSweepingSpaces(Callback callback) const {
@@ -157,10 +159,6 @@ class Sweeper {
 
   size_t ConcurrentSweepingPageCount();
 
-  // Concurrently sweeps many page from the given space. Returns true if there
-  // are no more pages to sweep in the given space.
-  bool ConcurrentSweepSpace(AllocationSpace identity, JobDelegate* delegate);
-
   Page* GetSweepingPageSafe(AllocationSpace space);
   bool TryRemoveSweepingPageSafe(AllocationSpace space, Page* page);
 
@@ -176,6 +174,8 @@ class Sweeper {
     return space - FIRST_GROWABLE_PAGED_SPACE;
   }
 
+  int NumberOfConcurrentSweepers() const;
+
   Heap* const heap_;
   NonAtomicMarkingState* marking_state_;
   std::unique_ptr<JobHandle> job_handle_;
@@ -183,6 +183,7 @@ class Sweeper {
   base::ConditionVariable cv_page_swept_;
   SweptList swept_list_[kNumberOfSweepingSpaces];
   SweepingList sweeping_list_[kNumberOfSweepingSpaces];
+  std::vector<ConcurrentSweeper> concurrent_sweepers_;
   // Main thread can finalize sweeping, while background threads allocation slow
   // path checks this flag to see whether it could support concurrent sweeping.
   std::atomic<bool> sweeping_in_progress_;