Reland "[heap] Reschedule concurrent marking tasks earlier"

This is a reland of b1c3ca2a

Original change's description:
> [heap] Reschedule concurrent marking tasks earlier
> 
> Currently we reschedule concurrent marking tasks if all tasks finish.
> This is too conservative and we can improve performance by rescheduling
> finished tasks without waiting for all other tasks.
> 
> As a drive-by this also changes task_count_ to total_task_count_.
> 
> Change-Id: If0b3bd45ce6d52f6bcd0065dd8d3efe9ea84184a
> Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1789142
> Commit-Queue: Ulan Degenbaev <ulan@chromium.org>
> Reviewed-by: Omer Katz <omerkatz@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#63593}

Change-Id: Id18bbb3cab85cd38bb7d2f21611825252ed4a1dc
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1789288Reviewed-by: Omer Katz <omerkatz@chromium.org>
Commit-Queue: Ulan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#63610}

src/heap/concurrent-marking.cc

@@ -864,8 +864,7 @@ void ConcurrentMarking::ScheduleTasks() {
   DCHECK(FLAG_parallel_marking || FLAG_concurrent_marking);
   DCHECK(!heap_->IsTearingDown());
   base::MutexGuard guard(&pending_lock_);
-  DCHECK_EQ(0, pending_task_count_);
-  if (task_count_ == 0) {
+  if (total_task_count_ == 0) {
     static const int num_cores =
         V8::GetCurrentPlatform()->NumberOfWorkerThreads() + 1;
 #if defined(V8_OS_MACOSX)
@@ -873,15 +872,18 @@ void ConcurrentMarking::ScheduleTasks() {
     // marking on competing hyper-threads (regresses Octane/Splay). As such,
     // only use num_cores/2, leaving one of those for the main thread.
     // TODO(ulan): Use all cores on Mac 10.12+.
-    task_count_ = Max(1, Min(kMaxTasks, (num_cores / 2) - 1));
+    total_task_count_ = Max(1, Min(kMaxTasks, (num_cores / 2) - 1));
 #else   // defined(OS_MACOSX)
     // On other platforms use all logical cores, leaving one for the main
     // thread.
-    task_count_ = Max(1, Min(kMaxTasks, num_cores - 1));
+    total_task_count_ = Max(1, Min(kMaxTasks, num_cores - 1));
 #endif  // defined(OS_MACOSX)
+    DCHECK_LE(total_task_count_, kMaxTasks);
+    // One task is for the main thread.
+    STATIC_ASSERT(kMaxTasks + 1 <= MarkingWorklist::kMaxNumTasks);
   }
   // Task id 0 is for the main thread.
-  for (int i = 1; i <= task_count_; i++) {
+  for (int i = 1; i <= total_task_count_; i++) {
     if (!is_pending_[i]) {
       if (FLAG_trace_concurrent_marking) {
         heap_->isolate()->PrintWithTimestamp(
@@ -899,7 +901,7 @@ void ConcurrentMarking::ScheduleTasks() {
       V8::GetCurrentPlatform()->CallOnWorkerThread(std::move(task));
     }
   }
-  DCHECK_EQ(task_count_, pending_task_count_);
+  DCHECK_EQ(total_task_count_, pending_task_count_);
 }
 
 void ConcurrentMarking::RescheduleTasksIfNeeded() {
@@ -907,11 +909,15 @@ void ConcurrentMarking::RescheduleTasksIfNeeded() {
   if (heap_->IsTearingDown()) return;
   {
     base::MutexGuard guard(&pending_lock_);
-    if (pending_task_count_ > 0) return;
+    // The total task count is initialized in ScheduleTasks from
+    // NumberOfWorkerThreads of the platform.
+    if (total_task_count_ > 0 && pending_task_count_ == total_task_count_) {
+      return;
+    }
   }
   if (!shared_->IsGlobalPoolEmpty() ||
-      !weak_objects_->current_ephemerons.IsEmpty() ||
-      !weak_objects_->discovered_ephemerons.IsEmpty()) {
+      !weak_objects_->current_ephemerons.IsGlobalPoolEmpty() ||
+      !weak_objects_->discovered_ephemerons.IsGlobalPoolEmpty()) {
     ScheduleTasks();
   }
 }
@@ -925,7 +931,7 @@ bool ConcurrentMarking::Stop(StopRequest stop_request) {
   if (stop_request != StopRequest::COMPLETE_TASKS_FOR_TESTING) {
     CancelableTaskManager* task_manager =
         heap_->isolate()->cancelable_task_manager();
-    for (int i = 1; i <= task_count_; i++) {
+    for (int i = 1; i <= total_task_count_; i++) {
       if (is_pending_[i]) {
         if (task_manager->TryAbort(cancelable_id_[i]) ==
             TryAbortResult::kTaskAborted) {
@@ -940,7 +946,7 @@ bool ConcurrentMarking::Stop(StopRequest stop_request) {
   while (pending_task_count_ > 0) {
     pending_condition_.Wait(&pending_lock_);
   }
-  for (int i = 1; i <= task_count_; i++) {
+  for (int i = 1; i <= total_task_count_; i++) {
     DCHECK(!is_pending_[i]);
   }
   return true;
@@ -956,7 +962,7 @@ bool ConcurrentMarking::IsStopped() {
 void ConcurrentMarking::FlushMemoryChunkData(
     MajorNonAtomicMarkingState* marking_state) {
   DCHECK_EQ(pending_task_count_, 0);
-  for (int i = 1; i <= task_count_; i++) {
+  for (int i = 1; i <= total_task_count_; i++) {
     MemoryChunkDataMap& memory_chunk_data = task_state_[i].memory_chunk_data;
     for (auto& pair : memory_chunk_data) {
       // ClearLiveness sets the live bytes to zero.
@@ -978,7 +984,7 @@ void ConcurrentMarking::FlushMemoryChunkData(
 }
 
 void ConcurrentMarking::ClearMemoryChunkData(MemoryChunk* chunk) {
-  for (int i = 1; i <= task_count_; i++) {
+  for (int i = 1; i <= total_task_count_; i++) {
     auto it = task_state_[i].memory_chunk_data.find(chunk);
     if (it != task_state_[i].memory_chunk_data.end()) {
       it->second.live_bytes = 0;
@@ -989,7 +995,7 @@ void ConcurrentMarking::ClearMemoryChunkData(MemoryChunk* chunk) {
 
 size_t ConcurrentMarking::TotalMarkedBytes() {
   size_t result = 0;
-  for (int i = 1; i <= task_count_; i++) {
+  for (int i = 1; i <= total_task_count_; i++) {
     result +=
         base::AsAtomicWord::Relaxed_Load<size_t>(&task_state_[i].marked_bytes);
   }

src/heap/concurrent-marking.h

@@ -86,8 +86,6 @@ class V8_EXPORT_PRIVATE ConcurrentMarking {
   // scavenge and is going to be re-used.
   void ClearMemoryChunkData(MemoryChunk* chunk);
 
-  int TaskCount() { return task_count_; }
-
   // Checks if all threads are stopped.
   bool IsStopped();
 
@@ -124,7 +122,7 @@ class V8_EXPORT_PRIVATE ConcurrentMarking {
   int pending_task_count_ = 0;
   bool is_pending_[kMaxTasks + 1] = {};
   CancelableTaskManager::Id cancelable_id_[kMaxTasks + 1] = {};
-  int task_count_ = 0;
+  int total_task_count_ = 0;
 };
 
 }  // namespace internal