[heap] Start concurrent marking simultaneously with incremental marking.

This patch hooks up concurrent marking (behind the flag) with the rest
of the GC:
1. Incremental marking spawns concurrent marking task seeded with the
root set.
2. Mark-compact waits for concurrent marking tasks to finish.
3. Scavenger does fast promotion if concurrent marking is pending.

BUG=chromium:694255

Review-Url: https://codereview.chromium.org/2735803005
Cr-Commit-Position: refs/heads/master@{#44526}

src/flag-definitions.h

@@ -654,6 +654,7 @@ DEFINE_BOOL(incremental_marking, true, "use incremental marking")
 DEFINE_BOOL(incremental_marking_wrappers, true,
             "use incremental marking for marking wrappers")
 DEFINE_BOOL(concurrent_marking, false, "use concurrent marking")
+DEFINE_BOOL(trace_concurrent_marking, false, "trace concurrent marking")
 DEFINE_INT(min_progress_during_incremental_marking_finalization, 32,
            "keep finalizing incremental marking as long as we discover at "
            "least this many unmarked objects")
@@ -1283,6 +1284,7 @@ DEFINE_VALUE_IMPLICATION(single_threaded, wasm_num_compilation_tasks, 0)
 
 DEFINE_BOOL(single_threaded, false, "disable the use of background tasks")
 DEFINE_NEG_IMPLICATION(single_threaded, concurrent_recompilation)
+DEFINE_NEG_IMPLICATION(single_threaded, concurrent_marking)
 DEFINE_NEG_IMPLICATION(single_threaded, concurrent_sweeping)
 DEFINE_NEG_IMPLICATION(single_threaded, parallel_compaction)
 DEFINE_NEG_IMPLICATION(single_threaded, parallel_pointer_update)

src/globals.h

@@ -118,6 +118,7 @@ namespace internal {
 #define V8_DOUBLE_FIELDS_UNBOXING 0
 #endif
 
+#define V8_CONCURRENT_MARKING 0
 
 typedef uint8_t byte;
 typedef byte* Address;

src/heap/concurrent-marking.cc

@@ -12,6 +12,8 @@
 #include "src/heap/marking.h"
 #include "src/isolate.h"
 #include "src/locked-queue-inl.h"
+#include "src/utils-inl.h"
+#include "src/utils.h"
 #include "src/v8.h"
 
 namespace v8 {
@@ -50,7 +52,7 @@ class ConcurrentMarkingMarkbits {
 
 class ConcurrentMarkingVisitor : public ObjectVisitor {
  public:
-  ConcurrentMarkingVisitor() {}
+  ConcurrentMarkingVisitor() : bytes_marked_(0) {}
 
   void VisitPointers(Object** start, Object** end) override {
     for (Object** p = start; p < end; p++) {
@@ -61,6 +63,7 @@ class ConcurrentMarkingVisitor : public ObjectVisitor {
 
   void MarkObject(HeapObject* obj) {
     if (markbits_.Mark(obj)) {
+      bytes_marked_ += obj->Size();
       marking_stack_.push(obj);
     }
   }
@@ -73,7 +76,10 @@ class ConcurrentMarkingVisitor : public ObjectVisitor {
     }
   }
 
+  size_t bytes_marked() { return bytes_marked_; }
+
  private:
+  size_t bytes_marked_;
   std::stack<HeapObject*> marking_stack_;
   ConcurrentMarkingMarkbits markbits_;
 };
@@ -92,11 +98,19 @@ class ConcurrentMarking::Task : public CancelableTask {
  private:
   // v8::internal::CancelableTask overrides.
   void RunInternal() override {
-    USE(heap_);
-    for (HeapObject* obj : *root_set_) {
-      marking_visitor_.MarkObject(obj);
+    double time_ms = heap_->MonotonicallyIncreasingTimeInMs();
+    {
+      TimedScope scope(&time_ms);
+      for (HeapObject* obj : *root_set_) {
+        marking_visitor_.MarkObject(obj);
+      }
+      marking_visitor_.MarkTransitively();
+    }
+    if (FLAG_trace_concurrent_marking) {
+      heap_->isolate()->PrintWithTimestamp(
+          "concurrently marked %dKB in %.2fms\n",
+          static_cast<int>(marking_visitor_.bytes_marked() / KB), time_ms);
     }
-    marking_visitor_.MarkTransitively();
     on_finish_->Signal();
   }
 
@@ -108,7 +122,12 @@ class ConcurrentMarking::Task : public CancelableTask {
 };
 
 ConcurrentMarking::ConcurrentMarking(Heap* heap)
-    : heap_(heap), pending_task_(0) {}
+    : heap_(heap), pending_task_semaphore_(0), is_task_pending_(false) {
+  // Concurrent marking does not work with double unboxing.
+  STATIC_ASSERT(!(V8_CONCURRENT_MARKING && V8_DOUBLE_FIELDS_UNBOXING));
+  // The runtime flag should be set only if the compile time flag was set.
+  CHECK(!FLAG_concurrent_marking || V8_CONCURRENT_MARKING);
+}
 
 ConcurrentMarking::~ConcurrentMarking() {}
 
@@ -116,17 +135,26 @@ void ConcurrentMarking::AddRoot(HeapObject* object) {
   root_set_.push_back(object);
 }
 
-void ConcurrentMarking::StartMarkingTask() {
+void ConcurrentMarking::StartTask() {
   if (!FLAG_concurrent_marking) return;
+  is_task_pending_ = true;
 
   V8::GetCurrentPlatform()->CallOnBackgroundThread(
-      new Task(heap_, &root_set_, &pending_task_),
+      new Task(heap_, &root_set_, &pending_task_semaphore_),
       v8::Platform::kShortRunningTask);
 }
 
 void ConcurrentMarking::WaitForTaskToComplete() {
   if (!FLAG_concurrent_marking) return;
-  pending_task_.Wait();
+  pending_task_semaphore_.Wait();
+  is_task_pending_ = false;
+  root_set_.clear();
+}
+
+void ConcurrentMarking::EnsureTaskCompleted() {
+  if (IsTaskPending()) {
+    WaitForTaskToComplete();
+  }
 }
 
 }  // namespace internal

src/heap/concurrent-marking.h

@@ -25,13 +25,16 @@ class ConcurrentMarking {
 
   void AddRoot(HeapObject* object);
 
-  void StartMarkingTask();
+  void StartTask();
   void WaitForTaskToComplete();
+  bool IsTaskPending() { return is_task_pending_; }
+  void EnsureTaskCompleted();
 
  private:
   class Task;
   Heap* heap_;
-  base::Semaphore pending_task_;
+  base::Semaphore pending_task_semaphore_;
+  bool is_task_pending_;
   std::vector<HeapObject*> root_set_;
 };
 

src/heap/heap.cc

@@ -1353,8 +1353,9 @@ bool Heap::PerformGarbageCollection(
         MinorMarkCompact();
         break;
       case SCAVENGER:
-        if (fast_promotion_mode_ &&
-            CanExpandOldGeneration(new_space()->Size())) {
+        if ((fast_promotion_mode_ &&
+             CanExpandOldGeneration(new_space()->Size())) ||
+            concurrent_marking_->IsTaskPending()) {
           tracer()->NotifyYoungGenerationHandling(
               YoungGenerationHandling::kFastPromotionDuringScavenge);
           EvacuateYoungGeneration();
@@ -1626,8 +1627,10 @@ class ScavengeWeakObjectRetainer : public WeakObjectRetainer {
 
 void Heap::EvacuateYoungGeneration() {
   TRACE_GC(tracer(), GCTracer::Scope::SCAVENGER_EVACUATE);
-  DCHECK(fast_promotion_mode_);
-  DCHECK(CanExpandOldGeneration(new_space()->Size()));
+  if (!FLAG_concurrent_marking) {
+    DCHECK(fast_promotion_mode_);
+    DCHECK(CanExpandOldGeneration(new_space()->Size()));
+  }
 
   mark_compact_collector()->sweeper().EnsureNewSpaceCompleted();
 

src/heap/heap.h

@@ -1239,6 +1239,12 @@ class Heap {
 
   IncrementalMarking* incremental_marking() { return incremental_marking_; }
 
+  // ===========================================================================
+  // Concurrent marking API. ===================================================
+  // ===========================================================================
+
+  ConcurrentMarking* concurrent_marking() { return concurrent_marking_; }
+
   // The runtime uses this function to notify potentially unsafe object layout
   // changes that require special synchronization with the concurrent marker.
   // A layout change is unsafe if

src/heap/incremental-marking.cc

@@ -7,6 +7,7 @@
 #include "src/code-stubs.h"
 #include "src/compilation-cache.h"
 #include "src/conversions.h"
+#include "src/heap/concurrent-marking.h"
 #include "src/heap/gc-idle-time-handler.h"
 #include "src/heap/gc-tracer.h"
 #include "src/heap/heap-inl.h"
@@ -545,6 +546,15 @@ void IncrementalMarking::StartMarking() {
   IncrementalMarkingRootMarkingVisitor visitor(this);
   heap_->IterateStrongRoots(&visitor, VISIT_ONLY_STRONG);
 
+  if (FLAG_concurrent_marking) {
+    ConcurrentMarking* concurrent_marking = heap_->concurrent_marking();
+    heap_->mark_compact_collector()->marking_deque()->Iterate(
+        [concurrent_marking](HeapObject* obj) {
+          concurrent_marking->AddRoot(obj);
+        });
+    concurrent_marking->StartTask();
+  }
+
   // Ready to start incremental marking.
   if (FLAG_trace_incremental_marking) {
     heap()->isolate()->PrintWithTimestamp("[IncrementalMarking] Running\n");

src/heap/mark-compact.cc

@@ -15,6 +15,7 @@
 #include "src/gdb-jit.h"
 #include "src/global-handles.h"
 #include "src/heap/array-buffer-tracker.h"
+#include "src/heap/concurrent-marking.h"
 #include "src/heap/gc-tracer.h"
 #include "src/heap/incremental-marking.h"
 #include "src/heap/mark-compact-inl.h"
@@ -847,6 +848,8 @@ void MarkCompactCollector::Prepare() {
   // them here.
   heap()->memory_allocator()->unmapper()->WaitUntilCompleted();
 
+  heap()->concurrent_marking()->EnsureTaskCompleted();
+
   // Clear marking bits if incremental marking is aborted.
   if (was_marked_incrementally_ && heap_->ShouldAbortIncrementalMarking()) {
     heap()->incremental_marking()->Stop();

src/heap/mark-compact.h

@@ -186,6 +186,15 @@ class MarkingDeque {
     }
   }
 
+  template <typename Callback>
+  void Iterate(Callback callback) {
+    int i = bottom_;
+    while (i != top_) {
+      callback(array_[i]);
+      i = (i + 1) & mask_;
+    }
+  }
+
   HeapObject** array() { return array_; }
   int bottom() { return bottom_; }
   int top() { return top_; }

test/cctest/heap/test-concurrent-marking.cc

@@ -15,12 +15,12 @@
 namespace v8 {
 namespace internal {
 TEST(ConcurrentMarking) {
-  i::FLAG_concurrent_marking = true;
+  if (!i::FLAG_concurrent_marking) return;
   CcTest::InitializeVM();
   Heap* heap = CcTest::heap();
   ConcurrentMarking* concurrent_marking = new ConcurrentMarking(heap);
   concurrent_marking->AddRoot(heap->undefined_value());
-  concurrent_marking->StartMarkingTask();
+  concurrent_marking->StartTask();
   concurrent_marking->WaitForTaskToComplete();
   delete concurrent_marking;
 }