[heap] Introduce WorkStealingMarkingDeque

Currently only relies on private stacks of segments, i.e., doesn't steal
anything, yet.

Bug: chromium:651354
Change-Id: Icedad3e3169b61afe988a1ece10f73f3a973bdb2
Reviewed-on: https://chromium-review.googlesource.com/508351
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: Hannes Payer <hpayer@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#45422}

BUILD.gn

@@ -1618,6 +1618,7 @@ v8_source_set("v8_base") {
     "src/heap/spaces.h",
     "src/heap/store-buffer.cc",
     "src/heap/store-buffer.h",
+    "src/heap/workstealing-marking-deque.h",
     "src/ic/access-compiler-data.h",
     "src/ic/access-compiler.cc",
     "src/ic/access-compiler.h",

src/heap/mark-compact.cc

@@ -25,6 +25,7 @@
 #include "src/heap/objects-visiting.h"
 #include "src/heap/page-parallel-job.h"
 #include "src/heap/spaces-inl.h"
+#include "src/heap/workstealing-marking-deque.h"
 #include "src/ic/ic.h"
 #include "src/ic/stub-cache.h"
 #include "src/tracing/tracing-category-observer.h"
@@ -388,9 +389,7 @@ void MarkCompactCollector::SetUp() {
   }
 }
 
-void MinorMarkCompactCollector::SetUp() {
-  for (int i = 0; i < kNumMarkers; i++) marking_deque(i)->SetUp();
-}
+void MinorMarkCompactCollector::SetUp() {}
 
 void MarkCompactCollector::TearDown() {
   AbortCompaction();
@@ -398,9 +397,7 @@ void MarkCompactCollector::TearDown() {
   delete code_flusher_;
 }
 
-void MinorMarkCompactCollector::TearDown() {
-  for (int i = 0; i < kNumMarkers; i++) marking_deque(i)->TearDown();
-}
+void MinorMarkCompactCollector::TearDown() {}
 
 void MarkCompactCollector::AddEvacuationCandidate(Page* p) {
   DCHECK(!p->NeverEvacuate());
@@ -2426,8 +2423,10 @@ class YoungGenerationMarkingVisitor final
  public:
   using BaseClass = HeapVisitor<int, YoungGenerationMarkingVisitor>;
 
-  YoungGenerationMarkingVisitor(Heap* heap, MarkingDeque* marking_deque)
-      : heap_(heap), marking_deque_(marking_deque) {}
+  YoungGenerationMarkingVisitor(Heap* heap,
+                                WorkStealingMarkingDeque* global_marking_deque,
+                                int task_id)
+      : heap_(heap), marking_deque_(global_marking_deque, task_id) {}
 
   void VisitPointers(HeapObject* host, Object** start, Object** end) final {
     const int kMinRangeForMarkingRecursion = 64;
@@ -2491,7 +2490,7 @@ class YoungGenerationMarkingVisitor final
     if (ObjectMarking::WhiteToBlack<MarkBit::ATOMIC>(object,
                                                      marking_state(object))) {
       // Marking deque overflow is unsupported for the young generation.
-      CHECK(marking_deque_->Push(object));
+      CHECK(marking_deque_.Push(object));
     }
   }
 
@@ -2512,7 +2511,7 @@ class YoungGenerationMarkingVisitor final
   }
 
   Heap* heap_;
-  MarkingDeque* marking_deque_;
+  LocalWorkStealingMarkingDeque marking_deque_;
 };
 
 class MinorMarkCompactCollector::RootMarkingVisitor : public RootVisitor {
@@ -2567,13 +2566,12 @@ class YoungGenerationMarkingTask : public ItemParallelJob::Task {
  public:
   YoungGenerationMarkingTask(Isolate* isolate,
                              MinorMarkCompactCollector* collector,
-                             MarkingDeque* marking_deque,
-                             YoungGenerationMarkingVisitor* visitor
-
-                             )
+                             WorkStealingMarkingDeque* marking_deque,
+                             YoungGenerationMarkingVisitor* visitor,
+                             int task_id)
       : ItemParallelJob::Task(isolate),
         collector_(collector),
-        marking_deque_(marking_deque),
+        marking_deque_(marking_deque, task_id),
         visitor_(visitor) {}
 
   void RunInParallel() override {
@@ -2584,9 +2582,10 @@ class YoungGenerationMarkingTask : public ItemParallelJob::Task {
       while ((item = GetItem<MarkingItem>()) != nullptr) {
         item->Process(this);
         item->MarkFinished();
-        collector_->EmptySpecificMarkingDeque(marking_deque_, visitor_);
+        EmptyLocalMarkingDeque();
       }
-      DCHECK(marking_deque_->IsEmpty());
+      EmptyMarkingDeque();
+      DCHECK(marking_deque_.IsEmpty());
     }
     if (FLAG_trace_minor_mc_parallel_marking) {
       PrintIsolate(collector_->isolate(), "marking[%p]: time=%f\n",
@@ -2604,8 +2603,24 @@ class YoungGenerationMarkingTask : public ItemParallelJob::Task {
   }
 
  private:
+  void EmptyLocalMarkingDeque() {
+    HeapObject* object = nullptr;
+    while (marking_deque_.Pop(&object)) {
+      visitor_->Visit(object);
+    }
+  }
+
+  void EmptyMarkingDeque() {
+    HeapObject* object = nullptr;
+    while (marking_deque_.WaitForMoreObjects()) {
+      while (marking_deque_.Pop(&object)) {
+        visitor_->Visit(object);
+      }
+    }
+  }
+
   MinorMarkCompactCollector* collector_;
-  MarkingDeque* marking_deque_;
+  LocalWorkStealingMarkingDeque marking_deque_;
   YoungGenerationMarkingVisitor* visitor_;
 };
 
@@ -2722,20 +2737,19 @@ class MinorMarkCompactCollector::RootMarkingVisitorSeedOnly
 
 MinorMarkCompactCollector::MinorMarkCompactCollector(Heap* heap)
     : MarkCompactCollectorBase(heap), page_parallel_job_semaphore_(0) {
+  marking_deque_ = new WorkStealingMarkingDeque();
   for (int i = 0; i < kNumMarkers; i++) {
-    marking_deque_[i] = new MarkingDeque(heap);
     marking_visitor_[i] =
-        new YoungGenerationMarkingVisitor(heap, marking_deque_[i]);
+        new YoungGenerationMarkingVisitor(heap, marking_deque_, i);
   }
 }
 
 MinorMarkCompactCollector::~MinorMarkCompactCollector() {
   for (int i = 0; i < kNumMarkers; i++) {
     DCHECK_NOT_NULL(marking_visitor_[i]);
-    DCHECK_NOT_NULL(marking_deque_[i]);
     delete marking_visitor_[i];
-    delete marking_deque_[i];
   }
+  delete marking_deque_;
 }
 
 SlotCallbackResult MinorMarkCompactCollector::CheckAndMarkObject(
@@ -2785,7 +2799,7 @@ void MinorMarkCompactCollector::MarkRootSetInParallel() {
     const int num_tasks = NumberOfMarkingTasks();
     for (int i = 0; i < num_tasks; i++) {
       job.AddTask(new YoungGenerationMarkingTask(
-          isolate(), this, marking_deque(i), marking_visitor(i)));
+          isolate(), this, marking_deque(), marking_visitor(i), i));
     }
     job.Run();
   }
@@ -2798,8 +2812,6 @@ void MinorMarkCompactCollector::MarkLiveObjects() {
 
   RootMarkingVisitor root_visitor(this);
 
-  for (int i = 0; i < kNumMarkers; i++) marking_deque(i)->StartUsing();
-
   {
     TRACE_GC(heap()->tracer(),
              GCTracer::Scope::MINOR_MC_MARK_IDENTIFY_GLOBAL_HANDLES);
@@ -2824,20 +2836,17 @@ void MinorMarkCompactCollector::MarkLiveObjects() {
         &root_visitor);
     ProcessMarkingDeque();
   }
-
-  for (int i = 0; i < kNumMarkers; i++) marking_deque(i)->StopUsing();
 }
 
 void MinorMarkCompactCollector::ProcessMarkingDeque() {
   EmptyMarkingDeque();
-  DCHECK(!marking_deque(kMainMarker)->overflowed());
-  DCHECK(marking_deque(kMainMarker)->IsEmpty());
 }
 
-void MinorMarkCompactCollector::EmptySpecificMarkingDeque(
-    MarkingDeque* marking_deque, YoungGenerationMarkingVisitor* visitor) {
-  while (!marking_deque->IsEmpty()) {
-    HeapObject* object = marking_deque->Pop();
+void MinorMarkCompactCollector::EmptyMarkingDeque() {
+  LocalWorkStealingMarkingDeque local_marking_deque(marking_deque(),
+                                                    kMainMarker);
+  HeapObject* object = nullptr;
+  while (local_marking_deque.Pop(&object)) {
     DCHECK(!object->IsFiller());
     DCHECK(object->IsHeapObject());
     DCHECK(heap()->Contains(object));
@@ -2845,15 +2854,9 @@ void MinorMarkCompactCollector::EmptySpecificMarkingDeque(
         object, marking_state(object))));
     DCHECK((ObjectMarking::IsBlack<MarkBit::NON_ATOMIC>(
         object, marking_state(object))));
-    visitor->Visit(object);
+    marking_visitor(kMainMarker)->Visit(object);
   }
-  DCHECK(!marking_deque->overflowed());
-  DCHECK(marking_deque->IsEmpty());
-}
-
-void MinorMarkCompactCollector::EmptyMarkingDeque() {
-  EmptySpecificMarkingDeque(marking_deque(kMainMarker),
-                            marking_visitor(kMainMarker));
+  DCHECK(local_marking_deque.IsEmpty());
 }
 
 void MinorMarkCompactCollector::CollectGarbage() {

src/heap/mark-compact.h

@@ -23,6 +23,7 @@ namespace internal {
 class CodeFlusher;
 class EvacuationJobTraits;
 class HeapObjectVisitor;
+class LocalWorkStealingMarkingDeque;
 class MarkCompactCollector;
 class MinorMarkCompactCollector;
 class MarkingVisitor;
@@ -31,6 +32,7 @@ template <typename JobTraits>
 class PageParallelJob;
 class RecordMigratedSlotVisitor;
 class ThreadLocalTop;
+class WorkStealingMarkingDeque;
 class YoungGenerationMarkingVisitor;
 
 #ifdef V8_CONCURRENT_MARKING
@@ -366,10 +368,7 @@ class MinorMarkCompactCollector final : public MarkCompactCollectorBase {
   static const int kNumMarkers = 4;
   static const int kMainMarker = 0;
 
-  inline MarkingDeque* marking_deque(int index) {
-    DCHECK_LT(index, kNumMarkers);
-    return marking_deque_[index];
-  }
+  inline WorkStealingMarkingDeque* marking_deque() { return marking_deque_; }
 
   inline YoungGenerationMarkingVisitor* marking_visitor(int index) {
     DCHECK_LT(index, kNumMarkers);
@@ -381,8 +380,6 @@ class MinorMarkCompactCollector final : public MarkCompactCollectorBase {
   void MarkRootSetInParallel();
   void ProcessMarkingDeque() override;
   void EmptyMarkingDeque() override;
-  void EmptySpecificMarkingDeque(MarkingDeque* marking_deque,
-                                 YoungGenerationMarkingVisitor* visitor);
   void ClearNonLiveReferences() override;
 
   void EvacuatePrologue() override;
@@ -393,7 +390,7 @@ class MinorMarkCompactCollector final : public MarkCompactCollectorBase {
 
   int NumberOfMarkingTasks();
 
-  MarkingDeque* marking_deque_[kNumMarkers];
+  WorkStealingMarkingDeque* marking_deque_;
   YoungGenerationMarkingVisitor* marking_visitor_[kNumMarkers];
   base::Semaphore page_parallel_job_semaphore_;
   List<Page*> new_space_evacuation_pages_;

src/heap/workstealing-marking-deque.h

+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_HEAP_WORKSTEALING_MARKING_DEQUE_
+#define V8_HEAP_WORKSTEALING_MARKING_DEQUE_
+
+#include <cstddef>
+
+#include "src/base/logging.h"
+#include "src/base/macros.h"
+
+namespace v8 {
+namespace internal {
+
+class HeapObject;
+
+class StackSegment {
+ public:
+  static const int kNumEntries = 64;
+
+  StackSegment(StackSegment* next, StackSegment* prev)
+      : next_(next), prev_(prev), index_(0) {}
+
+  bool Push(HeapObject* object) {
+    if (IsFull()) return false;
+
+    objects_[index_++] = object;
+    return true;
+  }
+
+  bool Pop(HeapObject** object) {
+    if (IsEmpty()) return false;
+
+    *object = objects_[--index_];
+    return true;
+  }
+
+  size_t Size() { return index_; }
+  bool IsEmpty() { return index_ == 0; }
+  bool IsFull() { return index_ == kNumEntries; }
+  void Clear() { index_ = 0; }
+
+  StackSegment* next() { return next_; }
+  StackSegment* prev() { return prev_; }
+  void set_next(StackSegment* next) { next_ = next; }
+  void set_prev(StackSegment* prev) { prev_ = prev; }
+
+  void Unlink() {
+    if (next() != nullptr) next()->set_prev(prev());
+    if (prev() != nullptr) prev()->set_next(next());
+  }
+
+ private:
+  StackSegment* next_;
+  StackSegment* prev_;
+  size_t index_;
+  HeapObject* objects_[kNumEntries];
+};
+
+class SegmentedStack {
+ public:
+  SegmentedStack()
+      : front_(new StackSegment(nullptr, nullptr)), back_(front_) {}
+
+  ~SegmentedStack() {
+    CHECK(IsEmpty());
+    delete front_;
+  }
+
+  bool Push(HeapObject* object) {
+    if (!front_->Push(object)) {
+      NewFront();
+      bool success = front_->Push(object);
+      USE(success);
+      DCHECK(success);
+    }
+    return true;
+  }
+
+  bool Pop(HeapObject** object) {
+    if (!front_->Pop(object)) {
+      if (IsEmpty()) return false;
+      DeleteFront();
+      bool success = front_->Pop(object);
+      USE(success);
+      DCHECK(success);
+    }
+    return object;
+  }
+
+  bool IsEmpty() { return front_ == back_ && front_->IsEmpty(); }
+
+ private:
+  void NewFront() {
+    StackSegment* s = new StackSegment(front_, nullptr);
+    front_->set_prev(s);
+    front_ = s;
+  }
+
+  void DeleteFront() { delete Unlink(front_); }
+
+  StackSegment* Unlink(StackSegment* segment) {
+    CHECK_NE(front_, back_);
+    if (segment == front_) front_ = front_->next();
+    if (segment == back_) back_ = back_->prev();
+    segment->Unlink();
+    return segment;
+  }
+
+  StackSegment* front_;
+  StackSegment* back_;
+};
+
+// TODO(mlippautz): Implement actual work stealing.
+class WorkStealingMarkingDeque {
+ public:
+  static const int kMaxNumTasks = 4;
+
+  bool Push(int task_id, HeapObject* object) {
+    DCHECK_LT(task_id, kMaxNumTasks);
+    return private_stacks_[task_id].Push(object);
+  }
+
+  bool Pop(int task_id, HeapObject** object) {
+    DCHECK_LT(task_id, kMaxNumTasks);
+    return private_stacks_[task_id].Pop(object);
+  }
+
+  bool IsLocalEmpty(int task_id) { return private_stacks_[task_id].IsEmpty(); }
+
+ private:
+  SegmentedStack private_stacks_[kMaxNumTasks];
+};
+
+class LocalWorkStealingMarkingDeque {
+ public:
+  LocalWorkStealingMarkingDeque(WorkStealingMarkingDeque* deque, int task_id)
+      : deque_(deque), task_id_(task_id) {}
+
+  // Pushes an object onto the marking deque.
+  bool Push(HeapObject* object) { return deque_->Push(task_id_, object); }
+
+  // Pops an object onto the marking deque.
+  bool Pop(HeapObject** object) { return deque_->Pop(task_id_, object); }
+
+  // Returns true if the local portion of the marking deque is empty.
+  bool IsEmpty() { return deque_->IsLocalEmpty(task_id_); }
+
+  // Blocks if there are no more objects available. Returns execution with
+  // |true| once new objects are available and |false| otherwise.
+  bool WaitForMoreObjects() {
+    // Return false once the local portion of the marking deque is drained.
+    // TODO(mlippautz): Implement a barrier that can be used to synchronize
+    // work stealing and emptiness.
+    return !IsEmpty();
+  }
+
+ private:
+  WorkStealingMarkingDeque* deque_;
+  int task_id_;
+};
+
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_HEAP_WORKSTEALING_MARKING_DEQUE_

src/v8.gyp

@@ -993,6 +993,7 @@
         'heap/spaces.h',
         'heap/store-buffer.cc',
         'heap/store-buffer.h',
+        'heap/workstealing-marking-deque.h',
         'intl.cc',
         'intl.h',
         'icu_util.cc',

test/unittests/BUILD.gn

@@ -114,6 +114,7 @@ v8_executable("unittests") {
     "heap/slot-set-unittest.cc",
     "heap/spaces-unittest.cc",
     "heap/unmapper-unittest.cc",
+    "heap/workstealing-marking-deque-unittest.cc",
     "interpreter/bytecode-array-builder-unittest.cc",
     "interpreter/bytecode-array-iterator-unittest.cc",
     "interpreter/bytecode-array-random-iterator-unittest.cc",

test/unittests/heap/workstealing-marking-deque-unittest.cc

+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/heap/workstealing-marking-deque.h"
+
+#include "test/unittests/test-utils.h"
+
+namespace v8 {
+namespace internal {
+
+class HeapObject {};
+
+TEST(WorkStealingMarkingDeque, LocalEmpty) {
+  WorkStealingMarkingDeque marking_deque;
+  LocalWorkStealingMarkingDeque local_marking_deque(&marking_deque, 0);
+  EXPECT_TRUE(local_marking_deque.IsEmpty());
+}
+
+TEST(WorkStealingMarkingDeque, LocalPushPop) {
+  WorkStealingMarkingDeque marking_deque;
+  LocalWorkStealingMarkingDeque local_marking_deque(&marking_deque, 0);
+  HeapObject* object1 = new HeapObject();
+  HeapObject* object2 = nullptr;
+  EXPECT_TRUE(local_marking_deque.Push(object1));
+  EXPECT_FALSE(local_marking_deque.IsEmpty());
+  EXPECT_TRUE(local_marking_deque.Pop(&object2));
+  EXPECT_EQ(object1, object2);
+  delete object1;
+}
+
+}  // namespace internal
+}  // namespace v8

test/unittests/unittests.gyp

@@ -111,6 +111,7 @@
       'heap/slot-set-unittest.cc',
       'heap/spaces-unittest.cc',
       'heap/unmapper-unittest.cc',
+      'heap/workstealing-marking-deque-unittest.cc',
       'interpreter/bytecodes-unittest.cc',
       'interpreter/bytecode-array-builder-unittest.cc',
       'interpreter/bytecode-array-iterator-unittest.cc',