[heap] Reland "Use shared markbits in the concurrent marker."

The concurrent marker now colors objects grey and black using
the atomic markbit operations.

The heap visitor is changed in two ways:
1) It iterates the map pointer of each object.
2) It guards object visitation with a predicate, which is overridden
in the concurrent marker with the result of GreyToBlack transition.

BUG=chromium:694255
CQ_INCLUDE_TRYBOTS=master.tryserver.chromium.linux:linux_chromium_rel_ng

Review-Url: https://codereview.chromium.org/2866473003
Cr-Commit-Position: refs/heads/master@{#45144}

src/flag-definitions.h

@@ -666,7 +666,7 @@ DEFINE_BOOL(age_code, true,
 DEFINE_BOOL(incremental_marking, true, "use incremental marking")
 DEFINE_BOOL(incremental_marking_wrappers, true,
             "use incremental marking for marking wrappers")
-DEFINE_BOOL(concurrent_marking, false, "use concurrent marking")
+DEFINE_BOOL(concurrent_marking, V8_CONCURRENT_MARKING, "use concurrent marking")
 DEFINE_BOOL(trace_concurrent_marking, false, "trace concurrent marking")
 DEFINE_INT(min_progress_during_incremental_marking_finalization, 32,
            "keep finalizing incremental marking as long as we discover at "

src/heap/concurrent-marking.cc

@@ -30,6 +30,11 @@ class ConcurrentMarkingVisitor final
   explicit ConcurrentMarkingVisitor(ConcurrentMarkingDeque* deque)
       : deque_(deque) {}
 
+  bool ShouldVisit(HeapObject* object) override {
+    return ObjectMarking::GreyToBlack<MarkBit::AccessMode::ATOMIC>(
+        object, marking_state(object));
+  }
+
   void VisitPointers(HeapObject* host, Object** start, Object** end) override {
     for (Object** p = start; p < end; p++) {
       if (!(*p)->IsHeapObject()) continue;
@@ -68,7 +73,7 @@ class ConcurrentMarkingVisitor final
   // ===========================================================================
 
   int VisitCode(Map* map, Code* object) override {
-    // TODO(ulan): push the object to the bail-out deque.
+    deque_->Push(object, MarkingThread::kConcurrent, TargetDeque::kBailout);
     return 0;
   }
 
@@ -77,58 +82,65 @@ class ConcurrentMarkingVisitor final
   // ===========================================================================
 
   int VisitBytecodeArray(Map* map, BytecodeArray* object) override {
-    // TODO(ulan): implement iteration of strong fields and push the object to
-    // the bailout deque.
+    // TODO(ulan): implement iteration of strong fields.
+    deque_->Push(object, MarkingThread::kConcurrent, TargetDeque::kBailout);
     return 0;
   }
 
   int VisitJSFunction(Map* map, JSFunction* object) override {
-    // TODO(ulan): implement iteration of strong fields and push the object to
-    // the bailout deque.
+    // TODO(ulan): implement iteration of strong fields.
+    deque_->Push(object, MarkingThread::kConcurrent, TargetDeque::kBailout);
     return 0;
   }
 
   int VisitMap(Map* map, Map* object) override {
-    // TODO(ulan): implement iteration of strong fields and push the object to
-    // the bailout deque.
+    // TODO(ulan): implement iteration of strong fields.
+    deque_->Push(object, MarkingThread::kConcurrent, TargetDeque::kBailout);
     return 0;
   }
 
   int VisitNativeContext(Map* map, Context* object) override {
-    // TODO(ulan): implement iteration of strong fields and push the object to
-    // the bailout deque.
+    // TODO(ulan): implement iteration of strong fields.
+    deque_->Push(object, MarkingThread::kConcurrent, TargetDeque::kBailout);
     return 0;
   }
 
   int VisitSharedFunctionInfo(Map* map, SharedFunctionInfo* object) override {
-    // TODO(ulan): implement iteration of strong fields and push the object to
-    // the bailout deque.
+    // TODO(ulan): implement iteration of strong fields.
+    deque_->Push(object, MarkingThread::kConcurrent, TargetDeque::kBailout);
     return 0;
   }
 
   int VisitTransitionArray(Map* map, TransitionArray* object) override {
-    // TODO(ulan): implement iteration of strong fields and push the object to
-    // the bailout deque.
+    // TODO(ulan): implement iteration of strong fields.
+    deque_->Push(object, MarkingThread::kConcurrent, TargetDeque::kBailout);
     return 0;
   }
 
   int VisitWeakCell(Map* map, WeakCell* object) override {
-    // TODO(ulan): implement iteration of strong fields and push the object to
-    // the bailout deque.
+    // TODO(ulan): implement iteration of strong fields.
+    deque_->Push(object, MarkingThread::kConcurrent, TargetDeque::kBailout);
     return 0;
   }
 
   int VisitJSWeakCollection(Map* map, JSWeakCollection* object) override {
-    // TODO(ulan): implement iteration of strong fields and push the object to
-    // the bailout deque.
+    // TODO(ulan): implement iteration of strong fields.
+    deque_->Push(object, MarkingThread::kConcurrent, TargetDeque::kBailout);
     return 0;
   }
 
-  void MarkObject(HeapObject* obj) {
-    deque_->Push(obj, MarkingThread::kConcurrent, TargetDeque::kShared);
+  void MarkObject(HeapObject* object) {
+    if (ObjectMarking::WhiteToGrey<MarkBit::AccessMode::ATOMIC>(
+            object, marking_state(object))) {
+      deque_->Push(object, MarkingThread::kConcurrent, TargetDeque::kShared);
+    }
   }
 
  private:
+  MarkingState marking_state(HeapObject* object) const {
+    return MarkingState::Internal(object);
+  }
+
   ConcurrentMarkingDeque* deque_;
 };
 
@@ -175,7 +187,7 @@ void ConcurrentMarking::Run() {
     TimedScope scope(&time_ms);
     HeapObject* object;
     while ((object = deque_->Pop(MarkingThread::kConcurrent)) != nullptr) {
-      bytes_marked += visitor_->IterateBody(object);
+      bytes_marked += visitor_->Visit(object);
     }
   }
   if (FLAG_trace_concurrent_marking) {

src/heap/objects-visiting-inl.h

@@ -637,8 +637,7 @@ void StaticMarkingVisitor<StaticVisitor>::VisitJSFunctionWeakCode(
 }
 
 template <typename ResultType, typename ConcreteVisitor>
-ResultType HeapVisitor<ResultType, ConcreteVisitor>::IterateBody(
-    HeapObject* object) {
+ResultType HeapVisitor<ResultType, ConcreteVisitor>::Visit(HeapObject* object) {
   Map* map = object->map();
   ConcreteVisitor* visitor = static_cast<ConcreteVisitor*>(this);
   switch (static_cast<VisitorId>(map->visitor_id())) {
@@ -669,14 +668,28 @@ ResultType HeapVisitor<ResultType, ConcreteVisitor>::IterateBody(
   return ResultType();
 }
 
-#define VISIT(type)                                                         \
-  template <typename ResultType, typename ConcreteVisitor>                  \
-  ResultType HeapVisitor<ResultType, ConcreteVisitor>::Visit##type(         \
-      Map* map, type* object) {                                             \
-    int size = type::BodyDescriptor::SizeOf(map, object);                   \
-    type::BodyDescriptor::IterateBody(object, size,                         \
-                                      static_cast<ConcreteVisitor*>(this)); \
-    return static_cast<ResultType>(size);                                   \
+template <typename ResultType, typename ConcreteVisitor>
+void HeapVisitor<ResultType, ConcreteVisitor>::VisitMapPointer(
+    HeapObject* host, HeapObject** map) {
+  static_cast<ConcreteVisitor*>(this)->VisitPointer(
+      host, reinterpret_cast<Object**>(map));
+}
+
+template <typename ResultType, typename ConcreteVisitor>
+bool HeapVisitor<ResultType, ConcreteVisitor>::ShouldVisit(HeapObject* object) {
+  return true;
+}
+
+#define VISIT(type)                                                 \
+  template <typename ResultType, typename ConcreteVisitor>          \
+  ResultType HeapVisitor<ResultType, ConcreteVisitor>::Visit##type( \
+      Map* map, type* object) {                                     \
+    ConcreteVisitor* visitor = static_cast<ConcreteVisitor*>(this); \
+    if (!visitor->ShouldVisit(object)) return ResultType();         \
+    int size = type::BodyDescriptor::SizeOf(map, object);           \
+    visitor->VisitMapPointer(object, object->map_slot());           \
+    type::BodyDescriptor::IterateBody(object, size, visitor);       \
+    return static_cast<ResultType>(size);                           \
   }
 TYPED_VISITOR_ID_LIST(VISIT)
 #undef VISIT
@@ -684,7 +697,10 @@ TYPED_VISITOR_ID_LIST(VISIT)
 template <typename ResultType, typename ConcreteVisitor>
 ResultType HeapVisitor<ResultType, ConcreteVisitor>::VisitShortcutCandidate(
     Map* map, ConsString* object) {
+  ConcreteVisitor* visitor = static_cast<ConcreteVisitor*>(this);
+  if (!visitor->ShouldVisit(object)) return ResultType();
   int size = ConsString::BodyDescriptor::SizeOf(map, object);
+  visitor->VisitMapPointer(object, object->map_slot());
   ConsString::BodyDescriptor::IterateBody(object, size,
                                           static_cast<ConcreteVisitor*>(this));
   return static_cast<ResultType>(size);
@@ -693,7 +709,10 @@ ResultType HeapVisitor<ResultType, ConcreteVisitor>::VisitShortcutCandidate(
 template <typename ResultType, typename ConcreteVisitor>
 ResultType HeapVisitor<ResultType, ConcreteVisitor>::VisitNativeContext(
     Map* map, Context* object) {
+  ConcreteVisitor* visitor = static_cast<ConcreteVisitor*>(this);
+  if (!visitor->ShouldVisit(object)) return ResultType();
   int size = Context::BodyDescriptor::SizeOf(map, object);
+  visitor->VisitMapPointer(object, object->map_slot());
   Context::BodyDescriptor::IterateBody(object, size,
                                        static_cast<ConcreteVisitor*>(this));
   return static_cast<ResultType>(size);
@@ -702,14 +721,20 @@ ResultType HeapVisitor<ResultType, ConcreteVisitor>::VisitNativeContext(
 template <typename ResultType, typename ConcreteVisitor>
 ResultType HeapVisitor<ResultType, ConcreteVisitor>::VisitDataObject(
     Map* map, HeapObject* object) {
+  ConcreteVisitor* visitor = static_cast<ConcreteVisitor*>(this);
+  if (!visitor->ShouldVisit(object)) return ResultType();
   int size = map->instance_size();
+  visitor->VisitMapPointer(object, object->map_slot());
   return static_cast<ResultType>(size);
 }
 
 template <typename ResultType, typename ConcreteVisitor>
 ResultType HeapVisitor<ResultType, ConcreteVisitor>::VisitJSObjectFast(
     Map* map, JSObject* object) {
+  ConcreteVisitor* visitor = static_cast<ConcreteVisitor*>(this);
+  if (!visitor->ShouldVisit(object)) return ResultType();
   int size = JSObject::FastBodyDescriptor::SizeOf(map, object);
+  visitor->VisitMapPointer(object, object->map_slot());
   JSObject::FastBodyDescriptor::IterateBody(
       object, size, static_cast<ConcreteVisitor*>(this));
   return static_cast<ResultType>(size);
@@ -717,7 +742,10 @@ ResultType HeapVisitor<ResultType, ConcreteVisitor>::VisitJSObjectFast(
 template <typename ResultType, typename ConcreteVisitor>
 ResultType HeapVisitor<ResultType, ConcreteVisitor>::VisitJSApiObject(
     Map* map, JSObject* object) {
+  ConcreteVisitor* visitor = static_cast<ConcreteVisitor*>(this);
+  if (!visitor->ShouldVisit(object)) return ResultType();
   int size = JSObject::BodyDescriptor::SizeOf(map, object);
+  visitor->VisitMapPointer(object, object->map_slot());
   JSObject::BodyDescriptor::IterateBody(object, size,
                                         static_cast<ConcreteVisitor*>(this));
   return static_cast<ResultType>(size);
@@ -725,7 +753,10 @@ ResultType HeapVisitor<ResultType, ConcreteVisitor>::VisitJSApiObject(
 template <typename ResultType, typename ConcreteVisitor>
 ResultType HeapVisitor<ResultType, ConcreteVisitor>::VisitStruct(
     Map* map, HeapObject* object) {
+  ConcreteVisitor* visitor = static_cast<ConcreteVisitor*>(this);
+  if (!visitor->ShouldVisit(object)) return ResultType();
   int size = map->instance_size();
+  visitor->VisitMapPointer(object, object->map_slot());
   StructBodyDescriptor::IterateBody(object, size,
                                     static_cast<ConcreteVisitor*>(this));
   return static_cast<ResultType>(size);
@@ -733,6 +764,9 @@ ResultType HeapVisitor<ResultType, ConcreteVisitor>::VisitStruct(
 template <typename ResultType, typename ConcreteVisitor>
 ResultType HeapVisitor<ResultType, ConcreteVisitor>::VisitFreeSpace(
     Map* map, FreeSpace* object) {
+  ConcreteVisitor* visitor = static_cast<ConcreteVisitor*>(this);
+  if (!visitor->ShouldVisit(object)) return ResultType();
+  visitor->VisitMapPointer(object, object->map_slot());
   return static_cast<ResultType>(FreeSpace::cast(object)->size());
 }
 

src/heap/objects-visiting.h

@@ -395,9 +395,16 @@ VisitorDispatchTable<typename StaticMarkingVisitor<StaticVisitor>::Callback>
 template <typename ResultType, typename ConcreteVisitor>
 class HeapVisitor : public ObjectVisitor {
  public:
-  ResultType IterateBody(HeapObject* object);
+  ResultType Visit(HeapObject* object);
 
  protected:
+  // A guard predicate for visiting the object.
+  // If it returns false then the default implementations of the Visit*
+  // functions bailout from iterating the object pointers.
+  virtual bool ShouldVisit(HeapObject* object);
+  // A callback for visiting the map pointer in the object header.
+  virtual void VisitMapPointer(HeapObject* host, HeapObject** map);
+
 #define VISIT(type) virtual ResultType Visit##type(Map* map, type* object);
   TYPED_VISITOR_ID_LIST(VISIT)
 #undef VISIT

src/objects-inl.h

@@ -1525,6 +1525,9 @@ void HeapObject::set_map_no_write_barrier(Map* value) {
   set_map_word(MapWord::FromMap(value));
 }
 
+HeapObject** HeapObject::map_slot() {
+  return reinterpret_cast<HeapObject**>(FIELD_ADDR(this, kMapOffset));
+}
 
 MapWord HeapObject::map_word() const {
   return MapWord(

src/objects.h

@@ -1669,6 +1669,9 @@ class HeapObject: public Object {
   // information.
   inline Map* map() const;
   inline void set_map(Map* value);
+
+  inline HeapObject** map_slot();
+
   // The no-write-barrier version.  This is OK if the object is white and in
   // new space, or if the value is an immortal immutable object, like the maps
   // of primitive (non-JS) objects like strings, heap numbers etc.