[heap] Allow concurrently transferring colors

Adds general support for concurrent modifications to live byte counters.

BUG=chromium:651354

Review-Url: https://codereview.chromium.org/2836583002
Cr-Commit-Position: refs/heads/master@{#44767}

src/heap/incremental-marking.h

@@ -32,6 +32,32 @@ class V8_EXPORT_PRIVATE IncrementalMarking {
 
   enum GCRequestType { NONE, COMPLETE_MARKING, FINALIZATION };
 
+  static void MarkGrey(Heap* heap, HeapObject* object);
+
+  static void MarkBlack(HeapObject* object, int size);
+
+  // Transfers mark bits without requiring proper object headers.
+  static void TransferMark(Heap* heap, HeapObject* from, HeapObject* to);
+
+  // Transfers color including live byte count, requiring properly set up
+  // objects.
+  template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
+  V8_INLINE static void TransferColor(HeapObject* from, HeapObject* to) {
+    if (ObjectMarking::IsBlack<access_mode>(to, MarkingState::Internal(to))) {
+      DCHECK(to->GetHeap()->incremental_marking()->black_allocation());
+      return;
+    }
+
+    DCHECK(ObjectMarking::IsWhite<access_mode>(to, MarkingState::Internal(to)));
+    if (ObjectMarking::IsGrey<access_mode>(from,
+                                           MarkingState::Internal(from))) {
+      ObjectMarking::WhiteToGrey<access_mode>(to, MarkingState::Internal(to));
+    } else if (ObjectMarking::IsBlack<access_mode>(
+                   from, MarkingState::Internal(from))) {
+      ObjectMarking::WhiteToBlack<access_mode>(to, MarkingState::Internal(to));
+    }
+  }
+
   explicit IncrementalMarking(Heap* heap);
 
   static void Initialize();
@@ -179,26 +205,6 @@ class V8_EXPORT_PRIVATE IncrementalMarking {
 
   bool IsIdleMarkingDelayCounterLimitReached();
 
-  static void MarkGrey(Heap* heap, HeapObject* object);
-
-  static void MarkBlack(HeapObject* object, int size);
-
-  static void TransferMark(Heap* heap, HeapObject* from, HeapObject* to);
-
-  V8_INLINE static void TransferColor(HeapObject* from, HeapObject* to) {
-    if (ObjectMarking::IsBlack(to, MarkingState::Internal(to))) {
-      DCHECK(to->GetHeap()->incremental_marking()->black_allocation());
-      return;
-    }
-
-    DCHECK(ObjectMarking::IsWhite(to, MarkingState::Internal(to)));
-    if (ObjectMarking::IsGrey(from, MarkingState::Internal(from))) {
-      ObjectMarking::WhiteToGrey(to, MarkingState::Internal(to));
-    } else if (ObjectMarking::IsBlack(from, MarkingState::Internal(from))) {
-      ObjectMarking::WhiteToBlack(to, MarkingState::Internal(to));
-    }
-  }
-
   void IterateBlackObject(HeapObject* object);
 
   Heap* heap() const { return heap_; }

src/heap/mark-compact.h

@@ -80,7 +80,7 @@ class ObjectMarking : public AllStatic {
         (access_mode == MarkBit::ATOMIC || IsBlack<access_mode>(obj, state)));
     MarkBit markbit = MarkBitFrom(obj, state);
     if (!Marking::BlackToGrey<access_mode>(markbit)) return false;
-    state.IncrementLiveBytes(-obj->Size());
+    state.IncrementLiveBytes<access_mode>(-obj->Size());
     return true;
   }
 
@@ -107,7 +107,7 @@ class ObjectMarking : public AllStatic {
     DCHECK((access_mode == MarkBit::ATOMIC || IsGrey<access_mode>(obj, state)));
     MarkBit markbit = MarkBitFrom(obj, state);
     if (!Marking::GreyToBlack<access_mode>(markbit)) return false;
-    state.IncrementLiveBytes(obj->Size());
+    state.IncrementLiveBytes<access_mode>(obj->Size());
     return true;
   }
 

src/heap/spaces.h

@@ -670,9 +670,9 @@ class MarkingState {
   MarkingState(Bitmap* bitmap, intptr_t* live_bytes)
       : bitmap_(bitmap), live_bytes_(live_bytes) {}
 
-  void IncrementLiveBytes(intptr_t by) const {
-    *live_bytes_ += static_cast<int>(by);
-  }
+  template <MarkBit::AccessMode mode = MarkBit::NON_ATOMIC>
+  inline void IncrementLiveBytes(intptr_t by) const;
+
   void SetLiveBytes(intptr_t value) const {
     *live_bytes_ = static_cast<int>(value);
   }
@@ -690,6 +690,18 @@ class MarkingState {
   intptr_t* live_bytes_;
 };
 
+template <>
+inline void MarkingState::IncrementLiveBytes<MarkBit::NON_ATOMIC>(
+    intptr_t by) const {
+  *live_bytes_ += by;
+}
+
+template <>
+inline void MarkingState::IncrementLiveBytes<MarkBit::ATOMIC>(
+    intptr_t by) const {
+  reinterpret_cast<base::AtomicNumber<intptr_t>*>(live_bytes_)->Increment(by);
+}
+
 // -----------------------------------------------------------------------------
 // A page is a memory chunk of a size 1MB. Large object pages may be larger.
 //