[heap] Implement concurrent marking of fixed arrays.

On left trimming the main thread ensures that the array
is black before changing its map and length.

The concurrent marker snapshots the length of the array
and then tries to change its color to black. If the change
is successful, then the array is iterated using the saved
length.


BUG=chromium:694255

Change-Id: Ib06c1c9c3240ace56c1897e5a73462aa447d41f5
Reviewed-on: https://chromium-review.googlesource.com/519323Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Commit-Queue: Ulan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#45656}

src/heap/concurrent-marking.cc

@@ -98,8 +98,12 @@ class ConcurrentMarkingVisitor final
   // ===========================================================================
 
   int VisitFixedArray(Map* map, FixedArray* object) override {
-    // TODO(ulan): implement iteration with prefetched length.
-    return BaseClass::VisitFixedArray(map, object);
+    int length = object->length();
+    int size = FixedArray::SizeFor(length);
+    if (!ShouldVisit(object)) return 0;
+    VisitMapPointer(object, object->map_slot());
+    FixedArray::BodyDescriptor::IterateBody(object, size, this);
+    return size;
   }
 
   // ===========================================================================

src/heap/heap.cc

@@ -41,6 +41,7 @@
 #include "src/heap/scavenger-inl.h"
 #include "src/heap/store-buffer.h"
 #include "src/interpreter/interpreter.h"
+#include "src/objects/object-macros.h"
 #include "src/objects/shared-function-info.h"
 #include "src/regexp/jsregexp.h"
 #include "src/runtime-profiler.h"
@@ -3209,14 +3210,9 @@ FixedArrayBase* Heap::LeftTrimFixedArray(FixedArrayBase* object,
   Address old_start = object->address();
   Address new_start = old_start + bytes_to_trim;
 
-  // Transfer the mark bits to their new location if the object is not within
-  // a black area.
-  if (!incremental_marking()->black_allocation() ||
-      !Marking::IsBlack(ObjectMarking::MarkBitFrom(
-          HeapObject::FromAddress(new_start),
-          MarkingState::Internal(HeapObject::FromAddress(new_start))))) {
-    incremental_marking()->TransferMark(this, object,
-                                        HeapObject::FromAddress(new_start));
+  if (incremental_marking()->IsMarking()) {
+    incremental_marking()->NotifyLeftTrimming(
+        object, HeapObject::FromAddress(new_start));
   }
 
   // Technically in new space this write might be omitted (except for
@@ -3227,10 +3223,9 @@ FixedArrayBase* Heap::LeftTrimFixedArray(FixedArrayBase* object,
   // Initialize header of the trimmed array. Since left trimming is only
   // performed on pages which are not concurrently swept creating a filler
   // object does not require synchronization.
-  Object** former_start = HeapObject::RawField(object, 0);
-  int new_start_index = elements_to_trim * (element_size / kPointerSize);
-  former_start[new_start_index] = map;
-  former_start[new_start_index + 1] = Smi::FromInt(len - elements_to_trim);
+  RELAXED_WRITE_FIELD(object, bytes_to_trim, map);
+  RELAXED_WRITE_FIELD(object, bytes_to_trim + kPointerSize,
+                      Smi::FromInt(len - elements_to_trim));
 
   FixedArrayBase* new_object =
       FixedArrayBase::cast(HeapObject::FromAddress(new_start));

src/heap/incremental-marking.cc

@@ -151,24 +151,38 @@ void IncrementalMarking::MarkBlackAndPush(HeapObject* obj) {
   }
 }
 
-void IncrementalMarking::TransferMark(Heap* heap, HeapObject* from,
-                                      HeapObject* to) {
+void IncrementalMarking::NotifyLeftTrimming(HeapObject* from, HeapObject* to) {
+  DCHECK(IsMarking());
   DCHECK(MemoryChunk::FromAddress(from->address())->SweepingDone());
-  // This is only used when resizing an object.
-  DCHECK(MemoryChunk::FromAddress(from->address()) ==
-         MemoryChunk::FromAddress(to->address()));
+  DCHECK_EQ(MemoryChunk::FromAddress(from->address()),
+            MemoryChunk::FromAddress(to->address()));
+  DCHECK_NE(from, to);
 
-  if (!IsMarking()) return;
+  MarkBit old_mark_bit = ObjectMarking::MarkBitFrom(from, marking_state(from));
+  MarkBit new_mark_bit = ObjectMarking::MarkBitFrom(to, marking_state(to));
 
-  // If the mark doesn't move, we don't check the color of the object.
-  // It doesn't matter whether the object is black, since it hasn't changed
-  // size, so the adjustment to the live data count will be zero anyway.
-  if (from == to) return;
+  if (black_allocation() && Marking::IsBlack(new_mark_bit)) {
+    // Nothing to do if the object is in black area.
+    return;
+  }
 
-  MarkBit new_mark_bit = ObjectMarking::MarkBitFrom(to, marking_state(to));
-  MarkBit old_mark_bit = ObjectMarking::MarkBitFrom(from, marking_state(from));
+  bool marked_black_due_to_left_trimming = false;
+  if (FLAG_concurrent_marking) {
+    // We need to mark the array black before overwriting its map and length
+    // so that the concurrent marker does not observe inconsistent state.
+    Marking::WhiteToGrey<kAtomicity>(old_mark_bit);
+    if (Marking::GreyToBlack<kAtomicity>(old_mark_bit)) {
+      // The concurrent marker will not mark the array. We need to push the
+      // new array start in marking deque to ensure that it will be marked.
+      marked_black_due_to_left_trimming = true;
+    }
+    DCHECK(Marking::IsBlack<kAtomicity>(old_mark_bit));
+  }
 
-  if (Marking::IsBlack<kAtomicity>(old_mark_bit)) {
+  if (Marking::IsBlack<kAtomicity>(old_mark_bit) &&
+      !marked_black_due_to_left_trimming) {
+    // The array was black before left trimming or was marked black by the
+    // concurrent marker. Simply transfer the color.
     if (from->address() + kPointerSize == to->address()) {
       // The old and the new markbits overlap. The |to| object has the
       // grey color. To make it black, we need to set the second bit.
@@ -179,12 +193,13 @@ void IncrementalMarking::TransferMark(Heap* heap, HeapObject* from,
       DCHECK(success);
       USE(success);
     }
-  } else if (Marking::IsGrey<kAtomicity>(old_mark_bit)) {
+  } else if (Marking::IsGrey<kAtomicity>(old_mark_bit) ||
+             marked_black_due_to_left_trimming) {
+    // The array was already grey or was marked black by this function.
+    // Mark the new array grey and push it to marking deque.
     if (from->address() + kPointerSize == to->address()) {
-      // The old and the new markbits overlap. The |to| object has the
-      // white color. To make it grey, we need to set the first bit.
-      // Note that Marking::WhiteToGrey does not work here because
-      // old_mark_bit.Next() can be set by the concurrent marker at any time.
+      // The old and the new markbits overlap. The |to| object is either white
+      // or grey.  Set the first bit to make sure that it is grey.
       new_mark_bit.Set();
       DCHECK(!new_mark_bit.Next().Get());
     } else {

src/heap/incremental-marking.h

@@ -65,8 +65,7 @@ class V8_EXPORT_PRIVATE IncrementalMarking {
     return MarkingState::Internal(chunk);
   }
 
-  // Transfers mark bits without requiring proper object headers.
-  void TransferMark(Heap* heap, HeapObject* from, HeapObject* to);
+  void NotifyLeftTrimming(HeapObject* from, HeapObject* to);
 
   // Transfers color including live byte count, requiring properly set up
   // objects.