[heap] Create untangled entry for elements write barrier

Implement independent write barrier for contents of objects.

Change-Id: I739745f524245fa61b2b1cd01263cc8fe3a3d373
Reviewed-on: https://chromium-review.googlesource.com/1152909
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: Camillo Bruni <cbruni@chromium.org>
Cr-Commit-Position: refs/heads/master@{#54979}

src/elements.cc

@@ -8,6 +8,7 @@
 #include "src/conversions.h"
 #include "src/frames.h"
 #include "src/heap/factory.h"
+#include "src/heap/heap-write-barrier-inl.h"
 #include "src/isolate-inl.h"
 #include "src/messages.h"
 #include "src/objects-inl.h"

src/heap/heap-inl.h

@@ -435,15 +435,6 @@ void Heap::RecordWriteIntoCode(Code* host, RelocInfo* rinfo, Object* value) {
   }
 }
 
-void Heap::RecordFixedArrayElements(FixedArray* array, int offset, int length) {
-  if (InNewSpace(array)) return;
-  for (int i = 0; i < length; i++) {
-    if (!InNewSpace(array->get(offset + i))) continue;
-    store_buffer()->InsertEntry(
-        reinterpret_cast<Address>(array->RawFieldOfElementAt(offset + i)));
-  }
-}
-
 Address* Heap::store_buffer_top_address() {
   return store_buffer()->top_address();
 }

src/heap/heap-write-barrier-inl.h

@@ -89,6 +89,15 @@ inline void GenerationalBarrier(HeapObject* object, MaybeObject** slot,
       object, reinterpret_cast<Address>(slot), value_heap_object);
 }
 
+inline void GenerationalBarrierForElements(Heap* heap, FixedArray* array,
+                                           int offset, int length) {
+  heap_internals::MemoryChunk* array_chunk =
+      heap_internals::MemoryChunk::FromHeapObject(array);
+  if (array_chunk->InNewSpace()) return;
+
+  Heap::GenerationalBarrierForElementsSlow(heap, array, offset, length);
+}
+
 inline void MarkingBarrier(HeapObject* object, Object** slot, Object* value) {
   DCHECK_IMPLIES(slot != nullptr, !HasWeakHeapObjectTag(*slot));
   DCHECK(!HasWeakHeapObjectTag(value));
@@ -105,6 +114,14 @@ inline void MarkingBarrier(HeapObject* object, MaybeObject** slot,
       object, reinterpret_cast<Address>(slot), value_heap_object);
 }
 
+inline void MarkingBarrierForElements(Heap* heap, HeapObject* object) {
+  heap_internals::MemoryChunk* object_chunk =
+      heap_internals::MemoryChunk::FromHeapObject(object);
+  if (!object_chunk->IsMarking()) return;
+
+  Heap::MarkingBarrierForElementsSlow(heap, object);
+}
+
 }  // namespace internal
 }  // namespace v8
 

src/heap/heap-write-barrier.h

@@ -8,6 +8,8 @@
 namespace v8 {
 namespace internal {
 
+class FixedArray;
+class Heap;
 class HeapObject;
 class MaybeObject;
 class Object;
@@ -15,14 +17,24 @@ class Object;
 // Note: In general it is preferred to use the macros defined in
 // object-macros.h.
 
+// Write barrier for FixedArray elements.
+#define FIXED_ARRAY_ELEMENTS_WRITE_BARRIER(heap, array, start, length) \
+  do {                                                                 \
+    GenerationalBarrierForElements(heap, array, start, length);        \
+    MarkingBarrierForElements(heap, array);                            \
+  } while (false)
+
 // Generational write barrier.
 void GenerationalBarrier(HeapObject* object, Object** slot, Object* value);
 void GenerationalBarrier(HeapObject* object, MaybeObject** slot,
                          MaybeObject* value);
+void GenerationalBarrierForElements(Heap* heap, FixedArray* array, int offset,
+                                    int length);
 
 // Marking write barrier.
 void MarkingBarrier(HeapObject* object, Object** slot, Object* value);
 void MarkingBarrier(HeapObject* object, MaybeObject** slot, MaybeObject* value);
+void MarkingBarrierForElements(Heap* heap, HeapObject* object);
 
 }  // namespace internal
 }  // namespace v8

src/heap/heap.cc

@@ -5833,6 +5833,15 @@ void Heap::GenerationalBarrierSlow(HeapObject* object, Address slot,
   heap->store_buffer()->InsertEntry(slot);
 }
 
+void Heap::GenerationalBarrierForElementsSlow(Heap* heap, FixedArray* array,
+                                              int offset, int length) {
+  for (int i = 0; i < length; i++) {
+    if (!InNewSpace(array->get(offset + i))) continue;
+    heap->store_buffer()->InsertEntry(
+        reinterpret_cast<Address>(array->RawFieldOfElementAt(offset + i)));
+  }
+}
+
 void Heap::MarkingBarrierSlow(HeapObject* object, Address slot,
                               HeapObject* value) {
   Heap* heap = Heap::FromWritableHeapObject(object);
@@ -5840,6 +5849,13 @@ void Heap::MarkingBarrierSlow(HeapObject* object, Address slot,
       object, reinterpret_cast<HeapObjectReference**>(slot), value);
 }
 
+void Heap::MarkingBarrierForElementsSlow(Heap* heap, HeapObject* object) {
+  if (FLAG_concurrent_marking ||
+      heap->incremental_marking()->marking_state()->IsBlack(object)) {
+    heap->incremental_marking()->RevisitObject(object);
+  }
+}
+
 bool Heap::PageFlagsAreConsistent(HeapObject* object) {
   Heap* heap = Heap::FromWritableHeapObject(object);
   MemoryChunk* chunk = MemoryChunk::FromHeapObject(object);

src/heap/heap.h

@@ -166,12 +166,6 @@ using v8::MemoryPressureLevel;
   V(empty_string)                           \
   PRIVATE_SYMBOL_LIST(V)
 
-#define FIXED_ARRAY_ELEMENTS_WRITE_BARRIER(heap, array, start, length) \
-  do {                                                                 \
-    heap->RecordFixedArrayElements(array, start, length);              \
-    heap->incremental_marking()->RecordWrites(array);                  \
-  } while (false)
-
 class AllocationObserver;
 class ArrayBufferCollector;
 class ArrayBufferTracker;
@@ -495,9 +489,13 @@ class Heap {
   V8_EXPORT_PRIVATE static void GenerationalBarrierSlow(HeapObject* object,
                                                         Address slot,
                                                         HeapObject* value);
+  V8_EXPORT_PRIVATE static void GenerationalBarrierForElementsSlow(
+      Heap* heap, FixedArray* array, int offset, int length);
   V8_EXPORT_PRIVATE static void MarkingBarrierSlow(HeapObject* object,
                                                    Address slot,
                                                    HeapObject* value);
+  V8_EXPORT_PRIVATE static void MarkingBarrierForElementsSlow(
+      Heap* heap, HeapObject* object);
   V8_EXPORT_PRIVATE static bool PageFlagsAreConsistent(HeapObject* object);
 
   // Notifies the heap that is ok to start marking or other activities that
@@ -991,8 +989,6 @@ class Heap {
   inline void RecordWriteIntoCode(Code* host, RelocInfo* rinfo, Object* target);
   void RecordWriteIntoCodeSlow(Code* host, RelocInfo* rinfo, Object* target);
   void RecordWritesIntoCode(Code* code);
-  inline void RecordFixedArrayElements(FixedArray* array, int offset,
-                                       int length);
 
   // Used for query incremental marking status in generated code.
   Address* IsMarkingFlagAddress() {

src/heap/incremental-marking-inl.h

@@ -36,14 +36,6 @@ void IncrementalMarking::RecordMaybeWeakWrite(HeapObject* obj,
   }
 }
 
-void IncrementalMarking::RecordWrites(HeapObject* obj) {
-  if (IsMarking()) {
-    if (FLAG_concurrent_marking || marking_state()->IsBlack(obj)) {
-      RevisitObject(obj);
-    }
-  }
-}
-
 void IncrementalMarking::RecordWriteIntoCode(Code* host, RelocInfo* rinfo,
                                              Object* value) {
   if (IsMarking() && value->IsHeapObject()) {

src/heap/incremental-marking.h

@@ -211,7 +211,7 @@ class V8_EXPORT_PRIVATE IncrementalMarking {
                                       MaybeObject* value);
   V8_INLINE void RecordWriteIntoCode(Code* host, RelocInfo* rinfo,
                                      Object* value);
-  V8_INLINE void RecordWrites(HeapObject* obj);
+  void RevisitObject(HeapObject* obj);
 
   void RecordWriteSlow(HeapObject* obj, HeapObjectReference** slot,
                        Object* value);
@@ -301,8 +301,6 @@ class V8_EXPORT_PRIVATE IncrementalMarking {
   // Visits the object and returns its size.
   V8_INLINE int VisitObject(Map* map, HeapObject* obj);
 
-  void RevisitObject(HeapObject* obj);
-
   void IncrementIdleMarkingDelayCounter();
 
   void AdvanceIncrementalMarkingOnAllocation();

src/objects.cc

@@ -4619,7 +4619,7 @@ void Map::ReplaceDescriptors(Isolate* isolate, DescriptorArray* new_descriptors,
   // Replace descriptors by new_descriptors in all maps that share it. The old
   // descriptors will not be trimmed in the mark-compactor, we need to mark
   // all its elements.
-  isolate->heap()->incremental_marking()->RecordWrites(to_replace);
+  MarkingBarrierForElements(isolate->heap(), to_replace);
   Map* current = this;
   while (current->instance_descriptors() == to_replace) {
     Object* next = current->GetBackPointer();
@@ -5400,7 +5400,7 @@ void Map::EnsureDescriptorSlack(Isolate* isolate, Handle<Map> map, int slack) {
   // Replace descriptors by new_descriptors in all maps that share it. The old
   // descriptors will not be trimmed in the mark-compactor, we need to mark
   // all its elements.
-  isolate->heap()->incremental_marking()->RecordWrites(*descriptors);
+  MarkingBarrierForElements(isolate->heap(), *descriptors);
 
   Map* current = *map;
   while (current->instance_descriptors() == *descriptors) {