[heap, iwyu] Refactor write-barrier for code.

This moves write-barrier for writes into code to heap-write-barrier.h
and adds four new functions:
- WriteBarrierForCode(host, rinfo, object) - combined generational
  and marking write barrier.
- WriteBarrierForCode(host) - combined write barrier that rescans
  all pointers in the host (former RecordWritesIntoCode).
- GenerationalWriteBarrierForCode.
- MarkingWriteBarrierForCode.

Bug: v8:8054,v8:7490
Change-Id: Ib1e07cfa1d5998fca2fa44e2ad08c52305f1373f
Reviewed-on: https://chromium-review.googlesource.com/1174436Reviewed-by: Yang Guo <yangguo@chromium.org>
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: Sigurd Schneider <sigurds@chromium.org>
Commit-Queue: Sigurd Schneider <sigurds@chromium.org>
Cr-Commit-Position: refs/heads/master@{#55185}

src/arm/assembler-arm-inl.h

@@ -120,8 +120,7 @@ void RelocInfo::set_target_object(Heap* heap, HeapObject* target,
                                    reinterpret_cast<Address>(target),
                                    icache_flush_mode);
   if (write_barrier_mode == UPDATE_WRITE_BARRIER && host() != nullptr) {
-    heap->incremental_marking()->RecordWriteIntoCode(host(), this, target);
-    heap->RecordWriteIntoCode(host(), this, target);
+    WriteBarrierForCode(host(), this, target);
   }
 }
 

src/arm64/assembler-arm64-inl.h

@@ -711,8 +711,7 @@ void RelocInfo::set_target_object(Heap* heap, HeapObject* target,
                                    reinterpret_cast<Address>(target),
                                    icache_flush_mode);
   if (write_barrier_mode == UPDATE_WRITE_BARRIER && host() != nullptr) {
-    heap->incremental_marking()->RecordWriteIntoCode(host(), this, target);
-    heap->RecordWriteIntoCode(host(), this, target);
+    WriteBarrierForCode(host(), this, target);
   }
 }
 

src/assembler.cc

@@ -508,8 +508,7 @@ void RelocInfo::set_target_address(Address target,
   if (write_barrier_mode == UPDATE_WRITE_BARRIER && host() != nullptr &&
       IsCodeTargetMode(rmode_)) {
     Code* target_code = Code::GetCodeFromTargetAddress(target);
-    host()->GetHeap()->incremental_marking()->RecordWriteIntoCode(host(), this,
-                                                                  target_code);
+    MarkingBarrierForCode(host(), this, target_code);
   }
 }
 

src/heap/factory.cc

@@ -2725,7 +2725,7 @@ Handle<Code> Factory::CopyCode(Handle<Code> code) {
   // allocation is on.
   heap->incremental_marking()->ProcessBlackAllocatedObject(*new_code);
   // Record all references to embedded objects in the new code object.
-  heap->RecordWritesIntoCode(*new_code);
+  WriteBarrierForCode(*new_code);
 
 #ifdef VERIFY_HEAP
   if (FLAG_verify_heap) new_code->ObjectVerify(isolate());

src/heap/heap-inl.h

@@ -429,12 +429,6 @@ bool Heap::ShouldBePromoted(Address old_address) {
          (!page->ContainsLimit(age_mark) || old_address < age_mark);
 }
 
-void Heap::RecordWriteIntoCode(Code* host, RelocInfo* rinfo, Object* value) {
-  if (InNewSpace(value)) {
-    RecordWriteIntoCodeSlow(host, rinfo, value);
-  }
-}
-
 Address* Heap::store_buffer_top_address() {
   return store_buffer()->top_address();
 }

src/heap/heap-write-barrier-inl.h

@@ -72,6 +72,18 @@ inline void MarkingBarrierInternal(HeapObject* object, Address slot,
 
 }  // namespace heap_internals
 
+inline void WriteBarrierForCode(Code* host, RelocInfo* rinfo, Object* value) {
+  DCHECK(!HasWeakHeapObjectTag(value));
+  if (!value->IsHeapObject()) return;
+  HeapObject* object = HeapObject::cast(value);
+  GenerationalBarrierForCode(host, rinfo, object);
+  MarkingBarrierForCode(host, rinfo, object);
+}
+
+inline void WriteBarrierForCode(Code* host) {
+  Heap::WriteBarrierForCodeSlow(host);
+}
+
 inline void GenerationalBarrier(HeapObject* object, Object** slot,
                                 Object* value) {
   DCHECK(!HasWeakHeapObjectTag(*slot));
@@ -98,6 +110,14 @@ inline void GenerationalBarrierForElements(Heap* heap, FixedArray* array,
   Heap::GenerationalBarrierForElementsSlow(heap, array, offset, length);
 }
 
+inline void GenerationalBarrierForCode(Code* host, RelocInfo* rinfo,
+                                       HeapObject* object) {
+  heap_internals::MemoryChunk* object_chunk =
+      heap_internals::MemoryChunk::FromHeapObject(object);
+  if (!object_chunk->InNewSpace()) return;
+  Heap::GenerationalBarrierForCodeSlow(host, rinfo, object);
+}
+
 inline void MarkingBarrier(HeapObject* object, Object** slot, Object* value) {
   DCHECK_IMPLIES(slot != nullptr, !HasWeakHeapObjectTag(*slot));
   DCHECK(!HasWeakHeapObjectTag(value));
@@ -122,6 +142,15 @@ inline void MarkingBarrierForElements(Heap* heap, HeapObject* object) {
   Heap::MarkingBarrierForElementsSlow(heap, object);
 }
 
+inline void MarkingBarrierForCode(Code* host, RelocInfo* rinfo,
+                                  HeapObject* object) {
+  DCHECK(!HasWeakHeapObjectTag(object));
+  heap_internals::MemoryChunk* object_chunk =
+      heap_internals::MemoryChunk::FromHeapObject(object);
+  if (!object_chunk->IsMarking()) return;
+  Heap::MarkingBarrierForCodeSlow(host, rinfo, object);
+}
+
 }  // namespace internal
 }  // namespace v8
 

src/heap/heap-write-barrier.h

@@ -8,11 +8,13 @@
 namespace v8 {
 namespace internal {
 
+class Code;
 class FixedArray;
 class Heap;
 class HeapObject;
 class MaybeObject;
 class Object;
+class RelocInfo;
 
 // Note: In general it is preferred to use the macros defined in
 // object-macros.h.
@@ -24,17 +26,24 @@ class Object;
     MarkingBarrierForElements(heap, array);                            \
   } while (false)
 
+// Combined write barriers.
+void WriteBarrierForCode(Code* host, RelocInfo* rinfo, Object* value);
+void WriteBarrierForCode(Code* host);
+
 // Generational write barrier.
 void GenerationalBarrier(HeapObject* object, Object** slot, Object* value);
 void GenerationalBarrier(HeapObject* object, MaybeObject** slot,
                          MaybeObject* value);
 void GenerationalBarrierForElements(Heap* heap, FixedArray* array, int offset,
                                     int length);
+void GenerationalBarrierForCode(Code* host, RelocInfo* rinfo,
+                                HeapObject* object);
 
 // Marking write barrier.
 void MarkingBarrier(HeapObject* object, Object** slot, Object* value);
 void MarkingBarrier(HeapObject* object, MaybeObject** slot, MaybeObject* value);
 void MarkingBarrierForElements(Heap* heap, HeapObject* object);
+void MarkingBarrierForCode(Code* host, RelocInfo* rinfo, HeapObject* object);
 
 }  // namespace internal
 }  // namespace v8

src/heap/heap.cc

@@ -5269,34 +5269,6 @@ void Heap::ClearRecordedSlotRange(Address start, Address end) {
   }
 }
 
-void Heap::RecordWriteIntoCodeSlow(Code* host, RelocInfo* rinfo,
-                                   Object* value) {
-  DCHECK(InNewSpace(value));
-  Page* source_page = Page::FromAddress(reinterpret_cast<Address>(host));
-  RelocInfo::Mode rmode = rinfo->rmode();
-  Address addr = rinfo->pc();
-  SlotType slot_type = SlotTypeForRelocInfoMode(rmode);
-  if (rinfo->IsInConstantPool()) {
-    addr = rinfo->constant_pool_entry_address();
-    if (RelocInfo::IsCodeTargetMode(rmode)) {
-      slot_type = CODE_ENTRY_SLOT;
-    } else {
-      DCHECK(RelocInfo::IsEmbeddedObject(rmode));
-      slot_type = OBJECT_SLOT;
-    }
-  }
-  RememberedSet<OLD_TO_NEW>::InsertTyped(
-      source_page, reinterpret_cast<Address>(host), slot_type, addr);
-}
-
-void Heap::RecordWritesIntoCode(Code* code) {
-  for (RelocIterator it(code, RelocInfo::ModeMask(RelocInfo::EMBEDDED_OBJECT));
-       !it.done(); it.next()) {
-    RecordWriteIntoCode(code, it.rinfo(), it.rinfo()->target_object());
-  }
-}
-
-
 PagedSpace* PagedSpaces::next() {
   switch (counter_++) {
     case RO_SPACE:
@@ -5870,6 +5842,14 @@ Code* Heap::GcSafeFindCodeForInnerPointer(Address inner_pointer) {
   }
 }
 
+void Heap::WriteBarrierForCodeSlow(Code* code) {
+  for (RelocIterator it(code, RelocInfo::ModeMask(RelocInfo::EMBEDDED_OBJECT));
+       !it.done(); it.next()) {
+    GenerationalBarrierForCode(code, it.rinfo(), it.rinfo()->target_object());
+    MarkingBarrierForCode(code, it.rinfo(), it.rinfo()->target_object());
+  }
+}
+
 void Heap::GenerationalBarrierSlow(HeapObject* object, Address slot,
                                    HeapObject* value) {
   Heap* heap = Heap::FromWritableHeapObject(object);
@@ -5885,6 +5865,26 @@ void Heap::GenerationalBarrierForElementsSlow(Heap* heap, FixedArray* array,
   }
 }
 
+void Heap::GenerationalBarrierForCodeSlow(Code* host, RelocInfo* rinfo,
+                                          HeapObject* object) {
+  DCHECK(InNewSpace(object));
+  Page* source_page = Page::FromAddress(reinterpret_cast<Address>(host));
+  RelocInfo::Mode rmode = rinfo->rmode();
+  Address addr = rinfo->pc();
+  SlotType slot_type = SlotTypeForRelocInfoMode(rmode);
+  if (rinfo->IsInConstantPool()) {
+    addr = rinfo->constant_pool_entry_address();
+    if (RelocInfo::IsCodeTargetMode(rmode)) {
+      slot_type = CODE_ENTRY_SLOT;
+    } else {
+      DCHECK(RelocInfo::IsEmbeddedObject(rmode));
+      slot_type = OBJECT_SLOT;
+    }
+  }
+  RememberedSet<OLD_TO_NEW>::InsertTyped(
+      source_page, reinterpret_cast<Address>(host), slot_type, addr);
+}
+
 void Heap::MarkingBarrierSlow(HeapObject* object, Address slot,
                               HeapObject* value) {
   Heap* heap = Heap::FromWritableHeapObject(object);
@@ -5899,6 +5899,13 @@ void Heap::MarkingBarrierForElementsSlow(Heap* heap, HeapObject* object) {
   }
 }
 
+void Heap::MarkingBarrierForCodeSlow(Code* host, RelocInfo* rinfo,
+                                     HeapObject* object) {
+  Heap* heap = Heap::FromWritableHeapObject(host);
+  DCHECK(heap->incremental_marking()->IsMarking());
+  heap->incremental_marking()->RecordWriteIntoCode(host, rinfo, object);
+}
+
 bool Heap::PageFlagsAreConsistent(HeapObject* object) {
   Heap* heap = Heap::FromWritableHeapObject(object);
   MemoryChunk* chunk = MemoryChunk::FromHeapObject(object);

src/heap/heap.h

@@ -484,16 +484,22 @@ class Heap {
   // by pointer size.
   static inline void CopyBlock(Address dst, Address src, int byte_size);
 
+  V8_EXPORT_PRIVATE static void WriteBarrierForCodeSlow(Code* host);
   V8_EXPORT_PRIVATE static void GenerationalBarrierSlow(HeapObject* object,
                                                         Address slot,
                                                         HeapObject* value);
   V8_EXPORT_PRIVATE static void GenerationalBarrierForElementsSlow(
       Heap* heap, FixedArray* array, int offset, int length);
+  V8_EXPORT_PRIVATE static void GenerationalBarrierForCodeSlow(
+      Code* host, RelocInfo* rinfo, HeapObject* value);
   V8_EXPORT_PRIVATE static void MarkingBarrierSlow(HeapObject* object,
                                                    Address slot,
                                                    HeapObject* value);
   V8_EXPORT_PRIVATE static void MarkingBarrierForElementsSlow(
       Heap* heap, HeapObject* object);
+  V8_EXPORT_PRIVATE static void MarkingBarrierForCodeSlow(Code* host,
+                                                          RelocInfo* rinfo,
+                                                          HeapObject* value);
   V8_EXPORT_PRIVATE static bool PageFlagsAreConsistent(HeapObject* object);
 
   // Notifies the heap that is ok to start marking or other activities that
@@ -982,12 +988,6 @@ class Heap {
   // Store buffer API. =========================================================
   // ===========================================================================
 
-  // Write barrier support for object[offset] = o;
-  // See heap/heap-write-barrier-inl.h for stand-alone methods.
-  inline void RecordWriteIntoCode(Code* host, RelocInfo* rinfo, Object* target);
-  void RecordWriteIntoCodeSlow(Code* host, RelocInfo* rinfo, Object* target);
-  void RecordWritesIntoCode(Code* code);
-
   // Used for query incremental marking status in generated code.
   Address* IsMarkingFlagAddress() {
     return reinterpret_cast<Address*>(&is_marking_flag_);

src/heap/incremental-marking-inl.h

@@ -36,13 +36,6 @@ void IncrementalMarking::RecordMaybeWeakWrite(HeapObject* obj,
   }
 }
 
-void IncrementalMarking::RecordWriteIntoCode(Code* host, RelocInfo* rinfo,
-                                             Object* value) {
-  if (IsMarking() && value->IsHeapObject()) {
-    RecordWriteIntoCodeSlow(host, rinfo, value);
-  }
-}
-
 void IncrementalMarking::RestartIfNotMarking() {
   if (state_ == COMPLETE) {
     state_ = MARKING;

src/heap/incremental-marking.cc

@@ -113,8 +113,9 @@ int IncrementalMarking::RecordWriteFromCode(HeapObject* obj, MaybeObject** slot,
   return 0;
 }
 
-void IncrementalMarking::RecordWriteIntoCodeSlow(Code* host, RelocInfo* rinfo,
-                                                 Object* value) {
+void IncrementalMarking::RecordWriteIntoCode(Code* host, RelocInfo* rinfo,
+                                             HeapObject* value) {
+  DCHECK(IsMarking());
   if (BaseRecordWrite(host, value)) {
     // Object is not going to be rescanned.  We need to record the slot.
     heap_->mark_compact_collector()->RecordRelocSlot(host, rinfo, value);

src/heap/incremental-marking.h

@@ -209,13 +209,11 @@ class V8_EXPORT_PRIVATE IncrementalMarking {
   V8_INLINE void RecordWrite(HeapObject* obj, Object** slot, Object* value);
   V8_INLINE void RecordMaybeWeakWrite(HeapObject* obj, MaybeObject** slot,
                                       MaybeObject* value);
-  V8_INLINE void RecordWriteIntoCode(Code* host, RelocInfo* rinfo,
-                                     Object* value);
   void RevisitObject(HeapObject* obj);
 
   void RecordWriteSlow(HeapObject* obj, HeapObjectReference** slot,
                        Object* value);
-  void RecordWriteIntoCodeSlow(Code* host, RelocInfo* rinfo, Object* value);
+  void RecordWriteIntoCode(Code* host, RelocInfo* rinfo, HeapObject* value);
 
   // Returns true if the function succeeds in transitioning the object
   // from white to grey.

src/heap/mark-compact.cc

@@ -1099,7 +1099,7 @@ class RecordMigratedSlotVisitor : public ObjectVisitor {
     DCHECK_EQ(host, rinfo->host());
     DCHECK(rinfo->rmode() == RelocInfo::EMBEDDED_OBJECT);
     HeapObject* object = HeapObject::cast(rinfo->target_object());
-    collector_->heap()->RecordWriteIntoCode(host, rinfo, object);
+    GenerationalBarrierForCode(host, rinfo, object);
     collector_->RecordRelocSlot(host, rinfo, object);
   }
 

src/ia32/assembler-ia32-inl.h

@@ -106,8 +106,7 @@ void RelocInfo::set_target_object(Heap* heap, HeapObject* target,
     Assembler::FlushICache(pc_, sizeof(Address));
   }
   if (write_barrier_mode == UPDATE_WRITE_BARRIER && host() != nullptr) {
-    heap->RecordWriteIntoCode(host(), this, target);
-    heap->incremental_marking()->RecordWriteIntoCode(host(), this, target);
+    WriteBarrierForCode(host(), this, target);
   }
 }
 

src/mips/assembler-mips-inl.h

@@ -198,9 +198,7 @@ void RelocInfo::set_target_object(Heap* heap, HeapObject* target,
                                    reinterpret_cast<Address>(target),
                                    icache_flush_mode);
   if (write_barrier_mode == UPDATE_WRITE_BARRIER && host() != nullptr) {
-    heap->incremental_marking()->RecordWriteIntoCode(host(), this,
-                                                     HeapObject::cast(target));
-    heap->RecordWriteIntoCode(host(), this, target);
+    WriteBarrierForCode(host(), this, target);
   }
 }
 

src/mips64/assembler-mips64-inl.h

@@ -171,11 +171,8 @@ void RelocInfo::set_target_object(Heap* heap, HeapObject* target,
   Assembler::set_target_address_at(pc_, constant_pool_,
                                    reinterpret_cast<Address>(target),
                                    icache_flush_mode);
-  if (write_barrier_mode == UPDATE_WRITE_BARRIER && host() != nullptr &&
-      target->IsHeapObject()) {
-    heap->incremental_marking()->RecordWriteIntoCode(host(), this,
-                                                     HeapObject::cast(target));
-    heap->RecordWriteIntoCode(host(), this, target);
+  if (write_barrier_mode == UPDATE_WRITE_BARRIER && host() != nullptr) {
+    WriteBarrierForCode(host(), this, target);
   }
 }
 

src/ppc/assembler-ppc-inl.h

@@ -181,8 +181,7 @@ void RelocInfo::set_target_object(Heap* heap, HeapObject* target,
                                    reinterpret_cast<Address>(target),
                                    icache_flush_mode);
   if (write_barrier_mode == UPDATE_WRITE_BARRIER && host() != nullptr) {
-    heap->incremental_marking()->RecordWriteIntoCode(host(), this, target);
-    heap->RecordWriteIntoCode(host(), this, target);
+    WriteBarrierForCode(host(), this, target);
   }
 }
 

src/s390/assembler-s390-inl.h

@@ -164,8 +164,7 @@ void RelocInfo::set_target_object(Heap* heap, HeapObject* target,
                                    reinterpret_cast<Address>(target),
                                    icache_flush_mode);
   if (write_barrier_mode == UPDATE_WRITE_BARRIER && host() != nullptr) {
-    heap->incremental_marking()->RecordWriteIntoCode(host(), this, target);
-    heap->RecordWriteIntoCode(host(), this, target);
+    WriteBarrierForCode(host(), this, target);
   }
 }
 

src/snapshot/object-deserializer.cc

@@ -59,7 +59,7 @@ void ObjectDeserializer::
   DCHECK(deserializing_user_code());
   for (Code* code : new_code_objects()) {
     // Record all references to embedded objects in the new code object.
-    isolate()->heap()->RecordWritesIntoCode(code);
+    WriteBarrierForCode(code);
     Assembler::FlushICache(code->raw_instruction_start(),
                            code->raw_instruction_size());
   }

src/snapshot/snapshot-common.cc

@@ -354,8 +354,10 @@ void FinalizeEmbeddedCodeTargets(Isolate* isolate, EmbeddedData* blob) {
       Code* target = Code::GetCodeFromTargetAddress(rinfo->target_address());
       CHECK(Builtins::IsIsolateIndependentBuiltin(target));
 
+      // Do not emit write-barrier for off-heap writes.
       off_heap_it.rinfo()->set_target_address(
-          blob->InstructionStartOfBuiltin(target->builtin_index()));
+          blob->InstructionStartOfBuiltin(target->builtin_index()),
+          SKIP_WRITE_BARRIER);
 
       on_heap_it.next();
       off_heap_it.next();

src/x64/assembler-x64-inl.h

@@ -362,8 +362,7 @@ void RelocInfo::set_target_object(Heap* heap, HeapObject* target,
     Assembler::FlushICache(pc_, sizeof(Address));
   }
   if (write_barrier_mode == UPDATE_WRITE_BARRIER && host() != nullptr) {
-    heap->incremental_marking()->RecordWriteIntoCode(host(), this, target);
-    heap->RecordWriteIntoCode(host(), this, target);
+    WriteBarrierForCode(host(), this, target);
   }
 }