[heap] Implement C++ shared heap write barrier

Record old-to-shared references in the C++ write barrier. When
an old-to-shared reference is created, this particular slot will be
atomically inserted into the old-to-new remembered set.

We already stopped clearing the old-to-new-remembered set after a
shared GC, so we already need to be able to handle such slots when
invalidating objects and in the sweeper.

Bug: v8:11708
Change-Id: I1b5854d58f6496228f3a3d9eb7acfd9492f09e68
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3557232Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Commit-Queue: Dominik Inführ <dinfuehr@chromium.org>
Cr-Commit-Position: refs/heads/main@{#80555}

src/heap/heap-write-barrier-inl.h

@@ -28,11 +28,17 @@ V8_EXPORT_PRIVATE bool Heap_ValueMightRequireGenerationalWriteBarrier(
 V8_EXPORT_PRIVATE void Heap_GenerationalBarrierSlow(HeapObject object,
                                                     Address slot,
                                                     HeapObject value);
+V8_EXPORT_PRIVATE void Heap_SharedHeapBarrierSlow(HeapObject object,
+                                                  Address slot,
+                                                  HeapObject value);
 V8_EXPORT_PRIVATE void Heap_WriteBarrierForCodeSlow(Code host);
 
 V8_EXPORT_PRIVATE void Heap_GenerationalBarrierForCodeSlow(Code host,
                                                            RelocInfo* rinfo,
                                                            HeapObject object);
+V8_EXPORT_PRIVATE void Heap_SharedHeapBarrierForCodeSlow(Code host,
+                                                         RelocInfo* rinfo,
+                                                         HeapObject object);
 
 V8_EXPORT_PRIVATE void Heap_GenerationalEphemeronKeyBarrierSlow(
     Heap* heap, EphemeronHashTable table, Address slot);
@@ -49,6 +55,7 @@ struct MemoryChunk {
   static constexpr uintptr_t kFromPageBit = uintptr_t{1} << 3;
   static constexpr uintptr_t kToPageBit = uintptr_t{1} << 4;
   static constexpr uintptr_t kReadOnlySpaceBit = uintptr_t{1} << 20;
+  static constexpr uintptr_t kInSharedHeapBit = uintptr_t{1} << 22;
 
   V8_INLINE static heap_internals::MemoryChunk* FromHeapObject(
       HeapObject object) {
@@ -58,6 +65,8 @@ struct MemoryChunk {
 
   V8_INLINE bool IsMarking() const { return GetFlags() & kMarkingBit; }
 
+  V8_INLINE bool InSharedHeap() const { return GetFlags() & kInSharedHeapBit; }
+
   V8_INLINE bool InYoungGeneration() const {
     if (V8_ENABLE_THIRD_PARTY_HEAP_BOOL) return false;
     constexpr uintptr_t kYoungGenerationMask = kFromPageBit | kToPageBit;
@@ -98,6 +107,23 @@ inline void GenerationalBarrierInternal(HeapObject object, Address slot,
   Heap_GenerationalBarrierSlow(object, slot, value);
 }
 
+inline void SharedHeapBarrierInternal(HeapObject object, Address slot,
+                                      HeapObject value) {
+  DCHECK(Heap_PageFlagsAreConsistent(object));
+  heap_internals::MemoryChunk* value_chunk =
+      heap_internals::MemoryChunk::FromHeapObject(value);
+  heap_internals::MemoryChunk* object_chunk =
+      heap_internals::MemoryChunk::FromHeapObject(object);
+
+  // We only care about pointers into the shared heap.
+  if (!value_chunk->InSharedHeap()) return;
+
+  // Do not record slots in new space or the shared heap itself.
+  if (object_chunk->InYoungGeneration() || object_chunk->InSharedHeap()) return;
+
+  Heap_SharedHeapBarrierSlow(object, slot, value);
+}
+
 inline void GenerationalEphemeronKeyBarrierInternal(EphemeronHashTable table,
                                                     Address slot,
                                                     HeapObject value) {
@@ -124,6 +150,7 @@ inline void WriteBarrierForCode(Code host, RelocInfo* rinfo, Object value) {
 
 inline void WriteBarrierForCode(Code host, RelocInfo* rinfo, HeapObject value) {
   GenerationalBarrierForCode(host, rinfo, value);
+  SharedHeapBarrierForCode(host, rinfo, value);
   WriteBarrier::Marking(host, rinfo, value);
 }
 
@@ -180,6 +207,32 @@ inline void GenerationalBarrierForCode(Code host, RelocInfo* rinfo,
   Heap_GenerationalBarrierForCodeSlow(host, rinfo, object);
 }
 
+inline void SharedHeapBarrier(HeapObject object, ObjectSlot slot,
+                              Object value) {
+  if (V8_ENABLE_THIRD_PARTY_HEAP_BOOL) return;
+  HeapObject value_heap_object;
+  if (!value.GetHeapObject(&value_heap_object)) return;
+  heap_internals::SharedHeapBarrierInternal(object, slot.address(),
+                                            value_heap_object);
+}
+inline void SharedHeapBarrier(HeapObject object, MaybeObjectSlot slot,
+                              MaybeObject value) {
+  if (V8_ENABLE_THIRD_PARTY_HEAP_BOOL) return;
+  HeapObject value_heap_object;
+  if (!value->GetHeapObject(&value_heap_object)) return;
+  heap_internals::SharedHeapBarrierInternal(object, slot.address(),
+                                            value_heap_object);
+}
+
+inline void SharedHeapBarrierForCode(Code host, RelocInfo* rinfo,
+                                     HeapObject object) {
+  if (V8_ENABLE_THIRD_PARTY_HEAP_BOOL) return;
+  heap_internals::MemoryChunk* object_chunk =
+      heap_internals::MemoryChunk::FromHeapObject(object);
+  if (!object_chunk->InSharedHeap()) return;
+  Heap_SharedHeapBarrierForCodeSlow(host, rinfo, object);
+}
+
 inline WriteBarrierMode GetWriteBarrierModeForObject(
     HeapObject object, const DisallowGarbageCollection* promise) {
   if (FLAG_disable_write_barriers) return SKIP_WRITE_BARRIER;

src/heap/heap-write-barrier.h

@@ -43,6 +43,12 @@ void GenerationalEphemeronKeyBarrier(EphemeronHashTable table, ObjectSlot slot,
                                      Object value);
 void GenerationalBarrierForCode(Code host, RelocInfo* rinfo, HeapObject object);
 
+// Shared heap write barrier.
+void SharedHeapBarrier(HeapObject object, ObjectSlot slot, Object value);
+void SharedHeapBarrier(HeapObject object, MaybeObjectSlot slot,
+                       MaybeObject value);
+void SharedHeapBarrierForCode(Code host, RelocInfo* rinfo, HeapObject object);
+
 inline bool IsReadOnlyHeapObject(HeapObject object);
 
 class V8_EXPORT_PRIVATE WriteBarrier {

src/heap/heap.cc

@@ -143,6 +143,11 @@ void Heap_GenerationalBarrierSlow(HeapObject object, Address slot,
   Heap::GenerationalBarrierSlow(object, slot, value);
 }
 
+void Heap_SharedHeapBarrierSlow(HeapObject object, Address slot,
+                                HeapObject value) {
+  Heap::SharedHeapBarrierSlow(object, slot, value);
+}
+
 void Heap_WriteBarrierForCodeSlow(Code host) {
   Heap::WriteBarrierForCodeSlow(host);
 }
@@ -152,6 +157,11 @@ void Heap_GenerationalBarrierForCodeSlow(Code host, RelocInfo* rinfo,
   Heap::GenerationalBarrierForCodeSlow(host, rinfo, object);
 }
 
+void Heap_SharedHeapBarrierForCodeSlow(Code host, RelocInfo* rinfo,
+                                       HeapObject object) {
+  Heap::SharedHeapBarrierForCodeSlow(host, rinfo, object);
+}
+
 void Heap_GenerationalEphemeronKeyBarrierSlow(Heap* heap,
                                               EphemeronHashTable table,
                                               Address slot) {
@@ -7310,6 +7320,12 @@ void Heap::GenerationalBarrierSlow(HeapObject object, Address slot,
   RememberedSet<OLD_TO_NEW>::Insert<AccessMode::NON_ATOMIC>(chunk, slot);
 }
 
+void Heap::SharedHeapBarrierSlow(HeapObject object, Address slot,
+                                 HeapObject value) {
+  MemoryChunk* chunk = MemoryChunk::FromHeapObject(object);
+  RememberedSet<OLD_TO_SHARED>::Insert<AccessMode::ATOMIC>(chunk, slot);
+}
+
 void Heap::RecordEphemeronKeyWrite(EphemeronHashTable table, Address slot) {
   DCHECK(ObjectInYoungGeneration(HeapObjectSlot(slot).ToHeapObject()));
   if (FLAG_minor_mc) {
@@ -7448,6 +7464,16 @@ void Heap::GenerationalBarrierForCodeSlow(Code host, RelocInfo* rinfo,
                                          info.offset);
 }
 
+void Heap::SharedHeapBarrierForCodeSlow(Code host, RelocInfo* rinfo,
+                                        HeapObject object) {
+  DCHECK(object.InSharedHeap());
+  const MarkCompactCollector::RecordRelocSlotInfo info =
+      MarkCompactCollector::ProcessRelocInfo(host, rinfo, object);
+
+  RememberedSet<OLD_TO_SHARED>::InsertTyped(info.memory_chunk, info.slot_type,
+                                            info.offset);
+}
+
 bool Heap::PageFlagsAreConsistent(HeapObject object) {
   if (V8_ENABLE_THIRD_PARTY_HEAP_BOOL) {
     return true;

src/heap/heap.h

@@ -494,12 +494,18 @@ class Heap {
   V8_EXPORT_PRIVATE static void GenerationalBarrierSlow(HeapObject object,
                                                         Address slot,
                                                         HeapObject value);
+  V8_EXPORT_PRIVATE static void SharedHeapBarrierSlow(HeapObject object,
+                                                      Address slot,
+                                                      HeapObject value);
   V8_EXPORT_PRIVATE inline void RecordEphemeronKeyWrite(
       EphemeronHashTable table, Address key_slot);
   V8_EXPORT_PRIVATE static void EphemeronKeyWriteBarrierFromCode(
       Address raw_object, Address address, Isolate* isolate);
   V8_EXPORT_PRIVATE static void GenerationalBarrierForCodeSlow(
       Code host, RelocInfo* rinfo, HeapObject value);
+  V8_EXPORT_PRIVATE static void SharedHeapBarrierForCodeSlow(Code host,
+                                                             RelocInfo* rinfo,
+                                                             HeapObject value);
   V8_EXPORT_PRIVATE static bool PageFlagsAreConsistent(HeapObject object);
 
   // Notifies the heap that is ok to start marking or other activities that

src/objects/object-macros.h

@@ -454,6 +454,7 @@
     DCHECK_NOT_NULL(GetHeapFromWritableObject(object));              \
     WriteBarrier::Marking(object, (object).RawField(offset), value); \
     GenerationalBarrier(object, (object).RawField(offset), value);   \
+    SharedHeapBarrier(object, (object).RawField(offset), value);     \
   } while (false)
 #endif
 
@@ -465,6 +466,7 @@
     DCHECK_NOT_NULL(GetHeapFromWritableObject(object));                       \
     WriteBarrier::Marking(object, (object).RawMaybeWeakField(offset), value); \
     GenerationalBarrier(object, (object).RawMaybeWeakField(offset), value);   \
+    SharedHeapBarrier(object, (object).RawMaybeWeakField(offset), value);     \
   } while (false)
 #endif
 
@@ -480,6 +482,7 @@
     EphemeronHashTable table = EphemeronHashTable::cast(object);              \
     WriteBarrier::Marking(object, (object).RawField(offset), value);          \
     GenerationalEphemeronKeyBarrier(table, (object).RawField(offset), value); \
+    SharedHeapBarrier(object, (object).RawField(offset), value);              \
   } while (false)
 #endif
 
@@ -498,6 +501,7 @@
         WriteBarrier::Marking(object, (object).RawField(offset), value); \
       }                                                                  \
       GenerationalBarrier(object, (object).RawField(offset), value);     \
+      SharedHeapBarrier(object, (object).RawField(offset), value);       \
     } else {                                                             \
       SLOW_DCHECK(!WriteBarrier::IsRequired(object, value));             \
     }                                                                    \
@@ -520,6 +524,7 @@
                               value);                                         \
       }                                                                       \
       GenerationalBarrier(object, (object).RawMaybeWeakField(offset), value); \
+      SharedHeapBarrier(object, (object).RawMaybeWeakField(offset), value);   \
     } else {                                                                  \
       SLOW_DCHECK(!WriteBarrier::IsRequired(object, value));                  \
     }                                                                         \
@@ -540,6 +545,7 @@
       }                                                                      \
       GenerationalEphemeronKeyBarrier(table, (object).RawField(offset),      \
                                       value);                                \
+      SharedHeapBarrier(table, (object).RawField(offset), value);            \
     } else {                                                                 \
       SLOW_DCHECK(!WriteBarrier::IsRequired(object, value));                 \
     }                                                                        \