[heap] Find references in client heaps to shared objects in shared GC

When performing a shared GC, we need to find references from the client
heaps into the shared heaps. For now we achieve this by simply
iterating all objects in client heaps.

We need to do this both for marking and when updating pointers after
evacuation.

Bug: v8:11708
Change-Id: Ic1dd94cc352be0404095e548979c37b1ef25682a
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3300142
Commit-Queue: Dominik Inführ <dinfuehr@chromium.org>
Reviewed-by: Shu-yu Guo <syg@chromium.org>
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Cr-Commit-Position: refs/heads/main@{#78240}

src/heap/heap.cc

@@ -4942,7 +4942,7 @@ void Heap::IterateRootsIncludingClients(RootVisitor* v,
                                         base::EnumSet<SkipRoot> options) {
   IterateRoots(v, options);
 
-  if (isolate()->global_safepoint()) {
+  if (isolate()->is_shared()) {
     isolate()->global_safepoint()->IterateClientIsolates(
         [v, options](Isolate* client) {
           client->heap()->IterateRoots(v, options);

src/heap/mark-compact.cc

@@ -21,6 +21,7 @@
 #include "src/heap/array-buffer-sweeper.h"
 #include "src/heap/code-object-registry.h"
 #include "src/heap/gc-tracer.h"
+#include "src/heap/heap.h"
 #include "src/heap/incremental-marking-inl.h"
 #include "src/heap/index-generator.h"
 #include "src/heap/invalidated-slots-inl.h"
@@ -1140,11 +1141,11 @@ class MarkCompactCollector::CustomRootBodyMarkingVisitor final
     UNREACHABLE();
   }
 
-  // VisitEmbedderPointer is defined by ObjectVisitor to call VisitPointers.
   void VisitCodeTarget(Code host, RelocInfo* rinfo) override {
     Code target = Code::GetCodeFromTargetAddress(rinfo->target_address());
     MarkObject(host, target);
   }
+
   void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override {
     MarkObject(host, rinfo->target_object(cage_base()));
   }
@@ -1158,6 +1159,72 @@ class MarkCompactCollector::CustomRootBodyMarkingVisitor final
   MarkCompactCollector* const collector_;
 };
 
+class MarkCompactCollector::SharedHeapObjectVisitor final
+    : public ObjectVisitorWithCageBases {
+ public:
+  explicit SharedHeapObjectVisitor(MarkCompactCollector* collector)
+      : ObjectVisitorWithCageBases(collector->isolate()),
+        collector_(collector) {}
+
+  void VisitPointer(HeapObject host, ObjectSlot p) final {
+    MarkObject(host, p.load(cage_base()));
+  }
+
+  void VisitPointer(HeapObject host, MaybeObjectSlot p) final {
+    MaybeObject object = p.load(cage_base());
+    HeapObject heap_object;
+    if (object.GetHeapObject(&heap_object)) MarkObject(host, heap_object);
+  }
+
+  void VisitMapPointer(HeapObject host) final {
+    MarkObject(host, host.map(cage_base()));
+  }
+
+  void VisitPointers(HeapObject host, ObjectSlot start, ObjectSlot end) final {
+    for (ObjectSlot p = start; p < end; ++p) {
+      // The map slot should be handled in VisitMapPointer.
+      DCHECK_NE(host.map_slot(), p);
+      DCHECK(!HasWeakHeapObjectTag(p.load(cage_base())));
+      MarkObject(host, p.load(cage_base()));
+    }
+  }
+
+  void VisitCodePointer(HeapObject host, CodeObjectSlot slot) override {
+    CHECK(V8_EXTERNAL_CODE_SPACE_BOOL);
+    // At the moment, custom roots cannot contain CodeDataContainers - the only
+    // objects that can contain Code pointers.
+    UNREACHABLE();
+  }
+
+  void VisitPointers(HeapObject host, MaybeObjectSlot start,
+                     MaybeObjectSlot end) final {
+    for (MaybeObjectSlot p = start; p < end; ++p) {
+      // The map slot should be handled in VisitMapPointer.
+      DCHECK_NE(host.map_slot(), ObjectSlot(p));
+      VisitPointer(host, p);
+    }
+  }
+
+  void VisitCodeTarget(Code host, RelocInfo* rinfo) override {
+    Code target = Code::GetCodeFromTargetAddress(rinfo->target_address());
+    MarkObject(host, target);
+  }
+  void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override {
+    MarkObject(host, rinfo->target_object(cage_base()));
+  }
+
+ private:
+  V8_INLINE void MarkObject(HeapObject host, Object object) {
+    DCHECK(!BasicMemoryChunk::FromHeapObject(host)->InSharedHeap());
+    if (!object.IsHeapObject()) return;
+    HeapObject heap_object = HeapObject::cast(object);
+    if (!BasicMemoryChunk::FromHeapObject(heap_object)->InSharedHeap()) return;
+    collector_->MarkObject(host, heap_object);
+  }
+
+  MarkCompactCollector* const collector_;
+};
+
 class InternalizedStringTableCleaner : public RootVisitor {
  public:
   explicit InternalizedStringTableCleaner(Heap* heap)
@@ -1777,7 +1844,7 @@ void MarkCompactCollector::MarkRoots(RootVisitor* root_visitor,
   // Custom marking for top optimized frame.
   ProcessTopOptimizedFrame(custom_root_body_visitor, isolate());
 
-  if (isolate()->global_safepoint()) {
+  if (isolate()->is_shared()) {
     isolate()->global_safepoint()->IterateClientIsolates(
         [this, custom_root_body_visitor](Isolate* client) {
           ProcessTopOptimizedFrame(custom_root_body_visitor, client);
@@ -1785,6 +1852,23 @@ void MarkCompactCollector::MarkRoots(RootVisitor* root_visitor,
   }
 }
 
+void MarkCompactCollector::MarkObjectsFromClientHeaps() {
+  if (!isolate()->is_shared()) return;
+
+  SharedHeapObjectVisitor visitor(this);
+
+  isolate()->global_safepoint()->IterateClientIsolates(
+      [&visitor](Isolate* client) {
+        Heap* heap = client->heap();
+        HeapObjectIterator iterator(heap, HeapObjectIterator::kNoFiltering);
+        for (HeapObject obj = iterator.Next(); !obj.is_null();
+             obj = iterator.Next()) {
+          PtrComprCageBase cage_base = GetPtrComprCageBase(obj);
+          obj.IterateFast(cage_base, &visitor);
+        }
+      });
+}
+
 void MarkCompactCollector::VisitObject(HeapObject obj) {
   marking_visitor_->Visit(obj.map(), obj);
 }
@@ -2177,6 +2261,11 @@ void MarkCompactCollector::MarkLiveObjects() {
     MarkRoots(&root_visitor, &custom_root_body_visitor);
   }
 
+  {
+    TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK_CLIENT_HEAPS);
+    MarkObjectsFromClientHeaps();
+  }
+
   {
     TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK_MAIN);
     if (FLAG_parallel_marking) {
@@ -3030,10 +3119,13 @@ static inline SlotCallbackResult UpdateStrongCodeSlot(
 
 }  // namespace
 
+static constexpr bool kClientHeap = true;
+
 // Visitor for updating root pointers and to-space pointers.
 // It does not expect to encounter pointers to dead objects.
-class PointersUpdatingVisitor : public ObjectVisitorWithCageBases,
-                                public RootVisitor {
+template <bool in_client_heap = false>
+class PointersUpdatingVisitor final : public ObjectVisitorWithCageBases,
+                                      public RootVisitor {
  public:
   explicit PointersUpdatingVisitor(Heap* heap)
       : ObjectVisitorWithCageBases(heap) {}
@@ -3087,14 +3179,34 @@ class PointersUpdatingVisitor : public ObjectVisitorWithCageBases,
     }
   }
 
-  void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override {
-    // This visitor nevers visits code objects.
-    UNREACHABLE();
+  void VisitMapPointer(HeapObject object) override {
+    if (in_client_heap) {
+      UpdateStrongSlotInternal(cage_base(), object.map_slot());
+    } else {
+      UNREACHABLE();
+    }
   }
 
   void VisitCodeTarget(Code host, RelocInfo* rinfo) override {
-    // This visitor nevers visits code objects.
-    UNREACHABLE();
+    if (in_client_heap) {
+      Code target = Code::GetCodeFromTargetAddress(rinfo->target_address());
+      CHECK_WITH_MSG(!target.InSharedHeap(),
+                     "refs into shared heap not yet supported here.");
+    } else {
+      // This visitor nevers visits code objects.
+      UNREACHABLE();
+    }
+  }
+
+  void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override {
+    if (in_client_heap) {
+      HeapObject target = rinfo->target_object(cage_base());
+      CHECK_WITH_MSG(!target.InSharedHeap(),
+                     "refs into shared heap not yet supported here.");
+    } else {
+      // This visitor nevers visits code objects.
+      UNREACHABLE();
+    }
   }
 
  private:
@@ -3929,7 +4041,7 @@ class ToSpaceUpdatingItem : public UpdatingItem {
   void ProcessVisitAll() {
     TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.gc"),
                  "ToSpaceUpdatingItem::ProcessVisitAll");
-    PointersUpdatingVisitor visitor(heap_);
+    PointersUpdatingVisitor<> visitor(heap_);
     for (Address cur = start_; cur < end_;) {
       HeapObject object = HeapObject::FromAddress(cur);
       Map map = object.map(visitor.cage_base());
@@ -3944,7 +4056,7 @@ class ToSpaceUpdatingItem : public UpdatingItem {
                  "ToSpaceUpdatingItem::ProcessVisitLive");
     // For young generation evacuations we want to visit grey objects, for
     // full MC, we need to visit black objects.
-    PointersUpdatingVisitor visitor(heap_);
+    PointersUpdatingVisitor<> visitor(heap_);
     for (auto object_and_size : LiveObjectRange<kAllLiveObjects>(
              chunk_, marking_state_->bitmap(chunk_))) {
       object_and_size.first.IterateBodyFast(visitor.cage_base(), &visitor);
@@ -4268,17 +4380,22 @@ class EphemeronTableUpdatingItem : public UpdatingItem {
 void MarkCompactCollector::UpdatePointersAfterEvacuation() {
   TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS);
 
-  PointersUpdatingVisitor updating_visitor(heap());
-
   {
     TRACE_GC(heap()->tracer(),
              GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_TO_NEW_ROOTS);
     // The external string table is updated at the end.
+    PointersUpdatingVisitor<> updating_visitor(heap());
     heap_->IterateRootsIncludingClients(
         &updating_visitor,
         base::EnumSet<SkipRoot>{SkipRoot::kExternalStringTable});
   }
 
+  {
+    TRACE_GC(heap()->tracer(),
+             GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_CLIENT_HEAPS);
+    UpdatePointersInClientHeaps();
+  }
+
   {
     TRACE_GC(heap()->tracer(),
              GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_SLOTS_MAIN);
@@ -4324,6 +4441,23 @@ void MarkCompactCollector::UpdatePointersAfterEvacuation() {
   }
 }
 
+void MarkCompactCollector::UpdatePointersInClientHeaps() {
+  if (!isolate()->is_shared()) return;
+
+  PointersUpdatingVisitor<kClientHeap> visitor(heap());
+
+  isolate()->global_safepoint()->IterateClientIsolates(
+      [&visitor](Isolate* client) {
+        Heap* heap = client->heap();
+        HeapObjectIterator iterator(heap, HeapObjectIterator::kNoFiltering);
+        for (HeapObject obj = iterator.Next(); !obj.is_null();
+             obj = iterator.Next()) {
+          PtrComprCageBase cage_base = GetPtrComprCageBase(obj);
+          obj.IterateFast(cage_base, &visitor);
+        }
+      });
+}
+
 void MarkCompactCollector::ReportAbortedEvacuationCandidateDueToOOM(
     Address failed_start, Page* page) {
   base::MutexGuard guard(&mutex_);
@@ -4832,7 +4966,7 @@ void MinorMarkCompactCollector::UpdatePointersAfterEvacuation() {
   TRACE_GC(heap()->tracer(),
            GCTracer::Scope::MINOR_MC_EVACUATE_UPDATE_POINTERS);
 
-  PointersUpdatingVisitor updating_visitor(heap());
+  PointersUpdatingVisitor<> updating_visitor(heap());
   std::vector<std::unique_ptr<UpdatingItem>> updating_items;
 
   // Create batches of global handles.

src/heap/mark-compact.h

@@ -456,6 +456,7 @@ class MarkCompactCollector final : public MarkCompactCollectorBase {
 
   class RootMarkingVisitor;
   class CustomRootBodyMarkingVisitor;
+  class SharedHeapObjectVisitor;
 
   enum IterationMode {
     kKeepMarking,
@@ -639,6 +640,13 @@ class MarkCompactCollector final : public MarkCompactCollectorBase {
   void MarkRoots(RootVisitor* root_visitor,
                  ObjectVisitor* custom_root_body_visitor);
 
+  // Mark all objects that are directly referenced from one of the clients
+  // heaps.
+  void MarkObjectsFromClientHeaps();
+
+  // Updates pointers to shared objects from client heaps.
+  void UpdatePointersInClientHeaps();
+
   // Marks object reachable from harmony weak maps and wrapper tracing.
   void ProcessEphemeronMarking();
 

src/heap/safepoint.cc

@@ -29,14 +29,16 @@ IsolateSafepoint::IsolateSafepoint(Heap* heap)
     : heap_(heap), local_heaps_head_(nullptr), active_safepoint_scopes_(0) {}
 
 void IsolateSafepoint::EnterLocalSafepointScope() {
-  // Safepoints need to be initiated on the main thread.
-  DCHECK_EQ(ThreadId::Current(), heap_->isolate()->thread_id());
+  // Safepoints need to be initiated on some main thread.
   DCHECK_NULL(LocalHeap::Current());
   DCHECK(AllowGarbageCollection::IsAllowed());
 
   LockMutex(heap_->isolate()->main_thread_local_heap());
   if (++active_safepoint_scopes_ > 1) return;
 
+  // Local safepoint can only be initiated on the isolate's main thread.
+  DCHECK_EQ(ThreadId::Current(), heap_->isolate()->thread_id());
+
   TimedHistogramScope timer(
       heap_->isolate()->counters()->gc_time_to_safepoint());
   TRACE_GC(heap_->tracer(), GCTracer::Scope::TIME_TO_SAFEPOINT);
@@ -47,6 +49,9 @@ void IsolateSafepoint::EnterLocalSafepointScope() {
 }
 
 void IsolateSafepoint::EnterGlobalSafepointScope(Isolate* initiator) {
+  // Safepoints need to be initiated on some main thread.
+  DCHECK_NULL(LocalHeap::Current());
+
   {
     IgnoreLocalGCRequests ignore_gc_requests(initiator->heap());
     LockMutex(initiator->main_thread_local_heap());

src/init/heap-symbols.h

@@ -526,11 +526,13 @@
   F(MC_EVACUATE_PROLOGUE)                            \
   F(MC_EVACUATE_REBALANCE)                           \
   F(MC_EVACUATE_UPDATE_POINTERS)                     \
+  F(MC_EVACUATE_UPDATE_POINTERS_CLIENT_HEAPS)        \
   F(MC_EVACUATE_UPDATE_POINTERS_PARALLEL)            \
   F(MC_EVACUATE_UPDATE_POINTERS_SLOTS_MAIN)          \
   F(MC_EVACUATE_UPDATE_POINTERS_TO_NEW_ROOTS)        \
   F(MC_EVACUATE_UPDATE_POINTERS_WEAK)                \
   F(MC_FINISH_SWEEP_ARRAY_BUFFERS)                   \
+  F(MC_MARK_CLIENT_HEAPS)                            \
   F(MC_MARK_EMBEDDER_PROLOGUE)                       \
   F(MC_MARK_EMBEDDER_TRACING)                        \
   F(MC_MARK_EMBEDDER_TRACING_CLOSURE)                \

test/cctest/heap/test-shared-heap.cc

@@ -169,24 +169,40 @@ void AllocateInSharedHeap(Isolate* shared_isolate, int iterations = 100) {
       shared_isolate,
       [iterations](v8::Isolate* client_isolate, Isolate* i_client_isolate) {
         HandleScope outer_scope(i_client_isolate);
-        std::vector<Handle<FixedArray>> arrays;
-        const int kKeptAliveArrays = 1000;
+        std::vector<Handle<FixedArray>> arrays_in_handles;
+        const int kKeptAliveInHandle = 1000;
+        const int kKeptAliveInHeap = 100;
+        Handle<FixedArray> arrays_in_heap =
+            i_client_isolate->factory()->NewFixedArray(kKeptAliveInHeap,
+                                                       AllocationType::kYoung);
 
         for (int i = 0; i < kNumIterations * iterations; i++) {
           HandleScope scope(i_client_isolate);
           Handle<FixedArray> array = i_client_isolate->factory()->NewFixedArray(
               100, AllocationType::kSharedOld);
-          if (i < kKeptAliveArrays) {
-            // Keep some of those arrays alive across GCs.
-            arrays.push_back(scope.CloseAndEscape(array));
+          if (i < kKeptAliveInHandle) {
+            // Keep some of those arrays alive across GCs through handles.
+            arrays_in_handles.push_back(scope.CloseAndEscape(array));
           }
+
+          if (i < kKeptAliveInHeap) {
+            // Keep some of those arrays alive across GCs through client heap
+            // references.
+            arrays_in_heap->set(i, *array);
+          }
+
           i_client_isolate->factory()->NewFixedArray(100,
                                                      AllocationType::kYoung);
         }
 
-        for (Handle<FixedArray> array : arrays) {
+        for (Handle<FixedArray> array : arrays_in_handles) {
           CHECK_EQ(array->length(), 100);
         }
+
+        for (int i = 0; i < kKeptAliveInHeap; i++) {
+          FixedArray array = FixedArray::cast(arrays_in_heap->get(i));
+          CHECK_EQ(array.length(), 100);
+        }
       });
 }