[heap] Introduce GlobalSafepoint and GlobalSafepointScope

Move logic to perform a global safepoint into GlobalSafepointScope
respectively GlobalSafepoint for easier reuse of this functionality in
the future.

Note that full functionality for a global safepoint will be provided
in a subsequent CL.

Bug: v8:11708
Change-Id: I80dd22c36ab01df573623aa36ead9cc373663b9b
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3259531Reviewed-by: Igor Sheludko <ishell@chromium.org>
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Commit-Queue: Dominik Inführ <dinfuehr@chromium.org>
Cr-Commit-Position: refs/heads/main@{#77719}

src/execution/isolate.cc

@@ -56,6 +56,7 @@
 #include "src/handles/persistent-handles.h"
 #include "src/heap/heap-inl.h"
 #include "src/heap/read-only-heap.h"
+#include "src/heap/safepoint.h"
 #include "src/ic/stub-cache.h"
 #include "src/init/bootstrapper.h"
 #include "src/init/setup-isolate.h"
@@ -3075,6 +3076,10 @@ Isolate::Isolate(std::unique_ptr<i::IsolateAllocator> isolate_allocator,
   InitializeDefaultEmbeddedBlob();
 
   MicrotaskQueue::SetUpDefaultMicrotaskQueue(this);
+
+  if (is_shared_) {
+    global_safepoint_ = std::make_unique<GlobalSafepoint>(this);
+  }
 }
 
 void Isolate::CheckIsolateLayout() {
@@ -3165,7 +3170,7 @@ void Isolate::Deinit() {
   }
 
   // All client isolates should already be detached.
-  DCHECK_NULL(client_isolate_head_);
+  if (is_shared()) global_safepoint()->AssertNoClients();
 
   if (FLAG_print_deopt_stress) {
     PrintF(stdout, "=== Stress deopt counter: %u\n", stress_deopt_count_);
@@ -5195,7 +5200,7 @@ void Isolate::AttachToSharedIsolate() {
 
   if (shared_isolate_) {
     DCHECK(shared_isolate_->is_shared());
-    shared_isolate_->AppendAsClientIsolate(this);
+    shared_isolate_->global_safepoint()->AppendClient(this);
   }
 
 #if DEBUG
@@ -5207,7 +5212,7 @@ void Isolate::DetachFromSharedIsolate() {
   DCHECK(attached_to_shared_isolate_);
 
   if (shared_isolate_) {
-    shared_isolate_->RemoveAsClientIsolate(this);
+    shared_isolate_->global_safepoint()->RemoveClient(this);
     shared_isolate_ = nullptr;
   }
 
@@ -5216,39 +5221,5 @@ void Isolate::DetachFromSharedIsolate() {
 #endif  // DEBUG
 }
 
-void Isolate::AppendAsClientIsolate(Isolate* client) {
-  base::MutexGuard guard(&client_isolate_mutex_);
-
-  DCHECK_NULL(client->prev_client_isolate_);
-  DCHECK_NULL(client->next_client_isolate_);
-  DCHECK_NE(client_isolate_head_, client);
-
-  if (client_isolate_head_) {
-    client_isolate_head_->prev_client_isolate_ = client;
-  }
-
-  client->prev_client_isolate_ = nullptr;
-  client->next_client_isolate_ = client_isolate_head_;
-
-  client_isolate_head_ = client;
-}
-
-void Isolate::RemoveAsClientIsolate(Isolate* client) {
-  base::MutexGuard guard(&client_isolate_mutex_);
-
-  if (client->next_client_isolate_) {
-    client->next_client_isolate_->prev_client_isolate_ =
-        client->prev_client_isolate_;
-  }
-
-  if (client->prev_client_isolate_) {
-    client->prev_client_isolate_->next_client_isolate_ =
-        client->next_client_isolate_;
-  } else {
-    DCHECK_EQ(client_isolate_head_, client);
-    client_isolate_head_ = client->next_client_isolate_;
-  }
-}
-
 }  // namespace internal
 }  // namespace v8

src/execution/isolate.h

@@ -95,6 +95,7 @@ class HandleScopeImplementer;
 class HeapObjectToIndexHashMap;
 class HeapProfiler;
 class GlobalHandles;
+class GlobalSafepoint;
 class InnerPointerToCodeCache;
 class LazyCompileDispatcher;
 class LocalIsolate;
@@ -1860,17 +1861,7 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory {
     shared_isolate_ = shared_isolate;
   }
 
-  bool HasClientIsolates() const { return client_isolate_head_; }
-
-  template <typename Callback>
-  void IterateClientIsolates(Callback callback) {
-    for (Isolate* current = client_isolate_head_; current;
-         current = current->next_client_isolate_) {
-      callback(current);
-    }
-  }
-
-  base::Mutex* client_isolate_mutex() { return &client_isolate_mutex_; }
+  GlobalSafepoint* global_safepoint() const { return global_safepoint_.get(); }
 
   bool OwnsStringTable() { return !FLAG_shared_string_table || is_shared(); }
 
@@ -2000,10 +1991,6 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory {
   void AttachToSharedIsolate();
   void DetachFromSharedIsolate();
 
-  // Methods for appending and removing to/from client isolates list.
-  void AppendAsClientIsolate(Isolate* client);
-  void RemoveAsClientIsolate(Isolate* client);
-
   // This class contains a collection of data accessible from both C++ runtime
   // and compiled code (including assembly stubs, builtins, interpreter bytecode
   // handlers and optimized code).
@@ -2308,15 +2295,12 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory {
   bool attached_to_shared_isolate_ = false;
 #endif  // DEBUG
 
-  // A shared isolate will use these two fields to track all its client
-  // isolates.
-  base::Mutex client_isolate_mutex_;
-  Isolate* client_isolate_head_ = nullptr;
-
-  // Used to form a linked list of all client isolates. Protected by
-  // client_isolate_mutex_.
-  Isolate* prev_client_isolate_ = nullptr;
-  Isolate* next_client_isolate_ = nullptr;
+  // Used to track and safepoint all client isolates attached to this shared
+  // isolate.
+  std::unique_ptr<GlobalSafepoint> global_safepoint_;
+  // Client isolates list managed by GlobalSafepoint.
+  Isolate* global_safepoint_prev_client_isolate_ = nullptr;
+  Isolate* global_safepoint_next_client_isolate_ = nullptr;
 
   // A signal-safe vector of heap pages containing code. Used with the
   // v8::Unwinder API.
@@ -2336,6 +2320,7 @@ class V8_EXPORT_PRIVATE Isolate final : private HiddenFactory {
   void* operator new(size_t, void* ptr) { return ptr; }
 
   friend class heap::HeapTester;
+  friend class GlobalSafepoint;
   friend class TestSerializer;
 };
 

src/heap/heap.cc

@@ -11,6 +11,7 @@
 #include <unordered_map>
 #include <unordered_set>
 
+#include "include/v8-locker.h"
 #include "src/api/api-inl.h"
 #include "src/base/bits.h"
 #include "src/base/flags.h"
@@ -2251,37 +2252,29 @@ void Heap::CollectSharedGarbage(GarbageCollectionReason gc_reason) {
 void Heap::PerformSharedGarbageCollection(Isolate* initiator,
                                           GarbageCollectionReason gc_reason) {
   DCHECK(IsShared());
-  base::MutexGuard guard(isolate()->client_isolate_mutex());
+
+  // Stop all client isolates attached to this isolate
+  GlobalSafepointScope global_safepoint(initiator);
+
+  // Migrate shared isolate to the main thread of the initiator isolate.
+  v8::Locker locker(reinterpret_cast<v8::Isolate*>(isolate()));
+  v8::Isolate::Scope isolate_scope(reinterpret_cast<v8::Isolate*>(isolate()));
 
   const char* collector_reason = nullptr;
   GarbageCollector collector = GarbageCollector::MARK_COMPACTOR;
 
   tracer()->Start(collector, gc_reason, collector_reason);
 
-  isolate()->IterateClientIsolates([initiator](Isolate* client) {
-    DCHECK_NOT_NULL(client->shared_isolate());
-    Heap* client_heap = client->heap();
-
-    IsolateSafepoint::StopMainThread stop_main_thread =
-        initiator == client ? IsolateSafepoint::StopMainThread::kNo
-                            : IsolateSafepoint::StopMainThread::kYes;
-
-    client_heap->safepoint()->EnterSafepointScope(stop_main_thread);
-    DCHECK(client_heap->deserialization_complete());
+  DCHECK_NOT_NULL(isolate()->global_safepoint());
 
+  isolate()->global_safepoint()->IterateClientIsolates([](Isolate* client) {
+    Heap* client_heap = client->heap();
     client_heap->shared_old_allocator_->FreeLinearAllocationArea();
     client_heap->shared_map_allocator_->FreeLinearAllocationArea();
   });
 
   PerformGarbageCollection(GarbageCollector::MARK_COMPACTOR);
 
-  isolate()->IterateClientIsolates([initiator](Isolate* client) {
-    IsolateSafepoint::StopMainThread stop_main_thread =
-        initiator == client ? IsolateSafepoint::StopMainThread::kNo
-                            : IsolateSafepoint::StopMainThread::kYes;
-    client->heap()->safepoint()->LeaveSafepointScope(stop_main_thread);
-  });
-
   tracer()->Stop(collector);
 }
 
@@ -4873,9 +4866,12 @@ void Heap::IterateRootsIncludingClients(RootVisitor* v,
                                         base::EnumSet<SkipRoot> options) {
   IterateRoots(v, options);
 
-  isolate()->IterateClientIsolates([v, options](Isolate* client) {
-    client->heap()->IterateRoots(v, options);
-  });
+  if (isolate()->global_safepoint()) {
+    isolate()->global_safepoint()->IterateClientIsolates(
+        [v, options](Isolate* client) {
+          client->heap()->IterateRoots(v, options);
+        });
+  }
 }
 
 void Heap::IterateWeakGlobalHandles(RootVisitor* v) {

src/heap/mark-compact.cc

@@ -1724,10 +1724,12 @@ void MarkCompactCollector::MarkRoots(RootVisitor* root_visitor,
   // Custom marking for top optimized frame.
   ProcessTopOptimizedFrame(custom_root_body_visitor, isolate());
 
-  isolate()->IterateClientIsolates(
-      [this, custom_root_body_visitor](Isolate* client) {
-        ProcessTopOptimizedFrame(custom_root_body_visitor, client);
-      });
+  if (isolate()->global_safepoint()) {
+    isolate()->global_safepoint()->IterateClientIsolates(
+        [this, custom_root_body_visitor](Isolate* client) {
+          ProcessTopOptimizedFrame(custom_root_body_visitor, client);
+        });
+  }
 }
 
 void MarkCompactCollector::VisitObject(HeapObject obj) {

src/heap/safepoint.cc

@@ -14,6 +14,7 @@
 #include "src/heap/heap-inl.h"
 #include "src/heap/heap.h"
 #include "src/heap/local-heap.h"
+#include "src/heap/parked-scope.h"
 #include "src/logging/counters-scopes.h"
 
 namespace v8 {
@@ -180,5 +181,92 @@ void IsolateSafepoint::Iterate(RootVisitor* visitor) {
   }
 }
 
+GlobalSafepoint::GlobalSafepoint(Isolate* isolate)
+    : shared_isolate_(isolate), shared_heap_(isolate->heap()) {}
+
+void GlobalSafepoint::AppendClient(Isolate* client) {
+  base::MutexGuard guard(&clients_mutex_);
+
+  DCHECK_NULL(client->global_safepoint_prev_client_isolate_);
+  DCHECK_NULL(client->global_safepoint_next_client_isolate_);
+  DCHECK_NE(clients_head_, client);
+
+  if (clients_head_) {
+    clients_head_->global_safepoint_prev_client_isolate_ = client;
+  }
+
+  client->global_safepoint_prev_client_isolate_ = nullptr;
+  client->global_safepoint_next_client_isolate_ = clients_head_;
+
+  clients_head_ = client;
+  client->shared_isolate_ = shared_isolate_;
+}
+
+void GlobalSafepoint::RemoveClient(Isolate* client) {
+  DCHECK_EQ(client->heap()->gc_state(), Heap::TEAR_DOWN);
+  base::MutexGuard guard(&clients_mutex_);
+
+  if (client->global_safepoint_next_client_isolate_) {
+    client->global_safepoint_next_client_isolate_
+        ->global_safepoint_prev_client_isolate_ =
+        client->global_safepoint_prev_client_isolate_;
+  }
+
+  if (client->global_safepoint_prev_client_isolate_) {
+    client->global_safepoint_prev_client_isolate_
+        ->global_safepoint_next_client_isolate_ =
+        client->global_safepoint_next_client_isolate_;
+  } else {
+    DCHECK_EQ(clients_head_, client);
+    clients_head_ = client->global_safepoint_next_client_isolate_;
+  }
+
+  client->shared_isolate_ = nullptr;
+}
+
+void GlobalSafepoint::AssertNoClients() { DCHECK_NULL(clients_head_); }
+
+void GlobalSafepoint::EnterGlobalSafepointScope(Isolate* initiator) {
+  if (!clients_mutex_.TryLock()) {
+    ParkedScope parked_scope(initiator->main_thread_local_heap());
+    clients_mutex_.Lock();
+  }
+
+  TimedHistogramScope timer(
+      initiator->counters()->gc_time_to_global_safepoint());
+  TRACE_GC(initiator->heap()->tracer(),
+           GCTracer::Scope::TIME_TO_GLOBAL_SAFEPOINT);
+
+  IterateClientIsolates([this, initiator](Isolate* client) {
+    Heap* client_heap = client->heap();
+    CHECK_EQ(initiator, client);
+    client_heap->safepoint()->EnterSafepointScope(
+        IsolateSafepoint::StopMainThread::kNo);
+
+    USE(this);
+    DCHECK_EQ(client->shared_isolate(), shared_isolate_);
+    DCHECK(client_heap->deserialization_complete());
+  });
+}
+
+void GlobalSafepoint::LeaveGlobalSafepointScope(Isolate* initiator) {
+  IterateClientIsolates([](Isolate* client) {
+    Heap* client_heap = client->heap();
+    client_heap->safepoint()->LeaveSafepointScope(
+        IsolateSafepoint::StopMainThread::kNo);
+  });
+
+  clients_mutex_.Unlock();
+}
+
+GlobalSafepointScope::GlobalSafepointScope(Isolate* initiator)
+    : initiator_(initiator), shared_isolate_(initiator->shared_isolate()) {
+  shared_isolate_->global_safepoint()->EnterGlobalSafepointScope(initiator_);
+}
+
+GlobalSafepointScope::~GlobalSafepointScope() {
+  shared_isolate_->global_safepoint()->LeaveGlobalSafepointScope(initiator_);
+}
+
 }  // namespace internal
 }  // namespace v8

src/heap/safepoint.h

@@ -119,6 +119,7 @@ class IsolateSafepoint final {
   int active_safepoint_scopes_;
 
   friend class Heap;
+  friend class GlobalSafepoint;
   friend class LocalHeap;
   friend class PersistentHandles;
   friend class SafepointScope;
@@ -133,6 +134,47 @@ class V8_NODISCARD SafepointScope {
   IsolateSafepoint* safepoint_;
 };
 
+// Used for reaching a global safepoint, a safepoint across all client isolates
+// of the shared isolate.
+class GlobalSafepoint final {
+ public:
+  explicit GlobalSafepoint(Isolate* isolate);
+
+  void AppendClient(Isolate* client);
+  void RemoveClient(Isolate* client);
+
+  template <typename Callback>
+  void IterateClientIsolates(Callback callback) {
+    for (Isolate* current = clients_head_; current;
+         current = current->global_safepoint_next_client_isolate_) {
+      callback(current);
+    }
+  }
+
+  void AssertNoClients();
+
+ private:
+  void EnterGlobalSafepointScope(Isolate* initiator);
+  void LeaveGlobalSafepointScope(Isolate* initiator);
+
+  Isolate* const shared_isolate_;
+  Heap* const shared_heap_;
+  base::Mutex clients_mutex_;
+  Isolate* clients_head_ = nullptr;
+
+  friend class GlobalSafepointScope;
+};
+
+class V8_NODISCARD GlobalSafepointScope {
+ public:
+  V8_EXPORT_PRIVATE explicit GlobalSafepointScope(Isolate* initiator);
+  V8_EXPORT_PRIVATE ~GlobalSafepointScope();
+
+ private:
+  Isolate* const initiator_;
+  Isolate* const shared_isolate_;
+};
+
 }  // namespace internal
 }  // namespace v8
 

src/init/heap-symbols.h

@@ -589,6 +589,7 @@
   F(SCAVENGER_SCAVENGE_WEAK)                         \
   F(SCAVENGER_SCAVENGE_FINALIZE)                     \
   F(SCAVENGER_SWEEP_ARRAY_BUFFERS)                   \
+  F(TIME_TO_GLOBAL_SAFEPOINT)                        \
   F(TIME_TO_SAFEPOINT)                               \
   F(UNMAPPER)
 

src/logging/counters-definitions.h

@@ -160,6 +160,8 @@ namespace internal {
   HT(gc_scavenger_foreground, V8.GCScavengerForeground, 10000, MILLISECOND)    \
   HT(measure_memory_delay_ms, V8.MeasureMemoryDelayMilliseconds, 100000,       \
      MILLISECOND)                                                              \
+  HT(gc_time_to_global_safepoint, V8.GC.TimeToGlobalSafepoint, 10000000,       \
+     MICROSECOND)                                                              \
   HT(gc_time_to_safepoint, V8.GC.TimeToSafepoint, 10000000, MICROSECOND)       \
   HT(gc_time_to_collection_on_background, V8.GC.TimeToCollectionOnBackground,  \
      10000000, MICROSECOND)                                                    \