heap,cppgc: Support for termination GC

Termination GCs are used to destroy remaining C++ object on the
managed heap to free potential off-heap memory. This is important for
gracefully shutting down workers.

Drive-by: Add guard prohibiting recursive sweeping calls on the
  mutator thread.

Bug: chromium:1056170
Change-Id: I02ea3b632d38f5beab18cc8f077cf717ed877909
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2631504
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: Omer Katz <omerkatz@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#72155}

include/cppgc/internal/persistent-node.h

@@ -90,23 +90,29 @@ class V8_EXPORT PersistentRegion final {
     PersistentNode* node = free_list_head_;
     free_list_head_ = free_list_head_->FreeListNext();
     node->InitializeAsUsedNode(owner, trace);
+    nodes_in_use_++;
     return node;
   }
 
   void FreeNode(PersistentNode* node) {
     node->InitializeAsFreeNode(free_list_head_);
     free_list_head_ = node;
+    CPPGC_DCHECK(nodes_in_use_ > 0);
+    nodes_in_use_--;
   }
 
   void Trace(Visitor*);
 
   size_t NodesInUse() const;
 
+  void ClearAllUsedNodes();
+
  private:
   void EnsureNodeSlots();
 
   std::vector<std::unique_ptr<PersistentNodeSlots>> nodes_;
   PersistentNode* free_list_head_ = nullptr;
+  size_t nodes_in_use_ = 0;
 };
 
 // CrossThreadPersistent uses PersistentRegion but protects it using this lock

include/v8-cppgc.h

@@ -48,6 +48,14 @@ class V8_EXPORT CppHeap {
    */
   cppgc::HeapHandle& GetHeapHandle();
 
+  /**
+   * Terminate clears all roots and performs multiple garbage collections to
+   * reclaim potentially newly created objects in destructors.
+   *
+   * After this call, object allocation is prohibited.
+   */
+  void Terminate();
+
  private:
   CppHeap() = default;
 

src/heap/cppgc-js/cpp-heap.cc

@@ -42,6 +42,8 @@ cppgc::HeapHandle& CppHeap::GetHeapHandle() {
   return *internal::CppHeap::From(this);
 }
 
+void CppHeap::Terminate() { internal::CppHeap::From(this)->Terminate(); }
+
 void JSHeapConsistency::DijkstraMarkingBarrierSlow(
     cppgc::HeapHandle& heap_handle, const TracedReferenceBase& ref) {
   auto& heap_base = cppgc::internal::HeapBase::From(heap_handle);
@@ -181,6 +183,15 @@ CppHeap::~CppHeap() {
   }
 }
 
+void CppHeap::Terminate() {
+  FinalizeIncrementalGarbageCollectionIfNeeded(
+      cppgc::Heap::StackState::kNoHeapPointers);
+  // Any future garbage collections will ignore the V8->C++ references.
+  isolate()->SetEmbedderHeapTracer(nullptr);
+  // Gracefully terminate the C++ heap invoking destructors.
+  HeapBase::Terminate();
+}
+
 void CppHeap::RegisterV8References(
     const std::vector<std::pair<void*, void*> >& embedder_fields) {
   DCHECK(marker_);

src/heap/cppgc-js/cpp-heap.h

@@ -47,6 +47,8 @@ class V8_EXPORT_PRIVATE CppHeap final : public cppgc::internal::HeapBase,
   void TraceEpilogue(TraceSummary* trace_summary) final;
   void EnterFinalPause(EmbedderStackState stack_state) final;
 
+  void Terminate();
+
  private:
   void FinalizeIncrementalGarbageCollectionIfNeeded(
       cppgc::Heap::StackState) final {

src/heap/cppgc/heap-base.cc

@@ -93,5 +93,34 @@ void HeapBase::AdvanceIncrementalGarbageCollectionOnAllocationIfNeeded() {
   if (marker_) marker_->AdvanceMarkingOnAllocation();
 }
 
+void HeapBase::Terminate() {
+  DCHECK(!IsMarking());
+  DCHECK(!in_no_gc_scope());
+
+  sweeper().FinishIfRunning();
+
+  constexpr size_t kMaxTerminationGCs = 20;
+  size_t gc_count = 0;
+  do {
+    CHECK_LT(gc_count++, kMaxTerminationGCs);
+
+    // Clear root sets.
+    strong_persistent_region_.ClearAllUsedNodes();
+    strong_cross_thread_persistent_region_.ClearAllUsedNodes();
+
+    stats_collector()->NotifyMarkingStarted(
+        GarbageCollector::Config::CollectionType::kMajor,
+        GarbageCollector::Config::IsForcedGC::kForced);
+    stats_collector()->NotifyMarkingCompleted(0);
+    object_allocator().ResetLinearAllocationBuffers();
+    sweeper().Start(
+        {Sweeper::SweepingConfig::SweepingType::kAtomic,
+         Sweeper::SweepingConfig::CompactableSpaceHandling::kSweep});
+    sweeper().NotifyDoneIfNeeded();
+  } while (strong_persistent_region_.NodesInUse() > 0);
+
+  object_allocator().Terminate();
+}
+
 }  // namespace internal
 }  // namespace cppgc

src/heap/cppgc/heap-base.h

@@ -156,12 +156,19 @@ class V8_EXPORT_PRIVATE HeapBase : public cppgc::HeapHandle {
   // Notifies the heap that a GC is done.
   virtual void PostGarbageCollection() = 0;
 
+  // Termination drops all roots (clears them out) and runs garbage collections
+  // in a bounded fixed point loop  until no new objects are created in
+  // destructors. Exceeding the loop bound results in a crash.
+  void Terminate();
+
  protected:
   virtual void FinalizeIncrementalGarbageCollectionIfNeeded(
       cppgc::Heap::StackState) = 0;
 
   bool in_no_gc_scope() const { return no_gc_scope_ > 0; }
 
+  bool IsMarking() const { return marker_.get(); }
+
   RawHeap raw_heap_;
   std::shared_ptr<cppgc::Platform> platform_;
 #if defined(CPPGC_CAGED_HEAP)

src/heap/cppgc/heap.cc

@@ -105,8 +105,6 @@ Heap::~Heap() {
   sweeper_.FinishIfRunning();
 }
 
-bool Heap::IsMarking() const { return marker_.get(); }
-
 void Heap::CollectGarbage(Config config) {
   DCHECK_EQ(Config::MarkingType::kAtomic, config.marking_type);
   CheckConfig(config, marking_support_, sweeping_support_);
@@ -149,7 +147,6 @@ void Heap::FinalizeIncrementalGarbageCollectionIfRunning(Config config) {
 
 void Heap::StartGarbageCollection(Config config) {
   DCHECK(!IsMarking());
-
   DCHECK(!in_no_gc_scope());
 
   // Finish sweeping in case it is still running.

src/heap/cppgc/heap.h

@@ -48,8 +48,6 @@ class V8_EXPORT_PRIVATE Heap final : public HeapBase,
 
   void PostGarbageCollection() final;
 
-  bool IsMarking() const;
-
   Config config_;
   GCInvoker gc_invoker_;
   HeapGrowing growing_;

src/heap/cppgc/object-allocator.cc

@@ -118,6 +118,8 @@ void* ObjectAllocator::OutOfLineAllocateImpl(NormalPageSpace* space,
                                              size_t size, GCInfoIndex gcinfo) {
   DCHECK_EQ(0, size & kAllocationMask);
   DCHECK_LE(kFreeListEntrySize, size);
+  // Out-of-line allocation allows for checking this is all situations.
+  CHECK(is_allocation_allowed());
 
   // 1. If this allocation is big enough, allocate a large object.
   if (size >= kLargeObjectSizeThreshold) {
@@ -189,6 +191,12 @@ void ObjectAllocator::ResetLinearAllocationBuffers() {
   visitor.Traverse(raw_heap_);
 }
 
+void ObjectAllocator::Terminate() {
+  ResetLinearAllocationBuffers();
+  // OutOfLineAllocateImpl checks is_allocation_allowed() unconditionally.
+  no_allocation_scope_++;
+}
+
 ObjectAllocator::NoAllocationScope::NoAllocationScope(
     ObjectAllocator& allocator)
     : allocator_(allocator) {

src/heap/cppgc/object-allocator.h

@@ -56,6 +56,9 @@ class V8_EXPORT_PRIVATE ObjectAllocator final : public cppgc::AllocationHandle {
 
   void ResetLinearAllocationBuffers();
 
+  // Terminate the allocator. Subsequent allocation calls result in a crash.
+  void Terminate();
+
  private:
   // Returns the initially tried SpaceType to allocate an object of |size| bytes
   // on. Returns the largest regular object size bucket for large objects.

src/heap/cppgc/persistent-node.cc

@@ -13,24 +13,37 @@
 namespace cppgc {
 namespace internal {
 
-PersistentRegion::~PersistentRegion() {
+PersistentRegion::~PersistentRegion() { ClearAllUsedNodes(); }
+
+void PersistentRegion::ClearAllUsedNodes() {
   for (auto& slots : nodes_) {
     for (auto& node : *slots) {
       if (node.IsUsed()) {
         static_cast<PersistentBase*>(node.owner())->ClearFromGC();
+        // Add nodes back to the free list to allow reusing for subsequent
+        // creation calls.
+        node.InitializeAsFreeNode(free_list_head_);
+        free_list_head_ = &node;
+        CPPGC_DCHECK(nodes_in_use_ > 0);
+        nodes_in_use_--;
       }
     }
   }
+  CPPGC_DCHECK(0u == nodes_in_use_);
 }
 
 size_t PersistentRegion::NodesInUse() const {
-  return std::accumulate(
+#ifdef DEBUG
+  const size_t accumulated_nodes_in_use_ = std::accumulate(
       nodes_.cbegin(), nodes_.cend(), 0u, [](size_t acc, const auto& slots) {
         return acc + std::count_if(slots->cbegin(), slots->cend(),
                                    [](const PersistentNode& node) {
                                      return node.IsUsed();
                                    });
       });
+  DCHECK_EQ(accumulated_nodes_in_use_, nodes_in_use_);
+#endif  // DEBUG
+  return nodes_in_use_;
 }
 
 void PersistentRegion::EnsureNodeSlots() {

src/heap/cppgc/sweeper.cc

@@ -525,6 +525,10 @@ class Sweeper::SweeperImpl final {
   void FinishIfRunning() {
     if (!is_in_progress_) return;
 
+    // Bail out for recursive sweeping calls. This can happen when finalizers
+    // allocate new memory.
+    if (is_sweeping_on_mutator_thread_) return;
+
     {
       StatsCollector::EnabledScope stats_scope(
           *heap_->heap(), StatsCollector::kIncrementalSweep);
@@ -543,6 +547,8 @@ class Sweeper::SweeperImpl final {
   void Finish() {
     DCHECK(is_in_progress_);
 
+    MutatorThreadSweepingScope sweeping_in_progresss(*this);
+
     // First, call finalizers on the mutator thread.
     SweepFinalizer finalizer(platform_);
     finalizer.FinalizeHeap(&space_states_);
@@ -580,6 +586,25 @@ class Sweeper::SweeperImpl final {
   }
 
  private:
+  class MutatorThreadSweepingScope final {
+   public:
+    explicit MutatorThreadSweepingScope(SweeperImpl& sweeper)
+        : sweeper_(sweeper) {
+      DCHECK(!sweeper_.is_sweeping_on_mutator_thread_);
+      sweeper_.is_sweeping_on_mutator_thread_ = true;
+    }
+    ~MutatorThreadSweepingScope() {
+      sweeper_.is_sweeping_on_mutator_thread_ = false;
+    }
+
+    MutatorThreadSweepingScope(const MutatorThreadSweepingScope&) = delete;
+    MutatorThreadSweepingScope& operator=(const MutatorThreadSweepingScope&) =
+        delete;
+
+   private:
+    SweeperImpl& sweeper_;
+  };
+
   class IncrementalSweepTask : public cppgc::IdleTask {
    public:
     using Handle = SingleThreadedHandle;
@@ -598,6 +623,8 @@ class Sweeper::SweeperImpl final {
     void Run(double deadline_in_seconds) override {
       if (handle_.IsCanceled() || !sweeper_->is_in_progress_) return;
 
+      MutatorThreadSweepingScope sweeping_in_progresss(*sweeper_);
+
       bool sweep_complete;
       {
         StatsCollector::EnabledScope stats_scope(
@@ -668,8 +695,12 @@ class Sweeper::SweeperImpl final {
   std::shared_ptr<cppgc::TaskRunner> foreground_task_runner_;
   IncrementalSweepTask::Handle incremental_sweeper_handle_;
   std::unique_ptr<cppgc::JobHandle> concurrent_sweeper_handle_;
+  // Indicates whether the sweeping phase is in progress.
   bool is_in_progress_ = false;
   bool notify_done_pending_ = false;
+  // Indicates whether whether the sweeper (or its finalization) is currently
+  // running on the main thread.
+  bool is_sweeping_on_mutator_thread_ = false;
 };
 
 Sweeper::Sweeper(RawHeap* heap, cppgc::Platform* platform,

test/unittests/heap/cppgc/heap-unittest.cc

@@ -9,6 +9,7 @@
 #include <numeric>
 
 #include "include/cppgc/allocation.h"
+#include "include/cppgc/persistent.h"
 #include "src/heap/cppgc/globals.h"
 #include "test/unittests/heap/cppgc/tests.h"
 #include "testing/gtest/include/gtest/gtest.h"
@@ -30,6 +31,8 @@ class GCHeapTest : public testing::TestWithHeap {
   }
 };
 
+class GCHeapDeathTest : public GCHeapTest {};
+
 class Foo : public GarbageCollected<Foo> {
  public:
   static size_t destructor_callcount;
@@ -153,5 +156,79 @@ TEST_F(GCHeapTest, AllocatedSizeDependOnAdditionalBytes) {
             HeapObjectHeader::FromPayload(object_with_more_bytes).GetSize());
 }
 
+TEST_F(GCHeapTest, TerminateEmptyHeap) { Heap::From(GetHeap())->Terminate(); }
+
+TEST_F(GCHeapTest, TerminateClearsPersistent) {
+  Persistent<Foo> foo = MakeGarbageCollected<Foo>(GetAllocationHandle());
+  EXPECT_TRUE(foo.Get());
+  Heap::From(GetHeap())->Terminate();
+  EXPECT_FALSE(foo.Get());
+}
+
+TEST_F(GCHeapTest, TerminateInvokesDestructor) {
+  Persistent<Foo> foo = MakeGarbageCollected<Foo>(GetAllocationHandle());
+  EXPECT_EQ(0u, Foo::destructor_callcount);
+  Heap::From(GetHeap())->Terminate();
+  EXPECT_EQ(1u, Foo::destructor_callcount);
+}
+
+namespace {
+
+class Cloner final : public GarbageCollected<Cloner> {
+ public:
+  static size_t destructor_count;
+
+  Cloner(cppgc::AllocationHandle& handle, size_t count)
+      : handle_(handle), count_(count) {}
+
+  ~Cloner() {
+    EXPECT_FALSE(new_instance_);
+    destructor_count++;
+    if (count_) {
+      new_instance_ =
+          MakeGarbageCollected<Cloner>(handle_, handle_, count_ - 1);
+    }
+  }
+
+  void Trace(Visitor*) const {}
+
+ private:
+  static Persistent<Cloner> new_instance_;
+
+  cppgc::AllocationHandle& handle_;
+  size_t count_;
+};
+
+Persistent<Cloner> Cloner::new_instance_;
+size_t Cloner::destructor_count;
+
+}  // namespace
+
+TEST_F(GCHeapTest, TerminateReclaimsNewState) {
+  Persistent<Cloner> cloner = MakeGarbageCollected<Cloner>(
+      GetAllocationHandle(), GetAllocationHandle(), 1);
+  Cloner::destructor_count = 0;
+  EXPECT_TRUE(cloner.Get());
+  Heap::From(GetHeap())->Terminate();
+  EXPECT_FALSE(cloner.Get());
+  EXPECT_EQ(2u, Cloner::destructor_count);
+}
+
+TEST_F(GCHeapDeathTest, TerminateProhibitsAllocation) {
+  Heap::From(GetHeap())->Terminate();
+  EXPECT_DEATH_IF_SUPPORTED(MakeGarbageCollected<Foo>(GetAllocationHandle()),
+                            "");
+}
+
+TEST_F(GCHeapDeathTest, LargeChainOfNewStates) {
+  Persistent<Cloner> cloner = MakeGarbageCollected<Cloner>(
+      GetAllocationHandle(), GetAllocationHandle(), 1000);
+  Cloner::destructor_count = 0;
+  EXPECT_TRUE(cloner.Get());
+  // Terminate() requires destructors to stop creating new state within a few
+  // garbage collections.
+  EXPECT_DEATH_IF_SUPPORTED(Heap::From(GetHeap())->Terminate(), "");
+}
+
 }  // namespace internal
 }  // namespace cppgc