heap: Move epilogue and weak callbacks to the epilogue of CollectGarbage

This is the 2. CL in a series of CollectGarbage refactoring CLs.

It moves two kinds of callbacks to the epilogue of CollectGarbage:
- Weak second pass phantom callbacks
- GCEpilogueCallbacks

As the prologue callbacks have also been moved outside the GC cycle,
with this CL the need for nesting tracers is eliminated. DCHECKs are
put in place to ensure this.

Bug: v8:12503
Change-Id: I0bc2c0fa0de1b0ffc32b5c648a612c0b3e0e1f3e
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3427199Reviewed-by: Omer Katz <omerkatz@chromium.org>
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Commit-Queue: Nikolaos Papaspyrou <nikolaos@chromium.org>
Cr-Commit-Position: refs/heads/main@{#78883}

src/heap/gc-tracer.cc

@@ -261,7 +261,8 @@ void GCTracer::ResetForTesting() {
 
 void GCTracer::NotifyYoungGenerationHandling(
     YoungGenerationHandling young_generation_handling) {
-  DCHECK(current_.type == Event::SCAVENGER || start_counter_ > 1);
+  DCHECK_GE(1, start_counter_);
+  DCHECK_EQ(Event::SCAVENGER, current_.type);
   heap_->isolate()->counters()->young_generation_handling()->AddSample(
       static_cast<int>(young_generation_handling));
 }
@@ -269,8 +270,8 @@ void GCTracer::NotifyYoungGenerationHandling(
 void GCTracer::Start(GarbageCollector collector,
                      GarbageCollectionReason gc_reason,
                      const char* collector_reason) {
+  DCHECK_EQ(0, start_counter_);
   start_counter_++;
-  if (start_counter_ != 1) return;
 
   previous_ = current_;
 
@@ -352,16 +353,7 @@ void GCTracer::StopInSafepoint() {
 
 void GCTracer::Stop(GarbageCollector collector) {
   start_counter_--;
-  if (start_counter_ != 0) {
-    if (FLAG_trace_gc_verbose) {
-      heap_->isolate()->PrintWithTimestamp(
-          "[Finished reentrant %s during %s.]\n",
-          Heap::CollectorName(collector), current_.TypeName(false));
-    }
-    return;
-  }
-
-  DCHECK_LE(0, start_counter_);
+  DCHECK_EQ(0, start_counter_);
   DCHECK((collector == GarbageCollector::SCAVENGER &&
           current_.type == Event::SCAVENGER) ||
          (collector == GarbageCollector::MINOR_MARK_COMPACTOR &&

src/heap/heap.cc

@@ -1722,6 +1722,8 @@ bool Heap::CollectGarbage(AllocationSpace space,
     }
   }
 
+  // Part 2: The main garbage collection phase.
+
   is_current_gc_forced_ = gc_callback_flags & v8::kGCCallbackFlagForced ||
                           current_gc_flags_ & kForcedGC ||
                           force_gc_on_next_allocation_;
@@ -1819,30 +1821,6 @@ bool Heap::CollectGarbage(AllocationSpace space,
       is_current_gc_forced_ = false;
       is_current_gc_for_heap_profiler_ = false;
 
-      {
-        TRACE_GC(tracer(), GCTracer::Scope::HEAP_EXTERNAL_WEAK_GLOBAL_HANDLES);
-        gc_post_processing_depth_++;
-        {
-          AllowGarbageCollection allow_gc;
-          AllowJavascriptExecution allow_js(isolate());
-          freed_global_handles +=
-              isolate_->global_handles()->PostGarbageCollectionProcessing(
-                  collector, gc_callback_flags);
-        }
-        gc_post_processing_depth_--;
-      }
-
-      {
-        GCCallbacksScope scope(this);
-        if (scope.CheckReenter()) {
-          AllowGarbageCollection allow_gc;
-          AllowJavascriptExecution allow_js(isolate());
-          TRACE_GC(tracer(), GCTracer::Scope::HEAP_EXTERNAL_EPILOGUE);
-          VMState<EXTERNAL> callback_state(isolate_);
-          HandleScope handle_scope(isolate_);
-          CallGCEpilogueCallbacks(gc_type, gc_callback_flags);
-        }
-      }
       if (collector == GarbageCollector::MARK_COMPACTOR ||
           collector == GarbageCollector::SCAVENGER) {
         tracer()->RecordGCPhasesHistograms(gc_type_timer);
@@ -1884,6 +1862,35 @@ bool Heap::CollectGarbage(AllocationSpace space,
     tracer()->Stop(collector);
   }
 
+  // Part 3: Invoke all callbacks which should happen after the actual garbage
+  // collection is triggered. Note that these callbacks may trigger another
+  // garbage collection since they may allocate.
+
+  {
+    TRACE_GC(tracer(), GCTracer::Scope::HEAP_EXTERNAL_WEAK_GLOBAL_HANDLES);
+    gc_post_processing_depth_++;
+    {
+      AllowGarbageCollection allow_gc;
+      AllowJavascriptExecution allow_js(isolate());
+      freed_global_handles +=
+          isolate_->global_handles()->PostGarbageCollectionProcessing(
+              collector, gc_callback_flags);
+    }
+    gc_post_processing_depth_--;
+  }
+
+  {
+    GCCallbacksScope scope(this);
+    if (scope.CheckReenter()) {
+      AllowGarbageCollection allow_gc;
+      AllowJavascriptExecution allow_js(isolate());
+      TRACE_GC(tracer(), GCTracer::Scope::HEAP_EXTERNAL_EPILOGUE);
+      VMState<EXTERNAL> callback_state(isolate_);
+      HandleScope handle_scope(isolate_);
+      CallGCEpilogueCallbacks(gc_type, gc_callback_flags);
+    }
+  }
+
   if (collector == GarbageCollector::MARK_COMPACTOR &&
       (gc_callback_flags & (kGCCallbackFlagForced |
                             kGCCallbackFlagCollectAllAvailableGarbage)) != 0) {