heap: Move embedder epilogue after first round of weak callbacks

Bug: chromium:843903, chromium:903586
Change-Id: I0441ab2cc0fcc0b2cb411be0c92860c64d808324
Reviewed-on: https://chromium-review.googlesource.com/c/1340254Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#57604}

src/heap-symbols.h

@@ -354,8 +354,8 @@
   F(MC_INCREMENTAL)                                                \
   F(MC_INCREMENTAL_START)                                          \
   F(MC_INCREMENTAL_SWEEPING)                                       \
-  F(MC_INCREMENTAL_WRAPPER_PROLOGUE)                               \
-  F(MC_INCREMENTAL_WRAPPER_TRACING)                                \
+  F(MC_INCREMENTAL_EMBEDDER_PROLOGUE)                              \
+  F(MC_INCREMENTAL_EMBEDDER_TRACING)                               \
   F(MC_INCREMENTAL_FINALIZE)                                       \
   F(MC_INCREMENTAL_FINALIZE_BODY)                                  \
   F(MC_INCREMENTAL_EXTERNAL_EPILOGUE)                              \
@@ -372,6 +372,7 @@
 
 #define TRACER_SCOPES(F)                             \
   INCREMENTAL_SCOPES(F)                              \
+  F(HEAP_EMBEDDER_TRACING_EPILOGUE)                  \
   F(HEAP_EPILOGUE)                                   \
   F(HEAP_EPILOGUE_REDUCE_NEW_SPACE)                  \
   F(HEAP_EXTERNAL_EPILOGUE)                          \
@@ -398,7 +399,9 @@
   F(MC_EVACUATE_UPDATE_POINTERS_SLOTS_MAP_SPACE)     \
   F(MC_EVACUATE_UPDATE_POINTERS_TO_NEW_ROOTS)        \
   F(MC_EVACUATE_UPDATE_POINTERS_WEAK)                \
-  F(MC_FINISH_WRAPPER_EPILOGUE)                      \
+  F(MC_MARK_EMBEDDER_PROLOGUE)                       \
+  F(MC_MARK_EMBEDDER_TRACING)                        \
+  F(MC_MARK_EMBEDDER_TRACING_CLOSURE)                \
   F(MC_MARK_FINISH_INCREMENTAL)                      \
   F(MC_MARK_MAIN)                                    \
   F(MC_MARK_ROOTS)                                   \
@@ -409,9 +412,6 @@
   F(MC_MARK_WEAK_CLOSURE_WEAK_HANDLES)               \
   F(MC_MARK_WEAK_CLOSURE_WEAK_ROOTS)                 \
   F(MC_MARK_WEAK_CLOSURE_HARMONY)                    \
-  F(MC_MARK_WRAPPERS)                                \
-  F(MC_MARK_WRAPPER_PROLOGUE)                        \
-  F(MC_MARK_WRAPPER_TRACING)                         \
   F(MC_SWEEP_CODE)                                   \
   F(MC_SWEEP_MAP)                                    \
   F(MC_SWEEP_OLD)                                    \

src/heap/gc-tracer.cc

@@ -656,6 +656,7 @@ void GCTracer::PrintNVP() const {
           "gc=%s "
           "reduce_memory=%d "
           "heap.prologue=%.2f "
+          "heap.embedder_tracing_epilogue=%.2f "
           "heap.epilogue=%.2f "
           "heap.epilogue.reduce_new_space=%.2f "
           "heap.external.prologue=%.1f "
@@ -684,7 +685,6 @@ void GCTracer::PrintNVP() const {
           "evacuate.update_pointers.slots.map_space=%.1f "
           "evacuate.update_pointers.weak=%.1f "
           "finish=%.1f "
-          "finish.wrapper_epilogue=%.1f "
           "mark=%.1f "
           "mark.finish_incremental=%.1f "
           "mark.roots=%.1f "
@@ -696,8 +696,8 @@ void GCTracer::PrintNVP() const {
           "mark.weak_closure.weak_handles=%.1f "
           "mark.weak_closure.weak_roots=%.1f "
           "mark.weak_closure.harmony=%.1f "
-          "mark.wrapper_prologue=%.1f "
-          "mark.wrapper_tracing=%.1f "
+          "mark.embedder_prologue=%.1f "
+          "mark.embedder_tracing=%.1f "
           "prologue=%.1f "
           "sweep=%.1f "
           "sweep.code=%.1f "
@@ -709,8 +709,8 @@ void GCTracer::PrintNVP() const {
           "incremental.finalize.external.prologue=%.1f "
           "incremental.finalize.external.epilogue=%.1f "
           "incremental.sweeping=%.1f "
-          "incremental.wrapper_prologue=%.1f "
-          "incremental.wrapper_tracing=%.1f "
+          "incremental.embedder_prologue=%.1f "
+          "incremental.embedder_tracing=%.1f "
           "incremental_wrapper_tracing_longest_step=%.1f "
           "incremental_finalize_longest_step=%.1f "
           "incremental_finalize_steps_count=%d "
@@ -752,6 +752,7 @@ void GCTracer::PrintNVP() const {
           "compaction_speed=%.f\n",
           duration, spent_in_mutator, current_.TypeName(true),
           current_.reduce_memory, current_.scopes[Scope::HEAP_PROLOGUE],
+          current_.scopes[Scope::HEAP_EMBEDDER_TRACING_EPILOGUE],
           current_.scopes[Scope::HEAP_EPILOGUE],
           current_.scopes[Scope::HEAP_EPILOGUE_REDUCE_NEW_SPACE],
           current_.scopes[Scope::HEAP_EXTERNAL_PROLOGUE],
@@ -779,9 +780,7 @@ void GCTracer::PrintNVP() const {
           current_.scopes[Scope::MC_EVACUATE_UPDATE_POINTERS_SLOTS_MAIN],
           current_.scopes[Scope::MC_EVACUATE_UPDATE_POINTERS_SLOTS_MAP_SPACE],
           current_.scopes[Scope::MC_EVACUATE_UPDATE_POINTERS_WEAK],
-          current_.scopes[Scope::MC_FINISH],
-          current_.scopes[Scope::MC_FINISH_WRAPPER_EPILOGUE],
-          current_.scopes[Scope::MC_MARK],
+          current_.scopes[Scope::MC_FINISH], current_.scopes[Scope::MC_MARK],
           current_.scopes[Scope::MC_MARK_FINISH_INCREMENTAL],
           current_.scopes[Scope::MC_MARK_ROOTS],
           current_.scopes[Scope::MC_MARK_MAIN],
@@ -792,8 +791,8 @@ void GCTracer::PrintNVP() const {
           current_.scopes[Scope::MC_MARK_WEAK_CLOSURE_WEAK_HANDLES],
           current_.scopes[Scope::MC_MARK_WEAK_CLOSURE_WEAK_ROOTS],
           current_.scopes[Scope::MC_MARK_WEAK_CLOSURE_HARMONY],
-          current_.scopes[Scope::MC_MARK_WRAPPER_PROLOGUE],
-          current_.scopes[Scope::MC_MARK_WRAPPER_TRACING],
+          current_.scopes[Scope::MC_MARK_EMBEDDER_PROLOGUE],
+          current_.scopes[Scope::MC_MARK_EMBEDDER_TRACING],
           current_.scopes[Scope::MC_PROLOGUE], current_.scopes[Scope::MC_SWEEP],
           current_.scopes[Scope::MC_SWEEP_CODE],
           current_.scopes[Scope::MC_SWEEP_MAP],
@@ -804,10 +803,11 @@ void GCTracer::PrintNVP() const {
           current_.scopes[Scope::MC_INCREMENTAL_EXTERNAL_PROLOGUE],
           current_.scopes[Scope::MC_INCREMENTAL_EXTERNAL_EPILOGUE],
           current_.scopes[Scope::MC_INCREMENTAL_SWEEPING],
-          current_.scopes[Scope::MC_INCREMENTAL_WRAPPER_PROLOGUE],
-          current_.scopes[Scope::MC_INCREMENTAL_WRAPPER_TRACING],
+          current_.scopes[Scope::MC_INCREMENTAL_EMBEDDER_PROLOGUE],
+          current_.scopes[Scope::MC_INCREMENTAL_EMBEDDER_TRACING],
           current_
-              .incremental_marking_scopes[Scope::MC_INCREMENTAL_WRAPPER_TRACING]
+              .incremental_marking_scopes
+                  [Scope::MC_INCREMENTAL_EMBEDDER_TRACING]
               .longest_step,
           current_
               .incremental_marking_scopes[Scope::MC_INCREMENTAL_FINALIZE_BODY]

src/heap/heap.cc

@@ -1700,7 +1700,20 @@ bool Heap::PerformGarbageCollection(
     // nested GCs.
     freed_global_handles =
         isolate_->global_handles()->InvokeFirstPassWeakCallbacks();
+  }
+
+  {
+    TRACE_GC(tracer(), GCTracer::Scope::HEAP_EMBEDDER_TRACING_EPILOGUE);
+    // TraceEpilogue may trigger operations that invalidate global handles. It
+    // has to be called *after* all other operations that potentially touch and
+    // reset global handles. It is also still part of the main garbage
+    // collection pause and thus needs to be called *before* any operation that
+    // can potentially trigger recursive garbage
+    local_embedder_heap_tracer()->TraceEpilogue();
+  }
 
+  {
+    TRACE_GC(tracer(), GCTracer::Scope::HEAP_EXTERNAL_WEAK_GLOBAL_HANDLES);
     gc_post_processing_depth_++;
     {
       AllowHeapAllocation allow_allocation;

src/heap/incremental-marking.cc

@@ -386,7 +386,7 @@ void IncrementalMarking::StartMarking() {
 
   {
     TRACE_GC(heap()->tracer(),
-             GCTracer::Scope::MC_INCREMENTAL_WRAPPER_PROLOGUE);
+             GCTracer::Scope::MC_INCREMENTAL_EMBEDDER_PROLOGUE);
     heap_->local_embedder_heap_tracer()->TracePrologue();
   }
 
@@ -802,7 +802,7 @@ intptr_t IncrementalMarking::ProcessMarkingWorklist(
 void IncrementalMarking::EmbedderStep(double duration_ms) {
   constexpr int kObjectsToProcessBeforeInterrupt = 100;
 
-  TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_INCREMENTAL_WRAPPER_TRACING);
+  TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_INCREMENTAL_EMBEDDER_TRACING);
 
   const double deadline =
       heap_->MonotonicallyIncreasingTimeInMs() + duration_ms;

src/heap/mark-compact.cc

@@ -758,7 +758,7 @@ void MarkCompactCollector::Prepare() {
   heap()->memory_allocator()->unmapper()->PrepareForMarkCompact();
 
   if (!was_marked_incrementally_) {
-    TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK_WRAPPER_PROLOGUE);
+    TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK_EMBEDDER_PROLOGUE);
     heap_->local_embedder_heap_tracer()->TracePrologue();
   }
 
@@ -845,14 +845,6 @@ void MarkCompactCollector::Finish() {
     Deoptimizer::DeoptimizeMarkedCode(isolate());
     have_code_to_deoptimize_ = false;
   }
-
-  {
-    TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_FINISH_WRAPPER_EPILOGUE);
-    // TraceEpilogue may trigger operations that invalidate global handles. It
-    // has to be called *after* all other operations that potentially touch and
-    // reset global handles.
-    heap()->local_embedder_heap_tracer()->TraceEpilogue();
-  }
 }
 
 class MarkCompactCollector::RootMarkingVisitor final : public RootVisitor {
@@ -1610,7 +1602,7 @@ void MarkCompactCollector::ProcessEphemeronsLinear() {
 
 void MarkCompactCollector::PerformWrapperTracing() {
   if (heap_->local_embedder_heap_tracer()->InUse()) {
-    TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK_WRAPPER_TRACING);
+    TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK_EMBEDDER_TRACING);
     HeapObject* object;
     while (marking_worklist()->embedder()->Pop(kMainThread, &object)) {
       heap_->TracePossibleWrapper(JSObject::cast(object));
@@ -1768,7 +1760,8 @@ void MarkCompactCollector::MarkLiveObjects() {
     // opportunistic as it may not discover graphs that are only reachable
     // through ephemerons.
     {
-      TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK_WRAPPERS);
+      TRACE_GC(heap()->tracer(),
+               GCTracer::Scope::MC_MARK_EMBEDDER_TRACING_CLOSURE);
       do {
         // PerformWrapperTracing() also empties the work items collected by
         // concurrent markers. As a result this call needs to happen at least