[heap] Add GC epilogue callbacks in LocalHeap

A background thread can register a callback that is guaranteed to be
invoked after each GC in a safepoint before background threads resume.
This will be allow the background compiler and parser to keep raw
pointers to frequently accessed objects and ensure that they are fixed
up after GC.

Note that the existing global GC epilogues are run after background
threads resume, so they are unsafe for background threads.

Change-Id: I1c782f912d63afc09c4982d393a6f3805a318962
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2675933
Commit-Queue: Ulan Degenbaev <ulan@chromium.org>
Reviewed-by: Toon Verwaest <verwaest@chromium.org>
Reviewed-by: Dominik Inführ <dinfuehr@chromium.org>
Cr-Commit-Position: refs/heads/master@{#72548}

src/heap/heap.cc

@@ -1121,6 +1121,10 @@ void Heap::GarbageCollectionEpilogueInSafepoint(GarbageCollector collector) {
 
   TRACE_GC(tracer(), GCTracer::Scope::HEAP_EPILOGUE_SAFEPOINT);
 
+  safepoint()->IterateLocalHeaps([](LocalHeap* local_heap) {
+    local_heap->InvokeGCEpilogueCallbacksInSafepoint();
+  });
+
 #define UPDATE_COUNTERS_FOR_SPACE(space)                \
   isolate_->counters()->space##_bytes_available()->Set( \
       static_cast<int>(space()->Available()));          \

src/heap/local-heap.cc

@@ -85,6 +85,8 @@ LocalHeap::~LocalHeap() {
     DCHECK_EQ(current_local_heap, this);
     current_local_heap = nullptr;
   }
+
+  DCHECK(gc_epilogue_callbacks_.empty());
 }
 
 void LocalHeap::EnsurePersistentHandles() {
@@ -204,5 +206,29 @@ Address LocalHeap::PerformCollectionAndAllocateAgain(
   heap_->FatalProcessOutOfMemory("LocalHeap: allocation failed");
 }
 
+void LocalHeap::AddGCEpilogueCallback(GCEpilogueCallback* callback,
+                                      void* data) {
+  std::pair<GCEpilogueCallback*, void*> callback_and_data(callback, data);
+  DCHECK_EQ(std::find(gc_epilogue_callbacks_.begin(),
+                      gc_epilogue_callbacks_.end(), callback_and_data),
+            gc_epilogue_callbacks_.end());
+  gc_epilogue_callbacks_.push_back(callback_and_data);
+}
+
+void LocalHeap::RemoveGCEpilogueCallback(GCEpilogueCallback* callback,
+                                         void* data) {
+  std::pair<GCEpilogueCallback*, void*> callback_and_data(callback, data);
+  auto it = std::find(gc_epilogue_callbacks_.begin(),
+                      gc_epilogue_callbacks_.end(), callback_and_data);
+  *it = gc_epilogue_callbacks_.back();
+  gc_epilogue_callbacks_.pop_back();
+}
+
+void LocalHeap::InvokeGCEpilogueCallbacksInSafepoint() {
+  for (auto callback_and_data : gc_epilogue_callbacks_) {
+    callback_and_data.first(callback_and_data.second);
+  }
+}
+
 }  // namespace internal
 }  // namespace v8

src/heap/local-heap.h

@@ -33,6 +33,8 @@ class LocalHandles;
 //            some time or for blocking operations like locking a mutex.
 class V8_EXPORT_PRIVATE LocalHeap {
  public:
+  using GCEpilogueCallback = void(void* data);
+
   explicit LocalHeap(
       Heap* heap, ThreadKind kind,
       std::unique_ptr<PersistentHandles> persistent_handles = nullptr);
@@ -133,6 +135,13 @@ class V8_EXPORT_PRIVATE LocalHeap {
   // Requests GC and blocks until the collection finishes.
   void PerformCollection();
 
+  // Adds a callback that is invoked with the given |data| after each GC.
+  // The callback is invoked on the main thread before any background thread
+  // resumes. The callback must not allocate or make any other calls that
+  // can trigger GC.
+  void AddGCEpilogueCallback(GCEpilogueCallback* callback, void* data);
+  void RemoveGCEpilogueCallback(GCEpilogueCallback* callback, void* data);
+
  private:
   enum class ThreadState {
     // Threads in this state need to be stopped in a safepoint.
@@ -164,6 +173,8 @@ class V8_EXPORT_PRIVATE LocalHeap {
 
   void EnterSafepoint();
 
+  void InvokeGCEpilogueCallbacksInSafepoint();
+
   Heap* heap_;
   bool is_main_thread_;
 
@@ -182,6 +193,8 @@ class V8_EXPORT_PRIVATE LocalHeap {
   std::unique_ptr<PersistentHandles> persistent_handles_;
   std::unique_ptr<MarkingBarrier> marking_barrier_;
 
+  std::vector<std::pair<GCEpilogueCallback*, void*>> gc_epilogue_callbacks_;
+
   ConcurrentAllocator old_space_allocator_;
 
   friend class Heap;

test/unittests/heap/local-heap-unittest.cc

@@ -3,7 +3,11 @@
 // found in the LICENSE file.
 
 #include "src/heap/local-heap.h"
+
+#include "src/base/platform/condition-variable.h"
+#include "src/base/platform/mutex.h"
 #include "src/heap/heap.h"
+#include "src/heap/parked-scope.h"
 #include "src/heap/safepoint.h"
 #include "test/unittests/test-utils.h"
 #include "testing/gtest/include/gtest/gtest.h"
@@ -72,5 +76,100 @@ TEST_F(LocalHeapTest, CurrentBackground) {
   CHECK_NULL(LocalHeap::Current());
 }
 
+namespace {
+
+class GCEpilogue {
+ public:
+  static void Callback(void* data) {
+    reinterpret_cast<GCEpilogue*>(data)->was_invoked_ = true;
+  }
+
+  void NotifyStarted() {
+    base::LockGuard<base::Mutex> lock_guard(&mutex_);
+    started_ = true;
+    cv_.NotifyOne();
+  }
+
+  void WaitUntilStarted() {
+    base::LockGuard<base::Mutex> lock_guard(&mutex_);
+    while (!started_) {
+      cv_.Wait(&mutex_);
+    }
+  }
+  void RequestStop() {
+    base::LockGuard<base::Mutex> lock_guard(&mutex_);
+    stop_requested_ = true;
+  }
+
+  bool StopRequested() {
+    base::LockGuard<base::Mutex> lock_guard(&mutex_);
+    return stop_requested_;
+  }
+
+  bool WasInvoked() { return was_invoked_; }
+
+ private:
+  bool was_invoked_ = false;
+  bool started_ = false;
+  bool stop_requested_ = false;
+  base::Mutex mutex_;
+  base::ConditionVariable cv_;
+};
+
+class BackgroundThreadForGCEpilogue final : public v8::base::Thread {
+ public:
+  explicit BackgroundThreadForGCEpilogue(Heap* heap, bool parked,
+                                         GCEpilogue* epilogue)
+      : v8::base::Thread(base::Thread::Options("BackgroundThread")),
+        heap_(heap),
+        parked_(parked),
+        epilogue_(epilogue) {}
+
+  void Run() override {
+    LocalHeap lh(heap_, ThreadKind::kBackground);
+    base::Optional<UnparkedScope> unparked_scope;
+    if (!parked_) {
+      unparked_scope.emplace(&lh);
+    }
+    epilogue_->NotifyStarted();
+    lh.AddGCEpilogueCallback(&GCEpilogue::Callback, epilogue_);
+    while (!epilogue_->StopRequested()) {
+      lh.Safepoint();
+    }
+    lh.RemoveGCEpilogueCallback(&GCEpilogue::Callback, epilogue_);
+  }
+
+  Heap* heap_;
+  bool parked_;
+  GCEpilogue* epilogue_;
+};
+
+}  // anonymous namespace
+
+TEST_F(LocalHeapTest, GCEpilogue) {
+  Heap* heap = i_isolate()->heap();
+  LocalHeap lh(heap, ThreadKind::kMain);
+  std::array<GCEpilogue, 3> epilogue;
+  lh.AddGCEpilogueCallback(&GCEpilogue::Callback, &epilogue[0]);
+  auto thread1 =
+      std::make_unique<BackgroundThreadForGCEpilogue>(heap, true, &epilogue[1]);
+  auto thread2 = std::make_unique<BackgroundThreadForGCEpilogue>(heap, false,
+                                                                 &epilogue[2]);
+  CHECK(thread1->Start());
+  CHECK(thread2->Start());
+  epilogue[1].WaitUntilStarted();
+  epilogue[2].WaitUntilStarted();
+  heap->PreciseCollectAllGarbage(Heap::kNoGCFlags,
+                                 GarbageCollectionReason::kTesting);
+  epilogue[1].RequestStop();
+  epilogue[2].RequestStop();
+  thread1->Join();
+  thread2->Join();
+  lh.RemoveGCEpilogueCallback(&GCEpilogue::Callback, &epilogue[0]);
+  for (auto& e : epilogue) {
+    CHECK(e.WasInvoked());
+  }
+}
+
 }  // namespace internal
 }  // namespace v8