[heap] Add linear ephemeron marking

After a certain number of iterations in the fixpoint
iteration switch to a linear algorithm. This
algorithm uses a key-to-values map for weak collections
contents to avoid checking all EphemeronHashTables
again.

Bug: chromium:844008
Change-Id: I044fede5911e0a780d088d1ba2bb5343317d9b7a
Reviewed-on: https://chromium-review.googlesource.com/1105835
Commit-Queue: Dominik Inführ <dinfuehr@google.com>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#54232}

src/flag-definitions.h

@@ -704,6 +704,9 @@ DEFINE_BOOL(parallel_ephemeron_visiting, true,
 DEFINE_BOOL(
     parallel_ephemeron_marking, true,
     "use parallel marking of objects after visiting ephemerons in atomic pause")
+DEFINE_INT(ephemeron_fixpoint_iterations, 0,
+           "number of fixpoint iterations it takes to switch to linear "
+           "ephemeron algorithm")
 DEFINE_BOOL(trace_concurrent_marking, false, "trace concurrent marking")
 DEFINE_BOOL(black_allocation, true, "use black allocation")
 DEFINE_BOOL(concurrent_store_buffer, true,

src/heap-symbols.h

@@ -336,6 +336,7 @@
   F(MC_MARK_WEAK_CLOSURE)                            \
   F(MC_MARK_WEAK_CLOSURE_EPHEMERON)                  \
   F(MC_MARK_WEAK_CLOSURE_EPHEMERON_MARKING)          \
+  F(MC_MARK_WEAK_CLOSURE_EPHEMERON_LINEAR)           \
   F(MC_MARK_WEAK_CLOSURE_WEAK_HANDLES)               \
   F(MC_MARK_WEAK_CLOSURE_WEAK_ROOTS)                 \
   F(MC_MARK_WEAK_CLOSURE_HARMONY)                    \

src/heap/gc-tracer.cc

@@ -699,6 +699,7 @@ void GCTracer::PrintNVP() const {
           "mark.weak_closure=%.1f "
           "mark.weak_closure.ephemeron=%.1f "
           "mark.weak_closure.ephemeron.marking=%.1f "
+          "mark.weak_closure.ephemeron.linear=%.1f "
           "mark.weak_closure.weak_handles=%.1f "
           "mark.weak_closure.weak_roots=%.1f "
           "mark.weak_closure.harmony=%.1f "
@@ -794,6 +795,7 @@ void GCTracer::PrintNVP() const {
           current_.scopes[Scope::MC_MARK_WEAK_CLOSURE],
           current_.scopes[Scope::MC_MARK_WEAK_CLOSURE_EPHEMERON],
           current_.scopes[Scope::MC_MARK_WEAK_CLOSURE_EPHEMERON_MARKING],
+          current_.scopes[Scope::MC_MARK_WEAK_CLOSURE_EPHEMERON_LINEAR],
           current_.scopes[Scope::MC_MARK_WEAK_CLOSURE_WEAK_HANDLES],
           current_.scopes[Scope::MC_MARK_WEAK_CLOSURE_WEAK_ROOTS],
           current_.scopes[Scope::MC_MARK_WEAK_CLOSURE_HARMONY],

src/heap/mark-compact.cc

@@ -1470,14 +1470,16 @@ void MarkCompactCollector::MarkRoots(RootVisitor* root_visitor,
 
 void MarkCompactCollector::ProcessEphemeronsUntilFixpoint() {
   bool work_to_do = true;
+  int iterations = 0;
+  int max_iterations = FLAG_ephemeron_fixpoint_iterations;
 
   while (work_to_do) {
-    if (heap_->local_embedder_heap_tracer()->InUse()) {
-      TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK_WRAPPER_TRACING);
-      heap_->local_embedder_heap_tracer()->RegisterWrappersWithRemoteTracer();
-      heap_->local_embedder_heap_tracer()->Trace(
-          0, EmbedderHeapTracer::AdvanceTracingActions(
-                 EmbedderHeapTracer::ForceCompletionAction::FORCE_COMPLETION));
+    PerformWrapperTracing();
+
+    if (iterations >= max_iterations) {
+      // Give up fixpoint iteration and switch to linear algorithm.
+      ProcessEphemeronsLinear();
+      break;
     }
 
     // Move ephemerons from next_ephemerons into current_ephemerons to
@@ -1506,6 +1508,7 @@ void MarkCompactCollector::ProcessEphemeronsUntilFixpoint() {
         work_to_do || !marking_worklist()->IsEmpty() ||
         heap()->concurrent_marking()->ephemeron_marked() ||
         heap()->local_embedder_heap_tracer()->NumberOfWrappersToTrace() > 0;
+    ++iterations;
   }
 
   CHECK(marking_worklist()->IsEmpty());
@@ -1539,12 +1542,113 @@ bool MarkCompactCollector::ProcessEphemerons() {
   }
 
   // Flush local ephemerons for main task to global pool.
+  weak_objects_.ephemeron_hash_tables.FlushToGlobal(kMainThread);
   weak_objects_.next_ephemerons.FlushToGlobal(kMainThread);
 
   return ephemeron_marked;
 }
 
+void MarkCompactCollector::ProcessEphemeronsLinear() {
+  TRACE_GC(heap()->tracer(),
+           GCTracer::Scope::MC_MARK_WEAK_CLOSURE_EPHEMERON_LINEAR);
+  CHECK(heap()->concurrent_marking()->IsStopped());
+  std::unordered_multimap<HeapObject*, HeapObject*> key_to_values;
+  Ephemeron ephemeron;
+
+  DCHECK(weak_objects_.current_ephemerons.IsEmpty());
+  weak_objects_.current_ephemerons.Swap(weak_objects_.next_ephemerons);
+
+  while (weak_objects_.current_ephemerons.Pop(kMainThread, &ephemeron)) {
+    VisitEphemeron(ephemeron.key, ephemeron.value);
+
+    if (non_atomic_marking_state()->IsWhite(ephemeron.value)) {
+      key_to_values.insert(std::make_pair(ephemeron.key, ephemeron.value));
+    }
+  }
+
+  ephemeron_marking_.newly_discovered_limit = key_to_values.size();
+  bool work_to_do = true;
+
+  while (work_to_do) {
+    PerformWrapperTracing();
+
+    ResetNewlyDiscovered();
+    ephemeron_marking_.newly_discovered_limit = key_to_values.size();
+
+    {
+      TRACE_GC(heap()->tracer(),
+               GCTracer::Scope::MC_MARK_WEAK_CLOSURE_EPHEMERON_MARKING);
+      // Drain marking worklist and push all discovered objects into
+      // newly_discovered.
+      ProcessMarkingWorklistInternal<
+          MarkCompactCollector::MarkingWorklistProcessingMode::
+              kTrackNewlyDiscoveredObjects>();
+    }
+
+    while (weak_objects_.discovered_ephemerons.Pop(kMainThread, &ephemeron)) {
+      VisitEphemeron(ephemeron.key, ephemeron.value);
+
+      if (non_atomic_marking_state()->IsWhite(ephemeron.value)) {
+        key_to_values.insert(std::make_pair(ephemeron.key, ephemeron.value));
+      }
+    }
+
+    if (ephemeron_marking_.newly_discovered_overflowed) {
+      // If newly_discovered was overflowed just visit all ephemerons in
+      // next_ephemerons.
+      weak_objects_.next_ephemerons.Iterate([&](Ephemeron ephemeron) {
+        if (non_atomic_marking_state()->IsBlackOrGrey(ephemeron.key) &&
+            non_atomic_marking_state()->WhiteToGrey(ephemeron.value)) {
+          marking_worklist()->Push(ephemeron.value);
+        }
+      });
+
+    } else {
+      // This is the good case: newly_discovered stores all discovered
+      // objects. Now use key_to_values to see if discovered objects keep more
+      // objects alive due to ephemeron semantics.
+      for (HeapObject* object : ephemeron_marking_.newly_discovered) {
+        auto range = key_to_values.equal_range(object);
+        for (auto it = range.first; it != range.second; ++it) {
+          HeapObject* value = it->second;
+          MarkObject(object, value);
+        }
+      }
+    }
+
+    // Do NOT drain marking worklist here, otherwise the current checks
+    // for work_to_do are not sufficient for determining if another iteration
+    // is necessary.
+
+    work_to_do =
+        !marking_worklist()->IsEmpty() ||
+        heap()->local_embedder_heap_tracer()->NumberOfWrappersToTrace() > 0;
+    CHECK(weak_objects_.discovered_ephemerons.IsEmpty());
+  }
+
+  ResetNewlyDiscovered();
+  ephemeron_marking_.newly_discovered.shrink_to_fit();
+
+  CHECK(marking_worklist()->IsEmpty());
+}
+
+void MarkCompactCollector::PerformWrapperTracing() {
+  if (heap_->local_embedder_heap_tracer()->InUse()) {
+    TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK_WRAPPER_TRACING);
+    heap_->local_embedder_heap_tracer()->RegisterWrappersWithRemoteTracer();
+    heap_->local_embedder_heap_tracer()->Trace(
+        0, EmbedderHeapTracer::AdvanceTracingActions(
+               EmbedderHeapTracer::ForceCompletionAction::FORCE_COMPLETION));
+  }
+}
+
 void MarkCompactCollector::ProcessMarkingWorklist() {
+  ProcessMarkingWorklistInternal<
+      MarkCompactCollector::MarkingWorklistProcessingMode::kDefault>();
+}
+
+template <MarkCompactCollector::MarkingWorklistProcessingMode mode>
+void MarkCompactCollector::ProcessMarkingWorklistInternal() {
   HeapObject* object;
   MarkCompactMarkingVisitor visitor(this, marking_state());
   while ((object = marking_worklist()->Pop()) != nullptr) {
@@ -1553,6 +1657,10 @@ void MarkCompactCollector::ProcessMarkingWorklist() {
     DCHECK(heap()->Contains(object));
     DCHECK(!(marking_state()->IsWhite(object)));
     marking_state()->GreyToBlack(object);
+    if (mode == MarkCompactCollector::MarkingWorklistProcessingMode::
+                    kTrackNewlyDiscoveredObjects) {
+      AddNewlyDiscovered(object);
+    }
     Map* map = object->map();
     MarkObject(object, map);
     visitor.Visit(map, object);
@@ -1955,6 +2063,15 @@ void MarkCompactCollector::ClearWeakCollections() {
   while (weak_objects_.ephemeron_hash_tables.Pop(kMainThread, &table)) {
     for (int i = 0; i < table->Capacity(); i++) {
       HeapObject* key = HeapObject::cast(table->KeyAt(i));
+#ifdef VERIFY_HEAP
+      Object* value = table->ValueAt(i);
+
+      if (value->IsHeapObject()) {
+        CHECK_IMPLIES(
+            non_atomic_marking_state()->IsBlackOrGrey(key),
+            non_atomic_marking_state()->IsBlackOrGrey(HeapObject::cast(value)));
+      }
+#endif
       if (!non_atomic_marking_state()->IsBlackOrGrey(key)) {
         table->RemoveEntry(i);
       }

src/heap/mark-compact.h

@@ -451,6 +451,12 @@ struct WeakObjects {
   Worklist<std::pair<HeapObject*, Code*>, 64> weak_objects_in_code;
 };
 
+struct EphemeronMarking {
+  std::vector<HeapObject*> newly_discovered;
+  bool newly_discovered_overflowed;
+  size_t newly_discovered_limit;
+};
+
 // Collector for young and old generation.
 class MarkCompactCollector final : public MarkCompactCollectorBase {
  public:
@@ -667,6 +673,22 @@ class MarkCompactCollector final : public MarkCompactCollectorBase {
                                             std::make_pair(object, code));
   }
 
+  void AddNewlyDiscovered(HeapObject* object) {
+    if (ephemeron_marking_.newly_discovered_overflowed) return;
+
+    if (ephemeron_marking_.newly_discovered.size() <
+        ephemeron_marking_.newly_discovered_limit) {
+      ephemeron_marking_.newly_discovered.push_back(object);
+    } else {
+      ephemeron_marking_.newly_discovered_overflowed = true;
+    }
+  }
+
+  void ResetNewlyDiscovered() {
+    ephemeron_marking_.newly_discovered_overflowed = false;
+    ephemeron_marking_.newly_discovered.clear();
+  }
+
   Sweeper* sweeper() { return sweeper_; }
 
 #ifdef DEBUG
@@ -735,6 +757,14 @@ class MarkCompactCollector final : public MarkCompactCollectorBase {
   // if no concurrent threads are running.
   void ProcessMarkingWorklist() override;
 
+  enum class MarkingWorklistProcessingMode {
+    kDefault,
+    kTrackNewlyDiscoveredObjects
+  };
+
+  template <MarkingWorklistProcessingMode mode>
+  void ProcessMarkingWorklistInternal();
+
   // Implements ephemeron semantics: Marks value if key is already reachable.
   // Returns true if value was actually marked.
   bool VisitEphemeron(HeapObject* key, HeapObject* value);
@@ -747,6 +777,13 @@ class MarkCompactCollector final : public MarkCompactCollectorBase {
   // fixpoint iteration.
   bool ProcessEphemerons();
 
+  // Mark ephemerons and drain marking worklist with a linear algorithm.
+  // Only used if fixpoint iteration doesn't finish within a few iterations.
+  void ProcessEphemeronsLinear();
+
+  // Perform Wrapper Tracing if in use.
+  void PerformWrapperTracing();
+
   // Callback function for telling whether the object *p is an unmarked
   // heap object.
   static bool IsUnmarkedHeapObject(Heap* heap, Object** p);
@@ -768,11 +805,6 @@ class MarkCompactCollector final : public MarkCompactCollectorBase {
   void TrimDescriptorArray(Map* map, DescriptorArray* descriptors);
   void TrimEnumCache(Map* map, DescriptorArray* descriptors);
 
-  // Mark all values associated with reachable keys in weak collections
-  // encountered so far.  This might push new object or even new weak maps onto
-  // the marking stack.
-  void ProcessWeakCollections();
-
   // After all reachable objects have been marked those weak map entries
   // with an unreachable key are removed from all encountered weak maps.
   // The linked list of all encountered weak maps is destroyed.
@@ -848,6 +880,7 @@ class MarkCompactCollector final : public MarkCompactCollectorBase {
 
   MarkingWorklist marking_worklist_;
   WeakObjects weak_objects_;
+  EphemeronMarking ephemeron_marking_;
 
   // Candidates for pages that should be evacuated.
   std::vector<Page*> evacuation_candidates_;