[heap] Introduce separate pass for pre-freeing empty buckets in SlotSet

This will allow us to concurrently allocate buckets during iteration.

Bug: chromium:738865
Change-Id: I88bd1ac152d1ef8b40395f0ba3e55a7c3e82f75d
Reviewed-on: https://chromium-review.googlesource.com/575990
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#46737}

src/heap/heap.cc

@@ -1755,9 +1755,11 @@ void Heap::Scavenge() {
     for (MemoryChunk* chunk : pages) {
       base::LockGuard<base::RecursiveMutex> guard(chunk->mutex());
       RememberedSet<OLD_TO_NEW>::Iterate(
-          chunk, [this, &scavenger](Address addr) {
+          chunk,
+          [this, &scavenger](Address addr) {
             return scavenger.CheckAndScavengeObject(this, addr);
-          });
+          },
+          SlotSet::KEEP_EMPTY_BUCKETS);
       RememberedSet<OLD_TO_NEW>::IterateTyped(
           chunk,
           [this, &scavenger](SlotType type, Address host_addr, Address addr) {
@@ -1806,6 +1808,10 @@ void Heap::Scavenge() {
 
   ArrayBufferTracker::FreeDeadInNewSpace(this);
 
+  RememberedSet<OLD_TO_NEW>::IterateMemoryChunks(this, [](MemoryChunk* chunk) {
+    RememberedSet<OLD_TO_NEW>::PreFreeEmptyBuckets(chunk);
+  });
+
   // Update how much has survived scavenge.
   DCHECK_GE(PromotedSpaceSizeOfObjects(), survived_watermark);
   IncrementYoungSurvivorsCounter(PromotedSpaceSizeOfObjects() +
@@ -4970,12 +4976,14 @@ template <RememberedSetType direction>
 void CollectSlots(MemoryChunk* chunk, Address start, Address end,
                   std::set<Address>* untyped,
                   std::set<std::pair<SlotType, Address> >* typed) {
-  RememberedSet<direction>::Iterate(chunk, [start, end, untyped](Address slot) {
-    if (start <= slot && slot < end) {
-      untyped->insert(slot);
-    }
-    return KEEP_SLOT;
-  });
+  RememberedSet<direction>::Iterate(chunk,
+                                    [start, end, untyped](Address slot) {
+                                      if (start <= slot && slot < end) {
+                                        untyped->insert(slot);
+                                      }
+                                      return KEEP_SLOT;
+                                    },
+                                    SlotSet::PREFREE_EMPTY_BUCKETS);
   RememberedSet<direction>::IterateTyped(
       chunk, [start, end, typed](SlotType type, Address host, Address slot) {
         if (start <= slot && slot < end) {

src/heap/mark-compact.cc

@@ -2325,9 +2325,10 @@ class PageMarkingItem : public MarkingItem {
   inline Heap* heap() { return chunk_->heap(); }
 
   void MarkUntypedPointers(YoungGenerationMarkingTask* task) {
-    RememberedSet<OLD_TO_NEW>::Iterate(chunk_, [this, task](Address slot) {
-      return CheckAndMarkObject(task, slot);
-    });
+    RememberedSet<OLD_TO_NEW>::Iterate(
+        chunk_,
+        [this, task](Address slot) { return CheckAndMarkObject(task, slot); },
+        SlotSet::PREFREE_EMPTY_BUCKETS);
   }
 
   void MarkTypedPointers(YoungGenerationMarkingTask* task) {
@@ -4260,27 +4261,39 @@ class RememberedSetUpdatingItem : public UpdatingItem {
     // those slots using atomics.
     if (chunk_->slot_set<OLD_TO_NEW, AccessMode::NON_ATOMIC>() != nullptr) {
       if (chunk_->owner() == heap_->map_space()) {
-        RememberedSet<OLD_TO_NEW>::Iterate(chunk_, [this](Address slot) {
-          return CheckAndUpdateOldToNewSlot<AccessMode::ATOMIC>(slot);
-        });
+        RememberedSet<OLD_TO_NEW>::Iterate(
+            chunk_,
+            [this](Address slot) {
+              return CheckAndUpdateOldToNewSlot<AccessMode::ATOMIC>(slot);
+            },
+            SlotSet::PREFREE_EMPTY_BUCKETS);
       } else {
-        RememberedSet<OLD_TO_NEW>::Iterate(chunk_, [this](Address slot) {
-          return CheckAndUpdateOldToNewSlot<AccessMode::NON_ATOMIC>(slot);
-        });
+        RememberedSet<OLD_TO_NEW>::Iterate(
+            chunk_,
+            [this](Address slot) {
+              return CheckAndUpdateOldToNewSlot<AccessMode::NON_ATOMIC>(slot);
+            },
+            SlotSet::PREFREE_EMPTY_BUCKETS);
       }
     }
     if ((updating_mode_ == RememberedSetUpdatingMode::ALL) &&
         (chunk_->slot_set<OLD_TO_OLD, AccessMode::NON_ATOMIC>() != nullptr)) {
       if (chunk_->owner() == heap_->map_space()) {
-        RememberedSet<OLD_TO_OLD>::Iterate(chunk_, [](Address slot) {
-          return UpdateSlot<AccessMode::ATOMIC>(
-              reinterpret_cast<Object**>(slot));
-        });
+        RememberedSet<OLD_TO_OLD>::Iterate(
+            chunk_,
+            [](Address slot) {
+              return UpdateSlot<AccessMode::ATOMIC>(
+                  reinterpret_cast<Object**>(slot));
+            },
+            SlotSet::PREFREE_EMPTY_BUCKETS);
       } else {
-        RememberedSet<OLD_TO_OLD>::Iterate(chunk_, [](Address slot) {
-          return UpdateSlot<AccessMode::NON_ATOMIC>(
-              reinterpret_cast<Object**>(slot));
-        });
+        RememberedSet<OLD_TO_OLD>::Iterate(
+            chunk_,
+            [](Address slot) {
+              return UpdateSlot<AccessMode::NON_ATOMIC>(
+                  reinterpret_cast<Object**>(slot));
+            },
+            SlotSet::PREFREE_EMPTY_BUCKETS);
       }
     }
   }

src/heap/remembered-set.h

@@ -129,15 +129,18 @@ class RememberedSet : public AllStatic {
   // Iterates and filters the remembered set in the given memory chunk with
   // the given callback. The callback should take (Address slot) and return
   // SlotCallbackResult.
+  //
+  // Notice that |mode| can only be of FREE* or PREFREE* if there are no other
+  // threads concurrently inserting slots.
   template <typename Callback>
-  static void Iterate(MemoryChunk* chunk, Callback callback) {
+  static void Iterate(MemoryChunk* chunk, Callback callback,
+                      SlotSet::EmptyBucketMode mode) {
     SlotSet* slots = chunk->slot_set<type>();
     if (slots != nullptr) {
       size_t pages = (chunk->size() + Page::kPageSize - 1) / Page::kPageSize;
       int new_count = 0;
       for (size_t page = 0; page < pages; page++) {
-        new_count +=
-            slots[page].Iterate(callback, SlotSet::PREFREE_EMPTY_BUCKETS);
+        new_count += slots[page].Iterate(callback, mode);
       }
       // Only old-to-old slot sets are released eagerly. Old-new-slot sets are
       // released by the sweeper threads.
@@ -147,6 +150,17 @@ class RememberedSet : public AllStatic {
     }
   }
 
+  static void PreFreeEmptyBuckets(MemoryChunk* chunk) {
+    DCHECK(type == OLD_TO_NEW);
+    SlotSet* slots = chunk->slot_set<type>();
+    if (slots != nullptr) {
+      size_t pages = (chunk->size() + Page::kPageSize - 1) / Page::kPageSize;
+      for (size_t page = 0; page < pages; page++) {
+        slots[page].PreFreeEmptyBuckets();
+      }
+    }
+  }
+
   // Given a page and a typed slot in that page, this function adds the slot
   // to the remembered set.
   static void InsertTyped(Page* page, Address host_addr, SlotType slot_type,

src/heap/slot-set.h

@@ -225,6 +225,25 @@ class SlotSet : public Malloced {
     return new_count;
   }
 
+  void PreFreeEmptyBuckets() {
+    for (int bucket_index = 0; bucket_index < kBuckets; bucket_index++) {
+      Bucket bucket = LoadBucket(&buckets_[bucket_index]);
+      if (bucket != nullptr) {
+        bool found_non_empty_cell = false;
+        int cell_offset = bucket_index * kBitsPerBucket;
+        for (int i = 0; i < kCellsPerBucket; i++, cell_offset += kBitsPerCell) {
+          if (LoadCell(&bucket[i])) {
+            found_non_empty_cell = true;
+            break;
+          }
+        }
+        if (!found_non_empty_cell) {
+          PreFreeEmptyBucket(bucket_index);
+        }
+      }
+    }
+  }
+
   void FreeToBeFreedBuckets() {
     base::LockGuard<base::Mutex> guard(&to_be_freed_buckets_mutex_);
     while (!to_be_freed_buckets_.empty()) {

test/cctest/heap/test-heap.cc

@@ -6100,46 +6100,56 @@ TEST(RememberedSetRemoveRange) {
     RememberedSet<OLD_TO_NEW>::Insert(chunk, x.first);
   }
 
-  RememberedSet<OLD_TO_NEW>::Iterate(chunk, [&slots](Address addr) {
-    CHECK(slots[addr]);
-    return KEEP_SLOT;
-  });
+  RememberedSet<OLD_TO_NEW>::Iterate(chunk,
+                                     [&slots](Address addr) {
+                                       CHECK(slots[addr]);
+                                       return KEEP_SLOT;
+                                     },
+                                     SlotSet::PREFREE_EMPTY_BUCKETS);
 
   RememberedSet<OLD_TO_NEW>::RemoveRange(chunk, start, start + kPointerSize,
                                          SlotSet::FREE_EMPTY_BUCKETS);
   slots[start] = false;
-  RememberedSet<OLD_TO_NEW>::Iterate(chunk, [&slots](Address addr) {
-    CHECK(slots[addr]);
-    return KEEP_SLOT;
-  });
+  RememberedSet<OLD_TO_NEW>::Iterate(chunk,
+                                     [&slots](Address addr) {
+                                       CHECK(slots[addr]);
+                                       return KEEP_SLOT;
+                                     },
+                                     SlotSet::PREFREE_EMPTY_BUCKETS);
 
   RememberedSet<OLD_TO_NEW>::RemoveRange(chunk, start + kPointerSize,
                                          start + Page::kPageSize,
                                          SlotSet::FREE_EMPTY_BUCKETS);
   slots[start + kPointerSize] = false;
   slots[start + Page::kPageSize - kPointerSize] = false;
-  RememberedSet<OLD_TO_NEW>::Iterate(chunk, [&slots](Address addr) {
-    CHECK(slots[addr]);
-    return KEEP_SLOT;
-  });
+  RememberedSet<OLD_TO_NEW>::Iterate(chunk,
+                                     [&slots](Address addr) {
+                                       CHECK(slots[addr]);
+                                       return KEEP_SLOT;
+                                     },
+                                     SlotSet::PREFREE_EMPTY_BUCKETS);
 
   RememberedSet<OLD_TO_NEW>::RemoveRange(chunk, start,
                                          start + Page::kPageSize + kPointerSize,
                                          SlotSet::FREE_EMPTY_BUCKETS);
   slots[start + Page::kPageSize] = false;
-  RememberedSet<OLD_TO_NEW>::Iterate(chunk, [&slots](Address addr) {
-    CHECK(slots[addr]);
-    return KEEP_SLOT;
-  });
+  RememberedSet<OLD_TO_NEW>::Iterate(chunk,
+                                     [&slots](Address addr) {
+                                       CHECK(slots[addr]);
+                                       return KEEP_SLOT;
+                                     },
+                                     SlotSet::PREFREE_EMPTY_BUCKETS);
 
   RememberedSet<OLD_TO_NEW>::RemoveRange(
       chunk, chunk->area_end() - kPointerSize, chunk->area_end(),
       SlotSet::FREE_EMPTY_BUCKETS);
   slots[chunk->area_end() - kPointerSize] = false;
-  RememberedSet<OLD_TO_NEW>::Iterate(chunk, [&slots](Address addr) {
-    CHECK(slots[addr]);
-    return KEEP_SLOT;
-  });
+  RememberedSet<OLD_TO_NEW>::Iterate(chunk,
+                                     [&slots](Address addr) {
+                                       CHECK(slots[addr]);
+                                       return KEEP_SLOT;
+                                     },
+                                     SlotSet::PREFREE_EMPTY_BUCKETS);
 }
 
 HEAP_TEST(Regress670675) {