[heap] Use the thread-safe free modes also for RemoveRange in SlotSet.

BUG=chromium:648568

Review-Url: https://codereview.chromium.org/2397373002
Cr-Commit-Position: refs/heads/master@{#40075}

src/heap/heap.cc

@@ -5951,8 +5951,10 @@ void Heap::ClearRecordedSlotRange(Address start, Address end) {
   if (!page->InNewSpace()) {
     store_buffer()->MoveEntriesToRememberedSet();
     DCHECK_EQ(page->owner()->identity(), OLD_SPACE);
-    RememberedSet<OLD_TO_NEW>::RemoveRange(page, start, end);
-    RememberedSet<OLD_TO_OLD>::RemoveRange(page, start, end);
+    RememberedSet<OLD_TO_NEW>::RemoveRange(page, start, end,
+                                           SlotSet::PREFREE_EMPTY_BUCKETS);
+    RememberedSet<OLD_TO_OLD>::RemoveRange(page, start, end,
+                                           SlotSet::FREE_EMPTY_BUCKETS);
   }
 }
 

src/heap/mark-compact.cc

@@ -3480,7 +3480,8 @@ bool MarkCompactCollector::VisitLiveObjects(MemoryChunk* page, Visitor* visitor,
             page->AddressToMarkbitIndex(object->address()));
         if (page->old_to_new_slots() != nullptr) {
           page->old_to_new_slots()->RemoveRange(
-              0, static_cast<int>(object->address() - page->address()));
+              0, static_cast<int>(object->address() - page->address()),
+              SlotSet::PREFREE_EMPTY_BUCKETS);
         }
         if (page->typed_old_to_new_slots() != nullptr) {
           RememberedSet<OLD_TO_NEW>::RemoveRangeTyped(page, page->address(),

src/heap/remembered-set.h

@@ -45,7 +45,8 @@ class RememberedSet {
 
   // Given a page and a range of slots in that page, this function removes the
   // slots from the remembered set.
-  static void RemoveRange(MemoryChunk* chunk, Address start, Address end) {
+  static void RemoveRange(MemoryChunk* chunk, Address start, Address end,
+                          SlotSet::EmptyBucketMode mode) {
     SlotSet* slot_set = GetSlotSet(chunk);
     if (slot_set != nullptr) {
       uintptr_t start_offset = start - chunk->address();
@@ -53,7 +54,7 @@ class RememberedSet {
       DCHECK_LT(start_offset, end_offset);
       if (end_offset < static_cast<uintptr_t>(Page::kPageSize)) {
         slot_set->RemoveRange(static_cast<int>(start_offset),
-                              static_cast<int>(end_offset));
+                              static_cast<int>(end_offset), mode);
       } else {
         // The large page has multiple slot sets.
         // Compute slot set indicies for the range [start_offset, end_offset).
@@ -67,17 +68,17 @@ class RememberedSet {
             end_offset - static_cast<uintptr_t>(end_chunk) * Page::kPageSize);
         if (start_chunk == end_chunk) {
           slot_set[start_chunk].RemoveRange(offset_in_start_chunk,
-                                            offset_in_end_chunk);
+                                            offset_in_end_chunk, mode);
         } else {
           // Clear all slots from start_offset to the end of first chunk.
           slot_set[start_chunk].RemoveRange(offset_in_start_chunk,
-                                            Page::kPageSize);
+                                            Page::kPageSize, mode);
           // Clear all slots in intermediate chunks.
           for (int i = start_chunk + 1; i < end_chunk; i++) {
-            slot_set[i].RemoveRange(0, Page::kPageSize);
+            slot_set[i].RemoveRange(0, Page::kPageSize, mode);
           }
           // Clear slots from the beginning of the last page to end_offset.
-          slot_set[end_chunk].RemoveRange(0, offset_in_end_chunk);
+          slot_set[end_chunk].RemoveRange(0, offset_in_end_chunk, mode);
         }
       }
     }

src/heap/slot-set.h

@@ -25,7 +25,13 @@ enum SlotCallbackResult { KEEP_SLOT, REMOVE_SLOT };
 // Each bucket is a bitmap with a bit corresponding to a single slot offset.
 class SlotSet : public Malloced {
  public:
-  enum IterationMode { PREFREE_EMPTY_BUCKETS, KEEP_EMPTY_BUCKETS };
+  enum EmptyBucketMode {
+    FREE_EMPTY_BUCKETS,     // An empty bucket will be deallocated immediately.
+    PREFREE_EMPTY_BUCKETS,  // An empty bucket will be unlinked from the slot
+                            // set, but deallocated on demand by a sweeper
+                            // thread.
+    KEEP_EMPTY_BUCKETS      // An empty bucket will be kept.
+  };
 
   SlotSet() {
     for (int i = 0; i < kBuckets; i++) {
@@ -74,9 +80,16 @@ class SlotSet : public Malloced {
     }
   }
 
+  void PreFreeEmptyBucket(int bucket_index) {
+    base::LockGuard<base::Mutex> guard(&to_be_freed_buckets_mutex_);
+    base::AtomicValue<uint32_t>* bucket_ptr = bucket[bucket_index].Value();
+    to_be_freed_buckets_.push(bucket_ptr);
+    bucket[bucket_index].SetValue(nullptr);
+  }
+
   // The slot offsets specify a range of slots at addresses:
   // [page_start_ + start_offset ... page_start_ + end_offset).
-  void RemoveRange(int start_offset, int end_offset) {
+  void RemoveRange(int start_offset, int end_offset, EmptyBucketMode mode) {
     CHECK_LE(end_offset, 1 << kPageSizeBits);
     DCHECK_LE(start_offset, end_offset);
     int start_bucket, start_cell, start_bit;
@@ -108,7 +121,11 @@ class SlotSet : public Malloced {
     DCHECK(current_bucket == end_bucket ||
            (current_bucket < end_bucket && current_cell == 0));
     while (current_bucket < end_bucket) {
-      ReleaseBucket(current_bucket);
+      if (mode == PREFREE_EMPTY_BUCKETS) {
+        PreFreeEmptyBucket(current_bucket);
+      } else if (mode == FREE_EMPTY_BUCKETS) {
+        ReleaseBucket(current_bucket);
+      }
       current_bucket++;
     }
     // All buckets between start_bucket and end_bucket are cleared.
@@ -148,7 +165,7 @@ class SlotSet : public Malloced {
   //    else return REMOVE_SLOT;
   // });
   template <typename Callback>
-  int Iterate(Callback callback, IterationMode mode) {
+  int Iterate(Callback callback, EmptyBucketMode mode) {
     int new_count = 0;
     for (int bucket_index = 0; bucket_index < kBuckets; bucket_index++) {
       if (bucket[bucket_index].Value() != nullptr) {
@@ -186,11 +203,7 @@ class SlotSet : public Malloced {
           }
         }
         if (mode == PREFREE_EMPTY_BUCKETS && in_bucket_count == 0) {
-          base::LockGuard<base::Mutex> guard(&to_be_freed_buckets_mutex_);
-          base::AtomicValue<uint32_t>* bucket_ptr =
-              bucket[bucket_index].Value();
-          to_be_freed_buckets_.push(bucket_ptr);
-          bucket[bucket_index].SetValue(nullptr);
+          PreFreeEmptyBucket(bucket_index);
         }
         new_count += in_bucket_count;
       }

src/heap/spaces.cc

@@ -2905,8 +2905,10 @@ Address LargePage::GetAddressToShrink() {
 }
 
 void LargePage::ClearOutOfLiveRangeSlots(Address free_start) {
-  RememberedSet<OLD_TO_NEW>::RemoveRange(this, free_start, area_end());
-  RememberedSet<OLD_TO_OLD>::RemoveRange(this, free_start, area_end());
+  RememberedSet<OLD_TO_NEW>::RemoveRange(this, free_start, area_end(),
+                                         SlotSet::FREE_EMPTY_BUCKETS);
+  RememberedSet<OLD_TO_OLD>::RemoveRange(this, free_start, area_end(),
+                                         SlotSet::FREE_EMPTY_BUCKETS);
   RememberedSet<OLD_TO_NEW>::RemoveRangeTyped(this, free_start, area_end());
   RememberedSet<OLD_TO_OLD>::RemoveRangeTyped(this, free_start, area_end());
 }

test/cctest/heap/test-heap.cc

@@ -7037,7 +7037,8 @@ TEST(RememberedSetRemoveRange) {
     return KEEP_SLOT;
   });
 
-  RememberedSet<OLD_TO_NEW>::RemoveRange(chunk, start, start + kPointerSize);
+  RememberedSet<OLD_TO_NEW>::RemoveRange(chunk, start, start + kPointerSize,
+                                         SlotSet::FREE_EMPTY_BUCKETS);
   slots[start] = false;
   RememberedSet<OLD_TO_NEW>::Iterate(chunk, [&slots](Address addr) {
     CHECK(slots[addr]);
@@ -7045,7 +7046,8 @@ TEST(RememberedSetRemoveRange) {
   });
 
   RememberedSet<OLD_TO_NEW>::RemoveRange(chunk, start + kPointerSize,
-                                         start + Page::kPageSize);
+                                         start + Page::kPageSize,
+                                         SlotSet::FREE_EMPTY_BUCKETS);
   slots[start + kPointerSize] = false;
   slots[start + Page::kPageSize - kPointerSize] = false;
   RememberedSet<OLD_TO_NEW>::Iterate(chunk, [&slots](Address addr) {
@@ -7053,8 +7055,9 @@ TEST(RememberedSetRemoveRange) {
     return KEEP_SLOT;
   });
 
-  RememberedSet<OLD_TO_NEW>::RemoveRange(
-      chunk, start, start + Page::kPageSize + kPointerSize);
+  RememberedSet<OLD_TO_NEW>::RemoveRange(chunk, start,
+                                         start + Page::kPageSize + kPointerSize,
+                                         SlotSet::FREE_EMPTY_BUCKETS);
   slots[start + Page::kPageSize] = false;
   RememberedSet<OLD_TO_NEW>::Iterate(chunk, [&slots](Address addr) {
     CHECK(slots[addr]);
@@ -7062,7 +7065,8 @@ TEST(RememberedSetRemoveRange) {
   });
 
   RememberedSet<OLD_TO_NEW>::RemoveRange(
-      chunk, chunk->area_end() - kPointerSize, chunk->area_end());
+      chunk, chunk->area_end() - kPointerSize, chunk->area_end(),
+      SlotSet::FREE_EMPTY_BUCKETS);
   slots[chunk->area_end() - kPointerSize] = false;
   RememberedSet<OLD_TO_NEW>::Iterate(chunk, [&slots](Address addr) {
     CHECK(slots[addr]);

test/unittests/heap/slot-set-unittest.cc

@@ -104,7 +104,7 @@ void CheckRemoveRangeOn(uint32_t start, uint32_t end) {
   for (uint32_t i = first; i <= last; i += kPointerSize) {
     set.Insert(i);
   }
-  set.RemoveRange(start, end);
+  set.RemoveRange(start, end, SlotSet::FREE_EMPTY_BUCKETS);
   if (first != start) {
     EXPECT_TRUE(set.Lookup(first));
   }
@@ -135,7 +135,7 @@ TEST(SlotSet, RemoveRange) {
   SlotSet set;
   set.SetPageStart(0);
   set.Insert(Page::kPageSize / 2);
-  set.RemoveRange(0, Page::kPageSize);
+  set.RemoveRange(0, Page::kPageSize, SlotSet::FREE_EMPTY_BUCKETS);
   for (uint32_t i = 0; i < Page::kPageSize; i += kPointerSize) {
     EXPECT_FALSE(set.Lookup(i));
   }