Commit beb4037a authored by Michael Lippautz's avatar Michael Lippautz Committed by Commit Bot

[heap] Introduce separate pass for pre-freeing empty buckets in SlotSet

This will allow us to concurrently allocate buckets during iteration.

Bug: chromium:738865
Change-Id: I88bd1ac152d1ef8b40395f0ba3e55a7c3e82f75d
Reviewed-on: https://chromium-review.googlesource.com/575990
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: 's avatarUlan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#46737}
parent cd7380bb
...@@ -1755,9 +1755,11 @@ void Heap::Scavenge() { ...@@ -1755,9 +1755,11 @@ void Heap::Scavenge() {
for (MemoryChunk* chunk : pages) { for (MemoryChunk* chunk : pages) {
base::LockGuard<base::RecursiveMutex> guard(chunk->mutex()); base::LockGuard<base::RecursiveMutex> guard(chunk->mutex());
RememberedSet<OLD_TO_NEW>::Iterate( RememberedSet<OLD_TO_NEW>::Iterate(
chunk, [this, &scavenger](Address addr) { chunk,
[this, &scavenger](Address addr) {
return scavenger.CheckAndScavengeObject(this, addr); return scavenger.CheckAndScavengeObject(this, addr);
}); },
SlotSet::KEEP_EMPTY_BUCKETS);
RememberedSet<OLD_TO_NEW>::IterateTyped( RememberedSet<OLD_TO_NEW>::IterateTyped(
chunk, chunk,
[this, &scavenger](SlotType type, Address host_addr, Address addr) { [this, &scavenger](SlotType type, Address host_addr, Address addr) {
...@@ -1806,6 +1808,10 @@ void Heap::Scavenge() { ...@@ -1806,6 +1808,10 @@ void Heap::Scavenge() {
ArrayBufferTracker::FreeDeadInNewSpace(this); ArrayBufferTracker::FreeDeadInNewSpace(this);
RememberedSet<OLD_TO_NEW>::IterateMemoryChunks(this, [](MemoryChunk* chunk) {
RememberedSet<OLD_TO_NEW>::PreFreeEmptyBuckets(chunk);
});
// Update how much has survived scavenge. // Update how much has survived scavenge.
DCHECK_GE(PromotedSpaceSizeOfObjects(), survived_watermark); DCHECK_GE(PromotedSpaceSizeOfObjects(), survived_watermark);
IncrementYoungSurvivorsCounter(PromotedSpaceSizeOfObjects() + IncrementYoungSurvivorsCounter(PromotedSpaceSizeOfObjects() +
...@@ -4970,12 +4976,14 @@ template <RememberedSetType direction> ...@@ -4970,12 +4976,14 @@ template <RememberedSetType direction>
void CollectSlots(MemoryChunk* chunk, Address start, Address end, void CollectSlots(MemoryChunk* chunk, Address start, Address end,
std::set<Address>* untyped, std::set<Address>* untyped,
std::set<std::pair<SlotType, Address> >* typed) { std::set<std::pair<SlotType, Address> >* typed) {
RememberedSet<direction>::Iterate(chunk, [start, end, untyped](Address slot) { RememberedSet<direction>::Iterate(chunk,
[start, end, untyped](Address slot) {
if (start <= slot && slot < end) { if (start <= slot && slot < end) {
untyped->insert(slot); untyped->insert(slot);
} }
return KEEP_SLOT; return KEEP_SLOT;
}); },
SlotSet::PREFREE_EMPTY_BUCKETS);
RememberedSet<direction>::IterateTyped( RememberedSet<direction>::IterateTyped(
chunk, [start, end, typed](SlotType type, Address host, Address slot) { chunk, [start, end, typed](SlotType type, Address host, Address slot) {
if (start <= slot && slot < end) { if (start <= slot && slot < end) {
......
...@@ -2325,9 +2325,10 @@ class PageMarkingItem : public MarkingItem { ...@@ -2325,9 +2325,10 @@ class PageMarkingItem : public MarkingItem {
inline Heap* heap() { return chunk_->heap(); } inline Heap* heap() { return chunk_->heap(); }
void MarkUntypedPointers(YoungGenerationMarkingTask* task) { void MarkUntypedPointers(YoungGenerationMarkingTask* task) {
RememberedSet<OLD_TO_NEW>::Iterate(chunk_, [this, task](Address slot) { RememberedSet<OLD_TO_NEW>::Iterate(
return CheckAndMarkObject(task, slot); chunk_,
}); [this, task](Address slot) { return CheckAndMarkObject(task, slot); },
SlotSet::PREFREE_EMPTY_BUCKETS);
} }
void MarkTypedPointers(YoungGenerationMarkingTask* task) { void MarkTypedPointers(YoungGenerationMarkingTask* task) {
...@@ -4260,27 +4261,39 @@ class RememberedSetUpdatingItem : public UpdatingItem { ...@@ -4260,27 +4261,39 @@ class RememberedSetUpdatingItem : public UpdatingItem {
// those slots using atomics. // those slots using atomics.
if (chunk_->slot_set<OLD_TO_NEW, AccessMode::NON_ATOMIC>() != nullptr) { if (chunk_->slot_set<OLD_TO_NEW, AccessMode::NON_ATOMIC>() != nullptr) {
if (chunk_->owner() == heap_->map_space()) { if (chunk_->owner() == heap_->map_space()) {
RememberedSet<OLD_TO_NEW>::Iterate(chunk_, [this](Address slot) { RememberedSet<OLD_TO_NEW>::Iterate(
chunk_,
[this](Address slot) {
return CheckAndUpdateOldToNewSlot<AccessMode::ATOMIC>(slot); return CheckAndUpdateOldToNewSlot<AccessMode::ATOMIC>(slot);
}); },
SlotSet::PREFREE_EMPTY_BUCKETS);
} else { } else {
RememberedSet<OLD_TO_NEW>::Iterate(chunk_, [this](Address slot) { RememberedSet<OLD_TO_NEW>::Iterate(
chunk_,
[this](Address slot) {
return CheckAndUpdateOldToNewSlot<AccessMode::NON_ATOMIC>(slot); return CheckAndUpdateOldToNewSlot<AccessMode::NON_ATOMIC>(slot);
}); },
SlotSet::PREFREE_EMPTY_BUCKETS);
} }
} }
if ((updating_mode_ == RememberedSetUpdatingMode::ALL) && if ((updating_mode_ == RememberedSetUpdatingMode::ALL) &&
(chunk_->slot_set<OLD_TO_OLD, AccessMode::NON_ATOMIC>() != nullptr)) { (chunk_->slot_set<OLD_TO_OLD, AccessMode::NON_ATOMIC>() != nullptr)) {
if (chunk_->owner() == heap_->map_space()) { if (chunk_->owner() == heap_->map_space()) {
RememberedSet<OLD_TO_OLD>::Iterate(chunk_, [](Address slot) { RememberedSet<OLD_TO_OLD>::Iterate(
chunk_,
[](Address slot) {
return UpdateSlot<AccessMode::ATOMIC>( return UpdateSlot<AccessMode::ATOMIC>(
reinterpret_cast<Object**>(slot)); reinterpret_cast<Object**>(slot));
}); },
SlotSet::PREFREE_EMPTY_BUCKETS);
} else { } else {
RememberedSet<OLD_TO_OLD>::Iterate(chunk_, [](Address slot) { RememberedSet<OLD_TO_OLD>::Iterate(
chunk_,
[](Address slot) {
return UpdateSlot<AccessMode::NON_ATOMIC>( return UpdateSlot<AccessMode::NON_ATOMIC>(
reinterpret_cast<Object**>(slot)); reinterpret_cast<Object**>(slot));
}); },
SlotSet::PREFREE_EMPTY_BUCKETS);
} }
} }
} }
......
...@@ -129,15 +129,18 @@ class RememberedSet : public AllStatic { ...@@ -129,15 +129,18 @@ class RememberedSet : public AllStatic {
// Iterates and filters the remembered set in the given memory chunk with // Iterates and filters the remembered set in the given memory chunk with
// the given callback. The callback should take (Address slot) and return // the given callback. The callback should take (Address slot) and return
// SlotCallbackResult. // SlotCallbackResult.
//
// Notice that |mode| can only be of FREE* or PREFREE* if there are no other
// threads concurrently inserting slots.
template <typename Callback> template <typename Callback>
static void Iterate(MemoryChunk* chunk, Callback callback) { static void Iterate(MemoryChunk* chunk, Callback callback,
SlotSet::EmptyBucketMode mode) {
SlotSet* slots = chunk->slot_set<type>(); SlotSet* slots = chunk->slot_set<type>();
if (slots != nullptr) { if (slots != nullptr) {
size_t pages = (chunk->size() + Page::kPageSize - 1) / Page::kPageSize; size_t pages = (chunk->size() + Page::kPageSize - 1) / Page::kPageSize;
int new_count = 0; int new_count = 0;
for (size_t page = 0; page < pages; page++) { for (size_t page = 0; page < pages; page++) {
new_count += new_count += slots[page].Iterate(callback, mode);
slots[page].Iterate(callback, SlotSet::PREFREE_EMPTY_BUCKETS);
} }
// Only old-to-old slot sets are released eagerly. Old-new-slot sets are // Only old-to-old slot sets are released eagerly. Old-new-slot sets are
// released by the sweeper threads. // released by the sweeper threads.
...@@ -147,6 +150,17 @@ class RememberedSet : public AllStatic { ...@@ -147,6 +150,17 @@ class RememberedSet : public AllStatic {
} }
} }
static void PreFreeEmptyBuckets(MemoryChunk* chunk) {
DCHECK(type == OLD_TO_NEW);
SlotSet* slots = chunk->slot_set<type>();
if (slots != nullptr) {
size_t pages = (chunk->size() + Page::kPageSize - 1) / Page::kPageSize;
for (size_t page = 0; page < pages; page++) {
slots[page].PreFreeEmptyBuckets();
}
}
}
// Given a page and a typed slot in that page, this function adds the slot // Given a page and a typed slot in that page, this function adds the slot
// to the remembered set. // to the remembered set.
static void InsertTyped(Page* page, Address host_addr, SlotType slot_type, static void InsertTyped(Page* page, Address host_addr, SlotType slot_type,
......
...@@ -225,6 +225,25 @@ class SlotSet : public Malloced { ...@@ -225,6 +225,25 @@ class SlotSet : public Malloced {
return new_count; return new_count;
} }
void PreFreeEmptyBuckets() {
for (int bucket_index = 0; bucket_index < kBuckets; bucket_index++) {
Bucket bucket = LoadBucket(&buckets_[bucket_index]);
if (bucket != nullptr) {
bool found_non_empty_cell = false;
int cell_offset = bucket_index * kBitsPerBucket;
for (int i = 0; i < kCellsPerBucket; i++, cell_offset += kBitsPerCell) {
if (LoadCell(&bucket[i])) {
found_non_empty_cell = true;
break;
}
}
if (!found_non_empty_cell) {
PreFreeEmptyBucket(bucket_index);
}
}
}
}
void FreeToBeFreedBuckets() { void FreeToBeFreedBuckets() {
base::LockGuard<base::Mutex> guard(&to_be_freed_buckets_mutex_); base::LockGuard<base::Mutex> guard(&to_be_freed_buckets_mutex_);
while (!to_be_freed_buckets_.empty()) { while (!to_be_freed_buckets_.empty()) {
......
...@@ -6100,46 +6100,56 @@ TEST(RememberedSetRemoveRange) { ...@@ -6100,46 +6100,56 @@ TEST(RememberedSetRemoveRange) {
RememberedSet<OLD_TO_NEW>::Insert(chunk, x.first); RememberedSet<OLD_TO_NEW>::Insert(chunk, x.first);
} }
RememberedSet<OLD_TO_NEW>::Iterate(chunk, [&slots](Address addr) { RememberedSet<OLD_TO_NEW>::Iterate(chunk,
[&slots](Address addr) {
CHECK(slots[addr]); CHECK(slots[addr]);
return KEEP_SLOT; return KEEP_SLOT;
}); },
SlotSet::PREFREE_EMPTY_BUCKETS);
RememberedSet<OLD_TO_NEW>::RemoveRange(chunk, start, start + kPointerSize, RememberedSet<OLD_TO_NEW>::RemoveRange(chunk, start, start + kPointerSize,
SlotSet::FREE_EMPTY_BUCKETS); SlotSet::FREE_EMPTY_BUCKETS);
slots[start] = false; slots[start] = false;
RememberedSet<OLD_TO_NEW>::Iterate(chunk, [&slots](Address addr) { RememberedSet<OLD_TO_NEW>::Iterate(chunk,
[&slots](Address addr) {
CHECK(slots[addr]); CHECK(slots[addr]);
return KEEP_SLOT; return KEEP_SLOT;
}); },
SlotSet::PREFREE_EMPTY_BUCKETS);
RememberedSet<OLD_TO_NEW>::RemoveRange(chunk, start + kPointerSize, RememberedSet<OLD_TO_NEW>::RemoveRange(chunk, start + kPointerSize,
start + Page::kPageSize, start + Page::kPageSize,
SlotSet::FREE_EMPTY_BUCKETS); SlotSet::FREE_EMPTY_BUCKETS);
slots[start + kPointerSize] = false; slots[start + kPointerSize] = false;
slots[start + Page::kPageSize - kPointerSize] = false; slots[start + Page::kPageSize - kPointerSize] = false;
RememberedSet<OLD_TO_NEW>::Iterate(chunk, [&slots](Address addr) { RememberedSet<OLD_TO_NEW>::Iterate(chunk,
[&slots](Address addr) {
CHECK(slots[addr]); CHECK(slots[addr]);
return KEEP_SLOT; return KEEP_SLOT;
}); },
SlotSet::PREFREE_EMPTY_BUCKETS);
RememberedSet<OLD_TO_NEW>::RemoveRange(chunk, start, RememberedSet<OLD_TO_NEW>::RemoveRange(chunk, start,
start + Page::kPageSize + kPointerSize, start + Page::kPageSize + kPointerSize,
SlotSet::FREE_EMPTY_BUCKETS); SlotSet::FREE_EMPTY_BUCKETS);
slots[start + Page::kPageSize] = false; slots[start + Page::kPageSize] = false;
RememberedSet<OLD_TO_NEW>::Iterate(chunk, [&slots](Address addr) { RememberedSet<OLD_TO_NEW>::Iterate(chunk,
[&slots](Address addr) {
CHECK(slots[addr]); CHECK(slots[addr]);
return KEEP_SLOT; return KEEP_SLOT;
}); },
SlotSet::PREFREE_EMPTY_BUCKETS);
RememberedSet<OLD_TO_NEW>::RemoveRange( RememberedSet<OLD_TO_NEW>::RemoveRange(
chunk, chunk->area_end() - kPointerSize, chunk->area_end(), chunk, chunk->area_end() - kPointerSize, chunk->area_end(),
SlotSet::FREE_EMPTY_BUCKETS); SlotSet::FREE_EMPTY_BUCKETS);
slots[chunk->area_end() - kPointerSize] = false; slots[chunk->area_end() - kPointerSize] = false;
RememberedSet<OLD_TO_NEW>::Iterate(chunk, [&slots](Address addr) { RememberedSet<OLD_TO_NEW>::Iterate(chunk,
[&slots](Address addr) {
CHECK(slots[addr]); CHECK(slots[addr]);
return KEEP_SLOT; return KEEP_SLOT;
}); },
SlotSet::PREFREE_EMPTY_BUCKETS);
} }
HEAP_TEST(Regress670675) { HEAP_TEST(Regress670675) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment