Commit 9847f9c8 authored by Dominik Inführ's avatar Dominik Inführ Committed by Commit Bot

[heap] Add linear ephemeron marking

After a certain number of iterations in the fixpoint
iteration switch to a linear algorithm. This
algorithm uses a key-to-values map for weak collections
contents to avoid checking all EphemeronHashTables
again.

Bug: chromium:844008
Change-Id: I044fede5911e0a780d088d1ba2bb5343317d9b7a
Reviewed-on: https://chromium-review.googlesource.com/1105835
Commit-Queue: Dominik Inführ <dinfuehr@google.com>
Reviewed-by: 's avatarUlan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#54232}
parent 0973a408
......@@ -704,6 +704,9 @@ DEFINE_BOOL(parallel_ephemeron_visiting, true,
DEFINE_BOOL(
parallel_ephemeron_marking, true,
"use parallel marking of objects after visiting ephemerons in atomic pause")
DEFINE_INT(ephemeron_fixpoint_iterations, 0,
"number of fixpoint iterations it takes to switch to linear "
"ephemeron algorithm")
DEFINE_BOOL(trace_concurrent_marking, false, "trace concurrent marking")
DEFINE_BOOL(black_allocation, true, "use black allocation")
DEFINE_BOOL(concurrent_store_buffer, true,
......
......@@ -336,6 +336,7 @@
F(MC_MARK_WEAK_CLOSURE) \
F(MC_MARK_WEAK_CLOSURE_EPHEMERON) \
F(MC_MARK_WEAK_CLOSURE_EPHEMERON_MARKING) \
F(MC_MARK_WEAK_CLOSURE_EPHEMERON_LINEAR) \
F(MC_MARK_WEAK_CLOSURE_WEAK_HANDLES) \
F(MC_MARK_WEAK_CLOSURE_WEAK_ROOTS) \
F(MC_MARK_WEAK_CLOSURE_HARMONY) \
......
......@@ -699,6 +699,7 @@ void GCTracer::PrintNVP() const {
"mark.weak_closure=%.1f "
"mark.weak_closure.ephemeron=%.1f "
"mark.weak_closure.ephemeron.marking=%.1f "
"mark.weak_closure.ephemeron.linear=%.1f "
"mark.weak_closure.weak_handles=%.1f "
"mark.weak_closure.weak_roots=%.1f "
"mark.weak_closure.harmony=%.1f "
......@@ -794,6 +795,7 @@ void GCTracer::PrintNVP() const {
current_.scopes[Scope::MC_MARK_WEAK_CLOSURE],
current_.scopes[Scope::MC_MARK_WEAK_CLOSURE_EPHEMERON],
current_.scopes[Scope::MC_MARK_WEAK_CLOSURE_EPHEMERON_MARKING],
current_.scopes[Scope::MC_MARK_WEAK_CLOSURE_EPHEMERON_LINEAR],
current_.scopes[Scope::MC_MARK_WEAK_CLOSURE_WEAK_HANDLES],
current_.scopes[Scope::MC_MARK_WEAK_CLOSURE_WEAK_ROOTS],
current_.scopes[Scope::MC_MARK_WEAK_CLOSURE_HARMONY],
......
......@@ -1470,14 +1470,16 @@ void MarkCompactCollector::MarkRoots(RootVisitor* root_visitor,
void MarkCompactCollector::ProcessEphemeronsUntilFixpoint() {
bool work_to_do = true;
int iterations = 0;
int max_iterations = FLAG_ephemeron_fixpoint_iterations;
while (work_to_do) {
if (heap_->local_embedder_heap_tracer()->InUse()) {
TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK_WRAPPER_TRACING);
heap_->local_embedder_heap_tracer()->RegisterWrappersWithRemoteTracer();
heap_->local_embedder_heap_tracer()->Trace(
0, EmbedderHeapTracer::AdvanceTracingActions(
EmbedderHeapTracer::ForceCompletionAction::FORCE_COMPLETION));
PerformWrapperTracing();
if (iterations >= max_iterations) {
// Give up fixpoint iteration and switch to linear algorithm.
ProcessEphemeronsLinear();
break;
}
// Move ephemerons from next_ephemerons into current_ephemerons to
......@@ -1506,6 +1508,7 @@ void MarkCompactCollector::ProcessEphemeronsUntilFixpoint() {
work_to_do || !marking_worklist()->IsEmpty() ||
heap()->concurrent_marking()->ephemeron_marked() ||
heap()->local_embedder_heap_tracer()->NumberOfWrappersToTrace() > 0;
++iterations;
}
CHECK(marking_worklist()->IsEmpty());
......@@ -1539,12 +1542,113 @@ bool MarkCompactCollector::ProcessEphemerons() {
}
// Flush local ephemerons for main task to global pool.
weak_objects_.ephemeron_hash_tables.FlushToGlobal(kMainThread);
weak_objects_.next_ephemerons.FlushToGlobal(kMainThread);
return ephemeron_marked;
}
void MarkCompactCollector::ProcessEphemeronsLinear() {
TRACE_GC(heap()->tracer(),
GCTracer::Scope::MC_MARK_WEAK_CLOSURE_EPHEMERON_LINEAR);
CHECK(heap()->concurrent_marking()->IsStopped());
std::unordered_multimap<HeapObject*, HeapObject*> key_to_values;
Ephemeron ephemeron;
DCHECK(weak_objects_.current_ephemerons.IsEmpty());
weak_objects_.current_ephemerons.Swap(weak_objects_.next_ephemerons);
while (weak_objects_.current_ephemerons.Pop(kMainThread, &ephemeron)) {
VisitEphemeron(ephemeron.key, ephemeron.value);
if (non_atomic_marking_state()->IsWhite(ephemeron.value)) {
key_to_values.insert(std::make_pair(ephemeron.key, ephemeron.value));
}
}
ephemeron_marking_.newly_discovered_limit = key_to_values.size();
bool work_to_do = true;
while (work_to_do) {
PerformWrapperTracing();
ResetNewlyDiscovered();
ephemeron_marking_.newly_discovered_limit = key_to_values.size();
{
TRACE_GC(heap()->tracer(),
GCTracer::Scope::MC_MARK_WEAK_CLOSURE_EPHEMERON_MARKING);
// Drain marking worklist and push all discovered objects into
// newly_discovered.
ProcessMarkingWorklistInternal<
MarkCompactCollector::MarkingWorklistProcessingMode::
kTrackNewlyDiscoveredObjects>();
}
while (weak_objects_.discovered_ephemerons.Pop(kMainThread, &ephemeron)) {
VisitEphemeron(ephemeron.key, ephemeron.value);
if (non_atomic_marking_state()->IsWhite(ephemeron.value)) {
key_to_values.insert(std::make_pair(ephemeron.key, ephemeron.value));
}
}
if (ephemeron_marking_.newly_discovered_overflowed) {
// If newly_discovered was overflowed just visit all ephemerons in
// next_ephemerons.
weak_objects_.next_ephemerons.Iterate([&](Ephemeron ephemeron) {
if (non_atomic_marking_state()->IsBlackOrGrey(ephemeron.key) &&
non_atomic_marking_state()->WhiteToGrey(ephemeron.value)) {
marking_worklist()->Push(ephemeron.value);
}
});
} else {
// This is the good case: newly_discovered stores all discovered
// objects. Now use key_to_values to see if discovered objects keep more
// objects alive due to ephemeron semantics.
for (HeapObject* object : ephemeron_marking_.newly_discovered) {
auto range = key_to_values.equal_range(object);
for (auto it = range.first; it != range.second; ++it) {
HeapObject* value = it->second;
MarkObject(object, value);
}
}
}
// Do NOT drain marking worklist here, otherwise the current checks
// for work_to_do are not sufficient for determining if another iteration
// is necessary.
work_to_do =
!marking_worklist()->IsEmpty() ||
heap()->local_embedder_heap_tracer()->NumberOfWrappersToTrace() > 0;
CHECK(weak_objects_.discovered_ephemerons.IsEmpty());
}
ResetNewlyDiscovered();
ephemeron_marking_.newly_discovered.shrink_to_fit();
CHECK(marking_worklist()->IsEmpty());
}
void MarkCompactCollector::PerformWrapperTracing() {
if (heap_->local_embedder_heap_tracer()->InUse()) {
TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK_WRAPPER_TRACING);
heap_->local_embedder_heap_tracer()->RegisterWrappersWithRemoteTracer();
heap_->local_embedder_heap_tracer()->Trace(
0, EmbedderHeapTracer::AdvanceTracingActions(
EmbedderHeapTracer::ForceCompletionAction::FORCE_COMPLETION));
}
}
void MarkCompactCollector::ProcessMarkingWorklist() {
ProcessMarkingWorklistInternal<
MarkCompactCollector::MarkingWorklistProcessingMode::kDefault>();
}
template <MarkCompactCollector::MarkingWorklistProcessingMode mode>
void MarkCompactCollector::ProcessMarkingWorklistInternal() {
HeapObject* object;
MarkCompactMarkingVisitor visitor(this, marking_state());
while ((object = marking_worklist()->Pop()) != nullptr) {
......@@ -1553,6 +1657,10 @@ void MarkCompactCollector::ProcessMarkingWorklist() {
DCHECK(heap()->Contains(object));
DCHECK(!(marking_state()->IsWhite(object)));
marking_state()->GreyToBlack(object);
if (mode == MarkCompactCollector::MarkingWorklistProcessingMode::
kTrackNewlyDiscoveredObjects) {
AddNewlyDiscovered(object);
}
Map* map = object->map();
MarkObject(object, map);
visitor.Visit(map, object);
......@@ -1955,6 +2063,15 @@ void MarkCompactCollector::ClearWeakCollections() {
while (weak_objects_.ephemeron_hash_tables.Pop(kMainThread, &table)) {
for (int i = 0; i < table->Capacity(); i++) {
HeapObject* key = HeapObject::cast(table->KeyAt(i));
#ifdef VERIFY_HEAP
Object* value = table->ValueAt(i);
if (value->IsHeapObject()) {
CHECK_IMPLIES(
non_atomic_marking_state()->IsBlackOrGrey(key),
non_atomic_marking_state()->IsBlackOrGrey(HeapObject::cast(value)));
}
#endif
if (!non_atomic_marking_state()->IsBlackOrGrey(key)) {
table->RemoveEntry(i);
}
......
......@@ -451,6 +451,12 @@ struct WeakObjects {
Worklist<std::pair<HeapObject*, Code*>, 64> weak_objects_in_code;
};
struct EphemeronMarking {
std::vector<HeapObject*> newly_discovered;
bool newly_discovered_overflowed;
size_t newly_discovered_limit;
};
// Collector for young and old generation.
class MarkCompactCollector final : public MarkCompactCollectorBase {
public:
......@@ -667,6 +673,22 @@ class MarkCompactCollector final : public MarkCompactCollectorBase {
std::make_pair(object, code));
}
void AddNewlyDiscovered(HeapObject* object) {
if (ephemeron_marking_.newly_discovered_overflowed) return;
if (ephemeron_marking_.newly_discovered.size() <
ephemeron_marking_.newly_discovered_limit) {
ephemeron_marking_.newly_discovered.push_back(object);
} else {
ephemeron_marking_.newly_discovered_overflowed = true;
}
}
void ResetNewlyDiscovered() {
ephemeron_marking_.newly_discovered_overflowed = false;
ephemeron_marking_.newly_discovered.clear();
}
Sweeper* sweeper() { return sweeper_; }
#ifdef DEBUG
......@@ -735,6 +757,14 @@ class MarkCompactCollector final : public MarkCompactCollectorBase {
// if no concurrent threads are running.
void ProcessMarkingWorklist() override;
enum class MarkingWorklistProcessingMode {
kDefault,
kTrackNewlyDiscoveredObjects
};
template <MarkingWorklistProcessingMode mode>
void ProcessMarkingWorklistInternal();
// Implements ephemeron semantics: Marks value if key is already reachable.
// Returns true if value was actually marked.
bool VisitEphemeron(HeapObject* key, HeapObject* value);
......@@ -747,6 +777,13 @@ class MarkCompactCollector final : public MarkCompactCollectorBase {
// fixpoint iteration.
bool ProcessEphemerons();
// Mark ephemerons and drain marking worklist with a linear algorithm.
// Only used if fixpoint iteration doesn't finish within a few iterations.
void ProcessEphemeronsLinear();
// Perform Wrapper Tracing if in use.
void PerformWrapperTracing();
// Callback function for telling whether the object *p is an unmarked
// heap object.
static bool IsUnmarkedHeapObject(Heap* heap, Object** p);
......@@ -768,11 +805,6 @@ class MarkCompactCollector final : public MarkCompactCollectorBase {
void TrimDescriptorArray(Map* map, DescriptorArray* descriptors);
void TrimEnumCache(Map* map, DescriptorArray* descriptors);
// Mark all values associated with reachable keys in weak collections
// encountered so far. This might push new object or even new weak maps onto
// the marking stack.
void ProcessWeakCollections();
// After all reachable objects have been marked those weak map entries
// with an unreachable key are removed from all encountered weak maps.
// The linked list of all encountered weak maps is destroyed.
......@@ -848,6 +880,7 @@ class MarkCompactCollector final : public MarkCompactCollectorBase {
MarkingWorklist marking_worklist_;
WeakObjects weak_objects_;
EphemeronMarking ephemeron_marking_;
// Candidates for pages that should be evacuated.
std::vector<Page*> evacuation_candidates_;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment