Commit 173109b9 authored by Michael Lippautz's avatar Michael Lippautz Committed by Commit Bot

[heap] Introduce explicit marking state and instance for minor MC

BUG=chromium:651354

Change-Id: Idcd7780f53ad07b3d782a66455f9c60addc2418d
Reviewed-on: https://chromium-review.googlesource.com/457317
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: 's avatarHannes Payer <hpayer@chromium.org>
Reviewed-by: 's avatarUlan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#43971}
parent 74daa15e
...@@ -657,6 +657,8 @@ DEFINE_INT(min_progress_during_incremental_marking_finalization, 32, ...@@ -657,6 +657,8 @@ DEFINE_INT(min_progress_during_incremental_marking_finalization, 32,
DEFINE_INT(max_incremental_marking_finalization_rounds, 3, DEFINE_INT(max_incremental_marking_finalization_rounds, 3,
"at most try this many times to finalize incremental marking") "at most try this many times to finalize incremental marking")
DEFINE_BOOL(minor_mc, false, "perform young generation mark compact GCs") DEFINE_BOOL(minor_mc, false, "perform young generation mark compact GCs")
DEFINE_NEG_IMPLICATION(minor_mc, page_promotion)
DEFINE_NEG_IMPLICATION(minor_mc, flush_code)
DEFINE_BOOL(black_allocation, true, "use black allocation") DEFINE_BOOL(black_allocation, true, "use black allocation")
DEFINE_BOOL(concurrent_store_buffer, true, DEFINE_BOOL(concurrent_store_buffer, true,
"use concurrent store buffer processing") "use concurrent store buffer processing")
......
...@@ -82,6 +82,7 @@ enum ScavengeSpeedMode { kForAllObjects, kForSurvivedObjects }; ...@@ -82,6 +82,7 @@ enum ScavengeSpeedMode { kForAllObjects, kForSurvivedObjects };
F(MC_SWEEP_CODE) \ F(MC_SWEEP_CODE) \
F(MC_SWEEP_MAP) \ F(MC_SWEEP_MAP) \
F(MC_SWEEP_OLD) \ F(MC_SWEEP_OLD) \
F(MC_MINOR_MC) \
F(MINOR_MC_MARK) \ F(MINOR_MC_MARK) \
F(MINOR_MC_MARK_CODE_FLUSH_CANDIDATES) \ F(MINOR_MC_MARK_CODE_FLUSH_CANDIDATES) \
F(MINOR_MC_MARK_GLOBAL_HANDLES) \ F(MINOR_MC_MARK_GLOBAL_HANDLES) \
......
...@@ -133,6 +133,7 @@ Heap::Heap() ...@@ -133,6 +133,7 @@ Heap::Heap()
last_gc_time_(0.0), last_gc_time_(0.0),
scavenge_collector_(nullptr), scavenge_collector_(nullptr),
mark_compact_collector_(nullptr), mark_compact_collector_(nullptr),
minor_mark_compact_collector_(nullptr),
memory_allocator_(nullptr), memory_allocator_(nullptr),
store_buffer_(nullptr), store_buffer_(nullptr),
incremental_marking_(nullptr), incremental_marking_(nullptr),
...@@ -1466,7 +1467,21 @@ void Heap::MarkCompact() { ...@@ -1466,7 +1467,21 @@ void Heap::MarkCompact() {
} }
} }
void Heap::MinorMarkCompact() { UNREACHABLE(); } void Heap::MinorMarkCompact() {
DCHECK(FLAG_minor_mc);
SetGCState(MINOR_MARK_COMPACT);
LOG(isolate_, ResourceEvent("MinorMarkCompact", "begin"));
TRACE_GC(tracer(), GCTracer::Scope::MC_MINOR_MC);
AlwaysAllocateScope always_allocate(isolate());
PauseAllocationObserversScope pause_observers(this);
minor_mark_compact_collector()->CollectGarbage();
LOG(isolate_, ResourceEvent("MinorMarkCompact", "end"));
SetGCState(NOT_IN_GC);
}
void Heap::MarkCompactEpilogue() { void Heap::MarkCompactEpilogue() {
TRACE_GC(tracer(), GCTracer::Scope::MC_EPILOGUE); TRACE_GC(tracer(), GCTracer::Scope::MC_EPILOGUE);
...@@ -5519,6 +5534,8 @@ bool Heap::SetUp() { ...@@ -5519,6 +5534,8 @@ bool Heap::SetUp() {
tracer_ = new GCTracer(this); tracer_ = new GCTracer(this);
scavenge_collector_ = new Scavenger(this); scavenge_collector_ = new Scavenger(this);
mark_compact_collector_ = new MarkCompactCollector(this); mark_compact_collector_ = new MarkCompactCollector(this);
if (FLAG_minor_mc)
minor_mark_compact_collector_ = new MinorMarkCompactCollector(this);
gc_idle_time_handler_ = new GCIdleTimeHandler(); gc_idle_time_handler_ = new GCIdleTimeHandler();
memory_reducer_ = new MemoryReducer(this); memory_reducer_ = new MemoryReducer(this);
if (V8_UNLIKELY(FLAG_gc_stats)) { if (V8_UNLIKELY(FLAG_gc_stats)) {
...@@ -5534,6 +5551,9 @@ bool Heap::SetUp() { ...@@ -5534,6 +5551,9 @@ bool Heap::SetUp() {
store_buffer()->SetUp(); store_buffer()->SetUp();
mark_compact_collector()->SetUp(); mark_compact_collector()->SetUp();
if (minor_mark_compact_collector() != nullptr) {
minor_mark_compact_collector()->SetUp();
}
idle_scavenge_observer_ = new IdleScavengeObserver( idle_scavenge_observer_ = new IdleScavengeObserver(
*this, ScavengeJob::kBytesAllocatedBeforeNextIdleTask); *this, ScavengeJob::kBytesAllocatedBeforeNextIdleTask);
...@@ -5655,6 +5675,12 @@ void Heap::TearDown() { ...@@ -5655,6 +5675,12 @@ void Heap::TearDown() {
mark_compact_collector_ = nullptr; mark_compact_collector_ = nullptr;
} }
if (minor_mark_compact_collector_ != nullptr) {
minor_mark_compact_collector_->TearDown();
delete minor_mark_compact_collector_;
minor_mark_compact_collector_ = nullptr;
}
delete incremental_marking_; delete incremental_marking_;
incremental_marking_ = nullptr; incremental_marking_ = nullptr;
......
...@@ -325,6 +325,7 @@ class Isolate; ...@@ -325,6 +325,7 @@ class Isolate;
class LocalEmbedderHeapTracer; class LocalEmbedderHeapTracer;
class MemoryAllocator; class MemoryAllocator;
class MemoryReducer; class MemoryReducer;
class MinorMarkCompactCollector;
class ObjectIterator; class ObjectIterator;
class ObjectStats; class ObjectStats;
class Page; class Page;
...@@ -559,7 +560,7 @@ class Heap { ...@@ -559,7 +560,7 @@ class Heap {
enum FindMementoMode { kForRuntime, kForGC }; enum FindMementoMode { kForRuntime, kForGC };
enum HeapState { NOT_IN_GC, SCAVENGE, MARK_COMPACT }; enum HeapState { NOT_IN_GC, SCAVENGE, MARK_COMPACT, MINOR_MARK_COMPACT };
enum UpdateAllocationSiteMode { kGlobal, kCached }; enum UpdateAllocationSiteMode { kGlobal, kCached };
...@@ -1040,6 +1041,10 @@ class Heap { ...@@ -1040,6 +1041,10 @@ class Heap {
return mark_compact_collector_; return mark_compact_collector_;
} }
MinorMarkCompactCollector* minor_mark_compact_collector() {
return minor_mark_compact_collector_;
}
// =========================================================================== // ===========================================================================
// Root set access. ========================================================== // Root set access. ==========================================================
// =========================================================================== // ===========================================================================
...@@ -2278,6 +2283,7 @@ class Heap { ...@@ -2278,6 +2283,7 @@ class Heap {
Scavenger* scavenge_collector_; Scavenger* scavenge_collector_;
MarkCompactCollector* mark_compact_collector_; MarkCompactCollector* mark_compact_collector_;
MinorMarkCompactCollector* minor_mark_compact_collector_;
MemoryAllocator* memory_allocator_; MemoryAllocator* memory_allocator_;
...@@ -2377,6 +2383,7 @@ class Heap { ...@@ -2377,6 +2383,7 @@ class Heap {
friend class IncrementalMarkingJob; friend class IncrementalMarkingJob;
friend class LargeObjectSpace; friend class LargeObjectSpace;
friend class MarkCompactCollector; friend class MarkCompactCollector;
friend class MinorMarkCompactCollector;
friend class MarkCompactMarkingVisitor; friend class MarkCompactMarkingVisitor;
friend class NewSpace; friend class NewSpace;
friend class ObjectStatsCollector; friend class ObjectStatsCollector;
......
...@@ -12,14 +12,20 @@ ...@@ -12,14 +12,20 @@
namespace v8 { namespace v8 {
namespace internal { namespace internal {
template <MarkingMode mode>
void MarkCompactCollector::PushBlack(HeapObject* obj) { void MarkCompactCollector::PushBlack(HeapObject* obj) {
DCHECK((ObjectMarking::IsBlack<MarkBit::NON_ATOMIC, mode>(obj))); DCHECK((ObjectMarking::IsBlack<MarkBit::NON_ATOMIC>(obj)));
if (!marking_deque<mode>()->Push(obj)) { if (!marking_deque()->Push(obj)) {
ObjectMarking::BlackToGrey<MarkBit::NON_ATOMIC, mode>(obj); ObjectMarking::BlackToGrey<MarkBit::NON_ATOMIC>(obj);
} }
} }
void MinorMarkCompactCollector::PushBlack(HeapObject* obj) {
DCHECK(
(ObjectMarking::IsBlack<MarkBit::NON_ATOMIC>(obj, StateForObject(obj))));
if (!marking_deque()->Push(obj)) {
ObjectMarking::BlackToGrey<MarkBit::NON_ATOMIC>(obj, StateForObject(obj));
}
}
void MarkCompactCollector::UnshiftBlack(HeapObject* obj) { void MarkCompactCollector::UnshiftBlack(HeapObject* obj) {
DCHECK(ObjectMarking::IsBlack(obj)); DCHECK(ObjectMarking::IsBlack(obj));
...@@ -28,11 +34,17 @@ void MarkCompactCollector::UnshiftBlack(HeapObject* obj) { ...@@ -28,11 +34,17 @@ void MarkCompactCollector::UnshiftBlack(HeapObject* obj) {
} }
} }
template <MarkingMode mode>
void MarkCompactCollector::MarkObject(HeapObject* obj) { void MarkCompactCollector::MarkObject(HeapObject* obj) {
if (ObjectMarking::IsWhite<MarkBit::NON_ATOMIC, mode>(obj)) { if (ObjectMarking::IsWhite<MarkBit::NON_ATOMIC>(obj)) {
ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC, mode>(obj); ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC>(obj);
PushBlack<mode>(obj); PushBlack(obj);
}
}
void MinorMarkCompactCollector::MarkObject(HeapObject* obj) {
if (ObjectMarking::IsWhite<MarkBit::NON_ATOMIC>(obj, StateForObject(obj))) {
ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC>(obj, StateForObject(obj));
PushBlack(obj);
} }
} }
......
...@@ -60,7 +60,6 @@ MarkCompactCollector::MarkCompactCollector(Heap* heap) ...@@ -60,7 +60,6 @@ MarkCompactCollector::MarkCompactCollector(Heap* heap)
black_allocation_(false), black_allocation_(false),
have_code_to_deoptimize_(false), have_code_to_deoptimize_(false),
marking_deque_(heap), marking_deque_(heap),
marking_deque_young_generation_(heap),
code_flusher_(nullptr), code_flusher_(nullptr),
sweeper_(heap) { sweeper_(heap) {
} }
...@@ -239,7 +238,6 @@ void MarkCompactCollector::SetUp() { ...@@ -239,7 +238,6 @@ void MarkCompactCollector::SetUp() {
DCHECK(strcmp(Marking::kGreyBitPattern, "10") == 0); DCHECK(strcmp(Marking::kGreyBitPattern, "10") == 0);
DCHECK(strcmp(Marking::kImpossibleBitPattern, "01") == 0); DCHECK(strcmp(Marking::kImpossibleBitPattern, "01") == 0);
marking_deque()->SetUp(); marking_deque()->SetUp();
marking_deque<MarkingMode::YOUNG_GENERATION>()->SetUp();
if (FLAG_flush_code) { if (FLAG_flush_code) {
code_flusher_ = new CodeFlusher(isolate()); code_flusher_ = new CodeFlusher(isolate());
...@@ -249,14 +247,15 @@ void MarkCompactCollector::SetUp() { ...@@ -249,14 +247,15 @@ void MarkCompactCollector::SetUp() {
} }
} }
void MinorMarkCompactCollector::SetUp() { marking_deque()->SetUp(); }
void MarkCompactCollector::TearDown() { void MarkCompactCollector::TearDown() {
AbortCompaction(); AbortCompaction();
marking_deque<MarkingMode::YOUNG_GENERATION>()->TearDown();
marking_deque()->TearDown(); marking_deque()->TearDown();
delete code_flusher_; delete code_flusher_;
} }
void MinorMarkCompactCollector::TearDown() { marking_deque()->TearDown(); }
void MarkCompactCollector::AddEvacuationCandidate(Page* p) { void MarkCompactCollector::AddEvacuationCandidate(Page* p) {
DCHECK(!p->NeverEvacuate()); DCHECK(!p->NeverEvacuate());
...@@ -1067,9 +1066,9 @@ class StaticYoungGenerationMarkingVisitor ...@@ -1067,9 +1066,9 @@ class StaticYoungGenerationMarkingVisitor
inline static void VisitPointer(Heap* heap, HeapObject* object, Object** p) { inline static void VisitPointer(Heap* heap, HeapObject* object, Object** p) {
Object* target = *p; Object* target = *p;
if (heap->InNewSpace(target)) { if (heap->InNewSpace(target)) {
if (MarkRecursively(heap, HeapObject::cast(target))) return; HeapObject* target_object = HeapObject::cast(target);
heap->mark_compact_collector()->MarkObject<MarkingMode::YOUNG_GENERATION>( if (MarkRecursively(heap, target_object)) return;
HeapObject::cast(target)); heap->minor_mark_compact_collector()->MarkObject(target_object);
} }
} }
...@@ -1078,11 +1077,11 @@ class StaticYoungGenerationMarkingVisitor ...@@ -1078,11 +1077,11 @@ class StaticYoungGenerationMarkingVisitor
StackLimitCheck check(heap->isolate()); StackLimitCheck check(heap->isolate());
if (check.HasOverflowed()) return false; if (check.HasOverflowed()) return false;
if (ObjectMarking::IsBlackOrGrey<MarkBit::NON_ATOMIC, const MarkingState state =
MarkingMode::YOUNG_GENERATION>(object)) MinorMarkCompactCollector::StateForObject(object);
if (ObjectMarking::IsBlackOrGrey<MarkBit::NON_ATOMIC>(object, state))
return true; return true;
ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC, ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC>(object, state);
MarkingMode::YOUNG_GENERATION>(object);
IterateBody(object->map(), object); IterateBody(object->map(), object);
return true; return true;
} }
...@@ -1330,13 +1329,49 @@ void MarkCompactCollector::PrepareForCodeFlushing() { ...@@ -1330,13 +1329,49 @@ void MarkCompactCollector::PrepareForCodeFlushing() {
heap()->isolate()->compilation_cache()->IterateFunctions(&visitor); heap()->isolate()->compilation_cache()->IterateFunctions(&visitor);
heap()->isolate()->handle_scope_implementer()->Iterate(&visitor); heap()->isolate()->handle_scope_implementer()->Iterate(&visitor);
ProcessMarkingDeque<MarkingMode::FULL>(); ProcessMarkingDeque();
} }
class MinorMarkCompactCollector::RootMarkingVisitor : public ObjectVisitor {
public:
explicit RootMarkingVisitor(MinorMarkCompactCollector* collector)
: collector_(collector) {}
void VisitPointer(Object** p) override { MarkObjectByPointer(p); }
void VisitPointers(Object** start, Object** end) override {
for (Object** p = start; p < end; p++) MarkObjectByPointer(p);
}
// Skip the weak next code link in a code object, which is visited in
// ProcessTopOptimizedFrame.
void VisitNextCodeLink(Object** p) override {}
private:
void MarkObjectByPointer(Object** p) {
if (!(*p)->IsHeapObject()) return;
HeapObject* object = HeapObject::cast(*p);
if (!collector_->heap()->InNewSpace(object)) return;
if (ObjectMarking::IsBlackOrGrey<MarkBit::NON_ATOMIC>(
object, StateForObject(object)))
return;
Map* map = object->map();
ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC>(object,
StateForObject(object));
StaticYoungGenerationMarkingVisitor::IterateBody(map, object);
collector_->EmptyMarkingDeque();
}
MinorMarkCompactCollector* collector_;
};
// Visitor class for marking heap roots. // Visitor class for marking heap roots.
template <MarkingMode mode> class MarkCompactCollector::RootMarkingVisitor : public ObjectVisitor {
class RootMarkingVisitor : public ObjectVisitor {
public: public:
explicit RootMarkingVisitor(Heap* heap) explicit RootMarkingVisitor(Heap* heap)
: collector_(heap->mark_compact_collector()) {} : collector_(heap->mark_compact_collector()) {}
...@@ -1357,30 +1392,19 @@ class RootMarkingVisitor : public ObjectVisitor { ...@@ -1357,30 +1392,19 @@ class RootMarkingVisitor : public ObjectVisitor {
HeapObject* object = HeapObject::cast(*p); HeapObject* object = HeapObject::cast(*p);
if (mode == MarkingMode::YOUNG_GENERATION && if (ObjectMarking::IsBlackOrGrey<MarkBit::NON_ATOMIC>(object)) return;
!collector_->heap()->InNewSpace(object))
return;
if (ObjectMarking::IsBlackOrGrey<MarkBit::NON_ATOMIC, mode>(object)) return;
Map* map = object->map(); Map* map = object->map();
// Mark the object. // Mark the object.
ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC, mode>(object); ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC>(object);
switch (mode) { // Mark the map pointer and body, and push them on the marking stack.
case MarkingMode::FULL: { collector_->MarkObject(map);
// Mark the map pointer and body, and push them on the marking stack. MarkCompactMarkingVisitor::IterateBody(map, object);
collector_->MarkObject(map);
MarkCompactMarkingVisitor::IterateBody(map, object);
} break;
case MarkingMode::YOUNG_GENERATION:
StaticYoungGenerationMarkingVisitor::IterateBody(map, object);
break;
}
// Mark all the objects reachable from the map and body. May leave // Mark all the objects reachable from the map and body. May leave
// overflowed objects in the heap. // overflowed objects in the heap.
collector_->EmptyMarkingDeque<mode>(); collector_->EmptyMarkingDeque();
} }
MarkCompactCollector* collector_; MarkCompactCollector* collector_;
...@@ -1482,7 +1506,7 @@ void MarkCompactCollector::DiscoverGreyObjectsWithIterator(T* it) { ...@@ -1482,7 +1506,7 @@ void MarkCompactCollector::DiscoverGreyObjectsWithIterator(T* it) {
void MarkCompactCollector::DiscoverGreyObjectsOnPage(MemoryChunk* p) { void MarkCompactCollector::DiscoverGreyObjectsOnPage(MemoryChunk* p) {
DCHECK(!marking_deque()->IsFull()); DCHECK(!marking_deque()->IsFull());
LiveObjectIterator<kGreyObjects> it(p); LiveObjectIterator<kGreyObjects> it(p, MarkingState::FromPageInternal(p));
HeapObject* object = NULL; HeapObject* object = NULL;
while ((object = it.Next()) != NULL) { while ((object = it.Next()) != NULL) {
DCHECK(ObjectMarking::IsGrey(object)); DCHECK(ObjectMarking::IsGrey(object));
...@@ -1941,8 +1965,7 @@ bool MarkCompactCollector::IsUnmarkedHeapObject(Object** p) { ...@@ -1941,8 +1965,7 @@ bool MarkCompactCollector::IsUnmarkedHeapObject(Object** p) {
return ObjectMarking::IsWhite(HeapObject::cast(o)); return ObjectMarking::IsWhite(HeapObject::cast(o));
} }
void MarkCompactCollector::MarkStringTable( void MarkCompactCollector::MarkStringTable(RootMarkingVisitor* visitor) {
RootMarkingVisitor<MarkingMode::FULL>* visitor) {
StringTable* string_table = heap()->string_table(); StringTable* string_table = heap()->string_table();
// Mark the string table itself. // Mark the string table itself.
if (ObjectMarking::IsWhite(string_table)) { if (ObjectMarking::IsWhite(string_table)) {
...@@ -1951,11 +1974,10 @@ void MarkCompactCollector::MarkStringTable( ...@@ -1951,11 +1974,10 @@ void MarkCompactCollector::MarkStringTable(
} }
// Explicitly mark the prefix. // Explicitly mark the prefix.
string_table->IteratePrefix(visitor); string_table->IteratePrefix(visitor);
ProcessMarkingDeque<MarkingMode::FULL>(); ProcessMarkingDeque();
} }
void MarkCompactCollector::MarkRoots( void MarkCompactCollector::MarkRoots(RootMarkingVisitor* visitor) {
RootMarkingVisitor<MarkingMode::FULL>* visitor) {
// Mark the heap roots including global variables, stack variables, // Mark the heap roots including global variables, stack variables,
// etc., and all objects reachable from them. // etc., and all objects reachable from them.
heap()->IterateStrongRoots(visitor, VISIT_ONLY_STRONG); heap()->IterateStrongRoots(visitor, VISIT_ONLY_STRONG);
...@@ -1964,9 +1986,9 @@ void MarkCompactCollector::MarkRoots( ...@@ -1964,9 +1986,9 @@ void MarkCompactCollector::MarkRoots(
MarkStringTable(visitor); MarkStringTable(visitor);
// There may be overflowed objects in the heap. Visit them now. // There may be overflowed objects in the heap. Visit them now.
while (marking_deque<MarkingMode::FULL>()->overflowed()) { while (marking_deque()->overflowed()) {
RefillMarkingDeque<MarkingMode::FULL>(); RefillMarkingDeque();
EmptyMarkingDeque<MarkingMode::FULL>(); EmptyMarkingDeque();
} }
} }
...@@ -1974,27 +1996,18 @@ void MarkCompactCollector::MarkRoots( ...@@ -1974,27 +1996,18 @@ void MarkCompactCollector::MarkRoots(
// Before: the marking stack contains zero or more heap object pointers. // Before: the marking stack contains zero or more heap object pointers.
// After: the marking stack is empty, and all objects reachable from the // After: the marking stack is empty, and all objects reachable from the
// marking stack have been marked, or are overflowed in the heap. // marking stack have been marked, or are overflowed in the heap.
template <MarkingMode mode>
void MarkCompactCollector::EmptyMarkingDeque() { void MarkCompactCollector::EmptyMarkingDeque() {
while (!marking_deque<mode>()->IsEmpty()) { while (!marking_deque()->IsEmpty()) {
HeapObject* object = marking_deque<mode>()->Pop(); HeapObject* object = marking_deque()->Pop();
DCHECK(!object->IsFiller()); DCHECK(!object->IsFiller());
DCHECK(object->IsHeapObject()); DCHECK(object->IsHeapObject());
DCHECK(heap()->Contains(object)); DCHECK(heap()->Contains(object));
DCHECK(!(ObjectMarking::IsWhite<MarkBit::NON_ATOMIC, mode>(object))); DCHECK(!(ObjectMarking::IsWhite<MarkBit::NON_ATOMIC>(object)));
Map* map = object->map(); Map* map = object->map();
switch (mode) { MarkObject(map);
case MarkingMode::FULL: { MarkCompactMarkingVisitor::IterateBody(map, object);
MarkObject(map);
MarkCompactMarkingVisitor::IterateBody(map, object);
} break;
case MarkingMode::YOUNG_GENERATION: {
DCHECK((ObjectMarking::IsBlack<MarkBit::NON_ATOMIC, mode>(object)));
StaticYoungGenerationMarkingVisitor::IterateBody(map, object);
} break;
}
} }
} }
...@@ -2004,43 +2017,35 @@ void MarkCompactCollector::EmptyMarkingDeque() { ...@@ -2004,43 +2017,35 @@ void MarkCompactCollector::EmptyMarkingDeque() {
// before sweeping completes. If sweeping completes, there are no remaining // before sweeping completes. If sweeping completes, there are no remaining
// overflowed objects in the heap so the overflow flag on the markings stack // overflowed objects in the heap so the overflow flag on the markings stack
// is cleared. // is cleared.
template <MarkingMode mode>
void MarkCompactCollector::RefillMarkingDeque() { void MarkCompactCollector::RefillMarkingDeque() {
// Young generation should never overflow.
DCHECK(mode != MarkingMode::YOUNG_GENERATION);
isolate()->CountUsage(v8::Isolate::UseCounterFeature::kMarkDequeOverflow); isolate()->CountUsage(v8::Isolate::UseCounterFeature::kMarkDequeOverflow);
DCHECK(marking_deque()->overflowed()); DCHECK(marking_deque()->overflowed());
DiscoverGreyObjectsInNewSpace(); DiscoverGreyObjectsInNewSpace();
if (marking_deque()->IsFull()) return; if (marking_deque()->IsFull()) return;
if (mode == MarkingMode::FULL) { DiscoverGreyObjectsInSpace(heap()->old_space());
DiscoverGreyObjectsInSpace(heap()->old_space()); if (marking_deque()->IsFull()) return;
if (marking_deque()->IsFull()) return; DiscoverGreyObjectsInSpace(heap()->code_space());
DiscoverGreyObjectsInSpace(heap()->code_space()); if (marking_deque()->IsFull()) return;
if (marking_deque()->IsFull()) return; DiscoverGreyObjectsInSpace(heap()->map_space());
DiscoverGreyObjectsInSpace(heap()->map_space()); if (marking_deque()->IsFull()) return;
if (marking_deque()->IsFull()) return; LargeObjectIterator lo_it(heap()->lo_space());
LargeObjectIterator lo_it(heap()->lo_space()); DiscoverGreyObjectsWithIterator(&lo_it);
DiscoverGreyObjectsWithIterator(&lo_it); if (marking_deque()->IsFull()) return;
if (marking_deque()->IsFull()) return;
}
marking_deque()->ClearOverflowed(); marking_deque()->ClearOverflowed();
} }
// Mark all objects reachable (transitively) from objects on the marking // Mark all objects reachable (transitively) from objects on the marking
// stack. Before: the marking stack contains zero or more heap object // stack. Before: the marking stack contains zero or more heap object
// pointers. After: the marking stack is empty and there are no overflowed // pointers. After: the marking stack is empty and there are no overflowed
// objects in the heap. // objects in the heap.
template <MarkingMode mode>
void MarkCompactCollector::ProcessMarkingDeque() { void MarkCompactCollector::ProcessMarkingDeque() {
EmptyMarkingDeque<mode>(); EmptyMarkingDeque();
while (marking_deque<mode>()->overflowed()) { while (marking_deque()->overflowed()) {
RefillMarkingDeque<mode>(); RefillMarkingDeque();
EmptyMarkingDeque<mode>(); EmptyMarkingDeque();
} }
DCHECK(marking_deque()->IsEmpty()); DCHECK(marking_deque()->IsEmpty());
} }
...@@ -2070,7 +2075,7 @@ void MarkCompactCollector::ProcessEphemeralMarking( ...@@ -2070,7 +2075,7 @@ void MarkCompactCollector::ProcessEphemeralMarking(
} }
ProcessWeakCollections(); ProcessWeakCollections();
work_to_do = !marking_deque()->IsEmpty(); work_to_do = !marking_deque()->IsEmpty();
ProcessMarkingDeque<MarkingMode::FULL>(); ProcessMarkingDeque();
} }
CHECK(marking_deque()->IsEmpty()); CHECK(marking_deque()->IsEmpty());
CHECK_EQ(0, heap()->local_embedder_heap_tracer()->NumberOfWrappersToTrace()); CHECK_EQ(0, heap()->local_embedder_heap_tracer()->NumberOfWrappersToTrace());
...@@ -2087,7 +2092,7 @@ void MarkCompactCollector::ProcessTopOptimizedFrame(ObjectVisitor* visitor) { ...@@ -2087,7 +2092,7 @@ void MarkCompactCollector::ProcessTopOptimizedFrame(ObjectVisitor* visitor) {
if (!code->CanDeoptAt(it.frame()->pc())) { if (!code->CanDeoptAt(it.frame()->pc())) {
Code::BodyDescriptor::IterateBody(code, visitor); Code::BodyDescriptor::IterateBody(code, visitor);
} }
ProcessMarkingDeque<MarkingMode::FULL>(); ProcessMarkingDeque();
return; return;
} }
} }
...@@ -2241,7 +2246,7 @@ void MarkCompactCollector::RecordObjectStats() { ...@@ -2241,7 +2246,7 @@ void MarkCompactCollector::RecordObjectStats() {
} }
} }
SlotCallbackResult MarkCompactCollector::CheckAndMarkObject( SlotCallbackResult MinorMarkCompactCollector::CheckAndMarkObject(
Heap* heap, Address slot_address) { Heap* heap, Address slot_address) {
Object* object = *reinterpret_cast<Object**>(slot_address); Object* object = *reinterpret_cast<Object**>(slot_address);
if (heap->InNewSpace(object)) { if (heap->InNewSpace(object)) {
...@@ -2249,13 +2254,12 @@ SlotCallbackResult MarkCompactCollector::CheckAndMarkObject( ...@@ -2249,13 +2254,12 @@ SlotCallbackResult MarkCompactCollector::CheckAndMarkObject(
// has to be in ToSpace. // has to be in ToSpace.
DCHECK(heap->InToSpace(object)); DCHECK(heap->InToSpace(object));
HeapObject* heap_object = reinterpret_cast<HeapObject*>(object); HeapObject* heap_object = reinterpret_cast<HeapObject*>(object);
if (ObjectMarking::IsBlackOrGrey<MarkBit::NON_ATOMIC, const MarkingState state =
MarkingMode::YOUNG_GENERATION>( MinorMarkCompactCollector::StateForObject(heap_object);
heap_object)) { if (ObjectMarking::IsBlackOrGrey<MarkBit::NON_ATOMIC>(heap_object, state)) {
return KEEP_SLOT; return KEEP_SLOT;
} }
ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC, ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC>(heap_object, state);
MarkingMode::YOUNG_GENERATION>(heap_object);
StaticYoungGenerationMarkingVisitor::IterateBody(heap_object->map(), StaticYoungGenerationMarkingVisitor::IterateBody(heap_object->map(),
heap_object); heap_object);
return KEEP_SLOT; return KEEP_SLOT;
...@@ -2268,15 +2272,15 @@ static bool IsUnmarkedObject(Heap* heap, Object** p) { ...@@ -2268,15 +2272,15 @@ static bool IsUnmarkedObject(Heap* heap, Object** p) {
return heap->InNewSpace(*p) && !ObjectMarking::IsBlack(HeapObject::cast(*p)); return heap->InNewSpace(*p) && !ObjectMarking::IsBlack(HeapObject::cast(*p));
} }
void MarkCompactCollector::MarkLiveObjectsInYoungGeneration() { void MinorMarkCompactCollector::MarkLiveObjects() {
TRACE_GC(heap()->tracer(), GCTracer::Scope::MINOR_MC_MARK); TRACE_GC(heap()->tracer(), GCTracer::Scope::MINOR_MC_MARK);
PostponeInterruptsScope postpone(isolate()); PostponeInterruptsScope postpone(isolate());
StaticYoungGenerationMarkingVisitor::Initialize(heap()); StaticYoungGenerationMarkingVisitor::Initialize(heap());
RootMarkingVisitor<MarkingMode::YOUNG_GENERATION> root_visitor(heap()); RootMarkingVisitor root_visitor(this);
marking_deque<MarkingMode::YOUNG_GENERATION>()->StartUsing(); marking_deque()->StartUsing();
for (Page* p : heap()->new_space()->to_space()) { for (Page* p : heap()->new_space()->to_space()) {
p->AllocateExternalBitmap(); p->AllocateExternalBitmap();
} }
...@@ -2287,7 +2291,7 @@ void MarkCompactCollector::MarkLiveObjectsInYoungGeneration() { ...@@ -2287,7 +2291,7 @@ void MarkCompactCollector::MarkLiveObjectsInYoungGeneration() {
{ {
TRACE_GC(heap()->tracer(), GCTracer::Scope::MINOR_MC_MARK_ROOTS); TRACE_GC(heap()->tracer(), GCTracer::Scope::MINOR_MC_MARK_ROOTS);
heap()->IterateRoots(&root_visitor, VISIT_ALL_IN_SCAVENGE); heap()->IterateRoots(&root_visitor, VISIT_ALL_IN_SCAVENGE);
ProcessMarkingDeque<MarkingMode::YOUNG_GENERATION>(); ProcessMarkingDeque();
} }
{ {
...@@ -2304,20 +2308,13 @@ void MarkCompactCollector::MarkLiveObjectsInYoungGeneration() { ...@@ -2304,20 +2308,13 @@ void MarkCompactCollector::MarkLiveObjectsInYoungGeneration() {
reinterpret_cast<Address>(addr)); reinterpret_cast<Address>(addr));
}); });
}); });
ProcessMarkingDeque<MarkingMode::YOUNG_GENERATION>(); ProcessMarkingDeque();
} }
{ {
TRACE_GC(heap()->tracer(), GCTracer::Scope::MINOR_MC_MARK_WEAK); TRACE_GC(heap()->tracer(), GCTracer::Scope::MINOR_MC_MARK_WEAK);
heap()->VisitEncounteredWeakCollections(&root_visitor); heap()->VisitEncounteredWeakCollections(&root_visitor);
ProcessMarkingDeque<MarkingMode::YOUNG_GENERATION>(); ProcessMarkingDeque();
}
if (is_code_flushing_enabled()) {
TRACE_GC(heap()->tracer(),
GCTracer::Scope::MINOR_MC_MARK_CODE_FLUSH_CANDIDATES);
code_flusher()->IteratePointersToFromSpace(&root_visitor);
ProcessMarkingDeque<MarkingMode::YOUNG_GENERATION>();
} }
{ {
...@@ -2328,7 +2325,7 @@ void MarkCompactCollector::MarkLiveObjectsInYoungGeneration() { ...@@ -2328,7 +2325,7 @@ void MarkCompactCollector::MarkLiveObjectsInYoungGeneration() {
->global_handles() ->global_handles()
->IterateNewSpaceWeakUnmodifiedRoots<GlobalHandles::VISIT_OTHERS>( ->IterateNewSpaceWeakUnmodifiedRoots<GlobalHandles::VISIT_OTHERS>(
&root_visitor); &root_visitor);
ProcessMarkingDeque<MarkingMode::YOUNG_GENERATION>(); ProcessMarkingDeque();
} }
// TODO(mlippautz): External bitmap should be deallocated after evacuation. // TODO(mlippautz): External bitmap should be deallocated after evacuation.
...@@ -2339,6 +2336,32 @@ void MarkCompactCollector::MarkLiveObjectsInYoungGeneration() { ...@@ -2339,6 +2336,32 @@ void MarkCompactCollector::MarkLiveObjectsInYoungGeneration() {
marking_deque()->StopUsing(); marking_deque()->StopUsing();
} }
void MinorMarkCompactCollector::ProcessMarkingDeque() {
EmptyMarkingDeque();
DCHECK(!marking_deque()->overflowed());
DCHECK(marking_deque()->IsEmpty());
}
void MinorMarkCompactCollector::EmptyMarkingDeque() {
while (!marking_deque()->IsEmpty()) {
HeapObject* object = marking_deque()->Pop();
DCHECK(!object->IsFiller());
DCHECK(object->IsHeapObject());
DCHECK(heap()->Contains(object));
DCHECK(!(ObjectMarking::IsWhite<MarkBit::NON_ATOMIC>(
object, StateForObject(object))));
Map* map = object->map();
DCHECK((ObjectMarking::IsBlack<MarkBit::NON_ATOMIC>(
object, StateForObject(object))));
StaticYoungGenerationMarkingVisitor::IterateBody(map, object);
}
}
void MinorMarkCompactCollector::CollectGarbage() { MarkLiveObjects(); }
void MarkCompactCollector::MarkLiveObjects() { void MarkCompactCollector::MarkLiveObjects() {
TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK); TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK);
// The recursive GC marker detects when it is nearing stack overflow, // The recursive GC marker detects when it is nearing stack overflow,
...@@ -2370,7 +2393,7 @@ void MarkCompactCollector::MarkLiveObjects() { ...@@ -2370,7 +2393,7 @@ void MarkCompactCollector::MarkLiveObjects() {
PrepareForCodeFlushing(); PrepareForCodeFlushing();
} }
RootMarkingVisitor<MarkingMode::FULL> root_visitor(heap()); RootMarkingVisitor root_visitor(heap());
{ {
TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK_ROOTS); TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK_ROOTS);
...@@ -2402,7 +2425,7 @@ void MarkCompactCollector::MarkLiveObjects() { ...@@ -2402,7 +2425,7 @@ void MarkCompactCollector::MarkLiveObjects() {
GCTracer::Scope::MC_MARK_WEAK_CLOSURE_WEAK_HANDLES); GCTracer::Scope::MC_MARK_WEAK_CLOSURE_WEAK_HANDLES);
heap()->isolate()->global_handles()->IdentifyWeakHandles( heap()->isolate()->global_handles()->IdentifyWeakHandles(
&IsUnmarkedHeapObject); &IsUnmarkedHeapObject);
ProcessMarkingDeque<MarkingMode::FULL>(); ProcessMarkingDeque();
} }
// Then we mark the objects. // Then we mark the objects.
...@@ -2410,7 +2433,7 @@ void MarkCompactCollector::MarkLiveObjects() { ...@@ -2410,7 +2433,7 @@ void MarkCompactCollector::MarkLiveObjects() {
TRACE_GC(heap()->tracer(), TRACE_GC(heap()->tracer(),
GCTracer::Scope::MC_MARK_WEAK_CLOSURE_WEAK_ROOTS); GCTracer::Scope::MC_MARK_WEAK_CLOSURE_WEAK_ROOTS);
heap()->isolate()->global_handles()->IterateWeakRoots(&root_visitor); heap()->isolate()->global_handles()->IterateWeakRoots(&root_visitor);
ProcessMarkingDeque<MarkingMode::FULL>(); ProcessMarkingDeque();
} }
// Repeat Harmony weak maps marking to mark unmarked objects reachable from // Repeat Harmony weak maps marking to mark unmarked objects reachable from
...@@ -3350,7 +3373,7 @@ int MarkCompactCollector::Sweeper::RawSweep( ...@@ -3350,7 +3373,7 @@ int MarkCompactCollector::Sweeper::RawSweep(
intptr_t max_freed_bytes = 0; intptr_t max_freed_bytes = 0;
int curr_region = -1; int curr_region = -1;
LiveObjectIterator<kBlackObjects> it(p); LiveObjectIterator<kBlackObjects> it(p, MarkingState::FromPageInternal(p));
HeapObject* object = NULL; HeapObject* object = NULL;
while ((object = it.Next()) != NULL) { while ((object = it.Next()) != NULL) {
...@@ -3467,7 +3490,8 @@ void MarkCompactCollector::RecordLiveSlotsOnPage(Page* page) { ...@@ -3467,7 +3490,8 @@ void MarkCompactCollector::RecordLiveSlotsOnPage(Page* page) {
template <class Visitor> template <class Visitor>
bool MarkCompactCollector::VisitLiveObjects(MemoryChunk* page, Visitor* visitor, bool MarkCompactCollector::VisitLiveObjects(MemoryChunk* page, Visitor* visitor,
IterationMode mode) { IterationMode mode) {
LiveObjectIterator<kBlackObjects> it(page); LiveObjectIterator<kBlackObjects> it(page,
MarkingState::FromPageInternal(page));
HeapObject* object = nullptr; HeapObject* object = nullptr;
while ((object = it.Next()) != nullptr) { while ((object = it.Next()) != nullptr) {
DCHECK(ObjectMarking::IsBlack(object)); DCHECK(ObjectMarking::IsBlack(object));
...@@ -3495,9 +3519,9 @@ bool MarkCompactCollector::VisitLiveObjects(MemoryChunk* page, Visitor* visitor, ...@@ -3495,9 +3519,9 @@ bool MarkCompactCollector::VisitLiveObjects(MemoryChunk* page, Visitor* visitor,
return true; return true;
} }
void MarkCompactCollector::RecomputeLiveBytes(MemoryChunk* page) { void MarkCompactCollector::RecomputeLiveBytes(MemoryChunk* page) {
LiveObjectIterator<kBlackObjects> it(page); LiveObjectIterator<kBlackObjects> it(page,
MarkingState::FromPageInternal(page));
int new_live_size = 0; int new_live_size = 0;
HeapObject* object = nullptr; HeapObject* object = nullptr;
while ((object = it.Next()) != nullptr) { while ((object = it.Next()) != nullptr) {
...@@ -3740,7 +3764,8 @@ class ToSpacePointerUpdateJobTraits { ...@@ -3740,7 +3764,8 @@ class ToSpacePointerUpdateJobTraits {
static void ProcessPageInParallelVisitLive(Heap* heap, PerTaskData visitor, static void ProcessPageInParallelVisitLive(Heap* heap, PerTaskData visitor,
MemoryChunk* chunk, MemoryChunk* chunk,
PerPageData limits) { PerPageData limits) {
LiveObjectIterator<kBlackObjects> it(chunk); LiveObjectIterator<kBlackObjects> it(chunk,
MarkingState::FromPageInternal(chunk));
HeapObject* object = NULL; HeapObject* object = NULL;
while ((object = it.Next()) != NULL) { while ((object = it.Next()) != NULL) {
Map* map = object->map(); Map* map = object->map();
......
...@@ -28,87 +28,181 @@ typedef void (*MarkObjectFunction)(Heap* heap, HeapObject* object); ...@@ -28,87 +28,181 @@ typedef void (*MarkObjectFunction)(Heap* heap, HeapObject* object);
// Forward declarations. // Forward declarations.
class CodeFlusher; class CodeFlusher;
class MarkCompactCollector; class MarkCompactCollector;
class MinorMarkCompactCollector;
class MarkingVisitor; class MarkingVisitor;
template <MarkingMode mode>
class RootMarkingVisitor;
class MarkingState {
public:
static MarkingState FromPageInternal(MemoryChunk* chunk) {
return MarkingState(chunk->markbits<MarkingMode::FULL>(),
chunk->live_bytes_address<MarkingMode::FULL>());
}
static MarkingState FromPageExternal(MemoryChunk* chunk) {
return MarkingState(
chunk->markbits<MarkingMode::YOUNG_GENERATION>(),
chunk->live_bytes_address<MarkingMode::YOUNG_GENERATION>());
}
MarkingState(Bitmap* bitmap, intptr_t* live_bytes)
: bitmap(bitmap), live_bytes(live_bytes) {}
void IncrementLiveBytes(intptr_t by) const { *live_bytes += by; }
void SetLiveBytes(intptr_t value) const { *live_bytes = value; }
void ClearLiveness() const {
bitmap->Clear();
*live_bytes = 0;
}
Bitmap* bitmap;
intptr_t* live_bytes;
};
// TODO(mlippautz): Remove duplicate accessors once the architecture for
// different markers is fixed.
class ObjectMarking : public AllStatic { class ObjectMarking : public AllStatic {
public: public:
template <MarkingMode mode = MarkingMode::FULL>
V8_INLINE static MarkBit MarkBitFrom(HeapObject* obj) { V8_INLINE static MarkBit MarkBitFrom(HeapObject* obj) {
const Address address = obj->address(); const Address address = obj->address();
const MemoryChunk* p = MemoryChunk::FromAddress(address); const MemoryChunk* p = MemoryChunk::FromAddress(address);
return p->markbits<mode>()->MarkBitFromIndex( return p->markbits()->MarkBitFromIndex(p->AddressToMarkbitIndex(address));
p->AddressToMarkbitIndex(address)); }
V8_INLINE static MarkBit MarkBitFrom(HeapObject* obj,
const MarkingState& state) {
const Address address = obj->address();
const MemoryChunk* p = MemoryChunk::FromAddress(address);
return state.bitmap->MarkBitFromIndex(p->AddressToMarkbitIndex(address));
} }
template <MarkingMode mode = MarkingMode::FULL>
static Marking::ObjectColor Color(HeapObject* obj) { static Marking::ObjectColor Color(HeapObject* obj) {
return Marking::Color(ObjectMarking::MarkBitFrom<mode>(obj)); return Marking::Color(ObjectMarking::MarkBitFrom(obj));
}
static Marking::ObjectColor Color(HeapObject* obj,
const MarkingState& state) {
return Marking::Color(ObjectMarking::MarkBitFrom(obj, state));
} }
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC, template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
MarkingMode mode = MarkingMode::FULL>
V8_INLINE static bool IsImpossible(HeapObject* obj) { V8_INLINE static bool IsImpossible(HeapObject* obj) {
return Marking::IsImpossible<access_mode>(MarkBitFrom<mode>(obj)); return Marking::IsImpossible<access_mode>(MarkBitFrom(obj));
} }
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC, template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
MarkingMode mode = MarkingMode::FULL> V8_INLINE static bool IsImpossible(HeapObject* obj,
const MarkingState& state) {
return Marking::IsImpossible<access_mode>(MarkBitFrom(obj, state));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool IsBlack(HeapObject* obj) { V8_INLINE static bool IsBlack(HeapObject* obj) {
return Marking::IsBlack<access_mode>(MarkBitFrom<mode>(obj)); return Marking::IsBlack<access_mode>(MarkBitFrom(obj));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool IsBlack(HeapObject* obj, const MarkingState& state) {
return Marking::IsBlack<access_mode>(MarkBitFrom(obj, state));
} }
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC, template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
MarkingMode mode = MarkingMode::FULL>
V8_INLINE static bool IsWhite(HeapObject* obj) { V8_INLINE static bool IsWhite(HeapObject* obj) {
return Marking::IsWhite<access_mode>(MarkBitFrom<mode>(obj)); return Marking::IsWhite<access_mode>(MarkBitFrom(obj));
} }
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC, template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
MarkingMode mode = MarkingMode::FULL> V8_INLINE static bool IsWhite(HeapObject* obj, const MarkingState& state) {
return Marking::IsWhite<access_mode>(MarkBitFrom(obj, state));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool IsGrey(HeapObject* obj) { V8_INLINE static bool IsGrey(HeapObject* obj) {
return Marking::IsGrey<access_mode>(MarkBitFrom<mode>(obj)); return Marking::IsGrey<access_mode>(MarkBitFrom(obj));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool IsGrey(HeapObject* obj, const MarkingState& state) {
return Marking::IsGrey<access_mode>(MarkBitFrom(obj, state));
} }
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC, template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
MarkingMode mode = MarkingMode::FULL>
V8_INLINE static bool IsBlackOrGrey(HeapObject* obj) { V8_INLINE static bool IsBlackOrGrey(HeapObject* obj) {
return Marking::IsBlackOrGrey<access_mode>(MarkBitFrom<mode>(obj)); return Marking::IsBlackOrGrey<access_mode>(MarkBitFrom(obj));
} }
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC, template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
MarkingMode mode = MarkingMode::FULL> V8_INLINE static bool IsBlackOrGrey(HeapObject* obj,
const MarkingState& state) {
return Marking::IsBlackOrGrey<access_mode>(MarkBitFrom(obj, state));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool BlackToGrey(HeapObject* obj) { V8_INLINE static bool BlackToGrey(HeapObject* obj) {
DCHECK((access_mode == MarkBit::ATOMIC || IsBlack<access_mode, mode>(obj))); DCHECK((access_mode == MarkBit::ATOMIC || IsBlack<access_mode>(obj)));
MarkBit markbit = MarkBitFrom<mode>(obj); MarkBit markbit = MarkBitFrom(obj);
if (!Marking::BlackToGrey<access_mode>(markbit)) return false;
MemoryChunk::IncrementLiveBytes(obj, -obj->Size());
return true;
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool BlackToGrey(HeapObject* obj,
const MarkingState& state) {
DCHECK(
(access_mode == MarkBit::ATOMIC || IsBlack<access_mode>(obj, state)));
MarkBit markbit = MarkBitFrom(obj, state);
if (!Marking::BlackToGrey<access_mode>(markbit)) return false; if (!Marking::BlackToGrey<access_mode>(markbit)) return false;
MemoryChunk::IncrementLiveBytes<mode>(obj, -obj->Size()); state.IncrementLiveBytes(-obj->Size());
return true; return true;
} }
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC, template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
MarkingMode mode = MarkingMode::FULL>
V8_INLINE static bool WhiteToGrey(HeapObject* obj) { V8_INLINE static bool WhiteToGrey(HeapObject* obj) {
DCHECK((access_mode == MarkBit::ATOMIC || IsWhite<access_mode, mode>(obj))); DCHECK((access_mode == MarkBit::ATOMIC || IsWhite<access_mode>(obj)));
return Marking::WhiteToGrey<access_mode>(MarkBitFrom<mode>(obj)); return Marking::WhiteToGrey<access_mode>(MarkBitFrom(obj));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool WhiteToGrey(HeapObject* obj,
const MarkingState& state) {
DCHECK(
(access_mode == MarkBit::ATOMIC || IsWhite<access_mode>(obj, state)));
return Marking::WhiteToGrey<access_mode>(MarkBitFrom(obj, state));
} }
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC, template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
MarkingMode mode = MarkingMode::FULL>
V8_INLINE static bool WhiteToBlack(HeapObject* obj) { V8_INLINE static bool WhiteToBlack(HeapObject* obj) {
DCHECK((access_mode == MarkBit::ATOMIC || IsWhite<access_mode, mode>(obj))); DCHECK((access_mode == MarkBit::ATOMIC || IsWhite<access_mode>(obj)));
if (!ObjectMarking::WhiteToGrey<access_mode, mode>(obj)) return false; if (!ObjectMarking::WhiteToGrey<access_mode>(obj)) return false;
return ObjectMarking::GreyToBlack<access_mode, mode>(obj); return ObjectMarking::GreyToBlack<access_mode>(obj);
} }
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC, template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
MarkingMode mode = MarkingMode::FULL> V8_INLINE static bool WhiteToBlack(HeapObject* obj,
const MarkingState& state) {
DCHECK(
(access_mode == MarkBit::ATOMIC || IsWhite<access_mode>(obj, state)));
if (!ObjectMarking::WhiteToGrey<access_mode>(obj, state)) return false;
return ObjectMarking::GreyToBlack<access_mode>(obj, state);
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool GreyToBlack(HeapObject* obj) { V8_INLINE static bool GreyToBlack(HeapObject* obj) {
DCHECK((access_mode == MarkBit::ATOMIC || IsGrey<access_mode, mode>(obj))); DCHECK((access_mode == MarkBit::ATOMIC || IsGrey<access_mode>(obj)));
MarkBit markbit = MarkBitFrom<mode>(obj); MarkBit markbit = MarkBitFrom(obj);
if (!Marking::GreyToBlack<access_mode>(markbit)) return false; if (!Marking::GreyToBlack<access_mode>(markbit)) return false;
MemoryChunk::IncrementLiveBytes<mode>(obj, obj->Size()); MemoryChunk::IncrementLiveBytes(obj, obj->Size());
return true;
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool GreyToBlack(HeapObject* obj,
const MarkingState& state) {
DCHECK((access_mode == MarkBit::ATOMIC || IsGrey<access_mode>(obj, state)));
MarkBit markbit = MarkBitFrom(obj, state);
if (!Marking::GreyToBlack<access_mode>(markbit)) return false;
state.IncrementLiveBytes(obj->Size());
return true; return true;
} }
...@@ -308,13 +402,13 @@ class ThreadLocalTop; ...@@ -308,13 +402,13 @@ class ThreadLocalTop;
class MarkBitCellIterator BASE_EMBEDDED { class MarkBitCellIterator BASE_EMBEDDED {
public: public:
explicit MarkBitCellIterator(MemoryChunk* chunk) : chunk_(chunk) { MarkBitCellIterator(MemoryChunk* chunk, MarkingState state) : chunk_(chunk) {
last_cell_index_ = Bitmap::IndexToCell(Bitmap::CellAlignIndex( last_cell_index_ = Bitmap::IndexToCell(Bitmap::CellAlignIndex(
chunk_->AddressToMarkbitIndex(chunk_->area_end()))); chunk_->AddressToMarkbitIndex(chunk_->area_end())));
cell_base_ = chunk_->area_start(); cell_base_ = chunk_->area_start();
cell_index_ = Bitmap::IndexToCell( cell_index_ = Bitmap::IndexToCell(
Bitmap::CellAlignIndex(chunk_->AddressToMarkbitIndex(cell_base_))); Bitmap::CellAlignIndex(chunk_->AddressToMarkbitIndex(cell_base_)));
cells_ = chunk_->markbits()->cells(); cells_ = state.bitmap->cells();
} }
inline bool Done() { return cell_index_ == last_cell_index_; } inline bool Done() { return cell_index_ == last_cell_index_; }
...@@ -377,12 +471,11 @@ enum LiveObjectIterationMode { ...@@ -377,12 +471,11 @@ enum LiveObjectIterationMode {
template <LiveObjectIterationMode T> template <LiveObjectIterationMode T>
class LiveObjectIterator BASE_EMBEDDED { class LiveObjectIterator BASE_EMBEDDED {
public: public:
explicit LiveObjectIterator(MemoryChunk* chunk) LiveObjectIterator(MemoryChunk* chunk, MarkingState state)
: chunk_(chunk), : chunk_(chunk),
it_(chunk_), it_(chunk_, state),
cell_base_(it_.CurrentCellBase()), cell_base_(it_.CurrentCellBase()),
current_cell_(*it_.CurrentCell()) { current_cell_(*it_.CurrentCell()) {}
}
HeapObject* Next(); HeapObject* Next();
...@@ -397,11 +490,47 @@ class LiveObjectIterator BASE_EMBEDDED { ...@@ -397,11 +490,47 @@ class LiveObjectIterator BASE_EMBEDDED {
enum PageEvacuationMode { NEW_TO_NEW, NEW_TO_OLD }; enum PageEvacuationMode { NEW_TO_NEW, NEW_TO_OLD };
class MinorMarkCompactCollector {
public:
explicit MinorMarkCompactCollector(Heap* heap)
: heap_(heap), marking_deque_(heap) {}
void SetUp();
void TearDown();
void CollectGarbage();
private:
class RootMarkingVisitor;
static MarkingState StateForObject(HeapObject* object) {
return MarkingState::FromPageExternal(Page::FromAddress(object->address()));
}
inline Heap* heap() { return heap_; }
inline Isolate* isolate() { return heap()->isolate(); }
inline MarkingDeque* marking_deque() { return &marking_deque_; }
V8_INLINE void MarkObject(HeapObject* obj);
V8_INLINE void PushBlack(HeapObject* obj);
SlotCallbackResult CheckAndMarkObject(Heap* heap, Address slot_address);
void MarkLiveObjects();
void ProcessMarkingDeque();
void EmptyMarkingDeque();
Heap* heap_;
MarkingDeque marking_deque_;
friend class StaticYoungGenerationMarkingVisitor;
};
// ------------------------------------------------------------------------- // -------------------------------------------------------------------------
// Mark-Compact collector // Mark-Compact collector
class MarkCompactCollector { class MarkCompactCollector {
public: public:
class Evacuator; class Evacuator;
class RootMarkingVisitor;
class Sweeper { class Sweeper {
public: public:
...@@ -567,11 +696,7 @@ class MarkCompactCollector { ...@@ -567,11 +696,7 @@ class MarkCompactCollector {
bool evacuation() const { return evacuation_; } bool evacuation() const { return evacuation_; }
template <MarkingMode mode = MarkingMode::FULL> MarkingDeque* marking_deque() { return &marking_deque_; }
MarkingDeque* marking_deque() {
return mode == MarkingMode::FULL ? &marking_deque_
: &marking_deque_young_generation_;
}
Sweeper& sweeper() { return sweeper_; } Sweeper& sweeper() { return sweeper_; }
...@@ -629,8 +754,6 @@ class MarkCompactCollector { ...@@ -629,8 +754,6 @@ class MarkCompactCollector {
friend class MarkCompactMarkingVisitor; friend class MarkCompactMarkingVisitor;
friend class MarkingVisitor; friend class MarkingVisitor;
friend class RecordMigratedSlotVisitor; friend class RecordMigratedSlotVisitor;
template <MarkingMode mode>
friend class RootMarkingVisitor;
friend class SharedFunctionInfoMarkingVisitor; friend class SharedFunctionInfoMarkingVisitor;
friend class StaticYoungGenerationMarkingVisitor; friend class StaticYoungGenerationMarkingVisitor;
...@@ -642,12 +765,9 @@ class MarkCompactCollector { ...@@ -642,12 +765,9 @@ class MarkCompactCollector {
// Marking operations for objects reachable from roots. // Marking operations for objects reachable from roots.
void MarkLiveObjects(); void MarkLiveObjects();
// Mark the young generation.
void MarkLiveObjectsInYoungGeneration();
// Pushes a black object onto the marking stack and accounts for live bytes. // Pushes a black object onto the marking stack and accounts for live bytes.
// Note that this assumes live bytes have not yet been counted. // Note that this assumes live bytes have not yet been counted.
template <MarkingMode mode = MarkingMode::FULL>
V8_INLINE void PushBlack(HeapObject* obj); V8_INLINE void PushBlack(HeapObject* obj);
// Unshifts a black object into the marking stack and accounts for live bytes. // Unshifts a black object into the marking stack and accounts for live bytes.
...@@ -656,19 +776,17 @@ class MarkCompactCollector { ...@@ -656,19 +776,17 @@ class MarkCompactCollector {
// Marks the object black and pushes it on the marking stack. // Marks the object black and pushes it on the marking stack.
// This is for non-incremental marking only. // This is for non-incremental marking only.
template <MarkingMode mode = MarkingMode::FULL>
V8_INLINE void MarkObject(HeapObject* obj); V8_INLINE void MarkObject(HeapObject* obj);
// Mark the heap roots and all objects reachable from them. // Mark the heap roots and all objects reachable from them.
void MarkRoots(RootMarkingVisitor<MarkingMode::FULL>* visitor); void MarkRoots(RootMarkingVisitor* visitor);
// Mark the string table specially. References to internalized strings from // Mark the string table specially. References to internalized strings from
// the string table are weak. // the string table are weak.
void MarkStringTable(RootMarkingVisitor<MarkingMode::FULL>* visitor); void MarkStringTable(RootMarkingVisitor* visitor);
// Mark objects reachable (transitively) from objects in the marking stack // Mark objects reachable (transitively) from objects in the marking stack
// or overflowed in the heap. // or overflowed in the heap.
template <MarkingMode mode>
void ProcessMarkingDeque(); void ProcessMarkingDeque();
// Mark objects reachable (transitively) from objects in the marking stack // Mark objects reachable (transitively) from objects in the marking stack
...@@ -692,13 +810,11 @@ class MarkCompactCollector { ...@@ -692,13 +810,11 @@ class MarkCompactCollector {
// stack. This function empties the marking stack, but may leave // stack. This function empties the marking stack, but may leave
// overflowed objects in the heap, in which case the marking stack's // overflowed objects in the heap, in which case the marking stack's
// overflow flag will be set. // overflow flag will be set.
template <MarkingMode mode>
void EmptyMarkingDeque(); void EmptyMarkingDeque();
// Refill the marking stack with overflowed objects from the heap. This // Refill the marking stack with overflowed objects from the heap. This
// function either leaves the marking stack full or clears the overflow // function either leaves the marking stack full or clears the overflow
// flag on the marking stack. // flag on the marking stack.
template <MarkingMode mode>
void RefillMarkingDeque(); void RefillMarkingDeque();
// Helper methods for refilling the marking stack by discovering grey objects // Helper methods for refilling the marking stack by discovering grey objects
...@@ -816,7 +932,6 @@ class MarkCompactCollector { ...@@ -816,7 +932,6 @@ class MarkCompactCollector {
bool have_code_to_deoptimize_; bool have_code_to_deoptimize_;
MarkingDeque marking_deque_; MarkingDeque marking_deque_;
MarkingDeque marking_deque_young_generation_;
CodeFlusher* code_flusher_; CodeFlusher* code_flusher_;
......
...@@ -526,6 +526,14 @@ class MemoryChunk { ...@@ -526,6 +526,14 @@ class MemoryChunk {
: young_generation_bitmap_; : young_generation_bitmap_;
} }
template <MarkingMode mode = MarkingMode::FULL>
inline intptr_t* live_bytes_address() {
// TODO(mlippautz): Fix type of live_byte_count_.
return mode == MarkingMode::FULL
? reinterpret_cast<intptr_t*>(&live_byte_count_)
: &young_generation_live_byte_count_;
}
inline uint32_t AddressToMarkbitIndex(Address addr) const { inline uint32_t AddressToMarkbitIndex(Address addr) const {
return static_cast<uint32_t>(addr - this->address()) >> kPointerSizeLog2; return static_cast<uint32_t>(addr - this->address()) >> kPointerSizeLog2;
} }
......
...@@ -354,7 +354,9 @@ TEST(Regress5829) { ...@@ -354,7 +354,9 @@ TEST(Regress5829) {
heap->CreateFillerObjectAt(old_end - kPointerSize, kPointerSize, heap->CreateFillerObjectAt(old_end - kPointerSize, kPointerSize,
ClearRecordedSlots::kNo); ClearRecordedSlots::kNo);
heap->old_space()->EmptyAllocationInfo(); heap->old_space()->EmptyAllocationInfo();
LiveObjectIterator<kGreyObjects> it(Page::FromAddress(array->address())); Page* page = Page::FromAddress(array->address());
LiveObjectIterator<kGreyObjects> it(page,
MarkingState::FromPageInternal(page));
HeapObject* object = nullptr; HeapObject* object = nullptr;
while ((object = it.Next()) != nullptr) { while ((object = it.Next()) != nullptr) {
CHECK(!object->IsFiller()); CHECK(!object->IsFiller());
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment