Commit 173109b9 authored by Michael Lippautz's avatar Michael Lippautz Committed by Commit Bot

[heap] Introduce explicit marking state and instance for minor MC

BUG=chromium:651354

Change-Id: Idcd7780f53ad07b3d782a66455f9c60addc2418d
Reviewed-on: https://chromium-review.googlesource.com/457317
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: 's avatarHannes Payer <hpayer@chromium.org>
Reviewed-by: 's avatarUlan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#43971}
parent 74daa15e
......@@ -657,6 +657,8 @@ DEFINE_INT(min_progress_during_incremental_marking_finalization, 32,
DEFINE_INT(max_incremental_marking_finalization_rounds, 3,
"at most try this many times to finalize incremental marking")
DEFINE_BOOL(minor_mc, false, "perform young generation mark compact GCs")
DEFINE_NEG_IMPLICATION(minor_mc, page_promotion)
DEFINE_NEG_IMPLICATION(minor_mc, flush_code)
DEFINE_BOOL(black_allocation, true, "use black allocation")
DEFINE_BOOL(concurrent_store_buffer, true,
"use concurrent store buffer processing")
......
......@@ -82,6 +82,7 @@ enum ScavengeSpeedMode { kForAllObjects, kForSurvivedObjects };
F(MC_SWEEP_CODE) \
F(MC_SWEEP_MAP) \
F(MC_SWEEP_OLD) \
F(MC_MINOR_MC) \
F(MINOR_MC_MARK) \
F(MINOR_MC_MARK_CODE_FLUSH_CANDIDATES) \
F(MINOR_MC_MARK_GLOBAL_HANDLES) \
......
......@@ -133,6 +133,7 @@ Heap::Heap()
last_gc_time_(0.0),
scavenge_collector_(nullptr),
mark_compact_collector_(nullptr),
minor_mark_compact_collector_(nullptr),
memory_allocator_(nullptr),
store_buffer_(nullptr),
incremental_marking_(nullptr),
......@@ -1466,7 +1467,21 @@ void Heap::MarkCompact() {
}
}
void Heap::MinorMarkCompact() { UNREACHABLE(); }
void Heap::MinorMarkCompact() {
DCHECK(FLAG_minor_mc);
SetGCState(MINOR_MARK_COMPACT);
LOG(isolate_, ResourceEvent("MinorMarkCompact", "begin"));
TRACE_GC(tracer(), GCTracer::Scope::MC_MINOR_MC);
AlwaysAllocateScope always_allocate(isolate());
PauseAllocationObserversScope pause_observers(this);
minor_mark_compact_collector()->CollectGarbage();
LOG(isolate_, ResourceEvent("MinorMarkCompact", "end"));
SetGCState(NOT_IN_GC);
}
void Heap::MarkCompactEpilogue() {
TRACE_GC(tracer(), GCTracer::Scope::MC_EPILOGUE);
......@@ -5519,6 +5534,8 @@ bool Heap::SetUp() {
tracer_ = new GCTracer(this);
scavenge_collector_ = new Scavenger(this);
mark_compact_collector_ = new MarkCompactCollector(this);
if (FLAG_minor_mc)
minor_mark_compact_collector_ = new MinorMarkCompactCollector(this);
gc_idle_time_handler_ = new GCIdleTimeHandler();
memory_reducer_ = new MemoryReducer(this);
if (V8_UNLIKELY(FLAG_gc_stats)) {
......@@ -5534,6 +5551,9 @@ bool Heap::SetUp() {
store_buffer()->SetUp();
mark_compact_collector()->SetUp();
if (minor_mark_compact_collector() != nullptr) {
minor_mark_compact_collector()->SetUp();
}
idle_scavenge_observer_ = new IdleScavengeObserver(
*this, ScavengeJob::kBytesAllocatedBeforeNextIdleTask);
......@@ -5655,6 +5675,12 @@ void Heap::TearDown() {
mark_compact_collector_ = nullptr;
}
if (minor_mark_compact_collector_ != nullptr) {
minor_mark_compact_collector_->TearDown();
delete minor_mark_compact_collector_;
minor_mark_compact_collector_ = nullptr;
}
delete incremental_marking_;
incremental_marking_ = nullptr;
......
......@@ -325,6 +325,7 @@ class Isolate;
class LocalEmbedderHeapTracer;
class MemoryAllocator;
class MemoryReducer;
class MinorMarkCompactCollector;
class ObjectIterator;
class ObjectStats;
class Page;
......@@ -559,7 +560,7 @@ class Heap {
enum FindMementoMode { kForRuntime, kForGC };
enum HeapState { NOT_IN_GC, SCAVENGE, MARK_COMPACT };
enum HeapState { NOT_IN_GC, SCAVENGE, MARK_COMPACT, MINOR_MARK_COMPACT };
enum UpdateAllocationSiteMode { kGlobal, kCached };
......@@ -1040,6 +1041,10 @@ class Heap {
return mark_compact_collector_;
}
MinorMarkCompactCollector* minor_mark_compact_collector() {
return minor_mark_compact_collector_;
}
// ===========================================================================
// Root set access. ==========================================================
// ===========================================================================
......@@ -2278,6 +2283,7 @@ class Heap {
Scavenger* scavenge_collector_;
MarkCompactCollector* mark_compact_collector_;
MinorMarkCompactCollector* minor_mark_compact_collector_;
MemoryAllocator* memory_allocator_;
......@@ -2377,6 +2383,7 @@ class Heap {
friend class IncrementalMarkingJob;
friend class LargeObjectSpace;
friend class MarkCompactCollector;
friend class MinorMarkCompactCollector;
friend class MarkCompactMarkingVisitor;
friend class NewSpace;
friend class ObjectStatsCollector;
......
......@@ -12,14 +12,20 @@
namespace v8 {
namespace internal {
template <MarkingMode mode>
void MarkCompactCollector::PushBlack(HeapObject* obj) {
DCHECK((ObjectMarking::IsBlack<MarkBit::NON_ATOMIC, mode>(obj)));
if (!marking_deque<mode>()->Push(obj)) {
ObjectMarking::BlackToGrey<MarkBit::NON_ATOMIC, mode>(obj);
DCHECK((ObjectMarking::IsBlack<MarkBit::NON_ATOMIC>(obj)));
if (!marking_deque()->Push(obj)) {
ObjectMarking::BlackToGrey<MarkBit::NON_ATOMIC>(obj);
}
}
void MinorMarkCompactCollector::PushBlack(HeapObject* obj) {
DCHECK(
(ObjectMarking::IsBlack<MarkBit::NON_ATOMIC>(obj, StateForObject(obj))));
if (!marking_deque()->Push(obj)) {
ObjectMarking::BlackToGrey<MarkBit::NON_ATOMIC>(obj, StateForObject(obj));
}
}
void MarkCompactCollector::UnshiftBlack(HeapObject* obj) {
DCHECK(ObjectMarking::IsBlack(obj));
......@@ -28,11 +34,17 @@ void MarkCompactCollector::UnshiftBlack(HeapObject* obj) {
}
}
template <MarkingMode mode>
void MarkCompactCollector::MarkObject(HeapObject* obj) {
if (ObjectMarking::IsWhite<MarkBit::NON_ATOMIC, mode>(obj)) {
ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC, mode>(obj);
PushBlack<mode>(obj);
if (ObjectMarking::IsWhite<MarkBit::NON_ATOMIC>(obj)) {
ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC>(obj);
PushBlack(obj);
}
}
void MinorMarkCompactCollector::MarkObject(HeapObject* obj) {
if (ObjectMarking::IsWhite<MarkBit::NON_ATOMIC>(obj, StateForObject(obj))) {
ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC>(obj, StateForObject(obj));
PushBlack(obj);
}
}
......
......@@ -60,7 +60,6 @@ MarkCompactCollector::MarkCompactCollector(Heap* heap)
black_allocation_(false),
have_code_to_deoptimize_(false),
marking_deque_(heap),
marking_deque_young_generation_(heap),
code_flusher_(nullptr),
sweeper_(heap) {
}
......@@ -239,7 +238,6 @@ void MarkCompactCollector::SetUp() {
DCHECK(strcmp(Marking::kGreyBitPattern, "10") == 0);
DCHECK(strcmp(Marking::kImpossibleBitPattern, "01") == 0);
marking_deque()->SetUp();
marking_deque<MarkingMode::YOUNG_GENERATION>()->SetUp();
if (FLAG_flush_code) {
code_flusher_ = new CodeFlusher(isolate());
......@@ -249,14 +247,15 @@ void MarkCompactCollector::SetUp() {
}
}
void MinorMarkCompactCollector::SetUp() { marking_deque()->SetUp(); }
void MarkCompactCollector::TearDown() {
AbortCompaction();
marking_deque<MarkingMode::YOUNG_GENERATION>()->TearDown();
marking_deque()->TearDown();
delete code_flusher_;
}
void MinorMarkCompactCollector::TearDown() { marking_deque()->TearDown(); }
void MarkCompactCollector::AddEvacuationCandidate(Page* p) {
DCHECK(!p->NeverEvacuate());
......@@ -1067,9 +1066,9 @@ class StaticYoungGenerationMarkingVisitor
inline static void VisitPointer(Heap* heap, HeapObject* object, Object** p) {
Object* target = *p;
if (heap->InNewSpace(target)) {
if (MarkRecursively(heap, HeapObject::cast(target))) return;
heap->mark_compact_collector()->MarkObject<MarkingMode::YOUNG_GENERATION>(
HeapObject::cast(target));
HeapObject* target_object = HeapObject::cast(target);
if (MarkRecursively(heap, target_object)) return;
heap->minor_mark_compact_collector()->MarkObject(target_object);
}
}
......@@ -1078,11 +1077,11 @@ class StaticYoungGenerationMarkingVisitor
StackLimitCheck check(heap->isolate());
if (check.HasOverflowed()) return false;
if (ObjectMarking::IsBlackOrGrey<MarkBit::NON_ATOMIC,
MarkingMode::YOUNG_GENERATION>(object))
const MarkingState state =
MinorMarkCompactCollector::StateForObject(object);
if (ObjectMarking::IsBlackOrGrey<MarkBit::NON_ATOMIC>(object, state))
return true;
ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC,
MarkingMode::YOUNG_GENERATION>(object);
ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC>(object, state);
IterateBody(object->map(), object);
return true;
}
......@@ -1330,13 +1329,49 @@ void MarkCompactCollector::PrepareForCodeFlushing() {
heap()->isolate()->compilation_cache()->IterateFunctions(&visitor);
heap()->isolate()->handle_scope_implementer()->Iterate(&visitor);
ProcessMarkingDeque<MarkingMode::FULL>();
ProcessMarkingDeque();
}
class MinorMarkCompactCollector::RootMarkingVisitor : public ObjectVisitor {
public:
explicit RootMarkingVisitor(MinorMarkCompactCollector* collector)
: collector_(collector) {}
void VisitPointer(Object** p) override { MarkObjectByPointer(p); }
void VisitPointers(Object** start, Object** end) override {
for (Object** p = start; p < end; p++) MarkObjectByPointer(p);
}
// Skip the weak next code link in a code object, which is visited in
// ProcessTopOptimizedFrame.
void VisitNextCodeLink(Object** p) override {}
private:
void MarkObjectByPointer(Object** p) {
if (!(*p)->IsHeapObject()) return;
HeapObject* object = HeapObject::cast(*p);
if (!collector_->heap()->InNewSpace(object)) return;
if (ObjectMarking::IsBlackOrGrey<MarkBit::NON_ATOMIC>(
object, StateForObject(object)))
return;
Map* map = object->map();
ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC>(object,
StateForObject(object));
StaticYoungGenerationMarkingVisitor::IterateBody(map, object);
collector_->EmptyMarkingDeque();
}
MinorMarkCompactCollector* collector_;
};
// Visitor class for marking heap roots.
template <MarkingMode mode>
class RootMarkingVisitor : public ObjectVisitor {
class MarkCompactCollector::RootMarkingVisitor : public ObjectVisitor {
public:
explicit RootMarkingVisitor(Heap* heap)
: collector_(heap->mark_compact_collector()) {}
......@@ -1357,30 +1392,19 @@ class RootMarkingVisitor : public ObjectVisitor {
HeapObject* object = HeapObject::cast(*p);
if (mode == MarkingMode::YOUNG_GENERATION &&
!collector_->heap()->InNewSpace(object))
return;
if (ObjectMarking::IsBlackOrGrey<MarkBit::NON_ATOMIC, mode>(object)) return;
if (ObjectMarking::IsBlackOrGrey<MarkBit::NON_ATOMIC>(object)) return;
Map* map = object->map();
// Mark the object.
ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC, mode>(object);
ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC>(object);
switch (mode) {
case MarkingMode::FULL: {
// Mark the map pointer and body, and push them on the marking stack.
collector_->MarkObject(map);
MarkCompactMarkingVisitor::IterateBody(map, object);
} break;
case MarkingMode::YOUNG_GENERATION:
StaticYoungGenerationMarkingVisitor::IterateBody(map, object);
break;
}
// Mark the map pointer and body, and push them on the marking stack.
collector_->MarkObject(map);
MarkCompactMarkingVisitor::IterateBody(map, object);
// Mark all the objects reachable from the map and body. May leave
// overflowed objects in the heap.
collector_->EmptyMarkingDeque<mode>();
collector_->EmptyMarkingDeque();
}
MarkCompactCollector* collector_;
......@@ -1482,7 +1506,7 @@ void MarkCompactCollector::DiscoverGreyObjectsWithIterator(T* it) {
void MarkCompactCollector::DiscoverGreyObjectsOnPage(MemoryChunk* p) {
DCHECK(!marking_deque()->IsFull());
LiveObjectIterator<kGreyObjects> it(p);
LiveObjectIterator<kGreyObjects> it(p, MarkingState::FromPageInternal(p));
HeapObject* object = NULL;
while ((object = it.Next()) != NULL) {
DCHECK(ObjectMarking::IsGrey(object));
......@@ -1941,8 +1965,7 @@ bool MarkCompactCollector::IsUnmarkedHeapObject(Object** p) {
return ObjectMarking::IsWhite(HeapObject::cast(o));
}
void MarkCompactCollector::MarkStringTable(
RootMarkingVisitor<MarkingMode::FULL>* visitor) {
void MarkCompactCollector::MarkStringTable(RootMarkingVisitor* visitor) {
StringTable* string_table = heap()->string_table();
// Mark the string table itself.
if (ObjectMarking::IsWhite(string_table)) {
......@@ -1951,11 +1974,10 @@ void MarkCompactCollector::MarkStringTable(
}
// Explicitly mark the prefix.
string_table->IteratePrefix(visitor);
ProcessMarkingDeque<MarkingMode::FULL>();
ProcessMarkingDeque();
}
void MarkCompactCollector::MarkRoots(
RootMarkingVisitor<MarkingMode::FULL>* visitor) {
void MarkCompactCollector::MarkRoots(RootMarkingVisitor* visitor) {
// Mark the heap roots including global variables, stack variables,
// etc., and all objects reachable from them.
heap()->IterateStrongRoots(visitor, VISIT_ONLY_STRONG);
......@@ -1964,9 +1986,9 @@ void MarkCompactCollector::MarkRoots(
MarkStringTable(visitor);
// There may be overflowed objects in the heap. Visit them now.
while (marking_deque<MarkingMode::FULL>()->overflowed()) {
RefillMarkingDeque<MarkingMode::FULL>();
EmptyMarkingDeque<MarkingMode::FULL>();
while (marking_deque()->overflowed()) {
RefillMarkingDeque();
EmptyMarkingDeque();
}
}
......@@ -1974,27 +1996,18 @@ void MarkCompactCollector::MarkRoots(
// Before: the marking stack contains zero or more heap object pointers.
// After: the marking stack is empty, and all objects reachable from the
// marking stack have been marked, or are overflowed in the heap.
template <MarkingMode mode>
void MarkCompactCollector::EmptyMarkingDeque() {
while (!marking_deque<mode>()->IsEmpty()) {
HeapObject* object = marking_deque<mode>()->Pop();
while (!marking_deque()->IsEmpty()) {
HeapObject* object = marking_deque()->Pop();
DCHECK(!object->IsFiller());
DCHECK(object->IsHeapObject());
DCHECK(heap()->Contains(object));
DCHECK(!(ObjectMarking::IsWhite<MarkBit::NON_ATOMIC, mode>(object)));
DCHECK(!(ObjectMarking::IsWhite<MarkBit::NON_ATOMIC>(object)));
Map* map = object->map();
switch (mode) {
case MarkingMode::FULL: {
MarkObject(map);
MarkCompactMarkingVisitor::IterateBody(map, object);
} break;
case MarkingMode::YOUNG_GENERATION: {
DCHECK((ObjectMarking::IsBlack<MarkBit::NON_ATOMIC, mode>(object)));
StaticYoungGenerationMarkingVisitor::IterateBody(map, object);
} break;
}
MarkObject(map);
MarkCompactMarkingVisitor::IterateBody(map, object);
}
}
......@@ -2004,43 +2017,35 @@ void MarkCompactCollector::EmptyMarkingDeque() {
// before sweeping completes. If sweeping completes, there are no remaining
// overflowed objects in the heap so the overflow flag on the markings stack
// is cleared.
template <MarkingMode mode>
void MarkCompactCollector::RefillMarkingDeque() {
// Young generation should never overflow.
DCHECK(mode != MarkingMode::YOUNG_GENERATION);
isolate()->CountUsage(v8::Isolate::UseCounterFeature::kMarkDequeOverflow);
DCHECK(marking_deque()->overflowed());
DiscoverGreyObjectsInNewSpace();
if (marking_deque()->IsFull()) return;
if (mode == MarkingMode::FULL) {
DiscoverGreyObjectsInSpace(heap()->old_space());
if (marking_deque()->IsFull()) return;
DiscoverGreyObjectsInSpace(heap()->code_space());
if (marking_deque()->IsFull()) return;
DiscoverGreyObjectsInSpace(heap()->map_space());
if (marking_deque()->IsFull()) return;
LargeObjectIterator lo_it(heap()->lo_space());
DiscoverGreyObjectsWithIterator(&lo_it);
if (marking_deque()->IsFull()) return;
}
DiscoverGreyObjectsInSpace(heap()->old_space());
if (marking_deque()->IsFull()) return;
DiscoverGreyObjectsInSpace(heap()->code_space());
if (marking_deque()->IsFull()) return;
DiscoverGreyObjectsInSpace(heap()->map_space());
if (marking_deque()->IsFull()) return;
LargeObjectIterator lo_it(heap()->lo_space());
DiscoverGreyObjectsWithIterator(&lo_it);
if (marking_deque()->IsFull()) return;
marking_deque()->ClearOverflowed();
}
// Mark all objects reachable (transitively) from objects on the marking
// stack. Before: the marking stack contains zero or more heap object
// pointers. After: the marking stack is empty and there are no overflowed
// objects in the heap.
template <MarkingMode mode>
void MarkCompactCollector::ProcessMarkingDeque() {
EmptyMarkingDeque<mode>();
while (marking_deque<mode>()->overflowed()) {
RefillMarkingDeque<mode>();
EmptyMarkingDeque<mode>();
EmptyMarkingDeque();
while (marking_deque()->overflowed()) {
RefillMarkingDeque();
EmptyMarkingDeque();
}
DCHECK(marking_deque()->IsEmpty());
}
......@@ -2070,7 +2075,7 @@ void MarkCompactCollector::ProcessEphemeralMarking(
}
ProcessWeakCollections();
work_to_do = !marking_deque()->IsEmpty();
ProcessMarkingDeque<MarkingMode::FULL>();
ProcessMarkingDeque();
}
CHECK(marking_deque()->IsEmpty());
CHECK_EQ(0, heap()->local_embedder_heap_tracer()->NumberOfWrappersToTrace());
......@@ -2087,7 +2092,7 @@ void MarkCompactCollector::ProcessTopOptimizedFrame(ObjectVisitor* visitor) {
if (!code->CanDeoptAt(it.frame()->pc())) {
Code::BodyDescriptor::IterateBody(code, visitor);
}
ProcessMarkingDeque<MarkingMode::FULL>();
ProcessMarkingDeque();
return;
}
}
......@@ -2241,7 +2246,7 @@ void MarkCompactCollector::RecordObjectStats() {
}
}
SlotCallbackResult MarkCompactCollector::CheckAndMarkObject(
SlotCallbackResult MinorMarkCompactCollector::CheckAndMarkObject(
Heap* heap, Address slot_address) {
Object* object = *reinterpret_cast<Object**>(slot_address);
if (heap->InNewSpace(object)) {
......@@ -2249,13 +2254,12 @@ SlotCallbackResult MarkCompactCollector::CheckAndMarkObject(
// has to be in ToSpace.
DCHECK(heap->InToSpace(object));
HeapObject* heap_object = reinterpret_cast<HeapObject*>(object);
if (ObjectMarking::IsBlackOrGrey<MarkBit::NON_ATOMIC,
MarkingMode::YOUNG_GENERATION>(
heap_object)) {
const MarkingState state =
MinorMarkCompactCollector::StateForObject(heap_object);
if (ObjectMarking::IsBlackOrGrey<MarkBit::NON_ATOMIC>(heap_object, state)) {
return KEEP_SLOT;
}
ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC,
MarkingMode::YOUNG_GENERATION>(heap_object);
ObjectMarking::WhiteToBlack<MarkBit::NON_ATOMIC>(heap_object, state);
StaticYoungGenerationMarkingVisitor::IterateBody(heap_object->map(),
heap_object);
return KEEP_SLOT;
......@@ -2268,15 +2272,15 @@ static bool IsUnmarkedObject(Heap* heap, Object** p) {
return heap->InNewSpace(*p) && !ObjectMarking::IsBlack(HeapObject::cast(*p));
}
void MarkCompactCollector::MarkLiveObjectsInYoungGeneration() {
void MinorMarkCompactCollector::MarkLiveObjects() {
TRACE_GC(heap()->tracer(), GCTracer::Scope::MINOR_MC_MARK);
PostponeInterruptsScope postpone(isolate());
StaticYoungGenerationMarkingVisitor::Initialize(heap());
RootMarkingVisitor<MarkingMode::YOUNG_GENERATION> root_visitor(heap());
RootMarkingVisitor root_visitor(this);
marking_deque<MarkingMode::YOUNG_GENERATION>()->StartUsing();
marking_deque()->StartUsing();
for (Page* p : heap()->new_space()->to_space()) {
p->AllocateExternalBitmap();
}
......@@ -2287,7 +2291,7 @@ void MarkCompactCollector::MarkLiveObjectsInYoungGeneration() {
{
TRACE_GC(heap()->tracer(), GCTracer::Scope::MINOR_MC_MARK_ROOTS);
heap()->IterateRoots(&root_visitor, VISIT_ALL_IN_SCAVENGE);
ProcessMarkingDeque<MarkingMode::YOUNG_GENERATION>();
ProcessMarkingDeque();
}
{
......@@ -2304,20 +2308,13 @@ void MarkCompactCollector::MarkLiveObjectsInYoungGeneration() {
reinterpret_cast<Address>(addr));
});
});
ProcessMarkingDeque<MarkingMode::YOUNG_GENERATION>();
ProcessMarkingDeque();
}
{
TRACE_GC(heap()->tracer(), GCTracer::Scope::MINOR_MC_MARK_WEAK);
heap()->VisitEncounteredWeakCollections(&root_visitor);
ProcessMarkingDeque<MarkingMode::YOUNG_GENERATION>();
}
if (is_code_flushing_enabled()) {
TRACE_GC(heap()->tracer(),
GCTracer::Scope::MINOR_MC_MARK_CODE_FLUSH_CANDIDATES);
code_flusher()->IteratePointersToFromSpace(&root_visitor);
ProcessMarkingDeque<MarkingMode::YOUNG_GENERATION>();
ProcessMarkingDeque();
}
{
......@@ -2328,7 +2325,7 @@ void MarkCompactCollector::MarkLiveObjectsInYoungGeneration() {
->global_handles()
->IterateNewSpaceWeakUnmodifiedRoots<GlobalHandles::VISIT_OTHERS>(
&root_visitor);
ProcessMarkingDeque<MarkingMode::YOUNG_GENERATION>();
ProcessMarkingDeque();
}
// TODO(mlippautz): External bitmap should be deallocated after evacuation.
......@@ -2339,6 +2336,32 @@ void MarkCompactCollector::MarkLiveObjectsInYoungGeneration() {
marking_deque()->StopUsing();
}
void MinorMarkCompactCollector::ProcessMarkingDeque() {
EmptyMarkingDeque();
DCHECK(!marking_deque()->overflowed());
DCHECK(marking_deque()->IsEmpty());
}
void MinorMarkCompactCollector::EmptyMarkingDeque() {
while (!marking_deque()->IsEmpty()) {
HeapObject* object = marking_deque()->Pop();
DCHECK(!object->IsFiller());
DCHECK(object->IsHeapObject());
DCHECK(heap()->Contains(object));
DCHECK(!(ObjectMarking::IsWhite<MarkBit::NON_ATOMIC>(
object, StateForObject(object))));
Map* map = object->map();
DCHECK((ObjectMarking::IsBlack<MarkBit::NON_ATOMIC>(
object, StateForObject(object))));
StaticYoungGenerationMarkingVisitor::IterateBody(map, object);
}
}
void MinorMarkCompactCollector::CollectGarbage() { MarkLiveObjects(); }
void MarkCompactCollector::MarkLiveObjects() {
TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK);
// The recursive GC marker detects when it is nearing stack overflow,
......@@ -2370,7 +2393,7 @@ void MarkCompactCollector::MarkLiveObjects() {
PrepareForCodeFlushing();
}
RootMarkingVisitor<MarkingMode::FULL> root_visitor(heap());
RootMarkingVisitor root_visitor(heap());
{
TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_MARK_ROOTS);
......@@ -2402,7 +2425,7 @@ void MarkCompactCollector::MarkLiveObjects() {
GCTracer::Scope::MC_MARK_WEAK_CLOSURE_WEAK_HANDLES);
heap()->isolate()->global_handles()->IdentifyWeakHandles(
&IsUnmarkedHeapObject);
ProcessMarkingDeque<MarkingMode::FULL>();
ProcessMarkingDeque();
}
// Then we mark the objects.
......@@ -2410,7 +2433,7 @@ void MarkCompactCollector::MarkLiveObjects() {
TRACE_GC(heap()->tracer(),
GCTracer::Scope::MC_MARK_WEAK_CLOSURE_WEAK_ROOTS);
heap()->isolate()->global_handles()->IterateWeakRoots(&root_visitor);
ProcessMarkingDeque<MarkingMode::FULL>();
ProcessMarkingDeque();
}
// Repeat Harmony weak maps marking to mark unmarked objects reachable from
......@@ -3350,7 +3373,7 @@ int MarkCompactCollector::Sweeper::RawSweep(
intptr_t max_freed_bytes = 0;
int curr_region = -1;
LiveObjectIterator<kBlackObjects> it(p);
LiveObjectIterator<kBlackObjects> it(p, MarkingState::FromPageInternal(p));
HeapObject* object = NULL;
while ((object = it.Next()) != NULL) {
......@@ -3467,7 +3490,8 @@ void MarkCompactCollector::RecordLiveSlotsOnPage(Page* page) {
template <class Visitor>
bool MarkCompactCollector::VisitLiveObjects(MemoryChunk* page, Visitor* visitor,
IterationMode mode) {
LiveObjectIterator<kBlackObjects> it(page);
LiveObjectIterator<kBlackObjects> it(page,
MarkingState::FromPageInternal(page));
HeapObject* object = nullptr;
while ((object = it.Next()) != nullptr) {
DCHECK(ObjectMarking::IsBlack(object));
......@@ -3495,9 +3519,9 @@ bool MarkCompactCollector::VisitLiveObjects(MemoryChunk* page, Visitor* visitor,
return true;
}
void MarkCompactCollector::RecomputeLiveBytes(MemoryChunk* page) {
LiveObjectIterator<kBlackObjects> it(page);
LiveObjectIterator<kBlackObjects> it(page,
MarkingState::FromPageInternal(page));
int new_live_size = 0;
HeapObject* object = nullptr;
while ((object = it.Next()) != nullptr) {
......@@ -3740,7 +3764,8 @@ class ToSpacePointerUpdateJobTraits {
static void ProcessPageInParallelVisitLive(Heap* heap, PerTaskData visitor,
MemoryChunk* chunk,
PerPageData limits) {
LiveObjectIterator<kBlackObjects> it(chunk);
LiveObjectIterator<kBlackObjects> it(chunk,
MarkingState::FromPageInternal(chunk));
HeapObject* object = NULL;
while ((object = it.Next()) != NULL) {
Map* map = object->map();
......
......@@ -28,87 +28,181 @@ typedef void (*MarkObjectFunction)(Heap* heap, HeapObject* object);
// Forward declarations.
class CodeFlusher;
class MarkCompactCollector;
class MinorMarkCompactCollector;
class MarkingVisitor;
template <MarkingMode mode>
class RootMarkingVisitor;
class MarkingState {
public:
static MarkingState FromPageInternal(MemoryChunk* chunk) {
return MarkingState(chunk->markbits<MarkingMode::FULL>(),
chunk->live_bytes_address<MarkingMode::FULL>());
}
static MarkingState FromPageExternal(MemoryChunk* chunk) {
return MarkingState(
chunk->markbits<MarkingMode::YOUNG_GENERATION>(),
chunk->live_bytes_address<MarkingMode::YOUNG_GENERATION>());
}
MarkingState(Bitmap* bitmap, intptr_t* live_bytes)
: bitmap(bitmap), live_bytes(live_bytes) {}
void IncrementLiveBytes(intptr_t by) const { *live_bytes += by; }
void SetLiveBytes(intptr_t value) const { *live_bytes = value; }
void ClearLiveness() const {
bitmap->Clear();
*live_bytes = 0;
}
Bitmap* bitmap;
intptr_t* live_bytes;
};
// TODO(mlippautz): Remove duplicate accessors once the architecture for
// different markers is fixed.
class ObjectMarking : public AllStatic {
public:
template <MarkingMode mode = MarkingMode::FULL>
V8_INLINE static MarkBit MarkBitFrom(HeapObject* obj) {
const Address address = obj->address();
const MemoryChunk* p = MemoryChunk::FromAddress(address);
return p->markbits<mode>()->MarkBitFromIndex(
p->AddressToMarkbitIndex(address));
return p->markbits()->MarkBitFromIndex(p->AddressToMarkbitIndex(address));
}
V8_INLINE static MarkBit MarkBitFrom(HeapObject* obj,
const MarkingState& state) {
const Address address = obj->address();
const MemoryChunk* p = MemoryChunk::FromAddress(address);
return state.bitmap->MarkBitFromIndex(p->AddressToMarkbitIndex(address));
}
template <MarkingMode mode = MarkingMode::FULL>
static Marking::ObjectColor Color(HeapObject* obj) {
return Marking::Color(ObjectMarking::MarkBitFrom<mode>(obj));
return Marking::Color(ObjectMarking::MarkBitFrom(obj));
}
static Marking::ObjectColor Color(HeapObject* obj,
const MarkingState& state) {
return Marking::Color(ObjectMarking::MarkBitFrom(obj, state));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC,
MarkingMode mode = MarkingMode::FULL>
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool IsImpossible(HeapObject* obj) {
return Marking::IsImpossible<access_mode>(MarkBitFrom<mode>(obj));
return Marking::IsImpossible<access_mode>(MarkBitFrom(obj));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC,
MarkingMode mode = MarkingMode::FULL>
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool IsImpossible(HeapObject* obj,
const MarkingState& state) {
return Marking::IsImpossible<access_mode>(MarkBitFrom(obj, state));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool IsBlack(HeapObject* obj) {
return Marking::IsBlack<access_mode>(MarkBitFrom<mode>(obj));
return Marking::IsBlack<access_mode>(MarkBitFrom(obj));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool IsBlack(HeapObject* obj, const MarkingState& state) {
return Marking::IsBlack<access_mode>(MarkBitFrom(obj, state));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC,
MarkingMode mode = MarkingMode::FULL>
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool IsWhite(HeapObject* obj) {
return Marking::IsWhite<access_mode>(MarkBitFrom<mode>(obj));
return Marking::IsWhite<access_mode>(MarkBitFrom(obj));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC,
MarkingMode mode = MarkingMode::FULL>
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool IsWhite(HeapObject* obj, const MarkingState& state) {
return Marking::IsWhite<access_mode>(MarkBitFrom(obj, state));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool IsGrey(HeapObject* obj) {
return Marking::IsGrey<access_mode>(MarkBitFrom<mode>(obj));
return Marking::IsGrey<access_mode>(MarkBitFrom(obj));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool IsGrey(HeapObject* obj, const MarkingState& state) {
return Marking::IsGrey<access_mode>(MarkBitFrom(obj, state));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC,
MarkingMode mode = MarkingMode::FULL>
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool IsBlackOrGrey(HeapObject* obj) {
return Marking::IsBlackOrGrey<access_mode>(MarkBitFrom<mode>(obj));
return Marking::IsBlackOrGrey<access_mode>(MarkBitFrom(obj));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC,
MarkingMode mode = MarkingMode::FULL>
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool IsBlackOrGrey(HeapObject* obj,
const MarkingState& state) {
return Marking::IsBlackOrGrey<access_mode>(MarkBitFrom(obj, state));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool BlackToGrey(HeapObject* obj) {
DCHECK((access_mode == MarkBit::ATOMIC || IsBlack<access_mode, mode>(obj)));
MarkBit markbit = MarkBitFrom<mode>(obj);
DCHECK((access_mode == MarkBit::ATOMIC || IsBlack<access_mode>(obj)));
MarkBit markbit = MarkBitFrom(obj);
if (!Marking::BlackToGrey<access_mode>(markbit)) return false;
MemoryChunk::IncrementLiveBytes(obj, -obj->Size());
return true;
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool BlackToGrey(HeapObject* obj,
const MarkingState& state) {
DCHECK(
(access_mode == MarkBit::ATOMIC || IsBlack<access_mode>(obj, state)));
MarkBit markbit = MarkBitFrom(obj, state);
if (!Marking::BlackToGrey<access_mode>(markbit)) return false;
MemoryChunk::IncrementLiveBytes<mode>(obj, -obj->Size());
state.IncrementLiveBytes(-obj->Size());
return true;
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC,
MarkingMode mode = MarkingMode::FULL>
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool WhiteToGrey(HeapObject* obj) {
DCHECK((access_mode == MarkBit::ATOMIC || IsWhite<access_mode, mode>(obj)));
return Marking::WhiteToGrey<access_mode>(MarkBitFrom<mode>(obj));
DCHECK((access_mode == MarkBit::ATOMIC || IsWhite<access_mode>(obj)));
return Marking::WhiteToGrey<access_mode>(MarkBitFrom(obj));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool WhiteToGrey(HeapObject* obj,
const MarkingState& state) {
DCHECK(
(access_mode == MarkBit::ATOMIC || IsWhite<access_mode>(obj, state)));
return Marking::WhiteToGrey<access_mode>(MarkBitFrom(obj, state));
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC,
MarkingMode mode = MarkingMode::FULL>
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool WhiteToBlack(HeapObject* obj) {
DCHECK((access_mode == MarkBit::ATOMIC || IsWhite<access_mode, mode>(obj)));
if (!ObjectMarking::WhiteToGrey<access_mode, mode>(obj)) return false;
return ObjectMarking::GreyToBlack<access_mode, mode>(obj);
DCHECK((access_mode == MarkBit::ATOMIC || IsWhite<access_mode>(obj)));
if (!ObjectMarking::WhiteToGrey<access_mode>(obj)) return false;
return ObjectMarking::GreyToBlack<access_mode>(obj);
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC,
MarkingMode mode = MarkingMode::FULL>
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool WhiteToBlack(HeapObject* obj,
const MarkingState& state) {
DCHECK(
(access_mode == MarkBit::ATOMIC || IsWhite<access_mode>(obj, state)));
if (!ObjectMarking::WhiteToGrey<access_mode>(obj, state)) return false;
return ObjectMarking::GreyToBlack<access_mode>(obj, state);
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool GreyToBlack(HeapObject* obj) {
DCHECK((access_mode == MarkBit::ATOMIC || IsGrey<access_mode, mode>(obj)));
MarkBit markbit = MarkBitFrom<mode>(obj);
DCHECK((access_mode == MarkBit::ATOMIC || IsGrey<access_mode>(obj)));
MarkBit markbit = MarkBitFrom(obj);
if (!Marking::GreyToBlack<access_mode>(markbit)) return false;
MemoryChunk::IncrementLiveBytes<mode>(obj, obj->Size());
MemoryChunk::IncrementLiveBytes(obj, obj->Size());
return true;
}
template <MarkBit::AccessMode access_mode = MarkBit::NON_ATOMIC>
V8_INLINE static bool GreyToBlack(HeapObject* obj,
const MarkingState& state) {
DCHECK((access_mode == MarkBit::ATOMIC || IsGrey<access_mode>(obj, state)));
MarkBit markbit = MarkBitFrom(obj, state);
if (!Marking::GreyToBlack<access_mode>(markbit)) return false;
state.IncrementLiveBytes(obj->Size());
return true;
}
......@@ -308,13 +402,13 @@ class ThreadLocalTop;
class MarkBitCellIterator BASE_EMBEDDED {
public:
explicit MarkBitCellIterator(MemoryChunk* chunk) : chunk_(chunk) {
MarkBitCellIterator(MemoryChunk* chunk, MarkingState state) : chunk_(chunk) {
last_cell_index_ = Bitmap::IndexToCell(Bitmap::CellAlignIndex(
chunk_->AddressToMarkbitIndex(chunk_->area_end())));
cell_base_ = chunk_->area_start();
cell_index_ = Bitmap::IndexToCell(
Bitmap::CellAlignIndex(chunk_->AddressToMarkbitIndex(cell_base_)));
cells_ = chunk_->markbits()->cells();
cells_ = state.bitmap->cells();
}
inline bool Done() { return cell_index_ == last_cell_index_; }
......@@ -377,12 +471,11 @@ enum LiveObjectIterationMode {
template <LiveObjectIterationMode T>
class LiveObjectIterator BASE_EMBEDDED {
public:
explicit LiveObjectIterator(MemoryChunk* chunk)
LiveObjectIterator(MemoryChunk* chunk, MarkingState state)
: chunk_(chunk),
it_(chunk_),
it_(chunk_, state),
cell_base_(it_.CurrentCellBase()),
current_cell_(*it_.CurrentCell()) {
}
current_cell_(*it_.CurrentCell()) {}
HeapObject* Next();
......@@ -397,11 +490,47 @@ class LiveObjectIterator BASE_EMBEDDED {
enum PageEvacuationMode { NEW_TO_NEW, NEW_TO_OLD };
class MinorMarkCompactCollector {
public:
explicit MinorMarkCompactCollector(Heap* heap)
: heap_(heap), marking_deque_(heap) {}
void SetUp();
void TearDown();
void CollectGarbage();
private:
class RootMarkingVisitor;
static MarkingState StateForObject(HeapObject* object) {
return MarkingState::FromPageExternal(Page::FromAddress(object->address()));
}
inline Heap* heap() { return heap_; }
inline Isolate* isolate() { return heap()->isolate(); }
inline MarkingDeque* marking_deque() { return &marking_deque_; }
V8_INLINE void MarkObject(HeapObject* obj);
V8_INLINE void PushBlack(HeapObject* obj);
SlotCallbackResult CheckAndMarkObject(Heap* heap, Address slot_address);
void MarkLiveObjects();
void ProcessMarkingDeque();
void EmptyMarkingDeque();
Heap* heap_;
MarkingDeque marking_deque_;
friend class StaticYoungGenerationMarkingVisitor;
};
// -------------------------------------------------------------------------
// Mark-Compact collector
class MarkCompactCollector {
public:
class Evacuator;
class RootMarkingVisitor;
class Sweeper {
public:
......@@ -567,11 +696,7 @@ class MarkCompactCollector {
bool evacuation() const { return evacuation_; }
template <MarkingMode mode = MarkingMode::FULL>
MarkingDeque* marking_deque() {
return mode == MarkingMode::FULL ? &marking_deque_
: &marking_deque_young_generation_;
}
MarkingDeque* marking_deque() { return &marking_deque_; }
Sweeper& sweeper() { return sweeper_; }
......@@ -629,8 +754,6 @@ class MarkCompactCollector {
friend class MarkCompactMarkingVisitor;
friend class MarkingVisitor;
friend class RecordMigratedSlotVisitor;
template <MarkingMode mode>
friend class RootMarkingVisitor;
friend class SharedFunctionInfoMarkingVisitor;
friend class StaticYoungGenerationMarkingVisitor;
......@@ -642,12 +765,9 @@ class MarkCompactCollector {
// Marking operations for objects reachable from roots.
void MarkLiveObjects();
// Mark the young generation.
void MarkLiveObjectsInYoungGeneration();
// Pushes a black object onto the marking stack and accounts for live bytes.
// Note that this assumes live bytes have not yet been counted.
template <MarkingMode mode = MarkingMode::FULL>
V8_INLINE void PushBlack(HeapObject* obj);
// Unshifts a black object into the marking stack and accounts for live bytes.
......@@ -656,19 +776,17 @@ class MarkCompactCollector {
// Marks the object black and pushes it on the marking stack.
// This is for non-incremental marking only.
template <MarkingMode mode = MarkingMode::FULL>
V8_INLINE void MarkObject(HeapObject* obj);
// Mark the heap roots and all objects reachable from them.
void MarkRoots(RootMarkingVisitor<MarkingMode::FULL>* visitor);
void MarkRoots(RootMarkingVisitor* visitor);
// Mark the string table specially. References to internalized strings from
// the string table are weak.
void MarkStringTable(RootMarkingVisitor<MarkingMode::FULL>* visitor);
void MarkStringTable(RootMarkingVisitor* visitor);
// Mark objects reachable (transitively) from objects in the marking stack
// or overflowed in the heap.
template <MarkingMode mode>
void ProcessMarkingDeque();
// Mark objects reachable (transitively) from objects in the marking stack
......@@ -692,13 +810,11 @@ class MarkCompactCollector {
// stack. This function empties the marking stack, but may leave
// overflowed objects in the heap, in which case the marking stack's
// overflow flag will be set.
template <MarkingMode mode>
void EmptyMarkingDeque();
// Refill the marking stack with overflowed objects from the heap. This
// function either leaves the marking stack full or clears the overflow
// flag on the marking stack.
template <MarkingMode mode>
void RefillMarkingDeque();
// Helper methods for refilling the marking stack by discovering grey objects
......@@ -816,7 +932,6 @@ class MarkCompactCollector {
bool have_code_to_deoptimize_;
MarkingDeque marking_deque_;
MarkingDeque marking_deque_young_generation_;
CodeFlusher* code_flusher_;
......
......@@ -526,6 +526,14 @@ class MemoryChunk {
: young_generation_bitmap_;
}
template <MarkingMode mode = MarkingMode::FULL>
inline intptr_t* live_bytes_address() {
// TODO(mlippautz): Fix type of live_byte_count_.
return mode == MarkingMode::FULL
? reinterpret_cast<intptr_t*>(&live_byte_count_)
: &young_generation_live_byte_count_;
}
inline uint32_t AddressToMarkbitIndex(Address addr) const {
return static_cast<uint32_t>(addr - this->address()) >> kPointerSizeLog2;
}
......
......@@ -354,7 +354,9 @@ TEST(Regress5829) {
heap->CreateFillerObjectAt(old_end - kPointerSize, kPointerSize,
ClearRecordedSlots::kNo);
heap->old_space()->EmptyAllocationInfo();
LiveObjectIterator<kGreyObjects> it(Page::FromAddress(array->address()));
Page* page = Page::FromAddress(array->address());
LiveObjectIterator<kGreyObjects> it(page,
MarkingState::FromPageInternal(page));
HeapObject* object = nullptr;
while ((object = it.Next()) != nullptr) {
CHECK(!object->IsFiller());
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment