// Copyright 2020 the V8 project authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "src/heap/local-heap.h" #include <atomic> #include <memory> #include "src/base/logging.h" #include "src/base/platform/mutex.h" #include "src/common/globals.h" #include "src/execution/isolate.h" #include "src/handles/local-handles.h" #include "src/heap/collection-barrier.h" #include "src/heap/gc-tracer.h" #include "src/heap/heap-inl.h" #include "src/heap/heap-write-barrier.h" #include "src/heap/local-heap-inl.h" #include "src/heap/marking-barrier.h" #include "src/heap/parked-scope.h" #include "src/heap/safepoint.h" namespace v8 { namespace internal { namespace { thread_local LocalHeap* current_local_heap = nullptr; } // namespace LocalHeap* LocalHeap::Current() { return current_local_heap; } #ifdef DEBUG void LocalHeap::VerifyCurrent() { LocalHeap* current = LocalHeap::Current(); if (is_main_thread()) DCHECK_NULL(current); else DCHECK_EQ(current, this); } #endif LocalHeap::LocalHeap(Heap* heap, ThreadKind kind, std::unique_ptr<PersistentHandles> persistent_handles) : heap_(heap), is_main_thread_(kind == ThreadKind::kMain), state_(ThreadState::Parked()), allocation_failed_(false), main_thread_parked_(false), prev_(nullptr), next_(nullptr), handles_(new LocalHandles), persistent_handles_(std::move(persistent_handles)), marking_barrier_(new MarkingBarrier(this)), old_space_allocator_(this, heap->old_space()), code_space_allocator_(this, heap->code_space()) { heap_->safepoint()->AddLocalHeap(this, [this] { if (!is_main_thread()) { WriteBarrier::SetForThread(marking_barrier_.get()); if (heap_->incremental_marking()->IsMarking()) { marking_barrier_->Activate( heap_->incremental_marking()->IsCompacting()); } } }); if (persistent_handles_) { persistent_handles_->Attach(this); } DCHECK_NULL(current_local_heap); if (!is_main_thread()) current_local_heap = this; } LocalHeap::~LocalHeap() { // Park thread since removing the local heap could block. EnsureParkedBeforeDestruction(); heap_->safepoint()->RemoveLocalHeap(this, [this] { old_space_allocator_.FreeLinearAllocationArea(); code_space_allocator_.FreeLinearAllocationArea(); if (!is_main_thread()) { marking_barrier_->Publish(); WriteBarrier::ClearForThread(marking_barrier_.get()); } }); if (!is_main_thread()) { DCHECK_EQ(current_local_heap, this); current_local_heap = nullptr; } DCHECK(gc_epilogue_callbacks_.empty()); } void LocalHeap::EnsurePersistentHandles() { if (!persistent_handles_) { persistent_handles_.reset( heap_->isolate()->NewPersistentHandles().release()); persistent_handles_->Attach(this); } } void LocalHeap::AttachPersistentHandles( std::unique_ptr<PersistentHandles> persistent_handles) { DCHECK_NULL(persistent_handles_); persistent_handles_ = std::move(persistent_handles); persistent_handles_->Attach(this); } std::unique_ptr<PersistentHandles> LocalHeap::DetachPersistentHandles() { if (persistent_handles_) persistent_handles_->Detach(); return std::move(persistent_handles_); } #ifdef DEBUG bool LocalHeap::ContainsPersistentHandle(Address* location) { return persistent_handles_ ? persistent_handles_->Contains(location) : false; } bool LocalHeap::ContainsLocalHandle(Address* location) { return handles_ ? handles_->Contains(location) : false; } bool LocalHeap::IsHandleDereferenceAllowed() { #ifdef DEBUG VerifyCurrent(); #endif return IsRunning(); } #endif bool LocalHeap::IsParked() { #ifdef DEBUG VerifyCurrent(); #endif return state_.load_relaxed().IsParked(); } bool LocalHeap::IsRunning() { #ifdef DEBUG VerifyCurrent(); #endif return state_.load_relaxed().IsRunning(); } void LocalHeap::ParkSlowPath() { while (true) { ThreadState current_state = ThreadState::Running(); if (state_.CompareExchangeStrong(current_state, ThreadState::Parked())) return; // CAS above failed, so state is Running with some additional flag. DCHECK(current_state.IsRunning()); if (is_main_thread()) { DCHECK(current_state.IsCollectionRequested()); heap_->CollectGarbageForBackground(this); } else { DCHECK(current_state.IsSafepointRequested()); DCHECK(!current_state.IsCollectionRequested()); CHECK(state_.CompareExchangeStrong(current_state, current_state.SetParked())); heap_->safepoint()->NotifyPark(); return; } } } void LocalHeap::UnparkSlowPath() { while (true) { ThreadState current_state = ThreadState::Parked(); if (state_.CompareExchangeStrong(current_state, ThreadState::Running())) return; // CAS above failed, so state is Parked with some additional flag. DCHECK(current_state.IsParked()); if (is_main_thread()) { DCHECK(current_state.IsCollectionRequested()); CHECK(state_.CompareExchangeStrong(current_state, current_state.SetRunning())); heap_->CollectGarbageForBackground(this); return; } else { DCHECK(current_state.IsSafepointRequested()); DCHECK(!current_state.IsCollectionRequested()); TRACE_GC1(heap_->tracer(), GCTracer::Scope::BACKGROUND_UNPARK, ThreadKind::kBackground); heap_->safepoint()->WaitInUnpark(); } } } void LocalHeap::EnsureParkedBeforeDestruction() { DCHECK_IMPLIES(!is_main_thread(), IsParked()); } void LocalHeap::SafepointSlowPath() { #ifdef DEBUG ThreadState current_state = state_.load_relaxed(); DCHECK(current_state.IsRunning()); #endif if (is_main_thread()) { DCHECK(current_state.IsCollectionRequested()); heap_->CollectGarbageForBackground(this); } else { DCHECK(current_state.IsSafepointRequested()); DCHECK(!current_state.IsCollectionRequested()); TRACE_GC1(heap_->tracer(), GCTracer::Scope::BACKGROUND_SAFEPOINT, ThreadKind::kBackground); // Parking the running thread here is an optimization. We do not need to // wake this thread up to reach the next safepoint. ThreadState old_state = state_.SetParked(); CHECK(old_state.IsRunning()); CHECK(old_state.IsSafepointRequested()); CHECK(!old_state.IsCollectionRequested()); heap_->safepoint()->WaitInSafepoint(); Unpark(); } } void LocalHeap::FreeLinearAllocationArea() { old_space_allocator_.FreeLinearAllocationArea(); code_space_allocator_.FreeLinearAllocationArea(); } void LocalHeap::MakeLinearAllocationAreaIterable() { old_space_allocator_.MakeLinearAllocationAreaIterable(); code_space_allocator_.MakeLinearAllocationAreaIterable(); } void LocalHeap::MarkLinearAllocationAreaBlack() { old_space_allocator_.MarkLinearAllocationAreaBlack(); code_space_allocator_.MarkLinearAllocationAreaBlack(); } void LocalHeap::UnmarkLinearAllocationArea() { old_space_allocator_.UnmarkLinearAllocationArea(); code_space_allocator_.UnmarkLinearAllocationArea(); } bool LocalHeap::TryPerformCollection() { if (is_main_thread()) { heap_->CollectGarbageForBackground(this); return true; } else { DCHECK(IsRunning()); heap_->collection_barrier_->RequestGC(); LocalHeap* main_thread = heap_->main_thread_local_heap(); const ThreadState old_state = main_thread->state_.SetCollectionRequested(); if (old_state.IsRunning()) { const bool performed_gc = heap_->collection_barrier_->AwaitCollectionBackground(this); return performed_gc; } else { DCHECK(old_state.IsParked()); return false; } } } Address LocalHeap::PerformCollectionAndAllocateAgain( int object_size, AllocationType type, AllocationOrigin origin, AllocationAlignment alignment) { CHECK(!allocation_failed_); CHECK(!main_thread_parked_); allocation_failed_ = true; static const int kMaxNumberOfRetries = 3; for (int i = 0; i < kMaxNumberOfRetries; i++) { if (!TryPerformCollection()) { main_thread_parked_ = true; } AllocationResult result = AllocateRaw(object_size, type, origin, alignment); if (!result.IsRetry()) { allocation_failed_ = false; main_thread_parked_ = false; return result.ToObjectChecked().address(); } } heap_->FatalProcessOutOfMemory("LocalHeap: allocation failed"); } void LocalHeap::AddGCEpilogueCallback(GCEpilogueCallback* callback, void* data) { DCHECK(!IsParked()); std::pair<GCEpilogueCallback*, void*> callback_and_data(callback, data); DCHECK_EQ(std::find(gc_epilogue_callbacks_.begin(), gc_epilogue_callbacks_.end(), callback_and_data), gc_epilogue_callbacks_.end()); gc_epilogue_callbacks_.push_back(callback_and_data); } void LocalHeap::RemoveGCEpilogueCallback(GCEpilogueCallback* callback, void* data) { DCHECK(!IsParked()); std::pair<GCEpilogueCallback*, void*> callback_and_data(callback, data); auto it = std::find(gc_epilogue_callbacks_.begin(), gc_epilogue_callbacks_.end(), callback_and_data); *it = gc_epilogue_callbacks_.back(); gc_epilogue_callbacks_.pop_back(); } void LocalHeap::InvokeGCEpilogueCallbacksInSafepoint() { for (auto callback_and_data : gc_epilogue_callbacks_) { callback_and_data.first(callback_and_data.second); } } } // namespace internal } // namespace v8