Commit 6f3aae99 authored by Michael Lippautz's avatar Michael Lippautz Committed by V8 LUCI CQ

cppgc: Move sweeper to regular tasks

- Use non-idle tasks to be able to process finalizers on time.
- Only process finalizers while concurrent marking is still running.

Bug: v8:13294
Change-Id: I1a2812c3fc350ea679c4c916c230cf736f2aa3ea
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3904648
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: 's avatarAnton Bikineev <bikineev@chromium.org>
Cr-Commit-Position: refs/heads/main@{#83367}
parent d5a94229
......@@ -3105,6 +3105,7 @@ filegroup(
"src/heap/cppgc/stats-collector.h",
"src/heap/cppgc/sweeper.cc",
"src/heap/cppgc/sweeper.h",
"src/heap/cppgc/heap-config.h",
"src/heap/cppgc/task-handle.h",
"src/heap/cppgc/trace-event.h",
"src/heap/cppgc/trace-trait.cc",
......
......@@ -5860,6 +5860,7 @@ v8_source_set("cppgc_base") {
"src/heap/cppgc/globals.h",
"src/heap/cppgc/heap-base.cc",
"src/heap/cppgc/heap-base.h",
"src/heap/cppgc/heap-config.h",
"src/heap/cppgc/heap-consistency.cc",
"src/heap/cppgc/heap-growing.cc",
"src/heap/cppgc/heap-growing.h",
......
......@@ -753,14 +753,14 @@ void CppHeap::TraceEpilogue() {
{
cppgc::subtle::NoGarbageCollectionScope no_gc(*this);
cppgc::internal::Sweeper::SweepingConfig::CompactableSpaceHandling
cppgc::internal::SweepingConfig::CompactableSpaceHandling
compactable_space_handling = compactor_.CompactSpacesIfEnabled();
const cppgc::internal::Sweeper::SweepingConfig sweeping_config{
const cppgc::internal::SweepingConfig sweeping_config{
SelectSweepingType(), compactable_space_handling,
ShouldReduceMemory(current_gc_flags_)
? cppgc::internal::Sweeper::SweepingConfig::FreeMemoryHandling::
? cppgc::internal::SweepingConfig::FreeMemoryHandling::
kDiscardWherePossible
: cppgc::internal::Sweeper::SweepingConfig::FreeMemoryHandling::
: cppgc::internal::SweepingConfig::FreeMemoryHandling::
kDoNotDiscard};
DCHECK_IMPLIES(!isolate_,
SweepingType::kAtomic == sweeping_config.sweeping_type);
......@@ -928,8 +928,8 @@ class CollectCustomSpaceStatisticsAtLastGCTask final : public v8::Task {
void Run() final {
cppgc::internal::Sweeper& sweeper = heap_.sweeper();
if (sweeper.PerformSweepOnMutatorThread(
heap_.platform()->MonotonicallyIncreasingTime() +
kStepSizeMs.InSecondsF())) {
kStepSizeMs,
cppgc::internal::StatsCollector::kSweepInTaskForStatistics)) {
// Sweeping is done.
DCHECK(!sweeper.IsSweepingInProgress());
ReportCustomSpaceStatistics(heap_.raw_heap(), std::move(custom_spaces_),
......
......@@ -13,8 +13,7 @@ namespace cppgc {
namespace internal {
class V8_EXPORT_PRIVATE Compactor final {
using CompactableSpaceHandling =
Sweeper::SweepingConfig::CompactableSpaceHandling;
using CompactableSpaceHandling = SweepingConfig::CompactableSpaceHandling;
public:
explicit Compactor(RawHeap&);
......
......@@ -6,8 +6,8 @@
#define V8_HEAP_CPPGC_GARBAGE_COLLECTOR_H_
#include "include/cppgc/common.h"
#include "src/heap/cppgc/heap-config.h"
#include "src/heap/cppgc/marker.h"
#include "src/heap/cppgc/sweeper.h"
namespace cppgc {
namespace internal {
......@@ -20,8 +20,8 @@ class GarbageCollector {
using CollectionType = Marker::MarkingConfig::CollectionType;
using StackState = cppgc::Heap::StackState;
using MarkingType = Marker::MarkingConfig::MarkingType;
using SweepingType = Sweeper::SweepingConfig::SweepingType;
using FreeMemoryHandling = Sweeper::SweepingConfig::FreeMemoryHandling;
using SweepingType = SweepingConfig::SweepingType;
using FreeMemoryHandling = SweepingConfig::FreeMemoryHandling;
using IsForcedGC = Marker::MarkingConfig::IsForcedGC;
static constexpr Config ConservativeAtomicConfig() {
......
......@@ -259,9 +259,8 @@ void HeapBase::Terminate() {
ExecutePreFinalizers();
// TODO(chromium:1029379): Prefinalizers may black-allocate objects (under a
// compile-time option). Run sweeping with forced finalization here.
sweeper().Start(
{Sweeper::SweepingConfig::SweepingType::kAtomic,
Sweeper::SweepingConfig::CompactableSpaceHandling::kSweep});
sweeper().Start({SweepingConfig::SweepingType::kAtomic,
SweepingConfig::CompactableSpaceHandling::kSweep});
in_atomic_pause_ = false;
sweeper().NotifyDoneIfNeeded();
......
// Copyright 2020 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef V8_HEAP_CPPGC_HEAP_CONFIG_H_
#define V8_HEAP_CPPGC_HEAP_CONFIG_H_
#include "include/cppgc/heap.h"
namespace cppgc::internal {
struct SweepingConfig {
using SweepingType = cppgc::Heap::SweepingType;
enum class CompactableSpaceHandling { kSweep, kIgnore };
enum class FreeMemoryHandling { kDoNotDiscard, kDiscardWherePossible };
SweepingType sweeping_type = SweepingType::kIncrementalAndConcurrent;
CompactableSpaceHandling compactable_space_handling =
CompactableSpaceHandling::kSweep;
FreeMemoryHandling free_memory_handling = FreeMemoryHandling::kDoNotDiscard;
};
} // namespace cppgc::internal
#endif // V8_HEAP_CPPGC_HEAP_CONFIG_H_
......@@ -203,9 +203,8 @@ void Heap::FinalizeGarbageCollection(Config::StackState stack_state) {
#endif // defined(CPPGC_YOUNG_GENERATION)
subtle::NoGarbageCollectionScope no_gc(*this);
const Sweeper::SweepingConfig sweeping_config{
config_.sweeping_type,
Sweeper::SweepingConfig::CompactableSpaceHandling::kSweep,
const SweepingConfig sweeping_config{
config_.sweeping_type, SweepingConfig::CompactableSpaceHandling::kSweep,
config_.free_memory_handling};
sweeper_.Start(sweeping_config);
in_atomic_pause_ = false;
......
......@@ -54,8 +54,8 @@ namespace internal {
V(MarkVisitRememberedSets) \
V(SweepFinishIfOutOfWork) \
V(SweepInvokePreFinalizers) \
V(SweepIdleStep) \
V(SweepInTask) \
V(SweepInTaskForStatistics) \
V(SweepOnAllocation) \
V(SweepFinalize)
......
......@@ -26,13 +26,41 @@
#include "src/heap/cppgc/stats-collector.h"
#include "src/heap/cppgc/task-handle.h"
namespace cppgc {
namespace internal {
namespace cppgc::internal {
namespace {
class DeadlineChecker final {
public:
explicit DeadlineChecker(v8::base::TimeTicks end) : end_(end) {}
bool Check() {
return (++count_ % kInterval == 0) && (end_ < v8::base::TimeTicks::Now());
}
private:
static constexpr size_t kInterval = 4;
const v8::base::TimeTicks end_;
size_t count_ = 0;
};
using v8::base::Optional;
enum class MutatorThreadSweepingMode {
kOnlyFinalizers,
kAll,
};
constexpr const char* ToString(MutatorThreadSweepingMode sweeping_mode) {
switch (sweeping_mode) {
case MutatorThreadSweepingMode::kAll:
return "all";
case MutatorThreadSweepingMode::kOnlyFinalizers:
return "only-finalizers";
}
}
enum class StickyBits : uint8_t {
kDisabled,
kEnabled,
......@@ -378,7 +406,7 @@ typename FinalizationBuilder::ResultType SweepNormalPage(
// - returns (unmaps) empty pages;
// - merges freelists to the space's freelist.
class SweepFinalizer final {
using FreeMemoryHandling = Sweeper::SweepingConfig::FreeMemoryHandling;
using FreeMemoryHandling = SweepingConfig::FreeMemoryHandling;
public:
SweepFinalizer(cppgc::Platform* platform,
......@@ -398,20 +426,13 @@ class SweepFinalizer final {
}
bool FinalizeSpaceWithDeadline(SpaceState* space_state,
double deadline_in_seconds) {
v8::base::TimeTicks deadline) {
DCHECK(platform_);
static constexpr size_t kDeadlineCheckInterval = 4;
size_t page_count = 1;
DeadlineChecker deadline_check(deadline);
while (auto page_state = space_state->swept_unfinalized_pages.Pop()) {
FinalizePage(&*page_state);
if (page_count % kDeadlineCheckInterval == 0 &&
deadline_in_seconds <= platform_->MonotonicallyIncreasingTime()) {
return false;
}
page_count++;
if (deadline_check.Check()) return false;
}
return true;
......@@ -489,7 +510,7 @@ class SweepFinalizer final {
class MutatorThreadSweeper final : private HeapVisitor<MutatorThreadSweeper> {
friend class HeapVisitor<MutatorThreadSweeper>;
using FreeMemoryHandling = Sweeper::SweepingConfig::FreeMemoryHandling;
using FreeMemoryHandling = SweepingConfig::FreeMemoryHandling;
public:
MutatorThreadSweeper(HeapBase* heap, SpaceStates* states,
......@@ -512,25 +533,23 @@ class MutatorThreadSweeper final : private HeapVisitor<MutatorThreadSweeper> {
void SweepPage(BasePage& page) { Traverse(page); }
bool SweepWithDeadline(double deadline_in_seconds) {
bool SweepWithDeadline(v8::base::TimeDelta max_duration,
MutatorThreadSweepingMode sweeping_mode) {
DCHECK(platform_);
static constexpr double kSlackInSeconds = 0.001;
for (SpaceState& state : *states_) {
// FinalizeSpaceWithDeadline() and SweepSpaceWithDeadline() won't check
// the deadline until it sweeps 10 pages. So we give a small slack for
// safety.
const double remaining_budget = deadline_in_seconds - kSlackInSeconds -
platform_->MonotonicallyIncreasingTime();
if (remaining_budget <= 0.) return false;
const auto deadline = v8::base::TimeTicks::Now() + max_duration;
// First, prioritize finalization of pages that were swept concurrently.
SweepFinalizer finalizer(platform_, free_memory_handling_);
if (!finalizer.FinalizeSpaceWithDeadline(&state, deadline_in_seconds)) {
if (!finalizer.FinalizeSpaceWithDeadline(&state, deadline)) {
return false;
}
if (sweeping_mode == MutatorThreadSweepingMode::kOnlyFinalizers)
return false;
// Help out the concurrent sweeper.
if (!SweepSpaceWithDeadline(&state, deadline_in_seconds)) {
if (!SweepSpaceWithDeadline(&state, deadline)) {
return false;
}
}
......@@ -542,16 +561,11 @@ class MutatorThreadSweeper final : private HeapVisitor<MutatorThreadSweeper> {
}
private:
bool SweepSpaceWithDeadline(SpaceState* state, double deadline_in_seconds) {
static constexpr size_t kDeadlineCheckInterval = 4;
size_t page_count = 1;
bool SweepSpaceWithDeadline(SpaceState* state, v8::base::TimeTicks deadline) {
DeadlineChecker deadline_check(deadline);
while (auto page = state->unswept_pages.Pop()) {
Traverse(**page);
if (page_count % kDeadlineCheckInterval == 0 &&
deadline_in_seconds <= platform_->MonotonicallyIncreasingTime()) {
return false;
}
page_count++;
if (deadline_check.Check()) return false;
}
return true;
......@@ -604,7 +618,7 @@ class ConcurrentSweepTask final : public cppgc::JobTask,
private HeapVisitor<ConcurrentSweepTask> {
friend class HeapVisitor<ConcurrentSweepTask>;
using FreeMemoryHandling = Sweeper::SweepingConfig::FreeMemoryHandling;
using FreeMemoryHandling = SweepingConfig::FreeMemoryHandling;
public:
ConcurrentSweepTask(HeapBase& heap, SpaceStates* states, Platform* platform,
......@@ -694,8 +708,7 @@ class ConcurrentSweepTask final : public cppgc::JobTask,
class PrepareForSweepVisitor final
: protected HeapVisitor<PrepareForSweepVisitor> {
friend class HeapVisitor<PrepareForSweepVisitor>;
using CompactableSpaceHandling =
Sweeper::SweepingConfig::CompactableSpaceHandling;
using CompactableSpaceHandling = SweepingConfig::CompactableSpaceHandling;
public:
PrepareForSweepVisitor(SpaceStates* states,
......@@ -747,7 +760,7 @@ class PrepareForSweepVisitor final
} // namespace
class Sweeper::SweeperImpl final {
using FreeMemoryHandling = Sweeper::SweepingConfig::FreeMemoryHandling;
using FreeMemoryHandling = SweepingConfig::FreeMemoryHandling;
public:
SweeperImpl(RawHeap& heap, StatsCollector* stats_collector)
......@@ -810,20 +823,7 @@ class Sweeper::SweeperImpl final {
StatsCollector::EnabledScope inner_scope(
stats_collector_, StatsCollector::kSweepOnAllocation);
MutatorThreadSweepingScope sweeping_in_progress(*this);
const auto deadline = v8::base::TimeTicks::Now() + max_duration;
size_t page_count = 0;
const auto ShouldYield = [&page_count, deadline]() {
constexpr size_t kPageInterruptInterval = 10;
if (page_count++ == kPageInterruptInterval) {
if (deadline < v8::base::TimeTicks::Now()) {
return true;
}
page_count = 0;
}
return false;
};
DeadlineChecker deadline_check(v8::base::TimeTicks::Now() + max_duration);
{
// First, process unfinalized pages as finalizing a page is faster than
// sweeping.
......@@ -833,7 +833,7 @@ class Sweeper::SweeperImpl final {
if (size <= finalizer.largest_new_free_list_entry()) {
return true;
}
if (ShouldYield()) {
if (deadline_check.Check()) {
return false;
}
}
......@@ -848,7 +848,7 @@ class Sweeper::SweeperImpl final {
if (size <= sweeper.largest_new_free_list_entry()) {
return true;
}
if (ShouldYield()) {
if (deadline_check.Check()) {
return false;
}
}
......@@ -880,6 +880,12 @@ class Sweeper::SweeperImpl final {
return true;
}
bool IsConcurrentSweepingDone() const {
return !concurrent_sweeper_handle_ ||
(concurrent_sweeper_handle_->IsValid() &&
!concurrent_sweeper_handle_->IsActive());
}
void FinishIfOutOfWork() {
if (is_in_progress_ && !is_sweeping_on_mutator_thread_ &&
concurrent_sweeper_handle_ && concurrent_sweeper_handle_->IsValid() &&
......@@ -899,10 +905,8 @@ class Sweeper::SweeperImpl final {
// deadline to see if sweeping can be fully finished.
MutatorThreadSweeper sweeper(heap_.heap(), &space_states_, platform_,
config_.free_memory_handling);
const double deadline =
platform_->MonotonicallyIncreasingTime() +
v8::base::TimeDelta::FromMilliseconds(2).InSecondsF();
if (sweeper.SweepWithDeadline(deadline)) {
if (sweeper.SweepWithDeadline(v8::base::TimeDelta::FromMilliseconds(2),
MutatorThreadSweepingMode::kAll)) {
FinalizeSweep();
}
}
......@@ -960,8 +964,9 @@ class Sweeper::SweeperImpl final {
bool IsSweepingInProgress() const { return is_in_progress_; }
bool PerformSweepOnMutatorThread(double deadline_in_seconds,
StatsCollector::ScopeId internal_scope_id) {
bool PerformSweepOnMutatorThread(v8::base::TimeDelta max_duration,
StatsCollector::ScopeId internal_scope_id,
MutatorThreadSweepingMode sweeping_mode) {
if (!is_in_progress_) return true;
MutatorThreadSweepingScope sweeping_in_progress(*this);
......@@ -975,10 +980,10 @@ class Sweeper::SweeperImpl final {
config_.free_memory_handling);
{
StatsCollector::EnabledScope inner_stats_scope(
stats_collector_, internal_scope_id, "deltaInSeconds",
deadline_in_seconds - platform_->MonotonicallyIncreasingTime());
sweep_complete = sweeper.SweepWithDeadline(deadline_in_seconds);
stats_collector_, internal_scope_id, "max_duration_ms",
max_duration.InMillisecondsF(), "sweeping_mode",
ToString(sweeping_mode));
sweep_complete = sweeper.SweepWithDeadline(max_duration, sweeping_mode);
}
if (sweep_complete) {
FinalizeSweep();
......@@ -1008,33 +1013,37 @@ class Sweeper::SweeperImpl final {
SweeperImpl& sweeper_;
};
class IncrementalSweepTask : public cppgc::IdleTask {
class IncrementalSweepTask final : public cppgc::Task {
public:
using Handle = SingleThreadedHandle;
explicit IncrementalSweepTask(SweeperImpl* sweeper)
explicit IncrementalSweepTask(SweeperImpl& sweeper)
: sweeper_(sweeper), handle_(Handle::NonEmptyTag{}) {}
static Handle Post(SweeperImpl* sweeper, cppgc::TaskRunner* runner) {
static Handle Post(SweeperImpl& sweeper, cppgc::TaskRunner* runner) {
auto task = std::make_unique<IncrementalSweepTask>(sweeper);
auto handle = task->GetHandle();
runner->PostIdleTask(std::move(task));
runner->PostTask(std::move(task));
return handle;
}
private:
void Run(double deadline_in_seconds) override {
void Run() override {
if (handle_.IsCanceled()) return;
if (!sweeper_->PerformSweepOnMutatorThread(
deadline_in_seconds, StatsCollector::kSweepIdleStep)) {
sweeper_->ScheduleIncrementalSweeping();
if (!sweeper_.PerformSweepOnMutatorThread(
v8::base::TimeDelta::FromMilliseconds(5),
StatsCollector::kSweepInTask,
sweeper_.IsConcurrentSweepingDone()
? MutatorThreadSweepingMode::kAll
: MutatorThreadSweepingMode::kOnlyFinalizers)) {
sweeper_.ScheduleIncrementalSweeping();
}
}
Handle GetHandle() const { return handle_; }
SweeperImpl* sweeper_;
SweeperImpl& sweeper_;
// TODO(chromium:1056170): Change to CancelableTask.
Handle handle_;
};
......@@ -1042,10 +1051,10 @@ class Sweeper::SweeperImpl final {
void ScheduleIncrementalSweeping() {
DCHECK(platform_);
auto runner = platform_->GetForegroundTaskRunner();
if (!runner || !runner->IdleTasksEnabled()) return;
if (!runner) return;
incremental_sweeper_handle_ =
IncrementalSweepTask::Post(this, runner.get());
IncrementalSweepTask::Post(*this, runner.get());
}
void ScheduleConcurrentSweeping() {
......@@ -1119,10 +1128,10 @@ bool Sweeper::IsSweepingInProgress() const {
return impl_->IsSweepingInProgress();
}
bool Sweeper::PerformSweepOnMutatorThread(double deadline_in_seconds) {
return impl_->PerformSweepOnMutatorThread(deadline_in_seconds,
StatsCollector::kSweepInTask);
bool Sweeper::PerformSweepOnMutatorThread(v8::base::TimeDelta max_duration,
StatsCollector::ScopeId scope_id) {
return impl_->PerformSweepOnMutatorThread(max_duration, scope_id,
MutatorThreadSweepingMode::kAll);
}
} // namespace internal
} // namespace cppgc
} // namespace cppgc::internal
......@@ -7,16 +7,13 @@
#include <memory>
#include "include/cppgc/heap.h"
#include "src/base/macros.h"
#include "src/base/platform/time.h"
#include "src/heap/cppgc/heap-config.h"
#include "src/heap/cppgc/memory.h"
#include "src/heap/cppgc/stats-collector.h"
namespace cppgc {
class Platform;
namespace internal {
namespace cppgc::internal {
class HeapBase;
class ConcurrentSweeperTest;
......@@ -24,17 +21,6 @@ class NormalPageSpace;
class V8_EXPORT_PRIVATE Sweeper final {
public:
struct SweepingConfig {
using SweepingType = cppgc::Heap::SweepingType;
enum class CompactableSpaceHandling { kSweep, kIgnore };
enum class FreeMemoryHandling { kDoNotDiscard, kDiscardWherePossible };
SweepingType sweeping_type = SweepingType::kIncrementalAndConcurrent;
CompactableSpaceHandling compactable_space_handling =
CompactableSpaceHandling::kSweep;
FreeMemoryHandling free_memory_handling = FreeMemoryHandling::kDoNotDiscard;
};
static constexpr bool CanDiscardMemory() {
return CheckMemoryIsInaccessibleIsNoop();
}
......@@ -63,7 +49,8 @@ class V8_EXPORT_PRIVATE Sweeper final {
bool IsSweepingInProgress() const;
// Assist with sweeping. Returns true if sweeping is done.
bool PerformSweepOnMutatorThread(double deadline_in_seconds);
bool PerformSweepOnMutatorThread(v8::base::TimeDelta max_duration,
StatsCollector::ScopeId);
private:
void WaitForConcurrentSweepingForTesting();
......@@ -76,7 +63,6 @@ class V8_EXPORT_PRIVATE Sweeper final {
friend class ConcurrentSweeperTest;
};
} // namespace internal
} // namespace cppgc
} // namespace cppgc::internal
#endif // V8_HEAP_CPPGC_SWEEPER_H_
......@@ -95,9 +95,9 @@ class CompactorTest : public testing::TestWithPlatform {
heap()->GetMarkerRefForTesting().reset();
FinishCompaction();
// Sweeping also verifies the object start bitmap.
const Sweeper::SweepingConfig sweeping_config{
Sweeper::SweepingConfig::SweepingType::kAtomic,
Sweeper::SweepingConfig::CompactableSpaceHandling::kIgnore};
const SweepingConfig sweeping_config{
SweepingConfig::SweepingType::kAtomic,
SweepingConfig::CompactableSpaceHandling::kIgnore};
heap()->sweeper().Start(sweeping_config);
}
......
......@@ -78,9 +78,9 @@ class ConcurrentSweeperTest : public testing::TestWithHeap {
GarbageCollector::Config::IsForcedGC::kNotForced);
heap->stats_collector()->NotifyMarkingCompleted(0);
Sweeper& sweeper = heap->sweeper();
const Sweeper::SweepingConfig sweeping_config{
Sweeper::SweepingConfig::SweepingType::kIncrementalAndConcurrent,
Sweeper::SweepingConfig::CompactableSpaceHandling::kSweep};
const SweepingConfig sweeping_config{
SweepingConfig::SweepingType::kIncrementalAndConcurrent,
SweepingConfig::CompactableSpaceHandling::kSweep};
sweeper.Start(sweeping_config);
}
......
......@@ -52,9 +52,9 @@ class SweeperTest : public testing::TestWithHeap {
GarbageCollector::Config::MarkingType::kAtomic,
GarbageCollector::Config::IsForcedGC::kNotForced);
heap->stats_collector()->NotifyMarkingCompleted(0);
const Sweeper::SweepingConfig sweeping_config{
Sweeper::SweepingConfig::SweepingType::kAtomic,
Sweeper::SweepingConfig::CompactableSpaceHandling::kSweep};
const SweepingConfig sweeping_config{
SweepingConfig::SweepingType::kAtomic,
SweepingConfig::CompactableSpaceHandling::kSweep};
sweeper.Start(sweeping_config);
sweeper.FinishIfRunning();
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment