cppgc: Port backing store compaction.

This CL ports the existing backing store compaction algorithm from
blink. It does not attempt to improve on the existing algorithm.

Currently only unified heap uses the compaction implementation. It is
never triggered through standalone GCs.

The compaction implementation resides within an internal "subtle" namespace.

Bug: v8:10990
Change-Id: I4aa781db1b711e7aafc34234c4fb142de84394d7
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2485228
Commit-Queue: Omer Katz <omerkatz@chromium.org>
Reviewed-by: Anton Bikineev <bikineev@chromium.org>
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#70714}

BUILD.gn

@@ -4385,6 +4385,10 @@ v8_source_set("cppgc_base") {
     "include/cppgc/visitor.h",
     "include/v8config.h",
     "src/heap/cppgc/allocation.cc",
+    "src/heap/cppgc/compaction-worklists.cc",
+    "src/heap/cppgc/compaction-worklists.h",
+    "src/heap/cppgc/compactor.cc",
+    "src/heap/cppgc/compactor.h",
     "src/heap/cppgc/concurrent-marker.cc",
     "src/heap/cppgc/concurrent-marker.h",
     "src/heap/cppgc/free-list.cc",

include/cppgc/custom-space.h

@@ -14,8 +14,6 @@ struct CustomSpaceIndex {
   size_t value;
 };
 
-enum class CustomSpaceCompactability { kNotCompactable, kCompactable };
-
 /**
  * Top-level base class for custom spaces. Users must inherit from CustomSpace
  * below.
@@ -66,6 +64,28 @@ struct SpaceTrait {
   using Space = void;
 };
 
+namespace internal {
+
+template <typename CustomSpace>
+struct IsAllocatedOnCompactableSpaceImpl {
+  static constexpr bool value = CustomSpace::kSupportsCompaction;
+};
+
+template <>
+struct IsAllocatedOnCompactableSpaceImpl<void> {
+  // Non-custom spaces are by default not compactable.
+  static constexpr bool value = false;
+};
+
+template <typename T>
+struct IsAllocatedOnCompactableSpace {
+ public:
+  static constexpr bool value =
+      IsAllocatedOnCompactableSpaceImpl<typename SpaceTrait<T>::Space>::value;
+};
+
+}  // namespace internal
+
 }  // namespace cppgc
 
 #endif  // INCLUDE_CPPGC_CUSTOM_SPACE_H_

include/cppgc/member.h

@@ -24,7 +24,7 @@ class MemberBase {
   MemberBase() = default;
   explicit MemberBase(void* value) : raw_(value) {}
 
-  void* const* GetRawSlot() const { return &raw_; }
+  void** GetRawSlot() const { return &raw_; }
   void* GetRaw() const { return raw_; }
   void SetRaw(void* value) { raw_ = value; }
 
@@ -178,6 +178,10 @@ class BasicMember final : private MemberBase, private CheckingPolicy {
     return result;
   }
 
+  const T** GetSlotForTesting() const {
+    return reinterpret_cast<const T**>(const_cast<const void**>(GetRawSlot()));
+  }
+
  private:
   T* GetRawAtomic() const {
     return static_cast<T*>(MemberBase::GetRawAtomic());

include/cppgc/visitor.h

@@ -5,6 +5,7 @@
 #ifndef INCLUDE_CPPGC_VISITOR_H_
 #define INCLUDE_CPPGC_VISITOR_H_
 
+#include "cppgc/custom-space.h"
 #include "cppgc/ephemeron-pair.h"
 #include "cppgc/garbage-collected.h"
 #include "cppgc/internal/logging.h"
@@ -13,6 +14,7 @@
 #include "cppgc/member.h"
 #include "cppgc/source-location.h"
 #include "cppgc/trace-trait.h"
+#include "cppgc/type-traits.h"
 
 namespace cppgc {
 
@@ -26,7 +28,6 @@ class BasicPersistent;
 class ConservativeTracingVisitor;
 class VisitorBase;
 class VisitorFactory;
-
 }  // namespace internal
 
 using WeakCallback = void (*)(const LivenessBroker&, const void*);
@@ -82,6 +83,8 @@ class V8_EXPORT Visitor {
     static_assert(sizeof(T), "Pointee type must be fully defined.");
     static_assert(internal::IsGarbageCollectedType<T>::value,
                   "T must be GarbageCollected or GarbageCollectedMixin type");
+    static_assert(!internal::IsAllocatedOnCompactableSpace<T>::value,
+                  "Weak references to compactable objects are not allowed");
 
     const T* value = weak_member.GetRawAtomic();
 
@@ -176,6 +179,22 @@ class V8_EXPORT Visitor {
                        data);
   }
 
+  /**
+   * Registers a slot containing a reference to an object allocated on a
+   * compactable space. Such references maybe be arbitrarily moved by the GC.
+   *
+   * \param slot location of reference to object that might be moved by the GC.
+   */
+  template <typename T>
+  void RegisterMovableReference(const T** slot) {
+    static_assert(internal::IsAllocatedOnCompactableSpace<T>::value,
+                  "Only references to objects allocated on compactable spaces "
+                  "should be registered as movable slots.");
+    static_assert(!internal::IsGarbageCollectedMixinTypeV<T>,
+                  "Mixin types do not support compaction.");
+    HandleMovableReference(reinterpret_cast<const void**>(slot));
+  }
+
   /**
    * Registers a weak callback that is invoked during garbage collection.
    *
@@ -214,6 +233,7 @@ class V8_EXPORT Visitor {
   virtual void VisitWeakContainer(const void* self, TraceDescriptor strong_desc,
                                   TraceDescriptor weak_desc,
                                   WeakCallback callback, const void* data) {}
+  virtual void HandleMovableReference(const void**) {}
 
  private:
   template <typename T, void (T::*method)(const LivenessBroker&)>
@@ -261,6 +281,8 @@ class V8_EXPORT Visitor {
     static_assert(internal::IsGarbageCollectedType<PointeeType>::value,
                   "Persistent's pointee type must be GarbageCollected or "
                   "GarbageCollectedMixin");
+    static_assert(!internal::IsAllocatedOnCompactableSpace<PointeeType>::value,
+                  "Weak references to compactable objects are not allowed");
     VisitWeakRoot(p.Get(), TraceTrait<PointeeType>::GetTraceDescriptor(p.Get()),
                   &HandleWeak<WeakPersistent>, &p, loc);
   }

src/heap/cppgc-js/cpp-heap.cc

@@ -177,6 +177,12 @@ void CppHeap::TracePrologue(TraceFlags flags) {
       UnifiedHeapMarker::MarkingConfig::CollectionType::kMajor,
       cppgc::Heap::StackState::kNoHeapPointers,
       UnifiedHeapMarker::MarkingConfig::MarkingType::kIncrementalAndConcurrent};
+  if ((flags == TraceFlags::kReduceMemory) || (flags == TraceFlags::kForced)) {
+    // Only enable compaction when in a memory reduction garbage collection as
+    // it may significantly increase the final garbage collection pause.
+    compactor_.InitializeIfShouldCompact(marking_config.marking_type,
+                                         marking_config.stack_state);
+  }
   marker_ =
       cppgc::internal::MarkerFactory::CreateAndStartMarking<UnifiedHeapMarker>(
           *isolate_.heap(), AsBase(), platform_.get(), marking_config);
@@ -194,7 +200,12 @@ bool CppHeap::AdvanceTracing(double deadline_in_ms) {
 bool CppHeap::IsTracingDone() { return marking_done_; }
 
 void CppHeap::EnterFinalPause(EmbedderStackState stack_state) {
-  marker_->EnterAtomicPause(cppgc::Heap::StackState::kNoHeapPointers);
+  marker_->EnterAtomicPause(stack_state);
+  if (compactor_.CancelIfShouldNotCompact(
+          UnifiedHeapMarker::MarkingConfig::MarkingType::kAtomic,
+          stack_state)) {
+    marker_->NotifyCompactionCancelled();
+  }
 }
 
 void CppHeap::TraceEpilogue(TraceSummary* trace_summary) {
@@ -213,10 +224,15 @@ void CppHeap::TraceEpilogue(TraceSummary* trace_summary) {
   UnifiedHeapMarkingVerifier verifier(*this);
   verifier.Run(cppgc::Heap::StackState::kNoHeapPointers);
 #endif
+  cppgc::internal::Sweeper::SweepingConfig::CompactableSpaceHandling
+      compactable_space_handling = compactor_.CompactSpacesIfEnabled();
   {
     NoGCScope no_gc(*this);
-    sweeper().Start(
-        cppgc::internal::Sweeper::Config::kIncrementalAndConcurrent);
+    const cppgc::internal::Sweeper::SweepingConfig sweeping_config{
+        cppgc::internal::Sweeper::SweepingConfig::SweepingType::
+            kIncrementalAndConcurrent,
+        compactable_space_handling};
+    sweeper().Start(sweeping_config);
   }
 }
 

src/heap/cppgc-js/unified-heap-marking-visitor.cc

@@ -48,6 +48,12 @@ void UnifiedHeapMarkingVisitorBase::RegisterWeakCallback(WeakCallback callback,
   marking_state_.RegisterWeakCallback(callback, object);
 }
 
+void UnifiedHeapMarkingVisitorBase::HandleMovableReference(const void** slot) {
+  auto* worklist = marking_state_.movable_slots_worklist();
+  if (!worklist) return;
+  worklist->Push(slot);
+}
+
 namespace {
 void DeferredTraceJSMember(cppgc::Visitor* visitor, const void* ref) {
   static_cast<JSVisitor*>(visitor)->Trace(

src/heap/cppgc-js/unified-heap-marking-visitor.h

@@ -48,6 +48,7 @@ class V8_EXPORT_PRIVATE UnifiedHeapMarkingVisitorBase : public JSVisitor {
                           TraceDescriptor weak_desc, WeakCallback callback,
                           const void* data) final;
   void RegisterWeakCallback(WeakCallback, const void*) final;
+  void HandleMovableReference(const void**) final;
 
   // JS handling.
   void Visit(const internal::JSMemberBase& ref) final;

src/heap/cppgc/compaction-worklists.cc

+
+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/heap/cppgc/compaction-worklists.h"
+
+namespace cppgc {
+namespace internal {
+
+void CompactionWorklists::ClearForTesting() { movable_slots_worklist_.Clear(); }
+
+}  // namespace internal
+}  // namespace cppgc

src/heap/cppgc/compaction-worklists.h

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_HEAP_CPPGC_COMPACTION_WORKLISTS_H_
+#define V8_HEAP_CPPGC_COMPACTION_WORKLISTS_H_
+
+#include <unordered_set>
+
+#include "src/heap/base/worklist.h"
+
+namespace cppgc {
+namespace internal {
+
+class CompactionWorklists {
+ public:
+  using MovableReference = const void*;
+
+  using MovableReferencesWorklist =
+      heap::base::Worklist<MovableReference*, 256 /* local entries */>;
+
+  MovableReferencesWorklist* movable_slots_worklist() {
+    return &movable_slots_worklist_;
+  }
+
+  void ClearForTesting();
+
+ private:
+  MovableReferencesWorklist movable_slots_worklist_;
+};
+
+}  // namespace internal
+}  // namespace cppgc
+
+#endif  // V8_HEAP_CPPGC_COMPACTION_WORKLISTS_H_

src/heap/cppgc/compactor.h

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_HEAP_CPPGC_COMPACTOR_H_
+#define V8_HEAP_CPPGC_COMPACTOR_H_
+
+#include "src/heap/cppgc/compaction-worklists.h"
+#include "src/heap/cppgc/garbage-collector.h"
+#include "src/heap/cppgc/raw-heap.h"
+
+namespace cppgc {
+namespace internal {
+
+class V8_EXPORT_PRIVATE Compactor final {
+  using CompactableSpaceHandling =
+      Sweeper::SweepingConfig::CompactableSpaceHandling;
+
+ public:
+  explicit Compactor(RawHeap&);
+  ~Compactor() { DCHECK(!is_enabled_); }
+
+  void InitializeIfShouldCompact(GarbageCollector::Config::MarkingType,
+                                 GarbageCollector::Config::StackState);
+  // Returns true is compaction was cancelled.
+  bool CancelIfShouldNotCompact(GarbageCollector::Config::MarkingType,
+                                GarbageCollector::Config::StackState);
+  CompactableSpaceHandling CompactSpacesIfEnabled();
+
+  CompactionWorklists* compaction_worklists() {
+    return compaction_worklists_.get();
+  }
+
+  void EnableForNextGCForTesting() { enable_for_next_gc_for_testing_ = true; }
+
+  bool IsEnabledForTesting() const { return is_enabled_; }
+
+ private:
+  bool ShouldCompact(GarbageCollector::Config::MarkingType,
+                     GarbageCollector::Config::StackState);
+
+  RawHeap& heap_;
+  // Compactor does not own the compactable spaces. The heap owns all spaces.
+  std::vector<NormalPageSpace*> compactable_spaces_;
+
+  std::unique_ptr<CompactionWorklists> compaction_worklists_;
+
+  bool is_enabled_ = false;
+
+  bool enable_for_next_gc_for_testing_ = false;
+};
+
+}  // namespace internal
+}  // namespace cppgc
+
+#endif  // V8_HEAP_CPPGC_COMPACTOR_H_

src/heap/cppgc/concurrent-marker.cc

@@ -74,7 +74,8 @@ void ConcurrentMarkingTask::Run(JobDelegate* job_delegate) {
   if (!HasWorkForConcurrentMarking(concurrent_marker_.marking_worklists()))
     return;
   ConcurrentMarkingState concurrent_marking_state(
-      concurrent_marker_.heap(), concurrent_marker_.marking_worklists());
+      concurrent_marker_.heap(), concurrent_marker_.marking_worklists(),
+      concurrent_marker_.heap().compactor().compaction_worklists());
   std::unique_ptr<Visitor> concurrent_marking_visitor =
       concurrent_marker_.CreateConcurrentMarkingVisitor(
           concurrent_marking_state);
@@ -186,6 +187,10 @@ void ConcurrentMarkerBase::JoinForTesting() {
     concurrent_marking_handle_->Join();
 }
 
+bool ConcurrentMarkerBase::IsActive() const {
+  return concurrent_marking_handle_ && concurrent_marking_handle_->IsRunning();
+}
+
 ConcurrentMarkerBase::~ConcurrentMarkerBase() {
   CHECK_IMPLIES(concurrent_marking_handle_,
                 !concurrent_marking_handle_->IsValid());

src/heap/cppgc/concurrent-marker.h

@@ -30,6 +30,8 @@ class V8_EXPORT_PRIVATE ConcurrentMarkerBase {
 
   bool NotifyIncrementalMutatorStepCompleted();
 
+  bool IsActive() const;
+
   HeapBase& heap() const { return heap_; }
   MarkingWorklists& marking_worklists() const { return marking_worklists_; }
   IncrementalMarkingSchedule& incremental_marking_schedule() const {

src/heap/cppgc/garbage-collector.h

@@ -19,7 +19,7 @@ class GarbageCollector {
     using CollectionType = Marker::MarkingConfig::CollectionType;
     using StackState = cppgc::Heap::StackState;
     using MarkingType = Marker::MarkingConfig::MarkingType;
-    using SweepingType = Sweeper::Config;
+    using SweepingType = Sweeper::SweepingConfig::SweepingType;
 
     static constexpr Config ConservativeAtomicConfig() {
       return {CollectionType::kMajor, StackState::kMayContainHeapPointers,

src/heap/cppgc/heap-base.cc

@@ -70,6 +70,7 @@ HeapBase::HeapBase(
       stack_(std::make_unique<heap::base::Stack>(
           v8::base::Stack::GetStackStart())),
       prefinalizer_handler_(std::make_unique<PreFinalizerHandler>()),
+      compactor_(raw_heap_),
       object_allocator_(&raw_heap_, page_backend_.get(),
                         stats_collector_.get()),
       sweeper_(&raw_heap_, platform_.get(), stats_collector_.get()),

src/heap/cppgc/heap-base.h

@@ -12,6 +12,7 @@
 #include "include/cppgc/internal/persistent-node.h"
 #include "include/cppgc/macros.h"
 #include "src/base/macros.h"
+#include "src/heap/cppgc/compactor.h"
 #include "src/heap/cppgc/marker.h"
 #include "src/heap/cppgc/object-allocator.h"
 #include "src/heap/cppgc/raw-heap.h"
@@ -97,6 +98,8 @@ class V8_EXPORT_PRIVATE HeapBase {
 
   MarkerBase* marker() const { return marker_.get(); }
 
+  Compactor& compactor() { return compactor_; }
+
   ObjectAllocator& object_allocator() { return object_allocator_; }
 
   Sweeper& sweeper() { return sweeper_; }
@@ -154,6 +157,7 @@ class V8_EXPORT_PRIVATE HeapBase {
   std::unique_ptr<PreFinalizerHandler> prefinalizer_handler_;
   std::unique_ptr<MarkerBase> marker_;
 
+  Compactor compactor_;
   ObjectAllocator object_allocator_;
   Sweeper sweeper_;
 

src/heap/cppgc/heap.cc

@@ -168,7 +168,10 @@ void Heap::FinalizeGarbageCollection(Config::StackState stack_state) {
 #endif
   {
     NoGCScope no_gc(*this);
-    sweeper_.Start(config_.sweeping_type);
+    const Sweeper::SweepingConfig sweeping_config{
+        config_.sweeping_type,
+        Sweeper::SweepingConfig::CompactableSpaceHandling::kSweep};
+    sweeper_.Start(sweeping_config);
   }
   gc_in_progress_ = false;
 }

src/heap/cppgc/marker.cc

@@ -158,7 +158,8 @@ MarkerBase::MarkerBase(Key, HeapBase& heap, cppgc::Platform* platform,
       config_(config),
       platform_(platform),
       foreground_task_runner_(platform_->GetForegroundTaskRunner()),
-      mutator_marking_state_(heap, marking_worklists_) {}
+      mutator_marking_state_(heap, marking_worklists_,
+                             heap.compactor().compaction_worklists()) {}
 
 MarkerBase::~MarkerBase() {
   // The fixed point iteration may have found not-fully-constructed objects.
@@ -435,6 +436,8 @@ void MarkerBase::MarkNotFullyConstructedObjects() {
 
 void MarkerBase::ClearAllWorklistsForTesting() {
   marking_worklists_.ClearForTesting();
+  auto* compaction_worklists = heap_.compactor().compaction_worklists();
+  if (compaction_worklists) compaction_worklists->ClearForTesting();
 }
 
 void MarkerBase::DisableIncrementalMarkingForTesting() {
@@ -445,6 +448,13 @@ void MarkerBase::WaitForConcurrentMarkingForTesting() {
   concurrent_marker_->JoinForTesting();
 }
 
+void MarkerBase::NotifyCompactionCancelled() {
+  // Compaction cannot be cancelled while concurrent marking is active.
+  DCHECK_EQ(MarkingConfig::MarkingType::kAtomic, config_.marking_type);
+  DCHECK_IMPLIES(concurrent_marker_, !concurrent_marker_->IsActive());
+  mutator_marking_state_.NotifyCompactionCancelled();
+}
+
 Marker::Marker(Key key, HeapBase& heap, cppgc::Platform* platform,
                MarkingConfig config)
     : MarkerBase(key, heap, platform, config),

src/heap/cppgc/marker.h

@@ -124,6 +124,8 @@ class V8_EXPORT_PRIVATE MarkerBase {
 
   void WaitForConcurrentMarkingForTesting();
 
+  void NotifyCompactionCancelled();
+
  protected:
   static constexpr v8::base::TimeDelta kMaximumIncrementalStepDuration =
       v8::base::TimeDelta::FromMilliseconds(2);

src/heap/cppgc/marking-state.h

@@ -6,6 +6,7 @@
 #define V8_HEAP_CPPGC_MARKING_STATE_H_
 
 #include "include/cppgc/trace-trait.h"
+#include "src/heap/cppgc/compaction-worklists.h"
 #include "src/heap/cppgc/globals.h"
 #include "src/heap/cppgc/heap-object-header.h"
 #include "src/heap/cppgc/heap-page.h"
@@ -18,7 +19,8 @@ namespace internal {
 // C++ marking implementation.
 class MarkingStateBase {
  public:
-  inline MarkingStateBase(HeapBase& heap, MarkingWorklists&);
+  inline MarkingStateBase(HeapBase& heap, MarkingWorklists&,
+                          CompactionWorklists*);
 
   MarkingStateBase(const MarkingStateBase&) = delete;
   MarkingStateBase& operator=(const MarkingStateBase&) = delete;
@@ -53,6 +55,7 @@ class MarkingStateBase {
     concurrent_marking_bailout_worklist_.Publish();
     discovered_ephemeron_pairs_worklist_.Publish();
     ephemeron_pairs_for_processing_worklist_.Publish();
+    if (IsCompactionEnabled()) movable_slots_worklist_->Publish();
   }
 
   MarkingWorklists::MarkingWorklist::Local& marking_worklist() {
@@ -88,6 +91,17 @@ class MarkingStateBase {
     return weak_containers_worklist_;
   }
 
+  CompactionWorklists::MovableReferencesWorklist::Local*
+  movable_slots_worklist() {
+    return movable_slots_worklist_.get();
+  }
+
+  void NotifyCompactionCancelled() {
+    DCHECK(IsCompactionEnabled());
+    movable_slots_worklist_->Clear();
+    movable_slots_worklist_.reset();
+  }
+
  protected:
   inline void MarkAndPush(HeapObjectHeader&, TraceDescriptor);
 
@@ -95,6 +109,10 @@ class MarkingStateBase {
 
   inline void RegisterWeakContainer(HeapObjectHeader&);
 
+  inline bool IsCompactionEnabled() const {
+    return movable_slots_worklist_.get();
+  }
+
 #ifdef DEBUG
   HeapBase& heap_;
 #endif  // DEBUG
@@ -113,12 +131,17 @@ class MarkingStateBase {
   MarkingWorklists::EphemeronPairsWorklist::Local
       ephemeron_pairs_for_processing_worklist_;
   MarkingWorklists::WeakContainersWorklist& weak_containers_worklist_;
+  // Existence of the worklist (|movable_slot_worklist_| != nullptr) denotes
+  // that compaction is currently enabled and slots must be recorded.
+  std::unique_ptr<CompactionWorklists::MovableReferencesWorklist::Local>
+      movable_slots_worklist_;
 
   size_t marked_bytes_ = 0;
 };
 
 MarkingStateBase::MarkingStateBase(HeapBase& heap,
-                                   MarkingWorklists& marking_worklists)
+                                   MarkingWorklists& marking_worklists,
+                                   CompactionWorklists* compaction_worklists)
     :
 #ifdef DEBUG
       heap_(heap),
@@ -137,6 +160,11 @@ MarkingStateBase::MarkingStateBase(HeapBase& heap,
       ephemeron_pairs_for_processing_worklist_(
           marking_worklists.ephemeron_pairs_for_processing_worklist()),
       weak_containers_worklist_(*marking_worklists.weak_containers_worklist()) {
+  if (compaction_worklists) {
+    movable_slots_worklist_ =
+        std::make_unique<CompactionWorklists::MovableReferencesWorklist::Local>(
+            compaction_worklists->movable_slots_worklist());
+  }
 }
 
 void MarkingStateBase::MarkAndPush(const void* object, TraceDescriptor desc) {
@@ -260,8 +288,9 @@ void MarkingStateBase::AccountMarkedBytes(size_t marked_bytes) {
 
 class MutatorMarkingState : public MarkingStateBase {
  public:
-  MutatorMarkingState(HeapBase& heap, MarkingWorklists& marking_worklists)
-      : MarkingStateBase(heap, marking_worklists) {}
+  MutatorMarkingState(HeapBase& heap, MarkingWorklists& marking_worklists,
+                      CompactionWorklists* compaction_worklists)
+      : MarkingStateBase(heap, marking_worklists, compaction_worklists) {}
 
   inline bool MarkNoPush(HeapObjectHeader& header) {
     return MutatorMarkingState::MarkingStateBase::MarkNoPush(header);
@@ -327,8 +356,9 @@ bool MutatorMarkingState::IsMarkedWeakContainer(HeapObjectHeader& header) {
 
 class ConcurrentMarkingState : public MarkingStateBase {
  public:
-  ConcurrentMarkingState(HeapBase& heap, MarkingWorklists& marking_worklists)
-      : MarkingStateBase(heap, marking_worklists) {}
+  ConcurrentMarkingState(HeapBase& heap, MarkingWorklists& marking_worklists,
+                         CompactionWorklists* compaction_worklists)
+      : MarkingStateBase(heap, marking_worklists, compaction_worklists) {}
 
   ~ConcurrentMarkingState() { DCHECK_EQ(last_marked_bytes_, marked_bytes_); }
 

src/heap/cppgc/marking-visitor.cc

@@ -43,6 +43,12 @@ void MarkingVisitorBase::RegisterWeakCallback(WeakCallback callback,
   marking_state_.RegisterWeakCallback(callback, object);
 }
 
+void MarkingVisitorBase::HandleMovableReference(const void** slot) {
+  auto* worklist = marking_state_.movable_slots_worklist();
+  if (!worklist) return;
+  worklist->Push(slot);
+}
+
 ConservativeMarkingVisitor::ConservativeMarkingVisitor(
     HeapBase& heap, MutatorMarkingState& marking_state, cppgc::Visitor& visitor)
     : ConservativeTracingVisitor(heap, *heap.page_backend(), visitor),

src/heap/cppgc/marking-visitor.h

@@ -33,6 +33,7 @@ class V8_EXPORT_PRIVATE MarkingVisitorBase : public VisitorBase {
                           TraceDescriptor weak_desc, WeakCallback callback,
                           const void* data) final;
   void RegisterWeakCallback(WeakCallback, const void*) final;
+  void HandleMovableReference(const void**) final;
 
   MarkingStateBase& marking_state_;
 };

src/heap/cppgc/sweeper.cc

@@ -446,10 +446,19 @@ class ConcurrentSweepTask final : public cppgc::JobTask,
 // - moves all Heap pages to local Sweeper's state (SpaceStates).
 class PrepareForSweepVisitor final
     : public HeapVisitor<PrepareForSweepVisitor> {
+  using CompactableSpaceHandling =
+      Sweeper::SweepingConfig::CompactableSpaceHandling;
+
  public:
-  explicit PrepareForSweepVisitor(SpaceStates* states) : states_(states) {}
+  PrepareForSweepVisitor(SpaceStates* states,
+                         CompactableSpaceHandling compactable_space_handling)
+      : states_(states),
+        compactable_space_handling_(compactable_space_handling) {}
 
   bool VisitNormalPageSpace(NormalPageSpace* space) {
+    if ((compactable_space_handling_ == CompactableSpaceHandling::kIgnore) &&
+        space->is_compactable())
+      return true;
     DCHECK(!space->linear_allocation_buffer().size());
     space->free_list().Clear();
     ExtractPages(space);
@@ -469,6 +478,7 @@ class PrepareForSweepVisitor final
   }
 
   SpaceStates* states_;
+  CompactableSpaceHandling compactable_space_handling_;
 };
 
 }  // namespace
@@ -485,17 +495,20 @@ class Sweeper::SweeperImpl final {
 
   ~SweeperImpl() { CancelSweepers(); }
 
-  void Start(Config config) {
+  void Start(SweepingConfig config) {
     is_in_progress_ = true;
 #if DEBUG
+    // Verify bitmap for all spaces regardless of |compactable_space_handling|.
     ObjectStartBitmapVerifier().Verify(heap_);
 #endif
-    PrepareForSweepVisitor(&space_states_).Traverse(heap_);
+    PrepareForSweepVisitor(&space_states_, config.compactable_space_handling)
+        .Traverse(heap_);
 
-    if (config == Config::kAtomic) {
+    if (config.sweeping_type == SweepingConfig::SweepingType::kAtomic) {
       Finish();
     } else {
-      DCHECK_EQ(Config::kIncrementalAndConcurrent, config);
+      DCHECK_EQ(SweepingConfig::SweepingType::kIncrementalAndConcurrent,
+                config.sweeping_type);
       ScheduleIncrementalSweeping();
       ScheduleConcurrentSweeping();
     }
@@ -620,7 +633,7 @@ Sweeper::Sweeper(RawHeap* heap, cppgc::Platform* platform,
 
 Sweeper::~Sweeper() = default;
 
-void Sweeper::Start(Config config) { impl_->Start(config); }
+void Sweeper::Start(SweepingConfig config) { impl_->Start(config); }
 void Sweeper::FinishIfRunning() { impl_->FinishIfRunning(); }
 void Sweeper::WaitForConcurrentSweepingForTesting() {
   impl_->WaitForConcurrentSweepingForTesting();

src/heap/cppgc/sweeper.h

@@ -21,7 +21,14 @@ class ConcurrentSweeperTest;
 
 class V8_EXPORT_PRIVATE Sweeper final {
  public:
-  enum class Config { kAtomic, kIncrementalAndConcurrent };
+  struct SweepingConfig {
+    enum class SweepingType : uint8_t { kAtomic, kIncrementalAndConcurrent };
+    enum class CompactableSpaceHandling { kSweep, kIgnore };
+
+    SweepingType sweeping_type = SweepingType::kIncrementalAndConcurrent;
+    CompactableSpaceHandling compactable_space_handling =
+        CompactableSpaceHandling::kSweep;
+  };
 
   Sweeper(RawHeap*, cppgc::Platform*, StatsCollector*);
   ~Sweeper();
@@ -30,7 +37,7 @@ class V8_EXPORT_PRIVATE Sweeper final {
   Sweeper& operator=(const Sweeper&) = delete;
 
   // Sweeper::Start assumes the heap holds no linear allocation buffers.
-  void Start(Config);
+  void Start(SweepingConfig);
   void FinishIfRunning();
 
  private:

test/unittests/BUILD.gn

@@ -80,6 +80,7 @@ v8_source_set("cppgc_unittests_sources") {
   testonly = true
 
   sources = [
+    "heap/cppgc/compactor-unittest.cc",
     "heap/cppgc/concurrent-marking-unittest.cc",
     "heap/cppgc/concurrent-sweeper-unittest.cc",
     "heap/cppgc/cross-thread-persistent-unittest.cc",

test/unittests/heap/cppgc/compactor-unittest.cc

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/heap/cppgc/compactor.h"
+
+#include "include/cppgc/allocation.h"
+#include "include/cppgc/custom-space.h"
+#include "include/cppgc/persistent.h"
+#include "src/heap/cppgc/heap-object-header.h"
+#include "src/heap/cppgc/heap-page.h"
+#include "src/heap/cppgc/marker.h"
+#include "src/heap/cppgc/stats-collector.h"
+#include "test/unittests/heap/cppgc/tests.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace cppgc {
+
+class CompactableCustomSpace : public CustomSpace<CompactableCustomSpace> {
+ public:
+  static constexpr size_t kSpaceIndex = 0;
+  static constexpr bool kSupportsCompaction = true;
+};
+
+namespace internal {
+
+namespace {
+
+struct CompactableGCed : public GarbageCollected<CompactableGCed> {
+ public:
+  ~CompactableGCed() { ++g_destructor_callcount; }
+  void Trace(Visitor* visitor) const {
+    visitor->Trace(other);
+    visitor->RegisterMovableReference(other.GetSlotForTesting());
+  }
+  static size_t g_destructor_callcount;
+  Member<CompactableGCed> other;
+  size_t id = 0;
+};
+// static
+size_t CompactableGCed::g_destructor_callcount = 0;
+
+template <int kNumObjects>
+struct CompactableHolder
+    : public GarbageCollected<CompactableHolder<kNumObjects>> {
+ public:
+  explicit CompactableHolder(cppgc::AllocationHandle& allocation_handle) {
+    for (int i = 0; i < kNumObjects; ++i)
+      objects[i] = MakeGarbageCollected<CompactableGCed>(allocation_handle);
+  }
+
+  void Trace(Visitor* visitor) const {
+    for (int i = 0; i < kNumObjects; ++i) {
+      visitor->Trace(objects[i]);
+      visitor->RegisterMovableReference(objects[i].GetSlotForTesting());
+    }
+  }
+  Member<CompactableGCed> objects[kNumObjects];
+};
+
+class CompactorTest : public testing::TestWithPlatform {
+ public:
+  CompactorTest() {
+    Heap::HeapOptions options;
+    options.custom_spaces.emplace_back(
+        std::make_unique<CompactableCustomSpace>());
+    heap_ = Heap::Create(platform_, std::move(options));
+  }
+
+  void StartCompaction() {
+    compactor().EnableForNextGCForTesting();
+    compactor().InitializeIfShouldCompact(
+        GarbageCollector::Config::MarkingType::kIncremental,
+        GarbageCollector::Config::StackState::kNoHeapPointers);
+    EXPECT_TRUE(compactor().IsEnabledForTesting());
+  }
+
+  void CancelCompaction() {
+    bool cancelled = compactor().CancelIfShouldNotCompact(
+        GarbageCollector::Config::MarkingType::kAtomic,
+        GarbageCollector::Config::StackState::kMayContainHeapPointers);
+    EXPECT_TRUE(cancelled);
+  }
+
+  void FinishCompaction() { compactor().CompactSpacesIfEnabled(); }
+
+  void StartGC() {
+    CompactableGCed::g_destructor_callcount = 0u;
+    StartCompaction();
+    heap()->StartIncrementalGarbageCollection(
+        GarbageCollector::Config::PreciseIncrementalConfig());
+  }
+
+  void EndGC() {
+    heap()->marker()->FinishMarking(
+        GarbageCollector::Config::StackState::kNoHeapPointers);
+    FinishCompaction();
+    // Sweeping also verifies the object start bitmap.
+    const Sweeper::SweepingConfig sweeping_config{
+        Sweeper::SweepingConfig::SweepingType::kAtomic,
+        Sweeper::SweepingConfig::CompactableSpaceHandling::kIgnore};
+    heap()->sweeper().Start(sweeping_config);
+  }
+
+  Heap* heap() { return Heap::From(heap_.get()); }
+  cppgc::AllocationHandle& GetAllocationHandle() {
+    return heap_->GetAllocationHandle();
+  }
+  Compactor& compactor() { return heap()->compactor(); }
+
+ private:
+  std::unique_ptr<cppgc::Heap> heap_;
+};
+
+}  // namespace
+
+}  // namespace internal
+
+template <>
+struct SpaceTrait<internal::CompactableGCed> {
+  using Space = CompactableCustomSpace;
+};
+
+namespace internal {
+
+TEST_F(CompactorTest, NothingToCompact) {
+  StartCompaction();
+  FinishCompaction();
+}
+
+TEST_F(CompactorTest, CancelledNothingToCompact) {
+  StartCompaction();
+  CancelCompaction();
+}
+
+TEST_F(CompactorTest, NonEmptySpaceAllLive) {
+  static constexpr int kNumObjects = 10;
+  Persistent<CompactableHolder<kNumObjects>> holder =
+      MakeGarbageCollected<CompactableHolder<kNumObjects>>(
+          GetAllocationHandle(), GetAllocationHandle());
+  CompactableGCed* references[kNumObjects] = {nullptr};
+  for (int i = 0; i < kNumObjects; ++i) {
+    references[i] = holder->objects[i];
+  }
+  StartGC();
+  EndGC();
+  EXPECT_EQ(0u, CompactableGCed::g_destructor_callcount);
+  for (int i = 0; i < kNumObjects; ++i) {
+    EXPECT_EQ(holder->objects[i], references[i]);
+  }
+}
+
+TEST_F(CompactorTest, NonEmptySpaceAllDead) {
+  static constexpr int kNumObjects = 10;
+  Persistent<CompactableHolder<kNumObjects>> holder =
+      MakeGarbageCollected<CompactableHolder<kNumObjects>>(
+          GetAllocationHandle(), GetAllocationHandle());
+  CompactableGCed::g_destructor_callcount = 0u;
+  StartGC();
+  for (int i = 0; i < kNumObjects; ++i) {
+    holder->objects[i] = nullptr;
+  }
+  EndGC();
+  EXPECT_EQ(10u, CompactableGCed::g_destructor_callcount);
+}
+
+TEST_F(CompactorTest, NonEmptySpaceHalfLive) {
+  static constexpr int kNumObjects = 10;
+  Persistent<CompactableHolder<kNumObjects>> holder =
+      MakeGarbageCollected<CompactableHolder<kNumObjects>>(
+          GetAllocationHandle(), GetAllocationHandle());
+  CompactableGCed* references[kNumObjects] = {nullptr};
+  for (int i = 0; i < kNumObjects; ++i) {
+    references[i] = holder->objects[i];
+  }
+  StartGC();
+  for (int i = 0; i < kNumObjects; i += 2) {
+    holder->objects[i] = nullptr;
+  }
+  EndGC();
+  // Half of object were destroyed.
+  EXPECT_EQ(5u, CompactableGCed::g_destructor_callcount);
+  // Remaining objects are compacted.
+  for (int i = 1; i < kNumObjects; i += 2) {
+    EXPECT_EQ(holder->objects[i], references[i / 2]);
+  }
+}
+
+TEST_F(CompactorTest, CompactAcrossPages) {
+  Persistent<CompactableHolder<1>> holder =
+      MakeGarbageCollected<CompactableHolder<1>>(GetAllocationHandle(),
+                                                 GetAllocationHandle());
+  CompactableGCed* reference = holder->objects[0];
+  static constexpr size_t kObjectsPerPage =
+      kPageSize / (sizeof(CompactableGCed) + sizeof(HeapObjectHeader));
+  for (size_t i = 0; i < kObjectsPerPage; ++i) {
+    holder->objects[0] =
+        MakeGarbageCollected<CompactableGCed>(GetAllocationHandle());
+  }
+  // Last allocated object should be on a new page.
+  EXPECT_NE(reference, holder->objects[0]);
+  EXPECT_NE(BasePage::FromInnerAddress(heap(), reference),
+            BasePage::FromInnerAddress(heap(), holder->objects[0].Get()));
+  StartGC();
+  EndGC();
+  // Half of object were destroyed.
+  EXPECT_EQ(kObjectsPerPage, CompactableGCed::g_destructor_callcount);
+  EXPECT_EQ(reference, holder->objects[0]);
+}
+
+TEST_F(CompactorTest, InteriorSlotToPreviousObject) {
+  static constexpr int kNumObjects = 3;
+  Persistent<CompactableHolder<kNumObjects>> holder =
+      MakeGarbageCollected<CompactableHolder<kNumObjects>>(
+          GetAllocationHandle(), GetAllocationHandle());
+  CompactableGCed* references[kNumObjects] = {nullptr};
+  for (int i = 0; i < kNumObjects; ++i) {
+    references[i] = holder->objects[i];
+  }
+  holder->objects[2]->other = holder->objects[1];
+  holder->objects[1] = nullptr;
+  holder->objects[0] = nullptr;
+  StartGC();
+  EndGC();
+  EXPECT_EQ(1u, CompactableGCed::g_destructor_callcount);
+  EXPECT_EQ(references[1], holder->objects[2]);
+  EXPECT_EQ(references[0], holder->objects[2]->other);
+}
+
+TEST_F(CompactorTest, InteriorSlotToNextObject) {
+  static constexpr int kNumObjects = 3;
+  Persistent<CompactableHolder<kNumObjects>> holder =
+      MakeGarbageCollected<CompactableHolder<kNumObjects>>(
+          GetAllocationHandle(), GetAllocationHandle());
+  CompactableGCed* references[kNumObjects] = {nullptr};
+  for (int i = 0; i < kNumObjects; ++i) {
+    references[i] = holder->objects[i];
+  }
+  holder->objects[1]->other = holder->objects[2];
+  holder->objects[2] = nullptr;
+  holder->objects[0] = nullptr;
+  StartGC();
+  EndGC();
+  EXPECT_EQ(1u, CompactableGCed::g_destructor_callcount);
+  EXPECT_EQ(references[0], holder->objects[1]);
+  EXPECT_EQ(references[1], holder->objects[1]->other);
+}
+
+}  // namespace internal
+}  // namespace cppgc

test/unittests/heap/cppgc/concurrent-sweeper-unittest.cc

@@ -75,7 +75,10 @@ class ConcurrentSweeperTest : public testing::TestWithHeap {
     heap->stats_collector()->NotifyMarkingStarted();
     heap->stats_collector()->NotifyMarkingCompleted(0);
     Sweeper& sweeper = heap->sweeper();
-    sweeper.Start(Sweeper::Config::kIncrementalAndConcurrent);
+    const Sweeper::SweepingConfig sweeping_config{
+        Sweeper::SweepingConfig::SweepingType::kIncrementalAndConcurrent,
+        Sweeper::SweepingConfig::CompactableSpaceHandling::kSweep};
+    sweeper.Start(sweeping_config);
   }
 
   void WaitForConcurrentSweeping() {

test/unittests/heap/cppgc/sweeper-unittest.cc

@@ -48,7 +48,10 @@ class SweeperTest : public testing::TestWithHeap {
     // methods are called in the right order.
     heap->stats_collector()->NotifyMarkingStarted();
     heap->stats_collector()->NotifyMarkingCompleted(0);
-    sweeper.Start(Sweeper::Config::kAtomic);
+    const Sweeper::SweepingConfig sweeping_config{
+        Sweeper::SweepingConfig::SweepingType::kAtomic,
+        Sweeper::SweepingConfig::CompactableSpaceHandling::kSweep};
+    sweeper.Start(sweeping_config);
     sweeper.FinishIfRunning();
   }