cppgc: Introduce Sweeper

This ports sweeper logic from Blink into a separate entity - Sweeper.
Concurrent sweeping is in a followup.

Bug: chromium:1056170
Change-Id: I41196225f0d882cb0ab5190d23e297ee2498df6b
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2167858
Commit-Queue: Anton Bikineev <bikineev@chromium.org>
Reviewed-by: Omer Katz <omerkatz@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Reviewed-by: Hannes Payer <hpayer@chromium.org>
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#67581}

BUILD.gn

@@ -4078,6 +4078,8 @@ v8_source_set("cppgc_base") {
     "src/heap/cppgc/source-location.cc",
     "src/heap/cppgc/stack.cc",
     "src/heap/cppgc/stack.h",
+    "src/heap/cppgc/sweeper.cc",
+    "src/heap/cppgc/sweeper.h",
     "src/heap/cppgc/worklist.h",
   ]
 

include/cppgc/heap.h

@@ -35,6 +35,9 @@ class V8_EXPORT Heap {
     kUserDefined4,
   };
 
+  static constexpr size_t kMaxNumberOfSpaces =
+      static_cast<size_t>(SpaceType::kUserDefined4) + 1;
+
   static std::unique_ptr<Heap> Create();
 
   virtual ~Heap() = default;

src/heap/cppgc/free-list.cc

@@ -10,6 +10,7 @@
 #include "src/base/bits.h"
 #include "src/heap/cppgc/globals.h"
 #include "src/heap/cppgc/heap-object-header-inl.h"
+#include "src/heap/cppgc/sanitizers.h"
 
 namespace cppgc {
 namespace internal {
@@ -64,6 +65,8 @@ void FreeList::Add(FreeList::Block block) {
   DCHECK_GT(kPageSize, size);
   DCHECK_LE(kFreeListEntrySize, size);
 
+  // Make sure the freelist header is writable.
+  SET_MEMORY_ACCESIBLE(block.address, sizeof(Entry));
   Entry* entry = new (block.address) Entry(size);
   const size_t index = BucketIndexForSize(static_cast<uint32_t>(size));
   entry->Link(&free_list_heads_[index]);

src/heap/cppgc/heap-space.cc

@@ -7,7 +7,7 @@
 #include <algorithm>
 
 #include "src/base/logging.h"
-#include "src/heap/cppgc/raw-heap.h"
+#include "src/heap/cppgc/heap-page.h"
 
 namespace cppgc {
 namespace internal {
@@ -29,6 +29,14 @@ void BaseSpace::RemovePage(BasePage* page) {
 NormalPageSpace::NormalPageSpace(RawHeap* heap, size_t index)
     : BaseSpace(heap, index, PageType::kNormal) {}
 
+void NormalPageSpace::ResetLinearAllocationBuffer() {
+  if (current_lab_.size()) {
+    DCHECK_NOT_NULL(current_lab_.start());
+    free_list_.Add({current_lab_.start(), current_lab_.size()});
+    current_lab_.Set(nullptr, 0);
+  }
+}
+
 LargePageSpace::LargePageSpace(RawHeap* heap, size_t index)
     : BaseSpace(heap, index, PageType::kLarge) {}
 

src/heap/cppgc/heap-space.h

@@ -91,6 +91,8 @@ class V8_EXPORT_PRIVATE NormalPageSpace final : public BaseSpace {
 
   NormalPageSpace(RawHeap* heap, size_t index);
 
+  void ResetLinearAllocationBuffer();
+
   LinearAllocationBuffer& linear_allocation_buffer() { return current_lab_; }
   const LinearAllocationBuffer& linear_allocation_buffer() const {
     return current_lab_;

src/heap/cppgc/heap.cc

@@ -12,6 +12,7 @@
 #include "src/heap/cppgc/heap-page.h"
 #include "src/heap/cppgc/heap-visitor.h"
 #include "src/heap/cppgc/stack.h"
+#include "src/heap/cppgc/sweeper.h"
 
 namespace cppgc {
 
@@ -87,13 +88,13 @@ Heap::Heap()
     : raw_heap_(this),
       page_backend_(std::make_unique<PageBackend>(&system_allocator_)),
       object_allocator_(&raw_heap_),
+      sweeper_(&raw_heap_),
       stack_(std::make_unique<Stack>(v8::base::Stack::GetStackStart())),
       prefinalizer_handler_(std::make_unique<PreFinalizerHandler>()) {}
 
 Heap::~Heap() {
-  for (HeapObjectHeader* header : objects_) {
-    header->Finalize();
-  }
+  // Finish already running GC if any, but don't finalize live objects.
+  sweeper_.Finish();
 }
 
 void Heap::CollectGarbage(GCConfig config) {
@@ -112,16 +113,7 @@ void Heap::CollectGarbage(GCConfig config) {
     NoAllocationScope no_allocation_scope_(this);
     prefinalizer_handler_->InvokePreFinalizers();
   }
-  for (auto it = objects_.begin(); it != objects_.end();) {
-    HeapObjectHeader* header = *it;
-    if (header->IsMarked()) {
-      header->Unmark();
-      ++it;
-    } else {
-      header->Finalize();
-      it = objects_.erase(it);
-    }
-  }
+  sweeper_.Start(Sweeper::Config::kAtomic);
 }
 
 size_t Heap::ObjectPayloadSize() const {

src/heap/cppgc/heap.h

@@ -18,6 +18,7 @@
 #include "src/heap/cppgc/page-memory.h"
 #include "src/heap/cppgc/prefinalizer-handler.h"
 #include "src/heap/cppgc/raw-heap.h"
+#include "src/heap/cppgc/sweeper.h"
 
 namespace cppgc {
 namespace internal {
@@ -98,6 +99,8 @@ class V8_EXPORT_PRIVATE Heap final : public cppgc::Heap {
   PageBackend* page_backend() { return page_backend_.get(); }
   const PageBackend* page_backend() const { return page_backend_.get(); }
 
+  Sweeper& sweeper() { return sweeper_; }
+
   size_t ObjectPayloadSize() const;
 
  private:
@@ -109,6 +112,7 @@ class V8_EXPORT_PRIVATE Heap final : public cppgc::Heap {
   v8::base::PageAllocator system_allocator_;
   std::unique_ptr<PageBackend> page_backend_;
   ObjectAllocator object_allocator_;
+  Sweeper sweeper_;
 
   std::unique_ptr<Stack> stack_;
   std::unique_ptr<PreFinalizerHandler> prefinalizer_handler_;

src/heap/cppgc/object-allocator-inl.h

@@ -5,13 +5,13 @@
 #ifndef V8_HEAP_CPPGC_OBJECT_ALLOCATOR_INL_H_
 #define V8_HEAP_CPPGC_OBJECT_ALLOCATOR_INL_H_
 
-#include "src/heap/cppgc/object-allocator.h"
-
 #include <new>
 
 #include "src/base/logging.h"
 #include "src/heap/cppgc/heap-object-header-inl.h"
 #include "src/heap/cppgc/heap-object-header.h"
+#include "src/heap/cppgc/object-allocator.h"
+#include "src/heap/cppgc/sanitizers.h"
 
 namespace cppgc {
 namespace internal {
@@ -37,13 +37,16 @@ inline RawHeap::SpaceType ObjectAllocator::GetSpaceIndexForSize(size_t size) {
 void* ObjectAllocator::AllocateObjectOnSpace(NormalPageSpace* space,
                                              size_t size, GCInfoIndex gcinfo) {
   DCHECK_LT(0u, gcinfo);
+
   NormalPageSpace::LinearAllocationBuffer& current_lab =
       space->linear_allocation_buffer();
   if (current_lab.size() < size) {
     return OutOfLineAllocate(space, size, gcinfo);
   }
-  auto* header =
-      new (current_lab.Allocate(size)) HeapObjectHeader(size, gcinfo);
+
+  void* raw = current_lab.Allocate(size);
+  SET_MEMORY_ACCESIBLE(raw, size);
+  auto* header = new (raw) HeapObjectHeader(size, gcinfo);
   return header->Payload();
 }
 }  // namespace internal

src/heap/cppgc/object-allocator.cc

@@ -3,7 +3,6 @@
 // found in the LICENSE file.
 
 #include "src/heap/cppgc/object-allocator.h"
-#include "src/heap/cppgc/object-allocator-inl.h"
 
 #include "src/heap/cppgc/globals.h"
 #include "src/heap/cppgc/heap-object-header-inl.h"
@@ -11,7 +10,9 @@
 #include "src/heap/cppgc/heap-page.h"
 #include "src/heap/cppgc/heap-space.h"
 #include "src/heap/cppgc/heap.h"
+#include "src/heap/cppgc/object-allocator-inl.h"
 #include "src/heap/cppgc/page-memory.h"
+#include "src/heap/cppgc/sweeper.h"
 
 namespace cppgc {
 namespace internal {
@@ -19,15 +20,6 @@ namespace {
 
 void* AllocateLargeObject(RawHeap* raw_heap, LargePageSpace* space, size_t size,
                           GCInfoIndex gcinfo) {
-  // 1. Try to sweep large objects more than size bytes before allocating a new
-  // large object.
-  // TODO(chromium:1056170): Add lazy sweep.
-
-  // 2. If we have failed in sweeping size bytes, we complete sweeping before
-  // allocating this large object.
-  // TODO(chromium:1056170):
-  // raw_heap->heap()->sweeper()->Complete(space);
-
   LargePage* page = LargePage::Create(space, size);
   auto* header = new (page->ObjectHeader())
       HeapObjectHeader(HeapObjectHeader::kLargeObjectSizeInHeader, gcinfo);
@@ -61,8 +53,7 @@ void* ObjectAllocator::OutOfLineAllocate(NormalPageSpace* space, size_t size,
   // TODO(chromium:1056170): Add lazy sweep.
 
   // 4. Complete sweeping.
-  // TODO(chromium:1056170):
-  // raw_heap_->heap()->sweeper()->Complete(space);
+  raw_heap_->heap()->sweeper().Finish();
 
   // 5. Add a new page to this heap.
   NormalPage::Create(space);

src/heap/cppgc/raw-heap.h

@@ -21,11 +21,10 @@ class BaseSpace;
 
 // RawHeap is responsible for space management.
 class V8_EXPORT_PRIVATE RawHeap final {
-  static constexpr size_t kNumberOfSpaces = 9;
-
  public:
   using SpaceType = cppgc::Heap::SpaceType;
-  using Spaces = std::array<std::unique_ptr<BaseSpace>, kNumberOfSpaces>;
+  using Spaces =
+      std::array<std::unique_ptr<BaseSpace>, cppgc::Heap::kMaxNumberOfSpaces>;
 
   using iterator = Spaces::iterator;
   using const_iterator = Spaces::const_iterator;

src/heap/cppgc/sanitizers.h

@@ -5,6 +5,9 @@
 #ifndef V8_HEAP_CPPGC_SANITIZERS_H_
 #define V8_HEAP_CPPGC_SANITIZERS_H_
 
+#include <stdint.h>
+#include <string.h>
+
 #include "src/base/macros.h"
 
 //
@@ -16,10 +19,15 @@
 #include <sanitizer/asan_interface.h>
 
 #define NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address))
+#if !defined(ASAN_POISON_MEMORY_REGION) || !defined(ASAN_UNPOISON_MEMORY_REGION)
+#error "ASAN_POISON_MEMORY_REGION must be defined"
+#endif
 
 #else  // !V8_USE_ADDRESS_SANITIZER
 
 #define NO_SANITIZE_ADDRESS
+#define ASAN_POISON_MEMORY_REGION(addr, size) ((void)(addr), (void)(size))
+#define ASAN_UNPOISON_MEMORY_REGION(addr, size) ((void)(addr), (void)(size))
 
 #endif  // V8_USE_ADDRESS_SANITIZER
 
@@ -27,12 +35,43 @@
 
 #include <sanitizer/msan_interface.h>
 
+#define MSAN_POISON(addr, size) __msan_allocated_memory(addr, size)
 #define MSAN_UNPOISON(addr, size) __msan_unpoison(addr, size)
 
 #else  // !V8_USE_MEMORY_SANITIZER
 
+#define MSAN_POISON(addr, size) ((void)(addr), (void)(size))
 #define MSAN_UNPOISON(addr, size) ((void)(addr), (void)(size))
 
 #endif  // V8_USE_MEMORY_SANITIZER
 
+// API for newly allocated or reclaimed memory.
+#if defined(V8_USE_MEMORY_SANITIZER)
+#define SET_MEMORY_ACCESIBLE(address, size) \
+  MSAN_UNPOISON(address, size);             \
+  memset((address), 0, (size))
+#define SET_MEMORY_INACCESIBLE(address, size) MSAN_POISON((address), (size))
+#elif DEBUG || defined(V8_USE_ADDRESS_SANITIZER)
+#define SET_MEMORY_ACCESIBLE(address, size)   \
+  ASAN_UNPOISON_MEMORY_REGION(address, size); \
+  memset((address), 0, (size))
+#define SET_MEMORY_INACCESIBLE(address, size)      \
+  ::cppgc::internal::ZapMemory((address), (size)); \
+  ASAN_POISON_MEMORY_REGION(address, size)
+#else
+#define SET_MEMORY_ACCESIBLE(address, size) memset((address), 0, (size))
+#define SET_MEMORY_INACCESIBLE(address, size) ((void)(address), (void)(size))
+#endif
+
+namespace cppgc {
+namespace internal {
+
+inline void ZapMemory(void* address, size_t size) {
+  static constexpr uint8_t kZappedValue = 0xcd;
+  memset(address, kZappedValue, size);
+}
+
+}  // namespace internal
+}  // namespace cppgc
+
 #endif  // V8_HEAP_CPPGC_SANITIZERS_H_

src/heap/cppgc/sweeper.cc

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/heap/cppgc/sweeper.h"
+
+#include <array>
+
+#include "src/heap/cppgc/free-list.h"
+#include "src/heap/cppgc/heap-object-header-inl.h"
+#include "src/heap/cppgc/heap-object-header.h"
+#include "src/heap/cppgc/heap-page.h"
+#include "src/heap/cppgc/heap-space.h"
+#include "src/heap/cppgc/heap-visitor.h"
+#include "src/heap/cppgc/raw-heap.h"
+#include "src/heap/cppgc/sanitizers.h"
+
+namespace cppgc {
+namespace internal {
+
+namespace {
+
+struct SpaceState {
+  BaseSpace::Pages unswept_pages;
+};
+using SpaceStates = std::array<SpaceState, cppgc::Heap::kMaxNumberOfSpaces>;
+
+bool SweepNormalPage(NormalPage* page) {
+  constexpr auto kAtomicAccess = HeapObjectHeader::AccessMode::kAtomic;
+
+  auto* space = NormalPageSpace::From(page->space());
+  Address start_of_gap = page->PayloadStart();
+  for (Address begin = page->PayloadStart(), end = page->PayloadEnd();
+       begin != end;) {
+    HeapObjectHeader* header = reinterpret_cast<HeapObjectHeader*>(begin);
+    const size_t size = header->GetSize();
+    // Check if this is a free list entry.
+    if (header->IsFree<kAtomicAccess>()) {
+      SET_MEMORY_INACCESIBLE(header, kFreeListEntrySize);
+      begin += size;
+      continue;
+    }
+    // Check if object is not marked (not reachable).
+    if (!header->IsMarked<kAtomicAccess>()) {
+      header->Finalize();
+      SET_MEMORY_INACCESIBLE(header, size);
+      begin += size;
+      continue;
+    }
+    // The object is alive.
+    const Address header_address = reinterpret_cast<Address>(header);
+    if (start_of_gap != header_address) {
+      space->free_list().Add(
+          {start_of_gap, static_cast<size_t>(header_address - start_of_gap)});
+    }
+    header->Unmark<kAtomicAccess>();
+    begin += size;
+    start_of_gap = begin;
+  }
+
+  if (start_of_gap != page->PayloadStart() &&
+      start_of_gap != page->PayloadEnd()) {
+    space->free_list().Add(
+        {start_of_gap, static_cast<size_t>(page->PayloadEnd() - start_of_gap)});
+  }
+
+  const bool is_empty = (start_of_gap == page->PayloadStart());
+  return is_empty;
+}
+
+// This visitor:
+// - resets linear allocation buffers and clears free lists for all spaces;
+// - moves all Heap pages to local Sweeper's state (SpaceStates).
+class PrepareForSweepVisitor final
+    : public HeapVisitor<PrepareForSweepVisitor> {
+ public:
+  explicit PrepareForSweepVisitor(SpaceStates* states) : states_(states) {}
+
+  bool VisitNormalPageSpace(NormalPageSpace* space) {
+    space->ResetLinearAllocationBuffer();
+    space->free_list().Clear();
+    return false;
+  }
+
+  bool VisitNormalPage(NormalPage* page) {
+    MovePageToSweeper(page);
+    return true;
+  }
+
+  bool VisitLargePage(LargePage* page) {
+    MovePageToSweeper(page);
+    return true;
+  }
+
+ private:
+  void MovePageToSweeper(BasePage* page) {
+    BaseSpace* space = page->space();
+    space->RemovePage(page);
+    (*states_)[space->index()].unswept_pages.push_back(page);
+  }
+
+  SpaceStates* states_;
+};
+
+class MutatorThreadSweepVisitor final
+    : private HeapVisitor<MutatorThreadSweepVisitor> {
+  friend class HeapVisitor<MutatorThreadSweepVisitor>;
+
+ public:
+  explicit MutatorThreadSweepVisitor(SpaceStates* space_states) {
+    for (SpaceState& state : *space_states) {
+      for (BasePage* page : state.unswept_pages) {
+        Traverse(page);
+      }
+      state.unswept_pages.clear();
+    }
+  }
+
+ private:
+  bool VisitNormalPage(NormalPage* page) {
+    const bool is_empty = SweepNormalPage(page);
+    if (is_empty) {
+      NormalPage::Destroy(page);
+    } else {
+      page->space()->AddPage(page);
+    }
+    return true;
+  }
+
+  bool VisitLargePage(LargePage* page) {
+    if (page->ObjectHeader()->IsMarked()) {
+      page->space()->AddPage(page);
+    } else {
+      page->ObjectHeader()->Finalize();
+      LargePage::Destroy(page);
+    }
+    return true;
+  }
+};
+
+}  // namespace
+
+class Sweeper::SweeperImpl final {
+ public:
+  explicit SweeperImpl(RawHeap* heap) : heap_(heap) {}
+
+  void Start(Config config) {
+    is_in_progress_ = true;
+    PrepareForSweepVisitor(&space_states_).Traverse(heap_);
+    if (config == Config::kAtomic) {
+      Finish();
+    } else {
+      DCHECK_EQ(Config::kIncrementalAndConcurrent, config);
+      // TODO(chromium:1056170): Schedule concurrent sweeping.
+    }
+  }
+
+  void Finish() {
+    if (!is_in_progress_) return;
+
+    MutatorThreadSweepVisitor s(&space_states_);
+
+    is_in_progress_ = false;
+  }
+
+ private:
+  SpaceStates space_states_;
+  RawHeap* heap_;
+  bool is_in_progress_ = false;
+};
+
+Sweeper::Sweeper(RawHeap* heap) : impl_(std::make_unique<SweeperImpl>(heap)) {}
+Sweeper::~Sweeper() = default;
+
+void Sweeper::Start(Config config) { impl_->Start(config); }
+void Sweeper::Finish() { impl_->Finish(); }
+
+}  // namespace internal
+}  // namespace cppgc

src/heap/cppgc/sweeper.h

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_HEAP_CPPGC_SWEEPER_H_
+#define V8_HEAP_CPPGC_SWEEPER_H_
+
+#include <memory>
+
+#include "src/base/macros.h"
+
+namespace cppgc {
+namespace internal {
+
+class RawHeap;
+
+class V8_EXPORT_PRIVATE Sweeper final {
+ public:
+  enum class Config { kAtomic, kIncrementalAndConcurrent };
+
+  explicit Sweeper(RawHeap*);
+  ~Sweeper();
+
+  Sweeper(const Sweeper&) = delete;
+  Sweeper& operator=(const Sweeper&) = delete;
+
+  void Start(Config);
+  void Finish();
+
+ private:
+  class SweeperImpl;
+  std::unique_ptr<SweeperImpl> impl_;
+};
+
+}  // namespace internal
+}  // namespace cppgc
+
+#endif  // V8_HEAP_CPPGC_SWEEPER_H_

test/unittests/BUILD.gn

@@ -58,6 +58,7 @@ v8_source_set("cppgc_unittests_sources") {
     "heap/cppgc/prefinalizer_unittests.cc",
     "heap/cppgc/source-location_unittest.cc",
     "heap/cppgc/stack_unittest.cc",
+    "heap/cppgc/sweeper_unittest.cc",
     "heap/cppgc/tests.cc",
     "heap/cppgc/tests.h",
     "heap/cppgc/visitor_unittest.cc",

test/unittests/heap/cppgc/heap-page_unittest.cc

@@ -239,6 +239,10 @@ TEST_F(PageTest, UnsweptPageDestruction) {
         static_cast<LargePageSpace*>(heap.Space(RawHeap::SpaceType::kLarge));
     auto* page = LargePage::Create(space, 2 * kLargeObjectSizeThreshold);
     EXPECT_DEATH_IF_SUPPORTED(LargePage::Destroy(page), "");
+    // Detach page and really destroy page in the parent process so that sweeper
+    // doesn't consider it.
+    space->RemovePage(page);
+    LargePage::Destroy(page);
   }
 }
 #endif

test/unittests/heap/cppgc/sweeper_unittest.cc

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/heap/cppgc/sweeper.h"
+
+#include <algorithm>
+
+#include "include/cppgc/allocation.h"
+#include "include/cppgc/persistent.h"
+#include "src/heap/cppgc/globals.h"
+#include "src/heap/cppgc/heap-object-header-inl.h"
+#include "src/heap/cppgc/heap-object-header.h"
+#include "src/heap/cppgc/heap-page.h"
+#include "src/heap/cppgc/heap.h"
+#include "src/heap/cppgc/page-memory-inl.h"
+#include "src/heap/cppgc/page-memory.h"
+#include "test/unittests/heap/cppgc/tests.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace cppgc {
+namespace internal {
+
+namespace {
+
+size_t g_destructor_callcount;
+
+template <size_t Size>
+class GCed : public GarbageCollected<GCed<Size>> {
+ public:
+  virtual ~GCed() { ++g_destructor_callcount; }
+
+  virtual void Trace(cppgc::Visitor*) const {}
+
+ private:
+  char array[Size];
+};
+
+class SweeperTest : public testing::TestWithHeap {
+ public:
+  SweeperTest() { g_destructor_callcount = 0; }
+
+  void Sweep() {
+    Sweeper& sweeper = Heap::From(GetHeap())->sweeper();
+    sweeper.Start(Sweeper::Config::kAtomic);
+    sweeper.Finish();
+  }
+
+  void MarkObject(void* payload) {
+    HeapObjectHeader& header = HeapObjectHeader::FromPayload(payload);
+    header.TryMarkAtomic();
+  }
+
+  PageBackend* GetBackend() { return Heap::From(GetHeap())->page_backend(); }
+};
+
+}  // namespace
+
+TEST_F(SweeperTest, SweepUnmarkedNormalObject) {
+  constexpr size_t kObjectSize = 8;
+  using Type = GCed<kObjectSize>;
+
+  MakeGarbageCollected<Type>(GetHeap());
+
+  EXPECT_EQ(0u, g_destructor_callcount);
+
+  Sweep();
+
+  EXPECT_EQ(1u, g_destructor_callcount);
+}
+
+TEST_F(SweeperTest, DontSweepMarkedNormalObject) {
+  constexpr size_t kObjectSize = 8;
+  using Type = GCed<kObjectSize>;
+
+  auto* object = MakeGarbageCollected<Type>(GetHeap());
+  MarkObject(object);
+  BasePage* page = BasePage::FromPayload(object);
+  BaseSpace* space = page->space();
+
+  EXPECT_EQ(0u, g_destructor_callcount);
+
+  Sweep();
+
+  EXPECT_EQ(0u, g_destructor_callcount);
+  // Check that page is returned back to the space.
+  EXPECT_NE(space->end(), std::find(space->begin(), space->end(), page));
+  EXPECT_NE(nullptr, GetBackend()->Lookup(reinterpret_cast<Address>(object)));
+}
+
+TEST_F(SweeperTest, SweepUnmarkedLargeObject) {
+  constexpr size_t kObjectSize = kLargeObjectSizeThreshold * 2;
+  using Type = GCed<kObjectSize>;
+
+  auto* object = MakeGarbageCollected<Type>(GetHeap());
+  BasePage* page = BasePage::FromPayload(object);
+  BaseSpace* space = page->space();
+
+  EXPECT_EQ(0u, g_destructor_callcount);
+
+  Sweep();
+
+  EXPECT_EQ(1u, g_destructor_callcount);
+  // Check that page is gone.
+  EXPECT_EQ(space->end(), std::find(space->begin(), space->end(), page));
+  EXPECT_EQ(nullptr, GetBackend()->Lookup(reinterpret_cast<Address>(object)));
+}
+
+TEST_F(SweeperTest, DontSweepMarkedLargeObject) {
+  constexpr size_t kObjectSize = kLargeObjectSizeThreshold * 2;
+  using Type = GCed<kObjectSize>;
+
+  auto* object = MakeGarbageCollected<Type>(GetHeap());
+  MarkObject(object);
+  BasePage* page = BasePage::FromPayload(object);
+  BaseSpace* space = page->space();
+
+  EXPECT_EQ(0u, g_destructor_callcount);
+
+  Sweep();
+
+  EXPECT_EQ(0u, g_destructor_callcount);
+  // Check that page is returned back to the space.
+  EXPECT_NE(space->end(), std::find(space->begin(), space->end(), page));
+  EXPECT_NE(nullptr, GetBackend()->Lookup(reinterpret_cast<Address>(object)));
+}
+
+TEST_F(SweeperTest, SweepMultipleObjectsOnPage) {
+  constexpr size_t kObjectSize = 8;
+  using Type = GCed<kObjectSize>;
+  const size_t kNumberOfObjects =
+      NormalPage::PayloadSize() / (sizeof(Type) + sizeof(HeapObjectHeader));
+
+  for (size_t i = 0; i < kNumberOfObjects; ++i) {
+    MakeGarbageCollected<Type>(GetHeap());
+  }
+
+  EXPECT_EQ(0u, g_destructor_callcount);
+
+  Sweep();
+
+  EXPECT_EQ(kNumberOfObjects, g_destructor_callcount);
+}
+
+TEST_F(SweeperTest, SweepObjectsOnAllArenas) {
+  MakeGarbageCollected<GCed<1>>(GetHeap());
+  MakeGarbageCollected<GCed<32>>(GetHeap());
+  MakeGarbageCollected<GCed<64>>(GetHeap());
+  MakeGarbageCollected<GCed<128>>(GetHeap());
+  MakeGarbageCollected<GCed<2 * kLargeObjectSizeThreshold>>(GetHeap());
+
+  EXPECT_EQ(0u, g_destructor_callcount);
+
+  Sweep();
+
+  EXPECT_EQ(5u, g_destructor_callcount);
+}
+
+TEST_F(SweeperTest, CoalesceFreeListEntries) {
+  constexpr size_t kObjectSize = 32;
+  using Type = GCed<kObjectSize>;
+
+  auto* object1 = MakeGarbageCollected<Type>(GetHeap());
+  auto* object2 = MakeGarbageCollected<Type>(GetHeap());
+  auto* object3 = MakeGarbageCollected<Type>(GetHeap());
+  auto* object4 = MakeGarbageCollected<Type>(GetHeap());
+
+  MarkObject(object1);
+  MarkObject(object4);
+
+  Address object2_start =
+      reinterpret_cast<Address>(&HeapObjectHeader::FromPayload(object2));
+  Address object3_end =
+      reinterpret_cast<Address>(&HeapObjectHeader::FromPayload(object3)) +
+      HeapObjectHeader::FromPayload(object3).GetSize();
+
+  const BasePage* page = BasePage::FromPayload(object2);
+  const FreeList& freelist = NormalPageSpace::From(page->space())->free_list();
+
+  const FreeList::Block coalesced_block = {object2_start,
+                                           object3_end - object2_start};
+
+  EXPECT_EQ(0u, g_destructor_callcount);
+  EXPECT_FALSE(freelist.Contains(coalesced_block));
+
+  Sweep();
+
+  EXPECT_EQ(2u, g_destructor_callcount);
+  EXPECT_TRUE(freelist.Contains(coalesced_block));
+}
+
+}  // namespace internal
+}  // namespace cppgc