cppgc: Implement allocation on custom spaces

This patch provides infrastructure to pin object types to specific
spaces. This allows embedders to create mutual exclusive arenas for
certain (base) types. In future, this will also be used to provide
sliding-window compaction on certain custom spaces.

We mainly preserve the existing infrastructure with the difference
that spaces are now slightly more dynamic than in Blink as they are
kept in a vector instead of a fixed-size array.

The mechanism differs from Blink in that it does not allow the user
object to call allocation methods directly but instead provides a
trait that can be overridden to specify a custom space.

The patch preserves templatization for objects that do not go into
custom spaces to safe a branch in the allocation hot path.

Change-Id: I08aa6932348e2d6258e19c4a32d189865f459f02
Bug: chromium:1056170
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2187611
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Reviewed-by: Omer Katz <omerkatz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#67666}

BUILD.gn

@@ -4020,6 +4020,7 @@ v8_source_set("cppgc_base") {
 
   sources = [
     "include/cppgc/allocation.h",
+    "include/cppgc/custom-space.h",
     "include/cppgc/garbage-collected.h",
     "include/cppgc/heap.h",
     "include/cppgc/internal/accessors.h",

include/cppgc/allocation.h

@@ -6,8 +6,10 @@
 #define INCLUDE_CPPGC_ALLOCATION_H_
 
 #include <stdint.h>
+
 #include <atomic>
 
+#include "cppgc/custom-space.h"
 #include "cppgc/garbage-collected.h"
 #include "cppgc/heap.h"
 #include "cppgc/internal/api-constants.h"
@@ -35,6 +37,8 @@ class V8_EXPORT MakeGarbageCollectedTraitInternal {
   }
 
   static void* Allocate(cppgc::Heap* heap, size_t size, GCInfoIndex index);
+  static void* Allocate(cppgc::Heap* heapx, size_t size, GCInfoIndex index,
+                        CustomSpaceIndex space_inde);
 
   friend class HeapObjectHeader;
 };
@@ -51,6 +55,28 @@ class V8_EXPORT MakeGarbageCollectedTraitInternal {
 template <typename T>
 class MakeGarbageCollectedTraitBase
     : private internal::MakeGarbageCollectedTraitInternal {
+ private:
+  template <typename U, typename CustomSpace>
+  struct SpacePolicy {
+    static void* Allocate(Heap* heap, size_t size) {
+      // Custom space.
+      static_assert(std::is_base_of<CustomSpaceBase, CustomSpace>::value,
+                    "Custom space must inherit from CustomSpaceBase.");
+      return internal::MakeGarbageCollectedTraitInternal::Allocate(
+          heap, size, internal::GCInfoTrait<T>::Index(),
+          CustomSpace::kSpaceIndex);
+    }
+  };
+
+  template <typename U>
+  struct SpacePolicy<U, void> {
+    static void* Allocate(Heap* heap, size_t size) {
+      // Default space.
+      return internal::MakeGarbageCollectedTraitInternal::Allocate(
+          heap, size, internal::GCInfoTrait<T>::Index());
+    }
+  };
+
  protected:
   /**
    * Allocates memory for an object of type T.
@@ -60,10 +86,7 @@ class MakeGarbageCollectedTraitBase
    * \returns the memory to construct an object of type T on.
    */
   static void* Allocate(Heap* heap, size_t size) {
-    // TODO(chromium:1056170): Allow specifying arena for specific embedder
-    // uses.
-    return internal::MakeGarbageCollectedTraitInternal::Allocate(
-        heap, size, internal::GCInfoTrait<T>::Index());
+    return SpacePolicy<T, typename SpaceTrait<T>::Space>::Allocate(heap, size);
   }
 
   /**

include/cppgc/custom-space.h

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef INCLUDE_CPPGC_CUSTOM_SPACE_H_
+#define INCLUDE_CPPGC_CUSTOM_SPACE_H_
+
+#include <stddef.h>
+
+namespace cppgc {
+
+using CustomSpaceIndex = size_t;
+
+/**
+ * Top-level base class for custom spaces. Users must inherit from CustomSpace
+ * below.
+ */
+class CustomSpaceBase {
+ public:
+  virtual ~CustomSpaceBase() = default;
+  virtual CustomSpaceIndex GetCustomSpaceIndex() const = 0;
+};
+
+/**
+ * Base class custom spaces should directly inherit from. The class inheriting
+ * from CustomSpace must define kSpaceIndex as unique space index. These
+ * indices need for form a sequence starting at 0.
+ *
+ * Example:
+ * \code
+ * class CustomSpace1 : public CustomSpace<CustomSpace1> {
+ *  public:
+ *   static constexpr CustomSpaceIndex kSpaceIndex = 0;
+ * };
+ * class CustomSpace2 : public CustomSpace<CustomSpace2> {
+ *  public:
+ *   static constexpr CustomSpaceIndex kSpaceIndex = 2;
+ * };
+ * \endcode
+ */
+template <typename ConcreteCustomSpace>
+class CustomSpace : public CustomSpaceBase {
+ public:
+  CustomSpaceIndex GetCustomSpaceIndex() const final {
+    return ConcreteCustomSpace::kSpaceIndex;
+  }
+};
+
+/**
+ * User-overridable trait that allows pinning types to custom spaces.
+ */
+template <typename T, typename = void>
+struct SpaceTrait {
+  using Space = void;
+};
+
+}  // namespace cppgc
+
+#endif  // INCLUDE_CPPGC_CUSTOM_SPACE_H_

include/cppgc/heap.h

@@ -6,7 +6,9 @@
 #define INCLUDE_CPPGC_HEAP_H_
 
 #include <memory>
+#include <vector>
 
+#include "cppgc/custom-space.h"
 #include "v8config.h"  // NOLINT(build/include_directory)
 
 namespace cppgc {
@@ -16,28 +18,6 @@ class Heap;
 
 class V8_EXPORT Heap {
  public:
-  // Normal spaces are used to store objects of different size classes:
-  // - kNormal1:  < 32 bytes
-  // - kNormal2:  < 64 bytes
-  // - kNormal3:  < 128 bytes
-  // - kNormal4: >= 128 bytes
-  // Objects of size greater than 2^16 get stored in the large space. Users can
-  // register up to 4 arenas for application specific needs.
-  enum class SpaceType {
-    kNormal1,
-    kNormal2,
-    kNormal3,
-    kNormal4,
-    kLarge,
-    kUserDefined1,
-    kUserDefined2,
-    kUserDefined3,
-    kUserDefined4,
-  };
-
-  static constexpr size_t kMaxNumberOfSpaces =
-      static_cast<size_t>(SpaceType::kUserDefined4) + 1;
-
   /**
    * Specifies the stack state the embedder is in.
    */
@@ -58,7 +38,18 @@ class V8_EXPORT Heap {
     kNonEmpty,
   };
 
-  static std::unique_ptr<Heap> Create();
+  struct HeapOptions {
+    static HeapOptions Default() { return {}; }
+
+    /**
+     * Custom spaces added to heap are required to have indices forming a
+     * numbered sequence starting at 0, i.e., their kSpaceIndex must correspond
+     * to the index they reside in the vector.
+     */
+    std::vector<std::unique_ptr<CustomSpaceBase>> custom_spaces;
+  };
+
+  static std::unique_ptr<Heap> Create(HeapOptions = HeapOptions::Default());
 
   virtual ~Heap() = default;
 

src/heap/cppgc/allocation.cc

@@ -22,5 +22,13 @@ void* MakeGarbageCollectedTraitInternal::Allocate(cppgc::Heap* heap,
   return Heap::From(heap)->Allocate(size, index);
 }
 
+// static
+void* MakeGarbageCollectedTraitInternal::Allocate(
+    cppgc::Heap* heap, size_t size, GCInfoIndex index,
+    CustomSpaceIndex space_index) {
+  DCHECK_NOT_NULL(heap);
+  return Heap::From(heap)->Allocate(size, index, space_index);
+}
+
 }  // namespace internal
 }  // namespace cppgc

src/heap/cppgc/heap-inl.h

@@ -20,6 +20,14 @@ void* Heap::Allocate(size_t size, GCInfoIndex index) {
   return result;
 }
 
+void* Heap::Allocate(size_t size, GCInfoIndex index,
+                     CustomSpaceIndex space_index) {
+  DCHECK(is_allocation_allowed());
+  void* result = object_allocator_.AllocateObject(size, index, space_index);
+  objects_.push_back(&HeapObjectHeader::FromPayload(result));
+  return result;
+}
+
 }  // namespace internal
 }  // namespace cppgc
 

src/heap/cppgc/heap.cc

@@ -16,8 +16,24 @@
 
 namespace cppgc {
 
-std::unique_ptr<Heap> Heap::Create() {
-  return std::make_unique<internal::Heap>();
+namespace {
+
+void VerifyCustomSpaces(
+    const std::vector<std::unique_ptr<CustomSpaceBase>>& custom_spaces) {
+  // Ensures that user-provided custom spaces have indices that form a sequence
+  // starting at 0.
+#ifdef DEBUG
+  for (size_t i = 0; i < custom_spaces.size(); ++i) {
+    DCHECK_EQ(i, custom_spaces[i]->GetCustomSpaceIndex());
+  }
+#endif  // DEBUG
+}
+
+}  // namespace
+
+std::unique_ptr<Heap> Heap::Create(cppgc::Heap::HeapOptions options) {
+  VerifyCustomSpaces(options.custom_spaces);
+  return std::make_unique<internal::Heap>(options.custom_spaces.size());
 }
 
 void Heap::ForceGarbageCollectionSlow(const char* source, const char* reason,
@@ -65,8 +81,8 @@ cppgc::LivenessBroker LivenessBrokerFactory::Create() {
   return cppgc::LivenessBroker();
 }
 
-Heap::Heap()
-    : raw_heap_(this),
+Heap::Heap(size_t custom_spaces)
+    : raw_heap_(this, custom_spaces),
       page_backend_(std::make_unique<PageBackend>(&system_allocator_)),
       object_allocator_(&raw_heap_),
       sweeper_(&raw_heap_),

src/heap/cppgc/heap.h

@@ -74,10 +74,12 @@ class V8_EXPORT_PRIVATE Heap final : public cppgc::Heap {
 
   static Heap* From(cppgc::Heap* heap) { return static_cast<Heap*>(heap); }
 
-  Heap();
+  explicit Heap(size_t custom_spaces);
   ~Heap() final;
 
   inline void* Allocate(size_t size, GCInfoIndex index);
+  inline void* Allocate(size_t size, GCInfoIndex index,
+                        CustomSpaceIndex space_index);
 
   void CollectGarbage(GCConfig config = GCConfig::Default());
 

src/heap/cppgc/object-allocator-inl.h

@@ -19,19 +19,32 @@ namespace internal {
 void* ObjectAllocator::AllocateObject(size_t size, GCInfoIndex gcinfo) {
   const size_t allocation_size =
       RoundUp(size + sizeof(HeapObjectHeader), kAllocationGranularity);
-  const RawHeap::SpaceType type = GetSpaceIndexForSize(allocation_size);
+  const RawHeap::RegularSpaceType type =
+      GetInitialSpaceIndexForSize(allocation_size);
   return AllocateObjectOnSpace(NormalPageSpace::From(raw_heap_->Space(type)),
                                allocation_size, gcinfo);
 }
 
+void* ObjectAllocator::AllocateObject(size_t size, GCInfoIndex gcinfo,
+                                      CustomSpaceIndex space_index) {
+  const size_t allocation_size =
+      RoundUp(size + sizeof(HeapObjectHeader), kAllocationGranularity);
+  const size_t internal_space_index =
+      raw_heap_->SpaceIndexForCustomSpace(space_index);
+  return AllocateObjectOnSpace(
+      NormalPageSpace::From(raw_heap_->Space(internal_space_index)),
+      allocation_size, gcinfo);
+}
+
 // static
-inline RawHeap::SpaceType ObjectAllocator::GetSpaceIndexForSize(size_t size) {
+inline RawHeap::RegularSpaceType ObjectAllocator::GetInitialSpaceIndexForSize(
+    size_t size) {
   if (size < 64) {
-    if (size < 32) return RawHeap::SpaceType::kNormal1;
-    return RawHeap::SpaceType::kNormal2;
+    if (size < 32) return RawHeap::RegularSpaceType::kNormal1;
+    return RawHeap::RegularSpaceType::kNormal2;
   }
-  if (size < 128) return RawHeap::SpaceType::kNormal3;
-  return RawHeap::SpaceType::kNormal4;
+  if (size < 128) return RawHeap::RegularSpaceType::kNormal3;
+  return RawHeap::RegularSpaceType::kNormal4;
 }
 
 void* ObjectAllocator::AllocateObjectOnSpace(NormalPageSpace* space,

src/heap/cppgc/object-allocator.cc

@@ -38,8 +38,8 @@ void* ObjectAllocator::OutOfLineAllocate(NormalPageSpace* space, size_t size,
 
   // 1. If this allocation is big enough, allocate a large object.
   if (size >= kLargeObjectSizeThreshold) {
-    auto* large_space =
-        LargePageSpace::From(raw_heap_->Space(RawHeap::SpaceType::kLarge));
+    auto* large_space = LargePageSpace::From(
+        raw_heap_->Space(RawHeap::RegularSpaceType::kLarge));
     return AllocateLargeObject(raw_heap_, large_space, size, gcinfo);
   }
 

src/heap/cppgc/object-allocator.h

@@ -17,9 +17,14 @@ class V8_EXPORT_PRIVATE ObjectAllocator final {
   explicit ObjectAllocator(RawHeap* heap);
 
   inline void* AllocateObject(size_t size, GCInfoIndex gcinfo);
+  inline void* AllocateObject(size_t size, GCInfoIndex gcinfo,
+                              CustomSpaceIndex space_index);
 
  private:
-  inline static RawHeap::SpaceType GetSpaceIndexForSize(size_t size);
+  // Returns the initially tried SpaceType to allocate an object of |size| bytes
+  // on. Returns the largest regular object size bucket for large objects.
+  inline static RawHeap::RegularSpaceType GetInitialSpaceIndexForSize(
+      size_t size);
 
   inline void* AllocateObjectOnSpace(NormalPageSpace* space, size_t size,
                                      GCInfoIndex gcinfo);

src/heap/cppgc/raw-heap.cc

@@ -9,14 +9,22 @@
 namespace cppgc {
 namespace internal {
 
-RawHeap::RawHeap(Heap* heap) : main_heap_(heap), used_spaces_(0) {
+// static
+constexpr size_t RawHeap::kNumberOfRegularSpaces;
+
+RawHeap::RawHeap(Heap* heap, size_t custom_spaces)
+    : main_heap_(heap), custom_spaces_(custom_spaces) {
   size_t i = 0;
-  for (; i < static_cast<size_t>(SpaceType::kLarge); ++i) {
-    spaces_[i] = std::make_unique<NormalPageSpace>(this, i);
+  for (; i < static_cast<size_t>(RegularSpaceType::kLarge); ++i) {
+    spaces_.push_back(std::make_unique<NormalPageSpace>(this, i));
+  }
+  spaces_.push_back(std::make_unique<LargePageSpace>(
+      this, static_cast<size_t>(RegularSpaceType::kLarge)));
+  DCHECK_EQ(kNumberOfRegularSpaces, spaces_.size());
+  for (size_t j = 0; j < custom_spaces; j++) {
+    spaces_.push_back(
+        std::make_unique<NormalPageSpace>(this, kNumberOfRegularSpaces + j));
   }
-  spaces_[i] = std::make_unique<LargePageSpace>(
-      this, static_cast<size_t>(SpaceType::kLarge));
-  used_spaces_ = i + 1;
 }
 
 RawHeap::~RawHeap() = default;

src/heap/cppgc/raw-heap.h

@@ -5,9 +5,9 @@
 #ifndef V8_HEAP_CPPGC_RAW_HEAP_H_
 #define V8_HEAP_CPPGC_RAW_HEAP_H_
 
-#include <array>
 #include <iterator>
 #include <memory>
+#include <vector>
 
 #include "include/cppgc/heap.h"
 #include "src/base/logging.h"
@@ -22,44 +22,79 @@ class BaseSpace;
 // RawHeap is responsible for space management.
 class V8_EXPORT_PRIVATE RawHeap final {
  public:
-  using SpaceType = cppgc::Heap::SpaceType;
-  using Spaces =
-      std::array<std::unique_ptr<BaseSpace>, cppgc::Heap::kMaxNumberOfSpaces>;
+  // Normal spaces are used to store objects of different size classes:
+  // - kNormal1:  < 32 bytes
+  // - kNormal2:  < 64 bytes
+  // - kNormal3:  < 128 bytes
+  // - kNormal4: >= 128 bytes
+  //
+  // Objects of size greater than 2^16 get stored in the large space.
+  //
+  // Users can override where objects are allocated via Heap::SpacePolicy and
+  // force allocation in one of the kUserDefined* spaces.
+  enum class RegularSpaceType : uint8_t {
+    kNormal1,
+    kNormal2,
+    kNormal3,
+    kNormal4,
+    kLarge,
+  };
 
+  static constexpr size_t kNumberOfRegularSpaces =
+      static_cast<size_t>(RegularSpaceType::kLarge) + 1;
+
+  using Spaces = std::vector<std::unique_ptr<BaseSpace>>;
   using iterator = Spaces::iterator;
   using const_iterator = Spaces::const_iterator;
 
-  explicit RawHeap(Heap* heap);
+  explicit RawHeap(Heap* heap, size_t custom_spaces);
   ~RawHeap();
 
   // Space iteration support.
   iterator begin() { return spaces_.begin(); }
   const_iterator begin() const { return spaces_.begin(); }
-  iterator end() { return std::next(spaces_.begin(), used_spaces_); }
-  const_iterator end() const {
-    return std::next(spaces_.begin(), used_spaces_);
-  }
+  iterator end() { return spaces_.end(); }
+  const_iterator end() const { return spaces_.end(); }
+
+  iterator custom_begin() { return std::next(begin(), kNumberOfRegularSpaces); }
+  iterator custom_end() { return end(); }
 
-  size_t size() const { return used_spaces_; }
+  size_t size() const { return spaces_.size(); }
+  size_t custom_spaces() const { return custom_spaces_; }
 
-  BaseSpace* Space(SpaceType type) {
+  BaseSpace* Space(RegularSpaceType type) {
     const size_t index = static_cast<size_t>(type);
     DCHECK_GT(spaces_.size(), index);
     BaseSpace* space = spaces_[index].get();
     DCHECK(space);
     return space;
   }
-  const BaseSpace* Space(SpaceType space) const {
+  const BaseSpace* Space(RegularSpaceType space) const {
     return const_cast<RawHeap&>(*this).Space(space);
   }
 
+  BaseSpace* Space(size_t space_index) {
+    DCHECK_GT(spaces_.size(), space_index);
+    BaseSpace* space = spaces_[space_index].get();
+    DCHECK(space);
+    return space;
+  }
+  const BaseSpace* Space(size_t space_index) const {
+    return const_cast<RawHeap&>(*this).Space(space_index);
+  }
+
+  size_t SpaceIndexForCustomSpace(CustomSpaceIndex space_index) {
+    DCHECK_LT(space_index, custom_spaces_);
+    return kNumberOfRegularSpaces + space_index;
+  }
+
   Heap* heap() { return main_heap_; }
   const Heap* heap() const { return main_heap_; }
 
  private:
   Heap* main_heap_;
   Spaces spaces_;
-  size_t used_spaces_;
+  size_t custom_spaces_;
 };
 
 }  // namespace internal

src/heap/cppgc/sweeper.cc

@@ -4,7 +4,7 @@
 
 #include "src/heap/cppgc/sweeper.h"
 
-#include <array>
+#include <vector>
 
 #include "src/heap/cppgc/free-list.h"
 #include "src/heap/cppgc/heap-object-header-inl.h"
@@ -23,7 +23,8 @@ namespace {
 struct SpaceState {
   BaseSpace::Pages unswept_pages;
 };
-using SpaceStates = std::array<SpaceState, cppgc::Heap::kMaxNumberOfSpaces>;
+// using SpaceStates = std::array<SpaceState, RawHeap::kMaxNumberOfSpaces>;
+using SpaceStates = std::vector<SpaceState>;
 
 bool SweepNormalPage(NormalPage* page) {
   constexpr auto kAtomicAccess = HeapObjectHeader::AccessMode::kAtomic;
@@ -142,7 +143,9 @@ class MutatorThreadSweepVisitor final
 
 class Sweeper::SweeperImpl final {
  public:
-  explicit SweeperImpl(RawHeap* heap) : heap_(heap) {}
+  explicit SweeperImpl(RawHeap* heap) : heap_(heap) {
+    space_states_.resize(heap_->size());
+  }
 
   void Start(Config config) {
     is_in_progress_ = true;

test/unittests/BUILD.gn

@@ -44,6 +44,7 @@ v8_source_set("cppgc_unittests_sources") {
   testonly = true
 
   sources = [
+    "heap/cppgc/custom-spaces_unittest.cc",
     "heap/cppgc/finalizer-trait_unittest.cc",
     "heap/cppgc/free-list_unittest.cc",
     "heap/cppgc/garbage-collected_unittest.cc",

test/unittests/heap/cppgc/custom-spaces_unittest.cc

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "include/cppgc/allocation.h"
+#include "include/cppgc/custom-space.h"
+#include "src/heap/cppgc/heap-page.h"
+#include "src/heap/cppgc/raw-heap.h"
+#include "test/unittests/heap/cppgc/tests.h"
+
+namespace cppgc {
+
+class CustomSpace1 : public CustomSpace<CustomSpace1> {
+ public:
+  static constexpr size_t kSpaceIndex = 0;
+};
+
+class CustomSpace2 : public CustomSpace<CustomSpace2> {
+ public:
+  static constexpr size_t kSpaceIndex = 1;
+};
+
+namespace internal {
+
+namespace {
+
+size_t g_destructor_callcount;
+
+class TestWithHeapWithCustomSpaces : public testing::TestWithPlatform {
+ protected:
+  TestWithHeapWithCustomSpaces() {
+    Heap::HeapOptions options;
+    options.custom_spaces.emplace_back(std::make_unique<CustomSpace1>());
+    options.custom_spaces.emplace_back(std::make_unique<CustomSpace2>());
+    heap_ = Heap::Create(std::move(options));
+    g_destructor_callcount = 0;
+  }
+
+  void PreciseGC() {
+    heap_->ForceGarbageCollectionSlow("TestWithHeapWithCustomSpaces", "Testing",
+                                      Heap::GCConfig::StackState::kEmpty);
+  }
+
+  cppgc::Heap* GetHeap() const { return heap_.get(); }
+
+ private:
+  std::unique_ptr<cppgc::Heap> heap_;
+};
+
+class RegularGCed final : public GarbageCollected<RegularGCed> {};
+
+class CustomGCed1 final : public GarbageCollected<CustomGCed1> {
+ public:
+  ~CustomGCed1() { g_destructor_callcount++; }
+};
+class CustomGCed2 final : public GarbageCollected<CustomGCed2> {
+ public:
+  ~CustomGCed2() { g_destructor_callcount++; }
+};
+
+class CustomGCedBase : public GarbageCollected<CustomGCedBase> {};
+class CustomGCedFinal1 final : public CustomGCedBase {
+ public:
+  ~CustomGCedFinal1() { g_destructor_callcount++; }
+};
+class CustomGCedFinal2 final : public CustomGCedBase {
+ public:
+  ~CustomGCedFinal2() { g_destructor_callcount++; }
+};
+
+}  // namespace
+
+}  // namespace internal
+
+template <>
+struct SpaceTrait<internal::CustomGCed1> {
+  using Space = CustomSpace1;
+};
+
+template <>
+struct SpaceTrait<internal::CustomGCed2> {
+  using Space = CustomSpace2;
+};
+
+template <typename T>
+struct SpaceTrait<
+    T, std::enable_if_t<std::is_base_of<internal::CustomGCedBase, T>::value>> {
+  using Space = CustomSpace1;
+};
+
+namespace internal {
+
+TEST_F(TestWithHeapWithCustomSpaces, AllocateOnCustomSpaces) {
+  auto* regular = MakeGarbageCollected<RegularGCed>(GetHeap());
+  auto* custom1 = MakeGarbageCollected<CustomGCed1>(GetHeap());
+  auto* custom2 = MakeGarbageCollected<CustomGCed2>(GetHeap());
+  EXPECT_EQ(RawHeap::kNumberOfRegularSpaces,
+            NormalPage::FromPayload(custom1)->space()->index());
+  EXPECT_EQ(RawHeap::kNumberOfRegularSpaces + 1,
+            NormalPage::FromPayload(custom2)->space()->index());
+  EXPECT_EQ(static_cast<size_t>(RawHeap::RegularSpaceType::kNormal1),
+            NormalPage::FromPayload(regular)->space()->index());
+}
+
+TEST_F(TestWithHeapWithCustomSpaces,
+       AllocateOnCustomSpacesSpecifiedThroughBase) {
+  auto* regular = MakeGarbageCollected<RegularGCed>(GetHeap());
+  auto* custom1 = MakeGarbageCollected<CustomGCedFinal1>(GetHeap());
+  auto* custom2 = MakeGarbageCollected<CustomGCedFinal2>(GetHeap());
+  EXPECT_EQ(RawHeap::kNumberOfRegularSpaces,
+            NormalPage::FromPayload(custom1)->space()->index());
+  EXPECT_EQ(RawHeap::kNumberOfRegularSpaces,
+            NormalPage::FromPayload(custom2)->space()->index());
+  EXPECT_EQ(static_cast<size_t>(RawHeap::RegularSpaceType::kNormal1),
+            NormalPage::FromPayload(regular)->space()->index());
+}
+
+TEST_F(TestWithHeapWithCustomSpaces, SweepCustomSpace) {
+  MakeGarbageCollected<CustomGCedFinal1>(GetHeap());
+  MakeGarbageCollected<CustomGCedFinal2>(GetHeap());
+  MakeGarbageCollected<CustomGCed1>(GetHeap());
+  MakeGarbageCollected<CustomGCed2>(GetHeap());
+  EXPECT_EQ(0u, g_destructor_callcount);
+  PreciseGC();
+  EXPECT_EQ(4u, g_destructor_callcount);
+}
+
+}  // namespace internal
+}  // namespace cppgc

test/unittests/heap/cppgc/garbage-collected_unittest.cc

@@ -66,7 +66,7 @@ TEST(GarbageCollectedTest, GarbageCollectedMixinTrait) {
   STATIC_ASSERT(IsGarbageCollectedMixinType<GCWithMergedMixins>::value);
 }
 
-TEST_F(GarbageCollectedTestWithHeap, GetObjectStartReturnsCorrentAddress) {
+TEST_F(GarbageCollectedTestWithHeap, GetObjectStartReturnsCurrentAddress) {
   GCed* gced = MakeGarbageCollected<GCed>(GetHeap());
   GCedWithMixin* gced_with_mixin =
       MakeGarbageCollected<GCedWithMixin>(GetHeap());

test/unittests/heap/cppgc/heap-page_unittest.cc

@@ -46,17 +46,16 @@ TEST_F(PageTest, SpaceIndexing) {
   RawHeap& heap = GetRawHeap();
   size_t space = 0u;
   for (const auto& ptr : heap) {
-    EXPECT_EQ(ptr.get(),
-              heap.Space(static_cast<cppgc::Heap::SpaceType>(space)));
+    EXPECT_EQ(ptr.get(), heap.Space(space));
     EXPECT_EQ(&heap, ptr.get()->raw_heap());
     EXPECT_EQ(space, ptr->index());
     ++space;
   }
-  EXPECT_EQ(space, static_cast<size_t>(cppgc::Heap::SpaceType::kUserDefined1));
+  EXPECT_GE(space, RawHeap::kNumberOfRegularSpaces);
 }
 
 TEST_F(PageTest, PredefinedSpaces) {
-  using SpaceType = RawHeap::SpaceType;
+  using SpaceType = RawHeap::RegularSpaceType;
   RawHeap& heap = GetRawHeap();
   {
     auto* gced = MakeGarbageCollected<GCed<1>>(GetHeap());
@@ -97,7 +96,7 @@ TEST_F(PageTest, PredefinedSpaces) {
 }
 
 TEST_F(PageTest, NormalPageIndexing) {
-  using SpaceType = RawHeap::SpaceType;
+  using SpaceType = RawHeap::RegularSpaceType;
   constexpr size_t kExpectedNumberOfPages = 10u;
   constexpr size_t kObjectSize = 8u;
   using Type = GCed<kObjectSize>;
@@ -125,7 +124,7 @@ TEST_F(PageTest, NormalPageIndexing) {
 }
 
 TEST_F(PageTest, LargePageIndexing) {
-  using SpaceType = RawHeap::SpaceType;
+  using SpaceType = RawHeap::RegularSpaceType;
   constexpr size_t kExpectedNumberOfPages = 10u;
   constexpr size_t kObjectSize = 2 * kLargeObjectSizeThreshold;
   using Type = GCed<kObjectSize>;
@@ -192,8 +191,8 @@ TEST_F(PageTest, HeapObjectHeaderOnLargePageIndexing) {
 TEST_F(PageTest, NormalPageCreationDestruction) {
   RawHeap& heap = GetRawHeap();
   const PageBackend* backend = Heap::From(GetHeap())->page_backend();
-  auto* space =
-      static_cast<NormalPageSpace*>(heap.Space(RawHeap::SpaceType::kNormal1));
+  auto* space = static_cast<NormalPageSpace*>(
+      heap.Space(RawHeap::RegularSpaceType::kNormal1));
   auto* page = NormalPage::Create(space);
   EXPECT_NE(space->end(), std::find(space->begin(), space->end(), page));
   EXPECT_TRUE(
@@ -213,8 +212,8 @@ TEST_F(PageTest, LargePageCreationDestruction) {
   constexpr size_t kObjectSize = 2 * kLargeObjectSizeThreshold;
   RawHeap& heap = GetRawHeap();
   const PageBackend* backend = Heap::From(GetHeap())->page_backend();
-  auto* space =
-      static_cast<LargePageSpace*>(heap.Space(RawHeap::SpaceType::kLarge));
+  auto* space = static_cast<LargePageSpace*>(
+      heap.Space(RawHeap::RegularSpaceType::kLarge));
   auto* page = LargePage::Create(space, kObjectSize);
   EXPECT_NE(space->end(), std::find(space->begin(), space->end(), page));
   EXPECT_NE(nullptr, backend->Lookup(page->PayloadStart()));
@@ -229,14 +228,14 @@ TEST_F(PageTest, LargePageCreationDestruction) {
 TEST_F(PageTest, UnsweptPageDestruction) {
   RawHeap& heap = GetRawHeap();
   {
-    auto* space =
-        static_cast<NormalPageSpace*>(heap.Space(RawHeap::SpaceType::kNormal1));
+    auto* space = static_cast<NormalPageSpace*>(
+        heap.Space(RawHeap::RegularSpaceType::kNormal1));
     auto* page = NormalPage::Create(space);
     EXPECT_DEATH_IF_SUPPORTED(NormalPage::Destroy(page), "");
   }
   {
-    auto* space =
-        static_cast<LargePageSpace*>(heap.Space(RawHeap::SpaceType::kLarge));
+    auto* space = static_cast<LargePageSpace*>(
+        heap.Space(RawHeap::RegularSpaceType::kLarge));
     auto* page = LargePage::Create(space, 2 * kLargeObjectSizeThreshold);
     EXPECT_DEATH_IF_SUPPORTED(LargePage::Destroy(page), "");
     // Detach page and really destroy page in the parent process so that sweeper