cppgc: young-gen: Take into account ages of adjacent cards

When setting a range for a newly allocated lab, consider adjacent cards.
If either is young, don't mark it as kMixed.

Bug: chromium:1029379
Change-Id: If7d1d920dd5769679de68800eae61f3a8dc1eb17
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3584116Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Commit-Queue: Anton Bikineev <bikineev@chromium.org>
Cr-Commit-Position: refs/heads/main@{#80175}

include/cppgc/internal/caged-heap-local-data.h

@@ -25,13 +25,17 @@ class HeapBase;
 // barrier. AgeTable contains entries that correspond to 512 bytes memory
 // regions (cards). Each entry in the table represents generation of the objects
 // that reside on the corresponding card (young, old or mixed).
-class AgeTable final {
+class V8_EXPORT AgeTable final {
   static constexpr size_t kRequiredSize = 1 * api_constants::kMB;
   static constexpr size_t kAllocationGranularity =
       api_constants::kAllocationGranularity;
 
  public:
+  // Represents age of the objects living on a single card.
   enum class Age : uint8_t { kOld, kYoung, kMixed };
+  // When setting age for a range, consider or ignore ages of the adjacent
+  // cards.
+  enum class AdjacentCardsPolicy : uint8_t { kConsider, kIgnore };
 
   static constexpr size_t kCardSizeInBytes =
       (api_constants::kCagedHeapReservationSize / kAllocationGranularity) /
@@ -40,6 +44,10 @@ class AgeTable final {
   void SetAge(uintptr_t cage_offset, Age age) {
     table_[card(cage_offset)] = age;
   }
+
+  void SetAgeForRange(uintptr_t cage_offset_begin, uintptr_t cage_offset_end,
+                      Age age, AdjacentCardsPolicy adjacent_cards_policy);
+
   V8_INLINE Age GetAge(uintptr_t cage_offset) const {
     return table_[card(cage_offset)];
   }

src/heap/cppgc/caged-heap-local-data.cc

@@ -27,6 +27,34 @@ static_assert(
     std::is_trivially_default_constructible<AgeTable>::value,
     "To support lazy committing, AgeTable must be trivially constructible");
 
+void AgeTable::SetAgeForRange(uintptr_t offset_begin, uintptr_t offset_end,
+                              Age age,
+                              AdjacentCardsPolicy adjacent_cards_policy) {
+  // First, mark inner cards.
+  const uintptr_t inner_card_offset_begin =
+      RoundUp(offset_begin, kCardSizeInBytes);
+  const uintptr_t outer_card_offset_end =
+      RoundDown(offset_end, kCardSizeInBytes);
+
+  for (auto inner_offset = inner_card_offset_begin;
+       inner_offset < outer_card_offset_end; inner_offset += kCardSizeInBytes)
+    SetAge(inner_offset, age);
+
+  // If outer cards are not card-aligned and are not of the same age, mark them
+  // as mixed.
+  const auto set_age_for_outer_card =
+      [this, age, adjacent_cards_policy](uintptr_t offset) {
+        if (IsAligned(offset, kCardSizeInBytes)) return;
+        if (adjacent_cards_policy == AdjacentCardsPolicy::kIgnore)
+          SetAge(offset, age);
+        else if (GetAge(offset) != age)
+          SetAge(offset, AgeTable::Age::kMixed);
+      };
+
+  set_age_for_outer_card(offset_begin);
+  set_age_for_outer_card(offset_end);
+}
+
 void AgeTable::Reset(PageAllocator* allocator) {
   // TODO(chromium:1029379): Consider MADV_DONTNEED instead of MADV_FREE on
   // POSIX platforms.

src/heap/cppgc/object-allocator.cc

@@ -30,31 +30,18 @@ void MarkRangeAsYoung(BasePage* page, Address begin, Address end) {
 #if defined(CPPGC_YOUNG_GENERATION)
   DCHECK_LT(begin, end);
 
-  static constexpr auto kEntrySize = AgeTable::kCardSizeInBytes;
-
-  const uintptr_t offset_begin = CagedHeap::OffsetFromAddress(begin);
-  const uintptr_t offset_end = CagedHeap::OffsetFromAddress(end);
-
-  const uintptr_t young_offset_begin = (begin == page->PayloadStart())
-                                           ? RoundDown(offset_begin, kEntrySize)
-                                           : RoundUp(offset_begin, kEntrySize);
-  const uintptr_t young_offset_end = (end == page->PayloadEnd())
-                                         ? RoundUp(offset_end, kEntrySize)
-                                         : RoundDown(offset_end, kEntrySize);
+  // Then, if the page is newly allocated, force the first and last cards to be
+  // marked as young.
+  const bool new_page =
+      (begin == page->PayloadStart()) && (end == page->PayloadEnd());
 
   auto& age_table = page->heap().caged_heap().local_data().age_table;
-  for (auto offset = young_offset_begin; offset < young_offset_end;
-       offset += AgeTable::kCardSizeInBytes) {
-    age_table.SetAge(offset, AgeTable::Age::kYoung);
-  }
-
-  // Set to kUnknown the first and the last regions of the newly allocated
-  // linear buffer.
-  if (begin != page->PayloadStart() && !IsAligned(offset_begin, kEntrySize))
-    age_table.SetAge(offset_begin, AgeTable::Age::kMixed);
-  if (end != page->PayloadEnd() && !IsAligned(offset_end, kEntrySize))
-    age_table.SetAge(offset_end, AgeTable::Age::kMixed);
-#endif
+  age_table.SetAgeForRange(CagedHeap::OffsetFromAddress(begin),
+                           CagedHeap::OffsetFromAddress(end),
+                           AgeTable::Age::kYoung,
+                           new_page ? AgeTable::AdjacentCardsPolicy::kIgnore
+                                    : AgeTable::AdjacentCardsPolicy::kConsider);
+#endif  // defined(CPPGC_YOUNG_GENERATION)
 }
 
 void AddToFreeList(NormalPageSpace& space, Address start, size_t size) {

test/unittests/BUILD.gn

@@ -142,6 +142,10 @@ v8_source_set("cppgc_unittests_sources") {
     "heap/cppgc/write-barrier-unittest.cc",
   ]
 
+  if (cppgc_enable_young_generation) {
+    sources += [ "heap/cppgc/age-table-unittest.cc" ]
+  }
+
   configs = [
     "../..:external_config",
     "../..:internal_config_base",

test/unittests/heap/cppgc/age-table-unittest.cc

+// Copyright 2022 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "include/cppgc/internal/caged-heap-local-data.h"
+#include "test/unittests/heap/cppgc/tests.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace cppgc {
+namespace internal {
+
+namespace {
+class AgeTableTest : public testing::TestWithHeap {
+ public:
+  using Age = AgeTable::Age;
+  using AdjacentCardsPolicy = AgeTable::AdjacentCardsPolicy;
+  static constexpr auto kCardSizeInBytes = AgeTable::kCardSizeInBytes;
+
+  AgeTableTest()
+      : disallow_gc_(GetHeapHandle()),
+        age_table_(Heap::From(GetHeap())->caged_heap().local_data().age_table) {
+  }
+
+  ~AgeTableTest() override {
+    age_table_.Reset(GetPlatform().GetPageAllocator());
+    // Collect all allocated pages.
+    for (auto* page : allocated_pages_) BasePage::Destroy(page);
+  }
+
+  NormalPage* AllocateNormalPage() {
+    RawHeap& heap = Heap::From(GetHeap())->raw_heap();
+    auto* space = static_cast<NormalPageSpace*>(
+        heap.Space(RawHeap::RegularSpaceType::kNormal1));
+    auto* page =
+        NormalPage::Create(*Heap::From(GetHeap())->page_backend(), *space);
+    allocated_pages_.push_back(page);
+    return page;
+  }
+
+  LargePage* AllocateLargePage() {
+    constexpr size_t kObjectSize = 2 * kLargeObjectSizeThreshold;
+    RawHeap& heap = Heap::From(GetHeap())->raw_heap();
+    auto* space = static_cast<LargePageSpace*>(
+        heap.Space(RawHeap::RegularSpaceType::kLarge));
+    auto* page = LargePage::Create(*Heap::From(GetHeap())->page_backend(),
+                                   *space, kObjectSize);
+    allocated_pages_.push_back(page);
+    return page;
+  }
+
+  void SetAgeForAddressRange(void* begin, void* end, Age age,
+                             AdjacentCardsPolicy adjacent_cards_policy) {
+    age_table_.SetAgeForRange(CagedHeap::OffsetFromAddress(begin),
+                              CagedHeap::OffsetFromAddress(end), age,
+                              adjacent_cards_policy);
+  }
+
+  Age GetAge(void* ptr) const {
+    return age_table_.GetAge(CagedHeap::OffsetFromAddress(ptr));
+  }
+
+  void SetAge(void* ptr, Age age) {
+    age_table_.SetAge(CagedHeap::OffsetFromAddress(ptr), age);
+  }
+
+  void AssertAgeForAddressRange(void* begin, void* end, Age age) {
+    const uintptr_t offset_begin = CagedHeap::OffsetFromAddress(begin);
+    const uintptr_t offset_end = CagedHeap::OffsetFromAddress(end);
+    for (auto offset = RoundDown(offset_begin, kCardSizeInBytes);
+         offset < RoundUp(offset_end, kCardSizeInBytes);
+         offset += kCardSizeInBytes)
+      EXPECT_EQ(age, age_table_.GetAge(offset));
+  }
+
+ private:
+  subtle::DisallowGarbageCollectionScope disallow_gc_;
+  std::vector<BasePage*> allocated_pages_;
+  AgeTable& age_table_;
+};
+
+}  // namespace
+
+TEST_F(AgeTableTest, SetAgeForNormalPage) {
+  auto* page = AllocateNormalPage();
+  // By default, everything is old.
+  AssertAgeForAddressRange(page->PayloadStart(), page->PayloadEnd(), Age::kOld);
+  // Set age for the entire page.
+  SetAgeForAddressRange(page->PayloadStart(), page->PayloadEnd(), Age::kYoung,
+                        AdjacentCardsPolicy::kIgnore);
+  // Check that all cards have been set as young.
+  AssertAgeForAddressRange(page->PayloadStart(), page->PayloadEnd(),
+                           Age::kYoung);
+}
+
+TEST_F(AgeTableTest, SetAgeForLargePage) {
+  auto* page = AllocateLargePage();
+  // By default, everything is old.
+  AssertAgeForAddressRange(page->PayloadStart(), page->PayloadEnd(), Age::kOld);
+  // Set age for the entire page.
+  SetAgeForAddressRange(page->PayloadStart(), page->PayloadEnd(), Age::kYoung,
+                        AdjacentCardsPolicy::kIgnore);
+  // Check that all cards have been set as young.
+  AssertAgeForAddressRange(page->PayloadStart(), page->PayloadEnd(),
+                           Age::kYoung);
+}
+
+TEST_F(AgeTableTest, SetAgeForSingleCardWithUnalignedAddresses) {
+  auto* page = AllocateNormalPage();
+  Address object_begin = reinterpret_cast<Address>(
+      RoundUp(reinterpret_cast<uintptr_t>(page->PayloadStart()),
+              kCardSizeInBytes) +
+      1);
+  Address object_end = object_begin + kCardSizeInBytes / 2;
+  EXPECT_EQ(Age::kOld, GetAge(object_begin));
+  // Try mark the card as young. This will mark the card as kMixed, since the
+  // card was previously marked as old.
+  SetAgeForAddressRange(object_begin, object_end, Age::kYoung,
+                        AdjacentCardsPolicy::kConsider);
+  EXPECT_EQ(Age::kMixed, GetAge(object_begin));
+  SetAge(object_begin, Age::kOld);
+  // Try mark as old, but ignore ages of outer cards.
+  SetAgeForAddressRange(object_begin, object_end, Age::kYoung,
+                        AdjacentCardsPolicy::kIgnore);
+  EXPECT_EQ(Age::kYoung, GetAge(object_begin));
+}
+
+TEST_F(AgeTableTest, SetAgeForSingleCardWithAlignedAddresses) {
+  auto* page = AllocateNormalPage();
+  Address object_begin = reinterpret_cast<Address>(RoundUp(
+      reinterpret_cast<uintptr_t>(page->PayloadStart()), kCardSizeInBytes));
+  Address object_end = object_begin + kCardSizeInBytes;
+  EXPECT_EQ(Age::kOld, GetAge(object_begin));
+  EXPECT_EQ(Age::kOld, GetAge(object_end));
+  // Try mark the card as young. This will mark the entire card as kYoung, since
+  // it's aligned.
+  SetAgeForAddressRange(object_begin, object_end, Age::kYoung,
+                        AdjacentCardsPolicy::kConsider);
+  EXPECT_EQ(Age::kYoung, GetAge(object_begin));
+  // The end card should not be touched.
+  EXPECT_EQ(Age::kOld, GetAge(object_end));
+}
+
+TEST_F(AgeTableTest, SetAgeForSingleCardWithAlignedBeginButUnalignedEnd) {
+  auto* page = AllocateNormalPage();
+  Address object_begin = reinterpret_cast<Address>(RoundUp(
+      reinterpret_cast<uintptr_t>(page->PayloadStart()), kCardSizeInBytes));
+  Address object_end = object_begin + kCardSizeInBytes + 1;
+  EXPECT_EQ(Age::kOld, GetAge(object_begin));
+  EXPECT_EQ(Age::kOld, GetAge(object_end));
+  // Try mark the card as young. This will mark the entire card as kYoung, since
+  // it's aligned.
+  SetAgeForAddressRange(object_begin, object_end, Age::kYoung,
+                        AdjacentCardsPolicy::kConsider);
+  EXPECT_EQ(Age::kYoung, GetAge(object_begin));
+  // The end card should be marked as mixed.
+  EXPECT_EQ(Age::kMixed, GetAge(object_end));
+}
+
+TEST_F(AgeTableTest, SetAgeForMultipleCardsWithUnalignedAddresses) {
+  static constexpr size_t kNumberOfCards = 4;
+  auto* page = AllocateNormalPage();
+  Address object_begin = reinterpret_cast<Address>(
+      RoundUp(reinterpret_cast<uintptr_t>(page->PayloadStart()),
+              kCardSizeInBytes) +
+      kCardSizeInBytes / 2);
+  Address object_end = object_begin + kNumberOfCards * kCardSizeInBytes;
+  AssertAgeForAddressRange(object_begin, object_end, Age::kOld);
+  // Try mark the cards as young. The inner 2 cards must be marked as young, the
+  // outer cards will be marked as mixed.
+  SetAgeForAddressRange(object_begin, object_end, Age::kYoung,
+                        AdjacentCardsPolicy::kConsider);
+  EXPECT_EQ(Age::kMixed, GetAge(object_begin));
+  EXPECT_EQ(Age::kYoung, GetAge(object_begin + kCardSizeInBytes));
+  EXPECT_EQ(Age::kYoung, GetAge(object_begin + 2 * kCardSizeInBytes));
+  EXPECT_EQ(Age::kMixed, GetAge(object_end));
+}
+
+TEST_F(AgeTableTest, SetAgeForMultipleCardsConsiderAdjacentCards) {
+  static constexpr size_t kNumberOfCards = 4;
+  auto* page = AllocateNormalPage();
+  Address object_begin = reinterpret_cast<Address>(
+      RoundUp(reinterpret_cast<uintptr_t>(page->PayloadStart()),
+              kCardSizeInBytes) +
+      kCardSizeInBytes / 2);
+  Address object_end = object_begin + kNumberOfCards * kCardSizeInBytes;
+  // Mark the first and the last card as young.
+  SetAge(object_begin, Age::kYoung);
+  SetAge(object_end, Age::kYoung);
+  // Mark all the cards as young. The inner 2 cards must be marked as young, the
+  // outer cards will also be marked as young.
+  SetAgeForAddressRange(object_begin, object_end, Age::kYoung,
+                        AdjacentCardsPolicy::kConsider);
+  EXPECT_EQ(Age::kYoung, GetAge(object_begin));
+  EXPECT_EQ(Age::kYoung, GetAge(object_begin + kCardSizeInBytes));
+  EXPECT_EQ(Age::kYoung, GetAge(object_begin + 2 * kCardSizeInBytes));
+  EXPECT_EQ(Age::kYoung, GetAge(object_end));
+}
+
+}  // namespace internal
+}  // namespace cppgc