heap: Make sure no pages are left with promotion markings

Bug: v8:12612
Change-Id: I120da76c304496b9b89ce764e5f1bfa4778e130f
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3460413Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Commit-Queue: Omer Katz <omerkatz@chromium.org>
Cr-Commit-Position: refs/heads/main@{#79098}

src/heap/large-spaces.cc

@@ -445,6 +445,9 @@ void LargeObjectSpace::Verify(Isolate* isolate) {
       ExternalBackingStoreType t = static_cast<ExternalBackingStoreType>(i);
       external_backing_store_bytes[t] += chunk->ExternalBackingStoreBytes(t);
     }
+
+    CHECK(!chunk->IsFlagSet(Page::PAGE_NEW_OLD_PROMOTION));
+    CHECK(!chunk->IsFlagSet(Page::PAGE_NEW_NEW_PROMOTION));
   }
   for (int i = 0; i < kNumTypes; i++) {
     ExternalBackingStoreType t = static_cast<ExternalBackingStoreType>(i);

src/heap/mark-compact.cc

@@ -3917,6 +3917,7 @@ void MarkCompactCollector::EvacuatePagesInParallel() {
       if (marking_state->IsBlack(object)) {
         heap_->lo_space()->PromoteNewLargeObject(current);
         current->SetFlag(Page::PAGE_NEW_OLD_PROMOTION);
+        promoted_large_pages_.push_back(current);
         evacuation_items.emplace_back(ParallelWorkItem{}, current);
       }
     }
@@ -4109,6 +4110,12 @@ void MarkCompactCollector::Evacuate() {
     }
     new_space_evacuation_pages_.clear();
 
+    for (LargePage* p : promoted_large_pages_) {
+      DCHECK(p->IsFlagSet(Page::PAGE_NEW_OLD_PROMOTION));
+      p->ClearFlag(Page::PAGE_NEW_OLD_PROMOTION);
+    }
+    promoted_large_pages_.clear();
+
     for (Page* p : old_space_evacuation_pages_) {
       if (p->IsFlagSet(Page::COMPACTION_WAS_ABORTED)) {
         sweeper()->AddPage(p->owner_identity(), p, Sweeper::REGULAR);
@@ -5095,6 +5102,11 @@ void MinorMarkCompactCollector::CleanupPromotedPages() {
     non_atomic_marking_state()->ClearLiveness(p);
   }
   promoted_pages_.clear();
+
+  for (LargePage* p : promoted_large_pages_) {
+    p->ClearFlag(Page::PAGE_NEW_OLD_PROMOTION);
+  }
+  promoted_large_pages_.clear();
 }
 
 void MinorMarkCompactCollector::SweepArrayBufferExtensions() {
@@ -5985,6 +5997,7 @@ void MinorMarkCompactCollector::EvacuatePagesInParallel() {
     if (non_atomic_marking_state_.IsGrey(object)) {
       heap_->lo_space()->PromoteNewLargeObject(current);
       current->SetFlag(Page::PAGE_NEW_OLD_PROMOTION);
+      promoted_large_pages_.push_back(current);
       evacuation_items.emplace_back(ParallelWorkItem{}, current);
     }
   }

src/heap/mark-compact.h

@@ -26,6 +26,7 @@ namespace internal {
 class EvacuationJobTraits;
 class HeapObjectVisitor;
 class ItemParallelJob;
+class LargePage;
 class MigrationObserver;
 class ReadOnlySpace;
 class RecordMigratedSlotVisitor;
@@ -807,6 +808,7 @@ class MarkCompactCollector final : public MarkCompactCollectorBase {
       aborted_evacuation_candidates_due_to_oom_;
   std::vector<std::pair<Address, Page*>>
       aborted_evacuation_candidates_due_to_flags_;
+  std::vector<LargePage*> promoted_large_pages_;
 
   MarkingState marking_state_;
   NonAtomicMarkingState non_atomic_marking_state_;
@@ -915,6 +917,7 @@ class MinorMarkCompactCollector final : public MarkCompactCollectorBase {
   base::Semaphore page_parallel_job_semaphore_;
   std::vector<Page*> new_space_evacuation_pages_;
   std::vector<Page*> promoted_pages_;
+  std::vector<LargePage*> promoted_large_pages_;
 
   friend class YoungGenerationMarkingTask;
   friend class YoungGenerationMarkingJob;

src/heap/new-spaces.cc

@@ -702,6 +702,11 @@ void NewSpace::Verify(Isolate* isolate) {
     external_space_bytes[static_cast<ExternalBackingStoreType>(i)] = 0;
   }
 
+  CHECK(!Page::FromAllocationAreaAddress(current)->IsFlagSet(
+      Page::PAGE_NEW_OLD_PROMOTION));
+  CHECK(!Page::FromAllocationAreaAddress(current)->IsFlagSet(
+      Page::PAGE_NEW_NEW_PROMOTION));
+
   PtrComprCageBase cage_base(isolate);
   while (current != top()) {
     if (!Page::IsAlignedToPageSize(current)) {
@@ -740,6 +745,8 @@ void NewSpace::Verify(Isolate* isolate) {
     } else {
       // At end of page, switch to next page.
       Page* page = Page::FromAllocationAreaAddress(current)->next_page();
+      CHECK(!page->IsFlagSet(Page::PAGE_NEW_OLD_PROMOTION));
+      CHECK(!page->IsFlagSet(Page::PAGE_NEW_NEW_PROMOTION));
       current = page->area_start();
     }
   }

src/heap/paged-spaces.cc

@@ -772,6 +772,9 @@ void PagedSpace::Verify(Isolate* isolate, ObjectVisitor* visitor) {
       CHECK_EQ(external_page_bytes[t], page->ExternalBackingStoreBytes(t));
       external_space_bytes[t] += external_page_bytes[t];
     }
+
+    CHECK(!page->IsFlagSet(Page::PAGE_NEW_OLD_PROMOTION));
+    CHECK(!page->IsFlagSet(Page::PAGE_NEW_NEW_PROMOTION));
   }
   for (int i = 0; i < kNumTypes; i++) {
     if (i == ExternalBackingStoreType::kArrayBuffer) continue;

src/heap/read-only-spaces.cc

@@ -533,6 +533,9 @@ void ReadOnlySpace::Verify(Isolate* isolate) {
       CHECK(!object.IsExternalString());
       CHECK(!object.IsJSArrayBuffer());
     }
+
+    CHECK(!page->IsFlagSet(Page::PAGE_NEW_OLD_PROMOTION));
+    CHECK(!page->IsFlagSet(Page::PAGE_NEW_NEW_PROMOTION));
   }
   CHECK(allocation_pointer_found_in_space);