[heap] Implement page lookup for IPR

This CL implements MemoryAllocator::LookupChunkContainingAddress, which
will be used for conservative stack scanning. The method determines
whether an address that may be an inner pointer is contained in some
allocated (normal or large) page. To achieve this, the CL introduces a
page database in the memory allocator.

Bug: v8:12851
Change-Id: I8b719a5f1b6e6b374ccf0666c91c2341c5f9856a
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3784986Reviewed-by: Omer Katz <omerkatz@chromium.org>
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Commit-Queue: Nikolaos Papaspyrou <nikolaos@chromium.org>
Cr-Commit-Position: refs/heads/main@{#82004}

src/heap/conservative-stack-visitor.cc

@@ -26,7 +26,8 @@ bool ConservativeStackVisitor::CheckPage(Address address, MemoryChunk* page) {
 #ifdef V8_ENABLE_INNER_POINTER_RESOLUTION_OSB
   base_ptr = page->object_start_bitmap()->FindBasePtr(address);
 #elif V8_ENABLE_INNER_POINTER_RESOLUTION_MB
-  base_ptr = MarkCompactCollector::FindBasePtrForMarking(address);
+  base_ptr = isolate_->heap()->mark_compact_collector()->FindBasePtrForMarking(
+      address);
 #else
 #error "Some inner pointer resolution mechanism is needed"
 #endif  // V8_ENABLE_INNER_POINTER_RESOLUTION_(OSB|MB)

src/heap/mark-compact.cc

@@ -2169,11 +2169,16 @@ Address FindPreviousObjectForConservativeMarking(const Page* page,
 }  // namespace
 
 Address MarkCompactCollector::FindBasePtrForMarking(Address maybe_inner_ptr) {
-  const Page* page = Page::FromAddress(maybe_inner_ptr);
-  // TODO(v8:12851): We need a mechanism for checking that this is a valid page,
-  // otherwise return kNullAddress.
-  DCHECK_LT(maybe_inner_ptr, page->area_end());
-  if (maybe_inner_ptr < page->area_start()) return kNullAddress;
+  // Check if the pointer is contained by a normal or large page owned by this
+  // heap. Bail out if it is not.
+  const MemoryChunk* chunk =
+      heap()->memory_allocator()->LookupChunkContainingAddress(maybe_inner_ptr);
+  if (chunk == nullptr) return kNullAddress;
+  DCHECK(chunk->Contains(maybe_inner_ptr));
+  // If it is contained in a large page, we want to mark the only object on it.
+  if (chunk->IsLargePage()) return chunk->area_start();
+  // Otherwise, we have a pointer inside a normal page.
+  const Page* page = static_cast<const Page*>(chunk);
   Address base_ptr =
       FindPreviousObjectForConservativeMarking(page, maybe_inner_ptr);
   // If the markbit is set, then we have an object that does not need be marked.

src/heap/mark-compact.h

@@ -568,7 +568,7 @@ class MarkCompactCollector final : public CollectorBase {
   // `kNullAddress` if the parameter does not point to (the interior of) a valid
   // heap object, or if it points to (the interior of) some object that is
   // already marked as live (black or grey).
-  static Address FindBasePtrForMarking(Address maybe_inner_ptr);
+  Address FindBasePtrForMarking(Address maybe_inner_ptr);
 #endif  // V8_ENABLE_INNER_POINTER_RESOLUTION_MB
 
  private:

src/heap/memory-allocator.cc

@@ -525,6 +525,10 @@ void MemoryAllocator::PerformFreeMemory(MemoryChunk* chunk) {
 }
 
 void MemoryAllocator::Free(MemoryAllocator::FreeMode mode, MemoryChunk* chunk) {
+  if (chunk->IsLargePage())
+    RecordLargePageDestroyed(*static_cast<LargePage*>(chunk));
+  else
+    RecordNormalPageDestroyed(*static_cast<Page*>(chunk));
   switch (mode) {
     case FreeMode::kImmediately:
       PreFreeMemory(chunk);
@@ -579,6 +583,7 @@ Page* MemoryAllocator::AllocatePage(MemoryAllocator::AllocationMode alloc_mode,
 #endif  // DEBUG
 
   space->InitializePage(page);
+  RecordNormalPageCreated(*page);
   return page;
 }
 
@@ -617,6 +622,7 @@ LargePage* MemoryAllocator::AllocateLargePage(LargeObjectSpace* space,
   if (page->executable()) RegisterExecutableMemoryChunk(page);
 #endif  // DEBUG
 
+  RecordLargePageCreated(*page);
   return page;
 }
 
@@ -748,5 +754,56 @@ bool MemoryAllocator::SetPermissionsOnExecutableMemoryChunk(VirtualMemory* vm,
   return false;
 }
 
+const MemoryChunk* MemoryAllocator::LookupChunkContainingAddress(
+    Address addr) const {
+  base::MutexGuard guard(&pages_mutex_);
+  BasicMemoryChunk* chunk = BasicMemoryChunk::FromAddress(addr);
+  if (auto it = normal_pages_.find(static_cast<Page*>(chunk));
+      it != normal_pages_.end()) {
+    // The chunk is a normal page.
+    DCHECK_LE(chunk->address(), addr);
+    DCHECK_GT(chunk->area_end(), addr);
+    if (chunk->Contains(addr)) return *it;
+  } else if (auto it = large_pages_.upper_bound(static_cast<LargePage*>(chunk));
+             it != large_pages_.begin()) {
+    // The chunk could be inside a large page.
+    DCHECK_IMPLIES(it != large_pages_.end(), addr < (*it)->address());
+    auto* large_page = *std::next(it, -1);
+    DCHECK_NOT_NULL(large_page);
+    DCHECK_LE(large_page->address(), addr);
+    if (large_page->Contains(addr)) return large_page;
+  }
+  // Not found in any page.
+  return nullptr;
+}
+
+void MemoryAllocator::RecordNormalPageCreated(const Page& page) {
+  base::MutexGuard guard(&pages_mutex_);
+  auto result = normal_pages_.insert(&page);
+  USE(result);
+  DCHECK(result.second);
+}
+
+void MemoryAllocator::RecordNormalPageDestroyed(const Page& page) {
+  base::MutexGuard guard(&pages_mutex_);
+  auto size = normal_pages_.erase(&page);
+  USE(size);
+  DCHECK_EQ(1u, size);
+}
+
+void MemoryAllocator::RecordLargePageCreated(const LargePage& page) {
+  base::MutexGuard guard(&pages_mutex_);
+  auto result = large_pages_.insert(&page);
+  USE(result);
+  DCHECK(result.second);
+}
+
+void MemoryAllocator::RecordLargePageDestroyed(const LargePage& page) {
+  base::MutexGuard guard(&pages_mutex_);
+  auto size = large_pages_.erase(&page);
+  USE(size);
+  DCHECK_EQ(1u, size);
+}
+
 }  // namespace internal
 }  // namespace v8

src/heap/memory-allocator.h

@@ -263,6 +263,16 @@ class MemoryAllocator {
 
   Address HandleAllocationFailure(Executability executable);
 
+  // Return the normal or large page that contains this address, if it is owned
+  // by this heap, otherwise a nullptr.
+  const MemoryChunk* LookupChunkContainingAddress(Address addr) const;
+
+  // Insert and remove normal and large pages that are owned by this heap.
+  void RecordNormalPageCreated(const Page& page);
+  void RecordNormalPageDestroyed(const Page& page);
+  void RecordLargePageCreated(const LargePage& page);
+  void RecordLargePageDestroyed(const LargePage& page);
+
  private:
   // Used to store all data about MemoryChunk allocation, e.g. in
   // AllocateUninitializedChunk.
@@ -411,6 +421,12 @@ class MemoryAllocator {
   base::Mutex executable_memory_mutex_;
 #endif  // DEBUG
 
+  // Allocated normal and large pages are stored here, to be used during
+  // conservative stack scanning.
+  std::unordered_set<const Page*> normal_pages_;
+  std::set<const LargePage*> large_pages_;
+  mutable base::Mutex pages_mutex_;
+
   V8_EXPORT_PRIVATE static size_t commit_page_size_;
   V8_EXPORT_PRIVATE static size_t commit_page_size_bits_;
 

test/unittests/heap/marking-inner-pointer-resolution-unittest.cc

@@ -15,7 +15,7 @@ class InnerPointerResolutionTest : public TestWithIsolate {
  public:
   struct ObjectRequest {
     int size;  // The only required field.
-    enum { REGULAR, FREE } type = REGULAR;
+    enum { REGULAR, FREE, LARGE } type = REGULAR;
     enum { WHITE, GREY, BLACK, BLACK_AREA } marked = WHITE;
     // If index_in_cell >= 0, the object is placed at the lowest address s.t.
     // Bitmap::IndexInCell(AddressToMarkbitIndex(address)) == index_in_cell.
@@ -46,9 +46,9 @@ class InnerPointerResolutionTest : public TestWithIsolate {
   MemoryAllocator* allocator() { return heap()->memory_allocator(); }
   MarkCompactCollector* collector() { return heap()->mark_compact_collector(); }
 
-  // Create, free and lookup pages.
+  // Create, free and lookup pages, normal or large.
 
-  int CreatePage() {
+  int CreateNormalPage() {
     OldSpace* old_space = heap()->old_space();
     DCHECK_NE(nullptr, old_space);
     auto* page = allocator()->AllocatePage(
@@ -60,6 +60,18 @@ class InnerPointerResolutionTest : public TestWithIsolate {
     return page_id;
   }
 
+  int CreateLargePage(size_t size) {
+    OldLargeObjectSpace* lo_space = heap()->lo_space();
+    EXPECT_NE(nullptr, lo_space);
+    LargePage* page =
+        allocator()->AllocateLargePage(lo_space, size, NOT_EXECUTABLE);
+    EXPECT_NE(nullptr, page);
+    int page_id = next_page_id_++;
+    DCHECK_EQ(pages_.end(), pages_.find(page_id));
+    pages_[page_id] = page;
+    return page_id;
+  }
+
   void FreePage(int page_id) {
     DCHECK_LE(0, page_id);
     auto it = pages_.find(page_id);
@@ -68,19 +80,25 @@ class InnerPointerResolutionTest : public TestWithIsolate {
     pages_.erase(it);
   }
 
-  Page* LookupPage(int page_id) {
+  MemoryChunk* LookupPage(int page_id) {
     DCHECK_LE(0, page_id);
     auto it = pages_.find(page_id);
     DCHECK_NE(pages_.end(), it);
     return it->second;
   }
 
+  bool IsPageAlive(int page_id) {
+    DCHECK_LE(0, page_id);
+    return pages_.find(page_id) != pages_.end();
+  }
+
   // Creates a list of objects in a page and ensures that the page is iterable.
-  void CreateObjectsInPage(const std::vector<ObjectRequest>& objects) {
-    int page_id = CreatePage();
-    Page* page = LookupPage(page_id);
+  int CreateObjectsInPage(const std::vector<ObjectRequest>& objects) {
+    int page_id = CreateNormalPage();
+    MemoryChunk* page = LookupPage(page_id);
     Address ptr = page->area_start();
     for (auto object : objects) {
+      DCHECK_NE(ObjectRequest::LARGE, object.type);
       DCHECK_EQ(0, object.size % kTaggedSize);
 
       // Check if padding is needed.
@@ -136,6 +154,23 @@ class InnerPointerResolutionTest : public TestWithIsolate {
                        page_id,
                        ptr};
     CreateObject(last);
+    return page_id;
+  }
+
+  std::vector<int> CreateLargeObjects(
+      const std::vector<ObjectRequest>& objects) {
+    std::vector<int> result;
+    for (auto object : objects) {
+      DCHECK_EQ(ObjectRequest::LARGE, object.type);
+      int page_id = CreateLargePage(object.size);
+      MemoryChunk* page = LookupPage(page_id);
+      object.page_id = page_id;
+      object.address = page->area_start();
+      CHECK_EQ(object.address + object.size, page->area_end());
+      CreateObject(object);
+      result.push_back(page_id);
+    }
+    return result;
   }
 
   void CreateObject(const ObjectRequest& object) {
@@ -144,7 +179,8 @@ class InnerPointerResolutionTest : public TestWithIsolate {
     // "Allocate" (i.e., manually place) the object in the page, set the map
     // and the size.
     switch (object.type) {
-      case ObjectRequest::REGULAR: {
+      case ObjectRequest::REGULAR:
+      case ObjectRequest::LARGE: {
         DCHECK_LE(2 * kTaggedSize, object.size);
         ReadOnlyRoots roots(heap());
         HeapObject heap_object(HeapObject::FromAddress(object.address));
@@ -174,7 +210,7 @@ class InnerPointerResolutionTest : public TestWithIsolate {
             HeapObject::FromAddress(object.address));
         break;
       case ObjectRequest::BLACK_AREA: {
-        Page* page = LookupPage(object.page_id);
+        MemoryChunk* page = LookupPage(object.page_id);
         collector()->marking_state()->bitmap(page)->SetRange(
             page->AddressToMarkbitIndex(object.address),
             page->AddressToMarkbitIndex(object.address + object.size));
@@ -189,10 +225,13 @@ class InnerPointerResolutionTest : public TestWithIsolate {
     DCHECK_GT(object.size, offset);
     Address base_ptr =
         collector()->FindBasePtrForMarking(object.address + offset);
-    if (object.type == ObjectRequest::FREE ||
-        object.marked == ObjectRequest::BLACK_AREA ||
-        (object.marked == ObjectRequest::BLACK && offset < 2 * kTaggedSize) ||
-        (object.marked == ObjectRequest::GREY && offset < kTaggedSize))
+    bool should_return_null =
+        !IsPageAlive(object.page_id) || (object.type == ObjectRequest::FREE) ||
+        (object.type == ObjectRequest::REGULAR &&
+         (object.marked == ObjectRequest::BLACK_AREA ||
+          (object.marked == ObjectRequest::BLACK && offset < 2 * kTaggedSize) ||
+          (object.marked == ObjectRequest::GREY && offset < kTaggedSize)));
+    if (should_return_null)
       EXPECT_EQ(kNullAddress, base_ptr);
     else
       EXPECT_EQ(object.address, base_ptr);
@@ -216,10 +255,13 @@ class InnerPointerResolutionTest : public TestWithIsolate {
       DCHECK_LE(page->address(), outside_ptr);
       RunTestOutside(outside_ptr);
     }
+    RunTestOutside(kNullAddress);
+    RunTestOutside(static_cast<Address>(42));
+    RunTestOutside(static_cast<Address>(kZapValue));
   }
 
  private:
-  std::map<int, Page*> pages_;
+  std::map<int, MemoryChunk*> pages_;
   int next_page_id_ = 0;
   std::vector<ObjectRequest> objects_;
 };
@@ -462,7 +504,7 @@ TEST_F(InnerPointerResolutionTest, BlackAreaOfManyCells) {
   TestAll();
 }
 
-// Test with more pages.
+// Test with more pages, normal and large.
 
 TEST_F(InnerPointerResolutionTest, TwoPages) {
   if (FLAG_enable_third_party_heap) return;
@@ -486,6 +528,79 @@ TEST_F(InnerPointerResolutionTest, TwoPages) {
   TestAll();
 }
 
+TEST_F(InnerPointerResolutionTest, OneLargePage) {
+  if (FLAG_enable_third_party_heap) return;
+  CreateLargeObjects({
+      {1 * MB, ObjectRequest::LARGE, ObjectRequest::WHITE},
+  });
+  TestAll();
+}
+
+TEST_F(InnerPointerResolutionTest, SeveralLargePages) {
+  if (FLAG_enable_third_party_heap) return;
+  CreateLargeObjects({
+      {1 * MB, ObjectRequest::LARGE, ObjectRequest::WHITE},
+      {32 * MB, ObjectRequest::LARGE, ObjectRequest::BLACK},
+  });
+  TestAll();
+}
+
+TEST_F(InnerPointerResolutionTest, PagesOfBothKind) {
+  if (FLAG_enable_third_party_heap) return;
+  CreateObjectsInPage({
+      {64, ObjectRequest::REGULAR, ObjectRequest::WHITE},
+      {52, ObjectRequest::REGULAR, ObjectRequest::BLACK},
+      {512, ObjectRequest::REGULAR, ObjectRequest::WHITE},
+      {60, ObjectRequest::REGULAR, ObjectRequest::BLACK},
+      {42176, ObjectRequest::REGULAR, ObjectRequest::GREY},
+  });
+  CreateObjectsInPage({
+      {512, ObjectRequest::REGULAR, ObjectRequest::BLACK_AREA},
+      {64, ObjectRequest::REGULAR, ObjectRequest::WHITE},
+      {48, ObjectRequest::REGULAR, ObjectRequest::BLACK_AREA},
+      {52, ObjectRequest::REGULAR, ObjectRequest::BLACK},
+      {4, ObjectRequest::FREE, ObjectRequest::GREY},
+      {8, ObjectRequest::FREE, ObjectRequest::GREY},
+      {8, ObjectRequest::REGULAR, ObjectRequest::WHITE},
+      {60, ObjectRequest::REGULAR, ObjectRequest::BLACK},
+  });
+  CreateLargeObjects({
+      {1 * MB, ObjectRequest::LARGE, ObjectRequest::WHITE},
+      {32 * MB, ObjectRequest::LARGE, ObjectRequest::BLACK},
+  });
+  TestAll();
+}
+
+TEST_F(InnerPointerResolutionTest, FreePages) {
+  if (FLAG_enable_third_party_heap) return;
+  int some_normal_page = CreateObjectsInPage({
+      {64, ObjectRequest::REGULAR, ObjectRequest::WHITE},
+      {52, ObjectRequest::REGULAR, ObjectRequest::BLACK},
+      {512, ObjectRequest::REGULAR, ObjectRequest::WHITE},
+      {60, ObjectRequest::REGULAR, ObjectRequest::BLACK},
+      {42176, ObjectRequest::REGULAR, ObjectRequest::GREY},
+  });
+  CreateObjectsInPage({
+      {512, ObjectRequest::REGULAR, ObjectRequest::BLACK_AREA},
+      {64, ObjectRequest::REGULAR, ObjectRequest::WHITE},
+      {48, ObjectRequest::REGULAR, ObjectRequest::BLACK_AREA},
+      {52, ObjectRequest::REGULAR, ObjectRequest::BLACK},
+      {4, ObjectRequest::FREE, ObjectRequest::GREY},
+      {8, ObjectRequest::FREE, ObjectRequest::GREY},
+      {8, ObjectRequest::REGULAR, ObjectRequest::WHITE},
+      {60, ObjectRequest::REGULAR, ObjectRequest::BLACK},
+  });
+  auto large_pages = CreateLargeObjects({
+      {1 * MB, ObjectRequest::LARGE, ObjectRequest::WHITE},
+      {32 * MB, ObjectRequest::LARGE, ObjectRequest::BLACK},
+  });
+  TestAll();
+  FreePage(some_normal_page);
+  TestAll();
+  FreePage(large_pages[0]);
+  TestAll();
+}
+
 #endif  // V8_ENABLE_INNER_POINTER_RESOLUTION_MB
 
 }  // namespace internal