[heap] Add a page-based code object registry.

The registry right now has no users. In a follow-up CL I will
remove the skip list for code pages and make users of the skip
list use the registry.

Bug: v8:9093
Change-Id: I23a2b9e0d4158e2ffa89626e71f58d3bb5a41201
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1593074Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Commit-Queue: Hannes Payer <hpayer@chromium.org>
Cr-Commit-Position: refs/heads/master@{#61184}

src/heap/heap-inl.h

@@ -224,6 +224,9 @@ AllocationResult Heap::AllocateRaw(int size_in_bytes, AllocationType type,
       // already.
       UnprotectAndRegisterMemoryChunk(object);
       ZapCodeObject(object->address(), size_in_bytes);
+      if (!large_object) {
+        MemoryChunk::FromHeapObject(object)->RegisterCodeObject(object);
+      }
     }
     OnAllocationEvent(object, size_in_bytes);
   }

src/heap/spaces.cc

@@ -602,6 +602,35 @@ void MemoryChunk::SetReadAndWritable() {
   }
 }
 
+void MemoryChunk::RegisterCodeObject(HeapObject code) {
+  DCHECK(MemoryChunk::FromHeapObject(code)->owner()->identity() == CODE_SPACE);
+  code_object_registry_->insert(code->address());
+}
+
+void MemoryChunk::RegisterCodeObjectInSwapRegistry(HeapObject code) {
+  DCHECK(MemoryChunk::FromHeapObject(code)->owner()->identity() == CODE_SPACE);
+  code_object_registry_swap_->insert(code->address());
+}
+
+void MemoryChunk::CreateSwapCodeObjectRegistry() {
+  DCHECK(!code_object_registry_swap_);
+  DCHECK(code_object_registry_);
+  code_object_registry_swap_ = new std::set<Address>();
+}
+
+void MemoryChunk::SwapCodeRegistries() {
+  DCHECK(code_object_registry_swap_);
+  DCHECK(code_object_registry_);
+  std::swap(code_object_registry_swap_, code_object_registry_);
+  delete code_object_registry_swap_;
+  code_object_registry_swap_ = nullptr;
+}
+
+bool MemoryChunk::CodeObjectRegistryContains(HeapObject object) {
+  return code_object_registry_->find(object->address()) !=
+         code_object_registry_->end();
+}
+
 namespace {
 
 PageAllocator::Permission DefaultWritableCodePermissions() {
@@ -688,6 +717,13 @@ MemoryChunk* MemoryChunk::Initialize(Heap* heap, Address base, size_t size,
 
   chunk->reservation_ = std::move(reservation);
 
+  if (owner->identity() == CODE_SPACE) {
+    chunk->code_object_registry_ = new std::set<Address>();
+  } else {
+    chunk->code_object_registry_ = nullptr;
+  }
+  chunk->code_object_registry_swap_ = nullptr;
+
   return chunk;
 }
 
@@ -1309,6 +1345,8 @@ void MemoryChunk::ReleaseAllocatedMemory() {
   if (local_tracker_ != nullptr) ReleaseLocalTracker();
   if (young_generation_bitmap_ != nullptr) ReleaseYoungGenerationBitmap();
   if (marking_bitmap_ != nullptr) ReleaseMarkingBitmap();
+  if (code_object_registry_ != nullptr) delete code_object_registry_;
+  DCHECK(!code_object_registry_swap_);
 
   if (!IsLargePage()) {
     Page* page = static_cast<Page*>(this);
@@ -1565,6 +1603,7 @@ void PagedSpace::RefillFreeList() {
         added += RelinkFreeListCategories(p);
       }
       added += p->wasted_memory();
+      if (identity() == CODE_SPACE) p->SwapCodeRegistries();
       if (is_local() && (added > kCompactionMemoryWanted)) break;
     }
   }

src/heap/spaces.h

@@ -399,7 +399,9 @@ class MemoryChunk {
       // FreeListCategory categories_[kNumberOfCategories]
       + kSystemPointerSize  // LocalArrayBufferTracker* local_tracker_
       + kIntptrSize  // std::atomic<intptr_t> young_generation_live_byte_count_
-      + kSystemPointerSize;  // Bitmap* young_generation_bitmap_
+      + kSystemPointerSize   // Bitmap* young_generation_bitmap_
+      + kSystemPointerSize   // std:set code_object_registry_
+      + kSystemPointerSize;  // std:set code_object_registry_swap_
 
   // Page size in bytes.  This must be a multiple of the OS page size.
   static const int kPageSize = 1 << kPageSizeBits;
@@ -671,6 +673,12 @@ class MemoryChunk {
 
   base::ListNode<MemoryChunk>& list_node() { return list_node_; }
 
+  V8_EXPORT_PRIVATE void RegisterCodeObject(HeapObject code);
+  V8_EXPORT_PRIVATE void RegisterCodeObjectInSwapRegistry(HeapObject code);
+  V8_EXPORT_PRIVATE void CreateSwapCodeObjectRegistry();
+  V8_EXPORT_PRIVATE void SwapCodeRegistries();
+  V8_EXPORT_PRIVATE bool CodeObjectRegistryContains(HeapObject code);
+
  protected:
   static MemoryChunk* Initialize(Heap* heap, Address base, size_t size,
                                  Address area_start, Address area_end,
@@ -779,6 +787,9 @@ class MemoryChunk {
   std::atomic<intptr_t> young_generation_live_byte_count_;
   Bitmap* young_generation_bitmap_;
 
+  std::set<Address>* code_object_registry_;
+  std::set<Address>* code_object_registry_swap_;
+
  private:
   void InitializeReservedMemory() { reservation_.Reset(); }
 

src/heap/sweeper.cc

@@ -249,6 +249,8 @@ int Sweeper::RawSweep(Page* p, FreeListRebuildingMode free_list_mode,
          space->identity() == CODE_SPACE || space->identity() == MAP_SPACE);
   DCHECK(!p->IsEvacuationCandidate() && !p->SweepingDone());
 
+  bool is_code_page = space->identity() == CODE_SPACE;
+
   // TODO(ulan): we don't have to clear type old-to-old slots in code space
   // because the concurrent marker doesn't mark code objects. This requires
   // the write barrier for code objects to check the color of the code object.
@@ -274,6 +276,8 @@ int Sweeper::RawSweep(Page* p, FreeListRebuildingMode free_list_mode,
     skip_list->Clear();
   }
 
+  if (is_code_page) p->CreateSwapCodeObjectRegistry();
+
   intptr_t live_bytes = 0;
   intptr_t freed_bytes = 0;
   intptr_t max_freed_bytes = 0;
@@ -287,6 +291,7 @@ int Sweeper::RawSweep(Page* p, FreeListRebuildingMode free_list_mode,
   for (auto object_and_size :
        LiveObjectRange<kBlackObjects>(p, marking_state_->bitmap(p))) {
     HeapObject const object = object_and_size.first;
+    if (is_code_page) p->RegisterCodeObjectInSwapRegistry(object);
     DCHECK(marking_state_->IsBlack(object));
     Address free_end = object->address();
     if (free_end != free_start) {

test/cctest/heap/test-heap.cc

@@ -6619,6 +6619,37 @@ HEAP_TEST(MemoryReducerActivationForSmallHeaps) {
   CHECK_EQ(heap->memory_reducer()->state_.action, MemoryReducer::Action::kWait);
 }
 
+TEST(CodeObjectRegistry) {
+  // We turn off compaction to ensure that code is not moving.
+  FLAG_never_compact = true;
+
+  Isolate* isolate = CcTest::i_isolate();
+  Heap* heap = isolate->heap();
+
+  Handle<Code> code1;
+  HandleScope outer_scope(heap->isolate());
+  Address code2_address;
+  {
+    code1 = DummyOptimizedCode(isolate);
+    Handle<Code> code2 = DummyOptimizedCode(isolate);
+    code2_address = code2->address();
+    // If this check breaks, change the allocation to ensure that both code
+    // objects are on the same page.
+    CHECK_EQ(MemoryChunk::FromHeapObject(*code1),
+             MemoryChunk::FromHeapObject(*code2));
+    CHECK(MemoryChunk::FromHeapObject(*code1)->CodeObjectRegistryContains(
+        *code1));
+    CHECK(MemoryChunk::FromHeapObject(*code2)->CodeObjectRegistryContains(
+        *code2));
+  }
+  CcTest::CollectAllAvailableGarbage();
+  CHECK(
+      MemoryChunk::FromHeapObject(*code1)->CodeObjectRegistryContains(*code1));
+  CHECK(
+      MemoryChunk::FromAddress(code2_address)
+          ->CodeObjectRegistryContains(HeapObject::FromAddress(code2_address)));
+}
+
 }  // namespace heap
 }  // namespace internal
 }  // namespace v8