[heap] Use a different code object registry data structure to speed up sweeping.

Bug: v8:9093, chromium:959824
Change-Id: I4c22149044d82d909454ec563203a0a2690e2251
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1609797
Commit-Queue: Hannes Payer <hpayer@chromium.org>
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: Yang Guo <yangguo@chromium.org>
Cr-Commit-Position: refs/heads/master@{#61457}

src/heap/heap-inl.h

@@ -225,7 +225,8 @@ AllocationResult Heap::AllocateRaw(int size_in_bytes, AllocationType type,
       UnprotectAndRegisterMemoryChunk(object);
       ZapCodeObject(object->address(), size_in_bytes);
       if (!large_object) {
-        MemoryChunk::FromHeapObject(object)->RegisterCodeObject(object);
+        MemoryChunk::FromHeapObject(object)->RegisterNewlyAllocatedCodeObject(
+            object);
       }
     }
     OnAllocationEvent(object, size_in_bytes);

src/heap/heap.cc

@@ -5768,23 +5768,19 @@ void AllocationObserver::AllocationStep(int bytes_allocated,
   DCHECK_GE(bytes_to_next_step_, 0);
 }
 
-namespace {
-
-Map GcSafeMapOfCodeSpaceObject(HeapObject object) {
+Map Heap::GcSafeMapOfCodeSpaceObject(HeapObject object) {
   MapWord map_word = object->map_word();
   return map_word.IsForwardingAddress() ? map_word.ToForwardingAddress()->map()
                                         : map_word.ToMap();
 }
 
-Code GcSafeCastToCode(Heap* heap, HeapObject object, Address inner_pointer) {
+Code Heap::GcSafeCastToCode(HeapObject object, Address inner_pointer) {
   Code code = Code::unchecked_cast(object);
   DCHECK(!code.is_null());
-  DCHECK(heap->GcSafeCodeContains(code, inner_pointer));
+  DCHECK(GcSafeCodeContains(code, inner_pointer));
   return code;
 }
 
-}  // namespace
-
 bool Heap::GcSafeCodeContains(Code code, Address addr) {
   Map map = GcSafeMapOfCodeSpaceObject(code);
   DCHECK(map == ReadOnlyRoots(this).code_map());
@@ -5801,7 +5797,7 @@ Code Heap::GcSafeFindCodeForInnerPointer(Address inner_pointer) {
   // Check if the inner pointer points into a large object chunk.
   LargePage* large_page = code_lo_space()->FindPage(inner_pointer);
   if (large_page != nullptr) {
-    return GcSafeCastToCode(this, large_page->GetObject(), inner_pointer);
+    return GcSafeCastToCode(large_page->GetObject(), inner_pointer);
   }
 
   DCHECK(code_space()->Contains(inner_pointer));
@@ -5811,7 +5807,7 @@ Code Heap::GcSafeFindCodeForInnerPointer(Address inner_pointer) {
   Page* page = Page::FromAddress(inner_pointer);
 
   HeapObject object = page->GetCodeObjectFromInnerAddress(inner_pointer);
-  return GcSafeCastToCode(this, object, inner_pointer);
+  return GcSafeCastToCode(object, inner_pointer);
 }
 
 void Heap::WriteBarrierForCodeSlow(Code code) {

src/heap/heap.h

@@ -1277,6 +1277,14 @@ class Heap {
   // Mostly useful for debugging.
   bool GcSafeCodeContains(Code code, Address addr);
 
+  // Casts a heap object to a code object and checks if the inner_pointer is
+  // within the object.
+  Code GcSafeCastToCode(HeapObject object, Address inner_pointer);
+
+  // Returns the map of an object. Can be used during garbage collection, i.e.
+  // it supports a forwarded map. Fails if the map is not the code map.
+  Map GcSafeMapOfCodeSpaceObject(HeapObject object);
+
 // =============================================================================
 #ifdef VERIFY_HEAP
   // Verify the heap is in its normal state before or after a GC.

src/heap/mark-compact.cc

@@ -1279,7 +1279,7 @@ class EvacuateVisitorBase : public HeapObjectVisitor {
       MigrateObject(*target_object, object, size, target_space);
       if (target_space == CODE_SPACE)
         MemoryChunk::FromHeapObject(*target_object)
-            ->RegisterCodeObject(*target_object);
+            ->RegisterNewlyAllocatedCodeObject(*target_object);
       return true;
     }
     return false;

src/heap/spaces.cc

@@ -25,7 +25,6 @@
 #include "src/objects-inl.h"
 #include "src/objects/free-space-inl.h"
 #include "src/objects/js-array-buffer-inl.h"
-#include "src/objects/js-array-inl.h"
 #include "src/ostreams.h"
 #include "src/snapshot/snapshot.h"
 #include "src/v8.h"
@@ -613,42 +612,63 @@ void MemoryChunk::SetReadAndWritable() {
   }
 }
 
-void MemoryChunk::RegisterCodeObject(HeapObject code) {
+void MemoryChunk::RegisterNewlyAllocatedCodeObject(HeapObject code) {
   DCHECK(Contains(code->address()));
   DCHECK(MemoryChunk::FromHeapObject(code)->owner()->identity() == CODE_SPACE);
-  code_object_registry_->insert(code->address());
+  auto result = code_object_registry_newly_allocated_->insert(code->address());
+  USE(result);
+  DCHECK(result.second);
 }
 
-void MemoryChunk::RegisterCodeObjectInSwapRegistry(HeapObject code) {
+void MemoryChunk::RegisterAlreadyExistingCodeObject(HeapObject code) {
   DCHECK(MemoryChunk::FromHeapObject(code)->owner()->identity() == CODE_SPACE);
-  code_object_registry_swap_->insert(code->address());
+  code_object_registry_already_existing_->push_back(code->address());
 }
 
-void MemoryChunk::CreateSwapCodeObjectRegistry() {
-  DCHECK(!code_object_registry_swap_);
-  DCHECK(code_object_registry_);
-  code_object_registry_swap_ = new std::set<Address>();
+void MemoryChunk::ClearCodeObjectRegistries() {
+  DCHECK(code_object_registry_already_existing_);
+  DCHECK(code_object_registry_newly_allocated_);
+  code_object_registry_already_existing_->clear();
+  code_object_registry_newly_allocated_->clear();
 }
 
-void MemoryChunk::SwapCodeRegistries() {
-  DCHECK(code_object_registry_swap_);
-  DCHECK(code_object_registry_);
-  std::swap(code_object_registry_swap_, code_object_registry_);
-  delete code_object_registry_swap_;
-  code_object_registry_swap_ = nullptr;
+void MemoryChunk::FinalizeCodeObjectRegistries() {
+  DCHECK(code_object_registry_already_existing_);
+  DCHECK(code_object_registry_newly_allocated_);
+  code_object_registry_already_existing_->shrink_to_fit();
 }
 
 bool MemoryChunk::CodeObjectRegistryContains(HeapObject object) {
-  return code_object_registry_->find(object->address()) !=
-         code_object_registry_->end();
+  return (code_object_registry_newly_allocated_->find(object->address()) !=
+          code_object_registry_newly_allocated_->end()) ||
+         (std::binary_search(code_object_registry_already_existing_->begin(),
+                             code_object_registry_already_existing_->end(),
+                             object->address()));
 }
 
-HeapObject MemoryChunk::GetCodeObjectFromInnerAddress(Address address) {
+Code MemoryChunk::GetCodeObjectFromInnerAddress(Address address) {
   DCHECK(Contains(address));
-  DCHECK(!code_object_registry_->empty());
-  auto it = code_object_registry_->upper_bound(address);
+
+  // Let's first try the std::vector which holds object that already existed
+  // since the last GC cycle.
+  if (!code_object_registry_already_existing_->empty()) {
+    auto it = std::upper_bound(code_object_registry_already_existing_->begin(),
+                               code_object_registry_already_existing_->end(),
+                               address);
+    if (it != code_object_registry_already_existing_->begin()) {
+      Code code = Code::unchecked_cast(HeapObject::FromAddress(*(--it)));
+      if (heap_->GcSafeCodeContains(code, address)) {
+        return code;
+      }
+    }
+  }
+
+  // If the address was not found, it has to be in the newly allocated code
+  // objects registry.
+  DCHECK(!code_object_registry_newly_allocated_->empty());
+  auto it = code_object_registry_newly_allocated_->upper_bound(address);
   HeapObject obj = HeapObject::FromAddress(*(--it));
-  return obj;
+  return heap_->GcSafeCastToCode(obj, address);
 }
 
 namespace {
@@ -737,11 +757,12 @@ MemoryChunk* MemoryChunk::Initialize(Heap* heap, Address base, size_t size,
   chunk->reservation_ = std::move(reservation);
 
   if (owner->identity() == CODE_SPACE) {
-    chunk->code_object_registry_ = new std::set<Address>();
+    chunk->code_object_registry_newly_allocated_ = new std::set<Address>();
+    chunk->code_object_registry_already_existing_ = new std::vector<Address>();
   } else {
-    chunk->code_object_registry_ = nullptr;
+    chunk->code_object_registry_newly_allocated_ = nullptr;
+    chunk->code_object_registry_already_existing_ = nullptr;
   }
-  chunk->code_object_registry_swap_ = nullptr;
 
   return chunk;
 }
@@ -1363,8 +1384,10 @@ void MemoryChunk::ReleaseAllocatedMemory() {
   if (local_tracker_ != nullptr) ReleaseLocalTracker();
   if (young_generation_bitmap_ != nullptr) ReleaseYoungGenerationBitmap();
   if (marking_bitmap_ != nullptr) ReleaseMarkingBitmap();
-  if (code_object_registry_ != nullptr) delete code_object_registry_;
-  DCHECK(!code_object_registry_swap_);
+  if (code_object_registry_newly_allocated_ != nullptr)
+    delete code_object_registry_newly_allocated_;
+  if (code_object_registry_already_existing_ != nullptr)
+    delete code_object_registry_already_existing_;
 
   if (!IsLargePage()) {
     Page* page = static_cast<Page*>(this);
@@ -1621,7 +1644,6 @@ void PagedSpace::RefillFreeList() {
         added += RelinkFreeListCategories(p);
       }
       added += p->wasted_memory();
-      if (identity() == CODE_SPACE) p->SwapCodeRegistries();
       if (is_local() && (added > kCompactionMemoryWanted)) break;
     }
   }

src/heap/spaces.h

@@ -401,9 +401,10 @@ class MemoryChunk {
       // FreeListCategory categories_[kNumberOfCategories]
       + kSystemPointerSize  // LocalArrayBufferTracker* local_tracker_
       + kIntptrSize  // std::atomic<intptr_t> young_generation_live_byte_count_
-      + kSystemPointerSize   // Bitmap* young_generation_bitmap_
-      + kSystemPointerSize   // std:set code_object_registry_
-      + kSystemPointerSize;  // std:set code_object_registry_swap_
+      + kSystemPointerSize  // Bitmap* young_generation_bitmap_
+      +
+      kSystemPointerSize  // std::vector code_object_registry_already_existing_
+      + kSystemPointerSize;  // std::set code_object_registry_newly_allocated_
 
   // Page size in bytes.  This must be a multiple of the OS page size.
   static const int kPageSize = 1 << kPageSizeBits;
@@ -674,12 +675,12 @@ class MemoryChunk {
 
   base::ListNode<MemoryChunk>& list_node() { return list_node_; }
 
-  V8_EXPORT_PRIVATE void RegisterCodeObject(HeapObject code);
-  V8_EXPORT_PRIVATE void RegisterCodeObjectInSwapRegistry(HeapObject code);
-  V8_EXPORT_PRIVATE void CreateSwapCodeObjectRegistry();
-  V8_EXPORT_PRIVATE void SwapCodeRegistries();
+  V8_EXPORT_PRIVATE void RegisterNewlyAllocatedCodeObject(HeapObject code);
+  V8_EXPORT_PRIVATE void RegisterAlreadyExistingCodeObject(HeapObject code);
+  V8_EXPORT_PRIVATE void ClearCodeObjectRegistries();
+  V8_EXPORT_PRIVATE void FinalizeCodeObjectRegistries();
   V8_EXPORT_PRIVATE bool CodeObjectRegistryContains(HeapObject code);
-  V8_EXPORT_PRIVATE HeapObject GetCodeObjectFromInnerAddress(Address address);
+  V8_EXPORT_PRIVATE Code GetCodeObjectFromInnerAddress(Address address);
 
  protected:
   static MemoryChunk* Initialize(Heap* heap, Address base, size_t size,
@@ -787,8 +788,8 @@ class MemoryChunk {
   std::atomic<intptr_t> young_generation_live_byte_count_;
   Bitmap* young_generation_bitmap_;
 
-  std::set<Address>* code_object_registry_;
-  std::set<Address>* code_object_registry_swap_;
+  std::vector<Address>* code_object_registry_already_existing_;
+  std::set<Address>* code_object_registry_newly_allocated_;
 
  private:
   void InitializeReservedMemory() { reservation_.Reset(); }

src/heap/sweeper.cc

@@ -266,8 +266,6 @@ int Sweeper::RawSweep(Page* p, FreeListRebuildingMode free_list_mode,
 
   Address free_start = p->area_start();
 
-  if (is_code_page) p->CreateSwapCodeObjectRegistry();
-
   intptr_t live_bytes = 0;
   intptr_t freed_bytes = 0;
   intptr_t max_freed_bytes = 0;
@@ -277,10 +275,12 @@ int Sweeper::RawSweep(Page* p, FreeListRebuildingMode free_list_mode,
   // live bytes and keep track of wasted_memory_.
   p->ResetAllocationStatistics();
 
+  if (is_code_page) p->ClearCodeObjectRegistries();
+
   for (auto object_and_size :
        LiveObjectRange<kBlackObjects>(p, marking_state_->bitmap(p))) {
     HeapObject const object = object_and_size.first;
-    if (is_code_page) p->RegisterCodeObjectInSwapRegistry(object);
+    if (is_code_page) p->RegisterAlreadyExistingCodeObject(object);
     DCHECK(marking_state_->IsBlack(object));
     Address free_end = object->address();
     if (free_end != free_start) {
@@ -367,6 +367,7 @@ int Sweeper::RawSweep(Page* p, FreeListRebuildingMode free_list_mode,
     DCHECK_EQ(live_bytes, p->allocated_bytes());
   }
   p->set_concurrent_sweeping_state(Page::kSweepingDone);
+  if (is_code_page) p->FinalizeCodeObjectRegistries();
   if (free_list_mode == IGNORE_FREE_LIST) return 0;
   return static_cast<int>(FreeList::GuaranteedAllocatable(max_freed_bytes));
 }

src/snapshot/deserializer-allocator.cc

@@ -45,7 +45,7 @@ Address DeserializerAllocator::AllocateRaw(AllocationSpace space, int size) {
     DCHECK_LE(high_water_[space], reservation[chunk_index].end);
 #endif
     if (space == CODE_SPACE)
-      MemoryChunk::FromAddress(address)->RegisterCodeObject(
+      MemoryChunk::FromAddress(address)->RegisterNewlyAllocatedCodeObject(
           HeapObject::FromAddress(address));
     return address;
   }