[heap] Optimize MemoryChunk::FromAnyPointerAddress

Currently this function requires the caller to hold a mutex for the
large page chunk hashtable and performs a hashtable lookup.

This patch adds a header sentinel field in each MemoryChunk. The field
is then used to distinguish large object slots from ordinary slots.

Bug: chromium:915233
Change-Id: I9fbeeb4f07f49573d0a21f9a2cc934370e417d68
Reviewed-on: https://chromium-review.googlesource.com/c/1391752
Commit-Queue: Ulan Degenbaev <ulan@chromium.org>
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#58732}

src/heap/heap-inl.h

@@ -617,13 +617,6 @@ void Heap::IncrementExternalBackingStoreBytes(ExternalBackingStoreType type,
   // trigger garbage collections.
 }
 
-bool Heap::IsWithinLargeObject(Address address) {
-  if (new_lo_space()->FindPage(address) || lo_space()->FindPage(address) ||
-      code_lo_space()->FindPage(address))
-    return true;
-  return false;
-}
-
 void Heap::DecrementExternalBackingStoreBytes(ExternalBackingStoreType type,
                                               size_t amount) {
   base::CheckedDecrement(&backing_store_bytes_, amount);

src/heap/heap.h

@@ -377,7 +377,6 @@ class Heap {
 
   bool IsLargeObject(HeapObject object);
   bool IsLargeMemoryChunk(MemoryChunk* chunk);
-  inline bool IsWithinLargeObject(Address address);
 
   bool IsInYoungGeneration(HeapObject object);
 

src/heap/spaces-inl.h

@@ -158,8 +158,7 @@ bool NewSpace::ToSpaceContains(Object o) { return to_space_.Contains(o); }
 bool NewSpace::FromSpaceContains(Object o) { return from_space_.Contains(o); }
 
 bool PagedSpace::Contains(Address addr) {
-  if (heap()->IsWithinLargeObject(addr)) return false;
-  return MemoryChunk::FromAnyPointerAddress(heap(), addr)->owner() == this;
+  return MemoryChunk::FromAnyPointerAddress(addr)->owner() == this;
 }
 
 bool PagedSpace::Contains(Object o) {
@@ -200,12 +199,18 @@ bool PagedSpace::TryFreeLast(HeapObject object, int object_size) {
   return false;
 }
 
-MemoryChunk* MemoryChunk::FromAnyPointerAddress(Heap* heap, Address addr) {
-  MemoryChunk* chunk = heap->lo_space()->FindPage(addr);
-  if (chunk == nullptr) {
-    chunk = MemoryChunk::FromAddress(addr);
+bool MemoryChunk::HasHeaderSentinel(Address slot_addr) {
+  Address base = BaseAddress(slot_addr);
+  if (slot_addr < base + kHeaderSize) return false;
+  return HeapObject::FromAddress(base) ==
+         ObjectSlot(base + kHeaderSentinelOffset).Relaxed_Load();
+}
+
+MemoryChunk* MemoryChunk::FromAnyPointerAddress(Address addr) {
+  while (!HasHeaderSentinel(addr)) {
+    addr = BaseAddress(addr) - 1;
   }
-  return chunk;
+  return FromAddress(addr);
 }
 
 void MemoryChunk::IncrementExternalBackingStoreBytes(

src/heap/spaces.cc

@@ -617,10 +617,12 @@ MemoryChunk* MemoryChunk::Initialize(Heap* heap, Address base, size_t size,
                                      VirtualMemory reservation) {
   MemoryChunk* chunk = FromAddress(base);
 
-  DCHECK(base == chunk->address());
+  DCHECK_EQ(base, chunk->address());
 
   chunk->heap_ = heap;
   chunk->size_ = size;
+  chunk->header_sentinel_ = HeapObject::FromAddress(base).ptr();
+  DCHECK(HasHeaderSentinel(area_start));
   chunk->area_start_ = area_start;
   chunk->area_end_ = area_end;
   chunk->flags_ = Flags(NO_FLAGS);

src/heap/spaces.h

@@ -363,6 +363,8 @@ class MemoryChunk {
       kMarkBitmapOffset + kSystemPointerSize;
   static const intptr_t kHeapOffset =
       kReservationOffset + 3 * kSystemPointerSize;
+  static const intptr_t kHeaderSentinelOffset =
+      kHeapOffset + kSystemPointerSize;
 
   static const size_t kHeaderSize =
       kSizeOffset               // NOLINT
@@ -371,6 +373,7 @@ class MemoryChunk {
       + kSystemPointerSize      // Bitmap* marking_bitmap_
       + 3 * kSystemPointerSize  // VirtualMemory reservation_
       + kSystemPointerSize      // Heap* heap_
+      + kSystemPointerSize      // Address header_sentinel_
       + kSystemPointerSize      // Address area_start_
       + kSystemPointerSize      // Address area_end_
       + kSystemPointerSize      // Address owner_
@@ -405,19 +408,21 @@ class MemoryChunk {
   // TODO(6792,mstarzinger): Drop to 3 or lower once WebAssembly is off heap.
   static const int kMaxWriteUnprotectCounter = 4;
 
+  static Address BaseAddress(Address a) { return a & ~kAlignmentMask; }
+
   // Only works if the pointer is in the first kPageSize of the MemoryChunk.
   static MemoryChunk* FromAddress(Address a) {
-    return reinterpret_cast<MemoryChunk*>(a & ~kAlignmentMask);
+    return reinterpret_cast<MemoryChunk*>(BaseAddress(a));
   }
   // Only works if the object is in the first kPageSize of the MemoryChunk.
   static MemoryChunk* FromHeapObject(const HeapObject o) {
-    return reinterpret_cast<MemoryChunk*>(o.ptr() & ~kAlignmentMask);
+    return reinterpret_cast<MemoryChunk*>(BaseAddress(o.ptr()));
   }
 
   void SetOldGenerationPageFlags(bool is_marking);
   void SetYoungGenerationPageFlags(bool is_marking);
 
-  static inline MemoryChunk* FromAnyPointerAddress(Heap* heap, Address addr);
+  static inline MemoryChunk* FromAnyPointerAddress(Address addr);
 
   static inline void UpdateHighWaterMark(Address mark) {
     if (mark == kNullAddress) return;
@@ -642,6 +647,8 @@ class MemoryChunk {
 
   void set_owner(Space* space) { owner_ = space; }
 
+  static inline bool HasHeaderSentinel(Address slot_addr);
+
   // Emits a memory barrier. For TSAN builds the other thread needs to perform
   // MemoryChunk::synchronized_heap() to simulate the barrier.
   void InitializationMemoryFence();
@@ -686,6 +693,12 @@ class MemoryChunk {
 
   Heap* heap_;
 
+  // This is used to distinguish the memory chunk header from the interior of a
+  // large page. The memory chunk header stores here an impossible tagged
+  // pointer: the tagger pointer of the page start. A field in a large object is
+  // guaranteed to not contain such a pointer.
+  Address header_sentinel_;
+
   // Start and end of allocatable memory on this chunk.
   Address area_start_;
   Address area_end_;

src/heap/store-buffer.cc

@@ -160,14 +160,14 @@ void StoreBuffer::MoveEntriesToRememberedSet(int index) {
   DCHECK_GE(index, 0);
   DCHECK_LT(index, kStoreBuffers);
   Address last_inserted_addr = kNullAddress;
+  MemoryChunk* chunk = nullptr;
 
-  // We are taking the chunk map mutex here because the page lookup of addr
-  // below may require us to check if addr is part of a large page.
-  base::MutexGuard guard(heap_->lo_space()->chunk_map_mutex());
   for (Address* current = start_[index]; current < lazy_top_[index];
        current++) {
     Address addr = *current;
-    MemoryChunk* chunk = MemoryChunk::FromAnyPointerAddress(heap_, addr);
+    if (MemoryChunk::BaseAddress(addr) != chunk->address()) {
+      chunk = MemoryChunk::FromAnyPointerAddress(addr);
+    }
     if (IsDeletionAddress(addr)) {
       last_inserted_addr = kNullAddress;
       current++;