[ext-code-space] Introduce MapWord::ToForwardingAddress(...)

... capable of computing the forwarding pointer for objects allocated
outside of the main pointer compression cage.

Drive-by: hoist computation of pointer compression cage base out of
certain loops in GC code.

Bug: v8:11880
Change-Id: I23efdffd1a237d9eedd0e2975e8e40811417ef31
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3204968Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Commit-Queue: Igor Sheludko <ishell@chromium.org>
Cr-Commit-Position: refs/heads/main@{#77244}

src/heap/heap.cc

@@ -978,10 +978,11 @@ size_t Heap::UsedGlobalHandlesSize() {
 
 void Heap::MergeAllocationSitePretenuringFeedback(
     const PretenuringFeedbackMap& local_pretenuring_feedback) {
+  PtrComprCageBase cage_base(isolate());
   AllocationSite site;
   for (auto& site_and_count : local_pretenuring_feedback) {
     site = site_and_count.first;
-    MapWord map_word = site_and_count.first.map_word(kRelaxedLoad);
+    MapWord map_word = site.map_word(cage_base, kRelaxedLoad);
     if (map_word.IsForwardingAddress()) {
       site = AllocationSite::cast(map_word.ToForwardingAddress());
     }
@@ -2731,8 +2732,9 @@ void Heap::UpdateExternalString(String string, size_t old_payload,
 
 String Heap::UpdateYoungReferenceInExternalStringTableEntry(Heap* heap,
                                                             FullObjectSlot p) {
+  PtrComprCageBase cage_base(heap->isolate());
   HeapObject obj = HeapObject::cast(*p);
-  MapWord first_word = obj.map_word(kRelaxedLoad);
+  MapWord first_word = obj.map_word(cage_base, kRelaxedLoad);
 
   String new_string;
 
@@ -2740,9 +2742,9 @@ String Heap::UpdateYoungReferenceInExternalStringTableEntry(Heap* heap,
     if (!first_word.IsForwardingAddress()) {
       // Unreachable external string can be finalized.
       String string = String::cast(obj);
-      if (!string.IsExternalString()) {
+      if (!string.IsExternalString(cage_base)) {
         // Original external string has been internalized.
-        DCHECK(string.IsThinString());
+        DCHECK(string.IsThinString(cage_base));
         return String();
       }
       heap->FinalizeExternalString(string);
@@ -2754,10 +2756,10 @@ String Heap::UpdateYoungReferenceInExternalStringTableEntry(Heap* heap,
   }
 
   // String is still reachable.
-  if (new_string.IsThinString()) {
+  if (new_string.IsThinString(cage_base)) {
     // Filtering Thin strings out of the external string table.
     return String();
-  } else if (new_string.IsExternalString()) {
+  } else if (new_string.IsExternalString(cage_base)) {
     MemoryChunk::MoveExternalBackingStoreBytes(
         ExternalBackingStoreType::kExternalString,
         Page::FromAddress((*p).ptr()), Page::FromHeapObject(new_string),
@@ -2766,7 +2768,7 @@ String Heap::UpdateYoungReferenceInExternalStringTableEntry(Heap* heap,
   }
 
   // Internalization can replace external strings with non-external strings.
-  return new_string.IsExternalString() ? new_string : String();
+  return new_string.IsExternalString(cage_base) ? new_string : String();
 }
 
 void Heap::ExternalStringTable::VerifyYoung() {
@@ -6995,9 +6997,17 @@ void Heap::CreateObjectStats() {
 }
 
 Map Heap::GcSafeMapOfCodeSpaceObject(HeapObject object) {
-  MapWord map_word = object.map_word(kRelaxedLoad);
-  return map_word.IsForwardingAddress() ? map_word.ToForwardingAddress().map()
-                                        : map_word.ToMap();
+  PtrComprCageBase cage_base(isolate());
+  MapWord map_word = object.map_word(cage_base, kRelaxedLoad);
+  if (map_word.IsForwardingAddress()) {
+#if V8_EXTERNAL_CODE_SPACE
+    PtrComprCageBase code_cage_base(isolate()->code_cage_base());
+#else
+    PtrComprCageBase code_cage_base = cage_base;
+#endif
+    return map_word.ToForwardingAddress(code_cage_base).map(cage_base);
+  }
+  return map_word.ToMap();
 }
 
 Code Heap::GcSafeCastToCode(HeapObject object, Address inner_pointer) {

src/heap/large-spaces.cc

@@ -354,6 +354,7 @@ void LargeObjectSpace::Verify(Isolate* isolate) {
     external_backing_store_bytes[static_cast<ExternalBackingStoreType>(i)] = 0;
   }
 
+  PtrComprCageBase cage_base(isolate);
   for (LargePage* chunk = first_page(); chunk != nullptr;
        chunk = chunk->next_page()) {
     // Each chunk contains an object that starts at the large object page's
@@ -364,23 +365,26 @@ void LargeObjectSpace::Verify(Isolate* isolate) {
 
     // The first word should be a map, and we expect all map pointers to be
     // in map space or read-only space.
-    Map map = object.map();
-    CHECK(map.IsMap());
+    Map map = object.map(cage_base);
+    CHECK(map.IsMap(cage_base));
     CHECK(ReadOnlyHeap::Contains(map) || heap()->map_space()->Contains(map));
 
     // We have only the following types in the large object space:
-    if (!(object.IsAbstractCode() || object.IsSeqString() ||
-          object.IsExternalString() || object.IsThinString() ||
-          object.IsFixedArray() || object.IsFixedDoubleArray() ||
-          object.IsWeakFixedArray() || object.IsWeakArrayList() ||
-          object.IsPropertyArray() || object.IsByteArray() ||
-          object.IsFeedbackVector() || object.IsBigInt() ||
-          object.IsFreeSpace() || object.IsFeedbackMetadata() ||
-          object.IsContext() || object.IsUncompiledDataWithoutPreparseData() ||
-          object.IsPreparseData()) &&
+    if (!(object.IsAbstractCode(cage_base) || object.IsSeqString(cage_base) ||
+          object.IsExternalString(cage_base) ||
+          object.IsThinString(cage_base) || object.IsFixedArray(cage_base) ||
+          object.IsFixedDoubleArray(cage_base) ||
+          object.IsWeakFixedArray(cage_base) ||
+          object.IsWeakArrayList(cage_base) ||
+          object.IsPropertyArray(cage_base) || object.IsByteArray(cage_base) ||
+          object.IsFeedbackVector(cage_base) || object.IsBigInt(cage_base) ||
+          object.IsFreeSpace(cage_base) ||
+          object.IsFeedbackMetadata(cage_base) || object.IsContext(cage_base) ||
+          object.IsUncompiledDataWithoutPreparseData(cage_base) ||
+          object.IsPreparseData(cage_base)) &&
         !FLAG_young_generation_large_objects) {
       FATAL("Found invalid Object (instance_type=%i) in large object space.",
-            object.map().instance_type());
+            object.map(cage_base).instance_type());
     }
 
     // The object itself should look OK.
@@ -391,27 +395,27 @@ void LargeObjectSpace::Verify(Isolate* isolate) {
     }
 
     // Byte arrays and strings don't have interior pointers.
-    if (object.IsAbstractCode()) {
+    if (object.IsAbstractCode(cage_base)) {
       VerifyPointersVisitor code_visitor(heap());
       object.IterateBody(map, object.Size(), &code_visitor);
-    } else if (object.IsFixedArray()) {
+    } else if (object.IsFixedArray(cage_base)) {
       FixedArray array = FixedArray::cast(object);
       for (int j = 0; j < array.length(); j++) {
         Object element = array.get(j);
         if (element.IsHeapObject()) {
           HeapObject element_object = HeapObject::cast(element);
           CHECK(IsValidHeapObject(heap(), element_object));
-          CHECK(element_object.map().IsMap());
+          CHECK(element_object.map(cage_base).IsMap(cage_base));
         }
       }
-    } else if (object.IsPropertyArray()) {
+    } else if (object.IsPropertyArray(cage_base)) {
       PropertyArray array = PropertyArray::cast(object);
       for (int j = 0; j < array.length(); j++) {
         Object property = array.get(j);
         if (property.IsHeapObject()) {
           HeapObject property_object = HeapObject::cast(property);
           CHECK(heap()->Contains(property_object));
-          CHECK(property_object.map().IsMap());
+          CHECK(property_object.map(cage_base).IsMap(cage_base));
         }
       }
     }

src/heap/mark-compact-inl.h

@@ -163,6 +163,7 @@ operator++(int) {
 
 template <LiveObjectIterationMode mode>
 void LiveObjectRange<mode>::iterator::AdvanceToNextValidObject() {
+  PtrComprCageBase cage_base(chunk_->heap()->isolate());
   while (!it_.Done()) {
     HeapObject object;
     int size = 0;
@@ -198,10 +199,10 @@ void LiveObjectRange<mode>::iterator::AdvanceToNextValidObject() {
         // make sure that we skip all set bits in the black area until the
         // object ends.
         HeapObject black_object = HeapObject::FromAddress(addr);
-        Object map_object = black_object.map(kAcquireLoad);
-        CHECK(map_object.IsMap());
+        Object map_object = black_object.map(cage_base, kAcquireLoad);
+        CHECK(map_object.IsMap(cage_base));
         map = Map::cast(map_object);
-        DCHECK(map.IsMap());
+        DCHECK(map.IsMap(cage_base));
         size = black_object.SizeFromMap(map);
         CHECK_LE(addr + size, chunk_->area_end());
         Address end = addr + size - kTaggedSize;
@@ -230,10 +231,10 @@ void LiveObjectRange<mode>::iterator::AdvanceToNextValidObject() {
         }
       } else if ((mode == kGreyObjects || mode == kAllLiveObjects)) {
         object = HeapObject::FromAddress(addr);
-        Object map_object = object.map(kAcquireLoad);
-        CHECK(map_object.IsMap());
+        Object map_object = object.map(cage_base, kAcquireLoad);
+        CHECK(map_object.IsMap(cage_base));
         map = Map::cast(map_object);
-        DCHECK(map.IsMap());
+        DCHECK(map.IsMap(cage_base));
         size = object.SizeFromMap(map);
         CHECK_LE(addr + size, chunk_->area_end());
       }

src/heap/mark-compact.cc

@@ -2879,8 +2879,10 @@ static inline SlotCallbackResult UpdateSlot(PtrComprCageBase cage_base,
                    MarkCompactCollector::IsOnEvacuationCandidate(heap_obj) ||
                        Page::FromHeapObject(heap_obj)->IsFlagSet(
                            Page::COMPACTION_WAS_ABORTED));
-    typename TSlot::TObject target =
-        MakeSlotValue<TSlot, reference_type>(map_word.ToForwardingAddress());
+    PtrComprCageBase host_cage_base =
+        V8_EXTERNAL_CODE_SPACE_BOOL ? GetPtrComprCageBase(heap_obj) : cage_base;
+    typename TSlot::TObject target = MakeSlotValue<TSlot, reference_type>(
+        map_word.ToForwardingAddress(host_cage_base));
     if (access_mode == AccessMode::NON_ATOMIC) {
       slot.store(target);
     } else {
@@ -4019,7 +4021,11 @@ class RememberedSetUpdatingItem : public UpdatingItem {
           (chunk_->slot_set<OLD_TO_CODE, AccessMode::NON_ATOMIC>() !=
            nullptr)) {
         PtrComprCageBase cage_base = heap_->isolate();
-        PtrComprCageBase code_cage_base = heap_->isolate();
+#if V8_EXTERNAL_CODE_SPACE
+        PtrComprCageBase code_cage_base(heap_->isolate()->code_cage_base());
+#else
+        PtrComprCageBase code_cage_base = cage_base;
+#endif
         RememberedSet<OLD_TO_CODE>::Iterate(
             chunk_,
             [=](MaybeObjectSlot slot) {
@@ -4153,25 +4159,26 @@ class EphemeronTableUpdatingItem : public UpdatingItem {
   void Process() override {
     TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.gc"),
                  "EphemeronTableUpdatingItem::Process");
+    PtrComprCageBase cage_base(heap_->isolate());
 
     for (auto it = heap_->ephemeron_remembered_set_.begin();
          it != heap_->ephemeron_remembered_set_.end();) {
       EphemeronHashTable table = it->first;
       auto& indices = it->second;
-      if (table.map_word(kRelaxedLoad).IsForwardingAddress()) {
+      if (table.map_word(cage_base, kRelaxedLoad).IsForwardingAddress()) {
         // The table has moved, and RecordMigratedSlotVisitor::VisitEphemeron
         // inserts entries for the moved table into ephemeron_remembered_set_.
         it = heap_->ephemeron_remembered_set_.erase(it);
         continue;
       }
-      DCHECK(table.map().IsMap());
-      DCHECK(table.Object::IsEphemeronHashTable());
+      DCHECK(table.map(cage_base).IsMap(cage_base));
+      DCHECK(table.IsEphemeronHashTable(cage_base));
       for (auto iti = indices.begin(); iti != indices.end();) {
         // EphemeronHashTable keys must be heap objects.
         HeapObjectSlot key_slot(table.RawFieldOfElementAt(
             EphemeronHashTable::EntryToIndex(InternalIndex(*iti))));
         HeapObject key = key_slot.ToHeapObject();
-        MapWord map_word = key.map_word(kRelaxedLoad);
+        MapWord map_word = key.map_word(cage_base, kRelaxedLoad);
         if (map_word.IsForwardingAddress()) {
           key = map_word.ToForwardingAddress();
           key_slot.StoreHeapObject(key);

src/heap/paged-spaces.cc

@@ -688,6 +688,7 @@ void PagedSpace::Verify(Isolate* isolate, ObjectVisitor* visitor) {
     external_space_bytes[static_cast<ExternalBackingStoreType>(i)] = 0;
   }
 
+  PtrComprCageBase cage_base(isolate);
   for (Page* page : *this) {
     CHECK_EQ(page->owner(), this);
 
@@ -729,7 +730,7 @@ void PagedSpace::Verify(Isolate* isolate, ObjectVisitor* visitor) {
       CHECK(object.address() + size <= top);
       end_of_previous_object = object.address() + size;
 
-      if (object.IsExternalString()) {
+      if (object.IsExternalString(cage_base)) {
         ExternalString external_string = ExternalString::cast(object);
         size_t payload_size = external_string.ExternalPayloadSize();
         external_page_bytes[ExternalBackingStoreType::kExternalString] +=

src/heap/sweeper.cc

@@ -366,6 +366,7 @@ int Sweeper::RawSweep(
   // Iterate over the page using the live objects and free the memory before
   // the given live object.
   Address free_start = p->area_start();
+  PtrComprCageBase cage_base(heap_->isolate());
   for (auto object_and_size :
        LiveObjectRange<kBlackObjects>(p, marking_state_->bitmap(p))) {
     HeapObject const object = object_and_size.first;
@@ -383,8 +384,8 @@ int Sweeper::RawSweep(
           free_start, free_end, p, non_empty_typed_slots, &free_ranges_map,
           &old_to_new_cleanup);
     }
-    Map map = object.map(kAcquireLoad);
-    DCHECK(map.IsMap());
+    Map map = object.map(cage_base, kAcquireLoad);
+    DCHECK(map.IsMap(cage_base));
     int size = object.SizeFromMap(map);
     live_bytes += size;
     free_start = free_end + size;

src/objects/objects-inl.h

@@ -18,6 +18,7 @@
 #include "src/builtins/builtins.h"
 #include "src/common/external-pointer-inl.h"
 #include "src/common/globals.h"
+#include "src/common/ptr-compr-inl.h"
 #include "src/handles/handles-inl.h"
 #include "src/heap/factory.h"
 #include "src/heap/heap-write-barrier-inl.h"
@@ -679,6 +680,22 @@ MapWord MapWord::FromForwardingAddress(HeapObject object) {
 
 HeapObject MapWord::ToForwardingAddress() {
   DCHECK(IsForwardingAddress());
+  HeapObject obj = HeapObject::FromAddress(value_);
+  // For objects allocated outside of the main pointer compression cage the
+  // variant with explicit cage base must be used.
+  DCHECK_IMPLIES(V8_EXTERNAL_CODE_SPACE_BOOL, !obj.IsCode());
+  return obj;
+}
+
+HeapObject MapWord::ToForwardingAddress(PtrComprCageBase host_cage_base) {
+  DCHECK(IsForwardingAddress());
+  if (V8_EXTERNAL_CODE_SPACE_BOOL) {
+    // Recompress value_ using proper host_cage_base since the map word
+    // has the upper 32 bits that correspond to the main cage base value.
+    Address value =
+        DecompressTaggedPointer(host_cage_base, CompressTagged(value_));
+    return HeapObject::FromAddress(value);
+  }
   return HeapObject::FromAddress(value_);
 }
 

src/objects/objects.h

@@ -787,8 +787,14 @@ class MapWord {
   // Create a map word from a forwarding address.
   static inline MapWord FromForwardingAddress(HeapObject object);
 
-  // View this map word as a forwarding address.
+  // View this map word as a forwarding address. The parameterless version
+  // is allowed to be used for objects allocated in the main pointer compression
+  // cage, while the second variant uses the value of the cage base explicitly
+  // and thus can be used in situations where one has to deal with both cases.
+  // Note, that the parameterless version is preferred because it avoids
+  // unnecessary recompressions.
   inline HeapObject ToForwardingAddress();
+  inline HeapObject ToForwardingAddress(PtrComprCageBase host_cage_base);
 
   inline Address ptr() { return value_; }