[ptr-compr] Introduce CompressedXxxSlots

for accessing compressed on-heap slots with corresponding contents.

Bug: v8:7703
Change-Id: Icb314f02f4d5e8d70b997dec02ee9a9568af5d5d
Reviewed-on: https://chromium-review.googlesource.com/c/1370040Reviewed-by: Toon Verwaest <verwaest@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Commit-Queue: Igor Sheludko <ishell@chromium.org>
Cr-Commit-Position: refs/heads/master@{#58215}

BUILD.gn

@@ -2467,6 +2467,7 @@ v8_source_set("v8_base") {
     "src/property.cc",
     "src/property.h",
     "src/prototype.h",
+    "src/ptr-compr-inl.h",
     "src/ptr-compr.h",
     "src/regexp/bytecodes-irregexp.h",
     "src/regexp/interpreter-irregexp.cc",

src/globals.h

@@ -573,6 +573,10 @@ class NewSpace;
 class NewLargeObjectSpace;
 class NumberDictionary;
 class Object;
+class CompressedObjectSlot;
+class CompressedMaybeObjectSlot;
+class CompressedMapWordSlot;
+class CompressedHeapObjectSlot;
 class FullObjectSlot;
 class FullMaybeObjectSlot;
 class FullHeapObjectSlot;
@@ -610,10 +614,17 @@ struct SlotTraits<SlotLocation::kOffHeap> {
 // on whether the pointer compression is enabled or not.
 template <>
 struct SlotTraits<SlotLocation::kOnHeap> {
+#ifdef V8_COMPRESS_POINTERS
+  using TObjectSlot = CompressedObjectSlot;
+  using TMapWordSlot = CompressedMapWordSlot;
+  using TMaybeObjectSlot = CompressedMaybeObjectSlot;
+  using THeapObjectSlot = CompressedHeapObjectSlot;
+#else
   using TObjectSlot = FullObjectSlot;
   using TMapWordSlot = FullObjectSlot;
   using TMaybeObjectSlot = FullMaybeObjectSlot;
   using THeapObjectSlot = FullHeapObjectSlot;
+#endif
 };
 
 // An ObjectSlot instance describes a kTaggedSize-sized on-heap field ("slot")

src/heap/slot-set.h

@@ -11,8 +11,13 @@
 #include "src/allocation.h"
 #include "src/base/atomic-utils.h"
 #include "src/base/bits.h"
+#include "src/objects/slots.h"
 #include "src/utils.h"
 
+#ifdef V8_COMPRESS_POINTERS
+#include "src/ptr-compr.h"
+#endif
+
 namespace v8 {
 namespace internal {
 

src/objects.h

@@ -27,6 +27,9 @@
 #include "src/roots.h"
 #include "src/utils.h"
 
+#ifdef V8_COMPRESS_POINTERS
+#include "src/ptr-compr.h"
+#endif
 
 // Has to be the last include (doesn't have include guards):
 #include "src/objects/object-macros.h"

src/objects/heap-object.h

@@ -143,6 +143,7 @@ class ObjectPtr {
 
  private:
   friend class FullObjectSlot;
+  friend class CompressedObjectSlot;
   Address ptr_;
 };
 

src/objects/slots-inl.h

@@ -13,6 +13,10 @@
 #include "src/objects/heap-object.h"
 #include "src/objects/maybe-object.h"
 
+#ifdef V8_COMPRESS_POINTERS
+#include "src/ptr-compr-inl.h"
+#endif
+
 namespace v8 {
 namespace internal {
 

src/objects/slots.h

@@ -177,7 +177,7 @@ class FullMaybeObjectSlot
 // In case it is known that that slot contains a strong heap object pointer,
 // ToHeapObject() can be used to retrieve that heap object.
 class FullHeapObjectSlot
-    : public SlotBase<HeapObjectSlot, Address, kSystemPointerSize> {
+    : public SlotBase<FullHeapObjectSlot, Address, kSystemPointerSize> {
  public:
   FullHeapObjectSlot() : SlotBase(kNullAddress) {}
   explicit FullHeapObjectSlot(Address ptr) : SlotBase(ptr) {}

src/ptr-compr-inl.h

+// Copyright 2018 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_PTR_COMPR_INL_H_
+#define V8_PTR_COMPR_INL_H_
+
+#if V8_TARGET_ARCH_64_BIT
+
+#include "src/ptr-compr.h"
+
+namespace v8 {
+namespace internal {
+
+// Compresses full-pointer representation of a tagged value to on-heap
+// representation.
+V8_INLINE Tagged_t CompressTagged(Address tagged) {
+  // The compression is no-op while we are using checked decompression.
+  STATIC_ASSERT(kTaggedSize == kSystemPointerSize);
+  // TODO(ishell): implement once kTaggedSize is equal to kInt32Size.
+  return tagged;
+}
+
+// Calculates isolate root value from any on-heap address.
+V8_INLINE Address GetRootFromOnHeapAddress(Address addr) {
+  return RoundDown(addr + kPtrComprIsolateRootBias,
+                   kPtrComprIsolateRootAlignment);
+}
+
+// Decompresses weak or strong heap object pointer or forwarding pointer,
+// preserving both weak- and smi- tags.
+V8_INLINE Address DecompressTaggedPointerImpl(Address on_heap_addr,
+                                              int32_t value) {
+  Address root = GetRootFromOnHeapAddress(on_heap_addr);
+  // Current compression scheme requires value to be sign-extended to inptr_t
+  // before adding the |root|.
+  return root + static_cast<Address>(static_cast<intptr_t>(value));
+}
+
+// Decompresses weak or strong heap object pointer or forwarding pointer,
+// preserving both weak- and smi- tags and checks that the result of
+// decompression matches full value stored in the field.
+// Checked decompression helps to find misuses of XxxSlots and FullXxxSlots.
+// TODO(ishell): remove in favour of DecompressTaggedPointerImpl() once
+// kTaggedSize is equal to kInt32Size.
+V8_INLINE Address DecompressTaggedPointer(Address on_heap_addr,
+                                          Tagged_t full_value) {
+  // Use only lower 32-bits of the value for decompression.
+  int32_t compressed = static_cast<int32_t>(full_value);
+  STATIC_ASSERT(kTaggedSize == kSystemPointerSize);
+  Address result = DecompressTaggedPointerImpl(on_heap_addr, compressed);
+#ifdef DEBUG
+  if (full_value != result) {
+    base::OS::DebugBreak();
+    result = DecompressTaggedPointerImpl(on_heap_addr, compressed);
+  }
+#endif
+  DCHECK_EQ(full_value, result);
+  return result;
+}
+
+// Decompresses any tagged value, preserving both weak- and smi- tags.
+V8_INLINE Address DecompressTaggedAnyImpl(Address on_heap_addr, int32_t value) {
+  // |root_mask| is 0 if the |value| was a smi or -1 otherwise.
+  Address root_mask = -static_cast<Address>(value & kSmiTagMask);
+  Address root_or_zero = root_mask & GetRootFromOnHeapAddress(on_heap_addr);
+  // Current compression scheme requires value to be sign-extended to inptr_t
+  // before adding the |root_or_zero|.
+  return root_or_zero + static_cast<Address>(static_cast<intptr_t>(value));
+}
+
+// Decompresses any tagged value, preserving both weak- and smi- tags and checks
+// that the result of decompression matches full value stored in the field.
+// Checked decompression helps to find misuses of XxxSlots and FullXxxSlots.
+// TODO(ishell): remove in favour of DecompressTaggedAnyImpl() once
+// kTaggedSize is equal to kInt32Size.
+V8_INLINE Address DecompressTaggedAny(Address on_heap_addr,
+                                      Tagged_t full_value) {
+  // Use only lower 32-bits of the value for decompression.
+  int32_t compressed = static_cast<int32_t>(full_value);
+  STATIC_ASSERT(kTaggedSize == kSystemPointerSize);
+  Address result = DecompressTaggedAnyImpl(on_heap_addr, compressed);
+#ifdef DEBUG
+  if (full_value != result) {
+    base::OS::DebugBreak();
+    result = DecompressTaggedAnyImpl(on_heap_addr, compressed);
+  }
+#endif
+  DCHECK_EQ(full_value, result);
+  return result;
+}
+
+//
+// CompressedObjectSlot implementation.
+//
+
+CompressedObjectSlot::CompressedObjectSlot(ObjectPtr* object)
+    : SlotBase(reinterpret_cast<Address>(&object->ptr_)) {}
+
+Object* CompressedObjectSlot::operator*() const {
+  Tagged_t value = *location();
+  return reinterpret_cast<Object*>(DecompressTaggedAny(address(), value));
+}
+
+ObjectPtr CompressedObjectSlot::load() const {
+  Tagged_t value = *location();
+  return ObjectPtr(DecompressTaggedAny(address(), value));
+}
+
+void CompressedObjectSlot::store(Object* value) const {
+  *location() = CompressTagged(value->ptr());
+}
+
+ObjectPtr CompressedObjectSlot::Acquire_Load() const {
+  AtomicTagged_t value = AsAtomicTagged::Acquire_Load(location());
+  return ObjectPtr(DecompressTaggedAny(address(), value));
+}
+
+Object* CompressedObjectSlot::Acquire_Load1() const {
+  AtomicTagged_t value = AsAtomicTagged::Acquire_Load(location());
+  return reinterpret_cast<Object*>(DecompressTaggedAny(address(), value));
+}
+
+ObjectPtr CompressedObjectSlot::Relaxed_Load() const {
+  AtomicTagged_t value = AsAtomicTagged::Relaxed_Load(location());
+  return ObjectPtr(DecompressTaggedAny(address(), value));
+}
+
+void CompressedObjectSlot::Relaxed_Store(ObjectPtr value) const {
+  Tagged_t ptr = CompressTagged(value->ptr());
+  AsAtomicTagged::Relaxed_Store(location(), ptr);
+}
+
+void CompressedObjectSlot::Relaxed_Store1(Object* value) const {
+  Tagged_t ptr = CompressTagged(value->ptr());
+  AsAtomicTagged::Relaxed_Store(location(), ptr);
+}
+
+void CompressedObjectSlot::Release_Store1(Object* value) const {
+  Tagged_t ptr = CompressTagged(value->ptr());
+  AsAtomicTagged::Release_Store(location(), ptr);
+}
+
+void CompressedObjectSlot::Release_Store(ObjectPtr value) const {
+  Tagged_t ptr = CompressTagged(value->ptr());
+  AsAtomicTagged::Release_Store(location(), ptr);
+}
+
+ObjectPtr CompressedObjectSlot::Release_CompareAndSwap(ObjectPtr old,
+                                                       ObjectPtr target) const {
+  Tagged_t old_ptr = CompressTagged(old->ptr());
+  Tagged_t target_ptr = CompressTagged(target->ptr());
+  Tagged_t result =
+      AsAtomicTagged::Release_CompareAndSwap(location(), old_ptr, target_ptr);
+  return ObjectPtr(DecompressTaggedAny(address(), result));
+}
+
+//
+// CompressedMapWordSlot implementation.
+//
+
+bool CompressedMapWordSlot::contains_value(Address raw_value) const {
+  Tagged_t value = *location();
+  return value == static_cast<Tagged_t>(raw_value);
+}
+
+ObjectPtr CompressedMapWordSlot::load() const {
+  Tagged_t value = *location();
+  return ObjectPtr(DecompressTaggedPointer(address(), value));
+}
+
+void CompressedMapWordSlot::store(ObjectPtr value) const {
+  *location() = CompressTagged(value.ptr());
+}
+
+ObjectPtr CompressedMapWordSlot::Relaxed_Load() const {
+  AtomicTagged_t value = AsAtomicTagged::Relaxed_Load(location());
+  return ObjectPtr(DecompressTaggedPointer(address(), value));
+}
+
+void CompressedMapWordSlot::Relaxed_Store(ObjectPtr value) const {
+  Tagged_t ptr = CompressTagged(value.ptr());
+  AsAtomicTagged::Relaxed_Store(location(), ptr);
+}
+
+ObjectPtr CompressedMapWordSlot::Acquire_Load() const {
+  AtomicTagged_t value = AsAtomicTagged::Acquire_Load(location());
+  return ObjectPtr(DecompressTaggedPointer(address(), value));
+}
+
+void CompressedMapWordSlot::Release_Store(ObjectPtr value) const {
+  Tagged_t ptr = CompressTagged(value->ptr());
+  AsAtomicTagged::Release_Store(location(), ptr);
+}
+
+ObjectPtr CompressedMapWordSlot::Release_CompareAndSwap(
+    ObjectPtr old, ObjectPtr target) const {
+  Tagged_t old_ptr = CompressTagged(old->ptr());
+  Tagged_t target_ptr = CompressTagged(target->ptr());
+  Tagged_t result =
+      AsAtomicTagged::Release_CompareAndSwap(location(), old_ptr, target_ptr);
+  return ObjectPtr(DecompressTaggedPointer(address(), result));
+}
+
+//
+// CompressedMaybeObjectSlot implementation.
+//
+
+MaybeObject CompressedMaybeObjectSlot::operator*() const {
+  Tagged_t value = *location();
+  return MaybeObject(DecompressTaggedAny(address(), value));
+}
+
+MaybeObject CompressedMaybeObjectSlot::load() const {
+  Tagged_t value = *location();
+  return MaybeObject(DecompressTaggedAny(address(), value));
+}
+
+void CompressedMaybeObjectSlot::store(MaybeObject value) const {
+  *location() = CompressTagged(value->ptr());
+}
+
+MaybeObject CompressedMaybeObjectSlot::Relaxed_Load() const {
+  AtomicTagged_t value = AsAtomicTagged::Relaxed_Load(location());
+  return MaybeObject(DecompressTaggedAny(address(), value));
+}
+
+void CompressedMaybeObjectSlot::Relaxed_Store(MaybeObject value) const {
+  Tagged_t ptr = CompressTagged(value->ptr());
+  AsAtomicTagged::Relaxed_Store(location(), ptr);
+}
+
+void CompressedMaybeObjectSlot::Release_CompareAndSwap(
+    MaybeObject old, MaybeObject target) const {
+  Tagged_t old_ptr = CompressTagged(old->ptr());
+  Tagged_t target_ptr = CompressTagged(target->ptr());
+  AsAtomicTagged::Release_CompareAndSwap(location(), old_ptr, target_ptr);
+}
+
+//
+// CompressedHeapObjectSlot implementation.
+//
+
+HeapObjectReference CompressedHeapObjectSlot::operator*() const {
+  Tagged_t value = *location();
+  return HeapObjectReference(DecompressTaggedPointer(address(), value));
+}
+
+void CompressedHeapObjectSlot::store(HeapObjectReference value) const {
+  *location() = CompressTagged(value.ptr());
+}
+
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_TARGET_ARCH_64_BIT
+
+#endif  // V8_PTR_COMPR_INL_H_

src/ptr-compr.h

@@ -8,6 +8,7 @@
 #if V8_TARGET_ARCH_64_BIT
 
 #include "src/globals.h"
+#include "src/objects/slots.h"
 
 namespace v8 {
 namespace internal {
@@ -16,6 +17,149 @@ constexpr size_t kPtrComprHeapReservationSize = size_t{4} * GB;
 constexpr size_t kPtrComprIsolateRootBias = kPtrComprHeapReservationSize / 2;
 constexpr size_t kPtrComprIsolateRootAlignment = size_t{4} * GB;
 
+// A CompressedObjectSlot instance describes a kTaggedSize-sized field ("slot")
+// holding a compressed tagged pointer (smi or heap object).
+// Its address() is the address of the slot.
+// The slot's contents can be read and written using operator* and store().
+class CompressedObjectSlot
+    : public SlotBase<CompressedObjectSlot, Tagged_t, kTaggedSize> {
+ public:
+  using TObject = ObjectPtr;
+  using THeapObjectSlot = CompressedHeapObjectSlot;
+
+  static constexpr bool kCanBeWeak = false;
+
+  CompressedObjectSlot() : SlotBase(kNullAddress) {}
+  explicit CompressedObjectSlot(Address ptr) : SlotBase(ptr) {}
+  explicit CompressedObjectSlot(Address* ptr)
+      : SlotBase(reinterpret_cast<Address>(ptr)) {}
+  inline explicit CompressedObjectSlot(ObjectPtr* object);
+  explicit CompressedObjectSlot(Object const* const* ptr)
+      : SlotBase(reinterpret_cast<Address>(ptr)) {}
+  template <typename T>
+  explicit CompressedObjectSlot(SlotBase<T, TData, kSlotDataSize> slot)
+      : SlotBase(slot.address()) {}
+
+  inline Object* operator*() const;
+  // TODO(3770): drop this in favor of operator* once migration is complete.
+  inline ObjectPtr load() const;
+  inline void store(Object* value) const;
+
+  inline ObjectPtr Acquire_Load() const;
+  inline ObjectPtr Relaxed_Load() const;
+  inline void Relaxed_Store(ObjectPtr value) const;
+  inline void Release_Store(ObjectPtr value) const;
+  inline ObjectPtr Release_CompareAndSwap(ObjectPtr old,
+                                          ObjectPtr target) const;
+  // Old-style alternative for the above, temporarily separate to allow
+  // incremental transition.
+  // TODO(3770): Get rid of the duplication when the migration is complete.
+  inline Object* Acquire_Load1() const;
+  inline void Relaxed_Store1(Object* value) const;
+  inline void Release_Store1(Object* value) const;
+};
+
+// A CompressedMapWordSlot instance describes a kTaggedSize-sized map-word field
+// ("slot") of heap objects holding a compressed tagged pointer or a Smi
+// representing forwaring pointer value.
+// This slot kind is similar to CompressedObjectSlot but decompression of
+// forwarding pointer is different.
+// Its address() is the address of the slot.
+// The slot's contents can be read and written using operator* and store().
+class CompressedMapWordSlot
+    : public SlotBase<CompressedMapWordSlot, Tagged_t, kTaggedSize> {
+ public:
+  using TObject = ObjectPtr;
+
+  static constexpr bool kCanBeWeak = false;
+
+  CompressedMapWordSlot() : SlotBase(kNullAddress) {}
+  explicit CompressedMapWordSlot(Address ptr) : SlotBase(ptr) {}
+
+  // Compares memory representation of a value stored in the slot with given
+  // raw value without decompression.
+  inline bool contains_value(Address raw_value) const;
+
+  inline ObjectPtr load() const;
+  inline void store(ObjectPtr value) const;
+
+  inline ObjectPtr Relaxed_Load() const;
+  inline void Relaxed_Store(ObjectPtr value) const;
+
+  inline ObjectPtr Acquire_Load() const;
+  inline void Release_Store(ObjectPtr value) const;
+  inline ObjectPtr Release_CompareAndSwap(ObjectPtr old,
+                                          ObjectPtr target) const;
+};
+
+// A CompressedMaybeObjectSlot instance describes a kTaggedSize-sized field
+// ("slot") holding a possibly-weak compressed tagged pointer
+// (think: MaybeObject).
+// Its address() is the address of the slot.
+// The slot's contents can be read and written using operator* and store().
+class CompressedMaybeObjectSlot
+    : public SlotBase<CompressedMaybeObjectSlot, Tagged_t, kTaggedSize> {
+ public:
+  using TObject = MaybeObject;
+  using THeapObjectSlot = CompressedHeapObjectSlot;
+
+  static constexpr bool kCanBeWeak = true;
+
+  CompressedMaybeObjectSlot() : SlotBase(kNullAddress) {}
+  explicit CompressedMaybeObjectSlot(Address ptr) : SlotBase(ptr) {}
+  explicit CompressedMaybeObjectSlot(ObjectPtr* ptr)
+      : SlotBase(reinterpret_cast<Address>(ptr)) {}
+  explicit CompressedMaybeObjectSlot(Object** ptr)
+      : SlotBase(reinterpret_cast<Address>(ptr)) {}
+  explicit CompressedMaybeObjectSlot(HeapObject** ptr)
+      : SlotBase(reinterpret_cast<Address>(ptr)) {}
+  template <typename T>
+  explicit CompressedMaybeObjectSlot(SlotBase<T, TData, kSlotDataSize> slot)
+      : SlotBase(slot.address()) {}
+
+  inline MaybeObject operator*() const;
+  // TODO(3770): drop this once ObjectSlot::load() is dropped.
+  inline MaybeObject load() const;
+  inline void store(MaybeObject value) const;
+
+  inline MaybeObject Relaxed_Load() const;
+  inline void Relaxed_Store(MaybeObject value) const;
+  inline void Release_CompareAndSwap(MaybeObject old, MaybeObject target) const;
+};
+
+// A CompressedHeapObjectSlot instance describes a kTaggedSize-sized field
+// ("slot") holding a weak or strong compressed pointer to a heap object (think:
+// HeapObjectReference).
+// Its address() is the address of the slot.
+// The slot's contents can be read and written using operator* and store().
+// In case it is known that that slot contains a strong heap object pointer,
+// ToHeapObject() can be used to retrieve that heap object.
+class CompressedHeapObjectSlot
+    : public SlotBase<CompressedHeapObjectSlot, Tagged_t, kTaggedSize> {
+ public:
+  CompressedHeapObjectSlot() : SlotBase(kNullAddress) {}
+  explicit CompressedHeapObjectSlot(Address ptr) : SlotBase(ptr) {}
+  explicit CompressedHeapObjectSlot(ObjectPtr* ptr)
+      : SlotBase(reinterpret_cast<Address>(ptr)) {}
+  explicit CompressedHeapObjectSlot(HeapObject** ptr)
+      : SlotBase(reinterpret_cast<Address>(ptr)) {}
+  template <typename T>
+  explicit CompressedHeapObjectSlot(SlotBase<T, TData, kSlotDataSize> slot)
+      : SlotBase(slot.address()) {}
+
+  inline HeapObjectReference operator*() const;
+  inline void store(HeapObjectReference value) const;
+
+  HeapObject* ToHeapObject() const {
+    DCHECK((*location() & kHeapObjectTagMask) == kHeapObjectTag);
+    return reinterpret_cast<HeapObject*>(*location());
+  }
+
+  void StoreHeapObject(HeapObject* value) const {
+    *reinterpret_cast<HeapObject**>(address()) = value;
+  }
+};
+
 }  // namespace internal
 }  // namespace v8
 

src/runtime/runtime-scopes.cc

@@ -596,10 +596,10 @@ RUNTIME_FUNCTION(Runtime_NewSloppyArguments) {
 RUNTIME_FUNCTION(Runtime_NewArgumentsElements) {
   HandleScope scope(isolate);
   DCHECK_EQ(3, args.length());
-  // Note that args[0] is the address of an array of object pointers (a.k.a.
-  // an ObjectSlot), which looks like a Smi because it's aligned.
+  // Note that args[0] is the address of an array of full object pointers
+  // (a.k.a. FullObjectSlot), which looks like a Smi because it's aligned.
   DCHECK(args[0].IsSmi());
-  ObjectSlot frame(args[0]->ptr());
+  FullObjectSlot frame(args[0]->ptr());
   CONVERT_SMI_ARG_CHECKED(length, 1);
   CONVERT_SMI_ARG_CHECKED(mapped_count, 2);
   Handle<FixedArray> result =

src/utils.h

@@ -21,7 +21,6 @@
 #include "src/base/platform/platform.h"
 #include "src/base/v8-fallthrough.h"
 #include "src/globals.h"
-#include "src/objects/slots.h"
 #include "src/vector.h"
 
 #if defined(V8_OS_AIX)