[ptr-compr] Let EmbedderDataSlot store raw data in a non-tagged part

and update visitors to not look at raw part. This will allow to have effecient
access to embedder data once kTaggedSize is switched to 32-bit value.

Bug: v8:8518
Change-Id: Ia1875a5ac5f3fb85df5c5555b970fd88d9e8d7a4
Reviewed-on: https://chromium-review.googlesource.com/c/1369957Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Commit-Queue: Igor Sheludko <ishell@chromium.org>
Cr-Commit-Position: refs/heads/master@{#58185}

include/v8-internal.h

@@ -122,6 +122,9 @@ class Internals {
   static const int kOddballKindOffset = 4 * kApiTaggedSize + kApiDoubleSize;
   static const int kForeignAddressOffset = kApiTaggedSize;
   static const int kJSObjectHeaderSize = 3 * kApiTaggedSize;
+  static const int kJSObjectHeaderSizeForEmbedderFields =
+      (kJSObjectHeaderSize + kApiSystemPointerSize - 1) &
+      -kApiSystemPointerSize;
   static const int kFixedArrayHeaderSize = 2 * kApiTaggedSize;
   static const int kEmbedderDataArrayHeaderSize = 2 * kApiTaggedSize;
   static const int kEmbedderDataSlotSize =

include/v8.h

@@ -9922,7 +9922,8 @@ Local<Value> Object::GetInternalField(int index) {
   if (instance_type == I::kJSObjectType ||
       instance_type == I::kJSApiObjectType ||
       instance_type == I::kJSSpecialApiObjectType) {
-    int offset = I::kJSObjectHeaderSize + (I::kEmbedderDataSlotSize * index);
+    int offset = I::kJSObjectHeaderSizeForEmbedderFields +
+                 (I::kEmbedderDataSlotSize * index);
     A value = I::ReadField<A>(obj, offset);
     A* result = HandleScope::CreateHandle(
         reinterpret_cast<internal::NeverReadOnlySpaceObject*>(obj), value);
@@ -9944,7 +9945,8 @@ void* Object::GetAlignedPointerFromInternalField(int index) {
   if (V8_LIKELY(instance_type == I::kJSObjectType ||
                 instance_type == I::kJSApiObjectType ||
                 instance_type == I::kJSSpecialApiObjectType)) {
-    int offset = I::kJSObjectHeaderSize + (I::kEmbedderDataSlotSize * index);
+    int offset = I::kJSObjectHeaderSizeForEmbedderFields +
+                 (I::kEmbedderDataSlotSize * index);
     return I::ReadField<void*>(obj, offset);
   }
 #endif

src/flag-definitions.h

@@ -116,6 +116,12 @@ struct MaybeBoolFlag {
 #define DEBUG_BOOL false
 #endif
 
+#ifdef V8_COMPRESS_POINTERS
+#define COMPRESS_POINTERS_BOOL true
+#else
+#define COMPRESS_POINTERS_BOOL false
+#endif
+
 // Supported ARM configurations are:
 //  "armv6":       ARMv6 + VFPv2
 //  "armv7":       ARMv7 + VFPv3-D32 + NEON

src/globals.h

@@ -884,7 +884,7 @@ constexpr int kIeeeDoubleExponentWordOffset = 0;
 // as a HeapObject pointer
 #define OBJECT_POINTER_PADDING(value) (OBJECT_POINTER_ALIGN(value) - (value))
 
-// POINTER_SIZE_ALIGN returns the value aligned as a pointer.
+// POINTER_SIZE_ALIGN returns the value aligned as a system pointer.
 #define POINTER_SIZE_ALIGN(value)                               \
   (((value) + kPointerAlignmentMask) & ~kPointerAlignmentMask)
 

src/heap/setup-heap-internal.cc

@@ -499,7 +499,8 @@ bool Heap::CreateInitialMaps() {
                  code_data_container)
 
     ALLOCATE_MAP(JS_MESSAGE_OBJECT_TYPE, JSMessageObject::kSize, message_object)
-    ALLOCATE_MAP(JS_OBJECT_TYPE, JSObject::kHeaderSize + kEmbedderDataSlotSize,
+    ALLOCATE_MAP(JS_OBJECT_TYPE,
+                 JSObject::kHeaderSizeForEmbedderFields + kEmbedderDataSlotSize,
                  external)
     external_map()->set_is_extensible(false);
 #undef ALLOCATE_PRIMITIVE_MAP

src/objects-body-descriptors-inl.h

@@ -31,11 +31,30 @@ int FlexibleWeakBodyDescriptor<start_offset>::SizeOf(Map map,
 
 bool BodyDescriptorBase::IsValidJSObjectSlotImpl(Map map, HeapObject* obj,
                                                  int offset) {
+#ifdef V8_COMPRESS_POINTERS
+  STATIC_ASSERT(kEmbedderDataSlotSize == 2 * kTaggedSize);
+  int embedder_fields_offset = JSObject::GetEmbedderFieldsStartOffset(map);
+  int inobject_fields_offset = map->GetInObjectPropertyOffset(0);
+  // |embedder_fields_offset| may be greater than |inobject_fields_offset| if
+  // the object does not have embedder fields but the check handles this
+  // case properly.
+  if (embedder_fields_offset <= offset && offset < inobject_fields_offset) {
+    // offset points to embedder fields area:
+    // [embedder_fields_offset, inobject_fields_offset).
+    STATIC_ASSERT(base::bits::IsPowerOfTwo(kEmbedderDataSlotSize));
+    return ((offset - embedder_fields_offset) & (kEmbedderDataSlotSize - 1)) ==
+           EmbedderDataSlot::kTaggedPayloadOffset;
+  }
+#else
+  // We store raw aligned pointers as Smis, so it's safe to treat the whole
+  // embedder field area as tagged slots.
+  STATIC_ASSERT(kEmbedderDataSlotSize == kTaggedSize);
+#endif
   if (!FLAG_unbox_double_fields || map->HasFastPointerLayout()) {
     return true;
   } else {
     DCHECK(FLAG_unbox_double_fields);
-    DCHECK(IsAligned(offset, kPointerSize));
+    DCHECK(IsAligned(offset, kSystemPointerSize));
 
     LayoutDescriptorHelper helper(map);
     DCHECK(!helper.all_fields_tagged());
@@ -48,12 +67,40 @@ void BodyDescriptorBase::IterateJSObjectBodyImpl(Map map, HeapObject* obj,
                                                  int start_offset,
                                                  int end_offset,
                                                  ObjectVisitor* v) {
+#ifdef V8_COMPRESS_POINTERS
+  STATIC_ASSERT(kEmbedderDataSlotSize == 2 * kTaggedSize);
+  int header_size = JSObject::GetHeaderSize(map);
+  int inobject_fields_offset = map->GetInObjectPropertyOffset(0);
+  // We are always requested to process header and embedder fields.
+  DCHECK_LE(inobject_fields_offset, end_offset);
+  // Embedder fields are located between header rouned up to the system pointer
+  // size and inobject properties.
+  if (header_size < inobject_fields_offset) {
+    // There are embedder fields.
+    IteratePointers(obj, start_offset, header_size, v);
+    // Iterate only tagged payload of the embedder slots and skip raw payload.
+    int embedder_fields_offset = RoundUp(header_size, kSystemPointerSize);
+    DCHECK_EQ(embedder_fields_offset,
+              JSObject::GetEmbedderFieldsStartOffset(map));
+    for (int offset =
+             embedder_fields_offset + EmbedderDataSlot::kTaggedPayloadOffset;
+         offset < inobject_fields_offset; offset += kEmbedderDataSlotSize) {
+      IteratePointer(obj, offset, v);
+    }
+    // Proceed processing inobject properties.
+    start_offset = inobject_fields_offset;
+  }
+#else
+  // We store raw aligned pointers as Smis, so it's safe to iterate the whole
+  // embedder field area as tagged slots.
+  STATIC_ASSERT(kEmbedderDataSlotSize == kTaggedSize);
+#endif
   if (!FLAG_unbox_double_fields || map->HasFastPointerLayout()) {
     IteratePointers(obj, start_offset, end_offset, v);
   } else {
     DCHECK(FLAG_unbox_double_fields);
-    DCHECK(IsAligned(start_offset, kPointerSize) &&
-           IsAligned(end_offset, kPointerSize));
+    DCHECK(IsAligned(start_offset, kSystemPointerSize) &&
+           IsAligned(end_offset, kSystemPointerSize));
 
     LayoutDescriptorHelper helper(map);
     DCHECK(!helper.all_fields_tagged());
@@ -723,17 +770,38 @@ class CodeDataContainer::BodyDescriptor final : public BodyDescriptorBase {
 class EmbedderDataArray::BodyDescriptor final : public BodyDescriptorBase {
  public:
   static bool IsValidSlot(Map map, HeapObject* obj, int offset) {
+#ifdef V8_COMPRESS_POINTERS
+    STATIC_ASSERT(kEmbedderDataSlotSize == 2 * kSystemPointerSize);
+    STATIC_ASSERT(base::bits::IsPowerOfTwo(kEmbedderDataSlotSize));
+    return (offset < EmbedderDataArray::kHeaderSize) ||
+           (((offset - EmbedderDataArray::kHeaderSize) &
+             (kEmbedderDataSlotSize - 1)) ==
+            EmbedderDataSlot::kTaggedPayloadOffset);
+#else
+    STATIC_ASSERT(kEmbedderDataSlotSize == kTaggedSize);
     // We store raw aligned pointers as Smis, so it's safe to iterate the whole
     // array.
     return true;
+#endif
   }
 
   template <typename ObjectVisitor>
   static inline void IterateBody(Map map, HeapObject* obj, int object_size,
                                  ObjectVisitor* v) {
+#ifdef V8_COMPRESS_POINTERS
+    STATIC_ASSERT(kEmbedderDataSlotSize == 2 * kSystemPointerSize);
+    // Iterate only tagged payload of the embedder slots and skip raw payload.
+    for (int offset = EmbedderDataArray::OffsetOfElementAt(0) +
+                      EmbedderDataSlot::kTaggedPayloadOffset;
+         offset < object_size; offset += kEmbedderDataSlotSize) {
+      IteratePointer(obj, offset, v);
+    }
+#else
     // We store raw aligned pointers as Smis, so it's safe to iterate the whole
     // array.
+    STATIC_ASSERT(kEmbedderDataSlotSize == kTaggedSize);
     IteratePointers(obj, EmbedderDataArray::kHeaderSize, object_size, v);
+#endif
   }
 
   static inline int SizeOf(Map map, HeapObject* object) {

src/objects.cc

@@ -3090,8 +3090,7 @@ VisitorId Map::GetVisitorId(Map map) {
   STATIC_ASSERT(kVisitorIdCount <= 256);
 
   const int instance_type = map->instance_type();
-  const bool has_unboxed_fields =
-      FLAG_unbox_double_fields && !map->HasFastPointerLayout();
+
   if (instance_type < FIRST_NONSTRING_TYPE) {
     switch (instance_type & kStringRepresentationMask) {
       case kSeqStringTag:
@@ -3298,8 +3297,12 @@ VisitorId Map::GetVisitorId(Map map) {
     case WASM_MEMORY_TYPE:
     case WASM_MODULE_TYPE:
     case WASM_TABLE_TYPE:
-    case JS_BOUND_FUNCTION_TYPE:
-      return has_unboxed_fields ? kVisitJSObject : kVisitJSObjectFast;
+    case JS_BOUND_FUNCTION_TYPE: {
+      const bool has_raw_data_fields =
+          (FLAG_unbox_double_fields && !map->HasFastPointerLayout()) ||
+          (COMPRESS_POINTERS_BOOL && JSObject::GetEmbedderFieldCount(map) > 0);
+      return has_raw_data_fields ? kVisitJSObject : kVisitJSObjectFast;
+    }
     case JS_API_OBJECT_TYPE:
     case JS_SPECIAL_API_OBJECT_TYPE:
       return kVisitJSApiObject;

src/objects/embedder-data-slot-inl.h

@@ -30,6 +30,9 @@ Object* EmbedderDataSlot::load_tagged() const {
 
 void EmbedderDataSlot::store_smi(Smi value) {
   ObjectSlot(address() + kTaggedPayloadOffset).Relaxed_Store(value);
+#ifdef V8_COMPRESS_POINTERS
+  ObjectSlot(address() + kRawPayloadOffset).Relaxed_Store(Smi::kZero);
+#endif
 }
 
 // static
@@ -39,6 +42,10 @@ void EmbedderDataSlot::store_tagged(EmbedderDataArray array, int entry_index,
   ObjectSlot(FIELD_ADDR(array, slot_offset + kTaggedPayloadOffset))
       .Relaxed_Store(ObjectPtr(value->ptr()));
   WRITE_BARRIER(array, slot_offset, value);
+#ifdef V8_COMPRESS_POINTERS
+  ObjectSlot(FIELD_ADDR(array, slot_offset + kRawPayloadOffset))
+      .Relaxed_Store(Smi::kZero);
+#endif
 }
 
 // static
@@ -48,27 +55,57 @@ void EmbedderDataSlot::store_tagged(JSObject object, int embedder_field_index,
   ObjectSlot(FIELD_ADDR(object, slot_offset + kTaggedPayloadOffset))
       .Relaxed_Store(ObjectPtr(value->ptr()));
   WRITE_BARRIER(object, slot_offset, value);
+#ifdef V8_COMPRESS_POINTERS
+  ObjectSlot(FIELD_ADDR(object, slot_offset + kRawPayloadOffset))
+      .Relaxed_Store(Smi::kZero);
+#endif
 }
 
 bool EmbedderDataSlot::ToAlignedPointer(void** out_pointer) const {
   Object* tagged_value =
       ObjectSlot(address() + kTaggedPayloadOffset).Relaxed_Load();
   if (!tagged_value->IsSmi()) return false;
+#ifdef V8_COMPRESS_POINTERS
+  STATIC_ASSERT(kSmiShiftSize == 0);
+  STATIC_ASSERT(SmiValuesAre31Bits());
+  Address value_lo = static_cast<uint32_t>(tagged_value->ptr());
+  STATIC_ASSERT(kTaggedSize == kSystemPointerSize);
+  Address value_hi =
+      FullObjectSlot(address() + kRawPayloadOffset).Relaxed_Load()->ptr();
+  Address value = value_lo | (value_hi << 32);
+  *out_pointer = reinterpret_cast<void*>(value);
+#else
   *out_pointer = reinterpret_cast<void*>(tagged_value);
+#endif
   return true;
 }
 
 bool EmbedderDataSlot::store_aligned_pointer(void* ptr) {
   Address value = reinterpret_cast<Address>(ptr);
   if (!HAS_SMI_TAG(value)) return false;
+#ifdef V8_COMPRESS_POINTERS
+  STATIC_ASSERT(kSmiShiftSize == 0);
+  STATIC_ASSERT(SmiValuesAre31Bits());
+  // Sign-extend lower 32-bits in order to form a proper Smi value.
+  STATIC_ASSERT(kTaggedSize == kSystemPointerSize);
+  Address lo = static_cast<intptr_t>(static_cast<int32_t>(value));
+  ObjectSlot(address() + kTaggedPayloadOffset).Relaxed_Store(Smi(lo));
+  Address hi = value >> 32;
+  ObjectSlot(address() + kRawPayloadOffset).Relaxed_Store(ObjectPtr(hi));
+#else
   ObjectSlot(address() + kTaggedPayloadOffset).Relaxed_Store(Smi(value));
+#endif
   return true;
 }
 
 EmbedderDataSlot::RawData EmbedderDataSlot::load_raw(
     const DisallowHeapAllocation& no_gc) const {
+  STATIC_ASSERT(kTaggedSize == kSystemPointerSize);
   return RawData{
       ObjectSlot(address() + kTaggedPayloadOffset).Relaxed_Load()->ptr(),
+#ifdef V8_COMPRESS_POINTERS
+      FullObjectSlot(address() + kRawPayloadOffset).Relaxed_Load()->ptr()
+#endif
   };
 }
 
@@ -76,6 +113,10 @@ void EmbedderDataSlot::store_raw(const EmbedderDataSlot::RawData& data,
                                  const DisallowHeapAllocation& no_gc) {
   ObjectSlot(address() + kTaggedPayloadOffset)
       .Relaxed_Store(ObjectPtr(data.data_[0]));
+#ifdef V8_COMPRESS_POINTERS
+  ObjectSlot(address() + kRawPayloadOffset)
+      .Relaxed_Store(ObjectPtr(data.data_[1]));
+#endif
 }
 
 }  // namespace internal

src/objects/embedder-data-slot.h

@@ -36,6 +36,7 @@ class EmbedderDataSlot
   V8_INLINE EmbedderDataSlot(EmbedderDataArray array, int entry_index);
   V8_INLINE EmbedderDataSlot(JSObject object, int embedder_field_index);
 
+  // TODO(ishell): these offsets are currently little-endian specific.
 #ifdef V8_COMPRESS_POINTERS
   static constexpr int kRawPayloadOffset = kTaggedSize;
 #endif

src/objects/js-objects-inl.h

@@ -263,13 +263,33 @@ int JSObject::GetHeaderSize(const Map map) {
              : GetHeaderSize(instance_type, map->has_prototype_slot());
 }
 
+// static
+int JSObject::GetEmbedderFieldsStartOffset(const Map map) {
+  // Embedder fields are located after the header size rounded up to the
+  // kSystemPointerSize, whereas in-object properties are at the end of the
+  // object.
+  int header_size = GetHeaderSize(map);
+  if (kTaggedSize == kSystemPointerSize) {
+    DCHECK(IsAligned(header_size, kSystemPointerSize));
+    return header_size;
+  } else {
+    return RoundUp(header_size, kSystemPointerSize);
+  }
+}
+
+int JSObject::GetEmbedderFieldsStartOffset() {
+  return GetEmbedderFieldsStartOffset(map());
+}
+
 // static
 int JSObject::GetEmbedderFieldCount(const Map map) {
   int instance_size = map->instance_size();
   if (instance_size == kVariableSizeSentinel) return 0;
-  // Internal objects do follow immediately after the header, whereas in-object
-  // properties are at the end of the object. Therefore there is no need
-  // to adjust the index here.
+  // Embedder fields are located after the header size rounded up to the
+  // kSystemPointerSize, whereas in-object properties are at the end of the
+  // object. We don't have to round up the header size here because division by
+  // kEmbedderDataSlotSizeInTaggedSlots will swallow potential padding in case
+  // of (kTaggedSize != kSystemPointerSize) anyway.
   return (((instance_size - GetHeaderSize(map)) >> kTaggedSizeLog2) -
           map->GetInObjectProperties()) /
          kEmbedderDataSlotSizeInTaggedSlots;
@@ -280,11 +300,9 @@ int JSObject::GetEmbedderFieldCount() const {
 }
 
 int JSObject::GetEmbedderFieldOffset(int index) {
-  DCHECK(index < GetEmbedderFieldCount() && index >= 0);
-  // Internal objects do follow immediately after the header, whereas in-object
-  // properties are at the end of the object. Therefore there is no need
-  // to adjust the index here.
-  return GetHeaderSize() + (kEmbedderDataSlotSize * index);
+  DCHECK_LT(static_cast<unsigned>(index),
+            static_cast<unsigned>(GetEmbedderFieldCount()));
+  return GetEmbedderFieldsStartOffset() + (kEmbedderDataSlotSize * index);
 }
 
 Object* JSObject::GetEmbedderField(int index) {

src/objects/js-objects.h

@@ -555,6 +555,9 @@ class JSObject : public JSReceiver {
   static inline int GetHeaderSize(const Map map);
   inline int GetHeaderSize() const;
 
+  static inline int GetEmbedderFieldsStartOffset(const Map map);
+  inline int GetEmbedderFieldsStartOffset();
+
   static inline int GetEmbedderFieldCount(const Map map);
   inline int GetEmbedderFieldCount() const;
   inline int GetEmbedderFieldOffset(int index);
@@ -743,10 +746,14 @@ class JSObject : public JSReceiver {
                 PropertyArray::kMaxLength);
 
 // Layout description.
-#define JS_OBJECT_FIELDS(V)       \
-  V(kElementsOffset, kTaggedSize) \
-  /* Header size. */              \
-  V(kHeaderSize, 0)
+#define JS_OBJECT_FIELDS(V)                              \
+  V(kElementsOffset, kTaggedSize)                        \
+  /* Header size. */                                     \
+  V(kHeaderSize, 0)                                      \
+  V(kOptionalEmbedderFieldPadding,                       \
+    POINTER_SIZE_PADDING(kOptionalEmbedderFieldPadding)) \
+  /* Header size aligned to kSystemPointerSize. */       \
+  V(kHeaderSizeForEmbedderFields, 0)
 
   DEFINE_FIELD_OFFSET_CONSTANTS(JSReceiver::kHeaderSize, JS_OBJECT_FIELDS)
 #undef JS_OBJECT_FIELDS
@@ -755,12 +762,17 @@ class JSObject : public JSReceiver {
   static const int kMaxInObjectProperties =
       (kMaxInstanceSize - kHeaderSize) >> kTaggedSizeLog2;
   STATIC_ASSERT(kMaxInObjectProperties <= kMaxNumberOfDescriptors);
-  // TODO(cbruni): Revisit calculation of the max supported embedder fields.
+
+  STATIC_ASSERT(kHeaderSizeForEmbedderFields ==
+                Internals::kJSObjectHeaderSizeForEmbedderFields);
+  static const int kMaxFirstInobjectPropertyOffset =
+      (1 << kFirstInobjectPropertyOffsetBitCount) - 1;
   static const int kMaxEmbedderFields =
-      (((1 << kFirstInobjectPropertyOffsetBitCount) - 1 - kHeaderSize) >>
-       kTaggedSizeLog2) /
-      kEmbedderDataSlotSizeInTaggedSlots;
-  STATIC_ASSERT(kMaxEmbedderFields <= kMaxInObjectProperties);
+      (kMaxFirstInobjectPropertyOffset - kHeaderSizeForEmbedderFields) /
+      kEmbedderDataSlotSize;
+  STATIC_ASSERT(kHeaderSizeForEmbedderFields +
+                    kMaxEmbedderFields * kEmbedderDataSlotSizeInTaggedSlots <=
+                kMaxInstanceSize);
 
   class BodyDescriptor;