[ptr-compr-8gb] Align Turbofan allocations to 8 bytes

In order to support a larger heap cage (8GB, 16GB), the cage offset
will take up more than 32 bits. As a consequence, for 8GB cages, the
least significant bit of the cage offset will overlap with the most
significant bit of the tagged offset. To avoid this, allocations need
to be aligned to 8 bytes to free up one bit from the offset.

All changes are deactivated behind the build flag
`v8_enable_pointer_compression_8gb`. Allocation folding is not yet
supported.

Bug: v8:13070
Change-Id: I602c71232e98eac4e2701b0922704a7adc31a662
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3817741
Commit-Queue: Teo Dutu <teodutu@google.com>
Reviewed-by: Marja Hölttä <marja@chromium.org>
Reviewed-by: Igor Sheludko <ishell@chromium.org>
Reviewed-by: Tobias Tebbi <tebbi@chromium.org>
Cr-Commit-Position: refs/heads/main@{#82331}

src/builtins/builtins-array.cc

@@ -7,6 +7,7 @@
 #include "src/builtins/builtins.h"
 #include "src/codegen/code-factory.h"
 #include "src/common/assert-scope.h"
+#include "src/common/globals.h"
 #include "src/debug/debug.h"
 #include "src/execution/isolate.h"
 #include "src/execution/protectors-inl.h"
@@ -603,7 +604,8 @@ BUILTIN(ArrayShift) {
     return ReadOnlyRoots(isolate).undefined_value();
   }
 
-  if (CanUseFastArrayShift(isolate, receiver)) {
+  if (!V8_COMPRESS_POINTERS_8GB_BOOL &&
+      CanUseFastArrayShift(isolate, receiver)) {
     Handle<JSArray> array = Handle<JSArray>::cast(receiver);
     RETURN_RESULT_OR_FAILURE(isolate,
                              array->GetElementsAccessor()->Shift(array));

src/builtins/builtins-constructor-gen.cc

@@ -5,6 +5,7 @@
 #include "src/builtins/builtins-constructor-gen.h"
 
 #include "src/ast/ast.h"
+#include "src/base/macros.h"
 #include "src/builtins/builtins-call-gen.h"
 #include "src/builtins/builtins-constructor.h"
 #include "src/builtins/builtins-utils-gen.h"
@@ -12,10 +13,13 @@
 #include "src/codegen/code-factory.h"
 #include "src/codegen/code-stub-assembler.h"
 #include "src/codegen/interface-descriptors.h"
+#include "src/codegen/machine-type.h"
 #include "src/codegen/macro-assembler.h"
+#include "src/codegen/tnode.h"
 #include "src/common/globals.h"
 #include "src/logging/counters.h"
 #include "src/objects/objects-inl.h"
+#include "src/objects/objects.h"
 
 namespace v8 {
 namespace internal {
@@ -596,17 +600,37 @@ TNode<HeapObject> ConstructorBuiltinsAssembler::CreateShallowObjectLiteral(
   static_assert(JSObject::kMaxInstanceSize < kMaxRegularHeapObjectSize);
   TNode<IntPtrT> instance_size =
       TimesTaggedSize(LoadMapInstanceSizeInWords(boilerplate_map));
-  TNode<IntPtrT> allocation_size = instance_size;
+  TVARIABLE(IntPtrT, aligned_instance_size, instance_size);
+  TVARIABLE(IntPtrT, allocation_size, instance_size);
+  TNode<BoolT> is_instance_size_aligned;
+  constexpr int filler_size = kDoubleSize - kTaggedSize;
   bool needs_allocation_memento = FLAG_allocation_site_pretenuring;
   if (needs_allocation_memento) {
     DCHECK(V8_ALLOCATION_SITE_TRACKING_BOOL);
     // Prepare for inner-allocating the AllocationMemento.
     allocation_size =
         IntPtrAdd(instance_size, IntPtrConstant(AllocationMemento::kSize));
+    if (V8_COMPRESS_POINTERS_8GB_BOOL) {
+      is_instance_size_aligned = WordIsAligned(instance_size, kDoubleAlignment);
+      Label size_is_aligned(this, {&aligned_instance_size, &allocation_size});
+      Label size_is_unaligned(this);
+      Branch(is_instance_size_aligned, &size_is_aligned, &size_is_unaligned);
+
+      BIND(&size_is_unaligned);
+      {
+        allocation_size =
+            IntPtrAdd(allocation_size.value(), IntPtrConstant(filler_size));
+        aligned_instance_size =
+            IntPtrAdd(instance_size, IntPtrConstant(filler_size));
+        Goto(&size_is_aligned);
+      }
+
+      BIND(&size_is_aligned);
+    }
   }
 
   TNode<HeapObject> copy =
-      UncheckedCast<HeapObject>(AllocateInNewSpace(allocation_size));
+      UncheckedCast<HeapObject>(AllocateInNewSpace(allocation_size.value()));
   {
     Comment("Initialize Literal Copy");
     // Initialize Object fields.
@@ -620,7 +644,19 @@ TNode<HeapObject> ConstructorBuiltinsAssembler::CreateShallowObjectLiteral(
   // Initialize the AllocationMemento before potential GCs due to heap number
   // allocation when copying the in-object properties.
   if (needs_allocation_memento) {
-    InitializeAllocationMemento(copy, instance_size, allocation_site);
+    if (V8_COMPRESS_POINTERS_8GB_BOOL) {
+      Label size_is_aligned(this), size_is_unaligned(this);
+      Branch(is_instance_size_aligned, &size_is_aligned, &size_is_unaligned);
+
+      BIND(&size_is_unaligned);
+      StoreObjectFieldNoWriteBarrier(copy, instance_size,
+                                     OnePointerFillerMapConstant());
+      Goto(&size_is_aligned);
+
+      BIND(&size_is_aligned);
+    }
+    InitializeAllocationMemento(copy, aligned_instance_size.value(),
+                                allocation_site);
   }
 
   {

src/codegen/code-stub-assembler.cc

@@ -15,6 +15,7 @@
 #include "src/execution/frames.h"
 #include "src/execution/protectors.h"
 #include "src/heap/heap-inl.h"  // For MemoryChunk. TODO(jkummerow): Drop.
+#include "src/heap/heap.h"
 #include "src/heap/memory-chunk.h"
 #include "src/logging/counters.h"
 #include "src/numbers/integer-literal-inl.h"
@@ -23,6 +24,7 @@
 #include "src/objects/descriptor-array.h"
 #include "src/objects/function-kind.h"
 #include "src/objects/heap-number.h"
+#include "src/objects/heap-object.h"
 #include "src/objects/instance-type.h"
 #include "src/objects/js-generator.h"
 #include "src/objects/oddball.h"
@@ -1343,7 +1345,10 @@ TNode<HeapObject> CodeStubAssembler::AllocateRawDoubleAligned(
 // unaligned access since both x64 and arm64 architectures (where pointer
 // compression is supported) allow unaligned access to doubles and full words.
 #endif  // V8_COMPRESS_POINTERS
-  // Allocation on 64 bit machine is naturally double aligned
+  if (V8_COMPRESS_POINTERS_8GB_BOOL) {
+    return AllocateRaw(size_in_bytes, flags | AllocationFlag::kDoubleAlignment,
+                       top_address, limit_address);
+  }
   return AllocateRaw(size_in_bytes, flags & ~AllocationFlag::kDoubleAlignment,
                      top_address, limit_address);
 #else
@@ -1412,7 +1417,8 @@ TNode<HeapObject> CodeStubAssembler::Allocate(TNode<IntPtrT> size_in_bytes,
       IntPtrAdd(ReinterpretCast<IntPtrT>(top_address),
                 IntPtrConstant(kSystemPointerSize));
 
-  if (flags & AllocationFlag::kDoubleAlignment) {
+  if (V8_COMPRESS_POINTERS_8GB_BOOL ||
+      (flags & AllocationFlag::kDoubleAlignment)) {
     return AllocateRawDoubleAligned(size_in_bytes, flags,
                                     ReinterpretCast<RawPtrT>(top_address),
                                     ReinterpretCast<RawPtrT>(limit_address));
@@ -4014,10 +4020,14 @@ CodeStubAssembler::AllocateUninitializedJSArrayWithElements(
       base_size += AllocationMemento::kSize;
     }
 
-    const int elements_offset = base_size;
+    const int elements_offset =
+        V8_COMPRESS_POINTERS_8GB_BOOL
+            ? RoundUp<kObjectAlignment8GbHeap>(base_size)
+            : base_size;
+    const int unaligned_elements_offset = base_size;
 
     // Compute space for elements
-    base_size += FixedArray::kHeaderSize;
+    base_size = elements_offset + FixedArray::kHeaderSize;
     TNode<IntPtrT> size = ElementOffsetFromIndex(capacity, kind, base_size);
 
     // For very large arrays in which the requested allocation exceeds the
@@ -4054,6 +4064,11 @@ CodeStubAssembler::AllocateUninitializedJSArrayWithElements(
     // Fold all objects into a single new space allocation.
     array =
         AllocateUninitializedJSArray(array_map, length, allocation_site, size);
+    if (V8_COMPRESS_POINTERS_8GB_BOOL &&
+        elements_offset != unaligned_elements_offset) {
+      StoreObjectFieldNoWriteBarrier(array.value(), unaligned_elements_offset,
+                                     OnePointerFillerMapConstant());
+    }
     elements = InnerAllocateElements(this, array.value(), elements_offset);
 
     StoreObjectFieldNoWriteBarrier(array.value(), JSObject::kElementsOffset,

src/common/globals.h

@@ -657,6 +657,14 @@ constexpr intptr_t kObjectAlignmentMask = kObjectAlignment - 1;
 constexpr intptr_t kObjectAlignment8GbHeap = 8;
 constexpr intptr_t kObjectAlignment8GbHeapMask = kObjectAlignment8GbHeap - 1;
 
+#ifdef V8_COMPRESS_POINTERS_8GB
+static_assert(
+    kObjectAlignment8GbHeap == 2 * kTaggedSize,
+    "When the 8GB heap is enabled, all allocations should be aligned to twice "
+    "the size of a tagged value. This enables aligning allocations by just "
+    "adding kTaggedSize to an unaligned address.");
+#endif
+
 // Desired alignment for system pointers.
 constexpr intptr_t kPointerAlignment = (1 << kSystemPointerSizeLog2);
 constexpr intptr_t kPointerAlignmentMask = kPointerAlignment - 1;

src/compiler/memory-lowering.cc

@@ -5,14 +5,19 @@
 #include "src/compiler/memory-lowering.h"
 
 #include "src/codegen/interface-descriptors-inl.h"
+#include "src/codegen/machine-type.h"
+#include "src/common/globals.h"
 #include "src/compiler/js-graph.h"
 #include "src/compiler/linkage.h"
 #include "src/compiler/node-matchers.h"
 #include "src/compiler/node-properties.h"
 #include "src/compiler/node.h"
 #include "src/compiler/simplified-operator.h"
+#include "src/compiler/write-barrier-kind.h"
+#include "src/heap/factory-inl.h"
 #include "src/roots/roots-inl.h"
 #include "src/sandbox/external-pointer-inl.h"
+#include "src/utils/utils.h"
 
 #if V8_ENABLE_WEBASSEMBLY
 #include "src/wasm/wasm-linkage.h"
@@ -131,6 +136,31 @@ Node* MemoryLowering::GetWasmInstanceNode() {
 
 #define __ gasm()->
 
+Node* MemoryLowering::AlignIf8GbCompression(Node* address) {
+  if (!V8_COMPRESS_POINTERS_8GB_BOOL) return address;
+
+  auto already_aligned = __ MakeLabel(MachineRepresentation::kWord64);
+  Node* alignment_check = __ WordEqual(
+      __ WordAnd(address, __ UintPtrConstant(kObjectAlignment8GbHeapMask)),
+      __ UintPtrConstant(0));
+
+  __ GotoIf(alignment_check, &already_aligned, address);
+  {
+    Node* one_pointer_filler_map =
+        __ HeapConstant(__ isolate()->factory()->one_pointer_filler_map());
+    __ Store(StoreRepresentation(MachineRepresentation::kCompressedPointer,
+                                 kNoWriteBarrier),
+             address, __ IntPtrConstant(0), one_pointer_filler_map);
+    Node* aligned_address =
+        __ IntPtrAdd(address, __ UintPtrConstant(kTaggedSize));
+    __ Goto(&already_aligned, aligned_address);
+  }
+
+  __ Bind(&already_aligned);
+
+  return already_aligned.PhiAt(0);
+}
+
 Reduction MemoryLowering::ReduceAllocateRaw(
     Node* node, AllocationType allocation_type,
     AllowLargeObjects allow_large_objects, AllocationState const** state_ptr) {
@@ -330,8 +360,8 @@ Reduction MemoryLowering::ReduceAllocateRaw(
     auto done = __ MakeLabel(MachineRepresentation::kTaggedPointer);
 
     // Load allocation top and limit.
-    Node* top =
-        __ Load(MachineType::Pointer(), top_address, __ IntPtrConstant(0));
+    Node* top = AlignIf8GbCompression(
+        __ Load(MachineType::Pointer(), top_address, __ IntPtrConstant(0)));
     Node* limit =
         __ Load(MachineType::Pointer(), limit_address, __ IntPtrConstant(0));
 

src/compiler/memory-lowering.h

@@ -113,6 +113,7 @@ class MemoryLowering final : public Reducer {
   Node* ComputeIndex(ElementAccess const& access, Node* node);
   void EnsureAllocateOperator();
   Node* GetWasmInstanceNode();
+  Node* AlignIf8GbCompression(Node* address);
 
   Graph* graph() const { return graph_; }
   Isolate* isolate() const { return isolate_; }