[ptr-compr][csa] Storing CompressedXXX (with Change node) instead of TaggedXXX

We translate stores with TaggedXXX (XXX in {"", "Signed", "Pointer"})
representation in CSA into stores of CompressedXXX with a
ChangeTaggedXXXToCompressedXXX in the raw-machine-assembler.

This way, CSA doesn't need to know about Compressed values since we
are introducing an explicit "compress" node.

Also, on ARM64, removed CheckPageFlagSet and CheckPageFlagClear since
CheckPageFlag can be used for both cases.

Moved CheckPageFlag to the TurboAssembler (from MacroAssembler) since it
was needed on code-generator-arm64.cc.

Bug: v8:8977, v8:7703
Cq-Include-Trybots: luci.v8.try:v8_linux64_pointer_compression_rel_ng
Cq-Include-Trybots: luci.v8.try:v8_linux64_arm64_pointer_compression_rel_ng
Change-Id: Ia3a41b09a4d715588a36461620be0432ed064d13
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1566517Reviewed-by: Michael Stanton <mvstanton@chromium.org>
Reviewed-by: Igor Sheludko <ishell@chromium.org>
Reviewed-by: Jaroslav Sevcik <jarin@chromium.org>
Commit-Queue: Santiago Aboy Solanes <solanes@chromium.org>
Cr-Commit-Position: refs/heads/master@{#60915}

src/arm64/macro-assembler-arm64.cc

@@ -2801,6 +2801,13 @@ void TurboAssembler::DecompressTaggedSigned(const Register& destination,
   RecordComment("]");
 }
 
+void TurboAssembler::DecompressTaggedPointer(const Register& destination,
+                                             const Register& source) {
+  RecordComment("[ DecompressTaggedPointer");
+  Add(destination, kRootRegister, Operand(source, SXTW));
+  RecordComment("]");
+}
+
 void TurboAssembler::DecompressTaggedPointer(const Register& destination,
                                              const MemOperand& field_operand) {
   RecordComment("[ DecompressTaggedPointer");
@@ -2919,7 +2926,7 @@ int MacroAssembler::SafepointRegisterStackIndex(int reg_code) {
   }
 }
 
-void MacroAssembler::CheckPageFlag(const Register& object,
+void TurboAssembler::CheckPageFlag(const Register& object,
                                    const Register& scratch, int mask,
                                    Condition cc, Label* condition_met) {
   And(scratch, object, ~kPageAlignmentMask);
@@ -2927,26 +2934,11 @@ void MacroAssembler::CheckPageFlag(const Register& object,
   if (cc == eq) {
     TestAndBranchIfAnySet(scratch, mask, condition_met);
   } else {
+    DCHECK_EQ(cc, ne);
     TestAndBranchIfAllClear(scratch, mask, condition_met);
   }
 }
 
-void TurboAssembler::CheckPageFlagSet(const Register& object,
-                                      const Register& scratch, int mask,
-                                      Label* if_any_set) {
-  And(scratch, object, ~kPageAlignmentMask);
-  Ldr(scratch, MemOperand(scratch, MemoryChunk::kFlagsOffset));
-  TestAndBranchIfAnySet(scratch, mask, if_any_set);
-}
-
-void TurboAssembler::CheckPageFlagClear(const Register& object,
-                                        const Register& scratch, int mask,
-                                        Label* if_all_clear) {
-  And(scratch, object, ~kPageAlignmentMask);
-  Ldr(scratch, MemOperand(scratch, MemoryChunk::kFlagsOffset));
-  TestAndBranchIfAllClear(scratch, mask, if_all_clear);
-}
-
 void MacroAssembler::RecordWriteField(Register object, int offset,
                                       Register value, Register scratch,
                                       LinkRegisterStatus lr_status,
@@ -3118,14 +3110,13 @@ void MacroAssembler::RecordWrite(Register object, Register address,
     DCHECK_EQ(0, kSmiTag);
     JumpIfSmi(value, &done);
   }
+  CheckPageFlag(value,
+                value,  // Used as scratch.
+                MemoryChunk::kPointersToHereAreInterestingMask, ne, &done);
 
-  CheckPageFlagClear(value,
-                     value,  // Used as scratch.
-                     MemoryChunk::kPointersToHereAreInterestingMask, &done);
-  CheckPageFlagClear(object,
-                     value,  // Used as scratch.
-                     MemoryChunk::kPointersFromHereAreInterestingMask,
-                     &done);
+  CheckPageFlag(object,
+                value,  // Used as scratch.
+                MemoryChunk::kPointersFromHereAreInterestingMask, ne, &done);
 
   // Record the actual write.
   if (lr_status == kLRHasNotBeenSaved) {

src/arm64/macro-assembler-arm64.h

@@ -783,11 +783,8 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
   Operand MoveImmediateForShiftedOp(const Register& dst, int64_t imm,
                                     PreShiftImmMode mode);
 
-  void CheckPageFlagSet(const Register& object, const Register& scratch,
-                        int mask, Label* if_any_set);
-
-  void CheckPageFlagClear(const Register& object, const Register& scratch,
-                          int mask, Label* if_all_clear);
+  void CheckPageFlag(const Register& object, const Register& scratch, int mask,
+                     Condition cc, Label* condition_met);
 
   // Test the bits of register defined by bit_pattern, and branch if ANY of
   // those bits are set. May corrupt the status flags.
@@ -1199,6 +1196,8 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
 
   void DecompressTaggedSigned(const Register& destination,
                               const MemOperand& field_operand);
+  void DecompressTaggedPointer(const Register& destination,
+                               const Register& source);
   void DecompressTaggedPointer(const Register& destination,
                                const MemOperand& field_operand);
   void DecompressAnyTagged(const Register& destination,
@@ -1925,9 +1924,6 @@ class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler {
   void PushSafepointRegisters();
   void PopSafepointRegisters();
 
-  void CheckPageFlag(const Register& object, const Register& scratch, int mask,
-                     Condition cc, Label* condition_met);
-
   // Notify the garbage collector that we wrote a pointer into an object.
   // |object| is the object being stored into, |value| is the object being
   // stored.  value and scratch registers are clobbered by the operation.

src/compiler/backend/arm64/code-generator-arm64.cc

@@ -280,9 +280,12 @@ class OutOfLineRecordWrite final : public OutOfLineCode {
     if (mode_ > RecordWriteMode::kValueIsPointer) {
       __ JumpIfSmi(value_, exit());
     }
-    __ CheckPageFlagClear(value_, scratch0_,
-                          MemoryChunk::kPointersToHereAreInterestingMask,
-                          exit());
+    if (COMPRESS_POINTERS_BOOL) {
+      __ DecompressTaggedPointer(value_, value_);
+    }
+    __ CheckPageFlag(value_, scratch0_,
+                     MemoryChunk::kPointersToHereAreInterestingMask, ne,
+                     exit());
     __ Add(scratch1_, object_, index_);
     RememberedSetAction const remembered_set_action =
         mode_ > RecordWriteMode::kValueIsMap ? EMIT_REMEMBERED_SET
@@ -862,9 +865,12 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
           this, object, index, value, scratch0, scratch1, mode,
           DetermineStubCallMode(), &unwinding_info_writer_);
       __ StoreTaggedField(value, MemOperand(object, index));
-      __ CheckPageFlagSet(object, scratch0,
-                          MemoryChunk::kPointersFromHereAreInterestingMask,
-                          ool->entry());
+      if (COMPRESS_POINTERS_BOOL) {
+        __ DecompressTaggedPointer(object, object);
+      }
+      __ CheckPageFlag(object, scratch0,
+                       MemoryChunk::kPointersFromHereAreInterestingMask, eq,
+                       ool->entry());
       __ Bind(ool->exit());
       break;
     }

src/compiler/backend/arm64/instruction-selector-arm64.cc

@@ -688,7 +688,7 @@ void InstructionSelector::VisitStore(Node* node) {
 
   // TODO(arm64): I guess this could be done in a better way.
   if (write_barrier_kind != kNoWriteBarrier) {
-    DCHECK(CanBeTaggedPointer(rep));
+    DCHECK(CanBeTaggedOrCompressedPointer(rep));
     AddressingMode addressing_mode;
     InstructionOperand inputs[3];
     size_t input_count = 0;

src/compiler/backend/x64/code-generator-x64.cc

@@ -247,6 +247,9 @@ class OutOfLineRecordWrite final : public OutOfLineCode {
     if (mode_ > RecordWriteMode::kValueIsPointer) {
       __ JumpIfSmi(value_, exit());
     }
+    if (COMPRESS_POINTERS_BOOL) {
+      __ DecompressTaggedPointer(value_, value_);
+    }
     __ CheckPageFlag(value_, scratch0_,
                      MemoryChunk::kPointersToHereAreInterestingMask, zero,
                      exit());
@@ -1018,6 +1021,9 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
           OutOfLineRecordWrite(this, object, operand, value, scratch0, scratch1,
                                mode, DetermineStubCallMode());
       __ StoreTaggedField(operand, value);
+      if (COMPRESS_POINTERS_BOOL) {
+        __ DecompressTaggedPointer(object, object);
+      }
       __ CheckPageFlag(object, scratch0,
                        MemoryChunk::kPointersFromHereAreInterestingMask,
                        not_zero, ool->entry());

src/compiler/backend/x64/instruction-selector-x64.cc

@@ -380,7 +380,7 @@ void InstructionSelector::VisitStore(Node* node) {
   WriteBarrierKind write_barrier_kind = store_rep.write_barrier_kind();
 
   if (write_barrier_kind != kNoWriteBarrier) {
-    DCHECK(CanBeTaggedPointer(store_rep.representation()));
+    DCHECK(CanBeTaggedOrCompressedPointer(store_rep.representation()));
     AddressingMode addressing_mode;
     InstructionOperand inputs[] = {
         g.UseUniqueRegister(base),

src/compiler/raw-machine-assembler.h

@@ -134,25 +134,24 @@ class V8_EXPORT_PRIVATE RawMachineAssembler {
              LoadSensitivity needs_poisoning = LoadSensitivity::kSafe) {
     // change_op is used below to change to the correct Tagged representation
     const Operator* change_op = nullptr;
-#ifdef V8_COMPRESS_POINTERS
-    switch (rep.representation()) {
-      case MachineRepresentation::kTaggedPointer:
-        rep = MachineType::CompressedPointer();
-        change_op = machine()->ChangeCompressedPointerToTaggedPointer();
-        break;
-      case MachineRepresentation::kTaggedSigned:
-        rep = MachineType::CompressedSigned();
-        change_op = machine()->ChangeCompressedSignedToTaggedSigned();
-        break;
-      case MachineRepresentation::kTagged:
-        rep = MachineType::AnyCompressed();
-        change_op = machine()->ChangeCompressedToTagged();
-        break;
-      default:
-        break;
+    if (COMPRESS_POINTERS_BOOL) {
+      switch (rep.representation()) {
+        case MachineRepresentation::kTaggedPointer:
+          rep = MachineType::CompressedPointer();
+          change_op = machine()->ChangeCompressedPointerToTaggedPointer();
+          break;
+        case MachineRepresentation::kTaggedSigned:
+          rep = MachineType::CompressedSigned();
+          change_op = machine()->ChangeCompressedSignedToTaggedSigned();
+          break;
+        case MachineRepresentation::kTagged:
+          rep = MachineType::AnyCompressed();
+          change_op = machine()->ChangeCompressedToTagged();
+          break;
+        default:
+          break;
+      }
     }
-#endif
-
     const Operator* op = machine()->Load(rep);
     CHECK_NE(PoisoningMitigationLevel::kPoisonAll, poisoning_level_);
     if (needs_poisoning == LoadSensitivity::kCritical &&
@@ -172,11 +171,51 @@ class V8_EXPORT_PRIVATE RawMachineAssembler {
   }
   Node* Store(MachineRepresentation rep, Node* base, Node* index, Node* value,
               WriteBarrierKind write_barrier) {
+    if (COMPRESS_POINTERS_BOOL) {
+      switch (rep) {
+        case MachineRepresentation::kTaggedPointer:
+          rep = MachineRepresentation::kCompressedPointer;
+          value = AddNode(machine()->ChangeTaggedPointerToCompressedPointer(),
+                          value);
+          break;
+        case MachineRepresentation::kTaggedSigned:
+          rep = MachineRepresentation::kCompressedSigned;
+          value =
+              AddNode(machine()->ChangeTaggedSignedToCompressedSigned(), value);
+          break;
+        case MachineRepresentation::kTagged:
+          rep = MachineRepresentation::kCompressed;
+          value = AddNode(machine()->ChangeTaggedToCompressed(), value);
+          break;
+        default:
+          break;
+      }
+    }
     return AddNode(machine()->Store(StoreRepresentation(rep, write_barrier)),
                    base, index, value);
   }
   void OptimizedStoreField(MachineRepresentation rep, Node* object, int offset,
                            Node* value, WriteBarrierKind write_barrier) {
+    if (COMPRESS_POINTERS_BOOL) {
+      switch (rep) {
+        case MachineRepresentation::kTaggedPointer:
+          rep = MachineRepresentation::kCompressedPointer;
+          value = AddNode(machine()->ChangeTaggedPointerToCompressedPointer(),
+                          value);
+          break;
+        case MachineRepresentation::kTaggedSigned:
+          rep = MachineRepresentation::kCompressedSigned;
+          value =
+              AddNode(machine()->ChangeTaggedSignedToCompressedSigned(), value);
+          break;
+        case MachineRepresentation::kTagged:
+          rep = MachineRepresentation::kCompressed;
+          value = AddNode(machine()->ChangeTaggedToCompressed(), value);
+          break;
+        default:
+          break;
+      }
+    }
     AddNode(simplified()->StoreField(FieldAccess(
                 BaseTaggedness::kTaggedBase, offset, MaybeHandle<Name>(),
                 MaybeHandle<Map>(), Type::Any(),

src/x64/macro-assembler-x64.cc

@@ -293,6 +293,14 @@ void TurboAssembler::DecompressTaggedSigned(Register destination,
   RecordComment("]");
 }
 
+void TurboAssembler::DecompressTaggedPointer(Register destination,
+                                             Register source) {
+  RecordComment("[ DecompressTaggedPointer");
+  movsxlq(destination, source);
+  addq(destination, kRootRegister);
+  RecordComment("]");
+}
+
 void TurboAssembler::DecompressTaggedPointer(Register destination,
                                              Operand field_operand) {
   RecordComment("[ DecompressTaggedPointer");

src/x64/macro-assembler-x64.h

@@ -510,6 +510,7 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
 
   // The following macros work even when pointer compression is not enabled.
   void DecompressTaggedSigned(Register destination, Operand field_operand);
+  void DecompressTaggedPointer(Register destination, Register source);
   void DecompressTaggedPointer(Register destination, Operand field_operand);
   void DecompressAnyTagged(Register destination, Operand field_operand,
                            Register scratch = kScratchRegister);