[ptr-compr][codegen] Combine loads and decompressions with poison too

Also add the poison to x64 decompresses which were missing.

Cq-Include-Trybots: luci.v8.try:v8_linux64_pointer_compression_rel_ng
Cq-Include-Trybots: luci.v8.try:v8_linux64_arm64_pointer_compression_rel_ng
Bug: v8:7703
Change-Id: I87fd8e56ad8132b4996749be034093566bb88301
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1700061
Commit-Queue: Santiago Aboy Solanes <solanes@chromium.org>
Reviewed-by: Sigurd Schneider <sigurds@chromium.org>
Cr-Commit-Position: refs/heads/master@{#62743}

src/compiler/backend/arm64/instruction-selector-arm64.cc

@@ -1646,11 +1646,16 @@ void InstructionSelector::VisitChangeTaggedSignedToCompressedSigned(
 void InstructionSelector::VisitChangeCompressedToTagged(Node* node) {
   Arm64OperandGenerator g(this);
   Node* const value = node->InputAt(0);
-  if (value->opcode() == IrOpcode::kLoad && CanCover(node, value)) {
-    // TODO(v8:7703): Make this work with poisoned loads as well.
+  if ((value->opcode() == IrOpcode::kLoad ||
+       value->opcode() == IrOpcode::kPoisonedLoad) &&
+      CanCover(node, value)) {
     DCHECK_EQ(LoadRepresentationOf(value->op()).representation(),
               MachineRepresentation::kCompressed);
     InstructionCode opcode = kArm64LdrDecompressAnyTagged;
+    if (value->opcode() == IrOpcode::kPoisonedLoad) {
+      CHECK_NE(poisoning_level_, PoisoningMitigationLevel::kDontPoison);
+      opcode |= MiscField::encode(kMemoryAccessPoisoned);
+    }
     ImmediateMode immediate_mode = kLoadStoreImm32;
     MachineRepresentation rep = MachineRepresentation::kCompressed;
     EmitLoad(this, value, opcode, immediate_mode, rep, node);
@@ -1663,10 +1668,16 @@ void InstructionSelector::VisitChangeCompressedPointerToTaggedPointer(
     Node* node) {
   Arm64OperandGenerator g(this);
   Node* const value = node->InputAt(0);
-  if (value->opcode() == IrOpcode::kLoad && CanCover(node, value)) {
+  if ((value->opcode() == IrOpcode::kLoad ||
+       value->opcode() == IrOpcode::kPoisonedLoad) &&
+      CanCover(node, value)) {
     DCHECK_EQ(LoadRepresentationOf(value->op()).representation(),
               MachineRepresentation::kCompressedPointer);
     InstructionCode opcode = kArm64LdrDecompressTaggedPointer;
+    if (value->opcode() == IrOpcode::kPoisonedLoad) {
+      CHECK_NE(poisoning_level_, PoisoningMitigationLevel::kDontPoison);
+      opcode |= MiscField::encode(kMemoryAccessPoisoned);
+    }
     ImmediateMode immediate_mode = kLoadStoreImm32;
     MachineRepresentation rep = MachineRepresentation::kCompressedPointer;
     EmitLoad(this, value, opcode, immediate_mode, rep, node);
@@ -1680,10 +1691,16 @@ void InstructionSelector::VisitChangeCompressedSignedToTaggedSigned(
     Node* node) {
   Arm64OperandGenerator g(this);
   Node* const value = node->InputAt(0);
-  if (value->opcode() == IrOpcode::kLoad && CanCover(node, value)) {
+  if ((value->opcode() == IrOpcode::kLoad ||
+       value->opcode() == IrOpcode::kPoisonedLoad) &&
+      CanCover(node, value)) {
     DCHECK_EQ(LoadRepresentationOf(value->op()).representation(),
               MachineRepresentation::kCompressedSigned);
     InstructionCode opcode = kArm64LdrDecompressTaggedSigned;
+    if (value->opcode() == IrOpcode::kPoisonedLoad) {
+      CHECK_NE(poisoning_level_, PoisoningMitigationLevel::kDontPoison);
+      opcode |= MiscField::encode(kMemoryAccessPoisoned);
+    }
     ImmediateMode immediate_mode = kLoadStoreImm32;
     MachineRepresentation rep = MachineRepresentation::kCompressedSigned;
     EmitLoad(this, value, opcode, immediate_mode, rep, node);

src/compiler/backend/instruction-selector.h

@@ -608,6 +608,8 @@ class V8_EXPORT_PRIVATE InstructionSelector final {
   MACHINE_SIMD_OP_LIST(DECLARE_GENERATOR)
 #undef DECLARE_GENERATOR
 
+  // Visit the load node with a value and opcode to replace with.
+  void VisitLoad(Node* node, Node* value, InstructionCode opcode);
   void VisitFinishRegion(Node* node);
   void VisitParameter(Node* node);
   void VisitIfException(Node* node);

src/compiler/backend/x64/code-generator-x64.cc

@@ -1953,16 +1953,19 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
     case kX64MovqDecompressTaggedSigned: {
       CHECK(instr->HasOutput());
       __ DecompressTaggedSigned(i.OutputRegister(), i.MemoryOperand());
+      EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
       break;
     }
     case kX64MovqDecompressTaggedPointer: {
       CHECK(instr->HasOutput());
       __ DecompressTaggedPointer(i.OutputRegister(), i.MemoryOperand());
+      EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
       break;
     }
     case kX64MovqDecompressAnyTagged: {
       CHECK(instr->HasOutput());
       __ DecompressAnyTagged(i.OutputRegister(), i.MemoryOperand());
+      EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
       break;
     }
     case kX64MovqCompressTagged: {
@@ -1979,16 +1982,19 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
     case kX64DecompressSigned: {
       CHECK(instr->HasOutput());
       ASSEMBLE_MOVX(DecompressTaggedSigned);
+      EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
       break;
     }
     case kX64DecompressPointer: {
       CHECK(instr->HasOutput());
       ASSEMBLE_MOVX(DecompressTaggedPointer);
+      EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
       break;
     }
     case kX64DecompressAny: {
       CHECK(instr->HasOutput());
       ASSEMBLE_MOVX(DecompressAnyTagged);
+      EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
       break;
     }
     case kX64CompressSigned:   // Fall through.

src/compiler/backend/x64/instruction-selector-x64.cc

@@ -314,16 +314,14 @@ void InstructionSelector::VisitAbortCSAAssert(Node* node) {
   Emit(kArchAbortCSAAssert, g.NoOutput(), g.UseFixed(node->InputAt(0), rdx));
 }
 
-void InstructionSelector::VisitLoad(Node* node) {
-  LoadRepresentation load_rep = LoadRepresentationOf(node->op());
+void InstructionSelector::VisitLoad(Node* node, Node* value,
+                                    InstructionCode opcode) {
   X64OperandGenerator g(this);
-
-  ArchOpcode opcode = GetLoadOpcode(load_rep);
   InstructionOperand outputs[] = {g.DefineAsRegister(node)};
   InstructionOperand inputs[3];
   size_t input_count = 0;
   AddressingMode mode =
-      g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
+      g.GetEffectiveAddressMemoryOperand(value, inputs, &input_count);
   InstructionCode code = opcode | AddressingModeField::encode(mode);
   if (node->opcode() == IrOpcode::kProtectedLoad) {
     code |= MiscField::encode(kMemoryAccessProtected);
@@ -334,6 +332,11 @@ void InstructionSelector::VisitLoad(Node* node) {
   Emit(code, 1, outputs, input_count, inputs);
 }
 
+void InstructionSelector::VisitLoad(Node* node) {
+  LoadRepresentation load_rep = LoadRepresentationOf(node->op());
+  VisitLoad(node, node, GetLoadOpcode(load_rep));
+}
+
 void InstructionSelector::VisitPoisonedLoad(Node* node) { VisitLoad(node); }
 
 void InstructionSelector::VisitProtectedLoad(Node* node) { VisitLoad(node); }
@@ -1255,60 +1258,45 @@ void InstructionSelector::VisitChangeTaggedSignedToCompressedSigned(
 }
 
 void InstructionSelector::VisitChangeCompressedToTagged(Node* node) {
-  X64OperandGenerator g(this);
   Node* const value = node->InputAt(0);
-  if (value->opcode() == IrOpcode::kLoad && CanCover(node, value)) {
+  if ((value->opcode() == IrOpcode::kLoad ||
+       value->opcode() == IrOpcode::kPoisonedLoad) &&
+      CanCover(node, value)) {
     DCHECK_EQ(LoadRepresentationOf(value->op()).representation(),
               MachineRepresentation::kCompressed);
-    InstructionCode opcode = kX64MovqDecompressAnyTagged;
-    InstructionOperand outputs[] = {g.DefineAsRegister(node)};
-    size_t input_count = 0;
-    InstructionOperand inputs[3];
-    AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
-        node->InputAt(0), inputs, &input_count);
-    opcode |= AddressingModeField::encode(mode);
-    Emit(opcode, 1, outputs, input_count, inputs);
+    VisitLoad(node, value, kX64MovqDecompressAnyTagged);
   } else {
+    X64OperandGenerator g(this);
     Emit(kX64DecompressAny, g.DefineAsRegister(node), g.Use(value));
   }
 }
 
 void InstructionSelector::VisitChangeCompressedPointerToTaggedPointer(
     Node* node) {
-  X64OperandGenerator g(this);
   Node* const value = node->InputAt(0);
-  if (value->opcode() == IrOpcode::kLoad && CanCover(node, value)) {
+  if ((value->opcode() == IrOpcode::kLoad ||
+       value->opcode() == IrOpcode::kPoisonedLoad) &&
+      CanCover(node, value)) {
     DCHECK_EQ(LoadRepresentationOf(value->op()).representation(),
               MachineRepresentation::kCompressedPointer);
-    InstructionCode opcode = kX64MovqDecompressTaggedPointer;
-    InstructionOperand outputs[] = {g.DefineAsRegister(node)};
-    size_t input_count = 0;
-    InstructionOperand inputs[3];
-    AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
-        node->InputAt(0), inputs, &input_count);
-    opcode |= AddressingModeField::encode(mode);
-    Emit(opcode, 1, outputs, input_count, inputs);
+    VisitLoad(node, value, kX64MovqDecompressTaggedPointer);
   } else {
+    X64OperandGenerator g(this);
     Emit(kX64DecompressPointer, g.DefineAsRegister(node), g.Use(value));
   }
 }
 
 void InstructionSelector::VisitChangeCompressedSignedToTaggedSigned(
     Node* node) {
-  X64OperandGenerator g(this);
   Node* const value = node->InputAt(0);
-  if (value->opcode() == IrOpcode::kLoad && CanCover(node, value)) {
+  if ((value->opcode() == IrOpcode::kLoad ||
+       value->opcode() == IrOpcode::kPoisonedLoad) &&
+      CanCover(node, value)) {
     DCHECK_EQ(LoadRepresentationOf(value->op()).representation(),
               MachineRepresentation::kCompressedSigned);
-    InstructionCode opcode = kX64MovqDecompressTaggedSigned;
-    InstructionOperand outputs[] = {g.DefineAsRegister(node)};
-    size_t input_count = 0;
-    InstructionOperand inputs[3];
-    AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
-        node->InputAt(0), inputs, &input_count);
-    opcode |= AddressingModeField::encode(mode);
-    Emit(opcode, 1, outputs, input_count, inputs);
+    VisitLoad(node, value, kX64MovqDecompressTaggedSigned);
   } else {
+    X64OperandGenerator g(this);
     Emit(kX64DecompressSigned, g.DefineAsRegister(node), g.Use(value));
   }
 }