[ptr-compr] Adding the branchful/branchless decompression choice to codegen

I missed these cases when adding the branchful decompression on codegen.

Cq-Include-Trybots: luci.v8.try:v8_linux64_pointer_compression_rel_ng
Cq-Include-Trybots: luci.v8.try:v8_linux64_arm64_pointer_compression_rel_ng
Bug: v8:7703
Change-Id: Idb3f5ca81e00bb17fa08ba2b2506b642ffbd7b4b
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1571623
Commit-Queue: Santiago Aboy Solanes <solanes@chromium.org>
Reviewed-by: Igor Sheludko <ishell@chromium.org>
Reviewed-by: Jaroslav Sevcik <jarin@chromium.org>
Cr-Commit-Position: refs/heads/master@{#61045}

src/arm64/macro-assembler-arm64.cc

@@ -2801,10 +2801,10 @@ void TurboAssembler::DecompressTaggedSigned(const Register& destination,
   RecordComment("]");
 }
 
-void TurboAssembler::DecompressTaggedPointer(const Register& destination,
-                                             const Register& source) {
-  RecordComment("[ DecompressTaggedPointer");
-  Add(destination, kRootRegister, Operand(source, SXTW));
+void TurboAssembler::DecompressTaggedSigned(const Register& destination,
+                                            const Register& source) {
+  RecordComment("[ DecompressTaggedSigned");
+  Sxtw(destination, source);
   RecordComment("]");
 }
 
@@ -2816,6 +2816,13 @@ void TurboAssembler::DecompressTaggedPointer(const Register& destination,
   RecordComment("]");
 }
 
+void TurboAssembler::DecompressTaggedPointer(const Register& destination,
+                                             const Register& source) {
+  RecordComment("[ DecompressTaggedPointer");
+  Add(destination, kRootRegister, Operand(source, SXTW));
+  RecordComment("]");
+}
+
 void TurboAssembler::DecompressAnyTagged(const Register& destination,
                                          const MemOperand& field_operand) {
   RecordComment("[ DecompressAnyTagged");
@@ -2841,6 +2848,31 @@ void TurboAssembler::DecompressAnyTagged(const Register& destination,
   RecordComment("]");
 }
 
+void TurboAssembler::DecompressAnyTagged(const Register& destination,
+                                         const Register& source) {
+  RecordComment("[ DecompressAnyTagged");
+  if (kUseBranchlessPtrDecompression) {
+    UseScratchRegisterScope temps(this);
+    // Branchlessly compute |masked_root|:
+    // masked_root = HAS_SMI_TAG(destination) ? 0 : kRootRegister;
+    STATIC_ASSERT((kSmiTagSize == 1) && (kSmiTag == 0));
+    Register masked_root = temps.AcquireX();
+    // Sign extend tag bit to entire register.
+    Sbfx(masked_root, source, 0, kSmiTagSize);
+    And(masked_root, masked_root, kRootRegister);
+    // Now this add operation will either leave the value unchanged if it is a
+    // smi or add the isolate root if it is a heap object.
+    Add(destination, masked_root, Operand(source, SXTW));
+  } else {
+    Label done;
+    Sxtw(destination, source);
+    JumpIfSmi(destination, &done);
+    Add(destination, kRootRegister, destination);
+    bind(&done);
+  }
+  RecordComment("]");
+}
+
 void MacroAssembler::CompareAndSplit(const Register& lhs,
                                      const Operand& rhs,
                                      Condition cond,

src/arm64/macro-assembler-arm64.h

@@ -1196,12 +1196,15 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
 
   void DecompressTaggedSigned(const Register& destination,
                               const MemOperand& field_operand);
-  void DecompressTaggedPointer(const Register& destination,
-                               const Register& source);
+  void DecompressTaggedSigned(const Register& destination,
+                              const Register& source);
   void DecompressTaggedPointer(const Register& destination,
                                const MemOperand& field_operand);
+  void DecompressTaggedPointer(const Register& destination,
+                               const Register& source);
   void DecompressAnyTagged(const Register& destination,
                            const MemOperand& field_operand);
+  void DecompressAnyTagged(const Register& destination, const Register& source);
 
  protected:
   // The actual Push and Pop implementations. These don't generate any code

src/compiler/backend/arm64/code-generator-arm64.cc

@@ -1581,27 +1581,15 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
       __ Str(i.InputOrZeroRegister64(0), i.MemoryOperand(1));
       break;
     case kArm64DecompressSigned: {
-      __ Sxtw(i.OutputRegister(), i.InputRegister(0));
+      __ DecompressTaggedSigned(i.OutputRegister(), i.InputRegister(0));
       break;
     }
     case kArm64DecompressPointer: {
-      __ Add(i.OutputRegister(), kRootRegister,
-             Operand(i.InputRegister(0), SXTW));
+      __ DecompressTaggedPointer(i.OutputRegister(), i.InputRegister(0));
       break;
     }
     case kArm64DecompressAny: {
-      // TODO(solanes): Do branchful compute?
-      // Branchlessly compute |masked_root|:
-      STATIC_ASSERT((kSmiTagSize == 1) && (kSmiTag == 0));
-      UseScratchRegisterScope temps(tasm());
-      Register masked_root = temps.AcquireX();
-      // Sign extend tag bit to entire register.
-      __ Sbfx(masked_root, i.InputRegister(0), 0, kSmiTagSize);
-      __ And(masked_root, masked_root, kRootRegister);
-      // Now this add operation will either leave the value unchanged if it is a
-      // smi or add the isolate root if it is a heap object.
-      __ Add(i.OutputRegister(), masked_root,
-             Operand(i.InputRegister(0), SXTW));
+      __ DecompressAnyTagged(i.OutputRegister(), i.InputRegister(0));
       break;
     }
     // TODO(solanes): Combine into one Compress? They seem to be identical.

src/compiler/backend/x64/code-generator-x64.cc

@@ -1961,29 +1961,17 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
     }
     case kX64DecompressSigned: {
       CHECK(instr->HasOutput());
-      ASSEMBLE_MOVX(movsxlq);
+      ASSEMBLE_MOVX(DecompressTaggedSigned);
       break;
     }
     case kX64DecompressPointer: {
       CHECK(instr->HasOutput());
-      ASSEMBLE_MOVX(movsxlq);
-      __ addq(i.OutputRegister(), kRootRegister);
+      ASSEMBLE_MOVX(DecompressTaggedPointer);
       break;
     }
     case kX64DecompressAny: {
       CHECK(instr->HasOutput());
-      ASSEMBLE_MOVX(movsxlq);
-      // TODO(solanes): Do branchful compute?
-      // Branchlessly compute |masked_root|:
-      STATIC_ASSERT((kSmiTagSize == 1) && (kSmiTag < 32));
-      Register masked_root = kScratchRegister;
-      __ movl(masked_root, i.OutputRegister());
-      __ andl(masked_root, Immediate(kSmiTagMask));
-      __ negq(masked_root);
-      __ andq(masked_root, kRootRegister);
-      // Now this add operation will either leave the value unchanged if it is a
-      // smi or add the isolate root if it is a heap object.
-      __ addq(i.OutputRegister(), masked_root);
+      ASSEMBLE_MOVX(DecompressAnyTagged);
       break;
     }
     // TODO(solanes): Combine into one Compress? They seem to be identical.

src/x64/macro-assembler-x64.cc

@@ -293,11 +293,10 @@ void TurboAssembler::DecompressTaggedSigned(Register destination,
   RecordComment("]");
 }
 
-void TurboAssembler::DecompressTaggedPointer(Register destination,
-                                             Register source) {
-  RecordComment("[ DecompressTaggedPointer");
+void TurboAssembler::DecompressTaggedSigned(Register destination,
+                                            Register source) {
+  RecordComment("[ DecompressTaggedSigned");
   movsxlq(destination, source);
-  addq(destination, kRootRegister);
   RecordComment("]");
 }
 
@@ -309,12 +308,16 @@ void TurboAssembler::DecompressTaggedPointer(Register destination,
   RecordComment("]");
 }
 
-void TurboAssembler::DecompressAnyTagged(Register destination,
-                                         Operand field_operand,
-                                         Register scratch) {
-  DCHECK(!AreAliased(destination, scratch));
-  RecordComment("[ DecompressAnyTagged");
-  movsxlq(destination, field_operand);
+void TurboAssembler::DecompressTaggedPointer(Register destination,
+                                             Register source) {
+  RecordComment("[ DecompressTaggedPointer");
+  movsxlq(destination, source);
+  addq(destination, kRootRegister);
+  RecordComment("]");
+}
+
+void TurboAssembler::DecompressRegisterAnyTagged(Register destination,
+                                                 Register scratch) {
   if (kUseBranchlessPtrDecompression) {
     // Branchlessly compute |masked_root|:
     // masked_root = HAS_SMI_TAG(destination) ? 0 : kRootRegister;
@@ -333,6 +336,24 @@ void TurboAssembler::DecompressAnyTagged(Register destination,
     addq(destination, kRootRegister);
     bind(&done);
   }
+}
+
+void TurboAssembler::DecompressAnyTagged(Register destination,
+                                         Operand field_operand,
+                                         Register scratch) {
+  DCHECK(!AreAliased(destination, scratch));
+  RecordComment("[ DecompressAnyTagged");
+  movsxlq(destination, field_operand);
+  DecompressRegisterAnyTagged(destination, scratch);
+  RecordComment("]");
+}
+
+void TurboAssembler::DecompressAnyTagged(Register destination, Register source,
+                                         Register scratch) {
+  DCHECK(!AreAliased(destination, scratch));
+  RecordComment("[ DecompressAnyTagged");
+  movsxlq(destination, source);
+  DecompressRegisterAnyTagged(destination, scratch);
   RecordComment("]");
 }
 

src/x64/macro-assembler-x64.h

@@ -510,10 +510,16 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
 
   // The following macros work even when pointer compression is not enabled.
   void DecompressTaggedSigned(Register destination, Operand field_operand);
-  void DecompressTaggedPointer(Register destination, Register source);
+  void DecompressTaggedSigned(Register destination, Register source);
   void DecompressTaggedPointer(Register destination, Operand field_operand);
+  void DecompressTaggedPointer(Register destination, Register source);
+  // Auxiliary function used by DecompressAnyTagged to perform the actual
+  // decompression. Assumes destination is already signed extended.
+  void DecompressRegisterAnyTagged(Register destination, Register scratch);
   void DecompressAnyTagged(Register destination, Operand field_operand,
                            Register scratch = kScratchRegister);
+  void DecompressAnyTagged(Register destination, Register source,
+                           Register scratch = kScratchRegister);
 
  protected:
   static const int kSmiShift = kSmiTagSize + kSmiShiftSize;