[codegen] Use separate TSANRelaxedStore stubs

Inline the SaveFPMode flag directly into the TSANRelaxedStore stubs:
 - Saves one register for input arguments
 - Avoid branches in the TSANRelaxedStore stubs

Bug: v8:7790, v8:11600
Change-Id: Ib1083f8c1a7e856028ff606ba8c2a93efb10db69
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2917037Reviewed-by: Ross McIlroy <rmcilroy@chromium.org>
Reviewed-by: Jakob Kummerow <jkummerow@chromium.org>
Reviewed-by: Jakob Gruber <jgruber@chromium.org>
Commit-Queue: Santiago Aboy Solanes <solanes@chromium.org>
Cr-Commit-Position: refs/heads/master@{#74781}

src/builtins/builtins-definitions.h

@@ -41,7 +41,8 @@ namespace internal {
   TFC(EphemeronKeyBarrierIgnoreFP, WriteBarrier)                               \
                                                                                \
   /* TSAN support for tagged stores in generated code.*/                       \
-  IF_TSAN(TFC, TSANRelaxedStore, TSANRelaxedStore)                             \
+  IF_TSAN(TFC, TSANRelaxedStoreIgnoreFP, TSANRelaxedStore)                     \
+  IF_TSAN(TFC, TSANRelaxedStoreSaveFP, TSANRelaxedStore)                       \
                                                                                \
   /* Adaptor for CPP builtin */                                                \
   TFC(AdaptorWithBuiltinExitFrame, CppBuiltinAdaptor)                          \

src/builtins/builtins-internal-gen.cc

@@ -414,41 +414,32 @@ class TSANRelaxedStoreCodeStubAssembler : public CodeStubAssembler {
   explicit TSANRelaxedStoreCodeStubAssembler(
       compiler::CodeAssemblerState* state)
       : CodeStubAssembler(state) {}
-
-  TNode<BoolT> ShouldSkipFPRegs(TNode<Smi> mode) {
-    return TaggedEqual(mode, SmiConstant(SaveFPRegsMode::kIgnore));
-  }
 };
 
-TF_BUILTIN(TSANRelaxedStore, TSANRelaxedStoreCodeStubAssembler) {
-  Label exit(this);
+TF_BUILTIN(TSANRelaxedStoreIgnoreFP, TSANRelaxedStoreCodeStubAssembler) {
   TNode<ExternalReference> function =
       ExternalConstant(ExternalReference::tsan_relaxed_store_function());
   auto address = UncheckedParameter<IntPtrT>(Descriptor::kAddress);
   TNode<IntPtrT> value =
       BitcastTaggedToWord(UncheckedParameter<Object>(Descriptor::kValue));
-  auto fp_mode = UncheckedParameter<Smi>(Descriptor::kFPMode);
-  Label dont_save_fp(this), save_fp(this);
-  Branch(ShouldSkipFPRegs(fp_mode), &dont_save_fp, &save_fp);
-  BIND(&dont_save_fp);
-  {
     CallCFunctionWithCallerSavedRegisters(
         function, MachineType::Int32(), SaveFPRegsMode::kIgnore,
         std::make_pair(MachineType::IntPtr(), address),
         std::make_pair(MachineType::IntPtr(), value));
-    Goto(&exit);
-  }
+    Return(UndefinedConstant());
+}
 
-  BIND(&save_fp);
-  {
-    CallCFunctionWithCallerSavedRegisters(
-        function, MachineType::Int32(), SaveFPRegsMode::kSave,
-        std::make_pair(MachineType::IntPtr(), address),
-        std::make_pair(MachineType::IntPtr(), value));
-    Goto(&exit);
-  }
-  BIND(&exit);
-  Return(TrueConstant());
+TF_BUILTIN(TSANRelaxedStoreSaveFP, TSANRelaxedStoreCodeStubAssembler) {
+  TNode<ExternalReference> function =
+      ExternalConstant(ExternalReference::tsan_relaxed_store_function());
+  auto address = UncheckedParameter<IntPtrT>(Descriptor::kAddress);
+  TNode<IntPtrT> value =
+      BitcastTaggedToWord(UncheckedParameter<Object>(Descriptor::kValue));
+  CallCFunctionWithCallerSavedRegisters(
+      function, MachineType::Int32(), SaveFPRegsMode::kSave,
+      std::make_pair(MachineType::IntPtr(), address),
+      std::make_pair(MachineType::IntPtr(), value));
+  Return(UndefinedConstant());
 }
 #endif  // V8_IS_TSAN
 

src/builtins/builtins.h

@@ -113,6 +113,14 @@ class Builtins {
     }
   }
 
+#ifdef V8_IS_TSAN
+  static Name GetTSANRelaxedStoreStub(SaveFPRegsMode fp_mode) {
+    return fp_mode == SaveFPRegsMode::kIgnore
+               ? Builtins::kTSANRelaxedStoreIgnoreFP
+               : Builtins::kTSANRelaxedStoreSaveFP;
+  }
+#endif  // V8_IS_TSAN
+
   // Convenience wrappers.
   Handle<Code> CallFunction(ConvertReceiverMode = ConvertReceiverMode::kAny);
   Handle<Code> Call(ConvertReceiverMode = ConvertReceiverMode::kAny);

src/codegen/interface-descriptors.h

@@ -1008,10 +1008,9 @@ class WriteBarrierDescriptor final
 class TSANRelaxedStoreDescriptor final
     : public StaticCallInterfaceDescriptor<TSANRelaxedStoreDescriptor> {
  public:
-  DEFINE_PARAMETERS_NO_CONTEXT(kAddress, kValue, kFPMode)
-  DEFINE_PARAMETER_TYPES(MachineType::Pointer(),       // kAddress
-                         MachineType::AnyTagged(),     // kValue
-                         MachineType::TaggedSigned())  // kFPMode
+  DEFINE_PARAMETERS_NO_CONTEXT(kAddress, kValue)
+  DEFINE_PARAMETER_TYPES(MachineType::Pointer(),    // kAddress
+                         MachineType::AnyTagged())  // kValue
 
   DECLARE_DESCRIPTOR(TSANRelaxedStoreDescriptor)
 

src/codegen/x64/interface-descriptors-x64-inl.h

@@ -27,7 +27,7 @@ constexpr auto WriteBarrierDescriptor::registers() {
 #ifdef V8_IS_TSAN
 // static
 constexpr auto TSANRelaxedStoreDescriptor::registers() {
-  return RegisterArray(arg_reg_1, arg_reg_2, arg_reg_3, kReturnRegister0);
+  return RegisterArray(arg_reg_1, arg_reg_2, kReturnRegister0);
 }
 #endif  // V8_IS_TSAN
 

src/codegen/x64/macro-assembler-x64.cc

@@ -473,7 +473,8 @@ void TurboAssembler::CallRecordWriteStub(
 #ifdef V8_IS_TSAN
 void TurboAssembler::CallTSANRelaxedStoreStub(Register address, Register value,
                                               SaveFPRegsMode fp_mode,
-                                              Address wasm_target) {
+                                              StubCallMode mode) {
+  DCHECK(!AreAliased(address, value));
   TSANRelaxedStoreDescriptor descriptor;
   RegList registers = descriptor.allocatable_registers();
 
@@ -483,22 +484,20 @@ void TurboAssembler::CallTSANRelaxedStoreStub(Register address, Register value,
       descriptor.GetRegisterParameter(TSANRelaxedStoreDescriptor::kAddress));
   Register value_parameter(
       descriptor.GetRegisterParameter(TSANRelaxedStoreDescriptor::kValue));
-  Register fp_mode_parameter(
-      descriptor.GetRegisterParameter(TSANRelaxedStoreDescriptor::kFPMode));
 
   // Prepare argument registers for calling RecordWrite
   // address_parameter   <= address
   // value_parameter <= value
   MovePair(address_parameter, address, value_parameter, value);
-  // fp_mode_parameter <= fp_mode
-  Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
 
-  if (wasm_target != kNullAddress) {
+  if (mode == StubCallMode::kCallWasmRuntimeStub) {
     // Use {near_call} for direct Wasm call within a module.
+    auto wasm_target = wasm::WasmCode::GetTSANRelaxedStoreStub(fp_mode);
     near_call(wasm_target, RelocInfo::WASM_STUB_CALL);
   } else {
+    auto builtin_index = Builtins::GetTSANRelaxedStoreStub(fp_mode);
     Handle<Code> code_target =
-        isolate()->builtins()->builtin_handle(Builtins::kTSANRelaxedStore);
+        isolate()->builtins()->builtin_handle(builtin_index);
     Call(code_target, RelocInfo::CODE_TARGET);
   }
 

src/codegen/x64/macro-assembler-x64.h

@@ -513,9 +513,9 @@ class V8_EXPORT_PRIVATE TurboAssembler : public SharedTurboAssembler {
       StubCallMode mode = StubCallMode::kCallBuiltinPointer);
 
 #ifdef V8_IS_TSAN
-  void CallTSANRelaxedStoreStub(Register address, Register value,
-                                SaveFPRegsMode fp_mode,
-                                Address wasm_target = kNullAddress);
+  void CallTSANRelaxedStoreStub(
+      Register address, Register value, SaveFPRegsMode fp_mode,
+      StubCallMode mode = StubCallMode::kCallBuiltinPointer);
 #endif  // V8_IS_TSAN
 
   void MoveNumber(Register dst, double value);

src/compiler/backend/x64/code-generator-x64.cc

@@ -335,6 +335,7 @@ class OutOfLineTSANRelaxedStore final : public OutOfLineCode {
         stub_mode_(stub_mode),
 #endif  // V8_ENABLE_WEBASSEMBLY
         zone_(gen->zone()) {
+    DCHECK(!AreAliased(value, scratch0));
   }
 
   void Generate() final {
@@ -349,7 +350,7 @@ class OutOfLineTSANRelaxedStore final : public OutOfLineCode {
       // Just encode the stub index. This will be patched when the code
       // is added to the native module and copied into wasm code space.
       __ CallTSANRelaxedStoreStub(scratch0_, value_, save_fp_mode,
-                                  wasm::WasmCode::kTSANRelaxedStore);
+                                  StubCallMode::kCallWasmRuntimeStub);
       return;
     }
 #endif  // V8_ENABLE_WEBASSEMBLY

src/debug/debug-evaluate.cc

@@ -1054,7 +1054,8 @@ static bool TransitivelyCalledBuiltinHasNoSideEffect(Builtins::Name caller,
     case Builtins::kToObject:
     case Builtins::kToString:
 #ifdef V8_IS_TSAN
-    case Builtins::kTSANRelaxedStore:
+    case Builtins::kTSANRelaxedStoreIgnoreFP:
+    case Builtins::kTSANRelaxedStoreSaveFP:
 #endif  // V8_IS_TSAN
     case Builtins::kWeakMapLookupHashIndex:
       return true;

src/wasm/wasm-code-manager.h

@@ -95,7 +95,8 @@ struct WasmModule;
   V(RecordWriteEmitRememberedSetIgnoreFP) \
   V(RecordWriteOmitRememberedSetIgnoreFP) \
   V(ToNumber)                             \
-  IF_TSAN(V, TSANRelaxedStore)            \
+  IF_TSAN(V, TSANRelaxedStoreIgnoreFP)    \
+  IF_TSAN(V, TSANRelaxedStoreSaveFP)      \
   V(WasmAllocateArrayWithRtt)             \
   V(WasmAllocateRtt)                      \
   V(WasmAllocateStructWithRtt)            \
@@ -172,6 +173,14 @@ class V8_EXPORT_PRIVATE WasmCode final {
     }
   }
 
+#ifdef V8_IS_TSAN
+  static RuntimeStubId GetTSANRelaxedStoreStub(SaveFPRegsMode fp_mode) {
+    return fp_mode == SaveFPRegsMode::kIgnore
+               ? RuntimeStubId::kTSANRelaxedStoreIgnoreFP
+               : RuntimeStubId::kTSANRelaxedStoreSaveFP;
+  }
+#endif  // V8_IS_TSAN
+
   Vector<byte> instructions() const {
     return VectorOf(instructions_, static_cast<size_t>(instructions_size_));
   }