[codegen] Use builtin calls for TSANRelaxedStore

Instead of calling the C function directly from codegen, we call a
builtin that calls the C function. This is done to encapsulate the
push/pop registers in the code in the builtin.

Bug: v8:7790, v8:11600
Change-Id: I4c77a80803d4eb44526b716901afe0e8ccbe077d
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2892663Reviewed-by: Georg Neis <neis@chromium.org>
Reviewed-by: Jakob Kummerow <jkummerow@chromium.org>
Reviewed-by: Ross McIlroy <rmcilroy@chromium.org>
Commit-Queue: Santiago Aboy Solanes <solanes@chromium.org>
Cr-Commit-Position: refs/heads/master@{#74599}

src/builtins/builtins-definitions.h

@@ -36,6 +36,9 @@ namespace internal {
   TFC(RecordWrite, RecordWrite)                                                \
   TFC(EphemeronKeyBarrier, EphemeronKeyBarrier)                                \
                                                                                \
+  /* TSAN support for tagged stores in generated code.*/                       \
+  IF_TSAN(TFC, TSANRelaxedStore, TSANRelaxedStore)                             \
+                                                                               \
   /* Adaptor for CPP builtin */                                                \
   TFC(AdaptorWithBuiltinExitFrame, CppBuiltinAdaptor)                          \
                                                                                \

src/builtins/builtins-internal-gen.cc

@@ -408,6 +408,50 @@ TF_BUILTIN(RecordWrite, RecordWriteCodeStubAssembler) {
   Return(TrueConstant());
 }
 
+#ifdef V8_IS_TSAN
+class TSANRelaxedStoreCodeStubAssembler : public CodeStubAssembler {
+ public:
+  explicit TSANRelaxedStoreCodeStubAssembler(
+      compiler::CodeAssemblerState* state)
+      : CodeStubAssembler(state) {}
+
+  TNode<BoolT> ShouldSkipFPRegs(TNode<Smi> mode) {
+    return TaggedEqual(mode, SmiConstant(SaveFPRegsMode::kIgnore));
+  }
+};
+
+TF_BUILTIN(TSANRelaxedStore, TSANRelaxedStoreCodeStubAssembler) {
+  Label exit(this);
+  TNode<ExternalReference> function =
+      ExternalConstant(ExternalReference::tsan_relaxed_store_function());
+  auto address = UncheckedParameter<IntPtrT>(Descriptor::kAddress);
+  TNode<IntPtrT> value =
+      BitcastTaggedToWord(UncheckedParameter<Object>(Descriptor::kValue));
+  auto fp_mode = UncheckedParameter<Smi>(Descriptor::kFPMode);
+  Label dont_save_fp(this), save_fp(this);
+  Branch(ShouldSkipFPRegs(fp_mode), &dont_save_fp, &save_fp);
+  BIND(&dont_save_fp);
+  {
+    CallCFunctionWithCallerSavedRegisters(
+        function, MachineType::Int32(), SaveFPRegsMode::kIgnore,
+        std::make_pair(MachineType::IntPtr(), address),
+        std::make_pair(MachineType::IntPtr(), value));
+    Goto(&exit);
+  }
+
+  BIND(&save_fp);
+  {
+    CallCFunctionWithCallerSavedRegisters(
+        function, MachineType::Int32(), SaveFPRegsMode::kSave,
+        std::make_pair(MachineType::IntPtr(), address),
+        std::make_pair(MachineType::IntPtr(), value));
+    Goto(&exit);
+  }
+  BIND(&exit);
+  Return(TrueConstant());
+}
+#endif  // V8_IS_TSAN
+
 TF_BUILTIN(EphemeronKeyBarrier, RecordWriteCodeStubAssembler) {
   Label exit(this);
 

src/codegen/interface-descriptors.h

@@ -113,6 +113,7 @@ namespace internal {
   V(StringAt)                            \
   V(StringAtAsString)                    \
   V(StringSubstring)                     \
+  IF_TSAN(V, TSANRelaxedStore)           \
   V(TypeConversion)                      \
   V(TypeConversionNoContext)             \
   V(TypeConversion_Baseline)             \
@@ -1007,6 +1008,22 @@ class RecordWriteDescriptor final
   static constexpr bool kRestrictAllocatableRegisters = true;
 };
 
+#ifdef V8_IS_TSAN
+class TSANRelaxedStoreDescriptor final
+    : public StaticCallInterfaceDescriptor<TSANRelaxedStoreDescriptor> {
+ public:
+  DEFINE_PARAMETERS_NO_CONTEXT(kAddress, kValue, kFPMode)
+  DEFINE_PARAMETER_TYPES(MachineType::Pointer(),       // kAddress
+                         MachineType::AnyTagged(),     // kValue
+                         MachineType::TaggedSigned())  // kFPMode
+
+  DECLARE_DESCRIPTOR(TSANRelaxedStoreDescriptor)
+
+  static constexpr auto registers();
+  static constexpr bool kRestrictAllocatableRegisters = true;
+};
+#endif  // V8_IS_TSAN
+
 class EphemeronKeyBarrierDescriptor final
     : public StaticCallInterfaceDescriptor<EphemeronKeyBarrierDescriptor> {
  public:

src/codegen/x64/interface-descriptors-x64-inl.h

@@ -24,6 +24,13 @@ constexpr auto RecordWriteDescriptor::registers() {
                        kReturnRegister0);
 }
 
+#ifdef V8_IS_TSAN
+// static
+constexpr auto TSANRelaxedStoreDescriptor::registers() {
+  return RegisterArray(arg_reg_1, arg_reg_2, arg_reg_3, kReturnRegister0);
+}
+#endif  // V8_IS_TSAN
+
 // static
 constexpr auto DynamicCheckMapsDescriptor::registers() {
   return RegisterArray(kReturnRegister0, arg_reg_1, arg_reg_2, arg_reg_3,

src/codegen/x64/macro-assembler-x64.cc

@@ -493,6 +493,42 @@ void TurboAssembler::CallRecordWriteStub(
   RestoreRegisters(registers);
 }
 
+#ifdef V8_IS_TSAN
+void TurboAssembler::CallTSANRelaxedStoreStub(Register address, Register value,
+                                              SaveFPRegsMode fp_mode,
+                                              Address wasm_target) {
+  TSANRelaxedStoreDescriptor descriptor;
+  RegList registers = descriptor.allocatable_registers();
+
+  SaveRegisters(registers);
+
+  Register address_parameter(
+      descriptor.GetRegisterParameter(TSANRelaxedStoreDescriptor::kAddress));
+  Register value_parameter(
+      descriptor.GetRegisterParameter(TSANRelaxedStoreDescriptor::kValue));
+  Register fp_mode_parameter(
+      descriptor.GetRegisterParameter(TSANRelaxedStoreDescriptor::kFPMode));
+
+  // Prepare argument registers for calling RecordWrite
+  // address_parameter   <= address
+  // value_parameter <= value
+  MovePair(address_parameter, address, value_parameter, value);
+  // fp_mode_parameter <= fp_mode
+  Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
+
+  if (wasm_target != kNullAddress) {
+    // Use {near_call} for direct Wasm call within a module.
+    near_call(wasm_target, RelocInfo::WASM_STUB_CALL);
+  } else {
+    Handle<Code> code_target =
+        isolate()->builtins()->builtin_handle(Builtins::kTSANRelaxedStore);
+    Call(code_target, RelocInfo::CODE_TARGET);
+  }
+
+  RestoreRegisters(registers);
+}
+#endif  // V8_IS_TSAN
+
 void MacroAssembler::RecordWrite(Register object, Register address,
                                  Register value, SaveFPRegsMode fp_mode,
                                  RememberedSetAction remembered_set_action,

src/codegen/x64/macro-assembler-x64.h

@@ -513,6 +513,12 @@ class V8_EXPORT_PRIVATE TurboAssembler : public SharedTurboAssembler {
   void CallEphemeronKeyBarrier(Register object, Register address,
                                SaveFPRegsMode fp_mode);
 
+#ifdef V8_IS_TSAN
+  void CallTSANRelaxedStoreStub(Register address, Register value,
+                                SaveFPRegsMode fp_mode,
+                                Address wasm_target = kNullAddress);
+#endif  // V8_IS_TSAN
+
   void MoveNumber(Register dst, double value);
   void MoveNonSmi(Register dst, double value);
 

src/compiler/backend/x64/code-generator-x64.cc

@@ -321,31 +321,47 @@ class OutOfLineRecordWrite final : public OutOfLineCode {
 };
 
 #ifdef V8_IS_TSAN
-class OutOfLineTSANWrite final : public OutOfLineCode {
+class OutOfLineTSANRelaxedStore final : public OutOfLineCode {
  public:
-  OutOfLineTSANWrite(CodeGenerator* gen, Operand operand, Register value)
+  OutOfLineTSANRelaxedStore(CodeGenerator* gen, Operand operand, Register value,
+                            Register scratch0, StubCallMode stub_mode)
       : OutOfLineCode(gen),
         operand_(operand),
         value_(value),
-        zone_(gen->zone()) {}
+        scratch0_(scratch0),
+#if V8_ENABLE_WEBASSEMBLY
+        stub_mode_(stub_mode),
+#endif  // V8_ENABLE_WEBASSEMBLY
+        zone_(gen->zone()) {
+  }
 
   void Generate() final {
     const SaveFPRegsMode save_fp_mode = frame()->DidAllocateDoubleRegisters()
                                             ? SaveFPRegsMode::kSave
                                             : SaveFPRegsMode::kIgnore;
-    const int number_of_args = 2;
-    __ PushCallerSaved(save_fp_mode);
-    __ PrepareCallCFunction(number_of_args);
-    __ leaq(arg_reg_1, operand_);
-    __ movq(arg_reg_2, value_);
-    __ CallCFunction(ExternalReference::tsan_relaxed_store_function(),
-                     number_of_args);
-    __ PopCallerSaved(save_fp_mode);
+    __ leaq(scratch0_, operand_);
+
+#if V8_ENABLE_WEBASSEMBLY
+    if (stub_mode_ == StubCallMode::kCallWasmRuntimeStub) {
+      // A direct call to a wasm runtime stub defined in this module.
+      // Just encode the stub index. This will be patched when the code
+      // is added to the native module and copied into wasm code space.
+      __ CallTSANRelaxedStoreStub(scratch0_, value_, save_fp_mode,
+                                  wasm::WasmCode::kTSANRelaxedStore);
+      return;
+    }
+#endif  // V8_ENABLE_WEBASSEMBLY
+
+    __ CallTSANRelaxedStoreStub(scratch0_, value_, save_fp_mode);
   }
 
  private:
   Operand const operand_;
   Register const value_;
+  Register const scratch0_;
+#if V8_ENABLE_WEBASSEMBLY
+  StubCallMode const stub_mode_;
+#endif  // V8_ENABLE_WEBASSEMBLY
   Zone* zone_;
 };
 #endif  // V8_IS_TSAN
@@ -1214,14 +1230,15 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
       size_t index = 0;
       Operand operand = i.MemoryOperand(&index);
       Register value = i.InputRegister(index);
+      Register scratch0 = i.TempRegister(0);
 
 #ifdef V8_IS_TSAN
-      auto tsan_ool = zone()->New<OutOfLineTSANWrite>(this, operand, value);
+      auto tsan_ool = zone()->New<OutOfLineTSANRelaxedStore>(
+          this, operand, value, scratch0, DetermineStubCallMode());
       __ jmp(tsan_ool->entry());
       __ bind(tsan_ool->exit());
 #endif  // V8_IS_TSAN
 
-      Register scratch0 = i.TempRegister(0);
       Register scratch1 = i.TempRegister(1);
       auto ool = zone()->New<OutOfLineRecordWrite>(this, object, operand, value,
                                                    scratch0, scratch1, mode,

src/debug/debug-evaluate.cc

@@ -1038,6 +1038,9 @@ static bool TransitivelyCalledBuiltinHasNoSideEffect(Builtins::Name caller,
     case Builtins::kToName:
     case Builtins::kToObject:
     case Builtins::kToString:
+#ifdef V8_IS_TSAN
+    case Builtins::kTSANRelaxedStore:
+#endif  // V8_IS_TSAN
     case Builtins::kWeakMapLookupHashIndex:
       return true;
     case Builtins::kJoinStackPop:

src/wasm/wasm-code-manager.h

@@ -92,6 +92,7 @@ struct WasmModule;
   V(I64ToBigInt)                         \
   V(RecordWrite)                         \
   V(ToNumber)                            \
+  IF_TSAN(V, TSANRelaxedStore)           \
   V(WasmAllocateArrayWithRtt)            \
   V(WasmAllocateRtt)                     \
   V(WasmAllocateStructWithRtt)           \