[compiler] Add TSAN support for generated code movb and movw

This is the last CL of the code generated stores.

Bug: v8:7790, v8:11600
Change-Id: If8bbabb422027f938c7acc0bdc12a233dfed580e
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2950760
Commit-Queue: Santiago Aboy Solanes <solanes@chromium.org>
Reviewed-by: Jakob Gruber <jgruber@chromium.org>
Reviewed-by: Andreas Haas <ahaas@chromium.org>
Cr-Commit-Position: refs/heads/master@{#75121}

src/builtins/builtins-definitions.h

@@ -40,7 +40,11 @@ namespace internal {
   TFC(EphemeronKeyBarrierSaveFP, WriteBarrier)                                 \
   TFC(EphemeronKeyBarrierIgnoreFP, WriteBarrier)                               \
                                                                                \
-  /* TSAN support for tagged stores in generated code.*/                       \
+  /* TSAN support for stores in generated code.*/                              \
+  IF_TSAN(TFC, TSANRelaxedStore8IgnoreFP, TSANRelaxedStore)                    \
+  IF_TSAN(TFC, TSANRelaxedStore8SaveFP, TSANRelaxedStore)                      \
+  IF_TSAN(TFC, TSANRelaxedStore16IgnoreFP, TSANRelaxedStore)                   \
+  IF_TSAN(TFC, TSANRelaxedStore16SaveFP, TSANRelaxedStore)                     \
   IF_TSAN(TFC, TSANRelaxedStore32IgnoreFP, TSANRelaxedStore)                   \
   IF_TSAN(TFC, TSANRelaxedStore32SaveFP, TSANRelaxedStore)                     \
   IF_TSAN(TFC, TSANRelaxedStore64IgnoreFP, TSANRelaxedStore)                   \

src/builtins/builtins-internal-gen.cc

@@ -421,7 +421,13 @@ class TSANRelaxedStoreCodeStubAssembler : public CodeStubAssembler {
       : CodeStubAssembler(state) {}
 
   TNode<ExternalReference> GetExternalReference(int size) {
-    if (size == kInt32Size) {
+    if (size == kInt8Size) {
+      return ExternalConstant(
+          ExternalReference::tsan_relaxed_store_function_8_bits());
+    } else if (size == kInt16Size) {
+      return ExternalConstant(
+          ExternalReference::tsan_relaxed_store_function_16_bits());
+    } else if (size == kInt32Size) {
       return ExternalConstant(
           ExternalReference::tsan_relaxed_store_function_32_bits());
     } else {
@@ -445,6 +451,22 @@ class TSANRelaxedStoreCodeStubAssembler : public CodeStubAssembler {
   }
 };
 
+TF_BUILTIN(TSANRelaxedStore8IgnoreFP, TSANRelaxedStoreCodeStubAssembler) {
+  GenerateTSANRelaxedStore(SaveFPRegsMode::kIgnore, kInt8Size);
+}
+
+TF_BUILTIN(TSANRelaxedStore8SaveFP, TSANRelaxedStoreCodeStubAssembler) {
+  GenerateTSANRelaxedStore(SaveFPRegsMode::kSave, kInt8Size);
+}
+
+TF_BUILTIN(TSANRelaxedStore16IgnoreFP, TSANRelaxedStoreCodeStubAssembler) {
+  GenerateTSANRelaxedStore(SaveFPRegsMode::kIgnore, kInt16Size);
+}
+
+TF_BUILTIN(TSANRelaxedStore16SaveFP, TSANRelaxedStoreCodeStubAssembler) {
+  GenerateTSANRelaxedStore(SaveFPRegsMode::kSave, kInt16Size);
+}
+
 TF_BUILTIN(TSANRelaxedStore32IgnoreFP, TSANRelaxedStoreCodeStubAssembler) {
   GenerateTSANRelaxedStore(SaveFPRegsMode::kIgnore, kInt32Size);
 }

src/builtins/builtins.h

@@ -117,7 +117,15 @@ class Builtins {
 
 #ifdef V8_IS_TSAN
   static Builtin GetTSANRelaxedStoreStub(SaveFPRegsMode fp_mode, int size) {
-    if (size == kInt32Size) {
+    if (size == kInt8Size) {
+      return fp_mode == SaveFPRegsMode::kIgnore
+                 ? Builtin::kTSANRelaxedStore8IgnoreFP
+                 : Builtin::kTSANRelaxedStore8SaveFP;
+    } else if (size == kInt16Size) {
+      return fp_mode == SaveFPRegsMode::kIgnore
+                 ? Builtin::kTSANRelaxedStore16IgnoreFP
+                 : Builtin::kTSANRelaxedStore16SaveFP;
+    } else if (size == kInt32Size) {
       return fp_mode == SaveFPRegsMode::kIgnore
                  ? Builtin::kTSANRelaxedStore32IgnoreFP
                  : Builtin::kTSANRelaxedStore32SaveFP;

src/codegen/external-reference.cc

@@ -1161,13 +1161,32 @@ FUNCTION_REFERENCE(atomic_pair_compare_exchange_function,
                    atomic_pair_compare_exchange)
 
 #ifdef V8_IS_TSAN
+namespace {
 // Mimics the store in generated code by having a relaxed store to the same
 // address, with the same value. This is done in order for TSAN to see these
 // stores from generated code.
 // Note that {value} is an int64_t irrespective of the store size. This is on
 // purpose to keep the function signatures the same accross stores. The
 // static_cast inside the method will ignore the bits which will not be stored.
-static void tsan_relaxed_store_32_bits(Address addr, int64_t value) {
+void tsan_relaxed_store_8_bits(Address addr, int64_t value) {
+#if V8_TARGET_ARCH_X64
+  base::Relaxed_Store(reinterpret_cast<base::Atomic8*>(addr),
+                      static_cast<base::Atomic8>(value));
+#else
+  UNREACHABLE();
+#endif  // V8_TARGET_ARCH_X64
+}
+
+void tsan_relaxed_store_16_bits(Address addr, int64_t value) {
+#if V8_TARGET_ARCH_X64
+  base::Relaxed_Store(reinterpret_cast<base::Atomic16*>(addr),
+                      static_cast<base::Atomic16>(value));
+#else
+  UNREACHABLE();
+#endif  // V8_TARGET_ARCH_X64
+}
+
+void tsan_relaxed_store_32_bits(Address addr, int64_t value) {
 #if V8_TARGET_ARCH_X64
     base::Relaxed_Store(reinterpret_cast<base::Atomic32*>(addr),
                         static_cast<base::Atomic32>(value));
@@ -1176,7 +1195,7 @@ static void tsan_relaxed_store_32_bits(Address addr, int64_t value) {
 #endif  // V8_TARGET_ARCH_X64
 }
 
-static void tsan_relaxed_store_64_bits(Address addr, int64_t value) {
+void tsan_relaxed_store_64_bits(Address addr, int64_t value) {
 #if V8_TARGET_ARCH_X64
   base::Relaxed_Store(reinterpret_cast<base::Atomic64*>(addr),
                       static_cast<base::Atomic64>(value));
@@ -1184,9 +1203,13 @@ static void tsan_relaxed_store_64_bits(Address addr, int64_t value) {
   UNREACHABLE();
 #endif  // V8_TARGET_ARCH_X64
 }
-
+}  // namespace
 #endif  // V8_IS_TSAN
 
+IF_TSAN(FUNCTION_REFERENCE, tsan_relaxed_store_function_8_bits,
+        tsan_relaxed_store_8_bits)
+IF_TSAN(FUNCTION_REFERENCE, tsan_relaxed_store_function_16_bits,
+        tsan_relaxed_store_16_bits)
 IF_TSAN(FUNCTION_REFERENCE, tsan_relaxed_store_function_32_bits,
         tsan_relaxed_store_32_bits)
 IF_TSAN(FUNCTION_REFERENCE, tsan_relaxed_store_function_64_bits,

src/codegen/external-reference.h

@@ -266,6 +266,10 @@ class StatsCounter;
   V(atomic_pair_exchange_function, "atomic_pair_exchange_function")            \
   V(atomic_pair_compare_exchange_function,                                     \
     "atomic_pair_compare_exchange_function")                                   \
+  IF_TSAN(V, tsan_relaxed_store_function_8_bits,                               \
+          "tsan_relaxed_store_function_8_bits")                                \
+  IF_TSAN(V, tsan_relaxed_store_function_16_bits,                              \
+          "tsan_relaxed_store_function_16_bits")                               \
   IF_TSAN(V, tsan_relaxed_store_function_32_bits,                              \
           "tsan_relaxed_store_function_32_bits")                               \
   IF_TSAN(V, tsan_relaxed_store_function_64_bits,                              \

src/common/globals.h

@@ -183,10 +183,12 @@ constexpr int kMinInt31 = kMinInt / 2;
 constexpr uint32_t kMaxUInt32 = 0xFFFFFFFFu;
 constexpr int kMinUInt32 = 0;
 
+constexpr int kInt8Size = sizeof(int8_t);
 constexpr int kUInt8Size = sizeof(uint8_t);
 constexpr int kByteSize = sizeof(byte);
 constexpr int kCharSize = sizeof(char);
 constexpr int kShortSize = sizeof(short);  // NOLINT
+constexpr int kInt16Size = sizeof(int16_t);
 constexpr int kUInt16Size = sizeof(uint16_t);
 constexpr int kIntSize = sizeof(int);
 constexpr int kInt32Size = sizeof(int32_t);

src/compiler/backend/x64/code-generator-x64.cc

@@ -2136,9 +2136,15 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
       size_t index = 0;
       Operand operand = i.MemoryOperand(&index);
       if (HasImmediateInput(instr, index)) {
-        __ movb(operand, Immediate(i.InputInt8(index)));
+        Immediate value(Immediate(i.InputInt8(index)));
+        __ movb(operand, value);
+        EmitTSANStoreOOLIfNeeded(zone(), this, tasm(), operand, value, i,
+                                 DetermineStubCallMode(), kInt8Size);
       } else {
-        __ movb(operand, i.InputRegister(index));
+        Register value(i.InputRegister(index));
+        __ movb(operand, value);
+        EmitTSANStoreOOLIfNeeded(zone(), this, tasm(), operand, value, i,
+                                 DetermineStubCallMode(), kInt8Size);
       }
       EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
       break;
@@ -2170,9 +2176,15 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
       size_t index = 0;
       Operand operand = i.MemoryOperand(&index);
       if (HasImmediateInput(instr, index)) {
-        __ movw(operand, Immediate(i.InputInt16(index)));
+        Immediate value(Immediate(i.InputInt16(index)));
+        __ movw(operand, value);
+        EmitTSANStoreOOLIfNeeded(zone(), this, tasm(), operand, value, i,
+                                 DetermineStubCallMode(), kInt16Size);
       } else {
-        __ movw(operand, i.InputRegister(index));
+        Register value(i.InputRegister(index));
+        __ movw(operand, value);
+        EmitTSANStoreOOLIfNeeded(zone(), this, tasm(), operand, value, i,
+                                 DetermineStubCallMode(), kInt16Size);
       }
       EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i);
       break;
@@ -2194,12 +2206,12 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
         size_t index = 0;
         Operand operand = i.MemoryOperand(&index);
         if (HasImmediateInput(instr, index)) {
-          Immediate value = i.InputImmediate(index);
+          Immediate value(i.InputImmediate(index));
           __ movl(operand, value);
           EmitTSANStoreOOLIfNeeded(zone(), this, tasm(), operand, value, i,
                                    DetermineStubCallMode(), kInt32Size);
         } else {
-          Register value = i.InputRegister(index);
+          Register value(i.InputRegister(index));
           __ movl(operand, value);
           EmitTSANStoreOOLIfNeeded(zone(), this, tasm(), operand, value, i,
                                    DetermineStubCallMode(), kInt32Size);
@@ -2235,12 +2247,12 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
       size_t index = 0;
       Operand operand = i.MemoryOperand(&index);
       if (HasImmediateInput(instr, index)) {
-        Immediate value = i.InputImmediate(index);
+        Immediate value(i.InputImmediate(index));
         __ StoreTaggedField(operand, value);
         EmitTSANStoreOOLIfNeeded(zone(), this, tasm(), operand, value, i,
                                  DetermineStubCallMode(), kTaggedSize);
       } else {
-        Register value = i.InputRegister(index);
+        Register value(i.InputRegister(index));
         __ StoreTaggedField(operand, value);
         EmitTSANStoreOOLIfNeeded(zone(), this, tasm(), operand, value, i,
                                  DetermineStubCallMode(), kTaggedSize);
@@ -2255,12 +2267,12 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
         size_t index = 0;
         Operand operand = i.MemoryOperand(&index);
         if (HasImmediateInput(instr, index)) {
-          Immediate value = i.InputImmediate(index);
+          Immediate value(i.InputImmediate(index));
           __ movq(operand, value);
           EmitTSANStoreOOLIfNeeded(zone(), this, tasm(), operand, value, i,
                                    DetermineStubCallMode(), kInt64Size);
         } else {
-          Register value = i.InputRegister(index);
+          Register value(i.InputRegister(index));
           __ movq(operand, value);
           EmitTSANStoreOOLIfNeeded(zone(), this, tasm(), operand, value, i,
                                    DetermineStubCallMode(), kInt64Size);

src/debug/debug-evaluate.cc

@@ -1055,6 +1055,10 @@ static bool TransitivelyCalledBuiltinHasNoSideEffect(Builtin caller,
     case Builtin::kToObject:
     case Builtin::kToString:
 #ifdef V8_IS_TSAN
+    case Builtin::kTSANRelaxedStore8IgnoreFP:
+    case Builtin::kTSANRelaxedStore8SaveFP:
+    case Builtin::kTSANRelaxedStore16IgnoreFP:
+    case Builtin::kTSANRelaxedStore16SaveFP:
     case Builtin::kTSANRelaxedStore32IgnoreFP:
     case Builtin::kTSANRelaxedStore32SaveFP:
     case Builtin::kTSANRelaxedStore64IgnoreFP:

src/snapshot/startup-serializer.cc

@@ -106,6 +106,10 @@ bool IsUnexpectedCodeObject(Isolate* isolate, HeapObject obj) {
     case Builtin::kRecordWriteEmitRememberedSetIgnoreFP:
     case Builtin::kRecordWriteOmitRememberedSetIgnoreFP:
 #ifdef V8_IS_TSAN
+    case Builtin::kTSANRelaxedStore8IgnoreFP:
+    case Builtin::kTSANRelaxedStore8SaveFP:
+    case Builtin::kTSANRelaxedStore16IgnoreFP:
+    case Builtin::kTSANRelaxedStore16SaveFP:
     case Builtin::kTSANRelaxedStore32IgnoreFP:
     case Builtin::kTSANRelaxedStore32SaveFP:
     case Builtin::kTSANRelaxedStore64IgnoreFP:

src/wasm/wasm-code-manager.h

@@ -95,6 +95,10 @@ struct WasmModule;
   V(RecordWriteEmitRememberedSetIgnoreFP) \
   V(RecordWriteOmitRememberedSetIgnoreFP) \
   V(ToNumber)                             \
+  IF_TSAN(V, TSANRelaxedStore8IgnoreFP)   \
+  IF_TSAN(V, TSANRelaxedStore8SaveFP)     \
+  IF_TSAN(V, TSANRelaxedStore16IgnoreFP)  \
+  IF_TSAN(V, TSANRelaxedStore16SaveFP)    \
   IF_TSAN(V, TSANRelaxedStore32IgnoreFP)  \
   IF_TSAN(V, TSANRelaxedStore32SaveFP)    \
   IF_TSAN(V, TSANRelaxedStore64IgnoreFP)  \
@@ -181,7 +185,15 @@ class V8_EXPORT_PRIVATE WasmCode final {
 #ifdef V8_IS_TSAN
   static RuntimeStubId GetTSANRelaxedStoreStub(SaveFPRegsMode fp_mode,
                                                int size) {
-    if (size == kInt32Size) {
+    if (size == kInt8Size) {
+      return fp_mode == SaveFPRegsMode::kIgnore
+                 ? RuntimeStubId::kTSANRelaxedStore8IgnoreFP
+                 : RuntimeStubId::kTSANRelaxedStore8SaveFP;
+    } else if (size == kInt16Size) {
+      return fp_mode == SaveFPRegsMode::kIgnore
+                 ? RuntimeStubId::kTSANRelaxedStore16IgnoreFP
+                 : RuntimeStubId::kTSANRelaxedStore16SaveFP;
+    } else if (size == kInt32Size) {
       return fp_mode == SaveFPRegsMode::kIgnore
                  ? RuntimeStubId::kTSANRelaxedStore32IgnoreFP
                  : RuntimeStubId::kTSANRelaxedStore32SaveFP;