[arm64] Use acquire/release memory accesses for atomics

Implement ASSEMBLE_ATOMIC_LOAD/STORE_INTEGER using acquire/release instructions,
to match the existing ASSEMBLE_ATOMIC_EXCHANGE_INTEGER macro.

BUG=v8:6097

Review-Url: https://codereview.chromium.org/2760963002
Cr-Commit-Position: refs/heads/master@{#44184}

src/arm64/assembler-arm64.cc

@@ -1698,19 +1698,19 @@ void Assembler::ldr(const CPURegister& rt, const Immediate& imm) {
 void Assembler::ldar(const Register& rt, const Register& rn) {
   DCHECK(rn.Is64Bits());
   LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? LDAR_w : LDAR_x;
-  Emit(op | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
+  Emit(op | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
 }
 
 void Assembler::ldaxr(const Register& rt, const Register& rn) {
   DCHECK(rn.Is64Bits());
   LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? LDAXR_w : LDAXR_x;
-  Emit(op | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
+  Emit(op | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
 }
 
 void Assembler::stlr(const Register& rt, const Register& rn) {
   DCHECK(rn.Is64Bits());
   LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? STLR_w : STLR_x;
-  Emit(op | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
+  Emit(op | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
 }
 
 void Assembler::stlxr(const Register& rs, const Register& rt,
@@ -1718,25 +1718,25 @@ void Assembler::stlxr(const Register& rs, const Register& rt,
   DCHECK(rs.Is32Bits());
   DCHECK(rn.Is64Bits());
   LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? STLXR_w : STLXR_x;
-  Emit(op | Rs(rs) | Rt2(x31) | Rn(rn) | Rt(rt));
+  Emit(op | Rs(rs) | Rt2(x31) | RnSP(rn) | Rt(rt));
 }
 
 void Assembler::ldarb(const Register& rt, const Register& rn) {
   DCHECK(rt.Is32Bits());
   DCHECK(rn.Is64Bits());
-  Emit(LDAR_b | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
+  Emit(LDAR_b | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
 }
 
 void Assembler::ldaxrb(const Register& rt, const Register& rn) {
   DCHECK(rt.Is32Bits());
   DCHECK(rn.Is64Bits());
-  Emit(LDAXR_b | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
+  Emit(LDAXR_b | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
 }
 
 void Assembler::stlrb(const Register& rt, const Register& rn) {
   DCHECK(rt.Is32Bits());
   DCHECK(rn.Is64Bits());
-  Emit(STLR_b | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
+  Emit(STLR_b | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
 }
 
 void Assembler::stlxrb(const Register& rs, const Register& rt,
@@ -1744,25 +1744,25 @@ void Assembler::stlxrb(const Register& rs, const Register& rt,
   DCHECK(rs.Is32Bits());
   DCHECK(rt.Is32Bits());
   DCHECK(rn.Is64Bits());
-  Emit(STLXR_b | Rs(rs) | Rt2(x31) | Rn(rn) | Rt(rt));
+  Emit(STLXR_b | Rs(rs) | Rt2(x31) | RnSP(rn) | Rt(rt));
 }
 
 void Assembler::ldarh(const Register& rt, const Register& rn) {
   DCHECK(rt.Is32Bits());
   DCHECK(rn.Is64Bits());
-  Emit(LDAR_h | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
+  Emit(LDAR_h | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
 }
 
 void Assembler::ldaxrh(const Register& rt, const Register& rn) {
   DCHECK(rt.Is32Bits());
   DCHECK(rn.Is64Bits());
-  Emit(LDAXR_h | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
+  Emit(LDAXR_h | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
 }
 
 void Assembler::stlrh(const Register& rt, const Register& rn) {
   DCHECK(rt.Is32Bits());
   DCHECK(rn.Is64Bits());
-  Emit(STLR_h | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
+  Emit(STLR_h | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
 }
 
 void Assembler::stlxrh(const Register& rs, const Register& rt,
@@ -1770,7 +1770,7 @@ void Assembler::stlxrh(const Register& rs, const Register& rt,
   DCHECK(rs.Is32Bits());
   DCHECK(rt.Is32Bits());
   DCHECK(rn.Is64Bits());
-  Emit(STLXR_h | Rs(rs) | Rt2(x31) | Rn(rn) | Rt(rt));
+  Emit(STLXR_h | Rs(rs) | Rt2(x31) | RnSP(rn) | Rt(rt));
 }
 
 void Assembler::mov(const Register& rd, const Register& rm) {

src/arm64/constants-arm64.h

@@ -864,7 +864,7 @@ enum LoadStoreRegisterOffset {
   #undef LOAD_STORE_REGISTER_OFFSET
 };
 
-// Load/store acquire/release
+// Load/store acquire/release.
 enum LoadStoreAcquireReleaseOp {
   LoadStoreAcquireReleaseFixed = 0x08000000,
   LoadStoreAcquireReleaseFMask = 0x3F000000,

src/arm64/disasm-arm64.cc

@@ -916,10 +916,10 @@ void DisassemblingDecoder::VisitLoadStorePairOffset(Instruction* instr) {
 
 void DisassemblingDecoder::VisitLoadStoreAcquireRelease(Instruction *instr) {
   const char *mnemonic = "unimplemented";
-  const char *form = "'Wt, ['Xn]";
-  const char *form_x = "'Xt, ['Xn]";
-  const char *form_stlx = "'Ws, 'Wt, ['Xn]";
-  const char *form_stlx_x = "'Ws, 'Xt, ['Xn]";
+  const char* form = "'Wt, ['Xns]";
+  const char* form_x = "'Xt, ['Xns]";
+  const char* form_stlx = "'Ws, 'Wt, ['Xns]";
+  const char* form_stlx_x = "'Ws, 'Xt, ['Xns]";
 
   switch (instr->Mask(LoadStoreAcquireReleaseMask)) {
     case LDAXR_b: mnemonic = "ldaxrb"; break;
@@ -938,7 +938,8 @@ void DisassemblingDecoder::VisitLoadStoreAcquireRelease(Instruction *instr) {
     case STLXR_b: mnemonic = "stlxrb"; form = form_stlx; break;
     case STLXR_w: mnemonic = "stlxr"; form = form_stlx; break;
     case STLXR_x: mnemonic = "stlxr"; form = form_stlx_x; break;
-    default: form = "(LoadStoreAcquireReleaseMask)";
+    default:
+      form = "(LoadStoreAcquireRelease)";
   }
   Format(instr, mnemonic, form);
 }

src/arm64/instrument-arm64.cc

@@ -61,7 +61,6 @@ typedef struct {
   CounterType type;
 } CounterDescriptor;
 
-
 static const CounterDescriptor kCounterList[] = {
     {"Instruction", Cumulative},
 
@@ -83,17 +82,18 @@ static const CounterDescriptor kCounterList[] = {
     {"Load FP", Gauge},
     {"Load Pair", Gauge},
     {"Load Literal", Gauge},
+    {"Load Acquire", Gauge},
 
     {"Store Integer", Gauge},
     {"Store FP", Gauge},
     {"Store Pair", Gauge},
+    {"Store Release", Gauge},
 
     {"PC Addressing", Gauge},
     {"Other", Gauge},
     {"SP Adjust", Gauge},
 };
 
-
 Instrument::Instrument(const char* datafile, uint64_t sample_period)
     : output_stream_(stderr), sample_period_(sample_period) {
 

src/arm64/simulator-arm64.cc

@@ -1953,36 +1953,40 @@ Simulator::TransactionSize Simulator::get_transaction_size(unsigned size) {
 }
 
 void Simulator::VisitLoadStoreAcquireRelease(Instruction* instr) {
-  unsigned rs = instr->Rs();
   unsigned rt = instr->Rt();
   unsigned rn = instr->Rn();
   LoadStoreAcquireReleaseOp op = static_cast<LoadStoreAcquireReleaseOp>(
       instr->Mask(LoadStoreAcquireReleaseMask));
   int32_t is_acquire_release = instr->LoadStoreXAcquireRelease();
-  int32_t is_not_exclusive = instr->LoadStoreXNotExclusive();
+  int32_t is_exclusive = (instr->LoadStoreXNotExclusive() == 0);
   int32_t is_load = instr->LoadStoreXLoad();
   int32_t is_pair = instr->LoadStoreXPair();
   USE(is_acquire_release);
-  USE(is_not_exclusive);
   USE(is_pair);
-  DCHECK_NE(is_acquire_release, 0);
-  DCHECK_EQ(is_not_exclusive, 0);  // Non exclusive unimplemented.
+  DCHECK_NE(is_acquire_release, 0);  // Non-acquire/release unimplemented.
   DCHECK_EQ(is_pair, 0);             // Pair unimplemented.
   unsigned access_size = 1 << instr->LoadStoreXSizeLog2();
   uintptr_t address = LoadStoreAddress(rn, 0, AddrMode::Offset);
-  DCHECK(address % access_size == 0);
+  DCHECK_EQ(address % access_size, 0);
   base::LockGuard<base::Mutex> lock_guard(&global_monitor_.Pointer()->mutex);
   if (is_load != 0) {
+    if (is_exclusive) {
       local_monitor_.NotifyLoadExcl(address, get_transaction_size(access_size));
       global_monitor_.Pointer()->NotifyLoadExcl_Locked(
           address, &global_monitor_processor_);
+    } else {
+      local_monitor_.NotifyLoad(address);
+    }
     switch (op) {
+      case LDAR_b:
       case LDAXR_b:
         set_wreg_no_log(rt, MemoryRead<uint8_t>(address));
         break;
+      case LDAR_h:
       case LDAXR_h:
         set_wreg_no_log(rt, MemoryRead<uint16_t>(address));
         break;
+      case LDAR_w:
       case LDAXR_w:
         set_wreg_no_log(rt, MemoryRead<uint32_t>(address));
         break;
@@ -1991,6 +1995,8 @@ void Simulator::VisitLoadStoreAcquireRelease(Instruction* instr) {
     }
     LogRead(address, access_size, rt);
   } else {
+    if (is_exclusive) {
+      unsigned rs = instr->Rs();
       if (local_monitor_.NotifyStoreExcl(address,
                                          get_transaction_size(access_size)) &&
           global_monitor_.Pointer()->NotifyStoreExcl_Locked(
@@ -2013,6 +2019,24 @@ void Simulator::VisitLoadStoreAcquireRelease(Instruction* instr) {
       } else {
         set_wreg(rs, 1);
       }
+    } else {
+      local_monitor_.NotifyStore(address);
+      global_monitor_.Pointer()->NotifyStore_Locked(address,
+                                                    &global_monitor_processor_);
+      switch (op) {
+        case STLR_b:
+          MemoryWrite<uint8_t>(address, wreg(rt));
+          break;
+        case STLR_h:
+          MemoryWrite<uint16_t>(address, wreg(rt));
+          break;
+        case STLR_w:
+          MemoryWrite<uint32_t>(address, wreg(rt));
+          break;
+        default:
+          UNIMPLEMENTED();
+      }
+    }
   }
 }
 

src/compiler/arm64/code-generator-arm64.cc

@@ -520,17 +520,14 @@ Condition FlagsConditionToCondition(FlagsCondition condition) {
 
 #define ASSEMBLE_ATOMIC_LOAD_INTEGER(asm_instr)                        \
   do {                                                                 \
-    __ asm_instr(i.OutputRegister(),                                  \
-                 MemOperand(i.InputRegister(0), i.InputRegister(1))); \
-    __ Dmb(InnerShareable, BarrierAll);                               \
+    __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
+    __ asm_instr(i.OutputRegister32(), i.TempRegister(0));             \
   } while (0)
 
 #define ASSEMBLE_ATOMIC_STORE_INTEGER(asm_instr)                       \
   do {                                                                 \
-    __ Dmb(InnerShareable, BarrierAll);                               \
-    __ asm_instr(i.InputRegister(2),                                  \
-                 MemOperand(i.InputRegister(0), i.InputRegister(1))); \
-    __ Dmb(InnerShareable, BarrierAll);                               \
+    __ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
+    __ asm_instr(i.InputRegister32(2), i.TempRegister(0));             \
   } while (0)
 
 #define ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(load_instr, store_instr)      \
@@ -1635,33 +1632,30 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
       ASSEMBLE_CHECKED_STORE_FLOAT(64);
       break;
     case kAtomicLoadInt8:
-      ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrsb);
+      ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarb);
+      __ Sxtb(i.OutputRegister(0), i.OutputRegister(0));
       break;
     case kAtomicLoadUint8:
-      ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrb);
+      ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarb);
       break;
     case kAtomicLoadInt16:
-      ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrsh);
+      ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarh);
+      __ Sxth(i.OutputRegister(0), i.OutputRegister(0));
       break;
     case kAtomicLoadUint16:
-      ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrh);
+      ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarh);
       break;
     case kAtomicLoadWord32:
-      __ Ldr(i.OutputRegister32(),
-             MemOperand(i.InputRegister(0), i.InputRegister(1)));
-      __ Dmb(InnerShareable, BarrierAll);
+      ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldar);
       break;
     case kAtomicStoreWord8:
-      ASSEMBLE_ATOMIC_STORE_INTEGER(Strb);
+      ASSEMBLE_ATOMIC_STORE_INTEGER(Stlrb);
       break;
     case kAtomicStoreWord16:
-      ASSEMBLE_ATOMIC_STORE_INTEGER(Strh);
+      ASSEMBLE_ATOMIC_STORE_INTEGER(Stlrh);
       break;
     case kAtomicStoreWord32:
-      __ Dmb(InnerShareable, BarrierAll);
-      __ Str(i.InputRegister32(2),
-             MemOperand(i.InputRegister(0), i.InputRegister(1)));
-      __ Dmb(InnerShareable, BarrierAll);
+      ASSEMBLE_ATOMIC_STORE_INTEGER(Stlr);
       break;
     case kAtomicExchangeInt8:
       ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrb, stlxrb);

src/compiler/arm64/instruction-selector-arm64.cc

@@ -2667,8 +2667,12 @@ void InstructionSelector::VisitAtomicLoad(Node* node) {
       UNREACHABLE();
       return;
   }
-  Emit(opcode | AddressingModeField::encode(kMode_MRR),
-       g.DefineAsRegister(node), g.UseRegister(base), g.UseRegister(index));
+  InstructionOperand inputs[] = {g.UseRegister(base), g.UseRegister(index)};
+  InstructionOperand outputs[] = {g.DefineAsRegister(node)};
+  InstructionOperand temps[] = {g.TempRegister()};
+  InstructionCode code = opcode | AddressingModeField::encode(kMode_MRR);
+  Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
+       arraysize(temps), temps);
 }
 
 void InstructionSelector::VisitAtomicStore(Node* node) {
@@ -2700,7 +2704,8 @@ void InstructionSelector::VisitAtomicStore(Node* node) {
   inputs[input_count++] = g.UseUniqueRegister(index);
   inputs[input_count++] = g.UseUniqueRegister(value);
   InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
-  Emit(code, 0, nullptr, input_count, inputs);
+  InstructionOperand temps[] = {g.TempRegister()};
+  Emit(code, 0, nullptr, input_count, inputs, arraysize(temps), temps);
 }
 
 void InstructionSelector::VisitAtomicExchange(Node* node) {
@@ -2733,10 +2738,9 @@ void InstructionSelector::VisitAtomicExchange(Node* node) {
   inputs[input_count++] = g.UseUniqueRegister(value);
   InstructionOperand outputs[1];
   outputs[0] = g.UseUniqueRegister(node);
-  InstructionOperand temp[2];
-  temp[0] = g.TempRegister();
+  InstructionOperand temps[] = {g.TempRegister()};
   InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
-  Emit(code, 1, outputs, input_count, inputs, 1, temp);
+  Emit(code, 1, outputs, input_count, inputs, arraysize(temps), temps);
 }
 
 void InstructionSelector::VisitAtomicCompareExchange(Node* node) {

test/cctest/test-disasm-arm64.cc

@@ -1276,6 +1276,19 @@ TEST_(load_store_acquire_release) {
   COMPARE(stlxrb(w21, w22, x23), "stlxrb w21, w22, [x23]");
   COMPARE(stlxrh(w24, w25, x26), "stlxrh w24, w25, [x26]");
 
+  COMPARE(ldarb(wzr, csp), "ldarb wzr, [csp]");
+  COMPARE(ldarh(wzr, csp), "ldarh wzr, [csp]");
+  COMPARE(ldar(wzr, csp), "ldar wzr, [csp]");
+  COMPARE(stlrb(wzr, csp), "stlrb wzr, [csp]");
+  COMPARE(stlrh(wzr, csp), "stlrh wzr, [csp]");
+  COMPARE(stlr(wzr, csp), "stlr wzr, [csp]");
+  COMPARE(ldaxrb(wzr, csp), "ldaxrb wzr, [csp]");
+  COMPARE(ldaxrh(wzr, csp), "ldaxrh wzr, [csp]");
+  COMPARE(ldaxr(wzr, csp), "ldaxr wzr, [csp]");
+  COMPARE(stlxrb(wzr, wzr, csp), "stlxrb wzr, wzr, [csp]");
+  COMPARE(stlxrh(wzr, wzr, csp), "stlxrh wzr, wzr, [csp]");
+  COMPARE(stlxr(wzr, wzr, csp), "stlxr wzr, wzr, [csp]");
+
   CLEANUP();
 }