Skip to content

V
V8
Commit 630c33c6 authored by martyn.capewell's avatar martyn.capewell Committed by Commit bot
Browse files
Options

[arm64] Use acquire/release memory accesses for atomics 

Implement ASSEMBLE_ATOMIC_LOAD/STORE_INTEGER using acquire/release instructions,
to match the existing ASSEMBLE_ATOMIC_EXCHANGE_INTEGER macro.

BUG=v8:6097

Review-Url: https://codereview.chromium.org/2760963002
Cr-Commit-Position: refs/heads/master@{#44184}
parent 88a7061a
Hide whitespace changes
Inline Side-by-side
Showing with 135 additions and 99 deletions
src/arm64/assembler-arm64.cc
View file @ 630c33c6
......@@ -1698,19 +1698,19 @@ void Assembler::ldr(const CPURegister& rt, const Immediate& imm) {
void Assembler::ldar(const Register& rt, const Register& rn) {
DCHECK(rn.Is64Bits());
LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? LDAR_w : LDAR_x;
Emit(op | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
Emit(op | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
}
void Assembler::ldaxr(const Register& rt, const Register& rn) {
DCHECK(rn.Is64Bits());
LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? LDAXR_w : LDAXR_x;
Emit(op | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
Emit(op | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
}
void Assembler::stlr(const Register& rt, const Register& rn) {
DCHECK(rn.Is64Bits());
LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? STLR_w : STLR_x;
Emit(op | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
Emit(op | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
}
void Assembler::stlxr(const Register& rs, const Register& rt,
......@@ -1718,25 +1718,25 @@ void Assembler::stlxr(const Register& rs, const Register& rt,
DCHECK(rs.Is32Bits());
DCHECK(rn.Is64Bits());
LoadStoreAcquireReleaseOp op = rt.Is32Bits() ? STLXR_w : STLXR_x;
Emit(op | Rs(rs) | Rt2(x31) | Rn(rn) | Rt(rt));
Emit(op | Rs(rs) | Rt2(x31) | RnSP(rn) | Rt(rt));
}
void Assembler::ldarb(const Register& rt, const Register& rn) {
DCHECK(rt.Is32Bits());
DCHECK(rn.Is64Bits());
Emit(LDAR_b | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
Emit(LDAR_b | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
}
void Assembler::ldaxrb(const Register& rt, const Register& rn) {
DCHECK(rt.Is32Bits());
DCHECK(rn.Is64Bits());
Emit(LDAXR_b | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
Emit(LDAXR_b | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
}
void Assembler::stlrb(const Register& rt, const Register& rn) {
DCHECK(rt.Is32Bits());
DCHECK(rn.Is64Bits());
Emit(STLR_b | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
Emit(STLR_b | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
}
void Assembler::stlxrb(const Register& rs, const Register& rt,
......@@ -1744,25 +1744,25 @@ void Assembler::stlxrb(const Register& rs, const Register& rt,
DCHECK(rs.Is32Bits());
DCHECK(rt.Is32Bits());
DCHECK(rn.Is64Bits());
Emit(STLXR_b | Rs(rs) | Rt2(x31) | Rn(rn) | Rt(rt));
Emit(STLXR_b | Rs(rs) | Rt2(x31) | RnSP(rn) | Rt(rt));
}
void Assembler::ldarh(const Register& rt, const Register& rn) {
DCHECK(rt.Is32Bits());
DCHECK(rn.Is64Bits());
Emit(LDAR_h | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
Emit(LDAR_h | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
}
void Assembler::ldaxrh(const Register& rt, const Register& rn) {
DCHECK(rt.Is32Bits());
DCHECK(rn.Is64Bits());
Emit(LDAXR_h | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
Emit(LDAXR_h | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
}
void Assembler::stlrh(const Register& rt, const Register& rn) {
DCHECK(rt.Is32Bits());
DCHECK(rn.Is64Bits());
Emit(STLR_h | Rs(x31) | Rt2(x31) | Rn(rn) | Rt(rt));
Emit(STLR_h | Rs(x31) | Rt2(x31) | RnSP(rn) | Rt(rt));
}
void Assembler::stlxrh(const Register& rs, const Register& rt,
......@@ -1770,7 +1770,7 @@ void Assembler::stlxrh(const Register& rs, const Register& rt,
DCHECK(rs.Is32Bits());
DCHECK(rt.Is32Bits());
DCHECK(rn.Is64Bits());
Emit(STLXR_h | Rs(rs) | Rt2(x31) | Rn(rn) | Rt(rt));
Emit(STLXR_h | Rs(rs) | Rt2(x31) | RnSP(rn) | Rt(rt));
}
void Assembler::mov(const Register& rd, const Register& rm) {
......
src/arm64/constants-arm64.h
View file @ 630c33c6
......@@ -864,7 +864,7 @@ enum LoadStoreRegisterOffset {
#undef LOAD_STORE_REGISTER_OFFSET
};
// Load/store acquire/release
// Load/store acquire/release.
enum LoadStoreAcquireReleaseOp {
LoadStoreAcquireReleaseFixed = 0x08000000,
LoadStoreAcquireReleaseFMask = 0x3F000000,
......
src/arm64/disasm-arm64.cc
View file @ 630c33c6
......@@ -916,10 +916,10 @@ void DisassemblingDecoder::VisitLoadStorePairOffset(Instruction* instr) {
void DisassemblingDecoder::VisitLoadStoreAcquireRelease(Instruction *instr) {
const char *mnemonic = "unimplemented";
const char *form = "'Wt, ['Xn]";
const char *form_x = "'Xt, ['Xn]";
const char *form_stlx = "'Ws, 'Wt, ['Xn]";
const char *form_stlx_x = "'Ws, 'Xt, ['Xn]";
const char* form = "'Wt, ['Xns]";
const char* form_x = "'Xt, ['Xns]";
const char* form_stlx = "'Ws, 'Wt, ['Xns]";
const char* form_stlx_x = "'Ws, 'Xt, ['Xns]";
switch (instr->Mask(LoadStoreAcquireReleaseMask)) {
case LDAXR_b: mnemonic = "ldaxrb"; break;
......@@ -938,7 +938,8 @@ void DisassemblingDecoder::VisitLoadStoreAcquireRelease(Instruction *instr) {
case STLXR_b: mnemonic = "stlxrb"; form = form_stlx; break;
case STLXR_w: mnemonic = "stlxr"; form = form_stlx; break;
case STLXR_x: mnemonic = "stlxr"; form = form_stlx_x; break;
default: form = "(LoadStoreAcquireReleaseMask)";
default:
form = "(LoadStoreAcquireRelease)";
}
Format(instr, mnemonic, form);
}
......
src/arm64/instrument-arm64.cc
View file @ 630c33c6
......@@ -61,39 +61,39 @@ typedef struct {
CounterType type;
} CounterDescriptor;
static const CounterDescriptor kCounterList[] = {
{"Instruction", Cumulative},
{"Move Immediate", Gauge},
{"Add/Sub DP", Gauge},
{"Logical DP", Gauge},
{"Other Int DP", Gauge},
{"FP DP", Gauge},
{"Conditional Select", Gauge},
{"Conditional Compare", Gauge},
{"Unconditional Branch", Gauge},
{"Compare and Branch", Gauge},
{"Test and Branch", Gauge},
{"Conditional Branch", Gauge},
{"Load Integer", Gauge},
{"Load FP", Gauge},
{"Load Pair", Gauge},
{"Load Literal", Gauge},
{"Store Integer", Gauge},
{"Store FP", Gauge},
{"Store Pair", Gauge},
{"PC Addressing", Gauge},
{"Other", Gauge},
{"SP Adjust", Gauge},
{"Instruction", Cumulative},
{"Move Immediate", Gauge},
{"Add/Sub DP", Gauge},
{"Logical DP", Gauge},
{"Other Int DP", Gauge},
{"FP DP", Gauge},
{"Conditional Select", Gauge},
{"Conditional Compare", Gauge},
{"Unconditional Branch", Gauge},
{"Compare and Branch", Gauge},
{"Test and Branch", Gauge},
{"Conditional Branch", Gauge},
{"Load Integer", Gauge},
{"Load FP", Gauge},
{"Load Pair", Gauge},
{"Load Literal", Gauge},
{"Load Acquire", Gauge},
{"Store Integer", Gauge},
{"Store FP", Gauge},
{"Store Pair", Gauge},
{"Store Release", Gauge},
{"PC Addressing", Gauge},
{"Other", Gauge},
{"SP Adjust", Gauge},
};
Instrument::Instrument(const char* datafile, uint64_t sample_period)
: output_stream_(stderr), sample_period_(sample_period) {
......
src/arm64/simulator-arm64.cc
View file @ 630c33c6
......@@ -1953,36 +1953,40 @@ Simulator::TransactionSize Simulator::get_transaction_size(unsigned size) {
}
void Simulator::VisitLoadStoreAcquireRelease(Instruction* instr) {
unsigned rs = instr->Rs();
unsigned rt = instr->Rt();
unsigned rn = instr->Rn();
LoadStoreAcquireReleaseOp op = static_cast<LoadStoreAcquireReleaseOp>(
instr->Mask(LoadStoreAcquireReleaseMask));
int32_t is_acquire_release = instr->LoadStoreXAcquireRelease();
int32_t is_not_exclusive = instr->LoadStoreXNotExclusive();
int32_t is_exclusive = (instr->LoadStoreXNotExclusive() == 0);
int32_t is_load = instr->LoadStoreXLoad();
int32_t is_pair = instr->LoadStoreXPair();
USE(is_acquire_release);
USE(is_not_exclusive);
USE(is_pair);
DCHECK_NE(is_acquire_release, 0);
DCHECK_EQ(is_not_exclusive, 0); // Non exclusive unimplemented.
DCHECK_EQ(is_pair, 0); // Pair unimplemented.
DCHECK_NE(is_acquire_release, 0); // Non-acquire/release unimplemented.
DCHECK_EQ(is_pair, 0); // Pair unimplemented.
unsigned access_size = 1 << instr->LoadStoreXSizeLog2();
uintptr_t address = LoadStoreAddress(rn, 0, AddrMode::Offset);
DCHECK(address % access_size == 0);
DCHECK_EQ(address % access_size, 0);
base::LockGuard<base::Mutex> lock_guard(&global_monitor_.Pointer()->mutex);
if (is_load != 0) {
local_monitor_.NotifyLoadExcl(address, get_transaction_size(access_size));
global_monitor_.Pointer()->NotifyLoadExcl_Locked(
address, &global_monitor_processor_);
if (is_exclusive) {
local_monitor_.NotifyLoadExcl(address, get_transaction_size(access_size));
global_monitor_.Pointer()->NotifyLoadExcl_Locked(
address, &global_monitor_processor_);
} else {
local_monitor_.NotifyLoad(address);
}
switch (op) {
case LDAR_b:
case LDAXR_b:
set_wreg_no_log(rt, MemoryRead<uint8_t>(address));
break;
case LDAR_h:
case LDAXR_h:
set_wreg_no_log(rt, MemoryRead<uint16_t>(address));
break;
case LDAR_w:
case LDAXR_w:
set_wreg_no_log(rt, MemoryRead<uint32_t>(address));
break;
......@@ -1991,27 +1995,47 @@ void Simulator::VisitLoadStoreAcquireRelease(Instruction* instr) {
}
LogRead(address, access_size, rt);
} else {
if (local_monitor_.NotifyStoreExcl(address,
get_transaction_size(access_size)) &&
global_monitor_.Pointer()->NotifyStoreExcl_Locked(
address, &global_monitor_processor_)) {
if (is_exclusive) {
unsigned rs = instr->Rs();
if (local_monitor_.NotifyStoreExcl(address,
get_transaction_size(access_size)) &&
global_monitor_.Pointer()->NotifyStoreExcl_Locked(
address, &global_monitor_processor_)) {
switch (op) {
case STLXR_b:
MemoryWrite<uint8_t>(address, wreg(rt));
break;
case STLXR_h:
MemoryWrite<uint16_t>(address, wreg(rt));
break;
case STLXR_w:
MemoryWrite<uint32_t>(address, wreg(rt));
break;
default:
UNIMPLEMENTED();
}
LogWrite(address, access_size, rt);
set_wreg(rs, 0);
} else {
set_wreg(rs, 1);
}
} else {
local_monitor_.NotifyStore(address);
global_monitor_.Pointer()->NotifyStore_Locked(address,
&global_monitor_processor_);
switch (op) {
case STLXR_b:
case STLR_b:
MemoryWrite<uint8_t>(address, wreg(rt));
break;
case STLXR_h:
case STLR_h:
MemoryWrite<uint16_t>(address, wreg(rt));
break;
case STLXR_w:
case STLR_w:
MemoryWrite<uint32_t>(address, wreg(rt));
break;
default:
UNIMPLEMENTED();
}
LogWrite(address, access_size, rt);
set_wreg(rs, 0);
} else {
set_wreg(rs, 1);
}
}
}
......
src/compiler/arm64/code-generator-arm64.cc
View file @ 630c33c6
......@@ -518,19 +518,16 @@ Condition FlagsConditionToCondition(FlagsCondition condition) {
} \
} while (0)
#define ASSEMBLE_ATOMIC_LOAD_INTEGER(asm_instr) \
do { \
__ asm_instr(i.OutputRegister(), \
MemOperand(i.InputRegister(0), i.InputRegister(1))); \
__ Dmb(InnerShareable, BarrierAll); \
#define ASSEMBLE_ATOMIC_LOAD_INTEGER(asm_instr) \
do { \
__ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
__ asm_instr(i.OutputRegister32(), i.TempRegister(0)); \
} while (0)
#define ASSEMBLE_ATOMIC_STORE_INTEGER(asm_instr) \
do { \
__ Dmb(InnerShareable, BarrierAll); \
__ asm_instr(i.InputRegister(2), \
MemOperand(i.InputRegister(0), i.InputRegister(1))); \
__ Dmb(InnerShareable, BarrierAll); \
#define ASSEMBLE_ATOMIC_STORE_INTEGER(asm_instr) \
do { \
__ Add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \
__ asm_instr(i.InputRegister32(2), i.TempRegister(0)); \
} while (0)
#define ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(load_instr, store_instr) \
......@@ -1635,33 +1632,30 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
ASSEMBLE_CHECKED_STORE_FLOAT(64);
break;
case kAtomicLoadInt8:
ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrsb);
ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarb);
__ Sxtb(i.OutputRegister(0), i.OutputRegister(0));
break;
case kAtomicLoadUint8:
ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrb);
ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarb);
break;
case kAtomicLoadInt16:
ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrsh);
ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarh);
__ Sxth(i.OutputRegister(0), i.OutputRegister(0));
break;
case kAtomicLoadUint16:
ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldrh);
ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldarh);
break;
case kAtomicLoadWord32:
__ Ldr(i.OutputRegister32(),
MemOperand(i.InputRegister(0), i.InputRegister(1)));
__ Dmb(InnerShareable, BarrierAll);
ASSEMBLE_ATOMIC_LOAD_INTEGER(Ldar);
break;
case kAtomicStoreWord8:
ASSEMBLE_ATOMIC_STORE_INTEGER(Strb);
ASSEMBLE_ATOMIC_STORE_INTEGER(Stlrb);
break;
case kAtomicStoreWord16:
ASSEMBLE_ATOMIC_STORE_INTEGER(Strh);
ASSEMBLE_ATOMIC_STORE_INTEGER(Stlrh);
break;
case kAtomicStoreWord32:
__ Dmb(InnerShareable, BarrierAll);
__ Str(i.InputRegister32(2),
MemOperand(i.InputRegister(0), i.InputRegister(1)));
__ Dmb(InnerShareable, BarrierAll);
ASSEMBLE_ATOMIC_STORE_INTEGER(Stlr);
break;
case kAtomicExchangeInt8:
ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(ldaxrb, stlxrb);
......
src/compiler/arm64/instruction-selector-arm64.cc
View file @ 630c33c6
......@@ -2667,8 +2667,12 @@ void InstructionSelector::VisitAtomicLoad(Node* node) {
UNREACHABLE();
return;
}
Emit(opcode | AddressingModeField::encode(kMode_MRR),
g.DefineAsRegister(node), g.UseRegister(base), g.UseRegister(index));
InstructionOperand inputs[] = {g.UseRegister(base), g.UseRegister(index)};
InstructionOperand outputs[] = {g.DefineAsRegister(node)};
InstructionOperand temps[] = {g.TempRegister()};
InstructionCode code = opcode | AddressingModeField::encode(kMode_MRR);
Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
arraysize(temps), temps);
}
void InstructionSelector::VisitAtomicStore(Node* node) {
......@@ -2700,7 +2704,8 @@ void InstructionSelector::VisitAtomicStore(Node* node) {
inputs[input_count++] = g.UseUniqueRegister(index);
inputs[input_count++] = g.UseUniqueRegister(value);
InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
Emit(code, 0, nullptr, input_count, inputs);
InstructionOperand temps[] = {g.TempRegister()};
Emit(code, 0, nullptr, input_count, inputs, arraysize(temps), temps);
}
void InstructionSelector::VisitAtomicExchange(Node* node) {
......@@ -2733,10 +2738,9 @@ void InstructionSelector::VisitAtomicExchange(Node* node) {
inputs[input_count++] = g.UseUniqueRegister(value);
InstructionOperand outputs[1];
outputs[0] = g.UseUniqueRegister(node);
InstructionOperand temp[2];
temp[0] = g.TempRegister();
InstructionOperand temps[] = {g.TempRegister()};
InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
Emit(code, 1, outputs, input_count, inputs, 1, temp);
Emit(code, 1, outputs, input_count, inputs, arraysize(temps), temps);
}
void InstructionSelector::VisitAtomicCompareExchange(Node* node) {
......
test/cctest/test-disasm-arm64.cc
View file @ 630c33c6
......@@ -1276,6 +1276,19 @@ TEST_(load_store_acquire_release) {
COMPARE(stlxrb(w21, w22, x23), "stlxrb w21, w22, [x23]");
COMPARE(stlxrh(w24, w25, x26), "stlxrh w24, w25, [x26]");
COMPARE(ldarb(wzr, csp), "ldarb wzr, [csp]");
COMPARE(ldarh(wzr, csp), "ldarh wzr, [csp]");
COMPARE(ldar(wzr, csp), "ldar wzr, [csp]");
COMPARE(stlrb(wzr, csp), "stlrb wzr, [csp]");
COMPARE(stlrh(wzr, csp), "stlrh wzr, [csp]");
COMPARE(stlr(wzr, csp), "stlr wzr, [csp]");
COMPARE(ldaxrb(wzr, csp), "ldaxrb wzr, [csp]");
COMPARE(ldaxrh(wzr, csp), "ldaxrh wzr, [csp]");
COMPARE(ldaxr(wzr, csp), "ldaxr wzr, [csp]");
COMPARE(stlxrb(wzr, wzr, csp), "stlxrb wzr, wzr, [csp]");
COMPARE(stlxrh(wzr, wzr, csp), "stlxrh wzr, wzr, [csp]");
COMPARE(stlxr(wzr, wzr, csp), "stlxr wzr, wzr, [csp]");
CLEANUP();
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment