Skip to content

V
V8
Commit 11245b26 authored by Deepti Gandluri's avatar Deepti Gandluri Committed by Commit Bot
Browse files
Options

[compiler] Fix use of projection nodes for I64Atomic ops 

Handle the case when one or both of the output nodes of an I64Atomic op
are optimized, for code-gen instructions that use a set of fixed
registers, use temp registers to ensure the registers are not
clobbered.

BUG:v8:6532


Change-Id: I52763c48d615cdf3ae8d754402b11da2df31a4a1
Reviewed-on: https://chromium-review.googlesource.com/1195910Reviewed-by: 's avatarBill Budge <bbudge@chromium.org>
Reviewed-by: 's avatarBen Titzer <titzer@chromium.org>
Commit-Queue: Deepti Gandluri <gdeepti@chromium.org>
Cr-Commit-Position: refs/heads/master@{#56004}
parent 15d6d7b4
Hide whitespace changes
Inline Side-by-side
Showing with 408 additions and 115 deletions
src/compiler/arm/code-generator-arm.cc
View file @ 11245b26
...@@ -416,41 +416,37 @@ void ComputePoisonedAddressForLoad(CodeGenerator* codegen, ...@@ -416,41 +416,37 @@ void ComputePoisonedAddressForLoad(CodeGenerator* codegen,
__ dmb(ISH); \ __ dmb(ISH); \
} while (0) } while (0)
#define ASSEMBLE_ATOMIC64_ARITH_BINOP(instr1, instr2) \ #define ASSEMBLE_ATOMIC64_ARITH_BINOP(instr1, instr2) \
do { \ do { \
Label binop; \ Label binop; \
__ add(i.TempRegister(0), i.InputRegister(2), i.InputRegister(3)); \ __ add(i.TempRegister(0), i.InputRegister(2), i.InputRegister(3)); \
__ dmb(ISH); \ __ dmb(ISH); \
__ bind(&binop); \ __ bind(&binop); \
__ ldrexd(i.OutputRegister(0), i.OutputRegister(1), i.TempRegister(0)); \ __ ldrexd(r2, r3, i.TempRegister(0)); \
__ instr1(i.TempRegister(1), i.OutputRegister(0), i.InputRegister(0), \ __ instr1(i.TempRegister(1), r2, i.InputRegister(0), SBit::SetCC); \
SBit::SetCC); \ __ instr2(i.TempRegister(2), r3, Operand(i.InputRegister(1))); \
__ instr2(i.TempRegister(2), i.OutputRegister(1), \ DCHECK_EQ(LeaveCC, i.OutputSBit()); \
Operand(i.InputRegister(1))); \ __ strexd(i.TempRegister(3), i.TempRegister(1), i.TempRegister(2), \
DCHECK_EQ(LeaveCC, i.OutputSBit()); \ i.TempRegister(0)); \
__ strexd(i.TempRegister(3), i.TempRegister(1), i.TempRegister(2), \ __ teq(i.TempRegister(3), Operand(0)); \
i.TempRegister(0)); \ __ b(ne, &binop); \
__ teq(i.TempRegister(3), Operand(0)); \ __ dmb(ISH); \
__ b(ne, &binop); \
__ dmb(ISH); \
} while (0) } while (0)
#define ASSEMBLE_ATOMIC64_LOGIC_BINOP(instr) \ #define ASSEMBLE_ATOMIC64_LOGIC_BINOP(foo) \
do { \ do { \
Label binop; \ Label binop; \
__ add(i.TempRegister(0), i.InputRegister(2), i.InputRegister(3)); \ __ add(i.TempRegister(0), i.InputRegister(2), i.InputRegister(3)); \
__ dmb(ISH); \ __ dmb(ISH); \
__ bind(&binop); \ __ bind(&binop); \
__ ldrexd(i.OutputRegister(0), i.OutputRegister(1), i.TempRegister(0)); \ __ ldrexd(r2, r3, i.TempRegister(0)); \
__ instr(i.TempRegister(1), i.OutputRegister(0), \ __ foo(i.TempRegister(1), r2, Operand(i.InputRegister(0))); \
Operand(i.InputRegister(0))); \ __ foo(i.TempRegister(2), r3, Operand(i.InputRegister(1))); \
__ instr(i.TempRegister(2), i.OutputRegister(1), \ __ strexd(i.TempRegister(3), i.TempRegister(1), i.TempRegister(2), \
Operand(i.InputRegister(1))); \ i.TempRegister(0)); \
__ strexd(i.TempRegister(3), i.TempRegister(1), i.TempRegister(2), \ __ teq(i.TempRegister(3), Operand(0)); \
i.TempRegister(0)); \ __ b(ne, &binop); \
__ teq(i.TempRegister(3), Operand(0)); \ __ dmb(ISH); \
__ b(ne, &binop); \
__ dmb(ISH); \
} while (0) } while (0)
#define ASSEMBLE_IEEE754_BINOP(name) \ #define ASSEMBLE_IEEE754_BINOP(name) \
...@@ -602,6 +598,19 @@ void AdjustStackPointerForTailCall( ...@@ -602,6 +598,19 @@ void AdjustStackPointerForTailCall(
} }
} }
#if DEBUG
bool VerifyOutputOfAtomicPairInstr(ArmOperandConverter* converter,
const Instruction* instr, Register low,
Register high) {
if (instr->OutputCount() > 0) {
if (converter->OutputRegister(0) != low) return false;
if (instr->OutputCount() == 2 && converter->OutputRegister(1) != high)
return false;
}
return true;
}
#endif
} // namespace } // namespace
void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr, void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
...@@ -2746,11 +2755,13 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( ...@@ -2746,11 +2755,13 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
ATOMIC_BINOP_CASE(Or, orr) ATOMIC_BINOP_CASE(Or, orr)
ATOMIC_BINOP_CASE(Xor, eor) ATOMIC_BINOP_CASE(Xor, eor)
#undef ATOMIC_BINOP_CASE #undef ATOMIC_BINOP_CASE
case kArmWord32AtomicPairLoad: case kArmWord32AtomicPairLoad: {
DCHECK(VerifyOutputOfAtomicPairInstr(&i, instr, r0, r1));
__ add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); __ add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1));
__ ldrexd(i.OutputRegister(0), i.OutputRegister(1), i.TempRegister(0)); __ ldrexd(r0, r1, i.TempRegister(0));
__ dmb(ISH); __ dmb(ISH);
break; break;
}
case kArmWord32AtomicPairStore: { case kArmWord32AtomicPairStore: {
Label store; Label store;
__ add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); __ add(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1));
...@@ -2764,28 +2775,32 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( ...@@ -2764,28 +2775,32 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
__ dmb(ISH); __ dmb(ISH);
break; break;
} }
#define ATOMIC_ARITH_BINOP_CASE(op, instr1, instr2) \ #define ATOMIC_ARITH_BINOP_CASE(op, instr1, instr2) \
case kArmWord32AtomicPair##op: { \ case kArmWord32AtomicPair##op: { \
ASSEMBLE_ATOMIC64_ARITH_BINOP(instr1, instr2); \ DCHECK(VerifyOutputOfAtomicPairInstr(&i, instr, r2, r3)); \
break; \ ASSEMBLE_ATOMIC64_ARITH_BINOP(instr1, instr2); \
break; \
} }
ATOMIC_ARITH_BINOP_CASE(Add, add, adc) ATOMIC_ARITH_BINOP_CASE(Add, add, adc)
ATOMIC_ARITH_BINOP_CASE(Sub, sub, sbc) ATOMIC_ARITH_BINOP_CASE(Sub, sub, sbc)
#undef ATOMIC_ARITH_BINOP_CASE #undef ATOMIC_ARITH_BINOP_CASE
#define ATOMIC_LOGIC_BINOP_CASE(op, instr) \ #define ATOMIC_LOGIC_BINOP_CASE(op, instr1) \
case kArmWord32AtomicPair##op: { \ case kArmWord32AtomicPair##op: { \
ASSEMBLE_ATOMIC64_LOGIC_BINOP(instr); \ DCHECK(VerifyOutputOfAtomicPairInstr(&i, instr, r2, r3)); \
break; \ ASSEMBLE_ATOMIC64_LOGIC_BINOP(instr1); \
break; \
} }
ATOMIC_LOGIC_BINOP_CASE(And, and_) ATOMIC_LOGIC_BINOP_CASE(And, and_)
ATOMIC_LOGIC_BINOP_CASE(Or, orr) ATOMIC_LOGIC_BINOP_CASE(Or, orr)
ATOMIC_LOGIC_BINOP_CASE(Xor, eor) ATOMIC_LOGIC_BINOP_CASE(Xor, eor)
#undef ATOMIC_LOGIC_BINOP_CASE
case kArmWord32AtomicPairExchange: { case kArmWord32AtomicPairExchange: {
DCHECK(VerifyOutputOfAtomicPairInstr(&i, instr, r6, r7));
Label exchange; Label exchange;
__ add(i.TempRegister(0), i.InputRegister(2), i.InputRegister(3)); __ add(i.TempRegister(0), i.InputRegister(2), i.InputRegister(3));
__ dmb(ISH); __ dmb(ISH);
__ bind(&exchange); __ bind(&exchange);
__ ldrexd(i.OutputRegister(0), i.OutputRegister(1), i.TempRegister(0)); __ ldrexd(r6, r7, i.TempRegister(0));
__ strexd(i.TempRegister(1), i.InputRegister(0), i.InputRegister(1), __ strexd(i.TempRegister(1), i.InputRegister(0), i.InputRegister(1),
i.TempRegister(0)); i.TempRegister(0));
__ teq(i.TempRegister(1), Operand(0)); __ teq(i.TempRegister(1), Operand(0));
...@@ -2794,15 +2809,16 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( ...@@ -2794,15 +2809,16 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
break; break;
} }
case kArmWord32AtomicPairCompareExchange: { case kArmWord32AtomicPairCompareExchange: {
DCHECK(VerifyOutputOfAtomicPairInstr(&i, instr, r2, r3));
__ add(i.TempRegister(0), i.InputRegister(4), i.InputRegister(5)); __ add(i.TempRegister(0), i.InputRegister(4), i.InputRegister(5));
Label compareExchange; Label compareExchange;
Label exit; Label exit;
__ dmb(ISH); __ dmb(ISH);
__ bind(&compareExchange); __ bind(&compareExchange);
__ ldrexd(i.OutputRegister(0), i.OutputRegister(1), i.TempRegister(0)); __ ldrexd(r2, r3, i.TempRegister(0));
__ teq(i.InputRegister(0), Operand(i.OutputRegister(0))); __ teq(i.InputRegister(0), Operand(r2));
__ b(ne, &exit); __ b(ne, &exit);
__ teq(i.InputRegister(1), Operand(i.OutputRegister(1))); __ teq(i.InputRegister(1), Operand(r3));
__ b(ne, &exit); __ b(ne, &exit);
__ strexd(i.TempRegister(1), i.InputRegister(2), i.InputRegister(3), __ strexd(i.TempRegister(1), i.InputRegister(2), i.InputRegister(3),
i.TempRegister(0)); i.TempRegister(0));
...@@ -2812,7 +2828,6 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( ...@@ -2812,7 +2828,6 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
__ dmb(ISH); __ dmb(ISH);
break; break;
} }
#undef ATOMIC_LOGIC_BINOP_CASE
#undef ASSEMBLE_ATOMIC_LOAD_INTEGER #undef ASSEMBLE_ATOMIC_LOAD_INTEGER
#undef ASSEMBLE_ATOMIC_STORE_INTEGER #undef ASSEMBLE_ATOMIC_STORE_INTEGER
#undef ASSEMBLE_ATOMIC_EXCHANGE_INTEGER #undef ASSEMBLE_ATOMIC_EXCHANGE_INTEGER
......
src/compiler/arm/instruction-selector-arm.cc
View file @ 11245b26
...@@ -414,14 +414,30 @@ void VisitPairAtomicBinOp(InstructionSelector* selector, Node* node, ...@@ -414,14 +414,30 @@ void VisitPairAtomicBinOp(InstructionSelector* selector, Node* node,
InstructionOperand inputs[] = {g.UseUniqueRegister(value), InstructionOperand inputs[] = {g.UseUniqueRegister(value),
g.UseUniqueRegister(value_high), g.UseUniqueRegister(value_high),
g.UseRegister(base), g.UseRegister(index)}; g.UseRegister(base), g.UseRegister(index)};
InstructionOperand outputs[] = { Node* projection0 = NodeProperties::FindProjection(node, 0);
g.DefineAsFixed(NodeProperties::FindProjection(node, 0), r2), Node* projection1 = NodeProperties::FindProjection(node, 1);
g.DefineAsFixed(NodeProperties::FindProjection(node, 1), r3)};
InstructionOperand temps[] = {g.TempRegister(), g.TempRegister(r6),
g.TempRegister(r7), g.TempRegister()};
InstructionCode code = opcode | AddressingModeField::encode(addressing_mode); InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs, if (projection1) {
arraysize(temps), temps); InstructionOperand outputs[] = {g.DefineAsFixed(projection0, r2),
g.DefineAsFixed(projection1, r3)};
InstructionOperand temps[] = {g.TempRegister(), g.TempRegister(r6),
g.TempRegister(r7), g.TempRegister()};
selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
arraysize(temps), temps);
} else if (projection0) {
InstructionOperand outputs[] = {g.DefineAsFixed(projection0, r2)};
InstructionOperand temps[] = {g.TempRegister(), g.TempRegister(r6),
g.TempRegister(r7), g.TempRegister(),
g.TempRegister(r3)};
selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
arraysize(temps), temps);
} else {
InstructionOperand temps[] = {g.TempRegister(), g.TempRegister(r6),
g.TempRegister(r7), g.TempRegister(),
g.TempRegister(r2), g.TempRegister(r3)};
selector->Emit(code, 0, nullptr, arraysize(inputs), inputs,
arraysize(temps), temps);
}
} }
} // namespace } // namespace
...@@ -2246,16 +2262,28 @@ void InstructionSelector::VisitWord32AtomicPairLoad(Node* node) { ...@@ -2246,16 +2262,28 @@ void InstructionSelector::VisitWord32AtomicPairLoad(Node* node) {
ArmOperandGenerator g(this); ArmOperandGenerator g(this);
Node* base = node->InputAt(0); Node* base = node->InputAt(0);
Node* index = node->InputAt(1); Node* index = node->InputAt(1);
InstructionOperand inputs[] = {g.UseRegister(base), g.UseRegister(index)};
InstructionOperand outputs[] = {
g.DefineAsFixed(NodeProperties::FindProjection(node, 0), r0),
g.DefineAsFixed(NodeProperties::FindProjection(node, 1), r1)};
InstructionOperand temps[] = {g.TempRegister()};
AddressingMode addressing_mode = kMode_Offset_RR; AddressingMode addressing_mode = kMode_Offset_RR;
InstructionCode code = InstructionCode code =
kArmWord32AtomicPairLoad | AddressingModeField::encode(addressing_mode); kArmWord32AtomicPairLoad | AddressingModeField::encode(addressing_mode);
Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs, InstructionOperand inputs[] = {g.UseRegister(base), g.UseRegister(index)};
arraysize(temps), temps); Node* projection0 = NodeProperties::FindProjection(node, 0);
Node* projection1 = NodeProperties::FindProjection(node, 1);
if (projection1) {
InstructionOperand outputs[] = {g.DefineAsFixed(projection0, r0),
g.DefineAsFixed(projection1, r1)};
InstructionOperand temps[] = {g.TempRegister()};
Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
arraysize(temps), temps);
} else if (projection0) {
InstructionOperand outputs[] = {g.DefineAsFixed(projection0, r0)};
InstructionOperand temps[] = {g.TempRegister(), g.TempRegister(r1)};
Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
arraysize(temps), temps);
} else {
InstructionOperand temps[] = {g.TempRegister(), g.TempRegister(r0),
g.TempRegister(r1)};
Emit(code, 0, nullptr, arraysize(inputs), inputs, arraysize(temps), temps);
}
} }
void InstructionSelector::VisitWord32AtomicPairStore(Node* node) { void InstructionSelector::VisitWord32AtomicPairStore(Node* node) {
...@@ -2305,14 +2333,27 @@ void InstructionSelector::VisitWord32AtomicPairExchange(Node* node) { ...@@ -2305,14 +2333,27 @@ void InstructionSelector::VisitWord32AtomicPairExchange(Node* node) {
InstructionOperand inputs[] = {g.UseFixed(value, r0), InstructionOperand inputs[] = {g.UseFixed(value, r0),
g.UseFixed(value_high, r1), g.UseFixed(value_high, r1),
g.UseRegister(base), g.UseRegister(index)}; g.UseRegister(base), g.UseRegister(index)};
InstructionOperand outputs[] = {
g.DefineAsFixed(NodeProperties::FindProjection(node, 0), r6),
g.DefineAsFixed(NodeProperties::FindProjection(node, 1), r7)};
InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
InstructionCode code = kArmWord32AtomicPairExchange | InstructionCode code = kArmWord32AtomicPairExchange |
AddressingModeField::encode(addressing_mode); AddressingModeField::encode(addressing_mode);
Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs, Node* projection0 = NodeProperties::FindProjection(node, 0);
arraysize(temps), temps); Node* projection1 = NodeProperties::FindProjection(node, 1);
if (projection1) {
InstructionOperand outputs[] = {g.DefineAsFixed(projection0, r6),
g.DefineAsFixed(projection1, r7)};
InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
arraysize(temps), temps);
} else if (projection0) {
InstructionOperand outputs[] = {g.DefineAsFixed(projection0, r6)};
InstructionOperand temps[] = {g.TempRegister(), g.TempRegister(),
g.TempRegister(r7)};
Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
arraysize(temps), temps);
} else {
InstructionOperand temps[] = {g.TempRegister(), g.TempRegister(),
g.TempRegister(r6), g.TempRegister(r7)};
Emit(code, 0, nullptr, arraysize(inputs), inputs, arraysize(temps), temps);
}
} }
void InstructionSelector::VisitWord32AtomicPairCompareExchange(Node* node) { void InstructionSelector::VisitWord32AtomicPairCompareExchange(Node* node) {
...@@ -2322,14 +2363,28 @@ void InstructionSelector::VisitWord32AtomicPairCompareExchange(Node* node) { ...@@ -2322,14 +2363,28 @@ void InstructionSelector::VisitWord32AtomicPairCompareExchange(Node* node) {
g.UseFixed(node->InputAt(2), r4), g.UseFixed(node->InputAt(3), r5), g.UseFixed(node->InputAt(2), r4), g.UseFixed(node->InputAt(3), r5),
g.UseFixed(node->InputAt(4), r8), g.UseFixed(node->InputAt(5), r9), g.UseFixed(node->InputAt(4), r8), g.UseFixed(node->InputAt(5), r9),
g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1))}; g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1))};
InstructionOperand outputs[] = {
g.DefineAsFixed(NodeProperties::FindProjection(node, 0), r2),
g.DefineAsFixed(NodeProperties::FindProjection(node, 1), r3)};
InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
InstructionCode code = kArmWord32AtomicPairCompareExchange | InstructionCode code = kArmWord32AtomicPairCompareExchange |
AddressingModeField::encode(addressing_mode); AddressingModeField::encode(addressing_mode);
Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs, Node* projection0 = NodeProperties::FindProjection(node, 0);
arraysize(temps), temps); Node* projection1 = NodeProperties::FindProjection(node, 1);
if (projection1) {
InstructionOperand outputs[] = {g.DefineAsFixed(projection0, r2),
g.DefineAsFixed(projection1, r3)};
InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
arraysize(temps), temps);
} else if (projection0) {
InstructionOperand outputs[] = {
g.DefineAsFixed(NodeProperties::FindProjection(node, 0), r2)};
InstructionOperand temps[] = {g.TempRegister(), g.TempRegister(),
g.TempRegister(r3)};
Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
arraysize(temps), temps);
} else {
InstructionOperand temps[] = {g.TempRegister(), g.TempRegister(),
g.TempRegister(r2), g.TempRegister(r3)};
Emit(code, 0, nullptr, arraysize(inputs), inputs, arraysize(temps), temps);
}
} }
#define SIMD_TYPE_LIST(V) \ #define SIMD_TYPE_LIST(V) \
......
src/compiler/ia32/code-generator-ia32.cc
View file @ 11245b26
...@@ -427,21 +427,21 @@ void EmitWordLoadPoisoningIfNeeded(CodeGenerator* codegen, ...@@ -427,21 +427,21 @@ void EmitWordLoadPoisoningIfNeeded(CodeGenerator* codegen,
__ j(not_equal, &binop); \ __ j(not_equal, &binop); \
} while (false) } while (false)
#define ASSEMBLE_I64ATOMIC_BINOP(instr1, instr2) \ #define ASSEMBLE_I64ATOMIC_BINOP(instr1, instr2) \
do { \ do { \
Label binop; \ Label binop; \
__ bind(&binop); \ __ bind(&binop); \
__ mov(i.OutputRegister(0), i.MemoryOperand(2)); \ __ mov(eax, i.MemoryOperand(2)); \
__ mov(i.OutputRegister(1), i.NextMemoryOperand(2)); \ __ mov(edx, i.NextMemoryOperand(2)); \
__ push(i.InputRegister(0)); \ __ push(i.InputRegister(0)); \
__ push(i.InputRegister(1)); \ __ push(i.InputRegister(1)); \
__ instr1(i.InputRegister(0), i.OutputRegister(0)); \ __ instr1(i.InputRegister(0), eax); \
__ instr2(i.InputRegister(1), i.OutputRegister(1)); \ __ instr2(i.InputRegister(1), edx); \
__ lock(); \ __ lock(); \
__ cmpxchg8b(i.MemoryOperand(2)); \ __ cmpxchg8b(i.MemoryOperand(2)); \
__ pop(i.InputRegister(1)); \ __ pop(i.InputRegister(1)); \
__ pop(i.InputRegister(0)); \ __ pop(i.InputRegister(0)); \
__ j(not_equal, &binop); \ __ j(not_equal, &binop); \
} while (false); } while (false);
#define ASSEMBLE_MOVX(mov_instr) \ #define ASSEMBLE_MOVX(mov_instr) \
...@@ -547,6 +547,18 @@ void AdjustStackPointerForTailCall(TurboAssembler* tasm, ...@@ -547,6 +547,18 @@ void AdjustStackPointerForTailCall(TurboAssembler* tasm,
} }
} }
#ifdef DEBUG
bool VerifyOutputOfAtomicPairInstr(IA32OperandConverter* converter,
const Instruction* instr) {
if (instr->OutputCount() > 0) {
if (converter->OutputRegister(0) != eax) return false;
if (instr->OutputCount() == 2 && converter->OutputRegister(1) != edx)
return false;
}
return true;
}
#endif
} // namespace } // namespace
void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr, void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
...@@ -3639,8 +3651,13 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( ...@@ -3639,8 +3651,13 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
case kIA32Word32AtomicPairLoad: { case kIA32Word32AtomicPairLoad: {
XMMRegister tmp = i.ToDoubleRegister(instr->TempAt(0)); XMMRegister tmp = i.ToDoubleRegister(instr->TempAt(0));
__ movq(tmp, i.MemoryOperand()); __ movq(tmp, i.MemoryOperand());
__ Pextrd(i.OutputRegister(0), tmp, 0); if (instr->OutputCount() == 2) {
__ Pextrd(i.OutputRegister(1), tmp, 1); __ Pextrd(i.OutputRegister(0), tmp, 0);
__ Pextrd(i.OutputRegister(1), tmp, 1);
} else if (instr->OutputCount() == 1) {
__ Pextrd(i.OutputRegister(0), tmp, 0);
__ Pextrd(i.TempRegister(1), tmp, 1);
}
break; break;
} }
case kIA32Word32AtomicPairStore: { case kIA32Word32AtomicPairStore: {
...@@ -3675,8 +3692,9 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( ...@@ -3675,8 +3692,9 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
break; break;
} }
case kIA32Word32AtomicPairExchange: { case kIA32Word32AtomicPairExchange: {
__ mov(i.OutputRegister(0), i.MemoryOperand(2)); DCHECK(VerifyOutputOfAtomicPairInstr(&i, instr));
__ mov(i.OutputRegister(1), i.NextMemoryOperand(2)); __ mov(eax, i.MemoryOperand(2));
__ mov(edx, i.NextMemoryOperand(2));
__ lock(); __ lock();
__ cmpxchg8b(i.MemoryOperand(2)); __ cmpxchg8b(i.MemoryOperand(2));
break; break;
...@@ -3746,10 +3764,11 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( ...@@ -3746,10 +3764,11 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
ATOMIC_BINOP_CASE(Or, or_) ATOMIC_BINOP_CASE(Or, or_)
ATOMIC_BINOP_CASE(Xor, xor_) ATOMIC_BINOP_CASE(Xor, xor_)
#undef ATOMIC_BINOP_CASE #undef ATOMIC_BINOP_CASE
#define ATOMIC_BINOP_CASE(op, instr1, instr2) \ #define ATOMIC_BINOP_CASE(op, instr1, instr2) \
case kIA32Word32AtomicPair##op: { \ case kIA32Word32AtomicPair##op: { \
ASSEMBLE_I64ATOMIC_BINOP(instr1, instr2) \ DCHECK(VerifyOutputOfAtomicPairInstr(&i, instr)); \
break; \ ASSEMBLE_I64ATOMIC_BINOP(instr1, instr2) \
break; \
} }
ATOMIC_BINOP_CASE(Add, add, adc) ATOMIC_BINOP_CASE(Add, add, adc)
ATOMIC_BINOP_CASE(And, and_, and_) ATOMIC_BINOP_CASE(And, and_, and_)
...@@ -3757,11 +3776,12 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( ...@@ -3757,11 +3776,12 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
ATOMIC_BINOP_CASE(Xor, xor_, xor_) ATOMIC_BINOP_CASE(Xor, xor_, xor_)
#undef ATOMIC_BINOP_CASE #undef ATOMIC_BINOP_CASE
case kIA32Word32AtomicPairSub: { case kIA32Word32AtomicPairSub: {
DCHECK(VerifyOutputOfAtomicPairInstr(&i, instr));
Label binop; Label binop;
__ bind(&binop); __ bind(&binop);
// Move memory operand into edx:eax // Move memory operand into edx:eax
__ mov(i.OutputRegister(0), i.MemoryOperand(2)); __ mov(eax, i.MemoryOperand(2));
__ mov(i.OutputRegister(1), i.NextMemoryOperand(2)); __ mov(edx, i.NextMemoryOperand(2));
// Save input registers temporarily on the stack. // Save input registers temporarily on the stack.
__ push(i.InputRegister(0)); __ push(i.InputRegister(0));
__ push(i.InputRegister(1)); __ push(i.InputRegister(1));
...@@ -3770,8 +3790,8 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( ...@@ -3770,8 +3790,8 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
__ adc(i.InputRegister(1), 0); __ adc(i.InputRegister(1), 0);
__ neg(i.InputRegister(1)); __ neg(i.InputRegister(1));
// Add memory operand, negated input. // Add memory operand, negated input.
__ add(i.InputRegister(0), i.OutputRegister(0)); __ add(i.InputRegister(0), eax);
__ adc(i.InputRegister(1), i.OutputRegister(1)); __ adc(i.InputRegister(1), edx);
__ lock(); __ lock();
__ cmpxchg8b(i.MemoryOperand(2)); __ cmpxchg8b(i.MemoryOperand(2));
// Restore input registers // Restore input registers
......
src/compiler/ia32/instruction-selector-ia32.cc
View file @ 11245b26
...@@ -1351,11 +1351,24 @@ void VisitPairAtomicBinOp(InstructionSelector* selector, Node* node, ...@@ -1351,11 +1351,24 @@ void VisitPairAtomicBinOp(InstructionSelector* selector, Node* node,
g.UseFixed(value, ebx), g.UseFixed(value_high, ecx), g.UseFixed(value, ebx), g.UseFixed(value_high, ecx),
g.UseUniqueRegister(base), g.UseUniqueRegister(base),
g.GetEffectiveIndexOperand(index, &addressing_mode)}; g.GetEffectiveIndexOperand(index, &addressing_mode)};
InstructionOperand outputs[] = {
g.DefineAsFixed(NodeProperties::FindProjection(node, 0), eax),
g.DefineAsFixed(NodeProperties::FindProjection(node, 1), edx)};
InstructionCode code = opcode | AddressingModeField::encode(addressing_mode); InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs); Node* projection0 = NodeProperties::FindProjection(node, 0);
Node* projection1 = NodeProperties::FindProjection(node, 1);
if (projection1) {
InstructionOperand outputs[] = {g.DefineAsFixed(projection0, eax),
g.DefineAsFixed(projection1, edx)};
selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs),
inputs);
} else if (projection0) {
InstructionOperand outputs[] = {g.DefineAsFixed(projection0, eax)};
InstructionOperand temps[] = {g.TempRegister(edx)};
selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
arraysize(temps), temps);
} else {
InstructionOperand temps[] = {g.TempRegister(eax), g.TempRegister(edx)};
selector->Emit(code, 0, nullptr, arraysize(inputs), inputs,
arraysize(temps), temps);
}
} }
} // namespace } // namespace
...@@ -1745,14 +1758,27 @@ void InstructionSelector::VisitWord32AtomicPairLoad(Node* node) { ...@@ -1745,14 +1758,27 @@ void InstructionSelector::VisitWord32AtomicPairLoad(Node* node) {
Node* index = node->InputAt(1); Node* index = node->InputAt(1);
InstructionOperand inputs[] = {g.UseUniqueRegister(base), InstructionOperand inputs[] = {g.UseUniqueRegister(base),
g.GetEffectiveIndexOperand(index, &mode)}; g.GetEffectiveIndexOperand(index, &mode)};
InstructionOperand temps[] = {g.TempDoubleRegister()}; Node* projection0 = NodeProperties::FindProjection(node, 0);
InstructionOperand outputs[] = { Node* projection1 = NodeProperties::FindProjection(node, 1);
g.DefineAsRegister(NodeProperties::FindProjection(node, 0)),
g.DefineAsRegister(NodeProperties::FindProjection(node, 1))};
InstructionCode code = InstructionCode code =
kIA32Word32AtomicPairLoad | AddressingModeField::encode(mode); kIA32Word32AtomicPairLoad | AddressingModeField::encode(mode);
Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
arraysize(temps), temps); if (projection1) {
InstructionOperand temps[] = {g.TempDoubleRegister()};
InstructionOperand outputs[] = {g.DefineAsRegister(projection0),
g.DefineAsRegister(projection1)};
Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
arraysize(temps), temps);
} else if (projection0) {
InstructionOperand temps[] = {g.TempDoubleRegister(), g.TempRegister()};
InstructionOperand outputs[] = {g.DefineAsRegister(projection0)};
Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
arraysize(temps), temps);
} else {
InstructionOperand temps[] = {g.TempDoubleRegister(), g.TempRegister(),
g.TempRegister()};
Emit(code, 0, nullptr, arraysize(inputs), inputs, arraysize(temps), temps);
}
} }
void InstructionSelector::VisitWord32AtomicPairStore(Node* node) { void InstructionSelector::VisitWord32AtomicPairStore(Node* node) {
...@@ -1812,12 +1838,23 @@ void InstructionSelector::VisitWord32AtomicPairCompareExchange(Node* node) { ...@@ -1812,12 +1838,23 @@ void InstructionSelector::VisitWord32AtomicPairCompareExchange(Node* node) {
// InputAt(0) => base // InputAt(0) => base
g.UseUniqueRegister(node->InputAt(0)), g.UseUniqueRegister(node->InputAt(0)),
g.GetEffectiveIndexOperand(index, &addressing_mode)}; g.GetEffectiveIndexOperand(index, &addressing_mode)};
InstructionOperand outputs[] = { Node* projection0 = NodeProperties::FindProjection(node, 0);
g.DefineAsFixed(NodeProperties::FindProjection(node, 0), eax), Node* projection1 = NodeProperties::FindProjection(node, 1);
g.DefineAsFixed(NodeProperties::FindProjection(node, 1), edx)};
InstructionCode code = kIA32Word32AtomicPairCompareExchange | InstructionCode code = kIA32Word32AtomicPairCompareExchange |
AddressingModeField::encode(addressing_mode); AddressingModeField::encode(addressing_mode);
Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs); if (projection1) {
InstructionOperand outputs[] = {g.DefineAsFixed(projection0, eax),
g.DefineAsFixed(projection1, edx)};
Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs);
} else if (projection0) {
InstructionOperand outputs[] = {g.DefineAsFixed(projection0, eax)};
InstructionOperand temps[] = {g.TempRegister(edx)};
Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
arraysize(temps), temps);
} else {
InstructionOperand temps[] = {g.TempRegister(eax), g.TempRegister(edx)};
Emit(code, 0, nullptr, arraysize(inputs), inputs, arraysize(temps), temps);
}
} }
#define SIMD_INT_TYPES(V) \ #define SIMD_INT_TYPES(V) \
......
test/cctest/wasm/test-run-wasm-atomics64.cc
View file @ 11245b26
...@@ -354,6 +354,172 @@ WASM_EXEC_TEST(I64AtomicStoreLoad8U) { ...@@ -354,6 +354,172 @@ WASM_EXEC_TEST(I64AtomicStoreLoad8U) {
} }
} }
// Drop tests verify atomic operations are run correctly when the
// entire 64-bit output is optimized out
void RunDropTest(ExecutionTier execution_tier, WasmOpcode wasm_op,
Uint64BinOp op) {
EXPERIMENTAL_FLAG_SCOPE(threads);
WasmRunner<uint64_t, uint64_t> r(execution_tier);
uint64_t* memory =
r.builder().AddMemoryElems<uint64_t>(kWasmPageSize / sizeof(uint64_t));
r.builder().SetHasSharedMemory();
BUILD(r,
WASM_ATOMICS_BINOP(wasm_op, WASM_I32V_1(0), WASM_GET_LOCAL(0),
MachineRepresentation::kWord64),
WASM_DROP, WASM_GET_LOCAL(0));
uint64_t initial = 0x1111222233334444, local = 0x1111111111111111;
r.builder().WriteMemory(&memory[0], initial);
CHECK_EQ(local, r.Call(local));
uint64_t expected = op(initial, local);
CHECK_EQ(expected, r.builder().ReadMemory(&memory[0]));
}
#define TEST_OPERATION(Name) \
WASM_EXEC_TEST(I64Atomic##Name##Drop) { \
RunDropTest(execution_tier, kExprI64Atomic##Name, Name); \
}
OPERATION_LIST(TEST_OPERATION)
#undef TEST_OPERATION
WASM_EXEC_TEST(I64AtomicSub16UDrop) {
EXPERIMENTAL_FLAG_SCOPE(threads);
WasmRunner<uint64_t, uint64_t> r(execution_tier);
uint16_t* memory =
r.builder().AddMemoryElems<uint16_t>(kWasmPageSize / sizeof(uint16_t));
r.builder().SetHasSharedMemory();
BUILD(r,
WASM_ATOMICS_BINOP(kExprI64AtomicSub16U, WASM_I32V_1(0),
WASM_GET_LOCAL(0), MachineRepresentation::kWord16),
WASM_DROP, WASM_GET_LOCAL(0));
uint16_t initial = 0x7, local = 0xffe0;
r.builder().WriteMemory(&memory[0], initial);
CHECK_EQ(local, r.Call(local));
uint16_t expected = Sub(initial, local);
CHECK_EQ(expected, r.builder().ReadMemory(&memory[0]));
}
WASM_EXEC_TEST(I64AtomicCompareExchangeDrop) {
EXPERIMENTAL_FLAG_SCOPE(threads);
WasmRunner<uint64_t, uint64_t, uint64_t> r(execution_tier);
r.builder().SetHasSharedMemory();
uint64_t* memory =
r.builder().AddMemoryElems<uint64_t>(kWasmPageSize / sizeof(uint64_t));
BUILD(r,
WASM_ATOMICS_TERNARY_OP(kExprI64AtomicCompareExchange, WASM_I32V_1(0),
WASM_GET_LOCAL(0), WASM_GET_LOCAL(1),
MachineRepresentation::kWord64),
WASM_DROP, WASM_GET_LOCAL(1));
uint64_t initial = 0x1111222233334444, local = 0x1111111111111111;
r.builder().WriteMemory(&memory[0], initial);
CHECK_EQ(local, r.Call(initial, local));
uint64_t expected = CompareExchange(initial, initial, local);
CHECK_EQ(expected, r.builder().ReadMemory(&memory[0]));
}
WASM_EXEC_TEST(I64AtomicStoreLoadDrop) {
EXPERIMENTAL_FLAG_SCOPE(threads);
WasmRunner<uint64_t, uint64_t, uint64_t> r(execution_tier);
r.builder().SetHasSharedMemory();
uint64_t* memory =
r.builder().AddMemoryElems<uint64_t>(kWasmPageSize / sizeof(uint64_t));
BUILD(r,
WASM_ATOMICS_STORE_OP(kExprI64AtomicStore, WASM_ZERO, WASM_GET_LOCAL(0),
MachineRepresentation::kWord64),
WASM_ATOMICS_LOAD_OP(kExprI64AtomicLoad, WASM_ZERO,
MachineRepresentation::kWord64),
WASM_DROP, WASM_GET_LOCAL(1));
uint64_t store_value = 0x1111111111111111, expected = 0xC0DE;
CHECK_EQ(expected, r.Call(store_value, expected));
CHECK_EQ(store_value, r.builder().ReadMemory(&memory[0]));
}
WASM_EXEC_TEST(I64AtomicAddConvertDrop) {
EXPERIMENTAL_FLAG_SCOPE(threads);
WasmRunner<uint64_t, uint64_t> r(execution_tier);
uint64_t* memory =
r.builder().AddMemoryElems<uint64_t>(kWasmPageSize / sizeof(uint64_t));
r.builder().SetHasSharedMemory();
BUILD(r,
WASM_ATOMICS_BINOP(kExprI64AtomicAdd, WASM_I32V_1(0), WASM_GET_LOCAL(0),
MachineRepresentation::kWord64),
kExprI32ConvertI64, WASM_DROP, WASM_GET_LOCAL(0));
uint64_t initial = 0x1111222233334444, local = 0x1111111111111111;
r.builder().WriteMemory(&memory[0], initial);
CHECK_EQ(local, r.Call(local));
uint64_t expected = Add(initial, local);
CHECK_EQ(expected, r.builder().ReadMemory(&memory[0]));
}
WASM_EXEC_TEST(I64AtomicLoadConvertDrop) {
EXPERIMENTAL_FLAG_SCOPE(threads);
WasmRunner<uint32_t, uint64_t> r(execution_tier);
uint64_t* memory =
r.builder().AddMemoryElems<uint64_t>(kWasmPageSize / sizeof(uint64_t));
r.builder().SetHasSharedMemory();
BUILD(r, WASM_I32_CONVERT_I64(WASM_ATOMICS_LOAD_OP(
kExprI64AtomicLoad, WASM_ZERO, MachineRepresentation::kWord64)));
uint64_t initial = 0x1111222233334444;
r.builder().WriteMemory(&memory[0], initial);
CHECK_EQ(static_cast<uint32_t>(initial), r.Call(initial));
}
// Convert tests verify atomic operations are run correctly when the
// upper half of the 64-bit output is optimized out
void RunConvertTest(ExecutionTier execution_tier, WasmOpcode wasm_op,
Uint64BinOp op) {
EXPERIMENTAL_FLAG_SCOPE(threads);
WasmRunner<uint32_t, uint64_t> r(execution_tier);
uint64_t* memory =
r.builder().AddMemoryElems<uint64_t>(kWasmPageSize / sizeof(uint64_t));
r.builder().SetHasSharedMemory();
BUILD(r, WASM_I32_CONVERT_I64(WASM_ATOMICS_BINOP(
kExprI64AtomicAdd, WASM_ZERO, WASM_GET_LOCAL(0),
MachineRepresentation::kWord64)));
uint64_t initial = 0x1111222233334444, local = 0x1111111111111111;
r.builder().WriteMemory(&memory[0], initial);
CHECK_EQ(static_cast<uint32_t>(initial), r.Call(local));
uint64_t expected = Add(initial, local);
CHECK_EQ(expected, r.builder().ReadMemory(&memory[0]));
}
#define TEST_OPERATION(Name) \
WASM_EXEC_TEST(I64AtomicConvert##Name) { \
RunConvertTest(execution_tier, kExprI64Atomic##Name, Name); \
}
OPERATION_LIST(TEST_OPERATION)
#undef TEST_OPERATION
WASM_EXEC_TEST(I64AtomicConvertCompareExchange) {
EXPERIMENTAL_FLAG_SCOPE(threads);
WasmRunner<uint32_t, uint64_t, uint64_t> r(execution_tier);
uint64_t* memory =
r.builder().AddMemoryElems<uint64_t>(kWasmPageSize / sizeof(uint64_t));
r.builder().SetHasSharedMemory();
BUILD(r, WASM_I32_CONVERT_I64(WASM_ATOMICS_TERNARY_OP(
kExprI64AtomicCompareExchange, WASM_I32V_1(0), WASM_GET_LOCAL(0),
WASM_GET_LOCAL(1), MachineRepresentation::kWord64)));
uint64_t initial = 0x1111222233334444, local = 0x1111111111111111;
r.builder().WriteMemory(&memory[0], initial);
CHECK_EQ(static_cast<uint32_t>(initial), r.Call(initial, local));
uint64_t expected = CompareExchange(initial, initial, local);
CHECK_EQ(expected, r.builder().ReadMemory(&memory[0]));
}
} // namespace test_run_wasm_atomics_64 } // namespace test_run_wasm_atomics_64
} // namespace wasm } // namespace wasm
} // namespace internal } // namespace internal
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment