Skip to content

V
V8
Commit 51025557 authored by Milad Fa's avatar Milad Fa Committed by V8 LUCI CQ
Browse files
Options

S390: Support Wasm atomic ops on big endian within turbofan 

Currently atomic ops on TF are using machine native byte order
and cannot be used by Wasm calls.

This Cl adds support for Little Endian enforced Wasm atomic ops
to S390 by reversing bytes where needed.

This CL does not change the behaviour on S390 simulator.

Change-Id: Iedb2c05a55f495409ee21a76713bf15e21108997
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3313444
Commit-Queue: Milad Farazmand <mfarazma@redhat.com>
Reviewed-by: 's avatarJunliang Yan <junyan@redhat.com>
Cr-Commit-Position: refs/heads/main@{#78254}
parent 6e2078d6
Show whitespace changes
Inline Side-by-side
Showing with 238 additions and 113 deletions
src/compiler/backend/s390/code-generator-s390.cc
View file @ 51025557
......@@ -734,6 +734,14 @@ static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) {
__ asm_instr(value, operand); \
} while (0)
static inline bool is_wasm_on_be(bool IsWasm) {
#if V8_TARGET_BIG_ENDIAN
return IsWasm;
#else
return false;
#endif
}
#define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_BYTE(load_and_ext) \
do { \
Register old_value = i.InputRegister(0); \
......@@ -762,7 +770,24 @@ static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) {
AddressingMode mode = kMode_None; \
MemOperand op = i.MemoryOperand(&mode, &index); \
__ lay(addr, op); \
__ AtomicCmpExchangeU16(addr, output, old_value, new_value, temp0, temp1); \
if (is_wasm_on_be(info()->IsWasm())) { \
Register temp2 = \
GetRegisterThatIsNotOneOf(output, old_value, new_value); \
Register temp3 = \
GetRegisterThatIsNotOneOf(output, old_value, new_value, temp2); \
__ Push(temp2, temp3); \
__ lrvr(temp2, old_value); \
__ lrvr(temp3, new_value); \
__ ShiftRightU32(temp2, temp2, Operand(16)); \
__ ShiftRightU32(temp3, temp3, Operand(16)); \
__ AtomicCmpExchangeU16(addr, output, temp2, temp3, temp0, temp1); \
__ lrvr(output, output); \
__ ShiftRightU32(output, output, Operand(16)); \
__ Pop(temp2, temp3); \
} else { \
__ AtomicCmpExchangeU16(addr, output, old_value, new_value, temp0, \
temp1); \
} \
__ load_and_ext(output, output); \
} while (false)
......@@ -775,11 +800,18 @@ static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) {
AddressingMode mode = kMode_None; \
MemOperand op = i.MemoryOperand(&mode, &index); \
__ lay(addr, op); \
if (is_wasm_on_be(info()->IsWasm())) { \
__ lrvr(r0, output); \
__ lrvr(r1, new_val); \
__ CmpAndSwap(r0, r1, MemOperand(addr)); \
__ lrvr(output, r0); \
} else { \
__ CmpAndSwap(output, new_val, MemOperand(addr)); \
} \
__ LoadU32(output, output); \
} while (false)
#define ASSEMBLE_ATOMIC_BINOP_WORD(load_and_op) \
#define ASSEMBLE_ATOMIC_BINOP_WORD(load_and_op, op) \
do { \
Register value = i.InputRegister(2); \
Register result = i.OutputRegister(0); \
......@@ -787,11 +819,23 @@ static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) {
AddressingMode mode = kMode_None; \
MemOperand op = i.MemoryOperand(&mode); \
__ lay(addr, op); \
if (is_wasm_on_be(info()->IsWasm())) { \
Label do_cs; \
__ bind(&do_cs); \
__ LoadU32(r0, MemOperand(addr)); \
__ lrvr(ip, r0); \
__ op(ip, ip, value); \
__ lrvr(ip, ip); \
__ CmpAndSwap(r0, ip, MemOperand(addr)); \
__ bne(&do_cs, Label::kNear); \
__ lrvr(result, r0); \
} else { \
__ load_and_op(result, value, MemOperand(addr)); \
} \
__ LoadU32(result, result); \
} while (false)
#define ASSEMBLE_ATOMIC_BINOP_WORD64(load_and_op) \
#define ASSEMBLE_ATOMIC_BINOP_WORD64(load_and_op, op) \
do { \
Register value = i.InputRegister(2); \
Register result = i.OutputRegister(0); \
......@@ -799,14 +843,49 @@ static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) {
AddressingMode mode = kMode_None; \
MemOperand op = i.MemoryOperand(&mode); \
__ lay(addr, op); \
if (is_wasm_on_be(info()->IsWasm())) { \
Label do_cs; \
__ bind(&do_cs); \
__ LoadU64(r0, MemOperand(addr)); \
__ lrvgr(ip, r0); \
__ op(ip, ip, value); \
__ lrvgr(ip, ip); \
__ CmpAndSwap64(r0, ip, MemOperand(addr)); \
__ bne(&do_cs, Label::kNear); \
__ lrvgr(result, r0); \
break; \
} \
__ load_and_op(result, value, MemOperand(addr)); \
} while (false)
#define ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end) \
#define ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end, \
maybe_reverse_bytes) \
do { \
/* At the moment this is only true when dealing with 2-byte values.*/ \
bool reverse_bytes = \
maybe_reverse_bytes && is_wasm_on_be(info()->IsWasm()); \
USE(reverse_bytes); \
Label do_cs; \
__ LoadU32(prev, MemOperand(addr, offset)); \
__ bind(&do_cs); \
if (reverse_bytes) { \
Register temp2 = GetRegisterThatIsNotOneOf(value, result, prev); \
__ Push(temp2); \
__ lrvr(temp2, prev); \
__ RotateInsertSelectBits(temp2, temp2, Operand(start), Operand(end), \
Operand(static_cast<intptr_t>(shift_amount)), \
true); \
__ RotateInsertSelectBits(temp, value, Operand(start), Operand(end), \
Operand(static_cast<intptr_t>(shift_amount)), \
true); \
__ bin_inst(new_val, temp2, temp); \
__ lrvr(temp2, new_val); \
__ lr(temp, prev); \
__ RotateInsertSelectBits(temp, temp2, Operand(start), Operand(end), \
Operand(static_cast<intptr_t>(shift_amount)), \
false); \
__ Pop(temp2); \
} else { \
__ RotateInsertSelectBits(temp, value, Operand(start), Operand(end), \
Operand(static_cast<intptr_t>(shift_amount)), \
true); \
......@@ -814,6 +893,7 @@ static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) {
__ lr(temp, prev); \
__ RotateInsertSelectBits(temp, new_val, Operand(start), Operand(end), \
Operand::Zero(), false); \
} \
__ CmpAndSwap(prev, temp, MemOperand(addr, offset)); \
__ bne(&do_cs, Label::kNear); \
} while (false)
......@@ -825,7 +905,7 @@ static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) {
constexpr int shift_amount = 16 - (index * 16); \
constexpr int start = 48 - shift_amount; \
constexpr int end = start + 15; \
ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end); \
ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end, true); \
extract_result(); \
}
#define ATOMIC_BIN_OP_BYTE(bin_inst, index, extract_result) \
......@@ -834,7 +914,7 @@ static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) {
constexpr int shift_amount = 24 - (index * 8); \
constexpr int start = 56 - shift_amount; \
constexpr int end = start + 7; \
ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end); \
ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end, false); \
extract_result(); \
}
#else
......@@ -844,7 +924,7 @@ static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) {
constexpr int shift_amount = index * 16; \
constexpr int start = 48 - shift_amount; \
constexpr int end = start + 15; \
ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end); \
ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end, false); \
extract_result(); \
}
#define ATOMIC_BIN_OP_BYTE(bin_inst, index, extract_result) \
......@@ -853,7 +933,7 @@ static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) {
constexpr int shift_amount = index * 8; \
constexpr int start = 56 - shift_amount; \
constexpr int end = start + 7; \
ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end); \
ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end, false); \
extract_result(); \
}
#endif // V8_TARGET_BIG_ENDIAN
......@@ -923,7 +1003,14 @@ static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) {
AddressingMode mode = kMode_None; \
MemOperand op = i.MemoryOperand(&mode, &index); \
__ lay(addr, op); \
if (is_wasm_on_be(info()->IsWasm())) { \
__ lrvgr(r0, output); \
__ lrvgr(r1, new_val); \
__ CmpAndSwap64(r0, r1, MemOperand(addr)); \
__ lrvgr(output, r0); \
} else { \
__ CmpAndSwap64(output, new_val, MemOperand(addr)); \
} \
} while (false)
void CodeGenerator::AssembleDeconstructFrame() {
......@@ -2308,13 +2395,24 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
Register index = i.InputRegister(1);
Register value = i.InputRegister(2);
Register output = i.OutputRegister();
bool reverse_bytes = is_wasm_on_be(info()->IsWasm());
__ la(r1, MemOperand(base, index));
__ AtomicExchangeU16(r1, value, output, r0);
Register value_ = value;
if (reverse_bytes) {
value_ = ip;
__ lrvr(value_, value);
__ ShiftRightU32(value_, value_, Operand(16));
}
__ AtomicExchangeU16(r1, value_, output, r0);
if (opcode == kAtomicExchangeInt16) {
__ lghr(output, output);
} else {
__ llghr(output, output);
}
if (reverse_bytes) {
__ lrvr(output, output);
__ ShiftRightU32(output, output, Operand(16));
}
break;
}
case kAtomicExchangeWord32: {
......@@ -2323,11 +2421,21 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
Register value = i.InputRegister(2);
Register output = i.OutputRegister();
Label do_cs;
bool reverse_bytes = is_wasm_on_be(info()->IsWasm());
__ lay(r1, MemOperand(base, index));
Register value_ = value;
if (reverse_bytes) {
value_ = ip;
__ lrvr(value_, value);
}
__ LoadU32(output, MemOperand(r1));
__ bind(&do_cs);
__ cs(output, value, MemOperand(r1));
__ cs(output, value_, MemOperand(r1));
__ bne(&do_cs, Label::kNear);
if (reverse_bytes) {
__ lrvr(output, output);
__ LoadU32(output, output);
}
break;
}
case kAtomicCompareExchangeInt8:
......@@ -2366,6 +2474,10 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
intptr_t shift_right = static_cast<intptr_t>(shift_amount); \
__ srlk(result, prev, Operand(shift_right)); \
__ LoadS16(result, result); \
if (is_wasm_on_be(info()->IsWasm())) { \
__ lrvr(result, result); \
__ ShiftRightS32(result, result, Operand(16)); \
} \
}); \
break; \
case kAtomic##op##Uint16: \
......@@ -2374,6 +2486,10 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
__ RotateInsertSelectBits(result, prev, Operand(48), Operand(63), \
Operand(static_cast<intptr_t>(rotate_left)), \
true); \
if (is_wasm_on_be(info()->IsWasm())) { \
__ lrvr(result, result); \
__ ShiftRightU32(result, result, Operand(16)); \
} \
}); \
break;
ATOMIC_BINOP_CASE(Add, AddS32)
......@@ -2383,46 +2499,55 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
ATOMIC_BINOP_CASE(Xor, Xor)
#undef ATOMIC_BINOP_CASE
case kAtomicAddWord32:
ASSEMBLE_ATOMIC_BINOP_WORD(laa);
ASSEMBLE_ATOMIC_BINOP_WORD(laa, AddS32);
break;
case kAtomicSubWord32:
ASSEMBLE_ATOMIC_BINOP_WORD(LoadAndSub32);
ASSEMBLE_ATOMIC_BINOP_WORD(LoadAndSub32, SubS32);
break;
case kAtomicAndWord32:
ASSEMBLE_ATOMIC_BINOP_WORD(lan);
ASSEMBLE_ATOMIC_BINOP_WORD(lan, AndP);
break;
case kAtomicOrWord32:
ASSEMBLE_ATOMIC_BINOP_WORD(lao);
ASSEMBLE_ATOMIC_BINOP_WORD(lao, OrP);
break;
case kAtomicXorWord32:
ASSEMBLE_ATOMIC_BINOP_WORD(lax);
ASSEMBLE_ATOMIC_BINOP_WORD(lax, XorP);
break;
case kS390_Word64AtomicAddUint64:
ASSEMBLE_ATOMIC_BINOP_WORD64(laag);
ASSEMBLE_ATOMIC_BINOP_WORD64(laag, AddS64);
break;
case kS390_Word64AtomicSubUint64:
ASSEMBLE_ATOMIC_BINOP_WORD64(LoadAndSub64);
ASSEMBLE_ATOMIC_BINOP_WORD64(LoadAndSub64, SubS64);
break;
case kS390_Word64AtomicAndUint64:
ASSEMBLE_ATOMIC_BINOP_WORD64(lang);
ASSEMBLE_ATOMIC_BINOP_WORD64(lang, AndP);
break;
case kS390_Word64AtomicOrUint64:
ASSEMBLE_ATOMIC_BINOP_WORD64(laog);
ASSEMBLE_ATOMIC_BINOP_WORD64(laog, OrP);
break;
case kS390_Word64AtomicXorUint64:
ASSEMBLE_ATOMIC_BINOP_WORD64(laxg);
ASSEMBLE_ATOMIC_BINOP_WORD64(laxg, XorP);
break;
case kS390_Word64AtomicExchangeUint64: {
Register base = i.InputRegister(0);
Register index = i.InputRegister(1);
Register value = i.InputRegister(2);
Register output = i.OutputRegister();
bool reverse_bytes = is_wasm_on_be(info()->IsWasm());
Label do_cs;
Register value_ = value;
__ la(r1, MemOperand(base, index));
if (reverse_bytes) {
value_ = ip;
__ lrvgr(value_, value);
}
__ lg(output, MemOperand(r1));
__ bind(&do_cs);
__ csg(output, value, MemOperand(r1));
__ csg(output, value_, MemOperand(r1));
__ bne(&do_cs, Label::kNear);
if (reverse_bytes) {
__ lrvgr(output, output);
}
break;
}
case kS390_Word64AtomicCompareExchangeUint64:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment