Skip to content

V
V8
Commit 51025557 authored by Milad Fa's avatar Milad Fa Committed by V8 LUCI CQ
Browse files
Options

S390: Support Wasm atomic ops on big endian within turbofan 

Currently atomic ops on TF are using machine native byte order
and cannot be used by Wasm calls.

This Cl adds support for Little Endian enforced Wasm atomic ops
to S390 by reversing bytes where needed.

This CL does not change the behaviour on S390 simulator.

Change-Id: Iedb2c05a55f495409ee21a76713bf15e21108997
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3313444
Commit-Queue: Milad Farazmand <mfarazma@redhat.com>
Reviewed-by: 's avatarJunliang Yan <junyan@redhat.com>
Cr-Commit-Position: refs/heads/main@{#78254}
parent 6e2078d6
Hide whitespace changes
Inline Side-by-side
Showing with 238 additions and 113 deletions
src/compiler/backend/s390/code-generator-s390.cc
View file @ 51025557
...@@ -734,6 +734,14 @@ static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) { ...@@ -734,6 +734,14 @@ static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) {
__ asm_instr(value, operand); \ __ asm_instr(value, operand); \
} while (0) } while (0)
static inline bool is_wasm_on_be(bool IsWasm) {
#if V8_TARGET_BIG_ENDIAN
return IsWasm;
#else
return false;
#endif
}
#define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_BYTE(load_and_ext) \ #define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_BYTE(load_and_ext) \
do { \ do { \
Register old_value = i.InputRegister(0); \ Register old_value = i.InputRegister(0); \
...@@ -750,111 +758,183 @@ static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) { ...@@ -750,111 +758,183 @@ static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) {
__ load_and_ext(output, output); \ __ load_and_ext(output, output); \
} while (false) } while (false)
#define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_HALFWORD(load_and_ext) \ #define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_HALFWORD(load_and_ext) \
do { \ do { \
Register old_value = i.InputRegister(0); \ Register old_value = i.InputRegister(0); \
Register new_value = i.InputRegister(1); \ Register new_value = i.InputRegister(1); \
Register output = i.OutputRegister(); \ Register output = i.OutputRegister(); \
Register addr = kScratchReg; \ Register addr = kScratchReg; \
Register temp0 = r0; \ Register temp0 = r0; \
Register temp1 = r1; \ Register temp1 = r1; \
size_t index = 2; \ size_t index = 2; \
AddressingMode mode = kMode_None; \ AddressingMode mode = kMode_None; \
MemOperand op = i.MemoryOperand(&mode, &index); \ MemOperand op = i.MemoryOperand(&mode, &index); \
__ lay(addr, op); \ __ lay(addr, op); \
__ AtomicCmpExchangeU16(addr, output, old_value, new_value, temp0, temp1); \ if (is_wasm_on_be(info()->IsWasm())) { \
__ load_and_ext(output, output); \ Register temp2 = \
GetRegisterThatIsNotOneOf(output, old_value, new_value); \
Register temp3 = \
GetRegisterThatIsNotOneOf(output, old_value, new_value, temp2); \
__ Push(temp2, temp3); \
__ lrvr(temp2, old_value); \
__ lrvr(temp3, new_value); \
__ ShiftRightU32(temp2, temp2, Operand(16)); \
__ ShiftRightU32(temp3, temp3, Operand(16)); \
__ AtomicCmpExchangeU16(addr, output, temp2, temp3, temp0, temp1); \
__ lrvr(output, output); \
__ ShiftRightU32(output, output, Operand(16)); \
__ Pop(temp2, temp3); \
} else { \
__ AtomicCmpExchangeU16(addr, output, old_value, new_value, temp0, \
temp1); \
} \
__ load_and_ext(output, output); \
} while (false) } while (false)
#define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_WORD() \ #define ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_WORD() \
do { \ do { \
Register new_val = i.InputRegister(1); \ Register new_val = i.InputRegister(1); \
Register output = i.OutputRegister(); \ Register output = i.OutputRegister(); \
Register addr = kScratchReg; \ Register addr = kScratchReg; \
size_t index = 2; \ size_t index = 2; \
AddressingMode mode = kMode_None; \ AddressingMode mode = kMode_None; \
MemOperand op = i.MemoryOperand(&mode, &index); \ MemOperand op = i.MemoryOperand(&mode, &index); \
__ lay(addr, op); \ __ lay(addr, op); \
__ CmpAndSwap(output, new_val, MemOperand(addr)); \ if (is_wasm_on_be(info()->IsWasm())) { \
__ LoadU32(output, output); \ __ lrvr(r0, output); \
__ lrvr(r1, new_val); \
__ CmpAndSwap(r0, r1, MemOperand(addr)); \
__ lrvr(output, r0); \
} else { \
__ CmpAndSwap(output, new_val, MemOperand(addr)); \
} \
__ LoadU32(output, output); \
} while (false) } while (false)
#define ASSEMBLE_ATOMIC_BINOP_WORD(load_and_op) \ #define ASSEMBLE_ATOMIC_BINOP_WORD(load_and_op, op) \
do { \ do { \
Register value = i.InputRegister(2); \ Register value = i.InputRegister(2); \
Register result = i.OutputRegister(0); \ Register result = i.OutputRegister(0); \
Register addr = r1; \ Register addr = r1; \
AddressingMode mode = kMode_None; \ AddressingMode mode = kMode_None; \
MemOperand op = i.MemoryOperand(&mode); \ MemOperand op = i.MemoryOperand(&mode); \
__ lay(addr, op); \ __ lay(addr, op); \
__ load_and_op(result, value, MemOperand(addr)); \ if (is_wasm_on_be(info()->IsWasm())) { \
__ LoadU32(result, result); \ Label do_cs; \
__ bind(&do_cs); \
__ LoadU32(r0, MemOperand(addr)); \
__ lrvr(ip, r0); \
__ op(ip, ip, value); \
__ lrvr(ip, ip); \
__ CmpAndSwap(r0, ip, MemOperand(addr)); \
__ bne(&do_cs, Label::kNear); \
__ lrvr(result, r0); \
} else { \
__ load_and_op(result, value, MemOperand(addr)); \
} \
__ LoadU32(result, result); \
} while (false) } while (false)
#define ASSEMBLE_ATOMIC_BINOP_WORD64(load_and_op) \ #define ASSEMBLE_ATOMIC_BINOP_WORD64(load_and_op, op) \
do { \ do { \
Register value = i.InputRegister(2); \ Register value = i.InputRegister(2); \
Register result = i.OutputRegister(0); \ Register result = i.OutputRegister(0); \
Register addr = r1; \ Register addr = r1; \
AddressingMode mode = kMode_None; \ AddressingMode mode = kMode_None; \
MemOperand op = i.MemoryOperand(&mode); \ MemOperand op = i.MemoryOperand(&mode); \
__ lay(addr, op); \ __ lay(addr, op); \
__ load_and_op(result, value, MemOperand(addr)); \ if (is_wasm_on_be(info()->IsWasm())) { \
Label do_cs; \
__ bind(&do_cs); \
__ LoadU64(r0, MemOperand(addr)); \
__ lrvgr(ip, r0); \
__ op(ip, ip, value); \
__ lrvgr(ip, ip); \
__ CmpAndSwap64(r0, ip, MemOperand(addr)); \
__ bne(&do_cs, Label::kNear); \
__ lrvgr(result, r0); \
break; \
} \
__ load_and_op(result, value, MemOperand(addr)); \
} while (false) } while (false)
#define ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end) \ #define ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end, \
do { \ maybe_reverse_bytes) \
Label do_cs; \ do { \
__ LoadU32(prev, MemOperand(addr, offset)); \ /* At the moment this is only true when dealing with 2-byte values.*/ \
__ bind(&do_cs); \ bool reverse_bytes = \
__ RotateInsertSelectBits(temp, value, Operand(start), Operand(end), \ maybe_reverse_bytes && is_wasm_on_be(info()->IsWasm()); \
Operand(static_cast<intptr_t>(shift_amount)), \ USE(reverse_bytes); \
true); \ Label do_cs; \
__ bin_inst(new_val, prev, temp); \ __ LoadU32(prev, MemOperand(addr, offset)); \
__ lr(temp, prev); \ __ bind(&do_cs); \
__ RotateInsertSelectBits(temp, new_val, Operand(start), Operand(end), \ if (reverse_bytes) { \
Operand::Zero(), false); \ Register temp2 = GetRegisterThatIsNotOneOf(value, result, prev); \
__ CmpAndSwap(prev, temp, MemOperand(addr, offset)); \ __ Push(temp2); \
__ bne(&do_cs, Label::kNear); \ __ lrvr(temp2, prev); \
__ RotateInsertSelectBits(temp2, temp2, Operand(start), Operand(end), \
Operand(static_cast<intptr_t>(shift_amount)), \
true); \
__ RotateInsertSelectBits(temp, value, Operand(start), Operand(end), \
Operand(static_cast<intptr_t>(shift_amount)), \
true); \
__ bin_inst(new_val, temp2, temp); \
__ lrvr(temp2, new_val); \
__ lr(temp, prev); \
__ RotateInsertSelectBits(temp, temp2, Operand(start), Operand(end), \
Operand(static_cast<intptr_t>(shift_amount)), \
false); \
__ Pop(temp2); \
} else { \
__ RotateInsertSelectBits(temp, value, Operand(start), Operand(end), \
Operand(static_cast<intptr_t>(shift_amount)), \
true); \
__ bin_inst(new_val, prev, temp); \
__ lr(temp, prev); \
__ RotateInsertSelectBits(temp, new_val, Operand(start), Operand(end), \
Operand::Zero(), false); \
} \
__ CmpAndSwap(prev, temp, MemOperand(addr, offset)); \
__ bne(&do_cs, Label::kNear); \
} while (false) } while (false)
#ifdef V8_TARGET_BIG_ENDIAN #ifdef V8_TARGET_BIG_ENDIAN
#define ATOMIC_BIN_OP_HALFWORD(bin_inst, index, extract_result) \ #define ATOMIC_BIN_OP_HALFWORD(bin_inst, index, extract_result) \
{ \ { \
constexpr int offset = -(2 * index); \ constexpr int offset = -(2 * index); \
constexpr int shift_amount = 16 - (index * 16); \ constexpr int shift_amount = 16 - (index * 16); \
constexpr int start = 48 - shift_amount; \ constexpr int start = 48 - shift_amount; \
constexpr int end = start + 15; \ constexpr int end = start + 15; \
ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end); \ ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end, true); \
extract_result(); \ extract_result(); \
} }
#define ATOMIC_BIN_OP_BYTE(bin_inst, index, extract_result) \ #define ATOMIC_BIN_OP_BYTE(bin_inst, index, extract_result) \
{ \ { \
constexpr int offset = -(index); \ constexpr int offset = -(index); \
constexpr int shift_amount = 24 - (index * 8); \ constexpr int shift_amount = 24 - (index * 8); \
constexpr int start = 56 - shift_amount; \ constexpr int start = 56 - shift_amount; \
constexpr int end = start + 7; \ constexpr int end = start + 7; \
ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end); \ ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end, false); \
extract_result(); \ extract_result(); \
} }
#else #else
#define ATOMIC_BIN_OP_HALFWORD(bin_inst, index, extract_result) \ #define ATOMIC_BIN_OP_HALFWORD(bin_inst, index, extract_result) \
{ \ { \
constexpr int offset = -(2 * index); \ constexpr int offset = -(2 * index); \
constexpr int shift_amount = index * 16; \ constexpr int shift_amount = index * 16; \
constexpr int start = 48 - shift_amount; \ constexpr int start = 48 - shift_amount; \
constexpr int end = start + 15; \ constexpr int end = start + 15; \
ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end); \ ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end, false); \
extract_result(); \ extract_result(); \
} }
#define ATOMIC_BIN_OP_BYTE(bin_inst, index, extract_result) \ #define ATOMIC_BIN_OP_BYTE(bin_inst, index, extract_result) \
{ \ { \
constexpr int offset = -(index); \ constexpr int offset = -(index); \
constexpr int shift_amount = index * 8; \ constexpr int shift_amount = index * 8; \
constexpr int start = 56 - shift_amount; \ constexpr int start = 56 - shift_amount; \
constexpr int end = start + 7; \ constexpr int end = start + 7; \
ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end); \ ATOMIC_BIN_OP(bin_inst, offset, shift_amount, start, end, false); \
extract_result(); \ extract_result(); \
} }
#endif // V8_TARGET_BIG_ENDIAN #endif // V8_TARGET_BIG_ENDIAN
...@@ -914,16 +994,23 @@ static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) { ...@@ -914,16 +994,23 @@ static inline int AssembleUnaryOp(Instruction* instr, _R _r, _M _m, _I _i) {
__ bind(&done); \ __ bind(&done); \
} while (false) } while (false)
#define ASSEMBLE_ATOMIC64_COMP_EXCHANGE_WORD64() \ #define ASSEMBLE_ATOMIC64_COMP_EXCHANGE_WORD64() \
do { \ do { \
Register new_val = i.InputRegister(1); \ Register new_val = i.InputRegister(1); \
Register output = i.OutputRegister(); \ Register output = i.OutputRegister(); \
Register addr = kScratchReg; \ Register addr = kScratchReg; \
size_t index = 2; \ size_t index = 2; \
AddressingMode mode = kMode_None; \ AddressingMode mode = kMode_None; \
MemOperand op = i.MemoryOperand(&mode, &index); \ MemOperand op = i.MemoryOperand(&mode, &index); \
__ lay(addr, op); \ __ lay(addr, op); \
__ CmpAndSwap64(output, new_val, MemOperand(addr)); \ if (is_wasm_on_be(info()->IsWasm())) { \
__ lrvgr(r0, output); \
__ lrvgr(r1, new_val); \
__ CmpAndSwap64(r0, r1, MemOperand(addr)); \
__ lrvgr(output, r0); \
} else { \
__ CmpAndSwap64(output, new_val, MemOperand(addr)); \
} \
} while (false) } while (false)
void CodeGenerator::AssembleDeconstructFrame() { void CodeGenerator::AssembleDeconstructFrame() {
...@@ -2308,13 +2395,24 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( ...@@ -2308,13 +2395,24 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
Register index = i.InputRegister(1); Register index = i.InputRegister(1);
Register value = i.InputRegister(2); Register value = i.InputRegister(2);
Register output = i.OutputRegister(); Register output = i.OutputRegister();
bool reverse_bytes = is_wasm_on_be(info()->IsWasm());
__ la(r1, MemOperand(base, index)); __ la(r1, MemOperand(base, index));
__ AtomicExchangeU16(r1, value, output, r0); Register value_ = value;
if (reverse_bytes) {
value_ = ip;
__ lrvr(value_, value);
__ ShiftRightU32(value_, value_, Operand(16));
}
__ AtomicExchangeU16(r1, value_, output, r0);
if (opcode == kAtomicExchangeInt16) { if (opcode == kAtomicExchangeInt16) {
__ lghr(output, output); __ lghr(output, output);
} else { } else {
__ llghr(output, output); __ llghr(output, output);
} }
if (reverse_bytes) {
__ lrvr(output, output);
__ ShiftRightU32(output, output, Operand(16));
}
break; break;
} }
case kAtomicExchangeWord32: { case kAtomicExchangeWord32: {
...@@ -2323,11 +2421,21 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( ...@@ -2323,11 +2421,21 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
Register value = i.InputRegister(2); Register value = i.InputRegister(2);
Register output = i.OutputRegister(); Register output = i.OutputRegister();
Label do_cs; Label do_cs;
bool reverse_bytes = is_wasm_on_be(info()->IsWasm());
__ lay(r1, MemOperand(base, index)); __ lay(r1, MemOperand(base, index));
Register value_ = value;
if (reverse_bytes) {
value_ = ip;
__ lrvr(value_, value);
}
__ LoadU32(output, MemOperand(r1)); __ LoadU32(output, MemOperand(r1));
__ bind(&do_cs); __ bind(&do_cs);
__ cs(output, value, MemOperand(r1)); __ cs(output, value_, MemOperand(r1));
__ bne(&do_cs, Label::kNear); __ bne(&do_cs, Label::kNear);
if (reverse_bytes) {
__ lrvr(output, output);
__ LoadU32(output, output);
}
break; break;
} }
case kAtomicCompareExchangeInt8: case kAtomicCompareExchangeInt8:
...@@ -2366,6 +2474,10 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( ...@@ -2366,6 +2474,10 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
intptr_t shift_right = static_cast<intptr_t>(shift_amount); \ intptr_t shift_right = static_cast<intptr_t>(shift_amount); \
__ srlk(result, prev, Operand(shift_right)); \ __ srlk(result, prev, Operand(shift_right)); \
__ LoadS16(result, result); \ __ LoadS16(result, result); \
if (is_wasm_on_be(info()->IsWasm())) { \
__ lrvr(result, result); \
__ ShiftRightS32(result, result, Operand(16)); \
} \
}); \ }); \
break; \ break; \
case kAtomic##op##Uint16: \ case kAtomic##op##Uint16: \
...@@ -2374,6 +2486,10 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( ...@@ -2374,6 +2486,10 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
__ RotateInsertSelectBits(result, prev, Operand(48), Operand(63), \ __ RotateInsertSelectBits(result, prev, Operand(48), Operand(63), \
Operand(static_cast<intptr_t>(rotate_left)), \ Operand(static_cast<intptr_t>(rotate_left)), \
true); \ true); \
if (is_wasm_on_be(info()->IsWasm())) { \
__ lrvr(result, result); \
__ ShiftRightU32(result, result, Operand(16)); \
} \
}); \ }); \
break; break;
ATOMIC_BINOP_CASE(Add, AddS32) ATOMIC_BINOP_CASE(Add, AddS32)
...@@ -2383,46 +2499,55 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( ...@@ -2383,46 +2499,55 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
ATOMIC_BINOP_CASE(Xor, Xor) ATOMIC_BINOP_CASE(Xor, Xor)
#undef ATOMIC_BINOP_CASE #undef ATOMIC_BINOP_CASE
case kAtomicAddWord32: case kAtomicAddWord32:
ASSEMBLE_ATOMIC_BINOP_WORD(laa); ASSEMBLE_ATOMIC_BINOP_WORD(laa, AddS32);
break; break;
case kAtomicSubWord32: case kAtomicSubWord32:
ASSEMBLE_ATOMIC_BINOP_WORD(LoadAndSub32); ASSEMBLE_ATOMIC_BINOP_WORD(LoadAndSub32, SubS32);
break; break;
case kAtomicAndWord32: case kAtomicAndWord32:
ASSEMBLE_ATOMIC_BINOP_WORD(lan); ASSEMBLE_ATOMIC_BINOP_WORD(lan, AndP);
break; break;
case kAtomicOrWord32: case kAtomicOrWord32:
ASSEMBLE_ATOMIC_BINOP_WORD(lao); ASSEMBLE_ATOMIC_BINOP_WORD(lao, OrP);
break; break;
case kAtomicXorWord32: case kAtomicXorWord32:
ASSEMBLE_ATOMIC_BINOP_WORD(lax); ASSEMBLE_ATOMIC_BINOP_WORD(lax, XorP);
break; break;
case kS390_Word64AtomicAddUint64: case kS390_Word64AtomicAddUint64:
ASSEMBLE_ATOMIC_BINOP_WORD64(laag); ASSEMBLE_ATOMIC_BINOP_WORD64(laag, AddS64);
break; break;
case kS390_Word64AtomicSubUint64: case kS390_Word64AtomicSubUint64:
ASSEMBLE_ATOMIC_BINOP_WORD64(LoadAndSub64); ASSEMBLE_ATOMIC_BINOP_WORD64(LoadAndSub64, SubS64);
break; break;
case kS390_Word64AtomicAndUint64: case kS390_Word64AtomicAndUint64:
ASSEMBLE_ATOMIC_BINOP_WORD64(lang); ASSEMBLE_ATOMIC_BINOP_WORD64(lang, AndP);
break; break;
case kS390_Word64AtomicOrUint64: case kS390_Word64AtomicOrUint64:
ASSEMBLE_ATOMIC_BINOP_WORD64(laog); ASSEMBLE_ATOMIC_BINOP_WORD64(laog, OrP);
break; break;
case kS390_Word64AtomicXorUint64: case kS390_Word64AtomicXorUint64:
ASSEMBLE_ATOMIC_BINOP_WORD64(laxg); ASSEMBLE_ATOMIC_BINOP_WORD64(laxg, XorP);
break; break;
case kS390_Word64AtomicExchangeUint64: { case kS390_Word64AtomicExchangeUint64: {
Register base = i.InputRegister(0); Register base = i.InputRegister(0);
Register index = i.InputRegister(1); Register index = i.InputRegister(1);
Register value = i.InputRegister(2); Register value = i.InputRegister(2);
Register output = i.OutputRegister(); Register output = i.OutputRegister();
bool reverse_bytes = is_wasm_on_be(info()->IsWasm());
Label do_cs; Label do_cs;
Register value_ = value;
__ la(r1, MemOperand(base, index)); __ la(r1, MemOperand(base, index));
if (reverse_bytes) {
value_ = ip;
__ lrvgr(value_, value);
}
__ lg(output, MemOperand(r1)); __ lg(output, MemOperand(r1));
__ bind(&do_cs); __ bind(&do_cs);
__ csg(output, value, MemOperand(r1)); __ csg(output, value_, MemOperand(r1));
__ bne(&do_cs, Label::kNear); __ bne(&do_cs, Label::kNear);
if (reverse_bytes) {
__ lrvgr(output, output);
}
break; break;
} }
case kS390_Word64AtomicCompareExchangeUint64: case kS390_Word64AtomicCompareExchangeUint64:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment