Commit 01724ace authored by Thibaud Michaud's avatar Thibaud Michaud Committed by Commit Bot

[assembler][x64] Inline Operand constructor

On x86-64, we spend significant time constructing and copying operands
in Liftoff (around 5% locally). Inlining the constructor and helper
functions removes most of the overhead.

R=clemensb@chromium.org

Bug: v8:10576
Change-Id: I1663e3e92abe7683eba9320e77fce9be8f84b4ca
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2225023
Commit-Queue: Thibaud Michaud <thibaudm@chromium.org>
Reviewed-by: 's avatarClemens Backes <clemensb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#68130}
parent b65fcfe9
...@@ -132,59 +132,7 @@ uint32_t RelocInfo::wasm_call_tag() const { ...@@ -132,59 +132,7 @@ uint32_t RelocInfo::wasm_call_tag() const {
// ----------------------------------------------------------------------------- // -----------------------------------------------------------------------------
// Implementation of Operand // Implementation of Operand
namespace { Operand::Operand(Operand operand, int32_t offset) {
class OperandBuilder {
public:
OperandBuilder(Register base, int32_t disp) {
if (base == rsp || base == r12) {
// SIB byte is needed to encode (rsp + offset) or (r12 + offset).
set_sib(times_1, rsp, base);
}
if (disp == 0 && base != rbp && base != r13) {
set_modrm(0, base);
} else if (is_int8(disp)) {
set_modrm(1, base);
set_disp8(disp);
} else {
set_modrm(2, base);
set_disp32(disp);
}
}
OperandBuilder(Register base, Register index, ScaleFactor scale,
int32_t disp) {
DCHECK(index != rsp);
set_sib(scale, index, base);
if (disp == 0 && base != rbp && base != r13) {
// This call to set_modrm doesn't overwrite the REX.B (or REX.X) bits
// possibly set by set_sib.
set_modrm(0, rsp);
} else if (is_int8(disp)) {
set_modrm(1, rsp);
set_disp8(disp);
} else {
set_modrm(2, rsp);
set_disp32(disp);
}
}
OperandBuilder(Register index, ScaleFactor scale, int32_t disp) {
DCHECK(index != rsp);
set_modrm(0, rsp);
set_sib(scale, index, rbp);
set_disp32(disp);
}
OperandBuilder(Label* label, int addend) {
data_.addend = addend;
DCHECK_NOT_NULL(label);
DCHECK(addend == 0 || (is_int8(addend) && label->is_bound()));
set_modrm(0, rbp);
set_disp64(reinterpret_cast<intptr_t>(label));
}
OperandBuilder(Operand operand, int32_t offset) {
DCHECK_GE(operand.data().len, 1); DCHECK_GE(operand.data().len, 1);
// Operand encodes REX ModR/M [SIB] [Disp]. // Operand encodes REX ModR/M [SIB] [Disp].
byte modrm = operand.data().buf[0]; byte modrm = operand.data().buf[0];
...@@ -195,8 +143,7 @@ class OperandBuilder { ...@@ -195,8 +143,7 @@ class OperandBuilder {
int base_reg = (has_sib ? operand.data().buf[1] : modrm) & 0x07; int base_reg = (has_sib ? operand.data().buf[1] : modrm) & 0x07;
// Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit // Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit
// displacement. // displacement.
bool is_baseless = bool is_baseless = (mode == 0) && (base_reg == 0x05); // No base or RIP base.
(mode == 0) && (base_reg == 0x05); // No base or RIP base.
int32_t disp_value = 0; int32_t disp_value = 0;
if (mode == 0x80 || is_baseless) { if (mode == 0x80 || is_baseless) {
// Mode 2 or mode 0 with rbp/r13 as base: Word displacement. // Mode 2 or mode 0 with rbp/r13 as base: Word displacement.
...@@ -231,69 +178,7 @@ class OperandBuilder { ...@@ -231,69 +178,7 @@ class OperandBuilder {
if (has_sib) { if (has_sib) {
data_.buf[1] = operand.data().buf[1]; data_.buf[1] = operand.data().buf[1];
} }
} }
void set_modrm(int mod, Register rm_reg) {
DCHECK(is_uint2(mod));
data_.buf[0] = mod << 6 | rm_reg.low_bits();
// Set REX.B to the high bit of rm.code().
data_.rex |= rm_reg.high_bit();
}
void set_sib(ScaleFactor scale, Register index, Register base) {
DCHECK_EQ(data_.len, 1);
DCHECK(is_uint2(scale));
// Use SIB with no index register only for base rsp or r12. Otherwise we
// would skip the SIB byte entirely.
DCHECK(index != rsp || base == rsp || base == r12);
data_.buf[1] = (scale << 6) | (index.low_bits() << 3) | base.low_bits();
data_.rex |= index.high_bit() << 1 | base.high_bit();
data_.len = 2;
}
void set_disp8(int disp) {
DCHECK(is_int8(disp));
DCHECK(data_.len == 1 || data_.len == 2);
int8_t* p = reinterpret_cast<int8_t*>(&data_.buf[data_.len]);
*p = disp;
data_.len += sizeof(int8_t);
}
void set_disp32(int disp) {
DCHECK(data_.len == 1 || data_.len == 2);
Address p = reinterpret_cast<Address>(&data_.buf[data_.len]);
WriteUnalignedValue(p, disp);
data_.len += sizeof(int32_t);
}
void set_disp64(int64_t disp) {
DCHECK_EQ(1, data_.len);
Address p = reinterpret_cast<Address>(&data_.buf[data_.len]);
WriteUnalignedValue(p, disp);
data_.len += sizeof(disp);
}
const Operand::Data& data() const { return data_; }
private:
Operand::Data data_;
};
} // namespace
Operand::Operand(Register base, int32_t disp)
: data_(OperandBuilder(base, disp).data()) {}
Operand::Operand(Register base, Register index, ScaleFactor scale, int32_t disp)
: data_(OperandBuilder(base, index, scale, disp).data()) {}
Operand::Operand(Register index, ScaleFactor scale, int32_t disp)
: data_(OperandBuilder(index, scale, disp).data()) {}
Operand::Operand(Label* label, int addend)
: data_(OperandBuilder(label, addend).data()) {}
Operand::Operand(Operand operand, int32_t offset)
: data_(OperandBuilder(operand, offset).data()) {}
bool Operand::AddressUsesRegister(Register reg) const { bool Operand::AddressUsesRegister(Register reg) const {
int code = reg.code(); int code = reg.code();
......
...@@ -173,13 +173,48 @@ class V8_EXPORT_PRIVATE Operand { ...@@ -173,13 +173,48 @@ class V8_EXPORT_PRIVATE Operand {
}; };
// [base + disp/r] // [base + disp/r]
Operand(Register base, int32_t disp); V8_INLINE Operand(Register base, int32_t disp) {
if (base == rsp || base == r12) {
// SIB byte is needed to encode (rsp + offset) or (r12 + offset).
set_sib(times_1, rsp, base);
}
if (disp == 0 && base != rbp && base != r13) {
set_modrm(0, base);
} else if (is_int8(disp)) {
set_modrm(1, base);
set_disp8(disp);
} else {
set_modrm(2, base);
set_disp32(disp);
}
}
// [base + index*scale + disp/r] // [base + index*scale + disp/r]
Operand(Register base, Register index, ScaleFactor scale, int32_t disp); V8_INLINE Operand(Register base, Register index, ScaleFactor scale,
int32_t disp) {
DCHECK(index != rsp);
set_sib(scale, index, base);
if (disp == 0 && base != rbp && base != r13) {
// This call to set_modrm doesn't overwrite the REX.B (or REX.X) bits
// possibly set by set_sib.
set_modrm(0, rsp);
} else if (is_int8(disp)) {
set_modrm(1, rsp);
set_disp8(disp);
} else {
set_modrm(2, rsp);
set_disp32(disp);
}
}
// [index*scale + disp/r] // [index*scale + disp/r]
Operand(Register index, ScaleFactor scale, int32_t disp); V8_INLINE Operand(Register index, ScaleFactor scale, int32_t disp) {
DCHECK(index != rsp);
set_modrm(0, rsp);
set_sib(scale, index, rbp);
set_disp32(disp);
}
// Offset from existing memory operand. // Offset from existing memory operand.
// Offset is added to existing displacement as 32-bit signed values and // Offset is added to existing displacement as 32-bit signed values and
...@@ -187,25 +222,64 @@ class V8_EXPORT_PRIVATE Operand { ...@@ -187,25 +222,64 @@ class V8_EXPORT_PRIVATE Operand {
Operand(Operand base, int32_t offset); Operand(Operand base, int32_t offset);
// [rip + disp/r] // [rip + disp/r]
explicit Operand(Label* label, int addend = 0); V8_INLINE explicit Operand(Label* label, int addend = 0) {
data_.addend = addend;
DCHECK_NOT_NULL(label);
DCHECK(addend == 0 || (is_int8(addend) && label->is_bound()));
set_modrm(0, rbp);
set_disp64(reinterpret_cast<intptr_t>(label));
}
Operand(const Operand&) V8_NOEXCEPT = default; Operand(const Operand&) V8_NOEXCEPT = default;
const Data& data() const { return data_; }
// Checks whether either base or index register is the given register. // Checks whether either base or index register is the given register.
// Does not check the "reg" part of the Operand. // Does not check the "reg" part of the Operand.
bool AddressUsesRegister(Register reg) const; bool AddressUsesRegister(Register reg) const;
// Queries related to the size of the generated instruction. private:
// Whether the generated instruction will have a REX prefix. V8_INLINE void set_modrm(int mod, Register rm_reg) {
bool requires_rex() const { return data_.rex != 0; } DCHECK(is_uint2(mod));
// Size of the ModR/M, SIB and displacement parts of the generated data_.buf[0] = mod << 6 | rm_reg.low_bits();
// instruction. // Set REX.B to the high bit of rm.code().
int operand_size() const { return data_.len; } data_.rex |= rm_reg.high_bit();
}
const Data& data() const { return data_; } V8_INLINE void set_sib(ScaleFactor scale, Register index, Register base) {
DCHECK_EQ(data_.len, 1);
DCHECK(is_uint2(scale));
// Use SIB with no index register only for base rsp or r12. Otherwise we
// would skip the SIB byte entirely.
DCHECK(index != rsp || base == rsp || base == r12);
data_.buf[1] = (scale << 6) | (index.low_bits() << 3) | base.low_bits();
data_.rex |= index.high_bit() << 1 | base.high_bit();
data_.len = 2;
}
private: V8_INLINE void set_disp8(int disp) {
const Data data_; DCHECK(is_int8(disp));
DCHECK(data_.len == 1 || data_.len == 2);
int8_t* p = reinterpret_cast<int8_t*>(&data_.buf[data_.len]);
*p = disp;
data_.len += sizeof(int8_t);
}
V8_INLINE void set_disp32(int disp) {
DCHECK(data_.len == 1 || data_.len == 2);
Address p = reinterpret_cast<Address>(&data_.buf[data_.len]);
WriteUnalignedValue(p, disp);
data_.len += sizeof(int32_t);
}
V8_INLINE void set_disp64(int64_t disp) {
DCHECK_EQ(1, data_.len);
Address p = reinterpret_cast<Address>(&data_.buf[data_.len]);
WriteUnalignedValue(p, disp);
data_.len += sizeof(disp);
}
Data data_;
}; };
ASSERT_TRIVIALLY_COPYABLE(Operand); ASSERT_TRIVIALLY_COPYABLE(Operand);
static_assert(sizeof(Operand) <= 2 * kSystemPointerSize, static_assert(sizeof(Operand) <= 2 * kSystemPointerSize,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment