Skip to content

V
V8
Commit 01724ace authored by Thibaud Michaud's avatar Thibaud Michaud Committed by Commit Bot
Browse files
Options

[assembler][x64] Inline Operand constructor 

On x86-64, we spend significant time constructing and copying operands
in Liftoff (around 5% locally). Inlining the constructor and helper
functions removes most of the overhead.

R=clemensb@chromium.org

Bug: v8:10576
Change-Id: I1663e3e92abe7683eba9320e77fce9be8f84b4ca
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2225023
Commit-Queue: Thibaud Michaud <thibaudm@chromium.org>
Reviewed-by: 's avatarClemens Backes <clemensb@chromium.org>
Cr-Commit-Position: refs/heads/master@{#68130}
parent b65fcfe9
Hide whitespace changes
Inline Side-by-side
Showing with 130 additions and 171 deletions
src/codegen/x64/assembler-x64.cc
View file @ 01724ace
...@@ -132,168 +132,53 @@ uint32_t RelocInfo::wasm_call_tag() const { ...@@ -132,168 +132,53 @@ uint32_t RelocInfo::wasm_call_tag() const {
// ----------------------------------------------------------------------------- // -----------------------------------------------------------------------------
// Implementation of Operand // Implementation of Operand
namespace { Operand::Operand(Operand operand, int32_t offset) {
class OperandBuilder { DCHECK_GE(operand.data().len, 1);
public: // Operand encodes REX ModR/M [SIB] [Disp].
OperandBuilder(Register base, int32_t disp) { byte modrm = operand.data().buf[0];
if (base == rsp || base == r12) { DCHECK_LT(modrm, 0xC0); // Disallow mode 3 (register target).
// SIB byte is needed to encode (rsp + offset) or (r12 + offset). bool has_sib = ((modrm & 0x07) == 0x04);
set_sib(times_1, rsp, base); byte mode = modrm & 0xC0;
} int disp_offset = has_sib ? 2 : 1;
int base_reg = (has_sib ? operand.data().buf[1] : modrm) & 0x07;
if (disp == 0 && base != rbp && base != r13) { // Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit
set_modrm(0, base); // displacement.
} else if (is_int8(disp)) { bool is_baseless = (mode == 0) && (base_reg == 0x05); // No base or RIP base.
set_modrm(1, base); int32_t disp_value = 0;
set_disp8(disp); if (mode == 0x80 || is_baseless) {
} else { // Mode 2 or mode 0 with rbp/r13 as base: Word displacement.
set_modrm(2, base); disp_value = ReadUnalignedValue<int32_t>(
set_disp32(disp); reinterpret_cast<Address>(&operand.data().buf[disp_offset]));
} } else if (mode == 0x40) {
// Mode 1: Byte displacement.
disp_value = static_cast<signed char>(operand.data().buf[disp_offset]);
} }
OperandBuilder(Register base, Register index, ScaleFactor scale, // Write new operand with same registers, but with modified displacement.
int32_t disp) { DCHECK(offset >= 0 ? disp_value + offset > disp_value
DCHECK(index != rsp); : disp_value + offset < disp_value); // No overflow.
set_sib(scale, index, base); disp_value += offset;
if (disp == 0 && base != rbp && base != r13) { data_.rex = operand.data().rex;
// This call to set_modrm doesn't overwrite the REX.B (or REX.X) bits if (!is_int8(disp_value) || is_baseless) {
// possibly set by set_sib. // Need 32 bits of displacement, mode 2 or mode 1 with register rbp/r13.
set_modrm(0, rsp); data_.buf[0] = (modrm & 0x3F) | (is_baseless ? 0x00 : 0x80);
} else if (is_int8(disp)) { data_.len = disp_offset + 4;
set_modrm(1, rsp); WriteUnalignedValue(reinterpret_cast<Address>(&data_.buf[disp_offset]),
set_disp8(disp); disp_value);
} else { } else if (disp_value != 0 || (base_reg == 0x05)) {
set_modrm(2, rsp); // Need 8 bits of displacement.
set_disp32(disp); data_.buf[0] = (modrm & 0x3F) | 0x40; // Mode 1.
} data_.len = disp_offset + 1;
} data_.buf[disp_offset] = static_cast<byte>(disp_value);
} else {
OperandBuilder(Register index, ScaleFactor scale, int32_t disp) { // Need no displacement.
DCHECK(index != rsp); data_.buf[0] = (modrm & 0x3F); // Mode 0.
set_modrm(0, rsp); data_.len = disp_offset;
set_sib(scale, index, rbp);
set_disp32(disp);
}
OperandBuilder(Label* label, int addend) {
data_.addend = addend;
DCHECK_NOT_NULL(label);
DCHECK(addend == 0 || (is_int8(addend) && label->is_bound()));
set_modrm(0, rbp);
set_disp64(reinterpret_cast<intptr_t>(label));
}
OperandBuilder(Operand operand, int32_t offset) {
DCHECK_GE(operand.data().len, 1);
// Operand encodes REX ModR/M [SIB] [Disp].
byte modrm = operand.data().buf[0];
DCHECK_LT(modrm, 0xC0); // Disallow mode 3 (register target).
bool has_sib = ((modrm & 0x07) == 0x04);
byte mode = modrm & 0xC0;
int disp_offset = has_sib ? 2 : 1;
int base_reg = (has_sib ? operand.data().buf[1] : modrm) & 0x07;
// Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit
// displacement.
bool is_baseless =
(mode == 0) && (base_reg == 0x05); // No base or RIP base.
int32_t disp_value = 0;
if (mode == 0x80 || is_baseless) {
// Mode 2 or mode 0 with rbp/r13 as base: Word displacement.
disp_value = ReadUnalignedValue<int32_t>(
reinterpret_cast<Address>(&operand.data().buf[disp_offset]));
} else if (mode == 0x40) {
// Mode 1: Byte displacement.
disp_value = static_cast<signed char>(operand.data().buf[disp_offset]);
}
// Write new operand with same registers, but with modified displacement.
DCHECK(offset >= 0 ? disp_value + offset > disp_value
: disp_value + offset < disp_value); // No overflow.
disp_value += offset;
data_.rex = operand.data().rex;
if (!is_int8(disp_value) || is_baseless) {
// Need 32 bits of displacement, mode 2 or mode 1 with register rbp/r13.
data_.buf[0] = (modrm & 0x3F) | (is_baseless ? 0x00 : 0x80);
data_.len = disp_offset + 4;
WriteUnalignedValue(reinterpret_cast<Address>(&data_.buf[disp_offset]),
disp_value);
} else if (disp_value != 0 || (base_reg == 0x05)) {
// Need 8 bits of displacement.
data_.buf[0] = (modrm & 0x3F) | 0x40; // Mode 1.
data_.len = disp_offset + 1;
data_.buf[disp_offset] = static_cast<byte>(disp_value);
} else {
// Need no displacement.
data_.buf[0] = (modrm & 0x3F); // Mode 0.
data_.len = disp_offset;
}
if (has_sib) {
data_.buf[1] = operand.data().buf[1];
}
}
void set_modrm(int mod, Register rm_reg) {
DCHECK(is_uint2(mod));
data_.buf[0] = mod << 6 | rm_reg.low_bits();
// Set REX.B to the high bit of rm.code().
data_.rex |= rm_reg.high_bit();
}
void set_sib(ScaleFactor scale, Register index, Register base) {
DCHECK_EQ(data_.len, 1);
DCHECK(is_uint2(scale));
// Use SIB with no index register only for base rsp or r12. Otherwise we
// would skip the SIB byte entirely.
DCHECK(index != rsp || base == rsp || base == r12);
data_.buf[1] = (scale << 6) | (index.low_bits() << 3) | base.low_bits();
data_.rex |= index.high_bit() << 1 | base.high_bit();
data_.len = 2;
}
void set_disp8(int disp) {
DCHECK(is_int8(disp));
DCHECK(data_.len == 1 || data_.len == 2);
int8_t* p = reinterpret_cast<int8_t*>(&data_.buf[data_.len]);
*p = disp;
data_.len += sizeof(int8_t);
}
void set_disp32(int disp) {
DCHECK(data_.len == 1 || data_.len == 2);
Address p = reinterpret_cast<Address>(&data_.buf[data_.len]);
WriteUnalignedValue(p, disp);
data_.len += sizeof(int32_t);
} }
if (has_sib) {
void set_disp64(int64_t disp) { data_.buf[1] = operand.data().buf[1];
DCHECK_EQ(1, data_.len);
Address p = reinterpret_cast<Address>(&data_.buf[data_.len]);
WriteUnalignedValue(p, disp);
data_.len += sizeof(disp);
} }
}
const Operand::Data& data() const { return data_; }
private:
Operand::Data data_;
};
} // namespace
Operand::Operand(Register base, int32_t disp)
: data_(OperandBuilder(base, disp).data()) {}
Operand::Operand(Register base, Register index, ScaleFactor scale, int32_t disp)
: data_(OperandBuilder(base, index, scale, disp).data()) {}
Operand::Operand(Register index, ScaleFactor scale, int32_t disp)
: data_(OperandBuilder(index, scale, disp).data()) {}
Operand::Operand(Label* label, int addend)
: data_(OperandBuilder(label, addend).data()) {}
Operand::Operand(Operand operand, int32_t offset)
: data_(OperandBuilder(operand, offset).data()) {}
bool Operand::AddressUsesRegister(Register reg) const { bool Operand::AddressUsesRegister(Register reg) const {
int code = reg.code(); int code = reg.code();
......
src/codegen/x64/assembler-x64.h
View file @ 01724ace
...@@ -173,13 +173,48 @@ class V8_EXPORT_PRIVATE Operand { ...@@ -173,13 +173,48 @@ class V8_EXPORT_PRIVATE Operand {
}; };
// [base + disp/r] // [base + disp/r]
Operand(Register base, int32_t disp); V8_INLINE Operand(Register base, int32_t disp) {
if (base == rsp || base == r12) {
// SIB byte is needed to encode (rsp + offset) or (r12 + offset).
set_sib(times_1, rsp, base);
}
if (disp == 0 && base != rbp && base != r13) {
set_modrm(0, base);
} else if (is_int8(disp)) {
set_modrm(1, base);
set_disp8(disp);
} else {
set_modrm(2, base);
set_disp32(disp);
}
}
// [base + index*scale + disp/r] // [base + index*scale + disp/r]
Operand(Register base, Register index, ScaleFactor scale, int32_t disp); V8_INLINE Operand(Register base, Register index, ScaleFactor scale,
int32_t disp) {
DCHECK(index != rsp);
set_sib(scale, index, base);
if (disp == 0 && base != rbp && base != r13) {
// This call to set_modrm doesn't overwrite the REX.B (or REX.X) bits
// possibly set by set_sib.
set_modrm(0, rsp);
} else if (is_int8(disp)) {
set_modrm(1, rsp);
set_disp8(disp);
} else {
set_modrm(2, rsp);
set_disp32(disp);
}
}
// [index*scale + disp/r] // [index*scale + disp/r]
Operand(Register index, ScaleFactor scale, int32_t disp); V8_INLINE Operand(Register index, ScaleFactor scale, int32_t disp) {
DCHECK(index != rsp);
set_modrm(0, rsp);
set_sib(scale, index, rbp);
set_disp32(disp);
}
// Offset from existing memory operand. // Offset from existing memory operand.
// Offset is added to existing displacement as 32-bit signed values and // Offset is added to existing displacement as 32-bit signed values and
...@@ -187,25 +222,64 @@ class V8_EXPORT_PRIVATE Operand { ...@@ -187,25 +222,64 @@ class V8_EXPORT_PRIVATE Operand {
Operand(Operand base, int32_t offset); Operand(Operand base, int32_t offset);
// [rip + disp/r] // [rip + disp/r]
explicit Operand(Label* label, int addend = 0); V8_INLINE explicit Operand(Label* label, int addend = 0) {
data_.addend = addend;
DCHECK_NOT_NULL(label);
DCHECK(addend == 0 || (is_int8(addend) && label->is_bound()));
set_modrm(0, rbp);
set_disp64(reinterpret_cast<intptr_t>(label));
}
Operand(const Operand&) V8_NOEXCEPT = default; Operand(const Operand&) V8_NOEXCEPT = default;
const Data& data() const { return data_; }
// Checks whether either base or index register is the given register. // Checks whether either base or index register is the given register.
// Does not check the "reg" part of the Operand. // Does not check the "reg" part of the Operand.
bool AddressUsesRegister(Register reg) const; bool AddressUsesRegister(Register reg) const;
// Queries related to the size of the generated instruction. private:
// Whether the generated instruction will have a REX prefix. V8_INLINE void set_modrm(int mod, Register rm_reg) {
bool requires_rex() const { return data_.rex != 0; } DCHECK(is_uint2(mod));
// Size of the ModR/M, SIB and displacement parts of the generated data_.buf[0] = mod << 6 | rm_reg.low_bits();
// instruction. // Set REX.B to the high bit of rm.code().
int operand_size() const { return data_.len; } data_.rex |= rm_reg.high_bit();
}
const Data& data() const { return data_; } V8_INLINE void set_sib(ScaleFactor scale, Register index, Register base) {
DCHECK_EQ(data_.len, 1);
DCHECK(is_uint2(scale));
// Use SIB with no index register only for base rsp or r12. Otherwise we
// would skip the SIB byte entirely.
DCHECK(index != rsp || base == rsp || base == r12);
data_.buf[1] = (scale << 6) | (index.low_bits() << 3) | base.low_bits();
data_.rex |= index.high_bit() << 1 | base.high_bit();
data_.len = 2;
}
private: V8_INLINE void set_disp8(int disp) {
const Data data_; DCHECK(is_int8(disp));
DCHECK(data_.len == 1 || data_.len == 2);
int8_t* p = reinterpret_cast<int8_t*>(&data_.buf[data_.len]);
*p = disp;
data_.len += sizeof(int8_t);
}
V8_INLINE void set_disp32(int disp) {
DCHECK(data_.len == 1 || data_.len == 2);
Address p = reinterpret_cast<Address>(&data_.buf[data_.len]);
WriteUnalignedValue(p, disp);
data_.len += sizeof(int32_t);
}
V8_INLINE void set_disp64(int64_t disp) {
DCHECK_EQ(1, data_.len);
Address p = reinterpret_cast<Address>(&data_.buf[data_.len]);
WriteUnalignedValue(p, disp);
data_.len += sizeof(disp);
}
Data data_;
}; };
ASSERT_TRIVIALLY_COPYABLE(Operand); ASSERT_TRIVIALLY_COPYABLE(Operand);
static_assert(sizeof(Operand) <= 2 * kSystemPointerSize, static_assert(sizeof(Operand) <= 2 * kSystemPointerSize,
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment