Commit 95a9b761 authored by danno's avatar danno Committed by Commit bot

[turbofan] Improve codegen for 8- and 16-bit memory comparisons on Intel platforms (reland)

Recognize and emit in-memory comparisons of 8-bit and 16-bit values with
immediate values that fit.

LOG=N
R=epertoso@chromium.org

Review-Url: https://codereview.chromium.org/2605863002
Cr-Commit-Position: refs/heads/master@{#42344}
parent ac9e6285
......@@ -1251,21 +1251,54 @@ void VisitCompare(InstructionSelector* selector, InstructionCode opcode,
VisitCompare(selector, opcode, g.UseRegister(left), g.Use(right), cont);
}
MachineType MachineTypeForNarrow(Node* node, Node* hint_node) {
if (hint_node->opcode() == IrOpcode::kLoad) {
MachineType hint = LoadRepresentationOf(hint_node->op());
if (node->opcode() == IrOpcode::kInt32Constant ||
node->opcode() == IrOpcode::kInt64Constant) {
int64_t constant = node->opcode() == IrOpcode::kInt32Constant
? OpParameter<int32_t>(node)
: OpParameter<int64_t>(node);
if (hint == MachineType::Int8()) {
if (constant >= std::numeric_limits<int8_t>::min() &&
constant <= std::numeric_limits<int8_t>::max()) {
return hint;
}
} else if (hint == MachineType::Uint8()) {
if (constant >= std::numeric_limits<uint8_t>::min() &&
constant <= std::numeric_limits<uint8_t>::max()) {
return hint;
}
} else if (hint == MachineType::Int16()) {
if (constant >= std::numeric_limits<int16_t>::min() &&
constant <= std::numeric_limits<int16_t>::max()) {
return hint;
}
} else if (hint == MachineType::Uint16()) {
if (constant >= std::numeric_limits<uint16_t>::min() &&
constant <= std::numeric_limits<uint16_t>::max()) {
return hint;
}
} else if (hint == MachineType::Int32()) {
return hint;
} else if (hint == MachineType::Uint32()) {
if (constant >= 0) return hint;
}
}
}
return node->opcode() == IrOpcode::kLoad ? LoadRepresentationOf(node->op())
: MachineType::None();
}
// Tries to match the size of the given opcode to that of the operands, if
// possible.
InstructionCode TryNarrowOpcodeSize(InstructionCode opcode, Node* left,
Node* right, FlagsContinuation* cont) {
// Currently, if one of the two operands is not a Load, we don't know what its
// machine representation is, so we bail out.
// TODO(epertoso): we can probably get some size information out of immediates
// and phi nodes.
if (left->opcode() != IrOpcode::kLoad || right->opcode() != IrOpcode::kLoad) {
return opcode;
}
// TODO(epertoso): we can probably get some size information out of phi nodes.
// If the load representations don't match, both operands will be
// zero/sign-extended to 32bit.
MachineType left_type = LoadRepresentationOf(left->op());
MachineType right_type = LoadRepresentationOf(right->op());
MachineType left_type = MachineTypeForNarrow(left, right);
MachineType right_type = MachineTypeForNarrow(right, left);
if (left_type == right_type) {
switch (left_type.representation()) {
case MachineRepresentation::kBit:
......@@ -1343,10 +1376,8 @@ void VisitWordCompare(InstructionSelector* selector, Node* node,
// Match immediates on right side of comparison.
if (g.CanBeImmediate(right)) {
if (g.CanBeMemoryOperand(opcode, node, left, effect_level)) {
// TODO(epertoso): we should use `narrowed_opcode' here once we match
// immediates too.
return VisitCompareWithMemoryOperand(selector, opcode, left,
if (g.CanBeMemoryOperand(narrowed_opcode, node, left, effect_level)) {
return VisitCompareWithMemoryOperand(selector, narrowed_opcode, left,
g.UseImmediate(right), cont);
}
return VisitCompare(selector, opcode, g.Use(left), g.UseImmediate(right),
......
......@@ -1727,21 +1727,54 @@ void VisitCompare(InstructionSelector* selector, InstructionCode opcode,
VisitCompare(selector, opcode, g.UseRegister(left), g.Use(right), cont);
}
MachineType MachineTypeForNarrow(Node* node, Node* hint_node) {
if (hint_node->opcode() == IrOpcode::kLoad) {
MachineType hint = LoadRepresentationOf(hint_node->op());
if (node->opcode() == IrOpcode::kInt32Constant ||
node->opcode() == IrOpcode::kInt64Constant) {
int64_t constant = node->opcode() == IrOpcode::kInt32Constant
? OpParameter<int32_t>(node)
: OpParameter<int64_t>(node);
if (hint == MachineType::Int8()) {
if (constant >= std::numeric_limits<int8_t>::min() &&
constant <= std::numeric_limits<int8_t>::max()) {
return hint;
}
} else if (hint == MachineType::Uint8()) {
if (constant >= std::numeric_limits<uint8_t>::min() &&
constant <= std::numeric_limits<uint8_t>::max()) {
return hint;
}
} else if (hint == MachineType::Int16()) {
if (constant >= std::numeric_limits<int16_t>::min() &&
constant <= std::numeric_limits<int16_t>::max()) {
return hint;
}
} else if (hint == MachineType::Uint16()) {
if (constant >= std::numeric_limits<uint16_t>::min() &&
constant <= std::numeric_limits<uint16_t>::max()) {
return hint;
}
} else if (hint == MachineType::Int32()) {
return hint;
} else if (hint == MachineType::Uint32()) {
if (constant >= 0) return hint;
}
}
}
return node->opcode() == IrOpcode::kLoad ? LoadRepresentationOf(node->op())
: MachineType::None();
}
// Tries to match the size of the given opcode to that of the operands, if
// possible.
InstructionCode TryNarrowOpcodeSize(InstructionCode opcode, Node* left,
Node* right, FlagsContinuation* cont) {
// Currently, if one of the two operands is not a Load, we don't know what its
// machine representation is, so we bail out.
// TODO(epertoso): we can probably get some size information out of immediates
// and phi nodes.
if (left->opcode() != IrOpcode::kLoad || right->opcode() != IrOpcode::kLoad) {
return opcode;
}
// TODO(epertoso): we can probably get some size information out phi nodes.
// If the load representations don't match, both operands will be
// zero/sign-extended to 32bit.
MachineType left_type = LoadRepresentationOf(left->op());
MachineType right_type = LoadRepresentationOf(right->op());
MachineType left_type = MachineTypeForNarrow(left, right);
MachineType right_type = MachineTypeForNarrow(right, left);
if (left_type == right_type) {
switch (left_type.representation()) {
case MachineRepresentation::kBit:
......
......@@ -2050,159 +2050,137 @@ void Assembler::store_rax(ExternalReference ref) {
void Assembler::testb(Register dst, Register src) {
EnsureSpace ensure_space(this);
if (src.low_bits() == 4) {
emit_rex_32(src, dst);
emit(0x84);
emit_modrm(src, dst);
} else {
if (!dst.is_byte_register() || !src.is_byte_register()) {
// Register is not one of al, bl, cl, dl. Its encoding needs REX.
emit_rex_32(dst, src);
}
emit(0x84);
emit_modrm(dst, src);
}
emit_test(dst, src, sizeof(int8_t));
}
void Assembler::testb(Register reg, Immediate mask) {
DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
EnsureSpace ensure_space(this);
if (reg.is(rax)) {
emit(0xA8);
emit(mask.value_); // Low byte emitted.
} else {
if (!reg.is_byte_register()) {
// Register is not one of al, bl, cl, dl. Its encoding needs REX.
emit_rex_32(reg);
}
emit(0xF6);
emit_modrm(0x0, reg);
emit(mask.value_); // Low byte emitted.
}
emit_test(reg, mask, sizeof(int8_t));
}
void Assembler::testb(const Operand& op, Immediate mask) {
DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
EnsureSpace ensure_space(this);
emit_optional_rex_32(rax, op);
emit(0xF6);
emit_operand(rax, op); // Operation code 0
emit(mask.value_); // Low byte emitted.
emit_test(op, mask, sizeof(int8_t));
}
void Assembler::testb(const Operand& op, Register reg) {
EnsureSpace ensure_space(this);
if (!reg.is_byte_register()) {
// Register is not one of al, bl, cl, dl. Its encoding needs REX.
emit_rex_32(reg, op);
} else {
emit_optional_rex_32(reg, op);
}
emit(0x84);
emit_operand(reg, op);
emit_test(op, reg, sizeof(int8_t));
}
void Assembler::testw(Register dst, Register src) {
EnsureSpace ensure_space(this);
emit(0x66);
if (!dst.is_byte_register() || !src.is_byte_register()) {
// Register is not one of al, bl, cl, dl. Its encoding needs REX.
emit_rex_32(dst, src);
}
emit(0x85);
emit_modrm(dst, src);
emit_test(dst, src, sizeof(uint16_t));
}
void Assembler::testw(Register reg, Immediate mask) {
DCHECK(is_int16(mask.value_) || is_uint16(mask.value_));
EnsureSpace ensure_space(this);
emit(0x66);
if (reg.is(rax)) {
emit(0xA9);
emitw(mask.value_);
} else {
if (!reg.is_byte_register()) {
emit_rex_32(reg);
}
emit(0xF7);
emit_modrm(0x0, reg);
emitw(mask.value_);
}
emit_test(reg, mask, sizeof(int16_t));
}
void Assembler::testw(const Operand& op, Immediate mask) {
DCHECK(is_int16(mask.value_) || is_uint16(mask.value_));
EnsureSpace ensure_space(this);
emit(0x66);
emit_optional_rex_32(rax, op);
emit(0xF7);
emit_operand(rax, op);
emitw(mask.value_);
emit_test(op, mask, sizeof(int16_t));
}
void Assembler::testw(const Operand& op, Register reg) {
EnsureSpace ensure_space(this);
emit(0x66);
emit_optional_rex_32(reg, op);
emit(0x85);
emit_operand(rax, op);
emit_test(op, reg, sizeof(int16_t));
}
void Assembler::emit_test(Register dst, Register src, int size) {
EnsureSpace ensure_space(this);
if (src.low_bits() == 4) {
emit_rex(src, dst, size);
emit(0x85);
emit_modrm(src, dst);
if (src.low_bits() == 4) std::swap(dst, src);
if (size == sizeof(int16_t)) {
emit(0x66);
size = sizeof(int32_t);
}
bool byte_operand = size == sizeof(int8_t);
if (byte_operand) {
size = sizeof(int32_t);
if (!src.is_byte_register() || !dst.is_byte_register()) {
emit_rex_32(dst, src);
}
} else {
emit_rex(dst, src, size);
emit(0x85);
emit_modrm(dst, src);
}
emit(byte_operand ? 0x84 : 0x85);
emit_modrm(dst, src);
}
void Assembler::emit_test(Register reg, Immediate mask, int size) {
// testl with a mask that fits in the low byte is exactly testb.
if (is_uint8(mask.value_)) {
testb(reg, mask);
return;
size = sizeof(int8_t);
} else if (is_uint16(mask.value_)) {
size = sizeof(int16_t);
}
EnsureSpace ensure_space(this);
if (reg.is(rax)) {
emit_rex(rax, size);
emit(0xA9);
emit(mask);
bool half_word = size == sizeof(int16_t);
if (half_word) {
emit(0x66);
size = sizeof(int32_t);
}
bool byte_operand = size == sizeof(int8_t);
if (byte_operand) {
size = sizeof(int32_t);
if (!reg.is_byte_register()) emit_rex_32(reg);
} else {
emit_rex(reg, size);
emit(0xF7);
}
if (reg.is(rax)) {
emit(byte_operand ? 0xA8 : 0xA9);
} else {
emit(byte_operand ? 0xF6 : 0xF7);
emit_modrm(0x0, reg);
}
if (byte_operand) {
emit(mask.value_);
} else if (half_word) {
emitw(mask.value_);
} else {
emit(mask);
}
}
void Assembler::emit_test(const Operand& op, Immediate mask, int size) {
// testl with a mask that fits in the low byte is exactly testb.
if (is_uint8(mask.value_)) {
testb(op, mask);
return;
size = sizeof(int8_t);
} else if (is_uint16(mask.value_)) {
size = sizeof(int16_t);
}
EnsureSpace ensure_space(this);
bool half_word = size == sizeof(int16_t);
if (half_word) {
emit(0x66);
size = sizeof(int32_t);
}
bool byte_operand = size == sizeof(int8_t);
if (byte_operand) {
size = sizeof(int32_t);
}
emit_rex(rax, op, size);
emit(0xF7);
emit(byte_operand ? 0xF6 : 0xF7);
emit_operand(rax, op); // Operation code 0
emit(mask);
if (byte_operand) {
emit(mask.value_);
} else if (half_word) {
emitw(mask.value_);
} else {
emit(mask);
}
}
void Assembler::emit_test(const Operand& op, Register reg, int size) {
EnsureSpace ensure_space(this);
emit_rex(reg, op, size);
emit(0x85);
if (size == sizeof(int16_t)) {
emit(0x66);
size = sizeof(int32_t);
}
bool byte_operand = size == sizeof(int8_t);
if (byte_operand) {
size = sizeof(int32_t);
if (!reg.is_byte_register()) emit_rex_32(reg, op);
} else {
emit_rex(reg, op, size);
}
emit(byte_operand ? 0x84 : 0x85);
emit_operand(reg, op);
}
......
......@@ -201,7 +201,7 @@ TEST(AssemblerX64ImulOperation) {
CHECK_EQ(-1, result);
}
TEST(AssemblerX64testbwOperation) {
TEST(AssemblerX64testbwqOperation) {
CcTest::InitializeVM();
v8::HandleScope scope(CcTest::isolate());
// Allocate an executable page of memory.
......@@ -338,6 +338,12 @@ TEST(AssemblerX64testbwOperation) {
__ testw(r11, r12);
__ j(zero, &bad);
// Test sign-extended imediate tests
__ movq(r11, Immediate(2));
__ shlq(r11, Immediate(32));
__ testq(r11, Immediate(-1));
__ j(zero, &bad);
// All tests passed
__ movq(rax, Immediate(1));
__ jmp(&done);
......
......@@ -2082,7 +2082,6 @@ TEST(NewElementsCapacitySmi) {
m.Return(m.CalculateNewElementsCapacity(m.Parameter(0),
CodeStubAssembler::SMI_PARAMETERS));
Handle<Code> code = data.GenerateCode();
code->Print();
CHECK(!code.is_null());
FunctionTester ft(code, 1);
Handle<Smi> test_value = Handle<Smi>(Smi::FromInt(0), isolate);
......@@ -2295,5 +2294,190 @@ TEST(NewPromiseCapability) {
}
}
TEST(DirectMemoryTest8BitWord32Immediate) {
Isolate* isolate(CcTest::InitIsolateOnce());
typedef CodeAssemblerLabel Label;
const int kNumParams = 0;
CodeAssemblerTester data(isolate, kNumParams);
CodeStubAssembler m(data.state());
int8_t buffer[] = {1, 2, 4, 8, 17, 33, 65, 127};
const int element_count = 8;
Label bad(&m);
Node* buffer_node = m.IntPtrConstant(reinterpret_cast<intptr_t>(buffer));
for (size_t i = 0; i < element_count; ++i) {
for (size_t j = 0; j < element_count; ++j) {
Node* loaded = m.LoadBufferObject(buffer_node, static_cast<int>(i),
MachineType::Uint8());
Node* masked = m.Word32And(loaded, m.Int32Constant(buffer[j]));
if ((buffer[j] & buffer[i]) != 0) {
m.GotoIf(m.Word32Equal(masked, m.Int32Constant(0)), &bad);
} else {
m.GotoIf(m.Word32NotEqual(masked, m.Int32Constant(0)), &bad);
}
}
}
m.Return(m.SmiConstant(1));
m.Bind(&bad);
m.Return(m.SmiConstant(0));
Handle<Code> code = data.GenerateCode();
CHECK(!code.is_null());
FunctionTester ft(code, kNumParams);
CHECK_EQ(1, Handle<Smi>::cast(ft.Call().ToHandleChecked())->value());
}
TEST(DirectMemoryTest16BitWord32Immediate) {
Isolate* isolate(CcTest::InitIsolateOnce());
typedef CodeAssemblerLabel Label;
const int kNumParams = 0;
CodeAssemblerTester data(isolate, kNumParams);
CodeStubAssembler m(data.state());
int16_t buffer[] = {156, 2234, 4544, 8444, 1723, 3888, 658, 1278};
const int element_count = 8;
Label bad(&m);
Node* buffer_node = m.IntPtrConstant(reinterpret_cast<intptr_t>(buffer));
for (size_t i = 0; i < element_count; ++i) {
for (size_t j = 0; j < element_count; ++j) {
Node* loaded =
m.LoadBufferObject(buffer_node, static_cast<int>(i * sizeof(int16_t)),
MachineType::Uint16());
Node* masked = m.Word32And(loaded, m.Int32Constant(buffer[j]));
if ((buffer[j] & buffer[i]) != 0) {
m.GotoIf(m.Word32Equal(masked, m.Int32Constant(0)), &bad);
} else {
m.GotoIf(m.Word32NotEqual(masked, m.Int32Constant(0)), &bad);
}
}
}
m.Return(m.SmiConstant(1));
m.Bind(&bad);
m.Return(m.SmiConstant(0));
Handle<Code> code = data.GenerateCode();
CHECK(!code.is_null());
FunctionTester ft(code, kNumParams);
CHECK_EQ(1, Handle<Smi>::cast(ft.Call().ToHandleChecked())->value());
}
TEST(DirectMemoryTest8BitWord32) {
Isolate* isolate(CcTest::InitIsolateOnce());
typedef CodeAssemblerLabel Label;
const int kNumParams = 0;
CodeAssemblerTester data(isolate, kNumParams);
CodeStubAssembler m(data.state());
int8_t buffer[] = {1, 2, 4, 8, 17, 33, 65, 127, 67, 38};
const int element_count = 10;
Label bad(&m);
Node* constants[element_count];
Node* buffer_node = m.IntPtrConstant(reinterpret_cast<intptr_t>(buffer));
for (size_t i = 0; i < element_count; ++i) {
constants[i] = m.LoadBufferObject(buffer_node, static_cast<int>(i),
MachineType::Uint8());
}
for (size_t i = 0; i < element_count; ++i) {
for (size_t j = 0; j < element_count; ++j) {
Node* loaded = m.LoadBufferObject(buffer_node, static_cast<int>(i),
MachineType::Uint8());
Node* masked = m.Word32And(loaded, constants[j]);
if ((buffer[j] & buffer[i]) != 0) {
m.GotoIf(m.Word32Equal(masked, m.Int32Constant(0)), &bad);
} else {
m.GotoIf(m.Word32NotEqual(masked, m.Int32Constant(0)), &bad);
}
masked = m.Word32And(constants[i], constants[j]);
if ((buffer[j] & buffer[i]) != 0) {
m.GotoIf(m.Word32Equal(masked, m.Int32Constant(0)), &bad);
} else {
m.GotoIf(m.Word32NotEqual(masked, m.Int32Constant(0)), &bad);
}
}
}
m.Return(m.SmiConstant(1));
m.Bind(&bad);
m.Return(m.SmiConstant(0));
Handle<Code> code = data.GenerateCode();
CHECK(!code.is_null());
FunctionTester ft(code, kNumParams);
CHECK_EQ(1, Handle<Smi>::cast(ft.Call().ToHandleChecked())->value());
}
TEST(DirectMemoryTest16BitWord32) {
Isolate* isolate(CcTest::InitIsolateOnce());
typedef CodeAssemblerLabel Label;
const int kNumParams = 0;
CodeAssemblerTester data(isolate, kNumParams);
CodeStubAssembler m(data.state());
int16_t buffer[] = {1, 2, 4, 8, 12345, 33, 65, 255, 67, 3823};
const int element_count = 10;
Label bad(&m);
Node* constants[element_count];
Node* buffer_node1 = m.IntPtrConstant(reinterpret_cast<intptr_t>(buffer));
for (size_t i = 0; i < element_count; ++i) {
constants[i] =
m.LoadBufferObject(buffer_node1, static_cast<int>(i * sizeof(int16_t)),
MachineType::Uint16());
}
Node* buffer_node2 = m.IntPtrConstant(reinterpret_cast<intptr_t>(buffer));
for (size_t i = 0; i < element_count; ++i) {
for (size_t j = 0; j < element_count; ++j) {
Node* loaded = m.LoadBufferObject(buffer_node1,
static_cast<int>(i * sizeof(int16_t)),
MachineType::Uint16());
Node* masked = m.Word32And(loaded, constants[j]);
if ((buffer[j] & buffer[i]) != 0) {
m.GotoIf(m.Word32Equal(masked, m.Int32Constant(0)), &bad);
} else {
m.GotoIf(m.Word32NotEqual(masked, m.Int32Constant(0)), &bad);
}
// Force a memory access relative to a high-number register.
loaded = m.LoadBufferObject(buffer_node2,
static_cast<int>(i * sizeof(int16_t)),
MachineType::Uint16());
masked = m.Word32And(loaded, constants[j]);
if ((buffer[j] & buffer[i]) != 0) {
m.GotoIf(m.Word32Equal(masked, m.Int32Constant(0)), &bad);
} else {
m.GotoIf(m.Word32NotEqual(masked, m.Int32Constant(0)), &bad);
}
masked = m.Word32And(constants[i], constants[j]);
if ((buffer[j] & buffer[i]) != 0) {
m.GotoIf(m.Word32Equal(masked, m.Int32Constant(0)), &bad);
} else {
m.GotoIf(m.Word32NotEqual(masked, m.Int32Constant(0)), &bad);
}
}
}
m.Return(m.SmiConstant(1));
m.Bind(&bad);
m.Return(m.SmiConstant(0));
Handle<Code> code = data.GenerateCode();
CHECK(!code.is_null());
FunctionTester ft(code, kNumParams);
CHECK_EQ(1, Handle<Smi>::cast(ft.Call().ToHandleChecked())->value());
}
} // namespace internal
} // namespace v8
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment