Fixed and improved code for integral division. Fixed and extended tests.

Arithmetic right shifting is *not* division in two's complement
representation, only in one's complement. So we convert to one's
complement, shift, and go back to two's complement. By permutating the
last steps, one can get efficient branch-free code. This insight comes
from the paleozoic era of computer science, see the paper from 1976:

   Guy Lewis Steele Jr.: "Arithmetic Shifting Considered Harmful"
   ftp://publications.ai.mit.edu/ai-publications/pdf/AIM-378.pdf

This results in better and more correct code than our previous
"neg/shift/neg" dance.

LOG=y
BUG=v8:3151
R=bmeurer@chromium.org

Review URL: https://codereview.chromium.org/166793002

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@19434 ce2b1a6d-e550-0410-aec6-3dcde31c8c00
parent 9ffe004a
......@@ -1381,7 +1381,7 @@ LInstruction* LChunkBuilder::DoDiv(HDiv* instr) {
ASSERT(instr->right()->representation().Equals(instr->representation()));
if (instr->RightIsPowerOf2()) {
ASSERT(!instr->CheckFlag(HValue::kCanBeDivByZero));
LOperand* value = UseRegisterAtStart(instr->left());
LOperand* value = UseRegister(instr->left());
LDivI* div = new(zone()) LDivI(value, UseConstant(instr->right()), NULL);
return AssignEnvironment(DefineAsRegister(div));
}
......
......@@ -1262,6 +1262,8 @@ class LDivI V8_FINAL : public LTemplateInstruction<1, 2, 1> {
LOperand* right() { return inputs_[1]; }
LOperand* temp() { return temps_[0]; }
bool is_flooring() { return hydrogen_value()->IsMathFloorOfDiv(); }
DECLARE_CONCRETE_INSTRUCTION(DivI, "div-i")
DECLARE_HYDROGEN_ACCESSOR(Div)
};
......
......@@ -2583,131 +2583,103 @@ void LCodeGen::DoDeoptimize(LDeoptimize* instr) {
void LCodeGen::DoDivI(LDivI* instr) {
Register dividend = ToRegister32(instr->left());
Register result = ToRegister32(instr->result());
bool has_power_of_2_divisor = instr->hydrogen()->RightIsPowerOf2();
bool can_overflow = instr->hydrogen()->CheckFlag(HValue::kCanOverflow);
bool bailout_on_minus_zero =
instr->hydrogen()->CheckFlag(HValue::kBailoutOnMinusZero);
bool can_be_div_by_zero =
instr->hydrogen()->CheckFlag(HValue::kCanBeDivByZero);
bool all_uses_truncating_to_int32 =
instr->hydrogen()->CheckFlag(HInstruction::kAllUsesTruncatingToInt32);
if (!instr->is_flooring() && instr->hydrogen()->RightIsPowerOf2()) {
HDiv* hdiv = instr->hydrogen();
Register dividend = ToRegister32(instr->left());
int32_t divisor = hdiv->right()->GetInteger32Constant();
Register result = ToRegister32(instr->result());
ASSERT(!result.is(dividend));
if (has_power_of_2_divisor) {
ASSERT(instr->temp() == NULL);
int32_t divisor = ToInteger32(LConstantOperand::cast(instr->right()));
int32_t power;
int32_t power_mask;
Label deopt, done;
ASSERT(divisor != 0);
if (divisor > 0) {
power = WhichPowerOf2(divisor);
power_mask = divisor - 1;
} else {
// Check for (0 / -x) as that will produce negative zero.
if (bailout_on_minus_zero) {
if (all_uses_truncating_to_int32) {
// If all uses truncate, and the dividend is zero, the truncated
// result is zero.
__ Mov(result, 0);
__ Cbz(dividend, &done);
} else {
__ Cbz(dividend, &deopt);
}
}
// Check for (kMinInt / -1).
if ((divisor == -1) && can_overflow && !all_uses_truncating_to_int32) {
// Check for kMinInt by subtracting one and checking for overflow.
__ Cmp(dividend, 1);
__ B(vs, &deopt);
}
power = WhichPowerOf2(-divisor);
power_mask = -divisor - 1;
// Check for (0 / -x) that will produce negative zero.
if (hdiv->left()->RangeCanInclude(0) && divisor < 0 &&
hdiv->CheckFlag(HValue::kBailoutOnMinusZero)) {
__ Cmp(dividend, 0);
DeoptimizeIf(eq, instr->environment());
}
if (power_mask != 0) {
if (all_uses_truncating_to_int32) {
__ Cmp(dividend, 0);
__ Cneg(result, dividend, lt);
__ Asr(result, result, power);
if (divisor > 0) __ Cneg(result, result, lt);
if (divisor < 0) __ Cneg(result, result, gt);
return; // Don't fall through to negation below.
} else {
// Deoptimize if remainder is not 0. If the least-significant
// power bits aren't 0, it's not a multiple of 2^power, and
// therefore, there will be a remainder.
__ TestAndBranchIfAnySet(dividend, power_mask, &deopt);
__ Asr(result, dividend, power);
if (divisor < 0) __ Neg(result, result);
}
// Check for (kMinInt / -1).
if (hdiv->left()->RangeCanInclude(kMinInt) && divisor == -1 &&
hdiv->CheckFlag(HValue::kCanOverflow)) {
__ Cmp(dividend, kMinInt);
DeoptimizeIf(eq, instr->environment());
}
// Deoptimize if remainder will not be 0.
if (!hdiv->CheckFlag(HInstruction::kAllUsesTruncatingToInt32) &&
Abs(divisor) != 1) {
__ Tst(dividend, Abs(divisor) - 1);
DeoptimizeIf(ne, instr->environment());
}
if (divisor == -1) { // Nice shortcut, not needed for correctness.
__ Neg(result, dividend);
return;
}
int32_t shift = WhichPowerOf2(Abs(divisor));
if (shift == 0) {
__ Mov(result, dividend);
} else if (shift == 1) {
__ Add(result, dividend, Operand(dividend, LSR, 31));
} else {
ASSERT((divisor == 1) || (divisor == -1));
if (divisor < 0) {
__ Neg(result, dividend);
} else {
__ Mov(result, dividend);
}
__ Mov(result, Operand(dividend, ASR, 31));
__ Add(result, dividend, Operand(result, LSR, 32 - shift));
}
__ B(&done);
__ Bind(&deopt);
Deoptimize(instr->environment());
__ Bind(&done);
} else {
Register divisor = ToRegister32(instr->right());
// Issue the division first, and then check for any deopt cases whilst the
// result is computed.
__ Sdiv(result, dividend, divisor);
if (shift > 0) __ Mov(result, Operand(result, ASR, shift));
if (divisor < 0) __ Neg(result, result);
return;
}
if (!all_uses_truncating_to_int32) {
Label deopt;
// Check for x / 0.
if (can_be_div_by_zero) {
__ Cbz(divisor, &deopt);
}
Register dividend = ToRegister32(instr->left());
Register divisor = ToRegister32(instr->right());
Register result = ToRegister32(instr->result());
HValue* hdiv = instr->hydrogen_value();
// Check for (0 / -x) as that will produce negative zero.
if (bailout_on_minus_zero) {
__ Cmp(divisor, 0);
// Issue the division first, and then check for any deopt cases whilst the
// result is computed.
__ Sdiv(result, dividend, divisor);
// If the divisor < 0 (mi), compare the dividend, and deopt if it is
// zero, ie. zero dividend with negative divisor deopts.
// If the divisor >= 0 (pl, the opposite of mi) set the flags to
// condition ne, so we don't deopt, ie. positive divisor doesn't deopt.
__ Ccmp(dividend, 0, NoFlag, mi);
__ B(eq, &deopt);
}
if (hdiv->CheckFlag(HInstruction::kAllUsesTruncatingToInt32)) {
ASSERT_EQ(NULL, instr->temp());
return;
}
// Check for (kMinInt / -1).
if (can_overflow) {
// Test dividend for kMinInt by subtracting one (cmp) and checking for
// overflow.
__ Cmp(dividend, 1);
// If overflow is set, ie. dividend = kMinInt, compare the divisor with
// -1. If overflow is clear, set the flags for condition ne, as the
// dividend isn't -1, and thus we shouldn't deopt.
__ Ccmp(divisor, -1, NoFlag, vs);
__ B(eq, &deopt);
}
Label deopt;
// Check for x / 0.
if (hdiv->CheckFlag(HValue::kCanBeDivByZero)) {
__ Cbz(divisor, &deopt);
}
// Compute remainder and deopt if it's not zero.
Register remainder = ToRegister32(instr->temp());
__ Msub(remainder, result, divisor, dividend);
__ Cbnz(remainder, &deopt);
// Check for (0 / -x) as that will produce negative zero.
if (hdiv->CheckFlag(HValue::kBailoutOnMinusZero)) {
__ Cmp(divisor, 0);
Label div_ok;
__ B(&div_ok);
__ Bind(&deopt);
Deoptimize(instr->environment());
__ Bind(&div_ok);
} else {
ASSERT(instr->temp() == NULL);
}
// If the divisor < 0 (mi), compare the dividend, and deopt if it is
// zero, ie. zero dividend with negative divisor deopts.
// If the divisor >= 0 (pl, the opposite of mi) set the flags to
// condition ne, so we don't deopt, ie. positive divisor doesn't deopt.
__ Ccmp(dividend, 0, NoFlag, mi);
__ B(eq, &deopt);
}
// Check for (kMinInt / -1).
if (hdiv->CheckFlag(HValue::kCanOverflow)) {
// Test dividend for kMinInt by subtracting one (cmp) and checking for
// overflow.
__ Cmp(dividend, 1);
// If overflow is set, ie. dividend = kMinInt, compare the divisor with
// -1. If overflow is clear, set the flags for condition ne, as the
// dividend isn't -1, and thus we shouldn't deopt.
__ Ccmp(divisor, -1, NoFlag, vs);
__ B(eq, &deopt);
}
// Compute remainder and deopt if it's not zero.
Register remainder = ToRegister32(instr->temp());
__ Msub(remainder, result, divisor, dividend);
__ Cbnz(remainder, &deopt);
Label div_ok;
__ B(&div_ok);
__ Bind(&deopt);
Deoptimize(instr->environment());
__ Bind(&div_ok);
}
......
......@@ -1236,7 +1236,7 @@ LInstruction* LChunkBuilder::DoDiv(HDiv* instr) {
ASSERT(instr->right()->representation().Equals(instr->representation()));
if (instr->RightIsPowerOf2()) {
ASSERT(!instr->CheckFlag(HValue::kCanBeDivByZero));
LOperand* value = UseRegisterAtStart(instr->left());
LOperand* value = UseRegister(instr->left());
LDivI* div = new(zone()) LDivI(value, UseConstant(instr->right()), NULL);
return AssignEnvironment(DefineAsRegister(div));
}
......
......@@ -1343,54 +1343,45 @@ void LCodeGen::EmitSignedIntegerDivisionByConstant(
void LCodeGen::DoDivI(LDivI* instr) {
if (!instr->is_flooring() && instr->hydrogen()->RightIsPowerOf2()) {
const Register dividend = ToRegister(instr->left());
const Register result = ToRegister(instr->result());
int32_t divisor = instr->hydrogen()->right()->GetInteger32Constant();
int32_t test_value = 0;
int32_t power = 0;
if (divisor > 0) {
test_value = divisor - 1;
power = WhichPowerOf2(divisor);
} else {
// Check for (0 / -x) that will produce negative zero.
if (instr->hydrogen()->CheckFlag(HValue::kBailoutOnMinusZero)) {
__ cmp(dividend, Operand::Zero());
DeoptimizeIf(eq, instr->environment());
}
// Check for (kMinInt / -1).
if (divisor == -1 && instr->hydrogen()->CheckFlag(HValue::kCanOverflow)) {
__ cmp(dividend, Operand(kMinInt));
DeoptimizeIf(eq, instr->environment());
}
test_value = - divisor - 1;
power = WhichPowerOf2(-divisor);
}
Register dividend = ToRegister(instr->left());
HDiv* hdiv = instr->hydrogen();
int32_t divisor = hdiv->right()->GetInteger32Constant();
Register result = ToRegister(instr->result());
ASSERT(!result.is(dividend));
if (test_value != 0) {
if (instr->hydrogen()->CheckFlag(
HInstruction::kAllUsesTruncatingToInt32)) {
__ sub(result, dividend, Operand::Zero(), SetCC);
__ rsb(result, result, Operand::Zero(), LeaveCC, lt);
__ mov(result, Operand(result, ASR, power));
if (divisor > 0) __ rsb(result, result, Operand::Zero(), LeaveCC, lt);
if (divisor < 0) __ rsb(result, result, Operand::Zero(), LeaveCC, gt);
return; // Don't fall through to "__ rsb" below.
} else {
// Deoptimize if remainder is not 0.
__ tst(dividend, Operand(test_value));
DeoptimizeIf(ne, instr->environment());
__ mov(result, Operand(dividend, ASR, power));
if (divisor < 0) __ rsb(result, result, Operand(0));
}
// Check for (0 / -x) that will produce negative zero.
if (hdiv->left()->RangeCanInclude(0) && divisor < 0 &&
hdiv->CheckFlag(HValue::kBailoutOnMinusZero)) {
__ cmp(dividend, Operand::Zero());
DeoptimizeIf(eq, instr->environment());
}
// Check for (kMinInt / -1).
if (hdiv->left()->RangeCanInclude(kMinInt) && divisor == -1 &&
hdiv->CheckFlag(HValue::kCanOverflow)) {
__ cmp(dividend, Operand(kMinInt));
DeoptimizeIf(eq, instr->environment());
}
// Deoptimize if remainder will not be 0.
if (!hdiv->CheckFlag(HInstruction::kAllUsesTruncatingToInt32) &&
Abs(divisor) != 1) {
__ tst(dividend, Operand(Abs(divisor) - 1));
DeoptimizeIf(ne, instr->environment());
}
if (divisor == -1) { // Nice shortcut, not needed for correctness.
__ rsb(result, dividend, Operand(0));
return;
}
int32_t shift = WhichPowerOf2(Abs(divisor));
if (shift == 0) {
__ mov(result, dividend);
} else if (shift == 1) {
__ add(result, dividend, Operand(dividend, LSR, 31));
} else {
if (divisor < 0) {
__ rsb(result, dividend, Operand(0));
} else {
__ Move(result, dividend);
}
__ mov(result, Operand(dividend, ASR, 31));
__ add(result, dividend, Operand(result, LSR, 32 - shift));
}
if (shift > 0) __ mov(result, Operand(result, ASR, shift));
if (divisor < 0) __ rsb(result, result, Operand(0));
return;
}
......
......@@ -1453,54 +1453,39 @@ void LCodeGen::DoModI(LModI* instr) {
void LCodeGen::DoDivI(LDivI* instr) {
if (!instr->is_flooring() && instr->hydrogen()->RightIsPowerOf2()) {
Register dividend = ToRegister(instr->left());
int32_t divisor = instr->hydrogen()->right()->GetInteger32Constant();
int32_t test_value = 0;
int32_t power = 0;
HDiv* hdiv = instr->hydrogen();
int32_t divisor = hdiv->right()->GetInteger32Constant();
Register result = ToRegister(instr->result());
ASSERT(!result.is(dividend));
if (divisor > 0) {
test_value = divisor - 1;
power = WhichPowerOf2(divisor);
} else {
// Check for (0 / -x) that will produce negative zero.
if (instr->hydrogen()->CheckFlag(HValue::kBailoutOnMinusZero)) {
__ test(dividend, Operand(dividend));
DeoptimizeIf(zero, instr->environment());
}
// Check for (kMinInt / -1).
if (divisor == -1 && instr->hydrogen()->CheckFlag(HValue::kCanOverflow)) {
__ cmp(dividend, kMinInt);
DeoptimizeIf(zero, instr->environment());
}
test_value = - divisor - 1;
power = WhichPowerOf2(-divisor);
// Check for (0 / -x) that will produce negative zero.
if (hdiv->left()->RangeCanInclude(0) && divisor < 0 &&
hdiv->CheckFlag(HValue::kBailoutOnMinusZero)) {
__ test(dividend, Operand(dividend));
DeoptimizeIf(zero, instr->environment());
}
if (test_value != 0) {
if (instr->hydrogen()->CheckFlag(
HInstruction::kAllUsesTruncatingToInt32)) {
Label done, negative;
__ cmp(dividend, 0);
__ j(less, &negative, Label::kNear);
__ sar(dividend, power);
if (divisor < 0) __ neg(dividend);
__ jmp(&done, Label::kNear);
__ bind(&negative);
__ neg(dividend);
__ sar(dividend, power);
if (divisor > 0) __ neg(dividend);
__ bind(&done);
return; // Don't fall through to "__ neg" below.
} else {
// Deoptimize if remainder is not 0.
__ test(dividend, Immediate(test_value));
// Check for (kMinInt / -1).
if (hdiv->left()->RangeCanInclude(kMinInt) && divisor == -1 &&
hdiv->CheckFlag(HValue::kCanOverflow)) {
__ cmp(dividend, kMinInt);
DeoptimizeIf(zero, instr->environment());
}
// Deoptimize if remainder will not be 0.
if (!hdiv->CheckFlag(HInstruction::kAllUsesTruncatingToInt32) &&
Abs(divisor) != 1) {
__ test(dividend, Immediate(Abs(divisor) - 1));
DeoptimizeIf(not_zero, instr->environment());
__ sar(dividend, power);
}
}
if (divisor < 0) __ neg(dividend);
__ Move(result, dividend);
int32_t shift = WhichPowerOf2(Abs(divisor));
if (shift > 0) {
// The arithmetic shift is always OK, the 'if' is an optimization only.
if (shift > 1) __ sar(result, 31);
__ shr(result, 32 - shift);
__ add(result, dividend);
__ sar(result, shift);
}
if (divisor < 0) __ neg(result);
return;
}
......
......@@ -1319,10 +1319,10 @@ LInstruction* LChunkBuilder::DoDiv(HDiv* instr) {
ASSERT(instr->right()->representation().Equals(instr->representation()));
if (instr->RightIsPowerOf2()) {
ASSERT(!instr->CheckFlag(HValue::kCanBeDivByZero));
LOperand* value = UseRegisterAtStart(instr->left());
LOperand* value = UseRegister(instr->left());
LDivI* div =
new(zone()) LDivI(value, UseOrConstant(instr->right()), NULL);
return AssignEnvironment(DefineSameAsFirst(div));
return AssignEnvironment(DefineAsRegister(div));
}
// The temporary operand is necessary to ensure that right is not allocated
// into edx.
......
......@@ -1152,55 +1152,38 @@ void LCodeGen::DoMathFloorOfDiv(LMathFloorOfDiv* instr) {
void LCodeGen::DoDivI(LDivI* instr) {
if (!instr->is_flooring() && instr->hydrogen()->RightIsPowerOf2()) {
Register dividend = ToRegister(instr->left());
int32_t divisor =
HConstant::cast(instr->hydrogen()->right())->Integer32Value();
int32_t test_value = 0;
int32_t power = 0;
if (divisor > 0) {
test_value = divisor - 1;
power = WhichPowerOf2(divisor);
} else {
// Check for (0 / -x) that will produce negative zero.
if (instr->hydrogen()->CheckFlag(HValue::kBailoutOnMinusZero)) {
__ testl(dividend, dividend);
DeoptimizeIf(zero, instr->environment());
}
// Check for (kMinInt / -1).
if (divisor == -1 && instr->hydrogen()->CheckFlag(HValue::kCanOverflow)) {
__ cmpl(dividend, Immediate(kMinInt));
DeoptimizeIf(zero, instr->environment());
}
test_value = - divisor - 1;
power = WhichPowerOf2(-divisor);
}
if (test_value != 0) {
if (instr->hydrogen()->CheckFlag(
HInstruction::kAllUsesTruncatingToInt32)) {
Label done, negative;
__ cmpl(dividend, Immediate(0));
__ j(less, &negative, Label::kNear);
__ sarl(dividend, Immediate(power));
if (divisor < 0) __ negl(dividend);
__ jmp(&done, Label::kNear);
HDiv* hdiv = instr->hydrogen();
int32_t divisor = hdiv->right()->GetInteger32Constant();
Register result = ToRegister(instr->result());
ASSERT(!result.is(dividend));
__ bind(&negative);
__ negl(dividend);
__ sarl(dividend, Immediate(power));
if (divisor > 0) __ negl(dividend);
__ bind(&done);
return; // Don't fall through to "__ neg" below.
} else {
// Deoptimize if remainder is not 0.
__ testl(dividend, Immediate(test_value));
DeoptimizeIf(not_zero, instr->environment());
__ sarl(dividend, Immediate(power));
}
// Check for (0 / -x) that will produce negative zero.
if (hdiv->left()->RangeCanInclude(0) && divisor < 0 &&
hdiv->CheckFlag(HValue::kBailoutOnMinusZero)) {
__ testl(dividend, dividend);
DeoptimizeIf(zero, instr->environment());
}
if (divisor < 0) __ negl(dividend);
// Check for (kMinInt / -1).
if (hdiv->left()->RangeCanInclude(kMinInt) && divisor == -1 &&
hdiv->CheckFlag(HValue::kCanOverflow)) {
__ cmpl(dividend, Immediate(kMinInt));
DeoptimizeIf(zero, instr->environment());
}
// Deoptimize if remainder will not be 0.
if (!hdiv->CheckFlag(HInstruction::kAllUsesTruncatingToInt32)) {
__ testl(dividend, Immediate(Abs(divisor) - 1));
DeoptimizeIf(not_zero, instr->environment());
}
__ Move(result, dividend);
int32_t shift = WhichPowerOf2(Abs(divisor));
if (shift > 0) {
// The arithmetic shift is always OK, the 'if' is an optimization only.
if (shift > 1) __ sarl(result, Immediate(31));
__ shrl(result, Immediate(32 - shift));
__ addl(result, dividend);
__ sarl(result, Immediate(shift));
}
if (divisor < 0) __ negl(result);
return;
}
......
......@@ -1240,10 +1240,10 @@ LInstruction* LChunkBuilder::DoDiv(HDiv* instr) {
ASSERT(instr->right()->representation().Equals(instr->representation()));
if (instr->RightIsPowerOf2()) {
ASSERT(!instr->CheckFlag(HValue::kCanBeDivByZero));
LOperand* value = UseRegisterAtStart(instr->left());
LOperand* value = UseRegister(instr->left());
LDivI* div =
new(zone()) LDivI(value, UseOrConstant(instr->right()), NULL);
return AssignEnvironment(DefineSameAsFirst(div));
return AssignEnvironment(DefineAsRegister(div));
}
// The temporary operand is necessary to ensure that right is not allocated
// into rdx.
......
......@@ -60,7 +60,7 @@ divn1(2);
divn1(2);
%OptimizeFunctionOnNextCall(divn1);
assertEquals(-2, divn1(2));
assertEquals(two_31, divn1(-two_31));
assertEquals(-two_31, divn1(two_31));
//Check for truncating to int32 case
......@@ -85,3 +85,14 @@ divn4t(8);
assertEquals(1, divn4t(-5));
assertEquals(-1, divn4t(5));
assertOptimized(divn4t);
// Check kMinInt case.
function div_by_two(x) {
return (x / 2) | 0;
}
div_by_two(12);
div_by_two(34);
%OptimizeFunctionOnNextCall(div_by_two);
div_by_two(56);
assertEquals(-(1 << 30), div_by_two(1 << 31));
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment