Fixed and improved code for integral division. Fixed and extended tests.

Arithmetic right shifting is *not* division in two's complement
representation, only in one's complement. So we convert to one's
complement, shift, and go back to two's complement. By permutating the
last steps, one can get efficient branch-free code. This insight comes
from the paleozoic era of computer science, see the paper from 1976:

   Guy Lewis Steele Jr.: "Arithmetic Shifting Considered Harmful"
   ftp://publications.ai.mit.edu/ai-publications/pdf/AIM-378.pdf

This results in better and more correct code than our previous
"neg/shift/neg" dance.

LOG=y
BUG=v8:3151
R=bmeurer@chromium.org

Review URL: https://codereview.chromium.org/166793002

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@19434 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/a64/lithium-a64.cc

@@ -1381,7 +1381,7 @@ LInstruction* LChunkBuilder::DoDiv(HDiv* instr) {
     ASSERT(instr->right()->representation().Equals(instr->representation()));
     if (instr->RightIsPowerOf2()) {
       ASSERT(!instr->CheckFlag(HValue::kCanBeDivByZero));
-      LOperand* value = UseRegisterAtStart(instr->left());
+      LOperand* value = UseRegister(instr->left());
       LDivI* div = new(zone()) LDivI(value, UseConstant(instr->right()), NULL);
       return AssignEnvironment(DefineAsRegister(div));
     }

src/a64/lithium-a64.h

@@ -1262,6 +1262,8 @@ class LDivI V8_FINAL : public LTemplateInstruction<1, 2, 1> {
   LOperand* right() { return inputs_[1]; }
   LOperand* temp() { return temps_[0]; }
 
+  bool is_flooring() { return hydrogen_value()->IsMathFloorOfDiv(); }
+
   DECLARE_CONCRETE_INSTRUCTION(DivI, "div-i")
   DECLARE_HYDROGEN_ACCESSOR(Div)
 };

src/a64/lithium-codegen-a64.cc

@@ -2583,95 +2583,71 @@ void LCodeGen::DoDeoptimize(LDeoptimize* instr) {
 
 
 void LCodeGen::DoDivI(LDivI* instr) {
+  if (!instr->is_flooring() && instr->hydrogen()->RightIsPowerOf2()) {
+    HDiv* hdiv = instr->hydrogen();
     Register dividend = ToRegister32(instr->left());
+    int32_t divisor = hdiv->right()->GetInteger32Constant();
     Register result = ToRegister32(instr->result());
+    ASSERT(!result.is(dividend));
 
-  bool has_power_of_2_divisor = instr->hydrogen()->RightIsPowerOf2();
-  bool can_overflow = instr->hydrogen()->CheckFlag(HValue::kCanOverflow);
-  bool bailout_on_minus_zero =
-      instr->hydrogen()->CheckFlag(HValue::kBailoutOnMinusZero);
-  bool can_be_div_by_zero =
-      instr->hydrogen()->CheckFlag(HValue::kCanBeDivByZero);
-  bool all_uses_truncating_to_int32 =
-      instr->hydrogen()->CheckFlag(HInstruction::kAllUsesTruncatingToInt32);
-
-  if (has_power_of_2_divisor) {
-    ASSERT(instr->temp() == NULL);
-    int32_t divisor = ToInteger32(LConstantOperand::cast(instr->right()));
-    int32_t power;
-    int32_t power_mask;
-    Label deopt, done;
-
-    ASSERT(divisor != 0);
-    if (divisor > 0) {
-      power = WhichPowerOf2(divisor);
-      power_mask = divisor - 1;
-    } else {
-      // Check for (0 / -x) as that will produce negative zero.
-      if (bailout_on_minus_zero) {
-        if (all_uses_truncating_to_int32) {
-         // If all uses truncate, and the dividend is zero, the truncated
-         // result is zero.
-         __ Mov(result, 0);
-         __ Cbz(dividend, &done);
-        } else {
-          __ Cbz(dividend, &deopt);
-        }
+    // Check for (0 / -x) that will produce negative zero.
+    if (hdiv->left()->RangeCanInclude(0) && divisor < 0 &&
+        hdiv->CheckFlag(HValue::kBailoutOnMinusZero)) {
+      __ Cmp(dividend, 0);
+      DeoptimizeIf(eq, instr->environment());
     }
     // Check for (kMinInt / -1).
-      if ((divisor == -1) && can_overflow && !all_uses_truncating_to_int32) {
-        // Check for kMinInt by subtracting one and checking for overflow.
-        __ Cmp(dividend, 1);
-        __ B(vs, &deopt);
-      }
-      power = WhichPowerOf2(-divisor);
-      power_mask = -divisor - 1;
+    if (hdiv->left()->RangeCanInclude(kMinInt) && divisor == -1 &&
+        hdiv->CheckFlag(HValue::kCanOverflow)) {
+      __ Cmp(dividend, kMinInt);
+      DeoptimizeIf(eq, instr->environment());
     }
-
-    if (power_mask != 0) {
-      if (all_uses_truncating_to_int32) {
-        __ Cmp(dividend, 0);
-        __ Cneg(result, dividend, lt);
-        __ Asr(result, result, power);
-        if (divisor > 0) __ Cneg(result, result, lt);
-        if (divisor < 0) __ Cneg(result, result, gt);
-        return;  // Don't fall through to negation below.
-      } else {
-        // Deoptimize if remainder is not 0. If the least-significant
-        // power bits aren't 0, it's not a multiple of 2^power, and
-        // therefore, there will be a remainder.
-        __ TestAndBranchIfAnySet(dividend, power_mask, &deopt);
-        __ Asr(result, dividend, power);
-        if (divisor < 0) __ Neg(result, result);
+    // Deoptimize if remainder will not be 0.
+    if (!hdiv->CheckFlag(HInstruction::kAllUsesTruncatingToInt32) &&
+        Abs(divisor) != 1) {
+      __ Tst(dividend, Abs(divisor) - 1);
+      DeoptimizeIf(ne, instr->environment());
     }
-    } else {
-      ASSERT((divisor == 1) || (divisor == -1));
-      if (divisor < 0) {
+    if (divisor == -1) {  // Nice shortcut, not needed for correctness.
       __ Neg(result, dividend);
-      } else {
+      return;
+    }
+    int32_t shift = WhichPowerOf2(Abs(divisor));
+    if (shift == 0) {
       __ Mov(result, dividend);
+    } else if (shift == 1) {
+      __ Add(result, dividend, Operand(dividend, LSR, 31));
+    } else {
+      __ Mov(result, Operand(dividend, ASR, 31));
+      __ Add(result, dividend, Operand(result, LSR, 32 - shift));
     }
+    if (shift > 0) __ Mov(result, Operand(result, ASR, shift));
+    if (divisor < 0) __ Neg(result, result);
+    return;
   }
-    __ B(&done);
-    __ Bind(&deopt);
-    Deoptimize(instr->environment());
-    __ Bind(&done);
-  } else {
+
+  Register dividend = ToRegister32(instr->left());
   Register divisor = ToRegister32(instr->right());
+  Register result = ToRegister32(instr->result());
+  HValue* hdiv = instr->hydrogen_value();
 
   // Issue the division first, and then check for any deopt cases whilst the
   // result is computed.
   __ Sdiv(result, dividend, divisor);
 
-    if (!all_uses_truncating_to_int32) {
+  if (hdiv->CheckFlag(HInstruction::kAllUsesTruncatingToInt32)) {
+    ASSERT_EQ(NULL, instr->temp());
+    return;
+  }
+
   Label deopt;
   // Check for x / 0.
-      if (can_be_div_by_zero) {
+  if (hdiv->CheckFlag(HValue::kCanBeDivByZero)) {
     __ Cbz(divisor, &deopt);
   }
 
   // Check for (0 / -x) as that will produce negative zero.
-      if (bailout_on_minus_zero) {
+  if (hdiv->CheckFlag(HValue::kBailoutOnMinusZero)) {
     __ Cmp(divisor, 0);
 
     // If the divisor < 0 (mi), compare the dividend, and deopt if it is
@@ -2683,7 +2659,7 @@ void LCodeGen::DoDivI(LDivI* instr) {
   }
 
   // Check for (kMinInt / -1).
-      if (can_overflow) {
+  if (hdiv->CheckFlag(HValue::kCanOverflow)) {
     // Test dividend for kMinInt by subtracting one (cmp) and checking for
     // overflow.
     __ Cmp(dividend, 1);
@@ -2704,10 +2680,6 @@ void LCodeGen::DoDivI(LDivI* instr) {
   __ Bind(&deopt);
   Deoptimize(instr->environment());
   __ Bind(&div_ok);
-    } else {
-      ASSERT(instr->temp() == NULL);
-    }
-  }
 }
 
 

src/arm/lithium-arm.cc

@@ -1236,7 +1236,7 @@ LInstruction* LChunkBuilder::DoDiv(HDiv* instr) {
     ASSERT(instr->right()->representation().Equals(instr->representation()));
     if (instr->RightIsPowerOf2()) {
       ASSERT(!instr->CheckFlag(HValue::kCanBeDivByZero));
-      LOperand* value = UseRegisterAtStart(instr->left());
+      LOperand* value = UseRegister(instr->left());
       LDivI* div = new(zone()) LDivI(value, UseConstant(instr->right()), NULL);
       return AssignEnvironment(DefineAsRegister(div));
     }

src/arm/lithium-codegen-arm.cc

@@ -1343,54 +1343,45 @@ void LCodeGen::EmitSignedIntegerDivisionByConstant(
 
 void LCodeGen::DoDivI(LDivI* instr) {
   if (!instr->is_flooring() && instr->hydrogen()->RightIsPowerOf2()) {
-    const Register dividend = ToRegister(instr->left());
-    const Register result = ToRegister(instr->result());
-    int32_t divisor = instr->hydrogen()->right()->GetInteger32Constant();
-    int32_t test_value = 0;
-    int32_t power = 0;
+    Register dividend = ToRegister(instr->left());
+    HDiv* hdiv = instr->hydrogen();
+    int32_t divisor = hdiv->right()->GetInteger32Constant();
+    Register result = ToRegister(instr->result());
+    ASSERT(!result.is(dividend));
 
-    if (divisor > 0) {
-      test_value = divisor - 1;
-      power = WhichPowerOf2(divisor);
-    } else {
     // Check for (0 / -x) that will produce negative zero.
-      if (instr->hydrogen()->CheckFlag(HValue::kBailoutOnMinusZero)) {
+    if (hdiv->left()->RangeCanInclude(0) && divisor < 0 &&
+        hdiv->CheckFlag(HValue::kBailoutOnMinusZero)) {
       __ cmp(dividend, Operand::Zero());
       DeoptimizeIf(eq, instr->environment());
     }
     // Check for (kMinInt / -1).
-      if (divisor == -1 && instr->hydrogen()->CheckFlag(HValue::kCanOverflow)) {
+    if (hdiv->left()->RangeCanInclude(kMinInt) && divisor == -1 &&
+        hdiv->CheckFlag(HValue::kCanOverflow)) {
       __ cmp(dividend, Operand(kMinInt));
       DeoptimizeIf(eq, instr->environment());
     }
-      test_value = - divisor - 1;
-      power = WhichPowerOf2(-divisor);
-    }
-
-    if (test_value != 0) {
-      if (instr->hydrogen()->CheckFlag(
-          HInstruction::kAllUsesTruncatingToInt32)) {
-        __ sub(result, dividend, Operand::Zero(), SetCC);
-        __ rsb(result, result, Operand::Zero(), LeaveCC, lt);
-        __ mov(result, Operand(result, ASR, power));
-        if (divisor > 0) __ rsb(result, result, Operand::Zero(), LeaveCC, lt);
-        if (divisor < 0) __ rsb(result, result, Operand::Zero(), LeaveCC, gt);
-        return;  // Don't fall through to "__ rsb" below.
-      } else {
-        // Deoptimize if remainder is not 0.
-        __ tst(dividend, Operand(test_value));
+    // Deoptimize if remainder will not be 0.
+    if (!hdiv->CheckFlag(HInstruction::kAllUsesTruncatingToInt32) &&
+        Abs(divisor) != 1) {
+      __ tst(dividend, Operand(Abs(divisor) - 1));
       DeoptimizeIf(ne, instr->environment());
-        __ mov(result, Operand(dividend, ASR, power));
-        if (divisor < 0) __ rsb(result, result, Operand(0));
     }
-    } else {
-      if (divisor < 0) {
+    if (divisor == -1) {  // Nice shortcut, not needed for correctness.
       __ rsb(result, dividend, Operand(0));
-      } else {
-        __ Move(result, dividend);
+      return;
     }
+    int32_t shift = WhichPowerOf2(Abs(divisor));
+    if (shift == 0) {
+      __ mov(result, dividend);
+    } else if (shift == 1) {
+      __ add(result, dividend, Operand(dividend, LSR, 31));
+    } else {
+      __ mov(result, Operand(dividend, ASR, 31));
+      __ add(result, dividend, Operand(result, LSR, 32 - shift));
     }
-
+    if (shift > 0) __ mov(result, Operand(result, ASR, shift));
+    if (divisor < 0) __ rsb(result, result, Operand(0));
     return;
   }
 

src/ia32/lithium-codegen-ia32.cc

@@ -1453,54 +1453,39 @@ void LCodeGen::DoModI(LModI* instr) {
 void LCodeGen::DoDivI(LDivI* instr) {
   if (!instr->is_flooring() && instr->hydrogen()->RightIsPowerOf2()) {
     Register dividend = ToRegister(instr->left());
-    int32_t divisor = instr->hydrogen()->right()->GetInteger32Constant();
-    int32_t test_value = 0;
-    int32_t power = 0;
+    HDiv* hdiv = instr->hydrogen();
+    int32_t divisor = hdiv->right()->GetInteger32Constant();
+    Register result = ToRegister(instr->result());
+    ASSERT(!result.is(dividend));
 
-    if (divisor > 0) {
-      test_value = divisor - 1;
-      power = WhichPowerOf2(divisor);
-    } else {
     // Check for (0 / -x) that will produce negative zero.
-      if (instr->hydrogen()->CheckFlag(HValue::kBailoutOnMinusZero)) {
+    if (hdiv->left()->RangeCanInclude(0) && divisor < 0 &&
+        hdiv->CheckFlag(HValue::kBailoutOnMinusZero)) {
       __ test(dividend, Operand(dividend));
       DeoptimizeIf(zero, instr->environment());
     }
     // Check for (kMinInt / -1).
-      if (divisor == -1 && instr->hydrogen()->CheckFlag(HValue::kCanOverflow)) {
+    if (hdiv->left()->RangeCanInclude(kMinInt) && divisor == -1 &&
+        hdiv->CheckFlag(HValue::kCanOverflow)) {
       __ cmp(dividend, kMinInt);
       DeoptimizeIf(zero, instr->environment());
     }
-      test_value = - divisor - 1;
-      power = WhichPowerOf2(-divisor);
-    }
-
-    if (test_value != 0) {
-      if (instr->hydrogen()->CheckFlag(
-          HInstruction::kAllUsesTruncatingToInt32)) {
-        Label done, negative;
-        __ cmp(dividend, 0);
-        __ j(less, &negative, Label::kNear);
-        __ sar(dividend, power);
-        if (divisor < 0) __ neg(dividend);
-        __ jmp(&done, Label::kNear);
-
-        __ bind(&negative);
-        __ neg(dividend);
-        __ sar(dividend, power);
-        if (divisor > 0) __ neg(dividend);
-        __ bind(&done);
-        return;  // Don't fall through to "__ neg" below.
-      } else {
-        // Deoptimize if remainder is not 0.
-        __ test(dividend, Immediate(test_value));
+    // Deoptimize if remainder will not be 0.
+    if (!hdiv->CheckFlag(HInstruction::kAllUsesTruncatingToInt32) &&
+        Abs(divisor) != 1) {
+        __ test(dividend, Immediate(Abs(divisor) - 1));
         DeoptimizeIf(not_zero, instr->environment());
-        __ sar(dividend, power);
-      }
     }
-
-    if (divisor < 0) __ neg(dividend);
-
+    __ Move(result, dividend);
+    int32_t shift = WhichPowerOf2(Abs(divisor));
+    if (shift > 0) {
+      // The arithmetic shift is always OK, the 'if' is an optimization only.
+      if (shift > 1) __ sar(result, 31);
+      __ shr(result, 32 - shift);
+      __ add(result, dividend);
+      __ sar(result, shift);
+    }
+    if (divisor < 0) __ neg(result);
     return;
   }
 

src/ia32/lithium-ia32.cc

@@ -1319,10 +1319,10 @@ LInstruction* LChunkBuilder::DoDiv(HDiv* instr) {
     ASSERT(instr->right()->representation().Equals(instr->representation()));
     if (instr->RightIsPowerOf2()) {
       ASSERT(!instr->CheckFlag(HValue::kCanBeDivByZero));
-      LOperand* value = UseRegisterAtStart(instr->left());
+      LOperand* value = UseRegister(instr->left());
       LDivI* div =
           new(zone()) LDivI(value, UseOrConstant(instr->right()), NULL);
-      return AssignEnvironment(DefineSameAsFirst(div));
+      return AssignEnvironment(DefineAsRegister(div));
     }
     // The temporary operand is necessary to ensure that right is not allocated
     // into edx.

src/x64/lithium-codegen-x64.cc

@@ -1152,55 +1152,38 @@ void LCodeGen::DoMathFloorOfDiv(LMathFloorOfDiv* instr) {
 void LCodeGen::DoDivI(LDivI* instr) {
   if (!instr->is_flooring() && instr->hydrogen()->RightIsPowerOf2()) {
     Register dividend = ToRegister(instr->left());
-    int32_t divisor =
-        HConstant::cast(instr->hydrogen()->right())->Integer32Value();
-    int32_t test_value = 0;
-    int32_t power = 0;
+    HDiv* hdiv = instr->hydrogen();
+    int32_t divisor = hdiv->right()->GetInteger32Constant();
+    Register result = ToRegister(instr->result());
+    ASSERT(!result.is(dividend));
 
-    if (divisor > 0) {
-      test_value = divisor - 1;
-      power = WhichPowerOf2(divisor);
-    } else {
     // Check for (0 / -x) that will produce negative zero.
-      if (instr->hydrogen()->CheckFlag(HValue::kBailoutOnMinusZero)) {
+    if (hdiv->left()->RangeCanInclude(0) && divisor < 0 &&
+          hdiv->CheckFlag(HValue::kBailoutOnMinusZero)) {
       __ testl(dividend, dividend);
       DeoptimizeIf(zero, instr->environment());
     }
     // Check for (kMinInt / -1).
-      if (divisor == -1 && instr->hydrogen()->CheckFlag(HValue::kCanOverflow)) {
+    if (hdiv->left()->RangeCanInclude(kMinInt) && divisor == -1 &&
+        hdiv->CheckFlag(HValue::kCanOverflow)) {
       __ cmpl(dividend, Immediate(kMinInt));
       DeoptimizeIf(zero, instr->environment());
     }
-      test_value = - divisor - 1;
-      power = WhichPowerOf2(-divisor);
-    }
-
-    if (test_value != 0) {
-      if (instr->hydrogen()->CheckFlag(
-          HInstruction::kAllUsesTruncatingToInt32)) {
-        Label done, negative;
-        __ cmpl(dividend, Immediate(0));
-        __ j(less, &negative, Label::kNear);
-        __ sarl(dividend, Immediate(power));
-        if (divisor < 0) __ negl(dividend);
-        __ jmp(&done, Label::kNear);
-
-        __ bind(&negative);
-        __ negl(dividend);
-        __ sarl(dividend, Immediate(power));
-        if (divisor > 0) __ negl(dividend);
-        __ bind(&done);
-        return;  // Don't fall through to "__ neg" below.
-      } else {
-        // Deoptimize if remainder is not 0.
-        __ testl(dividend, Immediate(test_value));
+    // Deoptimize if remainder will not be 0.
+    if (!hdiv->CheckFlag(HInstruction::kAllUsesTruncatingToInt32)) {
+      __ testl(dividend, Immediate(Abs(divisor) - 1));
       DeoptimizeIf(not_zero, instr->environment());
-        __ sarl(dividend, Immediate(power));
     }
+    __ Move(result, dividend);
+    int32_t shift = WhichPowerOf2(Abs(divisor));
+    if (shift > 0) {
+      // The arithmetic shift is always OK, the 'if' is an optimization only.
+      if (shift > 1) __ sarl(result, Immediate(31));
+      __ shrl(result, Immediate(32 - shift));
+      __ addl(result, dividend);
+      __ sarl(result, Immediate(shift));
     }
-
-    if (divisor < 0) __ negl(dividend);
-
+    if (divisor < 0) __ negl(result);
     return;
   }
 

src/x64/lithium-x64.cc

@@ -1240,10 +1240,10 @@ LInstruction* LChunkBuilder::DoDiv(HDiv* instr) {
     ASSERT(instr->right()->representation().Equals(instr->representation()));
     if (instr->RightIsPowerOf2()) {
       ASSERT(!instr->CheckFlag(HValue::kCanBeDivByZero));
-      LOperand* value = UseRegisterAtStart(instr->left());
+      LOperand* value = UseRegister(instr->left());
       LDivI* div =
           new(zone()) LDivI(value, UseOrConstant(instr->right()), NULL);
-      return AssignEnvironment(DefineSameAsFirst(div));
+      return AssignEnvironment(DefineAsRegister(div));
     }
     // The temporary operand is necessary to ensure that right is not allocated
     // into rdx.

test/mjsunit/shift-for-integer-div.js

@@ -60,7 +60,7 @@ divn1(2);
 divn1(2);
 %OptimizeFunctionOnNextCall(divn1);
 assertEquals(-2, divn1(2));
-assertEquals(two_31, divn1(-two_31));
+assertEquals(-two_31, divn1(two_31));
 
 
 //Check for truncating to int32 case
@@ -85,3 +85,14 @@ divn4t(8);
 assertEquals(1, divn4t(-5));
 assertEquals(-1, divn4t(5));
 assertOptimized(divn4t);
+
+// Check kMinInt case.
+function div_by_two(x) {
+  return (x / 2) | 0;
+}
+
+div_by_two(12);
+div_by_two(34);
+%OptimizeFunctionOnNextCall(div_by_two);
+div_by_two(56);
+assertEquals(-(1 << 30), div_by_two(1 << 31));