[wasm] Int64Lowering of Int64Mul on ia32 and arm.

Int64Mul is lowered to a new turbofan operator, Int32MulPair. The new
operator takes 4 inputs an generates 2 outputs. The inputs are the low
word of the left input, high word of the left input, the low word of the
right input, and high word of the right input. The ouputs are the low
and high word of the result of the multiplication.

R=titzer@chromium.org, v8-arm-ports@googlegroups.com

Review URL: https://codereview.chromium.org/1807273002

Cr-Commit-Position: refs/heads/master@{#35131}

src/compiler/arm/code-generator-arm.cc

@@ -809,6 +809,18 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
              Operand(i.InputRegister(3)));
       DCHECK_EQ(LeaveCC, i.OutputSBit());
       break;
+    case kArmMulPair:
+      // i.InputRegister(0) ... left low word.
+      // i.InputRegister(1) ... left high word.
+      // i.InputRegister(2) ... right low word.
+      // i.InputRegister(3) ... right high word.
+      __ umull(i.OutputRegister(0), i.OutputRegister(1), i.InputRegister(0),
+               i.InputRegister(2));
+      __ mla(i.OutputRegister(1), i.InputRegister(0), i.InputRegister(3),
+             i.OutputRegister(1));
+      __ mla(i.OutputRegister(1), i.InputRegister(2), i.InputRegister(1),
+             i.OutputRegister(1));
+      break;
     case kArmLslPair:
       if (instr->InputAt(2)->IsImmediate()) {
         __ LslPair(i.OutputRegister(0), i.OutputRegister(1), i.InputRegister(0),

src/compiler/arm/instruction-codes-arm.h

@@ -48,6 +48,7 @@ namespace compiler {
   V(ArmUxtah)                      \
   V(ArmAddPair)                    \
   V(ArmSubPair)                    \
+  V(ArmMulPair)                    \
   V(ArmLslPair)                    \
   V(ArmLsrPair)                    \
   V(ArmAsrPair)                    \

src/compiler/arm/instruction-scheduler-arm.cc

@@ -50,6 +50,7 @@ int InstructionScheduler::GetTargetInstructionFlags(
     case kArmRbit:
     case kArmAddPair:
     case kArmSubPair:
+    case kArmMulPair:
     case kArmLslPair:
     case kArmLsrPair:
     case kArmAsrPair:

src/compiler/arm/instruction-selector-arm.cc

@@ -799,6 +799,20 @@ void InstructionSelector::VisitInt32PairSub(Node* node) {
   Emit(kArmSubPair, 2, outputs, 4, inputs);
 }
 
+void InstructionSelector::VisitInt32PairMul(Node* node) {
+  ArmOperandGenerator g(this);
+  InstructionOperand inputs[] = {g.UseUniqueRegister(node->InputAt(0)),
+                                 g.UseUniqueRegister(node->InputAt(1)),
+                                 g.UseUniqueRegister(node->InputAt(2)),
+                                 g.UseUniqueRegister(node->InputAt(3))};
+
+  InstructionOperand outputs[] = {
+      g.DefineAsRegister(node),
+      g.DefineAsRegister(NodeProperties::FindProjection(node, 1))};
+
+  Emit(kArmMulPair, 2, outputs, 4, inputs);
+}
+
 void InstructionSelector::VisitWord32PairShl(Node* node) {
   ArmOperandGenerator g(this);
   // We use g.UseUniqueRegister here for InputAt(0) to guarantee that there is

src/compiler/ia32/code-generator-ia32.cc

@@ -727,6 +727,18 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
       }
       break;
     }
+    case kIA32MulPair: {
+      __ imul(i.OutputRegister(1), i.InputOperand(0));
+      __ mov(i.TempRegister(0), i.InputOperand(1));
+      __ imul(i.TempRegister(0), i.InputOperand(2));
+      __ add(i.OutputRegister(1), i.TempRegister(0));
+      __ mov(i.OutputRegister(0), i.InputOperand(0));
+      // Multiplies the low words and stores them in eax and edx.
+      __ mul(i.InputRegister(2));
+      __ add(i.OutputRegister(1), i.TempRegister(0));
+
+      break;
+    }
     case kIA32ShlPair:
       if (HasImmediateInput(instr, 2)) {
         __ ShlPair(i.InputRegister(1), i.InputRegister(0), i.InputInt6(2));

src/compiler/ia32/instruction-codes-ia32.h

@@ -31,6 +31,7 @@ namespace compiler {
   V(IA32Sar)                       \
   V(IA32AddPair)                   \
   V(IA32SubPair)                   \
+  V(IA32MulPair)                   \
   V(IA32ShlPair)                   \
   V(IA32ShrPair)                   \
   V(IA32SarPair)                   \

src/compiler/ia32/instruction-scheduler-ia32.cc

@@ -33,6 +33,7 @@ int InstructionScheduler::GetTargetInstructionFlags(
     case kIA32Sar:
     case kIA32AddPair:
     case kIA32SubPair:
+    case kIA32MulPair:
     case kIA32ShlPair:
     case kIA32ShrPair:
     case kIA32SarPair:

src/compiler/ia32/instruction-selector-ia32.cc

@@ -620,6 +620,24 @@ void InstructionSelector::VisitInt32PairSub(Node* node) {
   Emit(kIA32SubPair, 2, outputs, 4, inputs, 1, temps);
 }
 
+void InstructionSelector::VisitInt32PairMul(Node* node) {
+  IA32OperandGenerator g(this);
+
+  // InputAt(3) explicitly shares ecx with OutputRegister(1) to save one
+  // register and one mov instruction.
+  InstructionOperand inputs[] = {
+      g.UseUnique(node->InputAt(0)), g.UseUnique(node->InputAt(1)),
+      g.UseUniqueRegister(node->InputAt(2)), g.UseFixed(node->InputAt(3), ecx)};
+
+  InstructionOperand outputs[] = {
+      g.DefineAsFixed(node, eax),
+      g.DefineAsFixed(NodeProperties::FindProjection(node, 1), ecx)};
+
+  InstructionOperand temps[] = {g.TempRegister(edx)};
+
+  Emit(kIA32MulPair, 2, outputs, 4, inputs, 1, temps);
+}
+
 void InstructionSelector::VisitWord32PairShl(Node* node) {
   IA32OperandGenerator g(this);
 

src/compiler/instruction-selector.cc

@@ -1158,6 +1158,10 @@ void InstructionSelector::VisitNode(Node* node) {
       MarkAsWord32(NodeProperties::FindProjection(node, 0));
       MarkAsWord32(NodeProperties::FindProjection(node, 1));
       return VisitInt32PairSub(node);
+    case IrOpcode::kInt32PairMul:
+      MarkAsWord32(NodeProperties::FindProjection(node, 0));
+      MarkAsWord32(NodeProperties::FindProjection(node, 1));
+      return VisitInt32PairMul(node);
     case IrOpcode::kWord32PairShl:
       MarkAsWord32(NodeProperties::FindProjection(node, 0));
       MarkAsWord32(NodeProperties::FindProjection(node, 1));
@@ -1399,6 +1403,8 @@ void InstructionSelector::VisitInt32PairAdd(Node* node) { UNIMPLEMENTED(); }
 
 void InstructionSelector::VisitInt32PairSub(Node* node) { UNIMPLEMENTED(); }
 
+void InstructionSelector::VisitInt32PairMul(Node* node) { UNIMPLEMENTED(); }
+
 void InstructionSelector::VisitWord32PairShl(Node* node) { UNIMPLEMENTED(); }
 
 void InstructionSelector::VisitWord32PairShr(Node* node) { UNIMPLEMENTED(); }
@@ -1485,6 +1491,7 @@ void InstructionSelector::VisitProjection(Node* node) {
     case IrOpcode::kTryTruncateFloat64ToUint64:
     case IrOpcode::kInt32PairAdd:
     case IrOpcode::kInt32PairSub:
+    case IrOpcode::kInt32PairMul:
     case IrOpcode::kWord32PairShl:
     case IrOpcode::kWord32PairShr:
     case IrOpcode::kWord32PairSar:

src/compiler/int64-lowering.cc

@@ -303,6 +303,24 @@ void Int64Lowering::LowerNode(Node* node) {
       break;
     }
     // kExprI64Mul:
+    case IrOpcode::kInt64Mul: {
+      DCHECK(node->InputCount() == 2);
+
+      Node* right = node->InputAt(1);
+      node->ReplaceInput(1, GetReplacementLow(right));
+      node->AppendInput(zone(), GetReplacementHigh(right));
+
+      Node* left = node->InputAt(0);
+      node->ReplaceInput(0, GetReplacementLow(left));
+      node->InsertInput(zone(), 1, GetReplacementHigh(left));
+
+      NodeProperties::ChangeOp(node, machine()->Int32PairMul());
+      // We access the additional return values through projections.
+      Node* low_node = graph()->NewNode(common()->Projection(0), node);
+      Node* high_node = graph()->NewNode(common()->Projection(1), node);
+      ReplaceNode(node, low_node, high_node);
+      break;
+    }
     // kExprI64DivS:
     // kExprI64DivU:
     // kExprI64RemS:

src/compiler/int64-lowering.h

@@ -37,10 +37,13 @@ class Int64Lowering {
   CommonOperatorBuilder* common() const { return common_; }
   Signature<MachineRepresentation>* signature() const { return signature_; }
 
+  void PrepareReplacements(Node* node);
+  void PushNode(Node* node);
   void LowerNode(Node* node);
   bool DefaultLowering(Node* node);
   void LowerComparison(Node* node, const Operator* signed_op,
                        const Operator* unsigned_op);
+  void PrepareProjectionReplacements(Node* node);
 
   void ReplaceNode(Node* old, Node* new_low, Node* new_high);
   bool HasReplacementLow(Node* node);

src/compiler/machine-operator.cc

@@ -198,6 +198,7 @@ MachineRepresentation StackSlotRepresentationOf(Operator const* op) {
   V(LoadParentFramePointer, Operator::kNoProperties, 0, 0, 1)                 \
   V(Int32PairAdd, Operator::kNoProperties, 4, 0, 2)                           \
   V(Int32PairSub, Operator::kNoProperties, 4, 0, 2)                           \
+  V(Int32PairMul, Operator::kNoProperties, 4, 0, 2)                           \
   V(Word32PairShl, Operator::kNoProperties, 3, 0, 2)                          \
   V(Word32PairShr, Operator::kNoProperties, 3, 0, 2)                          \
   V(Word32PairSar, Operator::kNoProperties, 3, 0, 2)

src/compiler/machine-operator.h

@@ -183,6 +183,7 @@ class MachineOperatorBuilder final : public ZoneObject {
 
   const Operator* Int32PairAdd();
   const Operator* Int32PairSub();
+  const Operator* Int32PairMul();
   const Operator* Word32PairShl();
   const Operator* Word32PairShr();
   const Operator* Word32PairSar();

src/compiler/mips/instruction-selector-mips.cc

@@ -399,6 +399,8 @@ void InstructionSelector::VisitInt32PairAdd(Node* node) { UNIMPLEMENTED(); }
 
 void InstructionSelector::VisitInt32PairSub(Node* node) { UNIMPLEMENTED(); }
 
+void InstructionSelector::VisitInt32PairMul(Node* node) { UNIMPLEMENTED(); }
+
 void InstructionSelector::VisitWord32PairShl(Node* node) { UNIMPLEMENTED(); }
 
 void InstructionSelector::VisitWord32PairShr(Node* node) { UNIMPLEMENTED(); }

src/compiler/opcodes.h

@@ -338,6 +338,7 @@
   V(CheckedStore)               \
   V(Int32PairAdd)               \
   V(Int32PairSub)               \
+  V(Int32PairMul)               \
   V(Word32PairShl)              \
   V(Word32PairShr)              \
   V(Word32PairSar)

src/compiler/ppc/instruction-selector-ppc.cc

@@ -810,6 +810,8 @@ void InstructionSelector::VisitInt32PairSub(Node* node) {
   VisitPairBinop(this, kPPC_SubPair, node);
 }
 
+void InstructionSelector::VisitInt32PairMul(Node* node) { UNIMPLEMENTED(); }
+
 void VisitPairShift(InstructionSelector* selector, InstructionCode opcode,
                     Node* node) {
   PPCOperandGenerator g(selector);

src/compiler/raw-machine-assembler.h

@@ -330,6 +330,9 @@ class RawMachineAssembler {
   Node* Int32PairSub(Node* a_low, Node* a_high, Node* b_low, Node* b_high) {
     return AddNode(machine()->Int32PairSub(), a_low, a_high, b_low, b_high);
   }
+  Node* Int32PairMul(Node* a_low, Node* a_high, Node* b_low, Node* b_high) {
+    return AddNode(machine()->Int32PairMul(), a_low, a_high, b_low, b_high);
+  }
   Node* Word32PairShl(Node* low_word, Node* high_word, Node* shift) {
     return AddNode(machine()->Word32PairShl(), low_word, high_word, shift);
   }

src/compiler/typer.cc

@@ -2456,6 +2456,8 @@ Type* Typer::Visitor::TypeInt32PairAdd(Node* node) { return Type::Internal(); }
 
 Type* Typer::Visitor::TypeInt32PairSub(Node* node) { return Type::Internal(); }
 
+Type* Typer::Visitor::TypeInt32PairMul(Node* node) { return Type::Internal(); }
+
 Type* Typer::Visitor::TypeWord32PairShl(Node* node) { return Type::Internal(); }
 
 Type* Typer::Visitor::TypeWord32PairShr(Node* node) { return Type::Internal(); }

src/compiler/verifier.cc

@@ -958,6 +958,7 @@ void Verifier::Visitor::Check(Node* node) {
     case IrOpcode::kFloat64InsertHighWord32:
     case IrOpcode::kInt32PairAdd:
     case IrOpcode::kInt32PairSub:
+    case IrOpcode::kInt32PairMul:
     case IrOpcode::kWord32PairShl:
     case IrOpcode::kWord32PairShr:
     case IrOpcode::kWord32PairSar:

src/compiler/wasm-compiler.cc

@@ -500,6 +500,9 @@ Node* WasmGraphBuilder::Binop(wasm::WasmOpcode opcode, Node* left,
       op = m->Int64Sub();
       break;
     // kExprI64Mul:
+    case wasm::kExprI64Mul:
+      op = m->Int64Mul();
+      break;
     // kExprI64DivS:
     case wasm::kExprI64DivS:
       return BuildI64DivS(left, right);
@@ -571,9 +574,6 @@ Node* WasmGraphBuilder::Binop(wasm::WasmOpcode opcode, Node* left,
 #if WASM_64
     // Opcodes only supported on 64-bit platforms.
     // TODO(titzer): query the machine operator builder here instead of #ifdef.
-    case wasm::kExprI64Mul:
-      op = m->Int64Mul();
-      break;
     case wasm::kExprI64Ror:
       op = m->Word64Ror();
       break;

src/compiler/x87/instruction-selector-x87.cc

@@ -581,6 +581,8 @@ void InstructionSelector::VisitInt32PairSub(Node* node) {
   Emit(kX87SubPair, 2, outputs, 4, inputs, 1, temps);
 }
 
+void InstructionSelector::VisitInt32PairMul(Node* node) { UNIMPLEMENTED(); }
+
 void InstructionSelector::VisitWord32PairShl(Node* node) {
   X87OperandGenerator g(this);
 

test/cctest/compiler/test-run-machops.cc

@@ -4328,6 +4328,70 @@ TEST(RunInt32PairSubWithSharedInput) {
   TestInt32PairSubWithSharedInput(1, 1, 0, 0);
 }
 
+TEST(RunInt32PairMul) {
+  BufferedRawMachineAssemblerTester<int32_t> m(
+      MachineType::Uint32(), MachineType::Uint32(), MachineType::Uint32(),
+      MachineType::Uint32());
+
+  uint32_t high;
+  uint32_t low;
+
+  Node* PairMul = m.Int32PairMul(m.Parameter(0), m.Parameter(1), m.Parameter(2),
+                                 m.Parameter(3));
+
+  m.StoreToPointer(&low, MachineRepresentation::kWord32,
+                   m.Projection(0, PairMul));
+  m.StoreToPointer(&high, MachineRepresentation::kWord32,
+                   m.Projection(1, PairMul));
+  m.Return(m.Int32Constant(74));
+
+  FOR_UINT64_INPUTS(i) {
+    FOR_UINT64_INPUTS(j) {
+      m.Call(static_cast<uint32_t>(*i & 0xffffffff),
+             static_cast<uint32_t>(*i >> 32),
+             static_cast<uint32_t>(*j & 0xffffffff),
+             static_cast<uint32_t>(*j >> 32));
+      CHECK_EQ(*i * *j, ToInt64(low, high));
+    }
+  }
+}
+
+void TestInt32PairMulWithSharedInput(int a, int b, int c, int d) {
+  BufferedRawMachineAssemblerTester<int32_t> m(MachineType::Uint32(),
+                                               MachineType::Uint32());
+
+  uint32_t high;
+  uint32_t low;
+
+  Node* PairMul = m.Int32PairMul(m.Parameter(a), m.Parameter(b), m.Parameter(c),
+                                 m.Parameter(d));
+
+  m.StoreToPointer(&low, MachineRepresentation::kWord32,
+                   m.Projection(0, PairMul));
+  m.StoreToPointer(&high, MachineRepresentation::kWord32,
+                   m.Projection(1, PairMul));
+  m.Return(m.Int32Constant(74));
+
+  FOR_UINT32_INPUTS(i) {
+    FOR_UINT32_INPUTS(j) {
+      m.Call(*i, *j);
+      uint32_t inputs[] = {*i, *j};
+      CHECK_EQ(ToInt64(inputs[a], inputs[b]) * ToInt64(inputs[c], inputs[d]),
+               ToInt64(low, high));
+    }
+  }
+}
+
+TEST(RunInt32PairMulWithSharedInput) {
+  TestInt32PairMulWithSharedInput(0, 0, 0, 0);
+  TestInt32PairMulWithSharedInput(1, 0, 0, 0);
+  TestInt32PairMulWithSharedInput(0, 1, 0, 0);
+  TestInt32PairMulWithSharedInput(0, 0, 1, 0);
+  TestInt32PairMulWithSharedInput(0, 0, 0, 1);
+  TestInt32PairMulWithSharedInput(1, 1, 0, 0);
+  TestInt32PairMulWithSharedInput(0, 1, 1, 0);
+}
+
 TEST(RunWord32PairShl) {
   BufferedRawMachineAssemblerTester<int32_t> m(
       MachineType::Uint32(), MachineType::Uint32(), MachineType::Uint32());

test/cctest/wasm/test-run-wasm-64.cc

@@ -48,7 +48,7 @@
   V(I64LoadStore, true)         \
   V(I64Add, !MIPS_OR_X87)       \
   V(I64Sub, !MIPS_OR_X87)       \
-  V(I64Mul, false)              \
+  V(I64Mul, !MIPS_OR_X87)       \
   V(I64DivS, true)              \
   V(I64DivU, true)              \
   V(I64RemS, true)              \
@@ -873,6 +873,8 @@ TEST(Run_Wasm_I64Binops) {
   TEST_I64_BINOP(I64Add, 8888888888888LL, 3333333333333LL, 5555555555555LL);
   TEST_I64_BINOP(I64Sub, -111111111111LL, 777777777777LL, 888888888888LL);
   TEST_I64_BINOP(I64Mul, 65130756, 88734, 734);
+  TEST_I64_BINOP(I64Mul, 65130756, -88734, -734);
+  TEST_I64_BINOP(I64Mul, -65130756, -88734, 734);
   TEST_I64_BINOP(I64DivS, -66, -4777344, 72384);
   TEST_I64_BINOP(I64DivU, 805306368, 0xF0000000, 5);
   TEST_I64_BINOP(I64RemS, -3, -3003, 1000);

test/unittests/compiler/int64-lowering-unittest.cc

@@ -337,6 +337,23 @@ TEST_F(Int64LoweringTest, Int64Sub) {
 }
 
 // kExprI64Mul:
+TEST_F(Int64LoweringTest, Int64Mul) {
+  LowerGraph(graph()->NewNode(machine()->Int64Mul(), Int64Constant(value(0)),
+                              Int64Constant(value(1))),
+             MachineRepresentation::kWord64);
+
+  Capture<Node*> mul_capture;
+  Matcher<Node*> mul_matcher = IsInt32PairMul(
+      IsInt32Constant(low_word_value(0)), IsInt32Constant(high_word_value(0)),
+      IsInt32Constant(low_word_value(1)), IsInt32Constant(high_word_value(1)));
+
+  EXPECT_THAT(
+      graph()->end()->InputAt(1),
+      IsReturn2(IsProjection(0, AllOf(CaptureEq(&mul_capture), mul_matcher)),
+                IsProjection(1, AllOf(CaptureEq(&mul_capture), mul_matcher)),
+                start(), start()));
+}
+
 // kExprI64DivS:
 // kExprI64DivU:
 // kExprI64RemS:

test/unittests/compiler/node-test-utils.cc

@@ -2214,6 +2214,7 @@ Matcher<Node*> IsLoadFramePointer() {
 
 IS_QUADOP_MATCHER(Int32PairAdd)
 IS_QUADOP_MATCHER(Int32PairSub)
+IS_QUADOP_MATCHER(Int32PairMul)
 
 #define IS_TERNOP_MATCHER(Name)                                            \
   Matcher<Node*> Is##Name(const Matcher<Node*>& lhs_matcher,               \

test/unittests/compiler/node-test-utils.h

@@ -369,6 +369,10 @@ Matcher<Node*> IsInt32PairSub(const Matcher<Node*>& a_matcher,
                               const Matcher<Node*>& b_matcher,
                               const Matcher<Node*>& c_matcher,
                               const Matcher<Node*>& d_matcher);
+Matcher<Node*> IsInt32PairMul(const Matcher<Node*>& a_matcher,
+                              const Matcher<Node*>& b_matcher,
+                              const Matcher<Node*>& c_matcher,
+                              const Matcher<Node*>& d_matcher);
 
 Matcher<Node*> IsWord32PairShl(const Matcher<Node*>& lhs_matcher,
                                const Matcher<Node*>& mid_matcher,