CTZ instruction implemented as optional operator.

Review URL: https://codereview.chromium.org/1405453003

Cr-Commit-Position: refs/heads/master@{#31313}

src/compiler/arm/instruction-selector-arm.cc

@@ -697,6 +697,9 @@ void InstructionSelector::VisitWord32Clz(Node* node) {
 }
 
 
+void InstructionSelector::VisitWord32Ctz(Node* node) { UNREACHABLE(); }
+
+
 void InstructionSelector::VisitInt32Add(Node* node) {
   ArmOperandGenerator g(this);
   Int32BinopMatcher m(node);

src/compiler/arm64/instruction-selector-arm64.cc

@@ -927,6 +927,9 @@ void InstructionSelector::VisitWord32Clz(Node* node) {
 }
 
 
+void InstructionSelector::VisitWord32Ctz(Node* node) { UNREACHABLE(); }
+
+
 void InstructionSelector::VisitInt32Add(Node* node) {
   Arm64OperandGenerator g(this);
   Int32BinopMatcher m(node);

src/compiler/ia32/code-generator-ia32.cc

@@ -509,6 +509,9 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
     case kIA32Lzcnt:
       __ Lzcnt(i.OutputRegister(), i.InputOperand(0));
       break;
+    case kIA32Tzcnt:
+      __ Tzcnt(i.OutputRegister(), i.InputOperand(0));
+      break;
     case kSSEFloat32Cmp:
       __ ucomiss(i.InputDoubleRegister(0), i.InputOperand(1));
       break;

src/compiler/ia32/instruction-codes-ia32.h

@@ -31,6 +31,7 @@ namespace compiler {
   V(IA32Sar)                       \
   V(IA32Ror)                       \
   V(IA32Lzcnt)                     \
+  V(IA32Tzcnt)                     \
   V(SSEFloat32Cmp)                 \
   V(SSEFloat32Add)                 \
   V(SSEFloat32Sub)                 \

src/compiler/ia32/instruction-selector-ia32.cc

@@ -553,6 +553,12 @@ void InstructionSelector::VisitWord32Clz(Node* node) {
 }
 
 
+void InstructionSelector::VisitWord32Ctz(Node* node) {
+  IA32OperandGenerator g(this);
+  Emit(kIA32Tzcnt, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
+}
+
+
 void InstructionSelector::VisitInt32Add(Node* node) {
   IA32OperandGenerator g(this);
 
@@ -1357,7 +1363,8 @@ InstructionSelector::SupportedMachineOperatorFlags() {
       MachineOperatorBuilder::kFloat32Min |
       MachineOperatorBuilder::kFloat64Max |
       MachineOperatorBuilder::kFloat64Min |
-      MachineOperatorBuilder::kWord32ShiftIsSafe;
+      MachineOperatorBuilder::kWord32ShiftIsSafe |
+      MachineOperatorBuilder::kWord32Ctz;
   if (CpuFeatures::IsSupported(SSE4_1)) {
     flags |= MachineOperatorBuilder::kFloat64RoundDown |
              MachineOperatorBuilder::kFloat64RoundTruncate;

src/compiler/instruction-selector.cc

@@ -612,6 +612,8 @@ void InstructionSelector::VisitNode(Node* node) {
       return VisitWord32Equal(node);
     case IrOpcode::kWord32Clz:
       return MarkAsWord32(node), VisitWord32Clz(node);
+    case IrOpcode::kWord32Ctz:
+      return MarkAsWord32(node), VisitWord32Ctz(node);
     case IrOpcode::kWord64And:
       return MarkAsWord64(node), VisitWord64And(node);
     case IrOpcode::kWord64Or:

src/compiler/machine-operator.cc

@@ -167,6 +167,7 @@ CheckedStoreRepresentation CheckedStoreRepresentationOf(Operator const* op) {
   V(LoadFramePointer, Operator::kNoProperties, 0, 0, 1)
 
 #define PURE_OPTIONAL_OP_LIST(V)                            \
+  V(Word32Ctz, Operator::kNoProperties, 1, 0, 1)            \
   V(Float32Max, Operator::kNoProperties, 2, 0, 1)           \
   V(Float32Min, Operator::kNoProperties, 2, 0, 1)           \
   V(Float64Max, Operator::kNoProperties, 2, 0, 1)           \

src/compiler/machine-operator.h

@@ -117,9 +117,10 @@ class MachineOperatorBuilder final : public ZoneObject {
     kInt32DivIsSafe = 1u << 7,
     kUint32DivIsSafe = 1u << 8,
     kWord32ShiftIsSafe = 1u << 9,
+    kWord32Ctz = 1u << 10,
     kAllOptionalOps = kFloat32Max | kFloat32Min | kFloat64Max | kFloat64Min |
                       kFloat64RoundDown | kFloat64RoundTruncate |
-                      kFloat64RoundTiesAway
+                      kFloat64RoundTiesAway | kWord32Ctz
   };
   typedef base::Flags<Flag, unsigned> Flags;
 
@@ -135,6 +136,7 @@ class MachineOperatorBuilder final : public ZoneObject {
   const Operator* Word32Ror();
   const Operator* Word32Equal();
   const Operator* Word32Clz();
+  const OptionalOperator Word32Ctz();
   bool Word32ShiftIsSafe() const { return flags_ & kWord32ShiftIsSafe; }
 
   const Operator* Word64And();

src/compiler/mips/instruction-selector-mips.cc

@@ -269,6 +269,9 @@ void InstructionSelector::VisitWord32Clz(Node* node) {
 }
 
 
+void InstructionSelector::VisitWord32Ctz(Node* node) { UNREACHABLE(); }
+
+
 void InstructionSelector::VisitInt32Add(Node* node) {
   MipsOperandGenerator g(this);
 

src/compiler/mips64/instruction-selector-mips64.cc

@@ -321,6 +321,9 @@ void InstructionSelector::VisitWord32Clz(Node* node) {
 }
 
 
+void InstructionSelector::VisitWord32Ctz(Node* node) { UNREACHABLE(); }
+
+
 void InstructionSelector::VisitWord64Ror(Node* node) {
   VisitRRO(this, kMips64Dror, node);
 }

src/compiler/opcodes.h

@@ -226,6 +226,7 @@
   V(Word32Sar)                  \
   V(Word32Ror)                  \
   V(Word32Clz)                  \
+  V(Word32Ctz)                  \
   V(Word64And)                  \
   V(Word64Or)                   \
   V(Word64Xor)                  \

src/compiler/ppc/instruction-selector-ppc.cc

@@ -737,6 +737,9 @@ void InstructionSelector::VisitWord32Clz(Node* node) {
 }
 
 
+void InstructionSelector::VisitWord32Ctz(Node* node) { UNREACHABLE(); }
+
+
 void InstructionSelector::VisitInt32Add(Node* node) {
   VisitBinop<Int32BinopMatcher>(this, node, kPPC_Add, kInt16Imm);
 }

src/compiler/typer.cc

@@ -1698,6 +1698,9 @@ Type* Typer::Visitor::TypeWord32Equal(Node* node) { return Type::Boolean(); }
 Type* Typer::Visitor::TypeWord32Clz(Node* node) { return Type::Integral32(); }
 
 
+Type* Typer::Visitor::TypeWord32Ctz(Node* node) { return Type::Integral32(); }
+
+
 Type* Typer::Visitor::TypeWord64And(Node* node) { return Type::Internal(); }
 
 

src/compiler/verifier.cc

@@ -803,6 +803,7 @@ void Verifier::Visitor::Check(Node* node) {
     case IrOpcode::kWord32Ror:
     case IrOpcode::kWord32Equal:
     case IrOpcode::kWord32Clz:
+    case IrOpcode::kWord32Ctz:
     case IrOpcode::kWord64And:
     case IrOpcode::kWord64Or:
     case IrOpcode::kWord64Xor:

src/compiler/x64/code-generator-x64.cc

@@ -770,6 +770,13 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
         __ Lzcntl(i.OutputRegister(), i.InputOperand(0));
       }
       break;
+    case kX64Tzcnt32:
+      if (instr->InputAt(0)->IsRegister()) {
+        __ Tzcntl(i.OutputRegister(), i.InputRegister(0));
+      } else {
+        __ Tzcntl(i.OutputRegister(), i.InputOperand(0));
+      }
+      break;
     case kSSEFloat32Cmp:
       ASSEMBLE_SSE_BINOP(ucomiss);
       break;

src/compiler/x64/instruction-codes-x64.h

@@ -47,6 +47,7 @@ namespace compiler {
   V(X64Ror)                        \
   V(X64Ror32)                      \
   V(X64Lzcnt32)                    \
+  V(X64Tzcnt32)                    \
   V(SSEFloat32Cmp)                 \
   V(SSEFloat32Add)                 \
   V(SSEFloat32Sub)                 \

src/compiler/x64/instruction-selector-x64.cc

@@ -578,6 +578,12 @@ void InstructionSelector::VisitWord32Clz(Node* node) {
 }
 
 
+void InstructionSelector::VisitWord32Ctz(Node* node) {
+  X64OperandGenerator g(this);
+  Emit(kX64Tzcnt32, g.DefineAsRegister(node), g.Use(node->InputAt(0)));
+}
+
+
 void InstructionSelector::VisitInt32Add(Node* node) {
   X64OperandGenerator g(this);
 
@@ -1704,7 +1710,8 @@ InstructionSelector::SupportedMachineOperatorFlags() {
       MachineOperatorBuilder::kFloat32Min |
       MachineOperatorBuilder::kFloat64Max |
       MachineOperatorBuilder::kFloat64Min |
-      MachineOperatorBuilder::kWord32ShiftIsSafe;
+      MachineOperatorBuilder::kWord32ShiftIsSafe |
+      MachineOperatorBuilder::kWord32Ctz;
   if (CpuFeatures::IsSupported(SSE4_1)) {
     flags |= MachineOperatorBuilder::kFloat64RoundDown |
              MachineOperatorBuilder::kFloat64RoundTruncate;

src/compiler/x87/instruction-selector-x87.cc

@@ -514,6 +514,9 @@ void InstructionSelector::VisitWord32Clz(Node* node) {
 }
 
 
+void InstructionSelector::VisitWord32Ctz(Node* node) { UNREACHABLE(); }
+
+
 void InstructionSelector::VisitInt32Add(Node* node) {
   X87OperandGenerator g(this);
 

src/ia32/assembler-ia32.cc

@@ -1307,6 +1307,14 @@ void Assembler::bsr(Register dst, const Operand& src) {
 }
 
 
+void Assembler::bsf(Register dst, const Operand& src) {
+  EnsureSpace ensure_space(this);
+  EMIT(0x0F);
+  EMIT(0xBC);
+  emit_operand(dst, src);
+}
+
+
 void Assembler::hlt() {
   EnsureSpace ensure_space(this);
   EMIT(0xF4);

src/ia32/assembler-ia32.h

@@ -779,6 +779,8 @@ class Assembler : public AssemblerBase {
   void bts(const Operand& dst, Register src);
   void bsr(Register dst, Register src) { bsr(dst, Operand(src)); }
   void bsr(Register dst, const Operand& src);
+  void bsf(Register dst, Register src) { bsf(dst, Operand(src)); }
+  void bsf(Register dst, const Operand& src);
 
   // Miscellaneous
   void hlt();

src/ia32/macro-assembler-ia32.cc

@@ -2430,6 +2430,20 @@ void MacroAssembler::Lzcnt(Register dst, const Operand& src) {
 }
 
 
+void MacroAssembler::Tzcnt(Register dst, const Operand& src) {
+  if (CpuFeatures::IsSupported(BMI1)) {
+    CpuFeatureScope scope(this, BMI1);
+    tzcnt(dst, src);
+    return;
+  }
+  Label not_zero_src;
+  bsf(dst, src);
+  j(not_zero, &not_zero_src, Label::kNear);
+  Move(dst, Immediate(32));  // The result of tzcnt is 32 if src = 0.
+  bind(&not_zero_src);
+}
+
+
 void MacroAssembler::SetCounter(StatsCounter* counter, int value) {
   if (FLAG_native_code_counters && counter->Enabled()) {
     mov(Operand::StaticVariable(ExternalReference(counter)), Immediate(value));

src/ia32/macro-assembler-ia32.h

@@ -853,6 +853,9 @@ class MacroAssembler: public Assembler {
   void Lzcnt(Register dst, Register src) { Lzcnt(dst, Operand(src)); }
   void Lzcnt(Register dst, const Operand& src);
 
+  void Tzcnt(Register dst, Register src) { Tzcnt(dst, Operand(src)); }
+  void Tzcnt(Register dst, const Operand& src);
+
   // Emit call to the code we are currently generating.
   void CallSelf() {
     Handle<Code> self(reinterpret_cast<Code**>(CodeObject().location()));

src/x64/assembler-x64.cc

@@ -745,6 +745,24 @@ void Assembler::bsrl(Register dst, const Operand& src) {
 }
 
 
+void Assembler::bsfl(Register dst, Register src) {
+  EnsureSpace ensure_space(this);
+  emit_optional_rex_32(dst, src);
+  emit(0x0F);
+  emit(0xBC);
+  emit_modrm(dst, src);
+}
+
+
+void Assembler::bsfl(Register dst, const Operand& src) {
+  EnsureSpace ensure_space(this);
+  emit_optional_rex_32(dst, src);
+  emit(0x0F);
+  emit(0xBC);
+  emit_operand(dst, src);
+}
+
+
 void Assembler::call(Label* L) {
   positions_recorder()->WriteRecordedPositions();
   EnsureSpace ensure_space(this);

src/x64/assembler-x64.h

@@ -844,6 +844,8 @@ class Assembler : public AssemblerBase {
   void bts(const Operand& dst, Register src);
   void bsrl(Register dst, Register src);
   void bsrl(Register dst, const Operand& src);
+  void bsfl(Register dst, Register src);
+  void bsfl(Register dst, const Operand& src);
 
   // Miscellaneous
   void clc();

src/x64/macro-assembler-x64.cc

@@ -2866,6 +2866,34 @@ void MacroAssembler::Lzcntl(Register dst, const Operand& src) {
 }
 
 
+void MacroAssembler::Tzcntl(Register dst, Register src) {
+  if (CpuFeatures::IsSupported(BMI1)) {
+    CpuFeatureScope scope(this, BMI1);
+    tzcntl(dst, src);
+    return;
+  }
+  Label not_zero_src;
+  bsfl(dst, src);
+  j(not_zero, &not_zero_src, Label::kNear);
+  Set(dst, 32);  // The result of tzcnt is 32 if src = 0.
+  bind(&not_zero_src);
+}
+
+
+void MacroAssembler::Tzcntl(Register dst, const Operand& src) {
+  if (CpuFeatures::IsSupported(BMI1)) {
+    CpuFeatureScope scope(this, BMI1);
+    tzcntl(dst, src);
+    return;
+  }
+  Label not_zero_src;
+  bsfl(dst, src);
+  j(not_zero, &not_zero_src, Label::kNear);
+  Set(dst, 32);  // The result of tzcnt is 32 if src = 0.
+  bind(&not_zero_src);
+}
+
+
 void MacroAssembler::Pushad() {
   Push(rax);
   Push(rcx);

src/x64/macro-assembler-x64.h

@@ -944,6 +944,9 @@ class MacroAssembler: public Assembler {
   void Lzcntl(Register dst, Register src);
   void Lzcntl(Register dst, const Operand& src);
 
+  void Tzcntl(Register dst, Register src);
+  void Tzcntl(Register dst, const Operand& src);
+
   // Non-x64 instructions.
   // Push/pop all general purpose registers.
   // Does not push rsp/rbp nor any of the assembler's special purpose registers

test/cctest/compiler/test-run-machops.cc

@@ -27,6 +27,54 @@ TEST(RunInt32Add) {
 }
 
 
+void TestWord32Ctz(int32_t value, int32_t expected) {
+  RawMachineAssemblerTester<int32_t> m;
+  if (m.machine()->Word32Ctz().IsSupported()) {
+    Node* ctz =
+        m.AddNode(m.machine()->Word32Ctz().op(), m.Int32Constant(value));
+    m.Return(ctz);
+    CHECK_EQ(expected, m.Call());
+  }
+}
+
+
+TEST(RunInt32Ctz) {
+  TestWord32Ctz(0x00000000, 32);
+  TestWord32Ctz(0x80000000, 31);
+  TestWord32Ctz(0x40000000, 30);
+  TestWord32Ctz(0x20000000, 29);
+  TestWord32Ctz(0x10000000, 28);
+  TestWord32Ctz(0xa8000000, 27);
+  TestWord32Ctz(0xf4000000, 26);
+  TestWord32Ctz(0x62000000, 25);
+  TestWord32Ctz(0x91000000, 24);
+  TestWord32Ctz(0xcd800000, 23);
+  TestWord32Ctz(0x09400000, 22);
+  TestWord32Ctz(0xaf200000, 21);
+  TestWord32Ctz(0xac100000, 20);
+  TestWord32Ctz(0xe0b80000, 19);
+  TestWord32Ctz(0x9ce40000, 18);
+  TestWord32Ctz(0xc7920000, 17);
+  TestWord32Ctz(0xb8f10000, 16);
+  TestWord32Ctz(0x3b9f8000, 15);
+  TestWord32Ctz(0xdb4c4000, 14);
+  TestWord32Ctz(0xe9a32000, 13);
+  TestWord32Ctz(0xfca61000, 12);
+  TestWord32Ctz(0x6c8a7800, 11);
+  TestWord32Ctz(0x8ce5a400, 10);
+  TestWord32Ctz(0xcb7d0200, 9);
+  TestWord32Ctz(0xcb4dc100, 8);
+  TestWord32Ctz(0xdfbec580, 7);
+  TestWord32Ctz(0x27a9db40, 6);
+  TestWord32Ctz(0xde3bcb20, 5);
+  TestWord32Ctz(0xd7e8a610, 4);
+  TestWord32Ctz(0x9afdbc88, 3);
+  TestWord32Ctz(0x9afdbc84, 2);
+  TestWord32Ctz(0x9afdbc82, 1);
+  TestWord32Ctz(0x9afdbc81, 0);
+}
+
+
 static Node* Int32Input(RawMachineAssemblerTester<int32_t>* m, int index) {
   switch (index) {
     case 0: