[turbofan] Select tbz/tbnz when possible on ARM64.

R=bmeurer@chromium.org

Review URL: https://codereview.chromium.org/697653002

Cr-Commit-Position: refs/heads/master@{#25063}
git-svn-id: https://v8.googlecode.com/svn/branches/bleeding_edge@25063 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/compiler/arm64/code-generator-arm64.cc

@@ -170,7 +170,16 @@ class Arm64OperandConverter FINAL : public InstructionOperandConverter {
       int64_t imm = i.InputOperand##width(1).immediate().value();              \
       __ asm_instr(i.OutputRegister##width(), i.InputRegister##width(0), imm); \
     }                                                                          \
-  } while (0);
+  } while (0)
+
+
+#define ASSEMBLE_TEST_AND_BRANCH(asm_instr, width)             \
+  do {                                                         \
+    bool fallthrough = IsNextInAssemblyOrder(i.InputRpo(3));   \
+    __ asm_instr(i.InputRegister##width(0), i.InputInt6(1),    \
+                 code_->GetLabel(i.InputRpo(2)));              \
+    if (!fallthrough) __ B(code_->GetLabel(i.InputRpo(3)));    \
+  } while (0)
 
 
 // Assembles an instruction after register allocation, producing machine code.
@@ -421,6 +430,18 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
       __ Ubfx(i.OutputRegister32(), i.InputRegister32(0), i.InputInt8(1),
               i.InputInt8(2));
       break;
+    case kArm64Tbz:
+      ASSEMBLE_TEST_AND_BRANCH(Tbz, 64);
+      break;
+    case kArm64Tbz32:
+      ASSEMBLE_TEST_AND_BRANCH(Tbz, 32);
+      break;
+    case kArm64Tbnz:
+      ASSEMBLE_TEST_AND_BRANCH(Tbnz, 64);
+      break;
+    case kArm64Tbnz32:
+      ASSEMBLE_TEST_AND_BRANCH(Tbnz, 32);
+      break;
     case kArm64Claim: {
       int words = MiscField::decode(instr->opcode());
       __ Claim(words);

src/compiler/arm64/instruction-codes-arm64.h

@@ -68,6 +68,10 @@ namespace compiler {
   V(Arm64Sxtw)                     \
   V(Arm64Ubfx)                     \
   V(Arm64Ubfx32)                   \
+  V(Arm64Tbz)                      \
+  V(Arm64Tbz32)                    \
+  V(Arm64Tbnz)                     \
+  V(Arm64Tbnz32)                   \
   V(Arm64Claim)                    \
   V(Arm64Poke)                     \
   V(Arm64PokePairZero)             \

src/compiler/arm64/instruction-selector-arm64.cc

@@ -1204,12 +1204,44 @@ void InstructionSelector::VisitBranch(Node* branch, BasicBlock* tbranch,
       case IrOpcode::kInt32Sub:
         return VisitWordCompare(this, value, kArm64Cmp32, &cont, false,
                                 kArithmeticImm);
-      case IrOpcode::kWord32And:
+      case IrOpcode::kWord32And: {
+        Int32BinopMatcher m(value);
+        if (m.right().HasValue() &&
+            (base::bits::CountPopulation32(m.right().Value()) == 1)) {
+          // If the mask has only one bit set, we can use tbz/tbnz.
+          DCHECK((cont.condition() == kEqual) ||
+                 (cont.condition() == kNotEqual));
+          ArchOpcode opcode =
+              (cont.condition() == kEqual) ? kArm64Tbz32 : kArm64Tbnz32;
+          Emit(opcode, NULL, g.UseRegister(m.left().node()),
+               g.TempImmediate(
+                   base::bits::CountTrailingZeros32(m.right().Value())),
+               g.Label(cont.true_block()),
+               g.Label(cont.false_block()))->MarkAsControl();
+          return;
+        }
         return VisitWordCompare(this, value, kArm64Tst32, &cont, true,
                                 kLogical32Imm);
-      case IrOpcode::kWord64And:
+      }
+      case IrOpcode::kWord64And: {
+        Int64BinopMatcher m(value);
+        if (m.right().HasValue() &&
+            (base::bits::CountPopulation64(m.right().Value()) == 1)) {
+          // If the mask has only one bit set, we can use tbz/tbnz.
+          DCHECK((cont.condition() == kEqual) ||
+                 (cont.condition() == kNotEqual));
+          ArchOpcode opcode =
+              (cont.condition() == kEqual) ? kArm64Tbz : kArm64Tbnz;
+          Emit(opcode, NULL, g.UseRegister(m.left().node()),
+               g.TempImmediate(
+                   base::bits::CountTrailingZeros64(m.right().Value())),
+               g.Label(cont.true_block()),
+               g.Label(cont.false_block()))->MarkAsControl();
+          return;
+        }
         return VisitWordCompare(this, value, kArm64Tst, &cont, true,
                                 kLogical64Imm);
+      }
       default:
         break;
     }

test/unittests/compiler/arm64/instruction-selector-arm64-unittest.cc

@@ -649,6 +649,9 @@ INSTANTIATE_TEST_CASE_P(InstructionSelectorTest,
 
 TEST_F(InstructionSelectorTest, Word32AndBranchWithImmediateOnRight) {
   TRACED_FOREACH(int32_t, imm, kLogical32Immediates) {
+    // Skip the cases where the instruction selector would use tbz/tbnz.
+    if (base::bits::CountPopulation32(imm) == 1) continue;
+
     StreamBuilder m(this, kMachInt32, kMachInt32);
     MLabel a, b;
     m.Branch(m.Word32And(m.Parameter(0), m.Int32Constant(imm)), &a, &b);
@@ -669,6 +672,9 @@ TEST_F(InstructionSelectorTest, Word32AndBranchWithImmediateOnRight) {
 
 TEST_F(InstructionSelectorTest, Word64AndBranchWithImmediateOnRight) {
   TRACED_FOREACH(int64_t, imm, kLogical64Immediates) {
+    // Skip the cases where the instruction selector would use tbz/tbnz.
+    if (base::bits::CountPopulation64(imm) == 1) continue;
+
     StreamBuilder m(this, kMachInt64, kMachInt64);
     MLabel a, b;
     m.Branch(m.Word64And(m.Parameter(0), m.Int64Constant(imm)), &a, &b);
@@ -725,6 +731,9 @@ TEST_F(InstructionSelectorTest, SubBranchWithImmediateOnRight) {
 
 TEST_F(InstructionSelectorTest, Word32AndBranchWithImmediateOnLeft) {
   TRACED_FOREACH(int32_t, imm, kLogical32Immediates) {
+    // Skip the cases where the instruction selector would use tbz/tbnz.
+    if (base::bits::CountPopulation32(imm) == 1) continue;
+
     StreamBuilder m(this, kMachInt32, kMachInt32);
     MLabel a, b;
     m.Branch(m.Word32And(m.Int32Constant(imm), m.Parameter(0)), &a, &b);
@@ -746,6 +755,9 @@ TEST_F(InstructionSelectorTest, Word32AndBranchWithImmediateOnLeft) {
 
 TEST_F(InstructionSelectorTest, Word64AndBranchWithImmediateOnLeft) {
   TRACED_FOREACH(int64_t, imm, kLogical64Immediates) {
+    // Skip the cases where the instruction selector would use tbz/tbnz.
+    if (base::bits::CountPopulation64(imm) == 1) continue;
+
     StreamBuilder m(this, kMachInt64, kMachInt64);
     MLabel a, b;
     m.Branch(m.Word64And(m.Int64Constant(imm), m.Parameter(0)), &a, &b);
@@ -784,6 +796,162 @@ TEST_F(InstructionSelectorTest, AddBranchWithImmediateOnLeft) {
 }
 
 
+TEST_F(InstructionSelectorTest, Word32AndBranchWithOneBitMaskOnRight) {
+  TRACED_FORRANGE(int, bit, 0, 31) {
+    uint32_t mask = 1 << bit;
+    StreamBuilder m(this, kMachInt32, kMachInt32);
+    MLabel a, b;
+    m.Branch(m.Word32And(m.Parameter(0), m.Int32Constant(mask)), &a, &b);
+    m.Bind(&a);
+    m.Return(m.Int32Constant(1));
+    m.Bind(&b);
+    m.Return(m.Int32Constant(0));
+    Stream s = m.Build();
+    ASSERT_EQ(1U, s.size());
+    EXPECT_EQ(kArm64Tbnz32, s[0]->arch_opcode());
+    EXPECT_EQ(4U, s[0]->InputCount());
+    EXPECT_EQ(InstructionOperand::IMMEDIATE, s[0]->InputAt(1)->kind());
+    EXPECT_EQ(bit, s.ToInt32(s[0]->InputAt(1)));
+  }
+
+  TRACED_FORRANGE(int, bit, 0, 31) {
+    uint32_t mask = 1 << bit;
+    StreamBuilder m(this, kMachInt32, kMachInt32);
+    MLabel a, b;
+    m.Branch(
+        m.Word32BinaryNot(m.Word32And(m.Parameter(0), m.Int32Constant(mask))),
+        &a, &b);
+    m.Bind(&a);
+    m.Return(m.Int32Constant(1));
+    m.Bind(&b);
+    m.Return(m.Int32Constant(0));
+    Stream s = m.Build();
+    ASSERT_EQ(1U, s.size());
+    EXPECT_EQ(kArm64Tbz32, s[0]->arch_opcode());
+    EXPECT_EQ(4U, s[0]->InputCount());
+    EXPECT_EQ(InstructionOperand::IMMEDIATE, s[0]->InputAt(1)->kind());
+    EXPECT_EQ(bit, s.ToInt32(s[0]->InputAt(1)));
+  }
+}
+
+
+TEST_F(InstructionSelectorTest, Word32AndBranchWithOneBitMaskOnLeft) {
+  TRACED_FORRANGE(int, bit, 0, 31) {
+    uint32_t mask = 1 << bit;
+    StreamBuilder m(this, kMachInt32, kMachInt32);
+    MLabel a, b;
+    m.Branch(m.Word32And(m.Int32Constant(mask), m.Parameter(0)), &a, &b);
+    m.Bind(&a);
+    m.Return(m.Int32Constant(1));
+    m.Bind(&b);
+    m.Return(m.Int32Constant(0));
+    Stream s = m.Build();
+    ASSERT_EQ(1U, s.size());
+    EXPECT_EQ(kArm64Tbnz32, s[0]->arch_opcode());
+    EXPECT_EQ(4U, s[0]->InputCount());
+    EXPECT_EQ(InstructionOperand::IMMEDIATE, s[0]->InputAt(1)->kind());
+    EXPECT_EQ(bit, s.ToInt32(s[0]->InputAt(1)));
+  }
+
+  TRACED_FORRANGE(int, bit, 0, 31) {
+    uint32_t mask = 1 << bit;
+    StreamBuilder m(this, kMachInt32, kMachInt32);
+    MLabel a, b;
+    m.Branch(
+        m.Word32BinaryNot(m.Word32And(m.Int32Constant(mask), m.Parameter(0))),
+        &a, &b);
+    m.Bind(&a);
+    m.Return(m.Int32Constant(1));
+    m.Bind(&b);
+    m.Return(m.Int32Constant(0));
+    Stream s = m.Build();
+    ASSERT_EQ(1U, s.size());
+    EXPECT_EQ(kArm64Tbz32, s[0]->arch_opcode());
+    EXPECT_EQ(4U, s[0]->InputCount());
+    EXPECT_EQ(InstructionOperand::IMMEDIATE, s[0]->InputAt(1)->kind());
+    EXPECT_EQ(bit, s.ToInt32(s[0]->InputAt(1)));
+  }
+}
+
+
+TEST_F(InstructionSelectorTest, Word64AndBranchWithOneBitMaskOnRight) {
+  TRACED_FORRANGE(int, bit, 0, 63) {
+    uint64_t mask = 1L << bit;
+    StreamBuilder m(this, kMachInt64, kMachInt64);
+    MLabel a, b;
+    m.Branch(m.Word64And(m.Parameter(0), m.Int64Constant(mask)), &a, &b);
+    m.Bind(&a);
+    m.Return(m.Int32Constant(1));
+    m.Bind(&b);
+    m.Return(m.Int32Constant(0));
+    Stream s = m.Build();
+    ASSERT_EQ(1U, s.size());
+    EXPECT_EQ(kArm64Tbnz, s[0]->arch_opcode());
+    EXPECT_EQ(4U, s[0]->InputCount());
+    EXPECT_EQ(InstructionOperand::IMMEDIATE, s[0]->InputAt(1)->kind());
+    EXPECT_EQ(bit, s.ToInt64(s[0]->InputAt(1)));
+  }
+
+  TRACED_FORRANGE(int, bit, 0, 63) {
+    uint64_t mask = 1L << bit;
+    StreamBuilder m(this, kMachInt64, kMachInt64);
+    MLabel a, b;
+    m.Branch(
+        m.Word64BinaryNot(m.Word64And(m.Parameter(0), m.Int64Constant(mask))),
+        &a, &b);
+    m.Bind(&a);
+    m.Return(m.Int32Constant(1));
+    m.Bind(&b);
+    m.Return(m.Int32Constant(0));
+    Stream s = m.Build();
+    ASSERT_EQ(1U, s.size());
+    EXPECT_EQ(kArm64Tbz, s[0]->arch_opcode());
+    EXPECT_EQ(4U, s[0]->InputCount());
+    EXPECT_EQ(InstructionOperand::IMMEDIATE, s[0]->InputAt(1)->kind());
+    EXPECT_EQ(bit, s.ToInt64(s[0]->InputAt(1)));
+  }
+}
+
+
+TEST_F(InstructionSelectorTest, Word64AndBranchWithOneBitMaskOnLeft) {
+  TRACED_FORRANGE(int, bit, 0, 63) {
+    uint64_t mask = 1L << bit;
+    StreamBuilder m(this, kMachInt64, kMachInt64);
+    MLabel a, b;
+    m.Branch(m.Word64And(m.Int64Constant(mask), m.Parameter(0)), &a, &b);
+    m.Bind(&a);
+    m.Return(m.Int32Constant(1));
+    m.Bind(&b);
+    m.Return(m.Int32Constant(0));
+    Stream s = m.Build();
+    ASSERT_EQ(1U, s.size());
+    EXPECT_EQ(kArm64Tbnz, s[0]->arch_opcode());
+    EXPECT_EQ(4U, s[0]->InputCount());
+    EXPECT_EQ(InstructionOperand::IMMEDIATE, s[0]->InputAt(1)->kind());
+    EXPECT_EQ(bit, s.ToInt64(s[0]->InputAt(1)));
+  }
+
+  TRACED_FORRANGE(int, bit, 0, 63) {
+    uint64_t mask = 1L << bit;
+    StreamBuilder m(this, kMachInt64, kMachInt64);
+    MLabel a, b;
+    m.Branch(
+        m.Word64BinaryNot(m.Word64And(m.Int64Constant(mask), m.Parameter(0))),
+        &a, &b);
+    m.Bind(&a);
+    m.Return(m.Int32Constant(1));
+    m.Bind(&b);
+    m.Return(m.Int32Constant(0));
+    Stream s = m.Build();
+    ASSERT_EQ(1U, s.size());
+    EXPECT_EQ(kArm64Tbz, s[0]->arch_opcode());
+    EXPECT_EQ(4U, s[0]->InputCount());
+    EXPECT_EQ(InstructionOperand::IMMEDIATE, s[0]->InputAt(1)->kind());
+    EXPECT_EQ(bit, s.ToInt64(s[0]->InputAt(1)));
+  }
+}
+
+
 // -----------------------------------------------------------------------------
 // Add and subtract instructions with overflow.