Use TBZ/TBNZ regardless of CanCover() check.

This is useful even if there are other uses of the
arithmetic result, because it moves dependencies further back.

Change-Id: I6136a657b547198cb4ec92f38b89ddf5df334124
Reviewed-on: https://chromium-review.googlesource.com/1179662Reviewed-by: Jaroslav Sevcik <jarin@chromium.org>
Commit-Queue: Bogdan Lazarescu <bogdan.lazarescu@arm.com>
Cr-Commit-Position: refs/heads/master@{#55292}

src/compiler/arm64/instruction-selector-arm64.cc

@@ -2080,23 +2080,42 @@ void VisitWord64Test(InstructionSelector* selector, Node* node,
   VisitWordTest(selector, node, kArm64Tst, cont);
 }
 
-template <typename Matcher, ArchOpcode kOpcode>
-bool TryEmitTestAndBranch(InstructionSelector* selector, Node* node,
-                          FlagsContinuation* cont) {
-  Arm64OperandGenerator g(selector);
-  Matcher m(node);
-  if (cont->IsBranch() && !cont->IsPoisoned() && m.right().HasValue() &&
-      base::bits::IsPowerOfTwo(m.right().Value())) {
-    // If the mask has only one bit set, we can use tbz/tbnz.
-    DCHECK((cont->condition() == kEqual) || (cont->condition() == kNotEqual));
-    selector->EmitWithContinuation(
-        kOpcode, g.UseRegister(m.left().node()),
-        g.TempImmediate(base::bits::CountTrailingZeros(m.right().Value())),
-        cont);
-    return true;
+template <typename Matcher>
+struct TestAndBranchMatcher {
+  TestAndBranchMatcher(Node* node, FlagsContinuation* cont)
+      : matches_(false), cont_(cont), matcher_(node) {
+    Initialize();
+  }
+  bool Matches() const { return matches_; }
+
+  unsigned bit() const {
+    DCHECK(Matches());
+    return base::bits::CountTrailingZeros(matcher_.right().Value());
+  }
+
+  Node* input() const {
+    DCHECK(Matches());
+    return matcher_.left().node();
   }
-  return false;
-}
+
+ private:
+  bool matches_;
+  FlagsContinuation* cont_;
+  Matcher matcher_;
+
+  void Initialize() {
+    if (cont_->IsBranch() && !cont_->IsPoisoned() &&
+        matcher_.right().HasValue() &&
+        base::bits::IsPowerOfTwo(matcher_.right().Value())) {
+      // If the mask has only one bit set, we can use tbz/tbnz.
+      DCHECK((cont_->condition() == kEqual) ||
+             (cont_->condition() == kNotEqual));
+      matches_ = true;
+    } else {
+      matches_ = false;
+    }
+  }
+};
 
 // Shared routine for multiple float32 compare operations.
 void VisitFloat32Compare(InstructionSelector* selector, Node* node,
@@ -2226,6 +2245,58 @@ void InstructionSelector::VisitWordCompareZero(Node* user, Node* value,
     cont->Negate();
   }
 
+  // Try to match bit checks to create TBZ/TBNZ instructions.
+  // Unlike the switch below, CanCover check is not needed here.
+  // If there are several uses of the given operation, we will generate a TBZ
+  // instruction for each. This is useful even if there are other uses of the
+  // arithmetic result, because it moves dependencies further back.
+  switch (value->opcode()) {
+    case IrOpcode::kWord64Equal: {
+      Int64BinopMatcher m(value);
+      if (m.right().Is(0)) {
+        Node* const left = m.left().node();
+        if (left->opcode() == IrOpcode::kWord64And) {
+          // Attempt to merge the Word64Equal(Word64And(x, y), 0) comparison
+          // into a tbz/tbnz instruction.
+          TestAndBranchMatcher<Uint64BinopMatcher> tbm(left, cont);
+          if (tbm.Matches()) {
+            Arm64OperandGenerator gen(this);
+            cont->OverwriteAndNegateIfEqual(kEqual);
+            this->EmitWithContinuation(kArm64TestAndBranch,
+                                       gen.UseRegister(tbm.input()),
+                                       gen.TempImmediate(tbm.bit()), cont);
+            return;
+          }
+        }
+      }
+      break;
+    }
+    case IrOpcode::kWord32And: {
+      TestAndBranchMatcher<Uint32BinopMatcher> tbm(value, cont);
+      if (tbm.Matches()) {
+        Arm64OperandGenerator gen(this);
+        this->EmitWithContinuation(kArm64TestAndBranch32,
+                                   gen.UseRegister(tbm.input()),
+                                   gen.TempImmediate(tbm.bit()), cont);
+        return;
+      }
+      break;
+    }
+    case IrOpcode::kWord64And: {
+      TestAndBranchMatcher<Uint64BinopMatcher> tbm(value, cont);
+      if (tbm.Matches()) {
+        Arm64OperandGenerator gen(this);
+        this->EmitWithContinuation(kArm64TestAndBranch,
+                                   gen.UseRegister(tbm.input()),
+                                   gen.TempImmediate(tbm.bit()), cont);
+        return;
+      }
+      break;
+    }
+    default:
+      break;
+  }
+
   if (CanCover(user, value)) {
     switch (value->opcode()) {
       case IrOpcode::kWord32Equal:
@@ -2249,12 +2320,6 @@ void InstructionSelector::VisitWordCompareZero(Node* user, Node* value,
         if (m.right().Is(0)) {
           Node* const left = m.left().node();
           if (CanCover(value, left) && left->opcode() == IrOpcode::kWord64And) {
-            // Attempt to merge the Word64Equal(Word64And(x, y), 0) comparison
-            // into a tbz/tbnz instruction.
-            if (TryEmitTestAndBranch<Uint64BinopMatcher, kArm64TestAndBranch>(
-                    this, left, cont)) {
-              return;
-            }
             return VisitWordCompare(this, left, kArm64Tst, cont, true,
                                     kLogical64Imm);
           }
@@ -2351,17 +2416,9 @@ void InstructionSelector::VisitWordCompareZero(Node* user, Node* value,
       case IrOpcode::kInt32Sub:
         return VisitWord32Compare(this, value, cont);
       case IrOpcode::kWord32And:
-        if (TryEmitTestAndBranch<Uint32BinopMatcher, kArm64TestAndBranch32>(
-                this, value, cont)) {
-          return;
-        }
         return VisitWordCompare(this, value, kArm64Tst32, cont, true,
                                 kLogical32Imm);
       case IrOpcode::kWord64And:
-        if (TryEmitTestAndBranch<Uint64BinopMatcher, kArm64TestAndBranch>(
-                this, value, cont)) {
-          return;
-        }
         return VisitWordCompare(this, value, kArm64Tst, cont, true,
                                 kLogical64Imm);
       default:

test/unittests/compiler/arm64/instruction-selector-arm64-unittest.cc

@@ -1327,6 +1327,70 @@ TEST_F(InstructionSelectorTest, Word64AndBranchWithOneBitMaskOnLeft) {
   }
 }
 
+TEST_F(InstructionSelectorTest, TestAndBranch64EqualWhenCanCoverFalse) {
+  TRACED_FORRANGE(int, bit, 0, 63) {
+    uint64_t mask = uint64_t{1} << bit;
+    StreamBuilder m(this, MachineType::Int64(), MachineType::Int64());
+    RawMachineLabel a, b, c;
+    Node* n = m.Word64And(m.Parameter(0), m.Int64Constant(mask));
+    m.Branch(m.Word64Equal(n, m.Int64Constant(0)), &a, &b);
+    m.Bind(&a);
+    m.Branch(m.Word64Equal(n, m.Int64Constant(3)), &b, &c);
+    m.Bind(&c);
+    m.Return(m.Int64Constant(1));
+    m.Bind(&b);
+    m.Return(m.Int64Constant(0));
+
+    Stream s = m.Build();
+    ASSERT_EQ(3U, s.size());
+    EXPECT_EQ(kArm64And, s[0]->arch_opcode());
+    EXPECT_EQ(kEqual, s[0]->flags_condition());
+    EXPECT_EQ(kArm64TestAndBranch, s[1]->arch_opcode());
+    EXPECT_EQ(kEqual, s[1]->flags_condition());
+    EXPECT_EQ(kArm64Cmp, s[2]->arch_opcode());
+    EXPECT_EQ(kEqual, s[2]->flags_condition());
+    EXPECT_EQ(2U, s[0]->InputCount());
+  }
+}
+
+TEST_F(InstructionSelectorTest, TestAndBranch64AndWhenCanCoverFalse) {
+  TRACED_FORRANGE(int, bit, 0, 63) {
+    uint64_t mask = uint64_t{1} << bit;
+    StreamBuilder m(this, MachineType::Int64(), MachineType::Int64());
+    RawMachineLabel a, b, c;
+    m.Branch(m.Word64And(m.Parameter(0), m.Int64Constant(mask)), &a, &b);
+    m.Bind(&a);
+    m.Return(m.Int64Constant(1));
+    m.Bind(&b);
+    m.Return(m.Int64Constant(0));
+
+    Stream s = m.Build();
+    ASSERT_EQ(1U, s.size());
+    EXPECT_EQ(kArm64TestAndBranch, s[0]->arch_opcode());
+    EXPECT_EQ(InstructionOperand::IMMEDIATE, s[0]->InputAt(1)->kind());
+    EXPECT_EQ(4U, s[0]->InputCount());
+  }
+}
+
+TEST_F(InstructionSelectorTest, TestAndBranch32AndWhenCanCoverFalse) {
+  TRACED_FORRANGE(int, bit, 0, 31) {
+    uint32_t mask = uint32_t{1} << bit;
+    StreamBuilder m(this, MachineType::Int64(), MachineType::Int64());
+    RawMachineLabel a, b, c;
+    m.Branch(m.Word32And(m.Parameter(0), m.Int32Constant(mask)), &a, &b);
+    m.Bind(&a);
+    m.Return(m.Int32Constant(1));
+    m.Bind(&b);
+    m.Return(m.Int32Constant(0));
+
+    Stream s = m.Build();
+    ASSERT_EQ(1U, s.size());
+    EXPECT_EQ(kArm64TestAndBranch32, s[0]->arch_opcode());
+    EXPECT_EQ(InstructionOperand::IMMEDIATE, s[0]->InputAt(1)->kind());
+    EXPECT_EQ(4U, s[0]->InputCount());
+  }
+}
+
 TEST_F(InstructionSelectorTest, Word32EqualZeroAndBranchWithOneBitMask) {
   TRACED_FORRANGE(int, bit, 0, 31) {
     uint32_t mask = 1 << bit;