Reland "[compiler] Make ReduceWord32EqualForConstantRhs work for Word64Equal"

This is a reland of commit abd0adf1

Original change's description:
> [compiler] Make ReduceWord32EqualForConstantRhs work for Word64Equal
>
> Adds reduction case in MachineOperatorReducer for when the left-hand side of a
> Word64Equals is based on a 64-bit shift-and-mask operation, as is the case
> when Torque accesses 64-bit bitfields.
>
> This improves Speedometer2 by 0.15% on a Neoverse-N1 machine, with
> React-Redux being improved by 0.4%.
>
> Change-Id: Icd0451c00c1b25f7d370e81bddcfd668a5b2523c
> Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3834027
> Commit-Queue: George Wort <george.wort@arm.com>
> Reviewed-by: Nico Hartmann <nicohartmann@chromium.org>
> Cr-Commit-Position: refs/heads/main@{#82593}

Change-Id: I62393c062b2c785a5dfa3500b80fe44ec08f6f21
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3841569
Commit-Queue: George Wort <george.wort@arm.com>
Reviewed-by: Nico Hartmann <nicohartmann@chromium.org>
Cr-Commit-Position: refs/heads/main@{#82684}

src/compiler/machine-operator-reducer.cc

@@ -55,6 +55,10 @@ class Word32Adapter {
   static bool IsWordNSar(const T& x) {
     return x.IsWord32Sar();
   }
+  static bool IsWordNSarShiftOutZeros(const Operator* op) {
+    return op->opcode() == IrOpcode::kWord32Sar &&
+           OpParameter<ShiftKind>(op) == ShiftKind::kShiftOutZeros;
+  }
   template <typename T>
   static bool IsWordNXor(const T& x) {
     return x.IsWord32Xor();
@@ -115,6 +119,10 @@ class Word64Adapter {
   static bool IsWordNSar(const T& x) {
     return x.IsWord64Sar();
   }
+  static bool IsWordNSarShiftOutZeros(const Operator* op) {
+    return op->opcode() == IrOpcode::kWord64Sar &&
+           OpParameter<ShiftKind>(op) == ShiftKind::kShiftOutZeros;
+  }
   template <typename T>
   static bool IsWordNXor(const T& x) {
     return x.IsWord64Xor();
@@ -326,25 +334,10 @@ Reduction MachineOperatorReducer::Reduce(Node* node) {
       }
       break;
     }
-    case IrOpcode::kWord32Equal: {
+    case IrOpcode::kWord32Equal:
       return ReduceWord32Equal(node);
-    }
-    case IrOpcode::kWord64Equal: {
-      Int64BinopMatcher m(node);
-      if (m.IsFoldable()) {  // K == K => K  (K stands for arbitrary constants)
-        return ReplaceBool(m.left().ResolvedValue() ==
-                           m.right().ResolvedValue());
-      }
-      if (m.left().IsInt64Sub() && m.right().Is(0)) {  // x - y == 0 => x == y
-        Int64BinopMatcher msub(m.left().node());
-        node->ReplaceInput(0, msub.left().node());
-        node->ReplaceInput(1, msub.right().node());
-        return Changed(node);
-      }
-      // TODO(turbofan): fold HeapConstant, ExternalReference, pointer compares
-      if (m.LeftEqualsRight()) return ReplaceBool(true);  // x == x => true
-      break;
-    }
+    case IrOpcode::kWord64Equal:
+      return ReduceWord64Equal(node);
     case IrOpcode::kInt32Add:
       return ReduceInt32Add(node);
     case IrOpcode::kInt64Add:
@@ -1891,7 +1884,7 @@ struct BitfieldCheck {
     //    - the result may be truncated from 64 to 32
     // 2. Equality checks: `(val & mask) == expected`, where:
     //    - val may be truncated from 64 to 32 before masking (see
-    //      ReduceWord32EqualForConstantRhs)
+    //      ReduceWordEqualForConstantRhs)
     if (node->opcode() == IrOpcode::kWord32Equal) {
       Uint32BinopMatcher eq(node);
       if (eq.left().IsWord32And()) {
@@ -2152,11 +2145,11 @@ Reduction MachineOperatorReducer::ReduceWord32Equal(Node* node) {
   if (m.right().HasResolvedValue()) {
     base::Optional<std::pair<Node*, uint32_t>> replacements;
     if (m.left().IsTruncateInt64ToInt32()) {
-      replacements = ReduceWord32EqualForConstantRhs<Word64Adapter>(
+      replacements = ReduceWordEqualForConstantRhs<Word64Adapter, uint32_t>(
           NodeProperties::GetValueInput(m.left().node(), 0),
           static_cast<uint32_t>(m.right().ResolvedValue()));
     } else {
-      replacements = ReduceWord32EqualForConstantRhs<Word32Adapter>(
+      replacements = ReduceWordEqualForConstantRhs<Word32Adapter, uint32_t>(
           m.left().node(), static_cast<uint32_t>(m.right().ResolvedValue()));
     }
     if (replacements) {
@@ -2169,6 +2162,33 @@ Reduction MachineOperatorReducer::ReduceWord32Equal(Node* node) {
   return NoChange();
 }
 
+Reduction MachineOperatorReducer::ReduceWord64Equal(Node* node) {
+  Int64BinopMatcher m(node);
+  if (m.IsFoldable()) {  // K == K => K  (K stands for arbitrary constants)
+    return ReplaceBool(m.left().ResolvedValue() == m.right().ResolvedValue());
+  }
+  if (m.left().IsInt64Sub() && m.right().Is(0)) {  // x - y == 0 => x == y
+    Int64BinopMatcher msub(m.left().node());
+    node->ReplaceInput(0, msub.left().node());
+    node->ReplaceInput(1, msub.right().node());
+    return Changed(node);
+  }
+  // TODO(turbofan): fold HeapConstant, ExternalReference, pointer compares
+  if (m.LeftEqualsRight()) return ReplaceBool(true);  // x == x => true
+  if (m.right().HasResolvedValue()) {
+    base::Optional<std::pair<Node*, uint64_t>> replacements =
+        ReduceWordEqualForConstantRhs<Word64Adapter, uint64_t>(
+            m.left().node(), static_cast<uint64_t>(m.right().ResolvedValue()));
+    if (replacements) {
+      node->ReplaceInput(0, replacements->first);
+      node->ReplaceInput(1, Uint64Constant(replacements->second));
+      return Changed(node);
+    }
+  }
+
+  return NoChange();
+}
+
 Reduction MachineOperatorReducer::ReduceFloat64InsertLowWord32(Node* node) {
   DCHECK_EQ(IrOpcode::kFloat64InsertLowWord32, node->opcode());
   Float64Matcher mlhs(node->InputAt(0));
@@ -2472,16 +2492,15 @@ base::Optional<Node*> MachineOperatorReducer::ReduceConditionalN(Node* node) {
   // opposite of a 32-bit `x == 0` node. To avoid repetition, we can reuse logic
   // for Word32Equal: if `x == 0` can reduce to `y == 0`, then branch(x) can
   // reduce to branch(y).
-  auto replacements =
-      ReduceWord32EqualForConstantRhs<WordNAdapter>(condition.node(), 0);
+  auto replacements = ReduceWordEqualForConstantRhs<WordNAdapter, uint32_t>(
+      condition.node(), 0);
   if (replacements && replacements->second == 0) return replacements->first;
   return {};
 }
 
-template <typename WordNAdapter>
-base::Optional<std::pair<Node*, uint32_t>>
-MachineOperatorReducer::ReduceWord32EqualForConstantRhs(Node* lhs,
-                                                        uint32_t rhs) {
+template <typename WordNAdapter, typename uintN_t, typename intN_t>
+base::Optional<std::pair<Node*, uintN_t>>
+MachineOperatorReducer::ReduceWordEqualForConstantRhs(Node* lhs, uintN_t rhs) {
   if (WordNAdapter::IsWordNAnd(NodeMatcher(lhs))) {
     typename WordNAdapter::UintNBinopMatcher mand(lhs);
     if ((WordNAdapter::IsWordNShr(mand.left()) ||
@@ -2496,27 +2515,34 @@ MachineOperatorReducer::ReduceWord32EqualForConstantRhs(Node* lhs,
         // data ends up in the lower 32 bits for 64-bit mode.
         if (shift_bits <= base::bits::CountLeadingZeros(mask) &&
             shift_bits <= base::bits::CountLeadingZeros(rhs) &&
-            mask << shift_bits <= std::numeric_limits<uint32_t>::max()) {
+            (std::is_same_v<uintN_t, uint64_t> ||
+             mask << shift_bits <= std::numeric_limits<uintN_t>::max())) {
           Node* new_input = mshift.left().node();
-          uint32_t new_mask = static_cast<uint32_t>(mask << shift_bits);
-          uint32_t new_rhs = rhs << shift_bits;
-          if (WordNAdapter::WORD_SIZE == 64) {
+          uintN_t new_mask = static_cast<uintN_t>(mask << shift_bits);
+          uintN_t new_rhs = rhs << shift_bits;
+          if (std::is_same_v<uintN_t, uint32_t> &&
+              WordNAdapter::WORD_SIZE == 64) {
             // We can truncate before performing the And.
             new_input = TruncateInt64ToInt32(new_input);
+            return std::make_pair(Word32And(new_input, new_mask), new_rhs);
+          } else {
+            WordNAdapter a(this);
+            return std::make_pair(
+                a.WordNAnd(new_input, a.UintNConstant(new_mask)), new_rhs);
           }
-          return std::make_pair(Word32And(new_input, new_mask), new_rhs);
         }
       }
     }
   }
   // Replaces (x >> n) == k with x == k << n, with "k << n" being computed
   // here at compile time.
-  if (lhs->op() == machine()->Word32SarShiftOutZeros() &&
+  if (std::is_same_v<intN_t, typename WordNAdapter::intN_t> &&
+      WordNAdapter::IsWordNSarShiftOutZeros(lhs->op()) &&
       lhs->UseCount() == 1) {
     typename WordNAdapter::UintNBinopMatcher mshift(lhs);
     if (mshift.right().HasResolvedValue()) {
-      int32_t shift = static_cast<int32_t>(mshift.right().ResolvedValue());
-      if (CanRevertLeftShiftWithRightShift<int32_t>(rhs, shift)) {
+      intN_t shift = static_cast<intN_t>(mshift.right().ResolvedValue());
+      if (CanRevertLeftShiftWithRightShift<intN_t>(rhs, shift)) {
         return std::make_pair(mshift.left().node(), rhs << shift);
       }
     }

src/compiler/machine-operator-reducer.h

@@ -57,6 +57,9 @@ class V8_EXPORT_PRIVATE MachineOperatorReducer final
   Node* Word32Shr(Node* lhs, uint32_t rhs);
   Node* Word32Equal(Node* lhs, Node* rhs);
   Node* Word64And(Node* lhs, Node* rhs);
+  Node* Word64And(Node* lhs, uint64_t rhs) {
+    return Word64And(lhs, Uint64Constant(rhs));
+  }
   Node* Int32Add(Node* lhs, Node* rhs);
   Node* Int32Sub(Node* lhs, Node* rhs);
   Node* Int32Mul(Node* lhs, Node* rhs);
@@ -110,6 +113,7 @@ class V8_EXPORT_PRIVATE MachineOperatorReducer final
   Reduction ReduceWord32Xor(Node* node);
   Reduction ReduceWord64Xor(Node* node);
   Reduction ReduceWord32Equal(Node* node);
+  Reduction ReduceWord64Equal(Node* node);
   Reduction ReduceFloat64InsertLowWord32(Node* node);
   Reduction ReduceFloat64InsertHighWord32(Node* node);
   Reduction ReduceFloat64Compare(Node* node);
@@ -144,10 +148,14 @@ class V8_EXPORT_PRIVATE MachineOperatorReducer final
 
   // Helper for finding a reduced equality condition. Does not perform the
   // actual reduction; just returns a new pair that could be compared for the
-  // same outcome.
-  template <typename WordNAdapter>
-  base::Optional<std::pair<Node*, uint32_t>> ReduceWord32EqualForConstantRhs(
-      Node* lhs, uint32_t rhs);
+  // same outcome. uintN_t corresponds to the size of the Equal operator, and
+  // thus the size of rhs. While the size of the WordNAdaptor corresponds to the
+  // size of lhs, with the sizes being different for
+  // Word32Equal(TruncateInt64ToInt32(lhs), rhs).
+  template <typename WordNAdapter, typename uintN_t,
+            typename intN_t = typename std::make_signed<uintN_t>::type>
+  base::Optional<std::pair<Node*, uintN_t>> ReduceWordEqualForConstantRhs(
+      Node* lhs, uintN_t rhs);
 
   MachineGraph* mcgraph_;
   bool allow_signalling_nan_;

test/unittests/compiler/graph-unittest.h

@@ -43,6 +43,9 @@ class GraphTest : public TestWithNativeContextAndZone {
     return Int32Constant(base::bit_cast<int32_t>(value));
   }
   Node* Int64Constant(int64_t value);
+  Node* Uint64Constant(uint64_t value) {
+    return Int64Constant(base::bit_cast<int64_t>(value));
+  }
   Node* NumberConstant(double value);
   Node* HeapConstant(const Handle<HeapObject>& value);
   Node* FalseConstant();

test/unittests/compiler/machine-operator-reducer-unittest.cc

@@ -300,6 +300,22 @@ const uint32_t kUint32Values[] = {
     0x000FFFFF, 0x0007FFFF, 0x0003FFFF, 0x0001FFFF, 0x0000FFFF, 0x00007FFF,
     0x00003FFF, 0x00001FFF, 0x00000FFF, 0x000007FF, 0x000003FF, 0x000001FF};
 
+const uint64_t kUint64Values[] = {
+    0x0000000000000000, 0x0000000000000001, 0xFFFFFFFFFFFFFFFF,
+    0x1B09788B1B09788B, 0x0000000004C5FCE8, 0xCC0DE5BFCC0DE5BF,
+    0x273A798E273A798E, 0x187937A3187937A3, 0xECE3AF83ECE3AF83,
+    0x5495A16B5495A16B, 0x000000000B668ECC, 0x1122334455667788,
+    0x000000000000009E, 0x000000000000AF73, 0x000000000000116B,
+    0x0000000000658ECC, 0x00000000002B3B4C, 0x8877665588776655,
+    0x0720000000000000, 0x7FFFFFFFFFFFFFFF, 0x5612376156123761,
+    0x7FFFFFFFFFFF0000, 0x761C4761761C4761, 0x8000000000000000,
+    0xA000000000000000, 0xDDDDDDDDDDDDDDDD, 0xEEEEEEEEEEEEEEEE,
+    0xFFFFFFFFFFFFFFFD, 0xF000000000000000, 0x007FFFFFFFFFFFFF,
+    0x001FFFFFFFFFFFFF, 0x000FFFFFFFFFFFFF, 0x00007FFFFFFFFFFF,
+    0x00001FFFFFFFFFFF, 0x00000FFFFFFFFFFF, 0x000007FFFFFFFFFF,
+    0x000001FFFFFFFFFF, 0x00000000007FFFFF, 0x00000000001FFFFF,
+    0x00000000000FFFFF, 0x00000000000007FF, 0x00000000000001FF};
+
 struct ComparisonBinaryOperator {
   const Operator* (MachineOperatorBuilder::*constructor)();
   const char* constructor_name;
@@ -1359,6 +1375,44 @@ TEST_F(MachineOperatorReducerTest,
   }
 }
 
+// -----------------------------------------------------------------------------
+// Word64Equal
+
+TEST_F(MachineOperatorReducerTest,
+       Word64EqualWithShiftedMaskedValueAndConstant) {
+  // ((x >> K1) & K2) == K3 => (x & (K2 << K1)) == (K3 << K1)
+  Node* const p0 = Parameter(0);
+  TRACED_FOREACH(uint64_t, mask, kUint64Values) {
+    TRACED_FOREACH(uint64_t, rhs, kUint64Values) {
+      TRACED_FORRANGE(uint64_t, shift_bits, 1, 63) {
+        Node* node = graph()->NewNode(
+            machine()->Word64Equal(),
+            graph()->NewNode(machine()->Word64And(),
+                             graph()->NewNode(machine()->Word64Shr(), p0,
+                                              Uint64Constant(shift_bits)),
+                             Uint64Constant(mask)),
+            Uint64Constant(rhs));
+        Reduction r = Reduce(node);
+        uint64_t new_mask = mask << shift_bits;
+        uint64_t new_rhs = rhs << shift_bits;
+        if (new_mask >> shift_bits == mask && new_rhs >> shift_bits == rhs) {
+          ASSERT_TRUE(r.Changed());
+          // The left-hand side of the equality is now a Word64And operation,
+          // unless the mask is zero in which case the newly-created Word64And
+          // is immediately reduced away.
+          Matcher<Node*> lhs = mask == 0
+                                   ? IsInt64Constant(0)
+                                   : IsWord64And(p0, IsInt64Constant(new_mask));
+          EXPECT_THAT(r.replacement(),
+                      IsWord64Equal(lhs, IsInt64Constant(new_rhs)));
+        } else {
+          ASSERT_FALSE(r.Changed());
+        }
+      }
+    }
+  }
+}
+
 // -----------------------------------------------------------------------------
 // Branch