[wasm] New implementation of popcnt and ctz.

This patch provides a new implementation of popcnt and ctz in the case
where the platform does not provide these instructions. Instead of
building a TF graph which implements it we now call a C function.

Additionally I turned on additional tests in test-run-wasm-64.cc

R=titzer@chromium.org

Review URL: https://codereview.chromium.org/1857363003

Cr-Commit-Position: refs/heads/master@{#35685}

src/assembler.cc

@@ -1261,6 +1261,26 @@ ExternalReference ExternalReference::wasm_uint64_mod(Isolate* isolate) {
       Redirect(isolate, FUNCTION_ADDR(wasm::uint64_mod_wrapper)));
 }
 
+ExternalReference ExternalReference::wasm_word32_ctz(Isolate* isolate) {
+  return ExternalReference(
+      Redirect(isolate, FUNCTION_ADDR(wasm::word32_ctz_wrapper)));
+}
+
+ExternalReference ExternalReference::wasm_word64_ctz(Isolate* isolate) {
+  return ExternalReference(
+      Redirect(isolate, FUNCTION_ADDR(wasm::word64_ctz_wrapper)));
+}
+
+ExternalReference ExternalReference::wasm_word32_popcnt(Isolate* isolate) {
+  return ExternalReference(
+      Redirect(isolate, FUNCTION_ADDR(wasm::word32_popcnt_wrapper)));
+}
+
+ExternalReference ExternalReference::wasm_word64_popcnt(Isolate* isolate) {
+  return ExternalReference(
+      Redirect(isolate, FUNCTION_ADDR(wasm::word64_popcnt_wrapper)));
+}
+
 static void f64_acos_wrapper(double* param) { *param = std::acos(*param); }
 
 ExternalReference ExternalReference::f64_acos_wrapper_function(

src/assembler.h

@@ -951,6 +951,10 @@ class ExternalReference BASE_EMBEDDED {
   static ExternalReference wasm_int64_mod(Isolate* isolate);
   static ExternalReference wasm_uint64_div(Isolate* isolate);
   static ExternalReference wasm_uint64_mod(Isolate* isolate);
+  static ExternalReference wasm_word32_ctz(Isolate* isolate);
+  static ExternalReference wasm_word64_ctz(Isolate* isolate);
+  static ExternalReference wasm_word32_popcnt(Isolate* isolate);
+  static ExternalReference wasm_word64_popcnt(Isolate* isolate);
 
   static ExternalReference f64_acos_wrapper_function(Isolate* isolate);
   static ExternalReference f64_asin_wrapper_function(Isolate* isolate);

src/compiler/wasm-compiler.cc

@@ -1221,180 +1221,52 @@ Node* WasmGraphBuilder::BuildI32UConvertF64(Node* input) {
   return result;
 }
 
+Node* WasmGraphBuilder::BuildBitCountingCall(Node* input, ExternalReference ref,
+                                             MachineRepresentation input_type) {
+  Node* stack_slot_param =
+      graph()->NewNode(jsgraph()->machine()->StackSlot(input_type));
 
-Node* WasmGraphBuilder::BuildI32Ctz(Node* input) {
-  //// Implement the following code as TF graph.
-  // value = value | (value << 1);
-  // value = value | (value << 2);
-  // value = value | (value << 4);
-  // value = value | (value << 8);
-  // value = value | (value << 16);
-  // return CountPopulation32(0xffffffff XOR value);
-
-  Node* result =
-      Binop(wasm::kExprI32Ior, input,
-            Binop(wasm::kExprI32Shl, input, jsgraph()->Int32Constant(1)));
-
-  result = Binop(wasm::kExprI32Ior, result,
-                 Binop(wasm::kExprI32Shl, result, jsgraph()->Int32Constant(2)));
-
-  result = Binop(wasm::kExprI32Ior, result,
-                 Binop(wasm::kExprI32Shl, result, jsgraph()->Int32Constant(4)));
-
-  result = Binop(wasm::kExprI32Ior, result,
-                 Binop(wasm::kExprI32Shl, result, jsgraph()->Int32Constant(8)));
+  const Operator* store_op = jsgraph()->machine()->Store(
+      StoreRepresentation(input_type, kNoWriteBarrier));
+  *effect_ =
+      graph()->NewNode(store_op, stack_slot_param, jsgraph()->Int32Constant(0),
+                       input, *effect_, *control_);
 
-  result =
-      Binop(wasm::kExprI32Ior, result,
-            Binop(wasm::kExprI32Shl, result, jsgraph()->Int32Constant(16)));
+  MachineSignature::Builder sig_builder(jsgraph()->zone(), 1, 1);
+  sig_builder.AddReturn(MachineType::Int32());
+  sig_builder.AddParam(MachineType::Pointer());
 
-  result = BuildI32Popcnt(
-      Binop(wasm::kExprI32Xor, jsgraph()->Int32Constant(0xffffffff), result));
+  Node* function = graph()->NewNode(jsgraph()->common()->ExternalConstant(ref));
+  Node* args[] = {function, stack_slot_param};
 
-  return result;
+  return BuildCCall(sig_builder.Build(), args);
 }
 
+Node* WasmGraphBuilder::BuildI32Ctz(Node* input) {
+  return BuildBitCountingCall(
+      input, ExternalReference::wasm_word32_ctz(jsgraph()->isolate()),
+      MachineRepresentation::kWord32);
+}
 
 Node* WasmGraphBuilder::BuildI64Ctz(Node* input) {
-  //// Implement the following code as TF graph.
-  // value = value | (value << 1);
-  // value = value | (value << 2);
-  // value = value | (value << 4);
-  // value = value | (value << 8);
-  // value = value | (value << 16);
-  // value = value | (value << 32);
-  // return CountPopulation64(0xffffffffffffffff XOR value);
-
-  Node* result =
-      Binop(wasm::kExprI64Ior, input,
-            Binop(wasm::kExprI64Shl, input, jsgraph()->Int64Constant(1)));
-
-  result = Binop(wasm::kExprI64Ior, result,
-                 Binop(wasm::kExprI64Shl, result, jsgraph()->Int64Constant(2)));
-
-  result = Binop(wasm::kExprI64Ior, result,
-                 Binop(wasm::kExprI64Shl, result, jsgraph()->Int64Constant(4)));
-
-  result = Binop(wasm::kExprI64Ior, result,
-                 Binop(wasm::kExprI64Shl, result, jsgraph()->Int64Constant(8)));
-
-  result =
-      Binop(wasm::kExprI64Ior, result,
-            Binop(wasm::kExprI64Shl, result, jsgraph()->Int64Constant(16)));
-
-  result =
-      Binop(wasm::kExprI64Ior, result,
-            Binop(wasm::kExprI64Shl, result, jsgraph()->Int64Constant(32)));
-
-  result = BuildI64Popcnt(Binop(
-      wasm::kExprI64Xor, jsgraph()->Int64Constant(0xffffffffffffffff), result));
-
-  return result;
+  return Unop(wasm::kExprI64UConvertI32,
+              BuildBitCountingCall(input, ExternalReference::wasm_word64_ctz(
+                                              jsgraph()->isolate()),
+                                   MachineRepresentation::kWord64));
 }
 
-
 Node* WasmGraphBuilder::BuildI32Popcnt(Node* input) {
-  //// Implement the following code as a TF graph.
-  // value = ((value >> 1) & 0x55555555) + (value & 0x55555555);
-  // value = ((value >> 2) & 0x33333333) + (value & 0x33333333);
-  // value = ((value >> 4) & 0x0f0f0f0f) + (value & 0x0f0f0f0f);
-  // value = ((value >> 8) & 0x00ff00ff) + (value & 0x00ff00ff);
-  // value = ((value >> 16) & 0x0000ffff) + (value & 0x0000ffff);
-
-  Node* result = Binop(
-      wasm::kExprI32Add,
-      Binop(wasm::kExprI32And,
-            Binop(wasm::kExprI32ShrU, input, jsgraph()->Int32Constant(1)),
-            jsgraph()->Int32Constant(0x55555555)),
-      Binop(wasm::kExprI32And, input, jsgraph()->Int32Constant(0x55555555)));
-
-  result = Binop(
-      wasm::kExprI32Add,
-      Binop(wasm::kExprI32And,
-            Binop(wasm::kExprI32ShrU, result, jsgraph()->Int32Constant(2)),
-            jsgraph()->Int32Constant(0x33333333)),
-      Binop(wasm::kExprI32And, result, jsgraph()->Int32Constant(0x33333333)));
-
-  result = Binop(
-      wasm::kExprI32Add,
-      Binop(wasm::kExprI32And,
-            Binop(wasm::kExprI32ShrU, result, jsgraph()->Int32Constant(4)),
-            jsgraph()->Int32Constant(0x0f0f0f0f)),
-      Binop(wasm::kExprI32And, result, jsgraph()->Int32Constant(0x0f0f0f0f)));
-
-  result = Binop(
-      wasm::kExprI32Add,
-      Binop(wasm::kExprI32And,
-            Binop(wasm::kExprI32ShrU, result, jsgraph()->Int32Constant(8)),
-            jsgraph()->Int32Constant(0x00ff00ff)),
-      Binop(wasm::kExprI32And, result, jsgraph()->Int32Constant(0x00ff00ff)));
-
-  result = Binop(
-      wasm::kExprI32Add,
-      Binop(wasm::kExprI32And,
-            Binop(wasm::kExprI32ShrU, result, jsgraph()->Int32Constant(16)),
-            jsgraph()->Int32Constant(0x0000ffff)),
-      Binop(wasm::kExprI32And, result, jsgraph()->Int32Constant(0x0000ffff)));
-
-  return result;
+  return BuildBitCountingCall(
+      input, ExternalReference::wasm_word32_popcnt(jsgraph()->isolate()),
+      MachineRepresentation::kWord32);
 }
 
 
 Node* WasmGraphBuilder::BuildI64Popcnt(Node* input) {
-  //// Implement the following code as a TF graph.
-  // value = ((value >> 1) & 0x5555555555555555) + (value & 0x5555555555555555);
-  // value = ((value >> 2) & 0x3333333333333333) + (value & 0x3333333333333333);
-  // value = ((value >> 4) & 0x0f0f0f0f0f0f0f0f) + (value & 0x0f0f0f0f0f0f0f0f);
-  // value = ((value >> 8) & 0x00ff00ff00ff00ff) + (value & 0x00ff00ff00ff00ff);
-  // value = ((value >> 16) & 0x0000ffff0000ffff) + (value &
-  // 0x0000ffff0000ffff);
-  // value = ((value >> 32) & 0x00000000ffffffff) + (value &
-  // 0x00000000ffffffff);
-
-  Node* result =
-      Binop(wasm::kExprI64Add,
-            Binop(wasm::kExprI64And,
-                  Binop(wasm::kExprI64ShrU, input, jsgraph()->Int64Constant(1)),
-                  jsgraph()->Int64Constant(0x5555555555555555)),
-            Binop(wasm::kExprI64And, input,
-                  jsgraph()->Int64Constant(0x5555555555555555)));
-
-  result = Binop(wasm::kExprI64Add,
-                 Binop(wasm::kExprI64And, Binop(wasm::kExprI64ShrU, result,
-                                                jsgraph()->Int64Constant(2)),
-                       jsgraph()->Int64Constant(0x3333333333333333)),
-                 Binop(wasm::kExprI64And, result,
-                       jsgraph()->Int64Constant(0x3333333333333333)));
-
-  result = Binop(wasm::kExprI64Add,
-                 Binop(wasm::kExprI64And, Binop(wasm::kExprI64ShrU, result,
-                                                jsgraph()->Int64Constant(4)),
-                       jsgraph()->Int64Constant(0x0f0f0f0f0f0f0f0f)),
-                 Binop(wasm::kExprI64And, result,
-                       jsgraph()->Int64Constant(0x0f0f0f0f0f0f0f0f)));
-
-  result = Binop(wasm::kExprI64Add,
-                 Binop(wasm::kExprI64And, Binop(wasm::kExprI64ShrU, result,
-                                                jsgraph()->Int64Constant(8)),
-                       jsgraph()->Int64Constant(0x00ff00ff00ff00ff)),
-                 Binop(wasm::kExprI64And, result,
-                       jsgraph()->Int64Constant(0x00ff00ff00ff00ff)));
-
-  result = Binop(wasm::kExprI64Add,
-                 Binop(wasm::kExprI64And, Binop(wasm::kExprI64ShrU, result,
-                                                jsgraph()->Int64Constant(16)),
-                       jsgraph()->Int64Constant(0x0000ffff0000ffff)),
-                 Binop(wasm::kExprI64And, result,
-                       jsgraph()->Int64Constant(0x0000ffff0000ffff)));
-
-  result = Binop(wasm::kExprI64Add,
-                 Binop(wasm::kExprI64And, Binop(wasm::kExprI64ShrU, result,
-                                                jsgraph()->Int64Constant(32)),
-                       jsgraph()->Int64Constant(0x00000000ffffffff)),
-                 Binop(wasm::kExprI64And, result,
-                       jsgraph()->Int64Constant(0x00000000ffffffff)));
-
-  return result;
+  return Unop(wasm::kExprI64UConvertI32,
+              BuildBitCountingCall(input, ExternalReference::wasm_word64_popcnt(
+                                              jsgraph()->isolate()),
+                                   MachineRepresentation::kWord64));
 }
 
 Node* WasmGraphBuilder::BuildF32Trunc(Node* input) {

src/compiler/wasm-compiler.h

@@ -190,6 +190,9 @@ class WasmGraphBuilder {
   Node* BuildI32Popcnt(Node* input);
   Node* BuildI64Ctz(Node* input);
   Node* BuildI64Popcnt(Node* input);
+  Node* BuildBitCountingCall(Node* input, ExternalReference ref,
+                             MachineRepresentation input_type);
+
   Node* BuildCFuncInstruction(ExternalReference ref, MachineType type,
                               Node* input0, Node* input1 = nullptr);
   Node* BuildF32Trunc(Node* input);

src/external-reference-table.cc

@@ -136,6 +136,14 @@ ExternalReferenceTable::ExternalReferenceTable(Isolate* isolate) {
       "wasm::uint64_div");
   Add(ExternalReference::wasm_uint64_mod(isolate).address(),
       "wasm::uint64_mod");
+  Add(ExternalReference::wasm_word32_ctz(isolate).address(),
+      "wasm::word32_ctz");
+  Add(ExternalReference::wasm_word64_ctz(isolate).address(),
+      "wasm::word64_ctz");
+  Add(ExternalReference::wasm_word32_popcnt(isolate).address(),
+      "wasm::word32_popcnt");
+  Add(ExternalReference::wasm_word64_popcnt(isolate).address(),
+      "wasm::word64_popcnt");
   Add(ExternalReference::f64_acos_wrapper_function(isolate).address(),
       "f64_acos_wrapper");
   Add(ExternalReference::f64_asin_wrapper_function(isolate).address(),

src/wasm/wasm-external-refs.cc

@@ -9,6 +9,7 @@
 
 #include "include/v8config.h"
 
+#include "src/base/bits.h"
 #include "src/wasm/wasm-external-refs.h"
 
 namespace v8 {
@@ -176,6 +177,23 @@ int32_t uint64_mod_wrapper(uint64_t* dst, uint64_t* src) {
   *dst %= *src;
   return 1;
 }
+
+uint32_t word32_ctz_wrapper(uint32_t* input) {
+  return static_cast<uint32_t>(base::bits::CountTrailingZeros32(*input));
+}
+
+uint32_t word64_ctz_wrapper(uint64_t* input) {
+  return static_cast<uint32_t>(base::bits::CountTrailingZeros64(*input));
+}
+
+uint32_t word32_popcnt_wrapper(uint32_t* input) {
+  return static_cast<uint32_t>(base::bits::CountPopulation(*input));
+}
+
+uint32_t word64_popcnt_wrapper(uint64_t* input) {
+  return static_cast<uint32_t>(base::bits::CountPopulation(*input));
+}
+
 }  // namespace wasm
 }  // namespace internal
 }  // namespace v8

src/wasm/wasm-external-refs.h

@@ -50,8 +50,15 @@ int32_t int64_mod_wrapper(int64_t* dst, int64_t* src);
 int32_t uint64_div_wrapper(uint64_t* dst, uint64_t* src);
 
 int32_t uint64_mod_wrapper(uint64_t* dst, uint64_t* src);
+
+uint32_t word32_ctz_wrapper(uint32_t* input);
+
+uint32_t word64_ctz_wrapper(uint64_t* input);
+
+uint32_t word32_popcnt_wrapper(uint32_t* input);
+
+uint32_t word64_popcnt_wrapper(uint64_t* input);
 }  // namespace wasm
 }  // namespace internal
 }  // namespace v8
-
 #endif

test/cctest/compiler/test-run-calls-to-external-references.cc

@@ -48,6 +48,21 @@ void TestExternalReference(BufferedRawMachineAssemblerTester<int32_t>* m,
   CHECK_EQ(comparison_param2, param2);
 }
 
+template <typename R, typename P>
+void TestExternalReference(BufferedRawMachineAssemblerTester<R>* m,
+                           ExternalReference ref, R (*comparison)(P*),
+                           P param) {
+  P comparison_param = param;
+
+  Node* function = m->ExternalConstant(ref);
+  m->Return(m->CallCFunction1(MachineType::Pointer(), MachineType::Pointer(),
+                              function, m->PointerConstant(&param)));
+
+  CHECK_EQ(comparison(&comparison_param), m->Call());
+
+  CHECK_EQ(comparison_param, param);
+}
+
 template <typename R, typename P1, typename P2>
 void TestExternalReference(BufferedRawMachineAssemblerTester<R>* m,
                            ExternalReference ref, R (*comparison)(P1*, P2*),
@@ -201,6 +216,30 @@ TEST(RunCallUint64Mod) {
   TestExternalReference(&m, ref, wasm::uint64_mod_wrapper, uint64_t(1774),
                         uint64_t(21));
 }
+
+TEST(RunCallWord32Ctz) {
+  BufferedRawMachineAssemblerTester<uint32_t> m;
+  ExternalReference ref = ExternalReference::wasm_word32_ctz(m.isolate());
+  TestExternalReference(&m, ref, wasm::word32_ctz_wrapper, uint32_t(1774));
+}
+
+TEST(RunCallWord64Ctz) {
+  BufferedRawMachineAssemblerTester<uint32_t> m;
+  ExternalReference ref = ExternalReference::wasm_word64_ctz(m.isolate());
+  TestExternalReference(&m, ref, wasm::word64_ctz_wrapper, uint64_t(1774));
+}
+
+TEST(RunCallWord32Popcnt) {
+  BufferedRawMachineAssemblerTester<uint32_t> m;
+  ExternalReference ref = ExternalReference::wasm_word32_popcnt(m.isolate());
+  TestExternalReference(&m, ref, wasm::word32_popcnt_wrapper, uint32_t(1774));
+}
+
+TEST(RunCallWord64Popcnt) {
+  BufferedRawMachineAssemblerTester<uint32_t> m;
+  ExternalReference ref = ExternalReference::wasm_word64_popcnt(m.isolate());
+  TestExternalReference(&m, ref, wasm::word64_popcnt_wrapper, uint64_t(1774));
+}
 }  // namespace compiler
 }  // namespace internal
 }  // namespace v8

test/cctest/wasm/test-run-wasm-64.cc

@@ -28,22 +28,10 @@
 #define B1(a) kExprBlock, 1, a
 
 // Can't bridge macro land with nested macros.
-#if V8_TARGET_ARCH_MIPS || V8_TARGET_ARCH_X87
-#define MIPS_OR_X87 true
+#if V8_TARGET_ARCH_MIPS
+#define MIPS true
 #else
-#define MIPS_OR_X87 false
-#endif
-
-#if V8_TARGET_ARCH_X87
-#define X87 true
-#else
-#define X87 false
-#endif
-
-#if V8_TARGET_ARCH_MIPS || V8_TARGET_ARCH_X87 || V8_TARGET_ARCH_ARM
-#define MIPS_OR_ARM_OR_X87 true
-#else
-#define MIPS_OR_ARM_OR_X87 false
+#define MIPS false
 #endif
 
 #define FOREACH_I64_OPERATOR(V) \
@@ -53,9 +41,9 @@
   V(I64Return, true)            \
   V(I64Param, true)             \
   V(I64LoadStore, true)         \
-  V(I64Add, !X87)               \
-  V(I64Sub, !X87)               \
-  V(I64Mul, !MIPS_OR_X87)       \
+  V(I64Add, true)               \
+  V(I64Sub, true)               \
+  V(I64Mul, !MIPS)              \
   V(I64DivS, true)              \
   V(I64DivU, true)              \
   V(I64RemS, true)              \
@@ -63,9 +51,9 @@
   V(I64And, true)               \
   V(I64Ior, true)               \
   V(I64Xor, true)               \
-  V(I64Shl, !X87)               \
-  V(I64ShrU, !X87)              \
-  V(I64ShrS, !X87)              \
+  V(I64Shl, true)               \
+  V(I64ShrU, true)              \
+  V(I64ShrS, true)              \
   V(I64Eq, true)                \
   V(I64Ne, true)                \
   V(I64LtS, true)               \
@@ -78,7 +66,7 @@
   V(I64GeU, true)               \
   V(I64Ctz, true)               \
   V(I64Clz, true)               \
-  V(I64Popcnt, !X87)            \
+  V(I64Popcnt, true)            \
   V(I32ConvertI64, true)        \
   V(I64SConvertF32, true)       \
   V(I64SConvertF64, true)       \