MIPS32: Implement AtomicPair operators through runtime

This CL implements AtomicPair operators: Load, Store,
Add, Sub, Or, Xor, And, Exchange and CompareExchange using
runtime on MIPS32R2 and older. MIPS32R6 includes instructions
for 64-bit atomic access so they are implemented using those.

Change-Id: I1309c1ea4771480516ec5a92f7592533bdcb205c
Reviewed-on: https://chromium-review.googlesource.com/c/1326466Reviewed-by: Sreten Kovacevic <skovacevic@wavecomp.com>
Reviewed-by: Michael Starzinger <mstarzinger@chromium.org>
Commit-Queue: Ivica Bogosavljevic <ibogosavljevic@wavecomp.com>
Cr-Commit-Position: refs/heads/master@{#57496}

src/compiler/backend/mips/instruction-selector-mips.cc

@@ -235,29 +235,29 @@ static void VisitPairAtomicBinop(InstructionSelector* selector, Node* node,
   Node* value = node->InputAt(2);
   Node* value_high = node->InputAt(3);
 
-  InstructionOperand addr_reg = g.TempRegister();
-
-  selector->Emit(kMipsAdd | AddressingModeField::encode(kMode_None), addr_reg,
-                 g.UseRegister(index), g.UseRegister(base));
-
-  InstructionOperand inputs[] = {g.UseRegister(value),
-                                 g.UseRegister(value_high), addr_reg};
-  InstructionOperand temps[] = {g.TempRegister(), g.TempRegister(),
-                                g.TempRegister(), g.TempRegister()};
+  InstructionOperand inputs[] = {g.UseRegister(base), g.UseRegister(index),
+                                 g.UseFixed(value, a1),
+                                 g.UseFixed(value_high, a2)};
   Node* projection0 = NodeProperties::FindProjection(node, 0);
   Node* projection1 = NodeProperties::FindProjection(node, 1);
   if (projection1) {
-    InstructionOperand outputs[] = {g.DefineAsRegister(projection0),
-                                    g.DefineAsRegister(projection1)};
+    InstructionOperand outputs[] = {g.DefineAsFixed(projection0, v0),
+                                    g.DefineAsFixed(projection1, v1)};
+    InstructionOperand temps[] = {g.TempRegister(a0), g.TempRegister(),
+                                  g.TempRegister()};
     selector->Emit(opcode | AddressingModeField::encode(kMode_None),
                    arraysize(outputs), outputs, arraysize(inputs), inputs,
                    arraysize(temps), temps);
   } else if (projection0) {
-    InstructionOperand outputs[] = {g.DefineAsRegister(projection0)};
+    InstructionOperand outputs[] = {g.DefineAsFixed(projection0, v0)};
+    InstructionOperand temps[] = {g.TempRegister(a0), g.TempRegister(v1),
+                                  g.TempRegister()};
     selector->Emit(opcode | AddressingModeField::encode(kMode_None),
                    arraysize(outputs), outputs, arraysize(inputs), inputs,
                    arraysize(temps), temps);
   } else {
+    InstructionOperand temps[] = {g.TempRegister(a0), g.TempRegister(v0),
+                                  g.TempRegister(v1)};
     selector->Emit(opcode | AddressingModeField::encode(kMode_None), 0, nullptr,
                    arraysize(inputs), inputs, arraysize(temps), temps);
   }
@@ -688,24 +688,24 @@ void InstructionSelector::VisitWord32AtomicPairLoad(Node* node) {
   Node* projection0 = NodeProperties::FindProjection(node, 0);
   Node* projection1 = NodeProperties::FindProjection(node, 1);
 
-  InstructionOperand addr_reg = g.TempRegister();
-  Emit(kMipsAdd | AddressingModeField::encode(kMode_None), addr_reg,
-       g.UseRegister(index), g.UseRegister(base));
-  InstructionOperand inputs[] = {addr_reg};
+  InstructionOperand inputs[] = {g.UseRegister(base), g.UseRegister(index)};
 
-  InstructionOperand temps[] = {g.TempRegister()};
   if (projection1) {
-    InstructionOperand outputs[] = {g.DefineAsRegister(projection0),
-                                    g.DefineAsRegister(projection1)};
+    InstructionOperand outputs[] = {g.DefineAsFixed(projection0, v0),
+                                    g.DefineAsFixed(projection1, v1)};
+    InstructionOperand temps[] = {g.TempRegister(a0)};
     Emit(opcode | AddressingModeField::encode(kMode_MRI), arraysize(outputs),
-         outputs, arraysize(inputs), inputs, 1, temps);
+         outputs, arraysize(inputs), inputs, arraysize(temps), temps);
   } else if (projection0) {
-    InstructionOperand outputs[] = {g.DefineAsRegister(projection0)};
+    InstructionOperand outputs[] = {g.DefineAsFixed(projection0, v0)};
+    InstructionOperand temps[] = {g.TempRegister(a0), g.TempRegister(v1)};
     Emit(opcode | AddressingModeField::encode(kMode_MRI), arraysize(outputs),
-         outputs, arraysize(inputs), inputs, 1, temps);
+         outputs, arraysize(inputs), inputs, arraysize(temps), temps);
   } else {
+    InstructionOperand temps[] = {g.TempRegister(a0), g.TempRegister(v0),
+                                  g.TempRegister(v1)};
     Emit(opcode | AddressingModeField::encode(kMode_MRI), 0, nullptr,
-         arraysize(inputs), inputs, 1, temps);
+         arraysize(inputs), inputs, arraysize(temps), temps);
   }
 }
 
@@ -716,13 +716,11 @@ void InstructionSelector::VisitWord32AtomicPairStore(Node* node) {
   Node* value_low = node->InputAt(2);
   Node* value_high = node->InputAt(3);
 
-  InstructionOperand addr_reg = g.TempRegister();
-  Emit(kMipsAdd | AddressingModeField::encode(kMode_None), addr_reg,
-       g.UseRegister(index), g.UseRegister(base));
-
-  InstructionOperand inputs[] = {addr_reg, g.UseRegister(value_low),
-                                 g.UseRegister(value_high)};
-  InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
+  InstructionOperand inputs[] = {g.UseRegister(base), g.UseRegister(index),
+                                 g.UseFixed(value_low, a1),
+                                 g.UseFixed(value_high, a2)};
+  InstructionOperand temps[] = {g.TempRegister(a0), g.TempRegister(),
+                                g.TempRegister()};
   Emit(kMipsWord32AtomicPairStore | AddressingModeField::encode(kMode_MRI), 0,
        nullptr, arraysize(inputs), inputs, arraysize(temps), temps);
 }

src/external-reference.cc

@@ -991,6 +991,95 @@ ExternalReference ExternalReference::fixed_typed_array_base_data_offset() {
       FixedTypedArrayBase::kDataOffset - kHeapObjectTag));
 }
 
+static int64_t atomic_pair_load(intptr_t address) {
+  return std::atomic_load(reinterpret_cast<std::atomic<int64_t>*>(address));
+}
+
+ExternalReference ExternalReference::atomic_pair_load_function() {
+  return ExternalReference(Redirect(FUNCTION_ADDR(atomic_pair_load)));
+}
+
+static void atomic_pair_store(intptr_t address, int value_low, int value_high) {
+  int64_t value =
+      static_cast<int64_t>(value_high) << 32 | (value_low & 0xFFFFFFFF);
+  std::atomic_store(reinterpret_cast<std::atomic<int64_t>*>(address), value);
+}
+
+ExternalReference ExternalReference::atomic_pair_store_function() {
+  return ExternalReference(Redirect(FUNCTION_ADDR(atomic_pair_store)));
+}
+
+static int64_t atomic_pair_add(intptr_t address, int value_low,
+                               int value_high) {
+  int64_t value =
+      static_cast<int64_t>(value_high) << 32 | (value_low & 0xFFFFFFFF);
+  return std::atomic_fetch_add(reinterpret_cast<std::atomic<int64_t>*>(address),
+                               value);
+}
+
+ExternalReference ExternalReference::atomic_pair_add_function() {
+  return ExternalReference(Redirect(FUNCTION_ADDR(atomic_pair_add)));
+}
+
+static int64_t atomic_pair_sub(intptr_t address, int value_low,
+                               int value_high) {
+  int64_t value =
+      static_cast<int64_t>(value_high) << 32 | (value_low & 0xFFFFFFFF);
+  return std::atomic_fetch_sub(reinterpret_cast<std::atomic<int64_t>*>(address),
+                               value);
+}
+
+ExternalReference ExternalReference::atomic_pair_sub_function() {
+  return ExternalReference(Redirect(FUNCTION_ADDR(atomic_pair_sub)));
+}
+
+static int64_t atomic_pair_and(intptr_t address, int value_low,
+                               int value_high) {
+  int64_t value =
+      static_cast<int64_t>(value_high) << 32 | (value_low & 0xFFFFFFFF);
+  return std::atomic_fetch_and(reinterpret_cast<std::atomic<int64_t>*>(address),
+                               value);
+}
+
+ExternalReference ExternalReference::atomic_pair_and_function() {
+  return ExternalReference(Redirect(FUNCTION_ADDR(atomic_pair_and)));
+}
+
+static int64_t atomic_pair_or(intptr_t address, int value_low, int value_high) {
+  int64_t value =
+      static_cast<int64_t>(value_high) << 32 | (value_low & 0xFFFFFFFF);
+  return std::atomic_fetch_or(reinterpret_cast<std::atomic<int64_t>*>(address),
+                              value);
+}
+
+ExternalReference ExternalReference::atomic_pair_or_function() {
+  return ExternalReference(Redirect(FUNCTION_ADDR(atomic_pair_or)));
+}
+
+static int64_t atomic_pair_xor(intptr_t address, int value_low,
+                               int value_high) {
+  int64_t value =
+      static_cast<int64_t>(value_high) << 32 | (value_low & 0xFFFFFFFF);
+  return std::atomic_fetch_xor(reinterpret_cast<std::atomic<int64_t>*>(address),
+                               value);
+}
+
+ExternalReference ExternalReference::atomic_pair_xor_function() {
+  return ExternalReference(Redirect(FUNCTION_ADDR(atomic_pair_xor)));
+}
+
+static int64_t atomic_pair_exchange(intptr_t address, int value_low,
+                                    int value_high) {
+  int64_t value =
+      static_cast<int64_t>(value_high) << 32 | (value_low & 0xFFFFFFFF);
+  return std::atomic_exchange(reinterpret_cast<std::atomic<int64_t>*>(address),
+                              value);
+}
+
+ExternalReference ExternalReference::atomic_pair_exchange_function() {
+  return ExternalReference(Redirect(FUNCTION_ADDR(atomic_pair_exchange)));
+}
+
 static uint64_t atomic_pair_compare_exchange(intptr_t address,
                                              int old_value_low,
                                              int old_value_high,

src/external-reference.h

@@ -177,6 +177,14 @@ class StatsCounter;
   V(wasm_word32_ror, "wasm::word32_ror")                                      \
   V(wasm_word64_ctz, "wasm::word64_ctz")                                      \
   V(wasm_word64_popcnt, "wasm::word64_popcnt")                                \
+  V(atomic_pair_load_function, "atomic_pair_load_function")                   \
+  V(atomic_pair_store_function, "atomic_pair_store_function")                 \
+  V(atomic_pair_add_function, "atomic_pair_add_function")                     \
+  V(atomic_pair_sub_function, "atomic_pair_sub_function")                     \
+  V(atomic_pair_and_function, "atomic_pair_and_function")                     \
+  V(atomic_pair_or_function, "atomic_pair_or_function")                       \
+  V(atomic_pair_xor_function, "atomic_pair_xor_function")                     \
+  V(atomic_pair_exchange_function, "atomic_pair_exchange_function")           \
   V(atomic_pair_compare_exchange_function,                                    \
     "atomic_pair_compare_exchange_function")                                  \
   EXTERNAL_REFERENCE_LIST_INTL(V)

src/mips/assembler-mips.cc

@@ -2358,14 +2358,16 @@ void Assembler::sc(Register rd, const MemOperand& rs) {
   }
 }
 
-void Assembler::llwp(Register rd, Register rt, Register base) {
+void Assembler::llx(Register rd, const MemOperand& rs) {
   DCHECK(IsMipsArchVariant(kMips32r6));
-  GenInstrRegister(SPECIAL3, base, rt, rd, 1, LL_R6);
+  DCHECK(is_int9(rs.offset_));
+  GenInstrImmediate(SPECIAL3, rs.rm(), rd, rs.offset_, 1, LL_R6);
 }
 
-void Assembler::scwp(Register rd, Register rt, Register base) {
+void Assembler::scx(Register rd, const MemOperand& rs) {
   DCHECK(IsMipsArchVariant(kMips32r6));
-  GenInstrRegister(SPECIAL3, base, rt, rd, 1, SC_R6);
+  DCHECK(is_int9(rs.offset_));
+  GenInstrImmediate(SPECIAL3, rs.rm(), rd, rs.offset_, 1, SC_R6);
 }
 
 void Assembler::lui(Register rd, int32_t j) {

src/mips/assembler-mips.h

@@ -890,8 +890,8 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
 
   void ll(Register rd, const MemOperand& rs);
   void sc(Register rd, const MemOperand& rs);
-  void llwp(Register rd, Register rt, Register base);
-  void scwp(Register rd, Register rt, Register base);
+  void llx(Register rd, const MemOperand& rs);
+  void scx(Register rd, const MemOperand& rs);
 
   // ---------PC-Relative-instructions-----------
 

src/mips/disasm-mips.cc

@@ -1686,7 +1686,11 @@ void Decoder::DecodeTypeImmediateSPECIAL3(Instruction* instr) {
   switch (instr->FunctionFieldRaw()) {
     case LL_R6: {
       if (IsMipsArchVariant(kMips32r6)) {
-        Format(instr, "ll     'rt, 'imm9s('rs)");
+        if (instr->Bit(6)) {
+          Format(instr, "llx     'rt, 'imm9s('rs)");
+        } else {
+          Format(instr, "ll      'rt, 'imm9s('rs)");
+        }
       } else {
         Unknown(instr);
       }
@@ -1694,7 +1698,11 @@ void Decoder::DecodeTypeImmediateSPECIAL3(Instruction* instr) {
     }
     case SC_R6: {
       if (IsMipsArchVariant(kMips32r6)) {
-        Format(instr, "sc     'rt, 'imm9s('rs)");
+        if (instr->Bit(6)) {
+          Format(instr, "scx     'rt, 'imm9s('rs)");
+        } else {
+          Format(instr, "sc      'rt, 'imm9s('rs)");
+        }
       } else {
         Unknown(instr);
       }
@@ -1962,14 +1970,14 @@ void Decoder::DecodeTypeImmediate(Instruction* instr) {
       if (IsMipsArchVariant(kMips32r6)) {
         Unknown(instr);
       } else {
-        Format(instr, "ll     'rt, 'imm16s('rs)");
+        Format(instr, "ll      'rt, 'imm16s('rs)");
       }
       break;
     case SC:
       if (IsMipsArchVariant(kMips32r6)) {
         Unknown(instr);
       } else {
-        Format(instr, "sc     'rt, 'imm16s('rs)");
+        Format(instr, "sc      'rt, 'imm16s('rs)");
       }
       break;
     case LWC1:

src/mips/simulator-mips.cc

@@ -4240,21 +4240,6 @@ void Simulator::DecodeTypeRegisterSPECIAL3() {
       SetResult(rd_reg(), alu_out);
       break;
     }
-    case LL_R6: {
-      // LLWP/SCWP sequence cannot be simulated properly
-      DCHECK(IsMipsArchVariant(kMips32r6));
-      set_register(rd_reg(), ReadW(rs() + 4, instr_.instr()));
-      set_register(rt(), ReadW(rs(), instr_.instr()));
-      break;
-    }
-    case SC_R6: {
-      // LLWP/SCWP sequence cannot be simulated properly
-      DCHECK(IsMipsArchVariant(kMips32r6));
-      WriteW(rs() + 4, rd_reg(), instr_.instr());
-      WriteW(rs(), rt(), instr_.instr());
-      set_register(rt(), 1);
-      break;
-    }
     default:
       UNREACHABLE();
   }
@@ -6833,8 +6818,12 @@ void Simulator::DecodeTypeImmediate() {
           DCHECK(IsMipsArchVariant(kMips32r6));
           int32_t base = get_register(instr_.BaseValue());
           int32_t offset9 = instr_.Imm9Value();
+          int32_t bit6 = instr_.Bit(6);
           WriteW(base + offset9, rt, instr_.instr());
-          set_register(rt_reg, 1);
+          // Only SC (and not SCX) instruction modifies rt_reg
+          if (bit6 == 0) {
+            set_register(rt_reg, 1);
+          }
           break;
         }
         default:

test/cctest/cctest.status

@@ -356,7 +356,7 @@
 }],  # '(arch == mipsel or arch == mips64el or arch == mips or arch == mips64) and not simd_mips'
 
 ##############################################################################
-['arch == mipsel or arch == mips64el or arch == mips or arch == mips64', {
+['arch == mips or arch == mips64', {
   # TODO(mips-team): Implement I64Atomic operations on MIPS
   'test-run-wasm-atomics64/*': [SKIP],
 }],  # 'arch == mipsel or arch == mips64el or arch == mips or arch == mips64'

test/cctest/test-disasm-mips.cc

@@ -1123,11 +1123,11 @@ TEST(madd_msub_maddf_msubf) {
 TEST(atomic_load_store) {
   SET_UP();
   if (IsMipsArchVariant(kMips32r6)) {
-    COMPARE(ll(v0, MemOperand(v1, -1)), "7c62ffb6       ll     v0, -1(v1)");
-    COMPARE(sc(v0, MemOperand(v1, 1)), "7c6200a6       sc     v0, 1(v1)");
+    COMPARE(ll(v0, MemOperand(v1, -1)), "7c62ffb6       ll      v0, -1(v1)");
+    COMPARE(sc(v0, MemOperand(v1, 1)), "7c6200a6       sc      v0, 1(v1)");
   } else {
-    COMPARE(ll(v0, MemOperand(v1, -1)), "c062ffff       ll     v0, -1(v1)");
-    COMPARE(sc(v0, MemOperand(v1, 1)), "e0620001       sc     v0, 1(v1)");
+    COMPARE(ll(v0, MemOperand(v1, -1)), "c062ffff       ll      v0, -1(v1)");
+    COMPARE(sc(v0, MemOperand(v1, 1)), "e0620001       sc      v0, 1(v1)");
   }
   VERIFY_RUN();
 }