[mips] Implement AtomicPair operations on MIPS32R6

AtomicPair operations are only available with some instructions
introduced in version R6. Add support for needed instructions.

Change-Id: I808d6ed5b5efafd638846ec599941ebc71d90e23
Reviewed-on: https://chromium-review.googlesource.com/c/1251526Reviewed-by: Ivica Bogosavljevic <ibogosavljevic@wavecomp.com>
Reviewed-by: Ben Titzer <titzer@chromium.org>
Commit-Queue: Sreten Kovacevic <skovacevic@wavecomp.com>
Cr-Commit-Position: refs/heads/master@{#56379}

src/builtins/builtins-sharedarraybuffer-gen.cc

@@ -215,7 +215,7 @@ TF_BUILTIN(AtomicsLoad, SharedArrayBufferBuiltinsAssembler) {
   BIND(&u32);
   Return(ChangeUint32ToTagged(AtomicLoad(MachineType::Uint32(), backing_store,
                                          WordShl(index_word, 2))));
-#if V8_TARGET_ARCH_MIPS
+#if V8_TARGET_ARCH_MIPS && !_MIPS_ARCH_MIPS32R6
   BIND(&i64);
   Return(CallRuntime(Runtime::kAtomicsLoad64, context, array, index_integer));
 
@@ -292,7 +292,7 @@ TF_BUILTIN(AtomicsStore, SharedArrayBufferBuiltinsAssembler) {
   Return(value_integer);
 
   BIND(&u64);
-#if V8_TARGET_ARCH_MIPS
+#if V8_TARGET_ARCH_MIPS && !_MIPS_ARCH_MIPS32R6
   Return(CallRuntime(Runtime::kAtomicsStore64, context, array, index_integer,
                      value));
 #else

src/compiler/instruction-selector.cc

@@ -2389,7 +2389,7 @@ void InstructionSelector::VisitWord32PairShr(Node* node) { UNIMPLEMENTED(); }
 void InstructionSelector::VisitWord32PairSar(Node* node) { UNIMPLEMENTED(); }
 #endif  // V8_TARGET_ARCH_64_BIT
 
-#if !V8_TARGET_ARCH_IA32 && !V8_TARGET_ARCH_ARM
+#if !V8_TARGET_ARCH_IA32 && !V8_TARGET_ARCH_ARM && !V8_TARGET_ARCH_MIPS
 void InstructionSelector::VisitWord32AtomicPairLoad(Node* node) {
   UNIMPLEMENTED();
 }
@@ -2425,7 +2425,7 @@ void InstructionSelector::VisitWord32AtomicPairExchange(Node* node) {
 void InstructionSelector::VisitWord32AtomicPairCompareExchange(Node* node) {
   UNIMPLEMENTED();
 }
-#endif  // !V8_TARGET_ARCH_IA32 && !V8_TARGET_ARCH_ARM
+#endif  // !V8_TARGET_ARCH_IA32 && !V8_TARGET_ARCH_ARM && !V8_TARGET_ARCH_MIPS
 
 #if !V8_TARGET_ARCH_X64 && !V8_TARGET_ARCH_ARM64 && !V8_TARGET_ARCH_MIPS64 && \
     !V8_TARGET_ARCH_S390 && !V8_TARGET_ARCH_PPC

src/compiler/mips/code-generator-mips.cc

@@ -356,6 +356,41 @@ void EmitWordLoadPoisoningIfNeeded(CodeGenerator* codegen,
     __ sync();                                                          \
   } while (0)
 
+#define ASSEMBLE_ATOMIC64_LOGIC_BINOP(bin_instr)                               \
+  do {                                                                         \
+    if (IsMipsArchVariant(kMips32r6)) {                                        \
+      Label binop;                                                             \
+      __ sync();                                                               \
+      __ bind(&binop);                                                         \
+      __ llwp(i.TempRegister(0), i.TempRegister(1), i.InputRegister(2));       \
+      __ bin_instr(i.TempRegister(0), i.TempRegister(1), i.TempRegister(0),    \
+                   i.TempRegister(1), i.InputRegister(0), i.InputRegister(1)); \
+      __ scwp(i.TempRegister(0), i.TempRegister(1), i.InputRegister(2));       \
+      __ BranchShort(&binop, eq, i.TempRegister(1), Operand(zero_reg));        \
+      __ sync();                                                               \
+    } else {                                                                   \
+      UNREACHABLE();                                                           \
+    }                                                                          \
+  } while (0)
+
+#define ASSEMBLE_ATOMIC64_ARITH_BINOP(bin_instr)                              \
+  do {                                                                        \
+    if (IsMipsArchVariant(kMips32r6)) {                                       \
+      Label binop;                                                            \
+      __ sync();                                                              \
+      __ bind(&binop);                                                        \
+      __ llwp(i.TempRegister(0), i.TempRegister(1), i.InputRegister(2));      \
+      __ bin_instr(i.TempRegister(0), i.TempRegister(1), i.TempRegister(0),   \
+                   i.TempRegister(1), i.InputRegister(0), i.InputRegister(1), \
+                   i.TempRegister(2), i.TempRegister(3));                     \
+      __ scwp(i.TempRegister(0), i.TempRegister(1), i.InputRegister(2));      \
+      __ BranchShort(&binop, eq, i.TempRegister(1), Operand(zero_reg));       \
+      __ sync();                                                              \
+    } else {                                                                  \
+      UNREACHABLE();                                                          \
+    }                                                                         \
+  } while (0)
+
 #define ASSEMBLE_ATOMIC_BINOP_EXT(sign_extend, size, bin_instr)                \
   do {                                                                         \
     Label binop;                                                               \
@@ -1704,6 +1739,50 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
       ATOMIC_BINOP_CASE(Or, Or)
       ATOMIC_BINOP_CASE(Xor, Xor)
 #undef ATOMIC_BINOP_CASE
+    case kMipsWord32AtomicPairLoad: {
+      if (IsMipsArchVariant(kMips32r6)) {
+        Register second_output =
+            instr->OutputCount() == 2 ? i.OutputRegister(1) : i.TempRegister(0);
+        __ llwp(i.OutputRegister(0), second_output, i.InputRegister(0));
+        __ sync();
+      } else {
+        UNREACHABLE();
+      }
+      break;
+    }
+    case kMipsWord32AtomicPairStore: {
+      if (IsMipsArchVariant(kMips32r6)) {
+        Label store;
+        __ sync();
+        __ bind(&store);
+        __ llwp(i.TempRegister(0), i.TempRegister(1), i.InputRegister(0));
+        __ Move(i.TempRegister(0), i.InputRegister(2));
+        __ scwp(i.InputRegister(1), i.TempRegister(0), i.InputRegister(0));
+        __ BranchShort(&store, eq, i.TempRegister(0), Operand(zero_reg));
+        __ sync();
+      } else {
+        UNREACHABLE();
+      }
+      break;
+    }
+#define ATOMIC64_BINOP_ARITH_CASE(op, instr) \
+  case kMipsWord32AtomicPair##op:            \
+    ASSEMBLE_ATOMIC64_ARITH_BINOP(instr);    \
+    break;
+      ATOMIC64_BINOP_ARITH_CASE(Add, AddPair)
+      ATOMIC64_BINOP_ARITH_CASE(Sub, SubPair)
+#undef ATOMIC64_BINOP_ARITH_CASE
+#define ATOMIC64_BINOP_LOGIC_CASE(op, instr) \
+  case kMipsWord32AtomicPair##op:            \
+    ASSEMBLE_ATOMIC64_LOGIC_BINOP(instr);    \
+    break;
+      ATOMIC64_BINOP_LOGIC_CASE(And, AndPair)
+      ATOMIC64_BINOP_LOGIC_CASE(Or, OrPair)
+      ATOMIC64_BINOP_LOGIC_CASE(Xor, XorPair)
+#undef ATOMIC64_BINOP_LOGIC_CASE
+    case kMipsWord32AtomicPairExchange:
+    case kMipsWord32AtomicPairCompareExchange:
+      break;
     case kMipsS128Zero: {
       CpuFeatureScope msa_scope(tasm(), MIPS_SIMD);
       __ xor_v(i.OutputSimd128Register(), i.OutputSimd128Register(),

src/compiler/mips/instruction-codes-mips.h

@@ -276,7 +276,16 @@ namespace compiler {
   V(MipsI16x8UConvertI8x16Low)     \
   V(MipsI16x8UConvertI8x16High)    \
   V(MipsI8x16SConvertI16x8)        \
-  V(MipsI8x16UConvertI16x8)
+  V(MipsI8x16UConvertI16x8)        \
+  V(MipsWord32AtomicPairLoad)      \
+  V(MipsWord32AtomicPairStore)     \
+  V(MipsWord32AtomicPairAdd)       \
+  V(MipsWord32AtomicPairSub)       \
+  V(MipsWord32AtomicPairAnd)       \
+  V(MipsWord32AtomicPairOr)        \
+  V(MipsWord32AtomicPairXor)       \
+  V(MipsWord32AtomicPairExchange)  \
+  V(MipsWord32AtomicPairCompareExchange)
 
 // Addressing modes represent the "shape" of inputs to an instruction.
 // Many instructions support multiple addressing modes. Addressing modes

src/compiler/mips/instruction-scheduler-mips.cc

@@ -266,6 +266,7 @@ int InstructionScheduler::GetTargetInstructionFlags(
     case kMipsUlhu:
     case kMipsUlw:
     case kMipsUlwc1:
+    case kMipsWord32AtomicPairLoad:
       return kIsLoadOperation;
 
     case kMipsModD:
@@ -283,6 +284,14 @@ int InstructionScheduler::GetTargetInstructionFlags(
     case kMipsUsh:
     case kMipsUsw:
     case kMipsUswc1:
+    case kMipsWord32AtomicPairStore:
+    case kMipsWord32AtomicPairAdd:
+    case kMipsWord32AtomicPairSub:
+    case kMipsWord32AtomicPairAnd:
+    case kMipsWord32AtomicPairOr:
+    case kMipsWord32AtomicPairXor:
+    case kMipsWord32AtomicPairExchange:
+    case kMipsWord32AtomicPairCompareExchange:
       return kHasSideEffect;
 
 #define CASE(Name) case k##Name:

src/compiler/mips/instruction-selector-mips.cc

@@ -229,6 +229,42 @@ static void VisitBinop(InstructionSelector* selector, Node* node,
   VisitBinop(selector, node, opcode, false, kArchNop);
 }
 
+static void VisitPairAtomicBinop(InstructionSelector* selector, Node* node,
+                                 ArchOpcode opcode) {
+  MipsOperandGenerator g(selector);
+  Node* base = node->InputAt(0);
+  Node* index = node->InputAt(1);
+  Node* value = node->InputAt(2);
+  Node* value_high = node->InputAt(3);
+
+  InstructionOperand addr_reg = g.TempRegister();
+
+  selector->Emit(kMipsAdd | AddressingModeField::encode(kMode_None), addr_reg,
+                 g.UseRegister(index), g.UseRegister(base));
+
+  InstructionOperand inputs[] = {g.UseRegister(value),
+                                 g.UseRegister(value_high), addr_reg};
+  InstructionOperand temps[] = {g.TempRegister(), g.TempRegister(),
+                                g.TempRegister(), g.TempRegister()};
+  Node* projection0 = NodeProperties::FindProjection(node, 0);
+  Node* projection1 = NodeProperties::FindProjection(node, 1);
+  if (projection1) {
+    InstructionOperand outputs[] = {g.DefineAsRegister(projection0),
+                                    g.DefineAsRegister(projection1)};
+    selector->Emit(opcode | AddressingModeField::encode(kMode_None),
+                   arraysize(outputs), outputs, arraysize(inputs), inputs,
+                   arraysize(temps), temps);
+  } else if (projection0) {
+    InstructionOperand outputs[] = {g.DefineAsRegister(projection0)};
+    selector->Emit(opcode | AddressingModeField::encode(kMode_None),
+                   arraysize(outputs), outputs, arraysize(inputs), inputs,
+                   arraysize(temps), temps);
+  } else {
+    selector->Emit(opcode | AddressingModeField::encode(kMode_None), 0, nullptr,
+                   arraysize(inputs), inputs, arraysize(temps), temps);
+  }
+}
+
 void InstructionSelector::VisitStackSlot(Node* node) {
   StackSlotRepresentation rep = StackSlotRepresentationOf(node->op());
   int alignment = rep.alignment();
@@ -651,6 +687,81 @@ void InstructionSelector::VisitWord32Clz(Node* node) {
   VisitRR(this, kMipsClz, node);
 }
 
+void InstructionSelector::VisitWord32AtomicPairLoad(Node* node) {
+  MipsOperandGenerator g(this);
+  Node* base = node->InputAt(0);
+  Node* index = node->InputAt(1);
+  ArchOpcode opcode = kMipsWord32AtomicPairLoad;
+
+  Node* projection0 = NodeProperties::FindProjection(node, 0);
+  Node* projection1 = NodeProperties::FindProjection(node, 1);
+
+  InstructionOperand addr_reg = g.TempRegister();
+  Emit(kMipsAdd | AddressingModeField::encode(kMode_None), addr_reg,
+       g.UseRegister(index), g.UseRegister(base));
+  InstructionOperand inputs[] = {addr_reg};
+
+  InstructionOperand temps[] = {g.TempRegister()};
+  if (projection1) {
+    InstructionOperand outputs[] = {g.DefineAsRegister(projection0),
+                                    g.DefineAsRegister(projection1)};
+    Emit(opcode | AddressingModeField::encode(kMode_MRI), arraysize(outputs),
+         outputs, arraysize(inputs), inputs, 1, temps);
+  } else if (projection0) {
+    InstructionOperand outputs[] = {g.DefineAsRegister(projection0)};
+    Emit(opcode | AddressingModeField::encode(kMode_MRI), arraysize(outputs),
+         outputs, arraysize(inputs), inputs, 1, temps);
+  } else {
+    Emit(opcode | AddressingModeField::encode(kMode_MRI), 0, nullptr,
+         arraysize(inputs), inputs, 1, temps);
+  }
+}
+
+void InstructionSelector::VisitWord32AtomicPairStore(Node* node) {
+  MipsOperandGenerator g(this);
+  Node* base = node->InputAt(0);
+  Node* index = node->InputAt(1);
+  Node* value_low = node->InputAt(2);
+  Node* value_high = node->InputAt(3);
+
+  InstructionOperand addr_reg = g.TempRegister();
+  Emit(kMipsAdd | AddressingModeField::encode(kMode_None), addr_reg,
+       g.UseRegister(index), g.UseRegister(base));
+
+  InstructionOperand inputs[] = {addr_reg, g.UseRegister(value_low),
+                                 g.UseRegister(value_high)};
+  InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
+  Emit(kMipsWord32AtomicPairStore | AddressingModeField::encode(kMode_MRI), 0,
+       nullptr, arraysize(inputs), inputs, arraysize(temps), temps);
+}
+
+void InstructionSelector::VisitWord32AtomicPairAdd(Node* node) {
+  VisitPairAtomicBinop(this, node, kMipsWord32AtomicPairAdd);
+}
+
+void InstructionSelector::VisitWord32AtomicPairSub(Node* node) {
+  VisitPairAtomicBinop(this, node, kMipsWord32AtomicPairSub);
+}
+
+void InstructionSelector::VisitWord32AtomicPairAnd(Node* node) {
+  VisitPairAtomicBinop(this, node, kMipsWord32AtomicPairAnd);
+}
+
+void InstructionSelector::VisitWord32AtomicPairOr(Node* node) {
+  VisitPairAtomicBinop(this, node, kMipsWord32AtomicPairOr);
+}
+
+void InstructionSelector::VisitWord32AtomicPairXor(Node* node) {
+  VisitPairAtomicBinop(this, node, kMipsWord32AtomicPairXor);
+}
+
+void InstructionSelector::VisitWord32AtomicPairExchange(Node* node) {
+  VisitPairAtomicBinop(this, node, kMipsWord32AtomicPairExchange);
+}
+
+void InstructionSelector::VisitWord32AtomicPairCompareExchange(Node* node) {
+  VisitPairAtomicBinop(this, node, kMipsWord32AtomicPairCompareExchange);
+}
 
 void InstructionSelector::VisitWord32ReverseBits(Node* node) { UNREACHABLE(); }
 

src/mips/assembler-mips.cc

@@ -2335,6 +2335,16 @@ void Assembler::sc(Register rd, const MemOperand& rs) {
   }
 }
 
+void Assembler::llwp(Register rd, Register rt, Register base) {
+  DCHECK(IsMipsArchVariant(kMips32r6));
+  GenInstrRegister(SPECIAL3, base, rt, rd, 1, LL_R6);
+}
+
+void Assembler::scwp(Register rd, Register rt, Register base) {
+  DCHECK(IsMipsArchVariant(kMips32r6));
+  GenInstrRegister(SPECIAL3, base, rt, rd, 1, SC_R6);
+}
+
 void Assembler::lui(Register rd, int32_t j) {
   DCHECK(is_uint16(j) || is_int16(j));
   GenInstrImmediate(LUI, zero_reg, rd, j);

src/mips/assembler-mips.h

@@ -883,6 +883,8 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
 
   void ll(Register rd, const MemOperand& rs);
   void sc(Register rd, const MemOperand& rs);
+  void llwp(Register rd, Register rt, Register base);
+  void scwp(Register rd, Register rt, Register base);
 
   // ---------PC-Relative-instructions-----------
 

src/mips/disasm-mips.cc

@@ -1546,6 +1546,16 @@ void Decoder::DecodeTypeRegisterSPECIAL3(Instruction* instr) {
           }
           break;
         }
+        case LL_R6: {
+          DCHECK(IsMipsArchVariant(kMips32r6));
+          Format(instr, "llwp    'rd, 'rt, 0('rs)");
+          break;
+        }
+        case SC_R6: {
+          DCHECK(IsMipsArchVariant(kMips32r6));
+          Format(instr, "scwp    'rd, 'rt, 0('rs)");
+          break;
+        }
         default: {
           sa >>= kBp2Bits;
           switch (sa) {

src/mips/macro-assembler-mips.cc

@@ -1463,6 +1463,26 @@ void TurboAssembler::SubPair(Register dst_low, Register dst_high,
   Move(dst_low, scratch1);
 }
 
+void TurboAssembler::AndPair(Register dst_low, Register dst_high,
+                             Register left_low, Register left_high,
+                             Register right_low, Register right_high) {
+  And(dst_low, left_low, right_low);
+  And(dst_high, left_high, right_high);
+}
+
+void TurboAssembler::OrPair(Register dst_low, Register dst_high,
+                            Register left_low, Register left_high,
+                            Register right_low, Register right_high) {
+  Or(dst_low, left_low, right_low);
+  Or(dst_high, left_high, right_high);
+}
+void TurboAssembler::XorPair(Register dst_low, Register dst_high,
+                             Register left_low, Register left_high,
+                             Register right_low, Register right_high) {
+  Xor(dst_low, left_low, right_low);
+  Xor(dst_high, left_high, right_high);
+}
+
 void TurboAssembler::MulPair(Register dst_low, Register dst_high,
                              Register left_low, Register left_high,
                              Register right_low, Register right_high,

src/mips/macro-assembler-mips.h

@@ -582,6 +582,15 @@ class V8_EXPORT_PRIVATE TurboAssembler : public TurboAssemblerBase {
                Register left_high, Register right_low, Register right_high,
                Register scratch1, Register scratch2);
 
+  void AndPair(Register dst_low, Register dst_high, Register left_low,
+               Register left_high, Register right_low, Register right_high);
+
+  void OrPair(Register dst_low, Register dst_high, Register left_low,
+              Register left_high, Register right_low, Register right_high);
+
+  void XorPair(Register dst_low, Register dst_high, Register left_low,
+               Register left_high, Register right_low, Register right_high);
+
   void MulPair(Register dst_low, Register dst_high, Register left_low,
                Register left_high, Register right_low, Register right_high,
                Register scratch1, Register scratch2);

src/mips/simulator-mips.cc

@@ -4240,6 +4240,21 @@ void Simulator::DecodeTypeRegisterSPECIAL3() {
       SetResult(rd_reg(), alu_out);
       break;
     }
+    case LL_R6: {
+      // LLWP/SCWP sequence cannot be simulated properly
+      DCHECK(IsMipsArchVariant(kMips32r6));
+      set_register(rd_reg(), ReadW(rs() + 4, instr_.instr()));
+      set_register(rt(), ReadW(rs(), instr_.instr()));
+      break;
+    }
+    case SC_R6: {
+      // LLWP/SCWP sequence cannot be simulated properly
+      DCHECK(IsMipsArchVariant(kMips32r6));
+      WriteW(rs() + 4, rd_reg(), instr_.instr());
+      WriteW(rs(), rt(), instr_.instr());
+      set_register(rt(), 1);
+      break;
+    }
     default:
       UNREACHABLE();
   }