Add I64Atomic Load/Store ops for ia32

Bug: v8:6532
Change-Id: I6391c3d5e86d2b04735e241a1e0549a170ab4852
Reviewed-on: https://chromium-review.googlesource.com/1164640Reviewed-by: Ben Titzer <titzer@chromium.org>
Reviewed-by: Ben Smith <binji@chromium.org>
Commit-Queue: Deepti Gandluri <gdeepti@chromium.org>
Cr-Commit-Position: refs/heads/master@{#55027}

src/compiler/ia32/code-generator-ia32.cc

@@ -3632,6 +3632,20 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
       __ cmp(esp, Operand::StaticVariable(stack_limit));
       break;
     }
+    case kIA32Word32AtomicPairLoad: {
+      XMMRegister tmp = i.ToDoubleRegister(instr->TempAt(0));
+      __ movq(tmp, i.MemoryOperand());
+      __ Pextrd(i.OutputRegister(0), tmp, 0);
+      __ Pextrd(i.OutputRegister(1), tmp, 1);
+      break;
+    }
+    case kIA32Word32AtomicPairStore: {
+      __ mov(i.TempRegister(0), i.MemoryOperand(2));
+      __ mov(i.TempRegister(1), i.NextMemoryOperand(2));
+      __ lock();
+      __ cmpxchg8b(i.MemoryOperand(2));
+      break;
+    }
     case kWord32AtomicExchangeInt8: {
       __ xchg_b(i.InputRegister(0), i.MemoryOperand(1));
       __ movsx_b(i.InputRegister(0), i.InputRegister(0));

src/compiler/ia32/instruction-codes-ia32.h

@@ -351,6 +351,8 @@ namespace compiler {
   V(IA32S1x8AllTrue)                             \
   V(IA32S1x16AnyTrue)                            \
   V(IA32S1x16AllTrue)                            \
+  V(IA32Word32AtomicPairLoad)                    \
+  V(IA32Word32AtomicPairStore)                   \
   V(IA32Word32AtomicPairAdd)                     \
   V(IA32Word32AtomicPairSub)                     \
   V(IA32Word32AtomicPairAnd)                     \

src/compiler/ia32/instruction-scheduler-ia32.cc

@@ -368,6 +368,10 @@ int InstructionScheduler::GetTargetInstructionFlags(
     case kLFence:
       return kHasSideEffect;
 
+    case kIA32Word32AtomicPairLoad:
+      return kIsLoadOperation;
+
+    case kIA32Word32AtomicPairStore:
     case kIA32Word32AtomicPairAdd:
     case kIA32Word32AtomicPairSub:
     case kIA32Word32AtomicPairAnd:

src/compiler/ia32/instruction-selector-ia32.cc

@@ -1759,6 +1759,44 @@ VISIT_ATOMIC_BINOP(Or)
 VISIT_ATOMIC_BINOP(Xor)
 #undef VISIT_ATOMIC_BINOP
 
+void InstructionSelector::VisitWord32AtomicPairLoad(Node* node) {
+  IA32OperandGenerator g(this);
+  AddressingMode mode;
+  Node* base = node->InputAt(0);
+  Node* index = node->InputAt(1);
+  InstructionOperand inputs[] = {g.UseUniqueRegister(base),
+                                 g.GetEffectiveIndexOperand(index, &mode)};
+  InstructionOperand temps[] = {g.TempDoubleRegister()};
+  InstructionOperand outputs[] = {
+      g.DefineAsRegister(NodeProperties::FindProjection(node, 0)),
+      g.DefineAsRegister(NodeProperties::FindProjection(node, 1))};
+  InstructionCode code =
+      kIA32Word32AtomicPairLoad | AddressingModeField::encode(mode);
+  Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
+       arraysize(temps), temps);
+}
+
+void InstructionSelector::VisitWord32AtomicPairStore(Node* node) {
+  IA32OperandGenerator g(this);
+  Node* base = node->InputAt(0);
+  Node* index = node->InputAt(1);
+  Node* value = node->InputAt(2);
+  Node* value_high = node->InputAt(3);
+
+  AddressingMode addressing_mode;
+  InstructionOperand inputs[] = {
+      g.UseFixed(value, ebx), g.UseFixed(value_high, ecx),
+      g.UseUniqueRegister(base),
+      g.GetEffectiveIndexOperand(index, &addressing_mode)};
+  // Allocating temp registers here as stores are performed using an atomic
+  // exchange, the output of which is stored in edx:eax, which should be saved
+  // and restored at the end of the instruction.
+  InstructionOperand temps[] = {g.TempRegister(eax), g.TempRegister(edx)};
+  InstructionCode code =
+      kIA32Word32AtomicPairStore | AddressingModeField::encode(addressing_mode);
+  Emit(code, 0, nullptr, arraysize(inputs), inputs, arraysize(temps), temps);
+}
+
 void InstructionSelector::VisitWord32AtomicPairAdd(Node* node) {
   VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairAdd);
 }

src/compiler/instruction-selector.cc

@@ -1705,6 +1705,13 @@ void InstructionSelector::VisitNode(Node* node) {
       return VisitWord32AtomicStore(node);
     case IrOpcode::kWord64AtomicStore:
       return VisitWord64AtomicStore(node);
+    case IrOpcode::kWord32AtomicPairStore:
+      return VisitWord32AtomicPairStore(node);
+    case IrOpcode::kWord32AtomicPairLoad: {
+      MarkAsWord32(node);
+      MarkPairProjectionsAsWord32(node);
+      return VisitWord32AtomicPairLoad(node);
+    }
 #define ATOMIC_CASE(name, rep)                         \
   case IrOpcode::k##rep##Atomic##name: {               \
     MachineType type = AtomicOpType(node->op());       \
@@ -2383,6 +2390,14 @@ void InstructionSelector::VisitWord32PairSar(Node* node) { UNIMPLEMENTED(); }
 #endif  // V8_TARGET_ARCH_64_BIT
 
 #if !V8_TARGET_ARCH_IA32
+void InstructionSelector::VisitWord32AtomicPairLoad(Node* node) {
+  UNIMPLEMENTED();
+}
+
+void InstructionSelector::VisitWord32AtomicPairStore(Node* node) {
+  UNIMPLEMENTED();
+}
+
 void InstructionSelector::VisitWord32AtomicPairAdd(Node* node) {
   UNIMPLEMENTED();
 }

src/compiler/int64-lowering.cc

@@ -882,6 +882,32 @@ void Int64Lowering::LowerNode(Node* node) {
       node->NullAllInputs();
       break;
     }
+    case IrOpcode::kWord64AtomicLoad: {
+      DCHECK_EQ(4, node->InputCount());
+      MachineType type = AtomicOpType(node->op());
+      if (type == MachineType::Uint64()) {
+        NodeProperties::ChangeOp(node, machine()->Word32AtomicPairLoad());
+        ReplaceNodeWithProjections(node);
+      } else {
+        NodeProperties::ChangeOp(node, machine()->Word32AtomicLoad(type));
+        ReplaceNode(node, node, graph()->NewNode(common()->Int32Constant(0)));
+      }
+      break;
+    }
+    case IrOpcode::kWord64AtomicStore: {
+      DCHECK_EQ(5, node->InputCount());
+      MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
+      if (rep == MachineRepresentation::kWord64) {
+        Node* value = node->InputAt(2);
+        node->ReplaceInput(2, GetReplacementLow(value));
+        node->InsertInput(zone(), 3, GetReplacementHigh(value));
+        NodeProperties::ChangeOp(node, machine()->Word32AtomicPairStore());
+      } else {
+        DefaultLowering(node, true);
+        NodeProperties::ChangeOp(node, machine()->Word32AtomicStore(rep));
+      }
+      break;
+    }
 #define ATOMIC_CASE(name)                                                   \
   case IrOpcode::kWord64Atomic##name: {                                     \
     MachineType type = AtomicOpType(node->op());                            \

src/compiler/machine-operator.cc

@@ -39,6 +39,7 @@ LoadRepresentation LoadRepresentationOf(Operator const* op) {
          IrOpcode::kProtectedLoad == op->opcode() ||
          IrOpcode::kWord32AtomicLoad == op->opcode() ||
          IrOpcode::kWord64AtomicLoad == op->opcode() ||
+         IrOpcode::kWord32AtomicPairLoad == op->opcode() ||
          IrOpcode::kPoisonedLoad == op->opcode() ||
          IrOpcode::kUnalignedLoad == op->opcode());
   return OpParameter<LoadRepresentation>(op);
@@ -80,7 +81,8 @@ StackSlotRepresentation const& StackSlotRepresentationOf(Operator const* op) {
 
 MachineRepresentation AtomicStoreRepresentationOf(Operator const* op) {
   DCHECK(IrOpcode::kWord32AtomicStore == op->opcode() ||
-         IrOpcode::kWord64AtomicStore == op->opcode());
+         IrOpcode::kWord64AtomicStore == op->opcode() ||
+         IrOpcode::kWord32AtomicPairStore == op->opcode());
   return OpParameter<MachineRepresentation>(op);
 }
 
@@ -688,6 +690,22 @@ struct MachineOperatorGlobalCache {
   ATOMIC_U64_TYPE_LIST(ATOMIC_COMPARE_EXCHANGE)
 #undef ATOMIC_COMPARE_EXCHANGE
 
+  struct Word32AtomicPairLoadOperator : public Operator {
+    Word32AtomicPairLoadOperator()
+        : Operator(IrOpcode::kWord32AtomicPairLoad,
+                   Operator::kNoDeopt | Operator::kNoThrow,
+                   "Word32AtomicPairLoad", 2, 1, 1, 2, 1, 0) {}
+  };
+  Word32AtomicPairLoadOperator kWord32AtomicPairLoad;
+
+  struct Word32AtomicPairStoreOperator : public Operator {
+    Word32AtomicPairStoreOperator()
+        : Operator(IrOpcode::kWord32AtomicPairStore,
+                   Operator::kNoDeopt | Operator::kNoThrow,
+                   "Word32AtomicPairStore", 4, 1, 1, 0, 1, 0) {}
+  };
+  Word32AtomicPairStoreOperator kWord32AtomicPairStore;
+
 #define ATOMIC_PAIR_OP(op)                                      \
   struct Word32AtomicPair##op##Operator : public Operator {     \
     Word32AtomicPair##op##Operator()                            \
@@ -1191,6 +1209,14 @@ const Operator* MachineOperatorBuilder::Word64AtomicCompareExchange(
   UNREACHABLE();
 }
 
+const Operator* MachineOperatorBuilder::Word32AtomicPairLoad() {
+  return &cache_.kWord32AtomicPairLoad;
+}
+
+const Operator* MachineOperatorBuilder::Word32AtomicPairStore() {
+  return &cache_.kWord32AtomicPairStore;
+}
+
 const Operator* MachineOperatorBuilder::Word32AtomicPairAdd() {
   return &cache_.kWord32AtomicPairAdd;
 }

src/compiler/machine-operator.h

@@ -665,6 +665,10 @@ class V8_EXPORT_PRIVATE MachineOperatorBuilder final
   const Operator* Word64AtomicNarrowExchange(MachineType type);
   // atomic-narrow-compare-exchange [base + index], old_value, new_value
   const Operator* Word64AtomicNarrowCompareExchange(MachineType type);
+  // atomic-pair-load [base + index]
+  const Operator* Word32AtomicPairLoad();
+  // atomic-pair-sub [base + index], value_high, value-low
+  const Operator* Word32AtomicPairStore();
   // atomic-pair-add [base + index], value_high, value_low
   const Operator* Word32AtomicPairAdd();
   // atomic-pair-sub [base + index], value_high, value-low

src/compiler/opcodes.h

@@ -666,6 +666,8 @@
   V(Word32AtomicAnd)                 \
   V(Word32AtomicOr)                  \
   V(Word32AtomicXor)                 \
+  V(Word32AtomicPairLoad)            \
+  V(Word32AtomicPairStore)           \
   V(Word32AtomicPairAdd)             \
   V(Word32AtomicPairSub)             \
   V(Word32AtomicPairAnd)             \

src/compiler/verifier.cc

@@ -1738,6 +1738,8 @@ void Verifier::Visitor::Check(Node* node, const AllNodes& all) {
     case IrOpcode::kWord64AtomicXor:
     case IrOpcode::kWord64AtomicExchange:
     case IrOpcode::kWord64AtomicCompareExchange:
+    case IrOpcode::kWord32AtomicPairLoad:
+    case IrOpcode::kWord32AtomicPairStore:
     case IrOpcode::kWord32AtomicPairAdd:
     case IrOpcode::kWord32AtomicPairSub:
     case IrOpcode::kWord32AtomicPairAnd:

src/ia32/assembler-ia32.cc

@@ -690,6 +690,14 @@ void Assembler::movzx_w(Register dst, Operand src) {
   emit_operand(dst, src);
 }
 
+void Assembler::movq(XMMRegister dst, Operand src) {
+  EnsureSpace ensure_space(this);
+  EMIT(0xF3);
+  EMIT(0x0F);
+  EMIT(0x7E);
+  emit_operand(dst, src);
+}
+
 void Assembler::cmov(Condition cc, Register dst, Operand src) {
   EnsureSpace ensure_space(this);
   // Opcode: 0f 40 + cc /r.
@@ -3254,11 +3262,20 @@ void Assembler::emit_arith(int sel, Operand dst, const Immediate& x) {
 }
 
 void Assembler::emit_operand(Register reg, Operand adr) {
+  emit_operand(reg.code(), adr);
+}
+
+void Assembler::emit_operand(XMMRegister reg, Operand adr) {
+  Register ireg = Register::from_code(reg.code());
+  emit_operand(ireg, adr);
+}
+
+void Assembler::emit_operand(int code, Operand adr) {
   const unsigned length = adr.len_;
   DCHECK_GT(length, 0);
 
   // Emit updated ModRM byte containing the given register.
-  pc_[0] = (adr.buf_[0] & ~0x38) | (reg.code() << 3);
+  pc_[0] = (adr.buf_[0] & ~0x38) | (code << 3);
 
   // Emit the rest of the encoded operand.
   for (unsigned i = 1; i < length; i++) pc_[i] = adr.buf_[i];

src/ia32/assembler-ia32.h

@@ -640,6 +640,7 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
   void movzx_w(Register dst, Register src) { movzx_w(dst, Operand(src)); }
   void movzx_w(Register dst, Operand src);
 
+  void movq(XMMRegister dst, Operand src);
   // Conditional moves
   void cmov(Condition cc, Register dst, Register src) {
     cmov(cc, dst, Operand(src));
@@ -1821,7 +1822,9 @@ class V8_EXPORT_PRIVATE Assembler : public AssemblerBase {
   // sel specifies the /n in the modrm byte (see the Intel PRM).
   void emit_arith(int sel, Operand dst, const Immediate& x);
 
+  void emit_operand(int code, Operand adr);
   void emit_operand(Register reg, Operand adr);
+  void emit_operand(XMMRegister reg, Operand adr);
 
   void emit_label(Label* label);
 

src/ia32/disasm-ia32.cc

@@ -2501,6 +2501,9 @@ int DisassemblerIA32::InstructionDecode(v8::internal::Vector<char> out_buffer,
               case 0x5F:
                 mnem = "maxss";
                 break;
+              case 0x7E:
+                mnem = "movq";
+                break;
             }
             data += 3;
             int mod, regop, rm;

test/cctest/test-disasm-ia32.cc

@@ -391,6 +391,7 @@ TEST(DisasmIa320) {
     __ shufps(xmm0, xmm0, 0x0);
     __ cvtsd2ss(xmm0, xmm1);
     __ cvtsd2ss(xmm0, Operand(ebx, ecx, times_4, 10000));
+    __ movq(xmm0, Operand(edx, 4));
 
     // logic operation
     __ andps(xmm0, xmm1);

test/cctest/wasm/test-run-wasm-atomics64.cc

@@ -258,7 +258,6 @@ WASM_EXEC_TEST(I32AtomicCompareExchange8U) {
   }
 }
 
-#if V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_ARM64
 WASM_EXEC_TEST(I64AtomicLoad) {
   EXPERIMENTAL_FLAG_SCOPE(threads);
   WasmRunner<uint64_t> r(execution_mode);
@@ -402,7 +401,6 @@ WASM_EXEC_TEST(I64AtomicStoreLoad8U) {
     CHECK_EQ(*i, r.builder().ReadMemory(&memory[0]));
   }
 }
-#endif  // V8_TARGET_ARCH_X64 || V8_TARGET_ARCH_ARM64
 
 }  // namespace test_run_wasm_atomics_64
 }  // namespace wasm