[atomics] Implement 64-bit Atomics operations

Atomics.{load,store,add,sub,and,or,xor,exchange,compareExchange}
are updated to support BigInt and BigInt64Array/BigUint64Array inputs.

Atomics.{wait,wake,isLockFree} are left unchanged for now.

Bug: v8:8100
Cq-Include-Trybots: luci.v8.try:v8_linux_noi18n_rel_ng
Change-Id: I8862d7e18c58ae08784535e9c010ba94f067a0ee
Reviewed-on: https://chromium-review.googlesource.com/1237294
Commit-Queue: Jakob Kummerow <jkummerow@chromium.org>
Reviewed-by: Deepti Gandluri <gdeepti@chromium.org>
Reviewed-by: Georg Neis <neis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#56228}

src/code-stub-assembler.h

@@ -1103,6 +1103,12 @@ class V8_EXPORT_PRIVATE CodeStubAssembler : public compiler::CodeAssembler {
   Node* LoadFixedBigInt64ArrayElementAsTagged(Node* data_pointer, Node* offset);
   Node* LoadFixedBigUint64ArrayElementAsTagged(Node* data_pointer,
                                                Node* offset);
+  // 64-bit platforms only:
+  TNode<BigInt> BigIntFromInt64(TNode<IntPtrT> value);
+  TNode<BigInt> BigIntFromUint64(TNode<UintPtrT> value);
+  // 32-bit platforms only:
+  TNode<BigInt> BigIntFromInt32Pair(TNode<IntPtrT> low, TNode<IntPtrT> high);
+  TNode<BigInt> BigIntFromUint32Pair(TNode<UintPtrT> low, TNode<UintPtrT> high);
 
   void StoreFixedTypedArrayElementFromTagged(
       TNode<Context> context, TNode<FixedTypedArrayBase> elements,
@@ -2647,11 +2653,17 @@ class V8_EXPORT_PRIVATE CodeStubAssembler : public compiler::CodeAssembler {
                                      TNode<Object> value,
                                      TNode<Context> context,
                                      Label* opt_if_neutered);
-  // Part of the above, refactored out to reuse in another place
+  // Part of the above, refactored out to reuse in another place.
   void EmitBigTypedArrayElementStore(TNode<FixedTypedArrayBase> elements,
                                      TNode<RawPtrT> backing_store,
                                      TNode<IntPtrT> offset,
                                      TNode<BigInt> bigint_value);
+  // Implements the BigInt part of
+  // https://tc39.github.io/proposal-bigint/#sec-numbertorawbytes,
+  // including truncation to 64 bits (i.e. modulo 2^64).
+  // {var_high} is only used on 32-bit platforms.
+  void BigIntToRawBytes(TNode<BigInt> bigint, TVariable<UintPtrT>* var_low,
+                        TVariable<UintPtrT>* var_high);
 
   void EmitElementStore(Node* object, Node* key, Node* value, bool is_jsarray,
                         ElementsKind elements_kind,

src/compiler/code-assembler.cc

@@ -1010,14 +1010,16 @@ Node* CodeAssembler::StoreNoWriteBarrier(MachineRepresentation rep, Node* base,
 }
 
 Node* CodeAssembler::AtomicStore(MachineRepresentation rep, Node* base,
-                                 Node* offset, Node* value) {
-  return raw_assembler()->AtomicStore(rep, base, offset, value);
+                                 Node* offset, Node* value, Node* value_high) {
+  return raw_assembler()->AtomicStore(rep, base, offset, value, value_high);
 }
 
-#define ATOMIC_FUNCTION(name)                                        \
-  Node* CodeAssembler::Atomic##name(MachineType type, Node* base,    \
-                                    Node* offset, Node* value) {     \
-    return raw_assembler()->Atomic##name(type, base, offset, value); \
+#define ATOMIC_FUNCTION(name)                                       \
+  Node* CodeAssembler::Atomic##name(MachineType type, Node* base,   \
+                                    Node* offset, Node* value,      \
+                                    Node* value_high) {             \
+    return raw_assembler()->Atomic##name(type, base, offset, value, \
+                                         value_high);               \
   }
 ATOMIC_FUNCTION(Exchange);
 ATOMIC_FUNCTION(Add);
@@ -1029,9 +1031,11 @@ ATOMIC_FUNCTION(Xor);
 
 Node* CodeAssembler::AtomicCompareExchange(MachineType type, Node* base,
                                            Node* offset, Node* old_value,
-                                           Node* new_value) {
-  return raw_assembler()->AtomicCompareExchange(type, base, offset, old_value,
-                                                new_value);
+                                           Node* new_value,
+                                           Node* old_value_high,
+                                           Node* new_value_high) {
+  return raw_assembler()->AtomicCompareExchange(
+      type, base, offset, old_value, old_value_high, new_value, new_value_high);
 }
 
 Node* CodeAssembler::StoreRoot(RootIndex root_index, Node* value) {
@@ -1048,6 +1052,7 @@ Node* CodeAssembler::Retain(Node* value) {
 }
 
 Node* CodeAssembler::Projection(int index, Node* value) {
+  DCHECK(index < value->op()->ValueOutputCount());
   return raw_assembler()->Projection(index, value);
 }
 

src/compiler/code-assembler.h

@@ -833,25 +833,35 @@ class V8_EXPORT_PRIVATE CodeAssembler {
   Node* StoreNoWriteBarrier(MachineRepresentation rep, Node* base, Node* value);
   Node* StoreNoWriteBarrier(MachineRepresentation rep, Node* base, Node* offset,
                             Node* value);
+  // {value_high} is used for 64-bit stores on 32-bit platforms, must be
+  // nullptr in other cases.
   Node* AtomicStore(MachineRepresentation rep, Node* base, Node* offset,
-                    Node* value);
+                    Node* value, Node* value_high = nullptr);
 
   // Exchange value at raw memory location
-  Node* AtomicExchange(MachineType type, Node* base, Node* offset, Node* value);
+  Node* AtomicExchange(MachineType type, Node* base, Node* offset, Node* value,
+                       Node* value_high = nullptr);
 
   // Compare and Exchange value at raw memory location
   Node* AtomicCompareExchange(MachineType type, Node* base, Node* offset,
-                              Node* old_value, Node* new_value);
+                              Node* old_value, Node* new_value,
+                              Node* old_value_high = nullptr,
+                              Node* new_value_high = nullptr);
 
-  Node* AtomicAdd(MachineType type, Node* base, Node* offset, Node* value);
+  Node* AtomicAdd(MachineType type, Node* base, Node* offset, Node* value,
+                  Node* value_high = nullptr);
 
-  Node* AtomicSub(MachineType type, Node* base, Node* offset, Node* value);
+  Node* AtomicSub(MachineType type, Node* base, Node* offset, Node* value,
+                  Node* value_high = nullptr);
 
-  Node* AtomicAnd(MachineType type, Node* base, Node* offset, Node* value);
+  Node* AtomicAnd(MachineType type, Node* base, Node* offset, Node* value,
+                  Node* value_high = nullptr);
 
-  Node* AtomicOr(MachineType type, Node* base, Node* offset, Node* value);
+  Node* AtomicOr(MachineType type, Node* base, Node* offset, Node* value,
+                 Node* value_high = nullptr);
 
-  Node* AtomicXor(MachineType type, Node* base, Node* offset, Node* value);
+  Node* AtomicXor(MachineType type, Node* base, Node* offset, Node* value,
+                  Node* value_high = nullptr);
 
   // Store a value to the root array.
   Node* StoreRoot(RootIndex root_index, Node* value);

src/compiler/machine-graph-verifier.cc

@@ -65,6 +65,16 @@ class MachineRepresentationInferrer {
         auto call_descriptor = CallDescriptorOf(input->op());
         return call_descriptor->GetReturnType(index).representation();
       }
+      case IrOpcode::kWord32AtomicPairLoad:
+      case IrOpcode::kWord32AtomicPairAdd:
+      case IrOpcode::kWord32AtomicPairSub:
+      case IrOpcode::kWord32AtomicPairAnd:
+      case IrOpcode::kWord32AtomicPairOr:
+      case IrOpcode::kWord32AtomicPairXor:
+      case IrOpcode::kWord32AtomicPairExchange:
+      case IrOpcode::kWord32AtomicPairCompareExchange:
+        CHECK_LE(index, static_cast<size_t>(1));
+        return MachineRepresentation::kWord32;
       default:
         return MachineRepresentation::kNone;
     }
@@ -111,6 +121,7 @@ class MachineRepresentationInferrer {
             representation_vector_[node->id()] = MachineRepresentation::kNone;
             break;
           case IrOpcode::kWord32AtomicLoad:
+          case IrOpcode::kWord64AtomicLoad:
           case IrOpcode::kLoad:
           case IrOpcode::kProtectedLoad:
           case IrOpcode::kPoisonedLoad:
@@ -144,9 +155,21 @@ class MachineRepresentationInferrer {
             break;
           }
           case IrOpcode::kWord32AtomicStore:
+          case IrOpcode::kWord64AtomicStore:
             representation_vector_[node->id()] =
                 PromoteRepresentation(AtomicStoreRepresentationOf(node->op()));
             break;
+          case IrOpcode::kWord32AtomicPairLoad:
+          case IrOpcode::kWord32AtomicPairStore:
+          case IrOpcode::kWord32AtomicPairAdd:
+          case IrOpcode::kWord32AtomicPairSub:
+          case IrOpcode::kWord32AtomicPairAnd:
+          case IrOpcode::kWord32AtomicPairOr:
+          case IrOpcode::kWord32AtomicPairXor:
+          case IrOpcode::kWord32AtomicPairExchange:
+          case IrOpcode::kWord32AtomicPairCompareExchange:
+            representation_vector_[node->id()] = MachineRepresentation::kWord32;
+            break;
           case IrOpcode::kWord32AtomicExchange:
           case IrOpcode::kWord32AtomicCompareExchange:
           case IrOpcode::kWord32AtomicAdd:
@@ -154,6 +177,13 @@ class MachineRepresentationInferrer {
           case IrOpcode::kWord32AtomicAnd:
           case IrOpcode::kWord32AtomicOr:
           case IrOpcode::kWord32AtomicXor:
+          case IrOpcode::kWord64AtomicExchange:
+          case IrOpcode::kWord64AtomicCompareExchange:
+          case IrOpcode::kWord64AtomicAdd:
+          case IrOpcode::kWord64AtomicSub:
+          case IrOpcode::kWord64AtomicAnd:
+          case IrOpcode::kWord64AtomicOr:
+          case IrOpcode::kWord64AtomicXor:
             representation_vector_[node->id()] = PromoteRepresentation(
                 AtomicOpType(node->op()).representation());
             break;
@@ -487,11 +517,23 @@ class MachineRepresentationChecker {
             break;
           case IrOpcode::kLoad:
           case IrOpcode::kWord32AtomicLoad:
+          case IrOpcode::kWord32AtomicPairLoad:
+          case IrOpcode::kWord64AtomicLoad:
           case IrOpcode::kPoisonedLoad:
             CheckValueInputIsTaggedOrPointer(node, 0);
             CheckValueInputRepresentationIs(
                 node, 1, MachineType::PointerRepresentation());
             break;
+          case IrOpcode::kWord32AtomicPairAdd:
+          case IrOpcode::kWord32AtomicPairSub:
+          case IrOpcode::kWord32AtomicPairAnd:
+          case IrOpcode::kWord32AtomicPairOr:
+          case IrOpcode::kWord32AtomicPairXor:
+          case IrOpcode::kWord32AtomicPairStore:
+          case IrOpcode::kWord32AtomicPairExchange:
+            CheckValueInputRepresentationIs(node, 3,
+                                            MachineRepresentation::kWord32);
+            V8_FALLTHROUGH;
           case IrOpcode::kStore:
           case IrOpcode::kWord32AtomicStore:
           case IrOpcode::kWord32AtomicExchange:
@@ -500,6 +542,13 @@ class MachineRepresentationChecker {
           case IrOpcode::kWord32AtomicAnd:
           case IrOpcode::kWord32AtomicOr:
           case IrOpcode::kWord32AtomicXor:
+          case IrOpcode::kWord64AtomicStore:
+          case IrOpcode::kWord64AtomicExchange:
+          case IrOpcode::kWord64AtomicAdd:
+          case IrOpcode::kWord64AtomicSub:
+          case IrOpcode::kWord64AtomicAnd:
+          case IrOpcode::kWord64AtomicOr:
+          case IrOpcode::kWord64AtomicXor:
             CheckValueInputIsTaggedOrPointer(node, 0);
             CheckValueInputRepresentationIs(
                 node, 1, MachineType::PointerRepresentation());
@@ -514,7 +563,14 @@ class MachineRepresentationChecker {
                     node, 2, inferrer_->GetRepresentation(node));
             }
             break;
+          case IrOpcode::kWord32AtomicPairCompareExchange:
+            CheckValueInputRepresentationIs(node, 4,
+                                            MachineRepresentation::kWord32);
+            CheckValueInputRepresentationIs(node, 5,
+                                            MachineRepresentation::kWord32);
+            V8_FALLTHROUGH;
           case IrOpcode::kWord32AtomicCompareExchange:
+          case IrOpcode::kWord64AtomicCompareExchange:
             CheckValueInputIsTaggedOrPointer(node, 0);
             CheckValueInputRepresentationIs(
                 node, 1, MachineType::PointerRepresentation());

src/compiler/machine-operator.cc

@@ -81,8 +81,7 @@ StackSlotRepresentation const& StackSlotRepresentationOf(Operator const* op) {
 
 MachineRepresentation AtomicStoreRepresentationOf(Operator const* op) {
   DCHECK(IrOpcode::kWord32AtomicStore == op->opcode() ||
-         IrOpcode::kWord64AtomicStore == op->opcode() ||
-         IrOpcode::kWord32AtomicPairStore == op->opcode());
+         IrOpcode::kWord64AtomicStore == op->opcode());
   return OpParameter<MachineRepresentation>(op);
 }
 

src/compiler/raw-machine-assembler.h

@@ -173,15 +173,51 @@ class V8_EXPORT_PRIVATE RawMachineAssembler {
 
   // Atomic memory operations.
   Node* AtomicLoad(MachineType type, Node* base, Node* index) {
+    if (type.representation() == MachineRepresentation::kWord64) {
+      if (machine()->Is64()) {
+        return AddNode(machine()->Word64AtomicLoad(type), base, index);
+      } else {
+        return AddNode(machine()->Word32AtomicPairLoad(), base, index);
+      }
+    }
     return AddNode(machine()->Word32AtomicLoad(type), base, index);
   }
+
+#if defined(V8_TARGET_BIG_ENDIAN)
+#define VALUE_HALVES value_high, value
+#else
+#define VALUE_HALVES value, value_high
+#endif
+
   Node* AtomicStore(MachineRepresentation rep, Node* base, Node* index,
-                    Node* value) {
+                    Node* value, Node* value_high) {
+    if (rep == MachineRepresentation::kWord64) {
+      if (machine()->Is64()) {
+        DCHECK_NULL(value_high);
+        return AddNode(machine()->Word64AtomicStore(rep), base, index, value);
+      } else {
+        return AddNode(machine()->Word32AtomicPairStore(), base, index,
+                       VALUE_HALVES);
+      }
+    }
+    DCHECK_NULL(value_high);
     return AddNode(machine()->Word32AtomicStore(rep), base, index, value);
   }
-#define ATOMIC_FUNCTION(name)                                                 \
-  Node* Atomic##name(MachineType rep, Node* base, Node* index, Node* value) { \
-    return AddNode(machine()->Word32Atomic##name(rep), base, index, value);   \
+#define ATOMIC_FUNCTION(name)                                               \
+  Node* Atomic##name(MachineType rep, Node* base, Node* index, Node* value, \
+                     Node* value_high) {                                    \
+    if (rep.representation() == MachineRepresentation::kWord64) {           \
+      if (machine()->Is64()) {                                              \
+        DCHECK_NULL(value_high);                                            \
+        return AddNode(machine()->Word64Atomic##name(rep), base, index,     \
+                       value);                                              \
+      } else {                                                              \
+        return AddNode(machine()->Word32AtomicPair##name(), base, index,    \
+                       VALUE_HALVES);                                       \
+      }                                                                     \
+    }                                                                       \
+    DCHECK_NULL(value_high);                                                \
+    return AddNode(machine()->Word32Atomic##name(rep), base, index, value); \
   }
   ATOMIC_FUNCTION(Exchange);
   ATOMIC_FUNCTION(Add);
@@ -190,9 +226,25 @@ class V8_EXPORT_PRIVATE RawMachineAssembler {
   ATOMIC_FUNCTION(Or);
   ATOMIC_FUNCTION(Xor);
 #undef ATOMIC_FUNCTION
+#undef VALUE_HALVES
 
   Node* AtomicCompareExchange(MachineType rep, Node* base, Node* index,
-                              Node* old_value, Node* new_value) {
+                              Node* old_value, Node* old_value_high,
+                              Node* new_value, Node* new_value_high) {
+    if (rep.representation() == MachineRepresentation::kWord64) {
+      if (machine()->Is64()) {
+        DCHECK_NULL(old_value_high);
+        DCHECK_NULL(new_value_high);
+        return AddNode(machine()->Word64AtomicCompareExchange(rep), base, index,
+                       old_value, new_value);
+      } else {
+        return AddNode(machine()->Word32AtomicPairCompareExchange(), base,
+                       index, old_value, old_value_high, new_value,
+                       new_value_high);
+      }
+    }
+    DCHECK_NULL(old_value_high);
+    DCHECK_NULL(new_value_high);
     return AddNode(machine()->Word32AtomicCompareExchange(rep), base, index,
                    old_value, new_value);
   }

test/test262/test262.status

@@ -502,17 +502,8 @@
   'built-ins/TypedArray/prototype/set/typedarray-arg-set-values-same-buffer-same-type-sab': ['--harmony-sharedarraybuffer'],
 
   # https://bugs.chromium.org/p/v8/issues/detail?id=8100
-  'built-ins/Atomics/add/bigint/*': [SKIP],
-  'built-ins/Atomics/and/bigint/*': [SKIP],
-  'built-ins/Atomics/compareExchange/bigint/*': [SKIP],
-  'built-ins/Atomics/exchange/bigint/*': [SKIP],
-  'built-ins/Atomics/load/bigint/*': [SKIP],
   'built-ins/Atomics/notify/bigint/*': [SKIP],
-  'built-ins/Atomics/or/bigint/*': [SKIP],
-  'built-ins/Atomics/store/bigint/*': [SKIP],
-  'built-ins/Atomics/sub/bigint/*': [SKIP],
   'built-ins/Atomics/wait/bigint/*': [SKIP],
-  'built-ins/Atomics/xor/bigint/*': [SKIP],
 
   # https://bugs.chromium.org/p/v8/issues/detail?id=6049
   'built-ins/Object/internals/DefineOwnProperty/consistent-value-function-caller': [FAIL_SLOPPY],