[heap] Insert into old-to-new slots non-atomically

Since https://crrev.com/c/1771783 the mutator owns the old-to-new
remembered set, while the sweeper modifies the sweeping-slot-set.
This allows us to update the old-to-new remembered set non-atomically.

In this CL the mutator now inserts non-atomically into the remembered
set. The AccessMode is now explicit for Insert-operations as well.

Bug: v8:9454
Change-Id: I94730345f7dd34fe309839969330687c94b3080b
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1803652
Commit-Queue: Dominik Inführ <dinfuehr@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#63971}

src/heap/mark-compact-inl.h

@@ -485,7 +485,8 @@ void MarkCompactCollector::RecordSlot(HeapObject object, HeapObjectSlot slot,
   MemoryChunk* source_page = MemoryChunk::FromHeapObject(object);
   if (target_page->IsEvacuationCandidate<AccessMode::ATOMIC>() &&
       !source_page->ShouldSkipEvacuationSlotRecording<AccessMode::ATOMIC>()) {
-    RememberedSet<OLD_TO_OLD>::Insert(source_page, slot.address());
+    RememberedSet<OLD_TO_OLD>::Insert<AccessMode::ATOMIC>(source_page,
+                                                          slot.address());
   }
 }
 
@@ -493,7 +494,8 @@ void MarkCompactCollector::RecordSlot(MemoryChunk* source_page,
                                       HeapObjectSlot slot, HeapObject target) {
   MemoryChunk* target_page = MemoryChunk::FromHeapObject(target);
   if (target_page->IsEvacuationCandidate<AccessMode::ATOMIC>()) {
-    RememberedSet<OLD_TO_OLD>::Insert(source_page, slot.address());
+    RememberedSet<OLD_TO_OLD>::Insert<AccessMode::ATOMIC>(source_page,
+                                                          slot.address());
   }
 }
 

src/heap/remembered-set.h

@@ -93,7 +93,7 @@ class RememberedSet : public AllStatic {
  public:
   // Given a page and a slot in that page, this function adds the slot to the
   // remembered set.
-  template <AccessMode access_mode = AccessMode::ATOMIC>
+  template <AccessMode access_mode>
   static void Insert(MemoryChunk* chunk, Address slot_addr) {
     DCHECK(chunk->Contains(slot_addr));
     SlotSet* slot_set = chunk->slot_set<type, access_mode>();
@@ -374,7 +374,7 @@ class UpdateTypedSlotHelper {
 
 class RememberedSetSweeping {
  public:
-  template <AccessMode access_mode = AccessMode::ATOMIC>
+  template <AccessMode access_mode>
   static void Insert(MemoryChunk* chunk, Address slot_addr) {
     DCHECK(chunk->Contains(slot_addr));
     SlotSet* slot_set = chunk->sweeping_slot_set<access_mode>();

src/heap/scavenger.cc

@@ -158,9 +158,11 @@ class IterateAndScavengePromotedObjectsVisitor final : public ObjectVisitor {
         // Sweeper is stopped during scavenge, so we can directly
         // insert into its remembered set here.
         if (chunk->sweeping_slot_set()) {
-          RememberedSetSweeping::Insert(chunk, slot.address());
+          RememberedSetSweeping::Insert<AccessMode::ATOMIC>(chunk,
+                                                            slot.address());
         } else {
-          RememberedSet<OLD_TO_NEW>::Insert(chunk, slot.address());
+          RememberedSet<OLD_TO_NEW>::Insert<AccessMode::ATOMIC>(chunk,
+                                                                slot.address());
         }
       }
       SLOW_DCHECK(!MarkCompactCollector::IsOnEvacuationCandidate(
@@ -172,8 +174,8 @@ class IterateAndScavengePromotedObjectsVisitor final : public ObjectVisitor {
       // We cannot call MarkCompactCollector::RecordSlot because that checks
       // that the host page is not in young generation, which does not hold
       // for pending large pages.
-      RememberedSet<OLD_TO_OLD>::Insert(MemoryChunk::FromHeapObject(host),
-                                        slot.address());
+      RememberedSet<OLD_TO_OLD>::Insert<AccessMode::ATOMIC>(
+          MemoryChunk::FromHeapObject(host), slot.address());
     }
   }
 

src/heap/slot-set.h

@@ -55,7 +55,7 @@ class SlotSet : public Malloced {
   // The slot offset specifies a slot at address page_start_ + slot_offset.
   // AccessMode defines whether there can be concurrent access on the buckets
   // or not.
-  template <AccessMode access_mode = AccessMode::ATOMIC>
+  template <AccessMode access_mode>
   void Insert(int slot_offset) {
     int bucket_index, cell_index, bit_index;
     SlotToIndices(slot_offset, &bucket_index, &cell_index, &bit_index);

src/heap/spaces.cc

@@ -23,6 +23,7 @@
 #include "src/heap/read-only-heap.h"
 #include "src/heap/remembered-set.h"
 #include "src/heap/slot-set.h"
+#include "src/heap/store-buffer.h"
 #include "src/heap/sweeper.h"
 #include "src/init/v8.h"
 #include "src/logging/counters.h"
@@ -865,12 +866,13 @@ void Page::MoveOldToNewRememberedSetForSweeping() {
 
 void Page::MergeOldToNewRememberedSets() {
   if (sweeping_slot_set_ == nullptr) return;
+  DCHECK(heap()->store_buffer()->Empty());
 
   RememberedSet<OLD_TO_NEW>::Iterate(
       this,
       [this](MaybeObjectSlot slot) {
         Address address = slot.address();
-        RememberedSetSweeping::Insert(this, address);
+        RememberedSetSweeping::Insert<AccessMode::NON_ATOMIC>(this, address);
         return KEEP_SLOT;
       },
       SlotSet::KEEP_EMPTY_BUCKETS);

src/heap/store-buffer.cc

@@ -97,7 +97,8 @@ void StoreBuffer::InsertDuringRuntime(StoreBuffer* store_buffer, Address slot) {
 void StoreBuffer::InsertDuringGarbageCollection(StoreBuffer* store_buffer,
                                                 Address slot) {
   DCHECK(store_buffer->mode() != StoreBuffer::NOT_IN_GC);
-  RememberedSet<OLD_TO_NEW>::Insert(Page::FromAddress(slot), slot);
+  RememberedSet<OLD_TO_NEW>::Insert<AccessMode::ATOMIC>(Page::FromAddress(slot),
+                                                        slot);
 }
 
 void StoreBuffer::SetMode(StoreBufferMode mode) {
@@ -146,7 +147,7 @@ void StoreBuffer::MoveEntriesToRememberedSet(int index) {
       chunk = MemoryChunk::FromAnyPointerAddress(addr);
     }
     if (addr != last_inserted_addr) {
-      RememberedSet<OLD_TO_NEW>::Insert(chunk, addr);
+      RememberedSet<OLD_TO_NEW>::Insert<AccessMode::NON_ATOMIC>(chunk, addr);
       last_inserted_addr = addr;
     }
   }

test/cctest/heap/test-heap.cc

@@ -6037,7 +6037,7 @@ TEST(RememberedSetRemoveRange) {
   slots[chunk->area_end() - kTaggedSize] = true;
 
   for (auto x : slots) {
-    RememberedSet<OLD_TO_NEW>::Insert(chunk, x.first);
+    RememberedSet<OLD_TO_NEW>::Insert<AccessMode::ATOMIC>(chunk, x.first);
   }
 
   RememberedSet<OLD_TO_NEW>::Iterate(chunk,

test/unittests/heap/slot-set-unittest.cc

@@ -21,7 +21,7 @@ TEST(SlotSet, InsertAndLookup1) {
     EXPECT_FALSE(set.Lookup(i));
   }
   for (int i = 0; i < Page::kPageSize; i += kTaggedSize) {
-    set.Insert(i);
+    set.Insert<AccessMode::ATOMIC>(i);
   }
   for (int i = 0; i < Page::kPageSize; i += kTaggedSize) {
     EXPECT_TRUE(set.Lookup(i));
@@ -33,7 +33,7 @@ TEST(SlotSet, InsertAndLookup2) {
   set.SetPageStart(0);
   for (int i = 0; i < Page::kPageSize; i += kTaggedSize) {
     if (i % 7 == 0) {
-      set.Insert(i);
+      set.Insert<AccessMode::ATOMIC>(i);
     }
   }
   for (int i = 0; i < Page::kPageSize; i += kTaggedSize) {
@@ -50,7 +50,7 @@ TEST(SlotSet, Iterate) {
   set.SetPageStart(0);
   for (int i = 0; i < Page::kPageSize; i += kTaggedSize) {
     if (i % 7 == 0) {
-      set.Insert(i);
+      set.Insert<AccessMode::ATOMIC>(i);
     }
   }
 
@@ -78,7 +78,7 @@ TEST(SlotSet, Remove) {
   set.SetPageStart(0);
   for (int i = 0; i < Page::kPageSize; i += kTaggedSize) {
     if (i % 7 == 0) {
-      set.Insert(i);
+      set.Insert<AccessMode::ATOMIC>(i);
     }
   }
 
@@ -105,7 +105,7 @@ void CheckRemoveRangeOn(uint32_t start, uint32_t end) {
   for (const auto mode :
        {SlotSet::FREE_EMPTY_BUCKETS, SlotSet::KEEP_EMPTY_BUCKETS}) {
     for (uint32_t i = first; i <= last; i += kTaggedSize) {
-      set.Insert(i);
+      set.Insert<AccessMode::ATOMIC>(i);
     }
     set.RemoveRange(start, end, mode);
     if (first != start) {
@@ -140,7 +140,7 @@ TEST(SlotSet, RemoveRange) {
   set.SetPageStart(0);
   for (const auto mode :
        {SlotSet::FREE_EMPTY_BUCKETS, SlotSet::KEEP_EMPTY_BUCKETS}) {
-    set.Insert(Page::kPageSize / 2);
+    set.Insert<AccessMode::ATOMIC>(Page::kPageSize / 2);
     set.RemoveRange(0, Page::kPageSize, mode);
     for (uint32_t i = 0; i < Page::kPageSize; i += kTaggedSize) {
       EXPECT_FALSE(set.Lookup(i));