[heap] Separate map space pointers updating from main bulk

Evacuating and compacting LayoutDescriptor is meta-circular, i.e., we
need the descriptor to iterate the objects that are copied. Separate the
phases to avoid requiring a publishing store for object payloads.

Bug: v8:6884
Change-Id: Icc3bb80a938fbd9eec60c79d88ab8fdd9bda0840
Reviewed-on: https://chromium-review.googlesource.com/700437Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#48288}

src/heap-symbols.h

@@ -304,7 +304,8 @@
   F(MC_EVACUATE_PROLOGUE)                            \
   F(MC_EVACUATE_REBALANCE)                           \
   F(MC_EVACUATE_UPDATE_POINTERS)                     \
-  F(MC_EVACUATE_UPDATE_POINTERS_SLOTS)               \
+  F(MC_EVACUATE_UPDATE_POINTERS_SLOTS_MAIN)          \
+  F(MC_EVACUATE_UPDATE_POINTERS_SLOTS_MAP_SPACE)     \
   F(MC_EVACUATE_UPDATE_POINTERS_TO_NEW_ROOTS)        \
   F(MC_EVACUATE_UPDATE_POINTERS_WEAK)                \
   F(MC_FINISH)                                       \

src/heap/gc-tracer.cc

@@ -597,7 +597,8 @@ void GCTracer::PrintNVP() const {
           "evacuate.rebalance=%.1f "
           "evacuate.update_pointers=%.1f "
           "evacuate.update_pointers.to_new_roots=%.1f "
-          "evacuate.update_pointers.slots=%.1f "
+          "evacuate.update_pointers.slots.main=%.1f "
+          "evacuate.update_pointers.slots.map_space=%.1f "
           "evacuate.update_pointers.weak=%.1f "
           "finish=%.1f "
           "mark=%.1f "
@@ -682,7 +683,8 @@ void GCTracer::PrintNVP() const {
           current_.scopes[Scope::MC_EVACUATE_REBALANCE],
           current_.scopes[Scope::MC_EVACUATE_UPDATE_POINTERS],
           current_.scopes[Scope::MC_EVACUATE_UPDATE_POINTERS_TO_NEW_ROOTS],
-          current_.scopes[Scope::MC_EVACUATE_UPDATE_POINTERS_SLOTS],
+          current_.scopes[Scope::MC_EVACUATE_UPDATE_POINTERS_SLOTS_MAIN],
+          current_.scopes[Scope::MC_EVACUATE_UPDATE_POINTERS_SLOTS_MAP_SPACE],
           current_.scopes[Scope::MC_EVACUATE_UPDATE_POINTERS_WEAK],
           current_.scopes[Scope::MC_FINISH], current_.scopes[Scope::MC_MARK],
           current_.scopes[Scope::MC_MARK_FINISH_INCREMENTAL],

src/heap/mark-compact.cc

@@ -4129,30 +4129,25 @@ int MarkCompactCollectorBase::CollectToSpaceUpdatingItems(
   return NumberOfParallelToSpacePointerUpdateTasks(pages);
 }
 
+template <typename IterateableSpace>
 int MarkCompactCollectorBase::CollectRememberedSetUpdatingItems(
-    ItemParallelJob* job, RememberedSetUpdatingMode mode) {
+    ItemParallelJob* job, IterateableSpace* space,
+    RememberedSetUpdatingMode mode) {
   int pages = 0;
-  if (mode == RememberedSetUpdatingMode::ALL) {
-    RememberedSet<OLD_TO_OLD>::IterateMemoryChunks(
-        heap(), [this, &job, &pages, mode](MemoryChunk* chunk) {
-          job->AddItem(CreateRememberedSetUpdatingItem(chunk, mode));
-          pages++;
-        });
-  }
-  RememberedSet<OLD_TO_NEW>::IterateMemoryChunks(
-      heap(), [this, &job, &pages, mode](MemoryChunk* chunk) {
-        const bool contains_old_to_old_slots =
-            chunk->slot_set<OLD_TO_OLD>() != nullptr ||
-            chunk->typed_slot_set<OLD_TO_OLD>() != nullptr;
-        if (mode == RememberedSetUpdatingMode::OLD_TO_NEW_ONLY ||
-            !contains_old_to_old_slots) {
-          job->AddItem(CreateRememberedSetUpdatingItem(chunk, mode));
-          pages++;
-        }
-      });
-  return (pages == 0)
-             ? 0
-             : NumberOfParallelPointerUpdateTasks(pages, old_to_new_slots_);
+  for (MemoryChunk* chunk : *space) {
+    const bool contains_old_to_old_slots =
+        chunk->slot_set<OLD_TO_OLD>() != nullptr ||
+        chunk->typed_slot_set<OLD_TO_OLD>() != nullptr;
+    const bool contains_old_to_new_slots =
+        chunk->slot_set<OLD_TO_NEW>() != nullptr ||
+        chunk->typed_slot_set<OLD_TO_NEW>() != nullptr;
+    if (!contains_old_to_new_slots && !contains_old_to_old_slots) continue;
+    if (mode == RememberedSetUpdatingMode::ALL || contains_old_to_new_slots) {
+      job->AddItem(CreateRememberedSetUpdatingItem(chunk, mode));
+      pages++;
+    }
+  }
+  return pages;
 }
 
 void MinorMarkCompactCollector::CollectNewSpaceArrayBufferTrackerItems(
@@ -4187,31 +4182,65 @@ void MarkCompactCollector::UpdatePointersAfterEvacuation() {
   TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS);
 
   PointersUpdatingVisitor updating_visitor;
-  ItemParallelJob updating_job(isolate()->cancelable_task_manager(),
-                               &page_parallel_job_semaphore_);
-
-  CollectNewSpaceArrayBufferTrackerItems(&updating_job);
-  CollectOldSpaceArrayBufferTrackerItems(&updating_job);
-
-  const int to_space_tasks = CollectToSpaceUpdatingItems(&updating_job);
-  const int remembered_set_tasks = CollectRememberedSetUpdatingItems(
-      &updating_job, RememberedSetUpdatingMode::ALL);
-  const int num_tasks = Max(to_space_tasks, remembered_set_tasks);
-  for (int i = 0; i < num_tasks; i++) {
-    updating_job.AddTask(new PointersUpatingTask(isolate()));
-  }
 
   {
     TRACE_GC(heap()->tracer(),
              GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_TO_NEW_ROOTS);
     heap_->IterateRoots(&updating_visitor, VISIT_ALL_IN_SWEEP_NEWSPACE);
   }
+
   {
     TRACE_GC(heap()->tracer(),
-             GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_SLOTS);
+             GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_SLOTS_MAIN);
+    ItemParallelJob updating_job(isolate()->cancelable_task_manager(),
+                                 &page_parallel_job_semaphore_);
+
+    CollectNewSpaceArrayBufferTrackerItems(&updating_job);
+    CollectOldSpaceArrayBufferTrackerItems(&updating_job);
+
+    int remembered_set_pages = 0;
+    remembered_set_pages += CollectRememberedSetUpdatingItems(
+        &updating_job, heap()->old_space(), RememberedSetUpdatingMode::ALL);
+    remembered_set_pages += CollectRememberedSetUpdatingItems(
+        &updating_job, heap()->code_space(), RememberedSetUpdatingMode::ALL);
+    remembered_set_pages += CollectRememberedSetUpdatingItems(
+        &updating_job, heap()->lo_space(), RememberedSetUpdatingMode::ALL);
+    const int remembered_set_tasks =
+        remembered_set_pages == 0
+            ? 0
+            : NumberOfParallelPointerUpdateTasks(remembered_set_pages,
+                                                 old_to_new_slots_);
+    const int to_space_tasks = CollectToSpaceUpdatingItems(&updating_job);
+    const int num_tasks = Max(to_space_tasks, remembered_set_tasks);
+    for (int i = 0; i < num_tasks; i++) {
+      updating_job.AddTask(new PointersUpatingTask(isolate()));
+    }
     updating_job.Run();
   }
 
+  {
+    // Update pointers in map space in a separate phase to avoid data races
+    // with Map->LayoutDescriptor edge.
+    TRACE_GC(heap()->tracer(),
+             GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_SLOTS_MAP_SPACE);
+    ItemParallelJob updating_job(isolate()->cancelable_task_manager(),
+                                 &page_parallel_job_semaphore_);
+
+    int remembered_set_pages = 0;
+    remembered_set_pages += CollectRememberedSetUpdatingItems(
+        &updating_job, heap()->map_space(), RememberedSetUpdatingMode::ALL);
+    const int num_tasks = remembered_set_pages == 0
+                              ? 0
+                              : NumberOfParallelPointerUpdateTasks(
+                                    remembered_set_pages, old_to_new_slots_);
+    if (num_tasks > 0) {
+      for (int i = 0; i < num_tasks; i++) {
+        updating_job.AddTask(new PointersUpatingTask(isolate()));
+      }
+      updating_job.Run();
+    }
+  }
+
   {
     TRACE_GC(heap()->tracer(),
              GCTracer::Scope::MC_EVACUATE_UPDATE_POINTERS_WEAK);
@@ -4237,8 +4266,21 @@ void MinorMarkCompactCollector::UpdatePointersAfterEvacuation() {
   SeedGlobalHandles<GlobalHandlesUpdatingItem>(isolate()->global_handles(),
                                                &updating_job);
   const int to_space_tasks = CollectToSpaceUpdatingItems(&updating_job);
-  const int remembered_set_tasks = CollectRememberedSetUpdatingItems(
-      &updating_job, RememberedSetUpdatingMode::OLD_TO_NEW_ONLY);
+  int remembered_set_pages = 0;
+  remembered_set_pages += CollectRememberedSetUpdatingItems(
+      &updating_job, heap()->old_space(),
+      RememberedSetUpdatingMode::OLD_TO_NEW_ONLY);
+  remembered_set_pages += CollectRememberedSetUpdatingItems(
+      &updating_job, heap()->code_space(),
+      RememberedSetUpdatingMode::OLD_TO_NEW_ONLY);
+  remembered_set_pages += CollectRememberedSetUpdatingItems(
+      &updating_job, heap()->map_space(),
+      RememberedSetUpdatingMode::OLD_TO_NEW_ONLY);
+  remembered_set_pages += CollectRememberedSetUpdatingItems(
+      &updating_job, heap()->lo_space(),
+      RememberedSetUpdatingMode::OLD_TO_NEW_ONLY);
+  const int remembered_set_tasks = NumberOfParallelPointerUpdateTasks(
+      remembered_set_pages, old_to_new_slots_);
   const int num_tasks = Max(to_space_tasks, remembered_set_tasks);
   for (int i = 0; i < num_tasks; i++) {
     updating_job.AddTask(new PointersUpatingTask(isolate()));

src/heap/mark-compact.h

@@ -290,7 +290,9 @@ class MarkCompactCollectorBase {
   bool ShouldMovePage(Page* p, intptr_t live_bytes);
 
   int CollectToSpaceUpdatingItems(ItemParallelJob* job);
+  template <typename IterateableSpace>
   int CollectRememberedSetUpdatingItems(ItemParallelJob* job,
+                                        IterateableSpace* space,
                                         RememberedSetUpdatingMode mode);
 
   int NumberOfParallelCompactionTasks(int pages);