[heap] Sweep ArrayBufferExtensions concurrently to application

When ArrayBufferExtensions are enabled, sweep the extensions outside of
the GC pause concurrently to the application. The following GC will make
sure that the previous concurrent sweep operation is finished.

This CL introduces Heap::array_buffer_sweeper() that is both
responsible for tracking all extensions but also for sweeping.

Bug: v8:10064
Change-Id: I113cd625445a7d59ffb7a9de8b25a15a72b02b99
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2010107Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Commit-Queue: Dominik Inführ <dinfuehr@chromium.org>
Cr-Commit-Position: refs/heads/master@{#65984}

BUILD.gn

@@ -2324,6 +2324,8 @@ v8_source_set("v8_base_without_compiler") {
     "src/handles/maybe-handles.h",
     "src/heap/array-buffer-collector.cc",
     "src/heap/array-buffer-collector.h",
+    "src/heap/array-buffer-sweeper.cc",
+    "src/heap/array-buffer-sweeper.h",
     "src/heap/array-buffer-tracker-inl.h",
     "src/heap/array-buffer-tracker.cc",
     "src/heap/array-buffer-tracker.h",

src/flags/flag-definitions.h

@@ -892,6 +892,8 @@ DEFINE_BOOL(concurrent_marking, V8_CONCURRENT_MARKING_BOOL,
 DEFINE_BOOL_READONLY(array_buffer_extension, V8_ARRAY_BUFFER_EXTENSION_BOOL,
                      "enable array buffer tracking using extension objects")
 DEFINE_IMPLICATION(array_buffer_extension, always_promote_young_mc)
+DEFINE_BOOL(concurrent_array_buffer_sweeping, true,
+            "concurrently sweep array buffers")
 DEFINE_BOOL(parallel_marking, true, "use parallel marking in atomic pause")
 DEFINE_INT(ephemeron_fixpoint_iterations, 10,
            "number of fixpoint iterations it takes to switch to linear "

src/heap/array-buffer-sweeper.cc

+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/heap/array-buffer-sweeper.h"
+#include "src/heap/gc-tracer.h"
+#include "src/heap/heap-inl.h"
+#include "src/objects/js-array-buffer.h"
+#include "src/tasks/cancelable-task.h"
+#include "src/tasks/task-utils.h"
+
+namespace v8 {
+namespace internal {
+
+void ArrayBufferList::Append(ArrayBufferExtension* extension) {
+  if (head_ == nullptr) {
+    DCHECK_NULL(tail_);
+    head_ = tail_ = extension;
+  } else {
+    tail_->set_next(extension);
+    tail_ = extension;
+  }
+
+  extension->set_next(nullptr);
+}
+
+void ArrayBufferList::Append(ArrayBufferList* list) {
+  if (head_ == nullptr) {
+    DCHECK_NULL(tail_);
+    head_ = list->head_;
+    tail_ = list->tail_;
+  } else if (list->head_) {
+    DCHECK_NOT_NULL(list->tail_);
+    tail_->set_next(list->head_);
+    tail_ = list->tail_;
+  } else {
+    DCHECK_NULL(list->tail_);
+  }
+
+  list->Reset();
+}
+
+bool ArrayBufferList::Contains(ArrayBufferExtension* extension) {
+  ArrayBufferExtension* current = head_;
+
+  while (current) {
+    if (current == extension) return true;
+    current = current->next();
+  }
+
+  return false;
+}
+
+void ArrayBufferSweeper::EnsureFinished() {
+  if (!sweeping_in_progress_) return;
+
+  TryAbortResult abort_result =
+      heap_->isolate()->cancelable_task_manager()->TryAbort(job_.id);
+
+  switch (abort_result) {
+    case TryAbortResult::kTaskAborted: {
+      job_.Sweep();
+      Merge();
+      break;
+    }
+
+    case TryAbortResult::kTaskRemoved: {
+      CHECK_NE(job_.state, SweepingState::Uninitialized);
+      if (job_.state == SweepingState::Prepared) job_.Sweep();
+      Merge();
+      break;
+    }
+
+    case TryAbortResult::kTaskRunning: {
+      base::MutexGuard guard(&sweeping_mutex_);
+      CHECK_NE(job_.state, SweepingState::Uninitialized);
+      // Wait until task is finished with its work.
+      while (job_.state != SweepingState::Swept) {
+        job_finished_.Wait(&sweeping_mutex_);
+      }
+      Merge();
+      break;
+    }
+
+    default:
+      UNREACHABLE();
+  }
+
+  sweeping_in_progress_ = false;
+}
+
+void ArrayBufferSweeper::RequestSweepYoung() {
+  RequestSweep(SweepingScope::Young);
+}
+
+void ArrayBufferSweeper::RequestSweepFull() {
+  RequestSweep(SweepingScope::Full);
+}
+
+void ArrayBufferSweeper::RequestSweep(SweepingScope scope) {
+  DCHECK(!sweeping_in_progress_);
+
+  if (!heap_->IsTearingDown() && !heap_->ShouldReduceMemory() &&
+      FLAG_concurrent_array_buffer_sweeping) {
+    Prepare(scope);
+
+    auto task = MakeCancelableTask(heap_->isolate(), [this] {
+      TRACE_BACKGROUND_GC(
+          heap_->tracer(),
+          GCTracer::BackgroundScope::BACKGROUND_ARRAY_BUFFER_SWEEP);
+      base::MutexGuard guard(&sweeping_mutex_);
+      job_.Sweep();
+      job_finished_.NotifyAll();
+    });
+    job_.id = task->id();
+    V8::GetCurrentPlatform()->CallOnWorkerThread(std::move(task));
+    sweeping_in_progress_ = true;
+  } else {
+    Prepare(scope);
+    job_.Sweep();
+    Merge();
+  }
+}
+
+void ArrayBufferSweeper::Prepare(SweepingScope scope) {
+  CHECK_EQ(job_.state, SweepingState::Uninitialized);
+
+  if (scope == SweepingScope::Young) {
+    job_ =
+        SweepingJob::Prepare(young_, ArrayBufferList(), SweepingScope::Young);
+    young_.Reset();
+  } else {
+    CHECK_EQ(scope, SweepingScope::Full);
+    job_ = SweepingJob::Prepare(young_, old_, SweepingScope::Full);
+    young_.Reset();
+    old_.Reset();
+  }
+}
+
+void ArrayBufferSweeper::Merge() {
+  CHECK_EQ(job_.state, SweepingState::Swept);
+  young_.Append(&job_.young);
+  old_.Append(&job_.old);
+  job_.state = SweepingState::Uninitialized;
+}
+
+void ArrayBufferSweeper::ReleaseAll() {
+  EnsureFinished();
+  ReleaseAll(&old_);
+  ReleaseAll(&young_);
+}
+
+void ArrayBufferSweeper::ReleaseAll(ArrayBufferList* list) {
+  ArrayBufferExtension* current = list->head_;
+
+  while (current) {
+    ArrayBufferExtension* next = current->next();
+    delete current;
+    current = next;
+  }
+
+  list->Reset();
+}
+
+void ArrayBufferSweeper::Append(JSArrayBuffer object,
+                                ArrayBufferExtension* extension) {
+  if (Heap::InYoungGeneration(object)) {
+    young_.Append(extension);
+  } else {
+    old_.Append(extension);
+  }
+}
+
+ArrayBufferSweeper::SweepingJob::SweepingJob()
+    : state(SweepingState::Uninitialized) {}
+
+ArrayBufferSweeper::SweepingJob ArrayBufferSweeper::SweepingJob::Prepare(
+    ArrayBufferList young, ArrayBufferList old, SweepingScope scope) {
+  SweepingJob job;
+  job.young = young;
+  job.old = old;
+  job.scope = scope;
+  job.id = 0;
+  job.state = SweepingState::Prepared;
+  return job;
+}
+
+void ArrayBufferSweeper::SweepingJob::Sweep() {
+  CHECK_EQ(state, SweepingState::Prepared);
+
+  if (scope == SweepingScope::Young) {
+    SweepYoung();
+  } else {
+    CHECK_EQ(scope, SweepingScope::Full);
+    SweepFull();
+  }
+  state = SweepingState::Swept;
+}
+
+void ArrayBufferSweeper::SweepingJob::SweepFull() {
+  CHECK_EQ(scope, SweepingScope::Full);
+  ArrayBufferList promoted = SweepListFull(&young);
+  ArrayBufferList survived = SweepListFull(&old);
+
+  old = promoted;
+  old.Append(&survived);
+}
+
+ArrayBufferList ArrayBufferSweeper::SweepingJob::SweepListFull(
+    ArrayBufferList* list) {
+  ArrayBufferExtension* current = list->head_;
+  ArrayBufferList survived;
+
+  while (current) {
+    ArrayBufferExtension* next = current->next();
+
+    if (!current->IsMarked()) {
+      delete current;
+    } else {
+      current->Unmark();
+      survived.Append(current);
+    }
+
+    current = next;
+  }
+
+  list->Reset();
+  return survived;
+}
+
+void ArrayBufferSweeper::SweepingJob::SweepYoung() {
+  CHECK_EQ(scope, SweepingScope::Young);
+  ArrayBufferExtension* current = young.head_;
+
+  ArrayBufferList new_young;
+  ArrayBufferList new_old;
+
+  while (current) {
+    ArrayBufferExtension* next = current->next();
+
+    if (!current->IsYoungMarked()) {
+      delete current;
+    } else if (current->IsYoungPromoted()) {
+      current->YoungUnmark();
+      new_old.Append(current);
+    } else {
+      current->YoungUnmark();
+      new_young.Append(current);
+    }
+
+    current = next;
+  }
+
+  old = new_old;
+  young = new_young;
+}
+
+}  // namespace internal
+}  // namespace v8

src/heap/array-buffer-sweeper.h

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_HEAP_ARRAY_BUFFER_SWEEPER_H_
+#define V8_HEAP_ARRAY_BUFFER_SWEEPER_H_
+
+#include "src/base/platform/mutex.h"
+#include "src/objects/js-array-buffer.h"
+#include "src/tasks/cancelable-task.h"
+
+namespace v8 {
+namespace internal {
+
+class ArrayBufferExtension;
+class Heap;
+
+// Singly linked-list of ArrayBufferExtensions that stores head and tail of the
+// list to allow for concatenation of lists.
+struct ArrayBufferList {
+  ArrayBufferList() : head_(nullptr), tail_(nullptr) {}
+
+  ArrayBufferExtension* head_;
+  ArrayBufferExtension* tail_;
+
+  bool IsEmpty() {
+    DCHECK_IMPLIES(head_, tail_);
+    return head_ == nullptr;
+  }
+
+  void Reset() { head_ = tail_ = nullptr; }
+
+  void Append(ArrayBufferExtension* extension);
+  void Append(ArrayBufferList* list);
+
+  V8_EXPORT_PRIVATE bool Contains(ArrayBufferExtension* extension);
+};
+
+// The ArrayBufferSweeper iterates and deletes ArrayBufferExtensions
+// concurrently to the application.
+class ArrayBufferSweeper {
+ public:
+  explicit ArrayBufferSweeper(Heap* heap)
+      : heap_(heap), sweeping_in_progress_(false) {}
+  ~ArrayBufferSweeper() { ReleaseAll(); }
+
+  void EnsureFinished();
+  void RequestSweepYoung();
+  void RequestSweepFull();
+
+  void Append(JSArrayBuffer object, ArrayBufferExtension* extension);
+
+  ArrayBufferList young() { return young_; }
+  ArrayBufferList old() { return old_; }
+
+ private:
+  enum class SweepingScope { Young, Full };
+
+  enum class SweepingState { Uninitialized, Prepared, Swept };
+
+  struct SweepingJob {
+    CancelableTaskManager::Id id;
+    SweepingState state;
+    ArrayBufferList young;
+    ArrayBufferList old;
+    SweepingScope scope;
+
+    SweepingJob();
+
+    void Sweep();
+    void SweepYoung();
+    void SweepFull();
+    ArrayBufferList SweepListFull(ArrayBufferList* list);
+    static SweepingJob Prepare(ArrayBufferList young, ArrayBufferList old,
+                               SweepingScope scope);
+  } job_;
+
+  void Merge();
+
+  void RequestSweep(SweepingScope sweeping_task);
+  void Prepare(SweepingScope sweeping_task);
+
+  ArrayBufferList SweepListFull(ArrayBufferList* list);
+  ArrayBufferList SweepYoungGen();
+  void SweepOldGen(ArrayBufferExtension* extension);
+
+  void ReleaseAll();
+  void ReleaseAll(ArrayBufferList* extension);
+
+  Heap* const heap_;
+  bool sweeping_in_progress_;
+  base::Mutex sweeping_mutex_;
+  base::ConditionVariable job_finished_;
+
+  ArrayBufferList young_;
+  ArrayBufferList old_;
+};
+
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_HEAP_ARRAY_BUFFER_SWEEPER_H_

src/heap/gc-tracer.cc

@@ -562,6 +562,7 @@ void GCTracer::PrintNVP() const {
           "heap.external.epilogue=%.2f "
           "heap.external_weak_global_handles=%.2f "
           "fast_promote=%.2f "
+          "complete.sweep_array_buffers=%.2f "
           "scavenge=%.2f "
           "scavenge.process_array_buffers=%.2f "
           "scavenge.free_remembered_set=%.2f "
@@ -571,6 +572,7 @@ void GCTracer::PrintNVP() const {
           "scavenge.weak_global_handles.process=%.2f "
           "scavenge.parallel=%.2f "
           "scavenge.update_refs=%.2f "
+          "scavenge.sweep_array_buffers=%.2f "
           "background.scavenge.parallel=%.2f "
           "background.array_buffer_free=%.2f "
           "background.store_buffer=%.2f "
@@ -602,6 +604,7 @@ void GCTracer::PrintNVP() const {
           current_.scopes[Scope::HEAP_EXTERNAL_PROLOGUE],
           current_.scopes[Scope::HEAP_EXTERNAL_EPILOGUE],
           current_.scopes[Scope::HEAP_EXTERNAL_WEAK_GLOBAL_HANDLES],
+          current_.scopes[Scope::SCAVENGER_SWEEP_ARRAY_BUFFERS],
           current_.scopes[Scope::SCAVENGER_FAST_PROMOTE],
           current_.scopes[Scope::SCAVENGER_SCAVENGE],
           current_.scopes[Scope::SCAVENGER_PROCESS_ARRAY_BUFFERS],
@@ -614,6 +617,7 @@ void GCTracer::PrintNVP() const {
               .scopes[Scope::SCAVENGER_SCAVENGE_WEAK_GLOBAL_HANDLES_PROCESS],
           current_.scopes[Scope::SCAVENGER_SCAVENGE_PARALLEL],
           current_.scopes[Scope::SCAVENGER_SCAVENGE_UPDATE_REFS],
+          current_.scopes[Scope::SCAVENGER_SWEEP_ARRAY_BUFFERS],
           current_.scopes[Scope::SCAVENGER_BACKGROUND_SCAVENGE_PARALLEL],
           current_.scopes[Scope::BACKGROUND_ARRAY_BUFFER_FREE],
           current_.scopes[Scope::BACKGROUND_STORE_BUFFER],
@@ -712,6 +716,7 @@ void GCTracer::PrintNVP() const {
           "clear.weak_collections=%.1f "
           "clear.weak_lists=%.1f "
           "clear.weak_references=%.1f "
+          "complete.sweep_array_buffers=%.1f "
           "epilogue=%.1f "
           "evacuate=%.1f "
           "evacuate.candidates=%.1f "
@@ -726,6 +731,7 @@ void GCTracer::PrintNVP() const {
           "evacuate.update_pointers.slots.map_space=%.1f "
           "evacuate.update_pointers.weak=%.1f "
           "finish=%.1f "
+          "finish.sweep_array_buffers=%.1f "
           "mark=%.1f "
           "mark.finish_incremental=%.1f "
           "mark.roots=%.1f "
@@ -751,6 +757,7 @@ void GCTracer::PrintNVP() const {
           "incremental.finalize.external.epilogue=%.1f "
           "incremental.layout_change=%.1f "
           "incremental.start=%.1f "
+          "incremental.sweep_array_buffers=%.1f "
           "incremental.sweeping=%.1f "
           "incremental.embedder_prologue=%.1f "
           "incremental.embedder_tracing=%.1f "
@@ -803,6 +810,7 @@ void GCTracer::PrintNVP() const {
           current_.scopes[Scope::MC_CLEAR_WEAK_COLLECTIONS],
           current_.scopes[Scope::MC_CLEAR_WEAK_LISTS],
           current_.scopes[Scope::MC_CLEAR_WEAK_REFERENCES],
+          current_.scopes[Scope::MC_COMPLETE_SWEEP_ARRAY_BUFFERS],
           current_.scopes[Scope::MC_EPILOGUE],
           current_.scopes[Scope::MC_EVACUATE],
           current_.scopes[Scope::MC_EVACUATE_CANDIDATES],
@@ -816,7 +824,9 @@ void GCTracer::PrintNVP() const {
           current_.scopes[Scope::MC_EVACUATE_UPDATE_POINTERS_SLOTS_MAIN],
           current_.scopes[Scope::MC_EVACUATE_UPDATE_POINTERS_SLOTS_MAP_SPACE],
           current_.scopes[Scope::MC_EVACUATE_UPDATE_POINTERS_WEAK],
-          current_.scopes[Scope::MC_FINISH], current_.scopes[Scope::MC_MARK],
+          current_.scopes[Scope::MC_FINISH],
+          current_.scopes[Scope::MC_FINISH_SWEEP_ARRAY_BUFFERS],
+          current_.scopes[Scope::MC_MARK],
           current_.scopes[Scope::MC_MARK_FINISH_INCREMENTAL],
           current_.scopes[Scope::MC_MARK_ROOTS],
           current_.scopes[Scope::MC_MARK_MAIN],
@@ -840,6 +850,7 @@ void GCTracer::PrintNVP() const {
           current_.scopes[Scope::MC_INCREMENTAL_EXTERNAL_EPILOGUE],
           current_.scopes[Scope::MC_INCREMENTAL_LAYOUT_CHANGE],
           current_.scopes[Scope::MC_INCREMENTAL_START],
+          current_.scopes[Scope::MC_INCREMENTAL_SWEEP_ARRAY_BUFFERS],
           current_.scopes[Scope::MC_INCREMENTAL_SWEEPING],
           current_.scopes[Scope::MC_INCREMENTAL_EMBEDDER_PROLOGUE],
           current_.scopes[Scope::MC_INCREMENTAL_EMBEDDER_TRACING],

src/heap/heap.cc

@@ -26,6 +26,7 @@
 #include "src/execution/vm-state-inl.h"
 #include "src/handles/global-handles.h"
 #include "src/heap/array-buffer-collector.h"
+#include "src/heap/array-buffer-sweeper.h"
 #include "src/heap/array-buffer-tracker-inl.h"
 #include "src/heap/barrier.h"
 #include "src/heap/code-stats.h"
@@ -3796,30 +3797,7 @@ void Heap::RemoveNearHeapLimitCallback(v8::NearHeapLimitCallback callback,
 
 void Heap::AppendArrayBufferExtension(JSArrayBuffer object,
                                       ArrayBufferExtension* extension) {
-  if (Heap::InYoungGeneration(object)) {
-    extension->set_next(young_array_buffer_extensions_);
-    young_array_buffer_extensions_ = extension;
-  } else {
-    extension->set_next(old_array_buffer_extensions_);
-    old_array_buffer_extensions_ = extension;
-  }
-}
-
-void Heap::ReleaseAllArrayBufferExtensions() {
-  ReleaseAllArrayBufferExtensions(&old_array_buffer_extensions_);
-  ReleaseAllArrayBufferExtensions(&young_array_buffer_extensions_);
-}
-
-void Heap::ReleaseAllArrayBufferExtensions(ArrayBufferExtension** head) {
-  ArrayBufferExtension* current = *head;
-
-  while (current) {
-    ArrayBufferExtension* next = current->next();
-    delete current;
-    current = next;
-  }
-
-  *head = nullptr;
+  array_buffer_sweeper_->Append(object, extension);
 }
 
 void Heap::AutomaticallyRestoreInitialHeapLimit(double threshold_percent) {
@@ -5140,6 +5118,7 @@ void Heap::SetUpSpaces() {
   minor_mark_compact_collector_ = nullptr;
 #endif  // ENABLE_MINOR_MC
   array_buffer_collector_.reset(new ArrayBufferCollector(this));
+  array_buffer_sweeper_.reset(new ArrayBufferSweeper(this));
   gc_idle_time_handler_.reset(new GCIdleTimeHandler());
   memory_measurement_.reset(new MemoryMeasurement(isolate()));
   memory_reducer_.reset(new MemoryReducer(this));
@@ -5329,8 +5308,6 @@ void Heap::TearDown() {
   // It's too late for Heap::Verify() here, as parts of the Isolate are
   // already gone by the time this is called.
 
-  ReleaseAllArrayBufferExtensions();
-
   UpdateMaximumCommitted();
 
   if (FLAG_verify_predictable || FLAG_fuzzer_gc_analysis) {
@@ -5379,6 +5356,7 @@ void Heap::TearDown() {
 
   scavenger_collector_.reset();
   array_buffer_collector_.reset();
+  array_buffer_sweeper_.reset();
   incremental_marking_.reset();
   concurrent_marking_.reset();
 

src/heap/heap.h

@@ -60,6 +60,7 @@ using v8::MemoryPressureLevel;
 
 class AllocationObserver;
 class ArrayBufferCollector;
+class ArrayBufferSweeper;
 class CodeLargeObjectSpace;
 class ConcurrentMarking;
 class GCIdleTimeHandler;
@@ -588,27 +589,9 @@ class Heap {
   V8_EXPORT_PRIVATE void AutomaticallyRestoreInitialHeapLimit(
       double threshold_percent);
 
-  ArrayBufferExtension* old_array_buffer_extensions() {
-    return old_array_buffer_extensions_;
-  }
-
-  ArrayBufferExtension* young_array_buffer_extensions() {
-    return young_array_buffer_extensions_;
-  }
-
-  void set_old_array_buffer_extensions(ArrayBufferExtension* head) {
-    old_array_buffer_extensions_ = head;
-  }
-
-  void set_young_array_buffer_extensions(ArrayBufferExtension* head) {
-    young_array_buffer_extensions_ = head;
-  }
-
   void AppendArrayBufferExtension(JSArrayBuffer object,
                                   ArrayBufferExtension* extension);
 
-  void ReleaseAllArrayBufferExtensions();
-
   V8_EXPORT_PRIVATE double MonotonicallyIncreasingTimeInMs();
 
   void RecordStats(HeapStats* stats, bool take_snapshot = false);
@@ -763,6 +746,10 @@ class Heap {
     return array_buffer_collector_.get();
   }
 
+  ArrayBufferSweeper* array_buffer_sweeper() {
+    return array_buffer_sweeper_.get();
+  }
+
   // ===========================================================================
   // Root set access. ==========================================================
   // ===========================================================================
@@ -1423,8 +1410,6 @@ class Heap {
   static Isolate* GetIsolateFromWritableObject(HeapObject object);
 
  private:
-  void ReleaseAllArrayBufferExtensions(ArrayBufferExtension** head);
-
   using ExternalStringTableUpdaterCallback = String (*)(Heap* heap,
                                                         FullObjectSlot pointer);
 
@@ -2050,6 +2035,8 @@ class Heap {
   MinorMarkCompactCollector* minor_mark_compact_collector_ = nullptr;
   std::unique_ptr<ScavengerCollector> scavenger_collector_;
   std::unique_ptr<ArrayBufferCollector> array_buffer_collector_;
+  std::unique_ptr<ArrayBufferSweeper> array_buffer_sweeper_;
+
   std::unique_ptr<MemoryAllocator> memory_allocator_;
   std::unique_ptr<IncrementalMarking> incremental_marking_;
   std::unique_ptr<ConcurrentMarking> concurrent_marking_;
@@ -2151,6 +2138,7 @@ class Heap {
   // Classes in "heap" can be friends.
   friend class AlwaysAllocateScope;
   friend class ArrayBufferCollector;
+  friend class ArrayBufferSweeper;
   friend class ConcurrentMarking;
   friend class GCCallbacksScope;
   friend class GCTracer;

src/heap/incremental-marking.cc

@@ -6,6 +6,7 @@
 
 #include "src/codegen/compilation-cache.h"
 #include "src/execution/vm-state-inl.h"
+#include "src/heap/array-buffer-sweeper.h"
 #include "src/heap/concurrent-marking.h"
 #include "src/heap/embedder-tracing.h"
 #include "src/heap/gc-idle-time-handler.h"
@@ -293,6 +294,12 @@ void IncrementalMarking::Start(GarbageCollectionReason gc_reason) {
   should_hurry_ = false;
   was_activated_ = true;
 
+  {
+    TRACE_GC(heap()->tracer(),
+             GCTracer::Scope::MC_INCREMENTAL_SWEEP_ARRAY_BUFFERS);
+    heap_->array_buffer_sweeper()->EnsureFinished();
+  }
+
   if (!collector_->sweeping_in_progress()) {
     StartMarking();
   } else {

src/heap/mark-compact.cc

@@ -14,6 +14,7 @@
 #include "src/execution/vm-state-inl.h"
 #include "src/handles/global-handles.h"
 #include "src/heap/array-buffer-collector.h"
+#include "src/heap/array-buffer-sweeper.h"
 #include "src/heap/array-buffer-tracker-inl.h"
 #include "src/heap/gc-tracer.h"
 #include "src/heap/incremental-marking-inl.h"
@@ -837,6 +838,12 @@ void MarkCompactCollector::Prepare() {
   // Instead of waiting we could also abort the sweeper threads here.
   EnsureSweepingCompleted();
 
+  {
+    TRACE_GC(heap()->tracer(),
+             GCTracer::Scope::MC_COMPLETE_SWEEP_ARRAY_BUFFERS);
+    heap_->array_buffer_sweeper()->EnsureFinished();
+  }
+
   if (heap()->incremental_marking()->IsSweeping()) {
     heap()->incremental_marking()->Stop();
   }
@@ -937,52 +944,8 @@ void MarkCompactCollector::Finish() {
 }
 
 void MarkCompactCollector::SweepArrayBufferExtensions() {
-  ArrayBufferExtension* promoted_list = SweepYoungArrayBufferExtensions();
-  SweepOldArrayBufferExtensions(promoted_list);
-}
-
-void MarkCompactCollector::SweepOldArrayBufferExtensions(
-    ArrayBufferExtension* promoted_list) {
-  ArrayBufferExtension* current = heap_->old_array_buffer_extensions();
-  ArrayBufferExtension* last = promoted_list;
-
-  while (current) {
-    ArrayBufferExtension* next = current->next();
-
-    if (!current->IsMarked()) {
-      delete current;
-    } else {
-      current->Unmark();
-      current->set_next(last);
-      last = current;
-    }
-
-    current = next;
-  }
-
-  heap_->set_old_array_buffer_extensions(last);
-}
-
-ArrayBufferExtension* MarkCompactCollector::SweepYoungArrayBufferExtensions() {
-  ArrayBufferExtension* current = heap_->young_array_buffer_extensions();
-  ArrayBufferExtension* promoted_list = nullptr;
-
-  while (current) {
-    ArrayBufferExtension* next = current->next();
-
-    if (!current->IsMarked()) {
-      delete current;
-    } else {
-      current->Unmark();
-      current->set_next(promoted_list);
-      promoted_list = current;
-    }
-
-    current = next;
-  }
-
-  heap_->set_young_array_buffer_extensions(nullptr);
-  return promoted_list;
+  TRACE_GC(heap()->tracer(), GCTracer::Scope::MC_FINISH_SWEEP_ARRAY_BUFFERS);
+  heap_->array_buffer_sweeper()->RequestSweepFull();
 }
 
 class MarkCompactCollector::RootMarkingVisitor final : public RootVisitor {
@@ -4378,30 +4341,7 @@ void MinorMarkCompactCollector::CleanupSweepToIteratePages() {
 }
 
 void MinorMarkCompactCollector::SweepArrayBufferExtensions() {
-  ArrayBufferExtension* current = heap_->young_array_buffer_extensions();
-  ArrayBufferExtension* last_young = nullptr;
-  ArrayBufferExtension* last_old = heap_->old_array_buffer_extensions();
-
-  while (current) {
-    ArrayBufferExtension* next = current->next();
-
-    if (!current->IsYoungMarked()) {
-      delete current;
-    } else if (current->IsYoungPromoted()) {
-      current->YoungUnmark();
-      current->set_next(last_old);
-      last_old = current;
-    } else {
-      current->YoungUnmark();
-      current->set_next(last_young);
-      last_young = current;
-    }
-
-    current = next;
-  }
-
-  heap_->set_old_array_buffer_extensions(last_old);
-  heap_->set_young_array_buffer_extensions(last_young);
+  heap_->array_buffer_sweeper()->RequestSweepYoung();
 }
 
 class YoungGenerationMigrationObserver final : public MigrationObserver {
@@ -4566,6 +4506,8 @@ void MinorMarkCompactCollector::CollectGarbage() {
     CleanupSweepToIteratePages();
   }
 
+  heap()->array_buffer_sweeper()->EnsureFinished();
+
   MarkLiveObjects();
   ClearNonLiveReferences();
 #ifdef VERIFY_HEAP

src/heap/mark-compact.h

@@ -610,8 +610,6 @@ class MarkCompactCollector final : public MarkCompactCollectorBase {
 
   // Free unmarked ArrayBufferExtensions.
   void SweepArrayBufferExtensions();
-  void SweepOldArrayBufferExtensions(ArrayBufferExtension* promoted_list);
-  ArrayBufferExtension* SweepYoungArrayBufferExtensions();
 
   void MarkLiveObjects() override;
 

src/heap/scavenger.cc

@@ -5,6 +5,7 @@
 #include "src/heap/scavenger.h"
 
 #include "src/heap/array-buffer-collector.h"
+#include "src/heap/array-buffer-sweeper.h"
 #include "src/heap/barrier.h"
 #include "src/heap/gc-tracer.h"
 #include "src/heap/heap-inl.h"
@@ -236,6 +237,13 @@ class ScopedFullHeapCrashKey {
 
 void ScavengerCollector::CollectGarbage() {
   ScopedFullHeapCrashKey collect_full_heap_dump_if_crash(isolate_);
+
+  {
+    TRACE_GC(heap_->tracer(),
+             GCTracer::Scope::SCAVENGER_COMPLETE_SWEEP_ARRAY_BUFFERS);
+    heap_->array_buffer_sweeper()->EnsureFinished();
+  }
+
   DCHECK(surviving_new_large_objects_.empty());
   ItemParallelJob job(isolate_->cancelable_task_manager(),
                       &parallel_scavenge_semaphore_);
@@ -378,37 +386,17 @@ void ScavengerCollector::CollectGarbage() {
 #endif
   }
 
-  SweepArrayBufferExtensions();
+  {
+    TRACE_GC(heap_->tracer(), GCTracer::Scope::SCAVENGER_SWEEP_ARRAY_BUFFERS);
+    SweepArrayBufferExtensions();
+  }
 
   // Update how much has survived scavenge.
   heap_->IncrementYoungSurvivorsCounter(heap_->SurvivedYoungObjectSize());
 }
 
 void ScavengerCollector::SweepArrayBufferExtensions() {
-  ArrayBufferExtension* current = heap_->young_array_buffer_extensions();
-  ArrayBufferExtension* last_young = nullptr;
-  ArrayBufferExtension* last_old = heap_->old_array_buffer_extensions();
-
-  while (current) {
-    ArrayBufferExtension* next = current->next();
-
-    if (!current->IsYoungMarked()) {
-      delete current;
-    } else if (current->IsYoungPromoted()) {
-      current->YoungUnmark();
-      current->set_next(last_old);
-      last_old = current;
-    } else {
-      current->YoungUnmark();
-      current->set_next(last_young);
-      last_young = current;
-    }
-
-    current = next;
-  }
-
-  heap_->set_old_array_buffer_extensions(last_old);
-  heap_->set_young_array_buffer_extensions(last_young);
+  heap_->array_buffer_sweeper()->RequestSweepYoung();
 }
 
 void ScavengerCollector::HandleSurvivingNewLargeObjects() {

src/init/heap-symbols.h

@@ -394,6 +394,7 @@
   F(MC_INCREMENTAL_FINALIZE_BODY)                                  \
   F(MC_INCREMENTAL_LAYOUT_CHANGE)                                  \
   F(MC_INCREMENTAL_START)                                          \
+  F(MC_INCREMENTAL_SWEEP_ARRAY_BUFFERS)                            \
   F(MC_INCREMENTAL_SWEEPING)
 
 #define TOP_MC_SCOPES(F) \
@@ -425,6 +426,7 @@
   F(MC_CLEAR_WEAK_COLLECTIONS)                       \
   F(MC_CLEAR_WEAK_LISTS)                             \
   F(MC_CLEAR_WEAK_REFERENCES)                        \
+  F(MC_COMPLETE_SWEEP_ARRAY_BUFFERS)                 \
   F(MC_EVACUATE_CANDIDATES)                          \
   F(MC_EVACUATE_CLEAN_UP)                            \
   F(MC_EVACUATE_COPY)                                \
@@ -438,6 +440,7 @@
   F(MC_EVACUATE_UPDATE_POINTERS_SLOTS_MAP_SPACE)     \
   F(MC_EVACUATE_UPDATE_POINTERS_TO_NEW_ROOTS)        \
   F(MC_EVACUATE_UPDATE_POINTERS_WEAK)                \
+  F(MC_FINISH_SWEEP_ARRAY_BUFFERS)                   \
   F(MC_MARK_EMBEDDER_PROLOGUE)                       \
   F(MC_MARK_EMBEDDER_TRACING)                        \
   F(MC_MARK_EMBEDDER_TRACING_CLOSURE)                \
@@ -479,6 +482,7 @@
   F(MINOR_MC_MARKING_DEQUE)                          \
   F(MINOR_MC_RESET_LIVENESS)                         \
   F(MINOR_MC_SWEEPING)                               \
+  F(SCAVENGER_COMPLETE_SWEEP_ARRAY_BUFFERS)          \
   F(SCAVENGER_FAST_PROMOTE)                          \
   F(SCAVENGER_FREE_REMEMBERED_SET)                   \
   F(SCAVENGER_SCAVENGE)                              \
@@ -489,10 +493,12 @@
   F(SCAVENGER_SCAVENGE_ROOTS)                        \
   F(SCAVENGER_SCAVENGE_UPDATE_REFS)                  \
   F(SCAVENGER_SCAVENGE_WEAK)                         \
-  F(SCAVENGER_SCAVENGE_FINALIZE)
+  F(SCAVENGER_SCAVENGE_FINALIZE)                     \
+  F(SCAVENGER_SWEEP_ARRAY_BUFFERS)
 
 #define TRACER_BACKGROUND_SCOPES(F)               \
   F(BACKGROUND_ARRAY_BUFFER_FREE)                 \
+  F(BACKGROUND_ARRAY_BUFFER_SWEEP)                \
   F(BACKGROUND_STORE_BUFFER)                      \
   F(BACKGROUND_UNMAPPER)                          \
   F(MC_BACKGROUND_EVACUATE_COPY)                  \

test/cctest/heap/test-array-buffer-tracker.cc

@@ -4,6 +4,7 @@
 
 #include "src/api/api-inl.h"
 #include "src/execution/isolate.h"
+#include "src/heap/array-buffer-sweeper.h"
 #include "src/heap/array-buffer-tracker.h"
 #include "src/heap/heap-inl.h"
 #include "src/heap/spaces.h"
@@ -20,37 +21,25 @@ bool IsTracked(i::JSArrayBuffer buf) {
   return i::ArrayBufferTracker::IsTracked(buf);
 }
 
-bool LookupExtension(i::ArrayBufferExtension* head,
-                     i::ArrayBufferExtension* extension) {
-  i::ArrayBufferExtension* current = head;
-
-  while (current) {
-    if (current == extension) return true;
-    current = current->next();
-  }
-
-  return false;
-}
-
 bool IsTrackedYoung(i::Heap* heap, i::ArrayBufferExtension* extension) {
-  bool in_young =
-      LookupExtension(heap->young_array_buffer_extensions(), extension);
-  bool in_old = LookupExtension(heap->old_array_buffer_extensions(), extension);
-  return in_young && !in_old;
+  bool in_young = heap->array_buffer_sweeper()->young().Contains(extension);
+  bool in_old = heap->array_buffer_sweeper()->old().Contains(extension);
+  CHECK(!(in_young && in_old));
+  return in_young;
 }
 
 bool IsTrackedOld(i::Heap* heap, i::ArrayBufferExtension* extension) {
-  bool in_young =
-      LookupExtension(heap->young_array_buffer_extensions(), extension);
-  bool in_old = LookupExtension(heap->old_array_buffer_extensions(), extension);
-  return in_old && !in_young;
+  bool in_young = heap->array_buffer_sweeper()->young().Contains(extension);
+  bool in_old = heap->array_buffer_sweeper()->old().Contains(extension);
+  CHECK(!(in_young && in_old));
+  return in_old;
 }
 
 bool IsTracked(i::Heap* heap, i::ArrayBufferExtension* extension) {
-  bool in_young =
-      LookupExtension(heap->young_array_buffer_extensions(), extension);
-  bool in_old = LookupExtension(heap->old_array_buffer_extensions(), extension);
-  return in_young != in_old;
+  bool in_young = heap->array_buffer_sweeper()->young().Contains(extension);
+  bool in_old = heap->array_buffer_sweeper()->old().Contains(extension);
+  CHECK(!(in_young && in_old));
+  return in_young || in_old;
 }
 
 }  // namespace
@@ -93,6 +82,7 @@ TEST(ArrayBuffer_OnlyMC) {
 
 TEST(ArrayBuffer_OnlyMC_Extension) {
   if (!V8_ARRAY_BUFFER_EXTENSION_BOOL) return;
+  FLAG_concurrent_array_buffer_sweeping = false;
 
   ManualGCScope manual_gc_scope;
   CcTest::InitializeVM();
@@ -149,6 +139,7 @@ TEST(ArrayBuffer_OnlyScavenge) {
 
 TEST(ArrayBuffer_OnlyScavenge_Extension) {
   if (!V8_ARRAY_BUFFER_EXTENSION_BOOL) return;
+  FLAG_concurrent_array_buffer_sweeping = false;
 
   ManualGCScope manual_gc_scope;
   CcTest::InitializeVM();
@@ -209,6 +200,8 @@ TEST(ArrayBuffer_ScavengeAndMC) {
 
 TEST(ArrayBuffer_ScavengeAndMC_Extension) {
   if (!V8_ARRAY_BUFFER_EXTENSION_BOOL) return;
+  FLAG_concurrent_array_buffer_sweeping = false;
+
   ManualGCScope manual_gc_scope;
   CcTest::InitializeVM();
   LocalContext env;
@@ -424,6 +417,8 @@ TEST(ArrayBuffer_SemiSpaceCopyThenPagePromotion) {
 TEST(ArrayBuffer_PagePromotion_Extension) {
   if (!i::FLAG_incremental_marking || !V8_ARRAY_BUFFER_EXTENSION_BOOL) return;
   i::FLAG_always_promote_young_mc = true;
+  i::FLAG_concurrent_array_buffer_sweeping = false;
+
   ManualGCScope manual_gc_scope;
   // The test verifies that the marking state is preserved across semispace
   // copy.