[heap] Scavenger: Introduce list of copied objects

Bug: 
Change-Id: I6953e494632684b9e2bc9a1bf7f7801eea25cdcb
Reviewed-on: https://chromium-review.googlesource.com/563212
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#46479}

src/heap/heap.cc

@@ -1723,31 +1723,13 @@ void Heap::Scavenge() {
   new_space_->Flip();
   new_space_->ResetAllocationInfo();
 
-  // We need to sweep newly copied objects which can be either in the
-  // to space or promoted to the old generation.  For to-space
-  // objects, we treat the bottom of the to space as a queue.  Newly
-  // copied and unswept objects lie between a 'front' mark and the
-  // allocation pointer.
-  //
-  // Promoted objects can go into various old-generation spaces, and
-  // can be allocated internally in the spaces (from the free list).
-  // We treat the top of the to space as a queue of addresses of
-  // promoted objects.  The addresses of newly promoted and unswept
-  // objects lie between a 'front' mark and a 'rear' mark that is
-  // updated as a side effect of promoting an object.
-  //
-  // There is guaranteed to be enough room at the top of the to space
-  // for the addresses of promoted objects: every object promoted
-  // frees up its size in bytes from the top of the new space, and
-  // objects are at least one pointer in size.
-  Address new_space_front = new_space_->ToSpaceStart();
-
   const int kScavengerTasks = 1;
   const int kMainThreadId = 0;
+  CopiedList copied_list(kScavengerTasks);
   PromotionList promotion_list(kScavengerTasks);
   Scavenger scavenger(this, IsLogging(isolate()),
-                      incremental_marking()->IsMarking(), &promotion_list,
-                      kMainThreadId);
+                      incremental_marking()->IsMarking(), &copied_list,
+                      &promotion_list, kMainThreadId);
   RootScavengeVisitor root_scavenge_visitor(this, &scavenger);
 
   isolate()->global_handles()->IdentifyWeakUnmodifiedObjects(
@@ -1788,7 +1770,7 @@ void Heap::Scavenge() {
 
   {
     TRACE_GC(tracer(), GCTracer::Scope::SCAVENGER_SEMISPACE);
-    new_space_front = DoScavenge(&scavenger, new_space_front);
+    DoScavenge(&scavenger);
   }
 
   isolate()->global_handles()->MarkNewSpaceWeakUnmodifiedObjectsPending(
@@ -1796,7 +1778,7 @@ void Heap::Scavenge() {
 
   isolate()->global_handles()->IterateNewSpaceWeakUnmodifiedRoots(
       &root_scavenge_visitor);
-  new_space_front = DoScavenge(&scavenger, new_space_front);
+  DoScavenge(&scavenger);
 
   UpdateNewSpaceReferencesInExternalStringTable(
       &UpdateNewSpaceReferenceInExternalStringTableEntry);
@@ -1806,8 +1788,6 @@ void Heap::Scavenge() {
   ScavengeWeakObjectRetainer weak_object_retainer(this);
   ProcessYoungWeakReferences(&weak_object_retainer);
 
-  DCHECK(new_space_front == new_space_->top());
-
   // Set age mark.
   new_space_->set_age_mark(new_space_->top());
 
@@ -2002,43 +1982,38 @@ void Heap::VisitExternalResources(v8::ExternalResourceVisitor* visitor) {
   external_string_table_.IterateAll(&external_string_table_visitor);
 }
 
-Address Heap::DoScavenge(Scavenger* scavenger, Address new_space_front) {
-  // Threshold when to switch processing the promotion list to avoid allocating
-  // too much backing store in the worklist.
+void Heap::DoScavenge(Scavenger* scavenger) {
+  // Threshold when to switch processing the promotion list to avoid
+  // allocating too much backing store in the worklist.
   const int kProcessPromotionListThreshold = kPromotionListSegmentSize / 2;
   ScavengeVisitor scavenge_visitor(this, scavenger);
   PromotionList::View* promotion_list = scavenger->promotion_list();
+  CopiedRangesList* copied_list = scavenger->copied_list();
+
+  bool done;
   do {
-    SemiSpace::AssertValidRange(new_space_front, new_space_->top());
-    // The addresses new_space_front and new_space_.top() define a
-    // queue of unprocessed copied objects.  Process them until the
-    // queue is empty.
+    done = true;
+    AddressRange range;
     while ((promotion_list->LocalPushSegmentSize() <
             kProcessPromotionListThreshold) &&
-           new_space_front != new_space_->top()) {
-      if (!Page::IsAlignedToPageSize(new_space_front)) {
-        HeapObject* object = HeapObject::FromAddress(new_space_front);
-        new_space_front += scavenge_visitor.Visit(object);
-      } else {
-        new_space_front = Page::FromAllocationAreaAddress(new_space_front)
-                              ->next_page()
-                              ->area_start();
+           copied_list->Pop(&range)) {
+      for (Address current = range.first; current < range.second;) {
+        HeapObject* object = HeapObject::FromAddress(current);
+        int size = object->Size();
+        scavenge_visitor.Visit(object);
+        current += size;
       }
+      done = false;
     }
-
     ObjectAndSize object_and_size;
     while (promotion_list->Pop(&object_and_size)) {
       HeapObject* target = object_and_size.first;
       int size = object_and_size.second;
       DCHECK(!target->IsMap());
       IterateAndScavengePromotedObject(scavenger, target, size);
+      done = false;
     }
-
-    // Take another spin if there are now unswept objects in new space
-    // (there are currently no more unswept promoted objects).
-  } while (new_space_front != new_space_->top());
-
-  return new_space_front;
+  } while (!done);
 }
 
 

src/heap/heap.h

@@ -1783,7 +1783,7 @@ class Heap {
   void Scavenge();
   void EvacuateYoungGeneration();
 
-  Address DoScavenge(Scavenger* scavenger, Address new_space_front);
+  void DoScavenge(Scavenger* scavenger);
 
   void UpdateNewSpaceReferencesInExternalStringTable(
       ExternalStringTableUpdaterCallback updater_func);

src/heap/scavenger-inl.h

@@ -78,6 +78,7 @@ bool Scavenger::SemiSpaceCopyObject(Map* map, HeapObject** slot,
     // Update slot to new target.
     *slot = target;
 
+    copied_list()->Insert(target, object_size);
     heap()->IncrementSemiSpaceCopiedObjectSize(object_size);
     return true;
   }

src/heap/scavenger.h

@@ -12,17 +12,67 @@
 namespace v8 {
 namespace internal {
 
+static const int kCopiedListSegmentSize = 64;
 static const int kPromotionListSegmentSize = 64;
 
+using AddressRange = std::pair<Address, Address>;
+using CopiedList = Worklist<AddressRange, kCopiedListSegmentSize>;
 using ObjectAndSize = std::pair<HeapObject*, int>;
 using PromotionList = Worklist<ObjectAndSize, kPromotionListSegmentSize>;
 
+// A list of copied ranges. Keeps the last consecutive range local and announces
+// all other ranges to a global work list.
+class CopiedRangesList {
+ public:
+  CopiedRangesList(CopiedList* copied_list, int task_id)
+      : current_start_(nullptr),
+        current_end_(nullptr),
+        copied_list_(copied_list, task_id) {}
+
+  ~CopiedRangesList() {
+    CHECK_NULL(current_start_);
+    CHECK_NULL(current_end_);
+  }
+
+  void Insert(HeapObject* object, int size) {
+    const Address object_address = object->address();
+    if (current_end_ != object_address) {
+      if (current_start_ != nullptr) {
+        copied_list_.Push(AddressRange(current_start_, current_end_));
+      }
+      current_start_ = object_address;
+      current_end_ = current_start_ + size;
+      return;
+    }
+    DCHECK_EQ(current_end_, object_address);
+    current_end_ += size;
+    return;
+  }
+
+  bool Pop(AddressRange* entry) {
+    if (copied_list_.Pop(entry)) {
+      return true;
+    } else if (current_start_ != nullptr) {
+      *entry = AddressRange(current_start_, current_end_);
+      current_start_ = current_end_ = nullptr;
+      return true;
+    }
+    return false;
+  }
+
+ private:
+  Address current_start_;
+  Address current_end_;
+  CopiedList::View copied_list_;
+};
+
 class Scavenger {
  public:
   Scavenger(Heap* heap, bool is_logging, bool is_incremental_marking,
-            PromotionList* promotion_list, int task_id)
+            CopiedList* copied_list, PromotionList* promotion_list, int task_id)
       : heap_(heap),
         promotion_list_(promotion_list, task_id),
+        copied_list_(copied_list, task_id),
         is_logging_(is_logging),
         is_incremental_marking_(is_incremental_marking) {}
 
@@ -36,6 +86,7 @@ class Scavenger {
                                                    Address slot_address);
   inline Heap* heap() { return heap_; }
   inline PromotionList::View* promotion_list() { return &promotion_list_; }
+  inline CopiedRangesList* copied_list() { return &copied_list_; }
 
  private:
   V8_INLINE HeapObject* MigrateObject(HeapObject* source, HeapObject* target,
@@ -68,6 +119,7 @@ class Scavenger {
 
   Heap* const heap_;
   PromotionList::View promotion_list_;
+  CopiedRangesList copied_list_;
   bool is_logging_;
   bool is_incremental_marking_;
 };