[heap][ptr-compr] Use Heap::CopyRange() instead of tight copy loops

Also remove unused Factory::CopyFeedbackVector().

Bug: v8:7703
Change-Id: I75e16a55967e5970e4cbe3babae3a09d2a647313
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1611542Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Commit-Queue: Igor Sheludko <ishell@chromium.org>
Cr-Commit-Position: refs/heads/master@{#61512}

src/heap/factory.cc

@@ -2114,18 +2114,11 @@ Handle<T> Factory::CopyArrayWithMap(Handle<T> src, Handle<Map> map) {
   obj->set_map_after_allocation(*map, SKIP_WRITE_BARRIER);
 
   Handle<T> result(T::cast(obj), isolate());
+  initialize_length(result, len);
+
   DisallowHeapAllocation no_gc;
   WriteBarrierMode mode = result->GetWriteBarrierMode(no_gc);
-
-  if (mode == SKIP_WRITE_BARRIER) {
-    // Eliminate the write barrier if possible.
-    Heap::CopyBlock(obj->address() + kTaggedSize, src->address() + kTaggedSize,
-                    T::SizeFor(len) - kTaggedSize);
-  } else {
-    // Slow case: Just copy the content one-by-one.
-    initialize_length(result, len);
-    for (int i = 0; i < len; i++) result->set(i, src->get(i), mode);
-  }
+  result->CopyElements(isolate(), 0, *src, 0, len, mode);
   return result;
 }
 
@@ -2145,8 +2138,9 @@ Handle<T> Factory::CopyArrayAndGrow(Handle<T> src, int grow_by,
   // Copy the content.
   DisallowHeapAllocation no_gc;
   WriteBarrierMode mode = obj->GetWriteBarrierMode(no_gc);
-  for (int i = 0; i < old_len; i++) result->set(i, src->get(i), mode);
-  MemsetTagged(result->data_start() + old_len, *undefined_value(), grow_by);
+  result->CopyElements(isolate(), 0, *src, 0, old_len, mode);
+  MemsetTagged(ObjectSlot(result->data_start() + old_len),
+               ReadOnlyRoots(isolate()).undefined_value(), grow_by);
   return result;
 }
 
@@ -2163,25 +2157,8 @@ Handle<FixedArray> Factory::CopyFixedArrayAndGrow(Handle<FixedArray> array,
 
 Handle<WeakFixedArray> Factory::CopyWeakFixedArrayAndGrow(
     Handle<WeakFixedArray> src, int grow_by, AllocationType allocation) {
-  DCHECK(
-      !src->IsTransitionArray());  // Compacted by GC, this code doesn't work.
-  int old_len = src->length();
-  int new_len = old_len + grow_by;
-  DCHECK_GE(new_len, old_len);
-  HeapObject obj = AllocateRawFixedArray(new_len, allocation);
-  DCHECK_EQ(old_len, src->length());
-  obj->set_map_after_allocation(src->map(), SKIP_WRITE_BARRIER);
-
-  WeakFixedArray result = WeakFixedArray::cast(obj);
-  result->set_length(new_len);
-
-  // Copy the content.
-  DisallowHeapAllocation no_gc;
-  WriteBarrierMode mode = obj->GetWriteBarrierMode(no_gc);
-  for (int i = 0; i < old_len; i++) result->Set(i, src->Get(i), mode);
-  MemsetTagged(ObjectSlot(result->RawFieldOfElementAt(old_len)),
-               ReadOnlyRoots(isolate()).undefined_value(), grow_by);
-  return Handle<WeakFixedArray>(result, isolate());
+  DCHECK(!src->IsTransitionArray());  // Compacted by GC, this code doesn't work
+  return CopyArrayAndGrow(src, grow_by, allocation);
 }
 
 Handle<WeakArrayList> Factory::CopyWeakArrayListAndGrow(
@@ -2193,15 +2170,17 @@ Handle<WeakArrayList> Factory::CopyWeakArrayListAndGrow(
   obj->set_map_after_allocation(src->map(), SKIP_WRITE_BARRIER);
 
   WeakArrayList result = WeakArrayList::cast(obj);
-  result->set_length(src->length());
+  int old_len = src->length();
+  result->set_length(old_len);
   result->set_capacity(new_capacity);
 
   // Copy the content.
   DisallowHeapAllocation no_gc;
   WriteBarrierMode mode = obj->GetWriteBarrierMode(no_gc);
-  for (int i = 0; i < old_capacity; i++) result->Set(i, src->Get(i), mode);
-  MemsetTagged(ObjectSlot(result->data_start() + old_capacity),
-               ReadOnlyRoots(isolate()).undefined_value(), grow_by);
+  result->CopyElements(isolate(), 0, *src, 0, old_len, mode);
+  MemsetTagged(ObjectSlot(result->data_start() + old_len),
+               ReadOnlyRoots(isolate()).undefined_value(),
+               new_capacity - old_len);
   return Handle<WeakArrayList>(result, isolate());
 }
 
@@ -2225,7 +2204,7 @@ Handle<FixedArray> Factory::CopyFixedArrayUpTo(Handle<FixedArray> array,
   // Copy the content.
   DisallowHeapAllocation no_gc;
   WriteBarrierMode mode = result->GetWriteBarrierMode(no_gc);
-  for (int i = 0; i < new_len; i++) result->set(i, array->get(i), mode);
+  result->CopyElements(isolate(), 0, *array, 0, new_len, mode);
   return result;
 }
 
@@ -2260,34 +2239,6 @@ Handle<FixedDoubleArray> Factory::CopyFixedDoubleArray(
   return result;
 }
 
-Handle<FeedbackVector> Factory::CopyFeedbackVector(
-    Handle<FeedbackVector> array) {
-  int len = array->length();
-  HeapObject obj = AllocateRawWithImmortalMap(FeedbackVector::SizeFor(len),
-                                              AllocationType::kYoung,
-                                              *feedback_vector_map());
-  Handle<FeedbackVector> result(FeedbackVector::cast(obj), isolate());
-
-  DisallowHeapAllocation no_gc;
-  WriteBarrierMode mode = result->GetWriteBarrierMode(no_gc);
-
-  // Eliminate the write barrier if possible.
-  if (mode == SKIP_WRITE_BARRIER) {
-    Heap::CopyBlock(result->address() + kTaggedSize,
-                    result->address() + kTaggedSize,
-                    FeedbackVector::SizeFor(len) - kTaggedSize);
-  } else {
-    // Slow case: Just copy the content one-by-one.
-    result->set_shared_function_info(array->shared_function_info());
-    result->set_optimized_code_weak_or_smi(array->optimized_code_weak_or_smi());
-    result->set_invocation_count(array->invocation_count());
-    result->set_profiler_ticks(array->profiler_ticks());
-    result->set_deopt_count(array->deopt_count());
-    for (int i = 0; i < len; i++) result->set(i, array->get(i), mode);
-  }
-  return result;
-}
-
 Handle<Object> Factory::NewNumber(double value, AllocationType allocation) {
   // Materialize as a SMI if possible.
   int32_t int_value;

src/heap/factory.h

@@ -567,8 +567,6 @@ class V8_EXPORT_PRIVATE Factory {
 
   Handle<FixedDoubleArray> CopyFixedDoubleArray(Handle<FixedDoubleArray> array);
 
-  Handle<FeedbackVector> CopyFeedbackVector(Handle<FeedbackVector> array);
-
   // Numbers (e.g. literals) are pretenured by the parser.
   // The return value may be a smi or a heap number.
   Handle<Object> NewNumber(double value,

src/heap/heap.cc

@@ -1542,13 +1542,23 @@ void Heap::MoveRange(HeapObject dst_object, const ObjectSlot dst_slot,
   WriteBarrierForRange(dst_object, dst_slot, dst_end);
 }
 
-void Heap::CopyRange(HeapObject dst_object, const ObjectSlot dst_slot,
-                     const ObjectSlot src_slot, int len,
-                     WriteBarrierMode mode) {
+// Instantiate Heap::CopyRange() for ObjectSlot and MaybeObjectSlot.
+template void Heap::CopyRange<ObjectSlot>(HeapObject dst_object,
+                                          ObjectSlot dst_slot,
+                                          ObjectSlot src_slot, int len,
+                                          WriteBarrierMode mode);
+template void Heap::CopyRange<MaybeObjectSlot>(HeapObject dst_object,
+                                               MaybeObjectSlot dst_slot,
+                                               MaybeObjectSlot src_slot,
+                                               int len, WriteBarrierMode mode);
+
+template <typename TSlot>
+void Heap::CopyRange(HeapObject dst_object, const TSlot dst_slot,
+                     const TSlot src_slot, int len, WriteBarrierMode mode) {
   DCHECK_NE(len, 0);
 
   DCHECK_NE(dst_object->map(), ReadOnlyRoots(this).fixed_cow_array_map());
-  const ObjectSlot dst_end(dst_slot + len);
+  const TSlot dst_end(dst_slot + len);
   // Ensure ranges do not overlap.
   DCHECK(dst_end <= src_slot || (src_slot + len) <= dst_slot);
 
@@ -5844,10 +5854,9 @@ enum RangeWriteBarrierMode {
   kDoEvacuationSlotRecording = 1 << 2,
 };
 
-template <int kModeMask>
+template <int kModeMask, typename TSlot>
 void Heap::WriteBarrierForRangeImpl(MemoryChunk* source_page, HeapObject object,
-                                    ObjectSlot start_slot,
-                                    ObjectSlot end_slot) {
+                                    TSlot start_slot, TSlot end_slot) {
   // At least one of generational or marking write barrier should be requested.
   STATIC_ASSERT(kModeMask & (kDoGenerational | kDoMarking));
   // kDoEvacuationSlotRecording implies kDoMarking.
@@ -5858,8 +5867,8 @@ void Heap::WriteBarrierForRangeImpl(MemoryChunk* source_page, HeapObject object,
   IncrementalMarking* incremental_marking = this->incremental_marking();
   MarkCompactCollector* collector = this->mark_compact_collector();
 
-  for (ObjectSlot slot = start_slot; slot < end_slot; ++slot) {
-    Object value = *slot;
+  for (TSlot slot = start_slot; slot < end_slot; ++slot) {
+    typename TSlot::TObject value = *slot;
     HeapObject value_heap_object;
     if (!value.GetHeapObject(&value_heap_object)) continue;
 
@@ -5878,8 +5887,16 @@ void Heap::WriteBarrierForRangeImpl(MemoryChunk* source_page, HeapObject object,
   }
 }
 
-void Heap::WriteBarrierForRange(HeapObject object, ObjectSlot start_slot,
-                                ObjectSlot end_slot) {
+// Instantiate Heap::WriteBarrierForRange() for ObjectSlot and MaybeObjectSlot.
+template void Heap::WriteBarrierForRange<ObjectSlot>(HeapObject object,
+                                                     ObjectSlot start_slot,
+                                                     ObjectSlot end_slot);
+template void Heap::WriteBarrierForRange<MaybeObjectSlot>(
+    HeapObject object, MaybeObjectSlot start_slot, MaybeObjectSlot end_slot);
+
+template <typename TSlot>
+void Heap::WriteBarrierForRange(HeapObject object, TSlot start_slot,
+                                TSlot end_slot) {
   MemoryChunk* source_page = MemoryChunk::FromHeapObject(object);
   base::Flags<RangeWriteBarrierMode> mode;
 

src/heap/heap.h

@@ -353,8 +353,9 @@ class Heap {
 
   // Executes generational and/or marking write barrier for a [start, end) range
   // of non-weak slots inside |object|.
-  V8_EXPORT_PRIVATE void WriteBarrierForRange(HeapObject object,
-                                              ObjectSlot start, ObjectSlot end);
+  template <typename TSlot>
+  V8_EXPORT_PRIVATE void WriteBarrierForRange(HeapObject object, TSlot start,
+                                              TSlot end);
 
   V8_EXPORT_PRIVATE static void WriteBarrierForCodeSlow(Code host);
   V8_EXPORT_PRIVATE static void GenerationalBarrierSlow(HeapObject object,
@@ -397,8 +398,9 @@ class Heap {
 
   // Copy len non-weak tagged elements from src_slot to dst_slot of dst_object.
   // The source and destination memory ranges must not overlap.
-  void CopyRange(HeapObject dst_object, ObjectSlot dst_slot,
-                 ObjectSlot src_slot, int len, WriteBarrierMode mode);
+  template <typename TSlot>
+  void CopyRange(HeapObject dst_object, TSlot dst_slot, TSlot src_slot, int len,
+                 WriteBarrierMode mode);
 
   // Initialize a filler object to keep the ability to iterate over the heap
   // when introducing gaps within pages. If slots could have been recorded in
@@ -1504,11 +1506,10 @@ class Heap {
                                        int size_in_bytes);
 
   // Range write barrier implementation.
-  template <int kModeMask>
+  template <int kModeMask, typename TSlot>
   V8_INLINE void WriteBarrierForRangeImpl(MemoryChunk* source_page,
-                                          HeapObject object,
-                                          ObjectSlot start_slot,
-                                          ObjectSlot end_slot);
+                                          HeapObject object, TSlot start_slot,
+                                          TSlot end_slot);
 
   // Deopts all code that contains allocation instruction which are tenured or
   // not tenured. Moreover it clears the pretenuring allocation site statistics.

src/objects/fixed-array-inl.h

@@ -226,6 +226,8 @@ ObjectSlot FixedArray::RawFieldOfElementAt(int index) {
 void FixedArray::MoveElements(Isolate* isolate, int dst_index, int src_index,
                               int len, WriteBarrierMode mode) {
   if (len == 0) return;
+  DCHECK_LE(dst_index + len, length());
+  DCHECK_LE(src_index + len, length());
   DisallowHeapAllocation no_gc;
   ObjectSlot dst_slot(RawFieldOfElementAt(dst_index));
   ObjectSlot src_slot(RawFieldOfElementAt(src_index));
@@ -235,6 +237,8 @@ void FixedArray::MoveElements(Isolate* isolate, int dst_index, int src_index,
 void FixedArray::CopyElements(Isolate* isolate, int dst_index, FixedArray src,
                               int src_index, int len, WriteBarrierMode mode) {
   if (len == 0) return;
+  DCHECK_LE(dst_index + len, length());
+  DCHECK_LE(src_index + len, src.length());
   DisallowHeapAllocation no_gc;
 
   ObjectSlot dst_slot(RawFieldOfElementAt(dst_index));
@@ -443,6 +447,19 @@ MaybeObjectSlot WeakFixedArray::RawFieldOfElementAt(int index) {
   return RawMaybeWeakField(OffsetOfElementAt(index));
 }
 
+void WeakFixedArray::CopyElements(Isolate* isolate, int dst_index,
+                                  WeakFixedArray src, int src_index, int len,
+                                  WriteBarrierMode mode) {
+  if (len == 0) return;
+  DCHECK_LE(dst_index + len, length());
+  DCHECK_LE(src_index + len, src.length());
+  DisallowHeapAllocation no_gc;
+
+  MaybeObjectSlot dst_slot(data_start() + dst_index);
+  MaybeObjectSlot src_slot(src->data_start() + src_index);
+  isolate->heap()->CopyRange(*this, dst_slot, src_slot, len, mode);
+}
+
 MaybeObject WeakArrayList::Get(int index) const {
   DCHECK(index >= 0 && index < this->capacity());
   return RELAXED_READ_WEAK_FIELD(*this, OffsetOfElementAt(index));
@@ -460,6 +477,19 @@ MaybeObjectSlot WeakArrayList::data_start() {
   return RawMaybeWeakField(kHeaderSize);
 }
 
+void WeakArrayList::CopyElements(Isolate* isolate, int dst_index,
+                                 WeakArrayList src, int src_index, int len,
+                                 WriteBarrierMode mode) {
+  if (len == 0) return;
+  DCHECK_LE(dst_index + len, capacity());
+  DCHECK_LE(src_index + len, src.capacity());
+  DisallowHeapAllocation no_gc;
+
+  MaybeObjectSlot dst_slot(data_start() + dst_index);
+  MaybeObjectSlot src_slot(src->data_start() + src_index);
+  isolate->heap()->CopyRange(*this, dst_slot, src_slot, len, mode);
+}
+
 HeapObject WeakArrayList::Iterator::Next() {
   if (!array_.is_null()) {
     while (index_ < array_->length()) {

src/objects/fixed-array.h

@@ -296,6 +296,9 @@ class WeakFixedArray : public HeapObject {
 
   inline MaybeObjectSlot RawFieldOfElementAt(int index);
 
+  inline void CopyElements(Isolate* isolate, int dst_index, WeakFixedArray src,
+                           int src_index, int len, WriteBarrierMode mode);
+
   DECL_PRINTER(WeakFixedArray)
   DECL_VERIFIER(WeakFixedArray)
 
@@ -354,6 +357,9 @@ class WeakArrayList : public HeapObject {
   // Gives access to raw memory which stores the array's data.
   inline MaybeObjectSlot data_start();
 
+  inline void CopyElements(Isolate* isolate, int dst_index, WeakArrayList src,
+                           int src_index, int len, WriteBarrierMode mode);
+
   V8_EXPORT_PRIVATE bool IsFull();
 
   DECL_INT_ACCESSORS(capacity)

src/objects/property-array-inl.h

@@ -77,6 +77,17 @@ void PropertyArray::SetHash(int hash) {
   WRITE_FIELD(*this, kLengthAndHashOffset, Smi::FromInt(value));
 }
 
+void PropertyArray::CopyElements(Isolate* isolate, int dst_index,
+                                 PropertyArray src, int src_index, int len,
+                                 WriteBarrierMode mode) {
+  if (len == 0) return;
+  DisallowHeapAllocation no_gc;
+
+  ObjectSlot dst_slot(data_start() + dst_index);
+  ObjectSlot src_slot(src->data_start() + src_index);
+  isolate->heap()->CopyRange(*this, dst_slot, src_slot, len, mode);
+}
+
 }  // namespace internal
 }  // namespace v8
 

src/objects/property-array.h

@@ -35,6 +35,10 @@ class PropertyArray : public HeapObject {
   // Setter with explicit barrier mode.
   inline void set(int index, Object value, WriteBarrierMode mode);
 
+  // Signature must be in sync with FixedArray::CopyElements().
+  inline void CopyElements(Isolate* isolate, int dst_index, PropertyArray src,
+                           int src_index, int len, WriteBarrierMode mode);
+
   // Gives access to raw memory which stores the array's data.
   inline ObjectSlot data_start();
 

src/objects/slots-atomic-inl.h

@@ -80,6 +80,7 @@ class AtomicSlot : public SlotBase<AtomicSlot, Tagged_t> {
   AtomicSlot() : SlotBase(kNullAddress) {}
   explicit AtomicSlot(Address address) : SlotBase(address) {}
   explicit AtomicSlot(ObjectSlot slot) : SlotBase(slot.address()) {}
+  explicit AtomicSlot(MaybeObjectSlot slot) : SlotBase(slot.address()) {}
 
   Reference operator*() const {
     return Reference(reinterpret_cast<Tagged_t*>(address()));