[heap] Pass a force_promotion flag to the evacuation routine in the scavenger.

The {force_promotion} flag causes the scavenger to move an object to the
old generation instead of to the other semi-space. We use the flag to force
the promotion of objects which are referenced by code objects.

R=ulan@chromium.org

Review-Url: https://codereview.chromium.org/2002013002
Cr-Commit-Position: refs/heads/master@{#36443}

src/heap/heap.cc

@@ -1668,15 +1668,18 @@ void Heap::Scavenge() {
     // Copy objects reachable from the old generation.
     TRACE_GC(tracer(), GCTracer::Scope::SCAVENGER_OLD_TO_NEW_POINTERS);
     RememberedSet<OLD_TO_NEW>::Iterate(this, [this](Address addr) {
-      return Scavenger::CheckAndScavengeObject(this, addr);
+      return Scavenger::CheckAndScavengeObject(this, addr, DEFAULT_PROMOTION);
     });
 
     RememberedSet<OLD_TO_NEW>::IterateTyped(
         this, [this](SlotType type, Address addr) {
           return UpdateTypedSlotHelper::UpdateTypedSlot(
               isolate(), type, addr, [this](Object** addr) {
+                // We expect that objects referenced by code are long living.
+                // If we do not force promotion, then we need to clear
+                // old_to_new slots in dead code objects after mark-compact.
                 return Scavenger::CheckAndScavengeObject(
-                    this, reinterpret_cast<Address>(addr));
+                    this, reinterpret_cast<Address>(addr), FORCE_PROMOTION);
               });
         });
   }
@@ -4665,8 +4668,8 @@ void Heap::IteratePromotedObjectPointers(HeapObject* object, Address start,
     Object* target = *slot;
     if (target->IsHeapObject()) {
       if (Heap::InFromSpace(target)) {
-        callback(reinterpret_cast<HeapObject**>(slot),
-                 HeapObject::cast(target));
+        callback(reinterpret_cast<HeapObject**>(slot), HeapObject::cast(target),
+                 DEFAULT_PROMOTION);
         Object* new_target = *slot;
         if (InNewSpace(new_target)) {
           SLOW_DCHECK(Heap::InToSpace(new_target));

src/heap/heap.h

@@ -301,7 +301,10 @@ class Scavenger;
 class ScavengeJob;
 class WeakObjectRetainer;
 
-typedef void (*ObjectSlotCallback)(HeapObject** from, HeapObject* to);
+enum PromotionMode { FORCE_PROMOTION, DEFAULT_PROMOTION };
+
+typedef void (*ObjectSlotCallback)(HeapObject** from, HeapObject* to,
+                                   PromotionMode mode);
 
 // A queue of objects promoted during scavenge. Each object is accompanied
 // by it's size to avoid dereferencing a map pointer for scanning.

src/heap/scavenger-inl.h

@@ -10,7 +10,8 @@
 namespace v8 {
 namespace internal {
 
-void Scavenger::ScavengeObject(HeapObject** p, HeapObject* object) {
+void Scavenger::ScavengeObject(HeapObject** p, HeapObject* object,
+                               PromotionMode promotion_mode) {
   DCHECK(object->GetIsolate()->heap()->InFromSpace(object));
 
   // We use the first word (where the map pointer usually is) of a heap
@@ -34,18 +35,19 @@ void Scavenger::ScavengeObject(HeapObject** p, HeapObject* object) {
   // AllocationMementos are unrooted and shouldn't survive a scavenge
   DCHECK(object->map() != object->GetHeap()->allocation_memento_map());
   // Call the slow part of scavenge object.
-  return ScavengeObjectSlow(p, object);
+  return ScavengeObjectSlow(p, object, promotion_mode);
 }
 
-SlotCallbackResult Scavenger::CheckAndScavengeObject(Heap* heap,
-                                                     Address slot_address) {
+SlotCallbackResult Scavenger::CheckAndScavengeObject(
+    Heap* heap, Address slot_address, PromotionMode promotion_mode) {
   Object** slot = reinterpret_cast<Object**>(slot_address);
   Object* object = *slot;
   if (heap->InFromSpace(object)) {
     HeapObject* heap_object = reinterpret_cast<HeapObject*>(object);
     DCHECK(heap_object->IsHeapObject());
 
-    ScavengeObject(reinterpret_cast<HeapObject**>(slot), heap_object);
+    ScavengeObject(reinterpret_cast<HeapObject**>(slot), heap_object,
+                   promotion_mode);
 
     object = *slot;
     // If the object was in from space before and is after executing the
@@ -67,7 +69,8 @@ void StaticScavengeVisitor::VisitPointer(Heap* heap, HeapObject* obj,
   Object* object = *p;
   if (!heap->InNewSpace(object)) return;
   Scavenger::ScavengeObject(reinterpret_cast<HeapObject**>(p),
-                            reinterpret_cast<HeapObject*>(object));
+                            reinterpret_cast<HeapObject*>(object),
+                            DEFAULT_PROMOTION);
 }
 
 }  // namespace internal

src/heap/scavenger.cc

@@ -200,15 +200,16 @@ class ScavengingVisitor : public StaticVisitorBase {
     return false;
   }
 
-
   template <ObjectContents object_contents, AllocationAlignment alignment>
   static inline void EvacuateObject(Map* map, HeapObject** slot,
-                                    HeapObject* object, int object_size) {
+                                    HeapObject* object, int object_size,
+                                    PromotionMode promotion_mode) {
     SLOW_DCHECK(object_size <= Page::kAllocatableMemory);
     SLOW_DCHECK(object->Size() == object_size);
     Heap* heap = map->GetHeap();
 
-    if (!heap->ShouldBePromoted(object->address(), object_size)) {
+    if (promotion_mode != FORCE_PROMOTION &&
+        !heap->ShouldBePromoted(object->address(), object_size)) {
       // A semi-space copy may fail due to fragmentation. In that case, we
       // try to promote the object.
       if (SemiSpaceCopyObject<alignment>(map, slot, object, object_size)) {
@@ -220,17 +221,20 @@ class ScavengingVisitor : public StaticVisitorBase {
                                                   object_size)) {
       return;
     }
-
+    if (promotion_mode == FORCE_PROMOTION) {
+      FatalProcessOutOfMemory("Scavenger: forced promotion\n");
+    }
     // If promotion failed, we try to copy the object to the other semi-space
     if (SemiSpaceCopyObject<alignment>(map, slot, object, object_size)) return;
 
     FatalProcessOutOfMemory("Scavenger: semi-space copy\n");
   }
 
-
   static inline void EvacuateJSFunction(Map* map, HeapObject** slot,
-                                        HeapObject* object) {
-    ObjectEvacuationStrategy<POINTER_OBJECT>::Visit(map, slot, object);
+                                        HeapObject* object,
+                                        PromotionMode promotion_mode) {
+    ObjectEvacuationStrategy<POINTER_OBJECT>::Visit(map, slot, object,
+                                                    promotion_mode);
 
     if (marks_handling == IGNORE_MARKS) return;
 
@@ -252,71 +256,76 @@ class ScavengingVisitor : public StaticVisitorBase {
     }
   }
 
-
   static inline void EvacuateFixedArray(Map* map, HeapObject** slot,
-                                        HeapObject* object) {
+                                        HeapObject* object,
+                                        PromotionMode promotion_mode) {
     int length = reinterpret_cast<FixedArray*>(object)->synchronized_length();
     int object_size = FixedArray::SizeFor(length);
-    EvacuateObject<POINTER_OBJECT, kWordAligned>(map, slot, object,
-                                                 object_size);
+    EvacuateObject<POINTER_OBJECT, kWordAligned>(map, slot, object, object_size,
+                                                 promotion_mode);
   }
 
-
   static inline void EvacuateFixedDoubleArray(Map* map, HeapObject** slot,
-                                              HeapObject* object) {
+                                              HeapObject* object,
+                                              PromotionMode promotion_mode) {
     int length = reinterpret_cast<FixedDoubleArray*>(object)->length();
     int object_size = FixedDoubleArray::SizeFor(length);
-    EvacuateObject<DATA_OBJECT, kDoubleAligned>(map, slot, object, object_size);
+    EvacuateObject<DATA_OBJECT, kDoubleAligned>(map, slot, object, object_size,
+                                                promotion_mode);
   }
 
-
   static inline void EvacuateFixedTypedArray(Map* map, HeapObject** slot,
-                                             HeapObject* object) {
+                                             HeapObject* object,
+                                             PromotionMode promotion_mode) {
     int object_size = reinterpret_cast<FixedTypedArrayBase*>(object)->size();
-    EvacuateObject<POINTER_OBJECT, kWordAligned>(map, slot, object,
-                                                 object_size);
+    EvacuateObject<POINTER_OBJECT, kWordAligned>(map, slot, object, object_size,
+                                                 promotion_mode);
   }
 
-
   static inline void EvacuateFixedFloat64Array(Map* map, HeapObject** slot,
-                                               HeapObject* object) {
+                                               HeapObject* object,
+                                               PromotionMode promotion_mode) {
     int object_size = reinterpret_cast<FixedFloat64Array*>(object)->size();
     EvacuateObject<POINTER_OBJECT, kDoubleAligned>(map, slot, object,
-                                                   object_size);
+                                                   object_size, promotion_mode);
   }
 
-
   static inline void EvacuateJSArrayBuffer(Map* map, HeapObject** slot,
-                                           HeapObject* object) {
-    ObjectEvacuationStrategy<POINTER_OBJECT>::Visit(map, slot, object);
+                                           HeapObject* object,
+                                           PromotionMode promotion_mode) {
+    ObjectEvacuationStrategy<POINTER_OBJECT>::Visit(map, slot, object,
+                                                    promotion_mode);
   }
 
-
   static inline void EvacuateByteArray(Map* map, HeapObject** slot,
-                                       HeapObject* object) {
+                                       HeapObject* object,
+                                       PromotionMode promotion_mode) {
     int object_size = reinterpret_cast<ByteArray*>(object)->ByteArraySize();
-    EvacuateObject<DATA_OBJECT, kWordAligned>(map, slot, object, object_size);
+    EvacuateObject<DATA_OBJECT, kWordAligned>(map, slot, object, object_size,
+                                              promotion_mode);
   }
 
-
   static inline void EvacuateSeqOneByteString(Map* map, HeapObject** slot,
-                                              HeapObject* object) {
+                                              HeapObject* object,
+                                              PromotionMode promotion_mode) {
     int object_size = SeqOneByteString::cast(object)
                           ->SeqOneByteStringSize(map->instance_type());
-    EvacuateObject<DATA_OBJECT, kWordAligned>(map, slot, object, object_size);
+    EvacuateObject<DATA_OBJECT, kWordAligned>(map, slot, object, object_size,
+                                              promotion_mode);
   }
 
-
   static inline void EvacuateSeqTwoByteString(Map* map, HeapObject** slot,
-                                              HeapObject* object) {
+                                              HeapObject* object,
+                                              PromotionMode promotion_mode) {
     int object_size = SeqTwoByteString::cast(object)
                           ->SeqTwoByteStringSize(map->instance_type());
-    EvacuateObject<DATA_OBJECT, kWordAligned>(map, slot, object, object_size);
+    EvacuateObject<DATA_OBJECT, kWordAligned>(map, slot, object, object_size,
+                                              promotion_mode);
   }
 
-
   static inline void EvacuateShortcutCandidate(Map* map, HeapObject** slot,
-                                               HeapObject* object) {
+                                               HeapObject* object,
+                                               PromotionMode promotion_mode) {
     DCHECK(IsShortcutCandidate(map->instance_type()));
 
     Heap* heap = map->GetHeap();
@@ -342,14 +351,14 @@ class ScavengingVisitor : public StaticVisitorBase {
         return;
       }
 
-      Scavenger::ScavengeObjectSlow(slot, first);
+      Scavenger::ScavengeObjectSlow(slot, first, promotion_mode);
       object->set_map_word(MapWord::FromForwardingAddress(*slot));
       return;
     }
 
     int object_size = ConsString::kSize;
-    EvacuateObject<POINTER_OBJECT, kWordAligned>(map, slot, object,
-                                                 object_size);
+    EvacuateObject<POINTER_OBJECT, kWordAligned>(map, slot, object, object_size,
+                                                 promotion_mode);
   }
 
   template <ObjectContents object_contents>
@@ -357,15 +366,17 @@ class ScavengingVisitor : public StaticVisitorBase {
    public:
     template <int object_size>
     static inline void VisitSpecialized(Map* map, HeapObject** slot,
-                                        HeapObject* object) {
-      EvacuateObject<object_contents, kWordAligned>(map, slot, object,
-                                                    object_size);
+                                        HeapObject* object,
+                                        PromotionMode promotion_mode) {
+      EvacuateObject<object_contents, kWordAligned>(
+          map, slot, object, object_size, promotion_mode);
     }
 
-    static inline void Visit(Map* map, HeapObject** slot, HeapObject* object) {
+    static inline void Visit(Map* map, HeapObject** slot, HeapObject* object,
+                             PromotionMode promotion_mode) {
       int object_size = map->instance_size();
-      EvacuateObject<object_contents, kWordAligned>(map, slot, object,
-                                                    object_size);
+      EvacuateObject<object_contents, kWordAligned>(
+          map, slot, object, object_size, promotion_mode);
     }
   };
 
@@ -391,13 +402,15 @@ void Scavenger::Initialize() {
 
 
 // static
-void Scavenger::ScavengeObjectSlow(HeapObject** p, HeapObject* object) {
+void Scavenger::ScavengeObjectSlow(HeapObject** p, HeapObject* object,
+                                   PromotionMode promotion_mode) {
   SLOW_DCHECK(object->GetIsolate()->heap()->InFromSpace(object));
   MapWord first_word = object->map_word();
   SLOW_DCHECK(!first_word.IsForwardingAddress());
   Map* map = first_word.ToMap();
   Scavenger* scavenger = map->GetHeap()->scavenge_collector_;
-  scavenger->scavenging_visitors_table_.GetVisitor(map)(map, p, object);
+  scavenger->scavenging_visitors_table_.GetVisitor(map)(map, p, object,
+                                                        promotion_mode);
 }
 
 

src/heap/scavenger.h

@@ -12,7 +12,8 @@ namespace v8 {
 namespace internal {
 
 typedef void (*ScavengingCallback)(Map* map, HeapObject** slot,
-                                   HeapObject* object);
+                                   HeapObject* object,
+                                   PromotionMode promotion_mode);
 
 class Scavenger {
  public:
@@ -25,12 +26,15 @@ class Scavenger {
   // necessary, the object might be promoted to an old space.  The caller must
   // ensure the precondition that the object is (a) a heap object and (b) in
   // the heap's from space.
-  static inline void ScavengeObject(HeapObject** p, HeapObject* object);
-  static inline SlotCallbackResult CheckAndScavengeObject(Heap* heap,
-                                                          Address slot_address);
+  static inline void ScavengeObject(
+      HeapObject** p, HeapObject* object,
+      PromotionMode promotion_mode = DEFAULT_PROMOTION);
+  static inline SlotCallbackResult CheckAndScavengeObject(
+      Heap* heap, Address slot_address, PromotionMode promotion_mode);
 
   // Slow part of {ScavengeObject} above.
-  static void ScavengeObjectSlow(HeapObject** p, HeapObject* object);
+  static void ScavengeObjectSlow(HeapObject** p, HeapObject* object,
+                                 PromotionMode promotion_mode);
 
   // Chooses an appropriate static visitor table depending on the current state
   // of the heap (i.e. incremental marking, logging and profiling).