Commit 11e45684 authored by Igor Sheludko's avatar Igor Sheludko Committed by Commit Bot

[heap] Unify slots processing in performance-critical code

This CL unifies performance-critical slot processing code that was manually
specifaized for ObjectSlot. Now one templated implementation can be used
for processing both ObjectSlot and MaybeObjectSlot.

Bug: v8:8518
Change-Id: Ia4346a817911f8042459ce579741fe2308ef5e4d
Reviewed-on: https://chromium-review.googlesource.com/c/1354459
Commit-Queue: Igor Sheludko <ishell@chromium.org>
Reviewed-by: 's avatarJakob Kummerow <jkummerow@chromium.org>
Reviewed-by: 's avatarUlan Degenbaev <ulan@chromium.org>
Reviewed-by: 's avatarMichael Lippautz <mlippautz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#57933}
parent 4fb62411
......@@ -140,26 +140,26 @@ class ConcurrentMarkingVisitor final
void VisitPointers(HeapObject* host, ObjectSlot start,
ObjectSlot end) override {
for (ObjectSlot slot = start; slot < end; ++slot) {
Object* object = slot.Relaxed_Load();
DCHECK(!HasWeakHeapObjectTag(object));
if (object->IsHeapObject()) {
ProcessStrongHeapObject(host, slot, HeapObject::cast(object));
}
}
VisitPointersImpl(host, start, end);
}
void VisitPointers(HeapObject* host, MaybeObjectSlot start,
MaybeObjectSlot end) override {
for (MaybeObjectSlot slot = start; slot < end; ++slot) {
MaybeObject object = slot.Relaxed_Load();
VisitPointersImpl(host, start, end);
}
template <typename TSlot>
V8_INLINE void VisitPointersImpl(HeapObject* host, TSlot start, TSlot end) {
for (TSlot slot = start; slot < end; ++slot) {
typename TSlot::TObject object = slot.Relaxed_Load();
HeapObject* heap_object;
if (object->GetHeapObjectIfStrong(&heap_object)) {
if (object.GetHeapObjectIfStrong(&heap_object)) {
// If the reference changes concurrently from strong to weak, the write
// barrier will treat the weak reference as strong, so we won't miss the
// weak reference.
ProcessStrongHeapObject(host, ObjectSlot(slot), heap_object);
} else if (object->GetHeapObjectIfWeak(&heap_object)) {
} else if (TSlot::kCanBeWeek &&
object.GetHeapObjectIfWeak(&heap_object)) {
ProcessWeakHeapObject(host, HeapObjectSlot(slot), heap_object);
}
}
......
......@@ -200,58 +200,42 @@ int MarkingVisitor<fixed_array_mode, retaining_path_mode,
return size;
}
// class template arguments
template <FixedArrayVisitationMode fixed_array_mode,
TraceRetainingPathMode retaining_path_mode, typename MarkingState>
// method template arguments
template <typename TSlot>
void MarkingVisitor<fixed_array_mode, retaining_path_mode,
MarkingState>::VisitPointer(HeapObject* host,
ObjectSlot p) {
if (!(*p)->IsHeapObject()) return;
HeapObject* target_object = HeapObject::cast(*p);
collector_->RecordSlot(host, p, target_object);
MarkObject(host, target_object);
}
template <FixedArrayVisitationMode fixed_array_mode,
TraceRetainingPathMode retaining_path_mode, typename MarkingState>
void MarkingVisitor<fixed_array_mode, retaining_path_mode,
MarkingState>::VisitPointer(HeapObject* host,
MaybeObjectSlot p) {
MarkingState>::VisitPointerImpl(HeapObject* host,
TSlot slot) {
typename TSlot::TObject object = slot.load();
HeapObject* target_object;
if ((*p)->GetHeapObjectIfStrong(&target_object)) {
collector_->RecordSlot(host, HeapObjectSlot(p), target_object);
if (object.GetHeapObjectIfStrong(&target_object)) {
collector_->RecordSlot(host, HeapObjectSlot(slot), target_object);
MarkObject(host, target_object);
} else if ((*p)->GetHeapObjectIfWeak(&target_object)) {
} else if (TSlot::kCanBeWeek && object.GetHeapObjectIfWeak(&target_object)) {
if (marking_state()->IsBlackOrGrey(target_object)) {
// Weak references with live values are directly processed here to reduce
// the processing time of weak cells during the main GC pause.
collector_->RecordSlot(host, HeapObjectSlot(p), target_object);
collector_->RecordSlot(host, HeapObjectSlot(slot), target_object);
} else {
// If we do not know about liveness of values of weak cells, we have to
// process them when we know the liveness of the whole transitive
// closure.
collector_->AddWeakReference(host, HeapObjectSlot(p));
collector_->AddWeakReference(host, HeapObjectSlot(slot));
}
}
}
// class template arguments
template <FixedArrayVisitationMode fixed_array_mode,
TraceRetainingPathMode retaining_path_mode, typename MarkingState>
// method template arguments
template <typename TSlot>
void MarkingVisitor<fixed_array_mode, retaining_path_mode,
MarkingState>::VisitPointers(HeapObject* host,
ObjectSlot start,
ObjectSlot end) {
for (ObjectSlot p = start; p < end; ++p) {
VisitPointer(host, p);
}
}
template <FixedArrayVisitationMode fixed_array_mode,
TraceRetainingPathMode retaining_path_mode, typename MarkingState>
void MarkingVisitor<fixed_array_mode, retaining_path_mode,
MarkingState>::VisitPointers(HeapObject* host,
MaybeObjectSlot start,
MaybeObjectSlot end) {
for (MaybeObjectSlot p = start; p < end; ++p) {
MarkingState>::VisitPointersImpl(HeapObject* host,
TSlot start, TSlot end) {
for (TSlot p = start; p < end; ++p) {
VisitPointer(host, p);
}
}
......
......@@ -191,21 +191,11 @@ class FullMarkingVerifier : public MarkingVerifier {
}
void VerifyPointers(ObjectSlot start, ObjectSlot end) override {
for (ObjectSlot current = start; current < end; ++current) {
if ((*current)->IsHeapObject()) {
HeapObject* object = HeapObject::cast(*current);
CHECK(marking_state_->IsBlackOrGrey(object));
}
}
VerifyPointersImpl(start, end);
}
void VerifyPointers(MaybeObjectSlot start, MaybeObjectSlot end) override {
for (MaybeObjectSlot current = start; current < end; ++current) {
HeapObject* object;
if ((*current)->GetHeapObjectIfStrong(&object)) {
CHECK(marking_state_->IsBlackOrGrey(object));
}
}
VerifyPointersImpl(start, end);
}
void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) override {
......@@ -217,6 +207,17 @@ class FullMarkingVerifier : public MarkingVerifier {
}
private:
template <typename TSlot>
V8_INLINE void VerifyPointersImpl(TSlot start, TSlot end) {
for (TSlot slot = start; slot < end; ++slot) {
typename TSlot::TObject object = slot.load();
HeapObject* heap_object;
if (object.GetHeapObjectIfStrong(&heap_object)) {
CHECK(marking_state_->IsBlackOrGrey(heap_object));
}
}
}
MarkCompactCollector::NonAtomicMarkingState* marking_state_;
};
......@@ -305,27 +306,25 @@ class FullEvacuationVerifier : public EvacuationVerifier {
}
protected:
void VerifyPointers(ObjectSlot start, ObjectSlot end) override {
for (ObjectSlot current = start; current < end; ++current) {
if ((*current)->IsHeapObject()) {
HeapObject* object = HeapObject::cast(*current);
if (Heap::InNewSpace(object)) {
CHECK(Heap::InToSpace(object));
template <typename TSlot>
void VerifyPointersImpl(TSlot start, TSlot end) {
for (TSlot current = start; current < end; ++current) {
typename TSlot::TObject object = current.load();
HeapObject* heap_object;
if (object.GetHeapObjectIfStrong(&heap_object)) {
if (Heap::InNewSpace(heap_object)) {
CHECK(Heap::InToSpace(heap_object));
}
CHECK(!MarkCompactCollector::IsOnEvacuationCandidate(object));
CHECK(!MarkCompactCollector::IsOnEvacuationCandidate(heap_object));
}
}
}
void VerifyPointers(ObjectSlot start, ObjectSlot end) override {
VerifyPointersImpl(start, end);
}
void VerifyPointers(MaybeObjectSlot start, MaybeObjectSlot end) override {
for (MaybeObjectSlot current = start; current < end; ++current) {
HeapObject* object;
if ((*current)->GetHeapObjectIfStrong(&object)) {
if (Heap::InNewSpace(object)) {
CHECK(Heap::InToSpace(object));
}
CHECK(!MarkCompactCollector::IsOnEvacuationCandidate(object));
}
}
VerifyPointersImpl(start, end);
}
};
......@@ -3556,30 +3555,26 @@ class YoungGenerationMarkingVerifier : public MarkingVerifier {
}
void VerifyPointers(ObjectSlot start, ObjectSlot end) override {
for (ObjectSlot current = start; current < end; ++current) {
DCHECK(!HasWeakHeapObjectTag(*current));
if ((*current)->IsHeapObject()) {
HeapObject* object = HeapObject::cast(*current);
if (!Heap::InNewSpace(object)) return;
CHECK(IsMarked(object));
}
}
VerifyPointersImpl(start, end);
}
void VerifyPointers(MaybeObjectSlot start, MaybeObjectSlot end) override {
for (MaybeObjectSlot current = start; current < end; ++current) {
HeapObject* object;
VerifyPointersImpl(start, end);
}
private:
template <typename TSlot>
V8_INLINE void VerifyPointersImpl(TSlot start, TSlot end) {
for (TSlot slot = start; slot < end; ++slot) {
typename TSlot::TObject object = slot.load();
HeapObject* heap_object;
// Minor MC treats weak references as strong.
if ((*current)->GetHeapObject(&object)) {
if (!Heap::InNewSpace(object)) {
continue;
}
CHECK(IsMarked(object));
if (object.GetHeapObject(&heap_object)) {
CHECK_IMPLIES(Heap::InNewSpace(heap_object), IsMarked(heap_object));
}
}
}
private:
MinorMarkCompactCollector::NonAtomicMarkingState* marking_state_;
};
......@@ -3597,21 +3592,23 @@ class YoungGenerationEvacuationVerifier : public EvacuationVerifier {
}
protected:
void VerifyPointers(ObjectSlot start, ObjectSlot end) override {
for (ObjectSlot current = start; current < end; ++current) {
if ((*current)->IsHeapObject()) {
HeapObject* object = HeapObject::cast(*current);
CHECK_IMPLIES(Heap::InNewSpace(object), Heap::InToSpace(object));
template <typename TSlot>
void VerifyPointersImpl(TSlot start, TSlot end) {
for (TSlot current = start; current < end; ++current) {
typename TSlot::TObject object = current.load();
HeapObject* heap_object;
if (object.GetHeapObject(&heap_object)) {
CHECK_IMPLIES(Heap::InNewSpace(heap_object),
Heap::InToSpace(heap_object));
}
}
}
void VerifyPointers(ObjectSlot start, ObjectSlot end) override {
VerifyPointersImpl(start, end);
}
void VerifyPointers(MaybeObjectSlot start, MaybeObjectSlot end) override {
for (MaybeObjectSlot current = start; current < end; ++current) {
HeapObject* object;
if ((*current)->GetHeapObject(&object)) {
CHECK_IMPLIES(Heap::InNewSpace(object), Heap::InToSpace(object));
}
}
VerifyPointersImpl(start, end);
}
};
......@@ -3650,40 +3647,41 @@ class YoungGenerationMarkingVisitor final
V8_INLINE void VisitPointers(HeapObject* host, ObjectSlot start,
ObjectSlot end) final {
for (ObjectSlot p = start; p < end; ++p) {
VisitPointer(host, p);
}
VisitPointersImpl(host, start, end);
}
V8_INLINE void VisitPointers(HeapObject* host, MaybeObjectSlot start,
MaybeObjectSlot end) final {
for (MaybeObjectSlot p = start; p < end; ++p) {
VisitPointer(host, p);
}
VisitPointersImpl(host, start, end);
}
V8_INLINE void VisitPointer(HeapObject* host, ObjectSlot slot) final {
Object* target = *slot;
DCHECK(!HasWeakHeapObjectTag(target));
if (Heap::InNewSpace(target)) {
HeapObject* target_object = HeapObject::cast(target);
MarkObjectViaMarkingWorklist(target_object);
}
VisitPointerImpl(host, slot);
}
V8_INLINE void VisitPointer(HeapObject* host, MaybeObjectSlot slot) final {
MaybeObject target = *slot;
VisitPointerImpl(host, slot);
}
private:
template <typename TSlot>
V8_INLINE void VisitPointersImpl(HeapObject* host, TSlot start, TSlot end) {
for (TSlot slot = start; slot < end; ++slot) {
VisitPointer(host, slot);
}
}
template <typename TSlot>
V8_INLINE void VisitPointerImpl(HeapObject* host, TSlot slot) {
typename TSlot::TObject target = slot.load();
if (Heap::InNewSpace(target)) {
HeapObject* target_object;
// Treat weak references as strong. TODO(marja): Proper weakness handling
// for minor-mcs.
if (target->GetHeapObject(&target_object)) {
MarkObjectViaMarkingWorklist(target_object);
}
// Treat weak references as strong.
// TODO(marja): Proper weakness handling for minor-mcs.
HeapObject* target_object = target.GetHeapObject();
MarkObjectViaMarkingWorklist(target_object);
}
}
private:
inline void MarkObjectViaMarkingWorklist(HeapObject* object) {
if (marking_state_->WhiteToGrey(object)) {
// Marking deque overflow is unsupported for the young generation.
......
......@@ -932,12 +932,20 @@ class MarkingVisitor final
V8_INLINE int VisitJSWeakCell(Map map, JSWeakCell* object);
// ObjectVisitor implementation.
V8_INLINE void VisitPointer(HeapObject* host, ObjectSlot p) final;
V8_INLINE void VisitPointer(HeapObject* host, MaybeObjectSlot p) final;
V8_INLINE void VisitPointer(HeapObject* host, ObjectSlot p) final {
VisitPointerImpl(host, p);
}
V8_INLINE void VisitPointer(HeapObject* host, MaybeObjectSlot p) final {
VisitPointerImpl(host, p);
}
V8_INLINE void VisitPointers(HeapObject* host, ObjectSlot start,
ObjectSlot end) final;
ObjectSlot end) final {
VisitPointersImpl(host, start, end);
}
V8_INLINE void VisitPointers(HeapObject* host, MaybeObjectSlot start,
MaybeObjectSlot end) final;
MaybeObjectSlot end) final {
VisitPointersImpl(host, start, end);
}
V8_INLINE void VisitEmbeddedPointer(Code host, RelocInfo* rinfo) final;
V8_INLINE void VisitCodeTarget(Code host, RelocInfo* rinfo) final;
......@@ -951,6 +959,12 @@ class MarkingVisitor final
// is true.
static const int kProgressBarScanningChunk = 32 * 1024;
template <typename TSlot>
V8_INLINE void VisitPointerImpl(HeapObject* host, TSlot p);
template <typename TSlot>
V8_INLINE void VisitPointersImpl(HeapObject* host, TSlot start, TSlot end);
V8_INLINE int VisitFixedArrayIncremental(Map map, FixedArray object);
template <typename T>
......
......@@ -413,27 +413,27 @@ SlotCallbackResult Scavenger::CheckAndScavengeObject(Heap* heap,
return REMOVE_SLOT;
}
void ScavengeVisitor::VisitPointers(HeapObject* host, ObjectSlot start,
ObjectSlot end) {
for (ObjectSlot p = start; p < end; ++p) {
Object* object = *p;
if (!Heap::InNewSpace(object)) continue;
scavenger_->ScavengeObject(HeapObjectSlot(p),
reinterpret_cast<HeapObject*>(object));
}
V8_INLINE void ScavengeVisitor::VisitPointers(HeapObject* host,
ObjectSlot start,
ObjectSlot end) {
return VisitPointersImpl(host, start, end);
}
V8_INLINE void ScavengeVisitor::VisitPointers(HeapObject* host,
MaybeObjectSlot start,
MaybeObjectSlot end) {
return VisitPointersImpl(host, start, end);
}
void ScavengeVisitor::VisitPointers(HeapObject* host, MaybeObjectSlot start,
MaybeObjectSlot end) {
for (MaybeObjectSlot p = start; p < end; ++p) {
MaybeObject object = *p;
if (!Heap::InNewSpace(object)) continue;
// Treat the weak reference as strong.
template <typename TSlot>
void ScavengeVisitor::VisitPointersImpl(HeapObject* host, TSlot start,
TSlot end) {
for (TSlot slot = start; slot < end; ++slot) {
typename TSlot::TObject object = slot.load();
HeapObject* heap_object;
if (object->GetHeapObject(&heap_object)) {
scavenger_->ScavengeObject(HeapObjectSlot(p), heap_object);
} else {
UNREACHABLE();
// Treat weak references as strong.
if (object.GetHeapObject(&heap_object) && Heap::InNewSpace(heap_object)) {
scavenger_->ScavengeObject(HeapObjectSlot(slot), heap_object);
}
}
}
......
......@@ -78,23 +78,23 @@ class IterateAndScavengePromotedObjectsVisitor final : public ObjectVisitor {
inline void VisitPointers(HeapObject* host, ObjectSlot start,
ObjectSlot end) final {
for (ObjectSlot slot = start; slot < end; ++slot) {
Object* target = *slot;
DCHECK(!HasWeakHeapObjectTag(target));
if (target->IsHeapObject()) {
HandleSlot(host, HeapObjectSlot(slot), HeapObject::cast(target));
}
}
VisitPointersImpl(host, start, end);
}
inline void VisitPointers(HeapObject* host, MaybeObjectSlot start,
MaybeObjectSlot end) final {
// Treat weak references as strong. TODO(marja): Proper weakness handling in
// the young generation.
for (MaybeObjectSlot slot = start; slot < end; ++slot) {
MaybeObject target = *slot;
VisitPointersImpl(host, start, end);
}
private:
template <typename TSlot>
V8_INLINE void VisitPointersImpl(HeapObject* host, TSlot start, TSlot end) {
// Treat weak references as strong.
// TODO(marja): Proper weakness handling in the young generation.
for (TSlot slot = start; slot < end; ++slot) {
typename TSlot::TObject object = slot.load();
HeapObject* heap_object;
if (target->GetHeapObject(&heap_object)) {
if (object.GetHeapObject(&heap_object)) {
HandleSlot(host, HeapObjectSlot(slot), heap_object);
}
}
......@@ -124,7 +124,6 @@ class IterateAndScavengePromotedObjectsVisitor final : public ObjectVisitor {
}
}
private:
Heap* const heap_;
Scavenger* const scavenger_;
const bool record_slots_;
......
......@@ -230,10 +230,14 @@ class ScavengeVisitor final : public NewSpaceVisitor<ScavengeVisitor> {
V8_INLINE void VisitPointers(HeapObject* host, ObjectSlot start,
ObjectSlot end) final;
V8_INLINE void VisitPointers(HeapObject* host, MaybeObjectSlot start,
MaybeObjectSlot end) final;
private:
template <typename TSlot>
V8_INLINE void VisitPointersImpl(HeapObject* host, TSlot start, TSlot end);
Scavenger* const scavenger_;
};
......
......@@ -53,6 +53,21 @@ bool ObjectPtr::IsSmallOrderedHashTable() const {
IsSmallOrderedNameDictionary();
}
bool ObjectPtr::GetHeapObjectIfStrong(HeapObject** result) const {
return GetHeapObject(result);
}
bool ObjectPtr::GetHeapObject(HeapObject** result) const {
if (!IsHeapObject()) return false;
*result = reinterpret_cast<HeapObject*>(ptr());
return true;
}
HeapObject* ObjectPtr::GetHeapObject() const {
DCHECK(IsHeapObject());
return reinterpret_cast<HeapObject*>(ptr());
}
double ObjectPtr::Number() const {
return reinterpret_cast<Object*>(ptr())->Number();
}
......
......@@ -90,6 +90,31 @@ class ObjectPtr {
// allow kMaxUInt32.
V8_WARN_UNUSED_RESULT inline bool ToArrayIndex(uint32_t* index) const;
//
// The following GetHeapObjectXX methods mimic corresponding functionality
// in MaybeObject. Having them here allows us to unify code that processes
// ObjectSlots and MaybeObjectSlots.
//
// If this Object is a strong pointer to a HeapObject, returns true and
// sets *result. Otherwise returns false.
inline bool GetHeapObjectIfStrong(HeapObject** result) const;
// If this Object is a strong pointer to a HeapObject (weak pointers are not
// expected), returns true and sets *result. Otherwise returns false.
inline bool GetHeapObject(HeapObject** result) const;
// DCHECKs that this Object is a strong pointer to a HeapObject and returns
// the HeapObject.
inline HeapObject* GetHeapObject() const;
// Always returns false because Object is not expected to be a weak pointer
// to a HeapObject.
inline bool GetHeapObjectIfWeak(HeapObject** result) const {
DCHECK(!HasWeakHeapObjectTag(ptr()));
return false;
}
#ifdef VERIFY_HEAP
void ObjectVerify(Isolate* isolate) {
reinterpret_cast<Object*>(ptr())->ObjectVerify(isolate);
......
......@@ -18,6 +18,8 @@ namespace internal {
ObjectSlot::ObjectSlot(ObjectPtr* object)
: SlotBase(reinterpret_cast<Address>(&object->ptr_)) {}
ObjectPtr ObjectSlot::load() const { return ObjectPtr(*location()); }
void ObjectSlot::store(Object* value) const { *location() = value->ptr(); }
ObjectPtr ObjectSlot::Acquire_Load() const {
......@@ -59,6 +61,8 @@ MaybeObject MaybeObjectSlot::operator*() const {
return MaybeObject(*location());
}
MaybeObject MaybeObjectSlot::load() const { return MaybeObject(*location()); }
void MaybeObjectSlot::store(MaybeObject value) const {
*location() = value.ptr();
}
......
......@@ -16,6 +16,7 @@ template <typename Subclass, typename Data, size_t SlotDataSize>
class SlotBase {
public:
using TData = Data;
// TODO(ishell): This should eventually become just sizeof(TData) once
// pointer compression is implemented.
static constexpr size_t kSlotDataSize = SlotDataSize;
......@@ -84,11 +85,16 @@ class SlotBase {
};
// An ObjectSlot instance describes a kTaggedSize-sized field ("slot") holding
// a tagged pointer (smi or heap object).
// a tagged pointer (smi or strong heap object).
// Its address() is the address of the slot.
// The slot's contents can be read and written using operator* and store().
class ObjectSlot : public SlotBase<ObjectSlot, Tagged_t, kTaggedSize> {
public:
using TObject = ObjectPtr;
// Tagged value stored in this slot is guaranteed to never be a weak pointer.
static constexpr bool kCanBeWeek = false;
ObjectSlot() : SlotBase(kNullAddress) {}
explicit ObjectSlot(Address ptr) : SlotBase(ptr) {}
explicit ObjectSlot(Address* ptr)
......@@ -101,6 +107,8 @@ class ObjectSlot : public SlotBase<ObjectSlot, Tagged_t, kTaggedSize> {
: SlotBase(slot.address()) {}
Object* operator*() const { return *reinterpret_cast<Object**>(address()); }
// TODO(3770): drop this in favor of operator* once migration is complete.
inline ObjectPtr load() const;
inline void store(Object* value) const;
inline ObjectPtr Acquire_Load() const;
......@@ -124,6 +132,11 @@ class ObjectSlot : public SlotBase<ObjectSlot, Tagged_t, kTaggedSize> {
class MaybeObjectSlot
: public SlotBase<MaybeObjectSlot, Tagged_t, kTaggedSize> {
public:
using TObject = MaybeObject;
// Tagged value stored in this slot can be a weak pointer.
static constexpr bool kCanBeWeek = true;
explicit MaybeObjectSlot(Address ptr) : SlotBase(ptr) {}
explicit MaybeObjectSlot(Object** ptr)
: SlotBase(reinterpret_cast<Address>(ptr)) {}
......@@ -132,6 +145,8 @@ class MaybeObjectSlot
: SlotBase(slot.address()) {}
inline MaybeObject operator*() const;
// TODO(3770): drop this once ObjectSlot::load() is dropped.
inline MaybeObject load() const;
inline void store(MaybeObject value) const;
inline MaybeObject Relaxed_Load() const;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment