Commit 4d52549a authored by Michael Lippautz's avatar Michael Lippautz Committed by Commit Bot

cppgc: Remove -inl.h files

Move inlined methods to .h files accordingly, follwing style guide
rule:
https://google.github.io/styleguide/cppguide.html#Self_contained_Headers

Bug: chromium:1056170
Change-Id: Ia6c4f82bd4352d507eece36e540ad0d318e56920
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2273858Reviewed-by: 's avatarAnton Bikineev <bikineev@chromium.org>
Reviewed-by: 's avatarOmer Katz <omerkatz@chromium.org>
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#68627}
parent 86a63378
......@@ -4198,10 +4198,8 @@ v8_source_set("cppgc_base") {
"src/heap/cppgc/heap-base.h",
"src/heap/cppgc/heap-growing.cc",
"src/heap/cppgc/heap-growing.h",
"src/heap/cppgc/heap-object-header-inl.h",
"src/heap/cppgc/heap-object-header.cc",
"src/heap/cppgc/heap-object-header.h",
"src/heap/cppgc/heap-page-inl.h",
"src/heap/cppgc/heap-page.cc",
"src/heap/cppgc/heap-page.h",
"src/heap/cppgc/heap-space.cc",
......@@ -4216,12 +4214,9 @@ v8_source_set("cppgc_base") {
"src/heap/cppgc/marker.h",
"src/heap/cppgc/marking-visitor.cc",
"src/heap/cppgc/marking-visitor.h",
"src/heap/cppgc/object-allocator-inl.h",
"src/heap/cppgc/object-allocator.cc",
"src/heap/cppgc/object-allocator.h",
"src/heap/cppgc/object-start-bitmap-inl.h",
"src/heap/cppgc/object-start-bitmap.h",
"src/heap/cppgc/page-memory-inl.h",
"src/heap/cppgc/page-memory.cc",
"src/heap/cppgc/page-memory.h",
"src/heap/cppgc/persistent-node.cc",
......
......@@ -12,7 +12,7 @@
#include "src/flags/flags.h"
#include "src/heap/cppgc/gc-info-table.h"
#include "src/heap/cppgc/heap-base.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/marker.h"
#include "src/heap/cppgc/marking-visitor.h"
#include "src/heap/cppgc/object-allocator.h"
......
......@@ -6,7 +6,7 @@
#include "src/base/logging.h"
#include "src/base/macros.h"
#include "src/heap/cppgc/object-allocator-inl.h"
#include "src/heap/cppgc/object-allocator.h"
namespace cppgc {
namespace internal {
......
......@@ -9,7 +9,7 @@
#include "include/cppgc/internal/logging.h"
#include "src/base/bits.h"
#include "src/heap/cppgc/globals.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/sanitizers.h"
namespace cppgc {
......
......@@ -8,8 +8,8 @@
#include "src/base/platform/platform.h"
#include "src/heap/base/stack.h"
#include "src/heap/cppgc/globals.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-page-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap-page.h"
#include "src/heap/cppgc/heap-visitor.h"
#include "src/heap/cppgc/marker.h"
#include "src/heap/cppgc/page-memory.h"
......
// Copyright 2020 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef V8_HEAP_CPPGC_HEAP_OBJECT_HEADER_INL_H_
#define V8_HEAP_CPPGC_HEAP_OBJECT_HEADER_INL_H_
#include "include/cppgc/allocation.h"
#include "include/cppgc/internal/gc-info.h"
#include "src/base/atomic-utils.h"
#include "src/base/logging.h"
#include "src/base/macros.h"
#include "src/heap/cppgc/gc-info-table.h"
#include "src/heap/cppgc/globals.h"
#include "src/heap/cppgc/heap-object-header.h"
namespace cppgc {
namespace internal {
// static
HeapObjectHeader& HeapObjectHeader::FromPayload(void* payload) {
return *reinterpret_cast<HeapObjectHeader*>(static_cast<Address>(payload) -
sizeof(HeapObjectHeader));
}
// static
const HeapObjectHeader& HeapObjectHeader::FromPayload(const void* payload) {
return *reinterpret_cast<const HeapObjectHeader*>(
static_cast<ConstAddress>(payload) - sizeof(HeapObjectHeader));
}
HeapObjectHeader::HeapObjectHeader(size_t size, GCInfoIndex gc_info_index) {
#if defined(V8_TARGET_ARCH_64_BIT)
USE(padding_);
#endif // defined(V8_TARGET_ARCH_64_BIT)
DCHECK_LT(gc_info_index, GCInfoTable::kMaxIndex);
DCHECK_EQ(0u, size & (sizeof(HeapObjectHeader) - 1));
DCHECK_GE(kMaxSize, size);
encoded_high_ = GCInfoIndexField::encode(gc_info_index);
encoded_low_ = EncodeSize(size);
DCHECK(IsInConstruction());
#ifdef DEBUG
CheckApiConstants();
#endif // DEBUG
}
Address HeapObjectHeader::Payload() const {
return reinterpret_cast<Address>(const_cast<HeapObjectHeader*>(this)) +
sizeof(HeapObjectHeader);
}
template <HeapObjectHeader::AccessMode mode>
GCInfoIndex HeapObjectHeader::GetGCInfoIndex() const {
const uint16_t encoded =
LoadEncoded<mode, EncodedHalf::kHigh, std::memory_order_acquire>();
return GCInfoIndexField::decode(encoded);
}
template <HeapObjectHeader::AccessMode mode>
size_t HeapObjectHeader::GetSize() const {
// Size is immutable after construction while either marking or sweeping
// is running so relaxed load (if mode == kAtomic) is enough.
uint16_t encoded_low_value =
LoadEncoded<mode, EncodedHalf::kLow, std::memory_order_relaxed>();
const size_t size = DecodeSize(encoded_low_value);
return size;
}
void HeapObjectHeader::SetSize(size_t size) {
DCHECK(!IsMarked());
encoded_low_ |= EncodeSize(size);
}
template <HeapObjectHeader::AccessMode mode>
bool HeapObjectHeader::IsLargeObject() const {
return GetSize<mode>() == kLargeObjectSizeInHeader;
}
template <HeapObjectHeader::AccessMode mode>
bool HeapObjectHeader::IsInConstruction() const {
const uint16_t encoded =
LoadEncoded<mode, EncodedHalf::kHigh, std::memory_order_acquire>();
return !FullyConstructedField::decode(encoded);
}
void HeapObjectHeader::MarkAsFullyConstructed() {
MakeGarbageCollectedTraitInternal::MarkObjectAsFullyConstructed(Payload());
}
template <HeapObjectHeader::AccessMode mode>
bool HeapObjectHeader::IsMarked() const {
const uint16_t encoded =
LoadEncoded<mode, EncodedHalf::kLow, std::memory_order_relaxed>();
return MarkBitField::decode(encoded);
}
template <HeapObjectHeader::AccessMode mode>
void HeapObjectHeader::Unmark() {
DCHECK(IsMarked<mode>());
StoreEncoded<mode, EncodedHalf::kLow, std::memory_order_relaxed>(
MarkBitField::encode(false), MarkBitField::kMask);
}
bool HeapObjectHeader::TryMarkAtomic() {
auto* atomic_encoded = v8::base::AsAtomicPtr(&encoded_low_);
uint16_t old_value = atomic_encoded->load(std::memory_order_relaxed);
const uint16_t new_value = old_value | MarkBitField::encode(true);
if (new_value == old_value) {
return false;
}
return atomic_encoded->compare_exchange_strong(old_value, new_value,
std::memory_order_relaxed);
}
template <HeapObjectHeader::AccessMode mode>
bool HeapObjectHeader::IsYoung() const {
return !IsMarked<mode>();
}
template <HeapObjectHeader::AccessMode mode>
bool HeapObjectHeader::IsFree() const {
return GetGCInfoIndex() == kFreeListGCInfoIndex;
}
bool HeapObjectHeader::IsFinalizable() const {
const GCInfo& gc_info = GlobalGCInfoTable::GCInfoFromIndex(GetGCInfoIndex());
return gc_info.finalize;
}
template <HeapObjectHeader::AccessMode mode, HeapObjectHeader::EncodedHalf part,
std::memory_order memory_order>
uint16_t HeapObjectHeader::LoadEncoded() const {
const uint16_t& half =
part == EncodedHalf::kLow ? encoded_low_ : encoded_high_;
if (mode == AccessMode::kNonAtomic) return half;
return v8::base::AsAtomicPtr(&half)->load(memory_order);
}
template <HeapObjectHeader::AccessMode mode, HeapObjectHeader::EncodedHalf part,
std::memory_order memory_order>
void HeapObjectHeader::StoreEncoded(uint16_t bits, uint16_t mask) {
// Caveat: Not all changes to HeapObjectHeader's bitfields go through
// StoreEncoded. The following have their own implementations and need to be
// kept in sync:
// - HeapObjectHeader::TryMarkAtomic
// - MarkObjectAsFullyConstructed (API)
DCHECK_EQ(0u, bits & ~mask);
uint16_t& half = part == EncodedHalf::kLow ? encoded_low_ : encoded_high_;
if (mode == AccessMode::kNonAtomic) {
half = (half & ~mask) | bits;
return;
}
// We don't perform CAS loop here assuming that only none of the info that
// shares the same encoded halfs change at the same time.
auto* atomic_encoded = v8::base::AsAtomicPtr(&half);
uint16_t value = atomic_encoded->load(std::memory_order_relaxed);
value = (value & ~mask) | bits;
atomic_encoded->store(value, memory_order);
}
} // namespace internal
} // namespace cppgc
#endif // V8_HEAP_CPPGC_HEAP_OBJECT_HEADER_INL_H_
......@@ -7,7 +7,6 @@
#include "include/cppgc/internal/api-constants.h"
#include "src/base/macros.h"
#include "src/heap/cppgc/gc-info-table.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
namespace cppgc {
namespace internal {
......
......@@ -9,8 +9,13 @@
#include <atomic>
#include "include/cppgc/allocation.h"
#include "include/cppgc/internal/gc-info.h"
#include "src/base/atomic-utils.h"
#include "src/base/bit-field.h"
#include "src/base/logging.h"
#include "src/base/macros.h"
#include "src/heap/cppgc/gc-info-table.h"
#include "src/heap/cppgc/globals.h"
namespace cppgc {
......@@ -128,6 +133,147 @@ class HeapObjectHeader {
uint16_t encoded_low_;
};
// static
HeapObjectHeader& HeapObjectHeader::FromPayload(void* payload) {
return *reinterpret_cast<HeapObjectHeader*>(static_cast<Address>(payload) -
sizeof(HeapObjectHeader));
}
// static
const HeapObjectHeader& HeapObjectHeader::FromPayload(const void* payload) {
return *reinterpret_cast<const HeapObjectHeader*>(
static_cast<ConstAddress>(payload) - sizeof(HeapObjectHeader));
}
HeapObjectHeader::HeapObjectHeader(size_t size, GCInfoIndex gc_info_index) {
#if defined(V8_TARGET_ARCH_64_BIT)
USE(padding_);
#endif // defined(V8_TARGET_ARCH_64_BIT)
DCHECK_LT(gc_info_index, GCInfoTable::kMaxIndex);
DCHECK_EQ(0u, size & (sizeof(HeapObjectHeader) - 1));
DCHECK_GE(kMaxSize, size);
encoded_high_ = GCInfoIndexField::encode(gc_info_index);
encoded_low_ = EncodeSize(size);
DCHECK(IsInConstruction());
#ifdef DEBUG
CheckApiConstants();
#endif // DEBUG
}
Address HeapObjectHeader::Payload() const {
return reinterpret_cast<Address>(const_cast<HeapObjectHeader*>(this)) +
sizeof(HeapObjectHeader);
}
template <HeapObjectHeader::AccessMode mode>
GCInfoIndex HeapObjectHeader::GetGCInfoIndex() const {
const uint16_t encoded =
LoadEncoded<mode, EncodedHalf::kHigh, std::memory_order_acquire>();
return GCInfoIndexField::decode(encoded);
}
template <HeapObjectHeader::AccessMode mode>
size_t HeapObjectHeader::GetSize() const {
// Size is immutable after construction while either marking or sweeping
// is running so relaxed load (if mode == kAtomic) is enough.
uint16_t encoded_low_value =
LoadEncoded<mode, EncodedHalf::kLow, std::memory_order_relaxed>();
const size_t size = DecodeSize(encoded_low_value);
return size;
}
void HeapObjectHeader::SetSize(size_t size) {
DCHECK(!IsMarked());
encoded_low_ |= EncodeSize(size);
}
template <HeapObjectHeader::AccessMode mode>
bool HeapObjectHeader::IsLargeObject() const {
return GetSize<mode>() == kLargeObjectSizeInHeader;
}
template <HeapObjectHeader::AccessMode mode>
bool HeapObjectHeader::IsInConstruction() const {
const uint16_t encoded =
LoadEncoded<mode, EncodedHalf::kHigh, std::memory_order_acquire>();
return !FullyConstructedField::decode(encoded);
}
void HeapObjectHeader::MarkAsFullyConstructed() {
MakeGarbageCollectedTraitInternal::MarkObjectAsFullyConstructed(Payload());
}
template <HeapObjectHeader::AccessMode mode>
bool HeapObjectHeader::IsMarked() const {
const uint16_t encoded =
LoadEncoded<mode, EncodedHalf::kLow, std::memory_order_relaxed>();
return MarkBitField::decode(encoded);
}
template <HeapObjectHeader::AccessMode mode>
void HeapObjectHeader::Unmark() {
DCHECK(IsMarked<mode>());
StoreEncoded<mode, EncodedHalf::kLow, std::memory_order_relaxed>(
MarkBitField::encode(false), MarkBitField::kMask);
}
bool HeapObjectHeader::TryMarkAtomic() {
auto* atomic_encoded = v8::base::AsAtomicPtr(&encoded_low_);
uint16_t old_value = atomic_encoded->load(std::memory_order_relaxed);
const uint16_t new_value = old_value | MarkBitField::encode(true);
if (new_value == old_value) {
return false;
}
return atomic_encoded->compare_exchange_strong(old_value, new_value,
std::memory_order_relaxed);
}
template <HeapObjectHeader::AccessMode mode>
bool HeapObjectHeader::IsYoung() const {
return !IsMarked<mode>();
}
template <HeapObjectHeader::AccessMode mode>
bool HeapObjectHeader::IsFree() const {
return GetGCInfoIndex() == kFreeListGCInfoIndex;
}
bool HeapObjectHeader::IsFinalizable() const {
const GCInfo& gc_info = GlobalGCInfoTable::GCInfoFromIndex(GetGCInfoIndex());
return gc_info.finalize;
}
template <HeapObjectHeader::AccessMode mode, HeapObjectHeader::EncodedHalf part,
std::memory_order memory_order>
uint16_t HeapObjectHeader::LoadEncoded() const {
const uint16_t& half =
part == EncodedHalf::kLow ? encoded_low_ : encoded_high_;
if (mode == AccessMode::kNonAtomic) return half;
return v8::base::AsAtomicPtr(&half)->load(memory_order);
}
template <HeapObjectHeader::AccessMode mode, HeapObjectHeader::EncodedHalf part,
std::memory_order memory_order>
void HeapObjectHeader::StoreEncoded(uint16_t bits, uint16_t mask) {
// Caveat: Not all changes to HeapObjectHeader's bitfields go through
// StoreEncoded. The following have their own implementations and need to be
// kept in sync:
// - HeapObjectHeader::TryMarkAtomic
// - MarkObjectAsFullyConstructed (API)
DCHECK_EQ(0u, bits & ~mask);
uint16_t& half = part == EncodedHalf::kLow ? encoded_low_ : encoded_high_;
if (mode == AccessMode::kNonAtomic) {
half = (half & ~mask) | bits;
return;
}
// We don't perform CAS loop here assuming that only none of the info that
// shares the same encoded halfs change at the same time.
auto* atomic_encoded = v8::base::AsAtomicPtr(&half);
uint16_t value = atomic_encoded->load(std::memory_order_relaxed);
value = (value & ~mask) | bits;
atomic_encoded->store(value, memory_order);
}
} // namespace internal
} // namespace cppgc
......
// Copyright 2020 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef V8_HEAP_CPPGC_HEAP_PAGE_INL_H_
#define V8_HEAP_CPPGC_HEAP_PAGE_INL_H_
#include "src/heap/cppgc/heap-page.h"
namespace cppgc {
namespace internal {
// static
BasePage* BasePage::FromPayload(void* payload) {
return reinterpret_cast<BasePage*>(
(reinterpret_cast<uintptr_t>(payload) & kPageBaseMask) + kGuardPageSize);
}
// static
const BasePage* BasePage::FromPayload(const void* payload) {
return reinterpret_cast<const BasePage*>(
(reinterpret_cast<uintptr_t>(const_cast<void*>(payload)) &
kPageBaseMask) +
kGuardPageSize);
}
} // namespace internal
} // namespace cppgc
#endif // V8_HEAP_CPPGC_HEAP_PAGE_INL_H_
......@@ -9,12 +9,11 @@
#include "include/cppgc/internal/api-constants.h"
#include "src/base/logging.h"
#include "src/heap/cppgc/globals.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap-space.h"
#include "src/heap/cppgc/heap.h"
#include "src/heap/cppgc/object-start-bitmap-inl.h"
#include "src/heap/cppgc/object-start-bitmap.h"
#include "src/heap/cppgc/page-memory-inl.h"
#include "src/heap/cppgc/page-memory.h"
#include "src/heap/cppgc/raw-heap.h"
namespace cppgc {
......
......@@ -201,6 +201,20 @@ class V8_EXPORT_PRIVATE LargePage final : public BasePage {
size_t payload_size_;
};
// static
BasePage* BasePage::FromPayload(void* payload) {
return reinterpret_cast<BasePage*>(
(reinterpret_cast<uintptr_t>(payload) & kPageBaseMask) + kGuardPageSize);
}
// static
const BasePage* BasePage::FromPayload(const void* payload) {
return reinterpret_cast<const BasePage*>(
(reinterpret_cast<uintptr_t>(const_cast<void*>(payload)) &
kPageBaseMask) +
kGuardPageSize);
}
} // namespace internal
} // namespace cppgc
......
......@@ -8,8 +8,8 @@
#include "src/base/logging.h"
#include "src/base/platform/mutex.h"
#include "src/heap/cppgc/heap-page-inl.h"
#include "src/heap/cppgc/object-start-bitmap-inl.h"
#include "src/heap/cppgc/heap-page.h"
#include "src/heap/cppgc/object-start-bitmap.h"
namespace cppgc {
namespace internal {
......
......@@ -7,7 +7,7 @@
#include "src/heap/base/stack.h"
#include "src/heap/cppgc/garbage-collector.h"
#include "src/heap/cppgc/gc-invoker.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap-visitor.h"
#include "src/heap/cppgc/marker.h"
#include "src/heap/cppgc/prefinalizer-handler.h"
......
......@@ -2,11 +2,10 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "include/cppgc/liveness-broker.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/liveness-broker.h"
#include "src/heap/cppgc/heap-object-header.h"
namespace cppgc {
bool LivenessBroker::IsHeapObjectAliveImpl(const void* payload) const {
......
......@@ -5,8 +5,8 @@
#include "src/heap/cppgc/marker.h"
#include "include/cppgc/internal/process-heap.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-page-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap-page.h"
#include "src/heap/cppgc/heap-visitor.h"
#include "src/heap/cppgc/heap.h"
#include "src/heap/cppgc/liveness-broker.h"
......
......@@ -5,8 +5,8 @@
#include "src/heap/cppgc/marking-visitor.h"
#include "include/cppgc/garbage-collected.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-page-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap-page.h"
#include "src/heap/cppgc/heap.h"
#include "src/heap/cppgc/liveness-broker.h"
......
// Copyright 2020 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef V8_HEAP_CPPGC_OBJECT_ALLOCATOR_INL_H_
#define V8_HEAP_CPPGC_OBJECT_ALLOCATOR_INL_H_
#include <new>
#include "src/base/logging.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap-page-inl.h"
#include "src/heap/cppgc/object-allocator.h"
#include "src/heap/cppgc/object-start-bitmap-inl.h"
#include "src/heap/cppgc/object-start-bitmap.h"
#include "src/heap/cppgc/sanitizers.h"
namespace cppgc {
namespace internal {
void* ObjectAllocator::AllocateObject(size_t size, GCInfoIndex gcinfo) {
DCHECK(is_allocation_allowed());
const size_t allocation_size =
RoundUp<kAllocationGranularity>(size + sizeof(HeapObjectHeader));
const RawHeap::RegularSpaceType type =
GetInitialSpaceIndexForSize(allocation_size);
return AllocateObjectOnSpace(NormalPageSpace::From(raw_heap_->Space(type)),
allocation_size, gcinfo);
}
void* ObjectAllocator::AllocateObject(size_t size, GCInfoIndex gcinfo,
CustomSpaceIndex space_index) {
DCHECK(is_allocation_allowed());
const size_t allocation_size =
RoundUp<kAllocationGranularity>(size + sizeof(HeapObjectHeader));
return AllocateObjectOnSpace(
NormalPageSpace::From(raw_heap_->CustomSpace(space_index)),
allocation_size, gcinfo);
}
// static
RawHeap::RegularSpaceType ObjectAllocator::GetInitialSpaceIndexForSize(
size_t size) {
if (size < 64) {
if (size < 32) return RawHeap::RegularSpaceType::kNormal1;
return RawHeap::RegularSpaceType::kNormal2;
}
if (size < 128) return RawHeap::RegularSpaceType::kNormal3;
return RawHeap::RegularSpaceType::kNormal4;
}
void* ObjectAllocator::AllocateObjectOnSpace(NormalPageSpace* space,
size_t size, GCInfoIndex gcinfo) {
DCHECK_LT(0u, gcinfo);
NormalPageSpace::LinearAllocationBuffer& current_lab =
space->linear_allocation_buffer();
if (current_lab.size() < size) {
return OutOfLineAllocate(space, size, gcinfo);
}
void* raw = current_lab.Allocate(size);
SET_MEMORY_ACCESIBLE(raw, size);
auto* header = new (raw) HeapObjectHeader(size, gcinfo);
NormalPage::From(BasePage::FromPayload(header))
->object_start_bitmap()
.SetBit(reinterpret_cast<ConstAddress>(header));
return header->Payload();
}
} // namespace internal
} // namespace cppgc
#endif // V8_HEAP_CPPGC_OBJECT_ALLOCATOR_INL_H_
......@@ -8,13 +8,11 @@
#include "src/base/macros.h"
#include "src/heap/cppgc/free-list.h"
#include "src/heap/cppgc/globals.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap-page.h"
#include "src/heap/cppgc/heap-space.h"
#include "src/heap/cppgc/heap-visitor.h"
#include "src/heap/cppgc/heap.h"
#include "src/heap/cppgc/object-allocator-inl.h"
#include "src/heap/cppgc/object-start-bitmap.h"
#include "src/heap/cppgc/page-memory.h"
#include "src/heap/cppgc/stats-collector.h"
......
......@@ -8,8 +8,13 @@
#include "include/cppgc/allocation.h"
#include "include/cppgc/internal/gc-info.h"
#include "include/cppgc/macros.h"
#include "src/base/logging.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap-page.h"
#include "src/heap/cppgc/heap-space.h"
#include "src/heap/cppgc/object-start-bitmap.h"
#include "src/heap/cppgc/raw-heap.h"
#include "src/heap/cppgc/sanitizers.h"
namespace cppgc {
......@@ -71,6 +76,58 @@ class V8_EXPORT_PRIVATE ObjectAllocator final : public cppgc::AllocationHandle {
size_t no_allocation_scope_ = 0;
};
void* ObjectAllocator::AllocateObject(size_t size, GCInfoIndex gcinfo) {
DCHECK(is_allocation_allowed());
const size_t allocation_size =
RoundUp<kAllocationGranularity>(size + sizeof(HeapObjectHeader));
const RawHeap::RegularSpaceType type =
GetInitialSpaceIndexForSize(allocation_size);
return AllocateObjectOnSpace(NormalPageSpace::From(raw_heap_->Space(type)),
allocation_size, gcinfo);
}
void* ObjectAllocator::AllocateObject(size_t size, GCInfoIndex gcinfo,
CustomSpaceIndex space_index) {
DCHECK(is_allocation_allowed());
const size_t allocation_size =
RoundUp<kAllocationGranularity>(size + sizeof(HeapObjectHeader));
return AllocateObjectOnSpace(
NormalPageSpace::From(raw_heap_->CustomSpace(space_index)),
allocation_size, gcinfo);
}
// static
RawHeap::RegularSpaceType ObjectAllocator::GetInitialSpaceIndexForSize(
size_t size) {
if (size < 64) {
if (size < 32) return RawHeap::RegularSpaceType::kNormal1;
return RawHeap::RegularSpaceType::kNormal2;
}
if (size < 128) return RawHeap::RegularSpaceType::kNormal3;
return RawHeap::RegularSpaceType::kNormal4;
}
void* ObjectAllocator::AllocateObjectOnSpace(NormalPageSpace* space,
size_t size, GCInfoIndex gcinfo) {
DCHECK_LT(0u, gcinfo);
NormalPageSpace::LinearAllocationBuffer& current_lab =
space->linear_allocation_buffer();
if (current_lab.size() < size) {
return OutOfLineAllocate(space, size, gcinfo);
}
void* raw = current_lab.Allocate(size);
SET_MEMORY_ACCESIBLE(raw, size);
auto* header = new (raw) HeapObjectHeader(size, gcinfo);
NormalPage::From(BasePage::FromPayload(header))
->object_start_bitmap()
.SetBit(reinterpret_cast<ConstAddress>(header));
return header->Payload();
}
} // namespace internal
} // namespace cppgc
......
// Copyright 2020 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef V8_HEAP_CPPGC_OBJECT_START_BITMAP_INL_H_
#define V8_HEAP_CPPGC_OBJECT_START_BITMAP_INL_H_
#include <algorithm>
#include "src/base/bits.h"
#include "src/heap/cppgc/object-start-bitmap.h"
namespace cppgc {
namespace internal {
ObjectStartBitmap::ObjectStartBitmap(Address offset) : offset_(offset) {
Clear();
}
HeapObjectHeader* ObjectStartBitmap::FindHeader(
ConstAddress address_maybe_pointing_to_the_middle_of_object) const {
DCHECK_LE(offset_, address_maybe_pointing_to_the_middle_of_object);
size_t object_offset =
address_maybe_pointing_to_the_middle_of_object - offset_;
size_t object_start_number = object_offset / kAllocationGranularity;
size_t cell_index = object_start_number / kBitsPerCell;
DCHECK_GT(object_start_bit_map_.size(), cell_index);
const size_t bit = object_start_number & kCellMask;
uint8_t byte = object_start_bit_map_[cell_index] & ((1 << (bit + 1)) - 1);
while (!byte && cell_index) {
DCHECK_LT(0u, cell_index);
byte = object_start_bit_map_[--cell_index];
}
const int leading_zeroes = v8::base::bits::CountLeadingZeros(byte);
object_start_number =
(cell_index * kBitsPerCell) + (kBitsPerCell - 1) - leading_zeroes;
object_offset = object_start_number * kAllocationGranularity;
return reinterpret_cast<HeapObjectHeader*>(object_offset + offset_);
}
void ObjectStartBitmap::SetBit(ConstAddress header_address) {
size_t cell_index, object_bit;
ObjectStartIndexAndBit(header_address, &cell_index, &object_bit);
object_start_bit_map_[cell_index] |= (1 << object_bit);
}
void ObjectStartBitmap::ClearBit(ConstAddress header_address) {
size_t cell_index, object_bit;
ObjectStartIndexAndBit(header_address, &cell_index, &object_bit);
object_start_bit_map_[cell_index] &= ~(1 << object_bit);
}
bool ObjectStartBitmap::CheckBit(ConstAddress header_address) const {
size_t cell_index, object_bit;
ObjectStartIndexAndBit(header_address, &cell_index, &object_bit);
return object_start_bit_map_[cell_index] & (1 << object_bit);
}
void ObjectStartBitmap::ObjectStartIndexAndBit(ConstAddress header_address,
size_t* cell_index,
size_t* bit) const {
const size_t object_offset = header_address - offset_;
DCHECK(!(object_offset & kAllocationMask));
const size_t object_start_number = object_offset / kAllocationGranularity;
*cell_index = object_start_number / kBitsPerCell;
DCHECK_GT(kBitmapSize, *cell_index);
*bit = object_start_number & kCellMask;
}
template <typename Callback>
inline void ObjectStartBitmap::Iterate(Callback callback) const {
for (size_t cell_index = 0; cell_index < kReservedForBitmap; cell_index++) {
if (!object_start_bit_map_[cell_index]) continue;
uint8_t value = object_start_bit_map_[cell_index];
while (value) {
const int trailing_zeroes = v8::base::bits::CountTrailingZeros(value);
const size_t object_start_number =
(cell_index * kBitsPerCell) + trailing_zeroes;
const Address object_address =
offset_ + (kAllocationGranularity * object_start_number);
callback(object_address);
// Clear current object bit in temporary value to advance iteration.
value &= ~(1 << (object_start_number & kCellMask));
}
}
}
void ObjectStartBitmap::Clear() {
std::fill(object_start_bit_map_.begin(), object_start_bit_map_.end(), 0);
}
} // namespace internal
} // namespace cppgc
#endif // V8_HEAP_CPPGC_OBJECT_START_BITMAP_INL_H_
......@@ -10,6 +10,7 @@
#include <array>
#include "src/base/bits.h"
#include "src/base/macros.h"
#include "src/heap/cppgc/globals.h"
......@@ -74,6 +75,83 @@ class V8_EXPORT_PRIVATE ObjectStartBitmap {
std::array<uint8_t, kReservedForBitmap> object_start_bit_map_;
};
ObjectStartBitmap::ObjectStartBitmap(Address offset) : offset_(offset) {
Clear();
}
HeapObjectHeader* ObjectStartBitmap::FindHeader(
ConstAddress address_maybe_pointing_to_the_middle_of_object) const {
DCHECK_LE(offset_, address_maybe_pointing_to_the_middle_of_object);
size_t object_offset =
address_maybe_pointing_to_the_middle_of_object - offset_;
size_t object_start_number = object_offset / kAllocationGranularity;
size_t cell_index = object_start_number / kBitsPerCell;
DCHECK_GT(object_start_bit_map_.size(), cell_index);
const size_t bit = object_start_number & kCellMask;
uint8_t byte = object_start_bit_map_[cell_index] & ((1 << (bit + 1)) - 1);
while (!byte && cell_index) {
DCHECK_LT(0u, cell_index);
byte = object_start_bit_map_[--cell_index];
}
const int leading_zeroes = v8::base::bits::CountLeadingZeros(byte);
object_start_number =
(cell_index * kBitsPerCell) + (kBitsPerCell - 1) - leading_zeroes;
object_offset = object_start_number * kAllocationGranularity;
return reinterpret_cast<HeapObjectHeader*>(object_offset + offset_);
}
void ObjectStartBitmap::SetBit(ConstAddress header_address) {
size_t cell_index, object_bit;
ObjectStartIndexAndBit(header_address, &cell_index, &object_bit);
object_start_bit_map_[cell_index] |= (1 << object_bit);
}
void ObjectStartBitmap::ClearBit(ConstAddress header_address) {
size_t cell_index, object_bit;
ObjectStartIndexAndBit(header_address, &cell_index, &object_bit);
object_start_bit_map_[cell_index] &= ~(1 << object_bit);
}
bool ObjectStartBitmap::CheckBit(ConstAddress header_address) const {
size_t cell_index, object_bit;
ObjectStartIndexAndBit(header_address, &cell_index, &object_bit);
return object_start_bit_map_[cell_index] & (1 << object_bit);
}
void ObjectStartBitmap::ObjectStartIndexAndBit(ConstAddress header_address,
size_t* cell_index,
size_t* bit) const {
const size_t object_offset = header_address - offset_;
DCHECK(!(object_offset & kAllocationMask));
const size_t object_start_number = object_offset / kAllocationGranularity;
*cell_index = object_start_number / kBitsPerCell;
DCHECK_GT(kBitmapSize, *cell_index);
*bit = object_start_number & kCellMask;
}
template <typename Callback>
inline void ObjectStartBitmap::Iterate(Callback callback) const {
for (size_t cell_index = 0; cell_index < kReservedForBitmap; cell_index++) {
if (!object_start_bit_map_[cell_index]) continue;
uint8_t value = object_start_bit_map_[cell_index];
while (value) {
const int trailing_zeroes = v8::base::bits::CountTrailingZeros(value);
const size_t object_start_number =
(cell_index * kBitsPerCell) + trailing_zeroes;
const Address object_address =
offset_ + (kAllocationGranularity * object_start_number);
callback(object_address);
// Clear current object bit in temporary value to advance iteration.
value &= ~(1 << (object_start_number & kCellMask));
}
}
}
void ObjectStartBitmap::Clear() {
std::fill(object_start_bit_map_.begin(), object_start_bit_map_.end(), 0);
}
} // namespace internal
} // namespace cppgc
......
// Copyright 2020 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#ifndef V8_HEAP_CPPGC_PAGE_MEMORY_INL_H_
#define V8_HEAP_CPPGC_PAGE_MEMORY_INL_H_
#include "src/heap/cppgc/page-memory.h"
namespace cppgc {
namespace internal {
// Returns true if the provided allocator supports committing at the required
// granularity.
inline bool SupportsCommittingGuardPages(PageAllocator* allocator) {
return kGuardPageSize % allocator->CommitPageSize() == 0;
}
Address NormalPageMemoryRegion::Lookup(ConstAddress address) const {
size_t index = GetIndex(address);
if (!page_memories_in_use_[index]) return nullptr;
const MemoryRegion writeable_region = GetPageMemory(index).writeable_region();
return writeable_region.Contains(address) ? writeable_region.base() : nullptr;
}
Address LargePageMemoryRegion::Lookup(ConstAddress address) const {
const MemoryRegion writeable_region = GetPageMemory().writeable_region();
return writeable_region.Contains(address) ? writeable_region.base() : nullptr;
}
Address PageMemoryRegion::Lookup(ConstAddress address) const {
DCHECK(reserved_region().Contains(address));
return is_large()
? static_cast<const LargePageMemoryRegion*>(this)->Lookup(address)
: static_cast<const NormalPageMemoryRegion*>(this)->Lookup(
address);
}
PageMemoryRegion* PageMemoryRegionTree::Lookup(ConstAddress address) const {
auto it = set_.upper_bound(address);
// This check also covers set_.size() > 0, since for empty vectors it is
// guaranteed that begin() == end().
if (it == set_.begin()) return nullptr;
auto* result = std::next(it, -1)->second;
if (address < result->reserved_region().end()) return result;
return nullptr;
}
Address PageBackend::Lookup(ConstAddress address) const {
PageMemoryRegion* pmr = page_memory_region_tree_.Lookup(address);
return pmr ? pmr->Lookup(address) : nullptr;
}
} // namespace internal
} // namespace cppgc
#endif // V8_HEAP_CPPGC_PAGE_MEMORY_INL_H_
......@@ -5,7 +5,6 @@
#include "src/heap/cppgc/page-memory.h"
#include "src/base/macros.h"
#include "src/heap/cppgc/page-memory-inl.h"
namespace cppgc {
namespace internal {
......
......@@ -231,6 +231,47 @@ class V8_EXPORT_PRIVATE PageBackend final {
large_page_memory_regions_;
};
// Returns true if the provided allocator supports committing at the required
// granularity.
inline bool SupportsCommittingGuardPages(PageAllocator* allocator) {
return kGuardPageSize % allocator->CommitPageSize() == 0;
}
Address NormalPageMemoryRegion::Lookup(ConstAddress address) const {
size_t index = GetIndex(address);
if (!page_memories_in_use_[index]) return nullptr;
const MemoryRegion writeable_region = GetPageMemory(index).writeable_region();
return writeable_region.Contains(address) ? writeable_region.base() : nullptr;
}
Address LargePageMemoryRegion::Lookup(ConstAddress address) const {
const MemoryRegion writeable_region = GetPageMemory().writeable_region();
return writeable_region.Contains(address) ? writeable_region.base() : nullptr;
}
Address PageMemoryRegion::Lookup(ConstAddress address) const {
DCHECK(reserved_region().Contains(address));
return is_large()
? static_cast<const LargePageMemoryRegion*>(this)->Lookup(address)
: static_cast<const NormalPageMemoryRegion*>(this)->Lookup(
address);
}
PageMemoryRegion* PageMemoryRegionTree::Lookup(ConstAddress address) const {
auto it = set_.upper_bound(address);
// This check also covers set_.size() > 0, since for empty vectors it is
// guaranteed that begin() == end().
if (it == set_.begin()) return nullptr;
auto* result = std::next(it, -1)->second;
if (address < result->reserved_region().end()) return result;
return nullptr;
}
Address PageBackend::Lookup(ConstAddress address) const {
PageMemoryRegion* pmr = page_memory_region_tree_.Lookup(address);
return pmr ? pmr->Lookup(address) : nullptr;
}
} // namespace internal
} // namespace cppgc
......
......@@ -6,7 +6,7 @@
#include "include/cppgc/internal/persistent-node.h"
#include "src/base/macros.h"
#include "src/heap/cppgc/heap-page-inl.h"
#include "src/heap/cppgc/heap-page.h"
#include "src/heap/cppgc/heap.h"
namespace cppgc {
......
......@@ -8,7 +8,7 @@
#include <memory>
#include "src/base/platform/platform.h"
#include "src/heap/cppgc/heap-page-inl.h"
#include "src/heap/cppgc/heap-page.h"
#include "src/heap/cppgc/heap.h"
#include "src/heap/cppgc/liveness-broker.h"
......
......@@ -13,12 +13,10 @@
#include "src/base/platform/mutex.h"
#include "src/heap/cppgc/free-list.h"
#include "src/heap/cppgc/globals.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap-page.h"
#include "src/heap/cppgc/heap-space.h"
#include "src/heap/cppgc/heap-visitor.h"
#include "src/heap/cppgc/object-start-bitmap-inl.h"
#include "src/heap/cppgc/object-start-bitmap.h"
#include "src/heap/cppgc/raw-heap.h"
#include "src/heap/cppgc/sanitizers.h"
......
......@@ -5,9 +5,9 @@
#include "src/heap/cppgc/visitor.h"
#include "src/heap/cppgc/gc-info-table.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap-page.h"
#include "src/heap/cppgc/page-memory-inl.h"
#include "src/heap/cppgc/page-memory.h"
#include "src/heap/cppgc/sanitizers.h"
namespace cppgc {
......
......@@ -6,9 +6,8 @@
#include "include/cppgc/internal/pointer-policies.h"
#include "src/heap/cppgc/globals.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap-page-inl.h"
#include "src/heap/cppgc/heap-page.h"
#include "src/heap/cppgc/heap.h"
#include "src/heap/cppgc/marker.h"
#include "src/heap/cppgc/marking-visitor.h"
......
......@@ -10,11 +10,11 @@
#include "include/cppgc/platform.h"
#include "include/v8-platform.h"
#include "src/heap/cppgc/globals.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-page-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap-page.h"
#include "src/heap/cppgc/heap-space.h"
#include "src/heap/cppgc/heap-visitor.h"
#include "src/heap/cppgc/page-memory-inl.h"
#include "src/heap/cppgc/page-memory.h"
#include "src/heap/cppgc/raw-heap.h"
#include "src/heap/cppgc/stats-collector.h"
#include "src/heap/cppgc/sweeper.h"
......
......@@ -4,7 +4,7 @@
#include "include/cppgc/allocation.h"
#include "include/cppgc/custom-space.h"
#include "src/heap/cppgc/heap-page-inl.h"
#include "src/heap/cppgc/heap-page.h"
#include "src/heap/cppgc/raw-heap.h"
#include "test/unittests/heap/cppgc/tests.h"
......
......@@ -6,7 +6,7 @@
#include "include/cppgc/allocation.h"
#include "include/cppgc/type-traits.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap.h"
#include "test/unittests/heap/cppgc/tests.h"
#include "testing/gtest/include/gtest/gtest.h"
......
......@@ -12,7 +12,6 @@
#include "src/base/macros.h"
#include "src/base/platform/platform.h"
#include "src/heap/cppgc/globals.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "testing/gtest/include/gtest/gtest.h"
namespace cppgc {
......
......@@ -10,10 +10,7 @@
#include "include/cppgc/persistent.h"
#include "src/base/macros.h"
#include "src/heap/cppgc/globals.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap-page-inl.h"
#include "src/heap/cppgc/page-memory-inl.h"
#include "src/heap/cppgc/page-memory.h"
#include "src/heap/cppgc/raw-heap.h"
#include "test/unittests/heap/cppgc/tests.h"
......
......@@ -8,7 +8,7 @@
#include "include/cppgc/internal/pointer-policies.h"
#include "include/cppgc/member.h"
#include "include/cppgc/persistent.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/marking-visitor.h"
#include "src/heap/cppgc/stats-collector.h"
#include "test/unittests/heap/cppgc/tests.h"
......
......@@ -9,7 +9,7 @@
#include "include/cppgc/persistent.h"
#include "include/cppgc/source-location.h"
#include "src/heap/cppgc/globals.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/marker.h"
#include "test/unittests/heap/cppgc/tests.h"
#include "testing/gtest/include/gtest/gtest.h"
......
......@@ -6,7 +6,7 @@
#include "include/cppgc/allocation.h"
#include "include/cppgc/persistent.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap.h"
#include "test/unittests/heap/cppgc/tests.h"
#include "testing/gtest/include/gtest/gtest.h"
......
......@@ -7,10 +7,8 @@
#include "include/cppgc/allocation.h"
#include "src/base/macros.h"
#include "src/heap/cppgc/globals.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/object-start-bitmap-inl.h"
#include "src/heap/cppgc/page-memory-inl.h"
#include "src/heap/cppgc/page-memory.h"
#include "src/heap/cppgc/raw-heap.h"
#include "test/unittests/heap/cppgc/tests.h"
#include "testing/gtest/include/gtest/gtest.h"
......
......@@ -5,7 +5,6 @@
#include "src/heap/cppgc/page-memory.h"
#include "src/base/page-allocator.h"
#include "src/heap/cppgc/page-memory-inl.h"
#include "testing/gtest/include/gtest/gtest.h"
namespace cppgc {
......
......@@ -7,7 +7,7 @@
#include "include/cppgc/allocation.h"
#include "include/cppgc/garbage-collected.h"
#include "include/cppgc/persistent.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap.h"
#include "test/unittests/heap/cppgc/tests.h"
#include "testing/gtest/include/gtest/gtest.h"
......
......@@ -9,12 +9,10 @@
#include "include/cppgc/allocation.h"
#include "include/cppgc/persistent.h"
#include "src/heap/cppgc/globals.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/heap-page-inl.h"
#include "src/heap/cppgc/heap-page.h"
#include "src/heap/cppgc/heap-visitor.h"
#include "src/heap/cppgc/heap.h"
#include "src/heap/cppgc/page-memory-inl.h"
#include "src/heap/cppgc/page-memory.h"
#include "src/heap/cppgc/stats-collector.h"
#include "test/unittests/heap/cppgc/tests.h"
......
......@@ -9,7 +9,6 @@
#include <vector>
#include "include/cppgc/internal/pointer-policies.h"
#include "src/heap/cppgc/heap-object-header-inl.h"
#include "src/heap/cppgc/heap-object-header.h"
#include "src/heap/cppgc/marker.h"
#include "test/unittests/heap/cppgc/tests.h"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment