cppgc: Port FreeList implementation

- implemented as a single-linked list with head and tail
  pointers. The tail pointer is needed for freelist appending;
- stores entries in buckets, where bucket[log2(size)] stores
  entries >= size;
- implements worst fit allocation to amortize free list call;
- ported from Blink: https://bit.ly/2yC8XKJ.

Bug: chromium:1056170
Change-Id: I26cf62c948c95a7cbfecd5f7f22ad975e6b8c732
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2157376
Commit-Queue: Anton Bikineev <bikineev@chromium.org>
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Reviewed-by: Omer Katz <omerkatz@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#67310}

BUILD.gn

@@ -4014,6 +4014,8 @@ v8_source_set("cppgc_base") {
     "include/cppgc/visitor.h",
     "include/v8config.h",
     "src/heap/cppgc/allocation.cc",
+    "src/heap/cppgc/free-list.cc",
+    "src/heap/cppgc/free-list.h",
     "src/heap/cppgc/gc-info-table.cc",
     "src/heap/cppgc/gc-info-table.h",
     "src/heap/cppgc/gc-info.cc",

src/heap/cppgc/free-list.cc

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/heap/cppgc/free-list.h"
+
+#include <algorithm>
+
+#include "include/cppgc/internal/logging.h"
+#include "src/base/bits.h"
+#include "src/heap/cppgc/globals.h"
+#include "src/heap/cppgc/heap-object-header-inl.h"
+
+namespace cppgc {
+namespace internal {
+
+namespace {
+uint32_t BucketIndexForSize(uint32_t size) {
+  return v8::base::bits::WhichPowerOfTwo(
+      v8::base::bits::RoundDownToPowerOfTwo32(size));
+}
+}  // namespace
+
+class FreeList::Entry : public HeapObjectHeader {
+ public:
+  explicit Entry(size_t size) : HeapObjectHeader(size, kFreeListGCInfoIndex) {
+    static_assert(sizeof(Entry) == kFreeListEntrySize, "Sizes must match");
+  }
+
+  Entry* Next() const { return next_; }
+  void SetNext(Entry* next) { next_ = next; }
+
+  void Link(Entry** previous_next) {
+    next_ = *previous_next;
+    *previous_next = this;
+  }
+  void Unlink(Entry** previous_next) {
+    *previous_next = next_;
+    next_ = nullptr;
+  }
+
+ private:
+  Entry* next_ = nullptr;
+};
+
+FreeList::FreeList() { Clear(); }
+
+FreeList::FreeList(FreeList&& other) V8_NOEXCEPT
+    : free_list_heads_(std::move(other.free_list_heads_)),
+      free_list_tails_(std::move(other.free_list_tails_)),
+      biggest_free_list_index_(std::move(other.biggest_free_list_index_)) {
+  other.Clear();
+}
+
+FreeList& FreeList::operator=(FreeList&& other) V8_NOEXCEPT {
+  Clear();
+  Append(std::move(other));
+  DCHECK(other.IsEmpty());
+  return *this;
+}
+
+void FreeList::Add(FreeList::Block block) {
+  const size_t size = block.size;
+  DCHECK_GT(kPageSize, size);
+  DCHECK_LE(kFreeListEntrySize, size);
+
+  Entry* entry = new (block.address) Entry(size);
+  const size_t index = BucketIndexForSize(static_cast<uint32_t>(size));
+  entry->Link(&free_list_heads_[index]);
+  biggest_free_list_index_ = std::max(biggest_free_list_index_, index);
+  if (!entry->Next()) {
+    free_list_tails_[index] = entry;
+  }
+}
+
+void FreeList::Append(FreeList&& other) {
+#if DEBUG
+  const size_t expected_size = Size() + other.Size();
+#endif
+  // Newly created entries get added to the head.
+  for (size_t index = 0; index < free_list_tails_.size(); ++index) {
+    Entry* other_tail = other.free_list_tails_[index];
+    Entry*& this_head = this->free_list_heads_[index];
+    if (other_tail) {
+      other_tail->SetNext(this_head);
+      if (!this_head) {
+        this->free_list_tails_[index] = other_tail;
+      }
+      this_head = other.free_list_heads_[index];
+      other.free_list_heads_[index] = nullptr;
+      other.free_list_tails_[index] = nullptr;
+    }
+  }
+
+  biggest_free_list_index_ =
+      std::max(biggest_free_list_index_, other.biggest_free_list_index_);
+  other.biggest_free_list_index_ = 0;
+#if DEBUG
+  DCHECK_EQ(expected_size, Size());
+#endif
+  DCHECK(other.IsEmpty());
+}
+
+FreeList::Block FreeList::Allocate(size_t allocation_size) {
+  // Try reusing a block from the largest bin. The underlying reasoning
+  // being that we want to amortize this slow allocation call by carving
+  // off as a large a free block as possible in one go; a block that will
+  // service this block and let following allocations be serviced quickly
+  // by bump allocation.
+  // bucket_size represents minimal size of entries in a bucket.
+  size_t bucket_size = static_cast<size_t>(1) << biggest_free_list_index_;
+  size_t index = biggest_free_list_index_;
+  for (; index > 0; --index, bucket_size >>= 1) {
+    DCHECK(IsConsistent(index));
+    Entry* entry = free_list_heads_[index];
+    if (allocation_size > bucket_size) {
+      // Final bucket candidate; check initial entry if it is able
+      // to service this allocation. Do not perform a linear scan,
+      // as it is considered too costly.
+      if (!entry || entry->GetSize() < allocation_size) break;
+    }
+    if (entry) {
+      if (!entry->Next()) {
+        DCHECK_EQ(entry, free_list_tails_[index]);
+        free_list_tails_[index] = nullptr;
+      }
+      entry->Unlink(&free_list_heads_[index]);
+      biggest_free_list_index_ = index;
+      return {entry, entry->GetSize()};
+    }
+  }
+  biggest_free_list_index_ = index;
+  return {nullptr, 0u};
+}
+
+void FreeList::Clear() {
+  std::fill(free_list_heads_.begin(), free_list_heads_.end(), nullptr);
+  std::fill(free_list_tails_.begin(), free_list_tails_.end(), nullptr);
+  biggest_free_list_index_ = 0;
+}
+
+size_t FreeList::Size() const {
+  size_t size = 0;
+  for (auto* entry : free_list_heads_) {
+    while (entry) {
+      size += entry->GetSize();
+      entry = entry->Next();
+    }
+  }
+  return size;
+}
+
+bool FreeList::IsEmpty() const {
+  return std::all_of(free_list_heads_.cbegin(), free_list_heads_.cend(),
+                     [](const auto* entry) { return !entry; });
+}
+
+bool FreeList::IsConsistent(size_t index) const {
+  // Check that freelist head and tail pointers are consistent, i.e.
+  // - either both are nulls (no entries in the bucket);
+  // - or both are non-nulls and the tail points to the end.
+  return (!free_list_heads_[index] && !free_list_tails_[index]) ||
+         (free_list_heads_[index] && free_list_tails_[index] &&
+          !free_list_tails_[index]->Next());
+}
+
+}  // namespace internal
+}  // namespace cppgc

src/heap/cppgc/free-list.h

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_HEAP_CPPGC_FREE_LIST_H_
+#define V8_HEAP_CPPGC_FREE_LIST_H_
+
+#include <array>
+
+#include "src/base/macros.h"
+#include "src/heap/cppgc/globals.h"
+#include "src/heap/cppgc/heap-object-header.h"
+
+namespace cppgc {
+namespace internal {
+
+class V8_EXPORT_PRIVATE FreeList {
+ public:
+  struct Block {
+    void* address;
+    size_t size;
+  };
+
+  FreeList();
+
+  FreeList(const FreeList&) = delete;
+  FreeList& operator=(const FreeList&) = delete;
+
+  FreeList(FreeList&& freelist) V8_NOEXCEPT;
+  FreeList& operator=(FreeList&& freelist) V8_NOEXCEPT;
+
+  // Allocates entries which are at least of the provided size.
+  Block Allocate(size_t);
+
+  // Adds block to the freelist. The minimal block size is two words.
+  void Add(Block);
+
+  // Append other freelist into this.
+  void Append(FreeList&&);
+
+  void Clear();
+
+  size_t Size() const;
+  bool IsEmpty() const;
+
+ private:
+  class Entry;
+
+  bool IsConsistent(size_t) const;
+
+  // All |Entry|s in the nth list have size >= 2^n.
+  std::array<Entry*, kPageSizeLog2> free_list_heads_;
+  std::array<Entry*, kPageSizeLog2> free_list_tails_;
+  size_t biggest_free_list_index_ = 0;
+};
+
+}  // namespace internal
+}  // namespace cppgc
+
+#endif  // V8_HEAP_CPPGC_FREE_LIST_H_

src/heap/cppgc/globals.h

@@ -8,6 +8,8 @@
 #include <stddef.h>
 #include <stdint.h>
 
+#include "include/cppgc/internal/gc-info.h"
+
 namespace cppgc {
 namespace internal {
 
@@ -37,6 +39,9 @@ constexpr size_t kGuardPageSize = 4096;
 
 constexpr size_t kLargeObjectSizeThreshold = kPageSize / 2;
 
+constexpr GCInfoIndex kFreeListGCInfoIndex = 0;
+constexpr size_t kFreeListEntrySize = 2 * sizeof(uintptr_t);
+
 }  // namespace internal
 }  // namespace cppgc
 

src/heap/cppgc/heap-object-header-inl.h

@@ -11,6 +11,7 @@
 #include "src/base/logging.h"
 #include "src/base/macros.h"
 #include "src/heap/cppgc/gc-info-table.h"
+#include "src/heap/cppgc/globals.h"
 #include "src/heap/cppgc/heap-object-header.h"
 
 namespace cppgc {
@@ -33,7 +34,7 @@ HeapObjectHeader::HeapObjectHeader(size_t size, GCInfoIndex gc_info_index) {
   USE(padding_);
 #endif  // defined(V8_TARGET_ARCH_64_BIT)
   DCHECK_LT(gc_info_index, GCInfoTable::kMaxIndex);
-  DCHECK_EQ(0u, size & kAllocationMask);
+  DCHECK_EQ(0u, size & (sizeof(HeapObjectHeader) - 1));
   DCHECK_GE(kMaxSize, size);
   encoded_high_ = GCInfoIndexField::encode(gc_info_index);
   encoded_low_ = EncodeSize(size);
@@ -111,6 +112,11 @@ bool HeapObjectHeader::TryMarkAtomic() {
                                                  std::memory_order_relaxed);
 }
 
+template <HeapObjectHeader::AccessMode mode>
+bool HeapObjectHeader::IsFree() const {
+  return GetGCInfoIndex() == kFreeListGCInfoIndex;
+}
+
 template <HeapObjectHeader::AccessMode mode, HeapObjectHeader::EncodedHalf part,
           std::memory_order memory_order>
 uint16_t HeapObjectHeader::LoadEncoded() const {

src/heap/cppgc/heap-object-header.h

@@ -41,7 +41,7 @@ namespace internal {
 //   stored in |LargeObjectPage::PayloadSize()|.
 // - |mark bit| and |in construction| bits are located in separate 16-bit halves
 //    to allow potentially accessing them non-atomically.
-class HeapObjectHeader final {
+class HeapObjectHeader {
  public:
   enum class AccessMode : uint8_t { kNonAtomic, kAtomic };
 
@@ -77,6 +77,9 @@ class HeapObjectHeader final {
   void Unmark();
   inline bool TryMarkAtomic();
 
+  template <AccessMode = AccessMode::kNonAtomic>
+  bool IsFree() const;
+
   void Finalize();
 
  private:
@@ -102,7 +105,7 @@ class HeapObjectHeader final {
   static constexpr uint16_t EncodeSize(size_t size) {
     // Essentially, gets optimized to >> 1.
     using SizeField = UnusedField2::Next<size_t, 14>;
-    return SizeField::encode(size) / kAllocationGranularity;
+    return SizeField::encode(size / kAllocationGranularity);
   }
 
   V8_EXPORT_PRIVATE void CheckApiConstants();

test/unittests/BUILD.gn

@@ -45,6 +45,7 @@ v8_source_set("cppgc_unittests_sources") {
 
   sources = [
     "heap/cppgc/finalizer-trait_unittest.cc",
+    "heap/cppgc/free-list_unittest.cc",
     "heap/cppgc/garbage-collected_unittest.cc",
     "heap/cppgc/gc-info_unittest.cc",
     "heap/cppgc/heap-object-header_unittest.cc",

test/unittests/heap/cppgc/free-list_unittest.cc

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <memory>
+#include <numeric>
+#include <vector>
+
+#include "src/base/bits.h"
+#include "src/heap/cppgc/free-list.h"
+#include "src/heap/cppgc/globals.h"
+#include "src/heap/cppgc/heap-object-header.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace cppgc {
+namespace internal {
+namespace {
+
+class Block {
+ public:
+  Block() = default;
+  explicit Block(size_t size) : address_(calloc(1, size)), size_(size) {}
+
+  Block(Block&& other) V8_NOEXCEPT : address_(other.address_),
+                                     size_(other.size_) {
+    other.address_ = nullptr;
+    other.size_ = 0;
+  }
+
+  Block& operator=(Block&& other) V8_NOEXCEPT {
+    address_ = other.address_;
+    size_ = other.size_;
+    other.address_ = nullptr;
+    other.size_ = 0;
+    return *this;
+  }
+
+  ~Block() { free(address_); }
+
+  void* Address() const { return address_; }
+  size_t Size() const { return size_; }
+
+ private:
+  void* address_ = nullptr;
+  size_t size_ = 0;
+};
+
+std::vector<Block> CreateEntries() {
+  static constexpr size_t kFreeListEntrySizeLog2 =
+      v8::base::bits::WhichPowerOfTwo(kFreeListEntrySize);
+  std::vector<Block> vector;
+  vector.reserve(kPageSizeLog2);
+  for (size_t i = kFreeListEntrySizeLog2; i < kPageSizeLog2; ++i) {
+    vector.emplace_back(static_cast<size_t>(1u) << i);
+  }
+  return vector;
+}
+
+FreeList CreatePopulatedFreeList(const std::vector<Block>& blocks) {
+  FreeList list;
+  for (const auto& block : blocks) {
+    list.Add({block.Address(), block.Size()});
+  }
+  return list;
+}
+
+}  // namespace
+
+TEST(FreeListTest, Empty) {
+  FreeList list;
+  EXPECT_TRUE(list.IsEmpty());
+  EXPECT_EQ(0u, list.Size());
+
+  auto block = list.Allocate(16);
+  EXPECT_EQ(nullptr, block.address);
+  EXPECT_EQ(0u, block.size);
+}
+
+TEST(FreeListTest, Add) {
+  auto blocks = CreateEntries();
+  FreeList list = CreatePopulatedFreeList(blocks);
+  EXPECT_FALSE(list.IsEmpty());
+  const size_t allocated_size = std::accumulate(
+      blocks.cbegin(), blocks.cend(), 0u,
+      [](size_t acc, const Block& b) { return acc + b.Size(); });
+  EXPECT_EQ(allocated_size, list.Size());
+}
+
+TEST(FreeListTest, Clear) {
+  auto blocks = CreateEntries();
+  FreeList list = CreatePopulatedFreeList(blocks);
+  list.Clear();
+  EXPECT_EQ(0u, list.Size());
+  EXPECT_TRUE(list.IsEmpty());
+}
+
+TEST(FreeListTest, Move) {
+  {
+    auto blocks = CreateEntries();
+    FreeList list1 = CreatePopulatedFreeList(blocks);
+    const size_t expected_size = list1.Size();
+    FreeList list2 = std::move(list1);
+    EXPECT_EQ(expected_size, list2.Size());
+    EXPECT_FALSE(list2.IsEmpty());
+    EXPECT_EQ(0u, list1.Size());
+    EXPECT_TRUE(list1.IsEmpty());
+  }
+  {
+    auto blocks1 = CreateEntries();
+    FreeList list1 = CreatePopulatedFreeList(blocks1);
+    const size_t expected_size = list1.Size();
+
+    auto blocks2 = CreateEntries();
+    FreeList list2 = CreatePopulatedFreeList(blocks2);
+
+    list2 = std::move(list1);
+    EXPECT_EQ(expected_size, list2.Size());
+    EXPECT_FALSE(list2.IsEmpty());
+    EXPECT_EQ(0u, list1.Size());
+    EXPECT_TRUE(list1.IsEmpty());
+  }
+}
+
+TEST(FreeListTest, Append) {
+  auto blocks1 = CreateEntries();
+  FreeList list1 = CreatePopulatedFreeList(blocks1);
+  const size_t list1_size = list1.Size();
+
+  auto blocks2 = CreateEntries();
+  FreeList list2 = CreatePopulatedFreeList(blocks2);
+  const size_t list2_size = list1.Size();
+
+  list2.Append(std::move(list1));
+  EXPECT_EQ(list1_size + list2_size, list2.Size());
+  EXPECT_FALSE(list2.IsEmpty());
+  EXPECT_EQ(0u, list1.Size());
+  EXPECT_TRUE(list1.IsEmpty());
+}
+
+TEST(FreeListTest, Allocate) {
+  static constexpr size_t kFreeListEntrySizeLog2 =
+      v8::base::bits::WhichPowerOfTwo(kFreeListEntrySize);
+
+  std::vector<Block> blocks;
+  blocks.reserve(kPageSizeLog2);
+  for (size_t i = kFreeListEntrySizeLog2; i < kPageSizeLog2; ++i) {
+    blocks.emplace_back(static_cast<size_t>(1u) << i);
+  }
+
+  FreeList list = CreatePopulatedFreeList(blocks);
+
+  // Try allocate from the biggest block.
+  for (auto it = blocks.rbegin(); it < blocks.rend(); ++it) {
+    const auto result = list.Allocate(it->Size());
+    EXPECT_EQ(it->Address(), result.address);
+    EXPECT_EQ(it->Size(), result.size);
+  }
+
+  EXPECT_EQ(0u, list.Size());
+  EXPECT_TRUE(list.IsEmpty());
+
+  // Check that allocation fails for empty list:
+  const auto empty_block = list.Allocate(8);
+  EXPECT_EQ(nullptr, empty_block.address);
+  EXPECT_EQ(0u, empty_block.size);
+}
+
+}  // namespace internal
+}  // namespace cppgc