[heap] Make FreeListCategory lighter (size-wise)

A recent CL (1762292 and 1765533) changed the FreeList strategy,
switching to one that uses 46 categories rather than the previous 6 we
had. This caused a reduction of V8's heap size by about 1-2% on
average. However, because FreeListCategory is 56 bytes, rather than 4
bytes as one might expect (2 bytes offset, 1 byte for the category
type, and 1 byte padding), the overall memory improvement is actually
lower than that.

For instance, when 256M memory is allocated,
    1000 pages * 46 freelists * 56 bytes = 2.5M overhead
(ie, 1% overhead)

Ideally, FreeListCategory should only by 4 bytes: 2 bytes for the
offset of the top() on the page, 1 byte for the category type, and
1 byte padding.


This CL reduces the size of FreeListCategory by 24 bytes by removing
some fields.

More work should be done to reduce the size even further:

  - Remove the available_ counter (this require maintaining byte count
    at the page level rather than in each FreeListCategory; and
    maintaining that counter is not trivial, but doable).

  - Use a 16 bits offset to store the top() rather than a pointer.

  - Get rid of prev_ and next_: this change is the most
    complicated. It requires storing the space's pages in order, such
    that when one page's freelist is empty, we move on to the next
    page in the space.

  - the type_ field might even be removable, since this information can
    be deduced from the FreeListCategory's position in the page's array
    or FreeListCategory.

Bug: v8:9329
Change-Id: I8fd72cfa31ca12ba0dbf10be3948a72caee15b57
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1773270
Commit-Queue: Darius Mercadier <dmercadier@google.com>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#63448}

src/heap/spaces-inl.h

@@ -172,9 +172,7 @@ bool PagedSpace::Contains(Object o) {
 void PagedSpace::UnlinkFreeListCategories(Page* page) {
   DCHECK_EQ(this, page->owner());
   page->ForAllFreeListCategories([this](FreeListCategory* category) {
-    DCHECK_EQ(free_list(), category->owner());
     free_list()->RemoveCategory(category);
-    category->set_free_list(nullptr);
   });
 }
 
@@ -182,9 +180,8 @@ size_t PagedSpace::RelinkFreeListCategories(Page* page) {
   DCHECK_EQ(this, page->owner());
   size_t added = 0;
   page->ForAllFreeListCategories([this, &added](FreeListCategory* category) {
-    category->set_free_list(free_list());
     added += category->available();
-    category->Relink();
+    category->Relink(free_list());
   });
 
   DCHECK_IMPLIES(!page->IsFlagSet(Page::NEVER_ALLOCATE_ON_PAGE),
@@ -315,17 +312,51 @@ MemoryChunk* OldGenerationMemoryChunkIterator::next() {
   UNREACHABLE();
 }
 
-FreeList* FreeListCategory::owner() { return free_list_; }
-
-bool FreeListCategory::is_linked() {
+bool FreeListCategory::is_linked(FreeList* owner) const {
   return prev_ != nullptr || next_ != nullptr ||
-         free_list_->categories_[type_] == this;
+         owner->categories_[type_] == this;
 }
 
 void FreeListCategory::UpdateCountersAfterAllocation(size_t allocation_size) {
   available_ -= allocation_size;
-  length_--;
-  free_list_->DecreaseAvailableBytes(allocation_size);
+}
+
+Page* FreeList::GetPageForCategoryType(FreeListCategoryType type) {
+  FreeListCategory* category_top = top(type);
+  if (category_top != nullptr) {
+    DCHECK(!category_top->top().is_null());
+    return Page::FromHeapObject(category_top->top());
+  } else {
+    return nullptr;
+  }
+}
+
+Page* FreeListLegacy::GetPageForSize(size_t size_in_bytes) {
+  const int minimum_category =
+      static_cast<int>(SelectFreeListCategoryType(size_in_bytes));
+  Page* page = GetPageForCategoryType(kHuge);
+  if (!page && static_cast<int>(kLarge) >= minimum_category)
+    page = GetPageForCategoryType(kLarge);
+  if (!page && static_cast<int>(kMedium) >= minimum_category)
+    page = GetPageForCategoryType(kMedium);
+  if (!page && static_cast<int>(kSmall) >= minimum_category)
+    page = GetPageForCategoryType(kSmall);
+  if (!page && static_cast<int>(kTiny) >= minimum_category)
+    page = GetPageForCategoryType(kTiny);
+  if (!page && static_cast<int>(kTiniest) >= minimum_category)
+    page = GetPageForCategoryType(kTiniest);
+  return page;
+}
+
+Page* FreeListFastAlloc::GetPageForSize(size_t size_in_bytes) {
+  const int minimum_category =
+      static_cast<int>(SelectFreeListCategoryType(size_in_bytes));
+  Page* page = GetPageForCategoryType(kHuge);
+  if (!page && static_cast<int>(kLarge) >= minimum_category)
+    page = GetPageForCategoryType(kLarge);
+  if (!page && static_cast<int>(kMedium) >= minimum_category)
+    page = GetPageForCategoryType(kMedium);
+  return page;
 }
 
 AllocationResult LocalAllocationBuffer::AllocateRawAligned(

src/heap/spaces.cc

@@ -822,8 +822,7 @@ void Page::AllocateFreeListCategories() {
   categories_ = new FreeListCategory*[free_list()->number_of_categories()]();
   for (int i = kFirstCategory; i <= free_list()->last_category(); i++) {
     DCHECK_NULL(categories_[i]);
-    categories_[i] = new FreeListCategory(
-        reinterpret_cast<PagedSpace*>(owner())->free_list(), this);
+    categories_[i] = new FreeListCategory();
   }
 }
 
@@ -1664,8 +1663,9 @@ void PagedSpace::RefillFreeList() {
       // We regularly sweep NEVER_ALLOCATE_ON_PAGE pages. We drop the freelist
       // entries here to make them unavailable for allocations.
       if (p->IsFlagSet(Page::NEVER_ALLOCATE_ON_PAGE)) {
-        p->ForAllFreeListCategories(
-            [](FreeListCategory* category) { category->Reset(); });
+        p->ForAllFreeListCategories([this](FreeListCategory* category) {
+          category->Reset(free_list());
+        });
       }
       // Only during compaction pages can actually change ownership. This is
       // safe because there exists no other competing action on the page links
@@ -1982,7 +1982,6 @@ void PagedSpace::ReleasePage(Page* page) {
   DCHECK_EQ(page->owner(), this);
 
   free_list_->EvictFreeListItems(page);
-  DCHECK(!free_list_->ContainsPageFreeListItems(page));
 
   if (Page::FromAllocationAreaAddress(allocation_info_.top()) == page) {
     DCHECK(!top_on_previous_step_);
@@ -2967,23 +2966,21 @@ size_t NewSpace::CommittedPhysicalMemory() {
 // -----------------------------------------------------------------------------
 // Free lists for old object spaces implementation
 
-
-void FreeListCategory::Reset() {
-  if (is_linked() && !top().is_null()) {
-    owner()->DecreaseAvailableBytes(available_);
+void FreeListCategory::Reset(FreeList* owner) {
+  if (is_linked(owner) && !top().is_null()) {
+    owner->DecreaseAvailableBytes(available_);
   }
   set_top(FreeSpace());
   set_prev(nullptr);
   set_next(nullptr);
   available_ = 0;
-  length_ = 0;
 }
 
 FreeSpace FreeListCategory::PickNodeFromList(size_t minimum_size,
                                              size_t* node_size) {
-  DCHECK(page()->CanAllocate());
   FreeSpace node = top();
   DCHECK(!node.is_null());
+  DCHECK(Page::FromHeapObject(node)->CanAllocate());
   if (static_cast<size_t>(node.Size()) < minimum_size) {
     *node_size = 0;
     return FreeSpace();
@@ -2996,10 +2993,10 @@ FreeSpace FreeListCategory::PickNodeFromList(size_t minimum_size,
 
 FreeSpace FreeListCategory::SearchForNodeInList(size_t minimum_size,
                                                 size_t* node_size) {
-  DCHECK(page()->CanAllocate());
   FreeSpace prev_non_evac_node;
   for (FreeSpace cur_node = top(); !cur_node.is_null();
        cur_node = cur_node.next()) {
+    DCHECK(Page::FromHeapObject(cur_node)->CanAllocate());
     size_t size = cur_node.size();
     if (size >= minimum_size) {
       DCHECK_GE(available_, size);
@@ -3023,23 +3020,21 @@ FreeSpace FreeListCategory::SearchForNodeInList(size_t minimum_size,
   return FreeSpace();
 }
 
-void FreeListCategory::Free(Address start, size_t size_in_bytes,
-                            FreeMode mode) {
+void FreeListCategory::Free(Address start, size_t size_in_bytes, FreeMode mode,
+                            FreeList* owner) {
   FreeSpace free_space = FreeSpace::cast(HeapObject::FromAddress(start));
   free_space.set_next(top());
   set_top(free_space);
   available_ += size_in_bytes;
-  length_++;
   if (mode == kLinkCategory) {
-    if (is_linked()) {
-      owner()->IncreaseAvailableBytes(size_in_bytes);
+    if (is_linked(owner)) {
+      owner->IncreaseAvailableBytes(size_in_bytes);
     } else {
-      owner()->AddCategory(this);
+      owner->AddCategory(this);
     }
   }
 }
 
-
 void FreeListCategory::RepairFreeList(Heap* heap) {
   Map free_space_map = ReadOnlyRoots(heap).free_space_map();
   FreeSpace n = top();
@@ -3054,9 +3049,9 @@ void FreeListCategory::RepairFreeList(Heap* heap) {
   }
 }
 
-void FreeListCategory::Relink() {
-  DCHECK(!is_linked());
-  owner()->AddCategory(this);
+void FreeListCategory::Relink(FreeList* owner) {
+  DCHECK(!is_linked(owner));
+  owner->AddCategory(this);
 }
 
 // ------------------------------------------------
@@ -3087,6 +3082,7 @@ FreeSpace FreeList::TryFindNodeIn(FreeListCategoryType type,
   if (category == nullptr) return FreeSpace();
   FreeSpace node = category->PickNodeFromList(minimum_size, node_size);
   if (!node.is_null()) {
+    DecreaseAvailableBytes(*node_size);
     DCHECK(IsVeryLong() || Available() == SumFreeLists());
   }
   if (category->is_empty()) {
@@ -3104,6 +3100,7 @@ FreeSpace FreeList::SearchForNodeInList(FreeListCategoryType type,
     FreeListCategory* current = it.Next();
     node = current->SearchForNodeInList(minimum_size, node_size);
     if (!node.is_null()) {
+      DecreaseAvailableBytes(*node_size);
       DCHECK(IsVeryLong() || Available() == SumFreeLists());
       if (current->is_empty()) {
         RemoveCategory(current);
@@ -3128,7 +3125,7 @@ size_t FreeList::Free(Address start, size_t size_in_bytes, FreeMode mode) {
   // Insert other blocks at the head of a free list of the appropriate
   // magnitude.
   FreeListCategoryType type = SelectFreeListCategoryType(size_in_bytes);
-  page->free_list_category(type)->Free(start, size_in_bytes, mode);
+  page->free_list_category(type)->Free(start, size_in_bytes, mode, this);
   DCHECK_EQ(page->AvailableInFreeList(),
             page->AvailableInFreeListFromAllocatedBytes());
   return 0;
@@ -3348,7 +3345,7 @@ size_t FreeListManyCached::Free(Address start, size_t size_in_bytes,
   // Insert other blocks at the head of a free list of the appropriate
   // magnitude.
   FreeListCategoryType type = SelectFreeListCategoryType(size_in_bytes);
-  page->free_list_category(type)->Free(start, size_in_bytes, mode);
+  page->free_list_category(type)->Free(start, size_in_bytes, mode, this);
 
   // Updating cache
   if (mode == kLinkCategory) {
@@ -3529,7 +3526,7 @@ FreeSpace FreeListMap::Allocate(size_t size_in_bytes, size_t* node_size,
 
 void FreeList::Reset() {
   ForAllFreeListCategories(
-      [](FreeListCategory* category) { category->Reset(); });
+      [this](FreeListCategory* category) { category->Reset(this); });
   for (int i = kFirstCategory; i < number_of_categories_; i++) {
     categories_[i] = nullptr;
   }
@@ -3540,25 +3537,13 @@ void FreeList::Reset() {
 size_t FreeList::EvictFreeListItems(Page* page) {
   size_t sum = 0;
   page->ForAllFreeListCategories([this, &sum](FreeListCategory* category) {
-    DCHECK_EQ(this, category->owner());
     sum += category->available();
     RemoveCategory(category);
-    category->Reset();
+    category->Reset(this);
   });
   return sum;
 }
 
-bool FreeList::ContainsPageFreeListItems(Page* page) {
-  bool contained = false;
-  page->ForAllFreeListCategories(
-      [this, &contained](FreeListCategory* category) {
-        if (category->owner() == this && category->is_linked()) {
-          contained = true;
-        }
-      });
-  return contained;
-}
-
 void FreeList::RepairLists(Heap* heap) {
   ForAllFreeListCategories(
       [heap](FreeListCategory* category) { category->RepairFreeList(heap); });
@@ -3588,7 +3573,7 @@ void FreeList::RemoveCategory(FreeListCategory* category) {
   DCHECK_LT(type, number_of_categories_);
   FreeListCategory* top = categories_[type];
 
-  if (category->is_linked()) {
+  if (category->is_linked(this)) {
     DecreaseAvailableBytes(category->available());
   }
 
@@ -3633,13 +3618,25 @@ size_t FreeListCategory::SumFreeList() {
   while (!cur.is_null()) {
     // We can't use "cur->map()" here because both cur's map and the
     // root can be null during bootstrapping.
-    DCHECK(cur.map_slot().contains_value(
-        page()->heap()->isolate()->root(RootIndex::kFreeSpaceMap).ptr()));
+    DCHECK(cur.map_slot().contains_value(Page::FromHeapObject(cur)
+                                             ->heap()
+                                             ->isolate()
+                                             ->root(RootIndex::kFreeSpaceMap)
+                                             .ptr()));
     sum += cur.relaxed_read_size();
     cur = cur.next();
   }
   return sum;
 }
+int FreeListCategory::FreeListLength() {
+  int length = 0;
+  FreeSpace cur = top();
+  while (!cur.is_null()) {
+    length++;
+    cur = cur.next();
+  }
+  return length;
+}
 
 #ifdef DEBUG
 bool FreeList::IsVeryLong() {

src/heap/spaces.h

@@ -121,7 +121,7 @@ class Space;
 #define DCHECK_CODEOBJECT_SIZE(size, code_space) \
   DCHECK((0 < size) && (size <= code_space->AreaSize()))
 
-using FreeListCategoryType = int;
+using FreeListCategoryType = int32_t;
 
 static const FreeListCategoryType kFirstCategory = 0;
 static const FreeListCategoryType kInvalidCategory = -1;
@@ -139,32 +139,23 @@ enum RememberedSetType {
 // A free list category maintains a linked list of free memory blocks.
 class FreeListCategory {
  public:
-  FreeListCategory(FreeList* free_list, Page* page)
-      : free_list_(free_list),
-        page_(page),
-        type_(kInvalidCategory),
-        available_(0),
-        length_(0),
-        prev_(nullptr),
-        next_(nullptr) {}
-
   void Initialize(FreeListCategoryType type) {
     type_ = type;
     available_ = 0;
-    length_ = 0;
     prev_ = nullptr;
     next_ = nullptr;
   }
 
-  void Reset();
+  void Reset(FreeList* owner);
 
   void RepairFreeList(Heap* heap);
 
   // Relinks the category into the currently owning free list. Requires that the
   // category is currently unlinked.
-  void Relink();
+  void Relink(FreeList* owner);
 
-  void Free(Address address, size_t size_in_bytes, FreeMode mode);
+  void Free(Address address, size_t size_in_bytes, FreeMode mode,
+            FreeList* owner);
 
   // Performs a single try to pick a node of at least |minimum_size| from the
   // category. Stores the actual size in |node_size|. Returns nullptr if no
@@ -175,16 +166,12 @@ class FreeListCategory {
   // actual size in |node_size|. Returns nullptr if no node is found.
   FreeSpace SearchForNodeInList(size_t minimum_size, size_t* node_size);
 
-  inline FreeList* owner();
-  inline Page* page() const { return page_; }
-  inline bool is_linked();
+  inline bool is_linked(FreeList* owner) const;
   bool is_empty() { return top().is_null(); }
-  size_t available() const { return available_; }
-
-  void set_free_list(FreeList* free_list) { free_list_ = free_list; }
+  uint32_t available() const { return available_; }
 
   size_t SumFreeList();
-  int FreeListLength() { return length_; }
+  int FreeListLength();
 
  private:
   // For debug builds we accurately compute free lists lengths up until
@@ -202,34 +189,23 @@ class FreeListCategory {
   FreeListCategory* next() { return next_; }
   void set_next(FreeListCategory* next) { next_ = next; }
 
-  // This FreeListCategory is owned by the given free_list_.
-  FreeList* free_list_;
-
-  // This FreeListCategory holds free list entries of the given page_.
-  Page* const page_;
-
   // |type_|: The type of this free list category.
-  FreeListCategoryType type_;
+  FreeListCategoryType type_ = kInvalidCategory;
 
   // |available_|: Total available bytes in all blocks of this free list
   // category.
-  size_t available_;
-
-  // |length_|: Total blocks in this free list category.
-  int length_;
+  uint32_t available_ = 0;
 
   // |top_|: Points to the top FreeSpace in the free list category.
   FreeSpace top_;
 
-  FreeListCategory* prev_;
-  FreeListCategory* next_;
+  FreeListCategory* prev_ = nullptr;
+  FreeListCategory* next_ = nullptr;
 
   friend class FreeList;
   friend class FreeListManyCached;
   friend class PagedSpace;
   friend class MapSpace;
-
-  DISALLOW_IMPLICIT_CONSTRUCTORS(FreeListCategory);
 };
 
 // A free list maintains free blocks of memory. The free list is organized in
@@ -293,7 +269,6 @@ class FreeList {
   void RepairLists(Heap* heap);
 
   V8_EXPORT_PRIVATE size_t EvictFreeListItems(Page* page);
-  bool ContainsPageFreeListItems(Page* page);
 
   int number_of_categories() { return number_of_categories_; }
   FreeListCategoryType last_category() { return last_category_; }
@@ -364,9 +339,7 @@ class FreeList {
     return categories_[type];
   }
 
-  Page* GetPageForCategoryType(FreeListCategoryType type) {
-    return top(type) ? top(type)->page() : nullptr;
-  }
+  inline Page* GetPageForCategoryType(FreeListCategoryType type);
 
   int number_of_categories_ = 0;
   FreeListCategoryType last_category_ = 0;
@@ -1829,22 +1802,7 @@ class V8_EXPORT_PRIVATE FreeListLegacy : public FreeList {
     return maximum_freed;
   }
 
-  Page* GetPageForSize(size_t size_in_bytes) override {
-    const int minimum_category =
-        static_cast<int>(SelectFreeListCategoryType(size_in_bytes));
-    Page* page = GetPageForCategoryType(kHuge);
-    if (!page && static_cast<int>(kLarge) >= minimum_category)
-      page = GetPageForCategoryType(kLarge);
-    if (!page && static_cast<int>(kMedium) >= minimum_category)
-      page = GetPageForCategoryType(kMedium);
-    if (!page && static_cast<int>(kSmall) >= minimum_category)
-      page = GetPageForCategoryType(kSmall);
-    if (!page && static_cast<int>(kTiny) >= minimum_category)
-      page = GetPageForCategoryType(kTiny);
-    if (!page && static_cast<int>(kTiniest) >= minimum_category)
-      page = GetPageForCategoryType(kTiniest);
-    return page;
-  }
+  inline Page* GetPageForSize(size_t size_in_bytes) override;
 
   FreeListLegacy();
   ~FreeListLegacy();
@@ -1928,16 +1886,7 @@ class V8_EXPORT_PRIVATE FreeListFastAlloc : public FreeList {
     return kHugeAllocationMax;
   }
 
-  Page* GetPageForSize(size_t size_in_bytes) override {
-    const int minimum_category =
-        static_cast<int>(SelectFreeListCategoryType(size_in_bytes));
-    Page* page = GetPageForCategoryType(kHuge);
-    if (!page && static_cast<int>(kLarge) >= minimum_category)
-      page = GetPageForCategoryType(kLarge);
-    if (!page && static_cast<int>(kMedium) >= minimum_category)
-      page = GetPageForCategoryType(kMedium);
-    return page;
-  }
+  inline Page* GetPageForSize(size_t size_in_bytes) override;
 
   FreeListFastAlloc();
   ~FreeListFastAlloc();
@@ -1971,10 +1920,6 @@ class V8_EXPORT_PRIVATE FreeListFastAlloc : public FreeList {
     }
     return kHuge;
   }
-
-  Page* GetPageForCategoryType(FreeListCategoryType type) {
-    return top(type) ? top(type)->page() : nullptr;
-  }
 };
 
 // Use 47 Freelists: on per size between 24 and 256, and then a few ones for

src/heap/sweeper.cc

@@ -509,11 +509,14 @@ void Sweeper::AddPage(AllocationSpace space, Page* page,
 }
 
 void Sweeper::PrepareToBeSweptPage(AllocationSpace space, Page* page) {
+#ifdef DEBUG
   DCHECK_GE(page->area_size(),
             static_cast<size_t>(marking_state_->live_bytes(page)));
   DCHECK_EQ(Page::kSweepingDone, page->concurrent_sweeping_state());
-  page->ForAllFreeListCategories(
-      [](FreeListCategory* category) { DCHECK(!category->is_linked()); });
+  page->ForAllFreeListCategories([page](FreeListCategory* category) {
+    DCHECK(!category->is_linked(page->owner()->free_list()));
+  });
+#endif  // DEBUG
   page->set_concurrent_sweeping_state(Page::kSweepingPending);
   heap_->paged_space(space)->IncreaseAllocatedBytes(
       marking_state_->live_bytes(page), page);