[heap] Added per-page array buffer accouting (external memory).

Bug: chromium:845409
Change-Id: Ibc568cdc501edf5d84d9c6379aff58be069369af
Reviewed-on: https://chromium-review.googlesource.com/1114602
Commit-Queue: Rodrigo Bruno <rfbpb@google.com>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#54028}

src/heap/array-buffer-tracker-inl.h

@@ -75,8 +75,8 @@ void LocalArrayBufferTracker::Free(Callback should_free) {
     }
   }
   if (freed_memory > 0) {
-    // Update the Space with any freed backing-store bytes.
-    space()->DecrementExternalBackingStoreBytes(freed_memory);
+    page_->DecrementExternalBackingStoreBytes(
+        ExternalBackingStoreType::kArrayBuffer, freed_memory);
 
     // TODO(wez): Remove backing-store from external memory accounting.
     page_->heap()->update_external_memory_concurrently_freed(
@@ -98,8 +98,8 @@ void ArrayBufferTracker::FreeDead(Page* page, MarkingState* marking_state) {
 }
 
 void LocalArrayBufferTracker::Add(JSArrayBuffer* buffer, size_t length) {
-  // Track the backing-store usage against the owning Space.
-  space()->IncrementExternalBackingStoreBytes(length);
+  page_->IncrementExternalBackingStoreBytes(
+      ExternalBackingStoreType::kArrayBuffer, length);
 
   auto ret = array_buffers_.insert({buffer, {buffer->backing_store(), length}});
   USE(ret);
@@ -109,8 +109,8 @@ void LocalArrayBufferTracker::Add(JSArrayBuffer* buffer, size_t length) {
 }
 
 void LocalArrayBufferTracker::Remove(JSArrayBuffer* buffer, size_t length) {
-  // Remove the backing-store accounting from the owning Space.
-  space()->DecrementExternalBackingStoreBytes(length);
+  page_->DecrementExternalBackingStoreBytes(
+      ExternalBackingStoreType::kArrayBuffer, length);
 
   TrackingData::iterator it = array_buffers_.find(buffer);
   // Check that we indeed find a key to remove.

src/heap/array-buffer-tracker.cc

@@ -30,6 +30,7 @@ void LocalArrayBufferTracker::Process(Callback callback) {
   for (TrackingData::iterator it = array_buffers_.begin();
        it != array_buffers_.end(); ++it) {
     old_buffer = reinterpret_cast<JSArrayBuffer*>(it->first);
+    Page* old_page = Page::FromAddress(old_buffer->address());
     const CallbackResult result = callback(old_buffer, &new_buffer);
     if (result == kKeepEntry) {
       kept_array_buffers.insert(*it);
@@ -48,6 +49,9 @@ void LocalArrayBufferTracker::Process(Callback callback) {
         tracker->Add(new_buffer, size);
       }
       moved_memory += it->second.length;
+      old_page->DecrementExternalBackingStoreBytes(
+          ExternalBackingStoreType::kArrayBuffer, it->second.length);
+
     } else if (result == kRemoveEntry) {
       freed_memory += it->second.length;
       // We pass backing_store() and stored length to the collector for freeing
@@ -56,14 +60,14 @@ void LocalArrayBufferTracker::Process(Callback callback) {
       backing_stores_to_free.emplace_back(
           it->second.backing_store, it->second.length, it->second.backing_store,
           old_buffer->allocation_mode(), old_buffer->is_wasm_memory());
+      old_page->DecrementExternalBackingStoreBytes(
+          ExternalBackingStoreType::kArrayBuffer, it->second.length);
+
     } else {
       UNREACHABLE();
     }
   }
   if (moved_memory || freed_memory) {
-    // Update the Space with any moved or freed backing-store bytes.
-    space()->DecrementExternalBackingStoreBytes(freed_memory + moved_memory);
-
     // TODO(wez): Remove backing-store from external memory accounting.
     page_->heap()->update_external_memory_concurrently_freed(
         static_cast<intptr_t>(freed_memory));

src/heap/spaces.cc

@@ -631,7 +631,6 @@ MemoryChunk* MemoryChunk::Initialize(Heap* heap, Address base, size_t size,
   chunk->young_generation_bitmap_ = nullptr;
   chunk->local_tracker_ = nullptr;
 
-  chunk->external_backing_store_bytes_[ExternalBackingStoreType::kOther] = 0;
   chunk->external_backing_store_bytes_[ExternalBackingStoreType::kArrayBuffer] =
       0;
   chunk->external_backing_store_bytes_
@@ -1400,6 +1399,19 @@ void MemoryChunk::ReleaseYoungGenerationBitmap() {
   young_generation_bitmap_ = nullptr;
 }
 
+void MemoryChunk::IncrementExternalBackingStoreBytes(
+    ExternalBackingStoreType type, size_t amount) {
+  external_backing_store_bytes_[type] += amount;
+  owner()->IncrementExternalBackingStoreBytes(type, amount);
+}
+
+void MemoryChunk::DecrementExternalBackingStoreBytes(
+    ExternalBackingStoreType type, size_t amount) {
+  DCHECK_GE(external_backing_store_bytes_[type], amount);
+  external_backing_store_bytes_[type] -= amount;
+  owner()->DecrementExternalBackingStoreBytes(type, amount);
+}
+
 // -----------------------------------------------------------------------------
 // PagedSpace implementation
 
@@ -1500,9 +1512,6 @@ void PagedSpace::RefillFreeList() {
 void PagedSpace::MergeCompactionSpace(CompactionSpace* other) {
   base::LockGuard<base::Mutex> guard(mutex());
 
-  IncrementExternalBackingStoreBytes(other->ExternalBackingStoreBytes());
-  other->DecrementExternalBackingStoreBytes(other->ExternalBackingStoreBytes());
-
   DCHECK(identity() == other->identity());
   // Unmerged fields:
   //   area_size_
@@ -1592,6 +1601,10 @@ size_t PagedSpace::AddPage(Page* page) {
   AccountCommitted(page->size());
   IncreaseCapacity(page->area_size());
   IncreaseAllocatedBytes(page->allocated_bytes(), page);
+  for (size_t i = 0; i < ExternalBackingStoreType::kNumTypes; i++) {
+    ExternalBackingStoreType t = static_cast<ExternalBackingStoreType>(i);
+    IncrementExternalBackingStoreBytes(t, page->ExternalBackingStoreBytes(t));
+  }
   return RelinkFreeListCategories(page);
 }
 
@@ -1602,6 +1615,10 @@ void PagedSpace::RemovePage(Page* page) {
   DecreaseAllocatedBytes(page->allocated_bytes(), page);
   DecreaseCapacity(page->area_size());
   AccountUncommitted(page->size());
+  for (size_t i = 0; i < ExternalBackingStoreType::kNumTypes; i++) {
+    ExternalBackingStoreType t = static_cast<ExternalBackingStoreType>(i);
+    DecrementExternalBackingStoreBytes(t, page->ExternalBackingStoreBytes(t));
+  }
 }
 
 size_t PagedSpace::ShrinkPageToHighWaterMark(Page* page) {
@@ -2554,6 +2571,10 @@ void SemiSpace::RemovePage(Page* page) {
     }
   }
   memory_chunk_list_.Remove(page);
+  for (size_t i = 0; i < ExternalBackingStoreType::kNumTypes; i++) {
+    ExternalBackingStoreType t = static_cast<ExternalBackingStoreType>(i);
+    DecrementExternalBackingStoreBytes(t, page->ExternalBackingStoreBytes(t));
+  }
 }
 
 void SemiSpace::PrependPage(Page* page) {
@@ -2562,6 +2583,10 @@ void SemiSpace::PrependPage(Page* page) {
   page->set_owner(this);
   memory_chunk_list_.PushFront(page);
   pages_used_++;
+  for (size_t i = 0; i < ExternalBackingStoreType::kNumTypes; i++) {
+    ExternalBackingStoreType t = static_cast<ExternalBackingStoreType>(i);
+    IncrementExternalBackingStoreBytes(t, page->ExternalBackingStoreBytes(t));
+  }
 }
 
 void SemiSpace::Swap(SemiSpace* from, SemiSpace* to) {

src/heap/spaces.h

@@ -143,7 +143,6 @@ enum FreeMode { kLinkCategory, kDoNotLinkCategory };
 enum class SpaceAccountingMode { kSpaceAccounted, kSpaceUnaccounted };
 
 enum ExternalBackingStoreType {
-  kOther,
   kArrayBuffer,
   kExternalString,
   kNumTypes
@@ -538,13 +537,12 @@ class MemoryChunk {
     }
   }
 
-  void IncrementExternalBackingStoreBytes(size_t amount,
-                                          ExternalBackingStoreType type) {
-    external_backing_store_bytes_[type] += amount;
-  }
-  void DecrementExternalBackingStoreBytes(size_t amount,
-                                          ExternalBackingStoreType type) {
-    external_backing_store_bytes_[type] -= amount;
+  void IncrementExternalBackingStoreBytes(ExternalBackingStoreType type,
+                                          size_t amount);
+  void DecrementExternalBackingStoreBytes(ExternalBackingStoreType type,
+                                          size_t amount);
+  size_t ExternalBackingStoreBytes(ExternalBackingStoreType type) {
+    return external_backing_store_bytes_[type];
   }
 
   inline uint32_t AddressToMarkbitIndex(Address addr) const {
@@ -913,7 +911,6 @@ class Space : public Malloced {
         max_committed_(0) {
     external_backing_store_bytes_ =
         new std::atomic<size_t>[ExternalBackingStoreType::kNumTypes];
-    external_backing_store_bytes_[ExternalBackingStoreType::kOther] = 0;
     external_backing_store_bytes_[ExternalBackingStoreType::kArrayBuffer] = 0;
     external_backing_store_bytes_[ExternalBackingStoreType::kExternalString] =
         0;
@@ -960,7 +957,7 @@ class Space : public Malloced {
 
   // Returns amount of off-heap memory in-use by objects in this Space.
   virtual size_t ExternalBackingStoreBytes(
-      ExternalBackingStoreType type = ExternalBackingStoreType::kOther) const {
+      ExternalBackingStoreType type) const {
     return external_backing_store_bytes_[type];
   }
 
@@ -993,14 +990,13 @@ class Space : public Malloced {
     committed_ -= bytes;
   }
 
-  void IncrementExternalBackingStoreBytes(
-      size_t amount,
-      ExternalBackingStoreType type = ExternalBackingStoreType::kOther) {
+  void IncrementExternalBackingStoreBytes(ExternalBackingStoreType type,
+                                          size_t amount) {
     external_backing_store_bytes_[type] += amount;
   }
-  void DecrementExternalBackingStoreBytes(
-      size_t amount,
-      ExternalBackingStoreType type = ExternalBackingStoreType::kOther) {
+  void DecrementExternalBackingStoreBytes(ExternalBackingStoreType type,
+                                          size_t amount) {
+    DCHECK_GE(external_backing_store_bytes_[type], amount);
     external_backing_store_bytes_[type] -= amount;
   }
 
@@ -2626,8 +2622,7 @@ class NewSpace : public SpaceWithLinearArea {
   }
 
   size_t ExternalBackingStoreBytes(
-      ExternalBackingStoreType type =
-          ExternalBackingStoreType::kOther) const override {
+      ExternalBackingStoreType type) const override {
     DCHECK_EQ(0, from_space_.ExternalBackingStoreBytes(type));
     return to_space_.ExternalBackingStoreBytes(type);
   }

test/cctest/heap/test-array-buffer-tracker.cc

@@ -343,16 +343,17 @@ TEST(ArrayBuffer_ExternalBackingStoreSizeIncreases) {
   LocalContext env;
   v8::Isolate* isolate = env->GetIsolate();
   Heap* heap = reinterpret_cast<Isolate*>(isolate)->heap();
+  ExternalBackingStoreType type = ExternalBackingStoreType::kArrayBuffer;
 
   const size_t backing_store_before =
-      heap->new_space()->ExternalBackingStoreBytes();
+      heap->new_space()->ExternalBackingStoreBytes(type);
   {
     const size_t kArraybufferSize = 117;
     v8::HandleScope handle_scope(isolate);
     Local<v8::ArrayBuffer> ab = v8::ArrayBuffer::New(isolate, kArraybufferSize);
     USE(ab);
     const size_t backing_store_after =
-        heap->new_space()->ExternalBackingStoreBytes();
+        heap->new_space()->ExternalBackingStoreBytes(type);
     CHECK_EQ(kArraybufferSize, backing_store_after - backing_store_before);
   }
 }
@@ -362,9 +363,10 @@ TEST(ArrayBuffer_ExternalBackingStoreSizeDecreases) {
   LocalContext env;
   v8::Isolate* isolate = env->GetIsolate();
   Heap* heap = reinterpret_cast<Isolate*>(isolate)->heap();
+  ExternalBackingStoreType type = ExternalBackingStoreType::kArrayBuffer;
 
   const size_t backing_store_before =
-      heap->new_space()->ExternalBackingStoreBytes();
+      heap->new_space()->ExternalBackingStoreBytes(type);
   {
     const size_t kArraybufferSize = 117;
     v8::HandleScope handle_scope(isolate);
@@ -373,7 +375,7 @@ TEST(ArrayBuffer_ExternalBackingStoreSizeDecreases) {
   }
   heap::GcAndSweep(heap, OLD_SPACE);
   const size_t backing_store_after =
-      heap->new_space()->ExternalBackingStoreBytes();
+      heap->new_space()->ExternalBackingStoreBytes(type);
   CHECK_EQ(0, backing_store_after - backing_store_before);
 }
 
@@ -386,9 +388,10 @@ TEST(ArrayBuffer_ExternalBackingStoreSizeIncreasesMarkCompact) {
   v8::Isolate* isolate = env->GetIsolate();
   Heap* heap = reinterpret_cast<Isolate*>(isolate)->heap();
   heap::AbandonCurrentlyFreeMemory(heap->old_space());
+  ExternalBackingStoreType type = ExternalBackingStoreType::kArrayBuffer;
 
   const size_t backing_store_before =
-      heap->old_space()->ExternalBackingStoreBytes();
+      heap->old_space()->ExternalBackingStoreBytes(type);
 
   const size_t kArraybufferSize = 117;
   {
@@ -407,13 +410,13 @@ TEST(ArrayBuffer_ExternalBackingStoreSizeIncreasesMarkCompact) {
     CcTest::CollectAllGarbage();
 
     const size_t backing_store_after =
-        heap->old_space()->ExternalBackingStoreBytes();
+        heap->old_space()->ExternalBackingStoreBytes(type);
     CHECK_EQ(kArraybufferSize, backing_store_after - backing_store_before);
   }
 
   heap::GcAndSweep(heap, OLD_SPACE);
   const size_t backing_store_after =
-      heap->old_space()->ExternalBackingStoreBytes();
+      heap->old_space()->ExternalBackingStoreBytes(type);
   CHECK_EQ(0, backing_store_after - backing_store_before);
 }