[heap] Move external memory counters back into Heap

Move external memory counters out of IsolateData back into Heap.
The class ExternalMemoryAccounting now stores all counters and is
responsible for updates. This change will allow turning counters into
atomic variables.

Bug: v8:10315
Change-Id: I2abeda298d3cfcc630fd04ca78a3d6d703e3b419
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2346647Reviewed-by: Jakob Gruber <jgruber@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Commit-Queue: Dominik Inführ <dinfuehr@chromium.org>
Cr-Commit-Position: refs/heads/master@{#69356}

include/v8-internal.h

@@ -178,14 +178,8 @@ class Internals {
 
   // IsolateData layout guarantees.
   static const int kIsolateEmbedderDataOffset = 0;
-  static const int kExternalMemoryOffset =
-      kNumIsolateDataSlots * kApiSystemPointerSize;
-  static const int kExternalMemoryLimitOffset =
-      kExternalMemoryOffset + kApiInt64Size;
-  static const int kExternalMemoryLowSinceMarkCompactOffset =
-      kExternalMemoryLimitOffset + kApiInt64Size;
   static const int kIsolateFastCCallCallerFpOffset =
-      kExternalMemoryLowSinceMarkCompactOffset + kApiInt64Size;
+      kNumIsolateDataSlots * kApiSystemPointerSize;
   static const int kIsolateFastCCallCallerPcOffset =
       kIsolateFastCCallCallerFpOffset + kApiSystemPointerSize;
   static const int kIsolateStackGuardOffset =

include/v8.h

@@ -8777,8 +8777,7 @@ class V8_EXPORT Isolate {
    *   kept alive by JavaScript objects.
    * \returns the adjusted value.
    */
-  V8_INLINE int64_t
-      AdjustAmountOfExternalAllocatedMemory(int64_t change_in_bytes);
+  int64_t AdjustAmountOfExternalAllocatedMemory(int64_t change_in_bytes);
 
   /**
    * Returns the number of phantom handles without callbacks that were reset
@@ -12007,37 +12006,6 @@ MaybeLocal<T> Isolate::GetDataFromSnapshotOnce(size_t index) {
   return Local<T>(data);
 }
 
-int64_t Isolate::AdjustAmountOfExternalAllocatedMemory(
-    int64_t change_in_bytes) {
-  typedef internal::Internals I;
-  int64_t* external_memory = reinterpret_cast<int64_t*>(
-      reinterpret_cast<uint8_t*>(this) + I::kExternalMemoryOffset);
-  int64_t* external_memory_limit = reinterpret_cast<int64_t*>(
-      reinterpret_cast<uint8_t*>(this) + I::kExternalMemoryLimitOffset);
-  int64_t* external_memory_low_since_mc =
-      reinterpret_cast<int64_t*>(reinterpret_cast<uint8_t*>(this) +
-                                 I::kExternalMemoryLowSinceMarkCompactOffset);
-
-  // Embedders are weird: we see both over- and underflows here. Perform the
-  // addition with unsigned types to avoid undefined behavior.
-  const int64_t amount =
-      static_cast<int64_t>(static_cast<uint64_t>(change_in_bytes) +
-                           static_cast<uint64_t>(*external_memory));
-  *external_memory = amount;
-
-  if (amount < *external_memory_low_since_mc) {
-    *external_memory_low_since_mc = amount;
-    *external_memory_limit = amount + I::kExternalAllocationSoftLimit;
-  }
-
-  if (change_in_bytes <= 0) return *external_memory;
-
-  if (amount > *external_memory_limit) {
-    ReportExternalAllocationLimitReached();
-  }
-  return *external_memory;
-}
-
 Local<Value> Context::GetEmbedderData(int index) {
 #ifndef V8_ENABLE_CHECKS
   typedef internal::Address A;

src/api/api.cc

@@ -8665,6 +8665,19 @@ size_t Isolate::NumberOfPhantomHandleResetsSinceLastCall() {
   return isolate->global_handles()->GetAndResetGlobalHandleResetCount();
 }
 
+int64_t Isolate::AdjustAmountOfExternalAllocatedMemory(
+    int64_t change_in_bytes) {
+  i::Isolate* i_isolate = reinterpret_cast<i::Isolate*>(this);
+  int64_t amount = i_isolate->heap()->update_external_memory(change_in_bytes);
+
+  if (change_in_bytes <= 0) return amount;
+
+  if (amount > i_isolate->heap()->external_memory_limit()) {
+    ReportExternalAllocationLimitReached();
+  }
+  return amount;
+}
+
 void Isolate::SetEventLogger(LogEventCallback that) {
   // Do not overwrite the event logger if we want to log explicitly.
   if (i::FLAG_log_internal_timer_events) return;

src/execution/isolate-data.h

@@ -123,9 +123,6 @@ class IsolateData final {
   // beginning of IsolateData.
 #define FIELDS(V)                                                             \
   V(kEmbedderDataOffset, Internals::kNumIsolateDataSlots* kSystemPointerSize) \
-  V(kExternalMemoryOffset, kInt64Size)                                        \
-  V(kExternalMemoryLlimitOffset, kInt64Size)                                  \
-  V(kExternalMemoryLowSinceMarkCompactOffset, kInt64Size)                     \
   V(kFastCCallCallerFPOffset, kSystemPointerSize)                             \
   V(kFastCCallCallerPCOffset, kSystemPointerSize)                             \
   V(kStackGuardOffset, StackGuard::kSizeInBytes)                              \
@@ -150,17 +147,6 @@ class IsolateData final {
   // runtime checks.
   void* embedder_data_[Internals::kNumIsolateDataSlots] = {};
 
-  // TODO(ishell): Move these external memory counters back to Heap once the
-  // Node JS bot issue is solved.
-  // The amount of external memory registered through the API.
-  int64_t external_memory_ = 0;
-
-  // The limit when to trigger memory pressure from the API.
-  int64_t external_memory_limit_ = kExternalAllocationSoftLimit;
-
-  // Caches the amount of external memory registered at the last MC.
-  int64_t external_memory_low_since_mark_compact_ = 0;
-
   // Stores the state of the caller for TurboAssembler::CallCFunction so that
   // the sampling CPU profiler can iterate the stack during such calls. These
   // are stored on IsolateData so that they can be stored to with only one move
@@ -224,13 +210,6 @@ void IsolateData::AssertPredictableLayout() {
   STATIC_ASSERT(offsetof(IsolateData, thread_local_top_) ==
                 kThreadLocalTopOffset);
   STATIC_ASSERT(offsetof(IsolateData, builtins_) == kBuiltinsTableOffset);
-  STATIC_ASSERT(offsetof(IsolateData, external_memory_) ==
-                kExternalMemoryOffset);
-  STATIC_ASSERT(offsetof(IsolateData, external_memory_limit_) ==
-                kExternalMemoryLlimitOffset);
-  STATIC_ASSERT(
-      offsetof(IsolateData, external_memory_low_since_mark_compact_) ==
-      kExternalMemoryLowSinceMarkCompactOffset);
   STATIC_ASSERT(offsetof(IsolateData, fast_c_call_caller_fp_) ==
                 kFastCCallCallerFPOffset);
   STATIC_ASSERT(offsetof(IsolateData, fast_c_call_caller_pc_) ==

src/execution/isolate.cc

@@ -2971,17 +2971,6 @@ void Isolate::CheckIsolateLayout() {
            Internals::kIsolateStackGuardOffset);
   CHECK_EQ(static_cast<int>(OFFSET_OF(Isolate, isolate_data_.roots_)),
            Internals::kIsolateRootsOffset);
-  CHECK_EQ(Internals::kExternalMemoryOffset % 8, 0);
-  CHECK_EQ(static_cast<int>(OFFSET_OF(Isolate, isolate_data_.external_memory_)),
-           Internals::kExternalMemoryOffset);
-  CHECK_EQ(Internals::kExternalMemoryLimitOffset % 8, 0);
-  CHECK_EQ(static_cast<int>(
-               OFFSET_OF(Isolate, isolate_data_.external_memory_limit_)),
-           Internals::kExternalMemoryLimitOffset);
-  CHECK_EQ(Internals::kExternalMemoryLowSinceMarkCompactOffset % 8, 0);
-  CHECK_EQ(static_cast<int>(OFFSET_OF(
-               Isolate, isolate_data_.external_memory_low_since_mark_compact_)),
-           Internals::kExternalMemoryLowSinceMarkCompactOffset);
 }
 
 void Isolate::ClearSerializerData() {

src/heap/heap-inl.h

@@ -76,19 +76,10 @@ Isolate* Heap::isolate() {
       reinterpret_cast<size_t>(reinterpret_cast<Isolate*>(16)->heap()) + 16);
 }
 
-int64_t Heap::external_memory() {
-  return isolate()->isolate_data()->external_memory_;
-}
-
-void Heap::update_external_memory(int64_t delta) {
-  const int64_t amount = isolate()->isolate_data()->external_memory_ + delta;
-  isolate()->isolate_data()->external_memory_ = amount;
-  if (amount <
-      isolate()->isolate_data()->external_memory_low_since_mark_compact_) {
-    isolate()->isolate_data()->external_memory_low_since_mark_compact_ = amount;
-    isolate()->isolate_data()->external_memory_limit_ =
-        amount + kExternalAllocationSoftLimit;
-  }
+int64_t Heap::external_memory() { return external_memory_.total(); }
+
+int64_t Heap::update_external_memory(int64_t delta) {
+  return external_memory_.Update(delta);
 }
 
 RootsTable& Heap::roots_table() { return isolate()->roots_table(); }

src/heap/heap.cc

@@ -521,7 +521,7 @@ void Heap::PrintShortHeapStatistics() {
                memory_allocator()->unmapper()->NumberOfCommittedChunks(),
                CommittedMemoryOfUnmapper() / KB);
   PrintIsolate(isolate_, "External memory reported: %6" PRId64 " KB\n",
-               isolate()->isolate_data()->external_memory_ / KB);
+               external_memory_.total() / KB);
   PrintIsolate(isolate_, "Backing store memory: %6zu KB\n",
                backing_store_bytes_ / KB);
   PrintIsolate(isolate_, "External memory global %zu KB\n",
@@ -1412,10 +1412,9 @@ void Heap::ReportExternalMemoryPressure() {
       static_cast<GCCallbackFlags>(
           kGCCallbackFlagSynchronousPhantomCallbackProcessing |
           kGCCallbackFlagCollectAllExternalMemory);
-  int64_t current = isolate()->isolate_data()->external_memory_;
-  int64_t baseline =
-      isolate()->isolate_data()->external_memory_low_since_mark_compact_;
-  int64_t limit = isolate()->isolate_data()->external_memory_limit_;
+  int64_t current = external_memory_.total();
+  int64_t baseline = external_memory_.low_since_mark_compact();
+  int64_t limit = external_memory_.limit();
   TRACE_EVENT2(
       "devtools.timeline,v8", "V8.ExternalMemoryPressure", "external_memory_mb",
       static_cast<int>((current - baseline) / MB), "external_memory_limit_mb",
@@ -1454,6 +1453,8 @@ void Heap::ReportExternalMemoryPressure() {
   }
 }
 
+int64_t Heap::external_memory_limit() { return external_memory_.limit(); }
+
 void Heap::EnsureFillerObjectAtTop() {
   // There may be an allocation memento behind objects in new space. Upon
   // evacuation of a non-full new space (or if we are on the last page) there
@@ -2155,12 +2156,7 @@ void Heap::RecomputeLimits(GarbageCollector collector) {
   HeapGrowingMode mode = CurrentHeapGrowingMode();
 
   if (collector == MARK_COMPACTOR) {
-    // Register the amount of external allocated memory.
-    isolate()->isolate_data()->external_memory_low_since_mark_compact_ =
-        isolate()->isolate_data()->external_memory_;
-    isolate()->isolate_data()->external_memory_limit_ =
-        isolate()->isolate_data()->external_memory_ +
-        kExternalAllocationSoftLimit;
+    external_memory_.ResetAfterGC();
 
     old_generation_allocation_limit_ =
         MemoryController<V8HeapTrait>::CalculateAllocationLimit(
@@ -3839,8 +3835,8 @@ void Heap::CollectGarbageOnMemoryPressure() {
   double end = MonotonicallyIncreasingTimeInMs();
 
   // Estimate how much memory we can free.
-  int64_t potential_garbage = (CommittedMemory() - SizeOfObjects()) +
-                              isolate()->isolate_data()->external_memory_;
+  int64_t potential_garbage =
+      (CommittedMemory() - SizeOfObjects()) + external_memory_.total();
   // If we can potentially free large amount of memory, then start GC right
   // away instead of waiting for memory reducer.
   if (potential_garbage >= kGarbageThresholdInBytes &&
@@ -4860,15 +4856,8 @@ size_t Heap::GlobalSizeOfObjects() {
   return on_heap_size + embedder_size;
 }
 
-uint64_t Heap::PromotedExternalMemorySize() {
-  IsolateData* isolate_data = isolate()->isolate_data();
-  if (isolate_data->external_memory_ <=
-      isolate_data->external_memory_low_since_mark_compact_) {
-    return 0;
-  }
-  return static_cast<uint64_t>(
-      isolate_data->external_memory_ -
-      isolate_data->external_memory_low_since_mark_compact_);
+uint64_t Heap::AllocatedExternalMemorySinceMarkCompact() {
+  return external_memory_.AllocatedSinceMarkCompact();
 }
 
 bool Heap::AllocationLimitOvershotByLargeMargin() {
@@ -4876,12 +4865,12 @@ bool Heap::AllocationLimitOvershotByLargeMargin() {
   // The number is chosen based on v8.browsing_mobile on Nexus 7v2.
   constexpr size_t kMarginForSmallHeaps = 32u * MB;
 
-  const size_t v8_overshoot =
-      old_generation_allocation_limit_ <
-              OldGenerationObjectsAndPromotedExternalMemorySize()
-          ? OldGenerationObjectsAndPromotedExternalMemorySize() -
-                old_generation_allocation_limit_
-          : 0;
+  uint64_t size_now =
+      OldGenerationSizeOfObjects() + AllocatedExternalMemorySinceMarkCompact();
+
+  const size_t v8_overshoot = old_generation_allocation_limit_ < size_now
+                                  ? size_now - old_generation_allocation_limit_
+                                  : 0;
   const size_t global_overshoot =
       global_allocation_limit_ < GlobalSizeOfObjects()
           ? GlobalSizeOfObjects() - global_allocation_limit_
@@ -4979,7 +4968,8 @@ base::Optional<size_t> Heap::GlobalMemoryAvailable() {
 
 double Heap::PercentToOldGenerationLimit() {
   double size_at_gc = old_generation_size_at_last_gc_;
-  double size_now = OldGenerationObjectsAndPromotedExternalMemorySize();
+  double size_now =
+      OldGenerationSizeOfObjects() + AllocatedExternalMemorySinceMarkCompact();
   double current_bytes = size_now - size_at_gc;
   double total_bytes = old_generation_allocation_limit_ - size_at_gc;
   return total_bytes > 0 ? (current_bytes / total_bytes) * 100.0 : 0;
@@ -4987,7 +4977,8 @@ double Heap::PercentToOldGenerationLimit() {
 
 double Heap::PercentToGlobalMemoryLimit() {
   double size_at_gc = old_generation_size_at_last_gc_;
-  double size_now = OldGenerationObjectsAndPromotedExternalMemorySize();
+  double size_now =
+      OldGenerationSizeOfObjects() + AllocatedExternalMemorySinceMarkCompact();
   double current_bytes = size_now - size_at_gc;
   double total_bytes = old_generation_allocation_limit_ - size_at_gc;
   return total_bytes > 0 ? (current_bytes / total_bytes) * 100.0 : 0;

src/heap/heap.h

@@ -265,6 +265,44 @@ class Heap {
     const char* event_name_;
   };
 
+  class ExternalMemoryAccounting {
+   public:
+    int64_t total() { return total_; }
+    int64_t limit() { return limit_; }
+    int64_t low_since_mark_compact() { return low_since_mark_compact_; }
+
+    void ResetAfterGC() {
+      low_since_mark_compact_ = total_;
+      limit_ = total_ + kExternalAllocationSoftLimit;
+    }
+
+    int64_t Update(int64_t delta) {
+      const int64_t amount = total_ += delta;
+      if (amount < low_since_mark_compact_) {
+        low_since_mark_compact_ = amount;
+        limit_ = amount + kExternalAllocationSoftLimit;
+      }
+      return amount;
+    }
+
+    int64_t AllocatedSinceMarkCompact() {
+      if (total_ <= low_since_mark_compact_) {
+        return 0;
+      }
+      return static_cast<uint64_t>(total_ - low_since_mark_compact_);
+    }
+
+   private:
+    // The amount of external memory registered through the API.
+    int64_t total_ = 0;
+
+    // The limit when to trigger memory pressure from the API.
+    int64_t limit_ = kExternalAllocationSoftLimit;
+
+    // Caches the amount of external memory registered at the last MC.
+    int64_t low_since_mark_compact_ = 0;
+  };
+
   using PretenuringFeedbackMap =
       std::unordered_map<AllocationSite, size_t, Object::Hasher>;
 
@@ -667,7 +705,8 @@ class Heap {
   int64_t external_memory_hard_limit() { return max_old_generation_size_ / 2; }
 
   V8_INLINE int64_t external_memory();
-  V8_INLINE void update_external_memory(int64_t delta);
+  V8_EXPORT_PRIVATE int64_t external_memory_limit();
+  V8_INLINE int64_t update_external_memory(int64_t delta);
 
   V8_EXPORT_PRIVATE size_t YoungArrayBufferBytes();
   V8_EXPORT_PRIVATE size_t OldArrayBufferBytes();
@@ -1245,10 +1284,6 @@ class Heap {
     survived_since_last_expansion_ += survived;
   }
 
-  inline uint64_t OldGenerationObjectsAndPromotedExternalMemorySize() {
-    return OldGenerationSizeOfObjects() + PromotedExternalMemorySize();
-  }
-
   inline void UpdateNewSpaceAllocationCounter();
 
   inline size_t NewSpaceAllocationCounter();
@@ -1781,12 +1816,11 @@ class Heap {
   // ===========================================================================
 
   inline size_t OldGenerationSpaceAvailable() {
-    if (old_generation_allocation_limit_ <=
-        OldGenerationObjectsAndPromotedExternalMemorySize())
-      return 0;
-    return old_generation_allocation_limit_ -
-           static_cast<size_t>(
-               OldGenerationObjectsAndPromotedExternalMemorySize());
+    uint64_t bytes = OldGenerationSizeOfObjects() +
+                     AllocatedExternalMemorySinceMarkCompact();
+
+    if (old_generation_allocation_limit_ <= bytes) return 0;
+    return old_generation_allocation_limit_ - static_cast<size_t>(bytes);
   }
 
   void UpdateTotalGCTime(double duration);
@@ -1952,6 +1986,7 @@ class Heap {
 
   // The amount of memory that has been freed concurrently.
   std::atomic<uintptr_t> external_memory_concurrently_freed_{0};
+  ExternalMemoryAccounting external_memory_;
 
   // This can be calculated directly from a pointer to the heap; however, it is
   // more expedient to get at the isolate directly from within Heap methods.
@@ -2031,7 +2066,7 @@ class Heap {
   int gc_post_processing_depth_ = 0;
 
   // Returns the amount of external memory registered since last global gc.
-  V8_EXPORT_PRIVATE uint64_t PromotedExternalMemorySize();
+  V8_EXPORT_PRIVATE uint64_t AllocatedExternalMemorySinceMarkCompact();
 
   // How many "runtime allocations" happened.
   uint32_t allocations_count_ = 0;

test/unittests/heap/heap-unittest.cc

@@ -125,16 +125,14 @@ TEST_F(HeapTest, ASLR) {
 
 TEST_F(HeapTest, ExternalLimitDefault) {
   Heap* heap = i_isolate()->heap();
-  EXPECT_EQ(kExternalAllocationSoftLimit,
-            heap->isolate()->isolate_data()->external_memory_limit_);
+  EXPECT_EQ(kExternalAllocationSoftLimit, heap->external_memory_limit());
 }
 
 TEST_F(HeapTest, ExternalLimitStaysAboveDefaultForExplicitHandling) {
   v8_isolate()->AdjustAmountOfExternalAllocatedMemory(+10 * MB);
   v8_isolate()->AdjustAmountOfExternalAllocatedMemory(-10 * MB);
   Heap* heap = i_isolate()->heap();
-  EXPECT_GE(heap->isolate()->isolate_data()->external_memory_limit_,
-            kExternalAllocationSoftLimit);
+  EXPECT_GE(heap->external_memory_limit(), kExternalAllocationSoftLimit);
 }
 
 #if V8_TARGET_ARCH_64_BIT