[offthread] Check memory pressure after merging

Currently, there is no indication of memory pressure after merging pages
from off-thread space into the heap. This can mean that with lots of
off-thread compilations, we eventually run out of pages in the memory
allocator.

Now, trigger a critical memory pressure notification after merging if
the old generation is close to being unexpandable. We may want to add
other heuristics here (e.g. moderate memory pressure if the heap is
highly fragmented), or possibly add similar checks when the off-thread
space is initialized, but this works for now.

As a drive-by, inline the NotifyOffThreadSpaceMerged call into the
off-thread factory merge -- we already do a bunch of special merging
stuff in there, and nowhere else, so we may as well be explicit in what
we're actually doing. This also allows us to do a pre-check on old-space
size and hard limits just before the merge.

Bug: chromium:1011762
Change-Id: Ic2b46057cf751bd7eaf787f46beaaf4df36e78fe
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2131915
Commit-Queue: Leszek Swirski <leszeks@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Cr-Commit-Position: refs/heads/master@{#67357}

src/heap/heap.cc

@@ -5354,20 +5354,6 @@ void Heap::NotifyOldGenerationExpansion() {
   }
 }
 
-void Heap::NotifyOffThreadSpaceMerged() {
-  // TODO(leszeks): Ideally we would do this check during off-thread page
-  // allocation too, to proactively do GC. We should also probably do this check
-  // before merging rather than after.
-  if (!CanExpandOldGeneration(0)) {
-    // TODO(leszeks): We should try to invoke the near-heap limit callback and
-    // do a last-resort GC first.
-    FatalProcessOutOfMemory("Failed to merge off-thread pages into heap.");
-  }
-  StartIncrementalMarkingIfAllocationLimitIsReached(
-      GCFlagsForIncrementalMarking(), kGCCallbackScheduleIdleGarbageCollection);
-  NotifyOldGenerationExpansion();
-}
-
 void Heap::SetEmbedderHeapTracer(EmbedderHeapTracer* tracer) {
   DCHECK_EQ(gc_state_, HeapState::NOT_IN_GC);
   local_embedder_heap_tracer()->SetRemoteTracer(tracer);

src/heap/heap.h

@@ -434,9 +434,6 @@ class Heap {
 
   void NotifyOldGenerationExpansion();
 
-  // Notifies the heap that an off-thread space has been merged into it.
-  void NotifyOffThreadSpaceMerged();
-
   inline Address* NewSpaceAllocationTopAddress();
   inline Address* NewSpaceAllocationLimitAddress();
   inline Address* OldSpaceAllocationTopAddress();
@@ -2246,6 +2243,7 @@ class Heap {
 
   // The allocator interface.
   friend class Factory;
+  friend class OffThreadFactory;
 
   // The Isolate constructs us.
   friend class Isolate;

src/heap/off-thread-heap.cc

@@ -123,8 +123,39 @@ void OffThreadHeap::Publish(Heap* heap) {
     TRACE_EVENT0(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
                  "V8.OffThreadFinalization.Publish.Merge");
     Heap* heap = isolate->heap();
+
+    // Ensure that the old-space can expand do the size needed for the
+    // off-thread objects. Use capacity rather than size since we're adding
+    // entire pages.
+    size_t off_thread_size = space_.Capacity() + lo_space_.Size();
+    if (!heap->CanExpandOldGeneration(off_thread_size)) {
+      heap->InvokeNearHeapLimitCallback();
+      if (!heap->CanExpandOldGeneration(off_thread_size)) {
+        heap->CollectAllAvailableGarbage(GarbageCollectionReason::kLastResort);
+        if (!heap->CanExpandOldGeneration(off_thread_size)) {
+          heap->FatalProcessOutOfMemory(
+              "Can't expand old-space enough to merge off-thread pages.");
+        }
+      }
+    }
+
     heap->old_space()->MergeLocalSpace(&space_);
     heap->lo_space()->MergeOffThreadSpace(&lo_space_);
+
+    DCHECK(heap->CanExpandOldGeneration(0));
+    heap->NotifyOldGenerationExpansion();
+
+    // Possibly trigger a GC if we're close to exhausting the old generation.
+    // TODO(leszeks): Adjust the heuristics here.
+    heap->StartIncrementalMarkingIfAllocationLimitIsReached(
+        heap->GCFlagsForIncrementalMarking(),
+        kGCCallbackScheduleIdleGarbageCollection);
+
+    if (!heap->ShouldExpandOldGenerationOnSlowAllocation() ||
+        !heap->CanExpandOldGeneration(1 * MB)) {
+      heap->CollectGarbage(OLD_SPACE,
+                           GarbageCollectionReason::kAllocationFailure);
+    }
   }
 
   // Iterate the string slots, as an offset from the holders we have handles to.

src/heap/spaces.cc

@@ -1765,10 +1765,6 @@ void PagedSpace::MergeLocalSpace(LocalSpace* other) {
     // before shipping, which will likely be using LocalHeap.
   }
 
-  if (merging_from_off_thread) {
-    heap()->NotifyOffThreadSpaceMerged();
-  }
-
   DCHECK_EQ(0u, other->Size());
   DCHECK_EQ(0u, other->Capacity());
 }
@@ -4495,8 +4491,6 @@ void OldLargeObjectSpace::MergeOffThreadSpace(
         heap()->incremental_marking()->black_allocation(),
         heap()->incremental_marking()->marking_state()->IsBlack(object));
   }
-
-  heap()->NotifyOffThreadSpaceMerged();
 }
 
 NewLargeObjectSpace::NewLargeObjectSpace(Heap* heap, size_t capacity)