[heap, debugger] Introduce out-of-memory listener for debugger.

This API will allow DevTools to intercept out-of-memory condition,
increase the heap limit and schedule heap snapshot.

BUG=chromium:675911

Review-Url: https://codereview.chromium.org/2621873003
Cr-Commit-Position: refs/heads/master@{#42225}

src/api.cc

@@ -8819,7 +8819,6 @@ bool Debug::SetDebugEventListener(Isolate* isolate, EventCallback that,
   return true;
 }
 
-
 void Debug::DebugBreak(Isolate* isolate) {
   reinterpret_cast<i::Isolate*>(isolate)->stack_guard()->RequestDebugBreak();
 }
@@ -8987,6 +8986,13 @@ void debug::ChangeBreakOnException(Isolate* isolate, ExceptionBreakState type) {
                                                     type != NoBreakOnException);
 }
 
+void debug::SetOutOfMemoryCallback(Isolate* isolate,
+                                   OutOfMemoryCallback callback, void* data) {
+  i::Isolate* i_isolate = reinterpret_cast<i::Isolate*>(isolate);
+  ENTER_V8(i_isolate);
+  i_isolate->heap()->SetOutOfMemoryCallback(callback, data);
+}
+
 void debug::PrepareStep(Isolate* v8_isolate, StepAction action) {
   i::Isolate* isolate = reinterpret_cast<i::Isolate*>(v8_isolate);
   ENTER_V8(isolate);

src/debug/debug-interface.h

@@ -138,6 +138,16 @@ enum StepAction {
 void PrepareStep(Isolate* isolate, StepAction action);
 void ClearStepping(Isolate* isolate);
 
+/**
+ * Out-of-memory callback function.
+ * The function is invoked when the heap size is close to the hard limit.
+ *
+ * \param data the parameter provided during callback installation.
+ */
+typedef void (*OutOfMemoryCallback)(void* data);
+void SetOutOfMemoryCallback(Isolate* isolate, OutOfMemoryCallback callback,
+                            void* data);
+
 /**
  * Native wrapper around v8::internal::Script object.
  */

src/heap/heap.cc

@@ -95,6 +95,8 @@ Heap::Heap()
       survived_last_scavenge_(0),
       always_allocate_scope_count_(0),
       memory_pressure_level_(MemoryPressureLevel::kNone),
+      out_of_memory_callback_(nullptr),
+      out_of_memory_callback_data_(nullptr),
       contexts_disposed_(0),
       number_of_disposed_maps_(0),
       global_ic_age_(0),
@@ -867,6 +869,9 @@ void Heap::CollectAllAvailableGarbage(GarbageCollectionReason gc_reason) {
   // Note: as weak callbacks can execute arbitrary code, we cannot
   // hope that eventually there will be no weak callbacks invocations.
   // Therefore stop recollecting after several attempts.
+  if (gc_reason == GarbageCollectionReason::kLastResort) {
+    InvokeOutOfMemoryCallback();
+  }
   RuntimeCallTimerScope(isolate(), &RuntimeCallStats::GC_AllAvailableGarbage);
   if (isolate()->concurrent_recompilation_enabled()) {
     // The optimizing compiler may be unnecessarily holding on to memory.
@@ -4534,6 +4539,18 @@ void Heap::MemoryPressureNotification(MemoryPressureLevel level,
   }
 }
 
+void Heap::SetOutOfMemoryCallback(v8::debug::OutOfMemoryCallback callback,
+                                  void* data) {
+  out_of_memory_callback_ = callback;
+  out_of_memory_callback_data_ = data;
+}
+
+void Heap::InvokeOutOfMemoryCallback() {
+  if (out_of_memory_callback_) {
+    out_of_memory_callback_(out_of_memory_callback_data_);
+  }
+}
+
 void Heap::CollectCodeStatistics() {
   CodeStatistics::ResetCodeAndMetadataStatistics(isolate());
   // We do not look for code in new space, or map space.  If code

src/heap/heap.h

@@ -14,6 +14,7 @@
 #include "src/allocation.h"
 #include "src/assert-scope.h"
 #include "src/base/atomic-utils.h"
+#include "src/debug/debug-interface.h"
 #include "src/globals.h"
 #include "src/heap-symbols.h"
 #include "src/list.h"
@@ -846,6 +847,9 @@ class Heap {
                                   bool is_isolate_locked);
   void CheckMemoryPressure();
 
+  void SetOutOfMemoryCallback(v8::debug::OutOfMemoryCallback callback,
+                              void* data);
+
   double MonotonicallyIncreasingTimeInMs();
 
   void RecordStats(HeapStats* stats, bool take_snapshot = false);
@@ -1759,6 +1763,8 @@ class Heap {
 
   void CollectGarbageOnMemoryPressure();
 
+  void InvokeOutOfMemoryCallback();
+
   // Attempt to over-approximate the weak closure by marking object groups and
   // implicit references from global handles, but don't atomically complete
   // marking. If we continue to mark incrementally, we might have marked
@@ -2169,6 +2175,9 @@ class Heap {
   // and reset by a mark-compact garbage collection.
   base::AtomicValue<MemoryPressureLevel> memory_pressure_level_;
 
+  v8::debug::OutOfMemoryCallback out_of_memory_callback_;
+  void* out_of_memory_callback_data_;
+
   // For keeping track of context disposals.
   int contexts_disposed_;
 

test/cctest/test-debug.cc

@@ -6644,3 +6644,31 @@ TEST(DebugStepOverFunctionWithCaughtException) {
   v8::Debug::SetDebugEventListener(env->GetIsolate(), nullptr);
   CHECK_EQ(break_point_hit_count, 4);
 }
+
+bool out_of_memory_callback_called = false;
+void OutOfMemoryCallback(void* data) {
+  out_of_memory_callback_called = true;
+  reinterpret_cast<v8::Isolate*>(data)->IncreaseHeapLimitForDebugging();
+}
+
+UNINITIALIZED_TEST(DebugSetOutOfMemoryListener) {
+  v8::Isolate::CreateParams create_params;
+  create_params.array_buffer_allocator = CcTest::array_buffer_allocator();
+  create_params.constraints.set_max_old_space_size(10);
+  v8::Isolate* isolate = v8::Isolate::New(create_params);
+  i::Isolate* i_isolate = reinterpret_cast<i::Isolate*>(isolate);
+  {
+    v8::Isolate::Scope i_scope(isolate);
+    v8::HandleScope scope(isolate);
+    LocalContext context(isolate);
+    v8::debug::SetOutOfMemoryCallback(isolate, OutOfMemoryCallback,
+                                      reinterpret_cast<void*>(isolate));
+    CHECK(!out_of_memory_callback_called);
+    // The following allocation fails unless the out-of-memory callback
+    // increases the heap limit.
+    int length = 10 * i::MB / i::kPointerSize;
+    i_isolate->factory()->NewFixedArray(length, i::TENURED);
+    CHECK(out_of_memory_callback_called);
+  }
+  isolate->Dispose();
+}