Allow embedder to set global OOM handler

Embedders can currently specify a callback for OOM errors during
Isolate initialization. However, there are cases where an OOM error can
be thrown in a context where we don't have access to an Isolate, for
example on a task posted to a worker thread. This CL introduces an
initialization API to allow the embedder to specify a process-wide OOM
callback.

Bug: chromium:614440
Change-Id: I326753d80767679f677e85104d9edeef92e19086
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3561916Reviewed-by: Toon Verwaest <verwaest@chromium.org>
Commit-Queue: Kevin Babbitt <kbabbitt@microsoft.com>
Cr-Commit-Position: refs/heads/main@{#79832}

include/v8-initialization.h

@@ -8,10 +8,11 @@
 #include <stddef.h>
 #include <stdint.h>
 
-#include "v8-internal.h"  // NOLINT(build/include_directory)
-#include "v8-isolate.h"   // NOLINT(build/include_directory)
-#include "v8-platform.h"  // NOLINT(build/include_directory)
-#include "v8config.h"     // NOLINT(build/include_directory)
+#include "v8-callbacks.h"  // NOLINT(build/include_directory)
+#include "v8-internal.h"   // NOLINT(build/include_directory)
+#include "v8-isolate.h"    // NOLINT(build/include_directory)
+#include "v8-platform.h"   // NOLINT(build/include_directory)
+#include "v8config.h"      // NOLINT(build/include_directory)
 
 // We reserve the V8_* prefix for macros defined in V8 public API and
 // assume there are no name conflicts with the embedder's code.
@@ -275,6 +276,14 @@ class V8_EXPORT V8 {
       UnhandledExceptionCallback unhandled_exception_callback);
 #endif
 
+  /**
+   * Allows the host application to provide a callback that will be called when
+   * v8 has encountered a fatal failure to allocate memory and is about to
+   * terminate.
+   */
+
+  static void SetFatalMemoryErrorCallback(OOMErrorCallback oom_error_callback);
+
   /**
    * Get statistics about the shared memory usage.
    */

src/api/api.cc

@@ -172,6 +172,8 @@
 
 namespace v8 {
 
+static OOMErrorCallback g_oom_error_callback = nullptr;
+
 static ScriptOrigin GetScriptOriginForScript(i::Isolate* isolate,
                                              i::Handle<i::Script> script) {
   i::Handle<i::Object> scriptName(script->GetNameOrSourceURL(), isolate);
@@ -228,8 +230,9 @@ void i::V8::FatalProcessOutOfMemory(i::Isolate* isolate, const char* location,
     memset(last_few_messages, 0x0BADC0DE, Heap::kTraceRingBufferSize + 1);
     memset(js_stacktrace, 0x0BADC0DE, Heap::kStacktraceBufferSize + 1);
     memset(&heap_stats, 0xBADC0DE, sizeof(heap_stats));
-    // Note that the embedder's oom handler is also not available and therefore
-    // won't be called in this case. We just crash.
+    // Give the embedder a chance to handle the condition. If it doesn't,
+    // just crash.
+    if (g_oom_error_callback) g_oom_error_callback(location, is_heap_oom);
     FATAL("Fatal process out of memory: %s", location);
     UNREACHABLE();
   }
@@ -304,6 +307,7 @@ void i::V8::FatalProcessOutOfMemory(i::Isolate* isolate, const char* location,
     }
   }
   Utils::ReportOOMFailure(isolate, location, is_heap_oom);
+  if (g_oom_error_callback) g_oom_error_callback(location, is_heap_oom);
   // If the fatal error handler returns, we stop execution.
   FATAL("API fatal error handler returned after process out of memory");
 }
@@ -6106,6 +6110,11 @@ void V8::SetUnhandledExceptionCallback(
 }
 #endif  // V8_OS_WIN
 
+void v8::V8::SetFatalMemoryErrorCallback(
+    v8::OOMErrorCallback oom_error_callback) {
+  g_oom_error_callback = oom_error_callback;
+}
+
 void v8::V8::SetEntropySource(EntropySource entropy_source) {
   base::RandomNumberGenerator::SetEntropySource(entropy_source);
 }