[Memory] Add an OnCriticalMemoryPressure method to V8::Platform.

Adds virtual V8::Platform::OnCriticalMemoryPressure method, default
implementation does nothing.

Calls this method on first allocation failures in NewArray, Malloced,
and zone AccountingAllocator and adds retry logic.

Adds utility functions for allocating base::VirtualMemory to functions
in allocation.h, which call this method and add retry logic.

Calls these utility functions in heap CodeRange, Spaces, StoreBuffer
and SequentialMarkingDeque.

Bug: v8:6635
Cq-Include-Trybots: master.tryserver.chromium.linux:linux_chromium_rel_ng
Change-Id: I38afd394f3be556aca037d16675e9884658158cb
Reviewed-on: https://chromium-review.googlesource.com/583543
Commit-Queue: Bill Budge <bbudge@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#46988}

include/v8-platform.h

@@ -132,6 +132,15 @@ class Platform {
 
   virtual ~Platform() = default;
 
+  /**
+   * Enables the embedder to respond in cases where V8 can't allocate large
+   * blocks of memory. V8 retries the failed allocation once after calling this
+   * method. On success, execution continues; otherwise V8 exits with a fatal
+   * error.
+   * Embedder overrides of this function must NOT call back into V8.
+   */
+  virtual void OnCriticalMemoryPressure() {}
+
   /**
    * Gets the number of threads that are used to execute background tasks. Is
    * used to estimate the number of tasks a work package should be split into.

src/allocation.cc

@@ -18,10 +18,32 @@
 namespace v8 {
 namespace internal {
 
+namespace {
+
+void* AlignedAllocInternal(size_t size, size_t alignment) {
+  void* ptr;
+#if V8_OS_WIN
+  ptr = _aligned_malloc(size, alignment);
+#elif V8_LIBC_BIONIC
+  // posix_memalign is not exposed in some Android versions, so we fall back to
+  // memalign. See http://code.google.com/p/android/issues/detail?id=35391.
+  ptr = memalign(alignment, size);
+#else
+  if (posix_memalign(&ptr, alignment, size)) ptr = nullptr;
+#endif
+  return ptr;
+}
+
+}  // namespace
+
 void* Malloced::New(size_t size) {
   void* result = malloc(size);
-  if (result == NULL) {
-    V8::FatalProcessOutOfMemory("Malloced operator new");
+  if (result == nullptr) {
+    V8::GetCurrentPlatform()->OnCriticalMemoryPressure();
+    result = malloc(size);
+    if (result == nullptr) {
+      V8::FatalProcessOutOfMemory("Malloced operator new");
+    }
   }
   return result;
 }
@@ -54,17 +76,14 @@ char* StrNDup(const char* str, int n) {
 void* AlignedAlloc(size_t size, size_t alignment) {
   DCHECK_LE(V8_ALIGNOF(void*), alignment);
   DCHECK(base::bits::IsPowerOfTwo(alignment));
-  void* ptr;
-#if V8_OS_WIN
-  ptr = _aligned_malloc(size, alignment);
-#elif V8_LIBC_BIONIC
-  // posix_memalign is not exposed in some Android versions, so we fall back to
-  // memalign. See http://code.google.com/p/android/issues/detail?id=35391.
-  ptr = memalign(alignment, size);
-#else
-  if (posix_memalign(&ptr, alignment, size)) ptr = NULL;
-#endif
-  if (ptr == NULL) V8::FatalProcessOutOfMemory("AlignedAlloc");
+  void* ptr = AlignedAllocInternal(size, alignment);
+  if (ptr == nullptr) {
+    V8::GetCurrentPlatform()->OnCriticalMemoryPressure();
+    ptr = AlignedAllocInternal(size, alignment);
+    if (ptr == nullptr) {
+      V8::FatalProcessOutOfMemory("AlignedAlloc");
+    }
+  }
   return ptr;
 }
 
@@ -80,5 +99,32 @@ void AlignedFree(void *ptr) {
 #endif
 }
 
+bool AllocVirtualMemory(size_t size, void* hint, base::VirtualMemory* result) {
+  base::VirtualMemory first_try(size, hint);
+  if (first_try.IsReserved()) {
+    result->TakeControl(&first_try);
+    return true;
+  }
+
+  V8::GetCurrentPlatform()->OnCriticalMemoryPressure();
+  base::VirtualMemory second_try(size, hint);
+  result->TakeControl(&second_try);
+  return result->IsReserved();
+}
+
+bool AlignedAllocVirtualMemory(size_t size, size_t alignment, void* hint,
+                               base::VirtualMemory* result) {
+  base::VirtualMemory first_try(size, alignment, hint);
+  if (first_try.IsReserved()) {
+    result->TakeControl(&first_try);
+    return true;
+  }
+
+  V8::GetCurrentPlatform()->OnCriticalMemoryPressure();
+  base::VirtualMemory second_try(size, alignment, hint);
+  result->TakeControl(&second_try);
+  return result->IsReserved();
+}
+
 }  // namespace internal
 }  // namespace v8

src/allocation.h

@@ -5,15 +5,21 @@
 #ifndef V8_ALLOCATION_H_
 #define V8_ALLOCATION_H_
 
+#include "include/v8-platform.h"
 #include "src/base/compiler-specific.h"
+#include "src/base/platform/platform.h"
 #include "src/globals.h"
+#include "src/v8.h"
 
 namespace v8 {
 namespace internal {
 
-// Called when allocation routines fail to allocate.
-// This function should not return, but should terminate the current
-// processing.
+// This file defines memory allocation functions. If a first attempt at an
+// allocation fails, these functions call back into the embedder, then attempt
+// the allocation a second time. The embedder callback must not reenter V8.
+
+// Called when allocation routines fail to allocate, even with a possible retry.
+// This function should not return, but should terminate the current processing.
 V8_EXPORT_PRIVATE void FatalProcessOutOfMemory(const char* message);
 
 // Superclass for classes managed with new & delete.
@@ -26,29 +32,17 @@ class V8_EXPORT_PRIVATE Malloced {
   static void Delete(void* p);
 };
 
-// DEPRECATED
-// TODO(leszeks): Delete this during a quiet period
-#define BASE_EMBEDDED
-
-
-// Superclass for classes only using static method functions.
-// The subclass of AllStatic cannot be instantiated at all.
-class AllStatic {
-#ifdef DEBUG
- public:
-  AllStatic() = delete;
-#endif
-};
-
-
 template <typename T>
 T* NewArray(size_t size) {
-  T* result = new T[size];
-  if (result == NULL) FatalProcessOutOfMemory("NewArray");
+  T* result = new (std::nothrow) T[size];
+  if (result == nullptr) {
+    V8::GetCurrentPlatform()->OnCriticalMemoryPressure();
+    result = new (std::nothrow) T[size];
+    if (result == nullptr) FatalProcessOutOfMemory("NewArray");
+  }
   return result;
 }
 
-
 template <typename T>
 void DeleteArray(T* array) {
   delete[] array;
@@ -74,6 +68,10 @@ class FreeStoreAllocationPolicy {
 void* AlignedAlloc(size_t size, size_t alignment);
 void AlignedFree(void *ptr);
 
+bool AllocVirtualMemory(size_t size, void* hint, base::VirtualMemory* result);
+bool AlignedAllocVirtualMemory(size_t size, size_t alignment, void* hint,
+                               base::VirtualMemory* result);
+
 }  // namespace internal
 }  // namespace v8
 

src/base/DEPS

 include_rules = [
   "-include",
   "+include/v8config.h",
-  "+include/v8stdint.h",
   "-src",
   "+src/base",
 ]

src/globals.h

@@ -125,6 +125,19 @@ const int kStackSpaceRequiredForCompilation = 40;
 #define V8_SFI_HAS_UNIQUE_ID 1
 #endif
 
+// Superclass for classes only using static method functions.
+// The subclass of AllStatic cannot be instantiated at all.
+class AllStatic {
+#ifdef DEBUG
+ public:
+  AllStatic() = delete;
+#endif
+};
+
+// DEPRECATED
+// TODO(leszeks): Delete this during a quiet period
+#define BASE_EMBEDDED
+
 typedef uint8_t byte;
 typedef byte* Address;
 

src/heap/sequential-marking-deque.cc

@@ -13,8 +13,8 @@ namespace v8 {
 namespace internal {
 
 void SequentialMarkingDeque::SetUp() {
-  base::VirtualMemory reservation(kMaxSize, heap_->GetRandomMmapAddr());
-  if (!reservation.IsReserved()) {
+  base::VirtualMemory reservation;
+  if (!AllocVirtualMemory(kMaxSize, heap_->GetRandomMmapAddr(), &reservation)) {
     V8::FatalProcessOutOfMemory("SequentialMarkingDeque::SetUp");
   }
   backing_store_committed_size_ = 0;

src/heap/spaces.cc

@@ -25,7 +25,6 @@
 namespace v8 {
 namespace internal {
 
-
 // ----------------------------------------------------------------------------
 // HeapObjectIterator
 
@@ -119,12 +118,14 @@ bool CodeRange::SetUp(size_t requested) {
 
   DCHECK(!kRequiresCodeRange || requested <= kMaximalCodeRangeSize);
 
-  base::VirtualMemory reservation(
-      requested,
-      Max(kCodeRangeAreaAlignment,
-          static_cast<size_t>(base::OS::AllocateAlignment())),
-      base::OS::GetRandomMmapAddr());
-  if (!reservation.IsReserved()) return false;
+  base::VirtualMemory reservation;
+  if (!AlignedAllocVirtualMemory(
+          requested,
+          Max(kCodeRangeAreaAlignment,
+              static_cast<size_t>(base::OS::AllocateAlignment())),
+          base::OS::GetRandomMmapAddr(), &reservation)) {
+    return false;
+  }
 
   // We are sure that we have mapped a block of requested addresses.
   DCHECK(reservation.size() == requested);
@@ -453,9 +454,10 @@ void MemoryAllocator::FreeMemory(Address base, size_t size,
 Address MemoryAllocator::ReserveAlignedMemory(size_t size, size_t alignment,
                                               void* hint,
                                               base::VirtualMemory* controller) {
-  base::VirtualMemory reservation(size, alignment, hint);
+  base::VirtualMemory reservation;
+  if (!AlignedAllocVirtualMemory(size, alignment, hint, &reservation))
+    return nullptr;
 
-  if (!reservation.IsReserved()) return nullptr;
   const Address base =
       RoundUp(static_cast<Address>(reservation.address()), alignment);
   if (base + size != reservation.end()) {

src/heap/store-buffer.cc

@@ -31,8 +31,11 @@ void StoreBuffer::SetUp() {
   // Allocate 3x the buffer size, so that we can start the new store buffer
   // aligned to 2x the size.  This lets us use a bit test to detect the end of
   // the area.
-  base::VirtualMemory reservation(kStoreBufferSize * 3,
-                                  heap_->GetRandomMmapAddr());
+  base::VirtualMemory reservation;
+  if (!AllocVirtualMemory(kStoreBufferSize * 3, heap_->GetRandomMmapAddr(),
+                          &reservation)) {
+    V8::FatalProcessOutOfMemory("StoreBuffer::SetUp");
+  }
   uintptr_t start_as_int = reinterpret_cast<uintptr_t>(reservation.address());
   start_[0] =
       reinterpret_cast<Address*>(RoundUp(start_as_int, kStoreBufferSize));

src/v8.h

@@ -6,7 +6,6 @@
 #define V8_V8_H_
 
 #include "include/v8.h"
-#include "src/allocation.h"
 #include "src/globals.h"
 
 namespace v8 {

src/zone/accounting-allocator.cc

@@ -83,7 +83,11 @@ Segment* AccountingAllocator::GetSegment(size_t bytes) {
 
 Segment* AccountingAllocator::AllocateSegment(size_t bytes) {
   void* memory = malloc(bytes);
-  if (memory) {
+  if (memory == nullptr) {
+    V8::GetCurrentPlatform()->OnCriticalMemoryPressure();
+    memory = malloc(bytes);
+  }
+  if (memory != nullptr) {
     base::AtomicWord current =
         base::Relaxed_AtomicIncrement(&current_memory_usage_, bytes);
     base::AtomicWord max = base::Relaxed_Load(&max_memory_usage_);

test/cctest/BUILD.gn

@@ -108,6 +108,7 @@ v8_executable("cctest") {
     "test-access-checks.cc",
     "test-accessor-assembler.cc",
     "test-accessors.cc",
+    "test-allocation.cc",
     "test-api-accessors.cc",
     "test-api-interceptors.cc",
     "test-api.cc",

test/cctest/cctest.gyp

@@ -126,6 +126,7 @@
       'test-access-checks.cc',
       'test-accessor-assembler.cc',
       'test-accessors.cc',
+      'test-allocation.cc',
       'test-api.cc',
       'test-api.h',
       'test-api-accessors.cc',

test/cctest/test-allocation.cc

+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+#include <stdlib.h>
+#include <string.h>
+
+#include "src/v8.h"
+
+#include "test/cctest/cctest.h"
+
+using v8::internal::AccountingAllocator;
+
+using v8::IdleTask;
+using v8::Isolate;
+using v8::Task;
+
+#include "src/allocation.h"
+#include "src/zone/accounting-allocator.h"
+
+// ASAN isn't configured to return NULL, so skip all of these tests.
+#ifndef V8_USE_ADDRESS_SANITIZER
+
+namespace {
+
+// Minimal implementation of platform that can receive OOM callbacks.
+class MockAllocationPlatform : public v8::Platform {
+ public:
+  MockAllocationPlatform() { current_platform = this; }
+  virtual ~MockAllocationPlatform() {}
+
+  void OnCriticalMemoryPressure() override { oom_callback_called = true; }
+
+  void CallOnBackgroundThread(Task* task,
+                              ExpectedRuntime expected_runtime) override {}
+
+  void CallOnForegroundThread(Isolate* isolate, Task* task) override {}
+
+  void CallDelayedOnForegroundThread(Isolate* isolate, Task* task,
+                                     double delay_in_seconds) override {}
+
+  double MonotonicallyIncreasingTime() override { return 0.0; }
+
+  void CallIdleOnForegroundThread(Isolate* isolate, IdleTask* task) override {}
+
+  bool IdleTasksEnabled(Isolate* isolate) override { return false; }
+
+  v8::TracingController* GetTracingController() override {
+    return &tracing_controller_;
+  }
+
+  bool PendingIdleTask() { return false; }
+
+  void PerformIdleTask(double idle_time_in_seconds) {}
+
+  bool PendingDelayedTask() { return false; }
+
+  void PerformDelayedTask() {}
+
+  static MockAllocationPlatform* current_platform;
+  bool oom_callback_called = false;
+
+ private:
+  v8::TracingController tracing_controller_;
+
+  DISALLOW_COPY_AND_ASSIGN(MockAllocationPlatform);
+};
+
+MockAllocationPlatform* MockAllocationPlatform::current_platform = nullptr;
+
+bool DidCallOnCriticalMemoryPressure() {
+  return MockAllocationPlatform::current_platform &&
+         MockAllocationPlatform::current_platform->oom_callback_called;
+}
+
+// No OS should be able to malloc/new this number of bytes. Generate enough
+// random values in the address space to get a very large fraction of it. Using
+// even larger values is that overflow from rounding or padding can cause the
+// allocations to succeed somehow.
+size_t GetHugeMemoryAmount() {
+  static size_t huge_memory = 0;
+  if (!huge_memory) {
+    for (int i = 0; i < 100; i++) {
+      huge_memory |= bit_cast<size_t>(v8::base::OS::GetRandomMmapAddr());
+    }
+    // Make it larger than the available address space.
+    huge_memory *= 2;
+    CHECK_NE(0, huge_memory);
+  }
+  return huge_memory;
+}
+
+void OnMallocedOperatorNewOOM(const char* location, const char* message) {
+  // exit(0) if the OOM callback was called and location matches expectation.
+  if (DidCallOnCriticalMemoryPressure())
+    exit(strcmp(location, "Malloced operator new"));
+  exit(1);
+}
+
+void OnNewArrayOOM(const char* location, const char* message) {
+  // exit(0) if the OOM callback was called and location matches expectation.
+  if (DidCallOnCriticalMemoryPressure()) exit(strcmp(location, "NewArray"));
+  exit(1);
+}
+
+void OnAlignedAllocOOM(const char* location, const char* message) {
+  // exit(0) if the OOM callback was called and location matches expectation.
+  if (DidCallOnCriticalMemoryPressure()) exit(strcmp(location, "AlignedAlloc"));
+  exit(1);
+}
+
+}  // namespace
+
+TEST(AccountingAllocatorOOM) {
+  // TODO(bbudge) Implement a TemporaryPlatformScope to simplify test code.
+  v8::Platform* old_platform = i::V8::GetCurrentPlatform();
+  MockAllocationPlatform platform;
+  i::V8::SetPlatformForTesting(&platform);
+
+  v8::internal::AccountingAllocator allocator;
+  CHECK(!platform.oom_callback_called);
+  v8::internal::Segment* result = allocator.GetSegment(GetHugeMemoryAmount());
+  // On a few systems, allocation somehow succeeds.
+  CHECK_EQ(result == nullptr, platform.oom_callback_called);
+
+  i::V8::SetPlatformForTesting(old_platform);
+}
+
+TEST(MallocedOperatorNewOOM) {
+  v8::Platform* old_platform = i::V8::GetCurrentPlatform();
+  MockAllocationPlatform platform;
+  i::V8::SetPlatformForTesting(&platform);
+
+  CHECK(!platform.oom_callback_called);
+  CcTest::isolate()->SetFatalErrorHandler(OnMallocedOperatorNewOOM);
+  // On failure, this won't return, since a Malloced::New failure is fatal.
+  // In that case, behavior is checked in OnMallocedOperatorNewOOM before exit.
+  void* result = v8::internal::Malloced::New(GetHugeMemoryAmount());
+  // On a few systems, allocation somehow succeeds.
+  CHECK_EQ(result == nullptr, platform.oom_callback_called);
+
+  i::V8::SetPlatformForTesting(old_platform);
+}
+
+TEST(NewArrayOOM) {
+  v8::Platform* old_platform = i::V8::GetCurrentPlatform();
+  MockAllocationPlatform platform;
+  i::V8::SetPlatformForTesting(&platform);
+
+  CHECK(!platform.oom_callback_called);
+  CcTest::isolate()->SetFatalErrorHandler(OnNewArrayOOM);
+  // On failure, this won't return, since a NewArray failure is fatal.
+  // In that case, behavior is checked in OnNewArrayOOM before exit.
+  int8_t* result = v8::internal::NewArray<int8_t>(GetHugeMemoryAmount());
+  // On a few systems, allocation somehow succeeds.
+  CHECK_EQ(result == nullptr, platform.oom_callback_called);
+
+  i::V8::SetPlatformForTesting(old_platform);
+}
+
+TEST(AlignedAllocOOM) {
+  v8::Platform* old_platform = i::V8::GetCurrentPlatform();
+  MockAllocationPlatform platform;
+  i::V8::SetPlatformForTesting(&platform);
+
+  CHECK(!platform.oom_callback_called);
+  CcTest::isolate()->SetFatalErrorHandler(OnAlignedAllocOOM);
+  // On failure, this won't return, since an AlignedAlloc failure is fatal.
+  // In that case, behavior is checked in OnAlignedAllocOOM before exit.
+  void* result = v8::internal::AlignedAlloc(GetHugeMemoryAmount(),
+                                            v8::base::OS::AllocateAlignment());
+  // On a few systems, allocation somehow succeeds.
+  CHECK_EQ(result == nullptr, platform.oom_callback_called);
+
+  i::V8::SetPlatformForTesting(old_platform);
+}
+
+TEST(AllocVirtualMemoryOOM) {
+  v8::Platform* old_platform = i::V8::GetCurrentPlatform();
+  MockAllocationPlatform platform;
+  i::V8::SetPlatformForTesting(&platform);
+
+  CHECK(!platform.oom_callback_called);
+  v8::base::VirtualMemory result;
+  bool success =
+      v8::internal::AllocVirtualMemory(GetHugeMemoryAmount(), nullptr, &result);
+  // On a few systems, allocation somehow succeeds.
+  CHECK_IMPLIES(success, result.IsReserved());
+  CHECK_IMPLIES(!success, !result.IsReserved() && platform.oom_callback_called);
+
+  i::V8::SetPlatformForTesting(old_platform);
+}
+
+TEST(AlignedAllocVirtualMemoryOOM) {
+  v8::Platform* old_platform = i::V8::GetCurrentPlatform();
+  MockAllocationPlatform platform;
+  i::V8::SetPlatformForTesting(&platform);
+
+  CHECK(!platform.oom_callback_called);
+  v8::base::VirtualMemory result;
+  bool success = v8::internal::AlignedAllocVirtualMemory(
+      GetHugeMemoryAmount(), v8::base::OS::AllocateAlignment(), nullptr,
+      &result);
+  // On a few systems, allocation somehow succeeds.
+  CHECK_IMPLIES(success, result.IsReserved());
+  CHECK_IMPLIES(!success, !result.IsReserved() && platform.oom_callback_called);
+
+  i::V8::SetPlatformForTesting(old_platform);
+}
+
+#endif  // V8_USE_ADDRESS_SANITIZER