[heap] Reclaim inaccessible memory.

Bug: chromium:897074
Cq-Include-Trybots: luci.chromium.try:linux_chromium_rel_ng
Change-Id: I728572cda9a8914ee689eeee68a060b5713e4c6b
Reviewed-on: https://chromium-review.googlesource.com/c/1290972Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Commit-Queue: Hannes Payer <hpayer@chromium.org>
Cr-Commit-Position: refs/heads/master@{#56845}

include/v8-platform.h

@@ -250,6 +250,13 @@ class PageAllocator {
    */
   virtual bool SetPermissions(void* address, size_t length,
                               Permission permissions) = 0;
+
+  /**
+   * Frees memory in the given [address, address + size) range. address and size
+   * should be operating system page-aligned. The next write to this
+   * memory area brings the memory transparently back.
+   */
+  virtual bool DiscardSystemPages(void* address, size_t size) { return true; }
 };
 
 /**

src/base/bounded-page-allocator.cc

@@ -97,5 +97,9 @@ bool BoundedPageAllocator::SetPermissions(void* address, size_t size,
   return page_allocator_->SetPermissions(address, size, access);
 }
 
+bool BoundedPageAllocator::DiscardSystemPages(void* address, size_t size) {
+  return page_allocator_->DiscardSystemPages(address, size);
+}
+
 }  // namespace base
 }  // namespace v8

src/base/bounded-page-allocator.h

@@ -63,6 +63,8 @@ class V8_BASE_EXPORT BoundedPageAllocator : public v8::PageAllocator {
   bool SetPermissions(void* address, size_t size,
                       PageAllocator::Permission access) override;
 
+  bool DiscardSystemPages(void* address, size_t size) override;
+
  private:
   v8::base::Mutex mutex_;
   const size_t allocate_page_size_;

src/base/page-allocator.cc

@@ -58,5 +58,9 @@ bool PageAllocator::SetPermissions(void* address, size_t size,
       address, size, static_cast<base::OS::MemoryPermission>(access));
 }
 
+bool PageAllocator::DiscardSystemPages(void* address, size_t size) {
+  return base::OS::DiscardSystemPages(address, size);
+}
+
 }  // namespace base
 }  // namespace v8

src/base/page-allocator.h

@@ -36,6 +36,8 @@ class V8_BASE_EXPORT PageAllocator
   bool SetPermissions(void* address, size_t size,
                       PageAllocator::Permission access) override;
 
+  bool DiscardSystemPages(void* address, size_t size) override;
+
  private:
   const size_t allocate_page_size_;
   const size_t commit_page_size_;

src/base/platform/platform-cygwin.cc

@@ -171,6 +171,33 @@ bool OS::SetPermissions(void* address, size_t size, MemoryPermission access) {
   return VirtualAlloc(address, size, MEM_COMMIT, protect) != nullptr;
 }
 
+// static
+bool OS::DiscardSystemPages(void* address, size_t size) {
+  // On Windows, discarded pages are not returned to the system immediately and
+  // not guaranteed to be zeroed when returned to the application.
+  using DiscardVirtualMemoryFunction =
+      DWORD(WINAPI*)(PVOID virtualAddress, SIZE_T size);
+  static std::atomic<DiscardVirtualMemoryFunction> discard_virtual_memory(
+      reinterpret_cast<DiscardVirtualMemoryFunction>(-1));
+  if (discard_virtual_memory ==
+      reinterpret_cast<DiscardVirtualMemoryFunction>(-1))
+    discard_virtual_memory =
+        reinterpret_cast<DiscardVirtualMemoryFunction>(GetProcAddress(
+            GetModuleHandle(L"Kernel32.dll"), "DiscardVirtualMemory"));
+  // Use DiscardVirtualMemory when available because it releases faster than
+  // MEM_RESET.
+  DiscardVirtualMemoryFunction discard_function = discard_virtual_memory.load();
+  if (discard_function) {
+    DWORD ret = discard_function(address, size);
+    if (!ret) return true;
+  }
+  // DiscardVirtualMemory is buggy in Win10 SP0, so fall back to MEM_RESET on
+  // failure.
+  void* ptr = VirtualAlloc(address, size, MEM_RESET, PAGE_READWRITE);
+  CHECK(ptr);
+  return ptr;
+}
+
 // static
 bool OS::HasLazyCommits() {
   // TODO(alph): implement for the platform.

src/base/platform/platform-fuchsia.cc

@@ -119,6 +119,12 @@ bool OS::SetPermissions(void* address, size_t size, MemoryPermission access) {
                          reinterpret_cast<uintptr_t>(address), size) == ZX_OK;
 }
 
+// static
+bool OS::DiscardSystemPages(void* address, size_t size) {
+  // TODO(hpayer): Does Fuchsia have madvise?
+  return true;
+}
+
 // static
 bool OS::HasLazyCommits() {
   // TODO(scottmg): Port, https://crbug.com/731217.

src/base/platform/platform-posix.cc

@@ -145,32 +145,6 @@ void* Allocate(void* address, size_t size, OS::MemoryPermission access) {
   return result;
 }
 
-int ReclaimInaccessibleMemory(void* address, size_t size) {
-#if defined(OS_MACOSX)
-  // On OSX, MADV_FREE_REUSABLE has comparable behavior to MADV_FREE, but also
-  // marks the pages with the reusable bit, which allows both Activity Monitor
-  // and memory-infra to correctly track the pages.
-  int ret = madvise(address, size, MADV_FREE_REUSABLE);
-#elif defined(_AIX) || defined(V8_OS_SOLARIS)
-  int ret = madvise(reinterpret_cast<caddr_t>(address), size, MADV_FREE);
-#else
-  int ret = madvise(address, size, MADV_FREE);
-#endif
-  if (ret != 0 && errno == ENOSYS)
-    return 0;  // madvise is not available on all systems.
-  if (ret != 0 && errno == EINVAL) {
-    // MADV_FREE only works on Linux 4.5+ . If request failed, retry with older
-    // MADV_DONTNEED . Note that MADV_FREE being defined at compile time doesn't
-    // imply runtime support.
-#if defined(_AIX) || defined(V8_OS_SOLARIS)
-    ret = madvise(reinterpret_cast<caddr_t>(address), size, MADV_DONTNEED);
-#else
-    ret = madvise(address, size, MADV_DONTNEED);
-#endif
-  }
-  return ret;
-}
-
 #endif  // !V8_OS_FUCHSIA
 
 }  // namespace
@@ -352,7 +326,7 @@ bool OS::SetPermissions(void* address, size_t size, MemoryPermission access) {
   int ret = mprotect(address, size, prot);
   if (ret == 0 && access == OS::MemoryPermission::kNoAccess) {
     // This is advisory; ignore errors and continue execution.
-    ReclaimInaccessibleMemory(address, size);
+    USE(DiscardSystemPages(address, size));
   }
 
 // For accounting purposes, we want to call MADV_FREE_REUSE on macOS after
@@ -369,6 +343,34 @@ bool OS::SetPermissions(void* address, size_t size, MemoryPermission access) {
   return ret == 0;
 }
 
+bool OS::DiscardSystemPages(void* address, size_t size) {
+  DCHECK_EQ(0, reinterpret_cast<uintptr_t>(address) % CommitPageSize());
+  DCHECK_EQ(0, size % CommitPageSize());
+#if defined(OS_MACOSX)
+  // On OSX, MADV_FREE_REUSABLE has comparable behavior to MADV_FREE, but also
+  // marks the pages with the reusable bit, which allows both Activity Monitor
+  // and memory-infra to correctly track the pages.
+  int ret = madvise(address, size, MADV_FREE_REUSABLE);
+#elif defined(_AIX) || defined(V8_OS_SOLARIS)
+  int ret = madvise(reinterpret_cast<caddr_t>(address), size, MADV_FREE);
+#else
+  int ret = madvise(address, size, MADV_FREE);
+#endif
+  if (ret != 0 && errno == ENOSYS)
+    return true;  // madvise is not available on all systems.
+  if (ret != 0 && errno == EINVAL) {
+// MADV_FREE only works on Linux 4.5+ . If request failed, retry with older
+// MADV_DONTNEED . Note that MADV_FREE being defined at compile time doesn't
+// imply runtime support.
+#if defined(_AIX) || defined(V8_OS_SOLARIS)
+    ret = madvise(reinterpret_cast<caddr_t>(address), size, MADV_DONTNEED);
+#else
+    ret = madvise(address, size, MADV_DONTNEED);
+#endif
+  }
+  return ret == 0;
+}
+
 // static
 bool OS::HasLazyCommits() {
 #if V8_OS_AIX || V8_OS_LINUX || V8_OS_MACOSX

src/base/platform/platform-win32.cc

@@ -882,6 +882,33 @@ bool OS::SetPermissions(void* address, size_t size, MemoryPermission access) {
   return VirtualAlloc(address, size, MEM_COMMIT, protect) != nullptr;
 }
 
+// static
+bool OS::DiscardSystemPages(void* address, size_t size) {
+  // On Windows, discarded pages are not returned to the system immediately and
+  // not guaranteed to be zeroed when returned to the application.
+  using DiscardVirtualMemoryFunction =
+      DWORD(WINAPI*)(PVOID virtualAddress, SIZE_T size);
+  static std::atomic<DiscardVirtualMemoryFunction> discard_virtual_memory(
+      reinterpret_cast<DiscardVirtualMemoryFunction>(-1));
+  if (discard_virtual_memory ==
+      reinterpret_cast<DiscardVirtualMemoryFunction>(-1))
+    discard_virtual_memory =
+        reinterpret_cast<DiscardVirtualMemoryFunction>(GetProcAddress(
+            GetModuleHandle(L"Kernel32.dll"), "DiscardVirtualMemory"));
+  // Use DiscardVirtualMemory when available because it releases faster than
+  // MEM_RESET.
+  DiscardVirtualMemoryFunction discard_function = discard_virtual_memory.load();
+  if (discard_function) {
+    DWORD ret = discard_function(address, size);
+    if (!ret) return true;
+  }
+  // DiscardVirtualMemory is buggy in Win10 SP0, so fall back to MEM_RESET on
+  // failure.
+  void* ptr = VirtualAlloc(address, size, MEM_RESET, PAGE_READWRITE);
+  CHECK(ptr);
+  return ptr;
+}
+
 // static
 bool OS::HasLazyCommits() {
   // TODO(alph): implement for the platform.

src/base/platform/platform.h

@@ -273,6 +273,9 @@ class V8_BASE_EXPORT OS {
   V8_WARN_UNUSED_RESULT static bool SetPermissions(void* address, size_t size,
                                                    MemoryPermission access);
 
+  V8_WARN_UNUSED_RESULT static bool DiscardSystemPages(void* address,
+                                                       size_t size);
+
   static const int msPerSecond = 1000;
 
 #if V8_OS_POSIX

src/heap/spaces.cc

@@ -446,6 +446,18 @@ Address MemoryAllocator::AllocateAlignedMemory(
   return base;
 }
 
+void MemoryChunk::DiscardUnusedMemory(Address addr, size_t size) {
+  base::AddressRegion memory_area =
+      MemoryAllocator::ComputeDiscardMemoryArea(addr, size);
+  if (memory_area.size() != 0) {
+    MemoryAllocator* memory_allocator = heap_->memory_allocator();
+    v8::PageAllocator* page_allocator =
+        memory_allocator->page_allocator(executable());
+    CHECK(page_allocator->DiscardSystemPages(
+        reinterpret_cast<void*>(memory_area.begin()), memory_area.size()));
+  }
+}
+
 Heap* MemoryChunk::synchronized_heap() {
   return reinterpret_cast<Heap*>(
       base::Acquire_Load(reinterpret_cast<base::AtomicWord*>(&heap_)));
@@ -1129,6 +1141,19 @@ intptr_t MemoryAllocator::GetCommitPageSize() {
   }
 }
 
+base::AddressRegion MemoryAllocator::ComputeDiscardMemoryArea(Address addr,
+                                                              size_t size) {
+  size_t page_size = MemoryAllocator::GetCommitPageSize();
+  if (size < page_size + FreeSpace::kSize) {
+    return base::AddressRegion(0, 0);
+  }
+  Address discardable_start = RoundUp(addr + FreeSpace::kSize, page_size);
+  Address discardable_end = RoundDown(addr + size, page_size);
+  if (discardable_start >= discardable_end) return base::AddressRegion(0, 0);
+  return base::AddressRegion(discardable_start,
+                             discardable_end - discardable_start);
+}
+
 bool MemoryAllocator::CommitExecutableMemory(VirtualMemory* vm, Address start,
                                              size_t commit_size,
                                              size_t reserved_size) {

src/heap/spaces.h

@@ -431,6 +431,8 @@ class MemoryChunk {
       ExternalBackingStoreType type, MemoryChunk* from, MemoryChunk* to,
       size_t amount);
 
+  void DiscardUnusedMemory(Address addr, size_t size);
+
   Address address() const {
     return reinterpret_cast<Address>(const_cast<MemoryChunk*>(this));
   }
@@ -1287,6 +1289,12 @@ class V8_EXPORT_PRIVATE MemoryAllocator {
 
   static intptr_t GetCommitPageSize();
 
+  // Computes the memory area of discardable memory within a given memory area
+  // [addr, addr+size) and returns the result as base::AddressRegion. If the
+  // memory is not discardable base::AddressRegion is an empty region.
+  static base::AddressRegion ComputeDiscardMemoryArea(Address addr,
+                                                      size_t size);
+
   MemoryAllocator(Isolate* isolate, size_t max_capacity,
                   size_t code_range_size);
 

src/heap/sweeper.cc

@@ -289,6 +289,7 @@ int Sweeper::RawSweep(Page* p, FreeListRebuildingMode free_list_mode,
             free_start, static_cast<int>(size), ClearRecordedSlots::kNo,
             ClearFreedMemoryMode::kClearFreedMemory);
       }
+      p->DiscardUnusedMemory(free_start, size);
       RememberedSet<OLD_TO_NEW>::RemoveRange(p, free_start, free_end,
                                              SlotSet::KEEP_EMPTY_BUCKETS);
       RememberedSet<OLD_TO_OLD>::RemoveRange(p, free_start, free_end,
@@ -329,7 +330,7 @@ int Sweeper::RawSweep(Page* p, FreeListRebuildingMode free_list_mode,
                                       ClearRecordedSlots::kNo,
                                       ClearFreedMemoryMode::kClearFreedMemory);
     }
-
+    p->DiscardUnusedMemory(free_start, size);
     RememberedSet<OLD_TO_NEW>::RemoveRange(p, free_start, p->area_end(),
                                            SlotSet::KEEP_EMPTY_BUCKETS);
     RememberedSet<OLD_TO_OLD>::RemoveRange(p, free_start, p->area_end(),

test/cctest/heap/test-spaces.cc

@@ -213,6 +213,44 @@ TEST(MemoryAllocator) {
   delete memory_allocator;
 }
 
+TEST(ComputeDiscardMemoryAreas) {
+  base::AddressRegion memory_area;
+  size_t page_size = MemoryAllocator::GetCommitPageSize();
+  size_t free_header_size = FreeSpace::kSize;
+
+  memory_area = MemoryAllocator::ComputeDiscardMemoryArea(0, 0);
+  CHECK_EQ(memory_area.begin(), 0);
+  CHECK_EQ(memory_area.size(), 0);
+
+  memory_area = MemoryAllocator::ComputeDiscardMemoryArea(
+      0, page_size + free_header_size);
+  CHECK_EQ(memory_area.begin(), 0);
+  CHECK_EQ(memory_area.size(), 0);
+
+  memory_area = MemoryAllocator::ComputeDiscardMemoryArea(
+      page_size - free_header_size, page_size + free_header_size);
+  CHECK_EQ(memory_area.begin(), page_size);
+  CHECK_EQ(memory_area.size(), page_size);
+
+  memory_area = MemoryAllocator::ComputeDiscardMemoryArea(page_size, page_size);
+  CHECK_EQ(memory_area.begin(), 0);
+  CHECK_EQ(memory_area.size(), 0);
+
+  memory_area = MemoryAllocator::ComputeDiscardMemoryArea(
+      page_size / 2, page_size + page_size / 2);
+  CHECK_EQ(memory_area.begin(), page_size);
+  CHECK_EQ(memory_area.size(), page_size);
+
+  memory_area = MemoryAllocator::ComputeDiscardMemoryArea(
+      page_size / 2, page_size + page_size / 4);
+  CHECK_EQ(memory_area.begin(), 0);
+  CHECK_EQ(memory_area.size(), 0);
+
+  memory_area =
+      MemoryAllocator::ComputeDiscardMemoryArea(page_size / 2, page_size * 3);
+  CHECK_EQ(memory_area.begin(), page_size);
+  CHECK_EQ(memory_area.size(), page_size * 2);
+}
 
 TEST(NewSpace) {
   Isolate* isolate = CcTest::i_isolate();