[heap] Add conservative stack scanning

When enabled with the v8_enable_conservative_stack_scanning flag, a
snapshot of the call stack upon entry to GC is used to determine part of
the root-set. When the collector walks the stack, it looks at each value
and determines whether it could be a potential on-heap object pointer.

This is very experimental. For conservative stack scanning to work,
direct handles must be implemented.

Bug: v8:10614
Change-Id: Id4209cfbe76ef02239c903fabcb7f677b32fc977
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2375201
Commit-Queue: Anton Bikineev <bikineev@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Reviewed-by: Anton Bikineev <bikineev@chromium.org>
Reviewed-by: Igor Sheludko <ishell@chromium.org>
Cr-Commit-Position: refs/heads/master@{#69644}

BUILD.gn

@@ -3363,7 +3363,11 @@ v8_source_set("v8_base_without_compiler") {
   }
 
   if (v8_enable_conservative_stack_scanning) {
-    sources += [ "src/heap/object-start-bitmap.h" ]
+    sources += [
+      "src/heap/conservative-stack-visitor.cc",
+      "src/heap/conservative-stack-visitor.h",
+      "src/heap/object-start-bitmap.h",
+    ]
   }
 
   if (v8_enable_wasm_gdb_remote_debugging) {
@@ -3704,6 +3708,7 @@ v8_source_set("v8_base_without_compiler") {
   deps = [
     ":cppgc_base",
     ":torque_generated_definitions",
+    ":v8_cppgc_shared",
     ":v8_headers",
     ":v8_libbase",
     ":v8_libsampler",

src/DEPS

@@ -28,6 +28,8 @@ include_rules = [
   "+src/heap/read-only-heap-inl.h",
   "+src/heap/read-only-heap.h",
   "+src/heap/safepoint.h",
+  "+src/heap/base/stack.h",
+  "+src/heap/conservative-stack-visitor.h",
   "-src/inspector",
   "-src/interpreter",
   "+src/interpreter/bytecode-array-accessor.h",

src/execution/isolate.cc

@@ -99,6 +99,10 @@
 #include "src/diagnostics/unwinding-info-win64.h"
 #endif  // V8_OS_WIN64
 
+#ifdef V8_ENABLE_CONSERVATIVE_STACK_SCANNING
+#include "src/heap/conservative-stack-visitor.h"
+#endif
+
 extern "C" const uint8_t* v8_Default_embedded_blob_code_;
 extern "C" uint32_t v8_Default_embedded_blob_code_size_;
 extern "C" const uint8_t* v8_Default_embedded_blob_metadata_;
@@ -505,6 +509,11 @@ void Isolate::Iterate(RootVisitor* v, ThreadLocalTop* thread) {
         FullObjectSlot(reinterpret_cast<Address>(&(block->message_obj_))));
   }
 
+#ifdef V8_ENABLE_CONSERVATIVE_STACK_SCANNING
+  ConservativeStackVisitor stack_visitor(this, v);
+  thread_local_top()->stack_.IteratePointers(&stack_visitor);
+#endif
+
   // Iterate over pointers on native execution stack.
   wasm::WasmCodeRefScope wasm_code_ref_scope;
   for (StackFrameIterator it(this, thread); !it.done(); it.Advance()) {

src/execution/thread-local-top.cc

@@ -19,6 +19,10 @@ void ThreadLocalTop::Initialize(Isolate* isolate) {
   thread_id_ = ThreadId::Current();
   thread_in_wasm_flag_address_ = reinterpret_cast<Address>(
       trap_handler::GetThreadInWasmThreadLocalAddress());
+
+#ifdef V8_ENABLE_CONSERVATIVE_STACK_SCANNING
+  stack_ = ::heap::base::Stack(base::Stack::GetStackStart());
+#endif
 }
 
 void ThreadLocalTop::Free() {

src/execution/thread-local-top.h

@@ -10,6 +10,10 @@
 #include "src/objects/contexts.h"
 #include "src/utils/utils.h"
 
+#ifdef V8_ENABLE_CONSERVATIVE_STACK_SCANNING
+#include "src/heap/base/stack.h"
+#endif
+
 namespace v8 {
 
 class TryCatch;
@@ -26,7 +30,11 @@ class ThreadLocalTop {
   // TODO(all): This is not particularly beautiful. We should probably
   // refactor this to really consist of just Addresses and 32-bit
   // integer fields.
+#ifdef V8_ENABLE_CONSERVATIVE_STACK_SCANNING
+  static constexpr uint32_t kSizeInBytes = 25 * kSystemPointerSize;
+#else
   static constexpr uint32_t kSizeInBytes = 24 * kSystemPointerSize;
+#endif
 
   // Does early low-level initialization that does not depend on the
   // isolate being present.
@@ -142,6 +150,10 @@ class ThreadLocalTop {
 
   // Address of the thread-local "thread in wasm" flag.
   Address thread_in_wasm_flag_address_ = kNullAddress;
+
+#ifdef V8_ENABLE_CONSERVATIVE_STACK_SCANNING
+  ::heap::base::Stack stack_ = ::heap::base::Stack(nullptr);
+#endif
 };
 
 }  // namespace internal

src/heap/conservative-stack-visitor.cc

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/heap/conservative-stack-visitor.h"
+
+#include "src/execution/isolate-utils-inl.h"
+#include "src/heap/large-spaces.h"
+#include "src/heap/paged-spaces-inl.h"
+
+namespace v8 {
+namespace internal {
+
+ConservativeStackVisitor::ConservativeStackVisitor(Isolate* isolate,
+                                                   RootVisitor* delegate)
+    : isolate_(isolate), delegate_(delegate) {}
+
+void ConservativeStackVisitor::VisitPointer(const void* pointer) {
+  VisitConservativelyIfPointer(pointer);
+}
+
+bool ConservativeStackVisitor::CheckPage(Address address, MemoryChunk* page) {
+  if (address < page->area_start() || address >= page->area_end()) return false;
+
+  auto base_ptr = page->object_start_bitmap()->FindBasePtr(address);
+  if (base_ptr == kNullAddress) {
+    return false;
+  }
+
+  // At this point, base_ptr *must* refer to the valid object. We check if
+  // |address| resides inside the object or beyond it in unused memory.
+  HeapObject obj = HeapObject::FromAddress(base_ptr);
+  Address obj_end = obj.address() + obj.Size();
+
+  if (address >= obj_end) {
+    // |address| points to unused memory.
+    return false;
+  }
+
+  // TODO(jakehughes) Pinning is only required for the marking visitor. Other
+  // visitors (such as verify visitor) could work without pining. This should
+  // be moved to delegate_
+  page->SetFlag(BasicMemoryChunk::Flag::PINNED);
+
+  Object ptr = HeapObject::FromAddress(base_ptr);
+  FullObjectSlot root = FullObjectSlot(&ptr);
+  delegate_->VisitRootPointer(Root::kHandleScope, nullptr, root);
+  DCHECK(root == FullObjectSlot(reinterpret_cast<Address>(&base_ptr)));
+  return true;
+}
+
+void ConservativeStackVisitor::VisitConservativelyIfPointer(
+    const void* pointer) {
+  auto address = reinterpret_cast<Address>(pointer);
+  if (address > isolate_->heap()->old_space()->top() ||
+      address < isolate_->heap()->old_space()->limit()) {
+    return;
+  }
+
+  for (Page* page : *isolate_->heap()->old_space()) {
+    if (CheckPage(address, page)) {
+      return;
+    }
+  }
+
+  for (LargePage* page : *isolate_->heap()->lo_space()) {
+    if (address >= page->area_start() && address < page->area_end()) {
+      Object ptr = page->GetObject();
+      FullObjectSlot root = FullObjectSlot(&ptr);
+      delegate_->VisitRootPointer(Root::kHandleScope, nullptr, root);
+      DCHECK(root == FullObjectSlot(&ptr));
+      return;
+    }
+  }
+}
+
+}  // namespace internal
+}  // namespace v8

src/heap/conservative-stack-visitor.h

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_HEAP_CONSERVATIVE_STACK_VISITOR_H_
+#define V8_HEAP_CONSERVATIVE_STACK_VISITOR_H_
+
+#include "src/heap/base/stack.h"
+#include "src/heap/memory-chunk.h"
+
+namespace v8 {
+namespace internal {
+
+class ConservativeStackVisitor : public ::heap::base::StackVisitor {
+ public:
+  ConservativeStackVisitor(Isolate* isolate, RootVisitor* delegate);
+
+  void VisitPointer(const void* pointer) final;
+
+ private:
+  bool CheckPage(Address address, MemoryChunk* page);
+
+  void VisitConservativelyIfPointer(const void* pointer);
+
+  Isolate* isolate_ = nullptr;
+  RootVisitor* delegate_ = nullptr;
+};
+
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_HEAP_CONSERVATIVE_STACK_VISITOR_H_

src/heap/heap.cc

@@ -31,6 +31,7 @@
 #include "src/handles/global-handles.h"
 #include "src/heap/array-buffer-sweeper.h"
 #include "src/heap/barrier.h"
+#include "src/heap/base/stack.h"
 #include "src/heap/code-object-registry.h"
 #include "src/heap/code-stats.h"
 #include "src/heap/combined-heap.h"
@@ -89,6 +90,10 @@
 #include "src/utils/utils-inl.h"
 #include "src/utils/utils.h"
 
+#ifdef V8_ENABLE_CONSERVATIVE_STACK_SCANNING
+#include "src/heap/conservative-stack-visitor.h"
+#endif
+
 // Has to be the last include (doesn't have include guards):
 #include "src/objects/object-macros.h"
 
@@ -4534,8 +4539,10 @@ void Heap::IterateRoots(RootVisitor* v, base::EnumSet<SkipRoot> options) {
 
     // Iterate over local handles in handle scopes.
     FixStaleLeftTrimmedHandlesVisitor left_trim_visitor(this);
+#ifndef V8_ENABLE_CONSERVATIVE_STACK_SCANNING
     isolate_->handle_scope_implementer()->Iterate(&left_trim_visitor);
     isolate_->handle_scope_implementer()->Iterate(v);
+#endif
 
     if (FLAG_local_heaps) {
       safepoint_->Iterate(&left_trim_visitor);

src/heap/mark-compact.cc

@@ -750,6 +750,11 @@ void MarkCompactCollector::CollectEvacuationCandidates(PagedSpace* space) {
     } else {
       pages.push_back(std::make_pair(p->allocated_bytes(), p));
     }
+
+    // Unpin pages for the next GC
+    if (p->IsFlagSet(MemoryChunk::PINNED)) {
+      p->ClearFlag(MemoryChunk::PINNED);
+    }
   }
 
   int candidate_count = 0;

src/heap/object-start-bitmap.h

@@ -105,6 +105,10 @@ Address ObjectStartBitmap::FindBasePtr(Address maybe_inner_ptr) const {
     byte = load(--cell_index);
   }
   const int leading_zeroes = v8::base::bits::CountLeadingZeros(byte);
+  if (leading_zeroes == kBitsPerCell) {
+    return kNullAddress;
+  }
+
   object_start_number =
       (cell_index * kBitsPerCell) + (kBitsPerCell - 1) - leading_zeroes;
   Address base_ptr = StartIndexToAddress(object_start_number);