Reland "cppgc,heap: Implement atomic unified heap GC"

This is a reland of 539f0ed2

The reland fixes creating TimeDelta from double which requires
saturated_cast<>. Improvements to this constructions are tracked
in v8:10620.

Original change's description:
> cppgc,heap: Implement atomic unified heap GC
>
> Add v8::CppHeap as an implementation of a cppgc heap that
> integrates with V8's existing EmbedderHeapTracer API. The
> current implementation only supports non-incremental marking.
>
> Bug: chromium:1056170
> Change-Id: I4a09eb5ae57f5c7defe35eb3fe346627eb492473
> Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2245610
> Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
> Reviewed-by: Anton Bikineev <bikineev@chromium.org>
> Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
> Reviewed-by: Omer Katz <omerkatz@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#68374}

Bug: chromium:1056170,v8:10620
Change-Id: I39e15790e5cafe24da2a14d0bae6543391ebb536
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2248191Reviewed-by: Omer Katz <omerkatz@chromium.org>
Reviewed-by: Anton Bikineev <bikineev@chromium.org>
Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Commit-Queue: Michael Lippautz <mlippautz@chromium.org>
Cr-Commit-Position: refs/heads/master@{#68387}

BUILD.gn

@@ -2465,6 +2465,8 @@ v8_source_set("v8_base_without_compiler") {
     "src/heap/concurrent-allocator.h",
     "src/heap/concurrent-marking.cc",
     "src/heap/concurrent-marking.h",
+    "src/heap/cppgc-js/cpp-heap.cc",
+    "src/heap/cppgc-js/cpp-heap.h",
     "src/heap/embedder-tracing.cc",
     "src/heap/embedder-tracing.h",
     "src/heap/factory-base.cc",
@@ -3577,10 +3579,14 @@ v8_source_set("v8_base_without_compiler") {
     ]
   }
 
-  configs = [ ":internal_config" ]
+  configs = [
+    ":internal_config",
+    ":cppgc_base_config",
+  ]
 
   defines = []
   deps = [
+    ":cppgc_base",
     ":torque_generated_definitions",
     ":v8_headers",
     ":v8_libbase",
@@ -4206,29 +4212,29 @@ v8_source_set("cppgc_base") {
   }
 
   if (is_clang || !is_win) {
-    if (target_cpu == "x64") {
+    if (current_cpu == "x64") {
       sources += [ "src/heap/cppgc/asm/x64/push_registers_asm.cc" ]
-    } else if (target_cpu == "x86") {
+    } else if (current_cpu == "x86") {
       sources += [ "src/heap/cppgc/asm/ia32/push_registers_asm.cc" ]
-    } else if (target_cpu == "arm") {
+    } else if (current_cpu == "arm") {
       sources += [ "src/heap/cppgc/asm/arm/push_registers_asm.cc" ]
-    } else if (target_cpu == "arm64") {
+    } else if (current_cpu == "arm64") {
       sources += [ "src/heap/cppgc/asm/arm64/push_registers_asm.cc" ]
-    } else if (target_cpu == "ppc64") {
+    } else if (current_cpu == "ppc64") {
       sources += [ "src/heap/cppgc/asm/ppc/push_registers_asm.cc" ]
-    } else if (target_cpu == "s390x") {
+    } else if (current_cpu == "s390x") {
       sources += [ "src/heap/cppgc/asm/s390/push_registers_asm.cc" ]
-    } else if (target_cpu == "mipsel") {
+    } else if (current_cpu == "mipsel") {
       sources += [ "src/heap/cppgc/asm/mips/push_registers_asm.cc" ]
-    } else if (target_cpu == "mips64el") {
+    } else if (current_cpu == "mips64el") {
       sources += [ "src/heap/cppgc/asm/mips64/push_registers_asm.cc" ]
     }
   } else if (is_win) {
-    if (target_cpu == "x64") {
+    if (current_cpu == "x64") {
       sources += [ "src/heap/cppgc/asm/x64/push_registers_masm.S" ]
-    } else if (target_cpu == "x86") {
+    } else if (current_cpu == "x86") {
       sources += [ "src/heap/cppgc/asm/ia32/push_registers_masm.S" ]
-    } else if (target_cpu == "arm64") {
+    } else if (current_cpu == "arm64") {
       sources += [ "src/heap/cppgc/asm/arm64/push_registers_masm.S" ]
     }
   }

src/base/platform/time.h

@@ -14,6 +14,7 @@
 #include "src/base/base-export.h"
 #include "src/base/bits.h"
 #include "src/base/macros.h"
+#include "src/base/safe_conversions.h"
 #if V8_OS_WIN
 #include "src/base/win32-headers.h"
 #endif
@@ -90,6 +91,11 @@ class V8_BASE_EXPORT TimeDelta final {
     return TimeDelta(nanoseconds / TimeConstants::kNanosecondsPerMicrosecond);
   }
 
+  static TimeDelta FromMillisecondsD(double milliseconds) {
+    return FromDouble(milliseconds *
+                      TimeConstants::kMicrosecondsPerMillisecond);
+  }
+
   // Returns the maximum time delta, which should be greater than any reasonable
   // time delta we might compare it to. Adding or subtracting the maximum time
   // delta to a time or another time delta has an undefined result.
@@ -201,6 +207,9 @@ class V8_BASE_EXPORT TimeDelta final {
   }
 
  private:
+  // TODO(v8:10620): constexpr requires constexpr saturated_cast.
+  static inline TimeDelta FromDouble(double value);
+
   template<class TimeClass> friend class time_internal::TimeBase;
   // Constructs a delta given the duration in microseconds. This is private
   // to avoid confusion by callers with an integer constructor. Use
@@ -211,6 +220,11 @@ class V8_BASE_EXPORT TimeDelta final {
   int64_t delta_;
 };
 
+// static
+TimeDelta TimeDelta::FromDouble(double value) {
+  return TimeDelta(saturated_cast<int64_t>(value));
+}
+
 // static
 constexpr TimeDelta TimeDelta::Max() {
   return TimeDelta(std::numeric_limits<int64_t>::max());

src/heap/cppgc-js/cpp-heap.cc

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/heap/cppgc-js/cpp-heap.h"
+
+#include "include/cppgc/platform.h"
+#include "include/v8-platform.h"
+#include "include/v8.h"
+#include "src/base/macros.h"
+#include "src/base/platform/time.h"
+#include "src/flags/flags.h"
+#include "src/heap/cppgc/gc-info-table.h"
+#include "src/heap/cppgc/heap-base.h"
+#include "src/heap/cppgc/heap-object-header-inl.h"
+#include "src/heap/cppgc/marker.h"
+#include "src/heap/cppgc/marking-visitor.h"
+#include "src/heap/cppgc/object-allocator.h"
+#include "src/heap/cppgc/prefinalizer-handler.h"
+#include "src/heap/cppgc/stats-collector.h"
+#include "src/heap/cppgc/sweeper.h"
+#include "src/heap/marking-worklist.h"
+#include "src/heap/sweeper.h"
+#include "src/init/v8.h"
+
+namespace v8 {
+namespace internal {
+
+namespace {
+
+class CppgcPlatformAdapter final : public cppgc::Platform {
+ public:
+  explicit CppgcPlatformAdapter(v8::Isolate* isolate)
+      : platform_(V8::GetCurrentPlatform()), isolate_(isolate) {}
+
+  CppgcPlatformAdapter(const CppgcPlatformAdapter&) = delete;
+  CppgcPlatformAdapter& operator=(const CppgcPlatformAdapter&) = delete;
+
+  PageAllocator* GetPageAllocator() final {
+    return platform_->GetPageAllocator();
+  }
+
+  double MonotonicallyIncreasingTime() final {
+    return platform_->MonotonicallyIncreasingTime();
+  }
+
+  std::shared_ptr<TaskRunner> GetForegroundTaskRunner() final {
+    return platform_->GetForegroundTaskRunner(isolate_);
+  }
+
+  std::unique_ptr<JobHandle> PostJob(TaskPriority priority,
+                                     std::unique_ptr<JobTask> job_task) final {
+    return platform_->PostJob(priority, std::move(job_task));
+  }
+
+ private:
+  v8::Platform* platform_;
+  v8::Isolate* isolate_;
+};
+
+class UnifiedHeapMarker : public cppgc::internal::Marker {
+ public:
+  explicit UnifiedHeapMarker(
+      cppgc::internal::HeapBase& heap);  // NOLINT(runtime/references)
+
+  void AddObject(void*);
+
+  // TODO(chromium:1056170): Implement unified heap specific
+  // CreateMutatorThreadMarkingVisitor and AdvanceMarkingWithDeadline.
+};
+
+UnifiedHeapMarker::UnifiedHeapMarker(cppgc::internal::HeapBase& heap)
+    : cppgc::internal::Marker(heap) {}
+
+void UnifiedHeapMarker::AddObject(void* object) {
+  auto& header = cppgc::internal::HeapObjectHeader::FromPayload(object);
+  marking_visitor_->MarkObject(header);
+}
+
+}  // namespace
+
+CppHeap::CppHeap(v8::Isolate* isolate, size_t custom_spaces)
+    : cppgc::internal::HeapBase(std::make_shared<CppgcPlatformAdapter>(isolate),
+                                custom_spaces) {
+  CHECK(!FLAG_incremental_marking_wrappers);
+}
+
+void CppHeap::RegisterV8References(
+    const std::vector<std::pair<void*, void*> >& embedder_fields) {
+  DCHECK(marker_);
+  for (auto& tuple : embedder_fields) {
+    // First field points to type.
+    // Second field points to object.
+    static_cast<UnifiedHeapMarker*>(marker_.get())->AddObject(tuple.second);
+  }
+  marking_done_ = false;
+}
+
+void CppHeap::TracePrologue(TraceFlags flags) {
+  marker_ = std::make_unique<UnifiedHeapMarker>(AsBase());
+  const UnifiedHeapMarker::MarkingConfig marking_config{
+      UnifiedHeapMarker::MarkingConfig::CollectionType::kMajor,
+      cppgc::Heap::StackState::kNoHeapPointers,
+      UnifiedHeapMarker::MarkingConfig::MarkingType::kAtomic};
+  marker_->StartMarking(marking_config);
+  marking_done_ = false;
+}
+
+bool CppHeap::AdvanceTracing(double deadline_in_ms) {
+  marking_done_ = marker_->AdvanceMarkingWithDeadline(
+      v8::base::TimeDelta::FromMillisecondsD(deadline_in_ms));
+  return marking_done_;
+}
+
+bool CppHeap::IsTracingDone() { return marking_done_; }
+
+void CppHeap::EnterFinalPause(EmbedderStackState stack_state) {
+  const UnifiedHeapMarker::MarkingConfig marking_config{
+      UnifiedHeapMarker::MarkingConfig::CollectionType::kMajor,
+      cppgc::Heap::StackState::kNoHeapPointers,
+      UnifiedHeapMarker::MarkingConfig::MarkingType::kAtomic};
+  marker_->EnterAtomicPause(marking_config);
+}
+
+void CppHeap::TraceEpilogue(TraceSummary* trace_summary) {
+  CHECK(marking_done_);
+  marker_->LeaveAtomicPause();
+  {
+    // Pre finalizers are forbidden from allocating objects
+    cppgc::internal::ObjectAllocator::NoAllocationScope no_allocation_scope_(
+        object_allocator_);
+    marker()->ProcessWeakness();
+    prefinalizer_handler()->InvokePreFinalizers();
+  }
+  {
+    NoGCScope no_gc(*this);
+    sweeper().Start(cppgc::internal::Sweeper::Config::kAtomic);
+  }
+}
+
+}  // namespace internal
+}  // namespace v8

src/heap/cppgc-js/cpp-heap.h

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_HEAP_CPPGC_JS_CPP_HEAP_H_
+#define V8_HEAP_CPPGC_JS_CPP_HEAP_H_
+
+#include "include/v8.h"
+#include "src/base/macros.h"
+#include "src/heap/cppgc/heap-base.h"
+
+namespace v8 {
+
+class Isolate;
+
+namespace internal {
+
+// A C++ heap implementation used with V8 to implement unified heap.
+class V8_EXPORT_PRIVATE CppHeap final : public cppgc::internal::HeapBase,
+                                        public v8::EmbedderHeapTracer {
+ public:
+  CppHeap(v8::Isolate* isolate, size_t custom_spaces);
+
+  HeapBase& AsBase() { return *this; }
+  const HeapBase& AsBase() const { return *this; }
+
+  void RegisterV8References(
+      const std::vector<std::pair<void*, void*> >& embedder_fields) final;
+  void TracePrologue(TraceFlags flags) final;
+  bool AdvanceTracing(double deadline_in_ms) final;
+  bool IsTracingDone() final;
+  void TraceEpilogue(TraceSummary* trace_summary) final;
+  void EnterFinalPause(EmbedderStackState stack_state) final;
+
+ private:
+  bool marking_done_ = false;
+};
+
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_HEAP_CPPGC_JS_CPP_HEAP_H_

src/heap/cppgc/marker.cc

@@ -137,9 +137,8 @@ void Marker::EnterAtomicPause(MarkingConfig config) {
   ExitIncrementalMarkingIfNeeded(config_, heap());
   config_ = config;
 
-  // Reset LABs before trying to conservatively mark in-construction objects.
-  // This is also needed in preparation for sweeping.
-  heap().object_allocator().ResetLinearAllocationBuffers();
+  // VisitRoots also resets the LABs.
+  VisitRoots();
   if (config_.stack_state == MarkingConfig::StackState::kNoHeapPointers) {
     FlushNotFullyConstructedObjects();
   } else {

src/heap/cppgc/marker.h

@@ -83,6 +83,19 @@ class V8_EXPORT_PRIVATE Marker {
   // trigger incremental/concurrent marking if needed.
   void StartMarking(MarkingConfig config);
 
+  // Signals entering the atomic marking pause. The method
+  // - stops incremental/concurrent marking;
+  // - flushes back any in-construction worklists if needed;
+  // - Updates the MarkingConfig if the stack state has changed;
+  void EnterAtomicPause(MarkingConfig config);
+
+  // Makes marking progress.
+  virtual bool AdvanceMarkingWithDeadline(v8::base::TimeDelta);
+
+  // Signals leaving the atomic marking pause. This method expects no more
+  // objects to be marked and merely updates marking states if needed.
+  void LeaveAtomicPause();
+
   // Combines:
   // - EnterAtomicPause()
   // - AdvanceMarkingWithDeadline()
@@ -113,20 +126,6 @@ class V8_EXPORT_PRIVATE Marker {
   virtual std::unique_ptr<MutatorThreadMarkingVisitor>
   CreateMutatorThreadMarkingVisitor();
 
-  // Signals entering the atomic marking pause. The method
-  // - stops incremental/concurrent marking;
-  // - flushes back any in-construction worklists if needed;
-  // - Updates the MarkingConfig if the stack state has changed;
-  void EnterAtomicPause(MarkingConfig config);
-
-  // Makes marking progress.
-  virtual bool AdvanceMarkingWithDeadline(v8::base::TimeDelta);
-
-  // Signals leaving the atomic marking pause. This method expects no more
-  // objects to be marked and merely updates marking states if needed.
-  void LeaveAtomicPause();
-
- private:
   void VisitRoots();
 
   void FlushNotFullyConstructedObjects();

src/heap/cppgc/marking-visitor.cc

@@ -137,6 +137,13 @@ void MarkingVisitor::DynamicallyMarkAddress(ConstAddress address) {
   }
 }
 
+void MarkingVisitor::MarkObject(HeapObjectHeader& header) {
+  MarkHeader(
+      &header,
+      {header.Payload(),
+       GlobalGCInfoTable::GCInfoFromIndex(header.GetGCInfoIndex()).trace});
+}
+
 MutatorThreadMarkingVisitor::MutatorThreadMarkingVisitor(Marker* marker)
     : MarkingVisitor(marker->heap(), marker->marking_worklist(),
                      marker->not_fully_constructed_worklist(),

src/heap/cppgc/marking-visitor.h

@@ -34,6 +34,7 @@ class MarkingVisitor : public ConservativeTracingVisitor, public StackVisitor {
   void FlushWorklists();
 
   void DynamicallyMarkAddress(ConstAddress);
+  void MarkObject(HeapObjectHeader&);
 
   void AccountMarkedBytes(const HeapObjectHeader&);
   size_t marked_bytes() const { return marked_bytes_; }

src/heap/heap.h

@@ -1065,7 +1065,7 @@ class Heap {
     return local_embedder_heap_tracer_.get();
   }
 
-  void SetEmbedderHeapTracer(EmbedderHeapTracer* tracer);
+  V8_EXPORT_PRIVATE void SetEmbedderHeapTracer(EmbedderHeapTracer* tracer);
   EmbedderHeapTracer* GetEmbedderHeapTracer() const;
 
   void RegisterExternallyReferencedObject(Address* location);

test/cctest/BUILD.gn

@@ -394,6 +394,7 @@ v8_source_set("cctest_sources") {
   }
 
   configs = [
+    "../..:cppgc_base_config",
     "../..:external_config",
     "../..:internal_config_base",
     "../..:v8_tracing_config",

test/unittests/BUILD.gn

@@ -239,6 +239,7 @@ v8_source_set("unittests_sources") {
     "heap/gc-tracer-unittest.cc",
     "heap/heap-controller-unittest.cc",
     "heap/heap-unittest.cc",
+    "heap/heap-utils.h",
     "heap/item-parallel-job-unittest.cc",
     "heap/list-unittest.cc",
     "heap/local-heap-unittest.cc",
@@ -250,6 +251,7 @@ v8_source_set("unittests_sources") {
     "heap/safepoint-unittest.cc",
     "heap/slot-set-unittest.cc",
     "heap/spaces-unittest.cc",
+    "heap/unified-heap-unittest.cc",
     "heap/unmapper-unittest.cc",
     "heap/worklist-unittest.cc",
     "interpreter/bytecode-array-builder-unittest.cc",
@@ -380,6 +382,7 @@ v8_source_set("unittests_sources") {
   }
 
   configs = [
+    "../..:cppgc_base_config",
     "../..:external_config",
     "../..:internal_config_base",
   ]

test/unittests/heap/heap-utils.h

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_UNITTESTS_HEAP_HEAP_UTILS_H_
+#define V8_UNITTESTS_HEAP_HEAP_UTILS_H_
+
+#include "src/base/macros.h"
+#include "src/common/globals.h"
+#include "test/unittests/test-utils.h"
+#include "testing/gtest/include/gtest/gtest.h"
+
+namespace v8 {
+namespace internal {
+
+template <typename TMixin>
+class WithHeapInternals : public TMixin {
+ public:
+  WithHeapInternals() = default;
+
+  void CollectGarbage(i::AllocationSpace space) {
+    heap()->CollectGarbage(space, i::GarbageCollectionReason::kTesting);
+  }
+
+  Heap* heap() const { return this->i_isolate()->heap(); }
+
+ private:
+  DISALLOW_COPY_AND_ASSIGN(WithHeapInternals);
+};
+
+using TestWithHeapInternals =       //
+    WithHeapInternals<              //
+        WithInternalIsolateMixin<   //
+            WithIsolateScopeMixin<  //
+                WithIsolateMixin<   //
+                    ::testing::Test>>>>;
+
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_UNITTESTS_HEAP_HEAP_UTILS_H_

test/unittests/heap/unified-heap-unittest.cc

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "include/cppgc/allocation.h"
+#include "include/cppgc/garbage-collected.h"
+#include "include/cppgc/platform.h"
+#include "src/api/api-inl.h"
+#include "src/heap/cppgc-js/cpp-heap.h"
+#include "src/objects/objects-inl.h"
+#include "test/unittests/heap/heap-utils.h"
+
+namespace v8 {
+namespace internal {
+
+namespace {
+
+v8::Local<v8::Object> ConstructTraceableJSApiObject(
+    v8::Local<v8::Context> context, void* object) {
+  v8::EscapableHandleScope scope(context->GetIsolate());
+  v8::Local<v8::FunctionTemplate> function_t =
+      v8::FunctionTemplate::New(context->GetIsolate());
+  v8::Local<v8::ObjectTemplate> instance_t = function_t->InstanceTemplate();
+  instance_t->SetInternalFieldCount(2);
+  v8::Local<v8::Function> function =
+      function_t->GetFunction(context).ToLocalChecked();
+  v8::Local<v8::Object> instance =
+      function->NewInstance(context).ToLocalChecked();
+  instance->SetAlignedPointerInInternalField(0, object);
+  instance->SetAlignedPointerInInternalField(1, object);
+  CHECK(!instance.IsEmpty());
+  i::Handle<i::JSReceiver> js_obj = v8::Utils::OpenHandle(*instance);
+  CHECK_EQ(i::JS_API_OBJECT_TYPE, js_obj->map().instance_type());
+  return scope.Escape(instance);
+}
+
+void ResetWrappableConnection(v8::Local<v8::Object> api_object) {
+  api_object->SetAlignedPointerInInternalField(0, nullptr);
+  api_object->SetAlignedPointerInInternalField(1, nullptr);
+}
+
+class UnifiedHeapTest : public TestWithHeapInternals {
+ public:
+  UnifiedHeapTest()
+      : saved_incremental_marking_wrappers_(FLAG_incremental_marking_wrappers) {
+    FLAG_incremental_marking_wrappers = false;
+    cppgc::InitializeProcess(V8::GetCurrentPlatform()->GetPageAllocator());
+    cpp_heap_ = std::make_unique<CppHeap>(v8_isolate(), 0);
+    heap()->SetEmbedderHeapTracer(&cpp_heap());
+  }
+
+  ~UnifiedHeapTest() {
+    heap()->SetEmbedderHeapTracer(nullptr);
+    FLAG_incremental_marking_wrappers = saved_incremental_marking_wrappers_;
+    cppgc::ShutdownProcess();
+  }
+
+  CppHeap& cpp_heap() const { return *cpp_heap_.get(); }
+
+  cppgc::AllocationHandle& allocation_handle() {
+    return cpp_heap().object_allocator();
+  }
+
+ private:
+  std::unique_ptr<CppHeap> cpp_heap_;
+  bool saved_incremental_marking_wrappers_;
+};
+
+class Wrappable final : public cppgc::GarbageCollected<Wrappable> {
+ public:
+  static size_t destructor_callcount;
+
+  ~Wrappable() { destructor_callcount++; }
+
+  void Trace(cppgc::Visitor* visitor) const {}
+};
+
+size_t Wrappable::destructor_callcount = 0;
+
+}  // namespace
+
+TEST_F(UnifiedHeapTest, OnlyGC) { CollectGarbage(OLD_SPACE); }
+
+TEST_F(UnifiedHeapTest, FindingV8ToBlinkReference) {
+  v8::HandleScope scope(v8_isolate());
+  v8::Local<v8::Context> context = v8::Context::New(v8_isolate());
+  v8::Context::Scope context_scope(context);
+  v8::Local<v8::Object> api_object = ConstructTraceableJSApiObject(
+      context, cppgc::MakeGarbageCollected<Wrappable>(allocation_handle()));
+  EXPECT_FALSE(api_object.IsEmpty());
+  EXPECT_EQ(0u, Wrappable::destructor_callcount);
+  CollectGarbage(OLD_SPACE);
+  EXPECT_EQ(0u, Wrappable::destructor_callcount);
+  ResetWrappableConnection(api_object);
+  CollectGarbage(OLD_SPACE);
+  EXPECT_EQ(1u, Wrappable::destructor_callcount);
+}
+
+}  // namespace internal
+}  // namespace v8