cppgc: Speed up pointer decompression

With this CL, the decompression simply becomes:
       movsxd  rax, edi
       add     rax, rax
       and     rax, qword ptr fs:[base@TPOFF]

Bug: chromium:1325007

Change-Id: I931e4e667a9b9697671bccf14575420f8cb705e8
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3629871Reviewed-by: Michael Lippautz <mlippautz@chromium.org>
Commit-Queue: Anton Bikineev <bikineev@chromium.org>
Cr-Commit-Position: refs/heads/main@{#80521}

BUILD.bazel

@@ -3003,6 +3003,7 @@ filegroup(
         "src/heap/cppgc/marking-worklists.cc",
         "src/heap/cppgc/marking-worklists.h",
         "src/heap/cppgc/member.cc",
+        "src/heap/cppgc/member.h",
         "src/heap/cppgc/memory.cc",
         "src/heap/cppgc/memory.h",
         "src/heap/cppgc/metric-recorder.h",

BUILD.gn

@@ -5726,6 +5726,7 @@ v8_source_set("cppgc_base") {
     "src/heap/cppgc/marking-worklists.cc",
     "src/heap/cppgc/marking-worklists.h",
     "src/heap/cppgc/member.cc",
+    "src/heap/cppgc/member.h",
     "src/heap/cppgc/memory.cc",
     "src/heap/cppgc/memory.h",
     "src/heap/cppgc/metric-recorder.h",

include/cppgc/member.h

@@ -25,19 +25,34 @@ namespace internal {
 
 class CageBaseGlobal final {
  public:
-  V8_INLINE static void Update(uintptr_t base) {
-    CPPGC_DCHECK(0u ==
-                 (base & (api_constants::kCagedHeapReservationAlignment - 1)));
-    g_base_ = base;
+  V8_INLINE static uintptr_t Get() {
+    CPPGC_DCHECK(IsBaseConsistent());
+    return g_base_;
   }
 
-  V8_INLINE static uintptr_t Get() { return g_base_; }
+  V8_INLINE static bool IsSet() {
+    CPPGC_DCHECK(IsBaseConsistent());
+    return (g_base_ & ~kLowerHalfWordMask) != 0;
+  }
 
  private:
+  // We keep the lower halfword as ones to speed up decompression.
+  static constexpr uintptr_t kLowerHalfWordMask =
+      (api_constants::kCagedHeapReservationAlignment - 1);
+
   static thread_local V8_EXPORT uintptr_t g_base_
-      __attribute__((require_constant_initialization));
+#if !V8_CC_MSVC
+      __attribute__((require_constant_initialization))
+#endif  // !V8_CC_MSVC
+      ;
 
   CageBaseGlobal() = delete;
+
+  V8_INLINE static bool IsBaseConsistent() {
+    return kLowerHalfWordMask == (g_base_ & kLowerHalfWordMask);
+  }
+
+  friend class CageBaseGlobalUpdater;
 };
 
 class CompressedPointer final {
@@ -64,22 +79,36 @@ class CompressedPointer final {
   V8_INLINE void StoreRaw(Storage value) { value_ = value; }
 
   static V8_INLINE Storage Compress(const void* ptr) {
+    static_assert(
+        SentinelPointer::kSentinelValue == 0b10,
+        "The compression scheme relies on the sentinel encoded as 0b10");
     static constexpr size_t kGigaCageMask =
         ~(api_constants::kCagedHeapReservationAlignment - 1);
+
+    CPPGC_DCHECK(CageBaseGlobal::IsSet());
     const uintptr_t base = CageBaseGlobal::Get();
-    CPPGC_DCHECK(base);
     CPPGC_DCHECK(!ptr || ptr == kSentinelPointer ||
-                 base == (reinterpret_cast<uintptr_t>(ptr) & kGigaCageMask));
-    return static_cast<Storage>(reinterpret_cast<uintptr_t>(ptr));
+                 (base & kGigaCageMask) ==
+                     (reinterpret_cast<uintptr_t>(ptr) & kGigaCageMask));
+
+    const auto uptr = reinterpret_cast<uintptr_t>(ptr);
+    // Truncate the pointer and shift right by one.
+    auto compressed = static_cast<Storage>(uptr) >> 1;
+    // If the pointer is regular, set the most significant bit.
+    if (V8_LIKELY(compressed > 1)) {
+      CPPGC_DCHECK((reinterpret_cast<uintptr_t>(ptr) &
+                    (api_constants::kAllocationGranularity - 1)) == 0);
+      compressed |= 0x80000000;
+    }
+    return compressed;
   }
 
   static V8_INLINE void* Decompress(Storage ptr) {
+    CPPGC_DCHECK(CageBaseGlobal::IsSet());
     const uintptr_t base = CageBaseGlobal::Get();
-    CPPGC_DCHECK(base);
-    // We have to preserve nullptr and kSentinel, Members can't point to
-    // GigaCage metadata.
-    if (V8_UNLIKELY(ptr <= 1)) return reinterpret_cast<void*>(ptr);
-    return reinterpret_cast<void*>(base | static_cast<uintptr_t>(ptr));
+    // Sign extend the pointer and shift left by one.
+    const int64_t mask = static_cast<int64_t>(static_cast<int32_t>(ptr)) << 1;
+    return reinterpret_cast<void*>(mask & base);
   }
 
  private:

include/cppgc/sentinel-pointer.h

@@ -13,9 +13,9 @@ namespace internal {
 // Special tag type used to denote some sentinel member. The semantics of the
 // sentinel is defined by the embedder.
 struct SentinelPointer {
+  static constexpr intptr_t kSentinelValue = 0b10;
   template <typename T>
   operator T*() const {
-    static constexpr intptr_t kSentinelValue = 1;
     return reinterpret_cast<T*>(kSentinelValue);
   }
   // Hidden friends.

src/heap/cppgc/caged-heap.cc

@@ -16,6 +16,7 @@
 #include "src/base/platform/platform.h"
 #include "src/heap/cppgc/caged-heap.h"
 #include "src/heap/cppgc/globals.h"
+#include "src/heap/cppgc/member.h"
 
 namespace cppgc {
 namespace internal {
@@ -54,8 +55,9 @@ CagedHeap::CagedHeap(HeapBase& heap_base, PageAllocator& platform_allocator)
 
 #if defined(CPPGC_POINTER_COMPRESSION)
   // With pointer compression only single heap per thread is allowed.
-  CHECK(!CageBaseGlobal::Get());
-  CageBaseGlobal::Update(reinterpret_cast<uintptr_t>(reserved_area_.address()));
+  CHECK(!CageBaseGlobal::IsSet());
+  CageBaseGlobalUpdater::UpdateCageBase(
+      reinterpret_cast<uintptr_t>(reserved_area_.address()));
 #endif  // defined(CPPGC_POINTER_COMPRESSION)
 
   const bool is_not_oom = platform_allocator.SetPermissions(
@@ -86,8 +88,8 @@ CagedHeap::CagedHeap(HeapBase& heap_base, PageAllocator& platform_allocator)
 CagedHeap::~CagedHeap() {
 #if defined(CPPGC_POINTER_COMPRESSION)
   CHECK_EQ(reinterpret_cast<uintptr_t>(reserved_area_.address()),
-           CageBaseGlobal::Get());
-  CageBaseGlobal::Update(0u);
+           CageBaseGlobalUpdater::GetCageBase());
+  CageBaseGlobalUpdater::UpdateCageBase(0u);
 #endif  // defined(CPPGC_POINTER_COMPRESSION)
 }
 

src/heap/cppgc/concurrent-marker.cc

@@ -10,6 +10,7 @@
 #include "src/heap/cppgc/liveness-broker.h"
 #include "src/heap/cppgc/marking-state.h"
 #include "src/heap/cppgc/marking-visitor.h"
+#include "src/heap/cppgc/member.h"
 #include "src/heap/cppgc/stats-collector.h"
 
 namespace cppgc {
@@ -63,12 +64,14 @@ namespace {
 class PointerCompressionCageScope final {
  public:
   explicit PointerCompressionCageScope(HeapBase& heap)
-      : prev_cage_base_(CageBaseGlobal::Get()) {
-    CageBaseGlobal::Update(
+      : prev_cage_base_(CageBaseGlobalUpdater::GetCageBase()) {
+    CageBaseGlobalUpdater::UpdateCageBase(
         reinterpret_cast<uintptr_t>(heap.caged_heap().base()));
   }
 
-  ~PointerCompressionCageScope() { CageBaseGlobal::Update(prev_cage_base_); }
+  ~PointerCompressionCageScope() {
+    CageBaseGlobalUpdater::UpdateCageBase(prev_cage_base_);
+  }
 
  private:
   const uintptr_t prev_cage_base_;

src/heap/cppgc/member.cc

@@ -8,7 +8,8 @@ namespace cppgc {
 namespace internal {
 
 #if defined(CPPGC_POINTER_COMPRESSION)
-thread_local uintptr_t CageBaseGlobal::g_base_ = 0u;
+thread_local uintptr_t CageBaseGlobal::g_base_ =
+    CageBaseGlobal::kLowerHalfWordMask;
 #endif  // defined(CPPGC_POINTER_COMPRESSION)
 
 }  // namespace internal

src/heap/cppgc/member.h

+// Copyright 2022 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_HEAP_CPPGC_MEMBER_H_
+#define V8_HEAP_CPPGC_MEMBER_H_
+
+#include "include/cppgc/member.h"
+
+namespace cppgc {
+namespace internal {
+
+#if defined(CPPGC_POINTER_COMPRESSION)
+class CageBaseGlobalUpdater final {
+ public:
+  CageBaseGlobalUpdater() = delete;
+  static void UpdateCageBase(uintptr_t cage_base) {
+    CPPGC_DCHECK(CageBaseGlobal::IsBaseConsistent());
+    CPPGC_DCHECK(0u == (cage_base & CageBaseGlobal::kLowerHalfWordMask));
+    CageBaseGlobal::g_base_ = cage_base | CageBaseGlobal::kLowerHalfWordMask;
+  }
+
+  static uintptr_t GetCageBase() {
+    CPPGC_DCHECK(CageBaseGlobal::IsBaseConsistent());
+    return CageBaseGlobal::g_base_ & ~CageBaseGlobal::kLowerHalfWordMask;
+  }
+};
+#endif  // defined(CPPGC_POINTER_COMPRESSION)
+
+}  // namespace internal
+}  // namespace cppgc
+
+#endif  // V8_HEAP_CPPGC_MEMBER_H_

test/unittests/heap/cppgc/member-unittest.cc

@@ -566,6 +566,28 @@ TEST_F(MemberHeapDeathTest, CheckForOnHeapMemberCrashesOnInitialAssignment) {
 }
 #endif  // defined(CPPGC_POINTER_COMPRESSION)
 
+#if defined(CPPGC_POINTER_COMPRESSION)
+TEST_F(MemberTest, CompressDecompress) {
+  CompressedPointer cp;
+  EXPECT_EQ(nullptr, cp.Load());
+
+  Member<GCed> member;
+  cp.Store(member.Get());
+  EXPECT_EQ(nullptr, cp.Load());
+
+  cp.Store(kSentinelPointer);
+  EXPECT_EQ(kSentinelPointer, cp.Load());
+
+  member = kSentinelPointer;
+  cp.Store(member.Get());
+  EXPECT_EQ(kSentinelPointer, cp.Load());
+
+  member = MakeGarbageCollected<GCed>(GetAllocationHandle());
+  cp.Store(member.Get());
+  EXPECT_EQ(member.Get(), cp.Load());
+}
+#endif  // defined(CPPGC_POINTER_COMPRESSION)
+
 #endif  // V8_ENABLE_CHECKS
 
 }  // namespace internal