[zone-compr] Introduce ZoneTypeTraits and ZoneCompression

Also move zone compression flags to src/common/globals.h.

Bug: v8:9923
Change-Id: Id0a77720e735e2669a1e5eef48e1b4866ad99480
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2324255Reviewed-by: Leszek Swirski <leszeks@chromium.org>
Commit-Queue: Igor Sheludko <ishell@chromium.org>
Cr-Commit-Position: refs/heads/master@{#69160}

BUILD.gn

@@ -3307,14 +3307,15 @@ v8_source_set("v8_base_without_compiler") {
     "src/zone/type-stats.h",
     "src/zone/zone-allocator.h",
     "src/zone/zone-chunk-list.h",
+    "src/zone/zone-compression.h",
     "src/zone/zone-containers.h",
-    "src/zone/zone-fwd.h",
     "src/zone/zone-handle-set.h",
     "src/zone/zone-hashmap.h",
     "src/zone/zone-list-inl.h",
     "src/zone/zone-list.h",
     "src/zone/zone-segment.cc",
     "src/zone/zone-segment.h",
+    "src/zone/zone-type-traits.h",
     "src/zone/zone-utils.h",
     "src/zone/zone.cc",
     "src/zone/zone.h",

src/common/globals.h

@@ -243,6 +243,16 @@ constexpr size_t kReservedCodeRangePages = 0;
 
 STATIC_ASSERT(kSystemPointerSize == (1 << kSystemPointerSizeLog2));
 
+#ifdef V8_COMPRESS_ZONES
+#define COMPRESS_ZONES_BOOL true
+#else
+#define COMPRESS_ZONES_BOOL false
+#endif  // V8_COMPRESS_ZONES
+
+// The flag controls whether zones pointer compression should be enabled for
+// TurboFan graphs or not.
+static constexpr bool kCompressGraphZone = COMPRESS_ZONES_BOOL;
+
 #ifdef V8_COMPRESS_POINTERS
 static_assert(
     kSystemPointerSize == kInt64Size,

src/parsing/rewriter.h

@@ -7,7 +7,7 @@
 
 #include "src/base/macros.h"
 #include "src/base/optional.h"
-#include "src/zone/zone-fwd.h"
+#include "src/zone/zone-type-traits.h"
 
 namespace v8 {
 namespace internal {

src/zone/accounting-allocator.cc

@@ -10,28 +10,33 @@
 #include "src/base/logging.h"
 #include "src/base/macros.h"
 #include "src/utils/allocation.h"
-#include "src/zone/zone-fwd.h"
+#include "src/zone/zone-compression.h"
 #include "src/zone/zone-segment.h"
 
 namespace v8 {
 namespace internal {
 
+// These definitions are here in order to please the linker, which in debug mode
+// sometimes requires static constants to be defined in .cc files.
+const size_t ZoneCompression::kReservationSize;
+const size_t ZoneCompression::kReservationAlignment;
+
 namespace {
 
 static constexpr size_t kZonePageSize = 256 * KB;
 
 VirtualMemory ReserveAddressSpace(v8::PageAllocator* platform_allocator) {
-  DCHECK(
-      IsAligned(kZoneReservationSize, platform_allocator->AllocatePageSize()));
+  DCHECK(IsAligned(ZoneCompression::kReservationSize,
+                   platform_allocator->AllocatePageSize()));
 
   void* hint = reinterpret_cast<void*>(RoundDown(
       reinterpret_cast<uintptr_t>(platform_allocator->GetRandomMmapAddr()),
-      kZoneReservationAlignment));
+      ZoneCompression::kReservationAlignment));
 
-  VirtualMemory memory(platform_allocator, kZoneReservationSize, hint,
-                       kZoneReservationAlignment);
+  VirtualMemory memory(platform_allocator, ZoneCompression::kReservationSize,
+                       hint, ZoneCompression::kReservationAlignment);
   if (memory.IsReserved()) {
-    CHECK(IsAligned(memory.address(), kZoneReservationAlignment));
+    CHECK(IsAligned(memory.address(), ZoneCompression::kReservationAlignment));
     return memory;
   }
 
@@ -43,10 +48,11 @@ VirtualMemory ReserveAddressSpace(v8::PageAllocator* platform_allocator) {
 
 std::unique_ptr<v8::base::BoundedPageAllocator> CreateBoundedAllocator(
     v8::PageAllocator* platform_allocator, Address reservation_start) {
-  DCHECK(reservation_start);
+  CHECK(reservation_start);
+  CHECK(IsAligned(reservation_start, ZoneCompression::kReservationAlignment));
 
   auto allocator = std::make_unique<v8::base::BoundedPageAllocator>(
-      platform_allocator, reservation_start, kZoneReservationSize,
+      platform_allocator, reservation_start, ZoneCompression::kReservationSize,
       kZonePageSize);
 
   // Exclude first page from allocation to ensure that accesses through

src/zone/zone-allocator.h

@@ -40,10 +40,20 @@ class ZoneAllocator {
   // ZoneVector and friends or for ParallelMove.
   ZoneAllocator() : ZoneAllocator(nullptr) { UNREACHABLE(); }
 #endif
-  explicit ZoneAllocator(Zone* zone) : zone_(zone) {}
+  explicit ZoneAllocator(Zone* zone) : zone_(zone) {
+    // If we are going to allocate compressed pointers in the zone it must
+    // support compression.
+    DCHECK_IMPLIES(is_compressed_pointer<T>::value,
+                   zone_->supports_compression());
+  }
   template <typename U>
   ZoneAllocator(const ZoneAllocator<U>& other) V8_NOEXCEPT
-      : ZoneAllocator<T>(other.zone_) {}
+      : ZoneAllocator<T>(other.zone_) {
+    // If we are going to allocate compressed pointers in the zone it must
+    // support compression.
+    DCHECK_IMPLIES(is_compressed_pointer<T>::value,
+                   zone_->supports_compression());
+  }
   template <typename U>
   friend class ZoneAllocator;
 

src/zone/zone-compression.h

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_ZONE_ZONE_COMPRESSION_H_
+#define V8_ZONE_ZONE_COMPRESSION_H_
+
+#include "src/base/bits.h"
+#include "src/common/globals.h"
+
+namespace v8 {
+namespace internal {
+
+// This struct provides untyped implementation of zone compression scheme.
+//
+// The compression scheme relies on the following assumptions:
+// 1) all zones containing compressed pointers are allocated in the same "zone
+//    cage" of kReservationSize size and kReservationAlignment-aligned.
+//    Attempt to compress pointer to an object stored outside of the "cage"
+//    will silently succeed but it will later produce wrong result after
+//    decompression.
+// 2) compression is just a masking away bits above kReservationAlignment.
+// 3) nullptr is compressed to 0, thus there must be no valid objects allocated
+//    at the beginning of the "zone cage". Ideally, the first page of the cage
+//    should be unmapped in order to catch attempts to use decompressed nullptr
+//    value earlier.
+// 4) decompression requires "zone cage" address value, which is computed on
+//    the fly from an arbitrary address pointing somewhere to the "zone cage".
+// 5) decompression requires special casing for nullptr.
+struct ZoneCompression {
+  static const size_t kReservationSize = size_t{2} * GB;
+  static const size_t kReservationAlignment =
+      COMPRESS_ZONES_BOOL ? size_t{4} * GB : 1;
+
+  static_assert(base::bits::IsPowerOfTwo(kReservationAlignment),
+                "Bad zone alignment");
+
+  static const size_t kOffsetMask = kReservationAlignment - 1;
+
+  inline static Address base_of(const void* zone_pointer) {
+    return reinterpret_cast<Address>(zone_pointer) & ~kOffsetMask;
+  }
+
+  inline static bool CheckSameBase(const void* p1, const void* p2) {
+    if (p1 == nullptr || p2 == nullptr) return true;
+    CHECK_EQ(base_of(p1), base_of(p2));
+    return true;
+  }
+
+  inline static uint32_t Compress(const void* value) {
+    Address raw_value = reinterpret_cast<Address>(value);
+    uint32_t compressed_value = static_cast<uint32_t>(raw_value & kOffsetMask);
+    DCHECK_IMPLIES(compressed_value == 0, value == nullptr);
+    DCHECK_LT(compressed_value, kReservationSize);
+    return compressed_value;
+  }
+
+  inline static Address Decompress(const void* zone_pointer,
+                                   uint32_t compressed_value) {
+    if (compressed_value == 0) return kNullAddress;
+    return base_of(zone_pointer) + compressed_value;
+  }
+};
+
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_ZONE_ZONE_COMPRESSION_H_

src/zone/zone-fwd.h

-// Copyright 2020 the V8 project authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-#ifndef V8_ZONE_ZONE_FWD_H_
-#define V8_ZONE_ZONE_FWD_H_
-
-#include "src/common/globals.h"
-
-namespace v8 {
-namespace internal {
-
-//
-// This header contains forward declarations for Zone-related objects and
-// containers.
-//
-
-class Zone;
-
-template <typename T>
-class ZoneList;
-
-// ZonePtrList is a ZoneList of pointers to ZoneObjects allocated in the same
-// zone as the list object.
-template <typename T>
-using ZonePtrList = ZoneList<T*>;
-
-#ifdef V8_COMPRESS_ZONES
-static_assert(kSystemPointerSize == 8,
-              "Zone compression requires 64-bit architectures");
-#define COMPRESS_ZONES_BOOL true
-constexpr size_t kZoneReservationSize = static_cast<size_t>(2) * GB;
-constexpr size_t kZoneReservationAlignment = static_cast<size_t>(4) * GB;
-
-#else  // V8_COMPRESS_ZONES
-#define COMPRESS_ZONES_BOOL false
-// These constants must not be used when zone compression is not enabled.
-constexpr size_t kZoneReservationSize = 1;
-constexpr size_t kZoneReservationAlignment = 1;
-#endif  // V8_COMPRESS_ZONES
-
-// The flags controlling whether zones that will be used for allocating
-// TurboFan graphs should be compressed or not.
-static constexpr bool kCompressGraphZone = COMPRESS_ZONES_BOOL;
-
-}  // namespace internal
-}  // namespace v8
-
-#endif  // V8_ZONE_ZONE_FWD_H_

src/zone/zone-list-inl.h

@@ -31,7 +31,7 @@ template <typename T>
 void ZoneList<T>::AddAll(const Vector<const T>& other, Zone* zone) {
   int result_length = length_ + other.length();
   if (capacity_ < result_length) Resize(result_length, zone);
-  if (std::is_fundamental<T>()) {
+  if (std::is_trivially_copyable<T>::value) {
     memcpy(&data_[length_], other.begin(), sizeof(T) * other.length());
   } else {
     std::copy(other.begin(), other.end(), &data_[length_]);
@@ -64,7 +64,7 @@ void ZoneList<T>::Resize(int new_capacity, Zone* zone) {
   DCHECK_LE(length_, new_capacity);
   T* new_data = zone->NewArray<T>(new_capacity);
   if (length_ > 0) {
-    if (std::is_fundamental<T>()) {
+    if (std::is_trivially_copyable<T>::value) {
       MemCopy(new_data, data_, length_ * sizeof(T));
     } else {
       std::copy(&data_[0], &data_[length_], &new_data[0]);

src/zone/zone-list.h

@@ -6,7 +6,6 @@
 #define V8_ZONE_ZONE_LIST_H_
 
 #include "src/base/logging.h"
-#include "src/zone/zone-fwd.h"
 #include "src/zone/zone.h"
 
 namespace v8 {

src/zone/zone-type-traits.h

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_ZONE_ZONE_TYPE_TRAITS_H_
+#define V8_ZONE_ZONE_TYPE_TRAITS_H_
+
+#include "src/common/globals.h"
+
+namespace v8 {
+namespace internal {
+
+template <typename T>
+class ZoneList;
+
+// ZonePtrList is a ZoneList of pointers to ZoneObjects allocated in the same
+// zone as the list object.
+template <typename T>
+using ZonePtrList = ZoneList<T*>;
+
+template <typename T>
+using FullZonePtr = T*;
+
+template <typename T>
+class CompressedZonePtr;
+
+//
+// ZoneTypeTraits provides type aliases for compressed or full pointer
+// dependent types based on a static flag. It helps organizing fine-grained
+// control over which parts of the code base should use compressed zone
+// pointers.
+// For example:
+//   using ZoneNodePtr = typename ZoneTypeTraits<kCompressGraphZone>::Ptr<Node>;
+//
+// or
+//   template <typename T>
+//   using AstZonePtr = typename ZoneTypeTraits<kCompressAstZone>::Ptr<T>;
+//
+template <bool kEnableCompression>
+struct ZoneTypeTraits;
+
+template <>
+struct ZoneTypeTraits<false> {
+  template <typename T>
+  using Ptr = FullZonePtr<T>;
+};
+
+template <>
+struct ZoneTypeTraits<true> {
+  // TODO(ishell): use CompressedZonePtr<T> here
+  template <typename T>
+  using Ptr = FullZonePtr<T>;
+};
+
+// This requirement is necessary for being able to use memcopy in containers
+// of zone pointers.
+static_assert(
+    std::is_trivially_copyable<
+        ZoneTypeTraits<COMPRESS_ZONES_BOOL>::Ptr<int>>::value,
+    "ZoneTypeTraits<COMPRESS_ZONES_BOOL>::Ptr<T> must be trivially copyable");
+
+//
+// is_compressed_pointer<T> predicate can be used for checking if T is a
+// compressed pointer.
+//
+template <typename>
+struct is_compressed_pointer : std::false_type {};
+
+template <typename T>
+struct is_compressed_pointer<CompressedZonePtr<T>> : std::true_type {};
+
+template <typename T>
+struct is_compressed_pointer<const CompressedZonePtr<T>> : std::true_type {};
+
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_ZONE_ZONE_TYPE_TRAITS_H_

src/zone/zone-utils.h

@@ -20,7 +20,7 @@ Vector<T> CloneVector(Zone* zone, const Vector<const T>& other) {
   if (length == 0) return Vector<T>();
 
   T* data = zone->NewArray<T>(length);
-  if (std::is_fundamental<T>()) {
+  if (std::is_trivially_copyable<T>::value) {
     MemCopy(data, other.data(), length * sizeof(T));
   } else {
     std::copy(other.begin(), other.end(), data);

src/zone/zone.h

@@ -9,10 +9,11 @@
 
 #include "src/base/logging.h"
 #include "src/common/globals.h"
+#include "src/utils/utils.h"
 #include "src/zone/accounting-allocator.h"
 #include "src/zone/type-stats.h"
-#include "src/zone/zone-fwd.h"
 #include "src/zone/zone-segment.h"
+#include "src/zone/zone-type-traits.h"
 
 #ifndef ZONE_NAME
 #define ZONE_NAME __func__
@@ -116,6 +117,7 @@ class V8_EXPORT_PRIVATE Zone final {
   // distinguishable between each other.
   template <typename T, typename TypeTag = T[]>
   T* NewArray(size_t length) {
+    DCHECK_IMPLIES(is_compressed_pointer<T>::value, supports_compression());
     DCHECK_LT(length, std::numeric_limits<size_t>::max() / sizeof(T));
     return static_cast<T*>(Allocate<TypeTag>(length * sizeof(T)));
   }

test/cctest/compiler/codegen-tester.h

@@ -10,7 +10,6 @@
 #include "src/compiler/pipeline.h"
 #include "src/compiler/raw-machine-assembler.h"
 #include "src/execution/simulator.h"
-#include "src/zone/zone-fwd.h"
 #include "test/cctest/cctest.h"
 #include "test/cctest/compiler/call-tester.h"
 

test/cctest/compiler/test-graph-visualizer.cc

@@ -14,7 +14,6 @@
 #include "src/compiler/schedule.h"
 #include "src/compiler/scheduler.h"
 #include "src/compiler/verifier.h"
-#include "src/zone/zone-fwd.h"
 #include "test/cctest/cctest.h"
 
 namespace v8 {

test/cctest/compiler/test-instruction-scheduler.cc

@@ -5,7 +5,6 @@
 #include "src/compiler/backend/instruction-scheduler.h"
 #include "src/compiler/backend/instruction-selector-impl.h"
 #include "src/compiler/backend/instruction.h"
-#include "src/zone/zone-fwd.h"
 #include "test/cctest/cctest.h"
 
 namespace v8 {

test/cctest/compiler/test-node.cc

@@ -7,7 +7,6 @@
 #include "src/compiler/graph.h"
 #include "src/compiler/node.h"
 #include "src/compiler/operator.h"
-#include "src/zone/zone-fwd.h"
 #include "test/cctest/cctest.h"
 
 namespace v8 {

test/unittests/codegen/code-stub-assembler-unittest.h

@@ -6,7 +6,6 @@
 #define V8_UNITTESTS_CODE_STUB_ASSEMBLER_UNITTEST_H_
 
 #include "src/codegen/code-stub-assembler.h"
-#include "src/zone/zone-fwd.h"
 #include "test/unittests/test-utils.h"
 #include "testing/gmock-support.h"
 

test/unittests/compiler/graph-unittest.cc

@@ -8,7 +8,6 @@
 #include "src/compiler/node-properties.h"
 #include "src/heap/factory.h"
 #include "src/objects/objects-inl.h"  // TODO(everyone): Make typer.h IWYU compliant.
-#include "src/zone/zone-fwd.h"
 #include "test/unittests/compiler/node-test-utils.h"
 
 namespace v8 {

test/unittests/compiler/schedule-unittest.cc

@@ -5,7 +5,6 @@
 #include "src/compiler/schedule.h"
 
 #include "src/compiler/node.h"
-#include "src/zone/zone-fwd.h"
 #include "test/unittests/test-utils.h"
 #include "testing/gmock/include/gmock/gmock.h"
 

test/unittests/compiler/value-numbering-reducer-unittest.cc

@@ -9,7 +9,6 @@
 #include "src/compiler/graph.h"
 #include "src/compiler/node.h"
 #include "src/compiler/operator.h"
-#include "src/zone/zone-fwd.h"
 #include "test/unittests/test-utils.h"
 
 namespace v8 {

test/unittests/wasm/function-body-decoder-unittest.cc

@@ -15,6 +15,7 @@
 #include "src/wasm/wasm-limits.h"
 #include "src/wasm/wasm-module.h"
 #include "src/wasm/wasm-opcodes-inl.h"
+#include "src/zone/zone.h"
 #include "test/common/wasm/flag-utils.h"
 #include "test/common/wasm/test-signatures.h"
 #include "test/common/wasm/wasm-macro-gen.h"