Prepare StringForwardingTable for external strings

- Move StringForwardingTable implementation to own compilation unit.
- Refactoring preparing for layout change (Introduce explicit record
  class to make transition from contiguous Tagged_t fields to a
  heterogeneous record layout easier).
- Replace RootVisitor pattern for transitioning/cleanup during GC with
  callback.
- Minor cleanups.

Bug: v8:12957
Change-Id: Iae343393f470130eac0c54148a1303b67fb95aa4
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3845635Reviewed-by: Camillo Bruni <cbruni@chromium.org>
Commit-Queue: Patrick Thier <pthier@chromium.org>
Reviewed-by: Dominik Inführ <dinfuehr@chromium.org>
Cr-Commit-Position: refs/heads/main@{#82730}

BUILD.bazel

@@ -1855,6 +1855,9 @@ filegroup(
         "src/objects/string-inl.h",
         "src/objects/string-set-inl.h",
         "src/objects/string-set.h",
+        "src/objects/string-forwarding-table-inl.h",
+        "src/objects/string-forwarding-table.cc",
+        "src/objects/string-forwarding-table.h",
         "src/objects/string-table-inl.h",
         "src/objects/string-table.cc",
         "src/objects/symbol-table.cc",

BUILD.gn

@@ -3410,6 +3410,8 @@ v8_header_set("v8_internal_headers") {
     "src/objects/source-text-module-inl.h",
     "src/objects/source-text-module.h",
     "src/objects/string-comparator.h",
+    "src/objects/string-forwarding-table-inl.h",
+    "src/objects/string-forwarding-table.h",
     "src/objects/string-inl.h",
     "src/objects/string-set-inl.h",
     "src/objects/string-set.h",
@@ -4637,6 +4639,7 @@ v8_source_set("v8_base_without_compiler") {
     "src/objects/simd.cc",
     "src/objects/source-text-module.cc",
     "src/objects/string-comparator.cc",
+    "src/objects/string-forwarding-table.cc",
     "src/objects/string-table.cc",
     "src/objects/string.cc",
     "src/objects/swiss-name-dictionary.cc",

src/heap/mark-compact.cc

@@ -66,6 +66,7 @@
 #include "src/objects/objects.h"
 #include "src/objects/slots-inl.h"
 #include "src/objects/smi.h"
+#include "src/objects/string-forwarding-table-inl.h"
 #include "src/objects/transitions-inl.h"
 #include "src/objects/visitors.h"
 #include "src/snapshot/shared-heap-serializer.h"
@@ -1436,64 +1437,6 @@ class InternalizedStringTableCleaner final : public RootVisitor {
   int pointers_removed_ = 0;
 };
 
-class StringForwardingTableCleaner final : public RootVisitor {
- public:
-  explicit StringForwardingTableCleaner(Heap* heap) : heap_(heap) {}
-
-  void VisitRootPointers(Root root, const char* description,
-                         FullObjectSlot start, FullObjectSlot end) override {
-    UNREACHABLE();
-  }
-
-  void VisitRootPointers(Root root, const char* description,
-                         OffHeapObjectSlot start,
-                         OffHeapObjectSlot end) override {
-    DCHECK_EQ(root, Root::kStringForwardingTable);
-    // Visit all HeapObject pointers in [start, end).
-    // The forwarding table is organized in pairs of [orig string, forward
-    // string].
-    auto* marking_state =
-        heap_->mark_compact_collector()->non_atomic_marking_state();
-    Isolate* isolate = heap_->isolate();
-    for (OffHeapObjectSlot p = start; p < end; p += 2) {
-      Object original = p.load(isolate);
-      if (!original.IsHeapObject()) {
-        // Only if we always use the forwarding table, the string could be a
-        // smi, indicating that the entry died during scavenge.
-        DCHECK(FLAG_always_use_string_forwarding_table);
-        DCHECK_EQ(original, StringForwardingTable::deleted_element());
-        continue;
-      }
-      if (marking_state->IsBlack(HeapObject::cast(original))) {
-        String original_string = String::cast(original);
-        // Check if the string was already transitioned. This happens if we have
-        // multiple entries for the same string in the table.
-        if (original_string.IsThinString()) continue;
-
-        // The second slot of each record is the forward string.
-        Object forward = (p + 1).load(isolate);
-        String forward_string = String::cast(forward);
-
-        // Mark the forwarded string.
-        marking_state->WhiteToBlack(forward_string);
-
-        // Transition the original string to a ThinString and override the
-        // forwarding index with the correct hash.
-        original_string.MakeThin(isolate, forward_string);
-        original_string.set_raw_hash_field(forward_string.raw_hash_field());
-        // Record the slot in the old-to-old remembered set. This is required as
-        // the internalized string could be relocated during compaction.
-        ObjectSlot slot = ThinString::cast(original_string)
-                              .RawField(ThinString::kActualOffset);
-        MarkCompactCollector::RecordSlot(original_string, slot, forward_string);
-      }
-    }
-  }
-
- private:
-  Heap* heap_;
-};
-
 class ExternalStringTableCleaner : public RootVisitor {
  public:
   explicit ExternalStringTableCleaner(Heap* heap) : heap_(heap) {}
@@ -2866,6 +2809,62 @@ class ClearStringTableJobItem final : public ParallelClearingJob::ClearingItem {
   Isolate* const isolate_;
 };
 
+class StringForwardingTableCleaner final {
+ public:
+  explicit StringForwardingTableCleaner(Heap* heap)
+      : heap_(heap),
+        isolate_(heap_->isolate()),
+        marking_state_(
+            heap_->mark_compact_collector()->non_atomic_marking_state()) {}
+  void Run() {
+    StringForwardingTable* forwarding_table =
+        isolate_->string_forwarding_table();
+    forwarding_table->IterateElements(
+        isolate_, [&](StringForwardingTable::Record* record) {
+          TransitionStrings(record);
+        });
+    forwarding_table->Reset();
+  }
+
+ private:
+  void TransitionStrings(StringForwardingTable::Record* record) {
+    Object original = record->OriginalStringObject(isolate_);
+    if (!original.IsHeapObject()) {
+      // Only if we always use the forwarding table, the string could be a
+      // smi, indicating that the entry died during scavenge.
+      DCHECK(FLAG_always_use_string_forwarding_table);
+      DCHECK_EQ(original, StringForwardingTable::deleted_element());
+      return;
+    }
+    if (marking_state_->IsBlack(HeapObject::cast(original))) {
+      String original_string = String::cast(original);
+      TryInternalize(original_string, record);
+      original_string.set_raw_hash_field(record->raw_hash(isolate_));
+    }
+  }
+
+  void TryInternalize(String original_string,
+                      StringForwardingTable::Record* record) {
+    if (original_string.IsThinString()) return;
+    String forward_string = record->forward_string(isolate_);
+
+    // Mark the forwarded string to keep it alive.
+    marking_state_->WhiteToBlack(forward_string);
+    // Transition the original string to a ThinString and override the
+    // forwarding index with the correct hash.
+    original_string.MakeThin(isolate_, forward_string);
+    // Record the slot in the old-to-old remembered set. This is
+    // required as the internalized string could be relocated during
+    // compaction.
+    ObjectSlot slot =
+        ThinString::cast(original_string).RawField(ThinString::kActualOffset);
+    MarkCompactCollector::RecordSlot(original_string, slot, forward_string);
+  }
+  Heap* heap_;
+  Isolate* isolate_;
+  NonAtomicMarkingState* marking_state_;
+};
+
 }  // namespace
 
 void MarkCompactCollector::ClearNonLiveReferences() {
@@ -2879,11 +2878,8 @@ void MarkCompactCollector::ClearNonLiveReferences() {
     // Clearing the string forwarding table must happen before clearing the
     // string table, as entries in the forwarding table can keep internalized
     // strings alive.
-    StringForwardingTable* forwarding_table =
-        isolate()->string_forwarding_table();
-    StringForwardingTableCleaner visitor(heap());
-    forwarding_table->IterateElements(&visitor);
-    forwarding_table->Reset();
+    StringForwardingTableCleaner forwarding_table_cleaner(heap());
+    forwarding_table_cleaner.Run();
   }
 
   auto clearing_job = std::make_unique<ParallelClearingJob>();

src/objects/all-objects-inl.h

@@ -76,6 +76,7 @@
 #include "src/objects/shared-function-info-inl.h"
 #include "src/objects/slots-atomic-inl.h"
 #include "src/objects/slots-inl.h"
+#include "src/objects/string-forwarding-table-inl.h"
 #include "src/objects/string-inl.h"
 #include "src/objects/string-set-inl.h"
 #include "src/objects/string-table-inl.h"

src/objects/name-inl.h

@@ -10,8 +10,8 @@
 #include "src/objects/map-inl.h"
 #include "src/objects/name.h"
 #include "src/objects/primitive-heap-object-inl.h"
+#include "src/objects/string-forwarding-table.h"
 #include "src/objects/string-inl.h"
-#include "src/objects/string-table.h"
 
 // Has to be the last include (doesn't have include guards):
 #include "src/objects/object-macros.h"

src/objects/string-forwarding-table-inl.h

+// Copyright 2022 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_OBJECTS_STRING_FORWARDING_TABLE_INL_H_
+#define V8_OBJECTS_STRING_FORWARDING_TABLE_INL_H_
+
+#include "src/base/atomicops.h"
+#include "src/common/globals.h"
+#include "src/objects/name-inl.h"
+#include "src/objects/slots-inl.h"
+#include "src/objects/slots.h"
+#include "src/objects/string-forwarding-table.h"
+#include "src/objects/string-inl.h"
+// Has to be the last include (doesn't have include guards):
+#include "src/objects/object-macros.h"
+
+namespace v8 {
+namespace internal {
+
+class StringForwardingTable::Record final {
+ public:
+  String original_string(PtrComprCageBase cage_base) const {
+    return String::cast(OriginalStringObject(cage_base));
+  }
+
+  String forward_string(PtrComprCageBase cage_base) const {
+    return String::cast(ForwardStringObject(cage_base));
+  }
+
+  inline uint32_t raw_hash(PtrComprCageBase cage_base) const;
+
+  Object OriginalStringObject(PtrComprCageBase cage_base) const {
+    return OriginalStringSlot().Acquire_Load(cage_base);
+  }
+
+  Object ForwardStringObject(PtrComprCageBase cage_base) const {
+    return ForwardStringSlot().Acquire_Load(cage_base);
+  }
+
+  void set_original_string(Object object) {
+    OriginalStringSlot().Release_Store(object);
+  }
+
+  void set_forward_string(Object object) {
+    ForwardStringSlot().Release_Store(object);
+  }
+
+  inline void SetInternalized(String string, String forward_to);
+
+ private:
+  OffHeapObjectSlot OriginalStringSlot() const {
+    return OffHeapObjectSlot(&original_string_);
+  }
+
+  OffHeapObjectSlot ForwardStringSlot() const {
+    return OffHeapObjectSlot(&forward_string_);
+  }
+
+  Tagged_t original_string_;
+  Tagged_t forward_string_;
+
+  friend class StringForwardingTable::Block;
+};
+
+uint32_t StringForwardingTable::Record::raw_hash(
+    PtrComprCageBase cage_base) const {
+  String internalized = forward_string(cage_base);
+  uint32_t raw_hash = internalized.raw_hash_field();
+  DCHECK(Name::IsHashFieldComputed(raw_hash));
+  return raw_hash;
+}
+
+void StringForwardingTable::Record::SetInternalized(String string,
+                                                    String forward_to) {
+  set_original_string(string);
+  set_forward_string(forward_to);
+}
+
+class StringForwardingTable::Block {
+ public:
+  static std::unique_ptr<Block> New(int capacity);
+  explicit Block(int capacity);
+  int capacity() const { return capacity_; }
+  void* operator new(size_t size, int capacity);
+  void* operator new(size_t size) = delete;
+  void operator delete(void* data);
+
+  Record* record(int index) {
+    DCHECK_LT(index, capacity());
+    return &elements_[index];
+  }
+
+  const Record* record(int index) const {
+    DCHECK_LT(index, capacity());
+    return &elements_[index];
+  }
+
+  void UpdateAfterEvacuation(PtrComprCageBase cage_base);
+  void UpdateAfterEvacuation(PtrComprCageBase cage_base, int up_to_index);
+
+ private:
+  const int capacity_;
+  Record elements_[1];
+};
+
+class StringForwardingTable::BlockVector {
+ public:
+  using Block = StringForwardingTable::Block;
+  using Allocator = std::allocator<Block*>;
+
+  explicit BlockVector(size_t capacity);
+  ~BlockVector();
+  size_t capacity() const { return capacity_; }
+
+  Block* LoadBlock(size_t index, AcquireLoadTag) {
+    DCHECK_LT(index, size());
+    return base::AsAtomicPointer::Acquire_Load(&begin_[index]);
+  }
+
+  Block* LoadBlock(size_t index) {
+    DCHECK_LT(index, size());
+    return begin_[index];
+  }
+
+  void AddBlock(std::unique_ptr<Block> block) {
+    DCHECK_LT(size(), capacity());
+    base::AsAtomicPointer::Release_Store(&begin_[size_], block.release());
+    size_++;
+  }
+
+  static std::unique_ptr<BlockVector> Grow(BlockVector* data, size_t capacity,
+                                           const base::Mutex& mutex);
+
+  size_t size() const { return size_; }
+
+ private:
+  V8_NO_UNIQUE_ADDRESS Allocator allocator_;
+  const size_t capacity_;
+  std::atomic<size_t> size_;
+  Block** begin_;
+};
+
+int StringForwardingTable::size() const { return next_free_index_; }
+bool StringForwardingTable::empty() const { return size() == 0; }
+
+// static
+uint32_t StringForwardingTable::BlockForIndex(int index,
+                                              uint32_t* index_in_block) {
+  DCHECK_GE(index, 0);
+  DCHECK_NOT_NULL(index_in_block);
+  // The block is the leftmost set bit of the index, corrected by the size
+  // of the first block.
+  const uint32_t block_index =
+      kBitsPerInt -
+      base::bits::CountLeadingZeros(
+          static_cast<uint32_t>(index + kInitialBlockSize)) -
+      kInitialBlockSizeHighestBit - 1;
+  *index_in_block = IndexInBlock(index, block_index);
+  return block_index;
+}
+
+// static
+uint32_t StringForwardingTable::IndexInBlock(int index, uint32_t block_index) {
+  DCHECK_GE(index, 0);
+  // Clear out the leftmost set bit (the block index) to get the index within
+  // the block.
+  return static_cast<uint32_t>(index + kInitialBlockSize) &
+         ~(1u << (block_index + kInitialBlockSizeHighestBit));
+}
+
+// static
+uint32_t StringForwardingTable::CapacityForBlock(uint32_t block_index) {
+  return 1u << (block_index + kInitialBlockSizeHighestBit);
+}
+
+template <typename Func>
+void StringForwardingTable::IterateElements(Isolate* isolate, Func&& callback) {
+  isolate->heap()->safepoint()->AssertActive();
+  DCHECK_NE(isolate->heap()->gc_state(), Heap::NOT_IN_GC);
+
+  if (empty()) return;
+  BlockVector* blocks = blocks_.load(std::memory_order_relaxed);
+  const uint32_t last_block_index = static_cast<uint32_t>(blocks->size() - 1);
+  for (uint32_t block_index = 0; block_index < last_block_index;
+       ++block_index) {
+    Block* block = blocks->LoadBlock(block_index);
+    for (int index = 0; index < block->capacity(); ++index) {
+      Record* rec = block->record(index);
+      callback(rec);
+    }
+  }
+  // Handle last block separately, as it is not filled to capacity.
+  const uint32_t max_index = IndexInBlock(size() - 1, last_block_index) + 1;
+  Block* block = blocks->LoadBlock(last_block_index);
+  for (uint32_t index = 0; index < max_index; ++index) {
+    Record* rec = block->record(index);
+    callback(rec);
+  }
+}
+
+}  // namespace internal
+}  // namespace v8
+
+#include "src/objects/object-macros-undef.h"
+
+#endif  // V8_OBJECTS_STRING_FORWARDING_TABLE_INL_H_

src/objects/string-forwarding-table.cc

+// Copyright 2022 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/objects/string-forwarding-table.h"
+
+#include "src/base/atomicops.h"
+#include "src/common/globals.h"
+#include "src/objects/objects-inl.h"
+#include "src/objects/slots-inl.h"
+#include "src/objects/slots.h"
+#include "src/objects/string-forwarding-table-inl.h"
+#include "src/utils/allocation.h"
+
+namespace v8 {
+namespace internal {
+
+StringForwardingTable::Block::Block(int capacity) : capacity_(capacity) {
+  static_assert(unused_element().ptr() == 0);
+  static_assert(kNullAddress == 0);
+  static_assert(sizeof(Record) % sizeof(Address) == 0);
+  static_assert(offsetof(Record, original_string_) == 0);
+  constexpr int kRecordPointerSize = sizeof(Record) / sizeof(Address);
+  MemsetPointer(reinterpret_cast<Address*>(&elements_[0]), 0,
+                capacity_ * kRecordPointerSize);
+}
+
+void* StringForwardingTable::Block::operator new(size_t size, int capacity) {
+  // Make sure the size given is the size of the Block structure.
+  DCHECK_EQ(size, sizeof(StringForwardingTable::Block));
+  // Make sure the Record class is trivial and has standard layout.
+  static_assert(std::is_trivial_v<Record>);
+  static_assert(std::is_standard_layout_v<Record>);
+  // Make sure that the elements_ array is at the end of Block, with no padding,
+  // so that subsequent elements can be accessed as offsets from elements_.
+  static_assert(offsetof(StringForwardingTable::Block, elements_) ==
+                sizeof(StringForwardingTable::Block) - sizeof(Record));
+  // Make sure that elements_ is aligned when StringTable::Block is aligned.
+  static_assert((alignof(StringForwardingTable::Block) +
+                 offsetof(StringForwardingTable::Block, elements_)) %
+                    kTaggedSize ==
+                0);
+
+  const size_t elements_size = capacity * sizeof(Record);
+  // Storage for the first element is already supplied by elements_, so subtract
+  // sizeof(Record).
+  const size_t new_size = size + elements_size - sizeof(Record);
+  DCHECK_LE(alignof(StringForwardingTable::Block), kSystemPointerSize);
+  return AlignedAlloc(new_size, kSystemPointerSize);
+}
+
+void StringForwardingTable::Block::operator delete(void* block) {
+  AlignedFree(block);
+}
+
+std::unique_ptr<StringForwardingTable::Block> StringForwardingTable::Block::New(
+    int capacity) {
+  return std::unique_ptr<Block>(new (capacity) Block(capacity));
+}
+
+void StringForwardingTable::Block::UpdateAfterEvacuation(
+    PtrComprCageBase cage_base) {
+  UpdateAfterEvacuation(cage_base, capacity_);
+}
+
+void StringForwardingTable::Block::UpdateAfterEvacuation(
+    PtrComprCageBase cage_base, int up_to_index) {
+  // This is only used for Scavenger.
+  DCHECK(!FLAG_minor_mc);
+  DCHECK(FLAG_always_use_string_forwarding_table);
+  for (int index = 0; index < up_to_index; ++index) {
+    Object original = record(index)->OriginalStringObject(cage_base);
+    if (!original.IsHeapObject()) continue;
+    HeapObject object = HeapObject::cast(original);
+    if (Heap::InFromPage(object)) {
+      DCHECK(!object.InSharedWritableHeap());
+      MapWord map_word = object.map_word(kRelaxedLoad);
+      if (map_word.IsForwardingAddress()) {
+        HeapObject forwarded_object = map_word.ToForwardingAddress();
+        record(index)->set_original_string(forwarded_object);
+      } else {
+        record(index)->set_original_string(deleted_element());
+      }
+    } else {
+      DCHECK(!object.map_word(kRelaxedLoad).IsForwardingAddress());
+    }
+  }
+}
+
+StringForwardingTable::BlockVector::BlockVector(size_t capacity)
+    : allocator_(Allocator()), capacity_(capacity), size_(0) {
+  begin_ = allocator_.allocate(capacity);
+}
+
+StringForwardingTable::BlockVector::~BlockVector() {
+  allocator_.deallocate(begin_, capacity());
+}
+
+// static
+std::unique_ptr<StringForwardingTable::BlockVector>
+StringForwardingTable::BlockVector::Grow(
+    StringForwardingTable::BlockVector* data, size_t capacity,
+    const base::Mutex& mutex) {
+  mutex.AssertHeld();
+  std::unique_ptr<BlockVector> new_data =
+      std::make_unique<BlockVector>(capacity);
+  // Copy pointers to blocks from the old to the new vector.
+  for (size_t i = 0; i < data->size(); i++) {
+    new_data->begin_[i] = data->LoadBlock(i);
+  }
+  new_data->size_ = data->size();
+  return new_data;
+}
+
+StringForwardingTable::StringForwardingTable(Isolate* isolate)
+    : isolate_(isolate), next_free_index_(0) {
+  InitializeBlockVector();
+}
+
+StringForwardingTable::~StringForwardingTable() {
+  BlockVector* blocks = blocks_.load(std::memory_order_relaxed);
+  for (uint32_t block_index = 0; block_index < blocks->size(); block_index++) {
+    delete blocks->LoadBlock(block_index);
+  }
+}
+
+void StringForwardingTable::InitializeBlockVector() {
+  BlockVector* blocks = block_vector_storage_
+                            .emplace_back(std::make_unique<BlockVector>(
+                                kInitialBlockVectorCapacity))
+                            .get();
+  blocks->AddBlock(Block::New(kInitialBlockSize));
+  blocks_.store(blocks, std::memory_order_relaxed);
+}
+
+StringForwardingTable::BlockVector* StringForwardingTable::EnsureCapacity(
+    uint32_t block_index) {
+  BlockVector* blocks = blocks_.load(std::memory_order_acquire);
+  if V8_UNLIKELY (block_index >= blocks->size()) {
+    base::MutexGuard table_grow_guard(&grow_mutex_);
+    // Reload the vector, as another thread could have grown it.
+    blocks = blocks_.load(std::memory_order_relaxed);
+    // Check again if we need to grow under lock.
+    if (block_index >= blocks->size()) {
+      // Grow the vector if the block to insert is greater than the vectors
+      // capacity.
+      if (block_index >= blocks->capacity()) {
+        std::unique_ptr<BlockVector> new_blocks =
+            BlockVector::Grow(blocks, blocks->capacity() * 2, grow_mutex_);
+        block_vector_storage_.push_back(std::move(new_blocks));
+        blocks = block_vector_storage_.back().get();
+        blocks_.store(blocks, std::memory_order_release);
+      }
+      const uint32_t capacity = CapacityForBlock(block_index);
+      std::unique_ptr<Block> new_block = Block::New(capacity);
+      blocks->AddBlock(std::move(new_block));
+    }
+  }
+  return blocks;
+}
+
+int StringForwardingTable::AddForwardString(String string, String forward_to) {
+  DCHECK_IMPLIES(!FLAG_always_use_string_forwarding_table,
+                 string.InSharedHeap());
+  DCHECK_IMPLIES(!FLAG_always_use_string_forwarding_table,
+                 forward_to.InSharedHeap());
+  int index = next_free_index_++;
+  uint32_t index_in_block;
+  const uint32_t block_index = BlockForIndex(index, &index_in_block);
+
+  BlockVector* blocks = EnsureCapacity(block_index);
+  Block* block = blocks->LoadBlock(block_index, kAcquireLoad);
+  block->record(index_in_block)->SetInternalized(string, forward_to);
+  return index;
+}
+
+String StringForwardingTable::GetForwardString(PtrComprCageBase cage_base,
+                                               int index) const {
+  CHECK_LT(index, size());
+  uint32_t index_in_block;
+  const uint32_t block_index = BlockForIndex(index, &index_in_block);
+  Block* block = blocks_.load(std::memory_order_acquire)
+                     ->LoadBlock(block_index, kAcquireLoad);
+  return block->record(index_in_block)->forward_string(cage_base);
+}
+
+// static
+Address StringForwardingTable::GetForwardStringAddress(Isolate* isolate,
+                                                       int index) {
+  return isolate->string_forwarding_table()
+      ->GetForwardString(isolate, index)
+      .ptr();
+}
+
+uint32_t StringForwardingTable::GetRawHash(PtrComprCageBase cage_base,
+                                           int index) const {
+  CHECK_LT(index, size());
+  uint32_t index_in_block;
+  const uint32_t block_index = BlockForIndex(index, &index_in_block);
+  Block* block = blocks_.load(std::memory_order_acquire)
+                     ->LoadBlock(block_index, kAcquireLoad);
+  return block->record(index_in_block)->raw_hash(cage_base);
+}
+
+void StringForwardingTable::Reset() {
+  isolate_->heap()->safepoint()->AssertActive();
+  DCHECK_NE(isolate_->heap()->gc_state(), Heap::NOT_IN_GC);
+
+  BlockVector* blocks = blocks_.load(std::memory_order_relaxed);
+  for (uint32_t block_index = 0; block_index < blocks->size(); ++block_index) {
+    delete blocks->LoadBlock(block_index);
+  }
+
+  block_vector_storage_.clear();
+  InitializeBlockVector();
+  next_free_index_ = 0;
+}
+
+void StringForwardingTable::UpdateAfterEvacuation() {
+  DCHECK(FLAG_always_use_string_forwarding_table);
+
+  if (empty()) return;
+
+  BlockVector* blocks = blocks_.load(std::memory_order_relaxed);
+  const unsigned int last_block_index =
+      static_cast<unsigned int>(blocks->size() - 1);
+  for (unsigned int block_index = 0; block_index < last_block_index;
+       ++block_index) {
+    Block* block = blocks->LoadBlock(block_index, kAcquireLoad);
+    block->UpdateAfterEvacuation(isolate_);
+  }
+  // Handle last block separately, as it is not filled to capacity.
+  const int max_index = IndexInBlock(size() - 1, last_block_index) + 1;
+  blocks->LoadBlock(last_block_index, kAcquireLoad)
+      ->UpdateAfterEvacuation(isolate_, max_index);
+}
+
+}  // namespace internal
+}  // namespace v8

src/objects/string-forwarding-table.h

+// Copyright 2022 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_OBJECTS_STRING_FORWARDING_TABLE_H_
+#define V8_OBJECTS_STRING_FORWARDING_TABLE_H_
+
+#include "src/objects/string.h"
+
+// Has to be the last include (doesn't have include guards):
+#include "src/objects/object-macros.h"
+
+namespace v8 {
+namespace internal {
+
+// Mapping from forwarding indices (stored in a string's hash field) to
+// internalized strings.
+// The table is organised in "blocks". As writes only append new entries, the
+// organisation in blocks allows lock-free writes. We need a lock only for
+// growing the table (adding more blocks). When the vector holding the blocks
+// needs to grow, we keep a copy of the old vector alive to allow concurrent
+// reads while the vector is relocated.
+class StringForwardingTable {
+ public:
+  // Capacity for the first block.
+  static constexpr int kInitialBlockSize = 16;
+  static_assert(base::bits::IsPowerOfTwo(kInitialBlockSize));
+  static constexpr int kInitialBlockSizeHighestBit =
+      kBitsPerInt - base::bits::CountLeadingZeros32(kInitialBlockSize) - 1;
+  // Initial capacity in the block vector.
+  static constexpr int kInitialBlockVectorCapacity = 4;
+  static constexpr Smi unused_element() { return Smi::FromInt(0); }
+  static constexpr Smi deleted_element() { return Smi::FromInt(1); }
+
+  explicit StringForwardingTable(Isolate* isolate);
+  ~StringForwardingTable();
+
+  inline int size() const;
+  inline bool empty() const;
+  // Returns the index of the added record.
+  int AddForwardString(String string, String forward_to);
+  String GetForwardString(PtrComprCageBase cage_base, int index) const;
+  static Address GetForwardStringAddress(Isolate* isolate, int index);
+  V8_EXPORT_PRIVATE uint32_t GetRawHash(PtrComprCageBase cage_base,
+                                        int index) const;
+  template <typename Func>
+  V8_INLINE void IterateElements(Isolate* isolate, Func&& callback);
+  void Reset();
+  void UpdateAfterEvacuation();
+
+  class Record;
+
+ private:
+  class Block;
+  class BlockVector;
+
+  // Returns the block for a given index and sets the index within this block
+  // as out parameter.
+  static inline uint32_t BlockForIndex(int index, uint32_t* index_in_block_out);
+  static inline uint32_t IndexInBlock(int index, uint32_t block);
+  static inline uint32_t CapacityForBlock(uint32_t block);
+
+  void InitializeBlockVector();
+  // Ensure that |block| exists in the BlockVector already. If not, a new block
+  // is created (with capacity double the capacity of the last block) and
+  // inserted into the BlockVector. The BlockVector itself might grow (to double
+  // the capacity).
+  BlockVector* EnsureCapacity(uint32_t block);
+
+  Isolate* isolate_;
+  std::atomic<BlockVector*> blocks_;
+  // We need a vector of BlockVectors to keep old BlockVectors alive when we
+  // grow the table, due to concurrent reads that may still hold a pointer to
+  // them. |block_vector_sotrage_| is only accessed while we grow with the mutex
+  // held. All regular access go through |block_|, which holds a pointer to the
+  // current BlockVector.
+  std::vector<std::unique_ptr<BlockVector>> block_vector_storage_;
+  std::atomic<int> next_free_index_;
+  base::Mutex grow_mutex_;
+};
+
+}  // namespace internal
+}  // namespace v8
+
+#include "src/objects/object-macros-undef.h"
+
+#endif  // V8_OBJECTS_STRING_FORWARDING_TABLE_H_

src/objects/string-table-inl.h

@@ -24,37 +24,6 @@ uint32_t StringTableKey::hash() const {
   return Name::HashBits::decode(raw_hash_field_);
 }
 
-int StringForwardingTable::Size() const { return next_free_index_; }
-
-// static
-uint32_t StringForwardingTable::BlockForIndex(int index,
-                                              uint32_t* index_in_block) {
-  DCHECK_GE(index, 0);
-  DCHECK_NOT_NULL(index_in_block);
-  // The block is the leftmost set bit of the index, corrected by the size
-  // of the first block.
-  const uint32_t block = kBitsPerInt -
-                         base::bits::CountLeadingZeros(
-                             static_cast<uint32_t>(index + kInitialBlockSize)) -
-                         kInitialBlockSizeHighestBit - 1;
-  *index_in_block = IndexInBlock(index, block);
-  return block;
-}
-
-// static
-uint32_t StringForwardingTable::IndexInBlock(int index, uint32_t block) {
-  DCHECK_GE(index, 0);
-  // Clear out the leftmost set bit (the block) to get the index within the
-  // block.
-  return static_cast<uint32_t>(index + kInitialBlockSize) &
-         ~(1u << (block + kInitialBlockSizeHighestBit));
-}
-
-// static
-uint32_t StringForwardingTable::CapacityForBlock(uint32_t block) {
-  return 1u << (block + kInitialBlockSizeHighestBit);
-}
-
 }  // namespace internal
 }  // namespace v8
 

src/objects/string-table.h

@@ -95,66 +95,6 @@ class V8_EXPORT_PRIVATE StringTable {
   Isolate* isolate_;
 };
 
-// Mapping from forwarding indices (stored in a string's hash field) to
-// internalized strings.
-// The table is organised in "blocks". As writes only append new entries, the
-// organisation in blocks allows lock-free writes. We need a lock only for
-// growing the table (adding more blocks). When the vector holding the blocks
-// needs to grow, we keep a copy of the old vector alive to allow concurrent
-// reads while the vector is relocated.
-class StringForwardingTable {
- public:
-  // Capacity for the first block.
-  static constexpr int kInitialBlockSize = 16;
-  static_assert(base::bits::IsPowerOfTwo(kInitialBlockSize));
-  static constexpr int kInitialBlockSizeHighestBit =
-      kBitsPerInt - base::bits::CountLeadingZeros32(kInitialBlockSize) - 1;
-  // Initial capacity in the block vector.
-  static constexpr int kInitialBlockVectorCapacity = 4;
-  static constexpr Smi deleted_element() { return Smi::FromInt(0); }
-
-  explicit StringForwardingTable(Isolate* isolate);
-  ~StringForwardingTable();
-
-  inline int Size() const;
-  // Returns the index of the added string pair.
-  int Add(Isolate* isolate, String string, String forward_to);
-  String GetForwardString(Isolate* isolate, int index) const;
-  static Address GetForwardStringAddress(Isolate* isolate, int index);
-  V8_EXPORT_PRIVATE uint32_t GetRawHash(Isolate* isolate, int index) const;
-  void IterateElements(RootVisitor* visitor);
-  void Reset();
-  void UpdateAfterEvacuation();
-
- private:
-  class Block;
-  class BlockVector;
-
-  // Returns the block for a given index and sets the index within this block
-  // as out parameter.
-  static inline uint32_t BlockForIndex(int index, uint32_t* index_in_block_out);
-  static inline uint32_t IndexInBlock(int index, uint32_t block);
-  static inline uint32_t CapacityForBlock(uint32_t block);
-
-  void InitializeBlockVector();
-  // Ensure that |block| exists in the BlockVector already. If not, a new block
-  // is created (with capacity double the capacity of the last block) and
-  // inserted into the BlockVector. The BlockVector itself might grow (to double
-  // the capacity).
-  BlockVector* EnsureCapacity(uint32_t block);
-
-  Isolate* isolate_;
-  std::atomic<BlockVector*> blocks_;
-  // We need a vector of BlockVectors to keep old BlockVectors alive when we
-  // grow the table, due to concurrent reads that may still hold a pointer to
-  // them. |block_vector_sotrage_| is only accessed while we grow with the mutex
-  // held. All regular access go through |block_|, which holds a pointer to the
-  // current BlockVector.
-  std::vector<std::unique_ptr<BlockVector>> block_vector_storage_;
-  std::atomic<int> next_free_index_;
-  base::Mutex grow_mutex_;
-};
-
 }  // namespace internal
 }  // namespace v8
 

src/objects/visitors.h

@@ -18,7 +18,6 @@ class CodeDataContainer;
 
 #define ROOT_ID_LIST(V)                                 \
   V(kStringTable, "(Internalized strings)")             \
-  V(kStringForwardingTable, "(Forwarded strings)")      \
   V(kExternalStringsTable, "(External strings)")        \
   V(kReadOnlyRootList, "(Read-only roots)")             \
   V(kStrongRootList, "(Strong roots)")                  \

test/cctest/test-shared-strings.cc

@@ -13,6 +13,7 @@
 #include "src/heap/remembered-set.h"
 #include "src/objects/fixed-array.h"
 #include "src/objects/objects-inl.h"
+#include "src/objects/string-forwarding-table-inl.h"
 #include "test/cctest/cctest.h"
 #include "test/cctest/heap/heap-utils.h"
 
@@ -901,7 +902,7 @@ UNINITIALIZED_TEST(SharedStringsTransitionDuringGC) {
     i_isolate->heap()->CollectSharedGarbage(GarbageCollectionReason::kTesting);
 
     // Check that GC cleared the forwarding table.
-    CHECK_EQ(i_isolate->string_forwarding_table()->Size(), 0);
+    CHECK_EQ(i_isolate->string_forwarding_table()->size(), 0);
 
     // Check all strings are transitioned to ThinStrings
     for (int i = 0; i < shared_strings->length(); i++) {