global-handles.cc 57.1 KB
Newer Older
1
// Copyright 2009 the V8 project authors. All rights reserved.
2 3
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
4

5
#include "src/handles/global-handles.h"
6

7
#include <algorithm>
8 9
#include <map>

10
#include "src/api/api-inl.h"
11
#include "src/base/compiler-specific.h"
12
#include "src/execution/vm-state-inl.h"
13
#include "src/heap/embedder-tracing.h"
14
#include "src/heap/heap-write-barrier-inl.h"
15
#include "src/init/v8.h"
16
#include "src/logging/counters.h"
17
#include "src/objects/objects-inl.h"
18
#include "src/objects/slots.h"
19
#include "src/objects/visitors.h"
20
#include "src/sanitizer/asan.h"
21 22
#include "src/tasks/cancelable-task.h"
#include "src/tasks/task-utils.h"
23
#include "src/utils/utils.h"
24

25 26
namespace v8 {
namespace internal {
27

28 29
namespace {

30 31 32 33
// Specifies whether V8 expects the holder memory of a global handle to be live
// or dead.
enum class HandleHolder { kLive, kDead };

34 35 36 37 38 39 40 41 42 43
constexpr size_t kBlockSize = 256;

}  // namespace

template <class _NodeType>
class GlobalHandles::NodeBlock final {
 public:
  using BlockType = NodeBlock<_NodeType>;
  using NodeType = _NodeType;

44
  V8_INLINE static const NodeBlock* From(const NodeType* node);
45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
  V8_INLINE static NodeBlock* From(NodeType* node);

  NodeBlock(GlobalHandles* global_handles,
            GlobalHandles::NodeSpace<NodeType>* space,
            NodeBlock* next) V8_NOEXCEPT : next_(next),
                                           global_handles_(global_handles),
                                           space_(space) {}

  NodeType* at(size_t index) { return &nodes_[index]; }
  const NodeType* at(size_t index) const { return &nodes_[index]; }
  GlobalHandles::NodeSpace<NodeType>* space() const { return space_; }
  GlobalHandles* global_handles() const { return global_handles_; }

  V8_INLINE bool IncreaseUsage();
  V8_INLINE bool DecreaseUsage();

  V8_INLINE void ListAdd(NodeBlock** top);
  V8_INLINE void ListRemove(NodeBlock** top);

  NodeBlock* next() const { return next_; }
  NodeBlock* next_used() const { return next_used_; }

 private:
  NodeType nodes_[kBlockSize];
  NodeBlock* const next_;
  GlobalHandles* const global_handles_;
  GlobalHandles::NodeSpace<NodeType>* const space_;
  NodeBlock* next_used_ = nullptr;
  NodeBlock* prev_used_ = nullptr;
  uint32_t used_nodes_ = 0;

  DISALLOW_COPY_AND_ASSIGN(NodeBlock);
};

79 80 81 82 83 84 85 86 87 88
template <class NodeType>
const GlobalHandles::NodeBlock<NodeType>*
GlobalHandles::NodeBlock<NodeType>::From(const NodeType* node) {
  uintptr_t ptr = reinterpret_cast<const uintptr_t>(node) -
                  sizeof(NodeType) * node->index();
  const BlockType* block = reinterpret_cast<const BlockType*>(ptr);
  DCHECK_EQ(node, block->at(node->index()));
  return block;
}

89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
template <class NodeType>
GlobalHandles::NodeBlock<NodeType>* GlobalHandles::NodeBlock<NodeType>::From(
    NodeType* node) {
  uintptr_t ptr =
      reinterpret_cast<uintptr_t>(node) - sizeof(NodeType) * node->index();
  BlockType* block = reinterpret_cast<BlockType*>(ptr);
  DCHECK_EQ(node, block->at(node->index()));
  return block;
}

template <class NodeType>
bool GlobalHandles::NodeBlock<NodeType>::IncreaseUsage() {
  DCHECK_LT(used_nodes_, kBlockSize);
  return used_nodes_++ == 0;
}

template <class NodeType>
void GlobalHandles::NodeBlock<NodeType>::ListAdd(BlockType** top) {
  BlockType* old_top = *top;
  *top = this;
  next_used_ = old_top;
  prev_used_ = nullptr;
  if (old_top != nullptr) {
    old_top->prev_used_ = this;
  }
}

template <class NodeType>
bool GlobalHandles::NodeBlock<NodeType>::DecreaseUsage() {
  DCHECK_GT(used_nodes_, 0);
  return --used_nodes_ == 0;
}

template <class NodeType>
void GlobalHandles::NodeBlock<NodeType>::ListRemove(BlockType** top) {
  if (next_used_ != nullptr) next_used_->prev_used_ = prev_used_;
  if (prev_used_ != nullptr) prev_used_->next_used_ = next_used_;
  if (this == *top) {
    *top = next_used_;
  }
}

template <class BlockType>
class GlobalHandles::NodeIterator final {
 public:
  using NodeType = typename BlockType::NodeType;

  // Iterator traits.
  using iterator_category = std::forward_iterator_tag;
  using difference_type = std::ptrdiff_t;
  using value_type = NodeType*;
  using reference = value_type;
  using pointer = value_type*;

  explicit NodeIterator(BlockType* block) V8_NOEXCEPT : block_(block) {}
  NodeIterator(NodeIterator&& other) V8_NOEXCEPT : block_(other.block_),
                                                   index_(other.index_) {}

  bool operator==(const NodeIterator& other) const {
    return block_ == other.block_;
  }
  bool operator!=(const NodeIterator& other) const {
    return block_ != other.block_;
  }

  NodeIterator& operator++() {
    if (++index_ < kBlockSize) return *this;
    index_ = 0;
    block_ = block_->next_used();
    return *this;
  }

  NodeType* operator*() { return block_->at(index_); }
  NodeType* operator->() { return block_->at(index_); }

 private:
  BlockType* block_ = nullptr;
  size_t index_ = 0;

  DISALLOW_COPY_AND_ASSIGN(NodeIterator);
};

template <class NodeType>
class GlobalHandles::NodeSpace final {
 public:
  using BlockType = NodeBlock<NodeType>;
  using iterator = NodeIterator<BlockType>;

  static NodeSpace* From(NodeType* node);
  static void Release(NodeType* node);

  explicit NodeSpace(GlobalHandles* global_handles) V8_NOEXCEPT
      : global_handles_(global_handles) {}
  ~NodeSpace();

  V8_INLINE NodeType* Acquire(Object object);

  iterator begin() { return iterator(first_used_block_); }
  iterator end() { return iterator(nullptr); }

189
  size_t TotalSize() const { return blocks_ * sizeof(NodeType) * kBlockSize; }
190
  size_t handles_count() const { return handles_count_; }
191

192 193 194 195 196 197 198 199
 private:
  void PutNodesOnFreeList(BlockType* block);
  V8_INLINE void Free(NodeType* node);

  GlobalHandles* const global_handles_;
  BlockType* first_block_ = nullptr;
  BlockType* first_used_block_ = nullptr;
  NodeType* first_free_ = nullptr;
200
  size_t blocks_ = 0;
201
  size_t handles_count_ = 0;
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
};

template <class NodeType>
GlobalHandles::NodeSpace<NodeType>::~NodeSpace() {
  auto* block = first_block_;
  while (block != nullptr) {
    auto* tmp = block->next();
    delete block;
    block = tmp;
  }
}

template <class NodeType>
NodeType* GlobalHandles::NodeSpace<NodeType>::Acquire(Object object) {
  if (first_free_ == nullptr) {
    first_block_ = new BlockType(global_handles_, this, first_block_);
218
    blocks_++;
219 220 221 222 223 224 225 226 227 228 229
    PutNodesOnFreeList(first_block_);
  }
  DCHECK_NOT_NULL(first_free_);
  NodeType* node = first_free_;
  first_free_ = first_free_->next_free();
  node->Acquire(object);
  BlockType* block = BlockType::From(node);
  if (block->IncreaseUsage()) {
    block->ListAdd(&first_used_block_);
  }
  global_handles_->isolate()->counters()->global_handles()->Increment();
230
  handles_count_++;
231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261
  DCHECK(node->IsInUse());
  return node;
}

template <class NodeType>
void GlobalHandles::NodeSpace<NodeType>::PutNodesOnFreeList(BlockType* block) {
  for (int32_t i = kBlockSize - 1; i >= 0; --i) {
    NodeType* node = block->at(i);
    const uint8_t index = static_cast<uint8_t>(i);
    DCHECK_EQ(i, index);
    node->set_index(index);
    node->Free(first_free_);
    first_free_ = node;
  }
}

template <class NodeType>
void GlobalHandles::NodeSpace<NodeType>::Release(NodeType* node) {
  BlockType* block = BlockType::From(node);
  block->space()->Free(node);
}

template <class NodeType>
void GlobalHandles::NodeSpace<NodeType>::Free(NodeType* node) {
  node->Release(first_free_);
  first_free_ = node;
  BlockType* block = BlockType::From(node);
  if (block->DecreaseUsage()) {
    block->ListRemove(&first_used_block_);
  }
  global_handles_->isolate()->counters()->global_handles()->Decrement();
262
  handles_count_--;
263 264
}

265 266
template <class Child>
class NodeBase {
267
 public:
268 269 270 271
  static const Child* FromLocation(const Address* location) {
    return reinterpret_cast<const Child*>(location);
  }

272 273
  static Child* FromLocation(Address* location) {
    return reinterpret_cast<Child*>(location);
274 275
  }

276 277 278 279
  NodeBase() {
    DCHECK_EQ(offsetof(NodeBase, object_), 0);
    DCHECK_EQ(offsetof(NodeBase, class_id_), Internals::kNodeClassIdOffset);
    DCHECK_EQ(offsetof(NodeBase, flags_), Internals::kNodeFlagsOffset);
280
  }
281

282
#ifdef ENABLE_HANDLE_ZAPPING
283
  ~NodeBase() {
284
    ClearFields();
285
    data_.next_free = nullptr;
286
    index_ = 0;
287
  }
288
#endif
289

290
  void Free(Child* free_list) {
291
    ClearFields();
292
    AsChild()->MarkAsFree();
293
    data_.next_free = free_list;
294 295
  }

296
  void Acquire(Object object) {
297
    DCHECK(!AsChild()->IsInUse());
298
    CheckFieldsAreCleared();
299
    object_ = object.ptr();
300
    AsChild()->MarkAsUsed();
301
    data_.parameter = nullptr;
302
    DCHECK(AsChild()->IsInUse());
303 304
  }

305 306
  void Release(Child* free_list) {
    DCHECK(AsChild()->IsInUse());
307
    Free(free_list);
308
    DCHECK(!AsChild()->IsInUse());
309 310
  }

311
  Object object() const { return Object(object_); }
312
  FullObjectSlot location() { return FullObjectSlot(&object_); }
313
  Handle<Object> handle() { return Handle<Object>(&object_); }
314

315 316 317 318
  uint8_t index() const { return index_; }
  void set_index(uint8_t value) { index_ = value; }

  uint16_t wrapper_class_id() const { return class_id_; }
319 320 321
  bool has_wrapper_class_id() const {
    return class_id_ != v8::HeapProfiler::kPersistentHandleNoClassId;
  }
322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379

  // Accessors for next free node in the free list.
  Child* next_free() {
    DCHECK(!AsChild()->IsInUse());
    return data_.next_free;
  }

  void set_parameter(void* parameter) {
    DCHECK(AsChild()->IsInUse());
    data_.parameter = parameter;
  }
  void* parameter() const {
    DCHECK(AsChild()->IsInUse());
    return data_.parameter;
  }

 protected:
  Child* AsChild() { return reinterpret_cast<Child*>(this); }
  const Child* AsChild() const { return reinterpret_cast<const Child*>(this); }

  void ClearFields() {
    // Zap the values for eager trapping.
    object_ = kGlobalHandleZapValue;
    class_id_ = v8::HeapProfiler::kPersistentHandleNoClassId;
    AsChild()->ClearImplFields();
  }

  void CheckFieldsAreCleared() {
    DCHECK_EQ(kGlobalHandleZapValue, object_);
    DCHECK_EQ(v8::HeapProfiler::kPersistentHandleNoClassId, class_id_);
    AsChild()->CheckImplFieldsAreCleared();
  }

  // Storage for object pointer.
  //
  // Placed first to avoid offset computation. The stored data is equivalent to
  // an Object. It is stored as a plain Address for convenience (smallest number
  // of casts), and because it is a private implementation detail: the public
  // interface provides type safety.
  Address object_;

  // Class id set by the embedder.
  uint16_t class_id_;

  // Index in the containing handle block.
  uint8_t index_;

  uint8_t flags_;

  // The meaning of this field depends on node state:
  // - Node in free list: Stores next free node pointer.
  // - Otherwise, specific to the node implementation.
  union {
    Child* next_free;
    void* parameter;
  } data_;
};

380 381 382
namespace {

void ExtractInternalFields(JSObject jsobject, void** embedder_fields, int len) {
383
  int field_count = jsobject.GetEmbedderFieldCount();
384
  const Isolate* isolate = GetIsolateForPtrCompr(jsobject);
385 386 387
  for (int i = 0; i < len; ++i) {
    if (field_count == i) break;
    void* pointer;
388
    if (EmbedderDataSlot(jsobject, i).ToAlignedPointer(isolate, &pointer)) {
389 390 391 392 393 394 395
      embedder_fields[i] = pointer;
    }
  }
}

}  // namespace

396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
class GlobalHandles::Node final : public NodeBase<GlobalHandles::Node> {
 public:
  // State transition diagram:
  // FREE -> NORMAL <-> WEAK -> PENDING -> NEAR_DEATH -> { NORMAL, WEAK, FREE }
  enum State {
    FREE = 0,
    NORMAL,      // Normal global handle.
    WEAK,        // Flagged as weak but not yet finalized.
    PENDING,     // Has been recognized as only reachable by weak handles.
    NEAR_DEATH,  // Callback has informed the handle is near death.
    NUMBER_OF_NODE_STATES
  };

  Node() {
    STATIC_ASSERT(static_cast<int>(NodeState::kMask) ==
                  Internals::kNodeStateMask);
    STATIC_ASSERT(WEAK == Internals::kNodeStateIsWeakValue);
    STATIC_ASSERT(PENDING == Internals::kNodeStateIsPendingValue);
414
    set_in_young_list(false);
415 416 417 418 419 420 421 422 423 424 425 426
  }

  void Zap() {
    DCHECK(IsInUse());
    // Zap the values for eager trapping.
    object_ = kGlobalHandleZapValue;
  }

  const char* label() const {
    return state() == NORMAL ? reinterpret_cast<char*>(data_.parameter)
                             : nullptr;
  }
427 428 429

  // State and flag accessors.

430 431
  State state() const { return NodeState::decode(flags_); }
  void set_state(State state) { flags_ = NodeState::update(flags_, state); }
432

433 434
  bool is_in_young_list() const { return IsInYoungList::decode(flags_); }
  void set_in_young_list(bool v) { flags_ = IsInYoungList::update(flags_, v); }
435

436 437
  WeaknessType weakness_type() const {
    return NodeWeaknessType::decode(flags_);
438
  }
439 440
  void set_weakness_type(WeaknessType weakness_type) {
    flags_ = NodeWeaknessType::update(flags_, weakness_type);
441 442
  }

443
  bool IsWeak() const { return state() == WEAK; }
444

445 446
  bool IsInUse() const { return state() != FREE; }

447 448 449 450 451 452 453 454 455
  bool IsPhantomCallback() const {
    return weakness_type() == PHANTOM_WEAK ||
           weakness_type() == PHANTOM_WEAK_2_EMBEDDER_FIELDS;
  }

  bool IsPhantomResetHandle() const {
    return weakness_type() == PHANTOM_WEAK_RESET_HANDLE;
  }

456 457
  bool IsFinalizerHandle() const { return weakness_type() == FINALIZER_WEAK; }

458
  bool IsPendingPhantomCallback() const {
459
    return state() == PENDING && IsPhantomCallback();
460 461 462
  }

  bool IsPendingPhantomResetHandle() const {
463
    return state() == PENDING && IsPhantomResetHandle();
464 465
  }

466 467 468 469 470 471
  bool IsPendingFinalizer() const {
    return state() == PENDING && weakness_type() == FINALIZER_WEAK;
  }

  bool IsPending() const { return state() == PENDING; }

472 473
  bool IsRetainer() const {
    return state() != FREE &&
474
           !(state() == NEAR_DEATH && weakness_type() != FINALIZER_WEAK);
475
  }
476

477
  bool IsStrongRetainer() const { return state() == NORMAL; }
478 479

  bool IsWeakRetainer() const {
480
    return state() == WEAK || state() == PENDING ||
481
           (state() == NEAR_DEATH && weakness_type() == FINALIZER_WEAK);
482 483
  }

484
  void MarkPending() {
485
    DCHECK(state() == WEAK);
486
    set_state(PENDING);
487 488
  }

489 490
  bool has_callback() const { return weak_callback_ != nullptr; }

491 492
  // Accessors for next free node in the free list.
  Node* next_free() {
493
    DCHECK_EQ(FREE, state());
494
    return data_.next_free;
495 496
  }

497 498 499
  void MakeWeak(void* parameter,
                WeakCallbackInfo<void>::Callback phantom_callback,
                v8::WeakCallbackType type) {
500
    DCHECK_NOT_NULL(phantom_callback);
501
    DCHECK(IsInUse());
502
    CHECK_NE(object_, kGlobalHandleZapValue);
503
    set_state(WEAK);
504 505 506 507 508
    switch (type) {
      case v8::WeakCallbackType::kParameter:
        set_weakness_type(PHANTOM_WEAK);
        break;
      case v8::WeakCallbackType::kInternalFields:
509
        set_weakness_type(PHANTOM_WEAK_2_EMBEDDER_FIELDS);
510 511
        break;
      case v8::WeakCallbackType::kFinalizer:
512
        set_weakness_type(FINALIZER_WEAK);
513
        break;
514
    }
515
    set_parameter(parameter);
516
    weak_callback_ = phantom_callback;
517 518
  }

519
  void MakeWeak(Address** location_addr) {
520
    DCHECK(IsInUse());
521
    CHECK_NE(object_, kGlobalHandleZapValue);
522 523 524 525 526 527
    set_state(WEAK);
    set_weakness_type(PHANTOM_WEAK_RESET_HANDLE);
    set_parameter(location_addr);
    weak_callback_ = nullptr;
  }

528
  void* ClearWeakness() {
529
    DCHECK(IsInUse());
530
    void* p = parameter();
531
    set_state(NORMAL);
532
    set_parameter(nullptr);
533
    return p;
534 535
  }

536 537
  void AnnotateStrongRetainer(const char* label) {
    DCHECK_EQ(state(), NORMAL);
538
    data_.parameter = const_cast<char*>(label);
539 540
  }

541
  void CollectPhantomCallbackData(
542 543
      std::vector<std::pair<Node*, PendingPhantomCallback>>*
          pending_phantom_callbacks) {
544
    DCHECK(weakness_type() == PHANTOM_WEAK ||
545
           weakness_type() == PHANTOM_WEAK_2_EMBEDDER_FIELDS);
546
    DCHECK(state() == PENDING);
547
    DCHECK_NOT_NULL(weak_callback_);
548

549
    void* embedder_fields[v8::kEmbedderFieldsInWeakCallback] = {nullptr,
550
                                                                nullptr};
551
    if (weakness_type() != PHANTOM_WEAK && object().IsJSObject()) {
552 553
      ExtractInternalFields(JSObject::cast(object()), embedder_fields,
                            v8::kEmbedderFieldsInWeakCallback);
554
    }
555

556
    // Zap with something dangerous.
557
    location().store(Object(0xCA11));
558

559 560 561
    pending_phantom_callbacks->push_back(std::make_pair(
        this,
        PendingPhantomCallback(weak_callback_, parameter(), embedder_fields)));
562 563
    DCHECK(IsInUse());
    set_state(NEAR_DEATH);
564 565
  }

566 567
  void ResetPhantomHandle(HandleHolder handle_holder) {
    DCHECK_EQ(HandleHolder::kLive, handle_holder);
568 569
    DCHECK_EQ(PHANTOM_WEAK_RESET_HANDLE, weakness_type());
    DCHECK_EQ(PENDING, state());
570
    DCHECK_NULL(weak_callback_);
571
    Address** handle = reinterpret_cast<Address**>(parameter());
572
    *handle = nullptr;
573
    NodeSpace<Node>::Release(this);
574 575
  }

576 577
  void PostGarbageCollectionProcessing(Isolate* isolate) {
    // This method invokes a finalizer. Updating the method name would require
Dan Elphick's avatar
Dan Elphick committed
578
    // adjusting CFI blocklist as weak_callback_ is invoked on the wrong type.
579
    CHECK(IsPendingFinalizer());
580
    set_state(NEAR_DEATH);
581 582
    // Check that we are not passing a finalized external string to
    // the callback.
583 584 585 586
    DCHECK(!object().IsExternalOneByteString() ||
           ExternalOneByteString::cast(object()).resource() != nullptr);
    DCHECK(!object().IsExternalTwoByteString() ||
           ExternalTwoByteString::cast(object()).resource() != nullptr);
587 588 589
    // Leaving V8.
    VMState<EXTERNAL> vmstate(isolate);
    HandleScope handle_scope(isolate);
590
    void* embedder_fields[v8::kEmbedderFieldsInWeakCallback] = {nullptr,
591 592
                                                                nullptr};
    v8::WeakCallbackInfo<void> data(reinterpret_cast<v8::Isolate*>(isolate),
593
                                    parameter(), embedder_fields, nullptr);
594
    weak_callback_(data);
595 596 597
    // For finalizers the handle must have either been reset or made strong.
    // Both cases reset the state.
    CHECK_NE(NEAR_DEATH, state());
598 599
  }

600 601
  void MarkAsFree() { set_state(FREE); }
  void MarkAsUsed() { set_state(NORMAL); }
602

603 604 605
  GlobalHandles* global_handles() {
    return NodeBlock<Node>::From(this)->global_handles();
  }
606

607
 private:
608
  // Fields that are not used for managing node memory.
609
  void ClearImplFields() { weak_callback_ = nullptr; }
610

611
  void CheckImplFieldsAreCleared() { DCHECK_EQ(nullptr, weak_callback_); }
612

613
  // This stores three flags (independent, partially_dependent and
614
  // in_young_list) and a State.
615
  using NodeState = base::BitField8<State, 0, 3>;
616 617
  using IsInYoungList = NodeState::Next<bool, 1>;
  using NodeWeaknessType = IsInYoungList::Next<WeaknessType, 2>;
618

619
  // Handle specific callback - might be a weak reference in disguise.
620
  WeakCallbackInfo<void>::Callback weak_callback_;
621

622
  friend class NodeBase<Node>;
623

624 625 626
  DISALLOW_COPY_AND_ASSIGN(Node);
};

627 628 629
class GlobalHandles::TracedNode final
    : public NodeBase<GlobalHandles::TracedNode> {
 public:
630
  TracedNode() { set_in_young_list(false); }
631

632 633 634 635 636 637
  // Copy and move ctors are used when constructing a TracedNode when recording
  // a node for on-stack data structures. (Older compilers may refer to copy
  // instead of move ctor.)
  TracedNode(TracedNode&& other) V8_NOEXCEPT = default;
  TracedNode(const TracedNode& other) V8_NOEXCEPT = default;

638
  enum State { FREE = 0, NORMAL, NEAR_DEATH };
639 640 641 642 643 644 645

  State state() const { return NodeState::decode(flags_); }
  void set_state(State state) { flags_ = NodeState::update(flags_, state); }

  void MarkAsFree() { set_state(FREE); }
  void MarkAsUsed() { set_state(NORMAL); }
  bool IsInUse() const { return state() != FREE; }
646 647
  bool IsRetainer() const { return state() == NORMAL; }
  bool IsPhantomResetHandle() const { return callback_ == nullptr; }
648

649 650
  bool is_in_young_list() const { return IsInYoungList::decode(flags_); }
  void set_in_young_list(bool v) { flags_ = IsInYoungList::update(flags_, v); }
651 652 653 654

  bool is_root() const { return IsRoot::decode(flags_); }
  void set_root(bool v) { flags_ = IsRoot::update(flags_, v); }

655 656 657
  bool has_destructor() const { return HasDestructor::decode(flags_); }
  void set_has_destructor(bool v) { flags_ = HasDestructor::update(flags_, v); }

658 659 660 661
  bool markbit() const { return Markbit::decode(flags_); }
  void clear_markbit() { flags_ = Markbit::update(flags_, false); }
  void set_markbit() { flags_ = Markbit::update(flags_, true); }

662 663 664
  bool is_on_stack() const { return IsOnStack::decode(flags_); }
  void set_is_on_stack(bool v) { flags_ = IsOnStack::update(flags_, v); }

665 666 667 668 669
  void SetFinalizationCallback(void* parameter,
                               WeakCallbackInfo<void>::Callback callback) {
    set_parameter(parameter);
    callback_ = callback;
  }
670
  bool HasFinalizationCallback() const { return callback_ != nullptr; }
671

672 673
  void CopyObjectReference(const TracedNode& other) { object_ = other.object_; }

674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692
  void CollectPhantomCallbackData(
      std::vector<std::pair<TracedNode*, PendingPhantomCallback>>*
          pending_phantom_callbacks) {
    DCHECK(IsInUse());
    DCHECK_NOT_NULL(callback_);

    void* embedder_fields[v8::kEmbedderFieldsInWeakCallback] = {nullptr,
                                                                nullptr};
    ExtractInternalFields(JSObject::cast(object()), embedder_fields,
                          v8::kEmbedderFieldsInWeakCallback);

    // Zap with something dangerous.
    location().store(Object(0xCA11));

    pending_phantom_callbacks->push_back(std::make_pair(
        this, PendingPhantomCallback(callback_, parameter(), embedder_fields)));
    set_state(NEAR_DEATH);
  }

693
  void ResetPhantomHandle(HandleHolder handle_holder) {
694
    DCHECK(IsInUse());
695 696 697 698
    if (handle_holder == HandleHolder::kLive) {
      Address** handle = reinterpret_cast<Address**>(data_.parameter);
      *handle = nullptr;
    }
699 700 701 702
    NodeSpace<TracedNode>::Release(this);
    DCHECK(!IsInUse());
  }

703
  static void Verify(GlobalHandles* global_handles, const Address* const* slot);
704

705
 protected:
706
  using NodeState = base::BitField8<State, 0, 2>;
707 708
  using IsInYoungList = NodeState::Next<bool, 1>;
  using IsRoot = IsInYoungList::Next<bool, 1>;
709
  using HasDestructor = IsRoot::Next<bool, 1>;
710
  using Markbit = HasDestructor::Next<bool, 1>;
711
  using IsOnStack = Markbit::Next<bool, 1>;
712

713 714
  void ClearImplFields() {
    set_root(true);
715 716
    // Nodes are black allocated for simplicity.
    set_markbit();
717
    callback_ = nullptr;
718
    set_is_on_stack(false);
719
    set_has_destructor(false);
720 721 722 723
  }

  void CheckImplFieldsAreCleared() const {
    DCHECK(is_root());
724
    DCHECK(markbit());
725 726 727 728
    DCHECK_NULL(callback_);
  }

  WeakCallbackInfo<void>::Callback callback_;
729 730 731

  friend class NodeBase<GlobalHandles::TracedNode>;
};
732

733 734 735 736 737 738 739 740
// Space to keep track of on-stack handles (e.g. TracedReference). Such
// references are treated as root for any V8 garbage collection. The data
// structure is self healing and pessimistally filters outdated entries on
// insertion and iteration.
//
// Design doc: http://bit.ly/on-stack-traced-reference
class GlobalHandles::OnStackTracedNodeSpace final {
 public:
741 742 743 744 745
  static GlobalHandles* GetGlobalHandles(const TracedNode* on_stack_node) {
    DCHECK(on_stack_node->is_on_stack());
    return reinterpret_cast<const NodeEntry*>(on_stack_node)->global_handles;
  }

746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782
  explicit OnStackTracedNodeSpace(GlobalHandles* global_handles)
      : global_handles_(global_handles) {}

  void SetStackStart(void* stack_start) {
    CHECK(on_stack_nodes_.empty());
    stack_start_ =
        GetStackAddressForSlot(reinterpret_cast<uintptr_t>(stack_start));
  }

  bool IsOnStack(uintptr_t slot) const {
    const uintptr_t address = GetStackAddressForSlot(slot);
    return stack_start_ >= address && address > GetCurrentStackPosition();
  }

  void Iterate(RootVisitor* v);
  TracedNode* Acquire(Object value, uintptr_t address);
  void CleanupBelowCurrentStackPosition();
  void NotifyEmptyEmbedderStack();

  size_t NumberOfHandlesForTesting() const { return on_stack_nodes_.size(); }

 private:
  struct NodeEntry {
    TracedNode node;
    // Used to find back to GlobalHandles from a Node on copy. Needs to follow
    // node.
    GlobalHandles* global_handles;
  };

  uintptr_t GetStackAddressForSlot(uintptr_t slot) const;

  // Keeps track of registered handles and their stack address. The data
  // structure is cleaned on iteration and when adding new references using the
  // current stack address.
  std::map<uintptr_t, NodeEntry> on_stack_nodes_;
  uintptr_t stack_start_ = 0;
  GlobalHandles* global_handles_ = nullptr;
783
  size_t acquire_count_ = 0;
784 785 786 787 788 789 790
};

uintptr_t GlobalHandles::OnStackTracedNodeSpace::GetStackAddressForSlot(
    uintptr_t slot) const {
#ifdef V8_USE_ADDRESS_SANITIZER
  void* fake_stack = __asan_get_current_fake_stack();
  if (fake_stack) {
791 792 793 794 795 796
    void* fake_frame_start;
    void* real_frame = __asan_addr_is_in_fake_stack(
        fake_stack, reinterpret_cast<void*>(slot), &fake_frame_start, nullptr);
    if (real_frame) {
      return reinterpret_cast<uintptr_t>(real_frame) +
             (slot - reinterpret_cast<uintptr_t>(fake_frame_start));
797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820
    }
  }
#endif  // V8_USE_ADDRESS_SANITIZER
  return slot;
}

void GlobalHandles::OnStackTracedNodeSpace::NotifyEmptyEmbedderStack() {
  on_stack_nodes_.clear();
}

void GlobalHandles::OnStackTracedNodeSpace::Iterate(RootVisitor* v) {
  // Handles have been cleaned from the GC entry point which is higher up the
  // stack.
  for (auto& pair : on_stack_nodes_) {
    TracedNode& node = pair.second.node;
    if (node.IsRetainer()) {
      v->VisitRootPointer(Root::kGlobalHandles, "on-stack TracedReference",
                          node.location());
    }
  }
}

GlobalHandles::TracedNode* GlobalHandles::OnStackTracedNodeSpace::Acquire(
    Object value, uintptr_t slot) {
821 822 823
  constexpr size_t kAcquireCleanupThresholdLog2 = 8;
  constexpr size_t kAcquireCleanupThresholdMask =
      (size_t{1} << kAcquireCleanupThresholdLog2) - 1;
824
  DCHECK(IsOnStack(slot));
825 826 827
  if (((acquire_count_++) & kAcquireCleanupThresholdMask) == 0) {
    CleanupBelowCurrentStackPosition();
  }
828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851
  NodeEntry entry;
  entry.node.Free(nullptr);
  entry.global_handles = global_handles_;
  auto pair =
      on_stack_nodes_.insert({GetStackAddressForSlot(slot), std::move(entry)});
  if (!pair.second) {
    // Insertion failed because there already was an entry present for that
    // stack address. This can happen because cleanup is conservative in which
    // stack limits it used. Reusing the entry is fine as there's no aliasing of
    // different references with the same stack slot.
    pair.first->second.node.Free(nullptr);
  }
  TracedNode* result = &(pair.first->second.node);
  result->Acquire(value);
  result->set_is_on_stack(true);
  return result;
}

void GlobalHandles::OnStackTracedNodeSpace::CleanupBelowCurrentStackPosition() {
  if (on_stack_nodes_.empty()) return;
  const auto it = on_stack_nodes_.upper_bound(GetCurrentStackPosition());
  on_stack_nodes_.erase(on_stack_nodes_.begin(), it);
}

852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876
// static
void GlobalHandles::TracedNode::Verify(GlobalHandles* global_handles,
                                       const Address* const* slot) {
#ifdef DEBUG
  const TracedNode* node = FromLocation(*slot);
  DCHECK(node->IsInUse());
  DCHECK_IMPLIES(!node->has_destructor(), nullptr == node->parameter());
  DCHECK_IMPLIES(node->has_destructor() && !node->HasFinalizationCallback(),
                 node->parameter());
  bool slot_on_stack = global_handles->on_stack_nodes_->IsOnStack(
      reinterpret_cast<uintptr_t>(slot));
  DCHECK_EQ(slot_on_stack, node->is_on_stack());
  if (!node->is_on_stack()) {
    // On-heap nodes have seprate lists for young generation processing.
    bool is_young_gen_object = ObjectInYoungGeneration(node->object());
    DCHECK_IMPLIES(is_young_gen_object, node->is_in_young_list());
  }
  bool in_young_list =
      std::find(global_handles->traced_young_nodes_.begin(),
                global_handles->traced_young_nodes_.end(),
                node) != global_handles->traced_young_nodes_.end();
  DCHECK_EQ(in_young_list, node->is_in_young_list());
#endif  // DEBUG
}

877 878 879 880 881 882 883 884
void GlobalHandles::CleanupOnStackReferencesBelowCurrentStackPosition() {
  on_stack_nodes_->CleanupBelowCurrentStackPosition();
}

size_t GlobalHandles::NumberOfOnStackHandlesForTesting() {
  return on_stack_nodes_->NumberOfHandlesForTesting();
}

885 886 887 888
size_t GlobalHandles::TotalSize() const {
  return regular_nodes_->TotalSize() + traced_nodes_->TotalSize();
}

889 890 891 892 893 894 895 896 897
size_t GlobalHandles::UsedSize() const {
  return regular_nodes_->handles_count() * sizeof(Node) +
         traced_nodes_->handles_count() * sizeof(TracedNode);
}

size_t GlobalHandles::handles_count() const {
  return regular_nodes_->handles_count() + traced_nodes_->handles_count();
}

898 899 900 901 902 903 904 905
void GlobalHandles::SetStackStart(void* stack_start) {
  on_stack_nodes_->SetStackStart(stack_start);
}

void GlobalHandles::NotifyEmptyEmbedderStack() {
  on_stack_nodes_->NotifyEmptyEmbedderStack();
}

906 907
GlobalHandles::GlobalHandles(Isolate* isolate)
    : isolate_(isolate),
908
      regular_nodes_(new NodeSpace<GlobalHandles::Node>(this)),
909 910
      traced_nodes_(new NodeSpace<GlobalHandles::TracedNode>(this)),
      on_stack_nodes_(new OnStackTracedNodeSpace(this)) {}
911

912
GlobalHandles::~GlobalHandles() { regular_nodes_.reset(nullptr); }
913

914
Handle<Object> GlobalHandles::Create(Object value) {
915
  GlobalHandles::Node* result = regular_nodes_->Acquire(value);
916
  if (ObjectInYoungGeneration(value) && !result->is_in_young_list()) {
917 918
    young_nodes_.push_back(result);
    result->set_in_young_list(true);
919 920 921 922
  }
  return result->handle();
}

923
Handle<Object> GlobalHandles::Create(Address value) {
924
  return Create(Object(value));
925
}
926

927 928
Handle<Object> GlobalHandles::CreateTraced(Object value, Address* slot,
                                           bool has_destructor) {
929 930 931 932 933 934 935 936
  return CreateTraced(
      value, slot, has_destructor,
      on_stack_nodes_->IsOnStack(reinterpret_cast<uintptr_t>(slot)));
}

Handle<Object> GlobalHandles::CreateTraced(Object value, Address* slot,
                                           bool has_destructor,
                                           bool is_on_stack) {
937
  GlobalHandles::TracedNode* result;
938 939
  if (is_on_stack) {
    result = on_stack_nodes_->Acquire(value, reinterpret_cast<uintptr_t>(slot));
940 941 942 943 944 945
  } else {
    result = traced_nodes_->Acquire(value);
    if (ObjectInYoungGeneration(value) && !result->is_in_young_list()) {
      traced_young_nodes_.push_back(result);
      result->set_in_young_list(true);
    }
946
  }
947 948
  result->set_has_destructor(has_destructor);
  result->set_parameter(has_destructor ? slot : nullptr);
949 950 951
  return result->handle();
}

952 953 954
Handle<Object> GlobalHandles::CreateTraced(Address value, Address* slot,
                                           bool has_destructor) {
  return CreateTraced(Object(value), slot, has_destructor);
955 956
}

957
Handle<Object> GlobalHandles::CopyGlobal(Address* location) {
958
  DCHECK_NOT_NULL(location);
959
  GlobalHandles* global_handles =
960
      Node::FromLocation(location)->global_handles();
961 962
#ifdef VERIFY_HEAP
  if (i::FLAG_verify_heap) {
963
    Object(*location).ObjectVerify(global_handles->isolate());
964 965 966
  }
#endif  // VERIFY_HEAP
  return global_handles->Create(*location);
967 968
}

969 970 971 972 973 974 975 976
// static
void GlobalHandles::CopyTracedGlobal(const Address* const* from, Address** to) {
  DCHECK_NOT_NULL(*from);
  DCHECK_NULL(*to);
  const TracedNode* node = TracedNode::FromLocation(*from);
  // Copying a traced handle with finalization callback is prohibited because
  // the callback may require knowing about multiple copies of the traced
  // handle.
977 978 979
  CHECK_WITH_MSG(!node->HasFinalizationCallback(),
                 "Copying of references is not supported when "
                 "SetFinalizationCallback is set.");
980

981
  GlobalHandles* global_handles =
982
      GlobalHandles::From(const_cast<TracedNode*>(node));
983 984 985
  Handle<Object> o = global_handles->CreateTraced(
      node->object(), reinterpret_cast<Address*>(to), node->has_destructor());
  *to = o.location();
986 987
  TracedNode::Verify(global_handles, from);
  TracedNode::Verify(global_handles, to);
988 989 990 991 992 993 994
#ifdef VERIFY_HEAP
  if (i::FLAG_verify_heap) {
    Object(**to).ObjectVerify(global_handles->isolate());
  }
#endif  // VERIFY_HEAP
}

995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008
void GlobalHandles::MoveGlobal(Address** from, Address** to) {
  DCHECK_NOT_NULL(*from);
  DCHECK_NOT_NULL(*to);
  DCHECK_EQ(*from, *to);
  Node* node = Node::FromLocation(*from);
  if (node->IsWeak() && node->IsPhantomResetHandle()) {
    node->set_parameter(to);
  }

  // - Strong handles do not require fixups.
  // - Weak handles with finalizers and callbacks are too general to fix up. For
  //   those the callers need to ensure consistency.
}

1009
void GlobalHandles::MoveTracedGlobal(Address** from, Address** to) {
1010 1011 1012 1013 1014
  // Fast path for moving from an empty reference.
  if (!*from) {
    DestroyTraced(*to);
    *to = nullptr;
    return;
1015
  }
1016 1017 1018

  // Determining whether from or to are on stack.
  TracedNode* from_node = TracedNode::FromLocation(*from);
1019
  DCHECK(from_node->IsInUse());
1020 1021
  TracedNode* to_node = TracedNode::FromLocation(*to);
  GlobalHandles* global_handles = nullptr;
1022 1023 1024
#ifdef DEBUG
  global_handles = GlobalHandles::From(from_node);
#endif  // DEBUG
1025 1026 1027 1028 1029 1030 1031
  bool from_on_stack = from_node->is_on_stack();
  bool to_on_stack = false;
  if (!to_node) {
    // Figure out whether stack or heap to allow fast path for heap->heap move.
    global_handles = GlobalHandles::From(from_node);
    to_on_stack = global_handles->on_stack_nodes_->IsOnStack(
        reinterpret_cast<uintptr_t>(to));
1032 1033
  } else {
    to_on_stack = to_node->is_on_stack();
1034 1035
  }

1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046
  // Moving a traced handle with finalization callback is prohibited because
  // the callback may require knowing about multiple copies of the traced
  // handle.
  CHECK_WITH_MSG(!from_node->HasFinalizationCallback(),
                 "Moving of references is not supported when "
                 "SetFinalizationCallback is set.");
  // Types in v8.h ensure that we only copy/move handles that have the same
  // destructor behavior.
  DCHECK_IMPLIES(to_node,
                 to_node->has_destructor() == from_node->has_destructor());

1047 1048 1049 1050 1051 1052
  // Moving.
  if (from_on_stack || to_on_stack) {
    // Move involving a stack slot.
    if (!to_node) {
      DCHECK(global_handles);
      Handle<Object> o = global_handles->CreateTraced(
1053 1054
          from_node->object(), reinterpret_cast<Address*>(to),
          from_node->has_destructor(), to_on_stack);
1055
      *to = o.location();
1056 1057
      to_node = TracedNode::FromLocation(*to);
      DCHECK(to_node->markbit());
1058
    } else {
1059 1060 1061 1062 1063 1064 1065
      DCHECK(to_node->IsInUse());
      to_node->CopyObjectReference(*from_node);
      if (!to_node->is_on_stack() && !to_node->is_in_young_list() &&
          ObjectInYoungGeneration(to_node->object())) {
        global_handles = GlobalHandles::From(from_node);
        global_handles->traced_young_nodes_.push_back(to_node);
        to_node->set_in_young_list(true);
1066
      }
1067 1068 1069 1070 1071 1072 1073
    }
    DestroyTraced(*from);
    *from = nullptr;
  } else {
    // Pure heap move.
    DestroyTraced(*to);
    *to = *from;
1074
    to_node = from_node;
1075 1076 1077
    DCHECK_NOT_NULL(*from);
    DCHECK_NOT_NULL(*to);
    DCHECK_EQ(*from, *to);
1078 1079 1080
    // Fixup back reference for destructor.
    if (to_node->has_destructor()) {
      to_node->set_parameter(to);
1081 1082 1083
    }
    *from = nullptr;
  }
1084
  TracedNode::Verify(global_handles, to);
1085 1086 1087 1088 1089 1090 1091
}

// static
GlobalHandles* GlobalHandles::From(const TracedNode* node) {
  return node->is_on_stack()
             ? OnStackTracedNodeSpace::GetGlobalHandles(node)
             : NodeBlock<TracedNode>::From(node)->global_handles();
1092 1093
}

1094 1095 1096 1097 1098 1099
void GlobalHandles::MarkTraced(Address* location) {
  TracedNode* node = TracedNode::FromLocation(location);
  node->set_markbit();
  DCHECK(node->IsInUse());
}

1100
void GlobalHandles::Destroy(Address* location) {
1101 1102 1103
  if (location != nullptr) {
    NodeSpace<Node>::Release(Node::FromLocation(location));
  }
1104 1105
}

1106 1107
void GlobalHandles::DestroyTraced(Address* location) {
  if (location != nullptr) {
1108 1109 1110 1111 1112 1113
    TracedNode* node = TracedNode::FromLocation(location);
    if (node->is_on_stack()) {
      node->Release(nullptr);
    } else {
      NodeSpace<TracedNode>::Release(node);
    }
1114 1115
  }
}
1116

1117 1118 1119 1120 1121 1122 1123
void GlobalHandles::SetFinalizationCallbackForTraced(
    Address* location, void* parameter,
    WeakCallbackInfo<void>::Callback callback) {
  TracedNode::FromLocation(location)->SetFinalizationCallback(parameter,
                                                              callback);
}

1124
using GenericCallback = v8::WeakCallbackInfo<void>::Callback;
1125

1126 1127 1128
void GlobalHandles::MakeWeak(Address* location, void* parameter,
                             GenericCallback phantom_callback,
                             v8::WeakCallbackType type) {
1129
  Node::FromLocation(location)->MakeWeak(parameter, phantom_callback, type);
1130 1131
}

1132
void GlobalHandles::MakeWeak(Address** location_addr) {
1133
  Node::FromLocation(*location_addr)->MakeWeak(location_addr);
1134 1135
}

1136
void* GlobalHandles::ClearWeakness(Address* location) {
1137
  return Node::FromLocation(location)->ClearWeakness();
1138 1139
}

1140 1141
void GlobalHandles::AnnotateStrongRetainer(Address* location,
                                           const char* label) {
1142
  Node::FromLocation(location)->AnnotateStrongRetainer(label);
1143 1144
}

1145
bool GlobalHandles::IsWeak(Address* location) {
1146 1147 1148
  return Node::FromLocation(location)->IsWeak();
}

krasin's avatar
krasin committed
1149
DISABLE_CFI_PERF
1150
void GlobalHandles::IterateWeakRootsForFinalizers(RootVisitor* v) {
1151
  for (Node* node : *regular_nodes_) {
1152 1153 1154 1155
    if (node->IsWeakRetainer() && node->state() == Node::PENDING) {
      DCHECK(!node->IsPhantomCallback());
      DCHECK(!node->IsPhantomResetHandle());
      // Finalizers need to survive.
1156 1157
      v->VisitRootPointer(Root::kGlobalHandles, node->label(),
                          node->location());
1158 1159 1160 1161 1162 1163
    }
  }
}

DISABLE_CFI_PERF
void GlobalHandles::IterateWeakRootsForPhantomHandles(
1164
    WeakSlotCallbackWithHeap should_reset_handle) {
1165
  for (Node* node : *regular_nodes_) {
1166 1167
    if (node->IsWeakRetainer() &&
        should_reset_handle(isolate()->heap(), node->location())) {
1168 1169
      if (node->IsPhantomResetHandle()) {
        node->MarkPending();
1170
        node->ResetPhantomHandle(HandleHolder::kLive);
1171
        ++number_of_phantom_handle_resets_;
1172 1173
      } else if (node->IsPhantomCallback()) {
        node->MarkPending();
1174
        node->CollectPhantomCallbackData(&regular_pending_phantom_callbacks_);
1175 1176
      }
    }
1177
  }
1178
  for (TracedNode* node : *traced_nodes_) {
1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198
    if (!node->IsInUse()) continue;
    // Detect unreachable nodes first.
    if (!node->markbit() && node->IsPhantomResetHandle() &&
        !node->has_destructor()) {
      // The handle is unreachable and does not have a callback and a
      // destructor associated with it. We can clear it even if the target V8
      // object is alive. Note that the desctructor and the callback may
      // access the handle, that is why we avoid clearing it.
      node->ResetPhantomHandle(HandleHolder::kDead);
      ++number_of_phantom_handle_resets_;
      continue;
    } else if (node->markbit()) {
      // Clear the markbit for the next GC.
      node->clear_markbit();
    }
    DCHECK(node->IsInUse());
    // Detect nodes with unreachable target objects.
    if (should_reset_handle(isolate()->heap(), node->location())) {
      // If the node allows eager resetting, then reset it here. Otherwise,
      // collect its callback that will reset it.
1199
      if (node->IsPhantomResetHandle()) {
1200 1201
        node->ResetPhantomHandle(node->has_destructor() ? HandleHolder::kLive
                                                        : HandleHolder::kDead);
1202 1203 1204 1205
        ++number_of_phantom_handle_resets_;
      } else {
        node->CollectPhantomCallbackData(&traced_pending_phantom_callbacks_);
      }
1206 1207
    }
  }
1208 1209
}

1210
void GlobalHandles::IterateWeakRootsIdentifyFinalizers(
1211
    WeakSlotCallbackWithHeap should_reset_handle) {
1212
  for (Node* node : *regular_nodes_) {
1213 1214
    if (node->IsWeak() &&
        should_reset_handle(isolate()->heap(), node->location())) {
1215
      if (node->IsFinalizerHandle()) {
1216 1217
        node->MarkPending();
      }
1218 1219 1220 1221
    }
  }
}

1222 1223 1224 1225
void GlobalHandles::IdentifyWeakUnmodifiedObjects(
    WeakSlotCallback is_unmodified) {
  LocalEmbedderHeapTracer* const tracer =
      isolate()->heap()->local_embedder_heap_tracer();
1226
  for (TracedNode* node : traced_young_nodes_) {
1227 1228 1229 1230
    if (node->IsInUse()) {
      DCHECK(node->is_root());
      if (is_unmodified(node->location())) {
        v8::Value* value = ToApi<v8::Value>(node->handle());
1231 1232 1233 1234 1235 1236 1237
        if (node->has_destructor()) {
          node->set_root(tracer->IsRootForNonTracingGC(
              *reinterpret_cast<v8::TracedGlobal<v8::Value>*>(&value)));
        } else {
          node->set_root(tracer->IsRootForNonTracingGC(
              *reinterpret_cast<v8::TracedReference<v8::Value>*>(&value)));
        }
1238 1239 1240 1241 1242
      }
    }
  }
}

1243 1244
void GlobalHandles::IterateYoungStrongAndDependentRoots(RootVisitor* v) {
  for (Node* node : young_nodes_) {
1245
    if (node->IsStrongRetainer()) {
1246 1247
      v->VisitRootPointer(Root::kGlobalHandles, node->label(),
                          node->location());
1248 1249
    }
  }
1250
  for (TracedNode* node : traced_young_nodes_) {
1251 1252
    if (node->IsInUse() && node->is_root()) {
      v->VisitRootPointer(Root::kGlobalHandles, nullptr, node->location());
1253 1254 1255 1256
    }
  }
}

1257
void GlobalHandles::MarkYoungWeakUnmodifiedObjectsPending(
1258
    WeakSlotCallbackWithHeap is_dead) {
1259 1260
  for (Node* node : young_nodes_) {
    DCHECK(node->is_in_young_list());
1261
    if (node->IsWeak() && is_dead(isolate_->heap(), node->location())) {
1262 1263 1264
      if (!node->IsPhantomCallback() && !node->IsPhantomResetHandle()) {
        node->MarkPending();
      }
1265 1266 1267 1268
    }
  }
}

1269
void GlobalHandles::IterateYoungWeakUnmodifiedRootsForFinalizers(
1270
    RootVisitor* v) {
1271 1272
  for (Node* node : young_nodes_) {
    DCHECK(node->is_in_young_list());
1273
    if (node->IsWeakRetainer() && (node->state() == Node::PENDING)) {
1274 1275 1276
      DCHECK(!node->IsPhantomCallback());
      DCHECK(!node->IsPhantomResetHandle());
      // Finalizers need to survive.
1277 1278
      v->VisitRootPointer(Root::kGlobalHandles, node->label(),
                          node->location());
1279 1280 1281 1282
    }
  }
}

1283
void GlobalHandles::IterateYoungWeakUnmodifiedRootsForPhantomHandles(
1284
    RootVisitor* v, WeakSlotCallbackWithHeap should_reset_handle) {
1285 1286
  for (Node* node : young_nodes_) {
    DCHECK(node->is_in_young_list());
1287
    if (node->IsWeakRetainer() && (node->state() != Node::PENDING)) {
1288
      if (should_reset_handle(isolate_->heap(), node->location())) {
1289
        DCHECK(node->IsPhantomResetHandle() || node->IsPhantomCallback());
1290 1291
        if (node->IsPhantomResetHandle()) {
          node->MarkPending();
1292
          node->ResetPhantomHandle(HandleHolder::kLive);
1293 1294 1295
          ++number_of_phantom_handle_resets_;
        } else if (node->IsPhantomCallback()) {
          node->MarkPending();
1296
          node->CollectPhantomCallbackData(&regular_pending_phantom_callbacks_);
1297 1298 1299
        } else {
          UNREACHABLE();
        }
1300
      } else {
1301
        // Node survived and needs to be visited.
1302 1303
        v->VisitRootPointer(Root::kGlobalHandles, node->label(),
                            node->location());
1304 1305 1306
      }
    }
  }
1307 1308 1309

  LocalEmbedderHeapTracer* const tracer =
      isolate()->heap()->local_embedder_heap_tracer();
1310
  for (TracedNode* node : traced_young_nodes_) {
1311 1312 1313 1314 1315
    if (!node->IsInUse()) continue;

    DCHECK_IMPLIES(node->is_root(),
                   !should_reset_handle(isolate_->heap(), node->location()));
    if (should_reset_handle(isolate_->heap(), node->location())) {
1316
      if (node->IsPhantomResetHandle()) {
1317 1318 1319 1320 1321 1322 1323 1324
        if (node->has_destructor()) {
          // For handles with destructor it is guaranteed that the embedder
          // memory is still alive as the destructor would have otherwise
          // removed the memory.
          node->ResetPhantomHandle(HandleHolder::kLive);
        } else {
          v8::Value* value = ToApi<v8::Value>(node->handle());
          tracer->ResetHandleInNonTracingGC(
1325
              *reinterpret_cast<v8::TracedReference<v8::Value>*>(&value));
1326 1327 1328
          DCHECK(!node->IsInUse());
        }

1329 1330 1331 1332
        ++number_of_phantom_handle_resets_;
      } else {
        node->CollectPhantomCallbackData(&traced_pending_phantom_callbacks_);
      }
1333 1334 1335 1336 1337 1338 1339
    } else {
      if (!node->is_root()) {
        node->set_root(true);
        v->VisitRootPointer(Root::kGlobalHandles, nullptr, node->location());
      }
    }
  }
1340 1341
}

1342 1343 1344
void GlobalHandles::InvokeSecondPassPhantomCallbacksFromTask() {
  DCHECK(second_pass_callbacks_task_posted_);
  second_pass_callbacks_task_posted_ = false;
1345 1346
  Heap::DevToolsTraceEventScope devtools_trace_event_scope(
      isolate()->heap(), "MajorGC", "invoke weak phantom callbacks");
1347 1348 1349 1350 1351 1352 1353 1354 1355
  TRACE_EVENT0("v8", "V8.GCPhantomHandleProcessingCallback");
  isolate()->heap()->CallGCPrologueCallbacks(
      GCType::kGCTypeProcessWeakCallbacks, kNoGCCallbackFlags);
  InvokeSecondPassPhantomCallbacks();
  isolate()->heap()->CallGCEpilogueCallbacks(
      GCType::kGCTypeProcessWeakCallbacks, kNoGCCallbackFlags);
}

void GlobalHandles::InvokeSecondPassPhantomCallbacks() {
1356 1357 1358 1359 1360 1361 1362 1363
  // The callbacks may execute JS, which in turn may lead to another GC run.
  // If we are already processing the callbacks, we do not want to start over
  // from within the inner GC. Newly added callbacks will always be run by the
  // outermost GC run only.
  if (running_second_pass_callbacks_) return;
  running_second_pass_callbacks_ = true;

  AllowJavascriptExecution allow_js(isolate());
1364 1365 1366
  while (!second_pass_callbacks_.empty()) {
    auto callback = second_pass_callbacks_.back();
    second_pass_callbacks_.pop_back();
1367
    callback.Invoke(isolate(), PendingPhantomCallback::kSecondPass);
1368
  }
1369
  running_second_pass_callbacks_ = false;
1370 1371
}

1372 1373
size_t GlobalHandles::PostScavengeProcessing(unsigned post_processing_count) {
  size_t freed_nodes = 0;
1374
  for (Node* node : young_nodes_) {
1375 1376 1377 1378 1379 1380 1381
    // Filter free nodes.
    if (!node->IsRetainer()) continue;

    if (node->IsPending()) {
      DCHECK(node->has_callback());
      DCHECK(node->IsPendingFinalizer());
      node->PostGarbageCollectionProcessing(isolate_);
1382
    }
1383 1384 1385
    if (InRecursiveGC(post_processing_count)) return freed_nodes;

    if (!node->IsRetainer()) freed_nodes++;
1386 1387 1388 1389
  }
  return freed_nodes;
}

1390 1391
size_t GlobalHandles::PostMarkSweepProcessing(unsigned post_processing_count) {
  size_t freed_nodes = 0;
1392
  for (Node* node : *regular_nodes_) {
1393 1394 1395 1396 1397 1398 1399
    // Filter free nodes.
    if (!node->IsRetainer()) continue;

    if (node->IsPending()) {
      DCHECK(node->has_callback());
      DCHECK(node->IsPendingFinalizer());
      node->PostGarbageCollectionProcessing(isolate_);
1400
    }
1401 1402 1403
    if (InRecursiveGC(post_processing_count)) return freed_nodes;

    if (!node->IsRetainer()) freed_nodes++;
1404
  }
1405 1406 1407
  return freed_nodes;
}

1408
template <typename T>
1409
void GlobalHandles::UpdateAndCompactListOfYoungNode(
1410
    std::vector<T*>* node_list) {
1411
  size_t last = 0;
1412
  for (T* node : *node_list) {
1413
    DCHECK(node->is_in_young_list());
1414
    if (node->IsInUse()) {
1415
      if (ObjectInYoungGeneration(node->object())) {
1416
        (*node_list)[last++] = node;
1417
        isolate_->heap()->IncrementNodesCopiedInNewSpace();
1418
      } else {
1419
        node->set_in_young_list(false);
1420
        isolate_->heap()->IncrementNodesPromoted();
1421
      }
1422
    } else {
1423
      node->set_in_young_list(false);
1424
      isolate_->heap()->IncrementNodesDiedInNewSpace();
1425
    }
1426
  }
1427 1428 1429 1430 1431
  DCHECK_LE(last, node_list->size());
  node_list->resize(last);
  node_list->shrink_to_fit();
}

1432 1433 1434
void GlobalHandles::UpdateListOfYoungNodes() {
  UpdateAndCompactListOfYoungNode(&young_nodes_);
  UpdateAndCompactListOfYoungNode(&traced_young_nodes_);
1435 1436
}

1437 1438 1439
template <typename T>
size_t GlobalHandles::InvokeFirstPassWeakCallbacks(
    std::vector<std::pair<T*, PendingPhantomCallback>>* pending) {
1440
  size_t freed_nodes = 0;
1441 1442
  std::vector<std::pair<T*, PendingPhantomCallback>> pending_phantom_callbacks;
  pending_phantom_callbacks.swap(*pending);
dcarney's avatar
dcarney committed
1443 1444
  {
    // The initial pass callbacks must simply clear the nodes.
1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457
    for (auto& pair : pending_phantom_callbacks) {
      T* node = pair.first;
      DCHECK_EQ(T::NEAR_DEATH, node->state());
      pair.second.Invoke(isolate(), PendingPhantomCallback::kFirstPass);

      // Transition to second pass. It is required that the first pass callback
      // resets the handle using |v8::PersistentBase::Reset|. Also see comments
      // on |v8::WeakCallbackInfo|.
      CHECK_WITH_MSG(T::FREE == node->state(),
                     "Handle not reset in first callback. See comments on "
                     "|v8::WeakCallbackInfo|.");

      if (pair.second.callback()) second_pass_callbacks_.push_back(pair.second);
dcarney's avatar
dcarney committed
1458 1459 1460
      freed_nodes++;
    }
  }
1461 1462 1463
  return freed_nodes;
}

1464 1465 1466 1467 1468
size_t GlobalHandles::InvokeFirstPassWeakCallbacks() {
  return InvokeFirstPassWeakCallbacks(&regular_pending_phantom_callbacks_) +
         InvokeFirstPassWeakCallbacks(&traced_pending_phantom_callbacks_);
}

1469 1470
void GlobalHandles::InvokeOrScheduleSecondPassPhantomCallbacks(
    bool synchronous_second_pass) {
1471
  if (!second_pass_callbacks_.empty()) {
1472
    if (FLAG_optimize_for_size || FLAG_predictable || synchronous_second_pass) {
1473 1474
      Heap::DevToolsTraceEventScope devtools_trace_event_scope(
          isolate()->heap(), "MajorGC", "invoke weak phantom callbacks");
1475 1476
      isolate()->heap()->CallGCPrologueCallbacks(
          GCType::kGCTypeProcessWeakCallbacks, kNoGCCallbackFlags);
1477
      InvokeSecondPassPhantomCallbacks();
1478 1479
      isolate()->heap()->CallGCEpilogueCallbacks(
          GCType::kGCTypeProcessWeakCallbacks, kNoGCCallbackFlags);
1480 1481
    } else if (!second_pass_callbacks_task_posted_) {
      second_pass_callbacks_task_posted_ = true;
1482 1483
      auto taskrunner = V8::GetCurrentPlatform()->GetForegroundTaskRunner(
          reinterpret_cast<v8::Isolate*>(isolate()));
1484
      taskrunner->PostTask(MakeCancelableTask(
1485
          isolate(), [this] { InvokeSecondPassPhantomCallbacksFromTask(); }));
1486
    }
1487 1488 1489
  }
}

1490 1491
void GlobalHandles::PendingPhantomCallback::Invoke(Isolate* isolate,
                                                   InvocationType type) {
dcarney's avatar
dcarney committed
1492
  Data::Callback* callback_addr = nullptr;
1493
  if (type == kFirstPass) {
dcarney's avatar
dcarney committed
1494 1495 1496
    callback_addr = &callback_;
  }
  Data data(reinterpret_cast<v8::Isolate*>(isolate), parameter_,
1497
            embedder_fields_, callback_addr);
dcarney's avatar
dcarney committed
1498 1499 1500 1501 1502
  Data::Callback callback = callback_;
  callback_ = nullptr;
  callback(data);
}

1503 1504 1505 1506 1507
bool GlobalHandles::InRecursiveGC(unsigned gc_processing_counter) {
  return gc_processing_counter != post_gc_processing_count_;
}

size_t GlobalHandles::PostGarbageCollectionProcessing(
1508
    GarbageCollector collector, const v8::GCCallbackFlags gc_callback_flags) {
1509 1510 1511
  // Process weak global handle callbacks. This must be done after the
  // GC is completely done, because the callbacks may invoke arbitrary
  // API functions.
1512 1513 1514
  DCHECK_EQ(Heap::NOT_IN_GC, isolate_->heap()->gc_state());
  const unsigned post_processing_count = ++post_gc_processing_count_;
  size_t freed_nodes = 0;
1515
  bool synchronous_second_pass =
1516
      isolate_->heap()->IsTearingDown() ||
1517
      (gc_callback_flags &
1518
       (kGCCallbackFlagForced | kGCCallbackFlagCollectAllAvailableGarbage |
1519
        kGCCallbackFlagSynchronousPhantomCallbackProcessing)) != 0;
1520
  InvokeOrScheduleSecondPassPhantomCallbacks(synchronous_second_pass);
1521 1522 1523 1524 1525 1526 1527
  if (InRecursiveGC(post_processing_count)) return freed_nodes;

  freed_nodes += Heap::IsYoungGenerationCollector(collector)
                     ? PostScavengeProcessing(post_processing_count)
                     : PostMarkSweepProcessing(post_processing_count);
  if (InRecursiveGC(post_processing_count)) return freed_nodes;

1528
  UpdateListOfYoungNodes();
1529
  return freed_nodes;
1530 1531
}

1532
void GlobalHandles::IterateStrongRoots(RootVisitor* v) {
1533 1534 1535 1536
  for (Node* node : *regular_nodes_) {
    if (node->IsStrongRetainer()) {
      v->VisitRootPointer(Root::kGlobalHandles, node->label(),
                          node->location());
1537 1538
    }
  }
1539 1540 1541
}

void GlobalHandles::IterateStrongStackRoots(RootVisitor* v) {
1542
  on_stack_nodes_->Iterate(v);
1543 1544
}

1545
void GlobalHandles::IterateWeakRoots(RootVisitor* v) {
1546 1547 1548 1549
  for (Node* node : *regular_nodes_) {
    if (node->IsWeak()) {
      v->VisitRootPointer(Root::kGlobalHandles, node->label(),
                          node->location());
1550 1551
    }
  }
1552 1553 1554 1555 1556
  for (TracedNode* node : *traced_nodes_) {
    if (node->IsInUse()) {
      v->VisitRootPointer(Root::kGlobalHandles, nullptr, node->location());
    }
  }
1557
}
1558

krasin's avatar
krasin committed
1559
DISABLE_CFI_PERF
1560
void GlobalHandles::IterateAllRoots(RootVisitor* v) {
1561 1562 1563 1564
  for (Node* node : *regular_nodes_) {
    if (node->IsRetainer()) {
      v->VisitRootPointer(Root::kGlobalHandles, node->label(),
                          node->location());
1565 1566
    }
  }
1567
  for (TracedNode* node : *traced_nodes_) {
1568
    if (node->IsRetainer()) {
1569 1570 1571
      v->VisitRootPointer(Root::kGlobalHandles, nullptr, node->location());
    }
  }
1572
  on_stack_nodes_->Iterate(v);
1573 1574
}

1575
DISABLE_CFI_PERF
1576 1577
void GlobalHandles::IterateAllYoungRoots(RootVisitor* v) {
  for (Node* node : young_nodes_) {
1578
    if (node->IsRetainer()) {
1579 1580
      v->VisitRootPointer(Root::kGlobalHandles, node->label(),
                          node->location());
1581 1582
    }
  }
1583
  for (TracedNode* node : traced_young_nodes_) {
1584
    if (node->IsRetainer()) {
1585 1586 1587
      v->VisitRootPointer(Root::kGlobalHandles, nullptr, node->location());
    }
  }
1588
  on_stack_nodes_->Iterate(v);
1589
}
1590

krasin's avatar
krasin committed
1591
DISABLE_CFI_PERF
1592 1593
void GlobalHandles::ApplyPersistentHandleVisitor(
    v8::PersistentHandleVisitor* visitor, GlobalHandles::Node* node) {
1594
  v8::Value* value = ToApi<v8::Value>(node->handle());
1595 1596 1597 1598 1599 1600 1601 1602
  visitor->VisitPersistentHandle(
      reinterpret_cast<v8::Persistent<v8::Value>*>(&value),
      node->wrapper_class_id());
}

DISABLE_CFI_PERF
void GlobalHandles::IterateAllRootsWithClassIds(
    v8::PersistentHandleVisitor* visitor) {
1603 1604 1605
  for (Node* node : *regular_nodes_) {
    if (node->IsRetainer() && node->has_wrapper_class_id()) {
      ApplyPersistentHandleVisitor(visitor, node);
1606 1607 1608 1609
    }
  }
}

1610 1611 1612 1613 1614 1615
DISABLE_CFI_PERF
void GlobalHandles::IterateTracedNodes(
    v8::EmbedderHeapTracer::TracedGlobalHandleVisitor* visitor) {
  for (TracedNode* node : *traced_nodes_) {
    if (node->IsInUse()) {
      v8::Value* value = ToApi<v8::Value>(node->handle());
1616 1617 1618 1619 1620 1621 1622
      if (node->has_destructor()) {
        visitor->VisitTracedGlobalHandle(
            *reinterpret_cast<v8::TracedGlobal<v8::Value>*>(&value));
      } else {
        visitor->VisitTracedReference(
            *reinterpret_cast<v8::TracedReference<v8::Value>*>(&value));
      }
1623 1624 1625
    }
  }
}
1626

krasin's avatar
krasin committed
1627
DISABLE_CFI_PERF
1628
void GlobalHandles::IterateAllYoungRootsWithClassIds(
1629
    v8::PersistentHandleVisitor* visitor) {
1630
  for (Node* node : young_nodes_) {
1631
    if (node->IsRetainer() && node->has_wrapper_class_id()) {
1632
      ApplyPersistentHandleVisitor(visitor, node);
1633 1634 1635 1636
    }
  }
}

krasin's avatar
krasin committed
1637
DISABLE_CFI_PERF
1638
void GlobalHandles::IterateYoungWeakRootsWithClassIds(
1639
    v8::PersistentHandleVisitor* visitor) {
1640
  for (Node* node : young_nodes_) {
1641
    if (node->has_wrapper_class_id() && node->IsWeak()) {
1642
      ApplyPersistentHandleVisitor(visitor, node);
1643 1644 1645 1646
    }
  }
}

1647
void GlobalHandles::RecordStats(HeapStats* stats) {
1648 1649 1650 1651 1652
  *stats->global_handle_count = 0;
  *stats->weak_global_handle_count = 0;
  *stats->pending_global_handle_count = 0;
  *stats->near_death_global_handle_count = 0;
  *stats->free_global_handle_count = 0;
1653
  for (Node* node : *regular_nodes_) {
1654
    *stats->global_handle_count += 1;
1655
    if (node->state() == Node::WEAK) {
1656
      *stats->weak_global_handle_count += 1;
1657
    } else if (node->state() == Node::PENDING) {
1658
      *stats->pending_global_handle_count += 1;
1659
    } else if (node->state() == Node::NEAR_DEATH) {
1660
      *stats->near_death_global_handle_count += 1;
1661
    } else if (node->state() == Node::FREE) {
1662 1663 1664
      *stats->free_global_handle_count += 1;
    }
  }
1665 1666
}

1667 1668 1669
#ifdef DEBUG

void GlobalHandles::PrintStats() {
1670 1671 1672 1673 1674 1675
  int total = 0;
  int weak = 0;
  int pending = 0;
  int near_death = 0;
  int destroyed = 0;

1676
  for (Node* node : *regular_nodes_) {
1677
    total++;
1678 1679 1680 1681
    if (node->state() == Node::WEAK) weak++;
    if (node->state() == Node::PENDING) pending++;
    if (node->state() == Node::NEAR_DEATH) near_death++;
    if (node->state() == Node::FREE) destroyed++;
1682 1683
  }

1684
  PrintF("Global Handle Statistics:\n");
1685
  PrintF("  allocated memory = %zuB\n", total * sizeof(Node));
1686 1687 1688 1689
  PrintF("  # weak       = %d\n", weak);
  PrintF("  # pending    = %d\n", pending);
  PrintF("  # near_death = %d\n", near_death);
  PrintF("  # free       = %d\n", destroyed);
1690 1691 1692 1693 1694
  PrintF("  # total      = %d\n", total);
}

void GlobalHandles::Print() {
  PrintF("Global handles:\n");
1695 1696
  for (Node* node : *regular_nodes_) {
    PrintF("  handle %p to %p%s\n", node->location().ToVoidPtr(),
1697
           reinterpret_cast<void*>(node->object().ptr()),
1698
           node->IsWeak() ? " (weak)" : "");
1699 1700 1701 1702 1703
  }
}

#endif

1704
EternalHandles::~EternalHandles() {
1705
  for (Address* block : blocks_) delete[] block;
1706 1707
}

1708
void EternalHandles::IterateAllRoots(RootVisitor* visitor) {
1709
  int limit = size_;
1710
  for (Address* block : blocks_) {
1711
    DCHECK_GT(limit, 0);
1712
    visitor->VisitRootPointers(Root::kEternalHandles, nullptr,
1713 1714
                               FullObjectSlot(block),
                               FullObjectSlot(block + Min(limit, kSize)));
1715 1716 1717 1718
    limit -= kSize;
  }
}

1719 1720
void EternalHandles::IterateYoungRoots(RootVisitor* visitor) {
  for (int index : young_node_indices_) {
1721
    visitor->VisitRootPointer(Root::kEternalHandles, nullptr,
1722
                              FullObjectSlot(GetLocation(index)));
1723 1724 1725
  }
}

1726
void EternalHandles::PostGarbageCollectionProcessing() {
1727
  size_t last = 0;
1728
  for (int index : young_node_indices_) {
1729
    if (ObjectInYoungGeneration(Object(*GetLocation(index)))) {
1730
      young_node_indices_[last++] = index;
1731 1732
    }
  }
1733 1734
  DCHECK_LE(last, young_node_indices_.size());
  young_node_indices_.resize(last);
1735 1736
}

1737
void EternalHandles::Create(Isolate* isolate, Object object, int* index) {
1738
  DCHECK_EQ(kInvalidIndex, *index);
1739 1740
  if (object == Object()) return;
  Object the_hole = ReadOnlyRoots(isolate).the_hole_value();
1741
  DCHECK_NE(the_hole, object);
1742 1743
  int block = size_ >> kShift;
  int offset = size_ & kMask;
1744
  // Need to resize.
1745
  if (offset == 0) {
1746
    Address* next_block = new Address[kSize];
1747
    MemsetPointer(FullObjectSlot(next_block), the_hole, kSize);
1748
    blocks_.push_back(next_block);
1749
  }
1750 1751
  DCHECK_EQ(the_hole.ptr(), blocks_[block][offset]);
  blocks_[block][offset] = object.ptr();
1752
  if (ObjectInYoungGeneration(object)) {
1753
    young_node_indices_.push_back(size_);
1754
  }
1755
  *index = size_++;
1756 1757
}

1758 1759
}  // namespace internal
}  // namespace v8