global-handles.cc 57 KB
Newer Older
1
// Copyright 2009 the V8 project authors. All rights reserved.
2 3
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
4

5
#include "src/handles/global-handles.h"
6

7
#include <algorithm>
8 9
#include <map>

10
#include "src/api/api-inl.h"
11
#include "src/base/compiler-specific.h"
12
#include "src/execution/vm-state-inl.h"
13
#include "src/heap/embedder-tracing.h"
14
#include "src/heap/heap-write-barrier-inl.h"
15
#include "src/init/v8.h"
16
#include "src/logging/counters.h"
17
#include "src/objects/objects-inl.h"
18
#include "src/objects/slots.h"
19
#include "src/objects/visitors.h"
20
#include "src/sanitizer/asan.h"
21 22
#include "src/tasks/cancelable-task.h"
#include "src/tasks/task-utils.h"
23
#include "src/utils/utils.h"
24

25 26
namespace v8 {
namespace internal {
27

28 29
namespace {

30 31 32 33
// Specifies whether V8 expects the holder memory of a global handle to be live
// or dead.
enum class HandleHolder { kLive, kDead };

34 35 36 37 38 39 40 41 42 43
constexpr size_t kBlockSize = 256;

}  // namespace

template <class _NodeType>
class GlobalHandles::NodeBlock final {
 public:
  using BlockType = NodeBlock<_NodeType>;
  using NodeType = _NodeType;

44
  V8_INLINE static const NodeBlock* From(const NodeType* node);
45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78
  V8_INLINE static NodeBlock* From(NodeType* node);

  NodeBlock(GlobalHandles* global_handles,
            GlobalHandles::NodeSpace<NodeType>* space,
            NodeBlock* next) V8_NOEXCEPT : next_(next),
                                           global_handles_(global_handles),
                                           space_(space) {}

  NodeType* at(size_t index) { return &nodes_[index]; }
  const NodeType* at(size_t index) const { return &nodes_[index]; }
  GlobalHandles::NodeSpace<NodeType>* space() const { return space_; }
  GlobalHandles* global_handles() const { return global_handles_; }

  V8_INLINE bool IncreaseUsage();
  V8_INLINE bool DecreaseUsage();

  V8_INLINE void ListAdd(NodeBlock** top);
  V8_INLINE void ListRemove(NodeBlock** top);

  NodeBlock* next() const { return next_; }
  NodeBlock* next_used() const { return next_used_; }

 private:
  NodeType nodes_[kBlockSize];
  NodeBlock* const next_;
  GlobalHandles* const global_handles_;
  GlobalHandles::NodeSpace<NodeType>* const space_;
  NodeBlock* next_used_ = nullptr;
  NodeBlock* prev_used_ = nullptr;
  uint32_t used_nodes_ = 0;

  DISALLOW_COPY_AND_ASSIGN(NodeBlock);
};

79 80 81 82 83 84 85 86 87 88
template <class NodeType>
const GlobalHandles::NodeBlock<NodeType>*
GlobalHandles::NodeBlock<NodeType>::From(const NodeType* node) {
  uintptr_t ptr = reinterpret_cast<const uintptr_t>(node) -
                  sizeof(NodeType) * node->index();
  const BlockType* block = reinterpret_cast<const BlockType*>(ptr);
  DCHECK_EQ(node, block->at(node->index()));
  return block;
}

89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
template <class NodeType>
GlobalHandles::NodeBlock<NodeType>* GlobalHandles::NodeBlock<NodeType>::From(
    NodeType* node) {
  uintptr_t ptr =
      reinterpret_cast<uintptr_t>(node) - sizeof(NodeType) * node->index();
  BlockType* block = reinterpret_cast<BlockType*>(ptr);
  DCHECK_EQ(node, block->at(node->index()));
  return block;
}

template <class NodeType>
bool GlobalHandles::NodeBlock<NodeType>::IncreaseUsage() {
  DCHECK_LT(used_nodes_, kBlockSize);
  return used_nodes_++ == 0;
}

template <class NodeType>
void GlobalHandles::NodeBlock<NodeType>::ListAdd(BlockType** top) {
  BlockType* old_top = *top;
  *top = this;
  next_used_ = old_top;
  prev_used_ = nullptr;
  if (old_top != nullptr) {
    old_top->prev_used_ = this;
  }
}

template <class NodeType>
bool GlobalHandles::NodeBlock<NodeType>::DecreaseUsage() {
  DCHECK_GT(used_nodes_, 0);
  return --used_nodes_ == 0;
}

template <class NodeType>
void GlobalHandles::NodeBlock<NodeType>::ListRemove(BlockType** top) {
  if (next_used_ != nullptr) next_used_->prev_used_ = prev_used_;
  if (prev_used_ != nullptr) prev_used_->next_used_ = next_used_;
  if (this == *top) {
    *top = next_used_;
  }
}

template <class BlockType>
class GlobalHandles::NodeIterator final {
 public:
  using NodeType = typename BlockType::NodeType;

  // Iterator traits.
  using iterator_category = std::forward_iterator_tag;
  using difference_type = std::ptrdiff_t;
  using value_type = NodeType*;
  using reference = value_type;
  using pointer = value_type*;

  explicit NodeIterator(BlockType* block) V8_NOEXCEPT : block_(block) {}
  NodeIterator(NodeIterator&& other) V8_NOEXCEPT : block_(other.block_),
                                                   index_(other.index_) {}

  bool operator==(const NodeIterator& other) const {
    return block_ == other.block_;
  }
  bool operator!=(const NodeIterator& other) const {
    return block_ != other.block_;
  }

  NodeIterator& operator++() {
    if (++index_ < kBlockSize) return *this;
    index_ = 0;
    block_ = block_->next_used();
    return *this;
  }

  NodeType* operator*() { return block_->at(index_); }
  NodeType* operator->() { return block_->at(index_); }

 private:
  BlockType* block_ = nullptr;
  size_t index_ = 0;

  DISALLOW_COPY_AND_ASSIGN(NodeIterator);
};

template <class NodeType>
class GlobalHandles::NodeSpace final {
 public:
  using BlockType = NodeBlock<NodeType>;
  using iterator = NodeIterator<BlockType>;

  static NodeSpace* From(NodeType* node);
  static void Release(NodeType* node);

  explicit NodeSpace(GlobalHandles* global_handles) V8_NOEXCEPT
      : global_handles_(global_handles) {}
  ~NodeSpace();

  V8_INLINE NodeType* Acquire(Object object);

  iterator begin() { return iterator(first_used_block_); }
  iterator end() { return iterator(nullptr); }

189
  size_t TotalSize() const { return blocks_ * sizeof(NodeType) * kBlockSize; }
190
  size_t handles_count() const { return handles_count_; }
191

192 193 194 195 196 197 198 199
 private:
  void PutNodesOnFreeList(BlockType* block);
  V8_INLINE void Free(NodeType* node);

  GlobalHandles* const global_handles_;
  BlockType* first_block_ = nullptr;
  BlockType* first_used_block_ = nullptr;
  NodeType* first_free_ = nullptr;
200
  size_t blocks_ = 0;
201
  size_t handles_count_ = 0;
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
};

template <class NodeType>
GlobalHandles::NodeSpace<NodeType>::~NodeSpace() {
  auto* block = first_block_;
  while (block != nullptr) {
    auto* tmp = block->next();
    delete block;
    block = tmp;
  }
}

template <class NodeType>
NodeType* GlobalHandles::NodeSpace<NodeType>::Acquire(Object object) {
  if (first_free_ == nullptr) {
    first_block_ = new BlockType(global_handles_, this, first_block_);
218
    blocks_++;
219 220 221 222 223 224 225 226 227 228 229
    PutNodesOnFreeList(first_block_);
  }
  DCHECK_NOT_NULL(first_free_);
  NodeType* node = first_free_;
  first_free_ = first_free_->next_free();
  node->Acquire(object);
  BlockType* block = BlockType::From(node);
  if (block->IncreaseUsage()) {
    block->ListAdd(&first_used_block_);
  }
  global_handles_->isolate()->counters()->global_handles()->Increment();
230
  handles_count_++;
231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261
  DCHECK(node->IsInUse());
  return node;
}

template <class NodeType>
void GlobalHandles::NodeSpace<NodeType>::PutNodesOnFreeList(BlockType* block) {
  for (int32_t i = kBlockSize - 1; i >= 0; --i) {
    NodeType* node = block->at(i);
    const uint8_t index = static_cast<uint8_t>(i);
    DCHECK_EQ(i, index);
    node->set_index(index);
    node->Free(first_free_);
    first_free_ = node;
  }
}

template <class NodeType>
void GlobalHandles::NodeSpace<NodeType>::Release(NodeType* node) {
  BlockType* block = BlockType::From(node);
  block->space()->Free(node);
}

template <class NodeType>
void GlobalHandles::NodeSpace<NodeType>::Free(NodeType* node) {
  node->Release(first_free_);
  first_free_ = node;
  BlockType* block = BlockType::From(node);
  if (block->DecreaseUsage()) {
    block->ListRemove(&first_used_block_);
  }
  global_handles_->isolate()->counters()->global_handles()->Decrement();
262
  handles_count_--;
263 264
}

265 266
template <class Child>
class NodeBase {
267
 public:
268 269 270 271
  static const Child* FromLocation(const Address* location) {
    return reinterpret_cast<const Child*>(location);
  }

272 273
  static Child* FromLocation(Address* location) {
    return reinterpret_cast<Child*>(location);
274 275
  }

276 277 278 279
  NodeBase() {
    DCHECK_EQ(offsetof(NodeBase, object_), 0);
    DCHECK_EQ(offsetof(NodeBase, class_id_), Internals::kNodeClassIdOffset);
    DCHECK_EQ(offsetof(NodeBase, flags_), Internals::kNodeFlagsOffset);
280
  }
281

282
#ifdef ENABLE_HANDLE_ZAPPING
283
  ~NodeBase() {
284
    ClearFields();
285
    data_.next_free = nullptr;
286
    index_ = 0;
287
  }
288
#endif
289

290
  void Free(Child* free_list) {
291
    ClearFields();
292
    AsChild()->MarkAsFree();
293
    data_.next_free = free_list;
294 295
  }

296
  void Acquire(Object object) {
297
    DCHECK(!AsChild()->IsInUse());
298
    CheckFieldsAreCleared();
299
    object_ = object.ptr();
300
    AsChild()->MarkAsUsed();
301
    data_.parameter = nullptr;
302
    DCHECK(AsChild()->IsInUse());
303 304
  }

305 306
  void Release(Child* free_list) {
    DCHECK(AsChild()->IsInUse());
307
    Free(free_list);
308
    DCHECK(!AsChild()->IsInUse());
309 310
  }

311
  Object object() const { return Object(object_); }
312
  FullObjectSlot location() { return FullObjectSlot(&object_); }
313
  Handle<Object> handle() { return Handle<Object>(&object_); }
314

315 316 317 318
  uint8_t index() const { return index_; }
  void set_index(uint8_t value) { index_ = value; }

  uint16_t wrapper_class_id() const { return class_id_; }
319 320 321
  bool has_wrapper_class_id() const {
    return class_id_ != v8::HeapProfiler::kPersistentHandleNoClassId;
  }
322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379

  // Accessors for next free node in the free list.
  Child* next_free() {
    DCHECK(!AsChild()->IsInUse());
    return data_.next_free;
  }

  void set_parameter(void* parameter) {
    DCHECK(AsChild()->IsInUse());
    data_.parameter = parameter;
  }
  void* parameter() const {
    DCHECK(AsChild()->IsInUse());
    return data_.parameter;
  }

 protected:
  Child* AsChild() { return reinterpret_cast<Child*>(this); }
  const Child* AsChild() const { return reinterpret_cast<const Child*>(this); }

  void ClearFields() {
    // Zap the values for eager trapping.
    object_ = kGlobalHandleZapValue;
    class_id_ = v8::HeapProfiler::kPersistentHandleNoClassId;
    AsChild()->ClearImplFields();
  }

  void CheckFieldsAreCleared() {
    DCHECK_EQ(kGlobalHandleZapValue, object_);
    DCHECK_EQ(v8::HeapProfiler::kPersistentHandleNoClassId, class_id_);
    AsChild()->CheckImplFieldsAreCleared();
  }

  // Storage for object pointer.
  //
  // Placed first to avoid offset computation. The stored data is equivalent to
  // an Object. It is stored as a plain Address for convenience (smallest number
  // of casts), and because it is a private implementation detail: the public
  // interface provides type safety.
  Address object_;

  // Class id set by the embedder.
  uint16_t class_id_;

  // Index in the containing handle block.
  uint8_t index_;

  uint8_t flags_;

  // The meaning of this field depends on node state:
  // - Node in free list: Stores next free node pointer.
  // - Otherwise, specific to the node implementation.
  union {
    Child* next_free;
    void* parameter;
  } data_;
};

380 381 382
namespace {

void ExtractInternalFields(JSObject jsobject, void** embedder_fields, int len) {
383
  int field_count = jsobject.GetEmbedderFieldCount();
384 385 386 387 388 389 390 391 392 393 394
  for (int i = 0; i < len; ++i) {
    if (field_count == i) break;
    void* pointer;
    if (EmbedderDataSlot(jsobject, i).ToAlignedPointer(&pointer)) {
      embedder_fields[i] = pointer;
    }
  }
}

}  // namespace

395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412
class GlobalHandles::Node final : public NodeBase<GlobalHandles::Node> {
 public:
  // State transition diagram:
  // FREE -> NORMAL <-> WEAK -> PENDING -> NEAR_DEATH -> { NORMAL, WEAK, FREE }
  enum State {
    FREE = 0,
    NORMAL,      // Normal global handle.
    WEAK,        // Flagged as weak but not yet finalized.
    PENDING,     // Has been recognized as only reachable by weak handles.
    NEAR_DEATH,  // Callback has informed the handle is near death.
    NUMBER_OF_NODE_STATES
  };

  Node() {
    STATIC_ASSERT(static_cast<int>(NodeState::kMask) ==
                  Internals::kNodeStateMask);
    STATIC_ASSERT(WEAK == Internals::kNodeStateIsWeakValue);
    STATIC_ASSERT(PENDING == Internals::kNodeStateIsPendingValue);
413
    set_in_young_list(false);
414 415 416 417 418 419 420 421 422 423 424 425
  }

  void Zap() {
    DCHECK(IsInUse());
    // Zap the values for eager trapping.
    object_ = kGlobalHandleZapValue;
  }

  const char* label() const {
    return state() == NORMAL ? reinterpret_cast<char*>(data_.parameter)
                             : nullptr;
  }
426 427 428

  // State and flag accessors.

429 430
  State state() const { return NodeState::decode(flags_); }
  void set_state(State state) { flags_ = NodeState::update(flags_, state); }
431

432 433
  bool is_in_young_list() const { return IsInYoungList::decode(flags_); }
  void set_in_young_list(bool v) { flags_ = IsInYoungList::update(flags_, v); }
434

435 436
  WeaknessType weakness_type() const {
    return NodeWeaknessType::decode(flags_);
437
  }
438 439
  void set_weakness_type(WeaknessType weakness_type) {
    flags_ = NodeWeaknessType::update(flags_, weakness_type);
440 441
  }

442
  bool IsWeak() const { return state() == WEAK; }
443

444 445
  bool IsInUse() const { return state() != FREE; }

446 447 448 449 450 451 452 453 454
  bool IsPhantomCallback() const {
    return weakness_type() == PHANTOM_WEAK ||
           weakness_type() == PHANTOM_WEAK_2_EMBEDDER_FIELDS;
  }

  bool IsPhantomResetHandle() const {
    return weakness_type() == PHANTOM_WEAK_RESET_HANDLE;
  }

455 456
  bool IsFinalizerHandle() const { return weakness_type() == FINALIZER_WEAK; }

457
  bool IsPendingPhantomCallback() const {
458
    return state() == PENDING && IsPhantomCallback();
459 460 461
  }

  bool IsPendingPhantomResetHandle() const {
462
    return state() == PENDING && IsPhantomResetHandle();
463 464
  }

465 466 467 468 469 470
  bool IsPendingFinalizer() const {
    return state() == PENDING && weakness_type() == FINALIZER_WEAK;
  }

  bool IsPending() const { return state() == PENDING; }

471 472
  bool IsRetainer() const {
    return state() != FREE &&
473
           !(state() == NEAR_DEATH && weakness_type() != FINALIZER_WEAK);
474
  }
475

476
  bool IsStrongRetainer() const { return state() == NORMAL; }
477 478

  bool IsWeakRetainer() const {
479
    return state() == WEAK || state() == PENDING ||
480
           (state() == NEAR_DEATH && weakness_type() == FINALIZER_WEAK);
481 482
  }

483
  void MarkPending() {
484
    DCHECK(state() == WEAK);
485
    set_state(PENDING);
486 487
  }

488 489
  bool has_callback() const { return weak_callback_ != nullptr; }

490 491
  // Accessors for next free node in the free list.
  Node* next_free() {
492
    DCHECK_EQ(FREE, state());
493
    return data_.next_free;
494 495
  }

496 497 498
  void MakeWeak(void* parameter,
                WeakCallbackInfo<void>::Callback phantom_callback,
                v8::WeakCallbackType type) {
499
    DCHECK_NOT_NULL(phantom_callback);
500
    DCHECK(IsInUse());
501
    CHECK_NE(object_, kGlobalHandleZapValue);
502
    set_state(WEAK);
503 504 505 506 507
    switch (type) {
      case v8::WeakCallbackType::kParameter:
        set_weakness_type(PHANTOM_WEAK);
        break;
      case v8::WeakCallbackType::kInternalFields:
508
        set_weakness_type(PHANTOM_WEAK_2_EMBEDDER_FIELDS);
509 510
        break;
      case v8::WeakCallbackType::kFinalizer:
511
        set_weakness_type(FINALIZER_WEAK);
512
        break;
513
    }
514
    set_parameter(parameter);
515
    weak_callback_ = phantom_callback;
516 517
  }

518
  void MakeWeak(Address** location_addr) {
519
    DCHECK(IsInUse());
520
    CHECK_NE(object_, kGlobalHandleZapValue);
521 522 523 524 525 526
    set_state(WEAK);
    set_weakness_type(PHANTOM_WEAK_RESET_HANDLE);
    set_parameter(location_addr);
    weak_callback_ = nullptr;
  }

527
  void* ClearWeakness() {
528
    DCHECK(IsInUse());
529
    void* p = parameter();
530
    set_state(NORMAL);
531
    set_parameter(nullptr);
532
    return p;
533 534
  }

535 536
  void AnnotateStrongRetainer(const char* label) {
    DCHECK_EQ(state(), NORMAL);
537
    data_.parameter = const_cast<char*>(label);
538 539
  }

540
  void CollectPhantomCallbackData(
541 542
      std::vector<std::pair<Node*, PendingPhantomCallback>>*
          pending_phantom_callbacks) {
543
    DCHECK(weakness_type() == PHANTOM_WEAK ||
544
           weakness_type() == PHANTOM_WEAK_2_EMBEDDER_FIELDS);
545
    DCHECK(state() == PENDING);
546
    DCHECK_NOT_NULL(weak_callback_);
547

548
    void* embedder_fields[v8::kEmbedderFieldsInWeakCallback] = {nullptr,
549
                                                                nullptr};
550
    if (weakness_type() != PHANTOM_WEAK && object().IsJSObject()) {
551 552
      ExtractInternalFields(JSObject::cast(object()), embedder_fields,
                            v8::kEmbedderFieldsInWeakCallback);
553
    }
554

555
    // Zap with something dangerous.
556
    location().store(Object(0xCA11));
557

558 559 560
    pending_phantom_callbacks->push_back(std::make_pair(
        this,
        PendingPhantomCallback(weak_callback_, parameter(), embedder_fields)));
561 562
    DCHECK(IsInUse());
    set_state(NEAR_DEATH);
563 564
  }

565 566
  void ResetPhantomHandle(HandleHolder handle_holder) {
    DCHECK_EQ(HandleHolder::kLive, handle_holder);
567 568
    DCHECK_EQ(PHANTOM_WEAK_RESET_HANDLE, weakness_type());
    DCHECK_EQ(PENDING, state());
569
    DCHECK_NULL(weak_callback_);
570
    Address** handle = reinterpret_cast<Address**>(parameter());
571
    *handle = nullptr;
572
    NodeSpace<Node>::Release(this);
573 574
  }

575 576 577 578
  void PostGarbageCollectionProcessing(Isolate* isolate) {
    // This method invokes a finalizer. Updating the method name would require
    // adjusting CFI blacklist as weak_callback_ is invoked on the wrong type.
    CHECK(IsPendingFinalizer());
579
    set_state(NEAR_DEATH);
580 581
    // Check that we are not passing a finalized external string to
    // the callback.
582 583 584 585
    DCHECK(!object().IsExternalOneByteString() ||
           ExternalOneByteString::cast(object()).resource() != nullptr);
    DCHECK(!object().IsExternalTwoByteString() ||
           ExternalTwoByteString::cast(object()).resource() != nullptr);
586 587 588
    // Leaving V8.
    VMState<EXTERNAL> vmstate(isolate);
    HandleScope handle_scope(isolate);
589
    void* embedder_fields[v8::kEmbedderFieldsInWeakCallback] = {nullptr,
590 591
                                                                nullptr};
    v8::WeakCallbackInfo<void> data(reinterpret_cast<v8::Isolate*>(isolate),
592
                                    parameter(), embedder_fields, nullptr);
593
    weak_callback_(data);
594 595 596
    // For finalizers the handle must have either been reset or made strong.
    // Both cases reset the state.
    CHECK_NE(NEAR_DEATH, state());
597 598
  }

599 600
  void MarkAsFree() { set_state(FREE); }
  void MarkAsUsed() { set_state(NORMAL); }
601

602 603 604
  GlobalHandles* global_handles() {
    return NodeBlock<Node>::From(this)->global_handles();
  }
605

606
 private:
607
  // Fields that are not used for managing node memory.
608
  void ClearImplFields() { weak_callback_ = nullptr; }
609

610
  void CheckImplFieldsAreCleared() { DCHECK_EQ(nullptr, weak_callback_); }
611

612
  // This stores three flags (independent, partially_dependent and
613
  // in_young_list) and a State.
614
  using NodeState = base::BitField8<State, 0, 3>;
615 616
  using IsInYoungList = NodeState::Next<bool, 1>;
  using NodeWeaknessType = IsInYoungList::Next<WeaknessType, 2>;
617

618
  // Handle specific callback - might be a weak reference in disguise.
619
  WeakCallbackInfo<void>::Callback weak_callback_;
620

621
  friend class NodeBase<Node>;
622

623 624 625
  DISALLOW_COPY_AND_ASSIGN(Node);
};

626 627 628
class GlobalHandles::TracedNode final
    : public NodeBase<GlobalHandles::TracedNode> {
 public:
629
  TracedNode() { set_in_young_list(false); }
630

631 632 633 634 635 636
  // Copy and move ctors are used when constructing a TracedNode when recording
  // a node for on-stack data structures. (Older compilers may refer to copy
  // instead of move ctor.)
  TracedNode(TracedNode&& other) V8_NOEXCEPT = default;
  TracedNode(const TracedNode& other) V8_NOEXCEPT = default;

637
  enum State { FREE = 0, NORMAL, NEAR_DEATH };
638 639 640 641 642 643 644

  State state() const { return NodeState::decode(flags_); }
  void set_state(State state) { flags_ = NodeState::update(flags_, state); }

  void MarkAsFree() { set_state(FREE); }
  void MarkAsUsed() { set_state(NORMAL); }
  bool IsInUse() const { return state() != FREE; }
645 646
  bool IsRetainer() const { return state() == NORMAL; }
  bool IsPhantomResetHandle() const { return callback_ == nullptr; }
647

648 649
  bool is_in_young_list() const { return IsInYoungList::decode(flags_); }
  void set_in_young_list(bool v) { flags_ = IsInYoungList::update(flags_, v); }
650 651 652 653

  bool is_root() const { return IsRoot::decode(flags_); }
  void set_root(bool v) { flags_ = IsRoot::update(flags_, v); }

654 655 656
  bool has_destructor() const { return HasDestructor::decode(flags_); }
  void set_has_destructor(bool v) { flags_ = HasDestructor::update(flags_, v); }

657 658 659 660
  bool markbit() const { return Markbit::decode(flags_); }
  void clear_markbit() { flags_ = Markbit::update(flags_, false); }
  void set_markbit() { flags_ = Markbit::update(flags_, true); }

661 662 663
  bool is_on_stack() const { return IsOnStack::decode(flags_); }
  void set_is_on_stack(bool v) { flags_ = IsOnStack::update(flags_, v); }

664 665 666 667 668
  void SetFinalizationCallback(void* parameter,
                               WeakCallbackInfo<void>::Callback callback) {
    set_parameter(parameter);
    callback_ = callback;
  }
669
  bool HasFinalizationCallback() const { return callback_ != nullptr; }
670

671 672
  void CopyObjectReference(const TracedNode& other) { object_ = other.object_; }

673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691
  void CollectPhantomCallbackData(
      std::vector<std::pair<TracedNode*, PendingPhantomCallback>>*
          pending_phantom_callbacks) {
    DCHECK(IsInUse());
    DCHECK_NOT_NULL(callback_);

    void* embedder_fields[v8::kEmbedderFieldsInWeakCallback] = {nullptr,
                                                                nullptr};
    ExtractInternalFields(JSObject::cast(object()), embedder_fields,
                          v8::kEmbedderFieldsInWeakCallback);

    // Zap with something dangerous.
    location().store(Object(0xCA11));

    pending_phantom_callbacks->push_back(std::make_pair(
        this, PendingPhantomCallback(callback_, parameter(), embedder_fields)));
    set_state(NEAR_DEATH);
  }

692
  void ResetPhantomHandle(HandleHolder handle_holder) {
693
    DCHECK(IsInUse());
694 695 696 697
    if (handle_holder == HandleHolder::kLive) {
      Address** handle = reinterpret_cast<Address**>(data_.parameter);
      *handle = nullptr;
    }
698 699 700 701
    NodeSpace<TracedNode>::Release(this);
    DCHECK(!IsInUse());
  }

702
  static void Verify(GlobalHandles* global_handles, const Address* const* slot);
703

704
 protected:
705
  using NodeState = base::BitField8<State, 0, 2>;
706 707
  using IsInYoungList = NodeState::Next<bool, 1>;
  using IsRoot = IsInYoungList::Next<bool, 1>;
708
  using HasDestructor = IsRoot::Next<bool, 1>;
709
  using Markbit = HasDestructor::Next<bool, 1>;
710
  using IsOnStack = Markbit::Next<bool, 1>;
711

712 713
  void ClearImplFields() {
    set_root(true);
714 715
    // Nodes are black allocated for simplicity.
    set_markbit();
716
    callback_ = nullptr;
717
    set_is_on_stack(false);
718
    set_has_destructor(false);
719 720 721 722
  }

  void CheckImplFieldsAreCleared() const {
    DCHECK(is_root());
723
    DCHECK(markbit());
724 725 726 727
    DCHECK_NULL(callback_);
  }

  WeakCallbackInfo<void>::Callback callback_;
728 729 730

  friend class NodeBase<GlobalHandles::TracedNode>;
};
731

732 733 734 735 736 737 738 739
// Space to keep track of on-stack handles (e.g. TracedReference). Such
// references are treated as root for any V8 garbage collection. The data
// structure is self healing and pessimistally filters outdated entries on
// insertion and iteration.
//
// Design doc: http://bit.ly/on-stack-traced-reference
class GlobalHandles::OnStackTracedNodeSpace final {
 public:
740 741 742 743 744
  static GlobalHandles* GetGlobalHandles(const TracedNode* on_stack_node) {
    DCHECK(on_stack_node->is_on_stack());
    return reinterpret_cast<const NodeEntry*>(on_stack_node)->global_handles;
  }

745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781
  explicit OnStackTracedNodeSpace(GlobalHandles* global_handles)
      : global_handles_(global_handles) {}

  void SetStackStart(void* stack_start) {
    CHECK(on_stack_nodes_.empty());
    stack_start_ =
        GetStackAddressForSlot(reinterpret_cast<uintptr_t>(stack_start));
  }

  bool IsOnStack(uintptr_t slot) const {
    const uintptr_t address = GetStackAddressForSlot(slot);
    return stack_start_ >= address && address > GetCurrentStackPosition();
  }

  void Iterate(RootVisitor* v);
  TracedNode* Acquire(Object value, uintptr_t address);
  void CleanupBelowCurrentStackPosition();
  void NotifyEmptyEmbedderStack();

  size_t NumberOfHandlesForTesting() const { return on_stack_nodes_.size(); }

 private:
  struct NodeEntry {
    TracedNode node;
    // Used to find back to GlobalHandles from a Node on copy. Needs to follow
    // node.
    GlobalHandles* global_handles;
  };

  uintptr_t GetStackAddressForSlot(uintptr_t slot) const;

  // Keeps track of registered handles and their stack address. The data
  // structure is cleaned on iteration and when adding new references using the
  // current stack address.
  std::map<uintptr_t, NodeEntry> on_stack_nodes_;
  uintptr_t stack_start_ = 0;
  GlobalHandles* global_handles_ = nullptr;
782
  size_t acquire_count_ = 0;
783 784 785 786 787 788 789
};

uintptr_t GlobalHandles::OnStackTracedNodeSpace::GetStackAddressForSlot(
    uintptr_t slot) const {
#ifdef V8_USE_ADDRESS_SANITIZER
  void* fake_stack = __asan_get_current_fake_stack();
  if (fake_stack) {
790 791 792 793 794 795
    void* fake_frame_start;
    void* real_frame = __asan_addr_is_in_fake_stack(
        fake_stack, reinterpret_cast<void*>(slot), &fake_frame_start, nullptr);
    if (real_frame) {
      return reinterpret_cast<uintptr_t>(real_frame) +
             (slot - reinterpret_cast<uintptr_t>(fake_frame_start));
796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819
    }
  }
#endif  // V8_USE_ADDRESS_SANITIZER
  return slot;
}

void GlobalHandles::OnStackTracedNodeSpace::NotifyEmptyEmbedderStack() {
  on_stack_nodes_.clear();
}

void GlobalHandles::OnStackTracedNodeSpace::Iterate(RootVisitor* v) {
  // Handles have been cleaned from the GC entry point which is higher up the
  // stack.
  for (auto& pair : on_stack_nodes_) {
    TracedNode& node = pair.second.node;
    if (node.IsRetainer()) {
      v->VisitRootPointer(Root::kGlobalHandles, "on-stack TracedReference",
                          node.location());
    }
  }
}

GlobalHandles::TracedNode* GlobalHandles::OnStackTracedNodeSpace::Acquire(
    Object value, uintptr_t slot) {
820 821 822
  constexpr size_t kAcquireCleanupThresholdLog2 = 8;
  constexpr size_t kAcquireCleanupThresholdMask =
      (size_t{1} << kAcquireCleanupThresholdLog2) - 1;
823
  DCHECK(IsOnStack(slot));
824 825 826
  if (((acquire_count_++) & kAcquireCleanupThresholdMask) == 0) {
    CleanupBelowCurrentStackPosition();
  }
827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850
  NodeEntry entry;
  entry.node.Free(nullptr);
  entry.global_handles = global_handles_;
  auto pair =
      on_stack_nodes_.insert({GetStackAddressForSlot(slot), std::move(entry)});
  if (!pair.second) {
    // Insertion failed because there already was an entry present for that
    // stack address. This can happen because cleanup is conservative in which
    // stack limits it used. Reusing the entry is fine as there's no aliasing of
    // different references with the same stack slot.
    pair.first->second.node.Free(nullptr);
  }
  TracedNode* result = &(pair.first->second.node);
  result->Acquire(value);
  result->set_is_on_stack(true);
  return result;
}

void GlobalHandles::OnStackTracedNodeSpace::CleanupBelowCurrentStackPosition() {
  if (on_stack_nodes_.empty()) return;
  const auto it = on_stack_nodes_.upper_bound(GetCurrentStackPosition());
  on_stack_nodes_.erase(on_stack_nodes_.begin(), it);
}

851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875
// static
void GlobalHandles::TracedNode::Verify(GlobalHandles* global_handles,
                                       const Address* const* slot) {
#ifdef DEBUG
  const TracedNode* node = FromLocation(*slot);
  DCHECK(node->IsInUse());
  DCHECK_IMPLIES(!node->has_destructor(), nullptr == node->parameter());
  DCHECK_IMPLIES(node->has_destructor() && !node->HasFinalizationCallback(),
                 node->parameter());
  bool slot_on_stack = global_handles->on_stack_nodes_->IsOnStack(
      reinterpret_cast<uintptr_t>(slot));
  DCHECK_EQ(slot_on_stack, node->is_on_stack());
  if (!node->is_on_stack()) {
    // On-heap nodes have seprate lists for young generation processing.
    bool is_young_gen_object = ObjectInYoungGeneration(node->object());
    DCHECK_IMPLIES(is_young_gen_object, node->is_in_young_list());
  }
  bool in_young_list =
      std::find(global_handles->traced_young_nodes_.begin(),
                global_handles->traced_young_nodes_.end(),
                node) != global_handles->traced_young_nodes_.end();
  DCHECK_EQ(in_young_list, node->is_in_young_list());
#endif  // DEBUG
}

876 877 878 879 880 881 882 883
void GlobalHandles::CleanupOnStackReferencesBelowCurrentStackPosition() {
  on_stack_nodes_->CleanupBelowCurrentStackPosition();
}

size_t GlobalHandles::NumberOfOnStackHandlesForTesting() {
  return on_stack_nodes_->NumberOfHandlesForTesting();
}

884 885 886 887
size_t GlobalHandles::TotalSize() const {
  return regular_nodes_->TotalSize() + traced_nodes_->TotalSize();
}

888 889 890 891 892 893 894 895 896
size_t GlobalHandles::UsedSize() const {
  return regular_nodes_->handles_count() * sizeof(Node) +
         traced_nodes_->handles_count() * sizeof(TracedNode);
}

size_t GlobalHandles::handles_count() const {
  return regular_nodes_->handles_count() + traced_nodes_->handles_count();
}

897 898 899 900 901 902 903 904
void GlobalHandles::SetStackStart(void* stack_start) {
  on_stack_nodes_->SetStackStart(stack_start);
}

void GlobalHandles::NotifyEmptyEmbedderStack() {
  on_stack_nodes_->NotifyEmptyEmbedderStack();
}

905 906
GlobalHandles::GlobalHandles(Isolate* isolate)
    : isolate_(isolate),
907
      regular_nodes_(new NodeSpace<GlobalHandles::Node>(this)),
908 909
      traced_nodes_(new NodeSpace<GlobalHandles::TracedNode>(this)),
      on_stack_nodes_(new OnStackTracedNodeSpace(this)) {}
910

911
GlobalHandles::~GlobalHandles() { regular_nodes_.reset(nullptr); }
912

913
Handle<Object> GlobalHandles::Create(Object value) {
914
  GlobalHandles::Node* result = regular_nodes_->Acquire(value);
915
  if (ObjectInYoungGeneration(value) && !result->is_in_young_list()) {
916 917
    young_nodes_.push_back(result);
    result->set_in_young_list(true);
918 919 920 921
  }
  return result->handle();
}

922
Handle<Object> GlobalHandles::Create(Address value) {
923
  return Create(Object(value));
924
}
925

926 927
Handle<Object> GlobalHandles::CreateTraced(Object value, Address* slot,
                                           bool has_destructor) {
928 929 930 931 932 933 934 935
  return CreateTraced(
      value, slot, has_destructor,
      on_stack_nodes_->IsOnStack(reinterpret_cast<uintptr_t>(slot)));
}

Handle<Object> GlobalHandles::CreateTraced(Object value, Address* slot,
                                           bool has_destructor,
                                           bool is_on_stack) {
936
  GlobalHandles::TracedNode* result;
937 938
  if (is_on_stack) {
    result = on_stack_nodes_->Acquire(value, reinterpret_cast<uintptr_t>(slot));
939 940 941 942 943 944
  } else {
    result = traced_nodes_->Acquire(value);
    if (ObjectInYoungGeneration(value) && !result->is_in_young_list()) {
      traced_young_nodes_.push_back(result);
      result->set_in_young_list(true);
    }
945
  }
946 947
  result->set_has_destructor(has_destructor);
  result->set_parameter(has_destructor ? slot : nullptr);
948 949 950
  return result->handle();
}

951 952 953
Handle<Object> GlobalHandles::CreateTraced(Address value, Address* slot,
                                           bool has_destructor) {
  return CreateTraced(Object(value), slot, has_destructor);
954 955
}

956
Handle<Object> GlobalHandles::CopyGlobal(Address* location) {
957
  DCHECK_NOT_NULL(location);
958
  GlobalHandles* global_handles =
959
      Node::FromLocation(location)->global_handles();
960 961
#ifdef VERIFY_HEAP
  if (i::FLAG_verify_heap) {
962
    Object(*location).ObjectVerify(global_handles->isolate());
963 964 965
  }
#endif  // VERIFY_HEAP
  return global_handles->Create(*location);
966 967
}

968 969 970 971 972 973 974 975
// static
void GlobalHandles::CopyTracedGlobal(const Address* const* from, Address** to) {
  DCHECK_NOT_NULL(*from);
  DCHECK_NULL(*to);
  const TracedNode* node = TracedNode::FromLocation(*from);
  // Copying a traced handle with finalization callback is prohibited because
  // the callback may require knowing about multiple copies of the traced
  // handle.
976 977 978
  CHECK_WITH_MSG(!node->HasFinalizationCallback(),
                 "Copying of references is not supported when "
                 "SetFinalizationCallback is set.");
979

980
  GlobalHandles* global_handles =
981
      GlobalHandles::From(const_cast<TracedNode*>(node));
982 983 984
  Handle<Object> o = global_handles->CreateTraced(
      node->object(), reinterpret_cast<Address*>(to), node->has_destructor());
  *to = o.location();
985 986
  TracedNode::Verify(global_handles, from);
  TracedNode::Verify(global_handles, to);
987 988 989 990 991 992 993
#ifdef VERIFY_HEAP
  if (i::FLAG_verify_heap) {
    Object(**to).ObjectVerify(global_handles->isolate());
  }
#endif  // VERIFY_HEAP
}

994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007
void GlobalHandles::MoveGlobal(Address** from, Address** to) {
  DCHECK_NOT_NULL(*from);
  DCHECK_NOT_NULL(*to);
  DCHECK_EQ(*from, *to);
  Node* node = Node::FromLocation(*from);
  if (node->IsWeak() && node->IsPhantomResetHandle()) {
    node->set_parameter(to);
  }

  // - Strong handles do not require fixups.
  // - Weak handles with finalizers and callbacks are too general to fix up. For
  //   those the callers need to ensure consistency.
}

1008
void GlobalHandles::MoveTracedGlobal(Address** from, Address** to) {
1009 1010 1011 1012 1013
  // Fast path for moving from an empty reference.
  if (!*from) {
    DestroyTraced(*to);
    *to = nullptr;
    return;
1014
  }
1015 1016 1017

  // Determining whether from or to are on stack.
  TracedNode* from_node = TracedNode::FromLocation(*from);
1018
  DCHECK(from_node->IsInUse());
1019 1020
  TracedNode* to_node = TracedNode::FromLocation(*to);
  GlobalHandles* global_handles = nullptr;
1021 1022 1023
#ifdef DEBUG
  global_handles = GlobalHandles::From(from_node);
#endif  // DEBUG
1024 1025 1026 1027 1028 1029 1030
  bool from_on_stack = from_node->is_on_stack();
  bool to_on_stack = false;
  if (!to_node) {
    // Figure out whether stack or heap to allow fast path for heap->heap move.
    global_handles = GlobalHandles::From(from_node);
    to_on_stack = global_handles->on_stack_nodes_->IsOnStack(
        reinterpret_cast<uintptr_t>(to));
1031 1032
  } else {
    to_on_stack = to_node->is_on_stack();
1033 1034
  }

1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045
  // Moving a traced handle with finalization callback is prohibited because
  // the callback may require knowing about multiple copies of the traced
  // handle.
  CHECK_WITH_MSG(!from_node->HasFinalizationCallback(),
                 "Moving of references is not supported when "
                 "SetFinalizationCallback is set.");
  // Types in v8.h ensure that we only copy/move handles that have the same
  // destructor behavior.
  DCHECK_IMPLIES(to_node,
                 to_node->has_destructor() == from_node->has_destructor());

1046 1047 1048 1049 1050 1051
  // Moving.
  if (from_on_stack || to_on_stack) {
    // Move involving a stack slot.
    if (!to_node) {
      DCHECK(global_handles);
      Handle<Object> o = global_handles->CreateTraced(
1052 1053
          from_node->object(), reinterpret_cast<Address*>(to),
          from_node->has_destructor(), to_on_stack);
1054
      *to = o.location();
1055 1056
      to_node = TracedNode::FromLocation(*to);
      DCHECK(to_node->markbit());
1057
    } else {
1058 1059 1060 1061 1062 1063 1064
      DCHECK(to_node->IsInUse());
      to_node->CopyObjectReference(*from_node);
      if (!to_node->is_on_stack() && !to_node->is_in_young_list() &&
          ObjectInYoungGeneration(to_node->object())) {
        global_handles = GlobalHandles::From(from_node);
        global_handles->traced_young_nodes_.push_back(to_node);
        to_node->set_in_young_list(true);
1065
      }
1066 1067 1068 1069 1070 1071 1072
    }
    DestroyTraced(*from);
    *from = nullptr;
  } else {
    // Pure heap move.
    DestroyTraced(*to);
    *to = *from;
1073
    to_node = from_node;
1074 1075 1076
    DCHECK_NOT_NULL(*from);
    DCHECK_NOT_NULL(*to);
    DCHECK_EQ(*from, *to);
1077 1078 1079
    // Fixup back reference for destructor.
    if (to_node->has_destructor()) {
      to_node->set_parameter(to);
1080 1081 1082
    }
    *from = nullptr;
  }
1083
  TracedNode::Verify(global_handles, to);
1084 1085 1086 1087 1088 1089 1090
}

// static
GlobalHandles* GlobalHandles::From(const TracedNode* node) {
  return node->is_on_stack()
             ? OnStackTracedNodeSpace::GetGlobalHandles(node)
             : NodeBlock<TracedNode>::From(node)->global_handles();
1091 1092
}

1093 1094 1095 1096 1097 1098
void GlobalHandles::MarkTraced(Address* location) {
  TracedNode* node = TracedNode::FromLocation(location);
  node->set_markbit();
  DCHECK(node->IsInUse());
}

1099
void GlobalHandles::Destroy(Address* location) {
1100 1101 1102
  if (location != nullptr) {
    NodeSpace<Node>::Release(Node::FromLocation(location));
  }
1103 1104
}

1105 1106
void GlobalHandles::DestroyTraced(Address* location) {
  if (location != nullptr) {
1107 1108 1109 1110 1111 1112
    TracedNode* node = TracedNode::FromLocation(location);
    if (node->is_on_stack()) {
      node->Release(nullptr);
    } else {
      NodeSpace<TracedNode>::Release(node);
    }
1113 1114
  }
}
1115

1116 1117 1118 1119 1120 1121 1122
void GlobalHandles::SetFinalizationCallbackForTraced(
    Address* location, void* parameter,
    WeakCallbackInfo<void>::Callback callback) {
  TracedNode::FromLocation(location)->SetFinalizationCallback(parameter,
                                                              callback);
}

1123
using GenericCallback = v8::WeakCallbackInfo<void>::Callback;
1124

1125 1126 1127
void GlobalHandles::MakeWeak(Address* location, void* parameter,
                             GenericCallback phantom_callback,
                             v8::WeakCallbackType type) {
1128
  Node::FromLocation(location)->MakeWeak(parameter, phantom_callback, type);
1129 1130
}

1131
void GlobalHandles::MakeWeak(Address** location_addr) {
1132
  Node::FromLocation(*location_addr)->MakeWeak(location_addr);
1133 1134
}

1135
void* GlobalHandles::ClearWeakness(Address* location) {
1136
  return Node::FromLocation(location)->ClearWeakness();
1137 1138
}

1139 1140
void GlobalHandles::AnnotateStrongRetainer(Address* location,
                                           const char* label) {
1141
  Node::FromLocation(location)->AnnotateStrongRetainer(label);
1142 1143
}

1144
bool GlobalHandles::IsWeak(Address* location) {
1145 1146 1147
  return Node::FromLocation(location)->IsWeak();
}

krasin's avatar
krasin committed
1148
DISABLE_CFI_PERF
1149
void GlobalHandles::IterateWeakRootsForFinalizers(RootVisitor* v) {
1150
  for (Node* node : *regular_nodes_) {
1151 1152 1153 1154
    if (node->IsWeakRetainer() && node->state() == Node::PENDING) {
      DCHECK(!node->IsPhantomCallback());
      DCHECK(!node->IsPhantomResetHandle());
      // Finalizers need to survive.
1155 1156
      v->VisitRootPointer(Root::kGlobalHandles, node->label(),
                          node->location());
1157 1158 1159 1160 1161 1162
    }
  }
}

DISABLE_CFI_PERF
void GlobalHandles::IterateWeakRootsForPhantomHandles(
1163
    WeakSlotCallbackWithHeap should_reset_handle) {
1164
  for (Node* node : *regular_nodes_) {
1165 1166
    if (node->IsWeakRetainer() &&
        should_reset_handle(isolate()->heap(), node->location())) {
1167 1168
      if (node->IsPhantomResetHandle()) {
        node->MarkPending();
1169
        node->ResetPhantomHandle(HandleHolder::kLive);
1170
        ++number_of_phantom_handle_resets_;
1171 1172
      } else if (node->IsPhantomCallback()) {
        node->MarkPending();
1173
        node->CollectPhantomCallbackData(&regular_pending_phantom_callbacks_);
1174 1175
      }
    }
1176
  }
1177
  for (TracedNode* node : *traced_nodes_) {
1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197
    if (!node->IsInUse()) continue;
    // Detect unreachable nodes first.
    if (!node->markbit() && node->IsPhantomResetHandle() &&
        !node->has_destructor()) {
      // The handle is unreachable and does not have a callback and a
      // destructor associated with it. We can clear it even if the target V8
      // object is alive. Note that the desctructor and the callback may
      // access the handle, that is why we avoid clearing it.
      node->ResetPhantomHandle(HandleHolder::kDead);
      ++number_of_phantom_handle_resets_;
      continue;
    } else if (node->markbit()) {
      // Clear the markbit for the next GC.
      node->clear_markbit();
    }
    DCHECK(node->IsInUse());
    // Detect nodes with unreachable target objects.
    if (should_reset_handle(isolate()->heap(), node->location())) {
      // If the node allows eager resetting, then reset it here. Otherwise,
      // collect its callback that will reset it.
1198
      if (node->IsPhantomResetHandle()) {
1199 1200
        node->ResetPhantomHandle(node->has_destructor() ? HandleHolder::kLive
                                                        : HandleHolder::kDead);
1201 1202 1203 1204
        ++number_of_phantom_handle_resets_;
      } else {
        node->CollectPhantomCallbackData(&traced_pending_phantom_callbacks_);
      }
1205 1206
    }
  }
1207 1208
}

1209
void GlobalHandles::IterateWeakRootsIdentifyFinalizers(
1210
    WeakSlotCallbackWithHeap should_reset_handle) {
1211
  for (Node* node : *regular_nodes_) {
1212 1213
    if (node->IsWeak() &&
        should_reset_handle(isolate()->heap(), node->location())) {
1214
      if (node->IsFinalizerHandle()) {
1215 1216
        node->MarkPending();
      }
1217 1218 1219 1220
    }
  }
}

1221 1222 1223 1224
void GlobalHandles::IdentifyWeakUnmodifiedObjects(
    WeakSlotCallback is_unmodified) {
  LocalEmbedderHeapTracer* const tracer =
      isolate()->heap()->local_embedder_heap_tracer();
1225
  for (TracedNode* node : traced_young_nodes_) {
1226 1227 1228 1229
    if (node->IsInUse()) {
      DCHECK(node->is_root());
      if (is_unmodified(node->location())) {
        v8::Value* value = ToApi<v8::Value>(node->handle());
1230 1231 1232 1233 1234 1235 1236
        if (node->has_destructor()) {
          node->set_root(tracer->IsRootForNonTracingGC(
              *reinterpret_cast<v8::TracedGlobal<v8::Value>*>(&value)));
        } else {
          node->set_root(tracer->IsRootForNonTracingGC(
              *reinterpret_cast<v8::TracedReference<v8::Value>*>(&value)));
        }
1237 1238 1239 1240 1241
      }
    }
  }
}

1242 1243
void GlobalHandles::IterateYoungStrongAndDependentRoots(RootVisitor* v) {
  for (Node* node : young_nodes_) {
1244
    if (node->IsStrongRetainer()) {
1245 1246
      v->VisitRootPointer(Root::kGlobalHandles, node->label(),
                          node->location());
1247 1248
    }
  }
1249
  for (TracedNode* node : traced_young_nodes_) {
1250 1251
    if (node->IsInUse() && node->is_root()) {
      v->VisitRootPointer(Root::kGlobalHandles, nullptr, node->location());
1252 1253 1254 1255
    }
  }
}

1256
void GlobalHandles::MarkYoungWeakUnmodifiedObjectsPending(
1257
    WeakSlotCallbackWithHeap is_dead) {
1258 1259
  for (Node* node : young_nodes_) {
    DCHECK(node->is_in_young_list());
1260
    if (node->IsWeak() && is_dead(isolate_->heap(), node->location())) {
1261 1262 1263
      if (!node->IsPhantomCallback() && !node->IsPhantomResetHandle()) {
        node->MarkPending();
      }
1264 1265 1266 1267
    }
  }
}

1268
void GlobalHandles::IterateYoungWeakUnmodifiedRootsForFinalizers(
1269
    RootVisitor* v) {
1270 1271
  for (Node* node : young_nodes_) {
    DCHECK(node->is_in_young_list());
1272
    if (node->IsWeakRetainer() && (node->state() == Node::PENDING)) {
1273 1274 1275
      DCHECK(!node->IsPhantomCallback());
      DCHECK(!node->IsPhantomResetHandle());
      // Finalizers need to survive.
1276 1277
      v->VisitRootPointer(Root::kGlobalHandles, node->label(),
                          node->location());
1278 1279 1280 1281
    }
  }
}

1282
void GlobalHandles::IterateYoungWeakUnmodifiedRootsForPhantomHandles(
1283
    RootVisitor* v, WeakSlotCallbackWithHeap should_reset_handle) {
1284 1285
  for (Node* node : young_nodes_) {
    DCHECK(node->is_in_young_list());
1286
    if (node->IsWeakRetainer() && (node->state() != Node::PENDING)) {
1287
      if (should_reset_handle(isolate_->heap(), node->location())) {
1288
        DCHECK(node->IsPhantomResetHandle() || node->IsPhantomCallback());
1289 1290
        if (node->IsPhantomResetHandle()) {
          node->MarkPending();
1291
          node->ResetPhantomHandle(HandleHolder::kLive);
1292 1293 1294
          ++number_of_phantom_handle_resets_;
        } else if (node->IsPhantomCallback()) {
          node->MarkPending();
1295
          node->CollectPhantomCallbackData(&regular_pending_phantom_callbacks_);
1296 1297 1298
        } else {
          UNREACHABLE();
        }
1299
      } else {
1300
        // Node survived and needs to be visited.
1301 1302
        v->VisitRootPointer(Root::kGlobalHandles, node->label(),
                            node->location());
1303 1304 1305
      }
    }
  }
1306 1307 1308

  LocalEmbedderHeapTracer* const tracer =
      isolate()->heap()->local_embedder_heap_tracer();
1309
  for (TracedNode* node : traced_young_nodes_) {
1310 1311 1312 1313 1314
    if (!node->IsInUse()) continue;

    DCHECK_IMPLIES(node->is_root(),
                   !should_reset_handle(isolate_->heap(), node->location()));
    if (should_reset_handle(isolate_->heap(), node->location())) {
1315
      if (node->IsPhantomResetHandle()) {
1316 1317 1318 1319 1320 1321 1322 1323
        if (node->has_destructor()) {
          // For handles with destructor it is guaranteed that the embedder
          // memory is still alive as the destructor would have otherwise
          // removed the memory.
          node->ResetPhantomHandle(HandleHolder::kLive);
        } else {
          v8::Value* value = ToApi<v8::Value>(node->handle());
          tracer->ResetHandleInNonTracingGC(
1324
              *reinterpret_cast<v8::TracedReference<v8::Value>*>(&value));
1325 1326 1327
          DCHECK(!node->IsInUse());
        }

1328 1329 1330 1331
        ++number_of_phantom_handle_resets_;
      } else {
        node->CollectPhantomCallbackData(&traced_pending_phantom_callbacks_);
      }
1332 1333 1334 1335 1336 1337 1338
    } else {
      if (!node->is_root()) {
        node->set_root(true);
        v->VisitRootPointer(Root::kGlobalHandles, nullptr, node->location());
      }
    }
  }
1339 1340
}

1341 1342 1343
void GlobalHandles::InvokeSecondPassPhantomCallbacksFromTask() {
  DCHECK(second_pass_callbacks_task_posted_);
  second_pass_callbacks_task_posted_ = false;
1344 1345
  Heap::DevToolsTraceEventScope devtools_trace_event_scope(
      isolate()->heap(), "MajorGC", "invoke weak phantom callbacks");
1346 1347 1348 1349 1350 1351 1352 1353 1354
  TRACE_EVENT0("v8", "V8.GCPhantomHandleProcessingCallback");
  isolate()->heap()->CallGCPrologueCallbacks(
      GCType::kGCTypeProcessWeakCallbacks, kNoGCCallbackFlags);
  InvokeSecondPassPhantomCallbacks();
  isolate()->heap()->CallGCEpilogueCallbacks(
      GCType::kGCTypeProcessWeakCallbacks, kNoGCCallbackFlags);
}

void GlobalHandles::InvokeSecondPassPhantomCallbacks() {
1355 1356 1357 1358 1359 1360 1361 1362
  // The callbacks may execute JS, which in turn may lead to another GC run.
  // If we are already processing the callbacks, we do not want to start over
  // from within the inner GC. Newly added callbacks will always be run by the
  // outermost GC run only.
  if (running_second_pass_callbacks_) return;
  running_second_pass_callbacks_ = true;

  AllowJavascriptExecution allow_js(isolate());
1363 1364 1365
  while (!second_pass_callbacks_.empty()) {
    auto callback = second_pass_callbacks_.back();
    second_pass_callbacks_.pop_back();
1366
    callback.Invoke(isolate(), PendingPhantomCallback::kSecondPass);
1367
  }
1368
  running_second_pass_callbacks_ = false;
1369 1370
}

1371 1372
size_t GlobalHandles::PostScavengeProcessing(unsigned post_processing_count) {
  size_t freed_nodes = 0;
1373
  for (Node* node : young_nodes_) {
1374 1375 1376 1377 1378 1379 1380
    // Filter free nodes.
    if (!node->IsRetainer()) continue;

    if (node->IsPending()) {
      DCHECK(node->has_callback());
      DCHECK(node->IsPendingFinalizer());
      node->PostGarbageCollectionProcessing(isolate_);
1381
    }
1382 1383 1384
    if (InRecursiveGC(post_processing_count)) return freed_nodes;

    if (!node->IsRetainer()) freed_nodes++;
1385 1386 1387 1388
  }
  return freed_nodes;
}

1389 1390
size_t GlobalHandles::PostMarkSweepProcessing(unsigned post_processing_count) {
  size_t freed_nodes = 0;
1391
  for (Node* node : *regular_nodes_) {
1392 1393 1394 1395 1396 1397 1398
    // Filter free nodes.
    if (!node->IsRetainer()) continue;

    if (node->IsPending()) {
      DCHECK(node->has_callback());
      DCHECK(node->IsPendingFinalizer());
      node->PostGarbageCollectionProcessing(isolate_);
1399
    }
1400 1401 1402
    if (InRecursiveGC(post_processing_count)) return freed_nodes;

    if (!node->IsRetainer()) freed_nodes++;
1403
  }
1404 1405 1406
  return freed_nodes;
}

1407
template <typename T>
1408
void GlobalHandles::UpdateAndCompactListOfYoungNode(
1409
    std::vector<T*>* node_list) {
1410
  size_t last = 0;
1411
  for (T* node : *node_list) {
1412
    DCHECK(node->is_in_young_list());
1413
    if (node->IsInUse()) {
1414
      if (ObjectInYoungGeneration(node->object())) {
1415
        (*node_list)[last++] = node;
1416
        isolate_->heap()->IncrementNodesCopiedInNewSpace();
1417
      } else {
1418
        node->set_in_young_list(false);
1419
        isolate_->heap()->IncrementNodesPromoted();
1420
      }
1421
    } else {
1422
      node->set_in_young_list(false);
1423
      isolate_->heap()->IncrementNodesDiedInNewSpace();
1424
    }
1425
  }
1426 1427 1428 1429 1430
  DCHECK_LE(last, node_list->size());
  node_list->resize(last);
  node_list->shrink_to_fit();
}

1431 1432 1433
void GlobalHandles::UpdateListOfYoungNodes() {
  UpdateAndCompactListOfYoungNode(&young_nodes_);
  UpdateAndCompactListOfYoungNode(&traced_young_nodes_);
1434 1435
}

1436 1437 1438
template <typename T>
size_t GlobalHandles::InvokeFirstPassWeakCallbacks(
    std::vector<std::pair<T*, PendingPhantomCallback>>* pending) {
1439
  size_t freed_nodes = 0;
1440 1441
  std::vector<std::pair<T*, PendingPhantomCallback>> pending_phantom_callbacks;
  pending_phantom_callbacks.swap(*pending);
dcarney's avatar
dcarney committed
1442 1443
  {
    // The initial pass callbacks must simply clear the nodes.
1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456
    for (auto& pair : pending_phantom_callbacks) {
      T* node = pair.first;
      DCHECK_EQ(T::NEAR_DEATH, node->state());
      pair.second.Invoke(isolate(), PendingPhantomCallback::kFirstPass);

      // Transition to second pass. It is required that the first pass callback
      // resets the handle using |v8::PersistentBase::Reset|. Also see comments
      // on |v8::WeakCallbackInfo|.
      CHECK_WITH_MSG(T::FREE == node->state(),
                     "Handle not reset in first callback. See comments on "
                     "|v8::WeakCallbackInfo|.");

      if (pair.second.callback()) second_pass_callbacks_.push_back(pair.second);
dcarney's avatar
dcarney committed
1457 1458 1459
      freed_nodes++;
    }
  }
1460 1461 1462
  return freed_nodes;
}

1463 1464 1465 1466 1467
size_t GlobalHandles::InvokeFirstPassWeakCallbacks() {
  return InvokeFirstPassWeakCallbacks(&regular_pending_phantom_callbacks_) +
         InvokeFirstPassWeakCallbacks(&traced_pending_phantom_callbacks_);
}

1468 1469
void GlobalHandles::InvokeOrScheduleSecondPassPhantomCallbacks(
    bool synchronous_second_pass) {
1470
  if (!second_pass_callbacks_.empty()) {
1471
    if (FLAG_optimize_for_size || FLAG_predictable || synchronous_second_pass) {
1472 1473
      Heap::DevToolsTraceEventScope devtools_trace_event_scope(
          isolate()->heap(), "MajorGC", "invoke weak phantom callbacks");
1474 1475
      isolate()->heap()->CallGCPrologueCallbacks(
          GCType::kGCTypeProcessWeakCallbacks, kNoGCCallbackFlags);
1476
      InvokeSecondPassPhantomCallbacks();
1477 1478
      isolate()->heap()->CallGCEpilogueCallbacks(
          GCType::kGCTypeProcessWeakCallbacks, kNoGCCallbackFlags);
1479 1480
    } else if (!second_pass_callbacks_task_posted_) {
      second_pass_callbacks_task_posted_ = true;
1481 1482
      auto taskrunner = V8::GetCurrentPlatform()->GetForegroundTaskRunner(
          reinterpret_cast<v8::Isolate*>(isolate()));
1483
      taskrunner->PostTask(MakeCancelableTask(
1484
          isolate(), [this] { InvokeSecondPassPhantomCallbacksFromTask(); }));
1485
    }
1486 1487 1488
  }
}

1489 1490
void GlobalHandles::PendingPhantomCallback::Invoke(Isolate* isolate,
                                                   InvocationType type) {
dcarney's avatar
dcarney committed
1491
  Data::Callback* callback_addr = nullptr;
1492
  if (type == kFirstPass) {
dcarney's avatar
dcarney committed
1493 1494 1495
    callback_addr = &callback_;
  }
  Data data(reinterpret_cast<v8::Isolate*>(isolate), parameter_,
1496
            embedder_fields_, callback_addr);
dcarney's avatar
dcarney committed
1497 1498 1499 1500 1501
  Data::Callback callback = callback_;
  callback_ = nullptr;
  callback(data);
}

1502 1503 1504 1505 1506
bool GlobalHandles::InRecursiveGC(unsigned gc_processing_counter) {
  return gc_processing_counter != post_gc_processing_count_;
}

size_t GlobalHandles::PostGarbageCollectionProcessing(
1507
    GarbageCollector collector, const v8::GCCallbackFlags gc_callback_flags) {
1508 1509 1510
  // Process weak global handle callbacks. This must be done after the
  // GC is completely done, because the callbacks may invoke arbitrary
  // API functions.
1511 1512 1513
  DCHECK_EQ(Heap::NOT_IN_GC, isolate_->heap()->gc_state());
  const unsigned post_processing_count = ++post_gc_processing_count_;
  size_t freed_nodes = 0;
1514
  bool synchronous_second_pass =
1515
      isolate_->heap()->IsTearingDown() ||
1516
      (gc_callback_flags &
1517
       (kGCCallbackFlagForced | kGCCallbackFlagCollectAllAvailableGarbage |
1518
        kGCCallbackFlagSynchronousPhantomCallbackProcessing)) != 0;
1519
  InvokeOrScheduleSecondPassPhantomCallbacks(synchronous_second_pass);
1520 1521 1522 1523 1524 1525 1526
  if (InRecursiveGC(post_processing_count)) return freed_nodes;

  freed_nodes += Heap::IsYoungGenerationCollector(collector)
                     ? PostScavengeProcessing(post_processing_count)
                     : PostMarkSweepProcessing(post_processing_count);
  if (InRecursiveGC(post_processing_count)) return freed_nodes;

1527
  UpdateListOfYoungNodes();
1528
  return freed_nodes;
1529 1530
}

1531
void GlobalHandles::IterateStrongRoots(RootVisitor* v) {
1532 1533 1534 1535
  for (Node* node : *regular_nodes_) {
    if (node->IsStrongRetainer()) {
      v->VisitRootPointer(Root::kGlobalHandles, node->label(),
                          node->location());
1536 1537
    }
  }
1538 1539 1540
}

void GlobalHandles::IterateStrongStackRoots(RootVisitor* v) {
1541
  on_stack_nodes_->Iterate(v);
1542 1543
}

1544
void GlobalHandles::IterateWeakRoots(RootVisitor* v) {
1545 1546 1547 1548
  for (Node* node : *regular_nodes_) {
    if (node->IsWeak()) {
      v->VisitRootPointer(Root::kGlobalHandles, node->label(),
                          node->location());
1549 1550
    }
  }
1551 1552 1553 1554 1555
  for (TracedNode* node : *traced_nodes_) {
    if (node->IsInUse()) {
      v->VisitRootPointer(Root::kGlobalHandles, nullptr, node->location());
    }
  }
1556
}
1557

krasin's avatar
krasin committed
1558
DISABLE_CFI_PERF
1559
void GlobalHandles::IterateAllRoots(RootVisitor* v) {
1560 1561 1562 1563
  for (Node* node : *regular_nodes_) {
    if (node->IsRetainer()) {
      v->VisitRootPointer(Root::kGlobalHandles, node->label(),
                          node->location());
1564 1565
    }
  }
1566
  for (TracedNode* node : *traced_nodes_) {
1567
    if (node->IsRetainer()) {
1568 1569 1570
      v->VisitRootPointer(Root::kGlobalHandles, nullptr, node->location());
    }
  }
1571
  on_stack_nodes_->Iterate(v);
1572 1573
}

1574
DISABLE_CFI_PERF
1575 1576
void GlobalHandles::IterateAllYoungRoots(RootVisitor* v) {
  for (Node* node : young_nodes_) {
1577
    if (node->IsRetainer()) {
1578 1579
      v->VisitRootPointer(Root::kGlobalHandles, node->label(),
                          node->location());
1580 1581
    }
  }
1582
  for (TracedNode* node : traced_young_nodes_) {
1583
    if (node->IsRetainer()) {
1584 1585 1586
      v->VisitRootPointer(Root::kGlobalHandles, nullptr, node->location());
    }
  }
1587
  on_stack_nodes_->Iterate(v);
1588
}
1589

krasin's avatar
krasin committed
1590
DISABLE_CFI_PERF
1591 1592
void GlobalHandles::ApplyPersistentHandleVisitor(
    v8::PersistentHandleVisitor* visitor, GlobalHandles::Node* node) {
1593
  v8::Value* value = ToApi<v8::Value>(node->handle());
1594 1595 1596 1597 1598 1599 1600 1601
  visitor->VisitPersistentHandle(
      reinterpret_cast<v8::Persistent<v8::Value>*>(&value),
      node->wrapper_class_id());
}

DISABLE_CFI_PERF
void GlobalHandles::IterateAllRootsWithClassIds(
    v8::PersistentHandleVisitor* visitor) {
1602 1603 1604
  for (Node* node : *regular_nodes_) {
    if (node->IsRetainer() && node->has_wrapper_class_id()) {
      ApplyPersistentHandleVisitor(visitor, node);
1605 1606 1607 1608
    }
  }
}

1609 1610 1611 1612 1613 1614
DISABLE_CFI_PERF
void GlobalHandles::IterateTracedNodes(
    v8::EmbedderHeapTracer::TracedGlobalHandleVisitor* visitor) {
  for (TracedNode* node : *traced_nodes_) {
    if (node->IsInUse()) {
      v8::Value* value = ToApi<v8::Value>(node->handle());
1615 1616 1617 1618 1619 1620 1621
      if (node->has_destructor()) {
        visitor->VisitTracedGlobalHandle(
            *reinterpret_cast<v8::TracedGlobal<v8::Value>*>(&value));
      } else {
        visitor->VisitTracedReference(
            *reinterpret_cast<v8::TracedReference<v8::Value>*>(&value));
      }
1622 1623 1624
    }
  }
}
1625

krasin's avatar
krasin committed
1626
DISABLE_CFI_PERF
1627
void GlobalHandles::IterateAllYoungRootsWithClassIds(
1628
    v8::PersistentHandleVisitor* visitor) {
1629
  for (Node* node : young_nodes_) {
1630
    if (node->IsRetainer() && node->has_wrapper_class_id()) {
1631
      ApplyPersistentHandleVisitor(visitor, node);
1632 1633 1634 1635
    }
  }
}

krasin's avatar
krasin committed
1636
DISABLE_CFI_PERF
1637
void GlobalHandles::IterateYoungWeakRootsWithClassIds(
1638
    v8::PersistentHandleVisitor* visitor) {
1639
  for (Node* node : young_nodes_) {
1640
    if (node->has_wrapper_class_id() && node->IsWeak()) {
1641
      ApplyPersistentHandleVisitor(visitor, node);
1642 1643 1644 1645
    }
  }
}

1646
void GlobalHandles::RecordStats(HeapStats* stats) {
1647 1648 1649 1650 1651
  *stats->global_handle_count = 0;
  *stats->weak_global_handle_count = 0;
  *stats->pending_global_handle_count = 0;
  *stats->near_death_global_handle_count = 0;
  *stats->free_global_handle_count = 0;
1652
  for (Node* node : *regular_nodes_) {
1653
    *stats->global_handle_count += 1;
1654
    if (node->state() == Node::WEAK) {
1655
      *stats->weak_global_handle_count += 1;
1656
    } else if (node->state() == Node::PENDING) {
1657
      *stats->pending_global_handle_count += 1;
1658
    } else if (node->state() == Node::NEAR_DEATH) {
1659
      *stats->near_death_global_handle_count += 1;
1660
    } else if (node->state() == Node::FREE) {
1661 1662 1663
      *stats->free_global_handle_count += 1;
    }
  }
1664 1665
}

1666 1667 1668
#ifdef DEBUG

void GlobalHandles::PrintStats() {
1669 1670 1671 1672 1673 1674
  int total = 0;
  int weak = 0;
  int pending = 0;
  int near_death = 0;
  int destroyed = 0;

1675
  for (Node* node : *regular_nodes_) {
1676
    total++;
1677 1678 1679 1680
    if (node->state() == Node::WEAK) weak++;
    if (node->state() == Node::PENDING) pending++;
    if (node->state() == Node::NEAR_DEATH) near_death++;
    if (node->state() == Node::FREE) destroyed++;
1681 1682
  }

1683
  PrintF("Global Handle Statistics:\n");
1684
  PrintF("  allocated memory = %zuB\n", total * sizeof(Node));
1685 1686 1687 1688
  PrintF("  # weak       = %d\n", weak);
  PrintF("  # pending    = %d\n", pending);
  PrintF("  # near_death = %d\n", near_death);
  PrintF("  # free       = %d\n", destroyed);
1689 1690 1691 1692 1693
  PrintF("  # total      = %d\n", total);
}

void GlobalHandles::Print() {
  PrintF("Global handles:\n");
1694 1695
  for (Node* node : *regular_nodes_) {
    PrintF("  handle %p to %p%s\n", node->location().ToVoidPtr(),
1696
           reinterpret_cast<void*>(node->object().ptr()),
1697
           node->IsWeak() ? " (weak)" : "");
1698 1699 1700 1701 1702
  }
}

#endif

1703
EternalHandles::~EternalHandles() {
1704
  for (Address* block : blocks_) delete[] block;
1705 1706
}

1707
void EternalHandles::IterateAllRoots(RootVisitor* visitor) {
1708
  int limit = size_;
1709
  for (Address* block : blocks_) {
1710
    DCHECK_GT(limit, 0);
1711
    visitor->VisitRootPointers(Root::kEternalHandles, nullptr,
1712 1713
                               FullObjectSlot(block),
                               FullObjectSlot(block + Min(limit, kSize)));
1714 1715 1716 1717
    limit -= kSize;
  }
}

1718 1719
void EternalHandles::IterateYoungRoots(RootVisitor* visitor) {
  for (int index : young_node_indices_) {
1720
    visitor->VisitRootPointer(Root::kEternalHandles, nullptr,
1721
                              FullObjectSlot(GetLocation(index)));
1722 1723 1724
  }
}

1725
void EternalHandles::PostGarbageCollectionProcessing() {
1726
  size_t last = 0;
1727
  for (int index : young_node_indices_) {
1728
    if (ObjectInYoungGeneration(Object(*GetLocation(index)))) {
1729
      young_node_indices_[last++] = index;
1730 1731
    }
  }
1732 1733
  DCHECK_LE(last, young_node_indices_.size());
  young_node_indices_.resize(last);
1734 1735
}

1736
void EternalHandles::Create(Isolate* isolate, Object object, int* index) {
1737
  DCHECK_EQ(kInvalidIndex, *index);
1738 1739
  if (object == Object()) return;
  Object the_hole = ReadOnlyRoots(isolate).the_hole_value();
1740
  DCHECK_NE(the_hole, object);
1741 1742
  int block = size_ >> kShift;
  int offset = size_ & kMask;
1743
  // Need to resize.
1744
  if (offset == 0) {
1745
    Address* next_block = new Address[kSize];
1746
    MemsetPointer(FullObjectSlot(next_block), the_hole, kSize);
1747
    blocks_.push_back(next_block);
1748
  }
1749 1750
  DCHECK_EQ(the_hole.ptr(), blocks_[block][offset]);
  blocks_[block][offset] = object.ptr();
1751
  if (ObjectInYoungGeneration(object)) {
1752
    young_node_indices_.push_back(size_);
1753
  }
1754
  *index = size_++;
1755 1756
}

1757 1758
}  // namespace internal
}  // namespace v8