heap-inl.h 30 KB
Newer Older
1
// Copyright 2012 the V8 project authors. All rights reserved.
2 3
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
4

5 6
#ifndef V8_HEAP_HEAP_INL_H_
#define V8_HEAP_HEAP_INL_H_
7

8
#include <atomic>
9 10
#include <cmath>

11
// Clients of this interface shouldn't depend on lots of heap internals.
12
// Avoid including anything but `heap.h` from `src/heap` where possible.
13
#include "src/base/atomic-utils.h"
14
#include "src/base/atomicops.h"
15
#include "src/base/platform/mutex.h"
16
#include "src/base/platform/platform.h"
17
#include "src/base/sanitizer/msan.h"
18
#include "src/common/assert-scope.h"
19 20
#include "src/execution/isolate-data.h"
#include "src/execution/isolate.h"
21
#include "src/heap/code-object-registry.h"
22
#include "src/heap/concurrent-allocator-inl.h"
23 24
#include "src/heap/heap-write-barrier.h"
#include "src/heap/heap.h"
25
#include "src/heap/large-spaces.h"
26
#include "src/heap/memory-allocator.h"
27
#include "src/heap/memory-chunk-layout.h"
28
#include "src/heap/memory-chunk.h"
29
#include "src/heap/new-spaces-inl.h"
30
#include "src/heap/paged-spaces-inl.h"
31
#include "src/heap/read-only-heap.h"
32
#include "src/heap/read-only-spaces.h"
33
#include "src/heap/safepoint.h"
34
#include "src/heap/spaces-inl.h"
35
#include "src/heap/third-party/heap-api.h"
36
#include "src/objects/allocation-site-inl.h"
37
#include "src/objects/cell-inl.h"
38
#include "src/objects/descriptor-array.h"
39
#include "src/objects/feedback-cell-inl.h"
40
#include "src/objects/feedback-vector.h"
41
#include "src/objects/objects-inl.h"
42
#include "src/objects/oddball.h"
43
#include "src/objects/property-cell.h"
44
#include "src/objects/scope-info.h"
45
#include "src/objects/slots-inl.h"
46
#include "src/objects/struct-inl.h"
47
#include "src/objects/visitors-inl.h"
48
#include "src/profiler/heap-profiler.h"
49
#include "src/strings/string-hasher.h"
50
#include "src/utils/ostreams.h"
51
#include "src/zone/zone-list-inl.h"
52

53 54
namespace v8 {
namespace internal {
55

56 57 58 59 60 61 62 63 64 65 66 67 68
template <typename T>
T ForwardingAddress(T heap_obj) {
  MapWord map_word = heap_obj.map_word(kRelaxedLoad);

  if (map_word.IsForwardingAddress()) {
    return T::cast(map_word.ToForwardingAddress());
  } else if (Heap::InFromPage(heap_obj)) {
    return T();
  } else {
    return heap_obj;
  }
}

69 70
AllocationSpace AllocationResult::RetrySpace() {
  DCHECK(IsRetry());
jgruber's avatar
jgruber committed
71
  return static_cast<AllocationSpace>(Smi::ToInt(object_));
72 73
}

74
HeapObject AllocationResult::ToObjectChecked() {
75 76 77 78
  CHECK(!IsRetry());
  return HeapObject::cast(object_);
}

79 80 81 82 83
HeapObject AllocationResult::ToObject() {
  DCHECK(!IsRetry());
  return HeapObject::cast(object_);
}

84 85 86 87 88
Address AllocationResult::ToAddress() {
  DCHECK(!IsRetry());
  return HeapObject::cast(object_).address();
}

89
// static
90
base::EnumSet<CodeFlushMode> Heap::GetCodeFlushMode(Isolate* isolate) {
91
  if (isolate->disable_bytecode_flushing()) {
92
    return base::EnumSet<CodeFlushMode>();
93
  }
94 95 96 97 98 99 100 101 102 103

  base::EnumSet<CodeFlushMode> code_flush_mode;
  if (FLAG_flush_bytecode) {
    code_flush_mode.Add(CodeFlushMode::kFlushBytecode);
  }

  if (FLAG_flush_baseline_code) {
    code_flush_mode.Add(CodeFlushMode::kFlushBaselineCode);
  }

104
  if (FLAG_stress_flush_code) {
105 106 107 108
    // This is to check tests accidentally don't miss out on adding either flush
    // bytecode or flush code along with stress flush code. stress_flush_code
    // doesn't do anything if either one of them isn't enabled.
    DCHECK(FLAG_fuzzing || FLAG_flush_baseline_code || FLAG_flush_bytecode);
109
    code_flush_mode.Add(CodeFlushMode::kStressFlushCode);
110
  }
111 112

  return code_flush_mode;
113 114
}

115
Isolate* Heap::isolate() { return Isolate::FromHeap(this); }
116

117
int64_t Heap::external_memory() { return external_memory_.total(); }
118

119 120
int64_t Heap::update_external_memory(int64_t delta) {
  return external_memory_.Update(delta);
121 122 123 124
}

RootsTable& Heap::roots_table() { return isolate()->roots_table(); }

125 126 127
#define ROOT_ACCESSOR(Type, name, CamelName)                           \
  Type Heap::name() {                                                  \
    return Type::cast(Object(roots_table()[RootIndex::k##CamelName])); \
128
  }
129
MUTABLE_ROOT_LIST(ROOT_ACCESSOR)
130
#undef ROOT_ACCESSOR
131

132
#define ROOT_ACCESSOR(type, name, CamelName)                                   \
133
  void Heap::set_##name(type value) {                                          \
134 135 136 137 138 139
    /* The deserializer makes use of the fact that these common roots are */   \
    /* never in new space and never on a page that is being compacted.    */   \
    DCHECK_IMPLIES(deserialization_complete(),                                 \
                   !RootsTable::IsImmortalImmovable(RootIndex::k##CamelName)); \
    DCHECK_IMPLIES(RootsTable::IsImmortalImmovable(RootIndex::k##CamelName),   \
                   IsImmovable(HeapObject::cast(value)));                      \
140
    roots_table()[RootIndex::k##CamelName] = value.ptr();                      \
141 142 143 144
  }
ROOT_LIST(ROOT_ACCESSOR)
#undef ROOT_ACCESSOR

145
void Heap::SetRootMaterializedObjects(FixedArray objects) {
146
  roots_table()[RootIndex::kMaterializedObjects] = objects.ptr();
147 148
}

149
void Heap::SetRootScriptList(Object value) {
150
  roots_table()[RootIndex::kScriptList] = value.ptr();
151 152
}

153
void Heap::SetMessageListeners(TemplateList value) {
154
  roots_table()[RootIndex::kMessageListeners] = value.ptr();
155 156
}

157
void Heap::SetPendingOptimizeForTestBytecode(Object hash_table) {
158 159
  DCHECK(hash_table.IsObjectHashTable() || hash_table.IsUndefined(isolate()));
  roots_table()[RootIndex::kPendingOptimizeForTestBytecode] = hash_table.ptr();
160 161
}

mlippautz's avatar
mlippautz committed
162
PagedSpace* Heap::paged_space(int idx) {
163
  DCHECK(idx == OLD_SPACE || idx == CODE_SPACE || idx == MAP_SPACE);
164
  return static_cast<PagedSpace*>(space_[idx]);
mlippautz's avatar
mlippautz committed
165 166
}

167
Space* Heap::space(int idx) { return space_[idx]; }
mlippautz's avatar
mlippautz committed
168 169

Address* Heap::NewSpaceAllocationTopAddress() {
170
  return new_space_ ? new_space_->allocation_top_address() : nullptr;
mlippautz's avatar
mlippautz committed
171 172 173
}

Address* Heap::NewSpaceAllocationLimitAddress() {
174
  return new_space_ ? new_space_->allocation_limit_address() : nullptr;
mlippautz's avatar
mlippautz committed
175 176 177 178 179 180 181 182 183 184
}

Address* Heap::OldSpaceAllocationTopAddress() {
  return old_space_->allocation_top_address();
}

Address* Heap::OldSpaceAllocationLimitAddress() {
  return old_space_->allocation_limit_address();
}

185
inline const base::AddressRegion& Heap::code_region() {
186 187 188
#ifdef V8_ENABLE_THIRD_PARTY_HEAP
  return tp_heap_->GetCodeRange();
#else
189 190
  static constexpr base::AddressRegion kEmptyRegion;
  return code_range_ ? code_range_->reservation()->region() : kEmptyRegion;
191 192 193
#endif
}

194 195 196 197 198 199 200 201 202 203
int Heap::MaxRegularHeapObjectSize(AllocationType allocation) {
  if (!V8_ENABLE_THIRD_PARTY_HEAP_BOOL &&
      (allocation == AllocationType::kCode)) {
    DCHECK_EQ(MemoryChunkLayout::MaxRegularCodeObjectSize(),
              max_regular_code_object_size_);
    return max_regular_code_object_size_;
  }
  return kMaxRegularHeapObjectSize;
}

204
AllocationResult Heap::AllocateRaw(int size_in_bytes, AllocationType type,
205
                                   AllocationOrigin origin,
206
                                   AllocationAlignment alignment) {
207 208
  DCHECK(AllowHandleAllocation::IsAllowed());
  DCHECK(AllowHeapAllocation::IsAllowed());
209 210
  DCHECK_IMPLIES(type == AllocationType::kCode || type == AllocationType::kMap,
                 alignment == AllocationAlignment::kWordAligned);
211
  DCHECK_EQ(gc_state(), NOT_IN_GC);
212
#ifdef V8_ENABLE_ALLOCATION_TIMEOUT
213 214
  if (FLAG_random_gc_interval > 0 || FLAG_gc_interval >= 0) {
    if (!always_allocate() && Heap::allocation_timeout_-- <= 0) {
215 216
      AllocationSpace space = FLAG_single_generation ? OLD_SPACE : NEW_SPACE;
      return AllocationResult::Retry(space);
217
    }
218
  }
219 220
#endif
#ifdef DEBUG
221
  IncrementObjectCounters();
222
#endif
223

224 225 226 227
  if (CanSafepoint()) {
    main_thread_local_heap()->Safepoint();
  }

228
  size_t large_object_threshold = MaxRegularHeapObjectSize(type);
229 230
  bool large_object =
      static_cast<size_t>(size_in_bytes) > large_object_threshold;
231

232
  HeapObject object;
233
  AllocationResult allocation;
234

235
  if (FLAG_single_generation && type == AllocationType::kYoung) {
236
    type = AllocationType::kOld;
237
  }
238

239 240 241 242 243 244 245 246 247 248 249 250 251 252
  if (V8_ENABLE_THIRD_PARTY_HEAP_BOOL) {
    allocation = tp_heap_->Allocate(size_in_bytes, type, alignment);
  } else {
    if (AllocationType::kYoung == type) {
      if (large_object) {
        if (FLAG_young_generation_large_objects) {
          allocation = new_lo_space_->AllocateRaw(size_in_bytes);
        } else {
          // If young generation large objects are disalbed we have to tenure
          // the allocation and violate the given allocation type. This could be
          // dangerous. We may want to remove
          // FLAG_young_generation_large_objects and avoid patching.
          allocation = lo_space_->AllocateRaw(size_in_bytes);
        }
253
      } else {
254 255 256 257
        allocation = new_space_->AllocateRaw(size_in_bytes, alignment, origin);
      }
    } else if (AllocationType::kOld == type) {
      if (large_object) {
258
        allocation = lo_space_->AllocateRaw(size_in_bytes);
259 260
      } else {
        allocation = old_space_->AllocateRaw(size_in_bytes, alignment, origin);
261
      }
262
    } else if (AllocationType::kCode == type) {
263
      DCHECK(AllowCodeAllocation::IsAllowed());
264
      if (large_object) {
265
        allocation = code_lo_space_->AllocateRaw(size_in_bytes);
266 267
      } else {
        allocation = code_space_->AllocateRawUnaligned(size_in_bytes);
268 269 270 271 272 273 274
      }
    } else if (AllocationType::kMap == type) {
      allocation = map_space_->AllocateRawUnaligned(size_in_bytes);
    } else if (AllocationType::kReadOnly == type) {
      DCHECK(!large_object);
      DCHECK(CanAllocateInReadOnlySpace());
      DCHECK_EQ(AllocationOrigin::kRuntime, origin);
275
      allocation = read_only_space_->AllocateRaw(size_in_bytes, alignment);
276 277 278 279 280 281
    } else if (AllocationType::kSharedOld == type) {
      allocation =
          shared_old_allocator_->AllocateRaw(size_in_bytes, alignment, origin);
    } else if (AllocationType::kSharedMap == type) {
      allocation =
          shared_map_allocator_->AllocateRaw(size_in_bytes, alignment, origin);
282
    } else {
283
      UNREACHABLE();
284
    }
285
  }
286

287
  if (allocation.To(&object)) {
288
    if (AllocationType::kCode == type && !V8_ENABLE_THIRD_PARTY_HEAP_BOOL) {
289 290
      // Unprotect the memory chunk of the object if it was not unprotected
      // already.
291 292
      UnprotectAndRegisterMemoryChunk(object,
                                      UnprotectMemoryOrigin::kMainThread);
293
      ZapCodeObject(object.address(), size_in_bytes);
294
      if (!large_object) {
295 296
        MemoryChunk::FromHeapObject(object)
            ->GetCodeObjectRegistry()
297
            ->RegisterNewlyAllocatedCodeObject(object.address());
298
      }
299
    }
300 301 302 303 304 305 306 307 308

#ifdef V8_ENABLE_CONSERVATIVE_STACK_SCANNING
    if (AllocationType::kReadOnly != type) {
      DCHECK_TAG_ALIGNED(object.address());
      Page::FromHeapObject(object)->object_start_bitmap()->SetBit(
          object.address());
    }
#endif

309
    OnAllocationEvent(object, size_in_bytes);
310
  }
311

312
  return allocation;
313 314
}

315 316 317 318
template <Heap::AllocationRetryMode mode>
HeapObject Heap::AllocateRawWith(int size, AllocationType allocation,
                                 AllocationOrigin origin,
                                 AllocationAlignment alignment) {
319 320
  DCHECK(AllowHandleAllocation::IsAllowed());
  DCHECK(AllowHeapAllocation::IsAllowed());
321
  DCHECK_EQ(gc_state(), NOT_IN_GC);
322
  Heap* heap = isolate()->heap();
323
  if (allocation == AllocationType::kYoung &&
324
      alignment == AllocationAlignment::kWordAligned &&
325 326 327
      size <= MaxRegularHeapObjectSize(allocation) &&
      V8_LIKELY(!FLAG_single_generation && FLAG_inline_new &&
                FLAG_gc_interval == -1)) {
328 329
    Address* top = heap->NewSpaceAllocationTopAddress();
    Address* limit = heap->NewSpaceAllocationLimitAddress();
330
    if (*limit - *top >= static_cast<unsigned>(size)) {
331 332 333 334 335 336 337
      DCHECK(IsAligned(size, kTaggedSize));
      HeapObject obj = HeapObject::FromAddress(*top);
      *top += size;
      heap->CreateFillerObjectAt(obj.address(), size, ClearRecordedSlots::kNo);
      MSAN_ALLOCATED_UNINITIALIZED_MEMORY(obj.address(), size);
      return obj;
    }
338 339 340 341 342 343 344 345 346 347 348 349
  }
  switch (mode) {
    case kLightRetry:
      return AllocateRawWithLightRetrySlowPath(size, allocation, origin,
                                               alignment);
    case kRetryOrFail:
      return AllocateRawWithRetryOrFailSlowPath(size, allocation, origin,
                                                alignment);
  }
  UNREACHABLE();
}

350 351 352 353 354
Address Heap::AllocateRawOrFail(int size, AllocationType allocation,
                                AllocationOrigin origin,
                                AllocationAlignment alignment) {
  return AllocateRawWith<kRetryOrFail>(size, allocation, origin, alignment)
      .address();
355 356
}

357
void Heap::OnAllocationEvent(HeapObject object, int size_in_bytes) {
358
  for (auto& tracker : allocation_trackers_) {
359
    tracker->AllocationEvent(object.address(), size_in_bytes);
360 361
  }

362
  if (FLAG_verify_predictable) {
363 364 365 366
    ++allocations_count_;
    // Advance synthetic time by making a time request.
    MonotonicallyIncreasingTimeInMs();

367 368 369
    UpdateAllocationsHash(object);
    UpdateAllocationsHash(size_in_bytes);

370
    if (allocations_count_ % FLAG_dump_allocations_digest_at_alloc == 0) {
Clemens Hammacher's avatar
Clemens Hammacher committed
371
      PrintAllocationsHash();
372
    }
373 374 375 376
  } else if (FLAG_fuzzer_gc_analysis) {
    ++allocations_count_;
  } else if (FLAG_trace_allocation_stack_interval > 0) {
    ++allocations_count_;
377 378 379 380
    if (allocations_count_ % FLAG_trace_allocation_stack_interval == 0) {
      isolate()->PrintStack(stdout, Isolate::kPrintStackConcise);
    }
  }
381 382
}

383
bool Heap::CanAllocateInReadOnlySpace() {
384
  return read_only_space()->writable();
385 386
}

387
void Heap::UpdateAllocationsHash(HeapObject object) {
388
  Address object_address = object.address();
389
  MemoryChunk* memory_chunk = MemoryChunk::FromAddress(object_address);
390
  AllocationSpace allocation_space = memory_chunk->owner_identity();
391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408

  STATIC_ASSERT(kSpaceTagSize + kPageSizeBits <= 32);
  uint32_t value =
      static_cast<uint32_t>(object_address - memory_chunk->address()) |
      (static_cast<uint32_t>(allocation_space) << kPageSizeBits);

  UpdateAllocationsHash(value);
}

void Heap::UpdateAllocationsHash(uint32_t value) {
  uint16_t c1 = static_cast<uint16_t>(value);
  uint16_t c2 = static_cast<uint16_t>(value >> 16);
  raw_allocations_hash_ =
      StringHasher::AddCharacterCore(raw_allocations_hash_, c1);
  raw_allocations_hash_ =
      StringHasher::AddCharacterCore(raw_allocations_hash_, c2);
}

409
void Heap::RegisterExternalString(String string) {
410 411
  DCHECK(string.IsExternalString());
  DCHECK(!string.IsThinString());
412 413 414
  external_string_table_.AddString(string);
}

415
void Heap::FinalizeExternalString(String string) {
416
  DCHECK(string.IsExternalString());
417
  ExternalString ext_string = ExternalString::cast(string);
418

419 420 421 422 423 424
  if (!FLAG_enable_third_party_heap) {
    Page* page = Page::FromHeapObject(string);
    page->DecrementExternalBackingStoreBytes(
        ExternalBackingStoreType::kExternalString,
        ext_string.ExternalPayloadSize());
  }
425

426
  ext_string.DisposeResource(isolate());
427 428
}

429 430 431
Address Heap::NewSpaceTop() {
  return new_space_ ? new_space_->top() : kNullAddress;
}
432

433 434
bool Heap::InYoungGeneration(Object object) {
  DCHECK(!HasWeakHeapObjectTag(object));
435
  return object.IsHeapObject() && InYoungGeneration(HeapObject::cast(object));
436 437 438 439 440 441 442 443 444 445
}

// static
bool Heap::InYoungGeneration(MaybeObject object) {
  HeapObject heap_object;
  return object->GetHeapObject(&heap_object) && InYoungGeneration(heap_object);
}

// static
bool Heap::InYoungGeneration(HeapObject heap_object) {
446
  if (V8_ENABLE_THIRD_PARTY_HEAP_BOOL) return false;
447 448
  bool result =
      BasicMemoryChunk::FromHeapObject(heap_object)->InYoungGeneration();
449 450 451 452 453 454 455
#ifdef DEBUG
  // If in the young generation, then check we're either not in the middle of
  // GC or the object is in to-space.
  if (result) {
    // If the object is in the young generation, then it's not in RO_SPACE so
    // this is safe.
    Heap* heap = Heap::FromWritableHeapObject(heap_object);
456
    DCHECK_IMPLIES(heap->gc_state() == NOT_IN_GC, InToPage(heap_object));
457 458 459 460 461
  }
#endif
  return result;
}

462
// static
463
bool Heap::InFromPage(Object object) {
464
  DCHECK(!HasWeakHeapObjectTag(object));
465
  return object.IsHeapObject() && InFromPage(HeapObject::cast(object));
466 467
}

468
// static
469
bool Heap::InFromPage(MaybeObject object) {
470
  HeapObject heap_object;
471
  return object->GetHeapObject(&heap_object) && InFromPage(heap_object);
472 473
}

474
// static
475
bool Heap::InFromPage(HeapObject heap_object) {
476
  return BasicMemoryChunk::FromHeapObject(heap_object)->IsFromPage();
477 478
}

479
// static
480
bool Heap::InToPage(Object object) {
481
  DCHECK(!HasWeakHeapObjectTag(object));
482
  return object.IsHeapObject() && InToPage(HeapObject::cast(object));
483 484
}

485
// static
486
bool Heap::InToPage(MaybeObject object) {
487
  HeapObject heap_object;
488
  return object->GetHeapObject(&heap_object) && InToPage(heap_object);
489 490
}

491
// static
492
bool Heap::InToPage(HeapObject heap_object) {
493
  return BasicMemoryChunk::FromHeapObject(heap_object)->IsToPage();
494 495
}

496
bool Heap::InOldSpace(Object object) {
497
  if (V8_ENABLE_THIRD_PARTY_HEAP_BOOL) {
498 499
    return object.IsHeapObject() &&
           third_party_heap::Heap::InOldSpace(object.ptr());
500
  }
501 502
  return old_space_->Contains(object);
}
503

504
// static
505
Heap* Heap::FromWritableHeapObject(HeapObject obj) {
506 507 508
  if (V8_ENABLE_THIRD_PARTY_HEAP_BOOL) {
    return Heap::GetIsolateFromWritableObject(obj)->heap();
  }
509
  BasicMemoryChunk* chunk = BasicMemoryChunk::FromHeapObject(obj);
510 511 512
  // RO_SPACE can be shared between heaps, so we can't use RO_SPACE objects to
  // find a heap. The exception is when the ReadOnlySpace is writeable, during
  // bootstrapping, so explicitly allow this case.
513
  SLOW_DCHECK(chunk->IsWritable());
514 515 516 517 518
  Heap* heap = chunk->heap();
  SLOW_DCHECK(heap != nullptr);
  return heap;
}

519
bool Heap::ShouldBePromoted(Address old_address) {
520 521 522 523
  Page* page = Page::FromAddress(old_address);
  Address age_mark = new_space_->age_mark();
  return page->IsFlagSet(MemoryChunk::NEW_SPACE_BELOW_AGE_MARK) &&
         (!page->ContainsLimit(age_mark) || old_address < age_mark);
524 525
}

526
void Heap::CopyBlock(Address dst, Address src, int byte_size) {
527
  DCHECK(IsAligned(byte_size, kTaggedSize));
528
  CopyTagged(dst, src, static_cast<size_t>(byte_size / kTaggedSize));
529 530
}

531
template <Heap::FindMementoMode mode>
532
AllocationMemento Heap::FindAllocationMemento(Map map, HeapObject object) {
533 534
  Address object_address = object.address();
  Address memento_address = object_address + object.SizeFromMap(map);
535
  Address last_memento_word_address = memento_address + kTaggedSize;
536
  // If the memento would be on another page, bail out immediately.
537
  if (!Page::OnSamePage(object_address, last_memento_word_address)) {
538
    return AllocationMemento();
539
  }
540
  HeapObject candidate = HeapObject::FromAddress(memento_address);
541
  ObjectSlot candidate_map_slot = candidate.map_slot();
542 543 544
  // This fast check may peek at an uninitialized word. However, the slow check
  // below (memento_address == top) ensures that this is safe. Mark the word as
  // initialized to silence MemorySanitizer warnings.
545
  MSAN_MEMORY_IS_INITIALIZED(candidate_map_slot.address(), kTaggedSize);
546
  if (!candidate_map_slot.contains_map_value(
547
          ReadOnlyRoots(this).allocation_memento_map().ptr())) {
548
    return AllocationMemento();
549
  }
550 551 552 553

  // Bail out if the memento is below the age mark, which can happen when
  // mementos survived because a page got moved within new space.
  Page* object_page = Page::FromAddress(object_address);
554 555 556 557
  if (object_page->IsFlagSet(Page::NEW_SPACE_BELOW_AGE_MARK)) {
    Address age_mark =
        reinterpret_cast<SemiSpace*>(object_page->owner())->age_mark();
    if (!object_page->Contains(age_mark)) {
558
      return AllocationMemento();
559 560 561
    }
    // Do an exact check in the case where the age mark is on the same page.
    if (object_address < age_mark) {
562
      return AllocationMemento();
563
    }
564 565
  }

566
  AllocationMemento memento_candidate = AllocationMemento::cast(candidate);
567 568 569 570 571 572 573 574

  // Depending on what the memento is used for, we might need to perform
  // additional checks.
  Address top;
  switch (mode) {
    case Heap::kForGC:
      return memento_candidate;
    case Heap::kForRuntime:
575
      if (memento_candidate.is_null()) return AllocationMemento();
576 577 578 579 580 581
      // Either the object is the last object in the new space, or there is
      // another object of at least word size (the header map word) following
      // it, so suffices to compare ptr and top here.
      top = NewSpaceTop();
      DCHECK(memento_address == top ||
             memento_address + HeapObject::kHeaderSize <= top ||
582
             !Page::OnSamePage(memento_address, top - 1));
583
      if ((memento_address != top) && memento_candidate.IsValid()) {
584 585
        return memento_candidate;
      }
586
      return AllocationMemento();
587 588 589 590
    default:
      UNREACHABLE();
  }
  UNREACHABLE();
591 592
}

593
void Heap::UpdateAllocationSite(Map map, HeapObject object,
594 595
                                PretenuringFeedbackMap* pretenuring_feedback) {
  DCHECK_NE(pretenuring_feedback, &global_pretenuring_feedback_);
596
#ifdef DEBUG
597
  BasicMemoryChunk* chunk = BasicMemoryChunk::FromHeapObject(object);
598 599 600 601 602
  DCHECK_IMPLIES(chunk->IsToPage(),
                 chunk->IsFlagSet(MemoryChunk::PAGE_NEW_NEW_PROMOTION));
  DCHECK_IMPLIES(!chunk->InYoungGeneration(),
                 chunk->IsFlagSet(MemoryChunk::PAGE_NEW_OLD_PROMOTION));
#endif
603
  if (!FLAG_allocation_site_pretenuring ||
604
      !AllocationSite::CanTrack(map.instance_type())) {
605
    return;
606 607
  }
  AllocationMemento memento_candidate =
608
      FindAllocationMemento<kForGC>(map, object);
609
  if (memento_candidate.is_null()) return;
610

611 612 613
  // Entering cached feedback is used in the parallel case. We are not allowed
  // to dereference the allocation site and rather have to postpone all checks
  // till actually merging the data.
614
  Address key = memento_candidate.GetAllocationSiteUnchecked();
615
  (*pretenuring_feedback)[AllocationSite::unchecked_cast(Object(key))]++;
616
}
617

618
bool Heap::IsPendingAllocationInternal(HeapObject object) {
619
  DCHECK(deserialization_complete());
620

621 622 623 624
  if (V8_ENABLE_THIRD_PARTY_HEAP_BOOL) {
    return tp_heap_->IsPendingAllocation(object);
  }

625 626
  BasicMemoryChunk* chunk = BasicMemoryChunk::FromHeapObject(object);
  if (chunk->InReadOnlySpace()) return false;
627

628
  BaseSpace* base_space = chunk->owner();
629
  Address addr = object.address();
630

631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651
  switch (base_space->identity()) {
    case NEW_SPACE: {
      base::SharedMutexGuard<base::kShared> guard(
          new_space_->pending_allocation_mutex());
      Address top = new_space_->original_top_acquire();
      Address limit = new_space_->original_limit_relaxed();
      DCHECK_LE(top, limit);
      return top && top <= addr && addr < limit;
    }

    case OLD_SPACE:
    case CODE_SPACE:
    case MAP_SPACE: {
      PagedSpace* paged_space = static_cast<PagedSpace*>(base_space);
      base::SharedMutexGuard<base::kShared> guard(
          paged_space->pending_allocation_mutex());
      Address top = paged_space->original_top();
      Address limit = paged_space->original_limit();
      DCHECK_LE(top, limit);
      return top && top <= addr && addr < limit;
    }
652

653 654 655 656 657 658 659 660 661 662 663 664
    case LO_SPACE:
    case CODE_LO_SPACE:
    case NEW_LO_SPACE: {
      LargeObjectSpace* large_space =
          static_cast<LargeObjectSpace*>(base_space);
      base::SharedMutexGuard<base::kShared> guard(
          large_space->pending_allocation_mutex());
      return addr == large_space->pending_object();
    }

    case RO_SPACE:
      UNREACHABLE();
665
  }
666 667

  UNREACHABLE();
668 669
}

670 671 672 673 674 675 676 677 678
bool Heap::IsPendingAllocation(HeapObject object) {
  bool result = IsPendingAllocationInternal(object);
  if (FLAG_trace_pending_allocations && result) {
    StdoutStream{} << "Pending allocation: " << std::hex << "0x" << object.ptr()
                   << "\n";
  }
  return result;
}

679 680 681 682
bool Heap::IsPendingAllocation(Object object) {
  return object.IsHeapObject() && IsPendingAllocation(HeapObject::cast(object));
}

683
void Heap::ExternalStringTable::AddString(String string) {
684
  DCHECK(string.IsExternalString());
685 686
  DCHECK(!Contains(string));

687
  if (InYoungGeneration(string)) {
688
    young_strings_.push_back(string);
689
  } else {
690
    old_strings_.push_back(string);
691 692 693
  }
}

694
Oddball Heap::ToBoolean(bool condition) {
695 696
  ReadOnlyRoots roots(this);
  return condition ? roots.true_value() : roots.false_value();
697 698
}

699
int Heap::NextScriptId() {
700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720
  FullObjectSlot last_script_id_slot(&roots_table()[RootIndex::kLastScriptId]);
  Smi last_id = Smi::cast(last_script_id_slot.Relaxed_Load());
  Smi new_id, last_id_before_cas;
  do {
    if (last_id.value() == Smi::kMaxValue) {
      STATIC_ASSERT(v8::UnboundScript::kNoScriptId == 0);
      new_id = Smi::FromInt(1);
    } else {
      new_id = Smi::FromInt(last_id.value() + 1);
    }

    // CAS returns the old value on success, and the current value in the slot
    // on failure. Therefore, we want to break if the returned value matches the
    // old value (last_id), and keep looping (with the new last_id value) if it
    // doesn't.
    last_id_before_cas = last_id;
    last_id =
        Smi::cast(last_script_id_slot.Relaxed_CompareAndSwap(last_id, new_id));
  } while (last_id != last_id_before_cas);

  return new_id.value();
721 722
}

723
int Heap::NextDebuggingId() {
724
  int last_id = last_debugging_id().value();
725 726
  if (last_id == DebugInfo::DebuggingIdBits::kMax) {
    last_id = DebugInfo::kNoDebuggingId;
727 728 729 730 731 732
  }
  last_id++;
  set_last_debugging_id(Smi::FromInt(last_id));
  return last_id;
}

733
int Heap::GetNextTemplateSerialNumber() {
734 735
  int next_serial_number = next_template_serial_number().value();
  set_next_template_serial_number(Smi::FromInt(next_serial_number + 1));
736 737 738
  return next_serial_number;
}

739 740 741 742 743 744
int Heap::MaxNumberToStringCacheSize() const {
  // Compute the size of the number string cache based on the max newspace size.
  // The number string cache has a minimum size based on twice the initial cache
  // size to ensure that it is bigger after being made 'full size'.
  size_t number_string_cache_size = max_semi_space_size_ / 512;
  number_string_cache_size =
745 746
      std::max(static_cast<size_t>(kInitialNumberStringCacheSize * 2),
               std::min(static_cast<size_t>(0x4000), number_string_cache_size));
747 748 749 750
  // There is a string and a number per entry so the length is twice the number
  // of entries.
  return static_cast<int>(number_string_cache_size * 2);
}
751 752 753

void Heap::IncrementExternalBackingStoreBytes(ExternalBackingStoreType type,
                                              size_t amount) {
754 755
  base::CheckedIncrement(&backing_store_bytes_, static_cast<uint64_t>(amount),
                         std::memory_order_relaxed);
756 757 758 759 760 761
  // TODO(mlippautz): Implement interrupt for global memory allocations that can
  // trigger garbage collections.
}

void Heap::DecrementExternalBackingStoreBytes(ExternalBackingStoreType type,
                                              size_t amount) {
762 763
  base::CheckedDecrement(&backing_store_bytes_, static_cast<uint64_t>(amount),
                         std::memory_order_relaxed);
764 765
}

766 767
bool Heap::HasDirtyJSFinalizationRegistries() {
  return !dirty_js_finalization_registries_list().IsUndefined(isolate());
768 769
}

770 771 772
VerifyPointersVisitor::VerifyPointersVisitor(Heap* heap)
    : ObjectVisitorWithCageBases(heap), heap_(heap) {}

773
AlwaysAllocateScope::AlwaysAllocateScope(Heap* heap) : heap_(heap) {
774
  heap_->always_allocate_scope_count_++;
775 776
}

777
AlwaysAllocateScope::~AlwaysAllocateScope() {
778
  heap_->always_allocate_scope_count_--;
779 780 781 782 783 784 785 786 787
}

OptionalAlwaysAllocateScope::OptionalAlwaysAllocateScope(Heap* heap)
    : heap_(heap) {
  if (heap_) heap_->always_allocate_scope_count_++;
}

OptionalAlwaysAllocateScope::~OptionalAlwaysAllocateScope() {
  if (heap_) heap_->always_allocate_scope_count_--;
788 789
}

790 791 792
AlwaysAllocateScopeForTesting::AlwaysAllocateScopeForTesting(Heap* heap)
    : scope_(heap) {}

793 794
CodeSpaceMemoryModificationScope::CodeSpaceMemoryModificationScope(Heap* heap)
    : heap_(heap) {
795 796
  DCHECK_EQ(ThreadId::Current(), heap_->isolate()->thread_id());
  heap_->safepoint()->AssertActive();
797 798
  if (heap_->write_protect_code_memory()) {
    heap_->increment_code_space_memory_modification_scope_depth();
799
    heap_->code_space()->SetCodeModificationPermissions();
800
    LargePage* page = heap_->code_lo_space()->first_page();
801
    while (page != nullptr) {
802 803
      DCHECK(page->IsFlagSet(MemoryChunk::IS_EXECUTABLE));
      CHECK(heap_->memory_allocator()->IsMemoryChunkExecutable(page));
804
      page->SetCodeModificationPermissions();
805
      page = page->next_page();
806
    }
807 808 809 810
  }
}

CodeSpaceMemoryModificationScope::~CodeSpaceMemoryModificationScope() {
811 812
  if (heap_->write_protect_code_memory()) {
    heap_->decrement_code_space_memory_modification_scope_depth();
813
    heap_->code_space()->SetDefaultCodePermissions();
814
    LargePage* page = heap_->code_lo_space()->first_page();
815
    while (page != nullptr) {
816 817
      DCHECK(page->IsFlagSet(MemoryChunk::IS_EXECUTABLE));
      CHECK(heap_->memory_allocator()->IsMemoryChunkExecutable(page));
818
      page->SetDefaultCodePermissions();
819
      page = page->next_page();
820
    }
821 822 823
  }
}

824 825 826 827 828 829
CodePageCollectionMemoryModificationScope::
    CodePageCollectionMemoryModificationScope(Heap* heap)
    : heap_(heap) {
  if (heap_->write_protect_code_memory() &&
      !heap_->code_space_memory_modification_scope_depth()) {
    heap_->EnableUnprotectedMemoryChunksRegistry();
830
    heap_->IncrementCodePageCollectionMemoryModificationScopeDepth();
831 832 833 834 835 836 837
  }
}

CodePageCollectionMemoryModificationScope::
    ~CodePageCollectionMemoryModificationScope() {
  if (heap_->write_protect_code_memory() &&
      !heap_->code_space_memory_modification_scope_depth()) {
838 839 840 841 842
    heap_->DecrementCodePageCollectionMemoryModificationScopeDepth();
    if (heap_->code_page_collection_memory_modification_scope_depth() == 0) {
      heap_->ProtectUnprotectedMemoryChunks();
      heap_->DisableUnprotectedMemoryChunksRegistry();
    }
843 844 845
  }
}

846 847 848 849 850
#ifdef V8_ENABLE_THIRD_PARTY_HEAP
CodePageMemoryModificationScope::CodePageMemoryModificationScope(Code code)
    : chunk_(nullptr), scope_active_(false) {}
#else
CodePageMemoryModificationScope::CodePageMemoryModificationScope(Code code)
851
    : CodePageMemoryModificationScope(BasicMemoryChunk::FromHeapObject(code)) {}
852 853
#endif

854
CodePageMemoryModificationScope::CodePageMemoryModificationScope(
855
    BasicMemoryChunk* chunk)
856
    : chunk_(chunk),
857 858
      scope_active_(chunk_->heap()->write_protect_code_memory() &&
                    chunk_->IsFlagSet(MemoryChunk::IS_EXECUTABLE)) {
859
  if (scope_active_) {
860 861
    DCHECK(chunk_->owner()->identity() == CODE_SPACE ||
           (chunk_->owner()->identity() == CODE_LO_SPACE));
862
    MemoryChunk::cast(chunk_)->SetCodeModificationPermissions();
863 864 865 866
  }
}

CodePageMemoryModificationScope::~CodePageMemoryModificationScope() {
867
  if (scope_active_) {
868
    MemoryChunk::cast(chunk_)->SetDefaultCodePermissions();
869 870 871
  }
}

872 873
}  // namespace internal
}  // namespace v8
874

875
#endif  // V8_HEAP_HEAP_INL_H_