heap-inl.h 30 KB
Newer Older
1
// Copyright 2012 the V8 project authors. All rights reserved.
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
//       copyright notice, this list of conditions and the following
//       disclaimer in the documentation and/or other materials provided
//       with the distribution.
//     * Neither the name of Google Inc. nor the names of its
//       contributors may be used to endorse or promote products derived
//       from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#ifndef V8_HEAP_INL_H_
#define V8_HEAP_INL_H_

31
#include "heap.h"
32
#include "isolate.h"
33 34
#include "list-inl.h"
#include "objects.h"
35
#include "platform.h"
36
#include "v8-counters.h"
37 38
#include "store-buffer.h"
#include "store-buffer-inl.h"
39

40 41
namespace v8 {
namespace internal {
42

43
void PromotionQueue::insert(HeapObject* target, int size) {
44 45 46 47 48
  if (emergency_stack_ != NULL) {
    emergency_stack_->Add(Entry(target, size));
    return;
  }

49 50 51 52
  if (NewSpacePage::IsAtStart(reinterpret_cast<Address>(rear_))) {
    NewSpacePage* rear_page =
        NewSpacePage::FromAddress(reinterpret_cast<Address>(rear_));
    ASSERT(!rear_page->prev_page()->is_anchor());
53
    rear_ = reinterpret_cast<intptr_t*>(rear_page->prev_page()->area_end());
54 55 56 57 58 59 60 61 62 63 64 65
    ActivateGuardIfOnTheSamePage();
  }

  if (guard_) {
    ASSERT(GetHeadPage() ==
           Page::FromAllocationTop(reinterpret_cast<Address>(limit_)));

    if ((rear_ - 2) < limit_) {
      RelocateQueueHead();
      emergency_stack_->Add(Entry(target, size));
      return;
    }
66
  }
67

68 69 70
  *(--rear_) = reinterpret_cast<intptr_t>(target);
  *(--rear_) = size;
  // Assert no overflow into live objects.
71
#ifdef DEBUG
72
  SemiSpace::AssertValidRange(target->GetIsolate()->heap()->new_space()->top(),
73 74
                              reinterpret_cast<Address>(rear_));
#endif
75 76 77
}


78 79 80 81 82 83 84
void PromotionQueue::ActivateGuardIfOnTheSamePage() {
  guard_ = guard_ ||
      heap_->new_space()->active_space()->current_page()->address() ==
      GetHeadPage()->address();
}


85
MaybeObject* Heap::AllocateStringFromUtf8(Vector<const char> str,
86 87
                                          PretenureFlag pretenure) {
  // Check for ASCII first since this is the common case.
88 89 90 91
  const char* start = str.start();
  int length = str.length();
  int non_ascii_start = String::NonAsciiStart(start, length);
  if (non_ascii_start >= length) {
92 93
    // If the string is ASCII, we do not need to convert the characters
    // since UTF8 is backwards compatible with ASCII.
94
    return AllocateStringFromOneByte(str, pretenure);
95
  }
96
  // Non-ASCII and we need to decode.
97
  return AllocateStringFromUtf8Slow(str, non_ascii_start, pretenure);
98 99 100
}


101 102 103 104 105 106 107 108 109 110 111 112 113 114
template<>
bool inline Heap::IsOneByte(Vector<const char> str, int chars) {
  // TODO(dcarney): incorporate Latin-1 check when Latin-1 is supported?
  // ASCII only check.
  return chars == str.length();
}


template<>
bool inline Heap::IsOneByte(String* str, int chars) {
  return str->IsOneByteRepresentation();
}


115 116
MaybeObject* Heap::AllocateInternalizedStringFromUtf8(
    Vector<const char> str, int chars, uint32_t hash_field) {
117
  if (IsOneByte(str, chars)) {
118 119
    return AllocateOneByteInternalizedString(
        Vector<const uint8_t>::cast(str), hash_field);
120
  }
121
  return AllocateInternalizedStringImpl<false>(str, chars, hash_field);
122 123 124 125
}


template<typename T>
126 127
MaybeObject* Heap::AllocateInternalizedStringImpl(
    T t, int chars, uint32_t hash_field) {
128
  if (IsOneByte(t, chars)) {
129
    return AllocateInternalizedStringImpl<true>(t, chars, hash_field);
130
  }
131
  return AllocateInternalizedStringImpl<false>(t, chars, hash_field);
132 133 134
}


135 136
MaybeObject* Heap::AllocateOneByteInternalizedString(Vector<const uint8_t> str,
                                                     uint32_t hash_field) {
137
  if (str.length() > SeqOneByteString::kMaxLength) {
138
    return Failure::OutOfMemoryException(0x2);
139 140
  }
  // Compute map and object size.
141
  Map* map = ascii_internalized_string_map();
142
  int size = SeqOneByteString::SizeFor(str.length());
143
  AllocationSpace space = SelectSpace(size, OLD_DATA_SPACE, TENURED);
144 145 146

  // Allocate string.
  Object* result;
147
  { MaybeObject* maybe_result = AllocateRaw(size, space, OLD_DATA_SPACE);
148 149 150
    if (!maybe_result->ToObject(&result)) return maybe_result;
  }

151
  // String maps are all immortal immovable objects.
152
  reinterpret_cast<HeapObject*>(result)->set_map_no_write_barrier(map);
153 154 155 156 157 158 159 160
  // Set length and hash fields of the allocated string.
  String* answer = String::cast(result);
  answer->set_length(str.length());
  answer->set_hash_field(hash_field);

  ASSERT_EQ(size, answer->Size());

  // Fill in the characters.
161 162
  OS::MemCopy(answer->address() + SeqOneByteString::kHeaderSize,
              str.start(), str.length());
163 164 165 166 167

  return answer;
}


168 169
MaybeObject* Heap::AllocateTwoByteInternalizedString(Vector<const uc16> str,
                                                     uint32_t hash_field) {
170
  if (str.length() > SeqTwoByteString::kMaxLength) {
171
    return Failure::OutOfMemoryException(0x3);
172 173
  }
  // Compute map and object size.
174
  Map* map = internalized_string_map();
175
  int size = SeqTwoByteString::SizeFor(str.length());
176
  AllocationSpace space = SelectSpace(size, OLD_DATA_SPACE, TENURED);
177 178 179

  // Allocate string.
  Object* result;
180
  { MaybeObject* maybe_result = AllocateRaw(size, space, OLD_DATA_SPACE);
181 182 183 184 185 186 187 188 189 190 191 192
    if (!maybe_result->ToObject(&result)) return maybe_result;
  }

  reinterpret_cast<HeapObject*>(result)->set_map(map);
  // Set length and hash fields of the allocated string.
  String* answer = String::cast(result);
  answer->set_length(str.length());
  answer->set_hash_field(hash_field);

  ASSERT_EQ(size, answer->Size());

  // Fill in the characters.
193 194
  OS::MemCopy(answer->address() + SeqTwoByteString::kHeaderSize,
              str.start(), str.length() * kUC16Size);
195 196 197 198

  return answer;
}

199
MaybeObject* Heap::CopyFixedArray(FixedArray* src) {
200 201 202 203
  return CopyFixedArrayWithMap(src, src->map());
}


204 205 206 207 208
MaybeObject* Heap::CopyFixedDoubleArray(FixedDoubleArray* src) {
  return CopyFixedDoubleArrayWithMap(src, src->map());
}


209 210 211 212 213
MaybeObject* Heap::CopyConstantPoolArray(ConstantPoolArray* src) {
  return CopyConstantPoolArrayWithMap(src, src->map());
}


214 215 216
MaybeObject* Heap::AllocateRaw(int size_in_bytes,
                               AllocationSpace space,
                               AllocationSpace retry_space) {
217 218 219
  ASSERT(AllowHandleAllocation::IsAllowed());
  ASSERT(AllowHeapAllocation::IsAllowed());
  ASSERT(gc_state_ == NOT_IN_GC);
220 221
  ASSERT(space != NEW_SPACE ||
         retry_space == OLD_POINTER_SPACE ||
222 223
         retry_space == OLD_DATA_SPACE ||
         retry_space == LO_SPACE);
224 225 226 227
#ifdef DEBUG
  if (FLAG_gc_interval >= 0 &&
      !disallow_allocation_failure_ &&
      Heap::allocation_timeout_-- <= 0) {
228
    return Failure::RetryAfterGC(space);
229
  }
230 231
  isolate_->counters()->objs_since_last_full()->Increment();
  isolate_->counters()->objs_since_last_young()->Increment();
232
#endif
233
  MaybeObject* result;
234
  if (NEW_SPACE == space) {
235 236 237 238 239 240
    result = new_space_.AllocateRaw(size_in_bytes);
    if (always_allocate() && result->IsFailure()) {
      space = retry_space;
    } else {
      return result;
    }
241 242
  }

243 244 245 246
  if (OLD_POINTER_SPACE == space) {
    result = old_pointer_space_->AllocateRaw(size_in_bytes);
  } else if (OLD_DATA_SPACE == space) {
    result = old_data_space_->AllocateRaw(size_in_bytes);
247 248 249
  } else if (CODE_SPACE == space) {
    result = code_space_->AllocateRaw(size_in_bytes);
  } else if (LO_SPACE == space) {
250
    result = lo_space_->AllocateRaw(size_in_bytes, NOT_EXECUTABLE);
251 252
  } else if (CELL_SPACE == space) {
    result = cell_space_->AllocateRaw(size_in_bytes);
253 254
  } else if (PROPERTY_CELL_SPACE == space) {
    result = property_cell_space_->AllocateRaw(size_in_bytes);
255 256 257 258 259 260 261 262 263
  } else {
    ASSERT(MAP_SPACE == space);
    result = map_space_->AllocateRaw(size_in_bytes);
  }
  if (result->IsFailure()) old_gen_exhausted_ = true;
  return result;
}


264 265
MaybeObject* Heap::NumberFromInt32(
    int32_t value, PretenureFlag pretenure) {
266 267
  if (Smi::IsValid(value)) return Smi::FromInt(value);
  // Bypass NumberFromDouble to avoid various redundant checks.
268
  return AllocateHeapNumber(FastI2D(value), pretenure);
269 270 271
}


272 273
MaybeObject* Heap::NumberFromUint32(
    uint32_t value, PretenureFlag pretenure) {
274 275 276
  if (static_cast<int32_t>(value) >= 0 &&
      Smi::IsValid(static_cast<int32_t>(value))) {
    return Smi::FromInt(static_cast<int32_t>(value));
277 278
  }
  // Bypass NumberFromDouble to avoid various redundant checks.
279
  return AllocateHeapNumber(FastUI2D(value), pretenure);
280 281 282
}


283 284 285 286 287 288 289
void Heap::FinalizeExternalString(String* string) {
  ASSERT(string->IsExternalString());
  v8::String::ExternalStringResourceBase** resource_addr =
      reinterpret_cast<v8::String::ExternalStringResourceBase**>(
          reinterpret_cast<byte*>(string) +
          ExternalString::kResourceOffset -
          kHeapObjectTag);
290

291 292 293
  // Dispose of the C++ object if it has not already been disposed.
  if (*resource_addr != NULL) {
    (*resource_addr)->Dispose();
294
    *resource_addr = NULL;
295
  }
296 297 298
}


299
MaybeObject* Heap::AllocateRawMap() {
300
#ifdef DEBUG
301 302
  isolate_->counters()->objs_since_last_full()->Increment();
  isolate_->counters()->objs_since_last_young()->Increment();
303
#endif
304
  MaybeObject* result = map_space_->AllocateRaw(Map::kSize);
305 306 307 308 309
  if (result->IsFailure()) old_gen_exhausted_ = true;
  return result;
}


310
MaybeObject* Heap::AllocateRawCell() {
311
#ifdef DEBUG
312 313
  isolate_->counters()->objs_since_last_full()->Increment();
  isolate_->counters()->objs_since_last_young()->Increment();
314
#endif
315 316 317 318 319 320
  MaybeObject* result = cell_space_->AllocateRaw(Cell::kSize);
  if (result->IsFailure()) old_gen_exhausted_ = true;
  return result;
}


321
MaybeObject* Heap::AllocateRawPropertyCell() {
322 323 324 325 326
#ifdef DEBUG
  isolate_->counters()->objs_since_last_full()->Increment();
  isolate_->counters()->objs_since_last_young()->Increment();
#endif
  MaybeObject* result =
327
      property_cell_space_->AllocateRaw(PropertyCell::kSize);
328 329 330 331 332 333
  if (result->IsFailure()) old_gen_exhausted_ = true;
  return result;
}


bool Heap::InNewSpace(Object* object) {
334 335 336 337 338
  bool result = new_space_.Contains(object);
  ASSERT(!result ||                  // Either not in new space
         gc_state_ != NOT_IN_GC ||   // ... or in the middle of GC
         InToSpace(object));         // ... or in to-space (where we allocate).
  return result;
339 340 341
}


342 343
bool Heap::InNewSpace(Address address) {
  return new_space_.Contains(address);
344 345 346
}


347
bool Heap::InFromSpace(Object* object) {
348
  return new_space_.FromSpaceContains(object);
349 350 351 352
}


bool Heap::InToSpace(Object* object) {
353
  return new_space_.ToSpaceContains(object);
354 355 356
}


357 358 359 360 361 362 363 364 365 366
bool Heap::InOldPointerSpace(Address address) {
  return old_pointer_space_->Contains(address);
}


bool Heap::InOldPointerSpace(Object* object) {
  return InOldPointerSpace(reinterpret_cast<Address>(object));
}


367 368 369 370 371 372 373 374 375 376
bool Heap::InOldDataSpace(Address address) {
  return old_data_space_->Contains(address);
}


bool Heap::InOldDataSpace(Object* object) {
  return InOldDataSpace(reinterpret_cast<Address>(object));
}


377 378 379 380 381 382
bool Heap::OldGenerationAllocationLimitReached() {
  if (!incremental_marking()->IsStopped()) return false;
  return OldGenerationSpaceAvailable() < 0;
}


383 384 385 386
bool Heap::ShouldBePromoted(Address old_address, int object_size) {
  // An object should be promoted if:
  // - the object has survived a scavenge operation or
  // - to space is already 25% full.
387 388 389 390 391 392
  NewSpacePage* page = NewSpacePage::FromAddress(old_address);
  Address age_mark = new_space_.age_mark();
  bool below_mark = page->IsFlagSet(MemoryChunk::NEW_SPACE_BELOW_AGE_MARK) &&
      (!page->ContainsLimit(age_mark) || old_address < age_mark);
  return below_mark || (new_space_.Size() + object_size) >=
                        (new_space_.EffectiveCapacity() >> 2);
393 394 395 396
}


void Heap::RecordWrite(Address address, int offset) {
397
  if (!InNewSpace(address)) store_buffer_.Mark(address + offset);
398 399 400
}


401
void Heap::RecordWrites(Address address, int start, int len) {
402 403 404 405 406
  if (!InNewSpace(address)) {
    for (int i = 0; i < len; i++) {
      store_buffer_.Mark(address + start + i * kPointerSize);
    }
  }
407 408 409
}


410
OldSpace* Heap::TargetSpace(HeapObject* object) {
411 412 413 414 415 416 417 418 419
  InstanceType type = object->map()->instance_type();
  AllocationSpace space = TargetSpaceId(type);
  return (space == OLD_POINTER_SPACE)
      ? old_pointer_space_
      : old_data_space_;
}


AllocationSpace Heap::TargetSpaceId(InstanceType type) {
420 421
  // Heap numbers and sequential strings are promoted to old data space, all
  // other object types are promoted to old pointer space.  We do not use
422 423
  // object->IsHeapNumber() and object->IsSeqString() because we already
  // know that object has the heap object tag.
424 425 426 427 428

  // These objects are never allocated in new space.
  ASSERT(type != MAP_TYPE);
  ASSERT(type != CODE_TYPE);
  ASSERT(type != ODDBALL_TYPE);
429 430
  ASSERT(type != CELL_TYPE);
  ASSERT(type != PROPERTY_CELL_TYPE);
431

432 433 434
  if (type <= LAST_NAME_TYPE) {
    if (type == SYMBOL_TYPE) return OLD_POINTER_SPACE;
    ASSERT(type < FIRST_NONSTRING_TYPE);
435 436 437 438
    // There are four string representations: sequential strings, external
    // strings, cons strings, and sliced strings.
    // Only the latter two contain non-map-word pointers to heap objects.
    return ((type & kIsIndirectStringMask) == kIsIndirectStringTag)
439 440 441 442 443
        ? OLD_POINTER_SPACE
        : OLD_DATA_SPACE;
  } else {
    return (type <= LAST_DATA_TYPE) ? OLD_DATA_SPACE : OLD_POINTER_SPACE;
  }
444 445 446
}


447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483
bool Heap::AllowedToBeMigrated(HeapObject* object, AllocationSpace dst) {
  // Object migration is governed by the following rules:
  //
  // 1) Objects in new-space can be migrated to one of the old spaces
  //    that matches their target space or they stay in new-space.
  // 2) Objects in old-space stay in the same space when migrating.
  // 3) Fillers (two or more words) can migrate due to left-trimming of
  //    fixed arrays in new-space, old-data-space and old-pointer-space.
  // 4) Fillers (one word) can never migrate, they are skipped by
  //    incremental marking explicitly to prevent invalid pattern.
  //
  // Since this function is used for debugging only, we do not place
  // asserts here, but check everything explicitly.
  if (object->map() == one_pointer_filler_map()) return false;
  InstanceType type = object->map()->instance_type();
  MemoryChunk* chunk = MemoryChunk::FromAddress(object->address());
  AllocationSpace src = chunk->owner()->identity();
  switch (src) {
    case NEW_SPACE:
      return dst == src || dst == TargetSpaceId(type);
    case OLD_POINTER_SPACE:
      return dst == src && (dst == TargetSpaceId(type) || object->IsFiller());
    case OLD_DATA_SPACE:
      return dst == src && dst == TargetSpaceId(type);
    case CODE_SPACE:
      return dst == src && type == CODE_TYPE;
    case MAP_SPACE:
    case CELL_SPACE:
    case PROPERTY_CELL_SPACE:
    case LO_SPACE:
      return false;
  }
  UNREACHABLE();
  return false;
}


484 485 486
void Heap::CopyBlock(Address dst, Address src, int byte_size) {
  CopyWords(reinterpret_cast<Object**>(dst),
            reinterpret_cast<Object**>(src),
487
            static_cast<size_t>(byte_size / kPointerSize));
488 489 490
}


491
void Heap::MoveBlock(Address dst, Address src, int byte_size) {
492
  ASSERT(IsAligned(byte_size, kPointerSize));
493 494 495

  int size_in_words = byte_size / kPointerSize;

496
  if ((dst < src) || (dst >= (src + byte_size))) {
497 498 499
    Object** src_slot = reinterpret_cast<Object**>(src);
    Object** dst_slot = reinterpret_cast<Object**>(dst);
    Object** end_slot = src_slot + size_in_words;
500

501 502
    while (src_slot != end_slot) {
      *dst_slot++ = *src_slot++;
503 504
    }
  } else {
505
    OS::MemMove(dst, src, static_cast<size_t>(byte_size));
506 507 508 509
  }
}


510 511 512 513 514
void Heap::ScavengePointer(HeapObject** p) {
  ScavengeObject(p, *p);
}


515
void Heap::ScavengeObject(HeapObject** p, HeapObject* object) {
516
  ASSERT(object->GetIsolate()->heap()->InFromSpace(object));
517 518 519 520 521 522 523 524 525 526

  // We use the first word (where the map pointer usually is) of a heap
  // object to record the forwarding pointer.  A forwarding pointer can
  // point to an old space, the code space, or the to space of the new
  // generation.
  MapWord first_word = object->map_word();

  // If the first word is a forwarding address, the object has already been
  // copied.
  if (first_word.IsForwardingAddress()) {
527
    HeapObject* dest = first_word.ToForwardingAddress();
528
    ASSERT(object->GetIsolate()->heap()->InFromSpace(*p));
529
    *p = dest;
530 531 532
    return;
  }

533 534 535 536
  if (FLAG_trace_track_allocation_sites && object->IsJSObject()) {
    if (AllocationMemento::FindForJSObject(JSObject::cast(object), true) !=
        NULL) {
      object->GetIsolate()->heap()->allocation_mementos_found_++;
537 538 539
    }
  }

540 541
  // AllocationMementos are unrooted and shouldn't survive a scavenge
  ASSERT(object->map() != object->GetHeap()->allocation_memento_map());
542 543 544 545 546
  // Call the slow part of scavenge object.
  return ScavengeObjectSlow(p, object);
}


547 548 549 550
bool Heap::CollectGarbage(AllocationSpace space, const char* gc_reason) {
  const char* collector_reason = NULL;
  GarbageCollector collector = SelectGarbageCollector(space, &collector_reason);
  return CollectGarbage(space, collector, gc_reason, collector_reason);
551 552 553
}


554
MaybeObject* Heap::PrepareForCompare(String* str) {
555 556 557 558 559 560
  // Always flatten small strings and force flattening of long strings
  // after we have accumulated a certain amount we failed to flatten.
  static const int kMaxAlwaysFlattenLength = 32;
  static const int kFlattenLongThreshold = 16*KB;

  const int length = str->length();
561
  MaybeObject* obj = str->TryFlatten();
562
  if (length <= kMaxAlwaysFlattenLength ||
563
      unflattened_strings_length_ >= kFlattenLongThreshold) {
564 565 566
    return obj;
  }
  if (obj->IsFailure()) {
567
    unflattened_strings_length_ += length;
568 569 570 571 572
  }
  return str;
}


573 574
intptr_t Heap::AdjustAmountOfExternalAllocatedMemory(
    intptr_t change_in_bytes) {
575
  ASSERT(HasBeenSetUp());
576
  intptr_t amount = amount_of_external_allocated_memory_ + change_in_bytes;
577
  if (change_in_bytes > 0) {
578 579 580
    // Avoid overflow.
    if (amount > amount_of_external_allocated_memory_) {
      amount_of_external_allocated_memory_ = amount;
581 582 583 584
    } else {
      // Give up and reset the counters in case of an overflow.
      amount_of_external_allocated_memory_ = 0;
      amount_of_external_allocated_memory_at_last_global_gc_ = 0;
585
    }
586
    intptr_t amount_since_last_global_gc = PromotedExternalMemorySize();
587
    if (amount_since_last_global_gc > external_allocation_limit_) {
588
      CollectAllGarbage(kNoGCFlags, "external memory allocation limit reached");
589 590 591 592 593
    }
  } else {
    // Avoid underflow.
    if (amount >= 0) {
      amount_of_external_allocated_memory_ = amount;
594
    } else {
595
      // Give up and reset the counters in case of an underflow.
596 597
      amount_of_external_allocated_memory_ = 0;
      amount_of_external_allocated_memory_at_last_global_gc_ = 0;
598 599
    }
  }
600 601 602
  if (FLAG_trace_external_memory) {
    PrintPID("%8.0f ms: ", isolate()->time_millis_since_init());
    PrintF("Adjust amount of external memory: delta=%6" V8_PTR_PREFIX "d KB, "
603 604 605 606 607
           "amount=%6" V8_PTR_PREFIX "d KB, since_gc=%6" V8_PTR_PREFIX "d KB, "
           "isolate=0x%08" V8PRIxPTR ".\n",
           change_in_bytes / KB,
           amount_of_external_allocated_memory_ / KB,
           PromotedExternalMemorySize() / KB,
608 609
           reinterpret_cast<intptr_t>(isolate()));
  }
610 611 612 613 614
  ASSERT(amount_of_external_allocated_memory_ >= 0);
  return amount_of_external_allocated_memory_;
}


615 616 617 618 619
Isolate* Heap::isolate() {
  return reinterpret_cast<Isolate*>(reinterpret_cast<intptr_t>(this) -
      reinterpret_cast<size_t>(reinterpret_cast<Isolate*>(4)->heap()) + 4);
}

620

621
#ifdef DEBUG
622 623
#define GC_GREEDY_CHECK(ISOLATE) \
  if (FLAG_gc_greedy) (ISOLATE)->heap()->GarbageCollectionGreedyCheck()
624
#else
625
#define GC_GREEDY_CHECK(ISOLATE) { }
626
#endif
627

628 629 630 631
// Calls the FUNCTION_CALL function and retries it up to three times
// to guarantee that any allocations performed during the call will
// succeed if there's enough memory.

632 633
// Warning: Do not use the identifiers __object__, __maybe_object__ or
// __scope__ in a call to this macro.
634

635 636
#define CALL_AND_RETRY(ISOLATE, FUNCTION_CALL, RETURN_VALUE, RETURN_EMPTY, OOM)\
  do {                                                                         \
637
    GC_GREEDY_CHECK(ISOLATE);                                                  \
638 639 640 641 642 643 644
    MaybeObject* __maybe_object__ = FUNCTION_CALL;                             \
    Object* __object__ = NULL;                                                 \
    if (__maybe_object__->ToObject(&__object__)) RETURN_VALUE;                 \
    if (__maybe_object__->IsOutOfMemory()) {                                   \
      OOM;                                                                     \
    }                                                                          \
    if (!__maybe_object__->IsRetryAfterGC()) RETURN_EMPTY;                     \
645
    (ISOLATE)->heap()->CollectGarbage(Failure::cast(__maybe_object__)->        \
646 647 648 649 650 651 652 653
                                    allocation_space(),                        \
                                    "allocation failure");                     \
    __maybe_object__ = FUNCTION_CALL;                                          \
    if (__maybe_object__->ToObject(&__object__)) RETURN_VALUE;                 \
    if (__maybe_object__->IsOutOfMemory()) {                                   \
      OOM;                                                                     \
    }                                                                          \
    if (!__maybe_object__->IsRetryAfterGC()) RETURN_EMPTY;                     \
654 655
    (ISOLATE)->counters()->gc_last_resort_from_handles()->Increment();         \
    (ISOLATE)->heap()->CollectAllAvailableGarbage("last resort gc");           \
656 657 658 659 660 661 662 663 664 665 666 667 668
    {                                                                          \
      AlwaysAllocateScope __scope__;                                           \
      __maybe_object__ = FUNCTION_CALL;                                        \
    }                                                                          \
    if (__maybe_object__->ToObject(&__object__)) RETURN_VALUE;                 \
    if (__maybe_object__->IsOutOfMemory()) {                                   \
      OOM;                                                                     \
    }                                                                          \
    if (__maybe_object__->IsRetryAfterGC()) {                                  \
      /* TODO(1181417): Fix this. */                                           \
      v8::internal::V8::FatalProcessOutOfMemory("CALL_AND_RETRY_LAST", true);  \
    }                                                                          \
    RETURN_EMPTY;                                                              \
669 670
  } while (false)

671 672 673 674 675 676 677 678
#define CALL_AND_RETRY_OR_DIE(                                             \
     ISOLATE, FUNCTION_CALL, RETURN_VALUE, RETURN_EMPTY)                   \
  CALL_AND_RETRY(                                                          \
      ISOLATE,                                                             \
      FUNCTION_CALL,                                                       \
      RETURN_VALUE,                                                        \
      RETURN_EMPTY,                                                        \
      v8::internal::V8::FatalProcessOutOfMemory("CALL_AND_RETRY", true))
679

680 681 682 683 684
#define CALL_HEAP_FUNCTION(ISOLATE, FUNCTION_CALL, TYPE)                      \
  CALL_AND_RETRY_OR_DIE(ISOLATE,                                              \
                        FUNCTION_CALL,                                        \
                        return Handle<TYPE>(TYPE::cast(__object__), ISOLATE), \
                        return Handle<TYPE>())                                \
685 686


687 688
#define CALL_HEAP_FUNCTION_VOID(ISOLATE, FUNCTION_CALL)  \
  CALL_AND_RETRY_OR_DIE(ISOLATE, FUNCTION_CALL, return, return)
689 690


691 692 693 694
#define CALL_HEAP_FUNCTION_PASS_EXCEPTION(ISOLATE, FUNCTION_CALL) \
  CALL_AND_RETRY(ISOLATE,                                         \
                 FUNCTION_CALL,                                   \
                 return __object__,                               \
695
                 return __maybe_object__,                         \
696 697 698
                 return __maybe_object__)


699 700
void ExternalStringTable::AddString(String* string) {
  ASSERT(string->IsExternalString());
701
  if (heap_->InNewSpace(string)) {
702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725
    new_space_strings_.Add(string);
  } else {
    old_space_strings_.Add(string);
  }
}


void ExternalStringTable::Iterate(ObjectVisitor* v) {
  if (!new_space_strings_.is_empty()) {
    Object** start = &new_space_strings_[0];
    v->VisitPointers(start, start + new_space_strings_.length());
  }
  if (!old_space_strings_.is_empty()) {
    Object** start = &old_space_strings_[0];
    v->VisitPointers(start, start + old_space_strings_.length());
  }
}


// Verify() is inline to avoid ifdef-s around its calls in release
// mode.
void ExternalStringTable::Verify() {
#ifdef DEBUG
  for (int i = 0; i < new_space_strings_.length(); ++i) {
726 727
    Object* obj = Object::cast(new_space_strings_[i]);
    ASSERT(heap_->InNewSpace(obj));
728
    ASSERT(obj != heap_->the_hole_value());
729 730
  }
  for (int i = 0; i < old_space_strings_.length(); ++i) {
731 732
    Object* obj = Object::cast(old_space_strings_[i]);
    ASSERT(!heap_->InNewSpace(obj));
733
    ASSERT(obj != heap_->the_hole_value());
734 735 736 737 738 739 740
  }
#endif
}


void ExternalStringTable::AddOldString(String* string) {
  ASSERT(string->IsExternalString());
741
  ASSERT(!heap_->InNewSpace(string));
742 743 744 745 746 747
  old_space_strings_.Add(string);
}


void ExternalStringTable::ShrinkNewStrings(int position) {
  new_space_strings_.Rewind(position);
748
#ifdef VERIFY_HEAP
749 750 751
  if (FLAG_verify_heap) {
    Verify();
  }
752
#endif
753 754
}

755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774

void Heap::ClearInstanceofCache() {
  set_instanceof_cache_function(the_hole_value());
}


Object* Heap::ToBoolean(bool condition) {
  return condition ? true_value() : false_value();
}


void Heap::CompletelyClearInstanceofCache() {
  set_instanceof_cache_map(the_hole_value());
  set_instanceof_cache_function(the_hole_value());
}


MaybeObject* TranscendentalCache::Get(Type type, double input) {
  SubCache* cache = caches_[type];
  if (cache == NULL) {
775
    caches_[type] = cache = new SubCache(isolate_, type);
776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794
  }
  return cache->Get(input);
}


Address TranscendentalCache::cache_array_address() {
  return reinterpret_cast<Address>(caches_);
}


double TranscendentalCache::SubCache::Calculate(double input) {
  switch (type_) {
    case ACOS:
      return acos(input);
    case ASIN:
      return asin(input);
    case ATAN:
      return atan(input);
    case COS:
795
      return fast_cos(input);
796 797 798
    case EXP:
      return exp(input);
    case LOG:
799
      return fast_log(input);
800
    case SIN:
801
      return fast_sin(input);
802
    case TAN:
803
      return fast_tan(input);
804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834
    default:
      return 0.0;  // Never happens.
  }
}


MaybeObject* TranscendentalCache::SubCache::Get(double input) {
  Converter c;
  c.dbl = input;
  int hash = Hash(c);
  Element e = elements_[hash];
  if (e.in[0] == c.integers[0] &&
      e.in[1] == c.integers[1]) {
    ASSERT(e.output != NULL);
    isolate_->counters()->transcendental_cache_hit()->Increment();
    return e.output;
  }
  double answer = Calculate(input);
  isolate_->counters()->transcendental_cache_miss()->Increment();
  Object* heap_number;
  { MaybeObject* maybe_heap_number =
        isolate_->heap()->AllocateHeapNumber(answer);
    if (!maybe_heap_number->ToObject(&heap_number)) return maybe_heap_number;
  }
  elements_[hash].in[0] = c.integers[0];
  elements_[hash].in[1] = c.integers[1];
  elements_[hash].output = heap_number;
  return heap_number;
}


835 836 837 838 839
AlwaysAllocateScope::AlwaysAllocateScope() {
  // We shouldn't hit any nested scopes, because that requires
  // non-handle code to call handle code. The code still works but
  // performance will degrade, so we want to catch this situation
  // in debug mode.
840 841 842
  Isolate* isolate = Isolate::Current();
  ASSERT(isolate->heap()->always_allocate_scope_depth_ == 0);
  isolate->heap()->always_allocate_scope_depth_++;
843 844 845
}


846
AlwaysAllocateScope::~AlwaysAllocateScope() {
847 848 849
  Isolate* isolate = Isolate::Current();
  isolate->heap()->always_allocate_scope_depth_--;
  ASSERT(isolate->heap()->always_allocate_scope_depth_ == 0);
850 851 852
}


853
#ifdef VERIFY_HEAP
854
NoWeakObjectVerificationScope::NoWeakObjectVerificationScope() {
855
  Isolate* isolate = Isolate::Current();
856
  isolate->heap()->no_weak_object_verification_scope_depth_++;
857 858 859
}


860
NoWeakObjectVerificationScope::~NoWeakObjectVerificationScope() {
861
  Isolate* isolate = Isolate::Current();
862
  isolate->heap()->no_weak_object_verification_scope_depth_--;
863 864 865 866
}
#endif


867 868 869 870
void VerifyPointersVisitor::VisitPointers(Object** start, Object** end) {
  for (Object** current = start; current < end; current++) {
    if ((*current)->IsHeapObject()) {
      HeapObject* object = HeapObject::cast(*current);
871
      CHECK(object->GetIsolate()->heap()->Contains(object));
872
      CHECK(object->map()->IsMap());
873 874 875 876 877 878
    }
  }
}


double GCTracer::SizeOfHeapObjects() {
879
  return (static_cast<double>(heap_->SizeOfObjects())) / MB;
880 881 882 883
}


DisallowAllocationFailure::DisallowAllocationFailure() {
884
#ifdef DEBUG
885 886 887
  Isolate* isolate = Isolate::Current();
  old_state_ = isolate->heap()->disallow_allocation_failure_;
  isolate->heap()->disallow_allocation_failure_ = true;
888
#endif
889 890 891 892
}


DisallowAllocationFailure::~DisallowAllocationFailure() {
893
#ifdef DEBUG
894 895
  Isolate* isolate = Isolate::Current();
  isolate->heap()->disallow_allocation_failure_ = old_state_;
896
#endif
897
}
898 899


900 901 902
} }  // namespace v8::internal

#endif  // V8_HEAP_INL_H_