remembered-set.h 14 KB
Newer Older
1 2 3 4 5 6 7
// Copyright 2016 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef V8_REMEMBERED_SET_H
#define V8_REMEMBERED_SET_H

8
#include "src/assembler.h"
9 10 11
#include "src/heap/heap.h"
#include "src/heap/slot-set.h"
#include "src/heap/spaces.h"
12
#include "src/v8memory.h"
13 14 15 16

namespace v8 {
namespace internal {

17 18
enum RememberedSetIterationMode { SYNCHRONIZED, NON_SYNCHRONIZED };

19
// TODO(ulan): Investigate performance of de-templatizing this class.
20
template <RememberedSetType type>
21
class RememberedSet : public AllStatic {
22 23 24
 public:
  // Given a page and a slot in that page, this function adds the slot to the
  // remembered set.
25
  template <AccessMode access_mode = AccessMode::ATOMIC>
26 27
  static void Insert(MemoryChunk* chunk, Address slot_addr) {
    DCHECK(chunk->Contains(slot_addr));
28
    SlotSet* slot_set = chunk->slot_set<type, access_mode>();
29
    if (slot_set == nullptr) {
30
      slot_set = chunk->AllocateSlotSet<type>();
31
    }
32
    uintptr_t offset = slot_addr - chunk->address();
33 34
    slot_set[offset / Page::kPageSize].Insert<access_mode>(offset %
                                                           Page::kPageSize);
35 36
  }

37 38 39 40
  // Given a page and a slot in that page, this function returns true if
  // the remembered set contains the slot.
  static bool Contains(MemoryChunk* chunk, Address slot_addr) {
    DCHECK(chunk->Contains(slot_addr));
41
    SlotSet* slot_set = chunk->slot_set<type>();
42 43 44 45 46 47 48 49
    if (slot_set == nullptr) {
      return false;
    }
    uintptr_t offset = slot_addr - chunk->address();
    return slot_set[offset / Page::kPageSize].Contains(offset %
                                                       Page::kPageSize);
  }

50 51 52
  // Given a page and a slot in that page, this function removes the slot from
  // the remembered set.
  // If the slot was never added, then the function does nothing.
53 54
  static void Remove(MemoryChunk* chunk, Address slot_addr) {
    DCHECK(chunk->Contains(slot_addr));
55
    SlotSet* slot_set = chunk->slot_set<type>();
56
    if (slot_set != nullptr) {
57
      uintptr_t offset = slot_addr - chunk->address();
58 59 60 61
      slot_set[offset / Page::kPageSize].Remove(offset % Page::kPageSize);
    }
  }

62 63
  // Given a page and a range of slots in that page, this function removes the
  // slots from the remembered set.
64 65
  static void RemoveRange(MemoryChunk* chunk, Address start, Address end,
                          SlotSet::EmptyBucketMode mode) {
66
    SlotSet* slot_set = chunk->slot_set<type>();
67
    if (slot_set != nullptr) {
68 69
      uintptr_t start_offset = start - chunk->address();
      uintptr_t end_offset = end - chunk->address();
70
      DCHECK_LT(start_offset, end_offset);
71 72
      if (end_offset < static_cast<uintptr_t>(Page::kPageSize)) {
        slot_set->RemoveRange(static_cast<int>(start_offset),
73
                              static_cast<int>(end_offset), mode);
74 75 76 77 78 79 80 81 82 83 84 85 86
      } else {
        // The large page has multiple slot sets.
        // Compute slot set indicies for the range [start_offset, end_offset).
        int start_chunk = static_cast<int>(start_offset / Page::kPageSize);
        int end_chunk = static_cast<int>((end_offset - 1) / Page::kPageSize);
        int offset_in_start_chunk =
            static_cast<int>(start_offset % Page::kPageSize);
        // Note that using end_offset % Page::kPageSize would be incorrect
        // because end_offset is one beyond the last slot to clear.
        int offset_in_end_chunk = static_cast<int>(
            end_offset - static_cast<uintptr_t>(end_chunk) * Page::kPageSize);
        if (start_chunk == end_chunk) {
          slot_set[start_chunk].RemoveRange(offset_in_start_chunk,
87
                                            offset_in_end_chunk, mode);
88 89 90
        } else {
          // Clear all slots from start_offset to the end of first chunk.
          slot_set[start_chunk].RemoveRange(offset_in_start_chunk,
91
                                            Page::kPageSize, mode);
92 93
          // Clear all slots in intermediate chunks.
          for (int i = start_chunk + 1; i < end_chunk; i++) {
94
            slot_set[i].RemoveRange(0, Page::kPageSize, mode);
95 96
          }
          // Clear slots from the beginning of the last page to end_offset.
97
          slot_set[end_chunk].RemoveRange(0, offset_in_end_chunk, mode);
98 99
        }
      }
100 101 102
    }
  }

103
  // Iterates and filters the remembered set with the given callback.
104
  // The callback should take (Address slot) and return SlotCallbackResult.
105
  template <typename Callback>
106 107 108 109 110 111 112
  static void Iterate(Heap* heap, RememberedSetIterationMode mode,
                      Callback callback) {
    IterateMemoryChunks(heap, [mode, callback](MemoryChunk* chunk) {
      if (mode == SYNCHRONIZED) chunk->mutex()->Lock();
      Iterate(chunk, callback);
      if (mode == SYNCHRONIZED) chunk->mutex()->Unlock();
    });
113 114 115 116 117 118
  }

  // Iterates over all memory chunks that contains non-empty slot sets.
  // The callback should take (MemoryChunk* chunk) and return void.
  template <typename Callback>
  static void IterateMemoryChunks(Heap* heap, Callback callback) {
119
    MemoryChunkIterator it(heap);
120 121
    MemoryChunk* chunk;
    while ((chunk = it.next()) != nullptr) {
122 123
      SlotSet* slots = chunk->slot_set<type>();
      TypedSlotSet* typed_slots = chunk->typed_slot_set<type>();
124 125
      if (slots != nullptr || typed_slots != nullptr ||
          chunk->invalidated_slots() != nullptr) {
126 127 128 129 130 131 132 133
        callback(chunk);
      }
    }
  }

  // Iterates and filters the remembered set in the given memory chunk with
  // the given callback. The callback should take (Address slot) and return
  // SlotCallbackResult.
134 135 136
  //
  // Notice that |mode| can only be of FREE* or PREFREE* if there are no other
  // threads concurrently inserting slots.
137
  template <typename Callback>
138 139
  static void Iterate(MemoryChunk* chunk, Callback callback,
                      SlotSet::EmptyBucketMode mode) {
140
    SlotSet* slots = chunk->slot_set<type>();
141 142 143 144
    if (slots != nullptr) {
      size_t pages = (chunk->size() + Page::kPageSize - 1) / Page::kPageSize;
      int new_count = 0;
      for (size_t page = 0; page < pages; page++) {
145
        new_count += slots[page].Iterate(callback, mode);
146
      }
147 148
      // Only old-to-old slot sets are released eagerly. Old-new-slot sets are
      // released by the sweeper threads.
149 150
      if (type == OLD_TO_OLD && new_count == 0) {
        chunk->ReleaseSlotSet<OLD_TO_OLD>();
151 152 153 154
      }
    }
  }

155 156 157 158 159 160 161 162 163 164 165 166 167
  static int NumberOfPreFreedEmptyBuckets(MemoryChunk* chunk) {
    DCHECK(type == OLD_TO_NEW);
    int result = 0;
    SlotSet* slots = chunk->slot_set<type>();
    if (slots != nullptr) {
      size_t pages = (chunk->size() + Page::kPageSize - 1) / Page::kPageSize;
      for (size_t page = 0; page < pages; page++) {
        result += slots[page].NumberOfPreFreedEmptyBuckets();
      }
    }
    return result;
  }

168 169 170 171 172 173 174 175 176 177 178
  static void PreFreeEmptyBuckets(MemoryChunk* chunk) {
    DCHECK(type == OLD_TO_NEW);
    SlotSet* slots = chunk->slot_set<type>();
    if (slots != nullptr) {
      size_t pages = (chunk->size() + Page::kPageSize - 1) / Page::kPageSize;
      for (size_t page = 0; page < pages; page++) {
        slots[page].PreFreeEmptyBuckets();
      }
    }
  }

179 180 181 182 183 184 185 186 187 188 189 190
  static void FreeEmptyBuckets(MemoryChunk* chunk) {
    DCHECK(type == OLD_TO_NEW);
    SlotSet* slots = chunk->slot_set<type>();
    if (slots != nullptr) {
      size_t pages = (chunk->size() + Page::kPageSize - 1) / Page::kPageSize;
      for (size_t page = 0; page < pages; page++) {
        slots[page].FreeEmptyBuckets();
        slots[page].FreeToBeFreedBuckets();
      }
    }
  }

191 192
  // Given a page and a typed slot in that page, this function adds the slot
  // to the remembered set.
193 194
  static void InsertTyped(Page* page, Address host_addr, SlotType slot_type,
                          Address slot_addr) {
195
    TypedSlotSet* slot_set = page->typed_slot_set<type>();
196
    if (slot_set == nullptr) {
197
      slot_set = page->AllocateTypedSlotSet<type>();
198
    }
199 200 201
    if (host_addr == nullptr) {
      host_addr = page->address();
    }
202
    uintptr_t offset = slot_addr - page->address();
203
    uintptr_t host_offset = host_addr - page->address();
204
    DCHECK_LT(offset, static_cast<uintptr_t>(TypedSlotSet::kMaxOffset));
205 206 207
    DCHECK_LT(host_offset, static_cast<uintptr_t>(TypedSlotSet::kMaxOffset));
    slot_set->Insert(slot_type, static_cast<uint32_t>(host_offset),
                     static_cast<uint32_t>(offset));
208 209 210 211
  }

  // Given a page and a range of typed slots in that page, this function removes
  // the slots from the remembered set.
212
  static void RemoveRangeTyped(MemoryChunk* page, Address start, Address end) {
213
    TypedSlotSet* slots = page->typed_slot_set<type>();
214
    if (slots != nullptr) {
215 216 217 218 219 220 221
      slots->Iterate(
          [start, end](SlotType slot_type, Address host_addr,
                       Address slot_addr) {
            return start <= slot_addr && slot_addr < end ? REMOVE_SLOT
                                                         : KEEP_SLOT;
          },
          TypedSlotSet::PREFREE_EMPTY_CHUNKS);
222 223 224
    }
  }

225 226 227 228
  // Iterates and filters the remembered set with the given callback.
  // The callback should take (SlotType slot_type, SlotAddress slot) and return
  // SlotCallbackResult.
  template <typename Callback>
229 230 231 232
  static void IterateTyped(Heap* heap, RememberedSetIterationMode mode,
                           Callback callback) {
    IterateMemoryChunks(heap, [mode, callback](MemoryChunk* chunk) {
      if (mode == SYNCHRONIZED) chunk->mutex()->Lock();
233
      IterateTyped(chunk, callback);
234
      if (mode == SYNCHRONIZED) chunk->mutex()->Unlock();
235 236 237
    });
  }

238 239 240
  // Iterates and filters typed old to old pointers in the given memory chunk
  // with the given callback. The callback should take (SlotType slot_type,
  // Address slot_addr) and return SlotCallbackResult.
241
  template <typename Callback>
242
  static void IterateTyped(MemoryChunk* chunk, Callback callback) {
243
    TypedSlotSet* slots = chunk->typed_slot_set<type>();
244
    if (slots != nullptr) {
245
      int new_count = slots->Iterate(callback, TypedSlotSet::KEEP_EMPTY_CHUNKS);
246
      if (new_count == 0) {
247
        chunk->ReleaseTypedSlotSet<type>();
248 249 250 251 252 253
      }
    }
  }

  // Clear all old to old slots from the remembered set.
  static void ClearAll(Heap* heap) {
254
    STATIC_ASSERT(type == OLD_TO_OLD);
255
    MemoryChunkIterator it(heap);
256 257
    MemoryChunk* chunk;
    while ((chunk = it.next()) != nullptr) {
258 259
      chunk->ReleaseSlotSet<OLD_TO_OLD>();
      chunk->ReleaseTypedSlotSet<OLD_TO_OLD>();
260
      chunk->ReleaseInvalidatedSlots();
261 262 263
    }
  }

264 265 266 267
  // Eliminates all stale slots from the remembered set, i.e.
  // slots that are not part of live objects anymore. This method must be
  // called after marking, when the whole transitive closure is known and
  // must be called before sweeping when mark bits are still intact.
268 269
  static void ClearInvalidTypedSlots(Heap* heap, MemoryChunk* chunk);

270
 private:
271
  static bool IsValidSlot(Heap* heap, MemoryChunk* chunk, Object** slot);
272 273
};

274 275 276
class UpdateTypedSlotHelper {
 public:
  // Updates a code entry slot using an untyped slot callback.
Georg Neis's avatar
Georg Neis committed
277
  // The callback accepts Object** and returns SlotCallbackResult.
278 279 280 281 282 283 284 285 286 287 288 289 290 291
  template <typename Callback>
  static SlotCallbackResult UpdateCodeEntry(Address entry_address,
                                            Callback callback) {
    Object* code = Code::GetObjectFromEntryAddress(entry_address);
    Object* old_code = code;
    SlotCallbackResult result = callback(&code);
    if (code != old_code) {
      Memory::Address_at(entry_address) =
          reinterpret_cast<Code*>(code)->entry();
    }
    return result;
  }

  // Updates a code target slot using an untyped slot callback.
Georg Neis's avatar
Georg Neis committed
292
  // The callback accepts Object** and returns SlotCallbackResult.
293 294 295 296
  template <typename Callback>
  static SlotCallbackResult UpdateCodeTarget(RelocInfo* rinfo,
                                             Callback callback) {
    DCHECK(RelocInfo::IsCodeTarget(rinfo->rmode()));
297 298 299 300 301 302
    Code* old_target = Code::GetCodeFromTargetAddress(rinfo->target_address());
    Object* new_target = old_target;
    SlotCallbackResult result = callback(&new_target);
    if (new_target != old_target) {
      rinfo->set_target_address(old_target->GetIsolate(),
                                Code::cast(new_target)->instruction_start());
303 304 305 306 307
    }
    return result;
  }

  // Updates an embedded pointer slot using an untyped slot callback.
Georg Neis's avatar
Georg Neis committed
308
  // The callback accepts Object** and returns SlotCallbackResult.
309 310 311 312
  template <typename Callback>
  static SlotCallbackResult UpdateEmbeddedPointer(RelocInfo* rinfo,
                                                  Callback callback) {
    DCHECK(rinfo->rmode() == RelocInfo::EMBEDDED_OBJECT);
313 314 315 316 317
    HeapObject* old_target = rinfo->target_object();
    Object* new_target = old_target;
    SlotCallbackResult result = callback(&new_target);
    if (new_target != old_target) {
      rinfo->set_target_object(HeapObject::cast(new_target));
318 319 320 321 322
    }
    return result;
  }

  // Updates a typed slot using an untyped slot callback.
Georg Neis's avatar
Georg Neis committed
323
  // The callback accepts Object** and returns SlotCallbackResult.
324 325 326 327 328 329
  template <typename Callback>
  static SlotCallbackResult UpdateTypedSlot(Isolate* isolate,
                                            SlotType slot_type, Address addr,
                                            Callback callback) {
    switch (slot_type) {
      case CODE_TARGET_SLOT: {
330
        RelocInfo rinfo(addr, RelocInfo::CODE_TARGET, 0, NULL);
331 332 333 334 335 336
        return UpdateCodeTarget(&rinfo, callback);
      }
      case CODE_ENTRY_SLOT: {
        return UpdateCodeEntry(addr, callback);
      }
      case EMBEDDED_OBJECT_SLOT: {
337
        RelocInfo rinfo(addr, RelocInfo::EMBEDDED_OBJECT, 0, NULL);
338 339 340 341 342
        return UpdateEmbeddedPointer(&rinfo, callback);
      }
      case OBJECT_SLOT: {
        return callback(reinterpret_cast<Object**>(addr));
      }
343
      case CLEARED_SLOT:
344 345 346 347 348 349
        break;
    }
    UNREACHABLE();
  }
};

350 351 352 353 354 355 356 357 358
inline SlotType SlotTypeForRelocInfoMode(RelocInfo::Mode rmode) {
  if (RelocInfo::IsCodeTarget(rmode)) {
    return CODE_TARGET_SLOT;
  } else if (RelocInfo::IsEmbeddedObject(rmode)) {
    return EMBEDDED_OBJECT_SLOT;
  }
  UNREACHABLE();
}

359 360 361 362
}  // namespace internal
}  // namespace v8

#endif  // V8_REMEMBERED_SET_H