[heap] Make CompactTransitionArray deserializer friendly

Add a pre-loop over transition arrays during compaction, that checks
whether compaction is needed at all, and whether any of the entries are
still uninitialized values as part of deserialization (and therefore no
other targets can be dead). Bails out of compaction early if this is the
case.

Bug: v8:11305
Change-Id: I27af792a8a0bd3df17892f54ac95ed15e4bdfcc0
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2622910Reviewed-by: Ulan Degenbaev <ulan@chromium.org>
Commit-Queue: Leszek Swirski <leszeks@chromium.org>
Cr-Commit-Position: refs/heads/master@{#72038}

src/heap/mark-compact.cc

@@ -2277,11 +2277,45 @@ void MarkCompactCollector::ClearFullMapTransitions() {
   }
 }
 
+// Returns false if no maps have died, or if the transition array is
+// still being deserialized.
+bool MarkCompactCollector::TransitionArrayNeedsCompaction(
+    TransitionArray transitions, int num_transitions) {
+  for (int i = 0; i < num_transitions; ++i) {
+    MaybeObject raw_target = transitions.GetRawTarget(i);
+    if (raw_target.IsSmi()) {
+      // This target is still being deserialized,
+      DCHECK(isolate()->has_active_deserializer());
+      DCHECK_EQ(raw_target.ToSmi(), Deserializer::uninitialized_field_value());
+#ifdef DEBUG
+      // Targets can only be dead iff this array is fully deserialized.
+      for (int i = 0; i < num_transitions; ++i) {
+        DCHECK(!non_atomic_marking_state()->IsWhite(transitions.GetTarget(i)));
+      }
+#endif
+      return false;
+    } else if (non_atomic_marking_state()->IsWhite(
+                   TransitionsAccessor::GetTargetFromRaw(raw_target))) {
+#ifdef DEBUG
+      // Targets can only be dead iff this array is fully deserialized.
+      for (int i = 0; i < num_transitions; ++i) {
+        DCHECK(!transitions.GetRawTarget(i).IsSmi());
+      }
+#endif
+      return true;
+    }
+  }
+  return false;
+}
+
 bool MarkCompactCollector::CompactTransitionArray(Map map,
                                                   TransitionArray transitions,
                                                   DescriptorArray descriptors) {
   DCHECK(!map.is_prototype_map());
   int num_transitions = transitions.number_of_entries();
+  if (!TransitionArrayNeedsCompaction(transitions, num_transitions)) {
+    return false;
+  }
   bool descriptors_owner_died = false;
   int transition_index = 0;
   // Compact all live transitions to the left.

src/heap/mark-compact.h

@@ -679,6 +679,8 @@ class MarkCompactCollector final : public MarkCompactCollectorBase {
   void TrimEnumCache(Map map, DescriptorArray descriptors);
   bool CompactTransitionArray(Map map, TransitionArray transitions,
                               DescriptorArray descriptors);
+  bool TransitionArrayNeedsCompaction(TransitionArray transitions,
+                                      int num_transitions);
 
   // After all reachable objects have been marked those weak map entries
   // with an unreachable key are removed from all encountered weak maps.