[transitions] Change TraverseTransitionTree to iterative preorder DFS

Prior to this CL, TraverseTransitionTree was a recursive post-order
visitor. This led to stack overflows for deep transition trees.

This CL changes to an iterative DFS algorithm instead. Since no user
seems to rely on the visitation order, it was changed from postorder to
preorder for ease of implementation.

Bug: chromium:1224935
Change-Id: Ibda199422fb20fb4470c5c68947e0afbd9a5e596
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3084366
Commit-Queue: Jakob Gruber <jgruber@chromium.org>
Reviewed-by: Igor Sheludko <ishell@chromium.org>
Cr-Commit-Position: refs/heads/master@{#76195}

src/objects/transitions-inl.h

@@ -215,26 +215,32 @@ void TransitionsAccessor::Reload() {
 
 int TransitionsAccessor::Capacity() { return transitions().Capacity(); }
 
-void TransitionsAccessor::Initialize() {
-  raw_transitions_ = map_.raw_transitions(isolate_, kAcquireLoad);
+// static
+TransitionsAccessor::Encoding TransitionsAccessor::GetEncoding(
+    Isolate* isolate, MaybeObject raw_transitions) {
   HeapObject heap_object;
-  if (raw_transitions_->IsSmi() || raw_transitions_->IsCleared()) {
-    encoding_ = kUninitialized;
-  } else if (raw_transitions_->IsWeak()) {
-    encoding_ = kWeakRef;
-  } else if (raw_transitions_->GetHeapObjectIfStrong(isolate_, &heap_object)) {
+  if (raw_transitions->IsSmi() || raw_transitions->IsCleared()) {
+    return kUninitialized;
+  } else if (raw_transitions->IsWeak()) {
+    return kWeakRef;
+  } else if (raw_transitions->GetHeapObjectIfStrong(isolate, &heap_object)) {
     if (heap_object.IsTransitionArray()) {
-      encoding_ = kFullTransitionArray;
+      return kFullTransitionArray;
     } else if (heap_object.IsPrototypeInfo()) {
-      encoding_ = kPrototypeInfo;
+      return kPrototypeInfo;
     } else {
-      DCHECK(map_.is_deprecated());
       DCHECK(heap_object.IsMap());
-      encoding_ = kMigrationTarget;
+      return kMigrationTarget;
     }
   } else {
     UNREACHABLE();
   }
+}
+
+void TransitionsAccessor::Initialize() {
+  raw_transitions_ = map_.raw_transitions(isolate_, kAcquireLoad);
+  encoding_ = GetEncoding(isolate_, raw_transitions_);
+  DCHECK_IMPLIES(encoding_ == kMigrationTarget, map_.is_deprecated());
 #if DEBUG
   needs_reload_ = false;
 #endif

src/objects/transitions.cc

@@ -4,6 +4,7 @@
 
 #include "src/objects/transitions.h"
 
+#include "src/base/small-vector.h"
 #include "src/objects/objects-inl.h"
 #include "src/objects/transitions-inl.h"
 #include "src/utils/utils.h"
@@ -512,45 +513,59 @@ void TransitionsAccessor::EnsureHasFullTransitionArray() {
 }
 
 void TransitionsAccessor::TraverseTransitionTreeInternal(
-    TraverseCallback callback, DisallowGarbageCollection* no_gc) {
-  switch (encoding()) {
+    const TraverseCallback& callback, DisallowGarbageCollection* no_gc) {
+  // Mostly arbitrary but more than enough to run the test suite in static
+  // memory.
+  static constexpr int kStaticStackSize = 16;
+  base::SmallVector<Map, kStaticStackSize> stack;
+  stack.emplace_back(map_);
+
+  // Pre-order iterative depth-first-search.
+  while (!stack.empty()) {
+    Map current_map = stack.back();
+    stack.pop_back();
+
+    callback(current_map);
+
+    MaybeObject raw_transitions =
+        current_map.raw_transitions(isolate_, kAcquireLoad);
+    Encoding encoding = GetEncoding(isolate_, raw_transitions);
+
+    switch (encoding) {
       case kPrototypeInfo:
       case kUninitialized:
       case kMigrationTarget:
         break;
       case kWeakRef: {
-      Map simple_target =
-          Map::cast(raw_transitions_->GetHeapObjectAssumeWeak());
-      TransitionsAccessor(isolate_, simple_target, no_gc, concurrent_access_)
-          .TraverseTransitionTreeInternal(callback, no_gc);
+        stack.emplace_back(
+            Map::cast(raw_transitions->GetHeapObjectAssumeWeak()));
         break;
       }
       case kFullTransitionArray: {
-      if (transitions().HasPrototypeTransitions()) {
-        WeakFixedArray proto_trans = transitions().GetPrototypeTransitions();
-        int length = TransitionArray::NumberOfPrototypeTransitions(proto_trans);
+        TransitionArray transitions =
+            TransitionArray::cast(raw_transitions->GetHeapObjectAssumeStrong());
+        if (transitions.HasPrototypeTransitions()) {
+          WeakFixedArray proto_trans = transitions.GetPrototypeTransitions();
+          int length =
+              TransitionArray::NumberOfPrototypeTransitions(proto_trans);
           for (int i = 0; i < length; ++i) {
             int index = TransitionArray::kProtoTransitionHeaderSize + i;
             MaybeObject target = proto_trans.Get(index);
             HeapObject heap_object;
             if (target->GetHeapObjectIfWeak(&heap_object)) {
-            TransitionsAccessor(isolate_, Map::cast(heap_object), no_gc,
-                                concurrent_access_)
-                .TraverseTransitionTreeInternal(callback, no_gc);
+              stack.emplace_back(Map::cast(heap_object));
             } else {
               DCHECK(target->IsCleared());
             }
           }
         }
-      for (int i = 0; i < transitions().number_of_transitions(); ++i) {
-        TransitionsAccessor(isolate_, transitions().GetTarget(i), no_gc,
-                            concurrent_access_)
-            .TraverseTransitionTreeInternal(callback, no_gc);
+        for (int i = 0; i < transitions.number_of_transitions(); ++i) {
+          stack.emplace_back(transitions.GetTarget(i));
         }
         break;
       }
     }
-  callback(map_);
+  }
 }
 
 #ifdef DEBUG

src/objects/transitions.h

@@ -107,8 +107,8 @@ class V8_EXPORT_PRIVATE TransitionsAccessor {
   // ===== ITERATION =====
   using TraverseCallback = std::function<void(Map)>;
 
-  // Traverse the transition tree in postorder.
-  void TraverseTransitionTree(TraverseCallback callback) {
+  // Traverse the transition tree in preorder.
+  void TraverseTransitionTree(const TraverseCallback& callback) {
     // Make sure that we do not allocate in the callback.
     DisallowGarbageCollection no_gc;
     base::SharedMutexGuardIf<base::kShared> scope(
@@ -175,6 +175,9 @@ class V8_EXPORT_PRIVATE TransitionsAccessor {
   friend class third_party_heap::Impl;
   friend class TransitionArray;
 
+  static inline Encoding GetEncoding(Isolate* isolate,
+                                     MaybeObject raw_transitions);
+
   inline PropertyDetails GetSimpleTargetDetails(Map transition);
 
   static inline Name GetSimpleTransitionKey(Map transition);
@@ -200,7 +203,7 @@ class V8_EXPORT_PRIVATE TransitionsAccessor {
   void SetPrototypeTransitions(Handle<WeakFixedArray> proto_transitions);
   WeakFixedArray GetPrototypeTransitions();
 
-  void TraverseTransitionTreeInternal(TraverseCallback callback,
+  void TraverseTransitionTreeInternal(const TraverseCallback& callback,
                                       DisallowGarbageCollection* no_gc);
 
   Isolate* isolate_;