[compiler] Fix bug in JSBoundFunction::Serialize

Due to the previous change to that function, we can end up with
set (non-null) fields even when the overall serialized_ field is
unset. This can cause DCHECK failures (I don't think it's otherwise
observable).

Bug: chromium:1142240,v8:7790
Change-Id: I2711fae8a73438277caf7aa539f24d628b03153c
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2497170
Commit-Queue: Georg Neis <neis@chromium.org>
Reviewed-by: Maya Lekova <mslekova@chromium.org>
Cr-Commit-Position: refs/heads/master@{#70753}

src/compiler/js-heap-broker.cc

@@ -1521,21 +1521,30 @@ bool JSBoundFunctionData::Serialize(JSHeapBroker* broker) {
   TraceScope tracer(broker, this, "JSBoundFunctionData::Serialize");
   Handle<JSBoundFunction> function = Handle<JSBoundFunction>::cast(object());
 
-  // We set {serialized_} at the end in order to correctly handle the case where
-  // a recursive call to this method reaches the stack limit.
-  bool serialized = true;
+  // We don't immediately set {serialized_} in order to correctly handle the
+  // case where a recursive call to this method reaches the stack limit.
 
   DCHECK_NULL(bound_target_function_);
   bound_target_function_ =
       broker->GetOrCreateData(function->bound_target_function());
+  bool serialized_nested = true;
   if (!bound_target_function_->should_access_heap()) {
     if (bound_target_function_->IsJSBoundFunction()) {
-      serialized =
+      serialized_nested =
           bound_target_function_->AsJSBoundFunction()->Serialize(broker);
     } else if (bound_target_function_->IsJSFunction()) {
       bound_target_function_->AsJSFunction()->Serialize(broker);
     }
   }
+  if (!serialized_nested) {
+    // We couldn't serialize all nested bound functions due to stack
+    // overflow. Give up.
+    DCHECK(!serialized_);
+    bound_target_function_ = nullptr;  // Reset to sync with serialized_.
+    return false;
+  }
+
+  serialized_ = true;
 
   DCHECK_NULL(bound_arguments_);
   bound_arguments_ = broker->GetOrCreateData(function->bound_arguments());
@@ -1546,8 +1555,7 @@ bool JSBoundFunctionData::Serialize(JSHeapBroker* broker) {
   DCHECK_NULL(bound_this_);
   bound_this_ = broker->GetOrCreateData(function->bound_this());
 
-  serialized_ = serialized;
-  return serialized;
+  return true;
 }
 
 JSObjectData::JSObjectData(JSHeapBroker* broker, ObjectData** storage,

test/mjsunit/compiler/regress-1125145.js

@@ -11,6 +11,7 @@ for (let i = 0; i < 100000; ++i) {
 
 function main() {
   foo();
+  foo();
 }
 
 %PrepareFunctionForOptimization(main);

test/mjsunit/mjsunit.status

@@ -191,7 +191,6 @@
   # Skip slow tests in debug mode.
   'array-functions-prototype-misc': [SKIP],
   'compiler/regress-808472': [SKIP],
-  'compiler/regress-1125145': [SKIP],
   'es6/promise-all-overflow-2': [SKIP],
   'generated-transition-stub': [SKIP],
   'regress/regress-524': [SKIP],