[compiler] Check for stack overflow in recursive ReduceJSCall

Gracefully handle hugely nested JSBoundFunctions.

Bug: chromium:1125145
Change-Id: I08f136fa9d35cf16ea8da5132d4d483a75d0ba94
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2418091
Auto-Submit: Georg Neis <neis@chromium.org>
Reviewed-by: Maya Lekova <mslekova@chromium.org>
Commit-Queue: Georg Neis <neis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#70164}

src/compiler/heap-refs.h

@@ -13,6 +13,7 @@
 #include "src/utils/boxed-float.h"
 
 namespace v8 {
+
 class CFunctionInfo;
 
 namespace internal {
@@ -32,6 +33,7 @@ class NativeContext;
 class ScriptContextTable;
 
 namespace compiler {
+
 // Whether we are loading a property or storing to a property.
 // For a store during literal creation, do not walk up the prototype chain.
 enum class AccessMode { kLoad, kStore, kStoreInLiteral, kHas };
@@ -318,7 +320,7 @@ class JSBoundFunctionRef : public JSObjectRef {
 
   Handle<JSBoundFunction> object() const;
 
-  void Serialize();
+  bool Serialize();
   bool serialized() const;
 
   // The following are available only after calling Serialize().

src/compiler/js-call-reducer.cc

@@ -3975,6 +3975,8 @@ bool JSCallReducer::IsBuiltinOrApiFunction(JSFunctionRef function) const {
 }
 
 Reduction JSCallReducer::ReduceJSCall(Node* node) {
+  if (StackLimitCheck(isolate()).HasOverflowed()) return NoChange();
+
   JSCallNode n(node);
   CallParameters const& p = n.Parameters();
   Node* target = n.target();

src/compiler/js-heap-broker.cc

@@ -614,7 +614,7 @@ class JSBoundFunctionData : public JSObjectData {
   JSBoundFunctionData(JSHeapBroker* broker, ObjectData** storage,
                       Handle<JSBoundFunction> object);
 
-  void Serialize(JSHeapBroker* broker);
+  bool Serialize(JSHeapBroker* broker);
   bool serialized() const { return serialized_; }
 
   ObjectData* bound_target_function() const { return bound_target_function_; }
@@ -1492,19 +1492,24 @@ JSBoundFunctionData::JSBoundFunctionData(JSHeapBroker* broker,
                                          Handle<JSBoundFunction> object)
     : JSObjectData(broker, storage, object) {}
 
-void JSBoundFunctionData::Serialize(JSHeapBroker* broker) {
-  if (serialized_) return;
-  serialized_ = true;
+bool JSBoundFunctionData::Serialize(JSHeapBroker* broker) {
+  if (serialized_) return true;
+  if (StackLimitCheck(broker->isolate()).HasOverflowed()) return false;
 
   TraceScope tracer(broker, this, "JSBoundFunctionData::Serialize");
   Handle<JSBoundFunction> function = Handle<JSBoundFunction>::cast(object());
 
+  // We set {serialized_} at the end in order to correctly handle the case where
+  // a recursive call to this method reaches the stack limit.
+  bool serialized = true;
+
   DCHECK_NULL(bound_target_function_);
   bound_target_function_ =
       broker->GetOrCreateData(function->bound_target_function());
   if (!bound_target_function_->should_access_heap()) {
     if (bound_target_function_->IsJSBoundFunction()) {
-      bound_target_function_->AsJSBoundFunction()->Serialize(broker);
+      serialized =
+          bound_target_function_->AsJSBoundFunction()->Serialize(broker);
     } else if (bound_target_function_->IsJSFunction()) {
       bound_target_function_->AsJSFunction()->Serialize(broker);
     }
@@ -1518,6 +1523,9 @@ void JSBoundFunctionData::Serialize(JSHeapBroker* broker) {
 
   DCHECK_NULL(bound_this_);
   bound_this_ = broker->GetOrCreateData(function->bound_this());
+
+  serialized_ = serialized;
+  return serialized;
 }
 
 JSObjectData::JSObjectData(JSHeapBroker* broker, ObjectData** storage,
@@ -4366,10 +4374,10 @@ bool JSTypedArrayRef::serialized() const {
   return data()->AsJSTypedArray()->serialized();
 }
 
-void JSBoundFunctionRef::Serialize() {
-  if (data_->should_access_heap()) return;
+bool JSBoundFunctionRef::Serialize() {
+  if (data_->should_access_heap()) return true;
   CHECK_EQ(broker()->mode(), JSHeapBroker::kSerializing);
-  data()->AsJSBoundFunction()->Serialize(broker());
+  return data()->AsJSBoundFunction()->Serialize(broker());
 }
 
 void PropertyCellRef::Serialize() {

src/compiler/serializer-for-background-compilation.cc

@@ -2083,7 +2083,7 @@ void SerializerForBackgroundCompilation::ProcessCalleeForCallOrConstruct(
   if (callee->IsJSBoundFunction()) {
     JSBoundFunctionRef bound_function(broker(),
                                       Handle<JSBoundFunction>::cast(callee));
-    bound_function.Serialize();
+    if (!bound_function.Serialize()) return;
     callee = UnrollBoundFunction(bound_function, broker(), arguments,
                                  &expanded_arguments, zone())
                  .object();

test/mjsunit/compiler/regress-1125145.js

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --allow-natives-syntax --opt
+
+function foo() {}
+for (let i = 0; i < 100000; ++i) {
+  foo = foo.bind();
+}
+
+function main() {
+  foo();
+}
+
+%PrepareFunctionForOptimization(main);
+main();
+%OptimizeFunctionOnNextCall(main);
+main();

test/mjsunit/mjsunit.status

@@ -190,6 +190,7 @@
   # Skip slow tests in debug mode.
   'array-functions-prototype-misc': [SKIP],
   'compiler/regress-808472': [SKIP],
+  'compiler/regress-1125145': [SKIP],
   'es6/promise-all-overflow-2': [SKIP],
   'generated-transition-stub': [SKIP],
   'regress/regress-524': [SKIP],