Reland "[x64][ia32] Add stack overflow check in InvokePrologue"

This is a reland of adceb459
Redesign test to not be OS dependent.

Original change's description:
> [x64][ia32] Add stack overflow check in InvokePrologue
>
> In case of no arguments adaptor frame, we massage the arguments in InvokePrologue pushing undefined objects if the actual argument count is below the parameter count. This CL adds a stack overflow check before pushing these undefined objects to the stack.
>
> Change-Id: I2a88bf6fdfd17958f6f6884143a67d50ea842fd2
> Bug: v8:10201
> Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2491039
> Reviewed-by: Igor Sheludko <ishell@chromium.org>
> Reviewed-by: Georg Neis <neis@chromium.org>
> Commit-Queue: Victor Gomes <victorgomes@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#70927}

Bug: v8:10201
Change-Id: Ifab3413b748cdf3bb998a5080cd1fcb3b67a737b
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2517921Reviewed-by: Igor Sheludko <ishell@chromium.org>
Reviewed-by: Georg Neis <neis@chromium.org>
Commit-Queue: Victor Gomes <victorgomes@chromium.org>
Cr-Commit-Position: refs/heads/master@{#70973}

src/builtins/ia32/builtins-ia32.cc

@@ -72,48 +72,6 @@ static void GenerateTailCallToReturnedCode(MacroAssembler* masm,
 
 namespace {
 
-enum StackLimitKind { kInterruptStackLimit, kRealStackLimit };
-
-void CompareStackLimit(MacroAssembler* masm, Register with,
-                       StackLimitKind kind) {
-  DCHECK(masm->root_array_available());
-  Isolate* isolate = masm->isolate();
-  // Address through the root register. No load is needed.
-  ExternalReference limit =
-      kind == StackLimitKind::kRealStackLimit
-          ? ExternalReference::address_of_real_jslimit(isolate)
-          : ExternalReference::address_of_jslimit(isolate);
-  DCHECK(TurboAssembler::IsAddressableThroughRootRegister(isolate, limit));
-
-  intptr_t offset =
-      TurboAssembler::RootRegisterOffsetForExternalReference(isolate, limit);
-  __ cmp(with, Operand(kRootRegister, offset));
-}
-
-void Generate_StackOverflowCheck(MacroAssembler* masm, Register num_args,
-                                 Register scratch, Label* stack_overflow,
-                                 bool include_receiver = false) {
-  // Check the stack for overflow. We are not trying to catch
-  // interruptions (e.g. debug break and preemption) here, so the "real stack
-  // limit" is checked.
-  ExternalReference real_stack_limit =
-      ExternalReference::address_of_real_jslimit(masm->isolate());
-  // Compute the space that is left as a negative number in scratch. If
-  // we already overflowed, this will be a positive number.
-  __ mov(scratch, __ ExternalReferenceAsOperand(real_stack_limit, scratch));
-  __ sub(scratch, esp);
-  // Add the size of the arguments.
-  static_assert(kSystemPointerSize == 4,
-                "The next instruction assumes kSystemPointerSize == 4");
-  __ lea(scratch, Operand(scratch, num_args, times_system_pointer_size, 0));
-  if (include_receiver) {
-    __ add(scratch, Immediate(kSystemPointerSize));
-  }
-  // See if we overflowed, i.e. scratch is positive.
-  __ cmp(scratch, Immediate(0));
-  __ j(greater, stack_overflow);  // Signed comparison.
-}
-
 void Generate_JSBuiltinsConstructStubHelper(MacroAssembler* masm) {
   // ----------- S t a t e -------------
   //  -- eax: number of arguments
@@ -124,7 +82,7 @@ void Generate_JSBuiltinsConstructStubHelper(MacroAssembler* masm) {
 
   Label stack_overflow;
 
-  Generate_StackOverflowCheck(masm, eax, ecx, &stack_overflow);
+  __ StackOverflowCheck(eax, ecx, &stack_overflow);
 
   // Enter a construct frame.
   {
@@ -268,7 +226,7 @@ void Builtins::Generate_JSConstructStubGeneric(MacroAssembler* masm) {
     // Check if we have enough stack space to push all arguments.
     // Argument count in eax. Clobbers ecx.
     Label enough_stack_space, stack_overflow;
-    Generate_StackOverflowCheck(masm, eax, ecx, &stack_overflow);
+    __ StackOverflowCheck(eax, ecx, &stack_overflow);
     __ jmp(&enough_stack_space);
 
     __ bind(&stack_overflow);
@@ -524,7 +482,7 @@ static void Generate_JSEntryTrampolineHelper(MacroAssembler* masm,
     // Check if we have enough stack space to push all arguments.
     // Argument count in eax. Clobbers ecx.
     Label enough_stack_space, stack_overflow;
-    Generate_StackOverflowCheck(masm, eax, ecx, &stack_overflow);
+    __ StackOverflowCheck(eax, ecx, &stack_overflow);
     __ jmp(&enough_stack_space);
 
     __ bind(&stack_overflow);
@@ -635,7 +593,7 @@ void Builtins::Generate_ResumeGeneratorTrampoline(MacroAssembler* masm) {
   // Check the stack for overflow. We are not trying to catch interruptions
   // (i.e. debug break and preemption) here, so check the "real stack limit".
   Label stack_overflow;
-  CompareStackLimit(masm, esp, StackLimitKind::kRealStackLimit);
+  __ CompareStackLimit(esp, StackLimitKind::kRealStackLimit);
   __ j(below, &stack_overflow);
 
   // Pop return address.
@@ -1077,7 +1035,7 @@ void Builtins::Generate_InterpreterEntryTrampoline(MacroAssembler* masm) {
     // Do a stack check to ensure we don't go over the limit.
     __ mov(eax, esp);
     __ sub(eax, frame_size);
-    CompareStackLimit(masm, eax, StackLimitKind::kRealStackLimit);
+    __ CompareStackLimit(eax, StackLimitKind::kRealStackLimit);
     __ j(below, &stack_overflow);
 
     // If ok, push undefined as the initial value for all register file entries.
@@ -1108,7 +1066,7 @@ void Builtins::Generate_InterpreterEntryTrampoline(MacroAssembler* masm) {
   // Perform interrupt stack check.
   // TODO(solanes): Merge with the real stack limit check above.
   Label stack_check_interrupt, after_stack_check_interrupt;
-  CompareStackLimit(masm, esp, StackLimitKind::kInterruptStackLimit);
+  __ CompareStackLimit(esp, StackLimitKind::kInterruptStackLimit);
   __ j(below, &stack_check_interrupt);
   __ bind(&after_stack_check_interrupt);
 
@@ -1260,7 +1218,7 @@ void Builtins::Generate_InterpreterPushArgsThenCallImpl(
   }
 
   // Add a stack check before pushing the arguments.
-  Generate_StackOverflowCheck(masm, eax, scratch, &stack_overflow, true);
+  __ StackOverflowCheck(eax, scratch, &stack_overflow, true);
 
   __ movd(xmm0, eax);  // Spill number of arguments.
 
@@ -1338,7 +1296,7 @@ void Generate_InterpreterPushZeroAndArgsAndReturnAddress(
   // | addtl. slot |            | receiver slot |
 
   // Check for stack overflow before we increment the stack pointer.
-  Generate_StackOverflowCheck(masm, num_args, scratch1, stack_overflow, true);
+  __ StackOverflowCheck(num_args, scratch1, stack_overflow, true);
 
   // Step 1 - Update the stack pointer.
 
@@ -1988,7 +1946,7 @@ void Builtins::Generate_CallOrConstructVarargs(MacroAssembler* masm,
   // Check the stack for overflow. We are not trying to catch interruptions
   // (i.e. debug break and preemption) here, so check the "real stack limit".
   Label stack_overflow;
-  Generate_StackOverflowCheck(masm, kArgumentsLength, edx, &stack_overflow);
+  __ StackOverflowCheck(kArgumentsLength, edx, &stack_overflow);
 
   __ movd(xmm4, kArgumentsList);  // Spill the arguments list.
 
@@ -2140,7 +2098,7 @@ void Builtins::Generate_CallOrConstructForwardVarargs(MacroAssembler* masm,
 
     // Forward the arguments from the caller frame.
     __ movd(xmm2, edi);  // Preserve the target to call.
-    Generate_StackOverflowCheck(masm, edx, edi, &stack_overflow);
+    __ StackOverflowCheck(edx, edi, &stack_overflow);
     __ movd(xmm3, ebx);  // Preserve root register.
 
     Register scratch = ebx;
@@ -2344,7 +2302,7 @@ void Generate_PushBoundArguments(MacroAssembler* masm) {
     // Check the stack for overflow.
     {
       Label done, stack_overflow;
-      Generate_StackOverflowCheck(masm, edx, ecx, &stack_overflow);
+      __ StackOverflowCheck(edx, ecx, &stack_overflow);
       __ jmp(&done);
       __ bind(&stack_overflow);
       {
@@ -2613,8 +2571,8 @@ void Builtins::Generate_ArgumentsAdaptorTrampoline(MacroAssembler* masm) {
     EnterArgumentsAdaptorFrame(masm);
     // edi is used as a scratch register. It should be restored from the frame
     // when needed.
-    Generate_StackOverflowCheck(masm, kExpectedNumberOfArgumentsRegister, edi,
-                                &stack_overflow);
+    __ StackOverflowCheck(kExpectedNumberOfArgumentsRegister, edi,
+                          &stack_overflow);
 
     // Copy receiver and all expected arguments.
     const int offset = StandardFrameConstants::kCallerSPOffset;
@@ -2637,8 +2595,8 @@ void Builtins::Generate_ArgumentsAdaptorTrampoline(MacroAssembler* masm) {
     EnterArgumentsAdaptorFrame(masm);
     // edi is used as a scratch register. It should be restored from the frame
     // when needed.
-    Generate_StackOverflowCheck(masm, kExpectedNumberOfArgumentsRegister, edi,
-                                &stack_overflow);
+    __ StackOverflowCheck(kExpectedNumberOfArgumentsRegister, edi,
+                          &stack_overflow);
 
     // Remember expected arguments in xmm0.
     __ movd(xmm0, kExpectedNumberOfArgumentsRegister);

src/builtins/x64/builtins-x64.cc

@@ -75,43 +75,6 @@ static void GenerateTailCallToReturnedCode(MacroAssembler* masm,
 
 namespace {
 
-enum StackLimitKind { kInterruptStackLimit, kRealStackLimit };
-
-Operand StackLimitAsOperand(MacroAssembler* masm, StackLimitKind kind) {
-  DCHECK(masm->root_array_available());
-  Isolate* isolate = masm->isolate();
-  ExternalReference limit =
-      kind == StackLimitKind::kRealStackLimit
-          ? ExternalReference::address_of_real_jslimit(isolate)
-          : ExternalReference::address_of_jslimit(isolate);
-  DCHECK(TurboAssembler::IsAddressableThroughRootRegister(isolate, limit));
-
-  intptr_t offset =
-      TurboAssembler::RootRegisterOffsetForExternalReference(isolate, limit);
-  CHECK(is_int32(offset));
-  return Operand(kRootRegister, static_cast<int32_t>(offset));
-}
-
-void Generate_StackOverflowCheck(
-    MacroAssembler* masm, Register num_args, Register scratch,
-    Label* stack_overflow,
-    Label::Distance stack_overflow_distance = Label::kFar) {
-  // Check the stack for overflow. We are not trying to catch
-  // interruptions (e.g. debug break and preemption) here, so the "real stack
-  // limit" is checked.
-  __ movq(kScratchRegister,
-          StackLimitAsOperand(masm, StackLimitKind::kRealStackLimit));
-  __ movq(scratch, rsp);
-  // Make scratch the space we have left. The stack might already be overflowed
-  // here which will cause scratch to become negative.
-  __ subq(scratch, kScratchRegister);
-  __ sarq(scratch, Immediate(kSystemPointerSizeLog2));
-  // Check if the arguments will overflow the stack.
-  __ cmpq(scratch, num_args);
-  // Signed comparison.
-  __ j(less_equal, stack_overflow, stack_overflow_distance);
-}
-
 void Generate_JSBuiltinsConstructStubHelper(MacroAssembler* masm) {
   // ----------- S t a t e -------------
   //  -- rax: number of arguments
@@ -121,7 +84,7 @@ void Generate_JSBuiltinsConstructStubHelper(MacroAssembler* masm) {
   // -----------------------------------
 
   Label stack_overflow;
-  Generate_StackOverflowCheck(masm, rax, rcx, &stack_overflow, Label::kFar);
+  __ StackOverflowCheck(rax, rcx, &stack_overflow, Label::kFar);
 
   // Enter a construct frame.
   {
@@ -262,7 +225,7 @@ void Builtins::Generate_JSConstructStubGeneric(MacroAssembler* masm) {
     // Check if we have enough stack space to push all arguments.
     // Argument count in rax. Clobbers rcx.
     Label enough_stack_space, stack_overflow;
-    Generate_StackOverflowCheck(masm, rax, rcx, &stack_overflow, Label::kNear);
+    __ StackOverflowCheck(rax, rcx, &stack_overflow, Label::kNear);
     __ jmp(&enough_stack_space, Label::kNear);
 
     __ bind(&stack_overflow);
@@ -624,7 +587,7 @@ static void Generate_JSEntryTrampolineHelper(MacroAssembler* masm,
     // Check if we have enough stack space to push all arguments.
     // Argument count in rax. Clobbers rcx.
     Label enough_stack_space, stack_overflow;
-    Generate_StackOverflowCheck(masm, rax, rcx, &stack_overflow, Label::kNear);
+    __ StackOverflowCheck(rax, rcx, &stack_overflow, Label::kNear);
     __ jmp(&enough_stack_space, Label::kNear);
 
     __ bind(&stack_overflow);
@@ -735,7 +698,7 @@ void Builtins::Generate_ResumeGeneratorTrampoline(MacroAssembler* masm) {
   // Check the stack for overflow. We are not trying to catch interruptions
   // (i.e. debug break and preemption) here, so check the "real stack limit".
   Label stack_overflow;
-  __ cmpq(rsp, StackLimitAsOperand(masm, StackLimitKind::kRealStackLimit));
+  __ cmpq(rsp, __ StackLimitAsOperand(StackLimitKind::kRealStackLimit));
   __ j(below, &stack_overflow);
 
   // Pop return address.
@@ -1167,7 +1130,7 @@ void Builtins::Generate_InterpreterEntryTrampoline(MacroAssembler* masm) {
     // Do a stack check to ensure we don't go over the limit.
     __ movq(rax, rsp);
     __ subq(rax, rcx);
-    __ cmpq(rax, StackLimitAsOperand(masm, StackLimitKind::kRealStackLimit));
+    __ cmpq(rax, __ StackLimitAsOperand(StackLimitKind::kRealStackLimit));
     __ j(below, &stack_overflow);
 
     // If ok, push undefined as the initial value for all register file entries.
@@ -1199,7 +1162,7 @@ void Builtins::Generate_InterpreterEntryTrampoline(MacroAssembler* masm) {
   // Perform interrupt stack check.
   // TODO(solanes): Merge with the real stack limit check above.
   Label stack_check_interrupt, after_stack_check_interrupt;
-  __ cmpq(rsp, StackLimitAsOperand(masm, StackLimitKind::kInterruptStackLimit));
+  __ cmpq(rsp, __ StackLimitAsOperand(StackLimitKind::kInterruptStackLimit));
   __ j(below, &stack_check_interrupt);
   __ bind(&after_stack_check_interrupt);
 
@@ -1332,7 +1295,7 @@ void Builtins::Generate_InterpreterPushArgsThenCallImpl(
   __ leal(rcx, Operand(rax, 1));  // Add one for receiver.
 
   // Add a stack check before pushing arguments.
-  Generate_StackOverflowCheck(masm, rcx, rdx, &stack_overflow);
+  __ StackOverflowCheck(rcx, rdx, &stack_overflow);
 
   // Pop return address to allow tail-call after pushing arguments.
   __ PopReturnAddressTo(kScratchRegister);
@@ -1393,7 +1356,7 @@ void Builtins::Generate_InterpreterPushArgsThenConstructImpl(
   Label stack_overflow;
 
   // Add a stack check before pushing arguments.
-  Generate_StackOverflowCheck(masm, rax, r8, &stack_overflow);
+  __ StackOverflowCheck(rax, r8, &stack_overflow);
 
   // Pop return address to allow tail-call after pushing arguments.
   __ PopReturnAddressTo(kScratchRegister);
@@ -1930,7 +1893,7 @@ void Builtins::Generate_ArgumentsAdaptorTrampoline(MacroAssembler* masm) {
   // -------------------------------------------
   {
     EnterArgumentsAdaptorFrame(masm);
-    Generate_StackOverflowCheck(masm, rbx, rcx, &stack_overflow);
+    __ StackOverflowCheck(rbx, rcx, &stack_overflow);
 
     Label under_application, over_application, invoke;
     __ cmpq(rax, rbx);
@@ -2051,7 +2014,7 @@ void Builtins::Generate_CallOrConstructVarargs(MacroAssembler* masm,
   }
 
   Label stack_overflow;
-  Generate_StackOverflowCheck(masm, rcx, r8, &stack_overflow, Label::kNear);
+  __ StackOverflowCheck(rcx, r8, &stack_overflow, Label::kNear);
 
   // Push additional arguments onto the stack.
   // Move the arguments already in the stack,
@@ -2181,7 +2144,7 @@ void Builtins::Generate_CallOrConstructForwardVarargs(MacroAssembler* masm,
     // -----------------------------------
 
     // Check for stack overflow.
-    Generate_StackOverflowCheck(masm, r8, r12, &stack_overflow, Label::kNear);
+    __ StackOverflowCheck(r8, r12, &stack_overflow, Label::kNear);
 
     // Forward the arguments from the caller frame.
     // Move the arguments already in the stack,
@@ -2391,7 +2354,7 @@ void Generate_PushBoundArguments(MacroAssembler* masm) {
       // We are not trying to catch interruptions (i.e. debug break and
       // preemption) here, so check the "real stack limit".
       __ cmpq(kScratchRegister,
-              StackLimitAsOperand(masm, StackLimitKind::kRealStackLimit));
+              __ StackLimitAsOperand(StackLimitKind::kRealStackLimit));
       __ j(above_equal, &done, Label::kNear);
       {
         FrameScope scope(masm, StackFrame::MANUAL);

src/codegen/ia32/macro-assembler-ia32.cc

@@ -1112,6 +1112,47 @@ void TurboAssembler::PrepareForTailCall(
   mov(esp, new_sp_reg);
 }
 
+void MacroAssembler::CompareStackLimit(Register with, StackLimitKind kind) {
+  DCHECK(root_array_available());
+  Isolate* isolate = this->isolate();
+  // Address through the root register. No load is needed.
+  ExternalReference limit =
+      kind == StackLimitKind::kRealStackLimit
+          ? ExternalReference::address_of_real_jslimit(isolate)
+          : ExternalReference::address_of_jslimit(isolate);
+  DCHECK(TurboAssembler::IsAddressableThroughRootRegister(isolate, limit));
+
+  intptr_t offset =
+      TurboAssembler::RootRegisterOffsetForExternalReference(isolate, limit);
+  cmp(with, Operand(kRootRegister, offset));
+}
+
+void MacroAssembler::StackOverflowCheck(Register num_args, Register scratch,
+                                        Label* stack_overflow,
+                                        bool include_receiver) {
+  DCHECK_NE(num_args, scratch);
+  // Check the stack for overflow. We are not trying to catch
+  // interruptions (e.g. debug break and preemption) here, so the "real stack
+  // limit" is checked.
+  ExternalReference real_stack_limit =
+      ExternalReference::address_of_real_jslimit(isolate());
+  // Compute the space that is left as a negative number in scratch. If
+  // we already overflowed, this will be a positive number.
+  mov(scratch, ExternalReferenceAsOperand(real_stack_limit, scratch));
+  sub(scratch, esp);
+  // TODO(victorgomes): Remove {include_receiver} and always require one extra
+  // word of the stack space.
+  lea(scratch, Operand(scratch, num_args, times_system_pointer_size, 0));
+  if (include_receiver) {
+    add(scratch, Immediate(kSystemPointerSize));
+  }
+  // See if we overflowed, i.e. scratch is positive.
+  cmp(scratch, Immediate(0));
+  // TODO(victorgomes):  Save some bytes in the builtins that use stack checks
+  // by jumping to a builtin that throws the exception.
+  j(greater, stack_overflow);  // Signed comparison.
+}
+
 void MacroAssembler::InvokePrologue(Register expected_parameter_count,
                                     Register actual_parameter_count,
                                     Label* done, InvokeFlag flag) {
@@ -1120,13 +1161,15 @@ void MacroAssembler::InvokePrologue(Register expected_parameter_count,
     DCHECK_EQ(expected_parameter_count, ecx);
     Label regular_invoke;
 #ifdef V8_NO_ARGUMENTS_ADAPTOR
-    // Skip if adaptor sentinel.
+    // If the expected parameter count is equal to the adaptor sentinel, no need
+    // to push undefined value as arguments.
     cmp(expected_parameter_count, Immediate(kDontAdaptArgumentsSentinel));
-    j(equal, &regular_invoke, Label::kNear);
+    j(equal, &regular_invoke, Label::kFar);
 
-    // Skip if overapplication or if expected number of arguments.
+    // If overapplication or if the actual argument count is equal to the
+    // formal parameter count, no need to push extra undefined values.
     sub(expected_parameter_count, actual_parameter_count);
-    j(less_equal, &regular_invoke, Label::kNear);
+    j(less_equal, &regular_invoke, Label::kFar);
 
     // We need to preserve edx, edi, esi and ebx.
     movd(xmm0, edx);
@@ -1134,6 +1177,9 @@ void MacroAssembler::InvokePrologue(Register expected_parameter_count,
     movd(xmm2, esi);
     movd(xmm3, ebx);
 
+    Label stack_overflow;
+    StackOverflowCheck(expected_parameter_count, edx, &stack_overflow);
+
     Register scratch = esi;
 
     // Underapplication. Move the arguments already in the stack, including the
@@ -1176,6 +1222,16 @@ void MacroAssembler::InvokePrologue(Register expected_parameter_count,
     movd(esi, xmm2);
     movd(edi, xmm1);
     movd(edx, xmm0);
+
+    jmp(&regular_invoke);
+
+    bind(&stack_overflow);
+    {
+      FrameScope frame(this,
+                       has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
+      CallRuntime(Runtime::kThrowStackOverflow);
+      int3();  // This should be unreachable.
+    }
 #else
     cmp(expected_parameter_count, actual_parameter_count);
     j(equal, &regular_invoke);

src/codegen/ia32/macro-assembler-ia32.h

@@ -24,6 +24,10 @@ using MemOperand = Operand;
 enum RememberedSetAction { EMIT_REMEMBERED_SET, OMIT_REMEMBERED_SET };
 enum SmiCheck { INLINE_SMI_CHECK, OMIT_SMI_CHECK };
 
+// TODO(victorgomes): Move definition to macro-assembler.h, once all other
+// platforms are updated.
+enum class StackLimitKind { kInterruptStackLimit, kRealStackLimit };
+
 // Convenient class to access arguments below the stack pointer.
 class StackArgumentsAccessor {
  public:
@@ -840,6 +844,12 @@ class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler {
   void IncrementCounter(StatsCounter* counter, int value, Register scratch);
   void DecrementCounter(StatsCounter* counter, int value, Register scratch);
 
+  // ---------------------------------------------------------------------------
+  // Stack limit utilities
+  void CompareStackLimit(Register with, StackLimitKind kind);
+  void StackOverflowCheck(Register num_args, Register scratch,
+                          Label* stack_overflow, bool include_receiver = false);
+
   static int SafepointRegisterStackIndex(Register reg) {
     return SafepointRegisterStackIndex(reg.code());
   }

src/codegen/x64/macro-assembler-x64.cc

@@ -2439,19 +2439,62 @@ void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
   bind(&done);
 }
 
+Operand MacroAssembler::StackLimitAsOperand(StackLimitKind kind) {
+  DCHECK(root_array_available());
+  Isolate* isolate = this->isolate();
+  ExternalReference limit =
+      kind == StackLimitKind::kRealStackLimit
+          ? ExternalReference::address_of_real_jslimit(isolate)
+          : ExternalReference::address_of_jslimit(isolate);
+  DCHECK(TurboAssembler::IsAddressableThroughRootRegister(isolate, limit));
+
+  intptr_t offset =
+      TurboAssembler::RootRegisterOffsetForExternalReference(isolate, limit);
+  CHECK(is_int32(offset));
+  return Operand(kRootRegister, static_cast<int32_t>(offset));
+}
+
+void MacroAssembler::StackOverflowCheck(
+    Register num_args, Register scratch, Label* stack_overflow,
+    Label::Distance stack_overflow_distance) {
+  DCHECK_NE(num_args, scratch);
+  // Check the stack for overflow. We are not trying to catch
+  // interruptions (e.g. debug break and preemption) here, so the "real stack
+  // limit" is checked.
+  movq(kScratchRegister, StackLimitAsOperand(StackLimitKind::kRealStackLimit));
+  movq(scratch, rsp);
+  // Make scratch the space we have left. The stack might already be overflowed
+  // here which will cause scratch to become negative.
+  subq(scratch, kScratchRegister);
+  // TODO(victorgomes): Use ia32 approach with leaq, since it requires less
+  // instructions.
+  sarq(scratch, Immediate(kSystemPointerSizeLog2));
+  // Check if the arguments will overflow the stack.
+  cmpq(scratch, num_args);
+  // Signed comparison.
+  // TODO(victorgomes):  Save some bytes in the builtins that use stack checks
+  // by jumping to a builtin that throws the exception.
+  j(less_equal, stack_overflow, stack_overflow_distance);
+}
+
 void MacroAssembler::InvokePrologue(Register expected_parameter_count,
                                     Register actual_parameter_count,
                                     Label* done, InvokeFlag flag) {
   if (expected_parameter_count != actual_parameter_count) {
     Label regular_invoke;
 #ifdef V8_NO_ARGUMENTS_ADAPTOR
-    // Skip if adaptor sentinel.
+    // If the expected parameter count is equal to the adaptor sentinel, no need
+    // to push undefined value as arguments.
     cmpl(expected_parameter_count, Immediate(kDontAdaptArgumentsSentinel));
-    j(equal, &regular_invoke, Label::kNear);
+    j(equal, &regular_invoke, Label::kFar);
 
-    // Skip if overapplication or if expected number of arguments.
+    // If overapplication or if the actual argument count is equal to the
+    // formal parameter count, no need to push extra undefined values.
     subq(expected_parameter_count, actual_parameter_count);
-    j(less_equal, &regular_invoke, Label::kNear);
+    j(less_equal, &regular_invoke, Label::kFar);
+
+    Label stack_overflow;
+    StackOverflowCheck(expected_parameter_count, rcx, &stack_overflow);
 
     // Underapplication. Move the arguments already in the stack, including the
     // receiver and the return address.
@@ -2488,6 +2531,15 @@ void MacroAssembler::InvokePrologue(Register expected_parameter_count,
            kScratchRegister);
       j(greater, &loop, Label::kNear);
     }
+    jmp(&regular_invoke);
+
+    bind(&stack_overflow);
+    {
+      FrameScope frame(this,
+                       has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
+      CallRuntime(Runtime::kThrowStackOverflow);
+      int3();  // This should be unreachable.
+    }
 #else
     // Both expected and actual are in (different) registers. This
     // is the case when we invoke functions using call and apply.

src/codegen/x64/macro-assembler-x64.h

@@ -33,6 +33,10 @@ struct SmiIndex {
   ScaleFactor scale;
 };
 
+// TODO(victorgomes): Move definition to macro-assembler.h, once all other
+// platforms are updated.
+enum class StackLimitKind { kInterruptStackLimit, kRealStackLimit };
+
 // Convenient class to access arguments below the stack pointer.
 class StackArgumentsAccessor {
  public:
@@ -1030,6 +1034,13 @@ class V8_EXPORT_PRIVATE MacroAssembler : public TurboAssembler {
   void IncrementCounter(StatsCounter* counter, int value);
   void DecrementCounter(StatsCounter* counter, int value);
 
+  // ---------------------------------------------------------------------------
+  // Stack limit utilities
+  Operand StackLimitAsOperand(StackLimitKind kind);
+  void StackOverflowCheck(
+      Register num_args, Register scratch, Label* stack_overflow,
+      Label::Distance stack_overflow_distance = Label::kFar);
+
   // ---------------------------------------------------------------------------
   // In-place weak references.
   void LoadWeakValue(Register in_out, Label* target_if_cleared);

test/mjsunit/stackoverflow-underapplication.js

+// Copyright 2020 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --stack-size=100 --no-opt
+
+function f(
+  x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
+  x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
+  x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
+  x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
+  x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
+  x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
+  x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
+  x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
+  x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
+  x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
+  x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
+  x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
+  x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
+  x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
+  x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x,
+  x, x, x, x, x, x, x, x, x, x, x, x, x, x, x, x
+) { }
+
+function runNearStackLimit(f) {
+  let recursing_towards_stack_limit = true;
+  let f_succeeded = false;
+
+  function t() {
+    try {
+      t();
+      if (f_succeeded) return;
+      // Keep calling f until it stops throwing stack overflow exceptions.
+      f();
+      // f didn't throw, so we are done.
+      f_succeeded = true;
+    } catch(e) {
+      if (recursing_towards_stack_limit) {
+        recursing_towards_stack_limit = false;
+        // We reached the near stack limit state, call f first time.
+        f();
+        // f didn't throw, so we are done.
+        f_succeeded = true;
+      }
+    }
+  };
+
+  try {
+    t();
+  } catch(e) {}
+}
+
+runNearStackLimit(f);