X87: JSEntryTrampoline: check for stack space before pushing arguments

port 146598f4 (r27614)

original commit message:

  Optimistically pushing a lot of arguments can run into the stack limit of the
  process, at least on operating systems where this limit is close to the limit
  that V8 sets for itself.

BUG=

Review URL: https://codereview.chromium.org/1069283002

Cr-Commit-Position: refs/heads/master@{#27684}

src/x87/builtins-x87.cc

@@ -574,6 +574,47 @@ void Builtins::Generate_JSConstructStubForDerived(MacroAssembler* masm) {
 }
 
 
+enum IsTagged { kEaxIsSmiTagged, kEaxIsUntaggedInt };
+
+
+// Clobbers ecx, edx, edi; preserves all other registers.
+static void Generate_CheckStackOverflow(MacroAssembler* masm,
+                                        const int calleeOffset,
+                                        IsTagged eax_is_tagged) {
+  // eax   : the number of items to be pushed to the stack
+  //
+  // Check the stack for overflow. We are not trying to catch
+  // interruptions (e.g. debug break and preemption) here, so the "real stack
+  // limit" is checked.
+  Label okay;
+  ExternalReference real_stack_limit =
+      ExternalReference::address_of_real_stack_limit(masm->isolate());
+  __ mov(edi, Operand::StaticVariable(real_stack_limit));
+  // Make ecx the space we have left. The stack might already be overflowed
+  // here which will cause ecx to become negative.
+  __ mov(ecx, esp);
+  __ sub(ecx, edi);
+  // Make edx the space we need for the array when it is unrolled onto the
+  // stack.
+  __ mov(edx, eax);
+  int smi_tag = eax_is_tagged == kEaxIsSmiTagged ? kSmiTagSize : 0;
+  __ shl(edx, kPointerSizeLog2 - smi_tag);
+  // Check if the arguments will overflow the stack.
+  __ cmp(ecx, edx);
+  __ j(greater, &okay);  // Signed comparison.
+
+  // Out of stack space.
+  __ push(Operand(ebp, calleeOffset));  // push this
+  if (eax_is_tagged == kEaxIsUntaggedInt) {
+    __ SmiTag(eax);
+  }
+  __ push(eax);
+  __ InvokeBuiltin(Builtins::STACK_OVERFLOW, CALL_FUNCTION);
+
+  __ bind(&okay);
+}
+
+
 static void Generate_JSEntryTrampolineHelper(MacroAssembler* masm,
                                              bool is_construct) {
   ProfileEntryHookStub::MaybeCallEntryHook(masm);
@@ -599,6 +640,14 @@ static void Generate_JSEntryTrampolineHelper(MacroAssembler* masm,
     __ mov(eax, Operand(ebx, EntryFrameConstants::kArgcOffset));
     __ mov(ebx, Operand(ebx, EntryFrameConstants::kArgvOffset));
 
+    // Check if we have enough stack space to push all arguments.
+    // The function is the first thing that was pushed above after entering
+    // the internal frame.
+    const int kFunctionOffset =
+        InternalFrameConstants::kCodeOffset - kPointerSize;
+    // Expects argument count in eax. Clobbers ecx, edx, edi.
+    Generate_CheckStackOverflow(masm, kFunctionOffset, kEaxIsUntaggedInt);
+
     // Copy arguments to the stack in a loop.
     Label loop, entry;
     __ Move(ecx, Immediate(0));
@@ -990,38 +1039,6 @@ void Builtins::Generate_FunctionCall(MacroAssembler* masm) {
 }
 
 
-static void Generate_CheckStackOverflow(MacroAssembler* masm,
-                                        const int calleeOffset) {
-  // eax   : the number of items to be pushed to the stack
-  //
-  // Check the stack for overflow. We are not trying to catch
-  // interruptions (e.g. debug break and preemption) here, so the "real stack
-  // limit" is checked.
-  Label okay;
-  ExternalReference real_stack_limit =
-      ExternalReference::address_of_real_stack_limit(masm->isolate());
-  __ mov(edi, Operand::StaticVariable(real_stack_limit));
-  // Make ecx the space we have left. The stack might already be overflowed
-  // here which will cause ecx to become negative.
-  __ mov(ecx, esp);
-  __ sub(ecx, edi);
-  // Make edx the space we need for the array when it is unrolled onto the
-  // stack.
-  __ mov(edx, eax);
-  __ shl(edx, kPointerSizeLog2 - kSmiTagSize);
-  // Check if the arguments will overflow the stack.
-  __ cmp(ecx, edx);
-  __ j(greater, &okay);  // Signed comparison.
-
-  // Out of stack space.
-  __ push(Operand(ebp, calleeOffset));  // push this
-  __ push(eax);
-  __ InvokeBuiltin(Builtins::STACK_OVERFLOW, CALL_FUNCTION);
-
-  __ bind(&okay);
-}
-
-
 static void Generate_PushAppliedArguments(MacroAssembler* masm,
                                           const int argumentsOffset,
                                           const int indexOffset,
@@ -1099,7 +1116,7 @@ static void Generate_ApplyHelper(MacroAssembler* masm, bool targetIsArgument) {
       __ InvokeBuiltin(Builtins::APPLY_PREPARE, CALL_FUNCTION);
     }
 
-    Generate_CheckStackOverflow(masm, kFunctionOffset);
+    Generate_CheckStackOverflow(masm, kFunctionOffset, kEaxIsSmiTagged);
 
     // Push current index and limit.
     const int kLimitOffset =
@@ -1229,7 +1246,7 @@ static void Generate_ConstructHelper(MacroAssembler* masm) {
     __ push(Operand(ebp, kNewTargetOffset));
     __ InvokeBuiltin(Builtins::REFLECT_CONSTRUCT_PREPARE, CALL_FUNCTION);
 
-    Generate_CheckStackOverflow(masm, kFunctionOffset);
+    Generate_CheckStackOverflow(masm, kFunctionOffset, kEaxIsSmiTagged);
 
     // Push current index and limit.
     const int kLimitOffset =