[wasm][x64] Add an additional stack check for functions with big frames

This is the x64 implementation of the CL
https://codereview.chromium.org/2763593002

Original message:

[wasm][arm] Add an additional stack check for functions with big frames.

Stack overflow checks are typically implemented as part of the TurboFan
graph of a function. This means that the stack check code is executed
after frame construction. When a frame is too big, though, there may not
be enough space on the stack anymore to throw the stack overflow
exception after frame construction. With this CL we do an additional
stack check before frame construction for functions with big frames.

As discussed offline with mstarzinger, I do this change currently only
for WebAssembly.

This CL contains only the changes for arm. I will do the other platforms
in separate CLs.

R=titzer@chromium.org

Bug: v8:6318
Change-Id: Id4a8ea3ee76c37132e86a7c4e5d05d3bd86df44a
Reviewed-on: https://chromium-review.googlesource.com/565562
Commit-Queue: Andreas Haas <ahaas@chromium.org>
Reviewed-by: Ben Titzer <titzer@chromium.org>
Cr-Commit-Position: refs/heads/master@{#46640}

src/compiler/x64/code-generator-x64.cc

@@ -2841,9 +2841,7 @@ void CodeGenerator::AssembleArchTrap(Instruction* instr,
             new (gen_->zone()) ReferenceMap(gen_->zone());
         gen_->RecordSafepoint(reference_map, Safepoint::kSimple, 0,
                               Safepoint::kNoLazyDeopt);
-        if (FLAG_debug_code) {
-          __ ud2();
-        }
+        __ AssertUnreachable(kUnexpectedReturnFromWasmTrap);
       }
     }
 
@@ -3005,6 +3003,36 @@ void CodeGenerator::AssembleConstructFrame() {
 
   const RegList saves_fp = descriptor->CalleeSavedFPRegisters();
   if (shrink_slots > 0) {
+    if (info()->IsWasm() && shrink_slots > 128) {
+      // For WebAssembly functions with big frames we have to do the stack
+      // overflow check before we construct the frame. Otherwise we may not
+      // have enough space on the stack to call the runtime for the stack
+      // overflow.
+      Label done;
+
+      // If the frame is bigger than the stack, we throw the stack overflow
+      // exception unconditionally. Thereby we can avoid the integer overflow
+      // check in the condition code.
+      if (shrink_slots * kPointerSize < FLAG_stack_size * 1024) {
+        __ Move(kScratchRegister,
+                ExternalReference::address_of_real_stack_limit(__ isolate()));
+        __ movq(kScratchRegister, Operand(kScratchRegister, 0));
+        __ addq(kScratchRegister, Immediate(shrink_slots * kPointerSize));
+        __ cmpq(rsp, kScratchRegister);
+        __ j(above_equal, &done);
+      }
+      if (!frame_access_state()->has_frame()) {
+        __ set_has_frame(true);
+        __ EnterFrame(StackFrame::WASM_COMPILED);
+      }
+      __ Move(rsi, Smi::kZero);
+      __ CallRuntimeDelayed(zone(), Runtime::kThrowWasmStackOverflow);
+      ReferenceMap* reference_map = new (zone()) ReferenceMap(zone());
+      RecordSafepoint(reference_map, Safepoint::kSimple, 0,
+                      Safepoint::kNoLazyDeopt);
+      __ AssertUnreachable(kUnexpectedReturnFromWasmTrap);
+      __ bind(&done);
+    }
     __ subq(rsp, Immediate(shrink_slots * kPointerSize));
   }
 

src/x64/macro-assembler-x64.cc

@@ -527,6 +527,10 @@ void TurboAssembler::Assert(Condition cc, BailoutReason reason) {
   if (emit_debug_code()) Check(cc, reason);
 }
 
+void TurboAssembler::AssertUnreachable(BailoutReason reason) {
+  if (emit_debug_code()) Abort(reason);
+}
+
 void TurboAssembler::Check(Condition cc, BailoutReason reason) {
   Label L;
   j(cc, &L, Label::kNear);

src/x64/macro-assembler-x64.h

@@ -354,6 +354,10 @@ class TurboAssembler : public Assembler {
   // Use --debug_code to enable.
   void Assert(Condition cc, BailoutReason reason);
 
+  // Like Assert(), but without condition.
+  // Use --debug_code to enable.
+  void AssertUnreachable(BailoutReason reason);
+
   // Abort execution if a 64 bit register containing a 32 bit payload does not
   // have zeros in the top 32 bits, enabled via --debug-code.
   void AssertZeroExtended(Register reg);

test/wasm-spec-tests/wasm-spec-tests.status

@@ -6,7 +6,7 @@
 [ALWAYS, {
   #TODO(ahaas): Add additional stack checks on mips.
   # Issue 6318: Stack checks for functions with huge stack frames fail on x64 and ia32
-  'tests/skip-stack-guard-page': [PASS, ['arch == mipsel or arch == mips64el or arch == x64 or arch == ia32 or ((arch == ppc or arch == ppc64 or arch == s390 or arch == s390x) and simulator_run)', SKIP]],
+  'tests/skip-stack-guard-page': [PASS, ['arch == mipsel or arch == mips64el or arch == ia32 or ((arch == ppc or arch == ppc64 or arch == s390 or arch == s390x) and simulator_run)', SKIP]],
 }],  # ALWAYS
 
 ['arch == mipsel or arch == mips64el or arch == mips or arch == mips64', {