[wasm] Fix arm64 frame alignment.

    In JavaScript code and stubs, JSSP mirrors the CSP but may be unaligned.
    But in WASM code only CSP is used, like native code, and it must be
    aligned.
    Calls into WASM from JS need to carefully align the C stack
    pointer (csp) and restore the previous JSSP, while calls from WASM
    to JS need to compute a new JSSP and restore their CSP after the
    call.

R=ahaas@chromium.org
BUG=

Review URL: https://codereview.chromium.org/1811283003

Cr-Commit-Position: refs/heads/master@{#35096}

src/arm64/macro-assembler-arm64.cc

@@ -1355,6 +1355,14 @@ void MacroAssembler::AssertStackConsistency() {
   }
 }
 
+void MacroAssembler::AssertCspAligned() {
+  if (emit_debug_code() && use_real_aborts()) {
+    // TODO(titzer): use a real assert for alignment check?
+    UseScratchRegisterScope scope(this);
+    Register temp = scope.AcquireX();
+    ldr(temp, MemOperand(csp));
+  }
+}
 
 void MacroAssembler::AssertFPCRState(Register fpcr) {
   if (emit_debug_code()) {

src/arm64/macro-assembler-arm64.h

@@ -788,6 +788,9 @@ class MacroAssembler : public Assembler {
   // If emit_debug_code() is false, this emits no code.
   void AssertStackConsistency();
 
+  // Emits a runtime assert that the CSP is aligned.
+  void AssertCspAligned();
+
   // Preserve the callee-saved registers (as defined by AAPCS64).
   //
   // Higher-numbered registers are pushed before lower-numbered registers, and

src/compiler/arm64/code-generator-arm64.cc

@@ -530,16 +530,20 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
         __ Add(target, target, Code::kHeaderSize - kHeapObjectTag);
         __ Call(target);
       }
+      RecordCallPosition(instr);
       // TODO(titzer): this is ugly. JSSP should be a caller-save register
       // in this case, but it is not possible to express in the register
       // allocator.
-      CallDescriptor::Flags flags =
-          static_cast<CallDescriptor::Flags>(MiscField::decode(opcode));
+      CallDescriptor::Flags flags(MiscField::decode(opcode));
       if (flags & CallDescriptor::kRestoreJSSP) {
-        __ mov(jssp, csp);
+        __ Ldr(jssp, MemOperand(csp));
+        __ Mov(csp, jssp);
+      }
+      if (flags & CallDescriptor::kRestoreCSP) {
+        __ Mov(csp, jssp);
+        __ AssertCspAligned();
       }
       frame_access_state()->ClearSPDelta();
-      RecordCallPosition(instr);
       break;
     }
     case kArchTailCallCodeObjectFromJSFunction:
@@ -575,16 +579,20 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
       }
       __ Ldr(x10, FieldMemOperand(func, JSFunction::kCodeEntryOffset));
       __ Call(x10);
+      RecordCallPosition(instr);
       // TODO(titzer): this is ugly. JSSP should be a caller-save register
       // in this case, but it is not possible to express in the register
       // allocator.
-      CallDescriptor::Flags flags =
-          static_cast<CallDescriptor::Flags>(MiscField::decode(opcode));
+      CallDescriptor::Flags flags(MiscField::decode(opcode));
       if (flags & CallDescriptor::kRestoreJSSP) {
-        __ mov(jssp, csp);
+        __ Ldr(jssp, MemOperand(csp));
+        __ Mov(csp, jssp);
+      }
+      if (flags & CallDescriptor::kRestoreCSP) {
+        __ Mov(csp, jssp);
+        __ AssertCspAligned();
       }
       frame_access_state()->ClearSPDelta();
-      RecordCallPosition(instr);
       break;
     }
     case kArchTailCallJSFunctionFromJSFunction:
@@ -967,26 +975,46 @@ void CodeGenerator::AssembleArchInstruction(Instruction* instr) {
       // Pseudo instruction turned into cbz/cbnz in AssembleArchBranch.
       break;
     case kArm64ClaimCSP: {
-      int count = i.InputInt32(0);
+      int count = RoundUp(i.InputInt32(0), 2);
       Register prev = __ StackPointer();
       if (prev.Is(jssp)) {
-        __ AlignAndSetCSPForFrame();
-      }
-      if (count > 0) {
-        __ Claim(count);
+        // TODO(titzer): make this a macro-assembler method.
+        // Align the CSP and store the previous JSSP on the stack.
+        UseScratchRegisterScope scope(masm());
+        Register tmp = scope.AcquireX();
+
+        int sp_alignment = __ ActivationFrameAlignment();
+        __ Sub(tmp, jssp, kPointerSize);
+        __ And(tmp, tmp, Operand(~static_cast<uint64_t>(sp_alignment - 1)));
+        __ Mov(csp, tmp);
+        __ Str(jssp, MemOperand(csp));
+        if (count > 0) {
+          __ SetStackPointer(csp);
+          __ Claim(count);
+          __ SetStackPointer(prev);
+        }
+      } else {
+        __ AssertCspAligned();
+        if (count > 0) {
+          __ Claim(count);
+          frame_access_state()->IncreaseSPDelta(count);
+        }
       }
-      __ SetStackPointer(prev);
-      frame_access_state()->IncreaseSPDelta(count);
       break;
     }
     case kArm64ClaimJSSP: {
       int count = i.InputInt32(0);
       if (csp.Is(__ StackPointer())) {
-        // No JSP is set up. Compute it from the CSP.
-        int even = RoundUp(count, 2);
-        __ Sub(jssp, csp, count * kPointerSize);
-        __ Sub(csp, csp, even * kPointerSize);  // Must always be aligned.
-        frame_access_state()->IncreaseSPDelta(even);
+        // No JSSP is set up. Compute it from the CSP.
+        __ AssertCspAligned();
+        if (count > 0) {
+          int even = RoundUp(count, 2);
+          __ Sub(jssp, csp, count * kPointerSize);
+          __ Sub(csp, csp, even * kPointerSize);  // Must always be aligned.
+          frame_access_state()->IncreaseSPDelta(even);
+        } else {
+          __ Mov(jssp, csp);
+        }
       } else {
         // JSSP is the current stack pointer, just use regular Claim().
         __ Claim(count);
@@ -1467,6 +1495,10 @@ void CodeGenerator::AssembleDeoptimizerCall(
 
 void CodeGenerator::AssemblePrologue() {
   CallDescriptor* descriptor = linkage()->GetIncomingDescriptor();
+  if (descriptor->UseNativeStack()) {
+    __ AssertCspAligned();
+  }
+
   frame()->AlignFrame(16);
   int stack_shrink_slots = frame()->GetSpillSlotCount();
   if (frame()->needs_frame()) {
@@ -1579,6 +1611,10 @@ void CodeGenerator::AssembleReturn() {
     pop_count += (pop_count & 1);  // align
   }
   __ Drop(pop_count);
+
+  if (descriptor->UseNativeStack()) {
+    __ AssertCspAligned();
+  }
   __ Ret();
 }
 

src/compiler/arm64/instruction-selector-arm64.cc

@@ -1631,20 +1631,20 @@ void InstructionSelector::EmitPrepareArguments(
     Node* node) {
   Arm64OperandGenerator g(this);
 
+  bool from_native_stack = linkage()->GetIncomingDescriptor()->UseNativeStack();
   bool to_native_stack = descriptor->UseNativeStack();
 
+  bool always_claim = to_native_stack != from_native_stack;
+
   int claim_count = static_cast<int>(arguments->size());
   int slot = claim_count - 1;
-  if (to_native_stack) {
-    // Native stack must always be aligned to 16 (2 words).
-    claim_count = RoundUp(claim_count, 2);
-  }
-  // TODO(titzer): claim and poke probably take small immediates.
   // Bump the stack pointer(s).
-  if (claim_count > 0 || to_native_stack) {
+  if (claim_count > 0 || always_claim) {
+    // TODO(titzer): claim and poke probably take small immediates.
     // TODO(titzer): it would be better to bump the csp here only
     //                and emit paired stores with increment for non c frames.
     ArchOpcode claim = to_native_stack ? kArm64ClaimCSP : kArm64ClaimJSSP;
+    // Claim(0) isn't a nop if there is a mismatch between CSP and JSSP.
     Emit(claim, g.NoOutput(), g.TempImmediate(claim_count));
   }
 

src/compiler/c-linkage.cc

@@ -230,6 +230,11 @@ CallDescriptor* Linkage::GetSimplifiedCDescriptor(
   // The target for C calls is always an address (i.e. machine pointer).
   MachineType target_type = MachineType::Pointer();
   LinkageLocation target_loc = LinkageLocation::ForAnyRegister();
+  CallDescriptor::Flags flags = CallDescriptor::kUseNativeStack;
+  if (set_initialize_root_flag) {
+    flags |= CallDescriptor::kInitializeRootRegister;
+  }
+
   return new (zone) CallDescriptor(  // --
       CallDescriptor::kCallAddress,  // kind
       target_type,                   // target MachineType
@@ -240,10 +245,7 @@ CallDescriptor* Linkage::GetSimplifiedCDescriptor(
       Operator::kNoProperties,       // properties
       kCalleeSaveRegisters,          // callee-saved registers
       kCalleeSaveFPRegisters,        // callee-saved fp regs
-      set_initialize_root_flag ?     // flags
-          CallDescriptor::kInitializeRootRegister
-                               : CallDescriptor::kNoFlags,
-      "c-call");
+      flags, "c-call");
 }
 
 }  // namespace compiler

src/compiler/instruction-selector.cc

@@ -1543,13 +1543,15 @@ void InstructionSelector::VisitCall(Node* node, BasicBlock* handler) {
     buffer.instruction_args.push_back(g.Label(handler));
   }
 
-  // (arm64 only) caller uses JSSP but callee might destroy it.
-  if (descriptor->UseNativeStack() &&
-      !linkage()->GetIncomingDescriptor()->UseNativeStack()) {
-    flags |= CallDescriptor::kRestoreJSSP;
+  bool from_native_stack = linkage()->GetIncomingDescriptor()->UseNativeStack();
+  bool to_native_stack = descriptor->UseNativeStack();
+  if (from_native_stack != to_native_stack) {
+    // (arm64 only) Mismatch in the use of stack pointers. One or the other
+    // has to be restored manually by the code generator.
+    flags |= to_native_stack ? CallDescriptor::kRestoreJSSP
+                             : CallDescriptor::kRestoreCSP;
   }
 
-
   // Select the appropriate opcode based on the call type.
   InstructionCode opcode = kArchNop;
   switch (descriptor->kind()) {

src/compiler/linkage.h

@@ -160,10 +160,11 @@ class CallDescriptor final : public ZoneObject {
     kCanUseRoots = 1u << 6,
     // (arm64 only) native stack should be used for arguments.
     kUseNativeStack = 1u << 7,
-    // (arm64 only) call instruction has to restore JSSP.
+    // (arm64 only) call instruction has to restore JSSP or CSP.
     kRestoreJSSP = 1u << 8,
+    kRestoreCSP = 1u << 9,
     // Causes the code generator to initialize the root register.
-    kInitializeRootRegister = 1u << 9,
+    kInitializeRootRegister = 1u << 10,
     kPatchableCallSiteWithNop = kPatchableCallSite | kNeedsNopAfterCall
   };
   typedef base::Flags<Flag> Flags;

src/compiler/wasm-compiler.cc

@@ -2461,6 +2461,12 @@ Handle<JSFunction> CompileJSToWasmWrapper(
     CompilationInfo info(func_name, isolate, &zone, flags);
     Handle<Code> code =
         Pipeline::GenerateCodeForTesting(&info, incoming, &graph, nullptr);
+#ifdef ENABLE_DISASSEMBLER
+    if (FLAG_print_opt_code && !code.is_null()) {
+      OFStream os(stdout);
+      code->Disassemble(buffer.start(), os);
+    }
+#endif
     if (debugging) {
       buffer.Dispose();
     }
@@ -2541,6 +2547,12 @@ Handle<Code> CompileWasmToJSWrapper(Isolate* isolate, wasm::ModuleEnv* module,
 
     CompilationInfo info(func_name, isolate, &zone, flags);
     code = Pipeline::GenerateCodeForTesting(&info, incoming, &graph, nullptr);
+#ifdef ENABLE_DISASSEMBLER
+    if (FLAG_print_opt_code && !code.is_null()) {
+      OFStream os(stdout);
+      code->Disassemble(buffer.start(), os);
+    }
+#endif
     if (debugging) {
       buffer.Dispose();
     }

test/cctest/wasm/test-run-wasm-js.cc

@@ -66,11 +66,18 @@ uint32_t AddJsFunction(TestingModule* module, FunctionSig* sig,
 
 
 uint32_t AddJSSelector(TestingModule* module, FunctionSig* sig, int which) {
-  const int kMaxParams = 8;
-  static const char* formals[kMaxParams] = {
-      "",        "a",         "a,b",         "a,b,c",
-      "a,b,c,d", "a,b,c,d,e", "a,b,c,d,e,f", "a,b,c,d,e,f,g",
-  };
+  const int kMaxParams = 11;
+  static const char* formals[kMaxParams] = {"",
+                                            "a",
+                                            "a,b",
+                                            "a,b,c",
+                                            "a,b,c,d",
+                                            "a,b,c,d,e",
+                                            "a,b,c,d,e,f",
+                                            "a,b,c,d,e,f,g",
+                                            "a,b,c,d,e,f,g,h",
+                                            "a,b,c,d,e,f,g,h,i",
+                                            "a,b,c,d,e,f,g,h,i,j"};
   CHECK_LT(which, static_cast<int>(sig->parameter_count()));
   CHECK_LT(static_cast<int>(sig->parameter_count()), kMaxParams);
 
@@ -173,8 +180,6 @@ TEST(Run_I32Popcount_jswrapped) {
 }
 
 
-#if !V8_TARGET_ARCH_ARM64
-// TODO(titzer): dynamic frame alignment on arm64
 TEST(Run_CallJS_Add_jswrapped) {
   TestSignatures sigs;
   TestingModule module;
@@ -189,12 +194,9 @@ TEST(Run_CallJS_Add_jswrapped) {
   EXPECT_CALL(199, jsfunc, 100, -1);
   EXPECT_CALL(-666666801, jsfunc, -666666900, -1);
 }
-#endif
 
 
 void RunJSSelectTest(int which) {
-#if !V8_TARGET_ARCH_ARM
-  // TODO(titzer): fix tests on arm and reenable
   const int kMaxParams = 8;
   PredictableInputValues inputs(0x100);
   LocalType type = kAstF64;
@@ -225,7 +227,6 @@ void RunJSSelectTest(int which) {
     double expected = inputs.arg_d(which);
     EXPECT_CALL(expected, jsfunc, 0.0, 0.0);
   }
-#endif
 }
 
 
@@ -298,10 +299,11 @@ TEST(Run_WASMSelect_7) { RunWASMSelectTest(7); }
 void RunWASMSelectAlignTest(int num_args, int num_params) {
   PredictableInputValues inputs(0x300);
   Isolate* isolate = CcTest::InitIsolateOnce();
-  const int kMaxParams = 4;
+  const int kMaxParams = 10;
   DCHECK_LE(num_args, kMaxParams);
   LocalType type = kAstF64;
-  LocalType types[kMaxParams + 1] = {type, type, type, type, type};
+  LocalType types[kMaxParams + 1] = {type, type, type, type, type, type,
+                                     type, type, type, type, type};
   FunctionSig sig(1, num_params, types);
 
   for (int which = 0; which < num_params; which++) {
@@ -310,12 +312,16 @@ void RunWASMSelectAlignTest(int num_args, int num_params) {
     BUILD(t, WASM_GET_LOCAL(which));
     Handle<JSFunction> jsfunc = WrapCode(&module, t.CompileAndAdd());
 
-    Handle<Object> args[] = {
-        isolate->factory()->NewNumber(inputs.arg_d(0)),
-        isolate->factory()->NewNumber(inputs.arg_d(1)),
-        isolate->factory()->NewNumber(inputs.arg_d(2)),
-        isolate->factory()->NewNumber(inputs.arg_d(3)),
-    };
+    Handle<Object> args[] = {isolate->factory()->NewNumber(inputs.arg_d(0)),
+                             isolate->factory()->NewNumber(inputs.arg_d(1)),
+                             isolate->factory()->NewNumber(inputs.arg_d(2)),
+                             isolate->factory()->NewNumber(inputs.arg_d(3)),
+                             isolate->factory()->NewNumber(inputs.arg_d(4)),
+                             isolate->factory()->NewNumber(inputs.arg_d(5)),
+                             isolate->factory()->NewNumber(inputs.arg_d(6)),
+                             isolate->factory()->NewNumber(inputs.arg_d(7)),
+                             isolate->factory()->NewNumber(inputs.arg_d(8)),
+                             isolate->factory()->NewNumber(inputs.arg_d(9))};
 
     double nan = std::numeric_limits<double>::quiet_NaN();
     double expected = which < num_args ? inputs.arg_d(which) : nan;
@@ -353,16 +359,43 @@ TEST(Run_WASMSelectAlign_4) {
   RunWASMSelectAlignTest(4, 4);
 }
 
+TEST(Run_WASMSelectAlign_7) {
+  RunWASMSelectAlignTest(7, 5);
+  RunWASMSelectAlignTest(7, 6);
+  RunWASMSelectAlignTest(7, 7);
+}
+
+TEST(Run_WASMSelectAlign_8) {
+  RunWASMSelectAlignTest(8, 5);
+  RunWASMSelectAlignTest(8, 6);
+  RunWASMSelectAlignTest(8, 7);
+  RunWASMSelectAlignTest(8, 8);
+}
+
+TEST(Run_WASMSelectAlign_9) {
+  RunWASMSelectAlignTest(9, 6);
+  RunWASMSelectAlignTest(9, 7);
+  RunWASMSelectAlignTest(9, 8);
+  RunWASMSelectAlignTest(9, 9);
+}
+
+TEST(Run_WASMSelectAlign_10) {
+  RunWASMSelectAlignTest(10, 7);
+  RunWASMSelectAlignTest(10, 8);
+  RunWASMSelectAlignTest(10, 9);
+  RunWASMSelectAlignTest(10, 10);
+}
 
 void RunJSSelectAlignTest(int num_args, int num_params) {
   PredictableInputValues inputs(0x400);
   Isolate* isolate = CcTest::InitIsolateOnce();
   Factory* factory = isolate->factory();
-  const int kMaxParams = 4;
+  const int kMaxParams = 10;
   CHECK_LE(num_args, kMaxParams);
   CHECK_LE(num_params, kMaxParams);
   LocalType type = kAstF64;
-  LocalType types[kMaxParams + 1] = {type, type, type, type, type};
+  LocalType types[kMaxParams + 1] = {type, type, type, type, type, type,
+                                     type, type, type, type, type};
   FunctionSig sig(1, num_params, types);
 
   // Build the calling code.
@@ -392,6 +425,12 @@ void RunJSSelectAlignTest(int num_args, int num_params) {
         factory->NewNumber(inputs.arg_d(1)),
         factory->NewNumber(inputs.arg_d(2)),
         factory->NewNumber(inputs.arg_d(3)),
+        factory->NewNumber(inputs.arg_d(4)),
+        factory->NewNumber(inputs.arg_d(5)),
+        factory->NewNumber(inputs.arg_d(6)),
+        factory->NewNumber(inputs.arg_d(7)),
+        factory->NewNumber(inputs.arg_d(8)),
+        factory->NewNumber(inputs.arg_d(9)),
     };
 
     double nan = std::numeric_limits<double>::quiet_NaN();
@@ -406,29 +445,50 @@ TEST(Run_JSSelectAlign_0) {
   RunJSSelectAlignTest(0, 2);
 }
 
+TEST(Run_JSSelectAlign_1) {
+  RunJSSelectAlignTest(1, 2);
+  RunJSSelectAlignTest(1, 3);
+}
 
 TEST(Run_JSSelectAlign_2) {
   RunJSSelectAlignTest(2, 3);
   RunJSSelectAlignTest(2, 4);
 }
 
+TEST(Run_JSSelectAlign_3) {
+  RunJSSelectAlignTest(3, 3);
+  RunJSSelectAlignTest(3, 4);
+}
 
 TEST(Run_JSSelectAlign_4) {
   RunJSSelectAlignTest(4, 3);
   RunJSSelectAlignTest(4, 4);
 }
 
+TEST(Run_JSSelectAlign_7) {
+  RunJSSelectAlignTest(7, 3);
+  RunJSSelectAlignTest(7, 4);
+  RunJSSelectAlignTest(7, 4);
+  RunJSSelectAlignTest(7, 4);
+}
 
-#if !V8_TARGET_ARCH_ARM64
-// TODO(titzer): dynamic frame alignment on arm64
-TEST(Run_JSSelectAlign_1) {
-  RunJSSelectAlignTest(1, 2);
-  RunJSSelectAlignTest(1, 3);
+TEST(Run_JSSelectAlign_8) {
+  RunJSSelectAlignTest(8, 5);
+  RunJSSelectAlignTest(8, 6);
+  RunJSSelectAlignTest(8, 7);
+  RunJSSelectAlignTest(8, 8);
 }
 
+TEST(Run_JSSelectAlign_9) {
+  RunJSSelectAlignTest(9, 6);
+  RunJSSelectAlignTest(9, 7);
+  RunJSSelectAlignTest(9, 8);
+  RunJSSelectAlignTest(9, 9);
+}
 
-TEST(Run_JSSelectAlign_3) {
-  RunJSSelectAlignTest(3, 3);
-  RunJSSelectAlignTest(3, 4);
+TEST(Run_JSSelectAlign_10) {
+  RunJSSelectAlignTest(10, 7);
+  RunJSSelectAlignTest(10, 8);
+  RunJSSelectAlignTest(10, 9);
+  RunJSSelectAlignTest(10, 10);
 }
-#endif

test/cctest/wasm/test-run-wasm-module.cc

@@ -19,7 +19,6 @@ using namespace v8::internal::compiler;
 using namespace v8::internal::wasm;
 
 
-#if !V8_TARGET_ARCH_ARM64
 // TODO(titzer): fix arm64 frame alignment.
 namespace {
 void TestModule(WasmModuleIndex* module, int32_t expected_result) {
@@ -207,5 +206,3 @@ TEST(Run_WasmModule_Global) {
   TestModule(writer->WriteTo(&zone), 97);
 }
 #endif
-
-#endif  // !V8_TARGET_ARCH_ARM64

test/cctest/wasm/test-run-wasm.cc

@@ -2680,9 +2680,6 @@ TEST(Run_Wasm_F64CopySign) {
 }
 
 
-// TODO(tizer): Fix on arm and reenable.
-#if !V8_TARGET_ARCH_ARM && !V8_TARGET_ARCH_ARM64
-
 TEST(Run_Wasm_F32CopySign) {
   WasmRunner<float> r(MachineType::Float32(), MachineType::Float32());
   BUILD(r, WASM_F32_COPYSIGN(WASM_GET_LOCAL(0), WASM_GET_LOCAL(1)));
@@ -2692,10 +2689,6 @@ TEST(Run_Wasm_F32CopySign) {
   }
 }
 
-
-#endif
-
-
 void CompileCallIndirectMany(LocalType param) {
   // Make sure we don't run out of registers when compiling indirect calls
   // with many many parameters.

test/mjsunit/mjsunit.status

@@ -266,8 +266,7 @@
   'regress/regress-crbug-474297': [PASS, ['mode == debug', SLOW]],
   'es6/tail-call-megatest*': [PASS, FAST_VARIANTS],
 
-  # TODO(titzer): correct WASM adapter frame alignment on arm64
-  'wasm/*': [PASS, ['arch == arm64', SKIP]],
+  # TODO(titzer): ASM->WASM tests on these platforms
   'wasm/asm-wasm': [PASS, ['arch in [arm, arm64, mips, mipsel, mips64, mips64el]', SKIP]],
   # TODO(branelson): Figure out why ignition + asm-wasm-stdlib fails.
   'wasm/asm-wasm-stdlib': [PASS, ['arch in [arm, arm64, mips, mipsel, mips64, mips64el] or ignition == True', SKIP]],
@@ -276,7 +275,7 @@
   'wasm/asm-wasm-deopt': [PASS, ['arch in [arm, arm64, mips, mipsel, mips64, mips64el]', SKIP]],
 
   # TODO(branelson): Figure out why ignition + asm->wasm fails embenchen.
-  'wasm/embenchen/*': [PASS, ['ignition == True', SKIP]],
+  'wasm/embenchen/*': [PASS, ['arch == arm64', SKIP], ['ignition == True', SKIP]],
 
   # TODO(bradnelson) Fix and re-enable.
   'wasm/embenchen/box2d': [SKIP],   # hang