[arm64] Pad function arguments.

This patch updates the instruction selector and code generator to pad arguments
for arm64 and drop an even number of slots when dropping the arguments. It also
updates the builtins that handle arguments. These changes need to be made at
the same time.

It also adds some tests for forwarding varargs, as this was affected by the
builtin changes and the existing tests did not catch all issues.

Bug: v8:6644
Change-Id: I81318d1d1c9ab2568f84f2bb868d2a2d4cb56053
Reviewed-on: https://chromium-review.googlesource.com/829933
Commit-Queue: Georgia Kouveli <georgia.kouveli@arm.com>
Reviewed-by: Benedikt Meurer <bmeurer@chromium.org>
Cr-Commit-Position: refs/heads/master@{#50259}

src/arm64/assembler-arm64.h

@@ -295,9 +295,7 @@ class Register : public CPURegister {
 static_assert(IS_TRIVIALLY_COPYABLE(Register),
               "Register can efficiently be passed by value");
 
-// TODO(arm64): Switch this on when the rest of the argument padding changes
-//              are done.
-constexpr bool kPadArguments = false;
+constexpr bool kPadArguments = true;
 constexpr bool kSimpleFPAliasing = true;
 constexpr bool kSimdMaskRegisters = false;
 

src/arm64/macro-assembler-arm64-inl.h

@@ -1339,21 +1339,31 @@ void TurboAssembler::Drop(const Register& count, uint64_t unit_size) {
 
 void TurboAssembler::DropArguments(const Register& count,
                                    ArgumentsCountMode mode) {
+  int extra_slots = 1;  // Padding slot.
   if (mode == kCountExcludesReceiver) {
-    UseScratchRegisterScope temps(this);
-    Register tmp = temps.AcquireX();
-    Add(tmp, count, 1);
-    Drop(tmp);
-  } else {
-    Drop(count);
+    // Add a slot for the receiver.
+    ++extra_slots;
+  }
+  UseScratchRegisterScope temps(this);
+  Register tmp = temps.AcquireX();
+  Add(tmp, count, extra_slots);
+  Bic(tmp, tmp, 1);
+  Drop(tmp, kXRegSize);
+}
+
+void TurboAssembler::DropArguments(int64_t count, ArgumentsCountMode mode) {
+  if (mode == kCountExcludesReceiver) {
+    // Add a slot for the receiver.
+    ++count;
   }
+  Drop(RoundUp(count, 2), kXRegSize);
 }
 
-void TurboAssembler::DropSlots(int64_t count, uint64_t unit_size) {
-  Drop(count, unit_size);
+void TurboAssembler::DropSlots(int64_t count) {
+  Drop(RoundUp(count, 2), kXRegSize);
 }
 
-void TurboAssembler::PushArgument(const Register& arg) { Push(arg); }
+void TurboAssembler::PushArgument(const Register& arg) { Push(padreg, arg); }
 
 void MacroAssembler::DropBySMI(const Register& count_smi, uint64_t unit_size) {
   DCHECK(unit_size == 0 || base::bits::IsPowerOfTwo(unit_size));

src/arm64/macro-assembler-arm64.cc

@@ -2159,20 +2159,27 @@ void TurboAssembler::PrepareForTailCall(const ParameterCount& callee_args_count,
   // after we drop current frame. We add kPointerSize to count the receiver
   // argument which is not included into formal parameters count.
   Register dst_reg = scratch0;
-  add(dst_reg, fp, Operand(caller_args_count_reg, LSL, kPointerSizeLog2));
-  add(dst_reg, dst_reg,
-      Operand(StandardFrameConstants::kCallerSPOffset + kPointerSize));
+  Add(dst_reg, fp, Operand(caller_args_count_reg, LSL, kPointerSizeLog2));
+  Add(dst_reg, dst_reg, StandardFrameConstants::kCallerSPOffset + kPointerSize);
+  // Round dst_reg up to a multiple of 16 bytes, so that we overwrite any
+  // potential padding.
+  Add(dst_reg, dst_reg, 15);
+  Bic(dst_reg, dst_reg, 15);
 
   Register src_reg = caller_args_count_reg;
   // Calculate the end of source area. +kPointerSize is for the receiver.
   if (callee_args_count.is_reg()) {
-    add(src_reg, jssp, Operand(callee_args_count.reg(), LSL, kPointerSizeLog2));
-    add(src_reg, src_reg, Operand(kPointerSize));
+    Add(src_reg, jssp, Operand(callee_args_count.reg(), LSL, kPointerSizeLog2));
+    Add(src_reg, src_reg, kPointerSize);
   } else {
-    add(src_reg, jssp,
-        Operand((callee_args_count.immediate() + 1) * kPointerSize));
+    Add(src_reg, jssp, (callee_args_count.immediate() + 1) * kPointerSize);
   }
 
+  // Round src_reg up to a multiple of 16 bytes, so we include any potential
+  // padding in the copy.
+  Add(src_reg, src_reg, 15);
+  Bic(src_reg, src_reg, 15);
+
   if (FLAG_debug_code) {
     Cmp(src_reg, dst_reg);
     Check(lo, kStackAccessBelowStackPointer);

src/arm64/macro-assembler-arm64.h

@@ -711,25 +711,22 @@ class TurboAssembler : public Assembler {
   inline void Drop(int64_t count, uint64_t unit_size = kXRegSize);
   inline void Drop(const Register& count, uint64_t unit_size = kXRegSize);
 
-  // Drop arguments from stack without actually accessing memory.
-  // This will currently drop 'count' arguments from the stack.
+  // Drop 'count' arguments from the stack, rounded up to a multiple of two,
+  // without actually accessing memory.
   // We assume the size of the arguments is the pointer size.
   // An optional mode argument is passed, which can indicate we need to
   // explicitly add the receiver to the count.
-  // TODO(arm64): Update this to round up the number of bytes dropped to
-  // a multiple of 16, so that we can remove jssp.
   enum ArgumentsCountMode { kCountIncludesReceiver, kCountExcludesReceiver };
   inline void DropArguments(const Register& count,
                             ArgumentsCountMode mode = kCountIncludesReceiver);
+  inline void DropArguments(int64_t count,
+                            ArgumentsCountMode mode = kCountIncludesReceiver);
 
-  // Drop slots from stack without actually accessing memory.
-  // This will currently drop 'count' slots of the given size from the stack.
-  // TODO(arm64): Update this to round up the number of bytes dropped to
-  // a multiple of 16, so that we can remove jssp.
-  inline void DropSlots(int64_t count, uint64_t unit_size = kXRegSize);
+  // Drop 'count' slots from stack, rounded up to a multiple of two, without
+  // actually accessing memory.
+  inline void DropSlots(int64_t count);
 
-  // Push a single argument to the stack.
-  // TODO(arm64): Update this to push a padding slot above the argument.
+  // Push a single argument, with padding, to the stack.
   inline void PushArgument(const Register& arg);
 
   // Re-synchronizes the system stack pointer (csp) with the current stack

src/builtins/arm64/builtins-arm64.cc

@@ -418,19 +418,11 @@ void Generate_JSConstructStubGeneric(MacroAssembler* masm,
     __ InvokeFunction(x1, x3, actual, CALL_FUNCTION);
 
     // ----------- S t a t e -------------
-    // If argc is odd:
     //  -- sp[0*kPointerSize]: implicit receiver
     //  -- sp[1*kPointerSize]: padding
     //  -- sp[2*kPointerSize]: constructor function
     //  -- sp[3*kPointerSize]: number of arguments
     //  -- sp[4*kPointerSize]: context
-    // If argc is even:
-    //  -- sp[0*kPointerSize]: implicit receiver
-    //  -- sp[1*kPointerSize]: implicit receiver
-    //  -- sp[2*kPointerSize]: padding
-    //  -- sp[3*kPointerSize]: constructor function
-    //  -- sp[4*kPointerSize]: number of arguments
-    //  -- sp[5*kPointerSize]: context
     // -----------------------------------
 
     // Store offset of return address for deoptimizer.
@@ -568,10 +560,15 @@ void Builtins::Generate_ResumeGeneratorTrampoline(MacroAssembler* masm) {
   __ Ldr(w10,
          FieldMemOperand(x10, SharedFunctionInfo::kFormalParameterCountOffset));
 
-  // Claim slots for arguments and receiver.
-  __ Add(x11, x10, 1);
+  // Claim slots for arguments and receiver (rounded up to a multiple of two).
+  __ Add(x11, x10, 2);
+  __ Bic(x11, x11, 1);
   __ Claim(x11);
 
+  // Store padding (which might be replaced by the receiver).
+  __ Sub(x11, x11, 1);
+  __ Poke(padreg, Operand(x11, LSL, kPointerSizeLog2));
+
   // Poke receiver into highest claimed slot.
   __ Ldr(x5, FieldMemOperand(x1, JSGeneratorObject::kReceiverOffset));
   __ Poke(x5, Operand(x10, LSL, kPointerSizeLog2));
@@ -628,10 +625,10 @@ void Builtins::Generate_ResumeGeneratorTrampoline(MacroAssembler* masm) {
   __ Bind(&prepare_step_in_if_stepping);
   {
     FrameScope scope(masm, StackFrame::INTERNAL);
-    __ Push(x1);
+    __ Push(x1, padreg);
     __ PushArgument(x4);
     __ CallRuntime(Runtime::kDebugOnFunctionCall);
-    __ Pop(x1);
+    __ Pop(padreg, x1);
     __ Ldr(x4, FieldMemOperand(x1, JSGeneratorObject::kFunctionOffset));
   }
   __ B(&stepping_prepared);
@@ -639,9 +636,9 @@ void Builtins::Generate_ResumeGeneratorTrampoline(MacroAssembler* masm) {
   __ Bind(&prepare_step_in_suspended_generator);
   {
     FrameScope scope(masm, StackFrame::INTERNAL);
-    __ Push(x1);
+    __ Push(x1, padreg);
     __ CallRuntime(Runtime::kDebugPrepareStepInSuspendedGenerator);
-    __ Pop(x1);
+    __ Pop(padreg, x1);
     __ Ldr(x4, FieldMemOperand(x1, JSGeneratorObject::kFunctionOffset));
   }
   __ B(&stepping_prepared);
@@ -1187,10 +1184,19 @@ static void Generate_InterpreterPushArgs(MacroAssembler* masm,
   __ Unreachable();
   __ Bind(&done);
 
-  // TODO(arm64): Claim one extra slot for padding and store padreg to the
-  // padding slot.
+  // Round up to an even number of slots and claim them.
+  __ Add(slots_to_claim, slots_to_claim, 1);
+  __ Bic(slots_to_claim, slots_to_claim, 1);
   __ Claim(slots_to_claim);
 
+  {
+    // Store padding, which may be overwritten.
+    UseScratchRegisterScope temps(masm);
+    Register scratch = temps.AcquireX();
+    __ Sub(scratch, slots_to_claim, 1);
+    __ Poke(padreg, Operand(scratch, LSL, kPointerSizeLog2));
+  }
+
   if (receiver_mode == ConvertReceiverMode::kNullOrUndefined) {
     // Store "undefined" as the receiver arg if we need to.
     Register receiver = x14;
@@ -1861,45 +1867,55 @@ void Builtins::Generate_FunctionPrototypeApply(MacroAssembler* masm) {
 void Builtins::Generate_FunctionPrototypeCall(MacroAssembler* masm) {
   Register argc = x0;
   Register function = x1;
-  Register scratch1 = x10;
-  Register scratch2 = x11;
 
   ASM_LOCATION("Builtins::Generate_FunctionPrototypeCall");
 
-  // 1. Make sure we have at least one argument.
+  // 1. Get the callable to call (passed as receiver) from the stack.
+  __ Peek(function, Operand(argc, LSL, kXRegSizeLog2));
+
+  // 2. Handle case with no arguments.
   {
-    Label done;
-    __ Cbnz(argc, &done);
-    __ LoadRoot(scratch1, Heap::kUndefinedValueRootIndex);
-    __ Push(scratch1);
-    __ Mov(argc, 1);
-    __ Bind(&done);
+    Label non_zero;
+    Register scratch = x10;
+    __ Cbnz(argc, &non_zero);
+    __ LoadRoot(scratch, Heap::kUndefinedValueRootIndex);
+    // Overwrite receiver with undefined, which will be the new receiver.
+    // We do not need to overwrite the padding slot above it with anything.
+    __ Poke(scratch, 0);
+    // Call function. The argument count is already zero.
+    __ Jump(masm->isolate()->builtins()->Call(), RelocInfo::CODE_TARGET);
+    __ Bind(&non_zero);
   }
 
-  // 2. Get the callable to call (passed as receiver) from the stack.
-  __ Peek(function, Operand(argc, LSL, kXRegSizeLog2));
+  // 3. Overwrite the receiver with padding. If argc is odd, this is all we
+  //    need to do.
+  Label arguments_ready;
+  __ Poke(padreg, Operand(argc, LSL, kXRegSizeLog2));
+  __ Tbnz(argc, 0, &arguments_ready);
 
-  // 3. Shift arguments and return address one slot down on the stack
-  //    (overwriting the original receiver).  Adjust argument count to make
-  //    the original first argument the new receiver.
+  // 4. If argc is even:
+  //    Copy arguments two slots higher in memory, overwriting the original
+  //    receiver and padding.
   {
     Label loop;
-    // Calculate the copy start address (destination). Copy end address is jssp.
-    __ SlotAddress(scratch2, argc);
-    __ Sub(scratch1, scratch2, kPointerSize);
-
-    __ Bind(&loop);
-    __ Ldr(x12, MemOperand(scratch1, -kPointerSize, PostIndex));
-    __ Str(x12, MemOperand(scratch2, -kPointerSize, PostIndex));
-    __ Cmp(scratch1, jssp);
-    __ B(ge, &loop);
-    // Adjust the actual number of arguments and remove the top element
-    // (which is a copy of the last argument).
-    __ Sub(argc, argc, 1);
-    __ Drop(1);
+    Register copy_from = x10;
+    Register copy_to = x11;
+    Register count = x12;
+    Register last_arg_slot = x13;
+    __ Mov(count, argc);
+    __ Sub(last_arg_slot, argc, 1);
+    __ SlotAddress(copy_from, last_arg_slot);
+    __ Add(copy_to, copy_from, 2 * kPointerSize);
+    __ CopyDoubleWords(copy_to, copy_from, count,
+                       TurboAssembler::kSrcLessThanDst);
+    // Drop two slots. These are copies of the last two arguments.
+    __ Drop(2);
   }
 
-  // 4. Call the callable.
+  // 5. Adjust argument count to make the original first argument the new
+  //    receiver and call the callable.
+  __ Bind(&arguments_ready);
+  __ Sub(argc, argc, 1);
   __ Jump(masm->isolate()->builtins()->Call(), RelocInfo::CODE_TARGET);
 }
 
@@ -2066,7 +2082,9 @@ void Builtins::Generate_ReflectConstruct(MacroAssembler* masm) {
           RelocInfo::CODE_TARGET);
 }
 
-static void EnterArgumentsAdaptorFrame(MacroAssembler* masm) {
+namespace {
+
+void EnterArgumentsAdaptorFrame(MacroAssembler* masm) {
   __ Push(lr, fp);
   __ Mov(x11, StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR));
   __ Push(x11, x1);  // x1: function
@@ -2075,7 +2093,7 @@ static void EnterArgumentsAdaptorFrame(MacroAssembler* masm) {
   __ Add(fp, jssp, ArgumentsAdaptorFrameConstants::kFixedFrameSizeFromFp);
 }
 
-static void LeaveArgumentsAdaptorFrame(MacroAssembler* masm) {
+void LeaveArgumentsAdaptorFrame(MacroAssembler* masm) {
   // ----------- S t a t e -------------
   //  -- x0 : result being passed through
   // -----------------------------------
@@ -2090,6 +2108,67 @@ static void LeaveArgumentsAdaptorFrame(MacroAssembler* masm) {
   __ DropArguments(x10, TurboAssembler::kCountExcludesReceiver);
 }
 
+// Prepares the stack for copying the varargs. First we claim the necessary
+// slots, taking care of potential padding. Then we copy the existing arguments
+// one slot up or one slot down, as needed.
+void Generate_PrepareForCopyingVarargs(MacroAssembler* masm, Register argc,
+                                       Register len) {
+  Label len_odd, exit;
+  Register slots_to_copy = x10;  // If needed.
+  __ Add(slots_to_copy, argc, 1);
+  __ Add(argc, argc, len);
+  __ Tbnz(len, 0, &len_odd);
+  __ Claim(len);
+  __ B(&exit);
+
+  __ Bind(&len_odd);
+  // Claim space we need. If argc is even, slots_to_claim = len + 1, as we need
+  // one extra padding slot. If argc is odd, we know that the original arguments
+  // will have a padding slot we can reuse (since len is odd), so
+  // slots_to_claim = len - 1.
+  {
+    Register scratch = x11;
+    Register slots_to_claim = x12;
+    __ Add(slots_to_claim, len, 1);
+    __ And(scratch, argc, 1);
+    __ Sub(slots_to_claim, slots_to_claim, Operand(scratch, LSL, 1));
+    __ Claim(slots_to_claim);
+  }
+
+  Label copy_down;
+  __ Tbz(slots_to_copy, 0, &copy_down);
+
+  // Copy existing arguments one slot up.
+  {
+    Register src = x11;
+    Register dst = x12;
+    Register scratch = x13;
+    __ Sub(scratch, argc, 1);
+    __ SlotAddress(src, scratch);
+    __ SlotAddress(dst, argc);
+    __ CopyDoubleWords(dst, src, slots_to_copy,
+                       TurboAssembler::kSrcLessThanDst);
+  }
+  __ B(&exit);
+
+  // Copy existing arguments one slot down and add padding.
+  __ Bind(&copy_down);
+  {
+    Register src = x11;
+    Register dst = x12;
+    Register scratch = x13;
+    __ Add(src, len, 1);
+    __ Mov(dst, len);  // CopySlots will corrupt dst.
+    __ CopySlots(dst, src, slots_to_copy);
+    __ Add(scratch, argc, 1);
+    __ Poke(padreg, Operand(scratch, LSL, kPointerSizeLog2));  // Store padding.
+  }
+
+  __ Bind(&exit);
+}
+
+}  // namespace
+
 // static
 void Builtins::Generate_CallOrConstructVarargs(MacroAssembler* masm,
                                                Handle<Code> code) {
@@ -2122,30 +2201,34 @@ void Builtins::Generate_CallOrConstructVarargs(MacroAssembler* masm,
     __ Bind(&done);
   }
 
-  // Push arguments onto the stack (thisArgument is already on the stack).
-  {
-    Label done, push, loop;
-    Register src = x5;
+  // Skip argument setup if we don't need to push any varargs.
+  Label done;
+  __ Cbz(len, &done);
 
-    __ Add(src, arguments_list, FixedArray::kHeaderSize - kHeapObjectTag);
-    __ Add(argc, argc, len);  // The 'len' argument for Call() or Construct().
-    __ Cbz(len, &done);
+  Generate_PrepareForCopyingVarargs(masm, argc, len);
+
+  // Push varargs.
+  {
+    Label loop;
+    Register src = x10;
     Register the_hole_value = x11;
     Register undefined_value = x12;
-    // We do not use the CompareRoot macro as it would do a LoadRoot behind the
-    // scenes and we want to avoid that in a loop.
+    Register scratch = x13;
+    __ Add(src, arguments_list, FixedArray::kHeaderSize - kHeapObjectTag);
     __ LoadRoot(the_hole_value, Heap::kTheHoleValueRootIndex);
     __ LoadRoot(undefined_value, Heap::kUndefinedValueRootIndex);
-    __ Claim(len);
+    // We do not use the CompareRoot macro as it would do a LoadRoot behind the
+    // scenes and we want to avoid that in a loop.
+    // TODO(all): Consider using Ldp and Stp.
     __ Bind(&loop);
     __ Sub(len, len, 1);
-    __ Ldr(x10, MemOperand(src, kPointerSize, PostIndex));
-    __ Cmp(x10, the_hole_value);
-    __ Csel(x10, x10, undefined_value, ne);
-    __ Poke(x10, Operand(len, LSL, kPointerSizeLog2));
+    __ Ldr(scratch, MemOperand(src, kPointerSize, PostIndex));
+    __ Cmp(scratch, the_hole_value);
+    __ Csel(scratch, scratch, undefined_value, ne);
+    __ Poke(scratch, Operand(len, LSL, kPointerSizeLog2));
     __ Cbnz(len, &loop);
-    __ Bind(&done);
   }
+  __ Bind(&done);
 
   // Tail-call to the actual Call or Construct builtin.
   __ Jump(code, RelocInfo::CODE_TARGET);
@@ -2162,6 +2245,9 @@ void Builtins::Generate_CallOrConstructForwardVarargs(MacroAssembler* masm,
   //  -- x2 : start index (to support rest parameters)
   // -----------------------------------
 
+  Register argc = x0;
+  Register start_index = x2;
+
   // Check if new.target has a [[Construct]] internal method.
   if (mode == CallOrConstructMode::kConstruct) {
     Label new_target_constructor, new_target_not_constructor;
@@ -2181,49 +2267,57 @@ void Builtins::Generate_CallOrConstructForwardVarargs(MacroAssembler* masm,
   }
 
   // Check if we have an arguments adaptor frame below the function frame.
-  Label arguments_adaptor, arguments_done;
-  __ Ldr(x5, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
-  __ Ldr(x4, MemOperand(x5, CommonFrameConstants::kContextOrFrameTypeOffset));
-  __ Cmp(x4, StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR));
-  __ B(eq, &arguments_adaptor);
+  // args_fp will point to the frame that contains the actual arguments, which
+  // will be the current frame unless we have an arguments adaptor frame, in
+  // which case args_fp points to the arguments adaptor frame.
+  Register args_fp = x5;
+  Register len = x6;
   {
-    __ Ldr(x6, MemOperand(fp, JavaScriptFrameConstants::kFunctionOffset));
-    __ Ldr(x6, FieldMemOperand(x6, JSFunction::kSharedFunctionInfoOffset));
-    __ Ldrsw(x6, FieldMemOperand(
-                     x6, SharedFunctionInfo::kFormalParameterCountOffset));
-    __ Mov(x5, fp);
-  }
-  __ B(&arguments_done);
-  __ Bind(&arguments_adaptor);
-  {
-    // Just load the length from ArgumentsAdaptorFrame.
-    __ Ldrsw(x6, UntagSmiMemOperand(
-                     x5, ArgumentsAdaptorFrameConstants::kLengthOffset));
+    Label arguments_adaptor, arguments_done;
+    Register scratch = x10;
+    __ Ldr(args_fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
+    __ Ldr(x4, MemOperand(args_fp,
+                          CommonFrameConstants::kContextOrFrameTypeOffset));
+    __ Cmp(x4, StackFrame::TypeToMarker(StackFrame::ARGUMENTS_ADAPTOR));
+    __ B(eq, &arguments_adaptor);
+    {
+      __ Ldr(scratch,
+             MemOperand(fp, JavaScriptFrameConstants::kFunctionOffset));
+      __ Ldr(scratch,
+             FieldMemOperand(scratch, JSFunction::kSharedFunctionInfoOffset));
+      __ Ldrsw(len,
+               FieldMemOperand(
+                   scratch, SharedFunctionInfo::kFormalParameterCountOffset));
+      __ Mov(args_fp, fp);
+    }
+    __ B(&arguments_done);
+    __ Bind(&arguments_adaptor);
+    {
+      // Just load the length from ArgumentsAdaptorFrame.
+      __ Ldrsw(len,
+               UntagSmiMemOperand(
+                   args_fp, ArgumentsAdaptorFrameConstants::kLengthOffset));
+    }
+    __ Bind(&arguments_done);
   }
-  __ Bind(&arguments_done);
 
   Label stack_done, stack_overflow;
-  __ Subs(x6, x6, x2);
+  __ Subs(len, len, start_index);
   __ B(le, &stack_done);
-  {
-    // Check for stack overflow.
-    Generate_StackOverflowCheck(masm, x6, &stack_overflow);
+  // Check for stack overflow.
+  Generate_StackOverflowCheck(masm, x6, &stack_overflow);
 
-    // Forward the arguments from the caller frame.
-    {
-      Label loop;
-      __ Add(x5, x5, kPointerSize);
-      __ Add(x0, x0, x6);
-      __ Bind(&loop);
-      {
-        __ Ldr(x4, MemOperand(x5, x6, LSL, kPointerSizeLog2));
-        __ Push(x4);
-        __ Subs(x6, x6, 1);
-        __ B(ne, &loop);
-      }
-    }
+  Generate_PrepareForCopyingVarargs(masm, argc, len);
+
+  // Push varargs.
+  {
+    Register dst = x13;
+    __ Add(args_fp, args_fp, 2 * kPointerSize);
+    __ SlotAddress(dst, 0);
+    __ CopyDoubleWords(dst, args_fp, len);
   }
   __ B(&stack_done);
+
   __ Bind(&stack_overflow);
   __ TailCallRuntime(Runtime::kThrowStackOverflow);
   __ Bind(&stack_done);
@@ -2342,12 +2436,16 @@ void Generate_PushBoundArguments(MacroAssembler* masm) {
   //  -- x3 : new.target (only in case of [[Construct]])
   // -----------------------------------
 
+  Register bound_argc = x4;
+  Register bound_argv = x2;
+
   // Load [[BoundArguments]] into x2 and length of that into x4.
   Label no_bound_arguments;
-  __ Ldr(x2, FieldMemOperand(x1, JSBoundFunction::kBoundArgumentsOffset));
-  __ Ldrsw(x4, UntagSmiFieldMemOperand(x2, FixedArray::kLengthOffset));
-  __ Cmp(x4, 0);
-  __ B(eq, &no_bound_arguments);
+  __ Ldr(bound_argv,
+         FieldMemOperand(x1, JSBoundFunction::kBoundArgumentsOffset));
+  __ Ldrsw(bound_argc,
+           UntagSmiFieldMemOperand(bound_argv, FixedArray::kLengthOffset));
+  __ Cbz(bound_argc, &no_bound_arguments);
   {
     // ----------- S t a t e -------------
     //  -- x0 : the number of arguments (not including the receiver)
@@ -2357,44 +2455,97 @@ void Generate_PushBoundArguments(MacroAssembler* masm) {
     //  -- x4 : the number of [[BoundArguments]]
     // -----------------------------------
 
+    Register argc = x0;
+
+    // Check for stack overflow.
     {
-      Label done;
-      __ Claim(x4);
       // Check the stack for overflow. We are not trying to catch interruptions
       // (i.e. debug break and preemption) here, so check the "real stack
       // limit".
-      __ CompareRoot(jssp, Heap::kRealStackLimitRootIndex);
+      Label done;
+      __ LoadRoot(x10, Heap::kRealStackLimitRootIndex);
+      // Make x10 the space we have left. The stack might already be overflowed
+      // here which will cause x10 to become negative.
+      __ Sub(x10, masm->StackPointer(), x10);
+      // Check if the arguments will overflow the stack.
+      __ Cmp(x10, Operand(bound_argc, LSL, kPointerSizeLog2));
       __ B(gt, &done);  // Signed comparison.
-      // Restore the stack pointer.
-      __ Drop(x4);
-      {
-        FrameScope scope(masm, StackFrame::MANUAL);
-        __ EnterFrame(StackFrame::INTERNAL);
-        __ CallRuntime(Runtime::kThrowStackOverflow);
-      }
+      __ TailCallRuntime(Runtime::kThrowStackOverflow);
       __ Bind(&done);
     }
 
-    UseScratchRegisterScope temps(masm);
-    Register argc = temps.AcquireX();
-    // Relocate arguments down the stack.
-    __ Mov(argc, x0);
-    __ CopySlots(0, x4, argc);
+    // Check if we need padding.
+    Label copy_args, copy_bound_args;
+    Register total_argc = x15;
+    Register slots_to_claim = x12;
+    __ Add(total_argc, argc, bound_argc);
+    __ Mov(slots_to_claim, bound_argc);
+    __ Tbz(bound_argc, 0, &copy_args);
+
+    // Load receiver before we start moving the arguments. We will only
+    // need this in this path because the bound arguments are odd.
+    Register receiver = x14;
+    __ Peek(receiver, Operand(argc, LSL, kPointerSizeLog2));
 
-    // Copy [[BoundArguments]] to the stack (below the arguments). The first
-    // element of the array is copied to the highest address.
+    // Claim space we need. If argc is even, slots_to_claim = bound_argc + 1,
+    // as we need one extra padding slot. If argc is odd, we know that the
+    // original arguments will have a padding slot we can reuse (since
+    // bound_argc is odd), so slots_to_claim = bound_argc - 1.
     {
-      Label loop;
-      __ Ldrsw(x4, UntagSmiFieldMemOperand(x2, FixedArray::kLengthOffset));
-      __ Add(x2, x2, FixedArray::kHeaderSize - kHeapObjectTag);
-      __ SlotAddress(x11, x0);
-      __ Add(x0, x0, x4);
-      __ Bind(&loop);
-      __ Sub(x4, x4, 1);
-      __ Ldr(x10, MemOperand(x2, x4, LSL, kPointerSizeLog2));
-      // Poke into claimed area of stack.
-      __ Str(x10, MemOperand(x11, kPointerSize, PostIndex));
-      __ Cbnz(x4, &loop);
+      Register scratch = x11;
+      __ Add(slots_to_claim, bound_argc, 1);
+      __ And(scratch, total_argc, 1);
+      __ Sub(slots_to_claim, slots_to_claim, Operand(scratch, LSL, 1));
+    }
+
+    // Copy bound arguments.
+    __ Bind(&copy_args);
+    // Skip claim and copy of existing arguments in the special case where we
+    // do not need to claim any slots (this will be the case when
+    // bound_argc == 1 and the existing arguments have padding we can reuse).
+    __ Cbz(slots_to_claim, &copy_bound_args);
+    __ Claim(slots_to_claim);
+    {
+      Register count = x10;
+      // Relocate arguments to a lower address.
+      __ Mov(count, argc);
+      __ CopySlots(0, slots_to_claim, count);
+
+      __ Bind(&copy_bound_args);
+      // Copy [[BoundArguments]] to the stack (below the arguments). The first
+      // element of the array is copied to the highest address.
+      {
+        Label loop;
+        Register counter = x10;
+        Register scratch = x11;
+        Register copy_to = x12;
+        __ Add(bound_argv, bound_argv,
+               FixedArray::kHeaderSize - kHeapObjectTag);
+        __ SlotAddress(copy_to, argc);
+        __ Add(argc, argc,
+               bound_argc);  // Update argc to include bound arguments.
+        __ Lsl(counter, bound_argc, kPointerSizeLog2);
+        __ Bind(&loop);
+        __ Sub(counter, counter, kPointerSize);
+        __ Ldr(scratch, MemOperand(bound_argv, counter));
+        // Poke into claimed area of stack.
+        __ Str(scratch, MemOperand(copy_to, kPointerSize, PostIndex));
+        __ Cbnz(counter, &loop);
+      }
+
+      {
+        Label done;
+        Register scratch = x10;
+        __ Tbz(bound_argc, 0, &done);
+        // Store receiver.
+        __ Add(scratch, __ StackPointer(),
+               Operand(total_argc, LSL, kPointerSizeLog2));
+        __ Str(receiver, MemOperand(scratch, kPointerSize, PostIndex));
+        __ Tbnz(total_argc, 0, &done);
+        // Store padding.
+        __ Str(padreg, MemOperand(scratch));
+        __ Bind(&done);
+      }
     }
   }
   __ Bind(&no_bound_arguments);

src/compiler/arm64/code-generator-arm64.cc

@@ -597,6 +597,7 @@ void AdjustStackPointerForTailCall(TurboAssembler* tasm,
   int current_sp_offset = state->GetSPToFPSlotCount() +
                           StandardFrameConstants::kFixedSlotCountAboveFp;
   int stack_slot_delta = new_slot_above_sp - current_sp_offset;
+  DCHECK_EQ(stack_slot_delta % 2, 0);
   if (stack_slot_delta > 0) {
     tasm->Claim(stack_slot_delta);
     state->IncreaseSPDelta(stack_slot_delta);
@@ -616,8 +617,15 @@ void CodeGenerator::AssembleTailCallBeforeGap(Instruction* instr,
 
 void CodeGenerator::AssembleTailCallAfterGap(Instruction* instr,
                                              int first_unused_stack_slot) {
+  DCHECK_EQ(first_unused_stack_slot % 2, 0);
   AdjustStackPointerForTailCall(tasm(), frame_access_state(),
                                 first_unused_stack_slot);
+  DCHECK(instr->IsTailCall());
+  InstructionOperandConverter g(this, instr);
+  int optional_padding_slot = g.InputInt32(instr->InputCount() - 2);
+  if (optional_padding_slot % 2) {
+    __ Poke(padreg, optional_padding_slot * kPointerSize);
+  }
 }
 
 // Check if the code object is marked for deoptimization. If it is, then it
@@ -1369,7 +1377,7 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction(
       if (instr->InputAt(0)->IsFPRegister()) {
         __ Poke(i.InputFloat64Register(0), operand);
       } else {
-        __ Poke(i.InputRegister(0), operand);
+        __ Poke(i.InputOrZeroRegister64(0), operand);
       }
       __ SetStackPointer(prev);
       break;
@@ -2475,6 +2483,8 @@ void CodeGenerator::AssembleConstructFrame() {
     __ AssertCspAligned();
   }
 
+  // The frame has been previously padded in CodeGenerator::FinishFrame().
+  DCHECK_EQ(frame()->GetTotalFrameSlotCount() % 2, 0);
   int shrink_slots =
       frame()->GetTotalFrameSlotCount() - descriptor->CalculateFixedFrameSize();
 
@@ -2629,28 +2639,20 @@ void CodeGenerator::AssembleReturn(InstructionOperand* pop) {
       } else {
         __ Bind(&return_label_);
         AssembleDeconstructFrame();
-        if (descriptor->UseNativeStack()) {
-          pop_count += (pop_count & 1);  // align
-        }
       }
     } else {
       AssembleDeconstructFrame();
-      if (descriptor->UseNativeStack()) {
-        pop_count += (pop_count & 1);  // align
-      }
     }
-  } else if (descriptor->UseNativeStack()) {
-    pop_count += (pop_count & 1);  // align
   }
 
   if (pop->IsImmediate()) {
     DCHECK_EQ(Constant::kInt32, g.ToConstant(pop).type());
     pop_count += g.ToConstant(pop).ToInt32();
-    __ Drop(pop_count);
+    __ DropArguments(pop_count);
   } else {
     Register pop_reg = g.ToRegister(pop);
     __ Add(pop_reg, pop_reg, pop_count);
-    __ Drop(pop_reg);
+    __ DropArguments(pop_reg);
   }
 
   if (descriptor->UseNativeStack()) {

src/compiler/arm64/instruction-selector-arm64.cc

@@ -1740,6 +1740,7 @@ void InstructionSelector::EmitPrepareArguments(
 
   int claim_count = static_cast<int>(arguments->size());
   int slot = claim_count - 1;
+  claim_count = RoundUp(claim_count, 2);
   // Bump the stack pointer(s).
   if (claim_count > 0 || always_claim) {
     // TODO(titzer): claim and poke probably take small immediates.
@@ -1751,8 +1752,14 @@ void InstructionSelector::EmitPrepareArguments(
     Emit(claim, g.NoOutput(), g.TempImmediate(claim_count));
   }
 
-  // Poke the arguments into the stack.
   ArchOpcode poke = to_native_stack ? kArm64PokeCSP : kArm64PokeJSSP;
+  if (claim_count > 0) {
+    // Store padding, which might be overwritten.
+    Emit(poke, g.NoOutput(), g.UseImmediate(0),
+         g.TempImmediate(claim_count - 1));
+  }
+
+  // Poke the arguments into the stack.
   while (slot >= 0) {
     Emit(poke, g.NoOutput(), g.UseRegister((*arguments)[slot].node),
          g.TempImmediate(slot));

src/compiler/instruction-selector.cc

@@ -2505,6 +2505,14 @@ void InstructionSelector::VisitTailCall(Node* node) {
 
   Emit(kArchPrepareTailCall, g.NoOutput());
 
+  // Add an immediate operand that represents the first slot that is unused
+  // with respect to the stack pointer that has been updated for the tail call
+  // instruction. This is used by backends that need to pad arguments for stack
+  // alignment, in order to store an optional slot of padding above the
+  // arguments.
+  int optional_padding_slot = callee->GetFirstUnusedStackSlot();
+  buffer.instruction_args.push_back(g.TempImmediate(optional_padding_slot));
+
   int first_unused_stack_slot =
       (V8_TARGET_ARCH_STORES_RETURN_ADDRESS_ON_STACK ? 1 : 0) +
       stack_param_delta;

src/compiler/linkage.cc

@@ -75,33 +75,25 @@ bool CallDescriptor::HasSameReturnLocationsAs(
   return true;
 }
 
-int CallDescriptor::GetStackParameterDelta(
-    CallDescriptor const* tail_caller) const {
-  int callee_slots_above_sp = 0;
+int CallDescriptor::GetFirstUnusedStackSlot() const {
+  int slots_above_sp = 0;
   for (size_t i = 0; i < InputCount(); ++i) {
     LinkageLocation operand = GetInputLocation(i);
     if (!operand.IsRegister()) {
       int new_candidate =
           -operand.GetLocation() + operand.GetSizeInPointers() - 1;
-      if (new_candidate > callee_slots_above_sp) {
-        callee_slots_above_sp = new_candidate;
-      }
-    }
-  }
-  int tail_caller_slots_above_sp = 0;
-  if (tail_caller != nullptr) {
-    for (size_t i = 0; i < tail_caller->InputCount(); ++i) {
-      LinkageLocation operand = tail_caller->GetInputLocation(i);
-      if (!operand.IsRegister()) {
-        int new_candidate =
-            -operand.GetLocation() + operand.GetSizeInPointers() - 1;
-        if (new_candidate > tail_caller_slots_above_sp) {
-          tail_caller_slots_above_sp = new_candidate;
-        }
+      if (new_candidate > slots_above_sp) {
+        slots_above_sp = new_candidate;
       }
     }
   }
+  return slots_above_sp;
+}
 
+int CallDescriptor::GetStackParameterDelta(
+    CallDescriptor const* tail_caller) const {
+  int callee_slots_above_sp = GetFirstUnusedStackSlot();
+  int tail_caller_slots_above_sp = tail_caller->GetFirstUnusedStackSlot();
   int stack_param_delta = callee_slots_above_sp - tail_caller_slots_above_sp;
   if (kPadArguments) {
     // Adjust stack delta when it is odd.

src/compiler/linkage.h

@@ -298,6 +298,9 @@ class V8_EXPORT_PRIVATE CallDescriptor final
 
   bool HasSameReturnLocationsAs(const CallDescriptor* other) const;
 
+  // Returns the first stack slot that is not used by the stack parameters.
+  int GetFirstUnusedStackSlot() const;
+
   int GetStackParameterDelta(const CallDescriptor* tail_caller) const;
 
   bool CanTailCall(const Node* call) const;

test/cctest/compiler/test-code-generator.cc

@@ -780,10 +780,12 @@ class CodeGeneratorTester {
   }
 
   Instruction* CreateTailCall(int stack_slot_delta) {
+    int optional_padding_slot = stack_slot_delta;
     InstructionOperand callee[] = {
+        ImmediateOperand(ImmediateOperand::INLINE, optional_padding_slot),
         ImmediateOperand(ImmediateOperand::INLINE, stack_slot_delta)};
     Instruction* tail_call = Instruction::New(zone_, kArchTailCallCodeObject, 0,
-                                              nullptr, 1, callee, 0, nullptr);
+                                              nullptr, 2, callee, 0, nullptr);
     return tail_call;
   }
 
@@ -851,13 +853,15 @@ class CodeGeneratorTester {
     // We use either zero or one slots.
     int first_unused_stack_slot =
         V8_TARGET_ARCH_STORES_RETURN_ADDRESS_ON_STACK ? 1 : 0;
+    int optional_padding_slot = first_unused_stack_slot;
     InstructionOperand callee[] = {
         AllocatedOperand(LocationOperand::REGISTER,
                          MachineRepresentation::kTagged,
                          kReturnRegister0.code()),
+        ImmediateOperand(ImmediateOperand::INLINE, optional_padding_slot),
         ImmediateOperand(ImmediateOperand::INLINE, first_unused_stack_slot)};
     Instruction* tail_call = Instruction::New(zone_, kArchTailCallCodeObject, 0,
-                                              nullptr, 2, callee, 0, nullptr);
+                                              nullptr, 3, callee, 0, nullptr);
     sequence->AddInstruction(tail_call);
     sequence->EndBlock(RpoNumber::FromInt(0));
 

test/mjsunit/compiler/varargs.js

+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+x = "a";
+
+function test_varargs(...args) {
+  var sum = this.x;
+  for (i in args) {
+    sum += "," + args[i];
+  }
+  return sum;
+}
+
+assertEquals("a", test_varargs());
+assertEquals("a,b", test_varargs("b"));
+assertEquals("a,b,c", test_varargs("b", "c"));
+assertEquals("a,b,c,d", test_varargs("b", "c", "d"));
+assertEquals("a,b,c,d,e", test_varargs("b", "c", "d", "e"));
+
+function forward_varargs(...args) {
+  return test_varargs(...args);
+}
+
+assertEquals("a", forward_varargs());
+assertEquals("a,b", forward_varargs("b"));
+assertEquals("a,b,c", forward_varargs("b", "c"));
+assertEquals("a,b,c,d", forward_varargs("b", "c", "d"));
+assertEquals("a,b,c,d,e", forward_varargs("b", "c", "d", "e"));
+
+function forward_varargs_one_arg(x, ...args) {
+  return test_varargs(x, ...args);
+}
+
+assertEquals("a,undefined", forward_varargs_one_arg());
+assertEquals("a,b", forward_varargs_one_arg("b"));
+assertEquals("a,b,c", forward_varargs_one_arg("b", "c"));
+assertEquals("a,b,c,d", forward_varargs_one_arg("b", "c", "d"));
+assertEquals("a,b,c,d,e", forward_varargs_one_arg("b", "c", "d", "e"));
+
+function forward_varargs_two_args(x, y, ...args) {
+  return test_varargs(x, y, ...args);
+}
+
+assertEquals("a,undefined,undefined", forward_varargs_two_args());
+assertEquals("a,b,undefined", forward_varargs_two_args("b"));
+assertEquals("a,b,c", forward_varargs_two_args("b", "c"));
+assertEquals("a,b,c,d", forward_varargs_two_args("b", "c", "d"));
+assertEquals("a,b,c,d,e", forward_varargs_two_args("b", "c", "d", "e"));