[masm] Create helpers to manipulate arguments on the stack.

- Introduce helper to push arguments onto the stack (Standalone this
change doesn't make a lot of sense, but is in preparation for including
the receiver in argc).
- Introduce helper to shift arguments already on the stack to make room
for new arguments (Varargs).
- arm64 is not included because a) there was already a helper similar
to ShiftArguments and b) PushArguments is not similar enough to make
sense for arm64 because of small differences (e.g. also pushing the
function) in conjunction with stack alignment.

Drive-by: Use masm DropArguments in Sparkplug EmitReturn

Bug: v8:11112
Change-Id: Id7a3a5f025abb19e2a52dae27b3b484fe87e9faf
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3097275Reviewed-by: Victor Gomes <victorgomes@chromium.org>
Reviewed-by: Jakob Gruber <jgruber@chromium.org>
Commit-Queue: Patrick Thier <pthier@chromium.org>
Cr-Commit-Position: refs/heads/main@{#76392}

src/baseline/arm/baseline-assembler-arm-inl.h

@@ -501,9 +501,8 @@ void BaselineAssembler::EmitReturn(MacroAssembler* masm) {
   __ masm()->LeaveFrame(StackFrame::BASELINE);
 
   // Drop receiver + arguments.
-  __ masm()->add(params_size, params_size,
-                 Operand(1));  // Include the receiver.
-  __ masm()->Drop(params_size);
+  __ masm()->DropArguments(params_size, TurboAssembler::kCountIsInteger,
+                           TurboAssembler::kCountExcludesReceiver);
   __ masm()->Ret();
 }
 

src/baseline/arm64/baseline-assembler-arm64-inl.h

@@ -583,8 +583,7 @@ void BaselineAssembler::EmitReturn(MacroAssembler* masm) {
   __ masm()->LeaveFrame(StackFrame::BASELINE);
 
   // Drop receiver + arguments.
-  __ masm()->Add(params_size, params_size, 1);  // Include the receiver.
-  __ masm()->DropArguments(params_size);
+  __ masm()->DropArguments(params_size, TurboAssembler::kCountExcludesReceiver);
   __ masm()->Ret();
 }
 

src/baseline/ia32/baseline-assembler-ia32-inl.h

@@ -457,11 +457,9 @@ void BaselineAssembler::EmitReturn(MacroAssembler* masm) {
   __ masm()->LeaveFrame(StackFrame::BASELINE);
 
   // Drop receiver + arguments.
-  Register return_pc = scratch;
-  __ masm()->PopReturnAddressTo(return_pc);
-  __ masm()->lea(esp, MemOperand(esp, params_size, times_system_pointer_size,
-                                 kSystemPointerSize));
-  __ masm()->PushReturnAddressFrom(return_pc);
+  __ masm()->DropArguments(params_size, scratch,
+                           TurboAssembler::kCountIsInteger,
+                           TurboAssembler::kCountExcludesReceiver);
   __ masm()->Ret();
 }
 

src/baseline/x64/baseline-assembler-x64-inl.h

@@ -468,11 +468,9 @@ void BaselineAssembler::EmitReturn(MacroAssembler* masm) {
   __ masm()->LeaveFrame(StackFrame::BASELINE);
 
   // Drop receiver + arguments.
-  Register return_pc = scratch;
-  __ masm()->PopReturnAddressTo(return_pc);
-  __ masm()->leaq(rsp, MemOperand(rsp, params_size, times_system_pointer_size,
-                                  kSystemPointerSize));
-  __ masm()->PushReturnAddressFrom(return_pc);
+  __ masm()->DropArguments(params_size, scratch,
+                           TurboAssembler::kCountIsInteger,
+                           TurboAssembler::kCountExcludesReceiver);
   __ masm()->Ret();
 }
 

src/builtins/arm/builtins-arm.cc

@@ -76,6 +76,32 @@ static void GenerateTailCallToReturnedCode(MacroAssembler* masm,
 
 namespace {
 
+enum class ArgumentsElementType {
+  kRaw,    // Push arguments as they are.
+  kHandle  // Dereference arguments before pushing.
+};
+
+void Generate_PushArguments(MacroAssembler* masm, Register array, Register argc,
+                            Register scratch,
+                            ArgumentsElementType element_type) {
+  DCHECK(!AreAliased(array, argc, scratch));
+  UseScratchRegisterScope temps(masm);
+  Register counter = scratch;
+  Register value = temps.Acquire();
+  Label loop, entry;
+  __ mov(counter, argc);
+  __ b(&entry);
+  __ bind(&loop);
+  __ ldr(value, MemOperand(array, counter, LSL, kSystemPointerSizeLog2));
+  if (element_type == ArgumentsElementType::kHandle) {
+    __ ldr(value, MemOperand(value));
+  }
+  __ push(value);
+  __ bind(&entry);
+  __ sub(counter, counter, Operand(1), SetCC);
+  __ b(ge, &loop);
+}
+
 void Generate_JSBuiltinsConstructStubHelper(MacroAssembler* masm) {
   // ----------- S t a t e -------------
   //  -- r0     : number of arguments
@@ -106,12 +132,14 @@ void Generate_JSBuiltinsConstructStubHelper(MacroAssembler* masm) {
     // correct position (including any undefined), instead of delaying this to
     // InvokeFunction.
 
-    // Set up pointer to last argument (skip receiver).
+    // Set up pointer to first argument (skip receiver).
     __ add(
         r4, fp,
         Operand(StandardFrameConstants::kCallerSPOffset + kSystemPointerSize));
     // Copy arguments and receiver to the expression stack.
-    __ PushArray(r4, r0, r5);
+    // r4: Pointer to start of arguments.
+    // r0: Number of arguments.
+    Generate_PushArguments(masm, r4, r0, r5, ArgumentsElementType::kRaw);
     // The receiver for the builtin/api call.
     __ PushRoot(RootIndex::kTheHoleValue);
 
@@ -230,7 +258,9 @@ void Builtins::Generate_JSConstructStubGeneric(MacroAssembler* masm) {
   // InvokeFunction.
 
   // Copy arguments to the expression stack.
-  __ PushArray(r4, r0, r5);
+  // r4: Pointer to start of argument.
+  // r0: Number of arguments.
+  Generate_PushArguments(masm, r4, r0, r5, ArgumentsElementType::kRaw);
 
   // Push implicit receiver.
   __ Push(r6);
@@ -715,24 +745,13 @@ static void Generate_JSEntryTrampolineHelper(MacroAssembler* masm,
 
     __ bind(&enough_stack_space);
 
-    // Copy arguments to the stack in a loop.
+    // Copy arguments to the stack.
     // r1: new.target
     // r2: function
     // r3: receiver
     // r0: argc
     // r4: argv, i.e. points to first arg
-    Label loop, entry;
-    __ add(r6, r4, Operand(r0, LSL, kSystemPointerSizeLog2));
-    // r6 points past last arg.
-    __ b(&entry);
-    __ bind(&loop);
-    __ ldr(r5, MemOperand(r6, -kSystemPointerSize,
-                          PreIndex));  // read next parameter
-    __ ldr(r5, MemOperand(r5));        // dereference handle
-    __ push(r5);                       // push parameter
-    __ bind(&entry);
-    __ cmp(r4, r6);
-    __ b(ne, &loop);
+    Generate_PushArguments(masm, r4, r0, r5, ArgumentsElementType::kHandle);
 
     // Push the receiver.
     __ Push(r3);
@@ -2005,6 +2024,44 @@ void Builtins::Generate_ReflectConstruct(MacroAssembler* masm) {
           RelocInfo::CODE_TARGET);
 }
 
+namespace {
+
+// Allocate new stack space for |count| arguments and shift all existing
+// arguments already on the stack. |pointer_to_new_space_out| points to the
+// first free slot on the stack to copy additional arguments to and
+// |argc_in_out| is updated to include |count|.
+void Generate_AllocateSpaceAndShiftExistingArguments(
+    MacroAssembler* masm, Register count, Register argc_in_out,
+    Register pointer_to_new_space_out, Register scratch1, Register scratch2) {
+  DCHECK(!AreAliased(count, argc_in_out, pointer_to_new_space_out, scratch1,
+                     scratch2));
+  UseScratchRegisterScope temps(masm);
+  Register old_sp = scratch1;
+  Register new_space = scratch2;
+  __ mov(old_sp, sp);
+  __ lsl(new_space, count, Operand(kSystemPointerSizeLog2));
+  __ AllocateStackSpace(new_space);
+
+  Register end = scratch2;
+  Register value = temps.Acquire();
+  Register dest = pointer_to_new_space_out;
+  __ mov(dest, sp);
+  __ add(end, old_sp, Operand(argc_in_out, LSL, kSystemPointerSizeLog2));
+  Label loop, done;
+  __ bind(&loop);
+  __ cmp(old_sp, end);
+  __ b(gt, &done);
+  __ ldr(value, MemOperand(old_sp, kSystemPointerSize, PostIndex));
+  __ str(value, MemOperand(dest, kSystemPointerSize, PostIndex));
+  __ b(&loop);
+  __ bind(&done);
+
+  // Update total number of arguments.
+  __ add(argc_in_out, argc_in_out, count);
+}
+
+}  // namespace
+
 // static
 // TODO(v8:11615): Observe Code::kMaxArguments in CallOrConstructVarargs
 void Builtins::Generate_CallOrConstructVarargs(MacroAssembler* masm,
@@ -2042,23 +2099,10 @@ void Builtins::Generate_CallOrConstructVarargs(MacroAssembler* masm,
 
   // Move the arguments already in the stack,
   // including the receiver and the return address.
-  {
-    Label copy, check;
-    Register num = r5, src = r6, dest = r9;  // r7 and r8 are context and root.
-    __ mov(src, sp);
-    // Update stack pointer.
-    __ lsl(scratch, r4, Operand(kSystemPointerSizeLog2));
-    __ AllocateStackSpace(scratch);
-    __ mov(dest, sp);
-    __ mov(num, r0);
-    __ b(&check);
-    __ bind(&copy);
-    __ ldr(scratch, MemOperand(src, kSystemPointerSize, PostIndex));
-    __ str(scratch, MemOperand(dest, kSystemPointerSize, PostIndex));
-    __ sub(num, num, Operand(1), SetCC);
-    __ bind(&check);
-    __ b(ge, &copy);
-  }
+  // r4: Number of arguments to make room for.
+  // r0: Number of arguments already on the stack.
+  // r9: Points to first free slot on the stack after arguments were shifted.
+  Generate_AllocateSpaceAndShiftExistingArguments(masm, r4, r0, r9, r5, r6);
 
   // Copy arguments onto the stack (thisArgument is already on the stack).
   {
@@ -2077,7 +2121,6 @@ void Builtins::Generate_CallOrConstructVarargs(MacroAssembler* masm,
     __ add(r6, r6, Operand(1));
     __ b(&loop);
     __ bind(&done);
-    __ add(r0, r0, r6);
   }
 
   // Tail-call to the actual Call or Construct builtin.
@@ -2145,30 +2188,17 @@ void Builtins::Generate_CallOrConstructForwardVarargs(MacroAssembler* masm,
 
     // Move the arguments already in the stack,
     // including the receiver and the return address.
-    {
-      Label copy, check;
-      Register num = r8, src = r9,
-               dest = r2;  // r7 and r10 are context and root.
-      __ mov(src, sp);
-      // Update stack pointer.
-      __ lsl(scratch, r5, Operand(kSystemPointerSizeLog2));
-      __ AllocateStackSpace(scratch);
-      __ mov(dest, sp);
-      __ mov(num, r0);
-      __ b(&check);
-      __ bind(&copy);
-      __ ldr(scratch, MemOperand(src, kSystemPointerSize, PostIndex));
-      __ str(scratch, MemOperand(dest, kSystemPointerSize, PostIndex));
-      __ sub(num, num, Operand(1), SetCC);
-      __ bind(&check);
-      __ b(ge, &copy);
-    }
+    // r5: Number of arguments to make room for.
+    // r0: Number of arguments already on the stack.
+    // r2: Points to first free slot on the stack after arguments were shifted.
+    Generate_AllocateSpaceAndShiftExistingArguments(masm, r5, r0, r2, scratch,
+                                                    r8);
+
     // Copy arguments from the caller frame.
     // TODO(victorgomes): Consider using forward order as potentially more cache
     // friendly.
     {
       Label loop;
-      __ add(r0, r0, r5);
       __ bind(&loop);
       {
         __ sub(r5, r5, Operand(1), SetCC);

src/builtins/ia32/builtins-ia32.cc

@@ -78,6 +78,32 @@ static void GenerateTailCallToReturnedCode(MacroAssembler* masm,
 
 namespace {
 
+enum class ArgumentsElementType {
+  kRaw,    // Push arguments as they are.
+  kHandle  // Dereference arguments before pushing.
+};
+
+void Generate_PushArguments(MacroAssembler* masm, Register array, Register argc,
+                            Register scratch1, Register scratch2,
+                            ArgumentsElementType element_type) {
+  DCHECK(!AreAliased(array, argc, scratch1, scratch2));
+  Register counter = scratch1;
+  Label loop, entry;
+  __ mov(counter, argc);
+  __ jmp(&entry);
+  __ bind(&loop);
+  Operand value(array, counter, times_system_pointer_size, 0);
+  if (element_type == ArgumentsElementType::kHandle) {
+    DCHECK(scratch2 != no_reg);
+    __ mov(scratch2, value);
+    value = Operand(scratch2, 0);
+  }
+  __ Push(value);
+  __ bind(&entry);
+  __ dec(counter);
+  __ j(greater_equal, &loop, Label::kNear);
+}
+
 void Generate_JSBuiltinsConstructStubHelper(MacroAssembler* masm) {
   // ----------- S t a t e -------------
   //  -- eax: number of arguments
@@ -109,7 +135,10 @@ void Generate_JSBuiltinsConstructStubHelper(MacroAssembler* masm) {
     __ lea(esi, Operand(ebp, StandardFrameConstants::kCallerSPOffset +
                                  kSystemPointerSize));
     // Copy arguments to the expression stack.
-    __ PushArray(esi, eax, ecx);
+    // esi: Pointer to start of arguments.
+    // eax: Number of arguments.
+    Generate_PushArguments(masm, esi, eax, ecx, no_reg,
+                           ArgumentsElementType::kRaw);
     // The receiver for the builtin/api call.
     __ PushRoot(RootIndex::kTheHoleValue);
 
@@ -237,7 +266,10 @@ void Builtins::Generate_JSConstructStubGeneric(MacroAssembler* masm) {
   // InvokeFunction.
 
   // Copy arguments to the expression stack.
-  __ PushArray(edi, eax, ecx);
+  // edi: Pointer to start of arguments.
+  // eax: Number of arguments.
+  Generate_PushArguments(masm, edi, eax, ecx, no_reg,
+                         ArgumentsElementType::kRaw);
 
   // Push implicit receiver.
   __ movd(ecx, xmm0);
@@ -497,17 +529,11 @@ static void Generate_JSEntryTrampolineHelper(MacroAssembler* masm,
 
     __ bind(&enough_stack_space);
 
-    // Copy arguments to the stack in a loop.
-    Label loop, entry;
-    __ Move(ecx, eax);
-    __ jmp(&entry, Label::kNear);
-    __ bind(&loop);
-    // Push the parameter from argv.
-    __ mov(scratch2, Operand(scratch1, ecx, times_system_pointer_size, 0));
-    __ push(Operand(scratch2, 0));  // dereference handle
-    __ bind(&entry);
-    __ dec(ecx);
-    __ j(greater_equal, &loop);
+    // Copy arguments to the stack.
+    // scratch1 (edx): Pointer to start of arguments.
+    // eax: Number of arguments.
+    Generate_PushArguments(masm, scratch1, eax, ecx, scratch2,
+                           ArgumentsElementType::kHandle);
 
     // Load the previous frame pointer to access C arguments
     __ mov(scratch2, Operand(ebp, 0));
@@ -2105,6 +2131,52 @@ void Builtins::Generate_ReflectConstruct(MacroAssembler* masm) {
           RelocInfo::CODE_TARGET);
 }
 
+namespace {
+
+// Allocate new stack space for |count| arguments and shift all existing
+// arguments already on the stack. |pointer_to_new_space_out| points to the
+// first free slot on the stack to copy additional arguments to and
+// |argc_in_out| is updated to include |count|.
+void Generate_AllocateSpaceAndShiftExistingArguments(
+    MacroAssembler* masm, Register count, Register argc_in_out,
+    Register pointer_to_new_space_out, Register scratch1, Register scratch2) {
+  DCHECK(!AreAliased(count, argc_in_out, pointer_to_new_space_out, scratch1,
+                     scratch2));
+  // Use pointer_to_new_space_out as scratch until we set it to the correct
+  // value at the end.
+  Register old_esp = pointer_to_new_space_out;
+  Register new_space = scratch1;
+  __ mov(old_esp, esp);
+
+  __ lea(new_space, Operand(count, times_system_pointer_size, 0));
+  __ AllocateStackSpace(new_space);
+
+  __ inc(argc_in_out);  // Include the receiver.
+  Register current = scratch1;
+  Register value = scratch2;
+
+  Label loop, entry;
+  __ mov(current, 0);
+  __ jmp(&entry);
+  __ bind(&loop);
+  __ mov(value, Operand(old_esp, current, times_system_pointer_size, 0));
+  __ mov(Operand(esp, current, times_system_pointer_size, 0), value);
+  __ inc(current);
+  __ bind(&entry);
+  __ cmp(current, argc_in_out);
+  __ j(less_equal, &loop, Label::kNear);
+
+  // Point to the next free slot above the shifted arguments (argc + 1 slot for
+  // the return address).
+  __ lea(
+      pointer_to_new_space_out,
+      Operand(esp, argc_in_out, times_system_pointer_size, kSystemPointerSize));
+  // Update the total number of arguments, subtracting the receiver again.
+  __ lea(argc_in_out, Operand(argc_in_out, count, times_1, -1));
+}
+
+}  // namespace
+
 // static
 // TODO(v8:11615): Observe Code::kMaxArguments in CallOrConstructVarargs
 void Builtins::Generate_CallOrConstructVarargs(MacroAssembler* masm,
@@ -2114,15 +2186,13 @@ void Builtins::Generate_CallOrConstructVarargs(MacroAssembler* masm,
   //  -- esi    : context for the Call / Construct builtin
   //  -- eax    : number of parameters on the stack (not including the receiver)
   //  -- ecx    : len (number of elements to from args)
-  //  -- ecx    : new.target (checked to be constructor or undefined)
+  //  -- edx    : new.target (checked to be constructor or undefined)
   //  -- esp[4] : arguments list (a FixedArray)
   //  -- esp[0] : return address.
   // -----------------------------------
 
-  // We need to preserve eax, edi, esi and ebx.
-  __ movd(xmm0, edx);
-  __ movd(xmm1, edi);
-  __ movd(xmm2, eax);
+  __ movd(xmm0, edx);  // Spill new.target.
+  __ movd(xmm1, edi);  // Spill target.
   __ movd(xmm3, esi);  // Spill the context.
 
   const Register kArgumentsList = esi;
@@ -2157,32 +2227,15 @@ void Builtins::Generate_CallOrConstructVarargs(MacroAssembler* masm,
   __ StackOverflowCheck(kArgumentsLength, edx, &stack_overflow);
 
   __ movd(xmm4, kArgumentsList);  // Spill the arguments list.
-
   // Move the arguments already in the stack,
   // including the receiver and the return address.
-  {
-    Label copy, check;
-    Register src = edx, current = edi, tmp = esi;
-    // Update stack pointer.
-    __ mov(src, esp);
-    __ lea(tmp, Operand(kArgumentsLength, times_system_pointer_size, 0));
-    __ AllocateStackSpace(tmp);
-    // Include return address and receiver.
-    __ add(eax, Immediate(2));
-    __ mov(current, Immediate(0));
-    __ jmp(&check);
-    // Loop.
-    __ bind(&copy);
-    __ mov(tmp, Operand(src, current, times_system_pointer_size, 0));
-    __ mov(Operand(esp, current, times_system_pointer_size, 0), tmp);
-    __ inc(current);
-    __ bind(&check);
-    __ cmp(current, eax);
-    __ j(less, &copy);
-    __ lea(edx, Operand(esp, eax, times_system_pointer_size, 0));
-  }
-
+  // kArgumentsLength (ecx): Number of arguments to make room for.
+  // eax: Number of arguments already on the stack.
+  // edx: Points to first free slot on the stack after arguments were shifted.
+  Generate_AllocateSpaceAndShiftExistingArguments(masm, kArgumentsLength, eax,
+                                                  edx, edi, esi);
   __ movd(kArgumentsList, xmm4);  // Recover arguments list.
+  __ movd(xmm2, eax);             // Spill argument count.
 
   // Push additional arguments onto the stack.
   {
@@ -2207,12 +2260,9 @@ void Builtins::Generate_CallOrConstructVarargs(MacroAssembler* masm,
 
   // Restore eax, edi and edx.
   __ movd(esi, xmm3);  // Restore the context.
-  __ movd(eax, xmm2);
-  __ movd(edi, xmm1);
-  __ movd(edx, xmm0);
-
-  // Compute the actual parameter count.
-  __ add(eax, kArgumentsLength);
+  __ movd(eax, xmm2);  // Restore argument count.
+  __ movd(edi, xmm1);  // Restore target.
+  __ movd(edx, xmm0);  // Restore new.target.
 
   // Tail-call to the actual Call or Construct builtin.
   __ Jump(code, RelocInfo::CODE_TARGET);
@@ -2284,31 +2334,11 @@ void Builtins::Generate_CallOrConstructForwardVarargs(MacroAssembler* masm,
 
     // Move the arguments already in the stack,
     // including the receiver and the return address.
-    {
-      Label copy, check;
-      Register src = esi, current = edi;
-      // Update stack pointer.
-      __ mov(src, esp);
-      __ lea(scratch, Operand(edx, times_system_pointer_size, 0));
-      __ AllocateStackSpace(scratch);
-      // Include return address and receiver.
-      __ add(eax, Immediate(2));
-      __ Move(current, 0);
-      __ jmp(&check);
-      // Loop.
-      __ bind(&copy);
-      __ mov(scratch, Operand(src, current, times_system_pointer_size, 0));
-      __ mov(Operand(esp, current, times_system_pointer_size, 0), scratch);
-      __ inc(current);
-      __ bind(&check);
-      __ cmp(current, eax);
-      __ j(less, &copy);
-      __ lea(esi, Operand(esp, eax, times_system_pointer_size, 0));
-    }
-
-    // Update total number of arguments.
-    __ sub(eax, Immediate(2));
-    __ add(eax, edx);
+    // edx: Number of arguments to make room for.
+    // eax: Number of arguments already on the stack.
+    // esi: Points to first free slot on the stack after arguments were shifted.
+    Generate_AllocateSpaceAndShiftExistingArguments(masm, edx, eax, esi, ebx,
+                                                    edi);
 
     // Point to the first argument to copy (skipping receiver).
     __ lea(ecx, Operand(ecx, times_system_pointer_size,

src/builtins/x64/builtins-x64.cc

@@ -83,6 +83,31 @@ static void GenerateTailCallToReturnedCode(
 
 namespace {
 
+enum class ArgumentsElementType {
+  kRaw,    // Push arguments as they are.
+  kHandle  // Dereference arguments before pushing.
+};
+
+void Generate_PushArguments(MacroAssembler* masm, Register array, Register argc,
+                            Register scratch,
+                            ArgumentsElementType element_type) {
+  DCHECK(!AreAliased(array, argc, scratch, kScratchRegister));
+  Register counter = scratch;
+  Label loop, entry;
+  __ movq(counter, argc);
+  __ jmp(&entry);
+  __ bind(&loop);
+  Operand value(array, counter, times_system_pointer_size, 0);
+  if (element_type == ArgumentsElementType::kHandle) {
+    __ movq(kScratchRegister, value);
+    value = Operand(kScratchRegister, 0);
+  }
+  __ Push(value);
+  __ bind(&entry);
+  __ decq(counter);
+  __ j(greater_equal, &loop, Label::kNear);
+}
+
 void Generate_JSBuiltinsConstructStubHelper(MacroAssembler* masm) {
   // ----------- S t a t e -------------
   //  -- rax: number of arguments
@@ -112,7 +137,9 @@ void Generate_JSBuiltinsConstructStubHelper(MacroAssembler* masm) {
     __ leaq(rbx, Operand(rbp, StandardFrameConstants::kCallerSPOffset +
                                   kSystemPointerSize));
     // Copy arguments to the expression stack.
-    __ PushArray(rbx, rax, rcx);
+    // rbx: Pointer to start of arguments.
+    // rax: Number of arguments.
+    Generate_PushArguments(masm, rbx, rax, rcx, ArgumentsElementType::kRaw);
     // The receiver for the builtin/api call.
     __ PushRoot(RootIndex::kTheHoleValue);
 
@@ -236,7 +263,9 @@ void Builtins::Generate_JSConstructStubGeneric(MacroAssembler* masm) {
   // InvokeFunction.
 
   // Copy arguments to the expression stack.
-  __ PushArray(rbx, rax, rcx);
+  // rbx: Pointer to start of arguments.
+  // rax: Number of arguments.
+  Generate_PushArguments(masm, rbx, rax, rcx, ArgumentsElementType::kRaw);
 
   // Push implicit receiver.
   __ Push(r8);
@@ -607,18 +636,12 @@ static void Generate_JSEntryTrampolineHelper(MacroAssembler* masm,
 
     __ bind(&enough_stack_space);
 
-    // Copy arguments to the stack in a loop.
+    // Copy arguments to the stack.
     // Register rbx points to array of pointers to handle locations.
     // Push the values of these handles.
-    Label loop, entry;
-    __ movq(rcx, rax);
-    __ jmp(&entry, Label::kNear);
-    __ bind(&loop);
-    __ movq(kScratchRegister, Operand(rbx, rcx, times_system_pointer_size, 0));
-    __ Push(Operand(kScratchRegister, 0));  // dereference handle
-    __ bind(&entry);
-    __ decq(rcx);
-    __ j(greater_equal, &loop, Label::kNear);
+    // rbx: Pointer to start of arguments.
+    // rax: Number of arguments.
+    Generate_PushArguments(masm, rbx, rax, rcx, ArgumentsElementType::kHandle);
 
     // Push the receiver.
     __ Push(r9);
@@ -2076,6 +2099,58 @@ void Builtins::Generate_ReflectConstruct(MacroAssembler* masm) {
           RelocInfo::CODE_TARGET);
 }
 
+namespace {
+
+// Allocate new stack space for |count| arguments and shift all existing
+// arguments already on the stack. |pointer_to_new_space_out| points to the
+// first free slot on the stack to copy additional arguments to and
+// |argc_in_out| is updated to include |count|.
+void Generate_AllocateSpaceAndShiftExistingArguments(
+    MacroAssembler* masm, Register count, Register argc_in_out,
+    Register pointer_to_new_space_out, Register scratch1, Register scratch2) {
+  DCHECK(!AreAliased(count, argc_in_out, pointer_to_new_space_out, scratch1,
+                     scratch2, kScratchRegister));
+  // Use pointer_to_new_space_out as scratch until we set it to the correct
+  // value at the end.
+  Register old_rsp = pointer_to_new_space_out;
+  Register new_space = kScratchRegister;
+  __ movq(old_rsp, rsp);
+
+  __ leaq(new_space, Operand(count, times_system_pointer_size, 0));
+  __ AllocateStackSpace(new_space);
+
+  Register copy_count = scratch1;
+  // We have a spare register, so use it instead of clobbering argc.
+  // lea + add (to add the count to argc in the end) uses 1 less byte than
+  // inc + lea (with base, index and disp), at the cost of 1 extra register.
+  __ leaq(copy_count, Operand(argc_in_out, 1));  // Include the receiver.
+  Register current = scratch2;
+  Register value = kScratchRegister;
+
+  Label loop, entry;
+  __ Move(current, 0);
+  __ jmp(&entry);
+  __ bind(&loop);
+  __ movq(value, Operand(old_rsp, current, times_system_pointer_size, 0));
+  __ movq(Operand(rsp, current, times_system_pointer_size, 0), value);
+  __ incq(current);
+  __ bind(&entry);
+  __ cmpq(current, copy_count);
+  __ j(less_equal, &loop, Label::kNear);
+
+  // Point to the next free slot above the shifted arguments (copy_count + 1
+  // slot for the return address).
+  __ leaq(
+      pointer_to_new_space_out,
+      Operand(rsp, copy_count, times_system_pointer_size, kSystemPointerSize));
+  // We use addl instead of addq here because we can omit REX.W, saving 1 byte.
+  // We are especially constrained here because we are close to reaching the
+  // limit for a near jump to the stackoverflow label, so every byte counts.
+  __ addl(argc_in_out, count);  // Update total number of arguments.
+}
+
+}  // namespace
+
 // static
 // TODO(v8:11615): Observe Code::kMaxArguments in CallOrConstructVarargs
 void Builtins::Generate_CallOrConstructVarargs(MacroAssembler* masm,
@@ -2114,28 +2189,10 @@ void Builtins::Generate_CallOrConstructVarargs(MacroAssembler* masm,
   // Push additional arguments onto the stack.
   // Move the arguments already in the stack,
   // including the receiver and the return address.
-  {
-    Label copy, check;
-    Register src = r8, dest = rsp, num = r9, current = r12;
-    __ movq(src, rsp);
-    __ leaq(kScratchRegister, Operand(rcx, times_system_pointer_size, 0));
-    __ AllocateStackSpace(kScratchRegister);
-    __ leaq(num, Operand(rax, 2));  // Number of words to copy.
-                                    // +2 for receiver and return address.
-    __ Move(current, 0);
-    __ jmp(&check);
-    __ bind(&copy);
-    __ movq(kScratchRegister,
-            Operand(src, current, times_system_pointer_size, 0));
-    __ movq(Operand(dest, current, times_system_pointer_size, 0),
-            kScratchRegister);
-    __ incq(current);
-    __ bind(&check);
-    __ cmpq(current, num);
-    __ j(less, &copy);
-    __ leaq(r8, Operand(rsp, num, times_system_pointer_size, 0));
-  }
-
+  // rcx: Number of arguments to make room for.
+  // rax: Number of arguments already on the stack.
+  // r8: Points to first free slot on the stack after arguments were shifted.
+  Generate_AllocateSpaceAndShiftExistingArguments(masm, rcx, rax, r8, r9, r12);
   // Copy the additional arguments onto the stack.
   {
     Register value = r12;
@@ -2156,7 +2213,6 @@ void Builtins::Generate_CallOrConstructVarargs(MacroAssembler* masm,
     __ incl(current);
     __ jmp(&loop);
     __ bind(&done);
-    __ addq(rax, current);
   }
 
   // Tail-call to the actual Call or Construct builtin.
@@ -2216,29 +2272,11 @@ void Builtins::Generate_CallOrConstructForwardVarargs(MacroAssembler* masm,
     // Forward the arguments from the caller frame.
     // Move the arguments already in the stack,
     // including the receiver and the return address.
-    {
-      Label copy, check;
-      Register src = r9, dest = rsp, num = r12, current = r15;
-      __ movq(src, rsp);
-      __ leaq(kScratchRegister, Operand(r8, times_system_pointer_size, 0));
-      __ AllocateStackSpace(kScratchRegister);
-      __ leaq(num, Operand(rax, 2));  // Number of words to copy.
-                                      // +2 for receiver and return address.
-      __ Move(current, 0);
-      __ jmp(&check);
-      __ bind(&copy);
-      __ movq(kScratchRegister,
-              Operand(src, current, times_system_pointer_size, 0));
-      __ movq(Operand(dest, current, times_system_pointer_size, 0),
-              kScratchRegister);
-      __ incq(current);
-      __ bind(&check);
-      __ cmpq(current, num);
-      __ j(less, &copy);
-      __ leaq(r9, Operand(rsp, num, times_system_pointer_size, 0));
-    }
-
-    __ addl(rax, r8);  // Update total number of arguments.
+    // r8: Number of arguments to make room for.
+    // rax: Number of arguments already on the stack.
+    // r9: Points to first free slot on the stack after arguments were shifted.
+    Generate_AllocateSpaceAndShiftExistingArguments(masm, r8, rax, r9, r12,
+                                                    r15);
 
     // Point to the first argument to copy (skipping receiver).
     __ leaq(rcx, Operand(rcx, times_system_pointer_size,