macro-assembler-arm64.cc 111 KB
Newer Older
1
// Copyright 2013 the V8 project authors. All rights reserved.
2 3
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
4

5
#if V8_TARGET_ARCH_ARM64
6

7
#include "src/base/bits.h"
8
#include "src/base/division-by-constant.h"
9
#include "src/codegen/assembler.h"
10
#include "src/codegen/callable.h"
11
#include "src/codegen/code-factory.h"
12
#include "src/codegen/external-reference-table.h"
13 14
#include "src/codegen/macro-assembler-inl.h"
#include "src/codegen/register-configuration.h"
15
#include "src/debug/debug.h"
16
#include "src/deoptimizer/deoptimizer.h"
17 18
#include "src/execution/frame-constants.h"
#include "src/execution/frames-inl.h"
19
#include "src/heap/heap-inl.h"  // For MemoryChunk.
20
#include "src/init/bootstrapper.h"
21
#include "src/logging/counters.h"
22
#include "src/runtime/runtime.h"
23
#include "src/snapshot/embedded/embedded-data.h"
24
#include "src/snapshot/snapshot.h"
25
#include "src/wasm/wasm-code-manager.h"
26

27 28 29
// Satisfy cpplint check, but don't include platform-specific header. It is
// included recursively via macro-assembler.h.
#if 0
30
#include "src/codegen/arm64/macro-assembler-arm64.h"
31
#endif
32

33 34 35
namespace v8 {
namespace internal {

36
CPURegList TurboAssembler::DefaultTmpList() { return CPURegList(ip0, ip1); }
37

38
CPURegList TurboAssembler::DefaultFPTmpList() {
39 40 41
  return CPURegList(fp_scratch1, fp_scratch2);
}

42
int TurboAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
43
                                                    Register exclusion) const {
44
  auto list = kCallerSaved;
45 46
  list.Remove(exclusion);
  list.Align();
47

48
  int bytes = list.Count() * kXRegSizeInBits / 8;
49 50

  if (fp_mode == kSaveFPRegs) {
51
    DCHECK_EQ(kCallerSavedV.Count() % 2, 0);
52 53 54 55 56
    bytes += kCallerSavedV.Count() * kDRegSizeInBits / 8;
  }
  return bytes;
}

57 58
int TurboAssembler::PushCallerSaved(SaveFPRegsMode fp_mode,
                                    Register exclusion) {
59
  auto list = kCallerSaved;
60 61
  list.Remove(exclusion);
  list.Align();
62

63
  PushCPURegList(list);
64 65

  int bytes = list.Count() * kXRegSizeInBits / 8;
66 67

  if (fp_mode == kSaveFPRegs) {
68
    DCHECK_EQ(kCallerSavedV.Count() % 2, 0);
69
    PushCPURegList(kCallerSavedV);
70
    bytes += kCallerSavedV.Count() * kDRegSizeInBits / 8;
71
  }
72
  return bytes;
73 74
}

75
int TurboAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion) {
76
  int bytes = 0;
77
  if (fp_mode == kSaveFPRegs) {
78
    DCHECK_EQ(kCallerSavedV.Count() % 2, 0);
79
    PopCPURegList(kCallerSavedV);
80
    bytes += kCallerSavedV.Count() * kDRegSizeInBits / 8;
81 82 83
  }

  auto list = kCallerSaved;
84 85
  list.Remove(exclusion);
  list.Align();
86

87
  PopCPURegList(list);
88 89 90
  bytes += list.Count() * kXRegSizeInBits / 8;

  return bytes;
91 92
}

93 94
void TurboAssembler::LogicalMacro(const Register& rd, const Register& rn,
                                  const Operand& operand, LogicalOp op) {
95 96
  UseScratchRegisterScope temps(this);

97
  if (operand.NeedsRelocation(this)) {
98
    Register temp = temps.AcquireX();
99
    Ldr(temp, operand.immediate());
100
    Logical(rd, rn, temp, op);
101 102

  } else if (operand.IsImmediate()) {
103
    int64_t immediate = operand.ImmediateValue();
104 105 106 107 108 109 110 111
    unsigned reg_size = rd.SizeInBits();

    // If the operation is NOT, invert the operation and immediate.
    if ((op & NOT) == NOT) {
      op = static_cast<LogicalOp>(op & ~NOT);
      immediate = ~immediate;
    }

112 113 114
    // Ignore the top 32 bits of an immediate if we're moving to a W register.
    if (rd.Is32Bits()) {
      // Check that the top 32 bits are consistent.
115
      DCHECK(((immediate >> kWRegSizeInBits) == 0) ||
116 117 118 119
             ((immediate >> kWRegSizeInBits) == -1));
      immediate &= kWRegMask;
    }

120
    DCHECK(rd.Is64Bits() || is_uint32(immediate));
121

122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138
    // Special cases for all set or all clear immediates.
    if (immediate == 0) {
      switch (op) {
        case AND:
          Mov(rd, 0);
          return;
        case ORR:  // Fall through.
        case EOR:
          Mov(rd, rn);
          return;
        case ANDS:  // Fall through.
        case BICS:
          break;
        default:
          UNREACHABLE();
      }
    } else if ((rd.Is64Bits() && (immediate == -1L)) ||
139
               (rd.Is32Bits() && (immediate == 0xFFFFFFFFL))) {
140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163
      switch (op) {
        case AND:
          Mov(rd, rn);
          return;
        case ORR:
          Mov(rd, immediate);
          return;
        case EOR:
          Mvn(rd, rn);
          return;
        case ANDS:  // Fall through.
        case BICS:
          break;
        default:
          UNREACHABLE();
      }
    }

    unsigned n, imm_s, imm_r;
    if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
      // Immediate can be encoded in the instruction.
      LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
    } else {
      // Immediate can't be encoded: synthesize using move immediate.
164
      Register temp = temps.AcquireSameSizeAs(rn);
165 166 167

      // If the left-hand input is the stack pointer, we can't pre-shift the
      // immediate, as the encoding won't allow the subsequent post shift.
168
      PreShiftImmMode mode = rn == sp ? kNoShift : kAnyShift;
169 170
      Operand imm_operand = MoveImmediateForShiftedOp(temp, immediate, mode);

171
      if (rd.IsSP()) {
172 173
        // If rd is the stack pointer we cannot use it as the destination
        // register so we use the temp register as an intermediate again.
174
        Logical(temp, rn, imm_operand, op);
175
        Mov(sp, temp);
176
      } else {
177
        Logical(rd, rn, imm_operand, op);
178 179 180 181
      }
    }

  } else if (operand.IsExtendedRegister()) {
182
    DCHECK(operand.reg().SizeInBits() <= rd.SizeInBits());
183 184
    // Add/sub extended supports shift <= 4. We want to support exactly the
    // same modes here.
185
    DCHECK_LE(operand.shift_amount(), 4);
186
    DCHECK(operand.reg().Is64Bits() ||
187
           ((operand.extend() != UXTX) && (operand.extend() != SXTX)));
188
    Register temp = temps.AcquireSameSizeAs(rn);
189 190 191 192 193 194
    EmitExtendShift(temp, operand.reg(), operand.extend(),
                    operand.shift_amount());
    Logical(rd, rn, temp, op);

  } else {
    // The operand can be encoded in the instruction.
195
    DCHECK(operand.IsShiftedRegister());
196 197 198 199
    Logical(rd, rn, operand, op);
  }
}

200 201
void TurboAssembler::Mov(const Register& rd, uint64_t imm) {
  DCHECK(allow_macro_instructions());
202 203
  DCHECK(is_uint32(imm) || is_int32(imm) || rd.Is64Bits());
  DCHECK(!rd.IsZero());
204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221

  // TODO(all) extend to support more immediates.
  //
  // Immediates on Aarch64 can be produced using an initial value, and zero to
  // three move keep operations.
  //
  // Initial values can be generated with:
  //  1. 64-bit move zero (movz).
  //  2. 32-bit move inverted (movn).
  //  3. 64-bit move inverted.
  //  4. 32-bit orr immediate.
  //  5. 64-bit orr immediate.
  // Move-keep may then be used to modify each of the 16-bit half-words.
  //
  // The code below supports all five initial value generators, and
  // applying move-keep operations to move-zero and move-inverted initial
  // values.

222 223 224 225 226
  // Try to move the immediate in one instruction, and if that fails, switch to
  // using multiple instructions.
  if (!TryOneInstrMoveImmediate(rd, imm)) {
    unsigned reg_size = rd.SizeInBits();

227 228 229
    // Generic immediate case. Imm will be represented by
    //   [imm3, imm2, imm1, imm0], where each imm is 16 bits.
    // A move-zero or move-inverted is generated for the first non-zero or
230
    // non-0xFFFF immX, and a move-keep for subsequent non-zero immX.
231 232 233

    uint64_t ignored_halfword = 0;
    bool invert_move = false;
234
    // If the number of 0xFFFF halfwords is greater than the number of 0x0000
235 236 237
    // halfwords, it's more efficient to use move-inverted.
    if (CountClearHalfWords(~imm, reg_size) >
        CountClearHalfWords(imm, reg_size)) {
238
      ignored_halfword = 0xFFFFL;
239 240 241
      invert_move = true;
    }

242 243 244 245
    // Mov instructions can't move immediate values into the stack pointer, so
    // set up a temporary register, if needed.
    UseScratchRegisterScope temps(this);
    Register temp = rd.IsSP() ? temps.AcquireSameSizeAs(rd) : rd;
246 247 248

    // Iterate through the halfwords. Use movn/movz for the first non-ignored
    // halfword, and movk for subsequent halfwords.
249
    DCHECK_EQ(reg_size % 16, 0);
250
    bool first_mov_done = false;
251
    for (int i = 0; i < (rd.SizeInBits() / 16); i++) {
252
      uint64_t imm16 = (imm >> (16 * i)) & 0xFFFFL;
253 254 255
      if (imm16 != ignored_halfword) {
        if (!first_mov_done) {
          if (invert_move) {
256
            movn(temp, (~imm16) & 0xFFFFL, 16 * i);
257 258 259 260 261 262 263 264 265 266
          } else {
            movz(temp, imm16, 16 * i);
          }
          first_mov_done = true;
        } else {
          // Construct a wider constant.
          movk(temp, imm16, 16 * i);
        }
      }
    }
267
    DCHECK(first_mov_done);
268 269 270 271 272 273 274 275 276

    // Move the temporary if the original destination register was the stack
    // pointer.
    if (rd.IsSP()) {
      mov(rd, temp);
    }
  }
}

277
void TurboAssembler::Mov(const Register& rd, const Operand& operand,
278
                         DiscardMoveMode discard_mode) {
279
  DCHECK(allow_macro_instructions());
280
  DCHECK(!rd.IsZero());
281

282 283
  // Provide a swap register for instructions that need to write into the
  // system stack pointer (and can't do this inherently).
284 285
  UseScratchRegisterScope temps(this);
  Register dst = (rd.IsSP()) ? temps.AcquireSameSizeAs(rd) : rd;
286

287
  if (operand.NeedsRelocation(this)) {
288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
    // TODO(jgruber,v8:8887): Also consider a root-relative load when generating
    // non-isolate-independent code. In many cases it might be cheaper than
    // embedding the relocatable value.
    if (root_array_available_ && options().isolate_independent_code) {
      if (operand.ImmediateRMode() == RelocInfo::EXTERNAL_REFERENCE) {
        Address addr = static_cast<Address>(operand.ImmediateValue());
        ExternalReference reference = bit_cast<ExternalReference>(addr);
        IndirectLoadExternalReference(rd, reference);
        return;
      } else if (RelocInfo::IsEmbeddedObjectMode(operand.ImmediateRMode())) {
        Handle<HeapObject> x(
            reinterpret_cast<Address*>(operand.ImmediateValue()));
        // TODO(v8:9706): Fix-it! This load will always uncompress the value
        // even when we are loading a compressed embedded object.
        IndirectLoadConstant(rd.X(), x);
        return;
304 305
      }
    }
306
    Ldr(dst, operand);
307 308
  } else if (operand.IsImmediate()) {
    // Call the macro assembler for generic immediates.
309
    Mov(dst, operand.ImmediateValue());
310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328
  } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) {
    // Emit a shift instruction if moving a shifted register. This operation
    // could also be achieved using an orr instruction (like orn used by Mvn),
    // but using a shift instruction makes the disassembly clearer.
    EmitShift(dst, operand.reg(), operand.shift(), operand.shift_amount());
  } else if (operand.IsExtendedRegister()) {
    // Emit an extend instruction if moving an extended register. This handles
    // extend with post-shift operations, too.
    EmitExtendShift(dst, operand.reg(), operand.extend(),
                    operand.shift_amount());
  } else {
    // Otherwise, emit a register move only if the registers are distinct, or
    // if they are not X registers.
    //
    // Note that mov(w0, w0) is not a no-op because it clears the top word of
    // x0. A flag is provided (kDiscardForSameWReg) if a move between the same W
    // registers is not required to clear the top word of the X register. In
    // this case, the instruction is discarded.
    //
329
    // If sp is an operand, add #0 is emitted, otherwise, orr #0.
330
    if (rd != operand.reg() ||
331
        (rd.Is32Bits() && (discard_mode == kDontDiscardForSameWReg))) {
332 333 334 335 336 337 338
      Assembler::mov(rd, operand.reg());
    }
    // This case can handle writes into the system stack pointer directly.
    dst = rd;
  }

  // Copy the result to the system stack pointer.
339
  if (dst != rd) {
340
    DCHECK(rd.IsSP());
341 342 343 344
    Assembler::mov(rd, dst);
  }
}

345 346 347 348
void TurboAssembler::Mov(const Register& rd, Smi smi) {
  return Mov(rd, Operand(smi));
}

349
void TurboAssembler::Movi16bitHelper(const VRegister& vd, uint64_t imm) {
350
  DCHECK(is_uint16(imm));
351 352
  int byte1 = (imm & 0xFF);
  int byte2 = ((imm >> 8) & 0xFF);
353 354 355 356 357 358
  if (byte1 == byte2) {
    movi(vd.Is64Bits() ? vd.V8B() : vd.V16B(), byte1);
  } else if (byte1 == 0) {
    movi(vd, byte2, LSL, 8);
  } else if (byte2 == 0) {
    movi(vd, byte1);
359 360 361 362
  } else if (byte1 == 0xFF) {
    mvni(vd, ~byte2 & 0xFF, LSL, 8);
  } else if (byte2 == 0xFF) {
    mvni(vd, ~byte1 & 0xFF);
363 364 365 366 367 368 369 370
  } else {
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireW();
    movz(temp, imm);
    dup(vd, temp);
  }
}

371
void TurboAssembler::Movi32bitHelper(const VRegister& vd, uint64_t imm) {
372 373 374 375 376
  DCHECK(is_uint32(imm));

  uint8_t bytes[sizeof(imm)];
  memcpy(bytes, &imm, sizeof(imm));

377
  // All bytes are either 0x00 or 0xFF.
378 379 380
  {
    bool all0orff = true;
    for (int i = 0; i < 4; ++i) {
381
      if ((bytes[i] != 0) && (bytes[i] != 0xFF)) {
382 383 384 385 386 387 388 389 390 391 392 393 394
        all0orff = false;
        break;
      }
    }

    if (all0orff == true) {
      movi(vd.Is64Bits() ? vd.V1D() : vd.V2D(), ((imm << 32) | imm));
      return;
    }
  }

  // Of the 4 bytes, only one byte is non-zero.
  for (int i = 0; i < 4; i++) {
395
    if ((imm & (0xFF << (i * 8))) == imm) {
396 397 398 399 400
      movi(vd, bytes[i], LSL, i * 8);
      return;
    }
  }

401
  // Of the 4 bytes, only one byte is not 0xFF.
402
  for (int i = 0; i < 4; i++) {
403
    uint32_t mask = ~(0xFF << (i * 8));
404
    if ((imm & mask) == mask) {
405
      mvni(vd, ~bytes[i] & 0xFF, LSL, i * 8);
406 407 408 409 410
      return;
    }
  }

  // Immediate is of the form 0x00MMFFFF.
411
  if ((imm & 0xFF00FFFF) == 0x0000FFFF) {
412 413 414 415 416
    movi(vd, bytes[2], MSL, 16);
    return;
  }

  // Immediate is of the form 0x0000MMFF.
417
  if ((imm & 0xFFFF00FF) == 0x000000FF) {
418 419 420 421 422
    movi(vd, bytes[1], MSL, 8);
    return;
  }

  // Immediate is of the form 0xFFMM0000.
423 424
  if ((imm & 0xFF00FFFF) == 0xFF000000) {
    mvni(vd, ~bytes[2] & 0xFF, MSL, 16);
425 426 427
    return;
  }
  // Immediate is of the form 0xFFFFMM00.
428 429
  if ((imm & 0xFFFF00FF) == 0xFFFF0000) {
    mvni(vd, ~bytes[1] & 0xFF, MSL, 8);
430 431 432 433
    return;
  }

  // Top and bottom 16-bits are equal.
434 435
  if (((imm >> 16) & 0xFFFF) == (imm & 0xFFFF)) {
    Movi16bitHelper(vd.Is64Bits() ? vd.V4H() : vd.V8H(), imm & 0xFFFF);
436 437 438 439 440 441 442 443 444 445 446 447
    return;
  }

  // Default case.
  {
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireW();
    Mov(temp, imm);
    dup(vd, temp);
  }
}

448
void TurboAssembler::Movi64bitHelper(const VRegister& vd, uint64_t imm) {
449
  // All bytes are either 0x00 or 0xFF.
450 451 452
  {
    bool all0orff = true;
    for (int i = 0; i < 8; ++i) {
453 454
      int byteval = (imm >> (i * 8)) & 0xFF;
      if (byteval != 0 && byteval != 0xFF) {
455 456 457 458 459 460 461 462 463 464 465
        all0orff = false;
        break;
      }
    }
    if (all0orff == true) {
      movi(vd, imm);
      return;
    }
  }

  // Top and bottom 32-bits are equal.
466 467
  if (((imm >> 32) & 0xFFFFFFFF) == (imm & 0xFFFFFFFF)) {
    Movi32bitHelper(vd.Is64Bits() ? vd.V2S() : vd.V4S(), imm & 0xFFFFFFFF);
468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483
    return;
  }

  // Default case.
  {
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireX();
    Mov(temp, imm);
    if (vd.Is1D()) {
      mov(vd.D(), 0, temp);
    } else {
      dup(vd.V2D(), temp);
    }
  }
}

484
void TurboAssembler::Movi(const VRegister& vd, uint64_t imm, Shift shift,
485
                          int shift_amount) {
486
  DCHECK(allow_macro_instructions());
487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504
  if (shift_amount != 0 || shift != LSL) {
    movi(vd, imm, shift, shift_amount);
  } else if (vd.Is8B() || vd.Is16B()) {
    // 8-bit immediate.
    DCHECK(is_uint8(imm));
    movi(vd, imm);
  } else if (vd.Is4H() || vd.Is8H()) {
    // 16-bit immediate.
    Movi16bitHelper(vd, imm);
  } else if (vd.Is2S() || vd.Is4S()) {
    // 32-bit immediate.
    Movi32bitHelper(vd, imm);
  } else {
    // 64-bit immediate.
    Movi64bitHelper(vd, imm);
  }
}

505
void TurboAssembler::Movi(const VRegister& vd, uint64_t hi, uint64_t lo) {
506 507 508 509 510 511 512 513
  // TODO(all): Move 128-bit values in a more efficient way.
  DCHECK(vd.Is128Bits());
  UseScratchRegisterScope temps(this);
  Movi(vd.V2D(), lo);
  Register temp = temps.AcquireX();
  Mov(temp, hi);
  Ins(vd.V2D(), 1, temp);
}
514

515 516
void TurboAssembler::Mvn(const Register& rd, const Operand& operand) {
  DCHECK(allow_macro_instructions());
517

518
  if (operand.NeedsRelocation(this)) {
519
    Ldr(rd, operand.immediate());
520
    mvn(rd, rd);
521 522 523

  } else if (operand.IsImmediate()) {
    // Call the macro assembler for generic immediates.
524
    Mov(rd, ~operand.ImmediateValue());
525 526 527 528

  } else if (operand.IsExtendedRegister()) {
    // Emit two instructions for the extend case. This differs from Mov, as
    // the extend and invert can't be achieved in one instruction.
529
    EmitExtendShift(rd, operand.reg(), operand.extend(),
530
                    operand.shift_amount());
531
    mvn(rd, rd);
532 533 534 535 536 537

  } else {
    mvn(rd, operand);
  }
}

538
unsigned TurboAssembler::CountClearHalfWords(uint64_t imm, unsigned reg_size) {
539
  DCHECK_EQ(reg_size % 8, 0);
540 541
  int count = 0;
  for (unsigned i = 0; i < (reg_size / 16); i++) {
542
    if ((imm & 0xFFFF) == 0) {
543 544 545 546 547 548 549 550 551
      count++;
    }
    imm >>= 16;
  }
  return count;
}

// The movz instruction can generate immediates containing an arbitrary 16-bit
// half-word, with remaining bits clear, eg. 0x00001234, 0x0000123400000000.
552
bool TurboAssembler::IsImmMovz(uint64_t imm, unsigned reg_size) {
553
  DCHECK((reg_size == kXRegSizeInBits) || (reg_size == kWRegSizeInBits));
554 555 556 557
  return CountClearHalfWords(imm, reg_size) >= ((reg_size / 16) - 1);
}

// The movn instruction can generate immediates containing an arbitrary 16-bit
558
// half-word, with remaining bits set, eg. 0xFFFF1234, 0xFFFF1234FFFFFFFF.
559
bool TurboAssembler::IsImmMovn(uint64_t imm, unsigned reg_size) {
560 561 562
  return IsImmMovz(~imm, reg_size);
}

563
void TurboAssembler::ConditionalCompareMacro(const Register& rn,
564
                                             const Operand& operand,
565
                                             StatusFlags nzcv, Condition cond,
566
                                             ConditionalCompareOp op) {
567
  DCHECK((cond != al) && (cond != nv));
568
  if (operand.NeedsRelocation(this)) {
569 570
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireX();
571
    Ldr(temp, operand.immediate());
572
    ConditionalCompareMacro(rn, temp, nzcv, cond, op);
573 574

  } else if ((operand.IsShiftedRegister() && (operand.shift_amount() == 0)) ||
575 576
             (operand.IsImmediate() &&
              IsImmConditionalCompare(operand.ImmediateValue()))) {
577 578 579 580 581 582 583
    // The immediate can be encoded in the instruction, or the operand is an
    // unshifted register: call the assembler.
    ConditionalCompare(rn, operand, nzcv, cond, op);

  } else {
    // The operand isn't directly supported by the instruction: perform the
    // operation on a temporary register.
584 585
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireSameSizeAs(rn);
586 587 588 589 590
    Mov(temp, operand);
    ConditionalCompare(rn, temp, nzcv, cond, op);
  }
}

591 592
void TurboAssembler::Csel(const Register& rd, const Register& rn,
                          const Operand& operand, Condition cond) {
593
  DCHECK(allow_macro_instructions());
594 595
  DCHECK(!rd.IsZero());
  DCHECK((cond != al) && (cond != nv));
596 597 598
  if (operand.IsImmediate()) {
    // Immediate argument. Handle special cases of 0, 1 and -1 using zero
    // register.
599
    int64_t imm = operand.ImmediateValue();
600 601 602 603 604 605 606 607
    Register zr = AppropriateZeroRegFor(rn);
    if (imm == 0) {
      csel(rd, rn, zr, cond);
    } else if (imm == 1) {
      csinc(rd, rn, zr, cond);
    } else if (imm == -1) {
      csinv(rd, rn, zr, cond);
    } else {
608 609
      UseScratchRegisterScope temps(this);
      Register temp = temps.AcquireSameSizeAs(rn);
610
      Mov(temp, imm);
611 612 613 614 615 616 617
      csel(rd, rn, temp, cond);
    }
  } else if (operand.IsShiftedRegister() && (operand.shift_amount() == 0)) {
    // Unshifted register argument.
    csel(rd, rn, operand.reg(), cond);
  } else {
    // All other arguments.
618 619
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireSameSizeAs(rn);
620 621 622 623 624
    Mov(temp, operand);
    csel(rd, rn, temp, cond);
  }
}

625
bool TurboAssembler::TryOneInstrMoveImmediate(const Register& dst,
626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646
                                              int64_t imm) {
  unsigned n, imm_s, imm_r;
  int reg_size = dst.SizeInBits();
  if (IsImmMovz(imm, reg_size) && !dst.IsSP()) {
    // Immediate can be represented in a move zero instruction. Movz can't write
    // to the stack pointer.
    movz(dst, imm);
    return true;
  } else if (IsImmMovn(imm, reg_size) && !dst.IsSP()) {
    // Immediate can be represented in a move not instruction. Movn can't write
    // to the stack pointer.
    movn(dst, dst.Is64Bits() ? ~imm : (~imm & kWRegMask));
    return true;
  } else if (IsImmLogical(imm, reg_size, &n, &imm_s, &imm_r)) {
    // Immediate can be represented in a logical orr instruction.
    LogicalImmediate(dst, AppropriateZeroRegFor(dst), n, imm_s, imm_r, ORR);
    return true;
  }
  return false;
}

647
Operand TurboAssembler::MoveImmediateForShiftedOp(const Register& dst,
648 649
                                                  int64_t imm,
                                                  PreShiftImmMode mode) {
650 651 652 653 654 655
  int reg_size = dst.SizeInBits();
  // Encode the immediate in a single move instruction, if possible.
  if (TryOneInstrMoveImmediate(dst, imm)) {
    // The move was successful; nothing to do here.
  } else {
    // Pre-shift the immediate to the least-significant bits of the register.
656 657 658 659 660 661 662 663
    int shift_low;
    if (reg_size == 64) {
      shift_low = base::bits::CountTrailingZeros(imm);
    } else {
      DCHECK_EQ(reg_size, 32);
      shift_low = base::bits::CountTrailingZeros(static_cast<uint32_t>(imm));
    }

664 665 666 667 668 669 670
    if (mode == kLimitShiftForSP) {
      // When applied to the stack pointer, the subsequent arithmetic operation
      // can use the extend form to shift left by a maximum of four bits. Right
      // shifts are not allowed, so we filter them out later before the new
      // immediate is tested.
      shift_low = std::min(shift_low, 4);
    }
671 672 673 674 675 676 677 678
    int64_t imm_low = imm >> shift_low;

    // Pre-shift the immediate to the most-significant bits of the register. We
    // insert set bits in the least-significant bits, as this creates a
    // different immediate that may be encodable using movn or orr-immediate.
    // If this new immediate is encodable, the set bits will be eliminated by
    // the post shift on the following instruction.
    int shift_high = CountLeadingZeros(imm, reg_size);
679
    int64_t imm_high = (imm << shift_high) | ((INT64_C(1) << shift_high) - 1);
680

681
    if ((mode != kNoShift) && TryOneInstrMoveImmediate(dst, imm_low)) {
682 683 684
      // The new immediate has been moved into the destination's low bits:
      // return a new leftward-shifting operand.
      return Operand(dst, LSL, shift_low);
685
    } else if ((mode == kAnyShift) && TryOneInstrMoveImmediate(dst, imm_high)) {
686 687 688 689 690 691 692 693 694 695 696
      // The new immediate has been moved into the destination's high bits:
      // return a new rightward-shifting operand.
      return Operand(dst, LSR, shift_high);
    } else {
      // Use the generic move operation to set up the immediate.
      Mov(dst, imm);
    }
  }
  return Operand(dst);
}

697 698
void TurboAssembler::AddSubMacro(const Register& rd, const Register& rn,
                                 const Operand& operand, FlagsUpdate S,
699
                                 AddSubOp op) {
700
  if (operand.IsZero() && rd == rn && rd.Is64Bits() && rn.Is64Bits() &&
701
      !operand.NeedsRelocation(this) && (S == LeaveFlags)) {
702 703 704 705
    // The instruction would be a nop. Avoid generating useless code.
    return;
  }

706
  if (operand.NeedsRelocation(this)) {
707 708
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireX();
709
    Ldr(temp, operand.immediate());
710
    AddSubMacro(rd, rn, temp, S, op);
711
  } else if ((operand.IsImmediate() &&
712
              !IsImmAddSub(operand.ImmediateValue())) ||
713
             (rn.IsZero() && !operand.IsShiftedRegister()) ||
714
             (operand.IsShiftedRegister() && (operand.shift() == ROR))) {
715 716
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireSameSizeAs(rn);
717
    if (operand.IsImmediate()) {
718 719 720 721 722
      PreShiftImmMode mode = kAnyShift;

      // If the destination or source register is the stack pointer, we can
      // only pre-shift the immediate right by values supported in the add/sub
      // extend encoding.
723
      if (rd == sp) {
724 725 726
        // If the destination is SP and flags will be set, we can't pre-shift
        // the immediate at all.
        mode = (S == SetFlags) ? kNoShift : kLimitShiftForSP;
727
      } else if (rn == sp) {
728 729 730
        mode = kLimitShiftForSP;
      }

731
      Operand imm_operand =
732
          MoveImmediateForShiftedOp(temp, operand.ImmediateValue(), mode);
733 734 735 736 737
      AddSub(rd, rn, imm_operand, S, op);
    } else {
      Mov(temp, operand);
      AddSub(rd, rn, temp, S, op);
    }
738 739 740 741 742
  } else {
    AddSub(rd, rn, operand, S, op);
  }
}

743
void TurboAssembler::AddSubWithCarryMacro(const Register& rd,
744
                                          const Register& rn,
745
                                          const Operand& operand, FlagsUpdate S,
746
                                          AddSubWithCarryOp op) {
747
  DCHECK(rd.SizeInBits() == rn.SizeInBits());
748
  UseScratchRegisterScope temps(this);
749

750
  if (operand.NeedsRelocation(this)) {
751
    Register temp = temps.AcquireX();
752
    Ldr(temp, operand.immediate());
753
    AddSubWithCarryMacro(rd, rn, temp, S, op);
754 755 756 757

  } else if (operand.IsImmediate() ||
             (operand.IsShiftedRegister() && (operand.shift() == ROR))) {
    // Add/sub with carry (immediate or ROR shifted register.)
758
    Register temp = temps.AcquireSameSizeAs(rn);
759 760
    Mov(temp, operand);
    AddSubWithCarry(rd, rn, temp, S, op);
761

762 763
  } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) {
    // Add/sub with carry (shifted register).
764 765
    DCHECK(operand.reg().SizeInBits() == rd.SizeInBits());
    DCHECK(operand.shift() != ROR);
766 767 768
    DCHECK(is_uintn(operand.shift_amount(), rd.SizeInBits() == kXRegSizeInBits
                                                ? kXRegSizeInBitsLog2
                                                : kWRegSizeInBitsLog2));
769
    Register temp = temps.AcquireSameSizeAs(rn);
770 771 772 773 774
    EmitShift(temp, operand.reg(), operand.shift(), operand.shift_amount());
    AddSubWithCarry(rd, rn, temp, S, op);

  } else if (operand.IsExtendedRegister()) {
    // Add/sub with carry (extended register).
775
    DCHECK(operand.reg().SizeInBits() <= rd.SizeInBits());
776 777
    // Add/sub extended supports a shift <= 4. We want to support exactly the
    // same modes.
778
    DCHECK_LE(operand.shift_amount(), 4);
779
    DCHECK(operand.reg().Is64Bits() ||
780
           ((operand.extend() != UXTX) && (operand.extend() != SXTX)));
781
    Register temp = temps.AcquireSameSizeAs(rn);
782 783 784 785 786 787 788 789 790 791
    EmitExtendShift(temp, operand.reg(), operand.extend(),
                    operand.shift_amount());
    AddSubWithCarry(rd, rn, temp, S, op);

  } else {
    // The addressing mode is directly supported by the instruction.
    AddSubWithCarry(rd, rn, operand, S, op);
  }
}

792 793
void TurboAssembler::LoadStoreMacro(const CPURegister& rt,
                                    const MemOperand& addr, LoadStoreOp op) {
794
  int64_t offset = addr.offset();
795
  unsigned size = CalcLSDataSize(op);
796 797 798 799 800 801 802 803

  // Check if an immediate offset fits in the immediate field of the
  // appropriate instruction. If not, emit two instructions to perform
  // the operation.
  if (addr.IsImmediateOffset() && !IsImmLSScaled(offset, size) &&
      !IsImmLSUnscaled(offset)) {
    // Immediate offset that can't be encoded using unsigned or unscaled
    // addressing modes.
804 805
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireSameSizeAs(addr.base());
806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821
    Mov(temp, addr.offset());
    LoadStore(rt, MemOperand(addr.base(), temp), op);
  } else if (addr.IsPostIndex() && !IsImmLSUnscaled(offset)) {
    // Post-index beyond unscaled addressing range.
    LoadStore(rt, MemOperand(addr.base()), op);
    add(addr.base(), addr.base(), offset);
  } else if (addr.IsPreIndex() && !IsImmLSUnscaled(offset)) {
    // Pre-index beyond unscaled addressing range.
    add(addr.base(), addr.base(), offset);
    LoadStore(rt, MemOperand(addr.base()), op);
  } else {
    // Encodable in one load/store instruction.
    LoadStore(rt, addr, op);
  }
}

822
void TurboAssembler::LoadStorePairMacro(const CPURegister& rt,
823 824 825 826 827 828 829
                                        const CPURegister& rt2,
                                        const MemOperand& addr,
                                        LoadStorePairOp op) {
  // TODO(all): Should we support register offset for load-store-pair?
  DCHECK(!addr.IsRegisterOffset());

  int64_t offset = addr.offset();
830
  unsigned size = CalcLSPairDataSize(op);
831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854

  // Check if the offset fits in the immediate field of the appropriate
  // instruction. If not, emit two instructions to perform the operation.
  if (IsImmLSPair(offset, size)) {
    // Encodable in one load/store pair instruction.
    LoadStorePair(rt, rt2, addr, op);
  } else {
    Register base = addr.base();
    if (addr.IsImmediateOffset()) {
      UseScratchRegisterScope temps(this);
      Register temp = temps.AcquireSameSizeAs(base);
      Add(temp, base, offset);
      LoadStorePair(rt, rt2, MemOperand(temp), op);
    } else if (addr.IsPostIndex()) {
      LoadStorePair(rt, rt2, MemOperand(base), op);
      Add(base, base, offset);
    } else {
      DCHECK(addr.IsPreIndex());
      Add(base, base, offset);
      LoadStorePair(rt, rt2, MemOperand(base), op);
    }
  }
}

855 856
bool TurboAssembler::NeedExtraInstructionsOrRegisterBranch(
    Label* label, ImmBranchType b_type) {
857 858 859 860 861 862 863 864
  bool need_longer_range = false;
  // There are two situations in which we care about the offset being out of
  // range:
  //  - The label is bound but too far away.
  //  - The label is not bound but linked, and the previous branch
  //    instruction in the chain is too far away.
  if (label->is_bound() || label->is_linked()) {
    need_longer_range =
865
        !Instruction::IsValidImmPCOffset(b_type, label->pos() - pc_offset());
866 867 868
  }
  if (!need_longer_range && !label->is_bound()) {
    int max_reachable_pc = pc_offset() + Instruction::ImmBranchRange(b_type);
869 870
    unresolved_branches_.insert(std::pair<int, FarBranchInfo>(
        max_reachable_pc, FarBranchInfo(pc_offset(), label)));
871
    // Also maintain the next pool check.
872 873
    next_veneer_pool_check_ = Min(
        next_veneer_pool_check_, max_reachable_pc - kVeneerDistanceCheckMargin);
874 875 876 877
  }
  return need_longer_range;
}

878 879
void TurboAssembler::Adr(const Register& rd, Label* label, AdrHint hint) {
  DCHECK(allow_macro_instructions());
880
  DCHECK(!rd.IsZero());
881 882 883 884 885 886

  if (hint == kAdrNear) {
    adr(rd, label);
    return;
  }

887
  DCHECK_EQ(hint, kAdrFar);
888 889 890 891 892
  if (label->is_bound()) {
    int label_offset = label->pos() - pc_offset();
    if (Instruction::IsValidPCRelOffset(label_offset)) {
      adr(rd, label);
    } else {
893
      DCHECK_LE(label_offset, 0);
894 895 896 897 898
      int min_adr_offset = -(1 << (Instruction::ImmPCRelRangeBitwidth - 1));
      adr(rd, min_adr_offset);
      Add(rd, rd, label_offset - min_adr_offset);
    }
  } else {
899 900 901
    UseScratchRegisterScope temps(this);
    Register scratch = temps.AcquireX();

902 903
    InstructionAccurateScope scope(this,
                                   PatchingAssembler::kAdrFarPatchableNInstrs);
904 905 906 907 908 909 910 911
    adr(rd, label);
    for (int i = 0; i < PatchingAssembler::kAdrFarPatchableNNops; ++i) {
      nop(ADR_FAR_NOP);
    }
    movz(scratch, 0);
  }
}

912
void TurboAssembler::B(Label* label, BranchType type, Register reg, int bit) {
913
  DCHECK((reg == NoReg || type >= kBranchTypeFirstUsingReg) &&
914 915 916 917 918
         (bit == -1 || type >= kBranchTypeFirstUsingBit));
  if (kBranchTypeFirstCondition <= type && type <= kBranchTypeLastCondition) {
    B(static_cast<Condition>(type), label);
  } else {
    switch (type) {
919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935
      case always:
        B(label);
        break;
      case never:
        break;
      case reg_zero:
        Cbz(reg, label);
        break;
      case reg_not_zero:
        Cbnz(reg, label);
        break;
      case reg_bit_clear:
        Tbz(reg, bit, label);
        break;
      case reg_bit_set:
        Tbnz(reg, bit, label);
        break;
936 937 938 939 940 941
      default:
        UNREACHABLE();
    }
  }
}

942 943
void TurboAssembler::B(Label* label, Condition cond) {
  DCHECK(allow_macro_instructions());
944
  DCHECK((cond != al) && (cond != nv));
945 946 947

  Label done;
  bool need_extra_instructions =
948
      NeedExtraInstructionsOrRegisterBranch(label, CondBranchType);
949 950

  if (need_extra_instructions) {
951
    b(&done, NegateCondition(cond));
952
    B(label);
953 954 955 956 957 958
  } else {
    b(label, cond);
  }
  bind(&done);
}

959 960
void TurboAssembler::Tbnz(const Register& rt, unsigned bit_pos, Label* label) {
  DCHECK(allow_macro_instructions());
961 962 963

  Label done;
  bool need_extra_instructions =
964
      NeedExtraInstructionsOrRegisterBranch(label, TestBranchType);
965 966 967

  if (need_extra_instructions) {
    tbz(rt, bit_pos, &done);
968
    B(label);
969 970 971 972 973 974
  } else {
    tbnz(rt, bit_pos, label);
  }
  bind(&done);
}

975 976
void TurboAssembler::Tbz(const Register& rt, unsigned bit_pos, Label* label) {
  DCHECK(allow_macro_instructions());
977 978 979

  Label done;
  bool need_extra_instructions =
980
      NeedExtraInstructionsOrRegisterBranch(label, TestBranchType);
981 982 983

  if (need_extra_instructions) {
    tbnz(rt, bit_pos, &done);
984
    B(label);
985 986 987 988 989 990
  } else {
    tbz(rt, bit_pos, label);
  }
  bind(&done);
}

991 992
void TurboAssembler::Cbnz(const Register& rt, Label* label) {
  DCHECK(allow_macro_instructions());
993 994 995

  Label done;
  bool need_extra_instructions =
996
      NeedExtraInstructionsOrRegisterBranch(label, CompareBranchType);
997 998 999

  if (need_extra_instructions) {
    cbz(rt, &done);
1000
    B(label);
1001 1002 1003 1004 1005 1006
  } else {
    cbnz(rt, label);
  }
  bind(&done);
}

1007 1008
void TurboAssembler::Cbz(const Register& rt, Label* label) {
  DCHECK(allow_macro_instructions());
1009 1010 1011

  Label done;
  bool need_extra_instructions =
1012
      NeedExtraInstructionsOrRegisterBranch(label, CompareBranchType);
1013 1014 1015

  if (need_extra_instructions) {
    cbnz(rt, &done);
1016
    B(label);
1017 1018 1019 1020 1021 1022
  } else {
    cbz(rt, label);
  }
  bind(&done);
}

1023 1024
// Pseudo-instructions.

1025 1026 1027
void TurboAssembler::Abs(const Register& rd, const Register& rm,
                         Label* is_not_representable, Label* is_representable) {
  DCHECK(allow_macro_instructions());
1028
  DCHECK(AreSameSizeAndType(rd, rm));
1029 1030 1031 1032 1033 1034 1035

  Cmp(rm, 1);
  Cneg(rd, rm, lt);

  // If the comparison sets the v flag, the input was the smallest value
  // representable by rm, and the mathematical result of abs(rm) is not
  // representable using two's complement.
1036
  if ((is_not_representable != nullptr) && (is_representable != nullptr)) {
1037 1038
    B(is_not_representable, vs);
    B(is_representable);
1039
  } else if (is_not_representable != nullptr) {
1040
    B(is_not_representable, vs);
1041
  } else if (is_representable != nullptr) {
1042 1043 1044 1045 1046 1047
    B(is_representable, vc);
  }
}

// Abstracted stack operations.

1048
void TurboAssembler::Push(const CPURegister& src0, const CPURegister& src1,
1049
                          const CPURegister& src2, const CPURegister& src3) {
1050
  DCHECK(AreSameSizeAndType(src0, src1, src2, src3));
1051

1052
  int count = 1 + src1.is_valid() + src2.is_valid() + src3.is_valid();
1053
  int size = src0.SizeInBytes();
1054
  DCHECK_EQ(0, (size * count) % 16);
1055 1056 1057 1058

  PushHelper(count, size, src0, src1, src2, src3);
}

1059
void TurboAssembler::Push(const CPURegister& src0, const CPURegister& src1,
1060 1061 1062
                          const CPURegister& src2, const CPURegister& src3,
                          const CPURegister& src4, const CPURegister& src5,
                          const CPURegister& src6, const CPURegister& src7) {
1063
  DCHECK(AreSameSizeAndType(src0, src1, src2, src3, src4, src5, src6, src7));
1064

1065
  int count = 5 + src5.is_valid() + src6.is_valid() + src6.is_valid();
1066
  int size = src0.SizeInBytes();
1067
  DCHECK_EQ(0, (size * count) % 16);
1068 1069 1070 1071 1072

  PushHelper(4, size, src0, src1, src2, src3);
  PushHelper(count - 4, size, src4, src5, src6, src7);
}

1073
void TurboAssembler::Pop(const CPURegister& dst0, const CPURegister& dst1,
1074 1075 1076
                         const CPURegister& dst2, const CPURegister& dst3) {
  // It is not valid to pop into the same register more than once in one
  // instruction, not even into the zero register.
1077 1078
  DCHECK(!AreAliased(dst0, dst1, dst2, dst3));
  DCHECK(AreSameSizeAndType(dst0, dst1, dst2, dst3));
1079
  DCHECK(dst0.is_valid());
1080

1081
  int count = 1 + dst1.is_valid() + dst2.is_valid() + dst3.is_valid();
1082
  int size = dst0.SizeInBytes();
1083
  DCHECK_EQ(0, (size * count) % 16);
1084 1085 1086 1087

  PopHelper(count, size, dst0, dst1, dst2, dst3);
}

1088
void TurboAssembler::Pop(const CPURegister& dst0, const CPURegister& dst1,
1089 1090 1091 1092 1093 1094 1095
                         const CPURegister& dst2, const CPURegister& dst3,
                         const CPURegister& dst4, const CPURegister& dst5,
                         const CPURegister& dst6, const CPURegister& dst7) {
  // It is not valid to pop into the same register more than once in one
  // instruction, not even into the zero register.
  DCHECK(!AreAliased(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7));
  DCHECK(AreSameSizeAndType(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7));
1096
  DCHECK(dst0.is_valid());
1097

1098
  int count = 5 + dst5.is_valid() + dst6.is_valid() + dst7.is_valid();
1099
  int size = dst0.SizeInBytes();
1100
  DCHECK_EQ(0, (size * count) % 16);
1101 1102 1103 1104 1105

  PopHelper(4, size, dst0, dst1, dst2, dst3);
  PopHelper(count - 4, size, dst4, dst5, dst6, dst7);
}

1106
void TurboAssembler::Push(const Register& src0, const VRegister& src1) {
1107
  int size = src0.SizeInBytes() + src1.SizeInBytes();
1108
  DCHECK_EQ(0, size % 16);
1109 1110

  // Reserve room for src0 and push src1.
1111
  str(src1, MemOperand(sp, -size, PreIndex));
1112
  // Fill the gap with src0.
1113
  str(src0, MemOperand(sp, src1.SizeInBytes()));
1114 1115
}

1116
void TurboAssembler::PushCPURegList(CPURegList registers) {
1117
  int size = registers.RegisterSizeInBytes();
1118
  DCHECK_EQ(0, (size * registers.Count()) % 16);
1119

1120
  // Push up to four registers at a time.
1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131
  while (!registers.IsEmpty()) {
    int count_before = registers.Count();
    const CPURegister& src0 = registers.PopHighestIndex();
    const CPURegister& src1 = registers.PopHighestIndex();
    const CPURegister& src2 = registers.PopHighestIndex();
    const CPURegister& src3 = registers.PopHighestIndex();
    int count = count_before - registers.Count();
    PushHelper(count, size, src0, src1, src2, src3);
  }
}

1132
void TurboAssembler::PopCPURegList(CPURegList registers) {
1133
  int size = registers.RegisterSizeInBytes();
1134
  DCHECK_EQ(0, (size * registers.Count()) % 16);
1135

1136
  // Pop up to four registers at a time.
1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147
  while (!registers.IsEmpty()) {
    int count_before = registers.Count();
    const CPURegister& dst0 = registers.PopLowestIndex();
    const CPURegister& dst1 = registers.PopLowestIndex();
    const CPURegister& dst2 = registers.PopLowestIndex();
    const CPURegister& dst3 = registers.PopLowestIndex();
    int count = count_before - registers.Count();
    PopHelper(count, size, dst0, dst1, dst2, dst3);
  }
}

1148
void MacroAssembler::PushMultipleTimes(CPURegister src, Register count) {
1149 1150
  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireSameSizeAs(count);
1151

1152
  Label loop, leftover2, leftover1, done;
1153

1154 1155
  Subs(temp, count, 4);
  B(mi, &leftover2);
1156

1157 1158 1159 1160 1161
  // Push groups of four first.
  Bind(&loop);
  Subs(temp, temp, 4);
  PushHelper(4, src.SizeInBytes(), src, src, src, src);
  B(pl, &loop);
1162

1163 1164 1165 1166
  // Push groups of two.
  Bind(&leftover2);
  Tbz(count, 1, &leftover1);
  PushHelper(2, src.SizeInBytes(), src, src, NoReg, NoReg);
1167

1168 1169 1170 1171
  // Push the last one (if required).
  Bind(&leftover1);
  Tbz(count, 0, &done);
  PushHelper(1, src.SizeInBytes(), src, NoReg, NoReg, NoReg);
1172

1173
  Bind(&done);
1174 1175
}

1176
void TurboAssembler::PushHelper(int count, int size, const CPURegister& src0,
1177 1178 1179 1180 1181 1182
                                const CPURegister& src1,
                                const CPURegister& src2,
                                const CPURegister& src3) {
  // Ensure that we don't unintentially modify scratch or debug registers.
  InstructionAccurateScope scope(this);

1183 1184
  DCHECK(AreSameSizeAndType(src0, src1, src2, src3));
  DCHECK(size == src0.SizeInBytes());
1185 1186 1187 1188 1189

  // When pushing multiple registers, the store order is chosen such that
  // Push(a, b) is equivalent to Push(a) followed by Push(b).
  switch (count) {
    case 1:
1190
      DCHECK(src1.IsNone() && src2.IsNone() && src3.IsNone());
1191
      str(src0, MemOperand(sp, -1 * size, PreIndex));
1192 1193
      break;
    case 2:
1194
      DCHECK(src2.IsNone() && src3.IsNone());
1195
      stp(src1, src0, MemOperand(sp, -2 * size, PreIndex));
1196 1197
      break;
    case 3:
1198
      DCHECK(src3.IsNone());
1199 1200
      stp(src2, src1, MemOperand(sp, -3 * size, PreIndex));
      str(src0, MemOperand(sp, 2 * size));
1201 1202 1203
      break;
    case 4:
      // Skip over 4 * size, then fill in the gap. This allows four W registers
1204
      // to be pushed using sp, whilst maintaining 16-byte alignment for sp
1205
      // at all times.
1206 1207
      stp(src3, src2, MemOperand(sp, -4 * size, PreIndex));
      stp(src1, src0, MemOperand(sp, 2 * size));
1208 1209 1210 1211 1212 1213
      break;
    default:
      UNREACHABLE();
  }
}

1214 1215
void TurboAssembler::PopHelper(int count, int size, const CPURegister& dst0,
                               const CPURegister& dst1, const CPURegister& dst2,
1216 1217 1218 1219
                               const CPURegister& dst3) {
  // Ensure that we don't unintentially modify scratch or debug registers.
  InstructionAccurateScope scope(this);

1220 1221
  DCHECK(AreSameSizeAndType(dst0, dst1, dst2, dst3));
  DCHECK(size == dst0.SizeInBytes());
1222 1223 1224 1225 1226

  // When popping multiple registers, the load order is chosen such that
  // Pop(a, b) is equivalent to Pop(a) followed by Pop(b).
  switch (count) {
    case 1:
1227
      DCHECK(dst1.IsNone() && dst2.IsNone() && dst3.IsNone());
1228
      ldr(dst0, MemOperand(sp, 1 * size, PostIndex));
1229 1230
      break;
    case 2:
1231
      DCHECK(dst2.IsNone() && dst3.IsNone());
1232
      ldp(dst0, dst1, MemOperand(sp, 2 * size, PostIndex));
1233 1234
      break;
    case 3:
1235
      DCHECK(dst3.IsNone());
1236 1237
      ldr(dst2, MemOperand(sp, 2 * size));
      ldp(dst0, dst1, MemOperand(sp, 3 * size, PostIndex));
1238 1239 1240 1241
      break;
    case 4:
      // Load the higher addresses first, then load the lower addresses and
      // skip the whole block in the second instruction. This allows four W
1242 1243 1244 1245
      // registers to be popped using sp, whilst maintaining 16-byte alignment
      // for sp at all times.
      ldp(dst2, dst3, MemOperand(sp, 2 * size));
      ldp(dst0, dst1, MemOperand(sp, 4 * size, PostIndex));
1246 1247 1248 1249 1250 1251
      break;
    default:
      UNREACHABLE();
  }
}

1252
void TurboAssembler::Poke(const CPURegister& src, const Operand& offset) {
1253
  if (offset.IsImmediate()) {
1254
    DCHECK_GE(offset.ImmediateValue(), 0);
1255 1256
  } else if (emit_debug_code()) {
    Cmp(xzr, offset);
1257
    Check(le, AbortReason::kStackAccessBelowStackPointer);
1258 1259
  }

1260
  Str(src, MemOperand(sp, offset));
1261 1262
}

1263
void TurboAssembler::Peek(const CPURegister& dst, const Operand& offset) {
1264
  if (offset.IsImmediate()) {
1265
    DCHECK_GE(offset.ImmediateValue(), 0);
1266 1267
  } else if (emit_debug_code()) {
    Cmp(xzr, offset);
1268
    Check(le, AbortReason::kStackAccessBelowStackPointer);
1269 1270
  }

1271
  Ldr(dst, MemOperand(sp, offset));
1272 1273
}

1274
void TurboAssembler::PokePair(const CPURegister& src1, const CPURegister& src2,
1275
                              int offset) {
1276 1277
  DCHECK(AreSameSizeAndType(src1, src2));
  DCHECK((offset >= 0) && ((offset % src1.SizeInBytes()) == 0));
1278
  Stp(src1, src2, MemOperand(sp, offset));
1279 1280
}

1281
void MacroAssembler::PeekPair(const CPURegister& dst1, const CPURegister& dst2,
1282
                              int offset) {
1283 1284
  DCHECK(AreSameSizeAndType(dst1, dst2));
  DCHECK((offset >= 0) && ((offset % dst1.SizeInBytes()) == 0));
1285
  Ldp(dst1, dst2, MemOperand(sp, offset));
1286 1287 1288 1289 1290 1291
}

void MacroAssembler::PushCalleeSavedRegisters() {
  // Ensure that the macro-assembler doesn't use any scratch registers.
  InstructionAccurateScope scope(this);

1292
  MemOperand tos(sp, -2 * static_cast<int>(kXRegSize), PreIndex);
1293 1294 1295 1296 1297 1298

  stp(d14, d15, tos);
  stp(d12, d13, tos);
  stp(d10, d11, tos);
  stp(d8, d9, tos);

1299 1300 1301 1302 1303 1304 1305 1306 1307 1308
  STATIC_ASSERT(
      EntryFrameConstants::kCalleeSavedRegisterBytesPushedBeforeFpLrPair ==
      8 * kSystemPointerSize);

  stp(x29, x30, tos);  // fp, lr

  STATIC_ASSERT(
      EntryFrameConstants::kCalleeSavedRegisterBytesPushedAfterFpLrPair ==
      10 * kSystemPointerSize);

1309
  stp(x27, x28, tos);
1310 1311 1312 1313 1314 1315 1316 1317 1318 1319
  stp(x25, x26, tos);
  stp(x23, x24, tos);
  stp(x21, x22, tos);
  stp(x19, x20, tos);
}

void MacroAssembler::PopCalleeSavedRegisters() {
  // Ensure that the macro-assembler doesn't use any scratch registers.
  InstructionAccurateScope scope(this);

1320
  MemOperand tos(sp, 2 * kXRegSize, PostIndex);
1321 1322 1323 1324 1325

  ldp(x19, x20, tos);
  ldp(x21, x22, tos);
  ldp(x23, x24, tos);
  ldp(x25, x26, tos);
1326
  ldp(x27, x28, tos);
1327 1328 1329 1330 1331 1332 1333 1334
  ldp(x29, x30, tos);

  ldp(d8, d9, tos);
  ldp(d10, d11, tos);
  ldp(d12, d13, tos);
  ldp(d14, d15, tos);
}

1335
void TurboAssembler::AssertSpAligned() {
1336
  if (emit_debug_code()) {
1337
    HardAbortScope hard_abort(this);  // Avoid calls to Abort.
1338 1339
    // Arm64 requires the stack pointer to be 16-byte aligned prior to address
    // calculation.
1340 1341
    UseScratchRegisterScope scope(this);
    Register temp = scope.AcquireX();
1342 1343 1344
    Mov(temp, sp);
    Tst(temp, 15);
    Check(eq, AbortReason::kUnexpectedStackPointer);
1345 1346
  }
}
1347

1348 1349 1350 1351
void TurboAssembler::CopySlots(int dst, Register src, Register slot_count) {
  DCHECK(!src.IsZero());
  UseScratchRegisterScope scope(this);
  Register dst_reg = scope.AcquireX();
1352 1353
  SlotAddress(dst_reg, dst);
  SlotAddress(src, src);
1354 1355 1356 1357 1358 1359
  CopyDoubleWords(dst_reg, src, slot_count);
}

void TurboAssembler::CopySlots(Register dst, Register src,
                               Register slot_count) {
  DCHECK(!dst.IsZero() && !src.IsZero());
1360 1361
  SlotAddress(dst, dst);
  SlotAddress(src, src);
1362 1363 1364
  CopyDoubleWords(dst, src, slot_count);
}

1365 1366
void TurboAssembler::CopyDoubleWords(Register dst, Register src, Register count,
                                     CopyDoubleWordsMode mode) {
1367 1368
  DCHECK(!AreAliased(dst, src, count));

1369
  if (emit_debug_code()) {
1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380
    Register pointer1 = dst;
    Register pointer2 = src;
    if (mode == kSrcLessThanDst) {
      pointer1 = src;
      pointer2 = dst;
    }
    // Copy requires pointer1 < pointer2 || (pointer1 - pointer2) >= count.
    Label pointer1_below_pointer2;
    Subs(pointer1, pointer1, pointer2);
    B(lt, &pointer1_below_pointer2);
    Cmp(pointer1, count);
1381
    Check(ge, AbortReason::kOffsetOutOfRange);
1382 1383
    Bind(&pointer1_below_pointer2);
    Add(pointer1, pointer1, pointer2);
1384
  }
1385
  static_assert(kSystemPointerSize == kDRegSize,
1386
                "pointers must be the same size as doubles");
1387 1388

  int direction = (mode == kDstLessThanSrc) ? 1 : -1;
1389 1390 1391 1392
  UseScratchRegisterScope scope(this);
  VRegister temp0 = scope.AcquireD();
  VRegister temp1 = scope.AcquireD();

1393
  Label pairs, loop, done;
1394 1395

  Tbz(count, 0, &pairs);
1396
  Ldr(temp0, MemOperand(src, direction * kSystemPointerSize, PostIndex));
1397
  Sub(count, count, 1);
1398
  Str(temp0, MemOperand(dst, direction * kSystemPointerSize, PostIndex));
1399 1400

  Bind(&pairs);
1401 1402
  if (mode == kSrcLessThanDst) {
    // Adjust pointers for post-index ldp/stp with negative offset:
1403 1404
    Sub(dst, dst, kSystemPointerSize);
    Sub(src, src, kSystemPointerSize);
1405 1406
  }
  Bind(&loop);
1407
  Cbz(count, &done);
1408 1409
  Ldp(temp0, temp1,
      MemOperand(src, 2 * direction * kSystemPointerSize, PostIndex));
1410
  Sub(count, count, 2);
1411 1412
  Stp(temp0, temp1,
      MemOperand(dst, 2 * direction * kSystemPointerSize, PostIndex));
1413
  B(&loop);
1414 1415 1416 1417 1418 1419 1420

  // TODO(all): large copies may benefit from using temporary Q registers
  // to copy four double words per iteration.

  Bind(&done);
}

1421
void TurboAssembler::SlotAddress(Register dst, int slot_offset) {
1422
  Add(dst, sp, slot_offset << kSystemPointerSizeLog2);
1423 1424 1425
}

void TurboAssembler::SlotAddress(Register dst, Register slot_offset) {
1426
  Add(dst, sp, Operand(slot_offset, LSL, kSystemPointerSizeLog2));
1427 1428
}

1429
void TurboAssembler::AssertFPCRState(Register fpcr) {
1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446
  if (emit_debug_code()) {
    Label unexpected_mode, done;
    UseScratchRegisterScope temps(this);
    if (fpcr.IsNone()) {
      fpcr = temps.AcquireX();
      Mrs(fpcr, FPCR);
    }

    // Settings left to their default values:
    //   - Assert that flush-to-zero is not set.
    Tbnz(fpcr, FZ_offset, &unexpected_mode);
    //   - Assert that the rounding mode is nearest-with-ties-to-even.
    STATIC_ASSERT(FPTieEven == 0);
    Tst(fpcr, RMode_mask);
    B(eq, &done);

    Bind(&unexpected_mode);
1447
    Abort(AbortReason::kUnexpectedFPCRMode);
1448 1449 1450 1451 1452

    Bind(&done);
  }
}

1453
void TurboAssembler::CanonicalizeNaN(const VRegister& dst,
1454
                                     const VRegister& src) {
1455 1456
  AssertFPCRState();

1457 1458 1459
  // Subtracting 0.0 preserves all inputs except for signalling NaNs, which
  // become quiet NaNs. We use fsub rather than fadd because fsub preserves -0.0
  // inputs: -0.0 + 0.0 = 0.0, but -0.0 - 0.0 = -0.0.
1460 1461 1462
  Fsub(dst, src, fp_zero);
}

1463
void TurboAssembler::LoadRoot(Register destination, RootIndex index) {
1464 1465
  // TODO(jbramley): Most root values are constants, and can be synthesized
  // without a load. Refer to the ARM back end for details.
1466 1467
  Ldr(destination,
      MemOperand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
1468 1469
}

1470
void TurboAssembler::Move(Register dst, Smi src) { Mov(dst, src); }
1471

1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487
void TurboAssembler::MovePair(Register dst0, Register src0, Register dst1,
                              Register src1) {
  DCHECK_NE(dst0, dst1);
  if (dst0 != src1) {
    Mov(dst0, src0);
    Mov(dst1, src1);
  } else if (dst1 != src0) {
    // Swap the order of the moves to resolve the overlap.
    Mov(dst1, src1);
    Mov(dst0, src0);
  } else {
    // Worse case scenario, this is a swap.
    Swap(dst0, src0);
  }
}

1488 1489
void TurboAssembler::Swap(Register lhs, Register rhs) {
  DCHECK(lhs.IsSameSizeAndType(rhs));
1490
  DCHECK_NE(lhs, rhs);
1491 1492 1493 1494 1495 1496 1497 1498 1499
  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireX();
  Mov(temp, rhs);
  Mov(rhs, lhs);
  Mov(lhs, temp);
}

void TurboAssembler::Swap(VRegister lhs, VRegister rhs) {
  DCHECK(lhs.IsSameSizeAndType(rhs));
1500
  DCHECK_NE(lhs, rhs);
1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515
  UseScratchRegisterScope temps(this);
  VRegister temp = VRegister::no_reg();
  if (lhs.IsS()) {
    temp = temps.AcquireS();
  } else if (lhs.IsD()) {
    temp = temps.AcquireD();
  } else {
    DCHECK(lhs.IsQ());
    temp = temps.AcquireQ();
  }
  Mov(temp, rhs);
  Mov(rhs, lhs);
  Mov(lhs, temp);
}

1516
void TurboAssembler::AssertSmi(Register object, AbortReason reason) {
1517 1518 1519 1520 1521 1522 1523
  if (emit_debug_code()) {
    STATIC_ASSERT(kSmiTag == 0);
    Tst(object, kSmiTagMask);
    Check(eq, reason);
  }
}

1524
void MacroAssembler::AssertNotSmi(Register object, AbortReason reason) {
1525 1526 1527 1528 1529 1530 1531
  if (emit_debug_code()) {
    STATIC_ASSERT(kSmiTag == 0);
    Tst(object, kSmiTagMask);
    Check(ne, reason);
  }
}

1532 1533 1534 1535 1536 1537 1538
void MacroAssembler::AssertConstructor(Register object) {
  if (emit_debug_code()) {
    AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotAConstructor);

    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireX();

1539
    LoadMap(temp, object);
1540 1541 1542 1543 1544 1545 1546
    Ldrb(temp, FieldMemOperand(temp, Map::kBitFieldOffset));
    Tst(temp, Operand(Map::IsConstructorBit::kMask));

    Check(ne, AbortReason::kOperandIsNotAConstructor);
  }
}

1547 1548
void MacroAssembler::AssertFunction(Register object) {
  if (emit_debug_code()) {
1549
    AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotAFunction);
1550 1551 1552 1553 1554

    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireX();

    CompareObjectType(object, temp, temp, JS_FUNCTION_TYPE);
1555
    Check(eq, AbortReason::kOperandIsNotAFunction);
1556 1557 1558
  }
}

1559 1560
void MacroAssembler::AssertBoundFunction(Register object) {
  if (emit_debug_code()) {
1561
    AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotABoundFunction);
1562 1563 1564 1565 1566

    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireX();

    CompareObjectType(object, temp, temp, JS_BOUND_FUNCTION_TYPE);
1567
    Check(eq, AbortReason::kOperandIsNotABoundFunction);
1568 1569 1570
  }
}

1571
void MacroAssembler::AssertGeneratorObject(Register object) {
1572
  if (!emit_debug_code()) return;
1573
  AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotAGeneratorObject);
1574

1575 1576 1577
  // Load map
  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireX();
1578
  LoadMap(temp, object);
1579

1580
  Label do_check;
1581 1582
  // Load instance type and check if JSGeneratorObject
  CompareInstanceType(temp, temp, JS_GENERATOR_OBJECT_TYPE);
1583
  B(eq, &do_check);
1584

1585 1586 1587 1588
  // Check if JSAsyncFunctionObject
  Cmp(temp, JS_ASYNC_FUNCTION_OBJECT_TYPE);
  B(eq, &do_check);

1589 1590 1591 1592 1593
  // Check if JSAsyncGeneratorObject
  Cmp(temp, JS_ASYNC_GENERATOR_OBJECT_TYPE);

  bind(&do_check);
  // Restore generator object to register and perform assertion
1594
  Check(eq, AbortReason::kOperandIsNotAGeneratorObject);
1595
}
1596

1597
void MacroAssembler::AssertUndefinedOrAllocationSite(Register object) {
1598
  if (emit_debug_code()) {
1599 1600
    UseScratchRegisterScope temps(this);
    Register scratch = temps.AcquireX();
1601 1602
    Label done_checking;
    AssertNotSmi(object);
1603
    JumpIfRoot(object, RootIndex::kUndefinedValue, &done_checking);
1604
    LoadMap(scratch, object);
1605
    CompareInstanceType(scratch, scratch, ALLOCATION_SITE_TYPE);
1606
    Assert(eq, AbortReason::kExpectedUndefinedOrCell);
1607 1608 1609 1610
    Bind(&done_checking);
  }
}

1611
void TurboAssembler::AssertPositiveOrZero(Register value) {
1612 1613 1614 1615
  if (emit_debug_code()) {
    Label done;
    int sign_bit = value.Is64Bits() ? kXSignBit : kWSignBit;
    Tbz(value, sign_bit, &done);
1616
    Abort(AbortReason::kUnexpectedNegativeValue);
1617 1618
    Bind(&done);
  }
1619 1620
}

1621
void MacroAssembler::CallRuntime(const Runtime::Function* f, int num_arguments,
1622 1623 1624 1625 1626 1627
                                 SaveFPRegsMode save_doubles) {
  // All arguments must be on the stack before this function is called.
  // x0 holds the return value after the call.

  // Check that the number of arguments matches what the function expects.
  // If f->nargs is -1, the function can accept a variable number of arguments.
1628
  CHECK(f->nargs < 0 || f->nargs == num_arguments);
1629 1630 1631

  // Place the necessary arguments.
  Mov(x0, num_arguments);
1632
  Mov(x1, ExternalReference::Create(f));
1633

1634 1635 1636
  Handle<Code> code =
      CodeFactory::CEntry(isolate(), f->result_size, save_doubles);
  Call(code, RelocInfo::CODE_TARGET);
1637 1638
}

1639 1640
void MacroAssembler::JumpToExternalReference(const ExternalReference& builtin,
                                             bool builtin_exit_frame) {
1641
  Mov(x1, builtin);
1642 1643 1644
  Handle<Code> code = CodeFactory::CEntry(isolate(), 1, kDontSaveFPRegs,
                                          kArgvOnStack, builtin_exit_frame);
  Jump(code, RelocInfo::CODE_TARGET);
1645 1646
}

1647
void MacroAssembler::JumpToInstructionStream(Address entry) {
1648
  Ldr(kOffHeapTrampolineRegister, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
1649 1650 1651
  Br(kOffHeapTrampolineRegister);
}

1652 1653 1654 1655 1656 1657 1658 1659 1660 1661
void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid) {
  const Runtime::Function* function = Runtime::FunctionForId(fid);
  DCHECK_EQ(1, function->result_size);
  if (function->nargs >= 0) {
    // TODO(1236192): Most runtime routines don't need the number of
    // arguments passed in because it is constant. At some point we
    // should remove this need and make the runtime routine entry code
    // smarter.
    Mov(x0, function->nargs);
  }
1662
  JumpToExternalReference(ExternalReference::Create(fid));
1663 1664
}

1665
int TurboAssembler::ActivationFrameAlignment() {
1666
#if V8_HOST_ARCH_ARM64
1667 1668 1669 1670
  // Running on the real platform. Use the alignment as mandated by the local
  // environment.
  // Note: This will break if we ever start generating snapshots on one ARM
  // platform for another ARM platform with a different alignment.
1671
  return base::OS::ActivationFrameAlignment();
1672
#else   // V8_HOST_ARCH_ARM64
1673 1674 1675 1676 1677
  // If we are using the simulator then we should always align to the expected
  // alignment. As the simulator is used to generate snapshots we do not know
  // if the target platform will need alignment, so this is controlled from a
  // flag.
  return FLAG_sim_stack_alignment;
1678
#endif  // V8_HOST_ARCH_ARM64
1679 1680
}

1681
void TurboAssembler::CallCFunction(ExternalReference function,
1682 1683 1684 1685
                                   int num_of_reg_args) {
  CallCFunction(function, num_of_reg_args, 0);
}

1686
void TurboAssembler::CallCFunction(ExternalReference function,
1687 1688
                                   int num_of_reg_args,
                                   int num_of_double_args) {
1689 1690
  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireX();
1691
  Mov(temp, function);
1692
  CallCFunction(temp, num_of_reg_args, num_of_double_args);
1693 1694
}

1695
static const int kRegisterPassedArguments = 8;
1696

1697
void TurboAssembler::CallCFunction(Register function, int num_of_reg_args,
1698
                                   int num_of_double_args) {
1699
  DCHECK_LE(num_of_reg_args + num_of_double_args, kMaxCParameters);
1700
  DCHECK(has_frame());
1701 1702 1703 1704 1705 1706 1707 1708

  // If we're passing doubles, we're limited to the following prototypes
  // (defined by ExternalReference::Type):
  //  BUILTIN_COMPARE_CALL:  int f(double, double)
  //  BUILTIN_FP_FP_CALL:    double f(double, double)
  //  BUILTIN_FP_CALL:       double f(double)
  //  BUILTIN_FP_INT_CALL:   double f(double, int)
  if (num_of_double_args > 0) {
1709 1710
    DCHECK_LE(num_of_reg_args, 1);
    DCHECK_LE(num_of_double_args + num_of_reg_args, 2);
1711 1712
  }

1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731
  // Save the frame pointer and PC so that the stack layout remains iterable,
  // even without an ExitFrame which normally exists between JS and C frames.
  if (isolate() != nullptr) {
    Register scratch1 = x4;
    Register scratch2 = x5;
    Push(scratch1, scratch2);

    Label get_pc;
    Bind(&get_pc);
    Adr(scratch2, &get_pc);

    Mov(scratch1, ExternalReference::fast_c_call_caller_pc_address(isolate()));
    Str(scratch2, MemOperand(scratch1));
    Mov(scratch1, ExternalReference::fast_c_call_caller_fp_address(isolate()));
    Str(fp, MemOperand(scratch1));

    Pop(scratch2, scratch1);
  }

1732 1733 1734 1735
  // Call directly. The function called cannot cause a GC, or allow preemption,
  // so the return address in the link register stays correct.
  Call(function);

1736 1737 1738 1739 1740 1741 1742 1743 1744
  if (isolate() != nullptr) {
    // We don't unset the PC; the FP is the source of truth.
    Register scratch = x4;
    Push(scratch, xzr);
    Mov(scratch, ExternalReference::fast_c_call_caller_fp_address(isolate()));
    Str(xzr, MemOperand(scratch));
    Pop(xzr, scratch);
  }

1745 1746 1747 1748 1749
  if (num_of_reg_args > kRegisterPassedArguments) {
    // Drop the register passed arguments.
    int claim_slots = RoundUp(num_of_reg_args - kRegisterPassedArguments, 2);
    Drop(claim_slots);
  }
1750 1751
}

1752 1753
void TurboAssembler::LoadFromConstantsTable(Register destination,
                                            int constant_index) {
1754
  DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kBuiltinsConstantsTable));
1755
  LoadRoot(destination, RootIndex::kBuiltinsConstantsTable);
1756 1757 1758
  LoadTaggedPointerField(
      destination, FieldMemOperand(destination, FixedArray::OffsetOfElementAt(
                                                    constant_index)));
1759 1760
}

1761 1762
void TurboAssembler::LoadRootRelative(Register destination, int32_t offset) {
  Ldr(destination, MemOperand(kRootRegister, offset));
1763
}
1764 1765 1766 1767

void TurboAssembler::LoadRootRegisterOffset(Register destination,
                                            intptr_t offset) {
  if (offset == 0) {
1768
    Mov(destination, kRootRegister);
1769
  } else {
1770
    Add(destination, kRootRegister, offset);
1771 1772
  }
}
1773 1774 1775 1776 1777 1778 1779 1780

void TurboAssembler::Jump(Register target, Condition cond) {
  if (cond == nv) return;
  Label done;
  if (cond != al) B(NegateCondition(cond), &done);
  Br(target);
  Bind(&done);
}
1781

1782 1783
void TurboAssembler::JumpHelper(int64_t offset, RelocInfo::Mode rmode,
                                Condition cond) {
1784 1785 1786
  if (cond == nv) return;
  Label done;
  if (cond != al) B(NegateCondition(cond), &done);
1787 1788 1789 1790 1791 1792
  if (CanUseNearCallOrJump(rmode)) {
    DCHECK(IsNearCallOffset(offset));
    near_jump(static_cast<int>(offset), rmode);
  } else {
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireX();
1793
    uint64_t imm = reinterpret_cast<uint64_t>(pc_) + offset * kInstrSize;
1794 1795 1796
    Mov(temp, Immediate(imm, rmode));
    Br(temp);
  }
1797
  Bind(&done);
1798 1799
}

1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810
namespace {

// The calculated offset is either:
// * the 'target' input unmodified if this is a WASM call, or
// * the offset of the target from the current PC, in instructions, for any
//   other type of call.
static int64_t CalculateTargetOffset(Address target, RelocInfo::Mode rmode,
                                     byte* pc) {
  int64_t offset = static_cast<int64_t>(target);
  // The target of WebAssembly calls is still an index instead of an actual
  // address at this point, and needs to be encoded as-is.
1811
  if (rmode != RelocInfo::WASM_CALL && rmode != RelocInfo::WASM_STUB_CALL) {
1812
    offset -= reinterpret_cast<int64_t>(pc);
1813 1814
    DCHECK_EQ(offset % kInstrSize, 0);
    offset = offset / static_cast<int>(kInstrSize);
1815 1816 1817 1818 1819
  }
  return offset;
}
}  // namespace

1820
void TurboAssembler::Jump(Address target, RelocInfo::Mode rmode,
1821
                          Condition cond) {
1822
  JumpHelper(CalculateTargetOffset(target, rmode, pc_), rmode, cond);
1823 1824
}

1825
void TurboAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,
1826
                          Condition cond) {
1827
  DCHECK(RelocInfo::IsCodeTarget(rmode));
1828 1829 1830 1831 1832 1833 1834 1835 1836 1837
  DCHECK_IMPLIES(options().isolate_independent_code,
                 Builtins::IsIsolateIndependentBuiltin(*code));

  if (options().inline_offheap_trampolines) {
    int builtin_index = Builtins::kNoBuiltinId;
    if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index) &&
        Builtins::IsIsolateIndependent(builtin_index)) {
      // Inline the trampoline.
      RecordCommentForOffHeapTrampoline(builtin_index);
      CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
1838 1839
      UseScratchRegisterScope temps(this);
      Register scratch = temps.AcquireX();
1840 1841 1842
      EmbeddedData d = EmbeddedData::FromBlob();
      Address entry = d.InstructionStartOfBuiltin(builtin_index);
      Ldr(scratch, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
1843 1844 1845
      Jump(scratch, cond);
      return;
    }
1846
  }
1847

1848
  if (CanUseNearCallOrJump(rmode)) {
1849 1850 1851
    EmbeddedObjectIndex index = AddEmbeddedObject(code);
    DCHECK(is_int32(index));
    JumpHelper(static_cast<int64_t>(index), rmode, cond);
1852 1853 1854
  } else {
    Jump(code.address(), rmode, cond);
  }
1855 1856
}

1857 1858 1859 1860 1861 1862 1863
void TurboAssembler::Jump(const ExternalReference& reference) {
  UseScratchRegisterScope temps(this);
  Register scratch = temps.AcquireX();
  Mov(scratch, reference);
  Jump(scratch);
}

1864
void TurboAssembler::Call(Register target) {
1865
  BlockPoolsScope scope(this);
1866 1867 1868
  Blr(target);
}

1869
void TurboAssembler::Call(Address target, RelocInfo::Mode rmode) {
1870
  BlockPoolsScope scope(this);
1871

1872
  if (CanUseNearCallOrJump(rmode)) {
1873
    int64_t offset = CalculateTargetOffset(target, rmode, pc_);
1874 1875
    DCHECK(IsNearCallOffset(offset));
    near_call(static_cast<int>(offset), rmode);
1876
  } else {
1877
    IndirectCall(target, rmode);
1878 1879 1880
  }
}

1881
void TurboAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode) {
1882 1883
  DCHECK_IMPLIES(options().isolate_independent_code,
                 Builtins::IsIsolateIndependentBuiltin(*code));
1884
  BlockPoolsScope scope(this);
1885

1886 1887 1888 1889 1890
  if (options().inline_offheap_trampolines) {
    int builtin_index = Builtins::kNoBuiltinId;
    if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index) &&
        Builtins::IsIsolateIndependent(builtin_index)) {
      // Inline the trampoline.
1891
      CallBuiltin(builtin_index);
1892 1893
      return;
    }
1894
  }
1895

1896
  DCHECK(code->IsExecutable());
1897
  if (CanUseNearCallOrJump(rmode)) {
1898 1899 1900
    EmbeddedObjectIndex index = AddEmbeddedObject(code);
    DCHECK(is_int32(index));
    near_call(static_cast<int32_t>(index), rmode);
1901 1902 1903
  } else {
    IndirectCall(code.address(), rmode);
  }
1904 1905
}

1906 1907 1908
void TurboAssembler::Call(ExternalReference target) {
  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireX();
1909
  Mov(temp, target);
1910 1911 1912
  Call(temp);
}

1913
void TurboAssembler::LoadEntryFromBuiltinIndex(Register builtin_index) {
1914
  // The builtin_index register contains the builtin index as a Smi.
1915
  // Untagging is folded into the indexing operand below.
1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932
  if (SmiValuesAre32Bits()) {
    Asr(builtin_index, builtin_index, kSmiShift - kSystemPointerSizeLog2);
    Add(builtin_index, builtin_index,
        IsolateData::builtin_entry_table_offset());
    Ldr(builtin_index, MemOperand(kRootRegister, builtin_index));
  } else {
    DCHECK(SmiValuesAre31Bits());
    if (COMPRESS_POINTERS_BOOL) {
      Add(builtin_index, kRootRegister,
          Operand(builtin_index.W(), SXTW, kSystemPointerSizeLog2 - kSmiShift));
    } else {
      Add(builtin_index, kRootRegister,
          Operand(builtin_index, LSL, kSystemPointerSizeLog2 - kSmiShift));
    }
    Ldr(builtin_index,
        MemOperand(builtin_index, IsolateData::builtin_entry_table_offset()));
  }
1933 1934 1935 1936
}

void TurboAssembler::CallBuiltinByIndex(Register builtin_index) {
  LoadEntryFromBuiltinIndex(builtin_index);
1937
  Call(builtin_index);
1938 1939
}

1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951
void TurboAssembler::CallBuiltin(int builtin_index) {
  DCHECK(Builtins::IsBuiltinId(builtin_index));
  RecordCommentForOffHeapTrampoline(builtin_index);
  CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
  UseScratchRegisterScope temps(this);
  Register scratch = temps.AcquireX();
  EmbeddedData d = EmbeddedData::FromBlob();
  Address entry = d.InstructionStartOfBuiltin(builtin_index);
  Ldr(scratch, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
  Call(scratch);
}

1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964
void TurboAssembler::LoadCodeObjectEntry(Register destination,
                                         Register code_object) {
  // Code objects are called differently depending on whether we are generating
  // builtin code (which will later be embedded into the binary) or compiling
  // user JS code at runtime.
  // * Builtin code runs in --jitless mode and thus must not call into on-heap
  //   Code targets. Instead, we dispatch through the builtins entry table.
  // * Codegen at runtime does not have this restriction and we can use the
  //   shorter, branchless instruction sequence. The assumption here is that
  //   targets are usually generated code and not builtin Code objects.

  if (options().isolate_independent_code) {
    DCHECK(root_array_available());
1965
    Label if_code_is_off_heap, out;
1966 1967 1968 1969 1970 1971 1972

    UseScratchRegisterScope temps(this);
    Register scratch = temps.AcquireX();

    DCHECK(!AreAliased(destination, scratch));
    DCHECK(!AreAliased(code_object, scratch));

1973 1974 1975
    // Check whether the Code object is an off-heap trampoline. If so, call its
    // (off-heap) entry point directly without going through the (on-heap)
    // trampoline.  Otherwise, just call the Code object as always.
1976

1977 1978 1979
    Ldrsw(scratch, FieldMemOperand(code_object, Code::kFlagsOffset));
    Tst(scratch, Operand(Code::IsOffHeapTrampoline::kMask));
    B(ne, &if_code_is_off_heap);
1980

1981
    // Not an off-heap trampoline object, the entry point is at
1982 1983 1984 1985
    // Code::raw_instruction_start().
    Add(destination, code_object, Code::kHeaderSize - kHeapObjectTag);
    B(&out);

1986
    // An off-heap trampoline, the entry point is loaded from the builtin entry
1987
    // table.
1988 1989
    bind(&if_code_is_off_heap);
    Ldrsw(scratch, FieldMemOperand(code_object, Code::kBuiltinIndexOffset));
1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010
    Lsl(destination, scratch, kSystemPointerSizeLog2);
    Add(destination, destination, kRootRegister);
    Ldr(destination,
        MemOperand(destination, IsolateData::builtin_entry_table_offset()));

    bind(&out);
  } else {
    Add(destination, code_object, Code::kHeaderSize - kHeapObjectTag);
  }
}

void TurboAssembler::CallCodeObject(Register code_object) {
  LoadCodeObjectEntry(code_object, code_object);
  Call(code_object);
}

void TurboAssembler::JumpCodeObject(Register code_object) {
  LoadCodeObjectEntry(code_object, code_object);
  Jump(code_object);
}

2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038
void TurboAssembler::StoreReturnAddressAndCall(Register target) {
  // This generates the final instruction sequence for calls to C functions
  // once an exit frame has been constructed.
  //
  // Note that this assumes the caller code (i.e. the Code object currently
  // being generated) is immovable or that the callee function cannot trigger
  // GC, since the callee function will return to it.

  UseScratchRegisterScope temps(this);
  Register scratch1 = temps.AcquireX();

  Label return_location;
  Adr(scratch1, &return_location);
  Poke(scratch1, 0);

  if (emit_debug_code()) {
    // Verify that the slot below fp[kSPOffset]-8 points to the return location.
    Register scratch2 = temps.AcquireX();
    Ldr(scratch2, MemOperand(fp, ExitFrameConstants::kSPOffset));
    Ldr(scratch2, MemOperand(scratch2, -static_cast<int64_t>(kXRegSize)));
    Cmp(scratch2, scratch1);
    Check(eq, AbortReason::kReturnAddressNotFoundInFrame);
  }

  Blr(target);
  Bind(&return_location);
}

2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049
void TurboAssembler::IndirectCall(Address target, RelocInfo::Mode rmode) {
  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireX();
  Mov(temp, Immediate(target, rmode));
  Blr(temp);
}

bool TurboAssembler::IsNearCallOffset(int64_t offset) {
  return is_int26(offset);
}

2050
void TurboAssembler::CallForDeoptimization(Address target, int deopt_id) {
2051 2052
  BlockPoolsScope scope(this);
#ifdef DEBUG
2053
  Label start;
2054
  bind(&start);
2055
#endif
2056
  int64_t offset = static_cast<int64_t>(target) -
2057
                   static_cast<int64_t>(options().code_range_start);
2058 2059
  DCHECK_EQ(offset % kInstrSize, 0);
  offset = offset / static_cast<int>(kInstrSize);
2060
  DCHECK(IsNearCallOffset(offset));
2061
  near_call(static_cast<int>(offset), RelocInfo::RUNTIME_ENTRY);
2062
  DCHECK_EQ(SizeOfCodeGeneratedSince(&start), Deoptimizer::kDeoptExitSize);
2063 2064
}

2065 2066
void TurboAssembler::PrepareForTailCall(Register callee_args_count,
                                        Register caller_args_count,
2067
                                        Register scratch0, Register scratch1) {
2068
  DCHECK(!AreAliased(callee_args_count, caller_args_count, scratch0, scratch1));
2069 2070

  // Calculate the end of destination area where we will put the arguments
2071 2072
  // after we drop current frame. We add kSystemPointerSize to count the
  // receiver argument which is not included into formal parameters count.
2073
  Register dst_reg = scratch0;
2074
  Add(dst_reg, fp, Operand(caller_args_count, LSL, kSystemPointerSizeLog2));
2075 2076
  Add(dst_reg, dst_reg,
      StandardFrameConstants::kCallerSPOffset + kSystemPointerSize);
2077 2078 2079 2080
  // Round dst_reg up to a multiple of 16 bytes, so that we overwrite any
  // potential padding.
  Add(dst_reg, dst_reg, 15);
  Bic(dst_reg, dst_reg, 15);
2081

2082
  Register src_reg = caller_args_count;
2083
  // Calculate the end of source area. +kSystemPointerSize is for the receiver.
2084 2085
  Add(src_reg, sp, Operand(callee_args_count, LSL, kSystemPointerSizeLog2));
  Add(src_reg, src_reg, kSystemPointerSize);
2086

2087 2088 2089 2090 2091
  // Round src_reg up to a multiple of 16 bytes, so we include any potential
  // padding in the copy.
  Add(src_reg, src_reg, 15);
  Bic(src_reg, src_reg, 15);

2092
  if (FLAG_debug_code) {
2093
    Cmp(src_reg, dst_reg);
2094
    Check(lo, AbortReason::kStackAccessBelowStackPointer);
2095 2096 2097 2098
  }

  // Restore caller's frame pointer and return address now as they will be
  // overwritten by the copying loop.
2099 2100
  Ldr(lr, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
  Ldr(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
2101 2102 2103 2104 2105 2106 2107 2108

  // Now copy callee arguments to the caller frame going backwards to avoid
  // callee arguments corruption (source and destination areas could overlap).

  // Both src_reg and dst_reg are pointing to the word after the one to copy,
  // so they must be pre-decremented in the loop.
  Register tmp_reg = scratch1;
  Label loop, entry;
2109 2110
  B(&entry);
  bind(&loop);
2111 2112
  Ldr(tmp_reg, MemOperand(src_reg, -kSystemPointerSize, PreIndex));
  Str(tmp_reg, MemOperand(dst_reg, -kSystemPointerSize, PreIndex));
2113
  bind(&entry);
2114
  Cmp(sp, src_reg);
2115
  B(ne, &loop);
2116 2117

  // Leave current frame.
2118
  Mov(sp, dst_reg);
2119
}
2120

2121 2122 2123
void MacroAssembler::InvokePrologue(Register expected_parameter_count,
                                    Register actual_parameter_count,
                                    Label* done, InvokeFlag flag) {
2124 2125
  Label regular_invoke;

2126 2127
  // Check whether the expected and actual arguments count match. The registers
  // are set up according to contract with ArgumentsAdaptorTrampoline:
2128 2129 2130 2131
  //  x0: actual arguments count.
  //  x1: function (passed through to callee).
  //  x2: expected arguments count.
  // The code below is made a lot easier because the calling code already sets
2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148
  // up actual and expected registers according to the contract.
  DCHECK_EQ(actual_parameter_count, x0);
  DCHECK_EQ(expected_parameter_count, x2);

  // If actual == expected perform a regular invocation.
  Cmp(expected_parameter_count, actual_parameter_count);
  B(eq, &regular_invoke);

  // The argument counts mismatch, generate a call to the argument adaptor.
  Handle<Code> adaptor = BUILTIN_CODE(isolate(), ArgumentsAdaptorTrampoline);
  if (flag == CALL_FUNCTION) {
    Call(adaptor);
    // If the arg counts don't match, no extra code is emitted by
    // MAsm::InvokeFunctionCode and we can just fall through.
    B(done);
  } else {
    Jump(adaptor, RelocInfo::CODE_TARGET);
2149
  }
2150

2151 2152 2153
  Bind(&regular_invoke);
}

2154
void MacroAssembler::CallDebugOnFunctionCall(Register fun, Register new_target,
2155 2156
                                             Register expected_parameter_count,
                                             Register actual_parameter_count) {
2157
  // Load receiver to pass it later to DebugOnFunctionCall hook.
2158
  Ldr(x4, MemOperand(sp, actual_parameter_count, LSL, kSystemPointerSizeLog2));
2159
  FrameScope frame(this, has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
2160

2161
  if (!new_target.is_valid()) new_target = padreg;
2162

2163
  // Save values on stack.
2164 2165 2166
  SmiTag(expected_parameter_count);
  SmiTag(actual_parameter_count);
  Push(expected_parameter_count, actual_parameter_count, new_target, fun);
2167 2168
  Push(fun, x4);
  CallRuntime(Runtime::kDebugOnFunctionCall);
2169

2170
  // Restore values from stack.
2171 2172 2173
  Pop(fun, new_target, actual_parameter_count, expected_parameter_count);
  SmiUntag(actual_parameter_count);
  SmiUntag(expected_parameter_count);
2174 2175 2176
}

void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
2177 2178
                                        Register expected_parameter_count,
                                        Register actual_parameter_count,
2179
                                        InvokeFlag flag) {
2180
  // You can't call a function without a valid frame.
2181
  DCHECK(flag == JUMP_FUNCTION || has_frame());
2182 2183
  DCHECK_EQ(function, x1);
  DCHECK_IMPLIES(new_target.is_valid(), new_target == x3);
2184

2185
  // On function call, call into the debugger if necessary.
2186 2187 2188 2189 2190 2191 2192
  Label debug_hook, continue_after_hook;
  {
    Mov(x4, ExternalReference::debug_hook_on_function_call_address(isolate()));
    Ldrsb(x4, MemOperand(x4));
    Cbnz(x4, &debug_hook);
  }
  bind(&continue_after_hook);
2193 2194

  // Clear the new.target register if not given.
2195
  if (!new_target.is_valid()) {
2196
    LoadRoot(x3, RootIndex::kUndefinedValue);
2197
  }
2198

2199
  Label done;
2200
  InvokePrologue(expected_parameter_count, actual_parameter_count, &done, flag);
2201

2202 2203
  // If actual != expected, InvokePrologue will have handled the call through
  // the argument adaptor mechanism.
2204
  // The called function expects the call kind in x5.
2205 2206 2207 2208 2209 2210 2211 2212 2213 2214 2215
  // We call indirectly through the code field in the function to
  // allow recompilation to take effect without changing any of the
  // call sites.
  Register code = kJavaScriptCallCodeStartRegister;
  LoadTaggedPointerField(code,
                         FieldMemOperand(function, JSFunction::kCodeOffset));
  if (flag == CALL_FUNCTION) {
    CallCodeObject(code);
  } else {
    DCHECK(flag == JUMP_FUNCTION);
    JumpCodeObject(code);
2216
  }
2217 2218 2219 2220
  B(&done);

  // Deferred debug hook.
  bind(&debug_hook);
2221 2222
  CallDebugOnFunctionCall(function, new_target, expected_parameter_count,
                          actual_parameter_count);
2223
  B(&continue_after_hook);
2224 2225 2226 2227 2228 2229

  // Continue here if InvokePrologue does handle the invocation due to
  // mismatched parameter counts.
  Bind(&done);
}

2230 2231 2232
void MacroAssembler::InvokeFunctionWithNewTarget(
    Register function, Register new_target, Register actual_parameter_count,
    InvokeFlag flag) {
2233
  // You can't call a function without a valid frame.
2234
  DCHECK(flag == JUMP_FUNCTION || has_frame());
2235 2236 2237

  // Contract with called JS functions requires that function is passed in x1.
  // (See FullCodeGenerator::Generate().)
2238
  DCHECK_EQ(function, x1);
2239

2240
  Register expected_parameter_count = x2;
2241

2242 2243
  LoadTaggedPointerField(cp,
                         FieldMemOperand(function, JSFunction::kContextOffset));
2244 2245 2246
  // The number of arguments is stored as an int32_t, and -1 is a marker
  // (SharedFunctionInfo::kDontAdaptArgumentsSentinel), so we need sign
  // extension to correctly handle it.
2247
  LoadTaggedPointerField(
2248
      expected_parameter_count,
2249
      FieldMemOperand(function, JSFunction::kSharedFunctionInfoOffset));
2250 2251
  Ldrh(expected_parameter_count,
       FieldMemOperand(expected_parameter_count,
2252
                       SharedFunctionInfo::kFormalParameterCountOffset));
2253

2254 2255
  InvokeFunctionCode(function, new_target, expected_parameter_count,
                     actual_parameter_count, flag);
2256 2257 2258
}

void MacroAssembler::InvokeFunction(Register function,
2259 2260
                                    Register expected_parameter_count,
                                    Register actual_parameter_count,
2261
                                    InvokeFlag flag) {
2262
  // You can't call a function without a valid frame.
2263
  DCHECK(flag == JUMP_FUNCTION || has_frame());
2264 2265 2266

  // Contract with called JS functions requires that function is passed in x1.
  // (See FullCodeGenerator::Generate().)
2267
  DCHECK_EQ(function, x1);
2268 2269

  // Set up the context.
2270 2271
  LoadTaggedPointerField(cp,
                         FieldMemOperand(function, JSFunction::kContextOffset));
2272

2273 2274
  InvokeFunctionCode(function, no_reg, expected_parameter_count,
                     actual_parameter_count, flag);
2275 2276
}

2277
void TurboAssembler::TryConvertDoubleToInt64(Register result,
2278 2279 2280
                                             DoubleRegister double_input,
                                             Label* done) {
  // Try to convert with an FPU convert instruction. It's trivial to compute
2281
  // the modulo operation on an integer register so we convert to a 64-bit
2282
  // integer.
2283
  //
2284
  // Fcvtzs will saturate to INT64_MIN (0x800...00) or INT64_MAX (0x7FF...FF)
2285 2286
  // when the double is out of range. NaNs and infinities will be converted to 0
  // (as ECMA-262 requires).
2287
  Fcvtzs(result.X(), double_input);
2288

2289
  // The values INT64_MIN (0x800...00) or INT64_MAX (0x7FF...FF) are not
2290
  // representable using a double, so if the result is one of those then we know
2291
  // that saturation occurred, and we need to manually handle the conversion.
2292 2293 2294
  //
  // It is easy to detect INT64_MIN and INT64_MAX because adding or subtracting
  // 1 will cause signed overflow.
2295 2296
  Cmp(result.X(), 1);
  Ccmp(result.X(), -1, VFlag, vc);
2297

2298 2299
  B(vc, done);
}
2300

2301 2302
void TurboAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone,
                                       Register result,
2303 2304
                                       DoubleRegister double_input,
                                       StubCallMode stub_mode) {
2305
  Label done;
2306

2307 2308 2309
  // Try to convert the double to an int64. If successful, the bottom 32 bits
  // contain our truncated int32 result.
  TryConvertDoubleToInt64(result, double_input, &done);
2310

2311
  // If we fell through then inline version didn't succeed - call stub instead.
2312
  Push(lr, double_input);
2313

2314
  // DoubleToI preserves any registers it needs to clobber.
2315 2316
  if (stub_mode == StubCallMode::kCallWasmRuntimeStub) {
    Call(wasm::WasmCode::kDoubleToI, RelocInfo::WASM_STUB_CALL);
2317 2318
  } else if (options().inline_offheap_trampolines) {
    CallBuiltin(Builtins::kDoubleToI);
2319 2320 2321
  } else {
    Call(BUILTIN_CODE(isolate, DoubleToI), RelocInfo::CODE_TARGET);
  }
2322
  Ldr(result, MemOperand(sp, 0));
2323

2324 2325 2326
  DCHECK_EQ(xzr.SizeInBytes(), double_input.SizeInBytes());
  Pop(xzr, lr);  // xzr to drop the double input on the stack.

2327
  Bind(&done);
2328 2329
  // Keep our invariant that the upper 32 bits are zero.
  Uxtw(result.W(), result.W());
2330 2331
}

2332 2333
void TurboAssembler::Prologue() {
  Push(lr, fp, cp, x1);
2334
  Add(fp, sp, StandardFrameConstants::kFixedFrameSizeFromFp);
2335 2336
}

2337
void TurboAssembler::EnterFrame(StackFrame::Type type) {
2338 2339
  UseScratchRegisterScope temps(this);

2340
  if (type == StackFrame::INTERNAL) {
2341
    Register type_reg = temps.AcquireX();
2342
    Mov(type_reg, StackFrame::TypeToMarker(type));
2343 2344 2345
    // type_reg pushed twice for alignment.
    Push(lr, fp, type_reg, type_reg);
    const int kFrameSize =
2346
        TypedFrameConstants::kFixedFrameSizeFromFp + kSystemPointerSize;
2347
    Add(fp, sp, kFrameSize);
2348 2349
    // sp[3] : lr
    // sp[2] : fp
2350
    // sp[1] : type
2351
    // sp[0] : for alignment
2352
  } else if (type == StackFrame::WASM_COMPILED ||
2353 2354
             type == StackFrame::WASM_COMPILE_LAZY ||
             type == StackFrame::WASM_EXIT) {
2355
    Register type_reg = temps.AcquireX();
2356
    Mov(type_reg, StackFrame::TypeToMarker(type));
2357
    Push(lr, fp);
2358
    Mov(fp, sp);
2359
    Push(type_reg, padreg);
2360 2361 2362 2363
    // sp[3] : lr
    // sp[2] : fp
    // sp[1] : type
    // sp[0] : for alignment
2364
  } else {
2365
    DCHECK_EQ(type, StackFrame::CONSTRUCT);
2366
    Register type_reg = temps.AcquireX();
2367
    Mov(type_reg, StackFrame::TypeToMarker(type));
2368 2369 2370 2371 2372 2373 2374

    // Users of this frame type push a context pointer after the type field,
    // so do it here to keep the stack pointer aligned.
    Push(lr, fp, type_reg, cp);

    // The context pointer isn't part of the fixed frame, so add an extra slot
    // to account for it.
2375 2376
    Add(fp, sp,
        TypedFrameConstants::kFixedFrameSizeFromFp + kSystemPointerSize);
2377 2378 2379 2380
    // sp[3] : lr
    // sp[2] : fp
    // sp[1] : type
    // sp[0] : cp
2381
  }
2382 2383
}

2384
void TurboAssembler::LeaveFrame(StackFrame::Type type) {
2385 2386 2387 2388
  // Drop the execution stack down to the frame pointer and restore
  // the caller frame pointer and return address.
  Mov(sp, fp);
  Pop(fp, lr);
2389 2390 2391
}

void MacroAssembler::ExitFramePreserveFPRegs() {
2392
  DCHECK_EQ(kCallerSavedV.Count() % 2, 0);
2393
  PushCPURegList(kCallerSavedV);
2394 2395 2396 2397 2398
}

void MacroAssembler::ExitFrameRestoreFPRegs() {
  // Read the registers from the stack without popping them. The stack pointer
  // will be reset as part of the unwinding process.
2399
  CPURegList saved_fp_regs = kCallerSavedV;
2400
  DCHECK_EQ(saved_fp_regs.Count() % 2, 0);
2401 2402 2403 2404 2405

  int offset = ExitFrameConstants::kLastExitFrameField;
  while (!saved_fp_regs.IsEmpty()) {
    const CPURegister& dst0 = saved_fp_regs.PopHighestIndex();
    const CPURegister& dst1 = saved_fp_regs.PopHighestIndex();
2406
    offset -= 2 * kDRegSize;
2407 2408 2409 2410
    Ldp(dst1, dst0, MemOperand(fp, offset));
  }
}

2411 2412 2413 2414 2415
void MacroAssembler::EnterExitFrame(bool save_doubles, const Register& scratch,
                                    int extra_space,
                                    StackFrame::Type frame_type) {
  DCHECK(frame_type == StackFrame::EXIT ||
         frame_type == StackFrame::BUILTIN_EXIT);
2416 2417 2418

  // Set up the new stack frame.
  Push(lr, fp);
2419
  Mov(fp, sp);
2420
  Mov(scratch, StackFrame::TypeToMarker(frame_type));
2421
  Push(scratch, xzr);
2422 2423
  //          fp[8]: CallerPC (lr)
  //    fp -> fp[0]: CallerFP (old fp)
2424
  //          fp[-8]: STUB marker
2425
  //    sp -> fp[-16]: Space reserved for SPOffset.
2426 2427 2428 2429 2430 2431 2432
  STATIC_ASSERT((2 * kSystemPointerSize) ==
                ExitFrameConstants::kCallerSPOffset);
  STATIC_ASSERT((1 * kSystemPointerSize) ==
                ExitFrameConstants::kCallerPCOffset);
  STATIC_ASSERT((0 * kSystemPointerSize) ==
                ExitFrameConstants::kCallerFPOffset);
  STATIC_ASSERT((-2 * kSystemPointerSize) == ExitFrameConstants::kSPOffset);
2433 2434

  // Save the frame pointer and context pointer in the top frame.
2435 2436
  Mov(scratch,
      ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate()));
2437
  Str(fp, MemOperand(scratch));
2438 2439
  Mov(scratch,
      ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
2440 2441
  Str(cp, MemOperand(scratch));

2442
  STATIC_ASSERT((-2 * kSystemPointerSize) ==
2443
                ExitFrameConstants::kLastExitFrameField);
2444 2445 2446 2447
  if (save_doubles) {
    ExitFramePreserveFPRegs();
  }

2448 2449 2450
  // Round the number of space we need to claim to a multiple of two.
  int slots_to_claim = RoundUp(extra_space + 1, 2);

2451 2452 2453
  // Reserve space for the return address and for user requested memory.
  // We do this before aligning to make sure that we end up correctly
  // aligned with the minimum of wasted space.
2454
  Claim(slots_to_claim, kXRegSize);
2455 2456
  //         fp[8]: CallerPC (lr)
  //   fp -> fp[0]: CallerFP (old fp)
2457 2458
  //         fp[-8]: STUB marker
  //         fp[-16]: Space reserved for SPOffset.
2459
  //         fp[-16 - fp_size]: Saved doubles (if save_doubles is true).
2460 2461
  //         sp[8]: Extra space reserved for caller (if extra_space != 0).
  //   sp -> sp[0]: Space reserved for the return address.
2462 2463 2464 2465 2466

  // ExitFrame::GetStateForFramePointer expects to find the return address at
  // the memory address immediately below the pointer stored in SPOffset.
  // It is not safe to derive much else from SPOffset, because the size of the
  // padding can vary.
2467
  Add(scratch, sp, kXRegSize);
2468 2469 2470 2471 2472
  Str(scratch, MemOperand(fp, ExitFrameConstants::kSPOffset));
}

// Leave the current exit frame.
void MacroAssembler::LeaveExitFrame(bool restore_doubles,
2473 2474
                                    const Register& scratch,
                                    const Register& scratch2) {
2475 2476 2477 2478 2479
  if (restore_doubles) {
    ExitFrameRestoreFPRegs();
  }

  // Restore the context pointer from the top frame.
2480 2481
  Mov(scratch,
      ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
2482
  Ldr(cp, MemOperand(scratch));
2483 2484 2485

  if (emit_debug_code()) {
    // Also emit debug code to clear the cp in the top frame.
2486
    Mov(scratch2, Operand(Context::kInvalidContext));
2487 2488
    Mov(scratch, ExternalReference::Create(IsolateAddressId::kContextAddress,
                                           isolate()));
2489
    Str(scratch2, MemOperand(scratch));
2490 2491
  }
  // Clear the frame pointer from the top frame.
2492 2493
  Mov(scratch,
      ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate()));
2494 2495 2496 2497 2498 2499
  Str(xzr, MemOperand(scratch));

  // Pop the exit frame.
  //         fp[8]: CallerPC (lr)
  //   fp -> fp[0]: CallerFP (old fp)
  //         fp[...]: The rest of the frame.
2500
  Mov(sp, fp);
2501 2502 2503
  Pop(fp, lr);
}

2504 2505 2506 2507
void MacroAssembler::LoadGlobalProxy(Register dst) {
  LoadNativeContextSlot(Context::GLOBAL_PROXY_INDEX, dst);
}

2508 2509
void MacroAssembler::LoadWeakValue(Register out, Register in,
                                   Label* target_if_cleared) {
2510 2511
  CompareAndBranch(in.W(), Operand(kClearedWeakHeapObjectLower32), eq,
                   target_if_cleared);
2512 2513 2514

  and_(out, in, Operand(~kWeakHeapObjectMask));
}
2515 2516 2517

void MacroAssembler::IncrementCounter(StatsCounter* counter, int value,
                                      Register scratch1, Register scratch2) {
2518
  DCHECK_NE(value, 0);
2519
  if (FLAG_native_code_counters && counter->Enabled()) {
2520 2521 2522
    // This operation has to be exactly 32-bit wide in case the external
    // reference table redirects the counter to a uint32_t dummy_stats_counter_
    // field.
2523
    Mov(scratch2, ExternalReference::Create(counter));
2524 2525 2526
    Ldr(scratch1.W(), MemOperand(scratch2));
    Add(scratch1.W(), scratch1.W(), value);
    Str(scratch1.W(), MemOperand(scratch2));
2527 2528 2529 2530 2531 2532 2533 2534
  }
}

void MacroAssembler::DecrementCounter(StatsCounter* counter, int value,
                                      Register scratch1, Register scratch2) {
  IncrementCounter(counter, -value, scratch1, scratch2);
}

2535 2536
void MacroAssembler::MaybeDropFrames() {
  // Check whether we need to drop frames to restart a function on the stack.
2537
  Mov(x1, ExternalReference::debug_restart_fp_address(isolate()));
2538 2539
  Ldr(x1, MemOperand(x1));
  Tst(x1, x1);
2540
  Jump(BUILTIN_CODE(isolate(), FrameDropperTrampoline), RelocInfo::CODE_TARGET,
2541 2542
       ne);
}
2543

2544 2545 2546
void MacroAssembler::JumpIfObjectType(Register object, Register map,
                                      Register type_reg, InstanceType type,
                                      Label* if_cond_pass, Condition cond) {
2547 2548 2549 2550 2551
  CompareObjectType(object, map, type_reg, type);
  B(cond, if_cond_pass);
}

// Sets condition flags based on comparison, and returns type in type_reg.
2552 2553
void MacroAssembler::CompareObjectType(Register object, Register map,
                                       Register type_reg, InstanceType type) {
2554
  LoadMap(map, object);
2555 2556 2557
  CompareInstanceType(map, type_reg, type);
}

2558 2559 2560 2561
void MacroAssembler::LoadMap(Register dst, Register object) {
  LoadTaggedPointerField(dst, FieldMemOperand(object, HeapObject::kMapOffset));
}

2562
// Sets condition flags based on comparison, and returns type in type_reg.
2563
void MacroAssembler::CompareInstanceType(Register map, Register type_reg,
2564
                                         InstanceType type) {
2565
  Ldrh(type_reg, FieldMemOperand(map, Map::kInstanceTypeOffset));
2566 2567 2568
  Cmp(type_reg, type);
}

2569
void MacroAssembler::LoadElementsKindFromMap(Register result, Register map) {
2570
  // Load the map's "bit field 2".
2571
  Ldrb(result, FieldMemOperand(map, Map::kBitField2Offset));
2572
  // Retrieve elements_kind from bit field 2.
2573
  DecodeField<Map::ElementsKindBits>(result);
2574 2575
}

2576
void MacroAssembler::CompareRoot(const Register& obj, RootIndex index) {
2577 2578
  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireX();
2579
  DCHECK(!AreAliased(obj, temp));
2580
  LoadRoot(temp, index);
2581
  CmpTagged(obj, temp);
2582 2583
}

2584
void MacroAssembler::JumpIfRoot(const Register& obj, RootIndex index,
2585 2586 2587 2588 2589
                                Label* if_equal) {
  CompareRoot(obj, index);
  B(eq, if_equal);
}

2590
void MacroAssembler::JumpIfNotRoot(const Register& obj, RootIndex index,
2591 2592 2593 2594 2595
                                   Label* if_not_equal) {
  CompareRoot(obj, index);
  B(ne, if_not_equal);
}

2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611
void MacroAssembler::JumpIfIsInRange(const Register& value,
                                     unsigned lower_limit,
                                     unsigned higher_limit,
                                     Label* on_in_range) {
  if (lower_limit != 0) {
    UseScratchRegisterScope temps(this);
    Register scratch = temps.AcquireW();
    Sub(scratch, value, Operand(lower_limit));
    CompareAndBranch(scratch, Operand(higher_limit - lower_limit), ls,
                     on_in_range);
  } else {
    CompareAndBranch(value, Operand(higher_limit - lower_limit), ls,
                     on_in_range);
  }
}

2612 2613
void TurboAssembler::LoadTaggedPointerField(const Register& destination,
                                            const MemOperand& field_operand) {
2614 2615 2616 2617 2618
  if (COMPRESS_POINTERS_BOOL) {
    DecompressTaggedPointer(destination, field_operand);
  } else {
    Ldr(destination, field_operand);
  }
2619 2620 2621 2622
}

void TurboAssembler::LoadAnyTaggedField(const Register& destination,
                                        const MemOperand& field_operand) {
2623 2624 2625 2626 2627
  if (COMPRESS_POINTERS_BOOL) {
    DecompressAnyTagged(destination, field_operand);
  } else {
    Ldr(destination, field_operand);
  }
2628 2629
}

2630 2631 2632 2633
void TurboAssembler::SmiUntagField(Register dst, const MemOperand& src) {
  SmiUntag(dst, src);
}

2634 2635
void TurboAssembler::StoreTaggedField(const Register& value,
                                      const MemOperand& dst_field_operand) {
2636 2637 2638 2639 2640
  if (COMPRESS_POINTERS_BOOL) {
    Str(value.W(), dst_field_operand);
  } else {
    Str(value, dst_field_operand);
  }
2641 2642
}

2643 2644 2645
void TurboAssembler::DecompressTaggedSigned(const Register& destination,
                                            const MemOperand& field_operand) {
  RecordComment("[ DecompressTaggedSigned");
2646
  Ldr(destination.W(), field_operand);
2647 2648 2649
  RecordComment("]");
}

2650 2651 2652
void TurboAssembler::DecompressTaggedSigned(const Register& destination,
                                            const Register& source) {
  RecordComment("[ DecompressTaggedSigned");
2653
  Mov(destination.W(), source.W());
2654 2655 2656
  RecordComment("]");
}

2657 2658 2659
void TurboAssembler::DecompressTaggedPointer(const Register& destination,
                                             const MemOperand& field_operand) {
  RecordComment("[ DecompressTaggedPointer");
2660
  Ldr(destination.W(), field_operand);
2661
  Add(destination, kRootRegister, destination);
2662 2663 2664
  RecordComment("]");
}

2665 2666 2667
void TurboAssembler::DecompressTaggedPointer(const Register& destination,
                                             const Register& source) {
  RecordComment("[ DecompressTaggedPointer");
2668
  Add(destination, kRootRegister, Operand(source, UXTW));
2669 2670 2671
  RecordComment("]");
}

2672 2673 2674
void TurboAssembler::DecompressAnyTagged(const Register& destination,
                                         const MemOperand& field_operand) {
  RecordComment("[ DecompressAnyTagged");
2675
  Ldr(destination.W(), field_operand);
2676
  Add(destination, kRootRegister, destination);
2677 2678
  RecordComment("]");
}
2679

2680 2681 2682
void TurboAssembler::DecompressAnyTagged(const Register& destination,
                                         const Register& source) {
  RecordComment("[ DecompressAnyTagged");
2683
  Add(destination, kRootRegister, Operand(source, UXTW));
2684 2685 2686
  RecordComment("]");
}

2687 2688
int MacroAssembler::SafepointRegisterStackIndex(int reg_code) {
  // Make sure the safepoint registers list is what we expect.
2689
  DCHECK_EQ(CPURegList::GetSafepointSavedRegisters().list(), 0x6FFCFFFF);
2690 2691 2692 2693 2694

  // Safepoint registers are stored contiguously on the stack, but not all the
  // registers are saved. The following registers are excluded:
  //  - x16 and x17 (ip0 and ip1) because they shouldn't be preserved outside of
  //    the macro assembler.
2695
  //  - x31 (sp) because the system stack pointer doesn't need to be included
2696 2697 2698 2699 2700 2701
  //    in safepoint registers.
  //
  // This function implements the mapping of register code to index into the
  // safepoint register slots.
  if ((reg_code >= 0) && (reg_code <= 15)) {
    return reg_code;
2702
  } else if ((reg_code >= 18) && (reg_code <= 30)) {
2703 2704 2705 2706 2707 2708 2709 2710
    // Skip ip0 and ip1.
    return reg_code - 2;
  } else {
    // This register has no safepoint register slot.
    UNREACHABLE();
  }
}

2711
void TurboAssembler::CheckPageFlag(const Register& object, int mask,
2712
                                   Condition cc, Label* condition_met) {
2713 2714
  UseScratchRegisterScope temps(this);
  Register scratch = temps.AcquireX();
2715
  And(scratch, object, ~kPageAlignmentMask);
2716 2717 2718 2719
  Ldr(scratch, MemOperand(scratch, MemoryChunk::kFlagsOffset));
  if (cc == eq) {
    TestAndBranchIfAnySet(scratch, mask, condition_met);
  } else {
2720
    DCHECK_EQ(cc, ne);
2721 2722 2723
    TestAndBranchIfAllClear(scratch, mask, condition_met);
  }
}
2724

2725
void MacroAssembler::RecordWriteField(Register object, int offset,
2726
                                      Register value,
2727 2728 2729 2730
                                      LinkRegisterStatus lr_status,
                                      SaveFPRegsMode save_fp,
                                      RememberedSetAction remembered_set_action,
                                      SmiCheck smi_check) {
2731 2732 2733 2734 2735 2736 2737 2738 2739 2740
  // First, check if a write barrier is even needed. The tests below
  // catch stores of Smis.
  Label done;

  // Skip the barrier if writing a smi.
  if (smi_check == INLINE_SMI_CHECK) {
    JumpIfSmi(value, &done);
  }

  // Although the object register is tagged, the offset is relative to the start
2741 2742
  // of the object, so offset must be a multiple of kTaggedSize.
  DCHECK(IsAligned(offset, kTaggedSize));
2743 2744 2745

  if (emit_debug_code()) {
    Label ok;
2746 2747 2748
    UseScratchRegisterScope temps(this);
    Register scratch = temps.AcquireX();
    Add(scratch, object, offset - kHeapObjectTag);
2749
    Tst(scratch, kTaggedSize - 1);
2750
    B(eq, &ok);
2751
    Abort(AbortReason::kUnalignedCellInWriteBarrier);
2752 2753 2754
    Bind(&ok);
  }

2755 2756
  RecordWrite(object, Operand(offset - kHeapObjectTag), value, lr_status,
              save_fp, remembered_set_action, OMIT_SMI_CHECK);
2757 2758 2759 2760

  Bind(&done);
}

2761
void TurboAssembler::SaveRegisters(RegList registers) {
2762
  DCHECK_GT(NumRegs(registers), 0);
2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773
  CPURegList regs(lr);
  for (int i = 0; i < Register::kNumRegisters; ++i) {
    if ((registers >> i) & 1u) {
      regs.Combine(Register::XRegFromCode(i));
    }
  }

  PushCPURegList(regs);
}

void TurboAssembler::RestoreRegisters(RegList registers) {
2774
  DCHECK_GT(NumRegs(registers), 0);
2775 2776 2777 2778 2779 2780 2781 2782 2783 2784
  CPURegList regs(lr);
  for (int i = 0; i < Register::kNumRegisters; ++i) {
    if ((registers >> i) & 1u) {
      regs.Combine(Register::XRegFromCode(i));
    }
  }

  PopCPURegList(regs);
}

2785
void TurboAssembler::CallEphemeronKeyBarrier(Register object, Operand offset,
2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798
                                             SaveFPRegsMode fp_mode) {
  EphemeronKeyBarrierDescriptor descriptor;
  RegList registers = descriptor.allocatable_registers();

  SaveRegisters(registers);

  Register object_parameter(
      descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kObject));
  Register slot_parameter(descriptor.GetRegisterParameter(
      EphemeronKeyBarrierDescriptor::kSlotAddress));
  Register fp_mode_parameter(
      descriptor.GetRegisterParameter(EphemeronKeyBarrierDescriptor::kFPMode));

2799
  MoveObjectAndSlot(object_parameter, slot_parameter, object, offset);
2800 2801 2802 2803 2804 2805 2806

  Mov(fp_mode_parameter, Smi::FromEnum(fp_mode));
  Call(isolate()->builtins()->builtin_handle(Builtins::kEphemeronKeyBarrier),
       RelocInfo::CODE_TARGET);
  RestoreRegisters(registers);
}

2807
void TurboAssembler::CallRecordWriteStub(
2808 2809
    Register object, Operand offset, RememberedSetAction remembered_set_action,
    SaveFPRegsMode fp_mode) {
2810
  CallRecordWriteStub(
2811
      object, offset, remembered_set_action, fp_mode,
2812 2813 2814 2815 2816
      isolate()->builtins()->builtin_handle(Builtins::kRecordWrite),
      kNullAddress);
}

void TurboAssembler::CallRecordWriteStub(
2817 2818 2819
    Register object, Operand offset, RememberedSetAction remembered_set_action,
    SaveFPRegsMode fp_mode, Address wasm_target) {
  CallRecordWriteStub(object, offset, remembered_set_action, fp_mode,
2820 2821 2822 2823
                      Handle<Code>::null(), wasm_target);
}

void TurboAssembler::CallRecordWriteStub(
2824 2825
    Register object, Operand offset, RememberedSetAction remembered_set_action,
    SaveFPRegsMode fp_mode, Handle<Code> code_target, Address wasm_target) {
2826
  DCHECK_NE(code_target.is_null(), wasm_target == kNullAddress);
2827 2828 2829 2830 2831
  // TODO(albertnetymk): For now we ignore remembered_set_action and fp_mode,
  // i.e. always emit remember set and save FP registers in RecordWriteStub. If
  // large performance regression is observed, we should use these values to
  // avoid unnecessary work.

2832 2833
  RecordWriteDescriptor descriptor;
  RegList registers = descriptor.allocatable_registers();
2834 2835 2836

  SaveRegisters(registers);

2837 2838
  Register object_parameter(
      descriptor.GetRegisterParameter(RecordWriteDescriptor::kObject));
2839
  Register slot_parameter(
2840 2841 2842 2843 2844
      descriptor.GetRegisterParameter(RecordWriteDescriptor::kSlot));
  Register remembered_set_parameter(
      descriptor.GetRegisterParameter(RecordWriteDescriptor::kRememberedSet));
  Register fp_mode_parameter(
      descriptor.GetRegisterParameter(RecordWriteDescriptor::kFPMode));
2845

2846
  MoveObjectAndSlot(object_parameter, slot_parameter, object, offset);
2847

2848 2849
  Mov(remembered_set_parameter, Smi::FromEnum(remembered_set_action));
  Mov(fp_mode_parameter, Smi::FromEnum(fp_mode));
2850 2851 2852 2853 2854
  if (code_target.is_null()) {
    Call(wasm_target, RelocInfo::WASM_STUB_CALL);
  } else {
    Call(code_target, RelocInfo::CODE_TARGET);
  }
2855 2856 2857

  RestoreRegisters(registers);
}
2858

2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892
void TurboAssembler::MoveObjectAndSlot(Register dst_object, Register dst_slot,
                                       Register object, Operand offset) {
  DCHECK_NE(dst_object, dst_slot);
  // If `offset` is a register, it cannot overlap with `object`.
  DCHECK_IMPLIES(!offset.IsImmediate(), offset.reg() != object);

  // If the slot register does not overlap with the object register, we can
  // overwrite it.
  if (dst_slot != object) {
    Add(dst_slot, object, offset);
    Mov(dst_object, object);
    return;
  }

  DCHECK_EQ(dst_slot, object);

  // If the destination object register does not overlap with the offset
  // register, we can overwrite it.
  if (offset.IsImmediate() || (offset.reg() != dst_object)) {
    Mov(dst_object, dst_slot);
    Add(dst_slot, dst_slot, offset);
    return;
  }

  DCHECK_EQ(dst_object, offset.reg());

  // We only have `dst_slot` and `dst_object` left as distinct registers so we
  // have to swap them. We write this as a add+sub sequence to avoid using a
  // scratch register.
  Add(dst_slot, dst_slot, dst_object);
  Sub(dst_object, dst_slot, dst_object);
}

// If lr_status is kLRHasBeenSaved, lr will be clobbered.
2893 2894 2895
//
// The register 'object' contains a heap object pointer. The heap object tag is
// shifted away.
2896
void MacroAssembler::RecordWrite(Register object, Operand offset,
2897 2898 2899 2900
                                 Register value, LinkRegisterStatus lr_status,
                                 SaveFPRegsMode fp_mode,
                                 RememberedSetAction remembered_set_action,
                                 SmiCheck smi_check) {
2901
  ASM_LOCATION_IN_ASSEMBLER("MacroAssembler::RecordWrite");
2902
  DCHECK(!AreAliased(object, value));
2903 2904

  if (emit_debug_code()) {
2905 2906 2907
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireX();

2908 2909
    Add(temp, object, offset);
    LoadTaggedPointerField(temp, MemOperand(temp));
2910
    Cmp(temp, value);
2911
    Check(eq, AbortReason::kWrongAddressOrValuePassedToRecordWrite);
2912 2913
  }

2914 2915 2916 2917 2918 2919
  if ((remembered_set_action == OMIT_REMEMBERED_SET &&
       !FLAG_incremental_marking) ||
      FLAG_disable_write_barriers) {
    return;
  }

2920 2921 2922 2923 2924
  // First, check if a write barrier is even needed. The tests below
  // catch stores of smis and stores into the young generation.
  Label done;

  if (smi_check == INLINE_SMI_CHECK) {
2925
    DCHECK_EQ(0, kSmiTag);
2926 2927
    JumpIfSmi(value, &done);
  }
2928 2929
  CheckPageFlag(value, MemoryChunk::kPointersToHereAreInterestingMask, ne,
                &done);
2930

2931 2932
  CheckPageFlag(object, MemoryChunk::kPointersFromHereAreInterestingMask, ne,
                &done);
2933 2934 2935

  // Record the actual write.
  if (lr_status == kLRHasNotBeenSaved) {
2936
    Push(padreg, lr);
2937
  }
2938
  CallRecordWriteStub(object, offset, remembered_set_action, fp_mode);
2939
  if (lr_status == kLRHasNotBeenSaved) {
2940
    Pop(lr, padreg);
2941 2942 2943 2944 2945
  }

  Bind(&done);
}

2946
void TurboAssembler::Assert(Condition cond, AbortReason reason) {
2947 2948 2949 2950 2951
  if (emit_debug_code()) {
    Check(cond, reason);
  }
}

2952 2953 2954 2955
void TurboAssembler::AssertUnreachable(AbortReason reason) {
  if (emit_debug_code()) Abort(reason);
}

2956
void MacroAssembler::AssertRegisterIsRoot(Register reg, RootIndex index,
2957
                                          AbortReason reason) {
2958 2959 2960 2961 2962 2963
  if (emit_debug_code()) {
    CompareRoot(reg, index);
    Check(eq, reason);
  }
}

2964
void TurboAssembler::Check(Condition cond, AbortReason reason) {
2965 2966 2967 2968 2969 2970 2971
  Label ok;
  B(cond, &ok);
  Abort(reason);
  // Will not return here.
  Bind(&ok);
}

2972 2973
void TurboAssembler::Trap() { Brk(0); }

2974
void TurboAssembler::Abort(AbortReason reason) {
2975 2976
#ifdef DEBUG
  RecordComment("Abort message: ");
2977
  RecordComment(GetAbortReason(reason));
2978
#endif
2979

2980 2981
  // Avoid emitting call to builtin if requested.
  if (trap_on_abort()) {
2982 2983 2984 2985
    Brk(0);
    return;
  }

2986 2987 2988
  // We need some scratch registers for the MacroAssembler, so make sure we have
  // some. This is safe here because Abort never returns.
  RegList old_tmp_list = TmpList()->list();
2989
  TmpList()->Combine(MacroAssembler::DefaultTmpList());
2990

2991 2992 2993 2994 2995 2996 2997
  if (should_abort_hard()) {
    // We don't care if we constructed a frame. Just pretend we did.
    FrameScope assume_frame(this, StackFrame::NONE);
    Mov(w0, static_cast<int>(reason));
    Call(ExternalReference::abort_with_reason());
    return;
  }
2998

2999 3000
  // Avoid infinite recursion; Push contains some assertions that use Abort.
  HardAbortScope hard_aborts(this);
3001

3002
  Mov(x1, Smi::FromInt(static_cast<int>(reason)));
3003 3004 3005 3006 3007 3008

  if (!has_frame_) {
    // We don't actually want to generate a pile of code for this, so just
    // claim there is a stack frame, without generating one.
    FrameScope scope(this, StackFrame::NONE);
    Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
3009
  } else {
3010
    Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
3011
  }
3012

3013
  TmpList()->set_list(old_tmp_list);
3014 3015
}

3016
void MacroAssembler::LoadNativeContextSlot(int index, Register dst) {
3017 3018 3019 3020 3021
  LoadMap(dst, cp);
  LoadTaggedPointerField(
      dst, FieldMemOperand(
               dst, Map::kConstructorOrBackPointerOrNativeContextOffset));
  LoadTaggedPointerField(dst, MemOperand(dst, Context::SlotOffset(index)));
3022 3023 3024 3025
}

// This is the main Printf implementation. All other Printf variants call
// PrintfNoPreserve after setting up one or more PreserveRegisterScopes.
3026
void TurboAssembler::PrintfNoPreserve(const char* format,
3027 3028 3029 3030 3031 3032
                                      const CPURegister& arg0,
                                      const CPURegister& arg1,
                                      const CPURegister& arg2,
                                      const CPURegister& arg3) {
  // We cannot handle a caller-saved stack pointer. It doesn't make much sense
  // in most cases anyway, so this restriction shouldn't be too serious.
3033
  DCHECK(!kCallerSaved.IncludesAliasOf(sp));
3034

3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045
  // The provided arguments, and their proper procedure-call standard registers.
  CPURegister args[kPrintfMaxArgCount] = {arg0, arg1, arg2, arg3};
  CPURegister pcs[kPrintfMaxArgCount] = {NoReg, NoReg, NoReg, NoReg};

  int arg_count = kPrintfMaxArgCount;

  // The PCS varargs registers for printf. Note that x0 is used for the printf
  // format string.
  static const CPURegList kPCSVarargs =
      CPURegList(CPURegister::kRegister, kXRegSizeInBits, 1, arg_count);
  static const CPURegList kPCSVarargsFP =
3046
      CPURegList(CPURegister::kVRegister, kDRegSizeInBits, 0, arg_count - 1);
3047 3048 3049 3050

  // We can use caller-saved registers as scratch values, except for the
  // arguments and the PCS registers where they might need to go.
  CPURegList tmp_list = kCallerSaved;
3051
  tmp_list.Remove(x0);  // Used to pass the format string.
3052 3053 3054
  tmp_list.Remove(kPCSVarargs);
  tmp_list.Remove(arg0, arg1, arg2, arg3);

3055
  CPURegList fp_tmp_list = kCallerSavedV;
3056 3057 3058
  fp_tmp_list.Remove(kPCSVarargsFP);
  fp_tmp_list.Remove(arg0, arg1, arg2, arg3);

3059
  // Override the TurboAssembler's scratch register list. The lists will be
3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074
  // reset automatically at the end of the UseScratchRegisterScope.
  UseScratchRegisterScope temps(this);
  TmpList()->set_list(tmp_list.list());
  FPTmpList()->set_list(fp_tmp_list.list());

  // Copies of the printf vararg registers that we can pop from.
  CPURegList pcs_varargs = kPCSVarargs;
  CPURegList pcs_varargs_fp = kPCSVarargsFP;

  // Place the arguments. There are lots of clever tricks and optimizations we
  // could use here, but Printf is a debug tool so instead we just try to keep
  // it simple: Move each input that isn't already in the right place to a
  // scratch register, then move everything back.
  for (unsigned i = 0; i < kPrintfMaxArgCount; i++) {
    // Work out the proper PCS register for this argument.
3075
    if (args[i].IsRegister()) {
3076 3077 3078 3079
      pcs[i] = pcs_varargs.PopLowestIndex().X();
      // We might only need a W register here. We need to know the size of the
      // argument so we can properly encode it for the simulator call.
      if (args[i].Is32Bits()) pcs[i] = pcs[i].W();
3080
    } else if (args[i].IsVRegister()) {
3081 3082
      // In C, floats are always cast to doubles for varargs calls.
      pcs[i] = pcs_varargs_fp.PopLowestIndex().D();
3083
    } else {
3084
      DCHECK(args[i].IsNone());
3085 3086 3087 3088
      arg_count = i;
      break;
    }

3089 3090 3091 3092 3093 3094 3095 3096
    // If the argument is already in the right place, leave it where it is.
    if (args[i].Aliases(pcs[i])) continue;

    // Otherwise, if the argument is in a PCS argument register, allocate an
    // appropriate scratch register and then move it out of the way.
    if (kPCSVarargs.IncludesAliasOf(args[i]) ||
        kPCSVarargsFP.IncludesAliasOf(args[i])) {
      if (args[i].IsRegister()) {
3097
        Register old_arg = args[i].Reg();
3098 3099 3100 3101
        Register new_arg = temps.AcquireSameSizeAs(old_arg);
        Mov(new_arg, old_arg);
        args[i] = new_arg;
      } else {
3102
        VRegister old_arg = args[i].VReg();
3103
        VRegister new_arg = temps.AcquireSameSizeAs(old_arg);
3104 3105 3106
        Fmov(new_arg, old_arg);
        args[i] = new_arg;
      }
3107 3108 3109
    }
  }

3110 3111 3112
  // Do a second pass to move values into their final positions and perform any
  // conversions that may be required.
  for (int i = 0; i < arg_count; i++) {
3113
    DCHECK(pcs[i].type() == args[i].type());
3114
    if (pcs[i].IsRegister()) {
3115
      Mov(pcs[i].Reg(), args[i].Reg(), kDiscardForSameWReg);
3116
    } else {
3117
      DCHECK(pcs[i].IsVRegister());
3118
      if (pcs[i].SizeInBytes() == args[i].SizeInBytes()) {
3119
        Fmov(pcs[i].VReg(), args[i].VReg());
3120
      } else {
3121
        Fcvt(pcs[i].VReg(), args[i].VReg());
3122 3123
      }
    }
3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135
  }

  // Load the format string into x0, as per the procedure-call standard.
  //
  // To make the code as portable as possible, the format string is encoded
  // directly in the instruction stream. It might be cleaner to encode it in a
  // literal pool, but since Printf is usually used for debugging, it is
  // beneficial for it to be minimally dependent on other features.
  Label format_address;
  Adr(x0, &format_address);

  // Emit the format string directly in the instruction stream.
3136 3137
  {
    BlockPoolsScope scope(this);
3138 3139 3140 3141 3142 3143 3144 3145
    Label after_data;
    B(&after_data);
    Bind(&format_address);
    EmitStringData(format);
    Unreachable();
    Bind(&after_data);
  }

3146
  CallPrintf(arg_count, pcs);
3147 3148
}

3149 3150 3151 3152
void TurboAssembler::CallPrintf(int arg_count, const CPURegister* args) {
// A call to printf needs special handling for the simulator, since the system
// printf function will use a different instruction set and the procedure-call
// standard will not be compatible.
3153
#ifdef USE_SIMULATOR
3154 3155
  {
    InstructionAccurateScope scope(this, kPrintfLength / kInstrSize);
3156
    hlt(kImmExceptionIsPrintf);
3157
    dc32(arg_count);  // kPrintfArgCountOffset
3158 3159 3160 3161 3162 3163 3164 3165

    // Determine the argument pattern.
    uint32_t arg_pattern_list = 0;
    for (int i = 0; i < arg_count; i++) {
      uint32_t arg_pattern;
      if (args[i].IsRegister()) {
        arg_pattern = args[i].Is32Bits() ? kPrintfArgW : kPrintfArgX;
      } else {
3166
        DCHECK(args[i].Is64Bits());
3167 3168
        arg_pattern = kPrintfArgD;
      }
3169
      DCHECK(arg_pattern < (1 << kPrintfArgPatternBits));
3170 3171
      arg_pattern_list |= (arg_pattern << (kPrintfArgPatternBits * i));
    }
3172
    dc32(arg_pattern_list);  // kPrintfArgPatternListOffset
3173 3174
  }
#else
3175
  Call(ExternalReference::printf_function());
3176 3177 3178
#endif
}

3179
void TurboAssembler::Printf(const char* format, CPURegister arg0,
3180
                            CPURegister arg1, CPURegister arg2,
3181
                            CPURegister arg3) {
3182 3183 3184 3185 3186 3187 3188
  // Printf is expected to preserve all registers, so make sure that none are
  // available as scratch registers until we've preserved them.
  RegList old_tmp_list = TmpList()->list();
  RegList old_fp_tmp_list = FPTmpList()->list();
  TmpList()->set_list(0);
  FPTmpList()->set_list(0);

3189
  CPURegList saved_registers = kCallerSaved;
3190
  saved_registers.Align();
3191

3192
  // Preserve all caller-saved registers as well as NZCV.
3193 3194
  // PushCPURegList asserts that the size of each list is a multiple of 16
  // bytes.
3195
  PushCPURegList(saved_registers);
3196
  PushCPURegList(kCallerSavedV);
3197 3198

  // We can use caller-saved registers as scratch values (except for argN).
3199
  CPURegList tmp_list = saved_registers;
3200
  CPURegList fp_tmp_list = kCallerSavedV;
3201 3202 3203 3204 3205
  tmp_list.Remove(arg0, arg1, arg2, arg3);
  fp_tmp_list.Remove(arg0, arg1, arg2, arg3);
  TmpList()->set_list(tmp_list.list());
  FPTmpList()->set_list(fp_tmp_list.list());

3206 3207
  {
    UseScratchRegisterScope temps(this);
3208 3209 3210
    // If any of the arguments are the current stack pointer, allocate a new
    // register for them, and adjust the value to compensate for pushing the
    // caller-saved registers.
3211 3212 3213 3214
    bool arg0_sp = arg0.is_valid() && sp.Aliases(arg0);
    bool arg1_sp = arg1.is_valid() && sp.Aliases(arg1);
    bool arg2_sp = arg2.is_valid() && sp.Aliases(arg2);
    bool arg3_sp = arg3.is_valid() && sp.Aliases(arg3);
3215 3216 3217 3218
    if (arg0_sp || arg1_sp || arg2_sp || arg3_sp) {
      // Allocate a register to hold the original stack pointer value, to pass
      // to PrintfNoPreserve as an argument.
      Register arg_sp = temps.AcquireX();
3219
      Add(arg_sp, sp,
3220 3221
          saved_registers.TotalSizeInBytes() +
              kCallerSavedV.TotalSizeInBytes());
3222 3223 3224 3225 3226
      if (arg0_sp) arg0 = Register::Create(arg_sp.code(), arg0.SizeInBits());
      if (arg1_sp) arg1 = Register::Create(arg_sp.code(), arg1.SizeInBits());
      if (arg2_sp) arg2 = Register::Create(arg_sp.code(), arg2.SizeInBits());
      if (arg3_sp) arg3 = Register::Create(arg_sp.code(), arg3.SizeInBits());
    }
3227

3228
    // Preserve NZCV.
3229 3230
    {
      UseScratchRegisterScope temps(this);
3231 3232 3233 3234
      Register tmp = temps.AcquireX();
      Mrs(tmp, NZCV);
      Push(tmp, xzr);
    }
3235

3236 3237 3238
    PrintfNoPreserve(format, arg0, arg1, arg2, arg3);

    // Restore NZCV.
3239 3240
    {
      UseScratchRegisterScope temps(this);
3241 3242 3243 3244
      Register tmp = temps.AcquireX();
      Pop(xzr, tmp);
      Msr(NZCV, tmp);
    }
3245 3246
  }

3247
  PopCPURegList(kCallerSavedV);
3248
  PopCPURegList(saved_registers);
3249 3250 3251

  TmpList()->set_list(old_tmp_list);
  FPTmpList()->set_list(old_fp_tmp_list);
3252 3253
}

3254 3255 3256 3257 3258 3259 3260 3261 3262 3263
UseScratchRegisterScope::~UseScratchRegisterScope() {
  available_->set_list(old_available_);
  availablefp_->set_list(old_availablefp_);
}

Register UseScratchRegisterScope::AcquireSameSizeAs(const Register& reg) {
  int code = AcquireNextAvailable(available_).code();
  return Register::Create(code, reg.SizeInBits());
}

3264
VRegister UseScratchRegisterScope::AcquireSameSizeAs(const VRegister& reg) {
3265
  int code = AcquireNextAvailable(availablefp_).code();
3266
  return VRegister::Create(code, reg.SizeInBits());
3267 3268 3269 3270 3271 3272
}

CPURegister UseScratchRegisterScope::AcquireNextAvailable(
    CPURegList* available) {
  CHECK(!available->IsEmpty());
  CPURegister result = available->PopLowestIndex();
3273
  DCHECK(!AreAliased(result, xzr, sp));
3274 3275 3276
  return result;
}

3277 3278 3279 3280 3281
void TurboAssembler::ComputeCodeStartAddress(const Register& rd) {
  // We can use adr to load a pc relative location.
  adr(rd, -pc_offset());
}

3282 3283 3284
void TurboAssembler::ResetSpeculationPoisonRegister() {
  Mov(kSpeculationPoisonRegister, -1);
}
3285

3286 3287
}  // namespace internal
}  // namespace v8
3288

3289
#endif  // V8_TARGET_ARCH_ARM64