macro-assembler-arm64.cc 127 KB
Newer Older
1
// Copyright 2013 the V8 project authors. All rights reserved.
2 3
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
4

5
#if V8_TARGET_ARCH_ARM64
6

7
#include "src/base/bits.h"
8
#include "src/base/division-by-constant.h"
9
#include "src/codegen/assembler.h"
10
#include "src/codegen/callable.h"
11
#include "src/codegen/code-factory.h"
12
#include "src/codegen/external-reference-table.h"
13
#include "src/codegen/interface-descriptors-inl.h"
14 15
#include "src/codegen/macro-assembler-inl.h"
#include "src/codegen/register-configuration.h"
16
#include "src/codegen/reloc-info.h"
17
#include "src/debug/debug.h"
18
#include "src/deoptimizer/deoptimizer.h"
19 20
#include "src/execution/frame-constants.h"
#include "src/execution/frames-inl.h"
21
#include "src/heap/memory-chunk.h"
22
#include "src/init/bootstrapper.h"
23
#include "src/logging/counters.h"
24
#include "src/runtime/runtime.h"
25
#include "src/snapshot/snapshot.h"
26 27

#if V8_ENABLE_WEBASSEMBLY
28
#include "src/wasm/wasm-code-manager.h"
29
#endif  // V8_ENABLE_WEBASSEMBLY
30

31 32 33
// Satisfy cpplint check, but don't include platform-specific header. It is
// included recursively via macro-assembler.h.
#if 0
34
#include "src/base/platform/wrappers.h"
35
#include "src/codegen/arm64/macro-assembler-arm64.h"
36
#endif
37

38 39 40
namespace v8 {
namespace internal {

41
CPURegList TurboAssembler::DefaultTmpList() { return CPURegList(ip0, ip1); }
42

43
CPURegList TurboAssembler::DefaultFPTmpList() {
44 45 46
  return CPURegList(fp_scratch1, fp_scratch2);
}

47 48 49 50 51 52 53 54 55 56 57 58
namespace {

// For WebAssembly we care about the full floating point register. If we are not
// running Wasm, we can get away with saving half of those registers.
#if V8_ENABLE_WEBASSEMBLY
constexpr int kStackSavedSavedFPSizeInBits = kQRegSizeInBits;
#else
constexpr int kStackSavedSavedFPSizeInBits = kDRegSizeInBits;
#endif  // V8_ENABLE_WEBASSEMBLY

}  // namespace

59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
void TurboAssembler::PushCPURegList(CPURegList registers) {
  // If LR was stored here, we would need to sign it if
  // V8_ENABLE_CONTROL_FLOW_INTEGRITY is on.
  DCHECK(!registers.IncludesAliasOf(lr));

  int size = registers.RegisterSizeInBytes();
  DCHECK_EQ(0, (size * registers.Count()) % 16);

  // Push up to four registers at a time.
  while (!registers.IsEmpty()) {
    int count_before = registers.Count();
    const CPURegister& src0 = registers.PopHighestIndex();
    const CPURegister& src1 = registers.PopHighestIndex();
    const CPURegister& src2 = registers.PopHighestIndex();
    const CPURegister& src3 = registers.PopHighestIndex();
    int count = count_before - registers.Count();
    PushHelper(count, size, src0, src1, src2, src3);
  }
}

void TurboAssembler::PopCPURegList(CPURegList registers) {
  int size = registers.RegisterSizeInBytes();
  DCHECK_EQ(0, (size * registers.Count()) % 16);

  // If LR was loaded here, we would need to authenticate it if
  // V8_ENABLE_CONTROL_FLOW_INTEGRITY is on.
  DCHECK(!registers.IncludesAliasOf(lr));

  // Pop up to four registers at a time.
  while (!registers.IsEmpty()) {
    int count_before = registers.Count();
    const CPURegister& dst0 = registers.PopLowestIndex();
    const CPURegister& dst1 = registers.PopLowestIndex();
    const CPURegister& dst2 = registers.PopLowestIndex();
    const CPURegister& dst3 = registers.PopLowestIndex();
    int count = count_before - registers.Count();
    PopHelper(count, size, dst0, dst1, dst2, dst3);
  }
}

99
int TurboAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
100
                                                    Register exclusion) const {
101
  auto list = kCallerSaved;
102 103
  list.Remove(exclusion);
  list.Align();
104

105
  int bytes = list.TotalSizeInBytes();
106

107
  if (fp_mode == SaveFPRegsMode::kSave) {
108 109 110
    auto fp_list = CPURegList::GetCallerSavedV(kStackSavedSavedFPSizeInBits);
    DCHECK_EQ(fp_list.Count() % 2, 0);
    bytes += fp_list.TotalSizeInBytes();
111 112 113 114
  }
  return bytes;
}

115 116
int TurboAssembler::PushCallerSaved(SaveFPRegsMode fp_mode,
                                    Register exclusion) {
117
  ASM_CODE_COMMENT(this);
118
  auto list = kCallerSaved;
119 120
  list.Remove(exclusion);
  list.Align();
121

122
  PushCPURegList(list);
123

124
  int bytes = list.TotalSizeInBytes();
125

126
  if (fp_mode == SaveFPRegsMode::kSave) {
127 128 129 130
    auto fp_list = CPURegList::GetCallerSavedV(kStackSavedSavedFPSizeInBits);
    DCHECK_EQ(fp_list.Count() % 2, 0);
    PushCPURegList(fp_list);
    bytes += fp_list.TotalSizeInBytes();
131
  }
132
  return bytes;
133 134
}

135
int TurboAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion) {
136
  ASM_CODE_COMMENT(this);
137
  int bytes = 0;
138
  if (fp_mode == SaveFPRegsMode::kSave) {
139 140 141 142
    auto fp_list = CPURegList::GetCallerSavedV(kStackSavedSavedFPSizeInBits);
    DCHECK_EQ(fp_list.Count() % 2, 0);
    PopCPURegList(fp_list);
    bytes += fp_list.TotalSizeInBytes();
143 144 145
  }

  auto list = kCallerSaved;
146 147
  list.Remove(exclusion);
  list.Align();
148

149
  PopCPURegList(list);
150
  bytes += list.TotalSizeInBytes();
151 152

  return bytes;
153 154
}

155 156
void TurboAssembler::LogicalMacro(const Register& rd, const Register& rn,
                                  const Operand& operand, LogicalOp op) {
157
  ASM_CODE_COMMENT(this);
158 159
  UseScratchRegisterScope temps(this);

160
  if (operand.NeedsRelocation(this)) {
161
    Register temp = temps.AcquireX();
162
    Ldr(temp, operand.immediate());
163
    Logical(rd, rn, temp, op);
164 165

  } else if (operand.IsImmediate()) {
166
    int64_t immediate = operand.ImmediateValue();
167 168 169 170 171 172 173 174
    unsigned reg_size = rd.SizeInBits();

    // If the operation is NOT, invert the operation and immediate.
    if ((op & NOT) == NOT) {
      op = static_cast<LogicalOp>(op & ~NOT);
      immediate = ~immediate;
    }

175 176 177
    // Ignore the top 32 bits of an immediate if we're moving to a W register.
    if (rd.Is32Bits()) {
      // Check that the top 32 bits are consistent.
178
      DCHECK(((immediate >> kWRegSizeInBits) == 0) ||
179 180 181 182
             ((immediate >> kWRegSizeInBits) == -1));
      immediate &= kWRegMask;
    }

183
    DCHECK(rd.Is64Bits() || is_uint32(immediate));
184

185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201
    // Special cases for all set or all clear immediates.
    if (immediate == 0) {
      switch (op) {
        case AND:
          Mov(rd, 0);
          return;
        case ORR:  // Fall through.
        case EOR:
          Mov(rd, rn);
          return;
        case ANDS:  // Fall through.
        case BICS:
          break;
        default:
          UNREACHABLE();
      }
    } else if ((rd.Is64Bits() && (immediate == -1L)) ||
202
               (rd.Is32Bits() && (immediate == 0xFFFFFFFFL))) {
203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226
      switch (op) {
        case AND:
          Mov(rd, rn);
          return;
        case ORR:
          Mov(rd, immediate);
          return;
        case EOR:
          Mvn(rd, rn);
          return;
        case ANDS:  // Fall through.
        case BICS:
          break;
        default:
          UNREACHABLE();
      }
    }

    unsigned n, imm_s, imm_r;
    if (IsImmLogical(immediate, reg_size, &n, &imm_s, &imm_r)) {
      // Immediate can be encoded in the instruction.
      LogicalImmediate(rd, rn, n, imm_s, imm_r, op);
    } else {
      // Immediate can't be encoded: synthesize using move immediate.
227
      Register temp = temps.AcquireSameSizeAs(rn);
228 229 230

      // If the left-hand input is the stack pointer, we can't pre-shift the
      // immediate, as the encoding won't allow the subsequent post shift.
231
      PreShiftImmMode mode = rn == sp ? kNoShift : kAnyShift;
232 233
      Operand imm_operand = MoveImmediateForShiftedOp(temp, immediate, mode);

234
      if (rd.IsSP()) {
235 236
        // If rd is the stack pointer we cannot use it as the destination
        // register so we use the temp register as an intermediate again.
237
        Logical(temp, rn, imm_operand, op);
238
        Mov(sp, temp);
239
      } else {
240
        Logical(rd, rn, imm_operand, op);
241 242 243 244
      }
    }

  } else if (operand.IsExtendedRegister()) {
245
    DCHECK(operand.reg().SizeInBits() <= rd.SizeInBits());
246 247
    // Add/sub extended supports shift <= 4. We want to support exactly the
    // same modes here.
248
    DCHECK_LE(operand.shift_amount(), 4);
249
    DCHECK(operand.reg().Is64Bits() ||
250
           ((operand.extend() != UXTX) && (operand.extend() != SXTX)));
251
    Register temp = temps.AcquireSameSizeAs(rn);
252 253 254 255 256 257
    EmitExtendShift(temp, operand.reg(), operand.extend(),
                    operand.shift_amount());
    Logical(rd, rn, temp, op);

  } else {
    // The operand can be encoded in the instruction.
258
    DCHECK(operand.IsShiftedRegister());
259 260 261 262
    Logical(rd, rn, operand, op);
  }
}

263 264
void TurboAssembler::Mov(const Register& rd, uint64_t imm) {
  DCHECK(allow_macro_instructions());
265 266
  DCHECK(is_uint32(imm) || is_int32(imm) || rd.Is64Bits());
  DCHECK(!rd.IsZero());
267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284

  // TODO(all) extend to support more immediates.
  //
  // Immediates on Aarch64 can be produced using an initial value, and zero to
  // three move keep operations.
  //
  // Initial values can be generated with:
  //  1. 64-bit move zero (movz).
  //  2. 32-bit move inverted (movn).
  //  3. 64-bit move inverted.
  //  4. 32-bit orr immediate.
  //  5. 64-bit orr immediate.
  // Move-keep may then be used to modify each of the 16-bit half-words.
  //
  // The code below supports all five initial value generators, and
  // applying move-keep operations to move-zero and move-inverted initial
  // values.

285 286 287 288 289
  // Try to move the immediate in one instruction, and if that fails, switch to
  // using multiple instructions.
  if (!TryOneInstrMoveImmediate(rd, imm)) {
    unsigned reg_size = rd.SizeInBits();

290 291 292
    // Generic immediate case. Imm will be represented by
    //   [imm3, imm2, imm1, imm0], where each imm is 16 bits.
    // A move-zero or move-inverted is generated for the first non-zero or
293
    // non-0xFFFF immX, and a move-keep for subsequent non-zero immX.
294 295 296

    uint64_t ignored_halfword = 0;
    bool invert_move = false;
297
    // If the number of 0xFFFF halfwords is greater than the number of 0x0000
298
    // halfwords, it's more efficient to use move-inverted.
299
    if (CountSetHalfWords(imm, reg_size) > CountSetHalfWords(~imm, reg_size)) {
300
      ignored_halfword = 0xFFFFL;
301 302 303
      invert_move = true;
    }

304 305 306 307
    // Mov instructions can't move immediate values into the stack pointer, so
    // set up a temporary register, if needed.
    UseScratchRegisterScope temps(this);
    Register temp = rd.IsSP() ? temps.AcquireSameSizeAs(rd) : rd;
308 309 310

    // Iterate through the halfwords. Use movn/movz for the first non-ignored
    // halfword, and movk for subsequent halfwords.
311
    DCHECK_EQ(reg_size % 16, 0);
312
    bool first_mov_done = false;
313
    for (int i = 0; i < (rd.SizeInBits() / 16); i++) {
314
      uint64_t imm16 = (imm >> (16 * i)) & 0xFFFFL;
315 316 317
      if (imm16 != ignored_halfword) {
        if (!first_mov_done) {
          if (invert_move) {
318
            movn(temp, (~imm16) & 0xFFFFL, 16 * i);
319 320 321 322 323 324 325 326 327 328
          } else {
            movz(temp, imm16, 16 * i);
          }
          first_mov_done = true;
        } else {
          // Construct a wider constant.
          movk(temp, imm16, 16 * i);
        }
      }
    }
329
    DCHECK(first_mov_done);
330 331 332 333 334 335 336 337 338

    // Move the temporary if the original destination register was the stack
    // pointer.
    if (rd.IsSP()) {
      mov(rd, temp);
    }
  }
}

339
void TurboAssembler::Mov(const Register& rd, const Operand& operand,
340
                         DiscardMoveMode discard_mode) {
341
  DCHECK(allow_macro_instructions());
342
  DCHECK(!rd.IsZero());
343

344 345
  // Provide a swap register for instructions that need to write into the
  // system stack pointer (and can't do this inherently).
346 347
  UseScratchRegisterScope temps(this);
  Register dst = (rd.IsSP()) ? temps.AcquireSameSizeAs(rd) : rd;
348

349
  if (operand.NeedsRelocation(this)) {
350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365
    // TODO(jgruber,v8:8887): Also consider a root-relative load when generating
    // non-isolate-independent code. In many cases it might be cheaper than
    // embedding the relocatable value.
    if (root_array_available_ && options().isolate_independent_code) {
      if (operand.ImmediateRMode() == RelocInfo::EXTERNAL_REFERENCE) {
        Address addr = static_cast<Address>(operand.ImmediateValue());
        ExternalReference reference = bit_cast<ExternalReference>(addr);
        IndirectLoadExternalReference(rd, reference);
        return;
      } else if (RelocInfo::IsEmbeddedObjectMode(operand.ImmediateRMode())) {
        Handle<HeapObject> x(
            reinterpret_cast<Address*>(operand.ImmediateValue()));
        // TODO(v8:9706): Fix-it! This load will always uncompress the value
        // even when we are loading a compressed embedded object.
        IndirectLoadConstant(rd.X(), x);
        return;
366 367
      }
    }
368
    Ldr(dst, operand);
369 370
  } else if (operand.IsImmediate()) {
    // Call the macro assembler for generic immediates.
371
    Mov(dst, operand.ImmediateValue());
372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390
  } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) {
    // Emit a shift instruction if moving a shifted register. This operation
    // could also be achieved using an orr instruction (like orn used by Mvn),
    // but using a shift instruction makes the disassembly clearer.
    EmitShift(dst, operand.reg(), operand.shift(), operand.shift_amount());
  } else if (operand.IsExtendedRegister()) {
    // Emit an extend instruction if moving an extended register. This handles
    // extend with post-shift operations, too.
    EmitExtendShift(dst, operand.reg(), operand.extend(),
                    operand.shift_amount());
  } else {
    // Otherwise, emit a register move only if the registers are distinct, or
    // if they are not X registers.
    //
    // Note that mov(w0, w0) is not a no-op because it clears the top word of
    // x0. A flag is provided (kDiscardForSameWReg) if a move between the same W
    // registers is not required to clear the top word of the X register. In
    // this case, the instruction is discarded.
    //
391
    // If sp is an operand, add #0 is emitted, otherwise, orr #0.
392
    if (rd != operand.reg() ||
393
        (rd.Is32Bits() && (discard_mode == kDontDiscardForSameWReg))) {
394 395 396 397 398 399 400
      Assembler::mov(rd, operand.reg());
    }
    // This case can handle writes into the system stack pointer directly.
    dst = rd;
  }

  // Copy the result to the system stack pointer.
401
  if (dst != rd) {
402
    DCHECK(rd.IsSP());
403 404 405 406
    Assembler::mov(rd, dst);
  }
}

407 408 409 410
void TurboAssembler::Mov(const Register& rd, Smi smi) {
  return Mov(rd, Operand(smi));
}

411
void TurboAssembler::Movi16bitHelper(const VRegister& vd, uint64_t imm) {
412
  DCHECK(is_uint16(imm));
413 414
  int byte1 = (imm & 0xFF);
  int byte2 = ((imm >> 8) & 0xFF);
415 416 417 418 419 420
  if (byte1 == byte2) {
    movi(vd.Is64Bits() ? vd.V8B() : vd.V16B(), byte1);
  } else if (byte1 == 0) {
    movi(vd, byte2, LSL, 8);
  } else if (byte2 == 0) {
    movi(vd, byte1);
421 422 423 424
  } else if (byte1 == 0xFF) {
    mvni(vd, ~byte2 & 0xFF, LSL, 8);
  } else if (byte2 == 0xFF) {
    mvni(vd, ~byte1 & 0xFF);
425 426 427 428 429 430 431 432
  } else {
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireW();
    movz(temp, imm);
    dup(vd, temp);
  }
}

433
void TurboAssembler::Movi32bitHelper(const VRegister& vd, uint64_t imm) {
434 435 436
  DCHECK(is_uint32(imm));

  uint8_t bytes[sizeof(imm)];
437
  memcpy(bytes, &imm, sizeof(imm));
438

439
  // All bytes are either 0x00 or 0xFF.
440 441 442
  {
    bool all0orff = true;
    for (int i = 0; i < 4; ++i) {
443
      if ((bytes[i] != 0) && (bytes[i] != 0xFF)) {
444 445 446 447 448 449 450 451 452 453 454 455 456
        all0orff = false;
        break;
      }
    }

    if (all0orff == true) {
      movi(vd.Is64Bits() ? vd.V1D() : vd.V2D(), ((imm << 32) | imm));
      return;
    }
  }

  // Of the 4 bytes, only one byte is non-zero.
  for (int i = 0; i < 4; i++) {
457
    if ((imm & (0xFF << (i * 8))) == imm) {
458 459 460 461 462
      movi(vd, bytes[i], LSL, i * 8);
      return;
    }
  }

463
  // Of the 4 bytes, only one byte is not 0xFF.
464
  for (int i = 0; i < 4; i++) {
465
    uint32_t mask = ~(0xFF << (i * 8));
466
    if ((imm & mask) == mask) {
467
      mvni(vd, ~bytes[i] & 0xFF, LSL, i * 8);
468 469 470 471 472
      return;
    }
  }

  // Immediate is of the form 0x00MMFFFF.
473
  if ((imm & 0xFF00FFFF) == 0x0000FFFF) {
474 475 476 477 478
    movi(vd, bytes[2], MSL, 16);
    return;
  }

  // Immediate is of the form 0x0000MMFF.
479
  if ((imm & 0xFFFF00FF) == 0x000000FF) {
480 481 482 483 484
    movi(vd, bytes[1], MSL, 8);
    return;
  }

  // Immediate is of the form 0xFFMM0000.
485 486
  if ((imm & 0xFF00FFFF) == 0xFF000000) {
    mvni(vd, ~bytes[2] & 0xFF, MSL, 16);
487 488 489
    return;
  }
  // Immediate is of the form 0xFFFFMM00.
490 491
  if ((imm & 0xFFFF00FF) == 0xFFFF0000) {
    mvni(vd, ~bytes[1] & 0xFF, MSL, 8);
492 493 494 495
    return;
  }

  // Top and bottom 16-bits are equal.
496 497
  if (((imm >> 16) & 0xFFFF) == (imm & 0xFFFF)) {
    Movi16bitHelper(vd.Is64Bits() ? vd.V4H() : vd.V8H(), imm & 0xFFFF);
498 499 500 501 502 503 504 505 506 507 508 509
    return;
  }

  // Default case.
  {
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireW();
    Mov(temp, imm);
    dup(vd, temp);
  }
}

510
void TurboAssembler::Movi64bitHelper(const VRegister& vd, uint64_t imm) {
511
  // All bytes are either 0x00 or 0xFF.
512 513 514
  {
    bool all0orff = true;
    for (int i = 0; i < 8; ++i) {
515 516
      int byteval = (imm >> (i * 8)) & 0xFF;
      if (byteval != 0 && byteval != 0xFF) {
517 518 519 520 521 522 523 524 525 526 527
        all0orff = false;
        break;
      }
    }
    if (all0orff == true) {
      movi(vd, imm);
      return;
    }
  }

  // Top and bottom 32-bits are equal.
528 529
  if (((imm >> 32) & 0xFFFFFFFF) == (imm & 0xFFFFFFFF)) {
    Movi32bitHelper(vd.Is64Bits() ? vd.V2S() : vd.V4S(), imm & 0xFFFFFFFF);
530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545
    return;
  }

  // Default case.
  {
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireX();
    Mov(temp, imm);
    if (vd.Is1D()) {
      mov(vd.D(), 0, temp);
    } else {
      dup(vd.V2D(), temp);
    }
  }
}

546
void TurboAssembler::Movi(const VRegister& vd, uint64_t imm, Shift shift,
547
                          int shift_amount) {
548
  DCHECK(allow_macro_instructions());
549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566
  if (shift_amount != 0 || shift != LSL) {
    movi(vd, imm, shift, shift_amount);
  } else if (vd.Is8B() || vd.Is16B()) {
    // 8-bit immediate.
    DCHECK(is_uint8(imm));
    movi(vd, imm);
  } else if (vd.Is4H() || vd.Is8H()) {
    // 16-bit immediate.
    Movi16bitHelper(vd, imm);
  } else if (vd.Is2S() || vd.Is4S()) {
    // 32-bit immediate.
    Movi32bitHelper(vd, imm);
  } else {
    // 64-bit immediate.
    Movi64bitHelper(vd, imm);
  }
}

567
void TurboAssembler::Movi(const VRegister& vd, uint64_t hi, uint64_t lo) {
568
  // TODO(v8:11033): Move 128-bit values in a more efficient way.
569 570
  DCHECK(vd.Is128Bits());
  Movi(vd.V2D(), lo);
571 572 573 574 575 576
  if (lo != hi) {
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireX();
    Mov(temp, hi);
    Ins(vd.V2D(), 1, temp);
  }
577
}
578

579 580
void TurboAssembler::Mvn(const Register& rd, const Operand& operand) {
  DCHECK(allow_macro_instructions());
581

582
  if (operand.NeedsRelocation(this)) {
583
    Ldr(rd, operand.immediate());
584
    mvn(rd, rd);
585 586 587

  } else if (operand.IsImmediate()) {
    // Call the macro assembler for generic immediates.
588
    Mov(rd, ~operand.ImmediateValue());
589 590 591 592

  } else if (operand.IsExtendedRegister()) {
    // Emit two instructions for the extend case. This differs from Mov, as
    // the extend and invert can't be achieved in one instruction.
593
    EmitExtendShift(rd, operand.reg(), operand.extend(),
594
                    operand.shift_amount());
595
    mvn(rd, rd);
596 597 598 599 600 601

  } else {
    mvn(rd, operand);
  }
}

602 603 604 605 606 607 608 609 610 611 612
unsigned TurboAssembler::CountSetHalfWords(uint64_t imm, unsigned reg_size) {
  DCHECK_EQ(reg_size % 16, 0);

#define HALFWORD(idx) (((imm >> ((idx)*16)) & 0xFFFF) ? 1u : 0u)
  switch (reg_size / 16) {
    case 1:
      return HALFWORD(0);
    case 2:
      return HALFWORD(0) + HALFWORD(1);
    case 4:
      return HALFWORD(0) + HALFWORD(1) + HALFWORD(2) + HALFWORD(3);
613
  }
614 615
#undef HALFWORD
  UNREACHABLE();
616 617 618 619
}

// The movz instruction can generate immediates containing an arbitrary 16-bit
// half-word, with remaining bits clear, eg. 0x00001234, 0x0000123400000000.
620
bool TurboAssembler::IsImmMovz(uint64_t imm, unsigned reg_size) {
621
  DCHECK((reg_size == kXRegSizeInBits) || (reg_size == kWRegSizeInBits));
622
  return CountSetHalfWords(imm, reg_size) <= 1;
623 624 625
}

// The movn instruction can generate immediates containing an arbitrary 16-bit
626
// half-word, with remaining bits set, eg. 0xFFFF1234, 0xFFFF1234FFFFFFFF.
627
bool TurboAssembler::IsImmMovn(uint64_t imm, unsigned reg_size) {
628 629 630
  return IsImmMovz(~imm, reg_size);
}

631
void TurboAssembler::ConditionalCompareMacro(const Register& rn,
632
                                             const Operand& operand,
633
                                             StatusFlags nzcv, Condition cond,
634
                                             ConditionalCompareOp op) {
635
  DCHECK((cond != al) && (cond != nv));
636
  if (operand.NeedsRelocation(this)) {
637 638
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireX();
639
    Ldr(temp, operand.immediate());
640
    ConditionalCompareMacro(rn, temp, nzcv, cond, op);
641 642

  } else if ((operand.IsShiftedRegister() && (operand.shift_amount() == 0)) ||
643 644
             (operand.IsImmediate() &&
              IsImmConditionalCompare(operand.ImmediateValue()))) {
645 646 647 648 649 650 651
    // The immediate can be encoded in the instruction, or the operand is an
    // unshifted register: call the assembler.
    ConditionalCompare(rn, operand, nzcv, cond, op);

  } else {
    // The operand isn't directly supported by the instruction: perform the
    // operation on a temporary register.
652 653
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireSameSizeAs(rn);
654 655 656 657 658
    Mov(temp, operand);
    ConditionalCompare(rn, temp, nzcv, cond, op);
  }
}

659 660
void TurboAssembler::Csel(const Register& rd, const Register& rn,
                          const Operand& operand, Condition cond) {
661
  DCHECK(allow_macro_instructions());
662 663
  DCHECK(!rd.IsZero());
  DCHECK((cond != al) && (cond != nv));
664 665 666
  if (operand.IsImmediate()) {
    // Immediate argument. Handle special cases of 0, 1 and -1 using zero
    // register.
667
    int64_t imm = operand.ImmediateValue();
668 669 670 671 672 673 674 675
    Register zr = AppropriateZeroRegFor(rn);
    if (imm == 0) {
      csel(rd, rn, zr, cond);
    } else if (imm == 1) {
      csinc(rd, rn, zr, cond);
    } else if (imm == -1) {
      csinv(rd, rn, zr, cond);
    } else {
676 677
      UseScratchRegisterScope temps(this);
      Register temp = temps.AcquireSameSizeAs(rn);
678
      Mov(temp, imm);
679 680 681 682 683 684 685
      csel(rd, rn, temp, cond);
    }
  } else if (operand.IsShiftedRegister() && (operand.shift_amount() == 0)) {
    // Unshifted register argument.
    csel(rd, rn, operand.reg(), cond);
  } else {
    // All other arguments.
686 687
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireSameSizeAs(rn);
688 689 690 691 692
    Mov(temp, operand);
    csel(rd, rn, temp, cond);
  }
}

693
bool TurboAssembler::TryOneInstrMoveImmediate(const Register& dst,
694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714
                                              int64_t imm) {
  unsigned n, imm_s, imm_r;
  int reg_size = dst.SizeInBits();
  if (IsImmMovz(imm, reg_size) && !dst.IsSP()) {
    // Immediate can be represented in a move zero instruction. Movz can't write
    // to the stack pointer.
    movz(dst, imm);
    return true;
  } else if (IsImmMovn(imm, reg_size) && !dst.IsSP()) {
    // Immediate can be represented in a move not instruction. Movn can't write
    // to the stack pointer.
    movn(dst, dst.Is64Bits() ? ~imm : (~imm & kWRegMask));
    return true;
  } else if (IsImmLogical(imm, reg_size, &n, &imm_s, &imm_r)) {
    // Immediate can be represented in a logical orr instruction.
    LogicalImmediate(dst, AppropriateZeroRegFor(dst), n, imm_s, imm_r, ORR);
    return true;
  }
  return false;
}

715
Operand TurboAssembler::MoveImmediateForShiftedOp(const Register& dst,
716 717
                                                  int64_t imm,
                                                  PreShiftImmMode mode) {
718 719 720 721 722 723
  int reg_size = dst.SizeInBits();
  // Encode the immediate in a single move instruction, if possible.
  if (TryOneInstrMoveImmediate(dst, imm)) {
    // The move was successful; nothing to do here.
  } else {
    // Pre-shift the immediate to the least-significant bits of the register.
724 725 726 727 728 729 730 731
    int shift_low;
    if (reg_size == 64) {
      shift_low = base::bits::CountTrailingZeros(imm);
    } else {
      DCHECK_EQ(reg_size, 32);
      shift_low = base::bits::CountTrailingZeros(static_cast<uint32_t>(imm));
    }

732 733 734 735 736 737 738
    if (mode == kLimitShiftForSP) {
      // When applied to the stack pointer, the subsequent arithmetic operation
      // can use the extend form to shift left by a maximum of four bits. Right
      // shifts are not allowed, so we filter them out later before the new
      // immediate is tested.
      shift_low = std::min(shift_low, 4);
    }
739 740 741 742 743 744 745 746
    int64_t imm_low = imm >> shift_low;

    // Pre-shift the immediate to the most-significant bits of the register. We
    // insert set bits in the least-significant bits, as this creates a
    // different immediate that may be encodable using movn or orr-immediate.
    // If this new immediate is encodable, the set bits will be eliminated by
    // the post shift on the following instruction.
    int shift_high = CountLeadingZeros(imm, reg_size);
747
    int64_t imm_high = (imm << shift_high) | ((INT64_C(1) << shift_high) - 1);
748

749
    if ((mode != kNoShift) && TryOneInstrMoveImmediate(dst, imm_low)) {
750 751 752
      // The new immediate has been moved into the destination's low bits:
      // return a new leftward-shifting operand.
      return Operand(dst, LSL, shift_low);
753
    } else if ((mode == kAnyShift) && TryOneInstrMoveImmediate(dst, imm_high)) {
754 755 756 757 758 759 760 761 762 763 764
      // The new immediate has been moved into the destination's high bits:
      // return a new rightward-shifting operand.
      return Operand(dst, LSR, shift_high);
    } else {
      // Use the generic move operation to set up the immediate.
      Mov(dst, imm);
    }
  }
  return Operand(dst);
}

765 766
void TurboAssembler::AddSubMacro(const Register& rd, const Register& rn,
                                 const Operand& operand, FlagsUpdate S,
767
                                 AddSubOp op) {
768
  if (operand.IsZero() && rd == rn && rd.Is64Bits() && rn.Is64Bits() &&
769
      !operand.NeedsRelocation(this) && (S == LeaveFlags)) {
770 771 772 773
    // The instruction would be a nop. Avoid generating useless code.
    return;
  }

774
  if (operand.NeedsRelocation(this)) {
775 776
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireX();
777
    Ldr(temp, operand.immediate());
778
    AddSubMacro(rd, rn, temp, S, op);
779
  } else if ((operand.IsImmediate() &&
780
              !IsImmAddSub(operand.ImmediateValue())) ||
781
             (rn.IsZero() && !operand.IsShiftedRegister()) ||
782
             (operand.IsShiftedRegister() && (operand.shift() == ROR))) {
783 784
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireSameSizeAs(rn);
785
    if (operand.IsImmediate()) {
786 787 788 789 790
      PreShiftImmMode mode = kAnyShift;

      // If the destination or source register is the stack pointer, we can
      // only pre-shift the immediate right by values supported in the add/sub
      // extend encoding.
791
      if (rd == sp) {
792 793 794
        // If the destination is SP and flags will be set, we can't pre-shift
        // the immediate at all.
        mode = (S == SetFlags) ? kNoShift : kLimitShiftForSP;
795
      } else if (rn == sp) {
796 797 798
        mode = kLimitShiftForSP;
      }

799
      Operand imm_operand =
800
          MoveImmediateForShiftedOp(temp, operand.ImmediateValue(), mode);
801 802 803 804 805
      AddSub(rd, rn, imm_operand, S, op);
    } else {
      Mov(temp, operand);
      AddSub(rd, rn, temp, S, op);
    }
806 807 808 809 810
  } else {
    AddSub(rd, rn, operand, S, op);
  }
}

811
void TurboAssembler::AddSubWithCarryMacro(const Register& rd,
812
                                          const Register& rn,
813
                                          const Operand& operand, FlagsUpdate S,
814
                                          AddSubWithCarryOp op) {
815
  DCHECK(rd.SizeInBits() == rn.SizeInBits());
816
  UseScratchRegisterScope temps(this);
817

818
  if (operand.NeedsRelocation(this)) {
819
    Register temp = temps.AcquireX();
820
    Ldr(temp, operand.immediate());
821
    AddSubWithCarryMacro(rd, rn, temp, S, op);
822 823 824 825

  } else if (operand.IsImmediate() ||
             (operand.IsShiftedRegister() && (operand.shift() == ROR))) {
    // Add/sub with carry (immediate or ROR shifted register.)
826
    Register temp = temps.AcquireSameSizeAs(rn);
827 828
    Mov(temp, operand);
    AddSubWithCarry(rd, rn, temp, S, op);
829

830 831
  } else if (operand.IsShiftedRegister() && (operand.shift_amount() != 0)) {
    // Add/sub with carry (shifted register).
832 833
    DCHECK(operand.reg().SizeInBits() == rd.SizeInBits());
    DCHECK(operand.shift() != ROR);
834 835 836
    DCHECK(is_uintn(operand.shift_amount(), rd.SizeInBits() == kXRegSizeInBits
                                                ? kXRegSizeInBitsLog2
                                                : kWRegSizeInBitsLog2));
837
    Register temp = temps.AcquireSameSizeAs(rn);
838 839 840 841 842
    EmitShift(temp, operand.reg(), operand.shift(), operand.shift_amount());
    AddSubWithCarry(rd, rn, temp, S, op);

  } else if (operand.IsExtendedRegister()) {
    // Add/sub with carry (extended register).
843
    DCHECK(operand.reg().SizeInBits() <= rd.SizeInBits());
844 845
    // Add/sub extended supports a shift <= 4. We want to support exactly the
    // same modes.
846
    DCHECK_LE(operand.shift_amount(), 4);
847
    DCHECK(operand.reg().Is64Bits() ||
848
           ((operand.extend() != UXTX) && (operand.extend() != SXTX)));
849
    Register temp = temps.AcquireSameSizeAs(rn);
850 851 852 853 854 855 856 857 858 859
    EmitExtendShift(temp, operand.reg(), operand.extend(),
                    operand.shift_amount());
    AddSubWithCarry(rd, rn, temp, S, op);

  } else {
    // The addressing mode is directly supported by the instruction.
    AddSubWithCarry(rd, rn, operand, S, op);
  }
}

860 861
void TurboAssembler::LoadStoreMacro(const CPURegister& rt,
                                    const MemOperand& addr, LoadStoreOp op) {
862
  int64_t offset = addr.offset();
863
  unsigned size = CalcLSDataSize(op);
864 865 866 867 868 869 870 871

  // Check if an immediate offset fits in the immediate field of the
  // appropriate instruction. If not, emit two instructions to perform
  // the operation.
  if (addr.IsImmediateOffset() && !IsImmLSScaled(offset, size) &&
      !IsImmLSUnscaled(offset)) {
    // Immediate offset that can't be encoded using unsigned or unscaled
    // addressing modes.
872 873
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireSameSizeAs(addr.base());
874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889
    Mov(temp, addr.offset());
    LoadStore(rt, MemOperand(addr.base(), temp), op);
  } else if (addr.IsPostIndex() && !IsImmLSUnscaled(offset)) {
    // Post-index beyond unscaled addressing range.
    LoadStore(rt, MemOperand(addr.base()), op);
    add(addr.base(), addr.base(), offset);
  } else if (addr.IsPreIndex() && !IsImmLSUnscaled(offset)) {
    // Pre-index beyond unscaled addressing range.
    add(addr.base(), addr.base(), offset);
    LoadStore(rt, MemOperand(addr.base()), op);
  } else {
    // Encodable in one load/store instruction.
    LoadStore(rt, addr, op);
  }
}

890
void TurboAssembler::LoadStorePairMacro(const CPURegister& rt,
891 892 893 894 895 896 897
                                        const CPURegister& rt2,
                                        const MemOperand& addr,
                                        LoadStorePairOp op) {
  // TODO(all): Should we support register offset for load-store-pair?
  DCHECK(!addr.IsRegisterOffset());

  int64_t offset = addr.offset();
898
  unsigned size = CalcLSPairDataSize(op);
899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922

  // Check if the offset fits in the immediate field of the appropriate
  // instruction. If not, emit two instructions to perform the operation.
  if (IsImmLSPair(offset, size)) {
    // Encodable in one load/store pair instruction.
    LoadStorePair(rt, rt2, addr, op);
  } else {
    Register base = addr.base();
    if (addr.IsImmediateOffset()) {
      UseScratchRegisterScope temps(this);
      Register temp = temps.AcquireSameSizeAs(base);
      Add(temp, base, offset);
      LoadStorePair(rt, rt2, MemOperand(temp), op);
    } else if (addr.IsPostIndex()) {
      LoadStorePair(rt, rt2, MemOperand(base), op);
      Add(base, base, offset);
    } else {
      DCHECK(addr.IsPreIndex());
      Add(base, base, offset);
      LoadStorePair(rt, rt2, MemOperand(base), op);
    }
  }
}

923 924
bool TurboAssembler::NeedExtraInstructionsOrRegisterBranch(
    Label* label, ImmBranchType b_type) {
925 926 927 928 929 930 931 932
  bool need_longer_range = false;
  // There are two situations in which we care about the offset being out of
  // range:
  //  - The label is bound but too far away.
  //  - The label is not bound but linked, and the previous branch
  //    instruction in the chain is too far away.
  if (label->is_bound() || label->is_linked()) {
    need_longer_range =
933
        !Instruction::IsValidImmPCOffset(b_type, label->pos() - pc_offset());
934 935 936
  }
  if (!need_longer_range && !label->is_bound()) {
    int max_reachable_pc = pc_offset() + Instruction::ImmBranchRange(b_type);
937 938
    unresolved_branches_.insert(std::pair<int, FarBranchInfo>(
        max_reachable_pc, FarBranchInfo(pc_offset(), label)));
939
    // Also maintain the next pool check.
940
    next_veneer_pool_check_ = std::min(
941
        next_veneer_pool_check_, max_reachable_pc - kVeneerDistanceCheckMargin);
942 943 944 945
  }
  return need_longer_range;
}

946 947
void TurboAssembler::Adr(const Register& rd, Label* label, AdrHint hint) {
  DCHECK(allow_macro_instructions());
948
  DCHECK(!rd.IsZero());
949 950 951 952 953 954

  if (hint == kAdrNear) {
    adr(rd, label);
    return;
  }

955
  DCHECK_EQ(hint, kAdrFar);
956 957 958 959 960
  if (label->is_bound()) {
    int label_offset = label->pos() - pc_offset();
    if (Instruction::IsValidPCRelOffset(label_offset)) {
      adr(rd, label);
    } else {
961
      DCHECK_LE(label_offset, 0);
962 963 964 965 966
      int min_adr_offset = -(1 << (Instruction::ImmPCRelRangeBitwidth - 1));
      adr(rd, min_adr_offset);
      Add(rd, rd, label_offset - min_adr_offset);
    }
  } else {
967 968 969
    UseScratchRegisterScope temps(this);
    Register scratch = temps.AcquireX();

970 971
    InstructionAccurateScope scope(this,
                                   PatchingAssembler::kAdrFarPatchableNInstrs);
972 973 974 975 976 977 978 979
    adr(rd, label);
    for (int i = 0; i < PatchingAssembler::kAdrFarPatchableNNops; ++i) {
      nop(ADR_FAR_NOP);
    }
    movz(scratch, 0);
  }
}

980
void TurboAssembler::B(Label* label, BranchType type, Register reg, int bit) {
981
  DCHECK((reg == NoReg || type >= kBranchTypeFirstUsingReg) &&
982 983 984 985 986
         (bit == -1 || type >= kBranchTypeFirstUsingBit));
  if (kBranchTypeFirstCondition <= type && type <= kBranchTypeLastCondition) {
    B(static_cast<Condition>(type), label);
  } else {
    switch (type) {
987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003
      case always:
        B(label);
        break;
      case never:
        break;
      case reg_zero:
        Cbz(reg, label);
        break;
      case reg_not_zero:
        Cbnz(reg, label);
        break;
      case reg_bit_clear:
        Tbz(reg, bit, label);
        break;
      case reg_bit_set:
        Tbnz(reg, bit, label);
        break;
1004 1005 1006 1007 1008 1009
      default:
        UNREACHABLE();
    }
  }
}

1010 1011
void TurboAssembler::B(Label* label, Condition cond) {
  DCHECK(allow_macro_instructions());
1012
  DCHECK((cond != al) && (cond != nv));
1013 1014 1015

  Label done;
  bool need_extra_instructions =
1016
      NeedExtraInstructionsOrRegisterBranch(label, CondBranchType);
1017 1018

  if (need_extra_instructions) {
1019
    b(&done, NegateCondition(cond));
1020
    B(label);
1021 1022 1023 1024 1025 1026
  } else {
    b(label, cond);
  }
  bind(&done);
}

1027 1028
void TurboAssembler::Tbnz(const Register& rt, unsigned bit_pos, Label* label) {
  DCHECK(allow_macro_instructions());
1029 1030 1031

  Label done;
  bool need_extra_instructions =
1032
      NeedExtraInstructionsOrRegisterBranch(label, TestBranchType);
1033 1034 1035

  if (need_extra_instructions) {
    tbz(rt, bit_pos, &done);
1036
    B(label);
1037 1038 1039 1040 1041 1042
  } else {
    tbnz(rt, bit_pos, label);
  }
  bind(&done);
}

1043 1044
void TurboAssembler::Tbz(const Register& rt, unsigned bit_pos, Label* label) {
  DCHECK(allow_macro_instructions());
1045 1046 1047

  Label done;
  bool need_extra_instructions =
1048
      NeedExtraInstructionsOrRegisterBranch(label, TestBranchType);
1049 1050 1051

  if (need_extra_instructions) {
    tbnz(rt, bit_pos, &done);
1052
    B(label);
1053 1054 1055 1056 1057 1058
  } else {
    tbz(rt, bit_pos, label);
  }
  bind(&done);
}

1059 1060
void TurboAssembler::Cbnz(const Register& rt, Label* label) {
  DCHECK(allow_macro_instructions());
1061 1062 1063

  Label done;
  bool need_extra_instructions =
1064
      NeedExtraInstructionsOrRegisterBranch(label, CompareBranchType);
1065 1066 1067

  if (need_extra_instructions) {
    cbz(rt, &done);
1068
    B(label);
1069 1070 1071 1072 1073 1074
  } else {
    cbnz(rt, label);
  }
  bind(&done);
}

1075 1076
void TurboAssembler::Cbz(const Register& rt, Label* label) {
  DCHECK(allow_macro_instructions());
1077 1078 1079

  Label done;
  bool need_extra_instructions =
1080
      NeedExtraInstructionsOrRegisterBranch(label, CompareBranchType);
1081 1082 1083

  if (need_extra_instructions) {
    cbnz(rt, &done);
1084
    B(label);
1085 1086 1087 1088 1089 1090
  } else {
    cbz(rt, label);
  }
  bind(&done);
}

1091 1092
// Pseudo-instructions.

1093 1094 1095
void TurboAssembler::Abs(const Register& rd, const Register& rm,
                         Label* is_not_representable, Label* is_representable) {
  DCHECK(allow_macro_instructions());
1096
  DCHECK(AreSameSizeAndType(rd, rm));
1097 1098 1099 1100 1101 1102 1103

  Cmp(rm, 1);
  Cneg(rd, rm, lt);

  // If the comparison sets the v flag, the input was the smallest value
  // representable by rm, and the mathematical result of abs(rm) is not
  // representable using two's complement.
1104
  if ((is_not_representable != nullptr) && (is_representable != nullptr)) {
1105 1106
    B(is_not_representable, vs);
    B(is_representable);
1107
  } else if (is_not_representable != nullptr) {
1108
    B(is_not_representable, vs);
1109
  } else if (is_representable != nullptr) {
1110 1111 1112 1113 1114 1115
    B(is_representable, vc);
  }
}

// Abstracted stack operations.

1116
void TurboAssembler::Push(const CPURegister& src0, const CPURegister& src1,
1117 1118 1119
                          const CPURegister& src2, const CPURegister& src3,
                          const CPURegister& src4, const CPURegister& src5,
                          const CPURegister& src6, const CPURegister& src7) {
1120
  DCHECK(AreSameSizeAndType(src0, src1, src2, src3, src4, src5, src6, src7));
1121

1122
  int count = 5 + src5.is_valid() + src6.is_valid() + src6.is_valid();
1123
  int size = src0.SizeInBytes();
1124
  DCHECK_EQ(0, (size * count) % 16);
1125 1126 1127 1128 1129

  PushHelper(4, size, src0, src1, src2, src3);
  PushHelper(count - 4, size, src4, src5, src6, src7);
}

1130
void TurboAssembler::Pop(const CPURegister& dst0, const CPURegister& dst1,
1131 1132 1133 1134 1135 1136 1137
                         const CPURegister& dst2, const CPURegister& dst3,
                         const CPURegister& dst4, const CPURegister& dst5,
                         const CPURegister& dst6, const CPURegister& dst7) {
  // It is not valid to pop into the same register more than once in one
  // instruction, not even into the zero register.
  DCHECK(!AreAliased(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7));
  DCHECK(AreSameSizeAndType(dst0, dst1, dst2, dst3, dst4, dst5, dst6, dst7));
1138
  DCHECK(dst0.is_valid());
1139

1140
  int count = 5 + dst5.is_valid() + dst6.is_valid() + dst7.is_valid();
1141
  int size = dst0.SizeInBytes();
1142
  DCHECK_EQ(0, (size * count) % 16);
1143 1144 1145 1146 1147

  PopHelper(4, size, dst0, dst1, dst2, dst3);
  PopHelper(count - 4, size, dst4, dst5, dst6, dst7);
}

1148
void MacroAssembler::PushMultipleTimes(CPURegister src, Register count) {
1149 1150
  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireSameSizeAs(count);
1151

1152
  Label loop, leftover2, leftover1, done;
1153

1154 1155
  Subs(temp, count, 4);
  B(mi, &leftover2);
1156

1157 1158 1159 1160 1161
  // Push groups of four first.
  Bind(&loop);
  Subs(temp, temp, 4);
  PushHelper(4, src.SizeInBytes(), src, src, src, src);
  B(pl, &loop);
1162

1163 1164 1165 1166
  // Push groups of two.
  Bind(&leftover2);
  Tbz(count, 1, &leftover1);
  PushHelper(2, src.SizeInBytes(), src, src, NoReg, NoReg);
1167

1168 1169 1170 1171
  // Push the last one (if required).
  Bind(&leftover1);
  Tbz(count, 0, &done);
  PushHelper(1, src.SizeInBytes(), src, NoReg, NoReg, NoReg);
1172

1173
  Bind(&done);
1174 1175
}

1176
void TurboAssembler::PushHelper(int count, int size, const CPURegister& src0,
1177 1178 1179 1180 1181 1182
                                const CPURegister& src1,
                                const CPURegister& src2,
                                const CPURegister& src3) {
  // Ensure that we don't unintentially modify scratch or debug registers.
  InstructionAccurateScope scope(this);

1183 1184
  DCHECK(AreSameSizeAndType(src0, src1, src2, src3));
  DCHECK(size == src0.SizeInBytes());
1185 1186 1187 1188 1189

  // When pushing multiple registers, the store order is chosen such that
  // Push(a, b) is equivalent to Push(a) followed by Push(b).
  switch (count) {
    case 1:
1190
      DCHECK(src1.IsNone() && src2.IsNone() && src3.IsNone());
1191
      str(src0, MemOperand(sp, -1 * size, PreIndex));
1192 1193
      break;
    case 2:
1194
      DCHECK(src2.IsNone() && src3.IsNone());
1195
      stp(src1, src0, MemOperand(sp, -2 * size, PreIndex));
1196 1197
      break;
    case 3:
1198
      DCHECK(src3.IsNone());
1199 1200
      stp(src2, src1, MemOperand(sp, -3 * size, PreIndex));
      str(src0, MemOperand(sp, 2 * size));
1201 1202 1203
      break;
    case 4:
      // Skip over 4 * size, then fill in the gap. This allows four W registers
1204
      // to be pushed using sp, whilst maintaining 16-byte alignment for sp
1205
      // at all times.
1206 1207
      stp(src3, src2, MemOperand(sp, -4 * size, PreIndex));
      stp(src1, src0, MemOperand(sp, 2 * size));
1208 1209 1210 1211 1212 1213
      break;
    default:
      UNREACHABLE();
  }
}

1214 1215
void TurboAssembler::PopHelper(int count, int size, const CPURegister& dst0,
                               const CPURegister& dst1, const CPURegister& dst2,
1216 1217 1218 1219
                               const CPURegister& dst3) {
  // Ensure that we don't unintentially modify scratch or debug registers.
  InstructionAccurateScope scope(this);

1220 1221
  DCHECK(AreSameSizeAndType(dst0, dst1, dst2, dst3));
  DCHECK(size == dst0.SizeInBytes());
1222 1223 1224 1225 1226

  // When popping multiple registers, the load order is chosen such that
  // Pop(a, b) is equivalent to Pop(a) followed by Pop(b).
  switch (count) {
    case 1:
1227
      DCHECK(dst1.IsNone() && dst2.IsNone() && dst3.IsNone());
1228
      ldr(dst0, MemOperand(sp, 1 * size, PostIndex));
1229 1230
      break;
    case 2:
1231
      DCHECK(dst2.IsNone() && dst3.IsNone());
1232
      ldp(dst0, dst1, MemOperand(sp, 2 * size, PostIndex));
1233 1234
      break;
    case 3:
1235
      DCHECK(dst3.IsNone());
1236 1237
      ldr(dst2, MemOperand(sp, 2 * size));
      ldp(dst0, dst1, MemOperand(sp, 3 * size, PostIndex));
1238 1239 1240 1241
      break;
    case 4:
      // Load the higher addresses first, then load the lower addresses and
      // skip the whole block in the second instruction. This allows four W
1242 1243 1244 1245
      // registers to be popped using sp, whilst maintaining 16-byte alignment
      // for sp at all times.
      ldp(dst2, dst3, MemOperand(sp, 2 * size));
      ldp(dst0, dst1, MemOperand(sp, 4 * size, PostIndex));
1246 1247 1248 1249 1250 1251
      break;
    default:
      UNREACHABLE();
  }
}

1252
void TurboAssembler::PokePair(const CPURegister& src1, const CPURegister& src2,
1253
                              int offset) {
1254 1255
  DCHECK(AreSameSizeAndType(src1, src2));
  DCHECK((offset >= 0) && ((offset % src1.SizeInBytes()) == 0));
1256
  Stp(src1, src2, MemOperand(sp, offset));
1257 1258
}

1259
void MacroAssembler::PeekPair(const CPURegister& dst1, const CPURegister& dst2,
1260
                              int offset) {
1261 1262
  DCHECK(AreSameSizeAndType(dst1, dst2));
  DCHECK((offset >= 0) && ((offset % dst1.SizeInBytes()) == 0));
1263
  Ldp(dst1, dst2, MemOperand(sp, offset));
1264 1265 1266
}

void MacroAssembler::PushCalleeSavedRegisters() {
1267 1268 1269
  ASM_CODE_COMMENT(this);
  // Ensure that the macro-assembler doesn't use any scratch registers.
  InstructionAccurateScope scope(this);
1270

1271
  MemOperand tos(sp, -2 * static_cast<int>(kXRegSize), PreIndex);
1272

1273 1274 1275 1276
  stp(d14, d15, tos);
  stp(d12, d13, tos);
  stp(d10, d11, tos);
  stp(d8, d9, tos);
1277

1278 1279 1280 1281 1282
  stp(x27, x28, tos);
  stp(x25, x26, tos);
  stp(x23, x24, tos);
  stp(x21, x22, tos);
  stp(x19, x20, tos);
1283

1284 1285 1286
  STATIC_ASSERT(
      EntryFrameConstants::kCalleeSavedRegisterBytesPushedBeforeFpLrPair ==
      18 * kSystemPointerSize);
1287

1288 1289 1290 1291 1292 1293
#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
    // Use the stack pointer's value immediately before pushing the LR as the
    // context for signing it. This is what the StackFrameIterator expects.
    pacibsp();
#endif

1294 1295 1296 1297
    stp(x29, x30, tos);  // fp, lr

    STATIC_ASSERT(
        EntryFrameConstants::kCalleeSavedRegisterBytesPushedAfterFpLrPair == 0);
1298 1299 1300
}

void MacroAssembler::PopCalleeSavedRegisters() {
1301 1302 1303
  ASM_CODE_COMMENT(this);
  // Ensure that the macro-assembler doesn't use any scratch registers.
  InstructionAccurateScope scope(this);
1304

1305
  MemOperand tos(sp, 2 * kXRegSize, PostIndex);
1306

1307
  ldp(x29, x30, tos);  // fp, lr
1308

1309
#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
1310 1311 1312 1313
                       // The context (stack pointer value) for authenticating
                       // the LR here must
  // match the one used for signing it (see `PushCalleeSavedRegisters`).
  autibsp();
1314 1315
#endif

1316 1317 1318 1319 1320
    ldp(x19, x20, tos);
    ldp(x21, x22, tos);
    ldp(x23, x24, tos);
    ldp(x25, x26, tos);
    ldp(x27, x28, tos);
1321

1322 1323 1324 1325
    ldp(d8, d9, tos);
    ldp(d10, d11, tos);
    ldp(d12, d13, tos);
    ldp(d14, d15, tos);
1326 1327
}

1328
void TurboAssembler::AssertSpAligned() {
1329 1330 1331 1332 1333 1334 1335 1336 1337 1338
  if (!FLAG_debug_code) return;
  ASM_CODE_COMMENT(this);
  HardAbortScope hard_abort(this);  // Avoid calls to Abort.
  // Arm64 requires the stack pointer to be 16-byte aligned prior to address
  // calculation.
  UseScratchRegisterScope scope(this);
  Register temp = scope.AcquireX();
  Mov(temp, sp);
  Tst(temp, 15);
  Check(eq, AbortReason::kUnexpectedStackPointer);
1339
}
1340

1341 1342 1343 1344
void TurboAssembler::CopySlots(int dst, Register src, Register slot_count) {
  DCHECK(!src.IsZero());
  UseScratchRegisterScope scope(this);
  Register dst_reg = scope.AcquireX();
1345 1346
  SlotAddress(dst_reg, dst);
  SlotAddress(src, src);
1347 1348 1349 1350 1351 1352
  CopyDoubleWords(dst_reg, src, slot_count);
}

void TurboAssembler::CopySlots(Register dst, Register src,
                               Register slot_count) {
  DCHECK(!dst.IsZero() && !src.IsZero());
1353 1354
  SlotAddress(dst, dst);
  SlotAddress(src, src);
1355 1356 1357
  CopyDoubleWords(dst, src, slot_count);
}

1358 1359
void TurboAssembler::CopyDoubleWords(Register dst, Register src, Register count,
                                     CopyDoubleWordsMode mode) {
1360
  ASM_CODE_COMMENT(this);
1361 1362
  DCHECK(!AreAliased(dst, src, count));

1363
  if (FLAG_debug_code) {
1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374
    Register pointer1 = dst;
    Register pointer2 = src;
    if (mode == kSrcLessThanDst) {
      pointer1 = src;
      pointer2 = dst;
    }
    // Copy requires pointer1 < pointer2 || (pointer1 - pointer2) >= count.
    Label pointer1_below_pointer2;
    Subs(pointer1, pointer1, pointer2);
    B(lt, &pointer1_below_pointer2);
    Cmp(pointer1, count);
1375
    Check(ge, AbortReason::kOffsetOutOfRange);
1376 1377
    Bind(&pointer1_below_pointer2);
    Add(pointer1, pointer1, pointer2);
1378
  }
1379
  static_assert(kSystemPointerSize == kDRegSize,
1380
                "pointers must be the same size as doubles");
1381

1382 1383 1384 1385 1386 1387 1388 1389
  if (mode == kDstLessThanSrcAndReverse) {
    Add(src, src, Operand(count, LSL, kSystemPointerSizeLog2));
    Sub(src, src, kSystemPointerSize);
  }

  int src_direction = (mode == kDstLessThanSrc) ? 1 : -1;
  int dst_direction = (mode == kSrcLessThanDst) ? -1 : 1;

1390 1391 1392 1393
  UseScratchRegisterScope scope(this);
  VRegister temp0 = scope.AcquireD();
  VRegister temp1 = scope.AcquireD();

1394
  Label pairs, loop, done;
1395 1396

  Tbz(count, 0, &pairs);
1397
  Ldr(temp0, MemOperand(src, src_direction * kSystemPointerSize, PostIndex));
1398
  Sub(count, count, 1);
1399
  Str(temp0, MemOperand(dst, dst_direction * kSystemPointerSize, PostIndex));
1400 1401

  Bind(&pairs);
1402 1403
  if (mode == kSrcLessThanDst) {
    // Adjust pointers for post-index ldp/stp with negative offset:
1404 1405
    Sub(dst, dst, kSystemPointerSize);
    Sub(src, src, kSystemPointerSize);
1406 1407
  } else if (mode == kDstLessThanSrcAndReverse) {
    Sub(src, src, kSystemPointerSize);
1408 1409
  }
  Bind(&loop);
1410
  Cbz(count, &done);
1411
  Ldp(temp0, temp1,
1412
      MemOperand(src, 2 * src_direction * kSystemPointerSize, PostIndex));
1413
  Sub(count, count, 2);
1414 1415 1416 1417 1418 1419 1420
  if (mode == kDstLessThanSrcAndReverse) {
    Stp(temp1, temp0,
        MemOperand(dst, 2 * dst_direction * kSystemPointerSize, PostIndex));
  } else {
    Stp(temp0, temp1,
        MemOperand(dst, 2 * dst_direction * kSystemPointerSize, PostIndex));
  }
1421
  B(&loop);
1422 1423 1424 1425 1426 1427 1428

  // TODO(all): large copies may benefit from using temporary Q registers
  // to copy four double words per iteration.

  Bind(&done);
}

1429
void TurboAssembler::SlotAddress(Register dst, int slot_offset) {
1430
  Add(dst, sp, slot_offset << kSystemPointerSizeLog2);
1431 1432 1433
}

void TurboAssembler::SlotAddress(Register dst, Register slot_offset) {
1434
  Add(dst, sp, Operand(slot_offset, LSL, kSystemPointerSizeLog2));
1435 1436
}

1437
void TurboAssembler::AssertFPCRState(Register fpcr) {
1438 1439 1440 1441 1442 1443 1444 1445
  if (!FLAG_debug_code) return;
  ASM_CODE_COMMENT(this);
  Label unexpected_mode, done;
  UseScratchRegisterScope temps(this);
  if (fpcr.IsNone()) {
    fpcr = temps.AcquireX();
    Mrs(fpcr, FPCR);
  }
1446 1447 1448 1449 1450 1451 1452 1453 1454 1455

    // Settings left to their default values:
    //   - Assert that flush-to-zero is not set.
    Tbnz(fpcr, FZ_offset, &unexpected_mode);
    //   - Assert that the rounding mode is nearest-with-ties-to-even.
    STATIC_ASSERT(FPTieEven == 0);
    Tst(fpcr, RMode_mask);
    B(eq, &done);

    Bind(&unexpected_mode);
1456
    Abort(AbortReason::kUnexpectedFPCRMode);
1457 1458 1459 1460

    Bind(&done);
}

1461
void TurboAssembler::CanonicalizeNaN(const VRegister& dst,
1462
                                     const VRegister& src) {
1463 1464
  AssertFPCRState();

1465 1466 1467
  // Subtracting 0.0 preserves all inputs except for signalling NaNs, which
  // become quiet NaNs. We use fsub rather than fadd because fsub preserves -0.0
  // inputs: -0.0 + 0.0 = 0.0, but -0.0 - 0.0 = -0.0.
1468 1469 1470
  Fsub(dst, src, fp_zero);
}

1471
void TurboAssembler::LoadRoot(Register destination, RootIndex index) {
1472
  ASM_CODE_COMMENT(this);
1473 1474
  // TODO(jbramley): Most root values are constants, and can be synthesized
  // without a load. Refer to the ARM back end for details.
1475 1476
  Ldr(destination,
      MemOperand(kRootRegister, RootRegisterOffsetForRootIndex(index)));
1477 1478
}

1479
void TurboAssembler::PushRoot(RootIndex index) {
1480
  ASM_CODE_COMMENT(this);
1481 1482 1483 1484 1485 1486
  UseScratchRegisterScope temps(this);
  Register tmp = temps.AcquireX();
  LoadRoot(tmp, index);
  Push(tmp);
}

1487
void TurboAssembler::Move(Register dst, Smi src) { Mov(dst, src); }
1488 1489 1490 1491 1492
void TurboAssembler::Move(Register dst, MemOperand src) { Ldr(dst, src); }
void TurboAssembler::Move(Register dst, Register src) {
  if (dst == src) return;
  Mov(dst, src);
}
1493

1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509
void TurboAssembler::MovePair(Register dst0, Register src0, Register dst1,
                              Register src1) {
  DCHECK_NE(dst0, dst1);
  if (dst0 != src1) {
    Mov(dst0, src0);
    Mov(dst1, src1);
  } else if (dst1 != src0) {
    // Swap the order of the moves to resolve the overlap.
    Mov(dst1, src1);
    Mov(dst0, src0);
  } else {
    // Worse case scenario, this is a swap.
    Swap(dst0, src0);
  }
}

1510 1511
void TurboAssembler::Swap(Register lhs, Register rhs) {
  DCHECK(lhs.IsSameSizeAndType(rhs));
1512
  DCHECK_NE(lhs, rhs);
1513 1514 1515 1516 1517 1518 1519 1520 1521
  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireX();
  Mov(temp, rhs);
  Mov(rhs, lhs);
  Mov(lhs, temp);
}

void TurboAssembler::Swap(VRegister lhs, VRegister rhs) {
  DCHECK(lhs.IsSameSizeAndType(rhs));
1522
  DCHECK_NE(lhs, rhs);
1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537
  UseScratchRegisterScope temps(this);
  VRegister temp = VRegister::no_reg();
  if (lhs.IsS()) {
    temp = temps.AcquireS();
  } else if (lhs.IsD()) {
    temp = temps.AcquireD();
  } else {
    DCHECK(lhs.IsQ());
    temp = temps.AcquireQ();
  }
  Mov(temp, rhs);
  Mov(rhs, lhs);
  Mov(lhs, temp);
}

1538
void TurboAssembler::AssertSmi(Register object, AbortReason reason) {
1539 1540 1541 1542 1543
  if (!FLAG_debug_code) return;
  ASM_CODE_COMMENT(this);
  STATIC_ASSERT(kSmiTag == 0);
  Tst(object, kSmiTagMask);
  Check(eq, reason);
1544 1545
}

1546
void MacroAssembler::AssertNotSmi(Register object, AbortReason reason) {
1547 1548 1549 1550 1551
  if (!FLAG_debug_code) return;
  ASM_CODE_COMMENT(this);
  STATIC_ASSERT(kSmiTag == 0);
  Tst(object, kSmiTagMask);
  Check(ne, reason);
1552 1553
}

1554 1555 1556 1557 1558 1559 1560 1561
void MacroAssembler::AssertCodeT(Register object) {
  if (!FLAG_debug_code) return;
  ASM_CODE_COMMENT(this);
  AssertNotSmi(object, AbortReason::kOperandIsNotACodeT);

  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireX();

1562
  CompareObjectType(object, temp, temp, CODET_TYPE);
1563 1564 1565
  Check(eq, AbortReason::kOperandIsNotACodeT);
}

1566
void MacroAssembler::AssertConstructor(Register object) {
1567 1568 1569
  if (!FLAG_debug_code) return;
  ASM_CODE_COMMENT(this);
  AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotAConstructor);
1570

1571 1572
  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireX();
1573

1574 1575 1576
  LoadMap(temp, object);
  Ldrb(temp, FieldMemOperand(temp, Map::kBitFieldOffset));
  Tst(temp, Operand(Map::Bits1::IsConstructorBit::kMask));
1577

1578
  Check(ne, AbortReason::kOperandIsNotAConstructor);
1579 1580
}

1581
void MacroAssembler::AssertFunction(Register object) {
1582 1583 1584
  if (!FLAG_debug_code) return;
  ASM_CODE_COMMENT(this);
  AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotAFunction);
1585

1586 1587 1588 1589 1590 1591
  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireX();
  LoadMap(temp, object);
  CompareInstanceTypeRange(temp, temp, FIRST_JS_FUNCTION_TYPE,
                           LAST_JS_FUNCTION_TYPE);
  Check(ls, AbortReason::kOperandIsNotAFunction);
1592 1593
}

1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606
void MacroAssembler::AssertCallableFunction(Register object) {
  if (!FLAG_debug_code) return;
  ASM_CODE_COMMENT(this);
  AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotAFunction);

  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireX();
  LoadMap(temp, object);
  CompareInstanceTypeRange(temp, temp, FIRST_CALLABLE_JS_FUNCTION_TYPE,
                           LAST_CALLABLE_JS_FUNCTION_TYPE);
  Check(ls, AbortReason::kOperandIsNotACallableFunction);
}

1607
void MacroAssembler::AssertBoundFunction(Register object) {
1608 1609 1610
  if (!FLAG_debug_code) return;
  ASM_CODE_COMMENT(this);
  AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotABoundFunction);
1611

1612 1613
  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireX();
1614

1615 1616
  CompareObjectType(object, temp, temp, JS_BOUND_FUNCTION_TYPE);
  Check(eq, AbortReason::kOperandIsNotABoundFunction);
1617 1618
}

1619
void MacroAssembler::AssertGeneratorObject(Register object) {
1620
  if (!FLAG_debug_code) return;
1621
  ASM_CODE_COMMENT(this);
1622
  AssertNotSmi(object, AbortReason::kOperandIsASmiAndNotAGeneratorObject);
1623

1624 1625 1626
  // Load map
  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireX();
1627
  LoadMap(temp, object);
1628

1629
  Label do_check;
1630 1631
  // Load instance type and check if JSGeneratorObject
  CompareInstanceType(temp, temp, JS_GENERATOR_OBJECT_TYPE);
1632
  B(eq, &do_check);
1633

1634 1635 1636 1637
  // Check if JSAsyncFunctionObject
  Cmp(temp, JS_ASYNC_FUNCTION_OBJECT_TYPE);
  B(eq, &do_check);

1638 1639 1640 1641 1642
  // Check if JSAsyncGeneratorObject
  Cmp(temp, JS_ASYNC_GENERATOR_OBJECT_TYPE);

  bind(&do_check);
  // Restore generator object to register and perform assertion
1643
  Check(eq, AbortReason::kOperandIsNotAGeneratorObject);
1644
}
1645

1646
void MacroAssembler::AssertUndefinedOrAllocationSite(Register object) {
1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657
  if (!FLAG_debug_code) return;
  ASM_CODE_COMMENT(this);
  UseScratchRegisterScope temps(this);
  Register scratch = temps.AcquireX();
  Label done_checking;
  AssertNotSmi(object);
  JumpIfRoot(object, RootIndex::kUndefinedValue, &done_checking);
  LoadMap(scratch, object);
  CompareInstanceType(scratch, scratch, ALLOCATION_SITE_TYPE);
  Assert(eq, AbortReason::kExpectedUndefinedOrCell);
  Bind(&done_checking);
1658 1659
}

1660
void TurboAssembler::AssertPositiveOrZero(Register value) {
1661 1662 1663 1664 1665 1666 1667
  if (!FLAG_debug_code) return;
  ASM_CODE_COMMENT(this);
  Label done;
  int sign_bit = value.Is64Bits() ? kXSignBit : kWSignBit;
  Tbz(value, sign_bit, &done);
  Abort(AbortReason::kUnexpectedNegativeValue);
  Bind(&done);
1668 1669
}

1670
void MacroAssembler::CallRuntime(const Runtime::Function* f, int num_arguments,
1671
                                 SaveFPRegsMode save_doubles) {
1672
  ASM_CODE_COMMENT(this);
1673 1674 1675 1676 1677
  // All arguments must be on the stack before this function is called.
  // x0 holds the return value after the call.

  // Check that the number of arguments matches what the function expects.
  // If f->nargs is -1, the function can accept a variable number of arguments.
1678
  CHECK(f->nargs < 0 || f->nargs == num_arguments);
1679 1680 1681

  // Place the necessary arguments.
  Mov(x0, num_arguments);
1682
  Mov(x1, ExternalReference::Create(f));
1683

1684
  Handle<CodeT> code =
1685 1686
      CodeFactory::CEntry(isolate(), f->result_size, save_doubles);
  Call(code, RelocInfo::CODE_TARGET);
1687 1688
}

1689 1690
void MacroAssembler::JumpToExternalReference(const ExternalReference& builtin,
                                             bool builtin_exit_frame) {
1691
  ASM_CODE_COMMENT(this);
1692
  Mov(x1, builtin);
1693 1694 1695
  Handle<CodeT> code =
      CodeFactory::CEntry(isolate(), 1, SaveFPRegsMode::kIgnore,
                          ArgvMode::kStack, builtin_exit_frame);
1696
  Jump(code, RelocInfo::CODE_TARGET);
1697 1698
}

1699
void MacroAssembler::JumpToOffHeapInstructionStream(Address entry) {
1700
  Ldr(kOffHeapTrampolineRegister, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
1701 1702 1703
  Br(kOffHeapTrampolineRegister);
}

1704
void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid) {
1705
  ASM_CODE_COMMENT(this);
1706 1707 1708 1709 1710 1711 1712 1713 1714
  const Runtime::Function* function = Runtime::FunctionForId(fid);
  DCHECK_EQ(1, function->result_size);
  if (function->nargs >= 0) {
    // TODO(1236192): Most runtime routines don't need the number of
    // arguments passed in because it is constant. At some point we
    // should remove this need and make the runtime routine entry code
    // smarter.
    Mov(x0, function->nargs);
  }
1715
  JumpToExternalReference(ExternalReference::Create(fid));
1716 1717
}

1718
int TurboAssembler::ActivationFrameAlignment() {
1719
#if V8_HOST_ARCH_ARM64
1720 1721 1722 1723
  // Running on the real platform. Use the alignment as mandated by the local
  // environment.
  // Note: This will break if we ever start generating snapshots on one ARM
  // platform for another ARM platform with a different alignment.
1724
  return base::OS::ActivationFrameAlignment();
1725
#else   // V8_HOST_ARCH_ARM64
1726 1727 1728 1729 1730
  // If we are using the simulator then we should always align to the expected
  // alignment. As the simulator is used to generate snapshots we do not know
  // if the target platform will need alignment, so this is controlled from a
  // flag.
  return FLAG_sim_stack_alignment;
1731
#endif  // V8_HOST_ARCH_ARM64
1732 1733
}

1734
void TurboAssembler::CallCFunction(ExternalReference function,
1735 1736 1737 1738
                                   int num_of_reg_args) {
  CallCFunction(function, num_of_reg_args, 0);
}

1739
void TurboAssembler::CallCFunction(ExternalReference function,
1740 1741
                                   int num_of_reg_args,
                                   int num_of_double_args) {
1742
  ASM_CODE_COMMENT(this);
1743 1744
  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireX();
1745
  Mov(temp, function);
1746
  CallCFunction(temp, num_of_reg_args, num_of_double_args);
1747 1748
}

1749
static const int kRegisterPassedArguments = 8;
1750
static const int kFPRegisterPassedArguments = 8;
1751

1752
void TurboAssembler::CallCFunction(Register function, int num_of_reg_args,
1753
                                   int num_of_double_args) {
1754
  ASM_CODE_COMMENT(this);
1755
  DCHECK_LE(num_of_reg_args + num_of_double_args, kMaxCParameters);
1756
  DCHECK(has_frame());
1757

1758 1759
  // Save the frame pointer and PC so that the stack layout remains iterable,
  // even without an ExitFrame which normally exists between JS and C frames.
1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781
  Register pc_scratch = x4;
  Register addr_scratch = x5;
  Push(pc_scratch, addr_scratch);

  Label get_pc;
  Bind(&get_pc);
  Adr(pc_scratch, &get_pc);

  // See x64 code for reasoning about how to address the isolate data fields.
  if (root_array_available()) {
    Str(pc_scratch,
        MemOperand(kRootRegister, IsolateData::fast_c_call_caller_pc_offset()));
    Str(fp,
        MemOperand(kRootRegister, IsolateData::fast_c_call_caller_fp_offset()));
  } else {
    DCHECK_NOT_NULL(isolate());
    Mov(addr_scratch,
        ExternalReference::fast_c_call_caller_pc_address(isolate()));
    Str(pc_scratch, MemOperand(addr_scratch));
    Mov(addr_scratch,
        ExternalReference::fast_c_call_caller_fp_address(isolate()));
    Str(fp, MemOperand(addr_scratch));
1782 1783
  }

1784 1785
  Pop(addr_scratch, pc_scratch);

1786 1787 1788 1789
  // Call directly. The function called cannot cause a GC, or allow preemption,
  // so the return address in the link register stays correct.
  Call(function);

1790 1791 1792 1793 1794 1795 1796 1797 1798 1799 1800
  // We don't unset the PC; the FP is the source of truth.
  if (root_array_available()) {
    Str(xzr,
        MemOperand(kRootRegister, IsolateData::fast_c_call_caller_fp_offset()));
  } else {
    DCHECK_NOT_NULL(isolate());
    Push(addr_scratch, xzr);
    Mov(addr_scratch,
        ExternalReference::fast_c_call_caller_fp_address(isolate()));
    Str(xzr, MemOperand(addr_scratch));
    Pop(xzr, addr_scratch);
1801 1802
  }

1803 1804 1805 1806 1807
  if (num_of_reg_args > kRegisterPassedArguments) {
    // Drop the register passed arguments.
    int claim_slots = RoundUp(num_of_reg_args - kRegisterPassedArguments, 2);
    Drop(claim_slots);
  }
1808 1809 1810 1811 1812 1813 1814

  if (num_of_double_args > kFPRegisterPassedArguments) {
    // Drop the register passed arguments.
    int claim_slots =
        RoundUp(num_of_double_args - kFPRegisterPassedArguments, 2);
    Drop(claim_slots);
  }
1815 1816
}

1817 1818
void TurboAssembler::LoadFromConstantsTable(Register destination,
                                            int constant_index) {
1819
  ASM_CODE_COMMENT(this);
1820
  DCHECK(RootsTable::IsImmortalImmovable(RootIndex::kBuiltinsConstantsTable));
1821
  LoadRoot(destination, RootIndex::kBuiltinsConstantsTable);
1822 1823 1824
  LoadTaggedPointerField(
      destination, FieldMemOperand(destination, FixedArray::OffsetOfElementAt(
                                                    constant_index)));
1825 1826
}

1827 1828
void TurboAssembler::LoadRootRelative(Register destination, int32_t offset) {
  Ldr(destination, MemOperand(kRootRegister, offset));
1829
}
1830 1831 1832 1833

void TurboAssembler::LoadRootRegisterOffset(Register destination,
                                            intptr_t offset) {
  if (offset == 0) {
1834
    Mov(destination, kRootRegister);
1835
  } else {
1836
    Add(destination, kRootRegister, offset);
1837 1838
  }
}
1839 1840 1841 1842 1843 1844 1845 1846

void TurboAssembler::Jump(Register target, Condition cond) {
  if (cond == nv) return;
  Label done;
  if (cond != al) B(NegateCondition(cond), &done);
  Br(target);
  Bind(&done);
}
1847

1848 1849
void TurboAssembler::JumpHelper(int64_t offset, RelocInfo::Mode rmode,
                                Condition cond) {
1850 1851 1852
  if (cond == nv) return;
  Label done;
  if (cond != al) B(NegateCondition(cond), &done);
1853 1854 1855 1856 1857 1858
  if (CanUseNearCallOrJump(rmode)) {
    DCHECK(IsNearCallOffset(offset));
    near_jump(static_cast<int>(offset), rmode);
  } else {
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireX();
1859
    uint64_t imm = reinterpret_cast<uint64_t>(pc_) + offset * kInstrSize;
1860 1861 1862
    Mov(temp, Immediate(imm, rmode));
    Br(temp);
  }
1863
  Bind(&done);
1864 1865
}

1866
// The calculated offset is either:
1867
// * the 'target' input unmodified if this is a Wasm call, or
1868 1869
// * the offset of the target from the code range start, if this is a call to
//   un-embedded builtin, or
1870 1871
// * the offset of the target from the current PC, in instructions, for any
//   other type of call.
1872 1873
int64_t TurboAssembler::CalculateTargetOffset(Address target,
                                              RelocInfo::Mode rmode, byte* pc) {
1874
  int64_t offset = static_cast<int64_t>(target);
1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886
  if (rmode == RelocInfo::WASM_CALL || rmode == RelocInfo::WASM_STUB_CALL) {
    // The target of WebAssembly calls is still an index instead of an actual
    // address at this point, and needs to be encoded as-is.
    return offset;
  }
  if (RelocInfo::IsRuntimeEntry(rmode)) {
    // The runtime entry targets are used for generating short builtin calls
    // from JIT-compiled code (it's not used during snapshot creation).
    // The value is encoded as an offset from the code range (see
    // Assembler::runtime_entry_at()).
    // Note, that builtin-to-builitin calls use different OFF_HEAP_TARGET mode
    // and therefore are encoded differently.
1887 1888
    DCHECK_NE(options().code_range_base, 0);
    offset -= static_cast<int64_t>(options().code_range_base);
1889
  } else {
1890 1891
    offset -= reinterpret_cast<int64_t>(pc);
  }
1892 1893
  DCHECK_EQ(offset % kInstrSize, 0);
  offset = offset / static_cast<int>(kInstrSize);
1894 1895 1896
  return offset;
}

1897
void TurboAssembler::Jump(Address target, RelocInfo::Mode rmode,
1898
                          Condition cond) {
1899 1900
  int64_t offset = CalculateTargetOffset(target, rmode, pc_);
  JumpHelper(offset, rmode, cond);
1901 1902
}

1903
void TurboAssembler::Jump(Handle<CodeT> code, RelocInfo::Mode rmode,
1904
                          Condition cond) {
1905
  DCHECK(RelocInfo::IsCodeTarget(rmode));
1906
  DCHECK_IMPLIES(options().isolate_independent_code,
1907
                 Builtins::IsIsolateIndependentBuiltin(FromCodeT(*code)));
1908 1909

  if (options().inline_offheap_trampolines) {
1910 1911
    Builtin builtin = Builtin::kNoBuiltinId;
    if (isolate()->builtins()->IsBuiltinHandle(code, &builtin)) {
1912
      // Inline the trampoline.
1913
      CHECK_EQ(cond, Condition::al);  // Implement if necessary.
1914
      TailCallBuiltin(builtin);
1915 1916
      return;
    }
1917
  }
1918

1919
  if (CanUseNearCallOrJump(rmode)) {
1920 1921 1922
    EmbeddedObjectIndex index = AddEmbeddedObject(code);
    DCHECK(is_int32(index));
    JumpHelper(static_cast<int64_t>(index), rmode, cond);
1923 1924 1925
  } else {
    Jump(code.address(), rmode, cond);
  }
1926 1927
}

1928 1929 1930 1931 1932 1933 1934
void TurboAssembler::Jump(const ExternalReference& reference) {
  UseScratchRegisterScope temps(this);
  Register scratch = temps.AcquireX();
  Mov(scratch, reference);
  Jump(scratch);
}

1935
void TurboAssembler::Call(Register target) {
1936
  BlockPoolsScope scope(this);
1937 1938 1939
  Blr(target);
}

1940
void TurboAssembler::Call(Address target, RelocInfo::Mode rmode) {
1941
  BlockPoolsScope scope(this);
1942
  if (CanUseNearCallOrJump(rmode)) {
1943
    int64_t offset = CalculateTargetOffset(target, rmode, pc_);
1944 1945
    DCHECK(IsNearCallOffset(offset));
    near_call(static_cast<int>(offset), rmode);
1946
  } else {
1947
    IndirectCall(target, rmode);
1948 1949 1950
  }
}

1951
void TurboAssembler::Call(Handle<CodeT> code, RelocInfo::Mode rmode) {
1952
  DCHECK_IMPLIES(options().isolate_independent_code,
1953
                 Builtins::IsIsolateIndependentBuiltin(FromCodeT(*code)));
1954
  BlockPoolsScope scope(this);
1955

1956
  if (options().inline_offheap_trampolines) {
1957 1958
    Builtin builtin = Builtin::kNoBuiltinId;
    if (isolate()->builtins()->IsBuiltinHandle(code, &builtin)) {
1959
      // Inline the trampoline.
1960
      CallBuiltin(builtin);
1961 1962
      return;
    }
1963
  }
1964

1965
  DCHECK(FromCodeT(*code).IsExecutable());
1966
  if (CanUseNearCallOrJump(rmode)) {
1967 1968 1969
    EmbeddedObjectIndex index = AddEmbeddedObject(code);
    DCHECK(is_int32(index));
    near_call(static_cast<int32_t>(index), rmode);
1970 1971 1972
  } else {
    IndirectCall(code.address(), rmode);
  }
1973 1974
}

1975 1976 1977
void TurboAssembler::Call(ExternalReference target) {
  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireX();
1978
  Mov(temp, target);
1979 1980 1981
  Call(temp);
}

1982
void TurboAssembler::LoadEntryFromBuiltinIndex(Register builtin_index) {
1983
  ASM_CODE_COMMENT(this);
1984
  // The builtin_index register contains the builtin index as a Smi.
1985
  // Untagging is folded into the indexing operand below.
1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002
  if (SmiValuesAre32Bits()) {
    Asr(builtin_index, builtin_index, kSmiShift - kSystemPointerSizeLog2);
    Add(builtin_index, builtin_index,
        IsolateData::builtin_entry_table_offset());
    Ldr(builtin_index, MemOperand(kRootRegister, builtin_index));
  } else {
    DCHECK(SmiValuesAre31Bits());
    if (COMPRESS_POINTERS_BOOL) {
      Add(builtin_index, kRootRegister,
          Operand(builtin_index.W(), SXTW, kSystemPointerSizeLog2 - kSmiShift));
    } else {
      Add(builtin_index, kRootRegister,
          Operand(builtin_index, LSL, kSystemPointerSizeLog2 - kSmiShift));
    }
    Ldr(builtin_index,
        MemOperand(builtin_index, IsolateData::builtin_entry_table_offset()));
  }
2003 2004
}

2005
void TurboAssembler::LoadEntryFromBuiltin(Builtin builtin,
2006
                                          Register destination) {
2007
  Ldr(destination, EntryFromBuiltinAsOperand(builtin));
2008 2009
}

2010
MemOperand TurboAssembler::EntryFromBuiltinAsOperand(Builtin builtin) {
2011
  ASM_CODE_COMMENT(this);
2012 2013
  DCHECK(root_array_available());
  return MemOperand(kRootRegister,
2014
                    IsolateData::BuiltinEntrySlotOffset(builtin));
2015 2016
}

2017
void TurboAssembler::CallBuiltinByIndex(Register builtin_index) {
2018
  ASM_CODE_COMMENT(this);
2019
  LoadEntryFromBuiltinIndex(builtin_index);
2020
  Call(builtin_index);
2021 2022
}

2023
void TurboAssembler::CallBuiltin(Builtin builtin) {
2024
  ASM_CODE_COMMENT(this);
2025 2026 2027
  DCHECK(Builtins::IsBuiltinId(builtin));
  RecordCommentForOffHeapTrampoline(builtin);
  CHECK_NE(builtin, Builtin::kNoBuiltinId);
2028
  if (options().short_builtin_calls) {
2029
    Call(BuiltinEntry(builtin), RelocInfo::RUNTIME_ENTRY);
2030 2031 2032 2033

  } else {
    UseScratchRegisterScope temps(this);
    Register scratch = temps.AcquireX();
2034
    Ldr(scratch, Operand(BuiltinEntry(builtin), RelocInfo::OFF_HEAP_TARGET));
2035 2036 2037 2038
    Call(scratch);
  }
}

2039
void TurboAssembler::TailCallBuiltin(Builtin builtin) {
2040
  ASM_CODE_COMMENT(this);
2041 2042 2043
  DCHECK(Builtins::IsBuiltinId(builtin));
  RecordCommentForOffHeapTrampoline(builtin);
  CHECK_NE(builtin, Builtin::kNoBuiltinId);
2044
  if (options().short_builtin_calls) {
2045
    Jump(BuiltinEntry(builtin), RelocInfo::RUNTIME_ENTRY);
2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058

  } else {
    // The control flow integrity (CFI) feature allows us to "sign" code entry
    // points as a target for calls, jumps or both. Arm64 has special
    // instructions for this purpose, so-called "landing pads" (see
    // TurboAssembler::CallTarget(), TurboAssembler::JumpTarget() and
    // TurboAssembler::JumpOrCallTarget()). Currently, we generate "Call"
    // landing pads for CPP builtins. In order to allow tail calling to those
    // builtins we have to use a workaround.
    // x17 is used to allow using "Call" (i.e. `bti c`) rather than "Jump"
    // (i.e. `bti j`) landing pads for the tail-called code.
    Register temp = x17;

2059
    Ldr(temp, Operand(BuiltinEntry(builtin), RelocInfo::OFF_HEAP_TARGET));
2060 2061
    Jump(temp);
  }
2062 2063
}

2064 2065
void TurboAssembler::LoadCodeObjectEntry(Register destination,
                                         Register code_object) {
2066
  ASM_CODE_COMMENT(this);
2067 2068 2069 2070 2071
  if (V8_EXTERNAL_CODE_SPACE_BOOL) {
    LoadCodeDataContainerEntry(destination, code_object);
    return;
  }

2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082
  // Code objects are called differently depending on whether we are generating
  // builtin code (which will later be embedded into the binary) or compiling
  // user JS code at runtime.
  // * Builtin code runs in --jitless mode and thus must not call into on-heap
  //   Code targets. Instead, we dispatch through the builtins entry table.
  // * Codegen at runtime does not have this restriction and we can use the
  //   shorter, branchless instruction sequence. The assumption here is that
  //   targets are usually generated code and not builtin Code objects.

  if (options().isolate_independent_code) {
    DCHECK(root_array_available());
2083
    Label if_code_is_off_heap, out;
2084 2085 2086 2087 2088 2089 2090

    UseScratchRegisterScope temps(this);
    Register scratch = temps.AcquireX();

    DCHECK(!AreAliased(destination, scratch));
    DCHECK(!AreAliased(code_object, scratch));

2091 2092 2093
    // Check whether the Code object is an off-heap trampoline. If so, call its
    // (off-heap) entry point directly without going through the (on-heap)
    // trampoline.  Otherwise, just call the Code object as always.
2094

2095 2096 2097
    Ldr(scratch.W(), FieldMemOperand(code_object, Code::kFlagsOffset));
    TestAndBranchIfAnySet(scratch.W(), Code::IsOffHeapTrampoline::kMask,
                          &if_code_is_off_heap);
2098

2099
    // Not an off-heap trampoline object, the entry point is at
2100 2101 2102 2103
    // Code::raw_instruction_start().
    Add(destination, code_object, Code::kHeaderSize - kHeapObjectTag);
    B(&out);

2104
    // An off-heap trampoline, the entry point is loaded from the builtin entry
2105
    // table.
2106 2107
    bind(&if_code_is_off_heap);
    Ldrsw(scratch, FieldMemOperand(code_object, Code::kBuiltinIndexOffset));
2108 2109
    Add(destination, kRootRegister,
        Operand(scratch, LSL, kSystemPointerSizeLog2));
2110 2111 2112 2113 2114 2115 2116 2117 2118 2119
    Ldr(destination,
        MemOperand(destination, IsolateData::builtin_entry_table_offset()));

    bind(&out);
  } else {
    Add(destination, code_object, Code::kHeaderSize - kHeapObjectTag);
  }
}

void TurboAssembler::CallCodeObject(Register code_object) {
2120
  ASM_CODE_COMMENT(this);
2121 2122 2123 2124
  LoadCodeObjectEntry(code_object, code_object);
  Call(code_object);
}

2125
void TurboAssembler::JumpCodeObject(Register code_object, JumpMode jump_mode) {
2126
  ASM_CODE_COMMENT(this);
2127
  DCHECK_EQ(JumpMode::kJump, jump_mode);
2128
  LoadCodeObjectEntry(code_object, code_object);
2129 2130 2131 2132 2133 2134 2135

  UseScratchRegisterScope temps(this);
  if (code_object != x17) {
    temps.Exclude(x17);
    Mov(x17, code_object);
  }
  Jump(x17);
2136 2137
}

2138 2139 2140 2141
void TurboAssembler::LoadCodeDataContainerEntry(
    Register destination, Register code_data_container_object) {
  ASM_CODE_COMMENT(this);
  CHECK(V8_EXTERNAL_CODE_SPACE_BOOL);
2142 2143 2144 2145 2146 2147

  LoadExternalPointerField(
      destination,
      FieldMemOperand(code_data_container_object,
                      CodeDataContainer::kCodeEntryPointOffset),
      kCodeEntryPointTag);
2148 2149 2150 2151 2152
}

void TurboAssembler::LoadCodeDataContainerCodeNonBuiltin(
    Register destination, Register code_data_container_object) {
  ASM_CODE_COMMENT(this);
2153 2154 2155 2156 2157 2158 2159
  CHECK(V8_EXTERNAL_CODE_SPACE_BOOL);
  // Given the fields layout we can read the Code reference as a full word.
  STATIC_ASSERT(!V8_EXTERNAL_CODE_SPACE_BOOL ||
                (CodeDataContainer::kCodeCageBaseUpper32BitsOffset ==
                 CodeDataContainer::kCodeOffset + kTaggedSize));
  Ldr(destination, FieldMemOperand(code_data_container_object,
                                   CodeDataContainer::kCodeOffset));
2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208
}

void TurboAssembler::CallCodeDataContainerObject(
    Register code_data_container_object) {
  ASM_CODE_COMMENT(this);
  LoadCodeDataContainerEntry(code_data_container_object,
                             code_data_container_object);
  Call(code_data_container_object);
}

void TurboAssembler::JumpCodeDataContainerObject(
    Register code_data_container_object, JumpMode jump_mode) {
  ASM_CODE_COMMENT(this);
  DCHECK_EQ(JumpMode::kJump, jump_mode);
  LoadCodeDataContainerEntry(code_data_container_object,
                             code_data_container_object);
  UseScratchRegisterScope temps(this);
  if (code_data_container_object != x17) {
    temps.Exclude(x17);
    Mov(x17, code_data_container_object);
  }
  Jump(x17);
}

void TurboAssembler::LoadCodeTEntry(Register destination, Register code) {
  ASM_CODE_COMMENT(this);
  if (V8_EXTERNAL_CODE_SPACE_BOOL) {
    LoadCodeDataContainerEntry(destination, code);
  } else {
    Add(destination, code, Operand(Code::kHeaderSize - kHeapObjectTag));
  }
}

void TurboAssembler::CallCodeTObject(Register code) {
  if (V8_EXTERNAL_CODE_SPACE_BOOL) {
    CallCodeDataContainerObject(code);
  } else {
    CallCodeObject(code);
  }
}

void TurboAssembler::JumpCodeTObject(Register code, JumpMode jump_mode) {
  if (V8_EXTERNAL_CODE_SPACE_BOOL) {
    JumpCodeDataContainerObject(code, jump_mode);
  } else {
    JumpCodeObject(code, jump_mode);
  }
}

2209
void TurboAssembler::StoreReturnAddressAndCall(Register target) {
2210
  ASM_CODE_COMMENT(this);
2211 2212 2213 2214 2215 2216 2217 2218
  // This generates the final instruction sequence for calls to C functions
  // once an exit frame has been constructed.
  //
  // Note that this assumes the caller code (i.e. the Code object currently
  // being generated) is immovable or that the callee function cannot trigger
  // GC, since the callee function will return to it.

  UseScratchRegisterScope temps(this);
2219
  temps.Exclude(x16, x17);
2220 2221

  Label return_location;
2222 2223 2224
  Adr(x17, &return_location);
#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
  Add(x16, sp, kSystemPointerSize);
2225
  Pacib1716();
2226 2227
#endif
  Poke(x17, 0);
2228

2229
  if (FLAG_debug_code) {
2230
    ASM_CODE_COMMENT_STRING(this, "Verify fp[kSPOffset]-8");
2231 2232 2233 2234 2235
    // Verify that the slot below fp[kSPOffset]-8 points to the signed return
    // location.
    Ldr(x16, MemOperand(fp, ExitFrameConstants::kSPOffset));
    Ldr(x16, MemOperand(x16, -static_cast<int64_t>(kXRegSize)));
    Cmp(x16, x17);
2236 2237 2238 2239 2240 2241 2242
    Check(eq, AbortReason::kReturnAddressNotFoundInFrame);
  }

  Blr(target);
  Bind(&return_location);
}

2243
void TurboAssembler::IndirectCall(Address target, RelocInfo::Mode rmode) {
2244
  ASM_CODE_COMMENT(this);
2245 2246 2247 2248 2249 2250 2251 2252 2253 2254
  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireX();
  Mov(temp, Immediate(target, rmode));
  Blr(temp);
}

bool TurboAssembler::IsNearCallOffset(int64_t offset) {
  return is_int26(offset);
}

2255
void TurboAssembler::CallForDeoptimization(
2256 2257
    Builtin target, int deopt_id, Label* exit, DeoptimizeKind kind, Label* ret,
    Label* jump_deoptimization_entry_label) {
2258
  ASM_CODE_COMMENT(this);
2259
  BlockPoolsScope scope(this);
2260
  bl(jump_deoptimization_entry_label);
2261
  DCHECK_EQ(SizeOfCodeGeneratedSince(exit),
2262 2263
            (kind == DeoptimizeKind::kLazy) ? Deoptimizer::kLazyDeoptExitSize
                                            : Deoptimizer::kEagerDeoptExitSize);
2264 2265
}

2266
void MacroAssembler::LoadStackLimit(Register destination, StackLimitKind kind) {
2267
  ASM_CODE_COMMENT(this);
2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279
  DCHECK(root_array_available());
  Isolate* isolate = this->isolate();
  ExternalReference limit =
      kind == StackLimitKind::kRealStackLimit
          ? ExternalReference::address_of_real_jslimit(isolate)
          : ExternalReference::address_of_jslimit(isolate);
  DCHECK(TurboAssembler::IsAddressableThroughRootRegister(isolate, limit));

  intptr_t offset =
      TurboAssembler::RootRegisterOffsetForExternalReference(isolate, limit);
  Ldr(destination, MemOperand(kRootRegister, offset));
}
2280

2281 2282
void MacroAssembler::StackOverflowCheck(Register num_args,
                                        Label* stack_overflow) {
2283
  ASM_CODE_COMMENT(this);
2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301
  UseScratchRegisterScope temps(this);
  Register scratch = temps.AcquireX();

  // Check the stack for overflow.
  // We are not trying to catch interruptions (e.g. debug break and
  // preemption) here, so the "real stack limit" is checked.

  LoadStackLimit(scratch, StackLimitKind::kRealStackLimit);
  // Make scratch the space we have left. The stack might already be overflowed
  // here which will cause scratch to become negative.
  Sub(scratch, sp, scratch);
  // Check if the arguments will overflow the stack.
  Cmp(scratch, Operand(num_args, LSL, kSystemPointerSizeLog2));
  B(le, stack_overflow);
}

void MacroAssembler::InvokePrologue(Register formal_parameter_count,
                                    Register actual_argument_count, Label* done,
2302
                                    InvokeType type) {
2303
  ASM_CODE_COMMENT(this);
2304 2305 2306
  //  x0: actual arguments count.
  //  x1: function (passed through to callee).
  //  x2: expected arguments count.
2307 2308 2309 2310 2311 2312 2313
  //  x3: new target
  Label regular_invoke;
  DCHECK_EQ(actual_argument_count, x0);
  DCHECK_EQ(formal_parameter_count, x2);

  // If the formal parameter count is equal to the adaptor sentinel, no need
  // to push undefined value as arguments.
2314 2315 2316 2317
  if (kDontAdaptArgumentsSentinel != 0) {
    Cmp(formal_parameter_count, Operand(kDontAdaptArgumentsSentinel));
    B(eq, &regular_invoke);
  }
2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333

  // If overapplication or if the actual argument count is equal to the
  // formal parameter count, no need to push extra undefined values.
  Register extra_argument_count = x2;
  Subs(extra_argument_count, formal_parameter_count, actual_argument_count);
  B(le, &regular_invoke);

  // The stack pointer in arm64 needs to be 16-byte aligned. We might need to
  // (1) add an extra padding or (2) remove (re-use) the extra padding already
  // in the stack. Let {slots_to_copy} be the number of slots (arguments) to
  // move up in the stack and let {slots_to_claim} be the number of extra stack
  // slots to claim.
  Label even_extra_count, skip_move;
  Register slots_to_copy = x4;
  Register slots_to_claim = x5;

2334
  Mov(slots_to_copy, actual_argument_count);
2335 2336 2337 2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365
  Mov(slots_to_claim, extra_argument_count);
  Tbz(extra_argument_count, 0, &even_extra_count);

  // Calculate {slots_to_claim} when {extra_argument_count} is odd.
  // If {actual_argument_count} is even, we need one extra padding slot
  // {slots_to_claim = extra_argument_count + 1}.
  // If {actual_argument_count} is odd, we know that the
  // original arguments will have a padding slot that we can reuse
  // {slots_to_claim = extra_argument_count - 1}.
  {
    Register scratch = x11;
    Add(slots_to_claim, extra_argument_count, 1);
    And(scratch, actual_argument_count, 1);
    Sub(slots_to_claim, slots_to_claim, Operand(scratch, LSL, 1));
  }

  Bind(&even_extra_count);
  Cbz(slots_to_claim, &skip_move);

  Label stack_overflow;
  StackOverflowCheck(slots_to_claim, &stack_overflow);
  Claim(slots_to_claim);

  // Move the arguments already in the stack including the receiver.
  {
    Register src = x6;
    Register dst = x7;
    SlotAddress(src, slots_to_claim);
    SlotAddress(dst, 0);
    CopyDoubleWords(dst, src, slots_to_copy);
  }
2366

2367 2368 2369 2370 2371 2372 2373 2374 2375
  Bind(&skip_move);
  Register pointer_next_value = x5;

  // Copy extra arguments as undefined values.
  {
    Label loop;
    Register undefined_value = x6;
    Register count = x7;
    LoadRoot(undefined_value, RootIndex::kUndefinedValue);
2376
    SlotAddress(pointer_next_value, actual_argument_count);
2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388
    Mov(count, extra_argument_count);
    Bind(&loop);
    Str(undefined_value,
        MemOperand(pointer_next_value, kSystemPointerSize, PostIndex));
    Subs(count, count, 1);
    Cbnz(count, &loop);
  }

  // Set padding if needed.
  {
    Label skip;
    Register total_args_slots = x4;
2389
    Add(total_args_slots, actual_argument_count, extra_argument_count);
2390 2391 2392 2393 2394 2395 2396 2397
    Tbz(total_args_slots, 0, &skip);
    Str(padreg, MemOperand(pointer_next_value));
    Bind(&skip);
  }
  B(&regular_invoke);

  bind(&stack_overflow);
  {
2398 2399
    FrameScope frame(
        this, has_frame() ? StackFrame::NO_FRAME_TYPE : StackFrame::INTERNAL);
2400 2401 2402
    CallRuntime(Runtime::kThrowStackOverflow);
    Unreachable();
  }
2403

2404 2405 2406
  Bind(&regular_invoke);
}

2407
void MacroAssembler::CallDebugOnFunctionCall(Register fun, Register new_target,
2408 2409
                                             Register expected_parameter_count,
                                             Register actual_parameter_count) {
2410
  ASM_CODE_COMMENT(this);
2411
  // Load receiver to pass it later to DebugOnFunctionCall hook.
2412
  Peek(x4, ReceiverOperand(actual_parameter_count));
2413 2414
  FrameScope frame(
      this, has_frame() ? StackFrame::NO_FRAME_TYPE : StackFrame::INTERNAL);
2415

2416
  if (!new_target.is_valid()) new_target = padreg;
2417

2418
  // Save values on stack.
2419 2420 2421
  SmiTag(expected_parameter_count);
  SmiTag(actual_parameter_count);
  Push(expected_parameter_count, actual_parameter_count, new_target, fun);
2422 2423
  Push(fun, x4);
  CallRuntime(Runtime::kDebugOnFunctionCall);
2424

2425
  // Restore values from stack.
2426 2427 2428
  Pop(fun, new_target, actual_parameter_count, expected_parameter_count);
  SmiUntag(actual_parameter_count);
  SmiUntag(expected_parameter_count);
2429 2430 2431
}

void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
2432 2433
                                        Register expected_parameter_count,
                                        Register actual_parameter_count,
2434
                                        InvokeType type) {
2435
  ASM_CODE_COMMENT(this);
2436
  // You can't call a function without a valid frame.
2437
  DCHECK_IMPLIES(type == InvokeType::kCall, has_frame());
2438 2439
  DCHECK_EQ(function, x1);
  DCHECK_IMPLIES(new_target.is_valid(), new_target == x3);
2440

2441
  // On function call, call into the debugger if necessary.
2442 2443 2444 2445 2446 2447 2448
  Label debug_hook, continue_after_hook;
  {
    Mov(x4, ExternalReference::debug_hook_on_function_call_address(isolate()));
    Ldrsb(x4, MemOperand(x4));
    Cbnz(x4, &debug_hook);
  }
  bind(&continue_after_hook);
2449 2450

  // Clear the new.target register if not given.
2451
  if (!new_target.is_valid()) {
2452
    LoadRoot(x3, RootIndex::kUndefinedValue);
2453
  }
2454

2455
  Label done;
2456
  InvokePrologue(expected_parameter_count, actual_parameter_count, &done, type);
2457

2458 2459
  // If actual != expected, InvokePrologue will have handled the call through
  // the argument adaptor mechanism.
2460
  // The called function expects the call kind in x5.
2461 2462 2463 2464 2465 2466
  // We call indirectly through the code field in the function to
  // allow recompilation to take effect without changing any of the
  // call sites.
  Register code = kJavaScriptCallCodeStartRegister;
  LoadTaggedPointerField(code,
                         FieldMemOperand(function, JSFunction::kCodeOffset));
2467 2468
  switch (type) {
    case InvokeType::kCall:
2469
      CallCodeTObject(code);
2470 2471
      break;
    case InvokeType::kJump:
2472
      JumpCodeTObject(code);
2473
      break;
2474
  }
2475 2476 2477 2478
  B(&done);

  // Deferred debug hook.
  bind(&debug_hook);
2479 2480
  CallDebugOnFunctionCall(function, new_target, expected_parameter_count,
                          actual_parameter_count);
2481
  B(&continue_after_hook);
2482 2483 2484 2485 2486 2487

  // Continue here if InvokePrologue does handle the invocation due to
  // mismatched parameter counts.
  Bind(&done);
}

2488 2489 2490 2491
Operand MacroAssembler::ReceiverOperand(Register arg_count) {
  return Operand(0);
}

2492 2493
void MacroAssembler::InvokeFunctionWithNewTarget(
    Register function, Register new_target, Register actual_parameter_count,
2494
    InvokeType type) {
2495
  ASM_CODE_COMMENT(this);
2496
  // You can't call a function without a valid frame.
2497
  DCHECK(type == InvokeType::kJump || has_frame());
2498 2499 2500

  // Contract with called JS functions requires that function is passed in x1.
  // (See FullCodeGenerator::Generate().)
2501
  DCHECK_EQ(function, x1);
2502

2503
  Register expected_parameter_count = x2;
2504

2505 2506
  LoadTaggedPointerField(cp,
                         FieldMemOperand(function, JSFunction::kContextOffset));
2507
  // The number of arguments is stored as an int32_t, and -1 is a marker
2508
  // (kDontAdaptArgumentsSentinel), so we need sign
2509
  // extension to correctly handle it.
2510
  LoadTaggedPointerField(
2511
      expected_parameter_count,
2512
      FieldMemOperand(function, JSFunction::kSharedFunctionInfoOffset));
2513 2514
  Ldrh(expected_parameter_count,
       FieldMemOperand(expected_parameter_count,
2515
                       SharedFunctionInfo::kFormalParameterCountOffset));
2516

2517
  InvokeFunctionCode(function, new_target, expected_parameter_count,
2518
                     actual_parameter_count, type);
2519 2520 2521
}

void MacroAssembler::InvokeFunction(Register function,
2522 2523
                                    Register expected_parameter_count,
                                    Register actual_parameter_count,
2524
                                    InvokeType type) {
2525
  ASM_CODE_COMMENT(this);
2526
  // You can't call a function without a valid frame.
2527
  DCHECK(type == InvokeType::kJump || has_frame());
2528 2529 2530

  // Contract with called JS functions requires that function is passed in x1.
  // (See FullCodeGenerator::Generate().)
2531
  DCHECK_EQ(function, x1);
2532 2533

  // Set up the context.
2534 2535
  LoadTaggedPointerField(cp,
                         FieldMemOperand(function, JSFunction::kContextOffset));
2536

2537
  InvokeFunctionCode(function, no_reg, expected_parameter_count,
2538
                     actual_parameter_count, type);
2539 2540
}

2541
void TurboAssembler::TryConvertDoubleToInt64(Register result,
2542 2543
                                             DoubleRegister double_input,
                                             Label* done) {
2544
  ASM_CODE_COMMENT(this);
2545
  // Try to convert with an FPU convert instruction. It's trivial to compute
2546
  // the modulo operation on an integer register so we convert to a 64-bit
2547
  // integer.
2548
  //
2549
  // Fcvtzs will saturate to INT64_MIN (0x800...00) or INT64_MAX (0x7FF...FF)
2550 2551
  // when the double is out of range. NaNs and infinities will be converted to 0
  // (as ECMA-262 requires).
2552
  Fcvtzs(result.X(), double_input);
2553

2554
  // The values INT64_MIN (0x800...00) or INT64_MAX (0x7FF...FF) are not
2555
  // representable using a double, so if the result is one of those then we know
2556
  // that saturation occurred, and we need to manually handle the conversion.
2557 2558 2559
  //
  // It is easy to detect INT64_MIN and INT64_MAX because adding or subtracting
  // 1 will cause signed overflow.
2560 2561
  Cmp(result.X(), 1);
  Ccmp(result.X(), -1, VFlag, vc);
2562

2563 2564
  B(vc, done);
}
2565

2566 2567
void TurboAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone,
                                       Register result,
2568
                                       DoubleRegister double_input,
2569 2570
                                       StubCallMode stub_mode,
                                       LinkRegisterStatus lr_status) {
2571
  ASM_CODE_COMMENT(this);
2572 2573 2574 2575 2576
  if (CpuFeatures::IsSupported(JSCVT)) {
    Fjcvtzs(result.W(), double_input);
    return;
  }

2577
  Label done;
2578

2579 2580 2581
  // Try to convert the double to an int64. If successful, the bottom 32 bits
  // contain our truncated int32 result.
  TryConvertDoubleToInt64(result, double_input, &done);
2582

2583
  // If we fell through then inline version didn't succeed - call stub instead.
2584 2585 2586 2587 2588
  if (lr_status == kLRHasNotBeenSaved) {
    Push<TurboAssembler::kSignLR>(lr, double_input);
  } else {
    Push<TurboAssembler::kDontStoreLR>(xzr, double_input);
  }
2589

2590
  // DoubleToI preserves any registers it needs to clobber.
2591
#if V8_ENABLE_WEBASSEMBLY
2592
  if (stub_mode == StubCallMode::kCallWasmRuntimeStub) {
2593
    Call(wasm::WasmCode::kDoubleToI, RelocInfo::WASM_STUB_CALL);
2594
#else
2595 2596
  // For balance.
  if (false) {
2597
#endif  // V8_ENABLE_WEBASSEMBLY
2598
  } else if (options().inline_offheap_trampolines) {
2599
    CallBuiltin(Builtin::kDoubleToI);
2600 2601 2602
  } else {
    Call(BUILTIN_CODE(isolate, DoubleToI), RelocInfo::CODE_TARGET);
  }
2603
  Ldr(result, MemOperand(sp, 0));
2604

2605
  DCHECK_EQ(xzr.SizeInBytes(), double_input.SizeInBytes());
2606 2607 2608 2609 2610 2611 2612

  if (lr_status == kLRHasNotBeenSaved) {
    // Pop into xzr here to drop the double input on the stack:
    Pop<TurboAssembler::kAuthLR>(xzr, lr);
  } else {
    Drop(2);
  }
2613

2614
  Bind(&done);
2615 2616
  // Keep our invariant that the upper 32 bits are zero.
  Uxtw(result.W(), result.W());
2617 2618
}

2619
void TurboAssembler::Prologue() {
2620
  ASM_CODE_COMMENT(this);
2621 2622 2623 2624
  Push<TurboAssembler::kSignLR>(lr, fp);
  mov(fp, sp);
  STATIC_ASSERT(kExtraSlotClaimedByPrologue == 1);
  Push(cp, kJSFunctionRegister, kJavaScriptCallArgCountRegister, padreg);
2625 2626
}

2627
void TurboAssembler::EnterFrame(StackFrame::Type type) {
2628 2629
  UseScratchRegisterScope temps(this);

2630 2631 2632 2633 2634
  if (type == StackFrame::INTERNAL
#if V8_ENABLE_WEBASSEMBLY
      || type == StackFrame::WASM_DEBUG_BREAK
#endif  // V8_ENABLE_WEBASSEMBLY
  ) {
2635
    Register type_reg = temps.AcquireX();
2636
    Mov(type_reg, StackFrame::TypeToMarker(type));
2637
    Push<TurboAssembler::kSignLR>(lr, fp, type_reg, padreg);
2638
    const int kFrameSize =
2639
        TypedFrameConstants::kFixedFrameSizeFromFp + kSystemPointerSize;
2640
    Add(fp, sp, kFrameSize);
2641 2642
    // sp[3] : lr
    // sp[2] : fp
2643
    // sp[1] : type
2644
    // sp[0] : for alignment
2645
#if V8_ENABLE_WEBASSEMBLY
2646
  } else if (type == StackFrame::WASM ||
2647 2648
             type == StackFrame::WASM_COMPILE_LAZY ||
             type == StackFrame::WASM_EXIT) {
2649
    Register type_reg = temps.AcquireX();
2650
    Mov(type_reg, StackFrame::TypeToMarker(type));
2651
    Push<TurboAssembler::kSignLR>(lr, fp);
2652
    Mov(fp, sp);
2653
    Push(type_reg, kWasmInstanceRegister);
2654 2655 2656
    // sp[3] : lr
    // sp[2] : fp
    // sp[1] : type
2657
    // sp[0] : wasm instance
2658
#endif  // V8_ENABLE_WEBASSEMBLY
2659
  } else if (type == StackFrame::CONSTRUCT) {
2660
    Register type_reg = temps.AcquireX();
2661
    Mov(type_reg, StackFrame::TypeToMarker(type));
2662 2663 2664

    // Users of this frame type push a context pointer after the type field,
    // so do it here to keep the stack pointer aligned.
2665
    Push<TurboAssembler::kSignLR>(lr, fp, type_reg, cp);
2666 2667 2668

    // The context pointer isn't part of the fixed frame, so add an extra slot
    // to account for it.
2669 2670
    Add(fp, sp,
        TypedFrameConstants::kFixedFrameSizeFromFp + kSystemPointerSize);
2671 2672 2673 2674
    // sp[3] : lr
    // sp[2] : fp
    // sp[1] : type
    // sp[0] : cp
2675
  } else {
2676
    DCHECK(StackFrame::IsJavaScript(type));
2677 2678 2679 2680 2681 2682
    // Just push a minimal "machine frame", saving the frame pointer and return
    // address, without any markers.
    Push<TurboAssembler::kSignLR>(lr, fp);
    Mov(fp, sp);
    // sp[1] : lr
    // sp[0] : fp
2683
  }
2684 2685
}

2686
void TurboAssembler::LeaveFrame(StackFrame::Type type) {
2687
  ASM_CODE_COMMENT(this);
2688 2689 2690
  // Drop the execution stack down to the frame pointer and restore
  // the caller frame pointer and return address.
  Mov(sp, fp);
2691
  Pop<TurboAssembler::kAuthLR>(fp, lr);
2692 2693 2694
}

void MacroAssembler::ExitFramePreserveFPRegs() {
2695
  ASM_CODE_COMMENT(this);
2696
  DCHECK_EQ(kCallerSavedV.Count() % 2, 0);
2697
  PushCPURegList(kCallerSavedV);
2698 2699 2700 2701 2702
}

void MacroAssembler::ExitFrameRestoreFPRegs() {
  // Read the registers from the stack without popping them. The stack pointer
  // will be reset as part of the unwinding process.
2703
  ASM_CODE_COMMENT(this);
2704
  CPURegList saved_fp_regs = kCallerSavedV;
2705
  DCHECK_EQ(saved_fp_regs.Count() % 2, 0);
2706 2707 2708 2709 2710

  int offset = ExitFrameConstants::kLastExitFrameField;
  while (!saved_fp_regs.IsEmpty()) {
    const CPURegister& dst0 = saved_fp_regs.PopHighestIndex();
    const CPURegister& dst1 = saved_fp_regs.PopHighestIndex();
2711
    offset -= 2 * kDRegSize;
2712 2713 2714 2715
    Ldp(dst1, dst0, MemOperand(fp, offset));
  }
}

2716 2717 2718
void MacroAssembler::EnterExitFrame(bool save_doubles, const Register& scratch,
                                    int extra_space,
                                    StackFrame::Type frame_type) {
2719
  ASM_CODE_COMMENT(this);
2720 2721
  DCHECK(frame_type == StackFrame::EXIT ||
         frame_type == StackFrame::BUILTIN_EXIT);
2722 2723

  // Set up the new stack frame.
2724
  Push<TurboAssembler::kSignLR>(lr, fp);
2725
  Mov(fp, sp);
2726
  Mov(scratch, StackFrame::TypeToMarker(frame_type));
2727
  Push(scratch, xzr);
2728 2729
  //          fp[8]: CallerPC (lr)
  //    fp -> fp[0]: CallerFP (old fp)
2730
  //          fp[-8]: STUB marker
2731
  //    sp -> fp[-16]: Space reserved for SPOffset.
2732 2733 2734 2735 2736 2737 2738
  STATIC_ASSERT((2 * kSystemPointerSize) ==
                ExitFrameConstants::kCallerSPOffset);
  STATIC_ASSERT((1 * kSystemPointerSize) ==
                ExitFrameConstants::kCallerPCOffset);
  STATIC_ASSERT((0 * kSystemPointerSize) ==
                ExitFrameConstants::kCallerFPOffset);
  STATIC_ASSERT((-2 * kSystemPointerSize) == ExitFrameConstants::kSPOffset);
2739 2740

  // Save the frame pointer and context pointer in the top frame.
2741 2742
  Mov(scratch,
      ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate()));
2743
  Str(fp, MemOperand(scratch));
2744 2745
  Mov(scratch,
      ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
2746 2747
  Str(cp, MemOperand(scratch));

2748
  STATIC_ASSERT((-2 * kSystemPointerSize) ==
2749
                ExitFrameConstants::kLastExitFrameField);
2750 2751 2752 2753
  if (save_doubles) {
    ExitFramePreserveFPRegs();
  }

2754 2755 2756
  // Round the number of space we need to claim to a multiple of two.
  int slots_to_claim = RoundUp(extra_space + 1, 2);

2757 2758 2759
  // Reserve space for the return address and for user requested memory.
  // We do this before aligning to make sure that we end up correctly
  // aligned with the minimum of wasted space.
2760
  Claim(slots_to_claim, kXRegSize);
2761 2762
  //         fp[8]: CallerPC (lr)
  //   fp -> fp[0]: CallerFP (old fp)
2763 2764
  //         fp[-8]: STUB marker
  //         fp[-16]: Space reserved for SPOffset.
2765
  //         fp[-16 - fp_size]: Saved doubles (if save_doubles is true).
2766 2767
  //         sp[8]: Extra space reserved for caller (if extra_space != 0).
  //   sp -> sp[0]: Space reserved for the return address.
2768 2769 2770 2771 2772

  // ExitFrame::GetStateForFramePointer expects to find the return address at
  // the memory address immediately below the pointer stored in SPOffset.
  // It is not safe to derive much else from SPOffset, because the size of the
  // padding can vary.
2773
  Add(scratch, sp, kXRegSize);
2774 2775 2776 2777 2778
  Str(scratch, MemOperand(fp, ExitFrameConstants::kSPOffset));
}

// Leave the current exit frame.
void MacroAssembler::LeaveExitFrame(bool restore_doubles,
2779 2780
                                    const Register& scratch,
                                    const Register& scratch2) {
2781
  ASM_CODE_COMMENT(this);
2782 2783 2784 2785 2786
  if (restore_doubles) {
    ExitFrameRestoreFPRegs();
  }

  // Restore the context pointer from the top frame.
2787 2788
  Mov(scratch,
      ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
2789
  Ldr(cp, MemOperand(scratch));
2790

2791
  if (FLAG_debug_code) {
2792
    // Also emit debug code to clear the cp in the top frame.
2793
    Mov(scratch2, Operand(Context::kInvalidContext));
2794 2795
    Mov(scratch, ExternalReference::Create(IsolateAddressId::kContextAddress,
                                           isolate()));
2796
    Str(scratch2, MemOperand(scratch));
2797 2798
  }
  // Clear the frame pointer from the top frame.
2799 2800
  Mov(scratch,
      ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate()));
2801 2802 2803 2804 2805 2806
  Str(xzr, MemOperand(scratch));

  // Pop the exit frame.
  //         fp[8]: CallerPC (lr)
  //   fp -> fp[0]: CallerFP (old fp)
  //         fp[...]: The rest of the frame.
2807
  Mov(sp, fp);
2808
  Pop<TurboAssembler::kAuthLR>(fp, lr);
2809 2810
}

2811
void MacroAssembler::LoadGlobalProxy(Register dst) {
2812
  ASM_CODE_COMMENT(this);
2813
  LoadNativeContextSlot(dst, Context::GLOBAL_PROXY_INDEX);
2814 2815
}

2816 2817
void MacroAssembler::LoadWeakValue(Register out, Register in,
                                   Label* target_if_cleared) {
2818
  ASM_CODE_COMMENT(this);
2819 2820
  CompareAndBranch(in.W(), Operand(kClearedWeakHeapObjectLower32), eq,
                   target_if_cleared);
2821 2822 2823

  and_(out, in, Operand(~kWeakHeapObjectMask));
}
2824

2825 2826 2827
void MacroAssembler::EmitIncrementCounter(StatsCounter* counter, int value,
                                          Register scratch1,
                                          Register scratch2) {
2828
  ASM_CODE_COMMENT(this);
2829
  DCHECK_NE(value, 0);
2830
  if (FLAG_native_code_counters && counter->Enabled()) {
2831 2832 2833
    // This operation has to be exactly 32-bit wide in case the external
    // reference table redirects the counter to a uint32_t dummy_stats_counter_
    // field.
2834
    Mov(scratch2, ExternalReference::Create(counter));
2835 2836 2837
    Ldr(scratch1.W(), MemOperand(scratch2));
    Add(scratch1.W(), scratch1.W(), value);
    Str(scratch1.W(), MemOperand(scratch2));
2838 2839 2840
  }
}

2841 2842 2843
void MacroAssembler::JumpIfObjectType(Register object, Register map,
                                      Register type_reg, InstanceType type,
                                      Label* if_cond_pass, Condition cond) {
2844
  ASM_CODE_COMMENT(this);
2845 2846 2847 2848 2849
  CompareObjectType(object, map, type_reg, type);
  B(cond, if_cond_pass);
}

// Sets condition flags based on comparison, and returns type in type_reg.
2850 2851
void MacroAssembler::CompareObjectType(Register object, Register map,
                                       Register type_reg, InstanceType type) {
2852
  ASM_CODE_COMMENT(this);
2853
  LoadMap(map, object);
2854 2855 2856
  CompareInstanceType(map, type_reg, type);
}

2857
void TurboAssembler::LoadMap(Register dst, Register object) {
2858
  ASM_CODE_COMMENT(this);
2859 2860 2861
  LoadTaggedPointerField(dst, FieldMemOperand(object, HeapObject::kMapOffset));
}

2862
// Sets condition flags based on comparison, and returns type in type_reg.
2863
void MacroAssembler::CompareInstanceType(Register map, Register type_reg,
2864
                                         InstanceType type) {
2865
  ASM_CODE_COMMENT(this);
2866
  Ldrh(type_reg, FieldMemOperand(map, Map::kInstanceTypeOffset));
2867 2868 2869
  Cmp(type_reg, type);
}

2870 2871 2872 2873
// Sets condition flags based on comparison, and returns type in type_reg.
void MacroAssembler::CompareInstanceTypeRange(Register map, Register type_reg,
                                              InstanceType lower_limit,
                                              InstanceType higher_limit) {
2874
  ASM_CODE_COMMENT(this);
2875 2876 2877 2878 2879 2880 2881 2882
  DCHECK_LT(lower_limit, higher_limit);
  UseScratchRegisterScope temps(this);
  Register scratch = temps.AcquireX();
  Ldrh(type_reg, FieldMemOperand(map, Map::kInstanceTypeOffset));
  Sub(scratch, type_reg, Operand(lower_limit));
  Cmp(scratch, Operand(higher_limit - lower_limit));
}

2883
void MacroAssembler::LoadElementsKindFromMap(Register result, Register map) {
2884
  ASM_CODE_COMMENT(this);
2885
  // Load the map's "bit field 2".
2886
  Ldrb(result, FieldMemOperand(map, Map::kBitField2Offset));
2887
  // Retrieve elements_kind from bit field 2.
2888
  DecodeField<Map::Bits2::ElementsKindBits>(result);
2889 2890
}

2891
void MacroAssembler::CompareRoot(const Register& obj, RootIndex index) {
2892
  ASM_CODE_COMMENT(this);
2893 2894
  UseScratchRegisterScope temps(this);
  Register temp = temps.AcquireX();
2895
  DCHECK(!AreAliased(obj, temp));
2896
  LoadRoot(temp, index);
2897
  CmpTagged(obj, temp);
2898 2899
}

2900
void MacroAssembler::JumpIfRoot(const Register& obj, RootIndex index,
2901 2902 2903 2904 2905
                                Label* if_equal) {
  CompareRoot(obj, index);
  B(eq, if_equal);
}

2906
void MacroAssembler::JumpIfNotRoot(const Register& obj, RootIndex index,
2907 2908 2909 2910 2911
                                   Label* if_not_equal) {
  CompareRoot(obj, index);
  B(ne, if_not_equal);
}

2912 2913 2914 2915
void MacroAssembler::JumpIfIsInRange(const Register& value,
                                     unsigned lower_limit,
                                     unsigned higher_limit,
                                     Label* on_in_range) {
2916
  ASM_CODE_COMMENT(this);
2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928
  if (lower_limit != 0) {
    UseScratchRegisterScope temps(this);
    Register scratch = temps.AcquireW();
    Sub(scratch, value, Operand(lower_limit));
    CompareAndBranch(scratch, Operand(higher_limit - lower_limit), ls,
                     on_in_range);
  } else {
    CompareAndBranch(value, Operand(higher_limit - lower_limit), ls,
                     on_in_range);
  }
}

2929 2930
void TurboAssembler::LoadTaggedPointerField(const Register& destination,
                                            const MemOperand& field_operand) {
2931 2932 2933 2934 2935
  if (COMPRESS_POINTERS_BOOL) {
    DecompressTaggedPointer(destination, field_operand);
  } else {
    Ldr(destination, field_operand);
  }
2936 2937 2938 2939
}

void TurboAssembler::LoadAnyTaggedField(const Register& destination,
                                        const MemOperand& field_operand) {
2940 2941 2942 2943 2944
  if (COMPRESS_POINTERS_BOOL) {
    DecompressAnyTagged(destination, field_operand);
  } else {
    Ldr(destination, field_operand);
  }
2945 2946
}

2947 2948 2949 2950 2951 2952 2953 2954 2955
void TurboAssembler::LoadTaggedSignedField(const Register& destination,
                                           const MemOperand& field_operand) {
  if (COMPRESS_POINTERS_BOOL) {
    DecompressTaggedSigned(destination, field_operand);
  } else {
    Ldr(destination, field_operand);
  }
}

2956 2957 2958 2959
void TurboAssembler::SmiUntagField(Register dst, const MemOperand& src) {
  SmiUntag(dst, src);
}

2960 2961
void TurboAssembler::StoreTaggedField(const Register& value,
                                      const MemOperand& dst_field_operand) {
2962 2963 2964 2965 2966
  if (COMPRESS_POINTERS_BOOL) {
    Str(value.W(), dst_field_operand);
  } else {
    Str(value, dst_field_operand);
  }
2967 2968
}

2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980
void TurboAssembler::AtomicStoreTaggedField(const Register& value,
                                            const Register& dst_base,
                                            const Register& dst_index,
                                            const Register& temp) {
  Add(temp, dst_base, dst_index);
  if (COMPRESS_POINTERS_BOOL) {
    Stlr(value.W(), temp);
  } else {
    Stlr(value, temp);
  }
}

2981 2982
void TurboAssembler::DecompressTaggedSigned(const Register& destination,
                                            const MemOperand& field_operand) {
2983
  ASM_CODE_COMMENT(this);
2984
  Ldr(destination.W(), field_operand);
2985 2986 2987 2988 2989
  if (FLAG_debug_code) {
    // Corrupt the top 32 bits. Made up of 16 fixed bits and 16 pc offset bits.
    Add(destination, destination,
        ((kDebugZapValue << 16) | (pc_offset() & 0xffff)) << 32);
  }
2990 2991 2992 2993
}

void TurboAssembler::DecompressTaggedPointer(const Register& destination,
                                             const MemOperand& field_operand) {
2994
  ASM_CODE_COMMENT(this);
2995
  Ldr(destination.W(), field_operand);
2996
  Add(destination, kPtrComprCageBaseRegister, destination);
2997 2998
}

2999 3000
void TurboAssembler::DecompressTaggedPointer(const Register& destination,
                                             const Register& source) {
3001
  ASM_CODE_COMMENT(this);
3002
  Add(destination, kPtrComprCageBaseRegister, Operand(source, UXTW));
3003 3004
}

3005 3006
void TurboAssembler::DecompressAnyTagged(const Register& destination,
                                         const MemOperand& field_operand) {
3007
  ASM_CODE_COMMENT(this);
3008
  Ldr(destination.W(), field_operand);
3009
  Add(destination, kPtrComprCageBaseRegister, destination);
3010
}
3011

3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045
void TurboAssembler::AtomicDecompressTaggedSigned(const Register& destination,
                                                  const Register& base,
                                                  const Register& index,
                                                  const Register& temp) {
  ASM_CODE_COMMENT(this);
  Add(temp, base, index);
  Ldar(destination.W(), temp);
  if (FLAG_debug_code) {
    // Corrupt the top 32 bits. Made up of 16 fixed bits and 16 pc offset bits.
    Add(destination, destination,
        ((kDebugZapValue << 16) | (pc_offset() & 0xffff)) << 32);
  }
}

void TurboAssembler::AtomicDecompressTaggedPointer(const Register& destination,
                                                   const Register& base,
                                                   const Register& index,
                                                   const Register& temp) {
  ASM_CODE_COMMENT(this);
  Add(temp, base, index);
  Ldar(destination.W(), temp);
  Add(destination, kPtrComprCageBaseRegister, destination);
}

void TurboAssembler::AtomicDecompressAnyTagged(const Register& destination,
                                               const Register& base,
                                               const Register& index,
                                               const Register& temp) {
  ASM_CODE_COMMENT(this);
  Add(temp, base, index);
  Ldar(destination.W(), temp);
  Add(destination, kPtrComprCageBaseRegister, destination);
}

3046
void TurboAssembler::CheckPageFlag(const Register& object, int mask,
3047
                                   Condition cc, Label* condition_met) {
3048
  ASM_CODE_COMMENT(this);
3049 3050
  UseScratchRegisterScope temps(this);
  Register scratch = temps.AcquireX();
3051
  And(scratch, object, ~kPageAlignmentMask);
3052
  Ldr(scratch, MemOperand(scratch, BasicMemoryChunk::kFlagsOffset));
3053 3054 3055
  if (cc == eq) {
    TestAndBranchIfAnySet(scratch, mask, condition_met);
  } else {
3056
    DCHECK_EQ(cc, ne);
3057 3058 3059
    TestAndBranchIfAllClear(scratch, mask, condition_met);
  }
}
3060

3061
void MacroAssembler::RecordWriteField(Register object, int offset,
3062
                                      Register value,
3063 3064 3065 3066
                                      LinkRegisterStatus lr_status,
                                      SaveFPRegsMode save_fp,
                                      RememberedSetAction remembered_set_action,
                                      SmiCheck smi_check) {
3067
  ASM_CODE_COMMENT(this);
3068
  DCHECK(!AreAliased(object, value));
3069 3070 3071 3072 3073
  // First, check if a write barrier is even needed. The tests below
  // catch stores of Smis.
  Label done;

  // Skip the barrier if writing a smi.
3074
  if (smi_check == SmiCheck::kInline) {
3075 3076 3077 3078
    JumpIfSmi(value, &done);
  }

  // Although the object register is tagged, the offset is relative to the start
3079 3080
  // of the object, so offset must be a multiple of kTaggedSize.
  DCHECK(IsAligned(offset, kTaggedSize));
3081

3082
  if (FLAG_debug_code) {
3083
    ASM_CODE_COMMENT_STRING(this, "Verify slot_address");
3084
    Label ok;
3085 3086
    UseScratchRegisterScope temps(this);
    Register scratch = temps.AcquireX();
3087
    DCHECK(!AreAliased(object, value, scratch));
3088
    Add(scratch, object, offset - kHeapObjectTag);
3089
    Tst(scratch, kTaggedSize - 1);
3090
    B(eq, &ok);
3091
    Abort(AbortReason::kUnalignedCellInWriteBarrier);
3092 3093 3094
    Bind(&ok);
  }

3095
  RecordWrite(object, Operand(offset - kHeapObjectTag), value, lr_status,
3096
              save_fp, remembered_set_action, SmiCheck::kOmit);
3097 3098 3099 3100

  Bind(&done);
}

Samuel Groß's avatar
Samuel Groß committed
3101
void TurboAssembler::EncodeSandboxedPointer(const Register& value) {
Samuel Groß's avatar
Samuel Groß committed
3102
  ASM_CODE_COMMENT(this);
Samuel Groß's avatar
Samuel Groß committed
3103
#ifdef V8_SANDBOXED_POINTERS
Samuel Groß's avatar
Samuel Groß committed
3104
  Sub(value, value, kPtrComprCageBaseRegister);
Samuel Groß's avatar
Samuel Groß committed
3105
  Mov(value, Operand(value, LSL, kSandboxedPointerShift));
Samuel Groß's avatar
Samuel Groß committed
3106 3107 3108 3109 3110
#else
  UNREACHABLE();
#endif
}

Samuel Groß's avatar
Samuel Groß committed
3111
void TurboAssembler::DecodeSandboxedPointer(const Register& value) {
Samuel Groß's avatar
Samuel Groß committed
3112
  ASM_CODE_COMMENT(this);
Samuel Groß's avatar
Samuel Groß committed
3113
#ifdef V8_SANDBOXED_POINTERS
Samuel Groß's avatar
Samuel Groß committed
3114
  Add(value, kPtrComprCageBaseRegister,
Samuel Groß's avatar
Samuel Groß committed
3115
      Operand(value, LSR, kSandboxedPointerShift));
Samuel Groß's avatar
Samuel Groß committed
3116 3117 3118 3119 3120
#else
  UNREACHABLE();
#endif
}

Samuel Groß's avatar
Samuel Groß committed
3121 3122
void TurboAssembler::LoadSandboxedPointerField(
    const Register& destination, const MemOperand& field_operand) {
Samuel Groß's avatar
Samuel Groß committed
3123 3124
  ASM_CODE_COMMENT(this);
  Ldr(destination, field_operand);
Samuel Groß's avatar
Samuel Groß committed
3125
  DecodeSandboxedPointer(destination);
Samuel Groß's avatar
Samuel Groß committed
3126 3127
}

Samuel Groß's avatar
Samuel Groß committed
3128
void TurboAssembler::StoreSandboxedPointerField(
Samuel Groß's avatar
Samuel Groß committed
3129 3130 3131 3132 3133
    const Register& value, const MemOperand& dst_field_operand) {
  ASM_CODE_COMMENT(this);
  UseScratchRegisterScope temps(this);
  Register scratch = temps.AcquireX();
  Mov(scratch, value);
Samuel Groß's avatar
Samuel Groß committed
3134
  EncodeSandboxedPointer(scratch);
Samuel Groß's avatar
Samuel Groß committed
3135 3136 3137
  Str(scratch, dst_field_operand);
}

3138 3139 3140 3141 3142 3143
void TurboAssembler::LoadExternalPointerField(Register destination,
                                              MemOperand field_operand,
                                              ExternalPointerTag tag,
                                              Register isolate_root) {
  DCHECK(!AreAliased(destination, isolate_root));
  ASM_CODE_COMMENT(this);
Samuel Groß's avatar
Samuel Groß committed
3144
#ifdef V8_SANDBOXED_EXTERNAL_POINTERS
3145
  DCHECK_NE(kExternalPointerNullTag, tag);
3146 3147 3148 3149 3150 3151 3152 3153 3154 3155
  UseScratchRegisterScope temps(this);
  Register external_table = temps.AcquireX();
  if (isolate_root == no_reg) {
    DCHECK(root_array_available_);
    isolate_root = kRootRegister;
  }
  Ldr(external_table,
      MemOperand(isolate_root,
                 IsolateData::external_pointer_table_offset() +
                     Internals::kExternalPointerTableBufferOffset));
3156
  Ldr(destination.W(), field_operand);
3157 3158 3159 3160 3161 3162 3163
  // MemOperand doesn't support LSR currently (only LSL), so here we do the
  // offset computation separately first.
  STATIC_ASSERT(kExternalPointerIndexShift > kSystemPointerSizeLog2);
  int shift_amount = kExternalPointerIndexShift - kSystemPointerSizeLog2;
  Mov(destination, Operand(destination, LSR, shift_amount));
  Ldr(destination, MemOperand(external_table, destination));
  And(destination, destination, Immediate(~tag));
3164 3165
#else
  Ldr(destination, field_operand);
Samuel Groß's avatar
Samuel Groß committed
3166
#endif  // V8_SANDBOXED_EXTERNAL_POINTERS
3167 3168
}

3169
void TurboAssembler::MaybeSaveRegisters(RegList registers) {
3170
  if (registers.is_empty()) return;
3171
  ASM_CODE_COMMENT(this);
3172
  CPURegList regs(kXRegSizeInBits, registers);
3173 3174 3175
  // If we were saving LR, we might need to sign it.
  DCHECK(!regs.IncludesAliasOf(lr));
  regs.Align();
3176 3177 3178
  PushCPURegList(regs);
}

3179
void TurboAssembler::MaybeRestoreRegisters(RegList registers) {
3180
  if (registers.is_empty()) return;
3181
  ASM_CODE_COMMENT(this);
3182
  CPURegList regs(kXRegSizeInBits, registers);
3183 3184 3185
  // If we were saving LR, we might need to sign it.
  DCHECK(!regs.IncludesAliasOf(lr));
  regs.Align();
3186 3187 3188
  PopCPURegList(regs);
}

3189
void TurboAssembler::CallEphemeronKeyBarrier(Register object, Operand offset,
3190
                                             SaveFPRegsMode fp_mode) {
3191
  ASM_CODE_COMMENT(this);
3192 3193
  RegList registers = WriteBarrierDescriptor::ComputeSavedRegisters(object);
  MaybeSaveRegisters(registers);
3194

3195 3196 3197
  MoveObjectAndSlot(WriteBarrierDescriptor::ObjectRegister(),
                    WriteBarrierDescriptor::SlotAddressRegister(), object,
                    offset);
3198

3199
  Call(isolate()->builtins()->code_handle(
3200
           Builtins::GetEphemeronKeyBarrierStub(fp_mode)),
3201
       RelocInfo::CODE_TARGET);
3202
  MaybeRestoreRegisters(registers);
3203 3204
}

3205
void TurboAssembler::CallRecordWriteStubSaveRegisters(
3206
    Register object, Operand offset, RememberedSetAction remembered_set_action,
3207
    SaveFPRegsMode fp_mode, StubCallMode mode) {
3208
  ASM_CODE_COMMENT(this);
3209 3210
  RegList registers = WriteBarrierDescriptor::ComputeSavedRegisters(object);
  MaybeSaveRegisters(registers);
3211

3212 3213 3214 3215
  Register object_parameter = WriteBarrierDescriptor::ObjectRegister();
  Register slot_address_parameter =
      WriteBarrierDescriptor::SlotAddressRegister();
  MoveObjectAndSlot(object_parameter, slot_address_parameter, object, offset);
3216

3217 3218
  CallRecordWriteStub(object_parameter, slot_address_parameter,
                      remembered_set_action, fp_mode, mode);
3219

3220 3221 3222 3223 3224 3225 3226
  MaybeRestoreRegisters(registers);
}

void TurboAssembler::CallRecordWriteStub(
    Register object, Register slot_address,
    RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode,
    StubCallMode mode) {
3227
  ASM_CODE_COMMENT(this);
3228 3229
  DCHECK_EQ(WriteBarrierDescriptor::ObjectRegister(), object);
  DCHECK_EQ(WriteBarrierDescriptor::SlotAddressRegister(), slot_address);
3230
#if V8_ENABLE_WEBASSEMBLY
3231 3232 3233
  if (mode == StubCallMode::kCallWasmRuntimeStub) {
    auto wasm_target =
        wasm::WasmCode::GetRecordWriteStub(remembered_set_action, fp_mode);
3234
    Call(wasm_target, RelocInfo::WASM_STUB_CALL);
3235 3236 3237
#else
  if (false) {
#endif
3238
  } else {
3239
    Builtin builtin =
3240 3241
        Builtins::GetRecordWriteStub(remembered_set_action, fp_mode);
    if (options().inline_offheap_trampolines) {
3242
      CallBuiltin(builtin);
3243
    } else {
3244
      Handle<CodeT> code_target = isolate()->builtins()->code_handle(builtin);
3245 3246
      Call(code_target, RelocInfo::CODE_TARGET);
    }
3247
  }
3248
}
3249

3250 3251
void TurboAssembler::MoveObjectAndSlot(Register dst_object, Register dst_slot,
                                       Register object, Operand offset) {
3252
  ASM_CODE_COMMENT(this);
3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284
  DCHECK_NE(dst_object, dst_slot);
  // If `offset` is a register, it cannot overlap with `object`.
  DCHECK_IMPLIES(!offset.IsImmediate(), offset.reg() != object);

  // If the slot register does not overlap with the object register, we can
  // overwrite it.
  if (dst_slot != object) {
    Add(dst_slot, object, offset);
    Mov(dst_object, object);
    return;
  }

  DCHECK_EQ(dst_slot, object);

  // If the destination object register does not overlap with the offset
  // register, we can overwrite it.
  if (offset.IsImmediate() || (offset.reg() != dst_object)) {
    Mov(dst_object, dst_slot);
    Add(dst_slot, dst_slot, offset);
    return;
  }

  DCHECK_EQ(dst_object, offset.reg());

  // We only have `dst_slot` and `dst_object` left as distinct registers so we
  // have to swap them. We write this as a add+sub sequence to avoid using a
  // scratch register.
  Add(dst_slot, dst_slot, dst_object);
  Sub(dst_object, dst_slot, dst_object);
}

// If lr_status is kLRHasBeenSaved, lr will be clobbered.
3285 3286 3287
//
// The register 'object' contains a heap object pointer. The heap object tag is
// shifted away.
3288
void MacroAssembler::RecordWrite(Register object, Operand offset,
3289 3290 3291 3292
                                 Register value, LinkRegisterStatus lr_status,
                                 SaveFPRegsMode fp_mode,
                                 RememberedSetAction remembered_set_action,
                                 SmiCheck smi_check) {
3293
  ASM_CODE_COMMENT(this);
3294
  ASM_LOCATION_IN_ASSEMBLER("MacroAssembler::RecordWrite");
3295
  DCHECK(!AreAliased(object, value));
3296

3297
  if (FLAG_debug_code) {
3298
    ASM_CODE_COMMENT_STRING(this, "Verify slot_address");
3299 3300
    UseScratchRegisterScope temps(this);
    Register temp = temps.AcquireX();
3301
    DCHECK(!AreAliased(object, value, temp));
3302 3303
    Add(temp, object, offset);
    LoadTaggedPointerField(temp, MemOperand(temp));
3304
    Cmp(temp, value);
3305
    Check(eq, AbortReason::kWrongAddressOrValuePassedToRecordWrite);
3306 3307
  }

3308
  if ((remembered_set_action == RememberedSetAction::kOmit &&
3309 3310 3311 3312 3313
       !FLAG_incremental_marking) ||
      FLAG_disable_write_barriers) {
    return;
  }

3314 3315 3316 3317
  // First, check if a write barrier is even needed. The tests below
  // catch stores of smis and stores into the young generation.
  Label done;

3318
  if (smi_check == SmiCheck::kInline) {
3319
    DCHECK_EQ(0, kSmiTag);
3320 3321
    JumpIfSmi(value, &done);
  }
3322 3323
  CheckPageFlag(value, MemoryChunk::kPointersToHereAreInterestingMask, ne,
                &done);
3324

3325 3326
  CheckPageFlag(object, MemoryChunk::kPointersFromHereAreInterestingMask, ne,
                &done);
3327 3328 3329

  // Record the actual write.
  if (lr_status == kLRHasNotBeenSaved) {
3330
    Push<TurboAssembler::kSignLR>(padreg, lr);
3331
  }
3332 3333 3334 3335 3336 3337
  Register slot_address = WriteBarrierDescriptor::SlotAddressRegister();
  DCHECK(!AreAliased(object, slot_address, value));
  // TODO(cbruni): Turn offset into int.
  DCHECK(offset.IsImmediate());
  Add(slot_address, object, offset);
  CallRecordWriteStub(object, slot_address, remembered_set_action, fp_mode);
3338
  if (lr_status == kLRHasNotBeenSaved) {
3339
    Pop<TurboAssembler::kAuthLR>(lr, padreg);
3340
  }
3341
  if (FLAG_debug_code) Mov(slot_address, Operand(kZapValue));
3342 3343 3344 3345

  Bind(&done);
}

3346
void TurboAssembler::Assert(Condition cond, AbortReason reason) {
3347
  if (FLAG_debug_code) {
3348 3349 3350 3351
    Check(cond, reason);
  }
}

3352
void TurboAssembler::AssertUnreachable(AbortReason reason) {
3353
  if (FLAG_debug_code) Abort(reason);
3354 3355
}

3356
void TurboAssembler::Check(Condition cond, AbortReason reason) {
3357 3358 3359 3360 3361 3362 3363
  Label ok;
  B(cond, &ok);
  Abort(reason);
  // Will not return here.
  Bind(&ok);
}

3364
void TurboAssembler::Trap() { Brk(0); }
3365
void TurboAssembler::DebugBreak() { Debug("DebugBreak", 0, BREAK); }
3366

3367
void TurboAssembler::Abort(AbortReason reason) {
3368
  ASM_CODE_COMMENT(this);
3369 3370 3371 3372
  if (FLAG_code_comments) {
    RecordComment("Abort message: ");
    RecordComment(GetAbortReason(reason));
  }
3373

3374 3375
  // Avoid emitting call to builtin if requested.
  if (trap_on_abort()) {
3376 3377 3378 3379
    Brk(0);
    return;
  }

3380 3381
  // We need some scratch registers for the MacroAssembler, so make sure we have
  // some. This is safe here because Abort never returns.
3382
  uint64_t old_tmp_list = TmpList()->bits();
3383
  TmpList()->Combine(MacroAssembler::DefaultTmpList());
3384

3385 3386
  if (should_abort_hard()) {
    // We don't care if we constructed a frame. Just pretend we did.
3387
    FrameScope assume_frame(this, StackFrame::NO_FRAME_TYPE);
3388 3389 3390 3391
    Mov(w0, static_cast<int>(reason));
    Call(ExternalReference::abort_with_reason());
    return;
  }
3392

3393 3394
  // Avoid infinite recursion; Push contains some assertions that use Abort.
  HardAbortScope hard_aborts(this);
3395

3396
  Mov(x1, Smi::FromInt(static_cast<int>(reason)));
3397 3398 3399 3400

  if (!has_frame_) {
    // We don't actually want to generate a pile of code for this, so just
    // claim there is a stack frame, without generating one.
3401
    FrameScope scope(this, StackFrame::NO_FRAME_TYPE);
3402
    Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
3403
  } else {
3404
    Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
3405
  }
3406

3407
  TmpList()->set_bits(old_tmp_list);
3408 3409
}

3410
void MacroAssembler::LoadNativeContextSlot(Register dst, int index) {
3411 3412 3413 3414 3415
  LoadMap(dst, cp);
  LoadTaggedPointerField(
      dst, FieldMemOperand(
               dst, Map::kConstructorOrBackPointerOrNativeContextOffset));
  LoadTaggedPointerField(dst, MemOperand(dst, Context::SlotOffset(index)));
3416 3417 3418 3419
}

// This is the main Printf implementation. All other Printf variants call
// PrintfNoPreserve after setting up one or more PreserveRegisterScopes.
3420
void TurboAssembler::PrintfNoPreserve(const char* format,
3421 3422 3423 3424
                                      const CPURegister& arg0,
                                      const CPURegister& arg1,
                                      const CPURegister& arg2,
                                      const CPURegister& arg3) {
3425
  ASM_CODE_COMMENT(this);
3426 3427
  // We cannot handle a caller-saved stack pointer. It doesn't make much sense
  // in most cases anyway, so this restriction shouldn't be too serious.
3428
  DCHECK(!kCallerSaved.IncludesAliasOf(sp));
3429

3430 3431 3432 3433 3434 3435 3436 3437 3438 3439 3440
  // The provided arguments, and their proper procedure-call standard registers.
  CPURegister args[kPrintfMaxArgCount] = {arg0, arg1, arg2, arg3};
  CPURegister pcs[kPrintfMaxArgCount] = {NoReg, NoReg, NoReg, NoReg};

  int arg_count = kPrintfMaxArgCount;

  // The PCS varargs registers for printf. Note that x0 is used for the printf
  // format string.
  static const CPURegList kPCSVarargs =
      CPURegList(CPURegister::kRegister, kXRegSizeInBits, 1, arg_count);
  static const CPURegList kPCSVarargsFP =
3441
      CPURegList(CPURegister::kVRegister, kDRegSizeInBits, 0, arg_count - 1);
3442 3443 3444 3445

  // We can use caller-saved registers as scratch values, except for the
  // arguments and the PCS registers where they might need to go.
  CPURegList tmp_list = kCallerSaved;
3446
  tmp_list.Remove(x0);  // Used to pass the format string.
3447 3448 3449
  tmp_list.Remove(kPCSVarargs);
  tmp_list.Remove(arg0, arg1, arg2, arg3);

3450
  CPURegList fp_tmp_list = kCallerSavedV;
3451 3452 3453
  fp_tmp_list.Remove(kPCSVarargsFP);
  fp_tmp_list.Remove(arg0, arg1, arg2, arg3);

3454
  // Override the TurboAssembler's scratch register list. The lists will be
3455 3456
  // reset automatically at the end of the UseScratchRegisterScope.
  UseScratchRegisterScope temps(this);
3457 3458
  TmpList()->set_bits(tmp_list.bits());
  FPTmpList()->set_bits(fp_tmp_list.bits());
3459 3460 3461

  // Copies of the printf vararg registers that we can pop from.
  CPURegList pcs_varargs = kPCSVarargs;
3462
#ifndef V8_OS_WIN
3463
  CPURegList pcs_varargs_fp = kPCSVarargsFP;
3464
#endif
3465 3466 3467 3468 3469 3470 3471

  // Place the arguments. There are lots of clever tricks and optimizations we
  // could use here, but Printf is a debug tool so instead we just try to keep
  // it simple: Move each input that isn't already in the right place to a
  // scratch register, then move everything back.
  for (unsigned i = 0; i < kPrintfMaxArgCount; i++) {
    // Work out the proper PCS register for this argument.
3472
    if (args[i].IsRegister()) {
3473 3474 3475 3476
      pcs[i] = pcs_varargs.PopLowestIndex().X();
      // We might only need a W register here. We need to know the size of the
      // argument so we can properly encode it for the simulator call.
      if (args[i].Is32Bits()) pcs[i] = pcs[i].W();
3477
    } else if (args[i].IsVRegister()) {
3478
      // In C, floats are always cast to doubles for varargs calls.
3479 3480 3481 3482 3483 3484
#ifdef V8_OS_WIN
      // In case of variadic functions SIMD and Floating-point registers
      // aren't used. The general x0-x7 should be used instead.
      // https://docs.microsoft.com/en-us/cpp/build/arm64-windows-abi-conventions
      pcs[i] = pcs_varargs.PopLowestIndex().X();
#else
3485
      pcs[i] = pcs_varargs_fp.PopLowestIndex().D();
3486
#endif
3487
    } else {
3488
      DCHECK(args[i].IsNone());
3489 3490 3491 3492
      arg_count = i;
      break;
    }

3493 3494 3495 3496 3497 3498 3499 3500
    // If the argument is already in the right place, leave it where it is.
    if (args[i].Aliases(pcs[i])) continue;

    // Otherwise, if the argument is in a PCS argument register, allocate an
    // appropriate scratch register and then move it out of the way.
    if (kPCSVarargs.IncludesAliasOf(args[i]) ||
        kPCSVarargsFP.IncludesAliasOf(args[i])) {
      if (args[i].IsRegister()) {
3501
        Register old_arg = args[i].Reg();
3502 3503 3504 3505
        Register new_arg = temps.AcquireSameSizeAs(old_arg);
        Mov(new_arg, old_arg);
        args[i] = new_arg;
      } else {
3506
        VRegister old_arg = args[i].VReg();
3507
        VRegister new_arg = temps.AcquireSameSizeAs(old_arg);
3508 3509 3510
        Fmov(new_arg, old_arg);
        args[i] = new_arg;
      }
3511 3512 3513
    }
  }

3514 3515 3516
  // Do a second pass to move values into their final positions and perform any
  // conversions that may be required.
  for (int i = 0; i < arg_count; i++) {
3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532
#ifdef V8_OS_WIN
    if (args[i].IsVRegister()) {
      if (pcs[i].SizeInBytes() != args[i].SizeInBytes()) {
        // If the argument is half- or single-precision
        // converts to double-precision before that is
        // moved into the one of X scratch register.
        VRegister temp0 = temps.AcquireD();
        Fcvt(temp0.VReg(), args[i].VReg());
        Fmov(pcs[i].Reg(), temp0);
      } else {
        Fmov(pcs[i].Reg(), args[i].VReg());
      }
    } else {
      Mov(pcs[i].Reg(), args[i].Reg(), kDiscardForSameWReg);
    }
#else
3533
    DCHECK(pcs[i].type() == args[i].type());
3534
    if (pcs[i].IsRegister()) {
3535
      Mov(pcs[i].Reg(), args[i].Reg(), kDiscardForSameWReg);
3536
    } else {
3537
      DCHECK(pcs[i].IsVRegister());
3538
      if (pcs[i].SizeInBytes() == args[i].SizeInBytes()) {
3539
        Fmov(pcs[i].VReg(), args[i].VReg());
3540
      } else {
3541
        Fcvt(pcs[i].VReg(), args[i].VReg());
3542 3543
      }
    }
3544
#endif
3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556
  }

  // Load the format string into x0, as per the procedure-call standard.
  //
  // To make the code as portable as possible, the format string is encoded
  // directly in the instruction stream. It might be cleaner to encode it in a
  // literal pool, but since Printf is usually used for debugging, it is
  // beneficial for it to be minimally dependent on other features.
  Label format_address;
  Adr(x0, &format_address);

  // Emit the format string directly in the instruction stream.
3557 3558
  {
    BlockPoolsScope scope(this);
3559 3560 3561 3562 3563 3564 3565 3566
    Label after_data;
    B(&after_data);
    Bind(&format_address);
    EmitStringData(format);
    Unreachable();
    Bind(&after_data);
  }

3567
  CallPrintf(arg_count, pcs);
3568 3569
}

3570
void TurboAssembler::CallPrintf(int arg_count, const CPURegister* args) {
3571
  ASM_CODE_COMMENT(this);
3572 3573 3574 3575
  // A call to printf needs special handling for the simulator, since the system
  // printf function will use a different instruction set and the procedure-call
  // standard will not be compatible.
  if (options().enable_simulator_code) {
3576
    InstructionAccurateScope scope(this, kPrintfLength / kInstrSize);
3577
    hlt(kImmExceptionIsPrintf);
3578
    dc32(arg_count);  // kPrintfArgCountOffset
3579 3580 3581 3582 3583 3584 3585 3586

    // Determine the argument pattern.
    uint32_t arg_pattern_list = 0;
    for (int i = 0; i < arg_count; i++) {
      uint32_t arg_pattern;
      if (args[i].IsRegister()) {
        arg_pattern = args[i].Is32Bits() ? kPrintfArgW : kPrintfArgX;
      } else {
3587
        DCHECK(args[i].Is64Bits());
3588 3589
        arg_pattern = kPrintfArgD;
      }
3590
      DCHECK(arg_pattern < (1 << kPrintfArgPatternBits));
3591 3592
      arg_pattern_list |= (arg_pattern << (kPrintfArgPatternBits * i));
    }
3593
    dc32(arg_pattern_list);  // kPrintfArgPatternListOffset
3594
    return;
3595
  }
3596

3597
  Call(ExternalReference::printf_function());
3598 3599
}

3600
void TurboAssembler::Printf(const char* format, CPURegister arg0,
3601
                            CPURegister arg1, CPURegister arg2,
3602
                            CPURegister arg3) {
3603
  ASM_CODE_COMMENT(this);
3604 3605
  // Printf is expected to preserve all registers, so make sure that none are
  // available as scratch registers until we've preserved them.
3606 3607 3608 3609
  uint64_t old_tmp_list = TmpList()->bits();
  uint64_t old_fp_tmp_list = FPTmpList()->bits();
  TmpList()->set_bits(0);
  FPTmpList()->set_bits(0);
3610

3611
  CPURegList saved_registers = kCallerSaved;
3612
  saved_registers.Align();
3613

3614
  // Preserve all caller-saved registers as well as NZCV.
3615 3616
  // PushCPURegList asserts that the size of each list is a multiple of 16
  // bytes.
3617
  PushCPURegList(saved_registers);
3618
  PushCPURegList(kCallerSavedV);
3619 3620

  // We can use caller-saved registers as scratch values (except for argN).
3621
  CPURegList tmp_list = saved_registers;
3622
  CPURegList fp_tmp_list = kCallerSavedV;
3623 3624
  tmp_list.Remove(arg0, arg1, arg2, arg3);
  fp_tmp_list.Remove(arg0, arg1, arg2, arg3);
3625 3626
  TmpList()->set_bits(tmp_list.bits());
  FPTmpList()->set_bits(fp_tmp_list.bits());
3627

3628 3629
  {
    UseScratchRegisterScope temps(this);
3630 3631 3632
    // If any of the arguments are the current stack pointer, allocate a new
    // register for them, and adjust the value to compensate for pushing the
    // caller-saved registers.
3633 3634 3635 3636
    bool arg0_sp = arg0.is_valid() && sp.Aliases(arg0);
    bool arg1_sp = arg1.is_valid() && sp.Aliases(arg1);
    bool arg2_sp = arg2.is_valid() && sp.Aliases(arg2);
    bool arg3_sp = arg3.is_valid() && sp.Aliases(arg3);
3637 3638 3639 3640
    if (arg0_sp || arg1_sp || arg2_sp || arg3_sp) {
      // Allocate a register to hold the original stack pointer value, to pass
      // to PrintfNoPreserve as an argument.
      Register arg_sp = temps.AcquireX();
3641
      Add(arg_sp, sp,
3642 3643
          saved_registers.TotalSizeInBytes() +
              kCallerSavedV.TotalSizeInBytes());
3644 3645 3646 3647 3648
      if (arg0_sp) arg0 = Register::Create(arg_sp.code(), arg0.SizeInBits());
      if (arg1_sp) arg1 = Register::Create(arg_sp.code(), arg1.SizeInBits());
      if (arg2_sp) arg2 = Register::Create(arg_sp.code(), arg2.SizeInBits());
      if (arg3_sp) arg3 = Register::Create(arg_sp.code(), arg3.SizeInBits());
    }
3649

3650
    // Preserve NZCV.
3651 3652
    {
      UseScratchRegisterScope temps(this);
3653 3654 3655 3656
      Register tmp = temps.AcquireX();
      Mrs(tmp, NZCV);
      Push(tmp, xzr);
    }
3657

3658 3659 3660
    PrintfNoPreserve(format, arg0, arg1, arg2, arg3);

    // Restore NZCV.
3661 3662
    {
      UseScratchRegisterScope temps(this);
3663 3664 3665 3666
      Register tmp = temps.AcquireX();
      Pop(xzr, tmp);
      Msr(NZCV, tmp);
    }
3667 3668
  }

3669
  PopCPURegList(kCallerSavedV);
3670
  PopCPURegList(saved_registers);
3671

3672 3673
  TmpList()->set_bits(old_tmp_list);
  FPTmpList()->set_bits(old_fp_tmp_list);
3674 3675
}

3676
UseScratchRegisterScope::~UseScratchRegisterScope() {
3677 3678
  available_->set_bits(old_available_);
  availablefp_->set_bits(old_availablefp_);
3679 3680 3681 3682 3683 3684 3685
}

Register UseScratchRegisterScope::AcquireSameSizeAs(const Register& reg) {
  int code = AcquireNextAvailable(available_).code();
  return Register::Create(code, reg.SizeInBits());
}

3686
VRegister UseScratchRegisterScope::AcquireSameSizeAs(const VRegister& reg) {
3687
  int code = AcquireNextAvailable(availablefp_).code();
3688
  return VRegister::Create(code, reg.SizeInBits());
3689 3690 3691 3692 3693 3694
}

CPURegister UseScratchRegisterScope::AcquireNextAvailable(
    CPURegList* available) {
  CHECK(!available->IsEmpty());
  CPURegister result = available->PopLowestIndex();
3695
  DCHECK(!AreAliased(result, xzr, sp));
3696 3697 3698
  return result;
}

3699 3700 3701 3702 3703
void TurboAssembler::ComputeCodeStartAddress(const Register& rd) {
  // We can use adr to load a pc relative location.
  adr(rd, -pc_offset());
}

3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714
void TurboAssembler::RestoreFPAndLR() {
  static_assert(StandardFrameConstants::kCallerFPOffset + kSystemPointerSize ==
                    StandardFrameConstants::kCallerPCOffset,
                "Offsets must be consecutive for ldp!");
#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
  // Make sure we can use x16 and x17.
  UseScratchRegisterScope temps(this);
  temps.Exclude(x16, x17);
  // We can load the return address directly into x17.
  Add(x16, fp, StandardFrameConstants::kCallerSPOffset);
  Ldp(fp, x17, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
3715
  Autib1716();
3716 3717 3718 3719 3720 3721
  Mov(lr, x17);
#else
  Ldp(fp, lr, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
#endif
}

3722
#if V8_ENABLE_WEBASSEMBLY
3723 3724 3725 3726 3727 3728
void TurboAssembler::StoreReturnAddressInWasmExitFrame(Label* return_location) {
  UseScratchRegisterScope temps(this);
  temps.Exclude(x16, x17);
  Adr(x17, return_location);
#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY
  Add(x16, fp, WasmExitFrameConstants::kCallingPCOffset + kSystemPointerSize);
3729
  Pacib1716();
3730 3731 3732
#endif
  Str(x17, MemOperand(fp, WasmExitFrameConstants::kCallingPCOffset));
}
3733
#endif  // V8_ENABLE_WEBASSEMBLY
3734

3735 3736 3737 3738 3739 3740 3741 3742 3743 3744
void TurboAssembler::PopcntHelper(Register dst, Register src) {
  UseScratchRegisterScope temps(this);
  VRegister scratch = temps.AcquireV(kFormat8B);
  VRegister tmp = src.Is32Bits() ? scratch.S() : scratch.D();
  Fmov(tmp, src);
  Cnt(scratch, scratch);
  Addv(scratch.B(), scratch);
  Fmov(dst, tmp);
}

3745
void TurboAssembler::I64x2BitMask(Register dst, VRegister src) {
3746
  ASM_CODE_COMMENT(this);
3747 3748 3749 3750 3751 3752 3753 3754 3755
  UseScratchRegisterScope scope(this);
  VRegister tmp1 = scope.AcquireV(kFormat2D);
  Register tmp2 = scope.AcquireX();
  Ushr(tmp1.V2D(), src.V2D(), 63);
  Mov(dst.X(), tmp1.D(), 0);
  Mov(tmp2.X(), tmp1.D(), 1);
  Add(dst.W(), dst.W(), Operand(tmp2.W(), LSL, 1));
}

3756
void TurboAssembler::I64x2AllTrue(Register dst, VRegister src) {
3757
  ASM_CODE_COMMENT(this);
3758 3759 3760 3761 3762 3763 3764 3765
  UseScratchRegisterScope scope(this);
  VRegister tmp = scope.AcquireV(kFormat2D);
  Cmeq(tmp.V2D(), src.V2D(), 0);
  Addp(tmp.D(), tmp);
  Fcmp(tmp.D(), tmp.D());
  Cset(dst, eq);
}

3766 3767
}  // namespace internal
}  // namespace v8
3768

3769
#endif  // V8_TARGET_ARCH_ARM64