macro-assembler-mips64.cc 179 KB
Newer Older
1 2 3 4 5 6 7 8
// Copyright 2012 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include <limits.h>  // For LONG_MIN, LONG_MAX.

#if V8_TARGET_ARCH_MIPS64

9
#include "src/base/bits.h"
10
#include "src/base/division-by-constant.h"
11
#include "src/bootstrapper.h"
12
#include "src/callable.h"
13
#include "src/code-factory.h"
14
#include "src/code-stubs.h"
15
#include "src/debug/debug.h"
16
#include "src/external-reference-table.h"
17
#include "src/frames-inl.h"
18
#include "src/instruction-stream.h"
19
#include "src/mips64/assembler-mips64-inl.h"
20
#include "src/mips64/macro-assembler-mips64.h"
21
#include "src/register-configuration.h"
22
#include "src/runtime/runtime.h"
23
#include "src/snapshot/snapshot.h"
24 25 26 27

namespace v8 {
namespace internal {

28
MacroAssembler::MacroAssembler(Isolate* isolate, void* buffer, int size,
29
                               CodeObjectRequired create_code_object)
30 31 32 33 34 35 36 37 38 39 40
    : TurboAssembler(isolate, buffer, size, create_code_object) {
  if (create_code_object == CodeObjectRequired::kYes) {
    // Unlike TurboAssembler, which can be used off the main thread and may not
    // allocate, macro assembler creates its own copy of the self-reference
    // marker in order to disambiguate between self-references during nested
    // code generation (e.g.: codegen of the current object triggers stub
    // compilation through CodeStub::GetCode()).
    code_object_ = Handle<HeapObject>::New(
        *isolate->factory()->NewSelfReferenceMarker(), isolate);
  }
}
41

42 43 44 45 46 47 48 49
static inline bool IsZero(const Operand& rt) {
  if (rt.is_reg()) {
    return rt.rm() == zero_reg;
  } else {
    return rt.immediate() == 0;
  }
}

50 51 52 53 54
int TurboAssembler::RequiredStackSizeForCallerSaved(SaveFPRegsMode fp_mode,
                                                    Register exclusion1,
                                                    Register exclusion2,
                                                    Register exclusion3) const {
  int bytes = 0;
55
  RegList exclusions = 0;
56
  if (exclusion1 != no_reg) {
57
    exclusions |= exclusion1.bit();
58
    if (exclusion2 != no_reg) {
59
      exclusions |= exclusion2.bit();
60
      if (exclusion3 != no_reg) {
61 62 63 64 65
        exclusions |= exclusion3.bit();
      }
    }
  }

66 67
  RegList list = kJSCallerSaved & ~exclusions;
  bytes += NumRegs(list) * kPointerSize;
68 69

  if (fp_mode == kSaveFPRegs) {
70
    bytes += NumRegs(kCallerSavedFPU) * kDoubleSize;
71
  }
72 73

  return bytes;
74 75
}

76
int TurboAssembler::PushCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
77
                                    Register exclusion2, Register exclusion3) {
78 79
  int bytes = 0;
  RegList exclusions = 0;
80
  if (exclusion1 != no_reg) {
81
    exclusions |= exclusion1.bit();
82
    if (exclusion2 != no_reg) {
83
      exclusions |= exclusion2.bit();
84
      if (exclusion3 != no_reg) {
85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104
        exclusions |= exclusion3.bit();
      }
    }
  }

  RegList list = kJSCallerSaved & ~exclusions;
  MultiPush(list);
  bytes += NumRegs(list) * kPointerSize;

  if (fp_mode == kSaveFPRegs) {
    MultiPushFPU(kCallerSavedFPU);
    bytes += NumRegs(kCallerSavedFPU) * kDoubleSize;
  }

  return bytes;
}

int TurboAssembler::PopCallerSaved(SaveFPRegsMode fp_mode, Register exclusion1,
                                   Register exclusion2, Register exclusion3) {
  int bytes = 0;
105 106
  if (fp_mode == kSaveFPRegs) {
    MultiPopFPU(kCallerSavedFPU);
107
    bytes += NumRegs(kCallerSavedFPU) * kDoubleSize;
108 109 110
  }

  RegList exclusions = 0;
111
  if (exclusion1 != no_reg) {
112
    exclusions |= exclusion1.bit();
113
    if (exclusion2 != no_reg) {
114
      exclusions |= exclusion2.bit();
115
      if (exclusion3 != no_reg) {
116 117 118 119 120
        exclusions |= exclusion3.bit();
      }
    }
  }

121 122 123 124 125
  RegList list = kJSCallerSaved & ~exclusions;
  MultiPop(list);
  bytes += NumRegs(list) * kPointerSize;

  return bytes;
126 127
}

128
void TurboAssembler::LoadRoot(Register destination, Heap::RootListIndex index) {
129
  Ld(destination, MemOperand(s6, index << kPointerSizeLog2));
130 131
}

132 133 134
void TurboAssembler::LoadRoot(Register destination, Heap::RootListIndex index,
                              Condition cond, Register src1,
                              const Operand& src2) {
135
  Branch(2, NegateCondition(cond), src1, src2);
136
  Ld(destination, MemOperand(s6, index << kPointerSizeLog2));
137 138 139
}


140
void TurboAssembler::PushCommonFrame(Register marker_reg) {
141 142 143 144 145 146 147 148 149
  if (marker_reg.is_valid()) {
    Push(ra, fp, marker_reg);
    Daddu(fp, sp, Operand(kPointerSize));
  } else {
    Push(ra, fp);
    mov(fp, sp);
  }
}

150
void TurboAssembler::PushStandardFrame(Register function_reg) {
151 152 153 154 155 156 157 158 159
  int offset = -StandardFrameConstants::kContextOffset;
  if (function_reg.is_valid()) {
    Push(ra, fp, cp, function_reg);
    offset += kPointerSize;
  } else {
    Push(ra, fp, cp);
  }
  Daddu(fp, sp, Operand(offset));
}
160 161 162 163 164 165

// Push and pop all registers that can hold pointers.
void MacroAssembler::PushSafepointRegisters() {
  // Safepoints expect a block of kNumSafepointRegisters values on the
  // stack, so adjust the stack for unsaved registers.
  const int num_unsaved = kNumSafepointRegisters - kNumSafepointSavedRegisters;
166
  DCHECK_GE(num_unsaved, 0);
167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
  if (num_unsaved > 0) {
    Dsubu(sp, sp, Operand(num_unsaved * kPointerSize));
  }
  MultiPush(kSafepointSavedRegisters);
}


void MacroAssembler::PopSafepointRegisters() {
  const int num_unsaved = kNumSafepointRegisters - kNumSafepointSavedRegisters;
  MultiPop(kSafepointSavedRegisters);
  if (num_unsaved > 0) {
    Daddu(sp, sp, Operand(num_unsaved * kPointerSize));
  }
}

int MacroAssembler::SafepointRegisterStackIndex(int reg_code) {
  // The registers are pushed starting with the highest encoding,
  // which means that lowest encodings are closest to the stack pointer.
  return kSafepointRegisterStackIndexMap[reg_code];
}


189 190 191
// Clobbers object, dst, value, and ra, if (ra_status == kRAHasBeenSaved)
// The register 'object' contains a heap object pointer.  The heap object
// tag is shifted away.
192 193 194 195 196 197
void MacroAssembler::RecordWriteField(Register object, int offset,
                                      Register value, Register dst,
                                      RAStatus ra_status,
                                      SaveFPRegsMode save_fp,
                                      RememberedSetAction remembered_set_action,
                                      SmiCheck smi_check) {
198
  DCHECK(!AreAliased(value, dst, t8, object));
199 200 201 202 203 204 205 206 207 208 209
  // First, check if a write barrier is even needed. The tests below
  // catch stores of Smis.
  Label done;

  // Skip barrier if writing a smi.
  if (smi_check == INLINE_SMI_CHECK) {
    JumpIfSmi(value, &done);
  }

  // Although the object register is tagged, the offset is relative to the start
  // of the object, so so offset must be a multiple of kPointerSize.
210
  DCHECK(IsAligned(offset, kPointerSize));
211 212 213 214

  Daddu(dst, object, Operand(offset - kHeapObjectTag));
  if (emit_debug_code()) {
    Label ok;
215
    And(t8, dst, Operand(kPointerSize - 1));
216 217 218 219 220
    Branch(&ok, eq, t8, Operand(zero_reg));
    stop("Unaligned cell in write barrier");
    bind(&ok);
  }

221 222
  RecordWrite(object, dst, value, ra_status, save_fp, remembered_set_action,
              OMIT_SMI_CHECK);
223 224 225 226 227 228

  bind(&done);

  // Clobber clobbered input registers when running with the debug-code flag
  // turned on to provoke errors.
  if (emit_debug_code()) {
229 230
    li(value, Operand(bit_cast<int64_t>(kZapValue + 4)));
    li(dst, Operand(bit_cast<int64_t>(kZapValue + 8)));
231 232 233
  }
}

234
void TurboAssembler::SaveRegisters(RegList registers) {
235
  DCHECK_GT(NumRegs(registers), 0);
236 237 238 239 240 241 242 243 244 245
  RegList regs = 0;
  for (int i = 0; i < Register::kNumRegisters; ++i) {
    if ((registers >> i) & 1u) {
      regs |= Register::from_code(i).bit();
    }
  }
  MultiPush(regs);
}

void TurboAssembler::RestoreRegisters(RegList registers) {
246
  DCHECK_GT(NumRegs(registers), 0);
247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274
  RegList regs = 0;
  for (int i = 0; i < Register::kNumRegisters; ++i) {
    if ((registers >> i) & 1u) {
      regs |= Register::from_code(i).bit();
    }
  }
  MultiPop(regs);
}

void TurboAssembler::CallRecordWriteStub(
    Register object, Register address,
    RememberedSetAction remembered_set_action, SaveFPRegsMode fp_mode) {
  // TODO(albertnetymk): For now we ignore remembered_set_action and fp_mode,
  // i.e. always emit remember set and save FP registers in RecordWriteStub. If
  // large performance regression is observed, we should use these values to
  // avoid unnecessary work.

  Callable const callable =
      Builtins::CallableFor(isolate(), Builtins::kRecordWrite);
  RegList registers = callable.descriptor().allocatable_registers();

  SaveRegisters(registers);
  Register object_parameter(callable.descriptor().GetRegisterParameter(
      RecordWriteDescriptor::kObject));
  Register slot_parameter(
      callable.descriptor().GetRegisterParameter(RecordWriteDescriptor::kSlot));
  Register isolate_parameter(callable.descriptor().GetRegisterParameter(
      RecordWriteDescriptor::kIsolate));
275 276 277 278
  Register remembered_set_parameter(callable.descriptor().GetRegisterParameter(
      RecordWriteDescriptor::kRememberedSet));
  Register fp_mode_parameter(callable.descriptor().GetRegisterParameter(
      RecordWriteDescriptor::kFPMode));
279 280 281 282 283 284 285

  Push(object);
  Push(address);

  Pop(slot_parameter);
  Pop(object_parameter);

286
  li(isolate_parameter, ExternalReference::isolate_address(isolate()));
287 288
  Move(remembered_set_parameter, Smi::FromEnum(remembered_set_action));
  Move(fp_mode_parameter, Smi::FromEnum(fp_mode));
289 290 291 292
  Call(callable.code(), RelocInfo::CODE_TARGET);

  RestoreRegisters(registers);
}
293

294 295
// Clobbers object, address, value, and ra, if (ra_status == kRAHasBeenSaved)
// The register 'object' contains a heap object pointer.  The heap object
296
// tag is shifted away.
297 298 299 300 301
void MacroAssembler::RecordWrite(Register object, Register address,
                                 Register value, RAStatus ra_status,
                                 SaveFPRegsMode fp_mode,
                                 RememberedSetAction remembered_set_action,
                                 SmiCheck smi_check) {
302 303
  DCHECK(!AreAliased(object, address, value, t8));
  DCHECK(!AreAliased(object, address, value, t9));
304 305

  if (emit_debug_code()) {
306 307 308
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
    Ld(scratch, MemOperand(address));
309
    Assert(eq, AbortReason::kWrongAddressOrValuePassedToRecordWrite, scratch,
310
           Operand(value));
311 312 313 314 315 316 317 318 319 320 321 322
  }

  if (remembered_set_action == OMIT_REMEMBERED_SET &&
      !FLAG_incremental_marking) {
    return;
  }

  // First, check if a write barrier is even needed. The tests below
  // catch stores of smis and stores into the young generation.
  Label done;

  if (smi_check == INLINE_SMI_CHECK) {
323
    DCHECK_EQ(0, kSmiTag);
324 325 326
    JumpIfSmi(value, &done);
  }

327 328 329
  CheckPageFlag(value,
                value,  // Used as scratch.
                MemoryChunk::kPointersToHereAreInterestingMask, eq, &done);
330 331 332 333 334 335 336 337 338 339
  CheckPageFlag(object,
                value,  // Used as scratch.
                MemoryChunk::kPointersFromHereAreInterestingMask,
                eq,
                &done);

  // Record the actual write.
  if (ra_status == kRAHasNotBeenSaved) {
    push(ra);
  }
340
  CallRecordWriteStub(object, address, remembered_set_action, fp_mode);
341 342 343 344 345 346
  if (ra_status == kRAHasNotBeenSaved) {
    pop(ra);
  }

  bind(&done);

347 348 349 350 351 352 353 354
  {
    // Count number of write barriers in generated code.
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
    isolate()->counters()->write_barriers_static()->Increment();
    IncrementCounter(isolate()->counters()->write_barriers_dynamic(), 1,
                     scratch, value);
  }
355

356 357 358
  // Clobber clobbered registers when running with the debug-code flag
  // turned on to provoke errors.
  if (emit_debug_code()) {
359 360
    li(address, Operand(bit_cast<int64_t>(kZapValue + 12)));
    li(value, Operand(bit_cast<int64_t>(kZapValue + 16)));
361 362 363 364 365 366
  }
}

// ---------------------------------------------------------------------------
// Instruction macros.

367
void TurboAssembler::Addu(Register rd, Register rs, const Operand& rt) {
368 369 370
  if (rt.is_reg()) {
    addu(rd, rs, rt.rm());
  } else {
371
    if (is_int16(rt.immediate()) && !MustUseReg(rt.rmode())) {
372
      addiu(rd, rs, static_cast<int32_t>(rt.immediate()));
373 374
    } else {
      // li handles the relocation.
375 376
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
377
      DCHECK(rs != scratch);
378 379
      li(scratch, rt);
      addu(rd, rs, scratch);
380 381 382 383
    }
  }
}

384
void TurboAssembler::Daddu(Register rd, Register rs, const Operand& rt) {
385 386 387
  if (rt.is_reg()) {
    daddu(rd, rs, rt.rm());
  } else {
388
    if (is_int16(rt.immediate()) && !MustUseReg(rt.rmode())) {
389
      daddiu(rd, rs, static_cast<int32_t>(rt.immediate()));
390 391
    } else {
      // li handles the relocation.
392 393
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
394
      DCHECK(rs != scratch);
395 396
      li(scratch, rt);
      daddu(rd, rs, scratch);
397 398 399 400
    }
  }
}

401
void TurboAssembler::Subu(Register rd, Register rs, const Operand& rt) {
402 403 404
  if (rt.is_reg()) {
    subu(rd, rs, rt.rm());
  } else {
405
    DCHECK(is_int32(rt.immediate()));
406
    if (is_int16(-rt.immediate()) && !MustUseReg(rt.rmode())) {
407 408
      addiu(rd, rs,
            static_cast<int32_t>(
409
                -rt.immediate()));  // No subiu instr, use addiu(x, y, -imm).
410
    } else {
411 412
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
413
      DCHECK(rs != scratch);
414
      if (-rt.immediate() >> 16 == 0 && !MustUseReg(rt.rmode())) {
415
        // Use load -imm and addu when loading -imm generates one instruction.
416 417
        li(scratch, -rt.immediate());
        addu(rd, rs, scratch);
418 419
      } else {
        // li handles the relocation.
420 421
        li(scratch, rt);
        subu(rd, rs, scratch);
422
      }
423 424 425 426
    }
  }
}

427
void TurboAssembler::Dsubu(Register rd, Register rs, const Operand& rt) {
428 429
  if (rt.is_reg()) {
    dsubu(rd, rs, rt.rm());
430
  } else if (is_int16(-rt.immediate()) && !MustUseReg(rt.rmode())) {
431 432
    daddiu(rd, rs,
           static_cast<int32_t>(
433
               -rt.immediate()));  // No dsubiu instr, use daddiu(x, y, -imm).
434
  } else {
435
    DCHECK(rs != at);
436 437
    int li_count = InstrCountForLi64Bit(rt.immediate());
    int li_neg_count = InstrCountForLi64Bit(-rt.immediate());
438
    if (li_neg_count < li_count && !MustUseReg(rt.rmode())) {
439
      // Use load -imm and daddu when loading -imm generates one instruction.
440
      DCHECK(rt.immediate() != std::numeric_limits<int32_t>::min());
441 442 443 444
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
      li(scratch, Operand(-rt.immediate()));
      Daddu(rd, rs, scratch);
445 446
    } else {
      // li handles the relocation.
447 448 449 450
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
      li(scratch, rt);
      dsubu(rd, rs, scratch);
451 452 453 454
    }
  }
}

455
void TurboAssembler::Mul(Register rd, Register rs, const Operand& rt) {
456
  if (rt.is_reg()) {
457 458 459
    mul(rd, rs, rt.rm());
  } else {
    // li handles the relocation.
460 461
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
462
    DCHECK(rs != scratch);
463 464
    li(scratch, rt);
    mul(rd, rs, scratch);
465 466 467
  }
}

468
void TurboAssembler::Mulh(Register rd, Register rs, const Operand& rt) {
469 470
  if (rt.is_reg()) {
    if (kArchVariant != kMips64r6) {
471
      mult(rs, rt.rm());
472
      mfhi(rd);
473
    } else {
474
      muh(rd, rs, rt.rm());
475 476 477
    }
  } else {
    // li handles the relocation.
478 479
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
480
    DCHECK(rs != scratch);
481
    li(scratch, rt);
482
    if (kArchVariant != kMips64r6) {
483
      mult(rs, scratch);
484
      mfhi(rd);
485
    } else {
486
      muh(rd, rs, scratch);
487 488 489 490
    }
  }
}

491
void TurboAssembler::Mulhu(Register rd, Register rs, const Operand& rt) {
492 493 494 495 496 497 498 499 500
  if (rt.is_reg()) {
    if (kArchVariant != kMips64r6) {
      multu(rs, rt.rm());
      mfhi(rd);
    } else {
      muhu(rd, rs, rt.rm());
    }
  } else {
    // li handles the relocation.
501 502
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
503
    DCHECK(rs != scratch);
504
    li(scratch, rt);
505
    if (kArchVariant != kMips64r6) {
506
      multu(rs, scratch);
507 508
      mfhi(rd);
    } else {
509
      muhu(rd, rs, scratch);
510 511 512 513
    }
  }
}

514
void TurboAssembler::Dmul(Register rd, Register rs, const Operand& rt) {
515
  if (rt.is_reg()) {
516 517
    if (kArchVariant == kMips64r6) {
      dmul(rd, rs, rt.rm());
518 519 520 521 522 523
    } else {
      dmult(rs, rt.rm());
      mflo(rd);
    }
  } else {
    // li handles the relocation.
524 525
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
526
    DCHECK(rs != scratch);
527
    li(scratch, rt);
528
    if (kArchVariant == kMips64r6) {
529
      dmul(rd, rs, scratch);
530
    } else {
531
      dmult(rs, scratch);
532
      mflo(rd);
533 534 535 536
    }
  }
}

537
void TurboAssembler::Dmulh(Register rd, Register rs, const Operand& rt) {
538 539 540 541 542 543 544 545 546
  if (rt.is_reg()) {
    if (kArchVariant == kMips64r6) {
      dmuh(rd, rs, rt.rm());
    } else {
      dmult(rs, rt.rm());
      mfhi(rd);
    }
  } else {
    // li handles the relocation.
547 548
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
549
    DCHECK(rs != scratch);
550
    li(scratch, rt);
551
    if (kArchVariant == kMips64r6) {
552
      dmuh(rd, rs, scratch);
553
    } else {
554
      dmult(rs, scratch);
555
      mfhi(rd);
556 557 558 559
    }
  }
}

560
void TurboAssembler::Mult(Register rs, const Operand& rt) {
561 562 563 564
  if (rt.is_reg()) {
    mult(rs, rt.rm());
  } else {
    // li handles the relocation.
565 566
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
567
    DCHECK(rs != scratch);
568 569
    li(scratch, rt);
    mult(rs, scratch);
570 571 572
  }
}

573
void TurboAssembler::Dmult(Register rs, const Operand& rt) {
574 575 576 577
  if (rt.is_reg()) {
    dmult(rs, rt.rm());
  } else {
    // li handles the relocation.
578 579
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
580
    DCHECK(rs != scratch);
581 582
    li(scratch, rt);
    dmult(rs, scratch);
583 584 585
  }
}

586
void TurboAssembler::Multu(Register rs, const Operand& rt) {
587 588 589 590
  if (rt.is_reg()) {
    multu(rs, rt.rm());
  } else {
    // li handles the relocation.
591 592
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
593
    DCHECK(rs != scratch);
594 595
    li(scratch, rt);
    multu(rs, scratch);
596 597 598
  }
}

599
void TurboAssembler::Dmultu(Register rs, const Operand& rt) {
600 601 602 603
  if (rt.is_reg()) {
    dmultu(rs, rt.rm());
  } else {
    // li handles the relocation.
604 605
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
606
    DCHECK(rs != scratch);
607 608
    li(scratch, rt);
    dmultu(rs, scratch);
609 610 611
  }
}

612
void TurboAssembler::Div(Register rs, const Operand& rt) {
613 614 615 616
  if (rt.is_reg()) {
    div(rs, rt.rm());
  } else {
    // li handles the relocation.
617 618
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
619
    DCHECK(rs != scratch);
620 621
    li(scratch, rt);
    div(rs, scratch);
622 623 624
  }
}

625
void TurboAssembler::Div(Register res, Register rs, const Operand& rt) {
626 627 628 629 630 631 632 633 634
  if (rt.is_reg()) {
    if (kArchVariant != kMips64r6) {
      div(rs, rt.rm());
      mflo(res);
    } else {
      div(res, rs, rt.rm());
    }
  } else {
    // li handles the relocation.
635 636
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
637
    DCHECK(rs != scratch);
638
    li(scratch, rt);
639
    if (kArchVariant != kMips64r6) {
640
      div(rs, scratch);
641 642
      mflo(res);
    } else {
643
      div(res, rs, scratch);
644 645 646 647
    }
  }
}

648
void TurboAssembler::Mod(Register rd, Register rs, const Operand& rt) {
649 650 651 652 653 654 655 656 657
  if (rt.is_reg()) {
    if (kArchVariant != kMips64r6) {
      div(rs, rt.rm());
      mfhi(rd);
    } else {
      mod(rd, rs, rt.rm());
    }
  } else {
    // li handles the relocation.
658 659
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
660
    DCHECK(rs != scratch);
661
    li(scratch, rt);
662
    if (kArchVariant != kMips64r6) {
663
      div(rs, scratch);
664 665
      mfhi(rd);
    } else {
666
      mod(rd, rs, scratch);
667 668 669 670
    }
  }
}

671
void TurboAssembler::Modu(Register rd, Register rs, const Operand& rt) {
672 673 674 675 676 677 678 679 680
  if (rt.is_reg()) {
    if (kArchVariant != kMips64r6) {
      divu(rs, rt.rm());
      mfhi(rd);
    } else {
      modu(rd, rs, rt.rm());
    }
  } else {
    // li handles the relocation.
681 682
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
683
    DCHECK(rs != scratch);
684
    li(scratch, rt);
685
    if (kArchVariant != kMips64r6) {
686
      divu(rs, scratch);
687 688
      mfhi(rd);
    } else {
689
      modu(rd, rs, scratch);
690 691 692 693
    }
  }
}

694
void TurboAssembler::Ddiv(Register rs, const Operand& rt) {
695 696 697 698
  if (rt.is_reg()) {
    ddiv(rs, rt.rm());
  } else {
    // li handles the relocation.
699 700
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
701
    DCHECK(rs != scratch);
702 703
    li(scratch, rt);
    ddiv(rs, scratch);
704 705 706
  }
}

707
void TurboAssembler::Ddiv(Register rd, Register rs, const Operand& rt) {
708 709 710 711 712 713
  if (kArchVariant != kMips64r6) {
    if (rt.is_reg()) {
      ddiv(rs, rt.rm());
      mflo(rd);
    } else {
      // li handles the relocation.
714 715
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
716
      DCHECK(rs != scratch);
717 718
      li(scratch, rt);
      ddiv(rs, scratch);
719 720 721 722 723 724 725
      mflo(rd);
    }
  } else {
    if (rt.is_reg()) {
      ddiv(rd, rs, rt.rm());
    } else {
      // li handles the relocation.
726 727
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
728
      DCHECK(rs != scratch);
729 730
      li(scratch, rt);
      ddiv(rd, rs, scratch);
731 732 733 734
    }
  }
}

735
void TurboAssembler::Divu(Register rs, const Operand& rt) {
736 737 738 739
  if (rt.is_reg()) {
    divu(rs, rt.rm());
  } else {
    // li handles the relocation.
740 741
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
742
    DCHECK(rs != scratch);
743 744
    li(scratch, rt);
    divu(rs, scratch);
745 746 747
  }
}

748
void TurboAssembler::Divu(Register res, Register rs, const Operand& rt) {
749 750 751 752 753 754 755 756 757
  if (rt.is_reg()) {
    if (kArchVariant != kMips64r6) {
      divu(rs, rt.rm());
      mflo(res);
    } else {
      divu(res, rs, rt.rm());
    }
  } else {
    // li handles the relocation.
758 759
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
760
    DCHECK(rs != scratch);
761
    li(scratch, rt);
762
    if (kArchVariant != kMips64r6) {
763
      divu(rs, scratch);
764 765
      mflo(res);
    } else {
766
      divu(res, rs, scratch);
767 768 769 770
    }
  }
}

771
void TurboAssembler::Ddivu(Register rs, const Operand& rt) {
772 773 774 775
  if (rt.is_reg()) {
    ddivu(rs, rt.rm());
  } else {
    // li handles the relocation.
776 777
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
778
    DCHECK(rs != scratch);
779 780
    li(scratch, rt);
    ddivu(rs, scratch);
781 782 783
  }
}

784
void TurboAssembler::Ddivu(Register res, Register rs, const Operand& rt) {
785 786 787 788 789 790 791 792 793
  if (rt.is_reg()) {
    if (kArchVariant != kMips64r6) {
      ddivu(rs, rt.rm());
      mflo(res);
    } else {
      ddivu(res, rs, rt.rm());
    }
  } else {
    // li handles the relocation.
794 795
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
796
    DCHECK(rs != scratch);
797
    li(scratch, rt);
798
    if (kArchVariant != kMips64r6) {
799
      ddivu(rs, scratch);
800 801
      mflo(res);
    } else {
802
      ddivu(res, rs, scratch);
803 804 805 806
    }
  }
}

807
void TurboAssembler::Dmod(Register rd, Register rs, const Operand& rt) {
808 809 810 811 812 813
  if (kArchVariant != kMips64r6) {
    if (rt.is_reg()) {
      ddiv(rs, rt.rm());
      mfhi(rd);
    } else {
      // li handles the relocation.
814 815
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
816
      DCHECK(rs != scratch);
817 818
      li(scratch, rt);
      ddiv(rs, scratch);
819 820 821 822 823 824 825
      mfhi(rd);
    }
  } else {
    if (rt.is_reg()) {
      dmod(rd, rs, rt.rm());
    } else {
      // li handles the relocation.
826 827
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
828
      DCHECK(rs != scratch);
829 830
      li(scratch, rt);
      dmod(rd, rs, scratch);
831 832 833 834
    }
  }
}

835
void TurboAssembler::Dmodu(Register rd, Register rs, const Operand& rt) {
836 837 838 839 840 841
  if (kArchVariant != kMips64r6) {
    if (rt.is_reg()) {
      ddivu(rs, rt.rm());
      mfhi(rd);
    } else {
      // li handles the relocation.
842 843
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
844
      DCHECK(rs != scratch);
845 846
      li(scratch, rt);
      ddivu(rs, scratch);
847 848 849 850 851 852 853
      mfhi(rd);
    }
  } else {
    if (rt.is_reg()) {
      dmodu(rd, rs, rt.rm());
    } else {
      // li handles the relocation.
854 855
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
856
      DCHECK(rs != scratch);
857 858
      li(scratch, rt);
      dmodu(rd, rs, scratch);
859 860 861 862
    }
  }
}

863
void TurboAssembler::And(Register rd, Register rs, const Operand& rt) {
864 865 866
  if (rt.is_reg()) {
    and_(rd, rs, rt.rm());
  } else {
867
    if (is_uint16(rt.immediate()) && !MustUseReg(rt.rmode())) {
868
      andi(rd, rs, static_cast<int32_t>(rt.immediate()));
869 870
    } else {
      // li handles the relocation.
871 872
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
873
      DCHECK(rs != scratch);
874 875
      li(scratch, rt);
      and_(rd, rs, scratch);
876 877 878 879
    }
  }
}

880
void TurboAssembler::Or(Register rd, Register rs, const Operand& rt) {
881 882 883
  if (rt.is_reg()) {
    or_(rd, rs, rt.rm());
  } else {
884
    if (is_uint16(rt.immediate()) && !MustUseReg(rt.rmode())) {
885
      ori(rd, rs, static_cast<int32_t>(rt.immediate()));
886 887
    } else {
      // li handles the relocation.
888 889
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
890
      DCHECK(rs != scratch);
891 892
      li(scratch, rt);
      or_(rd, rs, scratch);
893 894 895 896
    }
  }
}

897
void TurboAssembler::Xor(Register rd, Register rs, const Operand& rt) {
898 899 900
  if (rt.is_reg()) {
    xor_(rd, rs, rt.rm());
  } else {
901
    if (is_uint16(rt.immediate()) && !MustUseReg(rt.rmode())) {
902
      xori(rd, rs, static_cast<int32_t>(rt.immediate()));
903 904
    } else {
      // li handles the relocation.
905 906
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
907
      DCHECK(rs != scratch);
908 909
      li(scratch, rt);
      xor_(rd, rs, scratch);
910 911 912 913
    }
  }
}

914
void TurboAssembler::Nor(Register rd, Register rs, const Operand& rt) {
915 916 917 918
  if (rt.is_reg()) {
    nor(rd, rs, rt.rm());
  } else {
    // li handles the relocation.
919 920
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
921
    DCHECK(rs != scratch);
922 923
    li(scratch, rt);
    nor(rd, rs, scratch);
924 925 926
  }
}

927
void TurboAssembler::Neg(Register rs, const Operand& rt) {
928
  dsubu(rs, zero_reg, rt.rm());
929 930
}

931
void TurboAssembler::Slt(Register rd, Register rs, const Operand& rt) {
932 933 934
  if (rt.is_reg()) {
    slt(rd, rs, rt.rm());
  } else {
935
    if (is_int16(rt.immediate()) && !MustUseReg(rt.rmode())) {
936
      slti(rd, rs, static_cast<int32_t>(rt.immediate()));
937 938
    } else {
      // li handles the relocation.
939 940
      UseScratchRegisterScope temps(this);
      Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
941
      DCHECK(rs != scratch);
942 943
      li(scratch, rt);
      slt(rd, rs, scratch);
944 945 946 947
    }
  }
}

948
void TurboAssembler::Sltu(Register rd, Register rs, const Operand& rt) {
949 950 951
  if (rt.is_reg()) {
    sltu(rd, rs, rt.rm());
  } else {
952
    const uint64_t int16_min = std::numeric_limits<int16_t>::min();
953
    if (is_uint15(rt.immediate()) && !MustUseReg(rt.rmode())) {
954
      // Imm range is: [0, 32767].
955 956
      sltiu(rd, rs, static_cast<int32_t>(rt.immediate()));
    } else if (is_uint15(rt.immediate() - int16_min) &&
957
               !MustUseReg(rt.rmode())) {
958
      // Imm range is: [max_unsigned-32767,max_unsigned].
959
      sltiu(rd, rs, static_cast<uint16_t>(rt.immediate()));
960 961
    } else {
      // li handles the relocation.
962 963
      UseScratchRegisterScope temps(this);
      Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
964
      DCHECK(rs != scratch);
965 966
      li(scratch, rt);
      sltu(rd, rs, scratch);
967 968 969 970
    }
  }
}

971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008
void TurboAssembler::Sle(Register rd, Register rs, const Operand& rt) {
  if (rt.is_reg()) {
    slt(rd, rt.rm(), rs);
  } else {
    // li handles the relocation.
    UseScratchRegisterScope temps(this);
    Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
    DCHECK(rs != scratch);
    li(scratch, rt);
    slt(rd, scratch, rs);
  }
  xori(rd, rd, 1);
}

void TurboAssembler::Sleu(Register rd, Register rs, const Operand& rt) {
  if (rt.is_reg()) {
    sltu(rd, rt.rm(), rs);
  } else {
    // li handles the relocation.
    UseScratchRegisterScope temps(this);
    Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
    DCHECK(rs != scratch);
    li(scratch, rt);
    sltu(rd, scratch, rs);
  }
  xori(rd, rd, 1);
}

void TurboAssembler::Sge(Register rd, Register rs, const Operand& rt) {
  Slt(rd, rs, rt);
  xori(rd, rd, 1);
}

void TurboAssembler::Sgeu(Register rd, Register rs, const Operand& rt) {
  Sltu(rd, rs, rt);
  xori(rd, rd, 1);
}

1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
void TurboAssembler::Sgt(Register rd, Register rs, const Operand& rt) {
  if (rt.is_reg()) {
    slt(rd, rt.rm(), rs);
  } else {
    // li handles the relocation.
    UseScratchRegisterScope temps(this);
    Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
    DCHECK(rs != scratch);
    li(scratch, rt);
    slt(rd, scratch, rs);
  }
}

void TurboAssembler::Sgtu(Register rd, Register rs, const Operand& rt) {
  if (rt.is_reg()) {
    sltu(rd, rt.rm(), rs);
  } else {
    // li handles the relocation.
    UseScratchRegisterScope temps(this);
    Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
    DCHECK(rs != scratch);
    li(scratch, rt);
    sltu(rd, scratch, rs);
  }
}

1035
void TurboAssembler::Ror(Register rd, Register rs, const Operand& rt) {
1036 1037
  if (rt.is_reg()) {
    rotrv(rd, rs, rt.rm());
1038
  } else {
1039
    int64_t ror_value = rt.immediate() % 32;
1040 1041 1042 1043
    if (ror_value < 0) {
      ror_value += 32;
    }
    rotr(rd, rs, ror_value);
1044 1045 1046
  }
}

1047
void TurboAssembler::Dror(Register rd, Register rs, const Operand& rt) {
1048 1049 1050
  if (rt.is_reg()) {
    drotrv(rd, rs, rt.rm());
  } else {
1051
    int64_t dror_value = rt.immediate() % 64;
1052 1053 1054 1055 1056 1057
    if (dror_value < 0) dror_value += 64;
    if (dror_value <= 31) {
      drotr(rd, rs, dror_value);
    } else {
      drotr32(rd, rs, dror_value - 32);
    }
1058 1059 1060 1061 1062 1063 1064 1065
  }
}


void MacroAssembler::Pref(int32_t hint, const MemOperand& rs) {
    pref(hint, rs);
}

1066
void TurboAssembler::Lsa(Register rd, Register rt, Register rs, uint8_t sa,
1067
                         Register scratch) {
1068
  DCHECK(sa >= 1 && sa <= 31);
1069
  if (kArchVariant == kMips64r6 && sa <= 4) {
1070
    lsa(rd, rt, rs, sa - 1);
1071
  } else {
1072 1073
    Register tmp = rd == rt ? scratch : rd;
    DCHECK(tmp != rt);
1074 1075 1076 1077 1078
    sll(tmp, rs, sa);
    Addu(rd, rt, tmp);
  }
}

1079
void TurboAssembler::Dlsa(Register rd, Register rt, Register rs, uint8_t sa,
1080
                          Register scratch) {
1081
  DCHECK(sa >= 1 && sa <= 31);
1082
  if (kArchVariant == kMips64r6 && sa <= 4) {
1083
    dlsa(rd, rt, rs, sa - 1);
1084
  } else {
1085 1086
    Register tmp = rd == rt ? scratch : rd;
    DCHECK(tmp != rt);
1087 1088 1089 1090 1091
    dsll(tmp, rs, sa);
    Daddu(rd, rt, tmp);
  }
}

1092
void TurboAssembler::Bovc(Register rs, Register rt, Label* L) {
1093 1094 1095 1096 1097 1098 1099 1100 1101 1102
  if (is_trampoline_emitted()) {
    Label skip;
    bnvc(rs, rt, &skip);
    BranchLong(L, PROTECT);
    bind(&skip);
  } else {
    bovc(rs, rt, L);
  }
}

1103
void TurboAssembler::Bnvc(Register rs, Register rt, Label* L) {
1104 1105 1106 1107 1108 1109 1110 1111 1112
  if (is_trampoline_emitted()) {
    Label skip;
    bovc(rs, rt, &skip);
    BranchLong(L, PROTECT);
    bind(&skip);
  } else {
    bnvc(rs, rt, L);
  }
}
1113

1114 1115
// ------------Pseudo-instructions-------------

1116
// Change endianness
1117
void TurboAssembler::ByteSwapSigned(Register dest, Register src,
1118
                                    int operand_size) {
1119 1120 1121 1122
  DCHECK(operand_size == 1 || operand_size == 2 || operand_size == 4 ||
         operand_size == 8);
  DCHECK(kArchVariant == kMips64r6 || kArchVariant == kMips64r2);
  if (operand_size == 1) {
1123 1124 1125 1126
    seb(src, src);
    sll(src, src, 0);
    dsbh(dest, src);
    dshd(dest, dest);
1127
  } else if (operand_size == 2) {
1128 1129 1130 1131
    seh(src, src);
    sll(src, src, 0);
    dsbh(dest, src);
    dshd(dest, dest);
1132
  } else if (operand_size == 4) {
1133 1134 1135
    sll(src, src, 0);
    dsbh(dest, src);
    dshd(dest, dest);
1136
  } else {
1137 1138
    dsbh(dest, src);
    dshd(dest, dest);
1139 1140 1141
  }
}

1142
void TurboAssembler::ByteSwapUnsigned(Register dest, Register src,
1143
                                      int operand_size) {
1144 1145
  DCHECK(operand_size == 1 || operand_size == 2 || operand_size == 4);
  if (operand_size == 1) {
1146 1147 1148
    andi(src, src, 0xFF);
    dsbh(dest, src);
    dshd(dest, dest);
1149
  } else if (operand_size == 2) {
1150 1151 1152
    andi(src, src, 0xFFFF);
    dsbh(dest, src);
    dshd(dest, dest);
1153
  } else {
1154 1155 1156 1157
    dsll32(src, src, 0);
    dsrl32(src, src, 0);
    dsbh(dest, src);
    dshd(dest, dest);
1158 1159 1160
  }
}

1161
void TurboAssembler::Ulw(Register rd, const MemOperand& rs) {
1162 1163
  DCHECK(rd != at);
  DCHECK(rs.rm() != at);
1164
  if (kArchVariant == kMips64r6) {
1165
    Lw(rd, rs);
1166
  } else {
1167
    DCHECK_EQ(kArchVariant, kMips64r2);
1168 1169 1170 1171
    DCHECK(kMipsLwrOffset <= 3 && kMipsLwlOffset <= 3);
    MemOperand source = rs;
    // Adjust offset for two accesses and check if offset + 3 fits into int16_t.
    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 3);
1172
    if (rd != source.rm()) {
1173 1174 1175
      lwr(rd, MemOperand(source.rm(), source.offset() + kMipsLwrOffset));
      lwl(rd, MemOperand(source.rm(), source.offset() + kMipsLwlOffset));
    } else {
1176 1177 1178 1179 1180
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
      lwr(scratch, MemOperand(rs.rm(), rs.offset() + kMipsLwrOffset));
      lwl(scratch, MemOperand(rs.rm(), rs.offset() + kMipsLwlOffset));
      mov(rd, scratch);
1181 1182 1183 1184
    }
  }
}

1185
void TurboAssembler::Ulwu(Register rd, const MemOperand& rs) {
1186
  if (kArchVariant == kMips64r6) {
1187
    Lwu(rd, rs);
1188
  } else {
1189
    DCHECK_EQ(kArchVariant, kMips64r2);
1190 1191 1192
    Ulw(rd, rs);
    Dext(rd, rd, 0, 32);
  }
1193 1194
}

1195
void TurboAssembler::Usw(Register rd, const MemOperand& rs) {
1196 1197 1198
  DCHECK(rd != at);
  DCHECK(rs.rm() != at);
  DCHECK(rd != rs.rm());
1199
  if (kArchVariant == kMips64r6) {
1200
    Sw(rd, rs);
1201
  } else {
1202
    DCHECK_EQ(kArchVariant, kMips64r2);
1203 1204 1205 1206 1207 1208
    DCHECK(kMipsSwrOffset <= 3 && kMipsSwlOffset <= 3);
    MemOperand source = rs;
    // Adjust offset for two accesses and check if offset + 3 fits into int16_t.
    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 3);
    swr(rd, MemOperand(source.rm(), source.offset() + kMipsSwrOffset));
    swl(rd, MemOperand(source.rm(), source.offset() + kMipsSwlOffset));
1209
  }
1210 1211
}

1212
void TurboAssembler::Ulh(Register rd, const MemOperand& rs) {
1213 1214
  DCHECK(rd != at);
  DCHECK(rs.rm() != at);
1215
  if (kArchVariant == kMips64r6) {
1216
    Lh(rd, rs);
1217
  } else {
1218
    DCHECK_EQ(kArchVariant, kMips64r2);
1219 1220 1221
    MemOperand source = rs;
    // Adjust offset for two accesses and check if offset + 1 fits into int16_t.
    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 1);
1222 1223
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
1224
    if (source.rm() == scratch) {
1225
#if defined(V8_TARGET_LITTLE_ENDIAN)
1226
      Lb(rd, MemOperand(source.rm(), source.offset() + 1));
1227
      Lbu(scratch, source);
1228
#elif defined(V8_TARGET_BIG_ENDIAN)
1229
      Lb(rd, source);
1230
      Lbu(scratch, MemOperand(source.rm(), source.offset() + 1));
1231
#endif
1232
    } else {
1233
#if defined(V8_TARGET_LITTLE_ENDIAN)
1234
      Lbu(scratch, source);
1235
      Lb(rd, MemOperand(source.rm(), source.offset() + 1));
1236
#elif defined(V8_TARGET_BIG_ENDIAN)
1237
      Lbu(scratch, MemOperand(source.rm(), source.offset() + 1));
1238
      Lb(rd, source);
1239 1240 1241
#endif
    }
    dsll(rd, rd, 8);
1242
    or_(rd, rd, scratch);
1243 1244
  }
}
1245

1246
void TurboAssembler::Ulhu(Register rd, const MemOperand& rs) {
1247 1248
  DCHECK(rd != at);
  DCHECK(rs.rm() != at);
1249
  if (kArchVariant == kMips64r6) {
1250
    Lhu(rd, rs);
1251
  } else {
1252
    DCHECK_EQ(kArchVariant, kMips64r2);
1253 1254 1255
    MemOperand source = rs;
    // Adjust offset for two accesses and check if offset + 1 fits into int16_t.
    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 1);
1256 1257
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
1258
    if (source.rm() == scratch) {
1259
#if defined(V8_TARGET_LITTLE_ENDIAN)
1260
      Lbu(rd, MemOperand(source.rm(), source.offset() + 1));
1261
      Lbu(scratch, source);
1262
#elif defined(V8_TARGET_BIG_ENDIAN)
1263
      Lbu(rd, source);
1264
      Lbu(scratch, MemOperand(source.rm(), source.offset() + 1));
1265
#endif
1266
    } else {
1267
#if defined(V8_TARGET_LITTLE_ENDIAN)
1268
      Lbu(scratch, source);
1269
      Lbu(rd, MemOperand(source.rm(), source.offset() + 1));
1270
#elif defined(V8_TARGET_BIG_ENDIAN)
1271
      Lbu(scratch, MemOperand(source.rm(), source.offset() + 1));
1272
      Lbu(rd, source);
1273 1274 1275
#endif
    }
    dsll(rd, rd, 8);
1276
    or_(rd, rd, scratch);
1277 1278 1279
  }
}

1280
void TurboAssembler::Ush(Register rd, const MemOperand& rs, Register scratch) {
1281 1282 1283 1284
  DCHECK(rd != at);
  DCHECK(rs.rm() != at);
  DCHECK(rs.rm() != scratch);
  DCHECK(scratch != at);
1285
  if (kArchVariant == kMips64r6) {
1286
    Sh(rd, rs);
1287
  } else {
1288
    DCHECK_EQ(kArchVariant, kMips64r2);
1289
    MemOperand source = rs;
1290 1291
    // Adjust offset for two accesses and check if offset + 1 fits into int16_t.
    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 1);
1292

1293
    if (scratch != rd) {
1294 1295 1296 1297
      mov(scratch, rd);
    }

#if defined(V8_TARGET_LITTLE_ENDIAN)
1298
    Sb(scratch, source);
1299
    srl(scratch, scratch, 8);
1300
    Sb(scratch, MemOperand(source.rm(), source.offset() + 1));
1301
#elif defined(V8_TARGET_BIG_ENDIAN)
1302
    Sb(scratch, MemOperand(source.rm(), source.offset() + 1));
1303
    srl(scratch, scratch, 8);
1304
    Sb(scratch, source);
1305 1306 1307 1308
#endif
  }
}

1309
void TurboAssembler::Uld(Register rd, const MemOperand& rs) {
1310 1311
  DCHECK(rd != at);
  DCHECK(rs.rm() != at);
1312
  if (kArchVariant == kMips64r6) {
1313
    Ld(rd, rs);
1314
  } else {
1315
    DCHECK_EQ(kArchVariant, kMips64r2);
1316 1317 1318 1319
    DCHECK(kMipsLdrOffset <= 7 && kMipsLdlOffset <= 7);
    MemOperand source = rs;
    // Adjust offset for two accesses and check if offset + 7 fits into int16_t.
    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 7);
1320
    if (rd != source.rm()) {
1321 1322 1323
      ldr(rd, MemOperand(source.rm(), source.offset() + kMipsLdrOffset));
      ldl(rd, MemOperand(source.rm(), source.offset() + kMipsLdlOffset));
    } else {
1324 1325 1326 1327 1328
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
      ldr(scratch, MemOperand(rs.rm(), rs.offset() + kMipsLdrOffset));
      ldl(scratch, MemOperand(rs.rm(), rs.offset() + kMipsLdlOffset));
      mov(rd, scratch);
1329
    }
1330 1331 1332 1333 1334 1335 1336 1337 1338
  }
}


// Load consequent 32-bit word pair in 64-bit reg. and put first word in low
// bits,
// second word in high bits.
void MacroAssembler::LoadWordPair(Register rd, const MemOperand& rs,
                                  Register scratch) {
1339 1340
  Lwu(rd, rs);
  Lw(scratch, MemOperand(rs.rm(), rs.offset() + kPointerSize / 2));
1341 1342 1343 1344
  dsll32(scratch, scratch, 0);
  Daddu(rd, rd, scratch);
}

1345
void TurboAssembler::Usd(Register rd, const MemOperand& rs) {
1346 1347
  DCHECK(rd != at);
  DCHECK(rs.rm() != at);
1348
  if (kArchVariant == kMips64r6) {
1349
    Sd(rd, rs);
1350
  } else {
1351
    DCHECK_EQ(kArchVariant, kMips64r2);
1352 1353 1354 1355 1356 1357
    DCHECK(kMipsSdrOffset <= 7 && kMipsSdlOffset <= 7);
    MemOperand source = rs;
    // Adjust offset for two accesses and check if offset + 7 fits into int16_t.
    AdjustBaseAndOffset(source, OffsetAccessType::TWO_ACCESSES, 7);
    sdr(rd, MemOperand(source.rm(), source.offset() + kMipsSdrOffset));
    sdl(rd, MemOperand(source.rm(), source.offset() + kMipsSdlOffset));
1358 1359 1360 1361 1362 1363 1364
  }
}


// Do 64-bit store as two consequent 32-bit stores to unaligned address.
void MacroAssembler::StoreWordPair(Register rd, const MemOperand& rs,
                                   Register scratch) {
1365
  Sw(rd, rs);
1366
  dsrl32(scratch, rd, 0);
1367
  Sw(scratch, MemOperand(rs.rm(), rs.offset() + kPointerSize / 2));
1368 1369
}

1370
void TurboAssembler::Ulwc1(FPURegister fd, const MemOperand& rs,
1371 1372
                           Register scratch) {
  if (kArchVariant == kMips64r6) {
1373
    Lwc1(fd, rs);
1374
  } else {
1375
    DCHECK_EQ(kArchVariant, kMips64r2);
1376 1377 1378 1379 1380
    Ulw(scratch, rs);
    mtc1(scratch, fd);
  }
}

1381
void TurboAssembler::Uswc1(FPURegister fd, const MemOperand& rs,
1382 1383
                           Register scratch) {
  if (kArchVariant == kMips64r6) {
1384
    Swc1(fd, rs);
1385
  } else {
1386
    DCHECK_EQ(kArchVariant, kMips64r2);
1387 1388 1389 1390 1391
    mfc1(scratch, fd);
    Usw(scratch, rs);
  }
}

1392
void TurboAssembler::Uldc1(FPURegister fd, const MemOperand& rs,
1393
                           Register scratch) {
1394
  DCHECK(scratch != at);
1395
  if (kArchVariant == kMips64r6) {
1396
    Ldc1(fd, rs);
1397
  } else {
1398
    DCHECK_EQ(kArchVariant, kMips64r2);
1399 1400 1401 1402 1403
    Uld(scratch, rs);
    dmtc1(scratch, fd);
  }
}

1404
void TurboAssembler::Usdc1(FPURegister fd, const MemOperand& rs,
1405
                           Register scratch) {
1406
  DCHECK(scratch != at);
1407
  if (kArchVariant == kMips64r6) {
1408
    Sdc1(fd, rs);
1409
  } else {
1410
    DCHECK_EQ(kArchVariant, kMips64r2);
1411 1412 1413 1414
    dmfc1(scratch, fd);
    Usd(scratch, rs);
  }
}
1415

1416
void TurboAssembler::Lb(Register rd, const MemOperand& rs) {
1417 1418 1419
  MemOperand source = rs;
  AdjustBaseAndOffset(source);
  lb(rd, source);
1420 1421
}

1422
void TurboAssembler::Lbu(Register rd, const MemOperand& rs) {
1423 1424 1425
  MemOperand source = rs;
  AdjustBaseAndOffset(source);
  lbu(rd, source);
1426 1427
}

1428
void TurboAssembler::Sb(Register rd, const MemOperand& rs) {
1429 1430 1431
  MemOperand source = rs;
  AdjustBaseAndOffset(source);
  sb(rd, source);
1432 1433
}

1434
void TurboAssembler::Lh(Register rd, const MemOperand& rs) {
1435 1436 1437
  MemOperand source = rs;
  AdjustBaseAndOffset(source);
  lh(rd, source);
1438 1439
}

1440
void TurboAssembler::Lhu(Register rd, const MemOperand& rs) {
1441 1442 1443
  MemOperand source = rs;
  AdjustBaseAndOffset(source);
  lhu(rd, source);
1444 1445
}

1446
void TurboAssembler::Sh(Register rd, const MemOperand& rs) {
1447 1448 1449
  MemOperand source = rs;
  AdjustBaseAndOffset(source);
  sh(rd, source);
1450 1451
}

1452
void TurboAssembler::Lw(Register rd, const MemOperand& rs) {
1453 1454 1455
  MemOperand source = rs;
  AdjustBaseAndOffset(source);
  lw(rd, source);
1456 1457
}

1458
void TurboAssembler::Lwu(Register rd, const MemOperand& rs) {
1459 1460 1461
  MemOperand source = rs;
  AdjustBaseAndOffset(source);
  lwu(rd, source);
1462 1463
}

1464
void TurboAssembler::Sw(Register rd, const MemOperand& rs) {
1465 1466 1467
  MemOperand source = rs;
  AdjustBaseAndOffset(source);
  sw(rd, source);
1468 1469
}

1470
void TurboAssembler::Ld(Register rd, const MemOperand& rs) {
1471 1472 1473
  MemOperand source = rs;
  AdjustBaseAndOffset(source);
  ld(rd, source);
1474 1475
}

1476
void TurboAssembler::Sd(Register rd, const MemOperand& rs) {
1477 1478 1479
  MemOperand source = rs;
  AdjustBaseAndOffset(source);
  sd(rd, source);
1480 1481
}

1482
void TurboAssembler::Lwc1(FPURegister fd, const MemOperand& src) {
1483 1484 1485
  MemOperand tmp = src;
  AdjustBaseAndOffset(tmp);
  lwc1(fd, tmp);
1486 1487
}

1488
void TurboAssembler::Swc1(FPURegister fs, const MemOperand& src) {
1489 1490 1491
  MemOperand tmp = src;
  AdjustBaseAndOffset(tmp);
  swc1(fs, tmp);
1492 1493
}

1494
void TurboAssembler::Ldc1(FPURegister fd, const MemOperand& src) {
1495 1496 1497
  MemOperand tmp = src;
  AdjustBaseAndOffset(tmp);
  ldc1(fd, tmp);
1498 1499
}

1500
void TurboAssembler::Sdc1(FPURegister fs, const MemOperand& src) {
1501 1502 1503
  MemOperand tmp = src;
  AdjustBaseAndOffset(tmp);
  sdc1(fs, tmp);
1504 1505
}

1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561
void TurboAssembler::Ll(Register rd, const MemOperand& rs) {
  bool is_one_instruction = (kArchVariant == kMips64r6) ? is_int9(rs.offset())
                                                        : is_int16(rs.offset());
  if (is_one_instruction) {
    ll(rd, rs);
  } else {
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
    li(scratch, rs.offset());
    daddu(scratch, scratch, rs.rm());
    ll(rd, MemOperand(scratch, 0));
  }
}

void TurboAssembler::Lld(Register rd, const MemOperand& rs) {
  bool is_one_instruction = (kArchVariant == kMips64r6) ? is_int9(rs.offset())
                                                        : is_int16(rs.offset());
  if (is_one_instruction) {
    lld(rd, rs);
  } else {
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
    li(scratch, rs.offset());
    daddu(scratch, scratch, rs.rm());
    lld(rd, MemOperand(scratch, 0));
  }
}

void TurboAssembler::Sc(Register rd, const MemOperand& rs) {
  bool is_one_instruction = (kArchVariant == kMips64r6) ? is_int9(rs.offset())
                                                        : is_int16(rs.offset());
  if (is_one_instruction) {
    sc(rd, rs);
  } else {
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
    li(scratch, rs.offset());
    daddu(scratch, scratch, rs.rm());
    sc(rd, MemOperand(scratch, 0));
  }
}

void TurboAssembler::Scd(Register rd, const MemOperand& rs) {
  bool is_one_instruction = (kArchVariant == kMips64r6) ? is_int9(rs.offset())
                                                        : is_int16(rs.offset());
  if (is_one_instruction) {
    scd(rd, rs);
  } else {
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
    li(scratch, rs.offset());
    daddu(scratch, scratch, rs.rm());
    scd(rd, MemOperand(scratch, 0));
  }
}

1562
void TurboAssembler::li(Register dst, Handle<HeapObject> value, LiFlags mode) {
1563
#ifdef V8_EMBEDDED_BUILTINS
1564
  if (root_array_available_ && isolate()->ShouldLoadConstantsFromRootList()) {
1565
    IndirectLoadConstant(dst, value);
1566 1567 1568 1569 1570 1571 1572 1573 1574
    return;
  }
#endif  // V8_EMBEDDED_BUILTINS
  li(dst, Operand(value), mode);
}

void TurboAssembler::li(Register dst, ExternalReference value, LiFlags mode) {
#ifdef V8_EMBEDDED_BUILTINS
  if (root_array_available_ && isolate()->ShouldLoadConstantsFromRootList()) {
1575
    IndirectLoadExternalReference(dst, value);
1576 1577 1578
    return;
  }
#endif  // V8_EMBEDDED_BUILTINS
1579
  li(dst, Operand(value), mode);
1580 1581
}

1582 1583 1584 1585 1586 1587 1588 1589 1590
static inline int InstrCountForLiLower32Bit(int64_t value) {
  if (!is_int16(static_cast<int32_t>(value)) && (value & kUpper16MaskOf64) &&
      (value & kImm16Mask)) {
    return 2;
  } else {
    return 1;
  }
}

1591
void TurboAssembler::LiLower32BitHelper(Register rd, Operand j) {
1592 1593 1594 1595
  if (is_int16(static_cast<int32_t>(j.immediate()))) {
    daddiu(rd, zero_reg, (j.immediate() & kImm16Mask));
  } else if (!(j.immediate() & kUpper16MaskOf64)) {
    ori(rd, zero_reg, j.immediate() & kImm16Mask);
1596
  } else {
1597 1598 1599
    lui(rd, j.immediate() >> kLuiShift & kImm16Mask);
    if (j.immediate() & kImm16Mask) {
      ori(rd, rd, j.immediate() & kImm16Mask);
1600
    }
1601 1602 1603
  }
}

1604 1605 1606 1607 1608 1609
static inline int InstrCountForLoadReplicatedConst32(int64_t value) {
  uint32_t x = static_cast<uint32_t>(value);
  uint32_t y = static_cast<uint32_t>(value >> 32);

  if (x == y) {
    return (is_uint16(x) || is_int16(x) || (x & kImm16Mask) == 0) ? 2 : 3;
1610
  }
1611 1612

  return INT_MAX;
1613
}
1614

1615
int TurboAssembler::InstrCountForLi64Bit(int64_t value) {
1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629
  if (is_int32(value)) {
    return InstrCountForLiLower32Bit(value);
  } else {
    int bit31 = value >> 31 & 0x1;
    if ((value & kUpper16MaskOf64) == 0 && is_int16(value >> 32) &&
        kArchVariant == kMips64r6) {
      return 2;
    } else if ((value & (kHigher16MaskOf64 | kUpper16MaskOf64)) == 0 &&
               kArchVariant == kMips64r6) {
      return 2;
    } else if ((value & kImm16Mask) == 0 && is_int16((value >> 32) + bit31) &&
               kArchVariant == kMips64r6) {
      return 2;
    } else if ((value & kImm16Mask) == 0 &&
1630
               ((value >> 31) & 0x1FFFF) == ((0x20000 - bit31) & 0x1FFFF) &&
1631 1632 1633 1634 1635 1636
               kArchVariant == kMips64r6) {
      return 2;
    } else if (is_int16(static_cast<int32_t>(value)) &&
               is_int16((value >> 32) + bit31) && kArchVariant == kMips64r6) {
      return 2;
    } else if (is_int16(static_cast<int32_t>(value)) &&
1637
               ((value >> 31) & 0x1FFFF) == ((0x20000 - bit31) & 0x1FFFF) &&
1638 1639
               kArchVariant == kMips64r6) {
      return 2;
1640 1641
    } else if (base::bits::IsPowerOfTwo(value + 1) ||
               value == std::numeric_limits<int64_t>::max()) {
1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707
      return 2;
    } else {
      int shift_cnt = base::bits::CountTrailingZeros64(value);
      int rep32_count = InstrCountForLoadReplicatedConst32(value);
      int64_t tmp = value >> shift_cnt;
      if (is_uint16(tmp)) {
        return 2;
      } else if (is_int16(tmp)) {
        return 2;
      } else if (rep32_count < 3) {
        return 2;
      } else if (is_int32(tmp)) {
        return 3;
      } else {
        shift_cnt = 16 + base::bits::CountTrailingZeros64(value >> 16);
        tmp = value >> shift_cnt;
        if (is_uint16(tmp)) {
          return 3;
        } else if (is_int16(tmp)) {
          return 3;
        } else if (rep32_count < 4) {
          return 3;
        } else if (kArchVariant == kMips64r6) {
          int64_t imm = value;
          int count = InstrCountForLiLower32Bit(imm);
          imm = (imm >> 32) + bit31;
          if (imm & kImm16Mask) {
            count++;
          }
          imm = (imm >> 16) + (imm >> 15 & 0x1);
          if (imm & kImm16Mask) {
            count++;
          }
          return count;
        } else {
          if (is_int48(value)) {
            int64_t k = value >> 16;
            int count = InstrCountForLiLower32Bit(k) + 1;
            if (value & kImm16Mask) {
              count++;
            }
            return count;
          } else {
            int64_t k = value >> 32;
            int count = InstrCountForLiLower32Bit(k);
            if ((value >> 16) & kImm16Mask) {
              count += 3;
              if (value & kImm16Mask) {
                count++;
              }
            } else {
              count++;
              if (value & kImm16Mask) {
                count++;
              }
            }
            return count;
          }
        }
      }
    }
  }
  UNREACHABLE();
  return INT_MAX;
}

1708 1709
// All changes to if...else conditions here must be added to
// InstrCountForLi64Bit as well.
1710
void TurboAssembler::li_optimized(Register rd, Operand j, LiFlags mode) {
1711
  DCHECK(!j.is_reg());
1712
  DCHECK(!MustUseReg(j.rmode()));
1713
  DCHECK(mode == OPTIMIZE_SIZE);
1714
  BlockTrampolinePoolScope block_trampoline_pool(this);
1715
  // Normal load of an immediate value which does not need Relocation Info.
1716
  if (is_int32(j.immediate())) {
1717 1718
    LiLower32BitHelper(rd, j);
  } else {
1719 1720 1721
    int bit31 = j.immediate() >> 31 & 0x1;
    if ((j.immediate() & kUpper16MaskOf64) == 0 &&
        is_int16(j.immediate() >> 32) && kArchVariant == kMips64r6) {
1722 1723 1724
      // 64-bit value which consists of an unsigned 16-bit value in its
      // least significant 32-bits, and a signed 16-bit value in its
      // most significant 32-bits.
1725 1726 1727
      ori(rd, zero_reg, j.immediate() & kImm16Mask);
      dahi(rd, j.immediate() >> 32 & kImm16Mask);
    } else if ((j.immediate() & (kHigher16MaskOf64 | kUpper16MaskOf64)) == 0 &&
1728 1729 1730 1731
               kArchVariant == kMips64r6) {
      // 64-bit value which consists of an unsigned 16-bit value in its
      // least significant 48-bits, and a signed 16-bit value in its
      // most significant 16-bits.
1732 1733 1734 1735
      ori(rd, zero_reg, j.immediate() & kImm16Mask);
      dati(rd, j.immediate() >> 48 & kImm16Mask);
    } else if ((j.immediate() & kImm16Mask) == 0 &&
               is_int16((j.immediate() >> 32) + bit31) &&
1736 1737 1738
               kArchVariant == kMips64r6) {
      // 16 LSBs (Least Significant Bits) all set to zero.
      // 48 MSBs (Most Significant Bits) hold a signed 32-bit value.
1739 1740 1741
      lui(rd, j.immediate() >> kLuiShift & kImm16Mask);
      dahi(rd, ((j.immediate() >> 32) + bit31) & kImm16Mask);
    } else if ((j.immediate() & kImm16Mask) == 0 &&
1742 1743
               ((j.immediate() >> 31) & 0x1FFFF) ==
                   ((0x20000 - bit31) & 0x1FFFF) &&
1744 1745 1746 1747
               kArchVariant == kMips64r6) {
      // 16 LSBs all set to zero.
      // 48 MSBs hold a signed value which can't be represented by signed
      // 32-bit number, and the middle 16 bits are all zero, or all one.
1748 1749 1750 1751
      lui(rd, j.immediate() >> kLuiShift & kImm16Mask);
      dati(rd, ((j.immediate() >> 48) + bit31) & kImm16Mask);
    } else if (is_int16(static_cast<int32_t>(j.immediate())) &&
               is_int16((j.immediate() >> 32) + bit31) &&
1752 1753 1754
               kArchVariant == kMips64r6) {
      // 32 LSBs contain a signed 16-bit number.
      // 32 MSBs contain a signed 16-bit number.
1755 1756 1757
      daddiu(rd, zero_reg, j.immediate() & kImm16Mask);
      dahi(rd, ((j.immediate() >> 32) + bit31) & kImm16Mask);
    } else if (is_int16(static_cast<int32_t>(j.immediate())) &&
1758 1759
               ((j.immediate() >> 31) & 0x1FFFF) ==
                   ((0x20000 - bit31) & 0x1FFFF) &&
1760 1761 1762
               kArchVariant == kMips64r6) {
      // 48 LSBs contain an unsigned 16-bit number.
      // 16 MSBs contain a signed 16-bit number.
1763 1764
      daddiu(rd, zero_reg, j.immediate() & kImm16Mask);
      dati(rd, ((j.immediate() >> 48) + bit31) & kImm16Mask);
1765 1766 1767 1768
    } else if (base::bits::IsPowerOfTwo(j.immediate() + 1) ||
               j.immediate() == std::numeric_limits<int64_t>::max()) {
      // 64-bit values which have their "n" LSBs set to one, and their
      // "64-n" MSBs set to zero. "n" must meet the restrictions 0 < n < 64.
1769
      int shift_cnt = 64 - base::bits::CountTrailingZeros64(j.immediate() + 1);
1770 1771 1772 1773 1774 1775
      daddiu(rd, zero_reg, -1);
      if (shift_cnt < 32) {
        dsrl(rd, rd, shift_cnt);
      } else {
        dsrl32(rd, rd, shift_cnt & 31);
      }
1776
    } else {
1777 1778 1779
      int shift_cnt = base::bits::CountTrailingZeros64(j.immediate());
      int rep32_count = InstrCountForLoadReplicatedConst32(j.immediate());
      int64_t tmp = j.immediate() >> shift_cnt;
1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792
      if (is_uint16(tmp)) {
        // Value can be computed by loading a 16-bit unsigned value, and
        // then shifting left.
        ori(rd, zero_reg, tmp & kImm16Mask);
        if (shift_cnt < 32) {
          dsll(rd, rd, shift_cnt);
        } else {
          dsll32(rd, rd, shift_cnt & 31);
        }
      } else if (is_int16(tmp)) {
        // Value can be computed by loading a 16-bit signed value, and
        // then shifting left.
        daddiu(rd, zero_reg, static_cast<int32_t>(tmp));
1793
        if (shift_cnt < 32) {
1794
          dsll(rd, rd, shift_cnt);
1795
        } else {
1796 1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813
          dsll32(rd, rd, shift_cnt & 31);
        }
      } else if (rep32_count < 3) {
        // Value being loaded has 32 LSBs equal to the 32 MSBs, and the
        // value loaded into the 32 LSBs can be loaded with a single
        // MIPS instruction.
        LiLower32BitHelper(rd, j);
        Dins(rd, rd, 32, 32);
      } else if (is_int32(tmp)) {
        // Loads with 3 instructions.
        // Value can be computed by loading a 32-bit signed value, and
        // then shifting left.
        lui(rd, tmp >> kLuiShift & kImm16Mask);
        ori(rd, rd, tmp & kImm16Mask);
        if (shift_cnt < 32) {
          dsll(rd, rd, shift_cnt);
        } else {
          dsll32(rd, rd, shift_cnt & 31);
1814
        }
1815
      } else {
1816 1817
        shift_cnt = 16 + base::bits::CountTrailingZeros64(j.immediate() >> 16);
        tmp = j.immediate() >> shift_cnt;
1818
        if (is_uint16(tmp)) {
1819 1820
          // Value can be computed by loading a 16-bit unsigned value,
          // shifting left, and "or"ing in another 16-bit unsigned value.
1821 1822 1823
          ori(rd, zero_reg, tmp & kImm16Mask);
          if (shift_cnt < 32) {
            dsll(rd, rd, shift_cnt);
1824
          } else {
1825
            dsll32(rd, rd, shift_cnt & 31);
1826
          }
1827
          ori(rd, rd, j.immediate() & kImm16Mask);
1828
        } else if (is_int16(tmp)) {
1829 1830
          // Value can be computed by loading a 16-bit signed value,
          // shifting left, and "or"ing in a 16-bit unsigned value.
1831 1832 1833 1834 1835
          daddiu(rd, zero_reg, static_cast<int32_t>(tmp));
          if (shift_cnt < 32) {
            dsll(rd, rd, shift_cnt);
          } else {
            dsll32(rd, rd, shift_cnt & 31);
1836
          }
1837
          ori(rd, rd, j.immediate() & kImm16Mask);
1838
        } else if (rep32_count < 4) {
1839
          // Value being loaded has 32 LSBs equal to the 32 MSBs, and the
1840
          // value in the 32 LSBs requires 2 MIPS instructions to load.
1841 1842
          LiLower32BitHelper(rd, j);
          Dins(rd, rd, 32, 32);
1843 1844 1845 1846
        } else if (kArchVariant == kMips64r6) {
          // Loads with 3-4 instructions.
          // Catch-all case to get any other 64-bit values which aren't
          // handled by special cases above.
1847
          int64_t imm = j.immediate();
1848 1849 1850 1851 1852 1853 1854 1855
          LiLower32BitHelper(rd, j);
          imm = (imm >> 32) + bit31;
          if (imm & kImm16Mask) {
            dahi(rd, imm & kImm16Mask);
          }
          imm = (imm >> 16) + (imm >> 15 & 0x1);
          if (imm & kImm16Mask) {
            dati(rd, imm & kImm16Mask);
1856
          }
1857
        } else {
1858 1859
          if (is_int48(j.immediate())) {
            Operand k = Operand(j.immediate() >> 16);
1860 1861
            LiLower32BitHelper(rd, k);
            dsll(rd, rd, 16);
1862 1863
            if (j.immediate() & kImm16Mask) {
              ori(rd, rd, j.immediate() & kImm16Mask);
1864
            }
1865
          } else {
1866
            Operand k = Operand(j.immediate() >> 32);
1867
            LiLower32BitHelper(rd, k);
1868
            if ((j.immediate() >> 16) & kImm16Mask) {
1869
              dsll(rd, rd, 16);
1870
              ori(rd, rd, (j.immediate() >> 16) & kImm16Mask);
1871
              dsll(rd, rd, 16);
1872 1873
              if (j.immediate() & kImm16Mask) {
                ori(rd, rd, j.immediate() & kImm16Mask);
1874
              }
1875
            } else {
1876
              dsll32(rd, rd, 0);
1877 1878
              if (j.immediate() & kImm16Mask) {
                ori(rd, rd, j.immediate() & kImm16Mask);
1879
              }
1880
            }
1881 1882 1883
          }
        }
      }
1884
    }
1885 1886 1887
  }
}

1888
void TurboAssembler::li(Register rd, Operand j, LiFlags mode) {
1889 1890
  DCHECK(!j.is_reg());
  BlockTrampolinePoolScope block_trampoline_pool(this);
1891
  if (!MustUseReg(j.rmode()) && mode == OPTIMIZE_SIZE) {
1892 1893 1894
    int li_count = InstrCountForLi64Bit(j.immediate());
    int li_neg_count = InstrCountForLi64Bit(-j.immediate());
    int li_not_count = InstrCountForLi64Bit(~j.immediate());
1895 1896 1897
    // Loading -MIN_INT64 could cause problems, but loading MIN_INT64 takes only
    // two instructions so no need to check for this.
    if (li_neg_count <= li_not_count && li_neg_count < li_count - 1) {
1898 1899
      DCHECK(j.immediate() != std::numeric_limits<int64_t>::min());
      li_optimized(rd, Operand(-j.immediate()), mode);
1900 1901
      Dsubu(rd, zero_reg, rd);
    } else if (li_neg_count > li_not_count && li_not_count < li_count - 1) {
1902 1903
      DCHECK(j.immediate() != std::numeric_limits<int64_t>::min());
      li_optimized(rd, Operand(~j.immediate()), mode);
1904 1905 1906 1907
      nor(rd, rd, rd);
    } else {
      li_optimized(rd, j, mode);
    }
1908
  } else if (MustUseReg(j.rmode())) {
1909 1910 1911 1912 1913 1914 1915 1916
    int64_t immediate;
    if (j.IsHeapObjectRequest()) {
      RequestHeapObject(j.heap_object_request());
      immediate = 0;
    } else {
      immediate = j.immediate();
    }

1917
    RecordRelocInfo(j.rmode(), immediate);
1918 1919
    lui(rd, (immediate >> 32) & kImm16Mask);
    ori(rd, rd, (immediate >> 16) & kImm16Mask);
1920
    dsll(rd, rd, 16);
1921
    ori(rd, rd, immediate & kImm16Mask);
1922 1923 1924
  } else if (mode == ADDRESS_LOAD)  {
    // We always need the same number of instructions as we may need to patch
    // this code to load another value which may need all 4 instructions.
1925 1926
    lui(rd, (j.immediate() >> 32) & kImm16Mask);
    ori(rd, rd, (j.immediate() >> 16) & kImm16Mask);
1927
    dsll(rd, rd, 16);
1928
    ori(rd, rd, j.immediate() & kImm16Mask);
1929 1930
  } else {  // mode == CONSTANT_SIZE - always emit the same instruction
            // sequence.
1931
    if (kArchVariant == kMips64r6) {
1932
      int64_t imm = j.immediate();
1933 1934 1935 1936 1937 1938
      lui(rd, imm >> kLuiShift & kImm16Mask);
      ori(rd, rd, (imm & kImm16Mask));
      imm = (imm >> 32) + ((imm >> 31) & 0x1);
      dahi(rd, imm & kImm16Mask & kImm16Mask);
      imm = (imm >> 16) + ((imm >> 15) & 0x1);
      dati(rd, imm & kImm16Mask & kImm16Mask);
1939
    } else {
1940 1941
      lui(rd, (j.immediate() >> 48) & kImm16Mask);
      ori(rd, rd, (j.immediate() >> 32) & kImm16Mask);
1942
      dsll(rd, rd, 16);
1943
      ori(rd, rd, (j.immediate() >> 16) & kImm16Mask);
1944
      dsll(rd, rd, 16);
1945
      ori(rd, rd, j.immediate() & kImm16Mask);
1946
    }
1947 1948 1949
  }
}

1950
void TurboAssembler::MultiPush(RegList regs) {
1951
  int16_t num_to_push = base::bits::CountPopulation(regs);
1952 1953 1954 1955 1956 1957
  int16_t stack_offset = num_to_push * kPointerSize;

  Dsubu(sp, sp, Operand(stack_offset));
  for (int16_t i = kNumRegisters - 1; i >= 0; i--) {
    if ((regs & (1 << i)) != 0) {
      stack_offset -= kPointerSize;
1958
      Sd(ToRegister(i), MemOperand(sp, stack_offset));
1959 1960 1961 1962 1963
    }
  }
}


1964
void TurboAssembler::MultiPop(RegList regs) {
1965 1966 1967 1968
  int16_t stack_offset = 0;

  for (int16_t i = 0; i < kNumRegisters; i++) {
    if ((regs & (1 << i)) != 0) {
1969
      Ld(ToRegister(i), MemOperand(sp, stack_offset));
1970 1971 1972 1973 1974 1975 1976
      stack_offset += kPointerSize;
    }
  }
  daddiu(sp, sp, stack_offset);
}


1977
void TurboAssembler::MultiPushFPU(RegList regs) {
1978
  int16_t num_to_push = base::bits::CountPopulation(regs);
1979 1980 1981 1982 1983 1984
  int16_t stack_offset = num_to_push * kDoubleSize;

  Dsubu(sp, sp, Operand(stack_offset));
  for (int16_t i = kNumRegisters - 1; i >= 0; i--) {
    if ((regs & (1 << i)) != 0) {
      stack_offset -= kDoubleSize;
1985
      Sdc1(FPURegister::from_code(i), MemOperand(sp, stack_offset));
1986 1987 1988 1989 1990
    }
  }
}


1991
void TurboAssembler::MultiPopFPU(RegList regs) {
1992 1993 1994 1995
  int16_t stack_offset = 0;

  for (int16_t i = 0; i < kNumRegisters; i++) {
    if ((regs & (1 << i)) != 0) {
1996
      Ldc1(FPURegister::from_code(i), MemOperand(sp, stack_offset));
1997 1998 1999 2000 2001 2002 2003
      stack_offset += kDoubleSize;
    }
  }
  daddiu(sp, sp, stack_offset);
}


2004
void TurboAssembler::Ext(Register rt, Register rs, uint16_t pos,
2005
                         uint16_t size) {
2006 2007
  DCHECK_LT(pos, 32);
  DCHECK_LT(pos + size, 33);
2008 2009 2010
  ext_(rt, rs, pos, size);
}

2011
void TurboAssembler::Dext(Register rt, Register rs, uint16_t pos,
2012
                          uint16_t size) {
2013 2014 2015 2016 2017 2018 2019 2020 2021
  DCHECK(pos < 64 && 0 < size && size <= 64 && 0 < pos + size &&
         pos + size <= 64);
  if (size > 32) {
    dextm_(rt, rs, pos, size);
  } else if (pos >= 32) {
    dextu_(rt, rs, pos, size);
  } else {
    dext_(rt, rs, pos, size);
  }
2022 2023
}

2024
void TurboAssembler::Ins(Register rt, Register rs, uint16_t pos,
2025
                         uint16_t size) {
2026 2027 2028
  DCHECK_LT(pos, 32);
  DCHECK_LE(pos + size, 32);
  DCHECK_NE(size, 0);
2029 2030 2031
  ins_(rt, rs, pos, size);
}

2032
void TurboAssembler::Dins(Register rt, Register rs, uint16_t pos,
2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044
                          uint16_t size) {
  DCHECK(pos < 64 && 0 < size && size <= 64 && 0 < pos + size &&
         pos + size <= 64);
  if (pos + size <= 32) {
    dins_(rt, rs, pos, size);
  } else if (pos < 32) {
    dinsm_(rt, rs, pos, size);
  } else {
    dinsu_(rt, rs, pos, size);
  }
}

2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079
void TurboAssembler::ExtractBits(Register dest, Register source, Register pos,
                                 int size, bool sign_extend) {
  srav(dest, source, pos);
  Dext(dest, dest, 0, size);
  if (sign_extend) {
    switch (size) {
      case 8:
        seb(dest, dest);
        break;
      case 16:
        seh(dest, dest);
        break;
      case 32:
        // sign-extend word
        sll(dest, dest, 0);
        break;
      default:
        UNREACHABLE();
    }
  }
}

void TurboAssembler::InsertBits(Register dest, Register source, Register pos,
                                int size) {
  Ror(dest, dest, pos);
  Dins(dest, source, 0, size);
  {
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
    Dsubu(scratch, pos, Operand(64));
    Neg(scratch, Operand(scratch));
    Ror(dest, dest, scratch);
  }
}

2080
void TurboAssembler::Neg_s(FPURegister fd, FPURegister fs) {
2081 2082 2083 2084
  if (kArchVariant == kMips64r6) {
    // r6 neg_s changes the sign for NaN-like operands as well.
    neg_s(fd, fs);
  } else {
2085
    DCHECK_EQ(kArchVariant, kMips64r2);
2086
    BlockTrampolinePoolScope block_trampoline_pool(this);
2087
    Label is_nan, done;
2088 2089
    Register scratch1 = t8;
    Register scratch2 = t9;
2090 2091
    CompareIsNanF32(fs, fs);
    BranchTrueShortF(&is_nan);
2092 2093 2094 2095 2096 2097
    Branch(USE_DELAY_SLOT, &done);
    // For NaN input, neg_s will return the same NaN value,
    // while the sign has to be changed separately.
    neg_s(fd, fs);  // In delay slot.
    bind(&is_nan);
    mfc1(scratch1, fs);
2098 2099 2100
    li(scratch2, kBinary32SignMask);
    Xor(scratch1, scratch1, scratch2);
    mtc1(scratch1, fd);
2101 2102 2103 2104
    bind(&done);
  }
}

2105
void TurboAssembler::Neg_d(FPURegister fd, FPURegister fs) {
2106 2107 2108 2109
  if (kArchVariant == kMips64r6) {
    // r6 neg_d changes the sign for NaN-like operands as well.
    neg_d(fd, fs);
  } else {
2110
    DCHECK_EQ(kArchVariant, kMips64r2);
2111
    BlockTrampolinePoolScope block_trampoline_pool(this);
2112
    Label is_nan, done;
2113 2114
    Register scratch1 = t8;
    Register scratch2 = t9;
2115 2116
    CompareIsNanF64(fs, fs);
    BranchTrueShortF(&is_nan);
2117 2118 2119 2120 2121 2122
    Branch(USE_DELAY_SLOT, &done);
    // For NaN input, neg_d will return the same NaN value,
    // while the sign has to be changed separately.
    neg_d(fd, fs);  // In delay slot.
    bind(&is_nan);
    dmfc1(scratch1, fs);
2123 2124 2125
    li(scratch2, Double::kSignMask);
    Xor(scratch1, scratch1, scratch2);
    dmtc1(scratch1, fd);
2126 2127 2128
    bind(&done);
  }
}
2129

2130
void TurboAssembler::Cvt_d_uw(FPURegister fd, FPURegister fs) {
2131 2132
  // Move the data from fs to t8.
  mfc1(t8, fs);
2133
  Cvt_d_uw(fd, t8);
2134 2135
}

2136
void TurboAssembler::Cvt_d_uw(FPURegister fd, Register rs) {
2137
  // Convert rs to a FP value in fd.
2138 2139
  DCHECK(rs != t9);
  DCHECK(rs != at);
2140

2141 2142 2143 2144
  // Zero extend int32 in rs.
  Dext(t9, rs, 0, 32);
  dmtc1(t9, fd);
  cvt_d_l(fd, fd);
2145 2146
}

2147
void TurboAssembler::Cvt_d_ul(FPURegister fd, FPURegister fs) {
2148 2149 2150 2151 2152
  // Move the data from fs to t8.
  dmfc1(t8, fs);
  Cvt_d_ul(fd, t8);
}

2153
void TurboAssembler::Cvt_d_ul(FPURegister fd, Register rs) {
2154 2155
  // Convert rs to a FP value in fd.

2156 2157
  DCHECK(rs != t9);
  DCHECK(rs != at);
2158

2159
  Label msb_clear, conversion_done;
2160

2161
  Branch(&msb_clear, ge, rs, Operand(zero_reg));
2162

2163
  // Rs >= 2^63
2164 2165 2166 2167 2168 2169 2170 2171
  andi(t9, rs, 1);
  dsrl(rs, rs, 1);
  or_(t9, t9, rs);
  dmtc1(t9, fd);
  cvt_d_l(fd, fd);
  Branch(USE_DELAY_SLOT, &conversion_done);
  add_d(fd, fd, fd);  // In delay slot.

2172 2173
  bind(&msb_clear);
  // Rs < 2^63, we can do simple conversion.
2174 2175 2176 2177 2178 2179
  dmtc1(rs, fd);
  cvt_d_l(fd, fd);

  bind(&conversion_done);
}

2180
void TurboAssembler::Cvt_s_uw(FPURegister fd, FPURegister fs) {
2181 2182 2183 2184 2185
  // Move the data from fs to t8.
  mfc1(t8, fs);
  Cvt_s_uw(fd, t8);
}

2186
void TurboAssembler::Cvt_s_uw(FPURegister fd, Register rs) {
2187
  // Convert rs to a FP value in fd.
2188 2189
  DCHECK(rs != t9);
  DCHECK(rs != at);
2190 2191 2192 2193 2194 2195

  // Zero extend int32 in rs.
  Dext(t9, rs, 0, 32);
  dmtc1(t9, fd);
  cvt_s_l(fd, fd);
}
2196

2197
void TurboAssembler::Cvt_s_ul(FPURegister fd, FPURegister fs) {
2198 2199 2200 2201 2202
  // Move the data from fs to t8.
  dmfc1(t8, fs);
  Cvt_s_ul(fd, t8);
}

2203
void TurboAssembler::Cvt_s_ul(FPURegister fd, Register rs) {
2204 2205
  // Convert rs to a FP value in fd.

2206 2207
  DCHECK(rs != t9);
  DCHECK(rs != at);
2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230

  Label positive, conversion_done;

  Branch(&positive, ge, rs, Operand(zero_reg));

  // Rs >= 2^31.
  andi(t9, rs, 1);
  dsrl(rs, rs, 1);
  or_(t9, t9, rs);
  dmtc1(t9, fd);
  cvt_s_l(fd, fd);
  Branch(USE_DELAY_SLOT, &conversion_done);
  add_s(fd, fd, fd);  // In delay slot.

  bind(&positive);
  // Rs < 2^31, we can do simple conversion.
  dmtc1(rs, fd);
  cvt_s_l(fd, fd);

  bind(&conversion_done);
}


2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256
void MacroAssembler::Round_l_d(FPURegister fd, FPURegister fs) {
  round_l_d(fd, fs);
}


void MacroAssembler::Floor_l_d(FPURegister fd, FPURegister fs) {
  floor_l_d(fd, fs);
}


void MacroAssembler::Ceil_l_d(FPURegister fd, FPURegister fs) {
  ceil_l_d(fd, fs);
}


void MacroAssembler::Trunc_l_d(FPURegister fd, FPURegister fs) {
  trunc_l_d(fd, fs);
}


void MacroAssembler::Trunc_l_ud(FPURegister fd,
                                FPURegister fs,
                                FPURegister scratch) {
  // Load to GPR.
  dmfc1(t8, fs);
  // Reset sign bit.
2257 2258 2259
  {
    UseScratchRegisterScope temps(this);
    Register scratch1 = temps.Acquire();
2260
    li(scratch1, 0x7FFFFFFFFFFFFFFF);
2261 2262
    and_(t8, t8, scratch1);
  }
2263 2264 2265 2266
  dmtc1(t8, fs);
  trunc_l_d(fd, fs);
}

2267
void TurboAssembler::Trunc_uw_d(FPURegister fd, FPURegister fs,
2268
                                FPURegister scratch) {
2269 2270
  BlockTrampolinePoolScope block_trampoline_pool(this);
  Trunc_uw_d(t8, fs, scratch);
2271 2272 2273
  mtc1(t8, fd);
}

2274
void TurboAssembler::Trunc_uw_s(FPURegister fd, FPURegister fs,
2275
                                FPURegister scratch) {
2276 2277
  BlockTrampolinePoolScope block_trampoline_pool(this);
  Trunc_uw_s(t8, fs, scratch);
2278 2279 2280
  mtc1(t8, fd);
}

2281
void TurboAssembler::Trunc_ul_d(FPURegister fd, FPURegister fs,
2282
                                FPURegister scratch, Register result) {
2283 2284
  BlockTrampolinePoolScope block_trampoline_pool(this);
  Trunc_ul_d(t8, fs, scratch, result);
2285 2286 2287
  dmtc1(t8, fd);
}

2288
void TurboAssembler::Trunc_ul_s(FPURegister fd, FPURegister fs,
2289
                                FPURegister scratch, Register result) {
2290 2291
  BlockTrampolinePoolScope block_trampoline_pool(this);
  Trunc_ul_s(t8, fs, scratch, result);
2292 2293 2294 2295
  dmtc1(t8, fd);
}


2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314
void MacroAssembler::Trunc_w_d(FPURegister fd, FPURegister fs) {
  trunc_w_d(fd, fs);
}


void MacroAssembler::Round_w_d(FPURegister fd, FPURegister fs) {
  round_w_d(fd, fs);
}


void MacroAssembler::Floor_w_d(FPURegister fd, FPURegister fs) {
  floor_w_d(fd, fs);
}


void MacroAssembler::Ceil_w_d(FPURegister fd, FPURegister fs) {
  ceil_w_d(fd, fs);
}

2315
void TurboAssembler::Trunc_uw_d(Register rd, FPURegister fs,
2316
                                FPURegister scratch) {
2317 2318
  DCHECK(fs != scratch);
  DCHECK(rd != at);
2319

2320 2321 2322 2323 2324 2325 2326 2327
  {
    // Load 2^31 into scratch as its float representation.
    UseScratchRegisterScope temps(this);
    Register scratch1 = temps.Acquire();
    li(scratch1, 0x41E00000);
    mtc1(zero_reg, scratch);
    mthc1(scratch1, scratch);
  }
2328 2329 2330
  // Test if scratch > fd.
  // If fd < 2^31 we can convert it normally.
  Label simple_convert;
2331
  CompareF64(OLT, fs, scratch);
2332
  BranchTrueShortF(&simple_convert);
2333 2334 2335

  // First we subtract 2^31 from fd, then trunc it to rs
  // and add 2^31 to rs.
2336
  sub_d(scratch, fs, scratch);
2337
  trunc_w_d(scratch, scratch);
2338 2339
  mfc1(rd, scratch);
  Or(rd, rd, 1 << 31);
2340 2341 2342 2343 2344

  Label done;
  Branch(&done);
  // Simple conversion.
  bind(&simple_convert);
2345 2346
  trunc_w_d(scratch, fs);
  mfc1(rd, scratch);
2347 2348 2349 2350

  bind(&done);
}

2351
void TurboAssembler::Trunc_uw_s(Register rd, FPURegister fs,
2352
                                FPURegister scratch) {
2353 2354
  DCHECK(fs != scratch);
  DCHECK(rd != at);
2355

2356 2357 2358 2359 2360 2361 2362
  {
    // Load 2^31 into scratch as its float representation.
    UseScratchRegisterScope temps(this);
    Register scratch1 = temps.Acquire();
    li(scratch1, 0x4F000000);
    mtc1(scratch1, scratch);
  }
2363 2364
  // Test if scratch > fs.
  // If fs < 2^31 we can convert it normally.
2365
  Label simple_convert;
2366
  CompareF32(OLT, fs, scratch);
2367
  BranchTrueShortF(&simple_convert);
2368

2369 2370 2371
  // First we subtract 2^31 from fs, then trunc it to rd
  // and add 2^31 to rd.
  sub_s(scratch, fs, scratch);
2372
  trunc_w_s(scratch, scratch);
2373 2374
  mfc1(rd, scratch);
  Or(rd, rd, 1 << 31);
2375 2376 2377 2378 2379

  Label done;
  Branch(&done);
  // Simple conversion.
  bind(&simple_convert);
2380 2381
  trunc_w_s(scratch, fs);
  mfc1(rd, scratch);
2382 2383 2384

  bind(&done);
}
2385

2386
void TurboAssembler::Trunc_ul_d(Register rd, FPURegister fs,
2387
                                FPURegister scratch, Register result) {
2388 2389
  DCHECK(fs != scratch);
  DCHECK(!AreAliased(rd, result, at));
2390

2391
  Label simple_convert, done, fail;
2392 2393
  if (result.is_valid()) {
    mov(result, zero_reg);
2394 2395
    Move(scratch, -1.0);
    // If fd =< -1 or unordered, then the conversion fails.
2396
    CompareF64(OLE, fs, scratch);
2397
    BranchTrueShortF(&fail);
2398
    CompareIsNanF64(fs, scratch);
2399
    BranchTrueShortF(&fail);
2400
  }
2401

2402
  // Load 2^63 into scratch as its double representation.
2403
  li(at, 0x43E0000000000000);
2404 2405
  dmtc1(at, scratch);

2406 2407 2408
  // Test if scratch > fs.
  // If fs < 2^63 we can convert it normally.
  CompareF64(OLT, fs, scratch);
2409
  BranchTrueShortF(&simple_convert);
2410

2411 2412 2413
  // First we subtract 2^63 from fs, then trunc it to rd
  // and add 2^63 to rd.
  sub_d(scratch, fs, scratch);
2414
  trunc_l_d(scratch, scratch);
2415 2416
  dmfc1(rd, scratch);
  Or(rd, rd, Operand(1UL << 63));
2417 2418 2419 2420
  Branch(&done);

  // Simple conversion.
  bind(&simple_convert);
2421 2422
  trunc_l_d(scratch, fs);
  dmfc1(rd, scratch);
2423 2424

  bind(&done);
2425
  if (result.is_valid()) {
2426
    // Conversion is failed if the result is negative.
2427 2428 2429 2430 2431 2432 2433 2434
    {
      UseScratchRegisterScope temps(this);
      Register scratch1 = temps.Acquire();
      addiu(scratch1, zero_reg, -1);
      dsrl(scratch1, scratch1, 1);  // Load 2^62.
      dmfc1(result, scratch);
      xor_(result, result, scratch1);
    }
2435
    Slt(result, zero_reg, result);
2436 2437 2438
  }

  bind(&fail);
2439 2440
}

2441
void TurboAssembler::Trunc_ul_s(Register rd, FPURegister fs,
2442
                                FPURegister scratch, Register result) {
2443 2444
  DCHECK(fs != scratch);
  DCHECK(!AreAliased(rd, result, at));
2445 2446 2447 2448

  Label simple_convert, done, fail;
  if (result.is_valid()) {
    mov(result, zero_reg);
2449 2450
    Move(scratch, -1.0f);
    // If fd =< -1 or unordered, then the conversion fails.
2451
    CompareF32(OLE, fs, scratch);
2452
    BranchTrueShortF(&fail);
2453
    CompareIsNanF32(fs, scratch);
2454
    BranchTrueShortF(&fail);
2455
  }
2456

2457 2458 2459 2460
  {
    // Load 2^63 into scratch as its float representation.
    UseScratchRegisterScope temps(this);
    Register scratch1 = temps.Acquire();
2461
    li(scratch1, 0x5F000000);
2462 2463
    mtc1(scratch1, scratch);
  }
2464

2465 2466 2467
  // Test if scratch > fs.
  // If fs < 2^63 we can convert it normally.
  CompareF32(OLT, fs, scratch);
2468
  BranchTrueShortF(&simple_convert);
2469

2470 2471 2472
  // First we subtract 2^63 from fs, then trunc it to rd
  // and add 2^63 to rd.
  sub_s(scratch, fs, scratch);
2473
  trunc_l_s(scratch, scratch);
2474 2475
  dmfc1(rd, scratch);
  Or(rd, rd, Operand(1UL << 63));
2476 2477 2478 2479
  Branch(&done);

  // Simple conversion.
  bind(&simple_convert);
2480 2481
  trunc_l_s(scratch, fs);
  dmfc1(rd, scratch);
2482 2483

  bind(&done);
2484 2485
  if (result.is_valid()) {
    // Conversion is failed if the result is negative or unordered.
2486 2487 2488 2489 2490 2491 2492 2493
    {
      UseScratchRegisterScope temps(this);
      Register scratch1 = temps.Acquire();
      addiu(scratch1, zero_reg, -1);
      dsrl(scratch1, scratch1, 1);  // Load 2^62.
      dmfc1(result, scratch);
      xor_(result, result, scratch1);
    }
2494 2495 2496 2497
    Slt(result, zero_reg, result);
  }

  bind(&fail);
2498 2499
}

2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573 2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616
template <typename RoundFunc>
void TurboAssembler::RoundDouble(FPURegister dst, FPURegister src,
                                 FPURoundingMode mode, RoundFunc round) {
  BlockTrampolinePoolScope block_trampoline_pool(this);
  Register scratch = t8;
  if (kArchVariant == kMips64r6) {
    cfc1(scratch, FCSR);
    li(at, Operand(mode));
    ctc1(at, FCSR);
    rint_d(dst, src);
    ctc1(scratch, FCSR);
  } else {
    Label done;
    mfhc1(scratch, src);
    Ext(at, scratch, HeapNumber::kExponentShift, HeapNumber::kExponentBits);
    Branch(USE_DELAY_SLOT, &done, hs, at,
           Operand(HeapNumber::kExponentBias + HeapNumber::kMantissaBits));
    mov_d(dst, src);
    round(this, dst, src);
    dmfc1(at, dst);
    Branch(USE_DELAY_SLOT, &done, ne, at, Operand(zero_reg));
    cvt_d_l(dst, dst);
    srl(at, scratch, 31);
    sll(at, at, 31);
    mthc1(at, dst);
    bind(&done);
  }
}

void TurboAssembler::Floor_d_d(FPURegister dst, FPURegister src) {
  RoundDouble(dst, src, mode_floor,
              [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
                tasm->floor_l_d(dst, src);
              });
}

void TurboAssembler::Ceil_d_d(FPURegister dst, FPURegister src) {
  RoundDouble(dst, src, mode_ceil,
              [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
                tasm->ceil_l_d(dst, src);
              });
}

void TurboAssembler::Trunc_d_d(FPURegister dst, FPURegister src) {
  RoundDouble(dst, src, mode_trunc,
              [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
                tasm->trunc_l_d(dst, src);
              });
}

void TurboAssembler::Round_d_d(FPURegister dst, FPURegister src) {
  RoundDouble(dst, src, mode_round,
              [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
                tasm->round_l_d(dst, src);
              });
}

template <typename RoundFunc>
void TurboAssembler::RoundFloat(FPURegister dst, FPURegister src,
                                FPURoundingMode mode, RoundFunc round) {
  BlockTrampolinePoolScope block_trampoline_pool(this);
  Register scratch = t8;
  if (kArchVariant == kMips64r6) {
    cfc1(scratch, FCSR);
    li(at, Operand(mode));
    ctc1(at, FCSR);
    rint_s(dst, src);
    ctc1(scratch, FCSR);
  } else {
    int32_t kFloat32ExponentBias = 127;
    int32_t kFloat32MantissaBits = 23;
    int32_t kFloat32ExponentBits = 8;
    Label done;
    mfc1(scratch, src);
    Ext(at, scratch, kFloat32MantissaBits, kFloat32ExponentBits);
    Branch(USE_DELAY_SLOT, &done, hs, at,
           Operand(kFloat32ExponentBias + kFloat32MantissaBits));
    mov_s(dst, src);
    round(this, dst, src);
    mfc1(at, dst);
    Branch(USE_DELAY_SLOT, &done, ne, at, Operand(zero_reg));
    cvt_s_w(dst, dst);
    srl(at, scratch, 31);
    sll(at, at, 31);
    mtc1(at, dst);
    bind(&done);
  }
}

void TurboAssembler::Floor_s_s(FPURegister dst, FPURegister src) {
  RoundFloat(dst, src, mode_floor,
             [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
               tasm->floor_w_s(dst, src);
             });
}

void TurboAssembler::Ceil_s_s(FPURegister dst, FPURegister src) {
  RoundFloat(dst, src, mode_ceil,
             [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
               tasm->ceil_w_s(dst, src);
             });
}

void TurboAssembler::Trunc_s_s(FPURegister dst, FPURegister src) {
  RoundFloat(dst, src, mode_trunc,
             [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
               tasm->trunc_w_s(dst, src);
             });
}

void TurboAssembler::Round_s_s(FPURegister dst, FPURegister src) {
  RoundFloat(dst, src, mode_round,
             [](TurboAssembler* tasm, FPURegister dst, FPURegister src) {
               tasm->round_w_s(dst, src);
             });
}

2617 2618
void MacroAssembler::Madd_s(FPURegister fd, FPURegister fr, FPURegister fs,
                            FPURegister ft, FPURegister scratch) {
2619
  DCHECK(fr != scratch && fs != scratch && ft != scratch);
2620 2621
  mul_s(scratch, fs, ft);
  add_s(fd, fr, scratch);
2622
}
2623

2624 2625
void MacroAssembler::Madd_d(FPURegister fd, FPURegister fr, FPURegister fs,
    FPURegister ft, FPURegister scratch) {
2626
  DCHECK(fr != scratch && fs != scratch && ft != scratch);
2627 2628
  mul_d(scratch, fs, ft);
  add_d(fd, fr, scratch);
2629 2630
}

2631 2632
void MacroAssembler::Msub_s(FPURegister fd, FPURegister fr, FPURegister fs,
                            FPURegister ft, FPURegister scratch) {
2633
  DCHECK(fr != scratch && fs != scratch && ft != scratch);
2634 2635
  mul_s(scratch, fs, ft);
  sub_s(fd, scratch, fr);
2636 2637 2638 2639
}

void MacroAssembler::Msub_d(FPURegister fd, FPURegister fr, FPURegister fs,
                            FPURegister ft, FPURegister scratch) {
2640
  DCHECK(fr != scratch && fs != scratch && ft != scratch);
2641 2642
  mul_d(scratch, fs, ft);
  sub_d(fd, scratch, fr);
2643
}
2644

2645 2646
void TurboAssembler::CompareF(SecondaryField sizeField, FPUCondition cc,
                              FPURegister cmp1, FPURegister cmp2) {
2647 2648
  if (kArchVariant == kMips64r6) {
    sizeField = sizeField == D ? L : W;
2649 2650 2651 2652
    DCHECK(cmp1 != kDoubleCompareReg && cmp2 != kDoubleCompareReg);
    cmp(cc, sizeField, kDoubleCompareReg, cmp1, cmp2);
  } else {
    c(cc, sizeField, cmp1, cmp2);
2653
  }
2654
}
2655

2656 2657 2658 2659 2660
void TurboAssembler::CompareIsNanF(SecondaryField sizeField, FPURegister cmp1,
                                   FPURegister cmp2) {
  CompareF(sizeField, UN, cmp1, cmp2);
}

2661
void TurboAssembler::BranchTrueShortF(Label* target, BranchDelaySlot bd) {
2662 2663 2664 2665
  if (kArchVariant == kMips64r6) {
    bc1nez(target, kDoubleCompareReg);
  } else {
    bc1t(target);
2666 2667
  }
  if (bd == PROTECT) {
2668
    nop();
2669
  }
2670
}
2671

2672
void TurboAssembler::BranchFalseShortF(Label* target, BranchDelaySlot bd) {
2673 2674 2675 2676
  if (kArchVariant == kMips64r6) {
    bc1eqz(target, kDoubleCompareReg);
  } else {
    bc1f(target);
2677 2678
  }
  if (bd == PROTECT) {
2679
    nop();
2680 2681 2682
  }
}

2683
void TurboAssembler::BranchTrueF(Label* target, BranchDelaySlot bd) {
2684 2685 2686 2687 2688
  bool long_branch =
      target->is_bound() ? !is_near(target) : is_trampoline_emitted();
  if (long_branch) {
    Label skip;
    BranchFalseShortF(&skip);
2689
    BranchLong(target, bd);
2690
    bind(&skip);
2691
  } else {
2692
    BranchTrueShortF(target, bd);
2693
  }
2694
}
2695

2696
void TurboAssembler::BranchFalseF(Label* target, BranchDelaySlot bd) {
2697 2698 2699 2700 2701
  bool long_branch =
      target->is_bound() ? !is_near(target) : is_trampoline_emitted();
  if (long_branch) {
    Label skip;
    BranchTrueShortF(&skip);
2702
    BranchLong(target, bd);
2703 2704
    bind(&skip);
  } else {
2705
    BranchFalseShortF(target, bd);
2706 2707 2708
  }
}

2709
void TurboAssembler::BranchMSA(Label* target, MSABranchDF df,
2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730
                               MSABranchCondition cond, MSARegister wt,
                               BranchDelaySlot bd) {
  {
    BlockTrampolinePoolScope block_trampoline_pool(this);

    if (target) {
      bool long_branch =
          target->is_bound() ? !is_near(target) : is_trampoline_emitted();
      if (long_branch) {
        Label skip;
        MSABranchCondition neg_cond = NegateMSABranchCondition(cond);
        BranchShortMSA(df, &skip, neg_cond, wt, bd);
        BranchLong(target, bd);
        bind(&skip);
      } else {
        BranchShortMSA(df, target, cond, wt, bd);
      }
    }
  }
}

2731
void TurboAssembler::BranchShortMSA(MSABranchDF df, Label* target,
2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784
                                    MSABranchCondition cond, MSARegister wt,
                                    BranchDelaySlot bd) {
  if (kArchVariant == kMips64r6) {
    BlockTrampolinePoolScope block_trampoline_pool(this);
    if (target) {
      switch (cond) {
        case all_not_zero:
          switch (df) {
            case MSA_BRANCH_D:
              bnz_d(wt, target);
              break;
            case MSA_BRANCH_W:
              bnz_w(wt, target);
              break;
            case MSA_BRANCH_H:
              bnz_h(wt, target);
              break;
            case MSA_BRANCH_B:
            default:
              bnz_b(wt, target);
          }
          break;
        case one_elem_not_zero:
          bnz_v(wt, target);
          break;
        case one_elem_zero:
          switch (df) {
            case MSA_BRANCH_D:
              bz_d(wt, target);
              break;
            case MSA_BRANCH_W:
              bz_w(wt, target);
              break;
            case MSA_BRANCH_H:
              bz_h(wt, target);
              break;
            case MSA_BRANCH_B:
            default:
              bz_b(wt, target);
          }
          break;
        case all_zero:
          bz_v(wt, target);
          break;
        default:
          UNREACHABLE();
      }
    }
  }
  if (bd == PROTECT) {
    nop();
  }
}
2785

2786
void TurboAssembler::FmoveLow(FPURegister dst, Register src_low) {
2787 2788
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
2789
  DCHECK(src_low != scratch);
2790
  mfhc1(scratch, dst);
2791
  mtc1(src_low, dst);
2792
  mthc1(scratch, dst);
2793 2794
}

2795
void TurboAssembler::Move(FPURegister dst, uint32_t src) {
2796 2797
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
2798
  li(scratch, Operand(static_cast<int32_t>(src)));
2799
  mtc1(scratch, dst);
2800 2801
}

2802
void TurboAssembler::Move(FPURegister dst, uint64_t src) {
2803
  // Handle special values first.
2804
  if (src == bit_cast<uint64_t>(0.0) && has_double_zero_reg_set_) {
2805
    mov_d(dst, kDoubleRegZero);
2806
  } else if (src == bit_cast<uint64_t>(-0.0) && has_double_zero_reg_set_) {
2807
    Neg_d(dst, kDoubleRegZero);
2808
  } else {
2809 2810 2811 2812
    uint32_t lo = src & 0xFFFFFFFF;
    uint32_t hi = src >> 32;
    // Move the low part of the double into the lower of the corresponding FPU
    // register of FPU register pair.
2813
    if (lo != 0) {
2814 2815
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
2816
      li(scratch, Operand(lo));
2817
      mtc1(scratch, dst);
2818 2819 2820
    } else {
      mtc1(zero_reg, dst);
    }
2821 2822
    // Move the high part of the double into the higher of the corresponding FPU
    // register of FPU register pair.
2823
    if (hi != 0) {
2824 2825
      UseScratchRegisterScope temps(this);
      Register scratch = temps.Acquire();
2826
      li(scratch, Operand(hi));
2827
      mthc1(scratch, dst);
2828 2829 2830
    } else {
      mthc1(zero_reg, dst);
    }
2831
    if (dst == kDoubleRegZero) has_double_zero_reg_set_ = true;
2832 2833 2834
  }
}

2835
void TurboAssembler::Movz(Register rd, Register rs, Register rt) {
2836
  if (kArchVariant == kMips64r6) {
2837 2838 2839 2840 2841 2842 2843 2844 2845
    Label done;
    Branch(&done, ne, rt, Operand(zero_reg));
    mov(rd, rs);
    bind(&done);
  } else {
    movz(rd, rs, rt);
  }
}

2846
void TurboAssembler::Movn(Register rd, Register rs, Register rt) {
2847
  if (kArchVariant == kMips64r6) {
2848 2849 2850 2851 2852 2853 2854 2855 2856
    Label done;
    Branch(&done, eq, rt, Operand(zero_reg));
    mov(rd, rs);
    bind(&done);
  } else {
    movn(rd, rs, rt);
  }
}

2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965
void TurboAssembler::LoadZeroOnCondition(Register rd, Register rs,
                                         const Operand& rt, Condition cond) {
  switch (cond) {
    case cc_always:
      mov(rd, zero_reg);
      break;
    case eq:
      if (rs == zero_reg) {
        if (rt.is_reg()) {
          LoadZeroIfConditionZero(rd, rt.rm());
        } else {
          if (rt.immediate() == 0) {
            mov(rd, zero_reg);
          } else {
            nop();
          }
        }
      } else if (IsZero(rt)) {
        LoadZeroIfConditionZero(rd, rs);
      } else {
        Dsubu(t9, rs, rt);
        LoadZeroIfConditionZero(rd, t9);
      }
      break;
    case ne:
      if (rs == zero_reg) {
        if (rt.is_reg()) {
          LoadZeroIfConditionNotZero(rd, rt.rm());
        } else {
          if (rt.immediate() != 0) {
            mov(rd, zero_reg);
          } else {
            nop();
          }
        }
      } else if (IsZero(rt)) {
        LoadZeroIfConditionNotZero(rd, rs);
      } else {
        Dsubu(t9, rs, rt);
        LoadZeroIfConditionNotZero(rd, t9);
      }
      break;

    // Signed comparison.
    case greater:
      Sgt(t9, rs, rt);
      LoadZeroIfConditionNotZero(rd, t9);
      break;
    case greater_equal:
      Sge(t9, rs, rt);
      LoadZeroIfConditionNotZero(rd, t9);
      // rs >= rt
      break;
    case less:
      Slt(t9, rs, rt);
      LoadZeroIfConditionNotZero(rd, t9);
      // rs < rt
      break;
    case less_equal:
      Sle(t9, rs, rt);
      LoadZeroIfConditionNotZero(rd, t9);
      // rs <= rt
      break;

    // Unsigned comparison.
    case Ugreater:
      Sgtu(t9, rs, rt);
      LoadZeroIfConditionNotZero(rd, t9);
      // rs > rt
      break;

    case Ugreater_equal:
      Sgeu(t9, rs, rt);
      LoadZeroIfConditionNotZero(rd, t9);
      // rs >= rt
      break;
    case Uless:
      Sltu(t9, rs, rt);
      LoadZeroIfConditionNotZero(rd, t9);
      // rs < rt
      break;
    case Uless_equal:
      Sleu(t9, rs, rt);
      LoadZeroIfConditionNotZero(rd, t9);
      // rs <= rt
      break;
    default:
      UNREACHABLE();
  }
}

void TurboAssembler::LoadZeroIfConditionNotZero(Register dest,
                                                Register condition) {
  if (kArchVariant == kMips64r6) {
    seleqz(dest, dest, condition);
  } else {
    Movn(dest, zero_reg, condition);
  }
}

void TurboAssembler::LoadZeroIfConditionZero(Register dest,
                                             Register condition) {
  if (kArchVariant == kMips64r6) {
    selnez(dest, dest, condition);
  } else {
    Movz(dest, zero_reg, condition);
  }
}

2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983
void TurboAssembler::LoadZeroIfFPUCondition(Register dest) {
  if (kArchVariant == kMips64r6) {
    dmfc1(kScratchReg, kDoubleCompareReg);
    LoadZeroIfConditionNotZero(dest, kScratchReg);
  } else {
    Movt(dest, zero_reg);
  }
}

void TurboAssembler::LoadZeroIfNotFPUCondition(Register dest) {
  if (kArchVariant == kMips64r6) {
    dmfc1(kScratchReg, kDoubleCompareReg);
    LoadZeroIfConditionZero(dest, kScratchReg);
  } else {
    Movf(dest, zero_reg);
  }
}

2984
void TurboAssembler::Movt(Register rd, Register rs, uint16_t cc) {
2985 2986 2987
  movt(rd, rs, cc);
}

2988
void TurboAssembler::Movf(Register rd, Register rs, uint16_t cc) {
2989 2990 2991
  movf(rd, rs, cc);
}

2992
void TurboAssembler::Clz(Register rd, Register rs) { clz(rd, rs); }
2993

2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022 3023 3024 3025 3026 3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123
void TurboAssembler::Ctz(Register rd, Register rs) {
  if (kArchVariant == kMips64r6) {
    // We don't have an instruction to count the number of trailing zeroes.
    // Start by flipping the bits end-for-end so we can count the number of
    // leading zeroes instead.
    rotr(rd, rs, 16);
    wsbh(rd, rd);
    bitswap(rd, rd);
    Clz(rd, rd);
  } else {
    // Convert trailing zeroes to trailing ones, and bits to their left
    // to zeroes.
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
    Daddu(scratch, rs, -1);
    Xor(rd, scratch, rs);
    And(rd, rd, scratch);
    // Count number of leading zeroes.
    Clz(rd, rd);
    // Subtract number of leading zeroes from 32 to get number of trailing
    // ones. Remember that the trailing ones were formerly trailing zeroes.
    li(scratch, 32);
    Subu(rd, scratch, rd);
  }
}

void TurboAssembler::Dctz(Register rd, Register rs) {
  if (kArchVariant == kMips64r6) {
    // We don't have an instruction to count the number of trailing zeroes.
    // Start by flipping the bits end-for-end so we can count the number of
    // leading zeroes instead.
    dsbh(rd, rs);
    dshd(rd, rd);
    dbitswap(rd, rd);
    dclz(rd, rd);
  } else {
    // Convert trailing zeroes to trailing ones, and bits to their left
    // to zeroes.
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
    Daddu(scratch, rs, -1);
    Xor(rd, scratch, rs);
    And(rd, rd, scratch);
    // Count number of leading zeroes.
    dclz(rd, rd);
    // Subtract number of leading zeroes from 64 to get number of trailing
    // ones. Remember that the trailing ones were formerly trailing zeroes.
    li(scratch, 64);
    Dsubu(rd, scratch, rd);
  }
}

void TurboAssembler::Popcnt(Register rd, Register rs) {
  // https://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetParallel
  //
  // A generalization of the best bit counting method to integers of
  // bit-widths up to 128 (parameterized by type T) is this:
  //
  // v = v - ((v >> 1) & (T)~(T)0/3);                           // temp
  // v = (v & (T)~(T)0/15*3) + ((v >> 2) & (T)~(T)0/15*3);      // temp
  // v = (v + (v >> 4)) & (T)~(T)0/255*15;                      // temp
  // c = (T)(v * ((T)~(T)0/255)) >> (sizeof(T) - 1) * BITS_PER_BYTE; //count
  //
  // For comparison, for 32-bit quantities, this algorithm can be executed
  // using 20 MIPS instructions (the calls to LoadConst32() generate two
  // machine instructions each for the values being used in this algorithm).
  // A(n unrolled) loop-based algorithm requires 25 instructions.
  //
  // For a 64-bit operand this can be performed in 24 instructions compared
  // to a(n unrolled) loop based algorithm which requires 38 instructions.
  //
  // There are algorithms which are faster in the cases where very few
  // bits are set but the algorithm here attempts to minimize the total
  // number of instructions executed even when a large number of bits
  // are set.
  uint32_t B0 = 0x55555555;     // (T)~(T)0/3
  uint32_t B1 = 0x33333333;     // (T)~(T)0/15*3
  uint32_t B2 = 0x0F0F0F0F;     // (T)~(T)0/255*15
  uint32_t value = 0x01010101;  // (T)~(T)0/255
  uint32_t shift = 24;          // (sizeof(T) - 1) * BITS_PER_BYTE

  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  Register scratch2 = t8;
  srl(scratch, rs, 1);
  li(scratch2, B0);
  And(scratch, scratch, scratch2);
  Subu(scratch, rs, scratch);
  li(scratch2, B1);
  And(rd, scratch, scratch2);
  srl(scratch, scratch, 2);
  And(scratch, scratch, scratch2);
  Addu(scratch, rd, scratch);
  srl(rd, scratch, 4);
  Addu(rd, rd, scratch);
  li(scratch2, B2);
  And(rd, rd, scratch2);
  li(scratch, value);
  Mul(rd, rd, scratch);
  srl(rd, rd, shift);
}

void TurboAssembler::Dpopcnt(Register rd, Register rs) {
  uint64_t B0 = 0x5555555555555555l;     // (T)~(T)0/3
  uint64_t B1 = 0x3333333333333333l;     // (T)~(T)0/15*3
  uint64_t B2 = 0x0F0F0F0F0F0F0F0Fl;     // (T)~(T)0/255*15
  uint64_t value = 0x0101010101010101l;  // (T)~(T)0/255
  uint64_t shift = 24;                   // (sizeof(T) - 1) * BITS_PER_BYTE

  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  Register scratch2 = t8;
  dsrl(scratch, rs, 1);
  li(scratch2, B0);
  And(scratch, scratch, scratch2);
  Dsubu(scratch, rs, scratch);
  li(scratch2, B1);
  And(rd, scratch, scratch2);
  dsrl(scratch, scratch, 2);
  And(scratch, scratch, scratch2);
  Daddu(scratch, rd, scratch);
  dsrl(rd, scratch, 4);
  Daddu(rd, rd, scratch);
  li(scratch2, B2);
  And(rd, rd, scratch2);
  li(scratch, value);
  Dmul(rd, rd, scratch);
  dsrl32(rd, rd, shift);
}

3124 3125 3126 3127 3128 3129 3130
void MacroAssembler::EmitFPUTruncate(FPURoundingMode rounding_mode,
                                     Register result,
                                     DoubleRegister double_input,
                                     Register scratch,
                                     DoubleRegister double_scratch,
                                     Register except_flag,
                                     CheckForInexactConversion check_inexact) {
3131 3132 3133
  DCHECK(result != scratch);
  DCHECK(double_input != double_scratch);
  DCHECK(except_flag != scratch);
3134 3135 3136 3137 3138 3139 3140 3141 3142 3143

  Label done;

  // Clear the except flag (0 = no exception)
  mov(except_flag, zero_reg);

  // Test for values that can be exactly represented as a signed 32-bit integer.
  cvt_w_d(double_scratch, double_input);
  mfc1(result, double_scratch);
  cvt_d_w(double_scratch, double_scratch);
3144 3145
  CompareF64(EQ, double_input, double_scratch);
  BranchTrueShortF(&done);
3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181 3182 3183 3184 3185 3186 3187

  int32_t except_mask = kFCSRFlagMask;  // Assume interested in all exceptions.

  if (check_inexact == kDontCheckForInexactConversion) {
    // Ignore inexact exceptions.
    except_mask &= ~kFCSRInexactFlagMask;
  }

  // Save FCSR.
  cfc1(scratch, FCSR);
  // Disable FPU exceptions.
  ctc1(zero_reg, FCSR);

  // Do operation based on rounding mode.
  switch (rounding_mode) {
    case kRoundToNearest:
      Round_w_d(double_scratch, double_input);
      break;
    case kRoundToZero:
      Trunc_w_d(double_scratch, double_input);
      break;
    case kRoundToPlusInf:
      Ceil_w_d(double_scratch, double_input);
      break;
    case kRoundToMinusInf:
      Floor_w_d(double_scratch, double_input);
      break;
  }  // End of switch-statement.

  // Retrieve FCSR.
  cfc1(except_flag, FCSR);
  // Restore FCSR.
  ctc1(scratch, FCSR);
  // Move the converted value into the result register.
  mfc1(result, double_scratch);

  // Check for fpu exceptions.
  And(except_flag, except_flag, Operand(except_mask));

  bind(&done);
}

3188
void TurboAssembler::TryInlineTruncateDoubleToI(Register result,
3189 3190
                                                DoubleRegister double_input,
                                                Label* done) {
3191
  DoubleRegister single_scratch = kScratchDoubleReg.low();
3192 3193
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207 3208 3209 3210 3211 3212
  Register scratch2 = t9;

  // Clear cumulative exception flags and save the FCSR.
  cfc1(scratch2, FCSR);
  ctc1(zero_reg, FCSR);
  // Try a conversion to a signed integer.
  trunc_w_d(single_scratch, double_input);
  mfc1(result, single_scratch);
  // Retrieve and restore the FCSR.
  cfc1(scratch, FCSR);
  ctc1(scratch2, FCSR);
  // Check for overflow and NaNs.
  And(scratch,
      scratch,
      kFCSROverflowFlagMask | kFCSRUnderflowFlagMask | kFCSRInvalidOpFlagMask);
  // If we had no exceptions we are done.
  Branch(done, eq, scratch, Operand(zero_reg));
}

3213 3214 3215
void TurboAssembler::TruncateDoubleToI(Isolate* isolate, Zone* zone,
                                       Register result,
                                       DoubleRegister double_input) {
3216 3217 3218 3219 3220 3221 3222
  Label done;

  TryInlineTruncateDoubleToI(result, double_input, &done);

  // If we fell through then inline version didn't succeed - call stub instead.
  push(ra);
  Dsubu(sp, sp, Operand(kDoubleSize));  // Put input on stack.
3223
  Sdc1(double_input, MemOperand(sp, 0));
3224

3225
  Call(BUILTIN_CODE(isolate, DoubleToI), RelocInfo::CODE_TARGET);
3226
  Ld(result, MemOperand(sp, 0));
3227 3228 3229 3230 3231 3232 3233 3234 3235 3236

  Daddu(sp, sp, Operand(kDoubleSize));
  pop(ra);

  bind(&done);
}

// Emulated condtional branches do not emit a nop in the branch delay slot.
//
// BRANCH_ARGS_CHECK checks that conditional jump arguments are correct.
3237 3238 3239
#define BRANCH_ARGS_CHECK(cond, rs, rt)                                  \
  DCHECK((cond == cc_always && rs == zero_reg && rt.rm() == zero_reg) || \
         (cond != cc_always && (rs != zero_reg || rt.rm() != zero_reg)))
3240

3241
void TurboAssembler::Branch(int32_t offset, BranchDelaySlot bdslot) {
3242
  DCHECK_EQ(kArchVariant, kMips64r6 ? is_int26(offset) : is_int16(offset));
3243 3244 3245
  BranchShort(offset, bdslot);
}

3246
void TurboAssembler::Branch(int32_t offset, Condition cond, Register rs,
3247 3248 3249 3250
                            const Operand& rt, BranchDelaySlot bdslot) {
  bool is_near = BranchShortCheck(offset, nullptr, cond, rs, rt, bdslot);
  DCHECK(is_near);
  USE(is_near);
3251 3252
}

3253
void TurboAssembler::Branch(Label* L, BranchDelaySlot bdslot) {
3254
  if (L->is_bound()) {
3255
    if (is_near_branch(L)) {
3256 3257
      BranchShort(L, bdslot);
    } else {
3258
      BranchLong(L, bdslot);
3259 3260 3261
    }
  } else {
    if (is_trampoline_emitted()) {
3262
      BranchLong(L, bdslot);
3263 3264 3265 3266 3267 3268
    } else {
      BranchShort(L, bdslot);
    }
  }
}

3269 3270
void TurboAssembler::Branch(Label* L, Condition cond, Register rs,
                            const Operand& rt, BranchDelaySlot bdslot) {
3271
  if (L->is_bound()) {
3272
    if (!BranchShortCheck(0, L, cond, rs, rt, bdslot)) {
3273 3274 3275 3276
      if (cond != cc_always) {
        Label skip;
        Condition neg_cond = NegateCondition(cond);
        BranchShort(&skip, neg_cond, rs, rt);
3277
        BranchLong(L, bdslot);
3278 3279
        bind(&skip);
      } else {
3280
        BranchLong(L, bdslot);
3281 3282 3283 3284 3285 3286 3287 3288
      }
    }
  } else {
    if (is_trampoline_emitted()) {
      if (cond != cc_always) {
        Label skip;
        Condition neg_cond = NegateCondition(cond);
        BranchShort(&skip, neg_cond, rs, rt);
3289
        BranchLong(L, bdslot);
3290 3291
        bind(&skip);
      } else {
3292
        BranchLong(L, bdslot);
3293 3294 3295 3296 3297 3298 3299
      }
    } else {
      BranchShort(L, cond, rs, rt, bdslot);
    }
  }
}

3300 3301
void TurboAssembler::Branch(Label* L, Condition cond, Register rs,
                            Heap::RootListIndex index, BranchDelaySlot bdslot) {
3302 3303 3304 3305
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  LoadRoot(scratch, index);
  Branch(L, cond, rs, Operand(scratch), bdslot);
3306 3307
}

3308
void TurboAssembler::BranchShortHelper(int16_t offset, Label* L,
3309 3310 3311
                                       BranchDelaySlot bdslot) {
  DCHECK(L == nullptr || offset == 0);
  offset = GetOffset(offset, L, OffsetSize::kOffset16);
3312 3313 3314 3315 3316 3317 3318
  b(offset);

  // Emit a nop in the branch delay slot if required.
  if (bdslot == PROTECT)
    nop();
}

3319
void TurboAssembler::BranchShortHelperR6(int32_t offset, Label* L) {
3320 3321 3322 3323
  DCHECK(L == nullptr || offset == 0);
  offset = GetOffset(offset, L, OffsetSize::kOffset26);
  bc(offset);
}
3324

3325
void TurboAssembler::BranchShort(int32_t offset, BranchDelaySlot bdslot) {
3326 3327 3328
  if (kArchVariant == kMips64r6 && bdslot == PROTECT) {
    DCHECK(is_int26(offset));
    BranchShortHelperR6(offset, nullptr);
3329
  } else {
3330 3331
    DCHECK(is_int16(offset));
    BranchShortHelper(offset, nullptr, bdslot);
3332 3333 3334
  }
}

3335
void TurboAssembler::BranchShort(Label* L, BranchDelaySlot bdslot) {
3336 3337 3338 3339 3340 3341
  if (kArchVariant == kMips64r6 && bdslot == PROTECT) {
    BranchShortHelperR6(0, L);
  } else {
    BranchShortHelper(0, L, bdslot);
  }
}
3342 3343


3344
int32_t TurboAssembler::GetOffset(int32_t offset, Label* L, OffsetSize bits) {
3345 3346 3347 3348 3349 3350 3351 3352
  if (L) {
    offset = branch_offset_helper(L, bits) >> 2;
  } else {
    DCHECK(is_intn(offset, bits));
  }
  return offset;
}

3353
Register TurboAssembler::GetRtAsRegisterHelper(const Operand& rt,
3354
                                               Register scratch) {
3355 3356
  Register r2 = no_reg;
  if (rt.is_reg()) {
3357
    r2 = rt.rm();
3358 3359 3360 3361 3362 3363 3364 3365
  } else {
    r2 = scratch;
    li(r2, rt);
  }

  return r2;
}

3366 3367 3368 3369 3370 3371 3372 3373 3374 3375 3376 3377 3378 3379 3380
bool TurboAssembler::CalculateOffset(Label* L, int32_t& offset,
                                     OffsetSize bits) {
  if (!is_near(L, bits)) return false;
  offset = GetOffset(offset, L, bits);
  return true;
}

bool TurboAssembler::CalculateOffset(Label* L, int32_t& offset, OffsetSize bits,
                                     Register& scratch, const Operand& rt) {
  if (!is_near(L, bits)) return false;
  scratch = GetRtAsRegisterHelper(rt, scratch);
  offset = GetOffset(offset, L, bits);
  return true;
}

3381
bool TurboAssembler::BranchShortHelperR6(int32_t offset, Label* L,
3382 3383 3384
                                         Condition cond, Register rs,
                                         const Operand& rt) {
  DCHECK(L == nullptr || offset == 0);
3385 3386
  UseScratchRegisterScope temps(this);
  Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
3387 3388 3389 3390 3391 3392

  // Be careful to always use shifted_branch_offset only just before the
  // branch instruction, as the location will be remember for patching the
  // target.
  {
    BlockTrampolinePoolScope block_trampoline_pool(this);
3393 3394
    switch (cond) {
      case cc_always:
3395
        if (!CalculateOffset(L, offset, OffsetSize::kOffset26)) return false;
3396
        bc(offset);
3397 3398
        break;
      case eq:
3399
        if (rt.is_reg() && rs.code() == rt.rm().code()) {
3400 3401
          // Pre R6 beq is used here to make the code patchable. Otherwise bc
          // should be used which has no condition field so is not patchable.
3402 3403
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
            return false;
3404 3405 3406
          beq(rs, scratch, offset);
          nop();
        } else if (IsZero(rt)) {
3407
          if (!CalculateOffset(L, offset, OffsetSize::kOffset21)) return false;
3408 3409 3410
          beqzc(rs, offset);
        } else {
          // We don't want any other register but scratch clobbered.
3411 3412
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
            return false;
3413 3414
          beqc(rs, scratch, offset);
        }
3415 3416
        break;
      case ne:
3417
        if (rt.is_reg() && rs.code() == rt.rm().code()) {
3418 3419
          // Pre R6 bne is used here to make the code patchable. Otherwise we
          // should not generate any instruction.
3420 3421
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
            return false;
3422 3423 3424
          bne(rs, scratch, offset);
          nop();
        } else if (IsZero(rt)) {
3425
          if (!CalculateOffset(L, offset, OffsetSize::kOffset21)) return false;
3426 3427 3428
          bnezc(rs, offset);
        } else {
          // We don't want any other register but scratch clobbered.
3429 3430
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
            return false;
3431 3432
          bnec(rs, scratch, offset);
        }
3433
        break;
3434

3435 3436
      // Signed comparison.
      case greater:
3437
        // rs > rt
3438
        if (rt.is_reg() && rs.code() == rt.rm().code()) {
3439
          break;  // No code needs to be emitted.
3440
        } else if (rs == zero_reg) {
3441 3442
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
            return false;
3443 3444
          bltzc(scratch, offset);
        } else if (IsZero(rt)) {
3445
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16)) return false;
3446
          bgtzc(rs, offset);
3447
        } else {
3448 3449
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
            return false;
3450
          DCHECK(rs != scratch);
3451
          bltc(scratch, rs, offset);
3452 3453 3454
        }
        break;
      case greater_equal:
3455
        // rs >= rt
3456
        if (rt.is_reg() && rs.code() == rt.rm().code()) {
3457
          if (!CalculateOffset(L, offset, OffsetSize::kOffset26)) return false;
3458
          bc(offset);
3459
        } else if (rs == zero_reg) {
3460 3461
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
            return false;
3462 3463
          blezc(scratch, offset);
        } else if (IsZero(rt)) {
3464
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16)) return false;
3465
          bgezc(rs, offset);
3466
        } else {
3467 3468
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
            return false;
3469
          DCHECK(rs != scratch);
3470
          bgec(rs, scratch, offset);
3471 3472 3473
        }
        break;
      case less:
3474
        // rs < rt
3475
        if (rt.is_reg() && rs.code() == rt.rm().code()) {
3476
          break;  // No code needs to be emitted.
3477
        } else if (rs == zero_reg) {
3478 3479
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
            return false;
3480 3481
          bgtzc(scratch, offset);
        } else if (IsZero(rt)) {
3482
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16)) return false;
3483
          bltzc(rs, offset);
3484
        } else {
3485 3486
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
            return false;
3487
          DCHECK(rs != scratch);
3488
          bltc(rs, scratch, offset);
3489 3490 3491
        }
        break;
      case less_equal:
3492
        // rs <= rt
3493
        if (rt.is_reg() && rs.code() == rt.rm().code()) {
3494
          if (!CalculateOffset(L, offset, OffsetSize::kOffset26)) return false;
3495
          bc(offset);
3496
        } else if (rs == zero_reg) {
3497 3498
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
            return false;
3499 3500
          bgezc(scratch, offset);
        } else if (IsZero(rt)) {
3501
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16)) return false;
3502
          blezc(rs, offset);
3503
        } else {
3504 3505
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
            return false;
3506
          DCHECK(rs != scratch);
3507
          bgec(scratch, rs, offset);
3508 3509
        }
        break;
3510

3511 3512
      // Unsigned comparison.
      case Ugreater:
3513
        // rs > rt
3514
        if (rt.is_reg() && rs.code() == rt.rm().code()) {
3515
          break;  // No code needs to be emitted.
3516
        } else if (rs == zero_reg) {
3517 3518
          if (!CalculateOffset(L, offset, OffsetSize::kOffset21, scratch, rt))
            return false;
3519 3520
          bnezc(scratch, offset);
        } else if (IsZero(rt)) {
3521
          if (!CalculateOffset(L, offset, OffsetSize::kOffset21)) return false;
3522
          bnezc(rs, offset);
3523
        } else {
3524 3525
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
            return false;
3526
          DCHECK(rs != scratch);
3527
          bltuc(scratch, rs, offset);
3528 3529 3530
        }
        break;
      case Ugreater_equal:
3531
        // rs >= rt
3532
        if (rt.is_reg() && rs.code() == rt.rm().code()) {
3533
          if (!CalculateOffset(L, offset, OffsetSize::kOffset26)) return false;
3534
          bc(offset);
3535
        } else if (rs == zero_reg) {
3536 3537
          if (!CalculateOffset(L, offset, OffsetSize::kOffset21, scratch, rt))
            return false;
3538 3539
          beqzc(scratch, offset);
        } else if (IsZero(rt)) {
3540
          if (!CalculateOffset(L, offset, OffsetSize::kOffset26)) return false;
3541
          bc(offset);
3542
        } else {
3543 3544
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
            return false;
3545
          DCHECK(rs != scratch);
3546
          bgeuc(rs, scratch, offset);
3547 3548 3549
        }
        break;
      case Uless:
3550
        // rs < rt
3551
        if (rt.is_reg() && rs.code() == rt.rm().code()) {
3552
          break;  // No code needs to be emitted.
3553
        } else if (rs == zero_reg) {
3554 3555
          if (!CalculateOffset(L, offset, OffsetSize::kOffset21, scratch, rt))
            return false;
3556 3557 3558
          bnezc(scratch, offset);
        } else if (IsZero(rt)) {
          break;  // No code needs to be emitted.
3559
        } else {
3560 3561
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
            return false;
3562
          DCHECK(rs != scratch);
3563
          bltuc(rs, scratch, offset);
3564 3565 3566
        }
        break;
      case Uless_equal:
3567
        // rs <= rt
3568
        if (rt.is_reg() && rs.code() == rt.rm().code()) {
3569
          if (!CalculateOffset(L, offset, OffsetSize::kOffset26)) return false;
3570
          bc(offset);
3571
        } else if (rs == zero_reg) {
3572 3573
          if (!CalculateOffset(L, offset, OffsetSize::kOffset26, scratch, rt))
            return false;
3574 3575
          bc(offset);
        } else if (IsZero(rt)) {
3576
          if (!CalculateOffset(L, offset, OffsetSize::kOffset21)) return false;
3577
          beqzc(rs, offset);
3578
        } else {
3579 3580
          if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
            return false;
3581
          DCHECK(rs != scratch);
3582
          bgeuc(scratch, rs, offset);
3583 3584 3585 3586 3587
        }
        break;
      default:
        UNREACHABLE();
    }
3588 3589 3590 3591 3592
  }
  CheckTrampolinePoolQuick(1);
  return true;
}

3593
bool TurboAssembler::BranchShortHelper(int16_t offset, Label* L, Condition cond,
3594 3595 3596 3597 3598
                                       Register rs, const Operand& rt,
                                       BranchDelaySlot bdslot) {
  DCHECK(L == nullptr || offset == 0);
  if (!is_near(L, OffsetSize::kOffset16)) return false;

3599 3600
  UseScratchRegisterScope temps(this);
  Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
3601 3602 3603 3604 3605 3606
  int32_t offset32;

  // Be careful to always use shifted_branch_offset only just before the
  // branch instruction, as the location will be remember for patching the
  // target.
  {
3607 3608 3609
    BlockTrampolinePoolScope block_trampoline_pool(this);
    switch (cond) {
      case cc_always:
3610 3611
        offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
        b(offset32);
3612 3613
        break;
      case eq:
3614 3615 3616
        if (IsZero(rt)) {
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          beq(rs, zero_reg, offset32);
3617
        } else {
3618 3619 3620 3621
          // We don't want any other register but scratch clobbered.
          scratch = GetRtAsRegisterHelper(rt, scratch);
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          beq(rs, scratch, offset32);
3622
        }
3623 3624
        break;
      case ne:
3625 3626 3627
        if (IsZero(rt)) {
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          bne(rs, zero_reg, offset32);
3628
        } else {
3629 3630 3631 3632
          // We don't want any other register but scratch clobbered.
          scratch = GetRtAsRegisterHelper(rt, scratch);
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          bne(rs, scratch, offset32);
3633
        }
3634
        break;
3635

3636 3637
      // Signed comparison.
      case greater:
3638 3639 3640
        if (IsZero(rt)) {
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          bgtz(rs, offset32);
3641
        } else {
3642 3643 3644
          Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          bne(scratch, zero_reg, offset32);
3645 3646 3647
        }
        break;
      case greater_equal:
3648 3649 3650
        if (IsZero(rt)) {
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          bgez(rs, offset32);
3651
        } else {
3652 3653 3654
          Slt(scratch, rs, rt);
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          beq(scratch, zero_reg, offset32);
3655 3656 3657
        }
        break;
      case less:
3658 3659 3660
        if (IsZero(rt)) {
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          bltz(rs, offset32);
3661
        } else {
3662 3663 3664
          Slt(scratch, rs, rt);
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          bne(scratch, zero_reg, offset32);
3665 3666 3667
        }
        break;
      case less_equal:
3668 3669 3670
        if (IsZero(rt)) {
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          blez(rs, offset32);
3671
        } else {
3672 3673 3674
          Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          beq(scratch, zero_reg, offset32);
3675 3676
        }
        break;
3677

3678 3679
      // Unsigned comparison.
      case Ugreater:
3680 3681 3682
        if (IsZero(rt)) {
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          bne(rs, zero_reg, offset32);
3683
        } else {
3684 3685 3686
          Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          bne(scratch, zero_reg, offset32);
3687 3688 3689
        }
        break;
      case Ugreater_equal:
3690 3691 3692
        if (IsZero(rt)) {
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          b(offset32);
3693
        } else {
3694 3695 3696
          Sltu(scratch, rs, rt);
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          beq(scratch, zero_reg, offset32);
3697 3698
        }
        break;
3699 3700 3701
      case Uless:
        if (IsZero(rt)) {
          return true;  // No code needs to be emitted.
3702
        } else {
3703 3704 3705
          Sltu(scratch, rs, rt);
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          bne(scratch, zero_reg, offset32);
3706 3707 3708
        }
        break;
      case Uless_equal:
3709 3710 3711
        if (IsZero(rt)) {
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          beq(rs, zero_reg, offset32);
3712
        } else {
3713 3714 3715
          Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
          offset32 = GetOffset(offset, L, OffsetSize::kOffset16);
          beq(scratch, zero_reg, offset32);
3716 3717 3718 3719 3720 3721
        }
        break;
      default:
        UNREACHABLE();
    }
  }
3722

3723 3724 3725
  // Emit a nop in the branch delay slot if required.
  if (bdslot == PROTECT)
    nop();
3726 3727

  return true;
3728 3729
}

3730
bool TurboAssembler::BranchShortCheck(int32_t offset, Label* L, Condition cond,
3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743
                                      Register rs, const Operand& rt,
                                      BranchDelaySlot bdslot) {
  BRANCH_ARGS_CHECK(cond, rs, rt);

  if (!L) {
    if (kArchVariant == kMips64r6 && bdslot == PROTECT) {
      DCHECK(is_int26(offset));
      return BranchShortHelperR6(offset, nullptr, cond, rs, rt);
    } else {
      DCHECK(is_int16(offset));
      return BranchShortHelper(offset, nullptr, cond, rs, rt, bdslot);
    }
  } else {
3744
    DCHECK_EQ(offset, 0);
3745 3746 3747 3748 3749 3750 3751 3752 3753
    if (kArchVariant == kMips64r6 && bdslot == PROTECT) {
      return BranchShortHelperR6(0, L, cond, rs, rt);
    } else {
      return BranchShortHelper(0, L, cond, rs, rt, bdslot);
    }
  }
  return false;
}

3754
void TurboAssembler::BranchShort(int32_t offset, Condition cond, Register rs,
3755 3756 3757 3758
                                 const Operand& rt, BranchDelaySlot bdslot) {
  BranchShortCheck(offset, nullptr, cond, rs, rt, bdslot);
}

3759
void TurboAssembler::BranchShort(Label* L, Condition cond, Register rs,
3760 3761 3762 3763
                                 const Operand& rt, BranchDelaySlot bdslot) {
  BranchShortCheck(0, L, cond, rs, rt, bdslot);
}

3764
void TurboAssembler::BranchAndLink(int32_t offset, BranchDelaySlot bdslot) {
3765 3766 3767
  BranchAndLinkShort(offset, bdslot);
}

3768
void TurboAssembler::BranchAndLink(int32_t offset, Condition cond, Register rs,
3769 3770 3771 3772
                                   const Operand& rt, BranchDelaySlot bdslot) {
  bool is_near = BranchAndLinkShortCheck(offset, nullptr, cond, rs, rt, bdslot);
  DCHECK(is_near);
  USE(is_near);
3773 3774
}

3775
void TurboAssembler::BranchAndLink(Label* L, BranchDelaySlot bdslot) {
3776
  if (L->is_bound()) {
3777
    if (is_near_branch(L)) {
3778 3779
      BranchAndLinkShort(L, bdslot);
    } else {
3780
      BranchAndLinkLong(L, bdslot);
3781 3782 3783
    }
  } else {
    if (is_trampoline_emitted()) {
3784
      BranchAndLinkLong(L, bdslot);
3785 3786 3787 3788 3789 3790
    } else {
      BranchAndLinkShort(L, bdslot);
    }
  }
}

3791 3792
void TurboAssembler::BranchAndLink(Label* L, Condition cond, Register rs,
                                   const Operand& rt, BranchDelaySlot bdslot) {
3793
  if (L->is_bound()) {
3794
    if (!BranchAndLinkShortCheck(0, L, cond, rs, rt, bdslot)) {
3795 3796 3797
      Label skip;
      Condition neg_cond = NegateCondition(cond);
      BranchShort(&skip, neg_cond, rs, rt);
3798
      BranchAndLinkLong(L, bdslot);
3799 3800 3801 3802 3803 3804 3805
      bind(&skip);
    }
  } else {
    if (is_trampoline_emitted()) {
      Label skip;
      Condition neg_cond = NegateCondition(cond);
      BranchShort(&skip, neg_cond, rs, rt);
3806
      BranchAndLinkLong(L, bdslot);
3807 3808
      bind(&skip);
    } else {
3809
      BranchAndLinkShortCheck(0, L, cond, rs, rt, bdslot);
3810 3811 3812 3813
    }
  }
}

3814
void TurboAssembler::BranchAndLinkShortHelper(int16_t offset, Label* L,
3815 3816 3817
                                              BranchDelaySlot bdslot) {
  DCHECK(L == nullptr || offset == 0);
  offset = GetOffset(offset, L, OffsetSize::kOffset16);
3818 3819 3820 3821 3822 3823 3824
  bal(offset);

  // Emit a nop in the branch delay slot if required.
  if (bdslot == PROTECT)
    nop();
}

3825
void TurboAssembler::BranchAndLinkShortHelperR6(int32_t offset, Label* L) {
3826 3827 3828 3829
  DCHECK(L == nullptr || offset == 0);
  offset = GetOffset(offset, L, OffsetSize::kOffset26);
  balc(offset);
}
3830

3831
void TurboAssembler::BranchAndLinkShort(int32_t offset,
3832 3833 3834 3835 3836 3837 3838 3839 3840
                                        BranchDelaySlot bdslot) {
  if (kArchVariant == kMips64r6 && bdslot == PROTECT) {
    DCHECK(is_int26(offset));
    BranchAndLinkShortHelperR6(offset, nullptr);
  } else {
    DCHECK(is_int16(offset));
    BranchAndLinkShortHelper(offset, nullptr, bdslot);
  }
}
3841

3842
void TurboAssembler::BranchAndLinkShort(Label* L, BranchDelaySlot bdslot) {
3843 3844 3845 3846
  if (kArchVariant == kMips64r6 && bdslot == PROTECT) {
    BranchAndLinkShortHelperR6(0, L);
  } else {
    BranchAndLinkShortHelper(0, L, bdslot);
3847 3848 3849
  }
}

3850
bool TurboAssembler::BranchAndLinkShortHelperR6(int32_t offset, Label* L,
3851 3852 3853
                                                Condition cond, Register rs,
                                                const Operand& rt) {
  DCHECK(L == nullptr || offset == 0);
3854 3855
  UseScratchRegisterScope temps(this);
  Register scratch = temps.hasAvailable() ? temps.Acquire() : t8;
3856
  OffsetSize bits = OffsetSize::kOffset16;
3857

3858 3859 3860 3861
  BlockTrampolinePoolScope block_trampoline_pool(this);
  DCHECK((cond == cc_always && is_int26(offset)) || is_int16(offset));
  switch (cond) {
    case cc_always:
3862
      if (!CalculateOffset(L, offset, OffsetSize::kOffset26)) return false;
3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876
      balc(offset);
      break;
    case eq:
      if (!is_near(L, bits)) return false;
      Subu(scratch, rs, rt);
      offset = GetOffset(offset, L, bits);
      beqzalc(scratch, offset);
      break;
    case ne:
      if (!is_near(L, bits)) return false;
      Subu(scratch, rs, rt);
      offset = GetOffset(offset, L, bits);
      bnezalc(scratch, offset);
      break;
3877

3878 3879 3880
    // Signed comparison.
    case greater:
      // rs > rt
3881
      if (rs.code() == rt.rm().code()) {
3882
        break;  // No code needs to be emitted.
3883
      } else if (rs == zero_reg) {
3884 3885
        if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
          return false;
3886 3887
        bltzalc(scratch, offset);
      } else if (IsZero(rt)) {
3888
        if (!CalculateOffset(L, offset, OffsetSize::kOffset16)) return false;
3889 3890 3891 3892 3893 3894 3895 3896 3897 3898
        bgtzalc(rs, offset);
      } else {
        if (!is_near(L, bits)) return false;
        Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
        offset = GetOffset(offset, L, bits);
        bnezalc(scratch, offset);
      }
      break;
    case greater_equal:
      // rs >= rt
3899
      if (rs.code() == rt.rm().code()) {
3900
        if (!CalculateOffset(L, offset, OffsetSize::kOffset26)) return false;
3901
        balc(offset);
3902
      } else if (rs == zero_reg) {
3903 3904
        if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
          return false;
3905 3906
        blezalc(scratch, offset);
      } else if (IsZero(rt)) {
3907
        if (!CalculateOffset(L, offset, OffsetSize::kOffset16)) return false;
3908 3909 3910 3911 3912 3913 3914 3915 3916 3917
        bgezalc(rs, offset);
      } else {
        if (!is_near(L, bits)) return false;
        Slt(scratch, rs, rt);
        offset = GetOffset(offset, L, bits);
        beqzalc(scratch, offset);
      }
      break;
    case less:
      // rs < rt
3918
      if (rs.code() == rt.rm().code()) {
3919
        break;  // No code needs to be emitted.
3920
      } else if (rs == zero_reg) {
3921 3922
        if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
          return false;
3923 3924
        bgtzalc(scratch, offset);
      } else if (IsZero(rt)) {
3925
        if (!CalculateOffset(L, offset, OffsetSize::kOffset16)) return false;
3926 3927 3928 3929 3930 3931 3932 3933 3934 3935
        bltzalc(rs, offset);
      } else {
        if (!is_near(L, bits)) return false;
        Slt(scratch, rs, rt);
        offset = GetOffset(offset, L, bits);
        bnezalc(scratch, offset);
      }
      break;
    case less_equal:
      // rs <= r2
3936
      if (rs.code() == rt.rm().code()) {
3937
        if (!CalculateOffset(L, offset, OffsetSize::kOffset26)) return false;
3938
        balc(offset);
3939
      } else if (rs == zero_reg) {
3940 3941
        if (!CalculateOffset(L, offset, OffsetSize::kOffset16, scratch, rt))
          return false;
3942 3943
        bgezalc(scratch, offset);
      } else if (IsZero(rt)) {
3944
        if (!CalculateOffset(L, offset, OffsetSize::kOffset16)) return false;
3945 3946 3947 3948 3949 3950 3951 3952
        blezalc(rs, offset);
      } else {
        if (!is_near(L, bits)) return false;
        Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
        offset = GetOffset(offset, L, bits);
        beqzalc(scratch, offset);
      }
      break;
3953 3954


3955 3956 3957 3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968 3969 3970 3971 3972 3973 3974 3975 3976 3977 3978 3979 3980 3981 3982 3983 3984 3985
    // Unsigned comparison.
    case Ugreater:
      // rs > r2
      if (!is_near(L, bits)) return false;
      Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
      offset = GetOffset(offset, L, bits);
      bnezalc(scratch, offset);
      break;
    case Ugreater_equal:
      // rs >= r2
      if (!is_near(L, bits)) return false;
      Sltu(scratch, rs, rt);
      offset = GetOffset(offset, L, bits);
      beqzalc(scratch, offset);
      break;
    case Uless:
      // rs < r2
      if (!is_near(L, bits)) return false;
      Sltu(scratch, rs, rt);
      offset = GetOffset(offset, L, bits);
      bnezalc(scratch, offset);
      break;
    case Uless_equal:
      // rs <= r2
      if (!is_near(L, bits)) return false;
      Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
      offset = GetOffset(offset, L, bits);
      beqzalc(scratch, offset);
      break;
    default:
      UNREACHABLE();
3986
  }
3987 3988
  return true;
}
3989

3990 3991 3992
// Pre r6 we need to use a bgezal or bltzal, but they can't be used directly
// with the slt instructions. We could use sub or add instead but we would miss
// overflow cases, so we keep slt and add an intermediate third instruction.
3993
bool TurboAssembler::BranchAndLinkShortHelper(int16_t offset, Label* L,
3994 3995 3996 3997 3998
                                              Condition cond, Register rs,
                                              const Operand& rt,
                                              BranchDelaySlot bdslot) {
  DCHECK(L == nullptr || offset == 0);
  if (!is_near(L, OffsetSize::kOffset16)) return false;
3999

4000 4001
  Register scratch = t8;
  BlockTrampolinePoolScope block_trampoline_pool(this);
4002

4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015 4016 4017 4018 4019
  switch (cond) {
    case cc_always:
      offset = GetOffset(offset, L, OffsetSize::kOffset16);
      bal(offset);
      break;
    case eq:
      bne(rs, GetRtAsRegisterHelper(rt, scratch), 2);
      nop();
      offset = GetOffset(offset, L, OffsetSize::kOffset16);
      bal(offset);
      break;
    case ne:
      beq(rs, GetRtAsRegisterHelper(rt, scratch), 2);
      nop();
      offset = GetOffset(offset, L, OffsetSize::kOffset16);
      bal(offset);
      break;
4020

4021 4022 4023 4024 4025 4026 4027 4028 4029 4030 4031 4032 4033 4034 4035 4036 4037 4038 4039 4040 4041 4042 4043 4044 4045 4046 4047 4048 4049 4050 4051 4052 4053 4054 4055 4056 4057 4058 4059 4060 4061 4062 4063 4064 4065 4066 4067 4068 4069 4070 4071 4072 4073 4074
    // Signed comparison.
    case greater:
      Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
      addiu(scratch, scratch, -1);
      offset = GetOffset(offset, L, OffsetSize::kOffset16);
      bgezal(scratch, offset);
      break;
    case greater_equal:
      Slt(scratch, rs, rt);
      addiu(scratch, scratch, -1);
      offset = GetOffset(offset, L, OffsetSize::kOffset16);
      bltzal(scratch, offset);
      break;
    case less:
      Slt(scratch, rs, rt);
      addiu(scratch, scratch, -1);
      offset = GetOffset(offset, L, OffsetSize::kOffset16);
      bgezal(scratch, offset);
      break;
    case less_equal:
      Slt(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
      addiu(scratch, scratch, -1);
      offset = GetOffset(offset, L, OffsetSize::kOffset16);
      bltzal(scratch, offset);
      break;

    // Unsigned comparison.
    case Ugreater:
      Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
      addiu(scratch, scratch, -1);
      offset = GetOffset(offset, L, OffsetSize::kOffset16);
      bgezal(scratch, offset);
      break;
    case Ugreater_equal:
      Sltu(scratch, rs, rt);
      addiu(scratch, scratch, -1);
      offset = GetOffset(offset, L, OffsetSize::kOffset16);
      bltzal(scratch, offset);
      break;
    case Uless:
      Sltu(scratch, rs, rt);
      addiu(scratch, scratch, -1);
      offset = GetOffset(offset, L, OffsetSize::kOffset16);
      bgezal(scratch, offset);
      break;
    case Uless_equal:
      Sltu(scratch, GetRtAsRegisterHelper(rt, scratch), rs);
      addiu(scratch, scratch, -1);
      offset = GetOffset(offset, L, OffsetSize::kOffset16);
      bltzal(scratch, offset);
      break;

    default:
      UNREACHABLE();
4075 4076 4077 4078 4079
  }

  // Emit a nop in the branch delay slot if required.
  if (bdslot == PROTECT)
    nop();
4080 4081 4082 4083

  return true;
}

4084
bool TurboAssembler::BranchAndLinkShortCheck(int32_t offset, Label* L,
4085 4086 4087 4088 4089 4090 4091 4092 4093 4094 4095 4096 4097 4098
                                             Condition cond, Register rs,
                                             const Operand& rt,
                                             BranchDelaySlot bdslot) {
  BRANCH_ARGS_CHECK(cond, rs, rt);

  if (!L) {
    if (kArchVariant == kMips64r6 && bdslot == PROTECT) {
      DCHECK(is_int26(offset));
      return BranchAndLinkShortHelperR6(offset, nullptr, cond, rs, rt);
    } else {
      DCHECK(is_int16(offset));
      return BranchAndLinkShortHelper(offset, nullptr, cond, rs, rt, bdslot);
    }
  } else {
4099
    DCHECK_EQ(offset, 0);
4100 4101 4102 4103 4104 4105 4106
    if (kArchVariant == kMips64r6 && bdslot == PROTECT) {
      return BranchAndLinkShortHelperR6(0, L, cond, rs, rt);
    } else {
      return BranchAndLinkShortHelper(0, L, cond, rs, rt, bdslot);
    }
  }
  return false;
4107 4108
}

4109
#ifdef V8_EMBEDDED_BUILTINS
4110 4111
void TurboAssembler::LoadFromConstantsTable(Register destination,
                                            int constant_index) {
4112 4113 4114
  DCHECK(isolate()->heap()->RootCanBeTreatedAsConstant(
      Heap::kBuiltinsConstantsTableRootIndex));
  LoadRoot(destination, Heap::kBuiltinsConstantsTableRootIndex);
4115 4116 4117
  Ld(destination,
     FieldMemOperand(destination,
                     FixedArray::kHeaderSize + constant_index * kPointerSize));
4118 4119
}

4120 4121
void TurboAssembler::LoadExternalReference(Register destination,
                                           int reference_index) {
4122 4123
  int32_t roots_to_external_reference_offset =
      Heap::roots_to_external_reference_table_offset() +
4124
      ExternalReferenceTable::OffsetOfEntry(reference_index);
4125 4126 4127
  Ld(destination,
     MemOperand(kRootRegister, roots_to_external_reference_offset));
}
4128 4129 4130 4131 4132 4133 4134

void TurboAssembler::LoadBuiltin(Register destination, int builtin_index) {
  DCHECK(Builtins::IsBuiltinId(builtin_index));
  int32_t roots_to_builtins_offset =
      Heap::roots_to_builtins_offset() + builtin_index * kPointerSize;
  Ld(destination, MemOperand(kRootRegister, roots_to_builtins_offset));
}
4135 4136 4137 4138 4139 4140 4141 4142 4143

void TurboAssembler::LoadRootRegisterOffset(Register destination,
                                            intptr_t offset) {
  if (offset == 0) {
    Move(destination, kRootRegister);
  } else {
    Daddu(destination, kRootRegister, Operand(offset));
  }
}
4144 4145
#endif  // V8_EMBEDDED_BUILTINS

4146 4147
void TurboAssembler::Jump(Register target, Condition cond, Register rs,
                          const Operand& rt, BranchDelaySlot bd) {
4148
  BlockTrampolinePoolScope block_trampoline_pool(this);
4149 4150 4151 4152 4153 4154 4155 4156
  if (kArchVariant == kMips64r6 && bd == PROTECT) {
    if (cond == cc_always) {
      jic(target, 0);
    } else {
      BRANCH_ARGS_CHECK(cond, rs, rt);
      Branch(2, NegateCondition(cond), rs, rt);
      jic(target, 0);
    }
4157
  } else {
4158 4159 4160 4161 4162 4163 4164 4165 4166
    if (cond == cc_always) {
      jr(target);
    } else {
      BRANCH_ARGS_CHECK(cond, rs, rt);
      Branch(2, NegateCondition(cond), rs, rt);
      jr(target);
    }
    // Emit a nop in the branch delay slot if required.
    if (bd == PROTECT) nop();
4167 4168 4169
  }
}

4170 4171
void TurboAssembler::Jump(intptr_t target, RelocInfo::Mode rmode,
                          Condition cond, Register rs, const Operand& rt,
4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183
                          BranchDelaySlot bd) {
  Label skip;
  if (cond != cc_always) {
    Branch(USE_DELAY_SLOT, &skip, NegateCondition(cond), rs, rt);
  }
  // The first instruction of 'li' may be placed in the delay slot.
  // This is not an issue, t9 is expected to be clobbered anyway.
  li(t9, Operand(target, rmode));
  Jump(t9, al, zero_reg, Operand(zero_reg), bd);
  bind(&skip);
}

4184 4185
void TurboAssembler::Jump(Address target, RelocInfo::Mode rmode, Condition cond,
                          Register rs, const Operand& rt, BranchDelaySlot bd) {
4186
  DCHECK(!RelocInfo::IsCodeTarget(rmode));
4187
  Jump(static_cast<intptr_t>(target), rmode, cond, rs, rt, bd);
4188 4189
}

4190 4191
void TurboAssembler::Jump(Handle<Code> code, RelocInfo::Mode rmode,
                          Condition cond, Register rs, const Operand& rt,
4192
                          BranchDelaySlot bd) {
4193
  DCHECK(RelocInfo::IsCodeTarget(rmode));
4194 4195
#ifdef V8_EMBEDDED_BUILTINS
  if (root_array_available_ && isolate()->ShouldLoadConstantsFromRootList()) {
4196
    IndirectLoadConstant(t9, code);
4197 4198
    Daddu(t9, t9, Operand(Code::kHeaderSize - kHeapObjectTag));
    Jump(t9, cond, rs, rt, bd);
4199
    return;
4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211
  } else if (!isolate()->serializer_enabled()) {
    int builtin_index = Builtins::kNoBuiltinId;
    if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index) &&
        Builtins::IsIsolateIndependent(builtin_index)) {
      // Inline the trampoline.
      CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
      EmbeddedData d = EmbeddedData::FromBlob();
      Address entry = d.InstructionStartOfBuiltin(builtin_index);
      li(t9, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
      Jump(t9, cond, rs, rt, bd);
      return;
    }
4212 4213
  }
#endif  // V8_EMBEDDED_BUILTINS
4214
  Jump(static_cast<intptr_t>(code.address()), rmode, cond, rs, rt, bd);
4215 4216
}

4217 4218
int TurboAssembler::CallSize(Register target, Condition cond, Register rs,
                             const Operand& rt, BranchDelaySlot bd) {
4219 4220 4221 4222 4223 4224 4225 4226
  int size = 0;

  if (cond == cc_always) {
    size += 1;
  } else {
    size += 3;
  }

4227
  if (bd == PROTECT && kArchVariant != kMips64r6) size += 1;
4228 4229 4230 4231 4232

  return size * kInstrSize;
}


4233
// Note: To call gcc-compiled C code on mips, you must call through t9.
4234 4235
void TurboAssembler::Call(Register target, Condition cond, Register rs,
                          const Operand& rt, BranchDelaySlot bd) {
4236 4237 4238 4239
#ifdef DEBUG
  int size = IsPrevInstrCompactBranch() ? kInstrSize : 0;
#endif

4240 4241 4242
  BlockTrampolinePoolScope block_trampoline_pool(this);
  Label start;
  bind(&start);
4243 4244 4245 4246 4247 4248 4249 4250
  if (kArchVariant == kMips64r6 && bd == PROTECT) {
    if (cond == cc_always) {
      jialc(target, 0);
    } else {
      BRANCH_ARGS_CHECK(cond, rs, rt);
      Branch(2, NegateCondition(cond), rs, rt);
      jialc(target, 0);
    }
4251
  } else {
4252 4253 4254 4255 4256 4257 4258 4259 4260
    if (cond == cc_always) {
      jalr(target);
    } else {
      BRANCH_ARGS_CHECK(cond, rs, rt);
      Branch(2, NegateCondition(cond), rs, rt);
      jalr(target);
    }
    // Emit a nop in the branch delay slot if required.
    if (bd == PROTECT) nop();
4261 4262
  }

4263
#ifdef DEBUG
4264 4265
  DCHECK_EQ(size + CallSize(target, cond, rs, rt, bd),
            SizeOfCodeGeneratedSince(&start));
4266
#endif
4267 4268
}

4269 4270
int TurboAssembler::CallSize(Address target, RelocInfo::Mode rmode,
                             Condition cond, Register rs, const Operand& rt,
4271 4272 4273 4274 4275
                             BranchDelaySlot bd) {
  int size = CallSize(t9, cond, rs, rt, bd);
  return size + 4 * kInstrSize;
}

4276 4277
void TurboAssembler::Call(Address target, RelocInfo::Mode rmode, Condition cond,
                          Register rs, const Operand& rt, BranchDelaySlot bd) {
4278 4279 4280
  BlockTrampolinePoolScope block_trampoline_pool(this);
  Label start;
  bind(&start);
4281
  li(t9, Operand(static_cast<int64_t>(target), rmode), ADDRESS_LOAD);
4282
  Call(t9, cond, rs, rt, bd);
4283
  DCHECK_EQ(CallSize(target, rmode, cond, rs, rt, bd),
4284 4285 4286
            SizeOfCodeGeneratedSince(&start));
}

4287 4288
int TurboAssembler::CallSize(Handle<Code> code, RelocInfo::Mode rmode,
                             Condition cond, Register rs, const Operand& rt,
4289
                             BranchDelaySlot bd) {
4290
  return CallSize(code.address(), rmode, cond, rs, rt, bd);
4291 4292
}

4293 4294
void TurboAssembler::Call(Handle<Code> code, RelocInfo::Mode rmode,
                          Condition cond, Register rs, const Operand& rt,
4295 4296
                          BranchDelaySlot bd) {
  BlockTrampolinePoolScope block_trampoline_pool(this);
4297 4298
#ifdef V8_EMBEDDED_BUILTINS
  if (root_array_available_ && isolate()->ShouldLoadConstantsFromRootList()) {
4299
    IndirectLoadConstant(t9, code);
4300 4301
    Daddu(t9, t9, Operand(Code::kHeaderSize - kHeapObjectTag));
    Call(t9, cond, rs, rt, bd);
4302
    return;
4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314
  } else if (!isolate()->serializer_enabled()) {
    int builtin_index = Builtins::kNoBuiltinId;
    if (isolate()->builtins()->IsBuiltinHandle(code, &builtin_index) &&
        Builtins::IsIsolateIndependent(builtin_index)) {
      // Inline the trampoline.
      CHECK_NE(builtin_index, Builtins::kNoBuiltinId);
      EmbeddedData d = EmbeddedData::FromBlob();
      Address entry = d.InstructionStartOfBuiltin(builtin_index);
      li(t9, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
      Call(t9, cond, rs, rt, bd);
      return;
    }
4315 4316
  }
#endif  // V8_EMBEDDED_BUILTINS
4317 4318
  Label start;
  bind(&start);
4319
  DCHECK(RelocInfo::IsCodeTarget(rmode));
4320
  Call(code.address(), rmode, cond, rs, rt, bd);
4321
  DCHECK_EQ(CallSize(code, rmode, cond, rs, rt, bd),
4322 4323 4324
            SizeOfCodeGeneratedSince(&start));
}

4325
void TurboAssembler::Ret(Condition cond, Register rs, const Operand& rt,
4326 4327 4328 4329
                         BranchDelaySlot bd) {
  Jump(ra, cond, rs, rt, bd);
}

4330
void TurboAssembler::BranchLong(Label* L, BranchDelaySlot bdslot) {
4331 4332 4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345
  if (kArchVariant == kMips64r6 && bdslot == PROTECT &&
      (!L->is_bound() || is_near_r6(L))) {
    BranchShortHelperR6(0, L);
  } else {
    EmitForbiddenSlotInstruction();
    BlockTrampolinePoolScope block_trampoline_pool(this);
    {
      BlockGrowBufferScope block_buf_growth(this);
      // Buffer growth (and relocation) must be blocked for internal references
      // until associated instructions are emitted and available to be patched.
      RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE_ENCODED);
      j(L);
    }
    // Emit a nop in the branch delay slot if required.
    if (bdslot == PROTECT) nop();
4346 4347 4348
  }
}

4349
void TurboAssembler::BranchAndLinkLong(Label* L, BranchDelaySlot bdslot) {
4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360 4361 4362 4363 4364
  if (kArchVariant == kMips64r6 && bdslot == PROTECT &&
      (!L->is_bound() || is_near_r6(L))) {
    BranchAndLinkShortHelperR6(0, L);
  } else {
    EmitForbiddenSlotInstruction();
    BlockTrampolinePoolScope block_trampoline_pool(this);
    {
      BlockGrowBufferScope block_buf_growth(this);
      // Buffer growth (and relocation) must be blocked for internal references
      // until associated instructions are emitted and available to be patched.
      RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE_ENCODED);
      jal(L);
    }
    // Emit a nop in the branch delay slot if required.
    if (bdslot == PROTECT) nop();
4365 4366 4367
  }
}

4368
void TurboAssembler::DropAndRet(int drop) {
4369
  DCHECK(is_int16(drop * kPointerSize));
4370 4371 4372 4373
  Ret(USE_DELAY_SLOT);
  daddiu(sp, sp, drop * kPointerSize);
}

4374
void TurboAssembler::DropAndRet(int drop, Condition cond, Register r1,
4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387 4388 4389
                                const Operand& r2) {
  // Both Drop and Ret need to be conditional.
  Label skip;
  if (cond != cc_always) {
    Branch(&skip, NegateCondition(cond), r1, r2);
  }

  Drop(drop);
  Ret();

  if (cond != cc_always) {
    bind(&skip);
  }
}

4390
void TurboAssembler::Drop(int count, Condition cond, Register reg,
4391 4392 4393 4394 4395 4396 4397 4398 4399 4400 4401
                          const Operand& op) {
  if (count <= 0) {
    return;
  }

  Label skip;

  if (cond != al) {
     Branch(&skip, NegateCondition(cond), reg, op);
  }

4402
  Daddu(sp, sp, Operand(count * kPointerSize));
4403 4404 4405 4406 4407 4408 4409 4410 4411 4412 4413

  if (cond != al) {
    bind(&skip);
  }
}



void MacroAssembler::Swap(Register reg1,
                          Register reg2,
                          Register scratch) {
4414
  if (scratch == no_reg) {
4415 4416 4417 4418 4419 4420 4421 4422 4423 4424
    Xor(reg1, reg1, Operand(reg2));
    Xor(reg2, reg2, Operand(reg1));
    Xor(reg1, reg1, Operand(reg2));
  } else {
    mov(scratch, reg1);
    mov(reg1, reg2);
    mov(reg2, scratch);
  }
}

4425
void TurboAssembler::Call(Label* target) { BranchAndLink(target); }
4426

4427
void TurboAssembler::Push(Smi* smi) {
4428 4429 4430 4431
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  li(scratch, Operand(smi));
  push(scratch);
4432
}
4433

4434
void TurboAssembler::Push(Handle<HeapObject> handle) {
4435 4436 4437 4438
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  li(scratch, Operand(handle));
  push(scratch);
4439 4440
}

4441 4442
void MacroAssembler::MaybeDropFrames() {
  // Check whether we need to drop frames to restart a function on the stack.
4443
  li(a1, ExternalReference::debug_restart_fp_address(isolate()));
4444
  Ld(a1, MemOperand(a1));
4445
  Jump(BUILTIN_CODE(isolate(), FrameDropperTrampoline), RelocInfo::CODE_TARGET,
4446 4447
       ne, a1, Operand(zero_reg));
}
4448 4449 4450 4451

// ---------------------------------------------------------------------------
// Exception handling.

4452
void MacroAssembler::PushStackHandler() {
4453
  // Adjust this code if not the case.
4454
  STATIC_ASSERT(StackHandlerConstants::kSize == 2 * kPointerSize);
4455 4456
  STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0 * kPointerSize);

4457 4458
  Push(Smi::kZero);  // Padding.

4459
  // Link the current handler as the next handler.
4460 4461
  li(a6,
     ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate()));
4462
  Ld(a5, MemOperand(a6));
4463
  push(a5);
4464

4465
  // Set this new handler as the current one.
4466
  Sd(sp, MemOperand(a6));
4467 4468 4469
}


4470
void MacroAssembler::PopStackHandler() {
4471 4472
  STATIC_ASSERT(StackHandlerConstants::kNextOffset == 0);
  pop(a1);
4473 4474
  Daddu(sp, sp, Operand(static_cast<int64_t>(StackHandlerConstants::kSize -
                                             kPointerSize)));
4475 4476
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
4477 4478
  li(scratch,
     ExternalReference::Create(IsolateAddressId::kHandlerAddress, isolate()));
4479
  Sd(a1, MemOperand(scratch));
4480 4481
}

4482
void TurboAssembler::FPUCanonicalizeNaN(const DoubleRegister dst,
4483 4484 4485
                                        const DoubleRegister src) {
  sub_d(dst, src, kDoubleRegZero);
}
4486

4487
void TurboAssembler::MovFromFloatResult(const DoubleRegister dst) {
4488
  if (IsMipsSoftFloatABI) {
4489 4490 4491 4492 4493
    if (kArchEndian == kLittle) {
      Move(dst, v0, v1);
    } else {
      Move(dst, v1, v0);
    }
4494 4495 4496 4497 4498
  } else {
    Move(dst, f0);  // Reg f0 is o32 ABI FP return value.
  }
}

4499
void TurboAssembler::MovFromFloatParameter(const DoubleRegister dst) {
4500
  if (IsMipsSoftFloatABI) {
4501 4502 4503 4504 4505
    if (kArchEndian == kLittle) {
      Move(dst, a0, a1);
    } else {
      Move(dst, a1, a0);
    }
4506
  } else {
4507
    Move(dst, f12);  // Reg f12 is n64 ABI FP first argument value.
4508 4509 4510
  }
}

4511
void TurboAssembler::MovToFloatParameter(DoubleRegister src) {
4512 4513 4514
  if (!IsMipsSoftFloatABI) {
    Move(f12, src);
  } else {
4515 4516 4517 4518 4519
    if (kArchEndian == kLittle) {
      Move(a0, a1, src);
    } else {
      Move(a1, a0, src);
    }
4520 4521 4522
  }
}

4523
void TurboAssembler::MovToFloatResult(DoubleRegister src) {
4524 4525 4526
  if (!IsMipsSoftFloatABI) {
    Move(f0, src);
  } else {
4527 4528 4529 4530 4531
    if (kArchEndian == kLittle) {
      Move(v0, v1, src);
    } else {
      Move(v1, v0, src);
    }
4532 4533 4534
  }
}

4535
void TurboAssembler::MovToFloatParameters(DoubleRegister src1,
4536 4537
                                          DoubleRegister src2) {
  if (!IsMipsSoftFloatABI) {
4538
    const DoubleRegister fparg2 = f13;
4539 4540
    if (src2 == f12) {
      DCHECK(src1 != fparg2);
4541 4542 4543 4544 4545 4546 4547
      Move(fparg2, src2);
      Move(f12, src1);
    } else {
      Move(f12, src1);
      Move(fparg2, src2);
    }
  } else {
4548 4549 4550 4551 4552 4553 4554
    if (kArchEndian == kLittle) {
      Move(a0, a1, src1);
      Move(a2, a3, src2);
    } else {
      Move(a1, a0, src1);
      Move(a3, a2, src2);
    }
4555 4556 4557 4558 4559 4560 4561
  }
}


// -----------------------------------------------------------------------------
// JavaScript invokes.

4562
void TurboAssembler::PrepareForTailCall(const ParameterCount& callee_args_count,
4563 4564 4565 4566 4567 4568 4569 4570 4571 4572 4573 4574 4575 4576 4577 4578 4579 4580 4581 4582 4583 4584 4585 4586 4587 4588 4589 4590 4591 4592
                                        Register caller_args_count_reg,
                                        Register scratch0, Register scratch1) {
#if DEBUG
  if (callee_args_count.is_reg()) {
    DCHECK(!AreAliased(callee_args_count.reg(), caller_args_count_reg, scratch0,
                       scratch1));
  } else {
    DCHECK(!AreAliased(caller_args_count_reg, scratch0, scratch1));
  }
#endif

  // Calculate the end of destination area where we will put the arguments
  // after we drop current frame. We add kPointerSize to count the receiver
  // argument which is not included into formal parameters count.
  Register dst_reg = scratch0;
  Dlsa(dst_reg, fp, caller_args_count_reg, kPointerSizeLog2);
  Daddu(dst_reg, dst_reg,
        Operand(StandardFrameConstants::kCallerSPOffset + kPointerSize));

  Register src_reg = caller_args_count_reg;
  // Calculate the end of source area. +kPointerSize is for the receiver.
  if (callee_args_count.is_reg()) {
    Dlsa(src_reg, sp, callee_args_count.reg(), kPointerSizeLog2);
    Daddu(src_reg, src_reg, Operand(kPointerSize));
  } else {
    Daddu(src_reg, sp,
          Operand((callee_args_count.immediate() + 1) * kPointerSize));
  }

  if (FLAG_debug_code) {
4593 4594
    Check(lo, AbortReason::kStackAccessBelowStackPointer, src_reg,
          Operand(dst_reg));
4595 4596 4597 4598
  }

  // Restore caller's frame pointer and return address now as they will be
  // overwritten by the copying loop.
4599 4600
  Ld(ra, MemOperand(fp, StandardFrameConstants::kCallerPCOffset));
  Ld(fp, MemOperand(fp, StandardFrameConstants::kCallerFPOffset));
4601 4602 4603 4604 4605 4606 4607 4608 4609 4610 4611 4612

  // Now copy callee arguments to the caller frame going backwards to avoid
  // callee arguments corruption (source and destination areas could overlap).

  // Both src_reg and dst_reg are pointing to the word after the one to copy,
  // so they must be pre-decremented in the loop.
  Register tmp_reg = scratch1;
  Label loop, entry;
  Branch(&entry);
  bind(&loop);
  Dsubu(src_reg, src_reg, Operand(kPointerSize));
  Dsubu(dst_reg, dst_reg, Operand(kPointerSize));
4613 4614
  Ld(tmp_reg, MemOperand(src_reg));
  Sd(tmp_reg, MemOperand(dst_reg));
4615 4616 4617 4618 4619 4620 4621
  bind(&entry);
  Branch(&loop, ne, sp, Operand(src_reg));

  // Leave current frame.
  mov(sp, dst_reg);
}

4622
void MacroAssembler::InvokePrologue(const ParameterCount& expected,
4623
                                    const ParameterCount& actual, Label* done,
4624
                                    bool* definitely_mismatches,
4625
                                    InvokeFlag flag) {
4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638
  bool definitely_matches = false;
  *definitely_mismatches = false;
  Label regular_invoke;

  // Check whether the expected and actual arguments count match. If not,
  // setup registers according to contract with ArgumentsAdaptorTrampoline:
  //  a0: actual arguments count
  //  a1: function (passed through to callee)
  //  a2: expected arguments count

  // The code below is made a lot easier because the calling code already sets
  // up actual and expected registers according to the contract if values are
  // passed in registers.
4639 4640
  DCHECK(actual.is_immediate() || actual.reg() == a0);
  DCHECK(expected.is_immediate() || expected.reg() == a2);
4641 4642

  if (expected.is_immediate()) {
4643
    DCHECK(actual.is_immediate());
4644
    li(a0, Operand(actual.immediate()));
4645 4646 4647 4648 4649 4650 4651 4652 4653 4654 4655 4656 4657 4658 4659 4660 4661
    if (expected.immediate() == actual.immediate()) {
      definitely_matches = true;
    } else {
      const int sentinel = SharedFunctionInfo::kDontAdaptArgumentsSentinel;
      if (expected.immediate() == sentinel) {
        // Don't worry about adapting arguments for builtins that
        // don't want that done. Skip adaption code by making it look
        // like we have a match between expected and actual number of
        // arguments.
        definitely_matches = true;
      } else {
        *definitely_mismatches = true;
        li(a2, Operand(expected.immediate()));
      }
    }
  } else if (actual.is_immediate()) {
    li(a0, Operand(actual.immediate()));
4662
    Branch(&regular_invoke, eq, expected.reg(), Operand(a0));
4663 4664 4665 4666 4667
  } else {
    Branch(&regular_invoke, eq, expected.reg(), Operand(actual.reg()));
  }

  if (!definitely_matches) {
4668
    Handle<Code> adaptor = BUILTIN_CODE(isolate(), ArgumentsAdaptorTrampoline);
4669 4670 4671 4672 4673 4674 4675 4676 4677 4678 4679 4680
    if (flag == CALL_FUNCTION) {
      Call(adaptor);
      if (!*definitely_mismatches) {
        Branch(done);
      }
    } else {
      Jump(adaptor, RelocInfo::CODE_TARGET);
    }
    bind(&regular_invoke);
  }
}

4681 4682 4683
void MacroAssembler::CheckDebugHook(Register fun, Register new_target,
                                    const ParameterCount& expected,
                                    const ParameterCount& actual) {
4684
  Label skip_hook;
4685

4686
  li(t0, ExternalReference::debug_hook_on_function_call_address(isolate()));
4687
  Lb(t0, MemOperand(t0));
4688
  Branch(&skip_hook, eq, t0, Operand(zero_reg));
4689

4690
  {
4691 4692 4693 4694 4695 4696 4697 4698
    // Load receiver to pass it later to DebugOnFunctionCall hook.
    if (actual.is_reg()) {
      mov(t0, actual.reg());
    } else {
      li(t0, actual.immediate());
    }
    Dlsa(t0, sp, t0, kPointerSizeLog2);
    Ld(t0, MemOperand(t0));
4699 4700 4701 4702 4703 4704 4705 4706 4707 4708 4709 4710 4711 4712 4713
    FrameScope frame(this,
                     has_frame() ? StackFrame::NONE : StackFrame::INTERNAL);
    if (expected.is_reg()) {
      SmiTag(expected.reg());
      Push(expected.reg());
    }
    if (actual.is_reg()) {
      SmiTag(actual.reg());
      Push(actual.reg());
    }
    if (new_target.is_valid()) {
      Push(new_target);
    }
    Push(fun);
    Push(fun);
4714
    Push(t0);
4715
    CallRuntime(Runtime::kDebugOnFunctionCall);
4716 4717 4718 4719 4720 4721 4722 4723 4724 4725 4726 4727 4728
    Pop(fun);
    if (new_target.is_valid()) {
      Pop(new_target);
    }
    if (actual.is_reg()) {
      Pop(actual.reg());
      SmiUntag(actual.reg());
    }
    if (expected.is_reg()) {
      Pop(expected.reg());
      SmiUntag(expected.reg());
    }
  }
4729
  bind(&skip_hook);
4730 4731 4732 4733 4734
}

void MacroAssembler::InvokeFunctionCode(Register function, Register new_target,
                                        const ParameterCount& expected,
                                        const ParameterCount& actual,
4735
                                        InvokeFlag flag) {
4736
  // You can't call a function without a valid frame.
4737
  DCHECK(flag == JUMP_FUNCTION || has_frame());
4738 4739
  DCHECK(function == a1);
  DCHECK_IMPLIES(new_target.is_valid(), new_target == a3);
4740

4741 4742
  // On function call, call into the debugger if necessary.
  CheckDebugHook(function, new_target, expected, actual);
4743 4744

  // Clear the new.target register if not given.
4745 4746 4747
  if (!new_target.is_valid()) {
    LoadRoot(a3, Heap::kUndefinedValueRootIndex);
  }
4748

4749
  Label done;
4750
  bool definitely_mismatches = false;
4751
  InvokePrologue(expected, actual, &done, &definitely_mismatches, flag);
4752
  if (!definitely_mismatches) {
4753 4754 4755
    // We call indirectly through the code field in the function to
    // allow recompilation to take effect without changing any of the
    // call sites.
4756
    Register code = kJavaScriptCallCodeStartRegister;
4757
    Ld(code, FieldMemOperand(function, JSFunction::kCodeOffset));
4758
    if (flag == CALL_FUNCTION) {
4759
      Daddu(code, code, Operand(Code::kHeaderSize - kHeapObjectTag));
4760 4761
      Call(code);
    } else {
4762
      DCHECK(flag == JUMP_FUNCTION);
4763
      Daddu(code, code, Operand(Code::kHeaderSize - kHeapObjectTag));
4764 4765 4766 4767 4768 4769 4770 4771
      Jump(code);
    }
    // Continue here if InvokePrologue does handle the invocation due to
    // mismatched parameter counts.
    bind(&done);
  }
}

4772
void MacroAssembler::InvokeFunction(Register function, Register new_target,
4773
                                    const ParameterCount& actual,
4774
                                    InvokeFlag flag) {
4775
  // You can't call a function without a valid frame.
4776
  DCHECK(flag == JUMP_FUNCTION || has_frame());
4777 4778

  // Contract with called JS functions requires that function is passed in a1.
4779
  DCHECK(function == a1);
4780
  Register expected_reg = a2;
4781
  Register temp_reg = t0;
4782 4783
  Ld(temp_reg, FieldMemOperand(a1, JSFunction::kSharedFunctionInfoOffset));
  Ld(cp, FieldMemOperand(a1, JSFunction::kContextOffset));
4784 4785 4786 4787
  // The argument count is stored as uint16_t
  Lhu(expected_reg,
      FieldMemOperand(temp_reg,
                      SharedFunctionInfo::kFormalParameterCountOffset));
4788
  ParameterCount expected(expected_reg);
4789
  InvokeFunctionCode(a1, new_target, expected, actual, flag);
4790 4791 4792 4793 4794
}

void MacroAssembler::InvokeFunction(Register function,
                                    const ParameterCount& expected,
                                    const ParameterCount& actual,
4795
                                    InvokeFlag flag) {
4796
  // You can't call a function without a valid frame.
4797
  DCHECK(flag == JUMP_FUNCTION || has_frame());
4798 4799

  // Contract with called JS functions requires that function is passed in a1.
4800
  DCHECK(function == a1);
4801 4802

  // Get the function and setup the context.
4803
  Ld(cp, FieldMemOperand(a1, JSFunction::kContextOffset));
4804

4805
  InvokeFunctionCode(a1, no_reg, expected, actual, flag);
4806 4807 4808 4809 4810 4811 4812 4813 4814
}


// ---------------------------------------------------------------------------
// Support functions.

void MacroAssembler::GetObjectType(Register object,
                                   Register map,
                                   Register type_reg) {
4815
  Ld(map, FieldMemOperand(object, HeapObject::kMapOffset));
4816
  Lhu(type_reg, FieldMemOperand(map, Map::kInstanceTypeOffset));
4817 4818 4819 4820 4821 4822 4823 4824 4825 4826 4827
}


// -----------------------------------------------------------------------------
// Runtime calls.

void MacroAssembler::CallStub(CodeStub* stub,
                              Condition cond,
                              Register r1,
                              const Operand& r2,
                              BranchDelaySlot bd) {
4828
  DCHECK(AllowThisStubCall(stub));  // Stub calls are not allowed in some stubs.
4829
  Call(stub->GetCode(), RelocInfo::CODE_TARGET, cond, r1, r2, bd);
4830 4831
}

4832
void TurboAssembler::CallStubDelayed(CodeStub* stub, Condition cond,
4833 4834 4835 4836 4837 4838
                                     Register r1, const Operand& r2,
                                     BranchDelaySlot bd) {
  DCHECK(AllowThisStubCall(stub));  // Stub calls are not allowed in some stubs.

  BlockTrampolinePoolScope block_trampoline_pool(this);

4839 4840 4841 4842
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  li(scratch, Operand::EmbeddedCode(stub));
  Call(scratch);
4843
}
4844 4845 4846 4847 4848 4849 4850 4851 4852

void MacroAssembler::TailCallStub(CodeStub* stub,
                                  Condition cond,
                                  Register r1,
                                  const Operand& r2,
                                  BranchDelaySlot bd) {
  Jump(stub->GetCode(), RelocInfo::CODE_TARGET, cond, r1, r2, bd);
}

4853 4854
bool TurboAssembler::AllowThisStubCall(CodeStub* stub) {
  return has_frame() || !stub->SometimesSetsUpAFrame();
4855 4856
}

4857 4858 4859 4860 4861 4862 4863
void TurboAssembler::DaddOverflow(Register dst, Register left,
                                  const Operand& right, Register overflow) {
  Register right_reg = no_reg;
  Register scratch = t8;
  if (!right.is_reg()) {
    li(at, Operand(right));
    right_reg = at;
4864
  } else {
4865
    right_reg = right.rm();
4866
  }
4867

4868 4869 4870 4871 4872 4873 4874 4875 4876 4877
  DCHECK(left != scratch && right_reg != scratch && dst != scratch &&
         overflow != scratch);
  DCHECK(overflow != left && overflow != right_reg);

  if (dst == left || dst == right_reg) {
    daddu(scratch, left, right_reg);
    xor_(overflow, scratch, left);
    xor_(at, scratch, right_reg);
    and_(overflow, overflow, at);
    mov(dst, scratch);
4878
  } else {
4879 4880 4881 4882
    daddu(dst, left, right_reg);
    xor_(overflow, dst, left);
    xor_(at, dst, right_reg);
    and_(overflow, overflow, at);
4883 4884 4885
  }
}

4886 4887 4888 4889 4890 4891 4892
void TurboAssembler::DsubOverflow(Register dst, Register left,
                                  const Operand& right, Register overflow) {
  Register right_reg = no_reg;
  Register scratch = t8;
  if (!right.is_reg()) {
    li(at, Operand(right));
    right_reg = at;
4893
  } else {
4894
    right_reg = right.rm();
4895 4896
  }

4897 4898 4899 4900 4901 4902 4903 4904 4905 4906
  DCHECK(left != scratch && right_reg != scratch && dst != scratch &&
         overflow != scratch);
  DCHECK(overflow != left && overflow != right_reg);

  if (dst == left || dst == right_reg) {
    dsubu(scratch, left, right_reg);
    xor_(overflow, left, scratch);
    xor_(at, left, right_reg);
    and_(overflow, overflow, at);
    mov(dst, scratch);
4907
  } else {
4908 4909 4910 4911
    dsubu(dst, left, right_reg);
    xor_(overflow, left, dst);
    xor_(at, left, right_reg);
    and_(overflow, overflow, at);
4912 4913 4914
  }
}

4915 4916 4917 4918 4919 4920 4921
void TurboAssembler::MulOverflow(Register dst, Register left,
                                 const Operand& right, Register overflow) {
  Register right_reg = no_reg;
  Register scratch = t8;
  if (!right.is_reg()) {
    li(at, Operand(right));
    right_reg = at;
4922
  } else {
4923
    right_reg = right.rm();
4924 4925
  }

4926 4927 4928
  DCHECK(left != scratch && right_reg != scratch && dst != scratch &&
         overflow != scratch);
  DCHECK(overflow != left && overflow != right_reg);
4929

4930 4931 4932
  if (dst == left || dst == right_reg) {
    Mul(scratch, left, right_reg);
    Mulh(overflow, left, right_reg);
4933 4934
    mov(dst, scratch);
  } else {
4935 4936
    Mul(dst, left, right_reg);
    Mulh(overflow, left, right_reg);
4937 4938 4939
  }

  dsra32(scratch, dst, 0);
4940
  xor_(overflow, overflow, scratch);
4941
}
4942

4943 4944
void TurboAssembler::CallRuntimeWithCEntry(Runtime::FunctionId fid,
                                           Register centry) {
4945 4946 4947 4948 4949 4950
  const Runtime::Function* f = Runtime::FunctionForId(fid);
  // TODO(1236192): Most runtime routines don't need the number of
  // arguments passed in because it is constant. At some point we
  // should remove this need and make the runtime routine entry code
  // smarter.
  PrepareCEntryArgs(f->nargs);
4951
  PrepareCEntryFunction(ExternalReference::Create(f));
4952 4953 4954
  DCHECK(!AreAliased(centry, a0, a1));
  Daddu(centry, centry, Operand(Code::kHeaderSize - kHeapObjectTag));
  Call(centry);
4955 4956
}

4957
void MacroAssembler::CallRuntime(const Runtime::Function* f, int num_arguments,
4958
                                 SaveFPRegsMode save_doubles) {
4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970
  // All parameters are on the stack. v0 has the return value after call.

  // If the expected number of arguments of the runtime function is
  // constant, we check that the actual number of arguments match the
  // expectation.
  CHECK(f->nargs < 0 || f->nargs == num_arguments);

  // TODO(1236192): Most runtime routines don't need the number of
  // arguments passed in because it is constant. At some point we
  // should remove this need and make the runtime routine entry code
  // smarter.
  PrepareCEntryArgs(num_arguments);
4971
  PrepareCEntryFunction(ExternalReference::Create(f));
4972 4973 4974
  Handle<Code> code =
      CodeFactory::CEntry(isolate(), f->result_size, save_doubles);
  Call(code, RelocInfo::CODE_TARGET);
4975 4976
}

4977 4978 4979 4980 4981 4982
void MacroAssembler::TailCallRuntime(Runtime::FunctionId fid) {
  const Runtime::Function* function = Runtime::FunctionForId(fid);
  DCHECK_EQ(1, function->result_size);
  if (function->nargs >= 0) {
    PrepareCEntryArgs(function->nargs);
  }
4983
  JumpToExternalReference(ExternalReference::Create(fid));
4984 4985 4986
}

void MacroAssembler::JumpToExternalReference(const ExternalReference& builtin,
4987 4988
                                             BranchDelaySlot bd,
                                             bool builtin_exit_frame) {
4989
  PrepareCEntryFunction(builtin);
4990 4991 4992
  Handle<Code> code = CodeFactory::CEntry(isolate(), 1, kDontSaveFPRegs,
                                          kArgvOnStack, builtin_exit_frame);
  Jump(code, RelocInfo::CODE_TARGET, al, zero_reg, Operand(zero_reg), bd);
4993 4994
}

4995
void MacroAssembler::JumpToInstructionStream(Address entry) {
4996
  li(kOffHeapTrampolineRegister, Operand(entry, RelocInfo::OFF_HEAP_TARGET));
4997 4998 4999
  Jump(kOffHeapTrampolineRegister);
}

5000 5001 5002 5003 5004 5005 5006
void MacroAssembler::LoadWeakValue(Register out, Register in,
                                   Label* target_if_cleared) {
  Branch(target_if_cleared, eq, in, Operand(kClearedWeakHeapObject));

  And(out, in, Operand(~kWeakHeapObjectMask));
}

5007 5008
void MacroAssembler::IncrementCounter(StatsCounter* counter, int value,
                                      Register scratch1, Register scratch2) {
5009
  DCHECK_GT(value, 0);
5010
  if (FLAG_native_code_counters && counter->Enabled()) {
5011
    li(scratch2, ExternalReference::Create(counter));
5012
    Lw(scratch1, MemOperand(scratch2));
5013
    Addu(scratch1, scratch1, Operand(value));
5014
    Sw(scratch1, MemOperand(scratch2));
5015 5016 5017 5018 5019 5020
  }
}


void MacroAssembler::DecrementCounter(StatsCounter* counter, int value,
                                      Register scratch1, Register scratch2) {
5021
  DCHECK_GT(value, 0);
5022
  if (FLAG_native_code_counters && counter->Enabled()) {
5023
    li(scratch2, ExternalReference::Create(counter));
5024
    Lw(scratch1, MemOperand(scratch2));
5025
    Subu(scratch1, scratch1, Operand(value));
5026
    Sw(scratch1, MemOperand(scratch2));
5027 5028 5029 5030 5031 5032 5033
  }
}


// -----------------------------------------------------------------------------
// Debugging.

5034
void TurboAssembler::Assert(Condition cc, AbortReason reason, Register rs,
5035
                            Operand rt) {
5036 5037 5038 5039
  if (emit_debug_code())
    Check(cc, reason, rs, rt);
}

5040
void TurboAssembler::Check(Condition cc, AbortReason reason, Register rs,
5041
                           Operand rt) {
5042 5043 5044 5045 5046 5047 5048
  Label L;
  Branch(&L, cc, rs, rt);
  Abort(reason);
  // Will not return here.
  bind(&L);
}

5049
void TurboAssembler::Abort(AbortReason reason) {
5050 5051
  Label abort_start;
  bind(&abort_start);
5052
  const char* msg = GetAbortReason(reason);
5053 5054 5055 5056
#ifdef DEBUG
  RecordComment("Abort message: ");
  RecordComment(msg);
#endif
5057

5058 5059
  // Avoid emitting call to builtin if requested.
  if (trap_on_abort()) {
5060 5061 5062 5063
    stop(msg);
    return;
  }

5064 5065
  Move(a0, Smi::FromInt(static_cast<int>(reason)));

5066
  // Disable stub call restrictions to always allow calls to abort.
5067
  if (!has_frame()) {
5068 5069 5070
    // We don't actually want to generate a pile of code for this, so just
    // claim there is a stack frame, without generating one.
    FrameScope scope(this, StackFrame::NONE);
5071
    Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
5072
  } else {
5073
    Call(BUILTIN_CODE(isolate(), Abort), RelocInfo::CODE_TARGET);
5074 5075 5076 5077 5078 5079 5080 5081 5082 5083
  }
  // Will not return here.
  if (is_trampoline_pool_blocked()) {
    // If the calling code cares about the exact number of
    // instructions generated, we insert padding here to keep the size
    // of the Abort macro constant.
    // Currently in debug mode with debug_code enabled the number of
    // generated instructions is 10, so we use this as a maximum value.
    static const int kExpectedAbortInstructions = 10;
    int abort_instructions = InstructionsGeneratedSince(&abort_start);
5084
    DCHECK_LE(abort_instructions, kExpectedAbortInstructions);
5085 5086 5087 5088 5089 5090
    while (abort_instructions++ < kExpectedAbortInstructions) {
      nop();
    }
  }
}

5091
void MacroAssembler::LoadNativeContextSlot(int index, Register dst) {
5092 5093
  Ld(dst, NativeContextMemOperand());
  Ld(dst, ContextMemOperand(dst, index));
5094 5095 5096
}


5097
void TurboAssembler::StubPrologue(StackFrame::Type type) {
5098 5099 5100 5101
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  li(scratch, Operand(StackFrame::TypeToMarker(type)));
  PushCommonFrame(scratch);
5102 5103
}

5104
void TurboAssembler::Prologue() { PushStandardFrame(a1); }
5105

5106
void TurboAssembler::EnterFrame(StackFrame::Type type) {
5107 5108
  int stack_offset = -3 * kPointerSize;
  const int fp_offset = 1 * kPointerSize;
5109 5110
  daddiu(sp, sp, stack_offset);
  stack_offset = -stack_offset - kPointerSize;
5111
  Sd(ra, MemOperand(sp, stack_offset));
5112
  stack_offset -= kPointerSize;
5113
  Sd(fp, MemOperand(sp, stack_offset));
5114
  stack_offset -= kPointerSize;
5115
  li(t9, Operand(StackFrame::TypeToMarker(type)));
5116
  Sd(t9, MemOperand(sp, stack_offset));
5117
  // Adjust FP to point to saved FP.
5118
  DCHECK_EQ(stack_offset, 0);
5119
  Daddu(fp, sp, Operand(fp_offset));
5120 5121
}

5122
void TurboAssembler::LeaveFrame(StackFrame::Type type) {
5123
  daddiu(sp, fp, 2 * kPointerSize);
5124 5125
  Ld(ra, MemOperand(fp, 1 * kPointerSize));
  Ld(fp, MemOperand(fp, 0 * kPointerSize));
5126 5127
}

5128 5129 5130 5131 5132 5133 5134 5135 5136 5137 5138 5139 5140
void MacroAssembler::EnterBuiltinFrame(Register context, Register target,
                                       Register argc) {
  Push(ra, fp);
  Move(fp, sp);
  Push(context, target, argc);
}

void MacroAssembler::LeaveBuiltinFrame(Register context, Register target,
                                       Register argc) {
  Pop(context, target, argc);
  Pop(ra, fp);
}

5141 5142 5143 5144 5145
void MacroAssembler::EnterExitFrame(bool save_doubles, int stack_space,
                                    StackFrame::Type frame_type) {
  DCHECK(frame_type == StackFrame::EXIT ||
         frame_type == StackFrame::BUILTIN_EXIT);

5146 5147 5148 5149 5150 5151 5152 5153 5154
  // Set up the frame structure on the stack.
  STATIC_ASSERT(2 * kPointerSize == ExitFrameConstants::kCallerSPDisplacement);
  STATIC_ASSERT(1 * kPointerSize == ExitFrameConstants::kCallerPCOffset);
  STATIC_ASSERT(0 * kPointerSize == ExitFrameConstants::kCallerFPOffset);

  // This is how the stack will look:
  // fp + 2 (==kCallerSPDisplacement) - old stack's end
  // [fp + 1 (==kCallerPCOffset)] - saved old ra
  // [fp + 0 (==kCallerFPOffset)] - saved old fp
5155 5156 5157
  // [fp - 1 StackFrame::EXIT Smi
  // [fp - 2 (==kSPOffset)] - sp of the called function
  // [fp - 3 (==kCodeOffset)] - CodeObject
5158 5159 5160
  // fp - (2 + stack_space + alignment) == sp == [fp - kSPOffset] - top of the
  //   new stack (will contain saved ra)

5161 5162
  // Save registers and reserve room for saved entry sp and code object.
  daddiu(sp, sp, -2 * kPointerSize - ExitFrameConstants::kFixedFrameSizeFromFp);
5163 5164
  Sd(ra, MemOperand(sp, 4 * kPointerSize));
  Sd(fp, MemOperand(sp, 3 * kPointerSize));
5165 5166 5167 5168 5169 5170
  {
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
    li(scratch, Operand(StackFrame::TypeToMarker(frame_type)));
    Sd(scratch, MemOperand(sp, 2 * kPointerSize));
  }
5171 5172
  // Set up new frame pointer.
  daddiu(fp, sp, ExitFrameConstants::kFixedFrameSizeFromFp);
5173 5174

  if (emit_debug_code()) {
5175
    Sd(zero_reg, MemOperand(fp, ExitFrameConstants::kSPOffset));
5176 5177 5178
  }

  // Accessed from ExitFrame::code_slot.
5179
  li(t8, CodeObject(), CONSTANT_SIZE);
5180
  Sd(t8, MemOperand(fp, ExitFrameConstants::kCodeOffset));
5181 5182

  // Save the frame pointer and the context in top.
5183 5184
  li(t8,
     ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate()));
5185
  Sd(fp, MemOperand(t8));
5186 5187
  li(t8,
     ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
5188
  Sd(cp, MemOperand(t8));
5189 5190 5191 5192

  const int frame_alignment = MacroAssembler::ActivationFrameAlignment();
  if (save_doubles) {
    // The stack is already aligned to 0 modulo 8 for stores with sdc1.
5193
    int kNumOfSavedRegisters = FPURegister::kNumRegisters / 2;
5194
    int space = kNumOfSavedRegisters * kDoubleSize;
5195 5196 5197 5198
    Dsubu(sp, sp, Operand(space));
    // Remember: we only need to save every 2nd double FPU value.
    for (int i = 0; i < kNumOfSavedRegisters; i++) {
      FPURegister reg = FPURegister::from_code(2 * i);
5199
      Sdc1(reg, MemOperand(sp, i * kDoubleSize));
5200 5201 5202 5203 5204 5205
    }
  }

  // Reserve place for the return address, stack space and an optional slot
  // (used by the DirectCEntryStub to hold the return value if a struct is
  // returned) and align the frame preparing for calling the runtime function.
5206
  DCHECK_GE(stack_space, 0);
5207 5208
  Dsubu(sp, sp, Operand((stack_space + 2) * kPointerSize));
  if (frame_alignment > 0) {
5209
    DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
5210 5211 5212 5213 5214
    And(sp, sp, Operand(-frame_alignment));  // Align stack.
  }

  // Set the exit frame sp value to point just before the return address
  // location.
5215 5216 5217 5218
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  daddiu(scratch, sp, kPointerSize);
  Sd(scratch, MemOperand(fp, ExitFrameConstants::kSPOffset));
5219 5220
}

5221
void MacroAssembler::LeaveExitFrame(bool save_doubles, Register argument_count,
5222
                                    bool do_return,
5223
                                    bool argument_count_is_length) {
5224 5225 5226
  // Optionally restore all double registers.
  if (save_doubles) {
    // Remember: we only need to restore every 2nd double FPU value.
5227
    int kNumOfSavedRegisters = FPURegister::kNumRegisters / 2;
5228 5229
    Dsubu(t8, fp, Operand(ExitFrameConstants::kFixedFrameSizeFromFp +
                          kNumOfSavedRegisters * kDoubleSize));
5230 5231
    for (int i = 0; i < kNumOfSavedRegisters; i++) {
      FPURegister reg = FPURegister::from_code(2 * i);
5232
      Ldc1(reg, MemOperand(t8, i * kDoubleSize));
5233 5234 5235 5236
    }
  }

  // Clear top frame.
5237 5238
  li(t8,
     ExternalReference::Create(IsolateAddressId::kCEntryFPAddress, isolate()));
5239
  Sd(zero_reg, MemOperand(t8));
5240 5241

  // Restore current context from top and clear it in debug mode.
5242 5243
  li(t8,
     ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
5244 5245
  Ld(cp, MemOperand(t8));

5246
#ifdef DEBUG
5247 5248
  li(t8,
     ExternalReference::Create(IsolateAddressId::kContextAddress, isolate()));
5249
  Sd(a3, MemOperand(t8));
5250 5251 5252 5253
#endif

  // Pop the arguments, restore registers, and return.
  mov(sp, fp);  // Respect ABI stack constraint.
5254 5255
  Ld(fp, MemOperand(sp, ExitFrameConstants::kCallerFPOffset));
  Ld(ra, MemOperand(sp, ExitFrameConstants::kCallerPCOffset));
5256 5257

  if (argument_count.is_valid()) {
5258 5259 5260
    if (argument_count_is_length) {
      daddu(sp, sp, argument_count);
    } else {
5261
      Dlsa(sp, sp, argument_count, kPointerSizeLog2, t8);
5262
    }
5263 5264 5265 5266 5267 5268 5269 5270 5271
  }

  if (do_return) {
    Ret(USE_DELAY_SLOT);
    // If returning, the instruction in the delay slot will be the addiu below.
  }
  daddiu(sp, sp, 2 * kPointerSize);
}

5272
int TurboAssembler::ActivationFrameAlignment() {
5273 5274 5275 5276 5277 5278 5279 5280 5281 5282 5283 5284 5285 5286 5287 5288 5289 5290 5291 5292 5293 5294 5295
#if V8_HOST_ARCH_MIPS || V8_HOST_ARCH_MIPS64
  // Running on the real platform. Use the alignment as mandated by the local
  // environment.
  // Note: This will break if we ever start generating snapshots on one Mips
  // platform for another Mips platform with a different alignment.
  return base::OS::ActivationFrameAlignment();
#else  // V8_HOST_ARCH_MIPS
  // If we are using the simulator then we should always align to the expected
  // alignment. As the simulator is used to generate snapshots we do not know
  // if the target platform will need alignment, so this is controlled from a
  // flag.
  return FLAG_sim_stack_alignment;
#endif  // V8_HOST_ARCH_MIPS
}


void MacroAssembler::AssertStackIsAligned() {
  if (emit_debug_code()) {
      const int frame_alignment = ActivationFrameAlignment();
      const int frame_alignment_mask = frame_alignment - 1;

      if (frame_alignment > kPointerSize) {
        Label alignment_as_expected;
5296
        DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
5297 5298 5299 5300 5301 5302
        {
          UseScratchRegisterScope temps(this);
          Register scratch = temps.Acquire();
          andi(scratch, sp, frame_alignment_mask);
          Branch(&alignment_as_expected, eq, scratch, Operand(zero_reg));
        }
5303 5304 5305 5306 5307 5308 5309
        // Don't use Check here, as it will call Runtime_Abort re-entering here.
        stop("Unexpected stack alignment");
        bind(&alignment_as_expected);
      }
    }
}

5310
void TurboAssembler::SmiUntag(Register dst, const MemOperand& src) {
5311
  if (SmiValuesAre32Bits()) {
5312
    Lw(dst, MemOperand(src.rm(), SmiWordOffset(src.offset())));
5313
  } else {
5314
    DCHECK(SmiValuesAre31Bits());
5315
    Lw(dst, src);
5316 5317 5318 5319 5320 5321 5322
    SmiUntag(dst);
  }
}

void MacroAssembler::UntagAndJumpIfSmi(Register dst,
                                       Register src,
                                       Label* smi_case) {
5323
  // DCHECK(dst!=src);
5324 5325 5326
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  JumpIfSmi(src, smi_case, scratch, USE_DELAY_SLOT);
5327 5328 5329
  SmiUntag(dst, src);
}

5330 5331
void TurboAssembler::JumpIfSmi(Register value, Label* smi_label,
                               Register scratch, BranchDelaySlot bd) {
5332
  DCHECK_EQ(0, kSmiTag);
5333 5334 5335 5336 5337 5338 5339 5340
  andi(scratch, value, kSmiTagMask);
  Branch(bd, smi_label, eq, scratch, Operand(zero_reg));
}

void MacroAssembler::JumpIfNotSmi(Register value,
                                  Label* not_smi_label,
                                  Register scratch,
                                  BranchDelaySlot bd) {
5341
  DCHECK_EQ(0, kSmiTag);
5342 5343 5344 5345 5346 5347 5348 5349 5350 5351 5352
  andi(scratch, value, kSmiTagMask);
  Branch(bd, not_smi_label, ne, scratch, Operand(zero_reg));
}


void MacroAssembler::JumpIfEitherSmi(Register reg1,
                                     Register reg2,
                                     Label* on_either_smi) {
  STATIC_ASSERT(kSmiTag == 0);
  // TODO(plind): Find some better to fix this assert issue.
#if defined(__APPLE__)
5353
  DCHECK_EQ(1, kSmiTagMask);
5354
#else
5355
  DCHECK_EQ((int64_t)1, kSmiTagMask);
5356 5357
#endif
  // Both Smi tags must be 1 (not Smi).
5358 5359 5360 5361
  UseScratchRegisterScope temps(this);
  Register scratch = temps.Acquire();
  and_(scratch, reg1, reg2);
  JumpIfSmi(scratch, on_either_smi);
5362 5363 5364 5365 5366
}

void MacroAssembler::AssertNotSmi(Register object) {
  if (emit_debug_code()) {
    STATIC_ASSERT(kSmiTag == 0);
5367 5368 5369
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
    andi(scratch, object, kSmiTagMask);
5370
    Check(ne, AbortReason::kOperandIsASmi, scratch, Operand(zero_reg));
5371 5372 5373 5374 5375 5376 5377
  }
}


void MacroAssembler::AssertSmi(Register object) {
  if (emit_debug_code()) {
    STATIC_ASSERT(kSmiTag == 0);
5378 5379 5380
    UseScratchRegisterScope temps(this);
    Register scratch = temps.Acquire();
    andi(scratch, object, kSmiTagMask);
5381
    Check(eq, AbortReason::kOperandIsASmi, scratch, Operand(zero_reg));
5382 5383 5384
  }
}

5385 5386 5387 5388
void MacroAssembler::AssertFixedArray(Register object) {
  if (emit_debug_code()) {
    STATIC_ASSERT(kSmiTag == 0);
    SmiTst(object, t8);
5389 5390
    Check(ne, AbortReason::kOperandIsASmiAndNotAFixedArray, t8,
          Operand(zero_reg));
5391
    GetObjectType(object, t8, t8);
5392 5393
    Check(eq, AbortReason::kOperandIsNotAFixedArray, t8,
          Operand(FIXED_ARRAY_TYPE));
5394 5395
  }
}
5396

5397 5398 5399 5400 5401 5402 5403 5404 5405 5406 5407 5408 5409 5410
void MacroAssembler::AssertConstructor(Register object) {
  if (emit_debug_code()) {
    STATIC_ASSERT(kSmiTag == 0);
    SmiTst(object, t8);
    Check(ne, AbortReason::kOperandIsASmiAndNotAConstructor, t8,
          Operand(zero_reg));

    ld(t8, FieldMemOperand(object, HeapObject::kMapOffset));
    Lbu(t8, FieldMemOperand(t8, Map::kBitFieldOffset));
    And(t8, t8, Operand(Map::IsConstructorBit::kMask));
    Check(ne, AbortReason::kOperandIsNotAConstructor, t8, Operand(zero_reg));
  }
}

5411 5412 5413
void MacroAssembler::AssertFunction(Register object) {
  if (emit_debug_code()) {
    STATIC_ASSERT(kSmiTag == 0);
5414
    SmiTst(object, t8);
5415 5416
    Check(ne, AbortReason::kOperandIsASmiAndNotAFunction, t8,
          Operand(zero_reg));
5417
    GetObjectType(object, t8, t8);
5418 5419
    Check(eq, AbortReason::kOperandIsNotAFunction, t8,
          Operand(JS_FUNCTION_TYPE));
5420 5421 5422 5423
  }
}


5424 5425 5426 5427
void MacroAssembler::AssertBoundFunction(Register object) {
  if (emit_debug_code()) {
    STATIC_ASSERT(kSmiTag == 0);
    SmiTst(object, t8);
5428 5429
    Check(ne, AbortReason::kOperandIsASmiAndNotABoundFunction, t8,
          Operand(zero_reg));
5430
    GetObjectType(object, t8, t8);
5431 5432
    Check(eq, AbortReason::kOperandIsNotABoundFunction, t8,
          Operand(JS_BOUND_FUNCTION_TYPE));
5433 5434 5435
  }
}

5436
void MacroAssembler::AssertGeneratorObject(Register object) {
5437 5438 5439
  if (!emit_debug_code()) return;
  STATIC_ASSERT(kSmiTag == 0);
  SmiTst(object, t8);
5440 5441
  Check(ne, AbortReason::kOperandIsASmiAndNotAGeneratorObject, t8,
        Operand(zero_reg));
5442

5443 5444
  GetObjectType(object, t8, t8);

5445
  Label done;
5446 5447 5448 5449 5450 5451 5452

  // Check if JSGeneratorObject
  Branch(&done, eq, t8, Operand(JS_GENERATOR_OBJECT_TYPE));

  // Check if JSAsyncGeneratorObject
  Branch(&done, eq, t8, Operand(JS_ASYNC_GENERATOR_OBJECT_TYPE));

5453
  Abort(AbortReason::kOperandIsNotAGeneratorObject);
5454 5455

  bind(&done);
5456
}
5457

5458 5459 5460 5461 5462 5463 5464
void MacroAssembler::AssertUndefinedOrAllocationSite(Register object,
                                                     Register scratch) {
  if (emit_debug_code()) {
    Label done_checking;
    AssertNotSmi(object);
    LoadRoot(scratch, Heap::kUndefinedValueRootIndex);
    Branch(&done_checking, eq, object, Operand(scratch));
5465
    Ld(t8, FieldMemOperand(object, HeapObject::kMapOffset));
5466
    LoadRoot(scratch, Heap::kAllocationSiteMapRootIndex);
5467
    Assert(eq, AbortReason::kExpectedUndefinedOrCell, t8, Operand(scratch));
5468 5469 5470 5471 5472
    bind(&done_checking);
  }
}


5473
void TurboAssembler::Float32Max(FPURegister dst, FPURegister src1,
5474
                                FPURegister src2, Label* out_of_line) {
5475
  if (src1 == src2) {
5476 5477 5478
    Move_s(dst, src1);
    return;
  }
5479 5480

  // Check if one of operands is NaN.
5481 5482
  CompareIsNanF32(src1, src2);
  BranchTrueF(out_of_line);
5483 5484 5485 5486 5487 5488

  if (kArchVariant >= kMips64r6) {
    max_s(dst, src1, src2);
  } else {
    Label return_left, return_right, done;

5489 5490 5491 5492
    CompareF32(OLT, src1, src2);
    BranchTrueShortF(&return_right);
    CompareF32(OLT, src2, src1);
    BranchTrueShortF(&return_left);
5493 5494 5495 5496

    // Operands are equal, but check for +/-0.
    mfc1(t8, src1);
    dsll32(t8, t8, 0);
5497 5498
    Branch(&return_left, eq, t8, Operand(zero_reg));
    Branch(&return_right);
5499 5500

    bind(&return_right);
5501
    if (src2 != dst) {
5502 5503
      Move_s(dst, src2);
    }
5504
    Branch(&done);
5505 5506

    bind(&return_left);
5507
    if (src1 != dst) {
5508 5509 5510 5511 5512 5513 5514
      Move_s(dst, src1);
    }

    bind(&done);
  }
}

5515
void TurboAssembler::Float32MaxOutOfLine(FPURegister dst, FPURegister src1,
5516 5517 5518 5519
                                         FPURegister src2) {
  add_s(dst, src1, src2);
}

5520
void TurboAssembler::Float32Min(FPURegister dst, FPURegister src1,
5521
                                FPURegister src2, Label* out_of_line) {
5522
  if (src1 == src2) {
5523 5524 5525
    Move_s(dst, src1);
    return;
  }
5526 5527

  // Check if one of operands is NaN.
5528 5529
  CompareIsNanF32(src1, src2);
  BranchTrueF(out_of_line);
5530 5531 5532 5533 5534 5535

  if (kArchVariant >= kMips64r6) {
    min_s(dst, src1, src2);
  } else {
    Label return_left, return_right, done;

5536 5537 5538 5539
    CompareF32(OLT, src1, src2);
    BranchTrueShortF(&return_left);
    CompareF32(OLT, src2, src1);
    BranchTrueShortF(&return_right);
5540 5541 5542 5543

    // Left equals right => check for -0.
    mfc1(t8, src1);
    dsll32(t8, t8, 0);
5544 5545
    Branch(&return_right, eq, t8, Operand(zero_reg));
    Branch(&return_left);
5546 5547

    bind(&return_right);
5548
    if (src2 != dst) {
5549 5550
      Move_s(dst, src2);
    }
5551
    Branch(&done);
5552 5553

    bind(&return_left);
5554
    if (src1 != dst) {
5555 5556 5557 5558 5559 5560 5561
      Move_s(dst, src1);
    }

    bind(&done);
  }
}

5562
void TurboAssembler::Float32MinOutOfLine(FPURegister dst, FPURegister src1,
5563 5564 5565 5566
                                         FPURegister src2) {
  add_s(dst, src1, src2);
}

5567
void TurboAssembler::Float64Max(FPURegister dst, FPURegister src1,
5568
                                FPURegister src2, Label* out_of_line) {
5569
  if (src1 == src2) {
5570 5571 5572
    Move_d(dst, src1);
    return;
  }
5573 5574

  // Check if one of operands is NaN.
5575 5576
  CompareIsNanF64(src1, src2);
  BranchTrueF(out_of_line);
5577 5578 5579 5580 5581 5582

  if (kArchVariant >= kMips64r6) {
    max_d(dst, src1, src2);
  } else {
    Label return_left, return_right, done;

5583 5584 5585 5586
    CompareF64(OLT, src1, src2);
    BranchTrueShortF(&return_right);
    CompareF64(OLT, src2, src1);
    BranchTrueShortF(&return_left);
5587 5588 5589

    // Left equals right => check for -0.
    dmfc1(t8, src1);
5590 5591
    Branch(&return_left, eq, t8, Operand(zero_reg));
    Branch(&return_right);
5592 5593

    bind(&return_right);
5594
    if (src2 != dst) {
5595 5596
      Move_d(dst, src2);
    }
5597
    Branch(&done);
5598 5599

    bind(&return_left);
5600
    if (src1 != dst) {
5601 5602 5603 5604 5605 5606 5607
      Move_d(dst, src1);
    }

    bind(&done);
  }
}

5608
void TurboAssembler::Float64MaxOutOfLine(FPURegister dst, FPURegister src1,
5609 5610 5611 5612
                                         FPURegister src2) {
  add_d(dst, src1, src2);
}

5613
void TurboAssembler::Float64Min(FPURegister dst, FPURegister src1,
5614
                                FPURegister src2, Label* out_of_line) {
5615
  if (src1 == src2) {
5616 5617 5618
    Move_d(dst, src1);
    return;
  }
5619 5620

  // Check if one of operands is NaN.
5621 5622
  CompareIsNanF64(src1, src2);
  BranchTrueF(out_of_line);
5623 5624 5625 5626 5627 5628

  if (kArchVariant >= kMips64r6) {
    min_d(dst, src1, src2);
  } else {
    Label return_left, return_right, done;

5629 5630 5631 5632
    CompareF64(OLT, src1, src2);
    BranchTrueShortF(&return_left);
    CompareF64(OLT, src2, src1);
    BranchTrueShortF(&return_right);
5633 5634 5635

    // Left equals right => check for -0.
    dmfc1(t8, src1);
5636 5637
    Branch(&return_right, eq, t8, Operand(zero_reg));
    Branch(&return_left);
5638 5639

    bind(&return_right);
5640
    if (src2 != dst) {
5641 5642
      Move_d(dst, src2);
    }
5643
    Branch(&done);
5644 5645

    bind(&return_left);
5646
    if (src1 != dst) {
5647 5648 5649 5650 5651 5652 5653
      Move_d(dst, src1);
    }

    bind(&done);
  }
}

5654
void TurboAssembler::Float64MinOutOfLine(FPURegister dst, FPURegister src1,
5655 5656 5657
                                         FPURegister src2) {
  add_d(dst, src1, src2);
}
5658

5659
static const int kRegisterPassedArguments = 8;
5660

5661
int TurboAssembler::CalculateStackPassedWords(int num_reg_arguments,
5662 5663 5664 5665 5666 5667 5668 5669 5670 5671 5672 5673 5674
                                              int num_double_arguments) {
  int stack_passed_words = 0;
  num_reg_arguments += 2 * num_double_arguments;

  // O32: Up to four simple arguments are passed in registers a0..a3.
  // N64: Up to eight simple arguments are passed in registers a0..a7.
  if (num_reg_arguments > kRegisterPassedArguments) {
    stack_passed_words += num_reg_arguments - kRegisterPassedArguments;
  }
  stack_passed_words += kCArgSlotCount;
  return stack_passed_words;
}

5675
void TurboAssembler::PrepareCallCFunction(int num_reg_arguments,
5676 5677 5678 5679 5680 5681 5682 5683 5684 5685 5686 5687 5688 5689 5690 5691 5692 5693
                                          int num_double_arguments,
                                          Register scratch) {
  int frame_alignment = ActivationFrameAlignment();

  // n64: Up to eight simple arguments in a0..a3, a4..a7, No argument slots.
  // O32: Up to four simple arguments are passed in registers a0..a3.
  // Those four arguments must have reserved argument slots on the stack for
  // mips, even though those argument slots are not normally used.
  // Both ABIs: Remaining arguments are pushed on the stack, above (higher
  // address than) the (O32) argument slots. (arg slot calculation handled by
  // CalculateStackPassedWords()).
  int stack_passed_arguments = CalculateStackPassedWords(
      num_reg_arguments, num_double_arguments);
  if (frame_alignment > kPointerSize) {
    // Make stack end at alignment and make room for num_arguments - 4 words
    // and the original value of sp.
    mov(scratch, sp);
    Dsubu(sp, sp, Operand((stack_passed_arguments + 1) * kPointerSize));
5694
    DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
5695
    And(sp, sp, Operand(-frame_alignment));
5696
    Sd(scratch, MemOperand(sp, stack_passed_arguments * kPointerSize));
5697 5698 5699 5700 5701
  } else {
    Dsubu(sp, sp, Operand(stack_passed_arguments * kPointerSize));
  }
}

5702
void TurboAssembler::PrepareCallCFunction(int num_reg_arguments,
5703 5704 5705 5706
                                          Register scratch) {
  PrepareCallCFunction(num_reg_arguments, 0, scratch);
}

5707
void TurboAssembler::CallCFunction(ExternalReference function,
5708 5709
                                   int num_reg_arguments,
                                   int num_double_arguments) {
5710
  li(t9, function);
5711
  CallCFunctionHelper(t9, num_reg_arguments, num_double_arguments);
5712 5713
}

5714
void TurboAssembler::CallCFunction(Register function, int num_reg_arguments,
5715 5716 5717 5718
                                   int num_double_arguments) {
  CallCFunctionHelper(function, num_reg_arguments, num_double_arguments);
}

5719
void TurboAssembler::CallCFunction(ExternalReference function,
5720 5721 5722 5723
                                   int num_arguments) {
  CallCFunction(function, num_arguments, 0);
}

5724
void TurboAssembler::CallCFunction(Register function, int num_arguments) {
5725 5726 5727
  CallCFunction(function, num_arguments, 0);
}

5728
void TurboAssembler::CallCFunctionHelper(Register function,
5729 5730
                                         int num_reg_arguments,
                                         int num_double_arguments) {
5731
  DCHECK_LE(num_reg_arguments + num_double_arguments, kMaxCParameters);
5732
  DCHECK(has_frame());
5733 5734 5735 5736 5737 5738 5739 5740 5741 5742 5743
  // Make sure that the stack is aligned before calling a C function unless
  // running in the simulator. The simulator has its own alignment check which
  // provides more information.
  // The argument stots are presumed to have been set up by
  // PrepareCallCFunction. The C function must be called via t9, for mips ABI.

#if V8_HOST_ARCH_MIPS || V8_HOST_ARCH_MIPS64
  if (emit_debug_code()) {
    int frame_alignment = base::OS::ActivationFrameAlignment();
    int frame_alignment_mask = frame_alignment - 1;
    if (frame_alignment > kPointerSize) {
5744
      DCHECK(base::bits::IsPowerOfTwo(frame_alignment));
5745
      Label alignment_as_expected;
5746 5747 5748 5749 5750 5751
      {
        UseScratchRegisterScope temps(this);
        Register scratch = temps.Acquire();
        And(scratch, sp, Operand(frame_alignment_mask));
        Branch(&alignment_as_expected, eq, scratch, Operand(zero_reg));
      }
5752 5753 5754 5755 5756 5757 5758 5759 5760 5761 5762 5763
      // Don't use Check here, as it will call Runtime_Abort possibly
      // re-entering here.
      stop("Unexpected alignment in CallCFunction");
      bind(&alignment_as_expected);
    }
  }
#endif  // V8_HOST_ARCH_MIPS

  // Just call directly. The function called cannot cause a GC, or
  // allow preemption, so the return address in the link register
  // stays correct.

5764
  if (function != t9) {
5765 5766 5767 5768 5769 5770 5771 5772 5773 5774
    mov(t9, function);
    function = t9;
  }

  Call(function);

  int stack_passed_arguments = CalculateStackPassedWords(
      num_reg_arguments, num_double_arguments);

  if (base::OS::ActivationFrameAlignment() > kPointerSize) {
5775
    Ld(sp, MemOperand(sp, stack_passed_arguments * kPointerSize));
5776 5777 5778 5779 5780 5781 5782 5783
  } else {
    Daddu(sp, sp, Operand(stack_passed_arguments * kPointerSize));
  }
}


#undef BRANCH_ARGS_CHECK

5784 5785
void TurboAssembler::CheckPageFlag(Register object, Register scratch, int mask,
                                   Condition cc, Label* condition_met) {
5786
  And(scratch, object, Operand(~Page::kPageAlignmentMask));
5787
  Ld(scratch, MemOperand(scratch, MemoryChunk::kFlagsOffset));
5788 5789 5790 5791 5792 5793 5794 5795 5796 5797 5798 5799 5800 5801 5802 5803 5804 5805 5806
  And(scratch, scratch, Operand(mask));
  Branch(condition_met, cc, scratch, Operand(zero_reg));
}


Register GetRegisterThatIsNotOneOf(Register reg1,
                                   Register reg2,
                                   Register reg3,
                                   Register reg4,
                                   Register reg5,
                                   Register reg6) {
  RegList regs = 0;
  if (reg1.is_valid()) regs |= reg1.bit();
  if (reg2.is_valid()) regs |= reg2.bit();
  if (reg3.is_valid()) regs |= reg3.bit();
  if (reg4.is_valid()) regs |= reg4.bit();
  if (reg5.is_valid()) regs |= reg5.bit();
  if (reg6.is_valid()) regs |= reg6.bit();

5807
  const RegisterConfiguration* config = RegisterConfiguration::Default();
5808 5809 5810
  for (int i = 0; i < config->num_allocatable_general_registers(); ++i) {
    int code = config->GetAllocatableGeneralCode(i);
    Register candidate = Register::from_code(code);
5811 5812 5813 5814 5815 5816
    if (regs & candidate.bit()) continue;
    return candidate;
  }
  UNREACHABLE();
}

5817 5818 5819 5820 5821 5822 5823
bool AreAliased(Register reg1, Register reg2, Register reg3, Register reg4,
                Register reg5, Register reg6, Register reg7, Register reg8,
                Register reg9, Register reg10) {
  int n_of_valid_regs = reg1.is_valid() + reg2.is_valid() + reg3.is_valid() +
                        reg4.is_valid() + reg5.is_valid() + reg6.is_valid() +
                        reg7.is_valid() + reg8.is_valid() + reg9.is_valid() +
                        reg10.is_valid();
5824 5825 5826 5827 5828 5829 5830 5831 5832 5833

  RegList regs = 0;
  if (reg1.is_valid()) regs |= reg1.bit();
  if (reg2.is_valid()) regs |= reg2.bit();
  if (reg3.is_valid()) regs |= reg3.bit();
  if (reg4.is_valid()) regs |= reg4.bit();
  if (reg5.is_valid()) regs |= reg5.bit();
  if (reg6.is_valid()) regs |= reg6.bit();
  if (reg7.is_valid()) regs |= reg7.bit();
  if (reg8.is_valid()) regs |= reg8.bit();
5834 5835
  if (reg9.is_valid()) regs |= reg9.bit();
  if (reg10.is_valid()) regs |= reg10.bit();
5836 5837 5838
  int n_of_non_aliasing_regs = NumRegs(regs);

  return n_of_valid_regs != n_of_non_aliasing_regs;
5839 5840
}

5841 5842 5843 5844 5845 5846 5847 5848 5849 5850 5851 5852 5853 5854 5855 5856 5857 5858
void TurboAssembler::ComputeCodeStartAddress(Register dst) {
  // This push on ra and the pop below together ensure that we restore the
  // register ra, which is needed while computing the code start address.
  push(ra);

  // The bal instruction puts the address of the current instruction into
  // the return address (ra) register, which we can use later on.
  Label current;
  bal(&current);
  nop();
  int pc = pc_offset();
  bind(&current);
  li(dst, Operand(pc));
  Dsubu(dst, ra, dst);

  pop(ra);  // Restore ra
}

5859 5860 5861
void TurboAssembler::ResetSpeculationPoisonRegister() {
  li(kSpeculationPoisonRegister, -1);
}
5862

5863 5864
}  // namespace internal
}  // namespace v8
5865 5866

#endif  // V8_TARGET_ARCH_MIPS64