assembler-x64.cc 88.8 KB
Newer Older
1
// Copyright 2012 the V8 project authors. All rights reserved.
2 3
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
4

5 6
#include "src/x64/assembler-x64.h"

7 8 9 10
#include <cstring>

#if V8_TARGET_ARCH_X64

11 12 13
#if V8_LIBC_MSVCRT
#include <intrin.h>  // _xgetbv()
#endif
14 15 16
#if V8_OS_MACOSX
#include <sys/sysctl.h>
#endif
17

18
#include "src/base/bits.h"
19
#include "src/macro-assembler.h"
20
#include "src/v8.h"
21

22 23
namespace v8 {
namespace internal {
24

25 26
// -----------------------------------------------------------------------------
// Implementation of CpuFeatures
27

28 29
namespace {

30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47
#if !V8_LIBC_MSVCRT

V8_INLINE uint64_t _xgetbv(unsigned int xcr) {
  unsigned eax, edx;
  // Check xgetbv; this uses a .byte sequence instead of the instruction
  // directly because older assemblers do not include support for xgetbv and
  // there is no easy way to conditionally compile based on the assembler
  // used.
  __asm__ volatile(".byte 0x0f, 0x01, 0xd0" : "=a"(eax), "=d"(edx) : "c"(xcr));
  return static_cast<uint64_t>(eax) | (static_cast<uint64_t>(edx) << 32);
}

#define _XCR_XFEATURE_ENABLED_MASK 0

#endif  // !V8_LIBC_MSVCRT


bool OSHasAVXSupport() {
48
#if V8_OS_MACOSX
49 50
  // Mac OS X up to 10.9 has a bug where AVX transitions were indeed being
  // caused by ISRs, so we detect that here and disable AVX in that case.
51 52
  char buffer[128];
  size_t buffer_size = arraysize(buffer);
53
  int ctl_name[] = {CTL_KERN, KERN_OSRELEASE};
54 55 56 57
  if (sysctl(ctl_name, 2, buffer, &buffer_size, nullptr, 0) != 0) {
    V8_Fatal(__FILE__, __LINE__, "V8 failed to get kernel version");
  }
  // The buffer now contains a string of the form XX.YY.ZZ, where
58 59 60 61 62 63
  // XX is the major kernel version component.
  char* period_pos = strchr(buffer, '.');
  DCHECK_NOT_NULL(period_pos);
  *period_pos = '\0';
  long kernel_version_major = strtol(buffer, nullptr, 10);  // NOLINT
  if (kernel_version_major <= 13) return false;
64
#endif  // V8_OS_MACOSX
65 66 67
  // Check whether OS claims to support AVX.
  uint64_t feature_mask = _xgetbv(_XCR_XFEATURE_ENABLED_MASK);
  return (feature_mask & 0x6) == 0x6;
68 69 70 71 72
}

}  // namespace


73
void CpuFeatures::ProbeImpl(bool cross_compile) {
74
  base::CPU cpu;
75 76
  CHECK(cpu.has_sse2());  // SSE2 support is mandatory.
  CHECK(cpu.has_cmov());  // CMOV support is mandatory.
77

78 79
  // Only use statically determined features for cross compile (snapshot).
  if (cross_compile) return;
80

81 82
  if (cpu.has_sse41() && FLAG_enable_sse4_1) supported_ |= 1u << SSE4_1;
  if (cpu.has_sse3() && FLAG_enable_sse3) supported_ |= 1u << SSE3;
83
  // SAHF is not generally available in long mode.
84
  if (cpu.has_sahf() && FLAG_enable_sahf) supported_ |= 1u << SAHF;
85 86 87 88 89 90 91 92
  if (cpu.has_avx() && FLAG_enable_avx && cpu.has_osxsave() &&
      OSHasAVXSupport()) {
    supported_ |= 1u << AVX;
  }
  if (cpu.has_fma3() && FLAG_enable_fma3 && cpu.has_osxsave() &&
      OSHasAVXSupport()) {
    supported_ |= 1u << FMA3;
  }
93 94 95 96
  if (cpu.has_bmi1() && FLAG_enable_bmi1) supported_ |= 1u << BMI1;
  if (cpu.has_bmi2() && FLAG_enable_bmi2) supported_ |= 1u << BMI2;
  if (cpu.has_lzcnt() && FLAG_enable_lzcnt) supported_ |= 1u << LZCNT;
  if (cpu.has_popcnt() && FLAG_enable_popcnt) supported_ |= 1u << POPCNT;
97 98 99 100 101
  if (strcmp(FLAG_mcpu, "auto") == 0) {
    if (cpu.is_atom()) supported_ |= 1u << ATOM;
  } else if (strcmp(FLAG_mcpu, "atom") == 0) {
    supported_ |= 1u << ATOM;
  }
102 103
}

104

105
void CpuFeatures::PrintTarget() { }
106
void CpuFeatures::PrintFeatures() {
107 108 109 110 111 112 113 114
  printf(
      "SSE3=%d SSE4_1=%d SAHF=%d AVX=%d FMA3=%d BMI1=%d BMI2=%d LZCNT=%d "
      "POPCNT=%d ATOM=%d\n",
      CpuFeatures::IsSupported(SSE3), CpuFeatures::IsSupported(SSE4_1),
      CpuFeatures::IsSupported(SAHF), CpuFeatures::IsSupported(AVX),
      CpuFeatures::IsSupported(FMA3), CpuFeatures::IsSupported(BMI1),
      CpuFeatures::IsSupported(BMI2), CpuFeatures::IsSupported(LZCNT),
      CpuFeatures::IsSupported(POPCNT), CpuFeatures::IsSupported(ATOM));
115
}
116 117


118 119 120
// -----------------------------------------------------------------------------
// Implementation of Operand

121
Operand::Operand(Register base, int32_t disp) : rex_(0) {
122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142
  len_ = 1;
  if (base.is(rsp) || base.is(r12)) {
    // SIB byte is needed to encode (rsp + offset) or (r12 + offset).
    set_sib(times_1, rsp, base);
  }

  if (disp == 0 && !base.is(rbp) && !base.is(r13)) {
    set_modrm(0, base);
  } else if (is_int8(disp)) {
    set_modrm(1, base);
    set_disp8(disp);
  } else {
    set_modrm(2, base);
    set_disp32(disp);
  }
}


Operand::Operand(Register base,
                 Register index,
                 ScaleFactor scale,
143
                 int32_t disp) : rex_(0) {
144
  DCHECK(!index.is(rsp));
145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160
  len_ = 1;
  set_sib(scale, index, base);
  if (disp == 0 && !base.is(rbp) && !base.is(r13)) {
    // This call to set_modrm doesn't overwrite the REX.B (or REX.X) bits
    // possibly set by set_sib.
    set_modrm(0, rsp);
  } else if (is_int8(disp)) {
    set_modrm(1, rsp);
    set_disp8(disp);
  } else {
    set_modrm(2, rsp);
    set_disp32(disp);
  }
}


161 162 163
Operand::Operand(Register index,
                 ScaleFactor scale,
                 int32_t disp) : rex_(0) {
164
  DCHECK(!index.is(rsp));
165 166 167 168 169 170 171
  len_ = 1;
  set_modrm(0, rsp);
  set_sib(scale, index, rbp);
  set_disp32(disp);
}


172 173 174 175 176 177 178
Operand::Operand(Label* label) : rex_(0), len_(1) {
  DCHECK_NOT_NULL(label);
  set_modrm(0, rbp);
  set_disp64(reinterpret_cast<intptr_t>(label));
}


179
Operand::Operand(const Operand& operand, int32_t offset) {
180
  DCHECK(operand.len_ >= 1);
181 182
  // Operand encodes REX ModR/M [SIB] [Disp].
  byte modrm = operand.buf_[0];
183
  DCHECK(modrm < 0xC0);  // Disallow mode 3 (register target).
184 185 186 187 188 189 190 191 192 193
  bool has_sib = ((modrm & 0x07) == 0x04);
  byte mode = modrm & 0xC0;
  int disp_offset = has_sib ? 2 : 1;
  int base_reg = (has_sib ? operand.buf_[1] : modrm) & 0x07;
  // Mode 0 with rbp/r13 as ModR/M or SIB base register always has a 32-bit
  // displacement.
  bool is_baseless = (mode == 0) && (base_reg == 0x05);  // No base or RIP base.
  int32_t disp_value = 0;
  if (mode == 0x80 || is_baseless) {
    // Mode 2 or mode 0 with rbp/r13 as base: Word displacement.
194
    disp_value = *bit_cast<const int32_t*>(&operand.buf_[disp_offset]);
195 196 197 198 199 200
  } else if (mode == 0x40) {
    // Mode 1: Byte displacement.
    disp_value = static_cast<signed char>(operand.buf_[disp_offset]);
  }

  // Write new operand with same registers, but with modified displacement.
201
  DCHECK(offset >= 0 ? disp_value + offset > disp_value
202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224
                     : disp_value + offset < disp_value);  // No overflow.
  disp_value += offset;
  rex_ = operand.rex_;
  if (!is_int8(disp_value) || is_baseless) {
    // Need 32 bits of displacement, mode 2 or mode 1 with register rbp/r13.
    buf_[0] = (modrm & 0x3f) | (is_baseless ? 0x00 : 0x80);
    len_ = disp_offset + 4;
    Memory::int32_at(&buf_[disp_offset]) = disp_value;
  } else if (disp_value != 0 || (base_reg == 0x05)) {
    // Need 8 bits of displacement.
    buf_[0] = (modrm & 0x3f) | 0x40;  // Mode 1.
    len_ = disp_offset + 1;
    buf_[disp_offset] = static_cast<byte>(disp_value);
  } else {
    // Need no displacement.
    buf_[0] = (modrm & 0x3f);  // Mode 0.
    len_ = disp_offset;
  }
  if (has_sib) {
    buf_[1] = operand.buf_[1];
  }
}

225 226 227

bool Operand::AddressUsesRegister(Register reg) const {
  int code = reg.code();
228
  DCHECK((buf_[0] & 0xC0) != 0xC0);  // Always a memory operand.
229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252
  // Start with only low three bits of base register. Initial decoding doesn't
  // distinguish on the REX.B bit.
  int base_code = buf_[0] & 0x07;
  if (base_code == rsp.code()) {
    // SIB byte present in buf_[1].
    // Check the index register from the SIB byte + REX.X prefix.
    int index_code = ((buf_[1] >> 3) & 0x07) | ((rex_ & 0x02) << 2);
    // Index code (including REX.X) of 0x04 (rsp) means no index register.
    if (index_code != rsp.code() && index_code == code) return true;
    // Add REX.B to get the full base register code.
    base_code = (buf_[1] & 0x07) | ((rex_ & 0x01) << 3);
    // A base register of 0x05 (rbp) with mod = 0 means no base register.
    if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false;
    return code == base_code;
  } else {
    // A base register with low bits of 0x05 (rbp or r13) and mod = 0 means
    // no base register.
    if (base_code == rbp.code() && ((buf_[0] & 0xC0) == 0)) return false;
    base_code |= ((rex_ & 0x01) << 3);
    return code == base_code;
  }
}


253
// -----------------------------------------------------------------------------
254
// Implementation of Assembler.
255

256 257 258
#ifdef GENERATED_CODE_COVERAGE
static void InitCoverageLog();
#endif
259

260 261
Assembler::Assembler(Isolate* isolate, void* buffer, int buffer_size)
    : AssemblerBase(isolate, buffer, buffer_size),
262
      code_targets_(100),
263
      positions_recorder_(this) {
264 265
  // Clear the buffer in debug mode unless it was provided by the
  // caller in which case we can't be sure it's okay to overwrite
266
  // existing code in it.
267 268
#ifdef DEBUG
  if (own_buffer_) {
269
    memset(buffer_, 0xCC, buffer_size_);  // int3
270 271 272
  }
#endif

273
  reloc_info_writer.Reposition(buffer_ + buffer_size_, pc_);
274

275

276 277 278 279 280 281 282
#ifdef GENERATED_CODE_COVERAGE
  InitCoverageLog();
#endif
}


void Assembler::GetCode(CodeDesc* desc) {
283 284
  // Finalize code (at this point overflow() may be true, but the gap ensures
  // that we are still not overlapping instructions and relocation info).
285
  reloc_info_writer.Finish();
286
  DCHECK(pc_ <= reloc_info_writer.pos());  // No overlap.
287
  // Set up code descriptor.
288 289 290
  desc->buffer = buffer_;
  desc->buffer_size = buffer_size_;
  desc->instr_size = pc_offset();
291
  DCHECK(desc->instr_size > 0);  // Zero-size code objects upset the system.
292 293
  desc->reloc_size =
      static_cast<int>((buffer_ + buffer_size_) - reloc_info_writer.pos());
294 295 296 297 298
  desc->origin = this;
}


void Assembler::Align(int m) {
299
  DCHECK(base::bits::IsPowerOfTwo32(m));
300
  int delta = (m - (pc_offset() & (m - 1))) & (m - 1);
301
  Nop(delta);
302 303
}

304

305 306 307 308 309
void Assembler::CodeTargetAlign() {
  Align(16);  // Preferred alignment of jump targets on x64.
}


310 311 312 313 314 315 316 317 318
bool Assembler::IsNop(Address addr) {
  Address a = addr;
  while (*a == 0x66) a++;
  if (*a == 0x90) return true;
  if (a[0] == 0xf && a[1] == 0x1f) return true;
  return false;
}


319
void Assembler::bind_to(Label* L, int pos) {
320 321
  DCHECK(!L->is_bound());  // Label may only be bound once.
  DCHECK(0 <= pos && pos <= pc_offset());  // Position must be valid.
322 323 324 325
  if (L->is_linked()) {
    int current = L->pos();
    int next = long_at(current);
    while (next != current) {
326 327 328 329 330 331 332 333 334 335
      if (current >= 4 && long_at(current - 4) == 0) {
        // Absolute address.
        intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos);
        *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64;
        internal_reference_positions_.push_back(current - 4);
      } else {
        // Relative address, relative to point after address.
        int imm32 = pos - (current + sizeof(int32_t));
        long_at_put(current, imm32);
      }
336 337 338 339
      current = next;
      next = long_at(next);
    }
    // Fix up last fixup on linked list.
340 341 342 343 344 345 346 347 348 349
    if (current >= 4 && long_at(current - 4) == 0) {
      // Absolute address.
      intptr_t imm64 = reinterpret_cast<intptr_t>(buffer_ + pos);
      *reinterpret_cast<intptr_t*>(addr_at(current - 4)) = imm64;
      internal_reference_positions_.push_back(current - 4);
    } else {
      // Relative address, relative to point after address.
      int imm32 = pos - (current + sizeof(int32_t));
      long_at_put(current, imm32);
    }
350
  }
351 352 353 354
  while (L->is_near_linked()) {
    int fixup_pos = L->near_link_pos();
    int offset_to_next =
        static_cast<int>(*reinterpret_cast<int8_t*>(addr_at(fixup_pos)));
355
    DCHECK(offset_to_next <= 0);
356
    int disp = pos - (fixup_pos + sizeof(int8_t));
357
    CHECK(is_int8(disp));
358 359 360 361 362 363 364
    set_byte_at(fixup_pos, disp);
    if (offset_to_next < 0) {
      L->link_to(fixup_pos + offset_to_next, Label::kNear);
    } else {
      L->UnuseNear();
    }
  }
365
  L->bind_to(pos);
366 367
}

368 369 370 371 372 373

void Assembler::bind(Label* L) {
  bind_to(L, pc_offset());
}


374
void Assembler::GrowBuffer() {
375
  DCHECK(buffer_overflow());
376
  if (!own_buffer_) FATAL("external code buffer is too small");
377

378
  // Compute new buffer size.
379
  CodeDesc desc;  // the new buffer
380 381
  desc.buffer_size = 2 * buffer_size_;

382 383 384
  // Some internal data structures overflow for very large buffers,
  // they must ensure that kMaximalBufferSize is not too large.
  if ((desc.buffer_size > kMaximalBufferSize) ||
385
      (desc.buffer_size > isolate()->heap()->MaxOldGenerationSize())) {
386 387
    V8::FatalProcessOutOfMemory("Assembler::GrowBuffer");
  }
388

389
  // Set up new buffer.
390 391
  desc.buffer = NewArray<byte>(desc.buffer_size);
  desc.instr_size = pc_offset();
392 393
  desc.reloc_size =
      static_cast<int>((buffer_ + buffer_size_) - (reloc_info_writer.pos()));
394

395 396 397 398 399
  // Clear the buffer in debug mode. Use 'int3' instructions to make
  // sure to get into problems if we ever run uninitialized code.
#ifdef DEBUG
  memset(desc.buffer, 0xCC, desc.buffer_size);
#endif
400

401
  // Copy the data.
402 403 404
  intptr_t pc_delta = desc.buffer - buffer_;
  intptr_t rc_delta = (desc.buffer + desc.buffer_size) -
      (buffer_ + buffer_size_);
405 406 407
  MemMove(desc.buffer, buffer_, desc.instr_size);
  MemMove(rc_delta + reloc_info_writer.pos(), reloc_info_writer.pos(),
          desc.reloc_size);
408

409
  // Switch buffers.
410
  DeleteArray(buffer_);
411 412 413 414 415 416
  buffer_ = desc.buffer;
  buffer_size_ = desc.buffer_size;
  pc_ += pc_delta;
  reloc_info_writer.Reposition(reloc_info_writer.pos() + rc_delta,
                               reloc_info_writer.last_pc() + pc_delta);

417 418 419 420
  // Relocate internal references.
  for (auto pos : internal_reference_positions_) {
    intptr_t* p = reinterpret_cast<intptr_t*>(buffer_ + pos);
    *p += pc_delta;
421
  }
422

423
  DCHECK(!buffer_overflow());
424 425 426
}


427
void Assembler::emit_operand(int code, const Operand& adr) {
428
  DCHECK(is_uint3(code));
429
  const unsigned length = adr.len_;
430
  DCHECK(length > 0);
431

432
  // Emit updated ModR/M byte containing the given register.
433
  DCHECK((adr.buf_[0] & 0x38) == 0);
434 435 436 437 438
  *pc_++ = adr.buf_[0] | code << 3;

  // Recognize RIP relative addressing.
  if (adr.buf_[0] == 5) {
    DCHECK_EQ(9u, length);
439
    Label* label = *bit_cast<Label* const*>(&adr.buf_[1]);
440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456
    if (label->is_bound()) {
      int offset = label->pos() - pc_offset() - sizeof(int32_t);
      DCHECK_GE(0, offset);
      emitl(offset);
    } else if (label->is_linked()) {
      emitl(label->pos());
      label->link_to(pc_offset() - sizeof(int32_t));
    } else {
      DCHECK(label->is_unused());
      int32_t current = pc_offset();
      emitl(current);
      label->link_to(current);
    }
  } else {
    // Emit the rest of the encoded operand.
    for (unsigned i = 1; i < length; i++) *pc_++ = adr.buf_[i];
  }
457 458 459
}


460
// Assembler Instruction implementations.
461

462 463 464 465
void Assembler::arithmetic_op(byte opcode,
                              Register reg,
                              const Operand& op,
                              int size) {
466
  EnsureSpace ensure_space(this);
467
  emit_rex(reg, op, size);
468
  emit(opcode);
469
  emit_operand(reg, op);
470 471 472
}


473 474 475 476
void Assembler::arithmetic_op(byte opcode,
                              Register reg,
                              Register rm_reg,
                              int size) {
477
  EnsureSpace ensure_space(this);
478
  DCHECK((opcode & 0xC6) == 2);
479 480
  if (rm_reg.low_bits() == 4)  {  // Forces SIB byte.
    // Swap reg and rm_reg and change opcode operand order.
481
    emit_rex(rm_reg, reg, size);
482 483 484
    emit(opcode ^ 0x02);
    emit_modrm(rm_reg, reg);
  } else {
485
    emit_rex(reg, rm_reg, size);
486 487 488
    emit(opcode);
    emit_modrm(reg, rm_reg);
  }
489 490
}

491

492
void Assembler::arithmetic_op_16(byte opcode, Register reg, Register rm_reg) {
493
  EnsureSpace ensure_space(this);
494
  DCHECK((opcode & 0xC6) == 2);
495 496 497 498 499 500 501 502 503 504 505 506
  if (rm_reg.low_bits() == 4) {  // Forces SIB byte.
    // Swap reg and rm_reg and change opcode operand order.
    emit(0x66);
    emit_optional_rex_32(rm_reg, reg);
    emit(opcode ^ 0x02);
    emit_modrm(rm_reg, reg);
  } else {
    emit(0x66);
    emit_optional_rex_32(reg, rm_reg);
    emit(opcode);
    emit_modrm(reg, rm_reg);
  }
507 508 509 510 511 512 513 514 515 516 517 518 519 520
}


void Assembler::arithmetic_op_16(byte opcode,
                                 Register reg,
                                 const Operand& rm_reg) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(reg, rm_reg);
  emit(opcode);
  emit_operand(reg, rm_reg);
}


521 522 523 524 525 526 527 528 529 530 531 532
void Assembler::arithmetic_op_8(byte opcode, Register reg, const Operand& op) {
  EnsureSpace ensure_space(this);
  if (!reg.is_byte_register()) {
    // Register is not one of al, bl, cl, dl.  Its encoding needs REX.
    emit_rex_32(reg);
  }
  emit(opcode);
  emit_operand(reg, op);
}


void Assembler::arithmetic_op_8(byte opcode, Register reg, Register rm_reg) {
533
  EnsureSpace ensure_space(this);
534
  DCHECK((opcode & 0xC6) == 2);
535
  if (rm_reg.low_bits() == 4)  {  // Forces SIB byte.
536
    // Swap reg and rm_reg and change opcode operand order.
537 538 539 540 541
    if (!rm_reg.is_byte_register() || !reg.is_byte_register()) {
      // Register is not one of al, bl, cl, dl.  Its encoding needs REX.
      emit_rex_32(rm_reg, reg);
    }
    emit(opcode ^ 0x02);
542 543
    emit_modrm(rm_reg, reg);
  } else {
544 545 546 547
    if (!reg.is_byte_register() || !rm_reg.is_byte_register()) {
      // Register is not one of al, bl, cl, dl.  Its encoding needs REX.
      emit_rex_32(reg, rm_reg);
    }
548 549 550
    emit(opcode);
    emit_modrm(reg, rm_reg);
  }
551 552 553
}


554 555
void Assembler::immediate_arithmetic_op(byte subcode,
                                        Register dst,
556 557
                                        Immediate src,
                                        int size) {
558
  EnsureSpace ensure_space(this);
559
  emit_rex(dst, size);
560
  if (is_int8(src.value_)) {
561
    emit(0x83);
562
    emit_modrm(subcode, dst);
563
    emit(src.value_);
564 565 566
  } else if (dst.is(rax)) {
    emit(0x05 | (subcode << 3));
    emitl(src.value_);
567
  } else {
568
    emit(0x81);
569
    emit_modrm(subcode, dst);
570 571 572 573 574 575
    emitl(src.value_);
  }
}

void Assembler::immediate_arithmetic_op(byte subcode,
                                        const Operand& dst,
576 577
                                        Immediate src,
                                        int size) {
578
  EnsureSpace ensure_space(this);
579
  emit_rex(dst, size);
580
  if (is_int8(src.value_)) {
581
    emit(0x83);
582
    emit_operand(subcode, dst);
583
    emit(src.value_);
584
  } else {
585
    emit(0x81);
586
    emit_operand(subcode, dst);
587 588 589 590
    emitl(src.value_);
  }
}

591

592 593 594 595 596 597 598 599 600 601 602 603
void Assembler::immediate_arithmetic_op_16(byte subcode,
                                           Register dst,
                                           Immediate src) {
  EnsureSpace ensure_space(this);
  emit(0x66);  // Operand size override prefix.
  emit_optional_rex_32(dst);
  if (is_int8(src.value_)) {
    emit(0x83);
    emit_modrm(subcode, dst);
    emit(src.value_);
  } else if (dst.is(rax)) {
    emit(0x05 | (subcode << 3));
604
    emitw(src.value_);
605 606 607
  } else {
    emit(0x81);
    emit_modrm(subcode, dst);
608
    emitw(src.value_);
609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625
  }
}


void Assembler::immediate_arithmetic_op_16(byte subcode,
                                           const Operand& dst,
                                           Immediate src) {
  EnsureSpace ensure_space(this);
  emit(0x66);  // Operand size override prefix.
  emit_optional_rex_32(dst);
  if (is_int8(src.value_)) {
    emit(0x83);
    emit_operand(subcode, dst);
    emit(src.value_);
  } else {
    emit(0x81);
    emit_operand(subcode, dst);
626
    emitw(src.value_);
627 628 629 630
  }
}


631
void Assembler::immediate_arithmetic_op_8(byte subcode,
632 633
                                          const Operand& dst,
                                          Immediate src) {
634 635
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst);
636
  DCHECK(is_int8(src.value_) || is_uint8(src.value_));
637 638 639 640 641 642
  emit(0x80);
  emit_operand(subcode, dst);
  emit(src.value_);
}


643 644 645 646
void Assembler::immediate_arithmetic_op_8(byte subcode,
                                          Register dst,
                                          Immediate src) {
  EnsureSpace ensure_space(this);
647
  if (!dst.is_byte_register()) {
648 649
    // Register is not one of al, bl, cl, dl.  Its encoding needs REX.
    emit_rex_32(dst);
650
  }
651
  DCHECK(is_int8(src.value_) || is_uint8(src.value_));
652 653 654 655 656 657
  emit(0x80);
  emit_modrm(subcode, dst);
  emit(src.value_);
}


658 659 660 661
void Assembler::shift(Register dst,
                      Immediate shift_amount,
                      int subcode,
                      int size) {
662
  EnsureSpace ensure_space(this);
663
  DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
664
                            : is_uint5(shift_amount.value_));
665
  if (shift_amount.value_ == 1) {
666
    emit_rex(dst, size);
667
    emit(0xD1);
668
    emit_modrm(subcode, dst);
669
  } else {
670
    emit_rex(dst, size);
671
    emit(0xC1);
672
    emit_modrm(subcode, dst);
673 674 675 676 677
    emit(shift_amount.value_);
  }
}


678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695
void Assembler::shift(Operand dst, Immediate shift_amount, int subcode,
                      int size) {
  EnsureSpace ensure_space(this);
  DCHECK(size == kInt64Size ? is_uint6(shift_amount.value_)
                            : is_uint5(shift_amount.value_));
  if (shift_amount.value_ == 1) {
    emit_rex(dst, size);
    emit(0xD1);
    emit_operand(subcode, dst);
  } else {
    emit_rex(dst, size);
    emit(0xC1);
    emit_operand(subcode, dst);
    emit(shift_amount.value_);
  }
}


696
void Assembler::shift(Register dst, int subcode, int size) {
697
  EnsureSpace ensure_space(this);
698
  emit_rex(dst, size);
699
  emit(0xD3);
700
  emit_modrm(subcode, dst);
701 702 703
}


704 705 706 707 708 709 710 711
void Assembler::shift(Operand dst, int subcode, int size) {
  EnsureSpace ensure_space(this);
  emit_rex(dst, size);
  emit(0xD3);
  emit_operand(subcode, dst);
}


712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729
void Assembler::bt(const Operand& dst, Register src) {
  EnsureSpace ensure_space(this);
  emit_rex_64(src, dst);
  emit(0x0F);
  emit(0xA3);
  emit_operand(src, dst);
}


void Assembler::bts(const Operand& dst, Register src) {
  EnsureSpace ensure_space(this);
  emit_rex_64(src, dst);
  emit(0x0F);
  emit(0xAB);
  emit_operand(src, dst);
}


730 731 732 733 734 735 736 737 738
void Assembler::bsrl(Register dst, Register src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0xBD);
  emit_modrm(dst, src);
}


739 740 741 742 743 744 745 746 747
void Assembler::bsrl(Register dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0xBD);
  emit_operand(dst, src);
}


748
void Assembler::call(Label* L) {
749
  positions_recorder()->WriteRecordedPositions();
750
  EnsureSpace ensure_space(this);
751
  // 1110 1000 #32-bit disp.
752
  emit(0xE8);
753 754
  if (L->is_bound()) {
    int offset = L->pos() - pc_offset() - sizeof(int32_t);
755
    DCHECK(offset <= 0);
756
    emitl(offset);
757
  } else if (L->is_linked()) {
758
    emitl(L->pos());
759 760
    L->link_to(pc_offset() - sizeof(int32_t));
  } else {
761
    DCHECK(L->is_unused());
762
    int32_t current = pc_offset();
763
    emitl(current);
764 765
    L->link_to(current);
  }
766 767 768
}


769
void Assembler::call(Address entry, RelocInfo::Mode rmode) {
770
  DCHECK(RelocInfo::IsRuntimeEntry(rmode));
771 772 773 774 775 776 777 778
  positions_recorder()->WriteRecordedPositions();
  EnsureSpace ensure_space(this);
  // 1110 1000 #32-bit disp.
  emit(0xE8);
  emit_runtime_entry(entry, rmode);
}


779 780
void Assembler::call(Handle<Code> target,
                     RelocInfo::Mode rmode,
781
                     TypeFeedbackId ast_id) {
782
  positions_recorder()->WriteRecordedPositions();
783
  EnsureSpace ensure_space(this);
784
  // 1110 1000 #32-bit disp.
785
  emit(0xE8);
786
  emit_code_target(target, rmode, ast_id);
787 788 789
}


790
void Assembler::call(Register adr) {
791
  positions_recorder()->WriteRecordedPositions();
792
  EnsureSpace ensure_space(this);
793
  // Opcode: FF /2 r64.
794
  emit_optional_rex_32(adr);
795
  emit(0xFF);
796
  emit_modrm(0x2, adr);
797 798
}

799

lrn@chromium.org's avatar
lrn@chromium.org committed
800
void Assembler::call(const Operand& op) {
801
  positions_recorder()->WriteRecordedPositions();
lrn@chromium.org's avatar
lrn@chromium.org committed
802
  EnsureSpace ensure_space(this);
803
  // Opcode: FF /2 m64.
804
  emit_optional_rex_32(op);
lrn@chromium.org's avatar
lrn@chromium.org committed
805
  emit(0xFF);
806
  emit_operand(0x2, op);
lrn@chromium.org's avatar
lrn@chromium.org committed
807 808 809
}


810 811 812 813 814 815 816 817 818 819 820
// Calls directly to the given address using a relative offset.
// Should only ever be used in Code objects for calls within the
// same Code object. Should not be used when generating new code (use labels),
// but only when patching existing code.
void Assembler::call(Address target) {
  positions_recorder()->WriteRecordedPositions();
  EnsureSpace ensure_space(this);
  // 1110 1000 #32-bit disp.
  emit(0xE8);
  Address source = pc_ + 4;
  intptr_t displacement = target - source;
821
  DCHECK(is_int32(displacement));
822 823 824 825
  emitl(static_cast<int32_t>(displacement));
}


826 827 828 829 830
void Assembler::clc() {
  EnsureSpace ensure_space(this);
  emit(0xF8);
}

831

832 833 834 835 836
void Assembler::cld() {
  EnsureSpace ensure_space(this);
  emit(0xFC);
}

837

838 839 840 841 842 843
void Assembler::cdq() {
  EnsureSpace ensure_space(this);
  emit(0x99);
}


844
void Assembler::cmovq(Condition cc, Register dst, Register src) {
845 846 847 848 849
  if (cc == always) {
    movq(dst, src);
  } else if (cc == never) {
    return;
  }
850 851
  // No need to check CpuInfo for CMOV support, it's a required part of the
  // 64-bit architecture.
852
  DCHECK(cc >= 0);  // Use mov for unconditional moves.
853
  EnsureSpace ensure_space(this);
854
  // Opcode: REX.W 0f 40 + cc /r.
855 856 857 858 859 860 861 862
  emit_rex_64(dst, src);
  emit(0x0f);
  emit(0x40 + cc);
  emit_modrm(dst, src);
}


void Assembler::cmovq(Condition cc, Register dst, const Operand& src) {
863 864 865 866 867
  if (cc == always) {
    movq(dst, src);
  } else if (cc == never) {
    return;
  }
868
  DCHECK(cc >= 0);
869
  EnsureSpace ensure_space(this);
870
  // Opcode: REX.W 0f 40 + cc /r.
871 872 873 874 875 876 877 878
  emit_rex_64(dst, src);
  emit(0x0f);
  emit(0x40 + cc);
  emit_operand(dst, src);
}


void Assembler::cmovl(Condition cc, Register dst, Register src) {
879 880 881 882 883
  if (cc == always) {
    movl(dst, src);
  } else if (cc == never) {
    return;
  }
884
  DCHECK(cc >= 0);
885
  EnsureSpace ensure_space(this);
886
  // Opcode: 0f 40 + cc /r.
887 888 889 890 891 892 893 894
  emit_optional_rex_32(dst, src);
  emit(0x0f);
  emit(0x40 + cc);
  emit_modrm(dst, src);
}


void Assembler::cmovl(Condition cc, Register dst, const Operand& src) {
895 896 897 898 899
  if (cc == always) {
    movl(dst, src);
  } else if (cc == never) {
    return;
  }
900
  DCHECK(cc >= 0);
901
  EnsureSpace ensure_space(this);
902
  // Opcode: 0f 40 + cc /r.
903 904 905 906 907 908 909
  emit_optional_rex_32(dst, src);
  emit(0x0f);
  emit(0x40 + cc);
  emit_operand(dst, src);
}


910
void Assembler::cmpb_al(Immediate imm8) {
911
  DCHECK(is_int8(imm8.value_) || is_uint8(imm8.value_));
912 913 914 915 916
  EnsureSpace ensure_space(this);
  emit(0x3c);
  emit(imm8.value_);
}

917

918 919 920 921 922 923 924
void Assembler::cpuid() {
  EnsureSpace ensure_space(this);
  emit(0x0F);
  emit(0xA2);
}


925 926 927 928 929 930 931
void Assembler::cqo() {
  EnsureSpace ensure_space(this);
  emit_rex_64();
  emit(0x99);
}


932
void Assembler::emit_dec(Register dst, int size) {
933
  EnsureSpace ensure_space(this);
934
  emit_rex(dst, size);
935 936 937 938 939
  emit(0xFF);
  emit_modrm(0x1, dst);
}


940
void Assembler::emit_dec(const Operand& dst, int size) {
941
  EnsureSpace ensure_space(this);
942
  emit_rex(dst, size);
943 944 945 946 947
  emit(0xFF);
  emit_operand(1, dst);
}


948 949
void Assembler::decb(Register dst) {
  EnsureSpace ensure_space(this);
950
  if (!dst.is_byte_register()) {
951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966
    // Register is not one of al, bl, cl, dl.  Its encoding needs REX.
    emit_rex_32(dst);
  }
  emit(0xFE);
  emit_modrm(0x1, dst);
}


void Assembler::decb(const Operand& dst) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst);
  emit(0xFE);
  emit_operand(1, dst);
}


967 968 969 970 971 972 973 974
void Assembler::enter(Immediate size) {
  EnsureSpace ensure_space(this);
  emit(0xC8);
  emitw(size.value_);  // 16 bit operand, always.
  emit(0);
}


975 976
void Assembler::hlt() {
  EnsureSpace ensure_space(this);
977
  emit(0xF4);
978 979 980
}


981
void Assembler::emit_idiv(Register src, int size) {
982
  EnsureSpace ensure_space(this);
983
  emit_rex(src, size);
984 985 986 987 988
  emit(0xF7);
  emit_modrm(0x7, src);
}


989 990 991 992 993 994 995 996
void Assembler::emit_div(Register src, int size) {
  EnsureSpace ensure_space(this);
  emit_rex(src, size);
  emit(0xF7);
  emit_modrm(0x6, src);
}


997
void Assembler::emit_imul(Register src, int size) {
998
  EnsureSpace ensure_space(this);
999
  emit_rex(src, size);
1000 1001 1002 1003 1004
  emit(0xF7);
  emit_modrm(0x5, src);
}


1005 1006 1007 1008 1009 1010 1011 1012
void Assembler::emit_imul(const Operand& src, int size) {
  EnsureSpace ensure_space(this);
  emit_rex(src, size);
  emit(0xF7);
  emit_operand(0x5, src);
}


1013
void Assembler::emit_imul(Register dst, Register src, int size) {
1014
  EnsureSpace ensure_space(this);
1015
  emit_rex(dst, src, size);
1016 1017 1018 1019 1020 1021
  emit(0x0F);
  emit(0xAF);
  emit_modrm(dst, src);
}


1022
void Assembler::emit_imul(Register dst, const Operand& src, int size) {
1023
  EnsureSpace ensure_space(this);
1024
  emit_rex(dst, src, size);
1025 1026 1027 1028 1029 1030
  emit(0x0F);
  emit(0xAF);
  emit_operand(dst, src);
}


1031
void Assembler::emit_imul(Register dst, Register src, Immediate imm, int size) {
1032
  EnsureSpace ensure_space(this);
1033
  emit_rex(dst, src, size);
1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045
  if (is_int8(imm.value_)) {
    emit(0x6B);
    emit_modrm(dst, src);
    emit(imm.value_);
  } else {
    emit(0x69);
    emit_modrm(dst, src);
    emitl(imm.value_);
  }
}


1046 1047 1048 1049 1050 1051
void Assembler::emit_imul(Register dst, const Operand& src, Immediate imm,
                          int size) {
  EnsureSpace ensure_space(this);
  emit_rex(dst, src, size);
  if (is_int8(imm.value_)) {
    emit(0x6B);
1052 1053
    emit_operand(dst, src);
    emit(imm.value_);
1054 1055
  } else {
    emit(0x69);
1056 1057
    emit_operand(dst, src);
    emitl(imm.value_);
1058 1059 1060 1061
  }
}


1062
void Assembler::emit_inc(Register dst, int size) {
1063
  EnsureSpace ensure_space(this);
1064
  emit_rex(dst, size);
1065
  emit(0xFF);
1066
  emit_modrm(0x0, dst);
1067 1068 1069
}


1070
void Assembler::emit_inc(const Operand& dst, int size) {
1071
  EnsureSpace ensure_space(this);
1072
  emit_rex(dst, size);
1073 1074 1075 1076 1077
  emit(0xFF);
  emit_operand(0, dst);
}


1078 1079
void Assembler::int3() {
  EnsureSpace ensure_space(this);
1080
  emit(0xCC);
1081 1082 1083
}


1084
void Assembler::j(Condition cc, Label* L, Label::Distance distance) {
1085 1086 1087 1088 1089 1090
  if (cc == always) {
    jmp(L);
    return;
  } else if (cc == never) {
    return;
  }
1091
  EnsureSpace ensure_space(this);
1092
  DCHECK(is_uint4(cc));
1093 1094 1095 1096
  if (L->is_bound()) {
    const int short_size = 2;
    const int long_size  = 6;
    int offs = L->pos() - pc_offset();
1097
    DCHECK(offs <= 0);
1098 1099 1100
    // Determine whether we can use 1-byte offsets for backwards branches,
    // which have a max range of 128 bytes.

1101 1102 1103 1104 1105 1106 1107
    // We also need to check predictable_code_size() flag here, because on x64,
    // when the full code generator recompiles code for debugging, some places
    // need to be padded out to a certain size. The debugger is keeping track of
    // how often it did this so that it can adjust return addresses on the
    // stack, but if the size of jump instructions can also change, that's not
    // enough and the calculated offsets would be incorrect.
    if (is_int8(offs - short_size) && !predictable_code_size()) {
1108
      // 0111 tttn #8-bit disp.
1109 1110
      emit(0x70 | cc);
      emit((offs - short_size) & 0xFF);
1111
    } else {
1112
      // 0000 1111 1000 tttn #32-bit disp.
1113 1114
      emit(0x0F);
      emit(0x80 | cc);
1115
      emitl(offs - long_size);
1116
    }
1117 1118 1119 1120 1121 1122
  } else if (distance == Label::kNear) {
    // 0111 tttn #8-bit disp
    emit(0x70 | cc);
    byte disp = 0x00;
    if (L->is_near_linked()) {
      int offset = L->near_link_pos() - pc_offset();
1123
      DCHECK(is_int8(offset));
1124 1125 1126 1127
      disp = static_cast<byte>(offset & 0xFF);
    }
    L->link_to(pc_offset(), Label::kNear);
    emit(disp);
1128
  } else if (L->is_linked()) {
1129
    // 0000 1111 1000 tttn #32-bit disp.
1130 1131
    emit(0x0F);
    emit(0x80 | cc);
1132
    emitl(L->pos());
1133 1134
    L->link_to(pc_offset() - sizeof(int32_t));
  } else {
1135
    DCHECK(L->is_unused());
1136 1137
    emit(0x0F);
    emit(0x80 | cc);
1138
    int32_t current = pc_offset();
1139
    emitl(current);
1140 1141 1142 1143 1144
    L->link_to(current);
  }
}


1145
void Assembler::j(Condition cc, Address entry, RelocInfo::Mode rmode) {
1146
  DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1147
  EnsureSpace ensure_space(this);
1148
  DCHECK(is_uint4(cc));
1149 1150 1151 1152 1153 1154
  emit(0x0F);
  emit(0x80 | cc);
  emit_runtime_entry(entry, rmode);
}


1155 1156 1157 1158
void Assembler::j(Condition cc,
                  Handle<Code> target,
                  RelocInfo::Mode rmode) {
  EnsureSpace ensure_space(this);
1159
  DCHECK(is_uint4(cc));
1160
  // 0000 1111 1000 tttn #32-bit disp.
1161 1162 1163 1164 1165 1166
  emit(0x0F);
  emit(0x80 | cc);
  emit_code_target(target, rmode);
}


1167
void Assembler::jmp(Label* L, Label::Distance distance) {
1168
  EnsureSpace ensure_space(this);
1169 1170
  const int short_size = sizeof(int8_t);
  const int long_size = sizeof(int32_t);
1171 1172
  if (L->is_bound()) {
    int offs = L->pos() - pc_offset() - 1;
1173
    DCHECK(offs <= 0);
1174
    if (is_int8(offs - short_size) && !predictable_code_size()) {
1175
      // 1110 1011 #8-bit disp.
1176
      emit(0xEB);
1177
      emit((offs - short_size) & 0xFF);
1178
    } else {
1179
      // 1110 1001 #32-bit disp.
1180
      emit(0xE9);
1181
      emitl(offs - long_size);
1182
    }
1183 1184 1185 1186 1187
  } else if (distance == Label::kNear) {
    emit(0xEB);
    byte disp = 0x00;
    if (L->is_near_linked()) {
      int offset = L->near_link_pos() - pc_offset();
1188
      DCHECK(is_int8(offset));
1189 1190 1191 1192 1193
      disp = static_cast<byte>(offset & 0xFF);
    }
    L->link_to(pc_offset(), Label::kNear);
    emit(disp);
  } else if (L->is_linked()) {
1194
    // 1110 1001 #32-bit disp.
1195
    emit(0xE9);
1196
    emitl(L->pos());
1197
    L->link_to(pc_offset() - long_size);
1198
  } else {
1199
    // 1110 1001 #32-bit disp.
1200
    DCHECK(L->is_unused());
1201
    emit(0xE9);
1202
    int32_t current = pc_offset();
1203
    emitl(current);
1204 1205 1206 1207 1208
    L->link_to(current);
  }
}


1209 1210
void Assembler::jmp(Handle<Code> target, RelocInfo::Mode rmode) {
  EnsureSpace ensure_space(this);
1211
  // 1110 1001 #32-bit disp.
1212 1213 1214 1215 1216
  emit(0xE9);
  emit_code_target(target, rmode);
}


1217
void Assembler::jmp(Address entry, RelocInfo::Mode rmode) {
1218
  DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1219
  EnsureSpace ensure_space(this);
1220
  DCHECK(RelocInfo::IsRuntimeEntry(rmode));
1221 1222 1223 1224 1225
  emit(0xE9);
  emit_runtime_entry(entry, rmode);
}


1226 1227
void Assembler::jmp(Register target) {
  EnsureSpace ensure_space(this);
1228
  // Opcode FF/4 r64.
1229
  emit_optional_rex_32(target);
1230
  emit(0xFF);
1231
  emit_modrm(0x4, target);
1232 1233 1234
}


1235 1236
void Assembler::jmp(const Operand& src) {
  EnsureSpace ensure_space(this);
1237
  // Opcode FF/4 m64.
1238 1239 1240 1241 1242 1243
  emit_optional_rex_32(src);
  emit(0xFF);
  emit_operand(0x4, src);
}


1244
void Assembler::emit_lea(Register dst, const Operand& src, int size) {
1245
  EnsureSpace ensure_space(this);
1246
  emit_rex(dst, src, size);
1247 1248 1249 1250 1251
  emit(0x8D);
  emit_operand(dst, src);
}


lrn@chromium.org's avatar
lrn@chromium.org committed
1252 1253
void Assembler::load_rax(void* value, RelocInfo::Mode mode) {
  EnsureSpace ensure_space(this);
1254 1255 1256 1257 1258
  if (kPointerSize == kInt64Size) {
    emit(0x48);  // REX.W
    emit(0xA1);
    emitp(value, mode);
  } else {
1259
    DCHECK(kPointerSize == kInt32Size);
1260 1261 1262 1263 1264 1265 1266
    emit(0xA1);
    emitp(value, mode);
    // In 64-bit mode, need to zero extend the operand to 8 bytes.
    // See 2.2.1.4 in Intel64 and IA32 Architectures Software
    // Developer's Manual Volume 2.
    emitl(0);
  }
lrn@chromium.org's avatar
lrn@chromium.org committed
1267 1268 1269 1270 1271 1272 1273 1274
}


void Assembler::load_rax(ExternalReference ref) {
  load_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
}


1275 1276 1277 1278 1279 1280
void Assembler::leave() {
  EnsureSpace ensure_space(this);
  emit(0xC9);
}


1281 1282
void Assembler::movb(Register dst, const Operand& src) {
  EnsureSpace ensure_space(this);
1283
  if (!dst.is_byte_register()) {
1284 1285 1286 1287 1288
    // Register is not one of al, bl, cl, dl.  Its encoding needs REX.
    emit_rex_32(dst, src);
  } else {
    emit_optional_rex_32(dst, src);
  }
1289 1290 1291 1292
  emit(0x8A);
  emit_operand(dst, src);
}

1293

1294 1295
void Assembler::movb(Register dst, Immediate imm) {
  EnsureSpace ensure_space(this);
1296
  if (!dst.is_byte_register()) {
1297
    // Register is not one of al, bl, cl, dl.  Its encoding needs REX.
1298 1299 1300
    emit_rex_32(dst);
  }
  emit(0xB0 + dst.low_bits());
1301 1302 1303
  emit(imm.value_);
}

1304

1305 1306
void Assembler::movb(const Operand& dst, Register src) {
  EnsureSpace ensure_space(this);
1307
  if (!src.is_byte_register()) {
1308
    // Register is not one of al, bl, cl, dl.  Its encoding needs REX.
1309 1310 1311 1312
    emit_rex_32(src, dst);
  } else {
    emit_optional_rex_32(src, dst);
  }
1313 1314 1315 1316
  emit(0x88);
  emit_operand(src, dst);
}

1317

1318 1319 1320 1321 1322 1323 1324 1325 1326
void Assembler::movb(const Operand& dst, Immediate imm) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst);
  emit(0xC6);
  emit_operand(0x0, dst);
  emit(static_cast<byte>(imm.value_));
}


1327 1328 1329 1330 1331 1332 1333 1334 1335
void Assembler::movw(Register dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(dst, src);
  emit(0x8B);
  emit_operand(dst, src);
}


1336 1337 1338 1339 1340 1341 1342 1343
void Assembler::movw(const Operand& dst, Register src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(src, dst);
  emit(0x89);
  emit_operand(src, dst);
}

1344

1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355
void Assembler::movw(const Operand& dst, Immediate imm) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(dst);
  emit(0xC7);
  emit_operand(0x0, dst);
  emit(static_cast<byte>(imm.value_ & 0xff));
  emit(static_cast<byte>(imm.value_ >> 8));
}


1356
void Assembler::emit_mov(Register dst, const Operand& src, int size) {
1357
  EnsureSpace ensure_space(this);
1358
  emit_rex(dst, src, size);
1359 1360 1361 1362 1363
  emit(0x8B);
  emit_operand(dst, src);
}


1364
void Assembler::emit_mov(Register dst, Register src, int size) {
1365
  EnsureSpace ensure_space(this);
1366
  if (src.low_bits() == 4) {
1367
    emit_rex(src, dst, size);
1368 1369 1370
    emit(0x89);
    emit_modrm(src, dst);
  } else {
1371
    emit_rex(dst, src, size);
1372 1373 1374
    emit(0x8B);
    emit_modrm(dst, src);
  }
1375 1376 1377
}


1378
void Assembler::emit_mov(const Operand& dst, Register src, int size) {
1379
  EnsureSpace ensure_space(this);
1380
  emit_rex(src, dst, size);
1381 1382 1383 1384 1385
  emit(0x89);
  emit_operand(src, dst);
}


1386
void Assembler::emit_mov(Register dst, Immediate value, int size) {
1387
  EnsureSpace ensure_space(this);
1388 1389 1390 1391
  emit_rex(dst, size);
  if (size == kInt64Size) {
    emit(0xC7);
    emit_modrm(0x0, dst);
1392
  } else {
1393
    DCHECK(size == kInt32Size);
1394
    emit(0xB8 + dst.low_bits());
1395
  }
1396
  emit(value);
1397 1398
}

1399

1400
void Assembler::emit_mov(const Operand& dst, Immediate value, int size) {
1401
  EnsureSpace ensure_space(this);
1402
  emit_rex(dst, size);
1403
  emit(0xC7);
1404 1405
  emit_operand(0x0, dst);
  emit(value);
lrn@chromium.org's avatar
lrn@chromium.org committed
1406 1407 1408
}


1409 1410 1411 1412 1413
void Assembler::movp(Register dst, void* value, RelocInfo::Mode rmode) {
  EnsureSpace ensure_space(this);
  emit_rex(dst, kPointerSize);
  emit(0xB8 | dst.low_bits());
  emitp(value, rmode);
1414 1415 1416
}


1417 1418 1419 1420 1421 1422 1423 1424
void Assembler::movq(Register dst, int64_t value) {
  EnsureSpace ensure_space(this);
  emit_rex_64(dst);
  emit(0xB8 | dst.low_bits());
  emitq(value);
}


1425 1426
void Assembler::movq(Register dst, uint64_t value) {
  movq(dst, static_cast<int64_t>(value));
1427 1428 1429
}


1430 1431
// Loads the ip-relative location of the src label into the target location
// (as a 32-bit offset sign extended to 64-bit).
1432 1433 1434 1435 1436 1437 1438
void Assembler::movl(const Operand& dst, Label* src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst);
  emit(0xC7);
  emit_operand(0, dst);
  if (src->is_bound()) {
    int offset = src->pos() - pc_offset() - sizeof(int32_t);
1439
    DCHECK(offset <= 0);
1440 1441 1442 1443 1444
    emitl(offset);
  } else if (src->is_linked()) {
    emitl(src->pos());
    src->link_to(pc_offset() - sizeof(int32_t));
  } else {
1445
    DCHECK(src->is_unused());
1446 1447 1448 1449 1450 1451 1452
    int32_t current = pc_offset();
    emitl(current);
    src->link_to(current);
  }
}


1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466
void Assembler::movsxbl(Register dst, Register src) {
  EnsureSpace ensure_space(this);
  if (!src.is_byte_register()) {
    // Register is not one of al, bl, cl, dl.  Its encoding needs REX.
    emit_rex_32(dst, src);
  } else {
    emit_optional_rex_32(dst, src);
  }
  emit(0x0F);
  emit(0xBE);
  emit_modrm(dst, src);
}


1467 1468 1469 1470 1471 1472 1473 1474 1475
void Assembler::movsxbl(Register dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0xBE);
  emit_operand(dst, src);
}


1476 1477
void Assembler::movsxbq(Register dst, const Operand& src) {
  EnsureSpace ensure_space(this);
1478
  emit_rex_64(dst, src);
1479 1480 1481 1482 1483 1484
  emit(0x0F);
  emit(0xBE);
  emit_operand(dst, src);
}


1485 1486 1487 1488 1489 1490 1491 1492 1493
void Assembler::movsxwl(Register dst, Register src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0xBF);
  emit_modrm(dst, src);
}


1494 1495 1496 1497 1498 1499 1500 1501 1502
void Assembler::movsxwl(Register dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0xBF);
  emit_operand(dst, src);
}


1503 1504 1505 1506 1507 1508 1509 1510 1511
void Assembler::movsxwq(Register dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit_rex_64(dst, src);
  emit(0x0F);
  emit(0xBF);
  emit_operand(dst, src);
}


1512 1513 1514 1515 1516
void Assembler::movsxlq(Register dst, Register src) {
  EnsureSpace ensure_space(this);
  emit_rex_64(dst, src);
  emit(0x63);
  emit_modrm(dst, src);
1517 1518 1519 1520 1521 1522 1523 1524
}


void Assembler::movsxlq(Register dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit_rex_64(dst, src);
  emit(0x63);
  emit_operand(dst, src);
1525 1526 1527
}


1528
void Assembler::emit_movzxb(Register dst, const Operand& src, int size) {
1529
  EnsureSpace ensure_space(this);
1530 1531
  // 32 bit operations zero the top 32 bits of 64 bit registers.  Therefore
  // there is no need to make this a 64 bit operation.
1532
  emit_optional_rex_32(dst, src);
1533 1534 1535 1536 1537 1538
  emit(0x0F);
  emit(0xB6);
  emit_operand(dst, src);
}


1539 1540 1541 1542
void Assembler::emit_movzxb(Register dst, Register src, int size) {
  EnsureSpace ensure_space(this);
  // 32 bit operations zero the top 32 bits of 64 bit registers.  Therefore
  // there is no need to make this a 64 bit operation.
1543 1544 1545 1546 1547 1548
  if (!src.is_byte_register()) {
    // Register is not one of al, bl, cl, dl.  Its encoding needs REX.
    emit_rex_32(dst, src);
  } else {
    emit_optional_rex_32(dst, src);
  }
1549 1550 1551 1552 1553 1554
  emit(0x0F);
  emit(0xB6);
  emit_modrm(dst, src);
}


1555
void Assembler::emit_movzxw(Register dst, const Operand& src, int size) {
1556
  EnsureSpace ensure_space(this);
1557 1558
  // 32 bit operations zero the top 32 bits of 64 bit registers.  Therefore
  // there is no need to make this a 64 bit operation.
1559 1560 1561 1562 1563 1564 1565
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0xB7);
  emit_operand(dst, src);
}


1566
void Assembler::emit_movzxw(Register dst, Register src, int size) {
1567
  EnsureSpace ensure_space(this);
1568 1569
  // 32 bit operations zero the top 32 bits of 64 bit registers.  Therefore
  // there is no need to make this a 64 bit operation.
1570 1571 1572 1573 1574 1575 1576
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0xB7);
  emit_modrm(dst, src);
}


1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591
void Assembler::repmovsb() {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit(0xA4);
}


void Assembler::repmovsw() {
  EnsureSpace ensure_space(this);
  emit(0x66);  // Operand size override.
  emit(0xF3);
  emit(0xA4);
}


1592
void Assembler::emit_repmovs(int size) {
1593 1594
  EnsureSpace ensure_space(this);
  emit(0xF3);
1595
  emit_rex(size);
1596 1597 1598 1599
  emit(0xA5);
}


1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616
void Assembler::mull(Register src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(src);
  emit(0xF7);
  emit_modrm(0x4, src);
}


void Assembler::mull(const Operand& src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(src);
  emit(0xF7);
  emit_operand(0x4, src);
}


void Assembler::mulq(Register src) {
1617 1618 1619 1620 1621 1622 1623
  EnsureSpace ensure_space(this);
  emit_rex_64(src);
  emit(0xF7);
  emit_modrm(0x4, src);
}


1624
void Assembler::emit_neg(Register dst, int size) {
1625
  EnsureSpace ensure_space(this);
1626
  emit_rex(dst, size);
1627 1628 1629 1630 1631
  emit(0xF7);
  emit_modrm(0x3, dst);
}


1632
void Assembler::emit_neg(const Operand& dst, int size) {
1633 1634 1635 1636 1637 1638 1639
  EnsureSpace ensure_space(this);
  emit_rex_64(dst);
  emit(0xF7);
  emit_operand(3, dst);
}


1640 1641
void Assembler::nop() {
  EnsureSpace ensure_space(this);
1642
  emit(0x90);
1643 1644
}

1645

1646
void Assembler::emit_not(Register dst, int size) {
1647
  EnsureSpace ensure_space(this);
1648
  emit_rex(dst, size);
1649
  emit(0xF7);
1650
  emit_modrm(0x2, dst);
1651 1652 1653
}


1654
void Assembler::emit_not(const Operand& dst, int size) {
1655
  EnsureSpace ensure_space(this);
1656
  emit_rex(dst, size);
1657 1658 1659 1660 1661
  emit(0xF7);
  emit_operand(2, dst);
}


1662
void Assembler::Nop(int n) {
1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677
  // The recommended muti-byte sequences of NOP instructions from the Intel 64
  // and IA-32 Architectures Software Developer's Manual.
  //
  // Length   Assembly                                Byte Sequence
  // 2 bytes  66 NOP                                  66 90H
  // 3 bytes  NOP DWORD ptr [EAX]                     0F 1F 00H
  // 4 bytes  NOP DWORD ptr [EAX + 00H]               0F 1F 40 00H
  // 5 bytes  NOP DWORD ptr [EAX + EAX*1 + 00H]       0F 1F 44 00 00H
  // 6 bytes  66 NOP DWORD ptr [EAX + EAX*1 + 00H]    66 0F 1F 44 00 00H
  // 7 bytes  NOP DWORD ptr [EAX + 00000000H]         0F 1F 80 00 00 00 00H
  // 8 bytes  NOP DWORD ptr [EAX + EAX*1 + 00000000H] 0F 1F 84 00 00 00 00 00H
  // 9 bytes  66 NOP DWORD ptr [EAX + EAX*1 +         66 0F 1F 84 00 00 00 00
  //          00000000H]                              00H

  EnsureSpace ensure_space(this);
1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734
  while (n > 0) {
    switch (n) {
      case 2:
        emit(0x66);
      case 1:
        emit(0x90);
        return;
      case 3:
        emit(0x0f);
        emit(0x1f);
        emit(0x00);
        return;
      case 4:
        emit(0x0f);
        emit(0x1f);
        emit(0x40);
        emit(0x00);
        return;
      case 6:
        emit(0x66);
      case 5:
        emit(0x0f);
        emit(0x1f);
        emit(0x44);
        emit(0x00);
        emit(0x00);
        return;
      case 7:
        emit(0x0f);
        emit(0x1f);
        emit(0x80);
        emit(0x00);
        emit(0x00);
        emit(0x00);
        emit(0x00);
        return;
      default:
      case 11:
        emit(0x66);
        n--;
      case 10:
        emit(0x66);
        n--;
      case 9:
        emit(0x66);
        n--;
      case 8:
        emit(0x0f);
        emit(0x1f);
        emit(0x84);
        emit(0x00);
        emit(0x00);
        emit(0x00);
        emit(0x00);
        emit(0x00);
        n -= 8;
    }
1735 1736 1737 1738
  }
}


1739
void Assembler::popq(Register dst) {
1740
  EnsureSpace ensure_space(this);
1741
  emit_optional_rex_32(dst);
1742
  emit(0x58 | dst.low_bits());
1743 1744 1745
}


1746
void Assembler::popq(const Operand& dst) {
1747
  EnsureSpace ensure_space(this);
1748
  emit_optional_rex_32(dst);
1749
  emit(0x8F);
1750
  emit_operand(0, dst);
1751 1752 1753
}


1754 1755 1756 1757 1758 1759
void Assembler::popfq() {
  EnsureSpace ensure_space(this);
  emit(0x9D);
}


1760
void Assembler::pushq(Register src) {
1761
  EnsureSpace ensure_space(this);
1762
  emit_optional_rex_32(src);
1763
  emit(0x50 | src.low_bits());
1764 1765 1766
}


1767
void Assembler::pushq(const Operand& src) {
1768
  EnsureSpace ensure_space(this);
1769
  emit_optional_rex_32(src);
1770
  emit(0xFF);
1771
  emit_operand(6, src);
1772 1773 1774
}


1775
void Assembler::pushq(Immediate value) {
1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786
  EnsureSpace ensure_space(this);
  if (is_int8(value.value_)) {
    emit(0x6A);
    emit(value.value_);  // Emit low byte of value.
  } else {
    emit(0x68);
    emitl(value.value_);
  }
}


1787
void Assembler::pushq_imm32(int32_t imm32) {
1788 1789 1790 1791 1792 1793
  EnsureSpace ensure_space(this);
  emit(0x68);
  emitl(imm32);
}


1794 1795 1796
void Assembler::pushfq() {
  EnsureSpace ensure_space(this);
  emit(0x9C);
1797 1798
}

1799

1800 1801
void Assembler::ret(int imm16) {
  EnsureSpace ensure_space(this);
1802
  DCHECK(is_uint16(imm16));
1803
  if (imm16 == 0) {
1804
    emit(0xC3);
1805
  } else {
1806 1807 1808
    emit(0xC2);
    emit(imm16 & 0xFF);
    emit((imm16 >> 8) & 0xFF);
1809 1810 1811
  }
}

1812

1813 1814 1815 1816 1817 1818 1819
void Assembler::ud2() {
  EnsureSpace ensure_space(this);
  emit(0x0F);
  emit(0x0B);
}


1820
void Assembler::setcc(Condition cc, Register reg) {
1821 1822 1823 1824
  if (cc > last_condition) {
    movb(reg, Immediate(cc == always ? 1 : 0));
    return;
  }
1825
  EnsureSpace ensure_space(this);
1826
  DCHECK(is_uint4(cc));
1827 1828
  if (!reg.is_byte_register()) {
    // Register is not one of al, bl, cl, dl.  Its encoding needs REX.
1829 1830 1831 1832 1833 1834 1835 1836
    emit_rex_32(reg);
  }
  emit(0x0F);
  emit(0x90 | cc);
  emit_modrm(0x0, reg);
}


1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854
void Assembler::shld(Register dst, Register src) {
  EnsureSpace ensure_space(this);
  emit_rex_64(src, dst);
  emit(0x0F);
  emit(0xA5);
  emit_modrm(src, dst);
}


void Assembler::shrd(Register dst, Register src) {
  EnsureSpace ensure_space(this);
  emit_rex_64(src, dst);
  emit(0x0F);
  emit(0xAD);
  emit_modrm(src, dst);
}


1855
void Assembler::emit_xchg(Register dst, Register src, int size) {
1856 1857 1858
  EnsureSpace ensure_space(this);
  if (src.is(rax) || dst.is(rax)) {  // Single-byte encoding
    Register other = src.is(rax) ? dst : src;
1859
    emit_rex(other, size);
1860
    emit(0x90 | other.low_bits());
1861
  } else if (dst.low_bits() == 4) {
1862
    emit_rex(dst, src, size);
1863 1864 1865
    emit(0x87);
    emit_modrm(dst, src);
  } else {
1866
    emit_rex(src, dst, size);
1867 1868 1869 1870 1871 1872
    emit(0x87);
    emit_modrm(src, dst);
  }
}


1873 1874 1875 1876 1877 1878 1879 1880
void Assembler::emit_xchg(Register dst, const Operand& src, int size) {
  EnsureSpace ensure_space(this);
  emit_rex(dst, src, size);
  emit(0x87);
  emit_operand(dst, src);
}


lrn@chromium.org's avatar
lrn@chromium.org committed
1881 1882
void Assembler::store_rax(void* dst, RelocInfo::Mode mode) {
  EnsureSpace ensure_space(this);
1883 1884 1885 1886 1887
  if (kPointerSize == kInt64Size) {
    emit(0x48);  // REX.W
    emit(0xA3);
    emitp(dst, mode);
  } else {
1888
    DCHECK(kPointerSize == kInt32Size);
1889 1890 1891 1892 1893 1894 1895
    emit(0xA3);
    emitp(dst, mode);
    // In 64-bit mode, need to zero extend the operand to 8 bytes.
    // See 2.2.1.4 in Intel64 and IA32 Architectures Software
    // Developer's Manual Volume 2.
    emitl(0);
  }
lrn@chromium.org's avatar
lrn@chromium.org committed
1896 1897 1898 1899 1900 1901 1902 1903
}


void Assembler::store_rax(ExternalReference ref) {
  store_rax(ref.address(), RelocInfo::EXTERNAL_REFERENCE);
}


1904 1905
void Assembler::testb(Register dst, Register src) {
  EnsureSpace ensure_space(this);
1906 1907 1908 1909 1910
  if (src.low_bits() == 4) {
    emit_rex_32(src, dst);
    emit(0x84);
    emit_modrm(src, dst);
  } else {
1911
    if (!dst.is_byte_register() || !src.is_byte_register()) {
1912 1913 1914 1915 1916
      // Register is not one of al, bl, cl, dl.  Its encoding needs REX.
      emit_rex_32(dst, src);
    }
    emit(0x84);
    emit_modrm(dst, src);
1917 1918 1919 1920
  }
}


1921
void Assembler::testb(Register reg, Immediate mask) {
1922
  DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
1923 1924 1925
  EnsureSpace ensure_space(this);
  if (reg.is(rax)) {
    emit(0xA8);
1926
    emit(mask.value_);  // Low byte emitted.
1927
  } else {
1928
    if (!reg.is_byte_register()) {
1929 1930
      // Register is not one of al, bl, cl, dl.  Its encoding needs REX.
      emit_rex_32(reg);
1931 1932
    }
    emit(0xF6);
1933
    emit_modrm(0x0, reg);
1934 1935 1936 1937 1938 1939
    emit(mask.value_);  // Low byte emitted.
  }
}


void Assembler::testb(const Operand& op, Immediate mask) {
1940
  DCHECK(is_int8(mask.value_) || is_uint8(mask.value_));
1941 1942 1943 1944 1945 1946 1947 1948
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(rax, op);
  emit(0xF6);
  emit_operand(rax, op);  // Operation code 0
  emit(mask.value_);  // Low byte emitted.
}


1949 1950
void Assembler::testb(const Operand& op, Register reg) {
  EnsureSpace ensure_space(this);
1951
  if (!reg.is_byte_register()) {
1952 1953 1954 1955 1956 1957 1958 1959 1960 1961
    // Register is not one of al, bl, cl, dl.  Its encoding needs REX.
    emit_rex_32(reg, op);
  } else {
    emit_optional_rex_32(reg, op);
  }
  emit(0x84);
  emit_operand(reg, op);
}


1962
void Assembler::emit_test(Register dst, Register src, int size) {
1963
  EnsureSpace ensure_space(this);
1964
  if (src.low_bits() == 4) {
1965
    emit_rex(src, dst, size);
1966 1967 1968
    emit(0x85);
    emit_modrm(src, dst);
  } else {
1969
    emit_rex(dst, src, size);
1970 1971 1972
    emit(0x85);
    emit_modrm(dst, src);
  }
1973 1974 1975
}


1976
void Assembler::emit_test(Register reg, Immediate mask, int size) {
1977 1978 1979 1980 1981
  // testl with a mask that fits in the low byte is exactly testb.
  if (is_uint8(mask.value_)) {
    testb(reg, mask);
    return;
  }
1982 1983
  EnsureSpace ensure_space(this);
  if (reg.is(rax)) {
1984
    emit_rex(rax, size);
1985 1986 1987
    emit(0xA9);
    emit(mask);
  } else {
1988
    emit_rex(reg, size);
1989
    emit(0xF7);
1990
    emit_modrm(0x0, reg);
1991 1992 1993 1994 1995
    emit(mask);
  }
}


1996
void Assembler::emit_test(const Operand& op, Immediate mask, int size) {
1997 1998 1999 2000 2001
  // testl with a mask that fits in the low byte is exactly testb.
  if (is_uint8(mask.value_)) {
    testb(op, mask);
    return;
  }
2002
  EnsureSpace ensure_space(this);
2003
  emit_rex(rax, op, size);
2004 2005 2006 2007 2008 2009
  emit(0xF7);
  emit_operand(rax, op);  // Operation code 0
  emit(mask);
}


2010
void Assembler::emit_test(const Operand& op, Register reg, int size) {
2011
  EnsureSpace ensure_space(this);
2012
  emit_rex(reg, op, size);
2013 2014 2015 2016 2017
  emit(0x85);
  emit_operand(reg, op);
}


2018
// FPU instructions.
2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040


void Assembler::fld(int i) {
  EnsureSpace ensure_space(this);
  emit_farith(0xD9, 0xC0, i);
}


void Assembler::fld1() {
  EnsureSpace ensure_space(this);
  emit(0xD9);
  emit(0xE8);
}


void Assembler::fldz() {
  EnsureSpace ensure_space(this);
  emit(0xD9);
  emit(0xEE);
}


2041 2042 2043 2044 2045 2046 2047
void Assembler::fldpi() {
  EnsureSpace ensure_space(this);
  emit(0xD9);
  emit(0xEB);
}


2048 2049 2050 2051 2052 2053 2054
void Assembler::fldln2() {
  EnsureSpace ensure_space(this);
  emit(0xD9);
  emit(0xED);
}


2055 2056
void Assembler::fld_s(const Operand& adr) {
  EnsureSpace ensure_space(this);
2057
  emit_optional_rex_32(adr);
2058 2059 2060 2061 2062 2063 2064
  emit(0xD9);
  emit_operand(0, adr);
}


void Assembler::fld_d(const Operand& adr) {
  EnsureSpace ensure_space(this);
2065
  emit_optional_rex_32(adr);
2066 2067 2068 2069 2070 2071 2072
  emit(0xDD);
  emit_operand(0, adr);
}


void Assembler::fstp_s(const Operand& adr) {
  EnsureSpace ensure_space(this);
2073
  emit_optional_rex_32(adr);
2074 2075 2076 2077 2078 2079 2080
  emit(0xD9);
  emit_operand(3, adr);
}


void Assembler::fstp_d(const Operand& adr) {
  EnsureSpace ensure_space(this);
2081
  emit_optional_rex_32(adr);
2082 2083 2084 2085 2086
  emit(0xDD);
  emit_operand(3, adr);
}


2087
void Assembler::fstp(int index) {
2088
  DCHECK(is_uint3(index));
2089 2090 2091 2092 2093
  EnsureSpace ensure_space(this);
  emit_farith(0xDD, 0xD8, index);
}


2094 2095
void Assembler::fild_s(const Operand& adr) {
  EnsureSpace ensure_space(this);
2096
  emit_optional_rex_32(adr);
2097 2098 2099 2100 2101 2102 2103
  emit(0xDB);
  emit_operand(0, adr);
}


void Assembler::fild_d(const Operand& adr) {
  EnsureSpace ensure_space(this);
2104
  emit_optional_rex_32(adr);
2105 2106 2107 2108 2109 2110 2111
  emit(0xDF);
  emit_operand(5, adr);
}


void Assembler::fistp_s(const Operand& adr) {
  EnsureSpace ensure_space(this);
2112
  emit_optional_rex_32(adr);
2113 2114 2115 2116 2117 2118
  emit(0xDB);
  emit_operand(3, adr);
}


void Assembler::fisttp_s(const Operand& adr) {
2119
  DCHECK(IsEnabled(SSE3));
2120
  EnsureSpace ensure_space(this);
2121
  emit_optional_rex_32(adr);
2122 2123 2124 2125 2126
  emit(0xDB);
  emit_operand(1, adr);
}


2127
void Assembler::fisttp_d(const Operand& adr) {
2128
  DCHECK(IsEnabled(SSE3));
2129 2130 2131 2132 2133 2134 2135
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(adr);
  emit(0xDD);
  emit_operand(1, adr);
}


2136 2137
void Assembler::fist_s(const Operand& adr) {
  EnsureSpace ensure_space(this);
2138
  emit_optional_rex_32(adr);
2139 2140 2141 2142 2143 2144 2145
  emit(0xDB);
  emit_operand(2, adr);
}


void Assembler::fistp_d(const Operand& adr) {
  EnsureSpace ensure_space(this);
2146
  emit_optional_rex_32(adr);
2147
  emit(0xDF);
2148
  emit_operand(7, adr);
2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179
}


void Assembler::fabs() {
  EnsureSpace ensure_space(this);
  emit(0xD9);
  emit(0xE1);
}


void Assembler::fchs() {
  EnsureSpace ensure_space(this);
  emit(0xD9);
  emit(0xE0);
}


void Assembler::fcos() {
  EnsureSpace ensure_space(this);
  emit(0xD9);
  emit(0xFF);
}


void Assembler::fsin() {
  EnsureSpace ensure_space(this);
  emit(0xD9);
  emit(0xFE);
}


2180 2181 2182 2183 2184 2185 2186
void Assembler::fptan() {
  EnsureSpace ensure_space(this);
  emit(0xD9);
  emit(0xF2);
}


2187 2188 2189 2190 2191 2192 2193
void Assembler::fyl2x() {
  EnsureSpace ensure_space(this);
  emit(0xD9);
  emit(0xF1);
}


2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208 2209 2210 2211 2212 2213 2214
void Assembler::f2xm1() {
  EnsureSpace ensure_space(this);
  emit(0xD9);
  emit(0xF0);
}


void Assembler::fscale() {
  EnsureSpace ensure_space(this);
  emit(0xD9);
  emit(0xFD);
}


void Assembler::fninit() {
  EnsureSpace ensure_space(this);
  emit(0xDB);
  emit(0xE3);
}


2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228
void Assembler::fadd(int i) {
  EnsureSpace ensure_space(this);
  emit_farith(0xDC, 0xC0, i);
}


void Assembler::fsub(int i) {
  EnsureSpace ensure_space(this);
  emit_farith(0xDC, 0xE8, i);
}


void Assembler::fisub_s(const Operand& adr) {
  EnsureSpace ensure_space(this);
2229
  emit_optional_rex_32(adr);
2230 2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329
  emit(0xDA);
  emit_operand(4, adr);
}


void Assembler::fmul(int i) {
  EnsureSpace ensure_space(this);
  emit_farith(0xDC, 0xC8, i);
}


void Assembler::fdiv(int i) {
  EnsureSpace ensure_space(this);
  emit_farith(0xDC, 0xF8, i);
}


void Assembler::faddp(int i) {
  EnsureSpace ensure_space(this);
  emit_farith(0xDE, 0xC0, i);
}


void Assembler::fsubp(int i) {
  EnsureSpace ensure_space(this);
  emit_farith(0xDE, 0xE8, i);
}


void Assembler::fsubrp(int i) {
  EnsureSpace ensure_space(this);
  emit_farith(0xDE, 0xE0, i);
}


void Assembler::fmulp(int i) {
  EnsureSpace ensure_space(this);
  emit_farith(0xDE, 0xC8, i);
}


void Assembler::fdivp(int i) {
  EnsureSpace ensure_space(this);
  emit_farith(0xDE, 0xF8, i);
}


void Assembler::fprem() {
  EnsureSpace ensure_space(this);
  emit(0xD9);
  emit(0xF8);
}


void Assembler::fprem1() {
  EnsureSpace ensure_space(this);
  emit(0xD9);
  emit(0xF5);
}


void Assembler::fxch(int i) {
  EnsureSpace ensure_space(this);
  emit_farith(0xD9, 0xC8, i);
}


void Assembler::fincstp() {
  EnsureSpace ensure_space(this);
  emit(0xD9);
  emit(0xF7);
}


void Assembler::ffree(int i) {
  EnsureSpace ensure_space(this);
  emit_farith(0xDD, 0xC0, i);
}


void Assembler::ftst() {
  EnsureSpace ensure_space(this);
  emit(0xD9);
  emit(0xE4);
}


void Assembler::fucomp(int i) {
  EnsureSpace ensure_space(this);
  emit_farith(0xDD, 0xE8, i);
}


void Assembler::fucompp() {
  EnsureSpace ensure_space(this);
  emit(0xDA);
  emit(0xE9);
}


2330 2331 2332 2333 2334 2335 2336
void Assembler::fucomi(int i) {
  EnsureSpace ensure_space(this);
  emit(0xDB);
  emit(0xE8 + i);
}


2337 2338 2339 2340 2341 2342 2343
void Assembler::fucomip() {
  EnsureSpace ensure_space(this);
  emit(0xDF);
  emit(0xE9);
}


2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370 2371 2372 2373 2374 2375 2376 2377
void Assembler::fcompp() {
  EnsureSpace ensure_space(this);
  emit(0xDE);
  emit(0xD9);
}


void Assembler::fnstsw_ax() {
  EnsureSpace ensure_space(this);
  emit(0xDF);
  emit(0xE0);
}


void Assembler::fwait() {
  EnsureSpace ensure_space(this);
  emit(0x9B);
}


void Assembler::frndint() {
  EnsureSpace ensure_space(this);
  emit(0xD9);
  emit(0xFC);
}


void Assembler::fnclex() {
  EnsureSpace ensure_space(this);
  emit(0xDB);
  emit(0xE2);
}


2378 2379 2380
void Assembler::sahf() {
  // TODO(X64): Test for presence. Not all 64-bit intel CPU's have sahf
  // in 64-bit mode. Test CpuID.
2381
  DCHECK(IsEnabled(SAHF));
2382 2383 2384 2385 2386
  EnsureSpace ensure_space(this);
  emit(0x9E);
}


2387
void Assembler::emit_farith(int b1, int b2, int i) {
2388 2389
  DCHECK(is_uint8(b1) && is_uint8(b2));  // wrong opcode
  DCHECK(is_uint3(i));  // illegal stack offset
2390 2391 2392 2393
  emit(b1);
  emit(b2 + i);
}

2394

2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405
// SSE operations.

void Assembler::andps(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x54);
  emit_sse_operand(dst, src);
}


2406 2407 2408 2409 2410 2411 2412 2413 2414
void Assembler::andps(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x54);
  emit_sse_operand(dst, src);
}


2415 2416 2417 2418 2419 2420 2421 2422 2423
void Assembler::orps(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x56);
  emit_sse_operand(dst, src);
}


2424 2425 2426 2427 2428 2429 2430 2431 2432
void Assembler::orps(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x56);
  emit_sse_operand(dst, src);
}


2433 2434 2435 2436 2437 2438 2439 2440 2441
void Assembler::xorps(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x57);
  emit_sse_operand(dst, src);
}


2442 2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458 2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522
void Assembler::xorps(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x57);
  emit_sse_operand(dst, src);
}


void Assembler::addps(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x58);
  emit_sse_operand(dst, src);
}


void Assembler::addps(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x58);
  emit_sse_operand(dst, src);
}


void Assembler::subps(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5C);
  emit_sse_operand(dst, src);
}


void Assembler::subps(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5C);
  emit_sse_operand(dst, src);
}


void Assembler::mulps(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x59);
  emit_sse_operand(dst, src);
}


void Assembler::mulps(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x59);
  emit_sse_operand(dst, src);
}


void Assembler::divps(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5E);
  emit_sse_operand(dst, src);
}


void Assembler::divps(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5E);
  emit_sse_operand(dst, src);
}


2523
// SSE 2 operations.
2524

2525 2526 2527 2528 2529 2530 2531 2532 2533 2534
void Assembler::movd(XMMRegister dst, Register src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x6E);
  emit_sse_operand(dst, src);
}


2535 2536 2537 2538 2539 2540 2541 2542 2543 2544
void Assembler::movd(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x6E);
  emit_sse_operand(dst, src);
}


2545 2546 2547
void Assembler::movd(Register dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
2548
  emit_optional_rex_32(src, dst);
2549 2550
  emit(0x0F);
  emit(0x7E);
2551
  emit_sse_operand(src, dst);
2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566 2567
}


void Assembler::movq(XMMRegister dst, Register src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_rex_64(dst, src);
  emit(0x0F);
  emit(0x6E);
  emit_sse_operand(dst, src);
}


void Assembler::movq(Register dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
2568
  emit_rex_64(src, dst);
2569 2570
  emit(0x0F);
  emit(0x7E);
2571
  emit_sse_operand(src, dst);
2572 2573 2574
}


lrn@chromium.org's avatar
lrn@chromium.org committed
2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592
void Assembler::movq(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  if (dst.low_bits() == 4) {
    // Avoid unnecessary SIB byte.
    emit(0xf3);
    emit_optional_rex_32(dst, src);
    emit(0x0F);
    emit(0x7e);
    emit_sse_operand(dst, src);
  } else {
    emit(0x66);
    emit_optional_rex_32(src, dst);
    emit(0x0F);
    emit(0xD6);
    emit_sse_operand(src, dst);
  }
}

2593

2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613
void Assembler::movdqa(const Operand& dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_rex_64(src, dst);
  emit(0x0F);
  emit(0x7F);
  emit_sse_operand(src, dst);
}


void Assembler::movdqa(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_rex_64(dst, src);
  emit(0x0F);
  emit(0x6F);
  emit_sse_operand(dst, src);
}


2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633
void Assembler::movdqu(const Operand& dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_rex_64(src, dst);
  emit(0x0F);
  emit(0x7F);
  emit_sse_operand(src, dst);
}


void Assembler::movdqu(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_rex_64(dst, src);
  emit(0x0F);
  emit(0x6F);
  emit_sse_operand(dst, src);
}


2634
void Assembler::extractps(Register dst, XMMRegister src, byte imm8) {
2635 2636
  DCHECK(IsEnabled(SSE4_1));
  DCHECK(is_uint8(imm8));
2637 2638
  EnsureSpace ensure_space(this);
  emit(0x66);
2639
  emit_optional_rex_32(src, dst);
2640 2641 2642
  emit(0x0F);
  emit(0x3A);
  emit(0x17);
2643
  emit_sse_operand(src, dst);
2644 2645 2646 2647
  emit(imm8);
}


2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686
void Assembler::pextrd(Register dst, XMMRegister src, int8_t imm8) {
  DCHECK(IsEnabled(SSE4_1));
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(src, dst);
  emit(0x0F);
  emit(0x3A);
  emit(0x16);
  emit_sse_operand(src, dst);
  emit(imm8);
}


void Assembler::pinsrd(XMMRegister dst, Register src, int8_t imm8) {
  DCHECK(IsEnabled(SSE4_1));
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x3A);
  emit(0x22);
  emit_sse_operand(dst, src);
  emit(imm8);
}


void Assembler::pinsrd(XMMRegister dst, const Operand& src, int8_t imm8) {
  DCHECK(IsEnabled(SSE4_1));
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x3A);
  emit(0x22);
  emit_sse_operand(dst, src);
  emit(imm8);
}


2687 2688 2689 2690 2691 2692 2693 2694 2695 2696
void Assembler::movsd(const Operand& dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);  // double
  emit_optional_rex_32(src, dst);
  emit(0x0F);
  emit(0x11);  // store
  emit_sse_operand(src, dst);
}


2697
void Assembler::movsd(XMMRegister dst, XMMRegister src) {
2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716
  EnsureSpace ensure_space(this);
  emit(0xF2);  // double
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x10);  // load
  emit_sse_operand(dst, src);
}


void Assembler::movsd(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);  // double
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x10);  // load
  emit_sse_operand(dst, src);
}


lrn@chromium.org's avatar
lrn@chromium.org committed
2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733
void Assembler::movaps(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  if (src.low_bits() == 4) {
    // Try to avoid an unnecessary SIB byte.
    emit_optional_rex_32(src, dst);
    emit(0x0F);
    emit(0x29);
    emit_sse_operand(src, dst);
  } else {
    emit_optional_rex_32(dst, src);
    emit(0x0F);
    emit(0x28);
    emit_sse_operand(dst, src);
  }
}


2734
void Assembler::shufps(XMMRegister dst, XMMRegister src, byte imm8) {
2735
  DCHECK(is_uint8(imm8));
2736 2737 2738 2739 2740 2741 2742 2743 2744
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(src, dst);
  emit(0x0F);
  emit(0xC6);
  emit_sse_operand(dst, src);
  emit(imm8);
}


lrn@chromium.org's avatar
lrn@chromium.org committed
2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763
void Assembler::movapd(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  if (src.low_bits() == 4) {
    // Try to avoid an unnecessary SIB byte.
    emit(0x66);
    emit_optional_rex_32(src, dst);
    emit(0x0F);
    emit(0x29);
    emit_sse_operand(src, dst);
  } else {
    emit(0x66);
    emit_optional_rex_32(dst, src);
    emit(0x0F);
    emit(0x28);
    emit_sse_operand(dst, src);
  }
}


2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843
void Assembler::addss(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x58);
  emit_sse_operand(dst, src);
}


void Assembler::addss(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x58);
  emit_sse_operand(dst, src);
}


void Assembler::subss(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5C);
  emit_sse_operand(dst, src);
}


void Assembler::subss(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5C);
  emit_sse_operand(dst, src);
}


void Assembler::mulss(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x59);
  emit_sse_operand(dst, src);
}


void Assembler::mulss(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x59);
  emit_sse_operand(dst, src);
}


void Assembler::divss(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5E);
  emit_sse_operand(dst, src);
}


void Assembler::divss(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5E);
  emit_sse_operand(dst, src);
}


2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903
void Assembler::maxss(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5F);
  emit_sse_operand(dst, src);
}


void Assembler::maxss(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5F);
  emit_sse_operand(dst, src);
}


void Assembler::minss(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5D);
  emit_sse_operand(dst, src);
}


void Assembler::minss(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5D);
  emit_sse_operand(dst, src);
}


void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x51);
  emit_sse_operand(dst, src);
}


void Assembler::sqrtss(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x51);
  emit_sse_operand(dst, src);
}


2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917 2918 2919 2920 2921
void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0f);
  emit(0x2e);
  emit_sse_operand(dst, src);
}


void Assembler::ucomiss(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0f);
  emit(0x2e);
  emit_sse_operand(dst, src);
}


2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937 2938 2939 2940 2941
void Assembler::movss(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);  // single
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x10);  // load
  emit_sse_operand(dst, src);
}


void Assembler::movss(const Operand& src, XMMRegister dst) {
  EnsureSpace ensure_space(this);
  emit(0xF3);  // single
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x11);  // store
  emit_sse_operand(dst, src);
}


2942 2943 2944
void Assembler::psllq(XMMRegister reg, byte imm8) {
  EnsureSpace ensure_space(this);
  emit(0x66);
2945
  emit_optional_rex_32(reg);
2946 2947 2948 2949 2950 2951 2952
  emit(0x0F);
  emit(0x73);
  emit_sse_operand(rsi, reg);  // rsi == 6
  emit(imm8);
}


2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970 2971 2972 2973 2974 2975 2976 2977 2978 2979 2980 2981 2982 2983 2984 2985
void Assembler::psrlq(XMMRegister reg, byte imm8) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(reg);
  emit(0x0F);
  emit(0x73);
  emit_sse_operand(rdx, reg);  // rdx == 2
  emit(imm8);
}


void Assembler::pslld(XMMRegister reg, byte imm8) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(reg);
  emit(0x0F);
  emit(0x72);
  emit_sse_operand(rsi, reg);  // rsi == 6
  emit(imm8);
}


void Assembler::psrld(XMMRegister reg, byte imm8) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(reg);
  emit(0x0F);
  emit(0x72);
  emit_sse_operand(rdx, reg);  // rdx == 2
  emit(imm8);
}


2986 2987 2988 2989 2990 2991 2992 2993 2994 2995
void Assembler::cvttss2si(Register dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x2C);
  emit_operand(dst, src);
}


2996 2997 2998 2999 3000 3001 3002 3003 3004 3005
void Assembler::cvttss2si(Register dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x2C);
  emit_sse_operand(dst, src);
}


3006 3007 3008 3009 3010 3011 3012 3013 3014 3015
void Assembler::cvttsd2si(Register dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x2C);
  emit_operand(dst, src);
}


3016 3017 3018 3019 3020 3021 3022 3023 3024 3025
void Assembler::cvttsd2si(Register dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x2C);
  emit_sse_operand(dst, src);
}


3026 3027 3028 3029 3030 3031 3032 3033 3034 3035
void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_rex_64(dst, src);
  emit(0x0F);
  emit(0x2C);
  emit_sse_operand(dst, src);
}


3036 3037 3038 3039 3040 3041 3042 3043 3044 3045
void Assembler::cvttsd2siq(Register dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_rex_64(dst, src);
  emit(0x0F);
  emit(0x2C);
  emit_sse_operand(dst, src);
}


3046 3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065
void Assembler::cvtlsi2sd(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x2A);
  emit_sse_operand(dst, src);
}


void Assembler::cvtlsi2sd(XMMRegister dst, Register src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x2A);
  emit_sse_operand(dst, src);
}


3066 3067 3068 3069 3070 3071 3072 3073 3074 3075
void Assembler::cvtlsi2ss(XMMRegister dst, Register src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x2A);
  emit_sse_operand(dst, src);
}


3076 3077 3078 3079 3080 3081 3082 3083 3084 3085
void Assembler::cvtqsi2sd(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_rex_64(dst, src);
  emit(0x0F);
  emit(0x2A);
  emit_sse_operand(dst, src);
}


3086 3087 3088 3089 3090 3091 3092 3093 3094 3095
void Assembler::cvtqsi2sd(XMMRegister dst, Register src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_rex_64(dst, src);
  emit(0x0F);
  emit(0x2A);
  emit_sse_operand(dst, src);
}


3096 3097 3098 3099 3100 3101 3102 3103 3104 3105
void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5A);
  emit_sse_operand(dst, src);
}


3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125
void Assembler::cvtss2sd(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5A);
  emit_sse_operand(dst, src);
}


void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5A);
  emit_sse_operand(dst, src);
}


3126 3127 3128 3129 3130 3131 3132 3133 3134 3135
void Assembler::cvtsd2ss(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5A);
  emit_sse_operand(dst, src);
}


3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155
void Assembler::cvtsd2si(Register dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x2D);
  emit_sse_operand(dst, src);
}


void Assembler::cvtsd2siq(Register dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_rex_64(dst, src);
  emit(0x0F);
  emit(0x2D);
  emit_sse_operand(dst, src);
}


3156 3157 3158 3159 3160 3161 3162 3163 3164 3165
void Assembler::addsd(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x58);
  emit_sse_operand(dst, src);
}


3166 3167 3168 3169 3170 3171 3172 3173 3174 3175
void Assembler::addsd(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x58);
  emit_sse_operand(dst, src);
}


3176 3177 3178 3179 3180 3181 3182 3183 3184 3185
void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x59);
  emit_sse_operand(dst, src);
}


3186 3187 3188 3189 3190 3191 3192 3193 3194 3195
void Assembler::mulsd(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x59);
  emit_sse_operand(dst, src);
}


3196 3197 3198 3199 3200 3201 3202 3203 3204 3205
void Assembler::subsd(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5C);
  emit_sse_operand(dst, src);
}


3206 3207 3208 3209 3210 3211 3212 3213 3214 3215
void Assembler::subsd(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5C);
  emit_sse_operand(dst, src);
}


3216 3217 3218 3219 3220 3221 3222 3223 3224 3225
void Assembler::divsd(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5E);
  emit_sse_operand(dst, src);
}


3226 3227 3228 3229 3230 3231 3232 3233 3234 3235
void Assembler::divsd(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5E);
  emit_sse_operand(dst, src);
}


3236 3237 3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260 3261 3262 3263 3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275
void Assembler::maxsd(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5F);
  emit_sse_operand(dst, src);
}


void Assembler::maxsd(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5F);
  emit_sse_operand(dst, src);
}


void Assembler::minsd(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5D);
  emit_sse_operand(dst, src);
}


void Assembler::minsd(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x5D);
  emit_sse_operand(dst, src);
}


3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292 3293 3294 3295
void Assembler::andpd(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x54);
  emit_sse_operand(dst, src);
}


void Assembler::orpd(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x56);
  emit_sse_operand(dst, src);
}


3296 3297 3298 3299
void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(dst, src);
3300
  emit(0x0F);
lrn@chromium.org's avatar
lrn@chromium.org committed
3301 3302 3303 3304 3305
  emit(0x57);
  emit_sse_operand(dst, src);
}


3306 3307 3308 3309 3310 3311 3312 3313 3314 3315
void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x51);
  emit_sse_operand(dst, src);
}


3316 3317 3318 3319 3320 3321 3322 3323 3324 3325
void Assembler::sqrtsd(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x51);
  emit_sse_operand(dst, src);
}


3326 3327 3328 3329 3330 3331 3332 3333 3334
void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(dst, src);
  emit(0x0f);
  emit(0x2e);
  emit_sse_operand(dst, src);
}

3335

3336 3337 3338 3339 3340 3341 3342 3343 3344 3345
void Assembler::ucomisd(XMMRegister dst, const Operand& src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(dst, src);
  emit(0x0f);
  emit(0x2e);
  emit_sse_operand(dst, src);
}


3346 3347 3348 3349 3350 3351 3352 3353 3354 3355 3356
void Assembler::cmpltsd(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0xF2);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0xC2);
  emit_sse_operand(dst, src);
  emit(0x01);  // LT == 1
}


3357
void Assembler::roundsd(XMMRegister dst, XMMRegister src, RoundingMode mode) {
3358
  DCHECK(IsEnabled(SSE4_1));
3359 3360 3361 3362 3363 3364 3365 3366 3367 3368 3369 3370
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(dst, src);
  emit(0x0f);
  emit(0x3a);
  emit(0x0b);
  emit_sse_operand(dst, src);
  // Mask precision exeption.
  emit(static_cast<byte>(mode) | 0x8);
}


3371 3372 3373 3374 3375 3376 3377 3378 3379
void Assembler::movmskpd(Register dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(dst, src);
  emit(0x0f);
  emit(0x50);
  emit_sse_operand(dst, src);
}

3380

3381 3382 3383 3384 3385 3386 3387 3388 3389
void Assembler::movmskps(Register dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit_optional_rex_32(dst, src);
  emit(0x0f);
  emit(0x50);
  emit_sse_operand(dst, src);
}


3390 3391 3392 3393 3394 3395 3396 3397 3398 3399
void Assembler::pcmpeqd(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x76);
  emit_sse_operand(dst, src);
}


3400 3401 3402 3403 3404 3405 3406 3407 3408 3409 3410 3411 3412 3413 3414 3415 3416 3417 3418 3419
void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x62);
  emit_sse_operand(dst, src);
}


void Assembler::punpckhdq(XMMRegister dst, XMMRegister src) {
  EnsureSpace ensure_space(this);
  emit(0x66);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0x6A);
  emit_sse_operand(dst, src);
}


3420
// AVX instructions
3421 3422 3423 3424
void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1,
                       XMMRegister src2) {
  DCHECK(IsEnabled(FMA3));
  EnsureSpace ensure_space(this);
3425
  emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1);
3426 3427 3428 3429 3430 3431 3432 3433 3434
  emit(op);
  emit_sse_operand(dst, src2);
}


void Assembler::vfmasd(byte op, XMMRegister dst, XMMRegister src1,
                       const Operand& src2) {
  DCHECK(IsEnabled(FMA3));
  EnsureSpace ensure_space(this);
3435
  emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW1);
3436 3437 3438 3439 3440 3441 3442 3443 3444
  emit(op);
  emit_sse_operand(dst, src2);
}


void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1,
                       XMMRegister src2) {
  DCHECK(IsEnabled(FMA3));
  EnsureSpace ensure_space(this);
3445
  emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0);
3446 3447 3448 3449 3450 3451 3452 3453 3454
  emit(op);
  emit_sse_operand(dst, src2);
}


void Assembler::vfmass(byte op, XMMRegister dst, XMMRegister src1,
                       const Operand& src2) {
  DCHECK(IsEnabled(FMA3));
  EnsureSpace ensure_space(this);
3455 3456 3457 3458 3459 3460
  emit_vex_prefix(dst, src1, src2, kLIG, k66, k0F38, kW0);
  emit(op);
  emit_sse_operand(dst, src2);
}


3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478
void Assembler::vucomisd(XMMRegister dst, XMMRegister src) {
  DCHECK(IsEnabled(AVX));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(dst, xmm0, src, kLIG, k66, k0F, kWIG);
  emit(0x2e);
  emit_sse_operand(dst, src);
}


void Assembler::vucomisd(XMMRegister dst, const Operand& src) {
  DCHECK(IsEnabled(AVX));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(dst, xmm0, src, kLIG, k66, k0F, kWIG);
  emit(0x2e);
  emit_sse_operand(dst, src);
}


3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489 3490 3491 3492 3493
void Assembler::vsd(byte op, XMMRegister dst, XMMRegister src1,
                    XMMRegister src2) {
  DCHECK(IsEnabled(AVX));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(dst, src1, src2, kLIG, kF2, k0F, kWIG);
  emit(op);
  emit_sse_operand(dst, src2);
}


void Assembler::vsd(byte op, XMMRegister dst, XMMRegister src1,
                    const Operand& src2) {
  DCHECK(IsEnabled(AVX));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(dst, src1, src2, kLIG, kF2, k0F, kWIG);
3494 3495 3496 3497 3498
  emit(op);
  emit_sse_operand(dst, src2);
}


3499 3500 3501 3502 3503 3504 3505 3506 3507 3508 3509 3510 3511 3512 3513 3514 3515 3516 3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536 3537 3538
void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
                    XMMRegister src2) {
  DCHECK(IsEnabled(AVX));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
  emit(op);
  emit_sse_operand(dst, src2);
}


void Assembler::vps(byte op, XMMRegister dst, XMMRegister src1,
                    const Operand& src2) {
  DCHECK(IsEnabled(AVX));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(dst, src1, src2, kL128, kNone, k0F, kWIG);
  emit(op);
  emit_sse_operand(dst, src2);
}


void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1,
                    XMMRegister src2) {
  DCHECK(IsEnabled(AVX));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG);
  emit(op);
  emit_sse_operand(dst, src2);
}


void Assembler::vpd(byte op, XMMRegister dst, XMMRegister src1,
                    const Operand& src2) {
  DCHECK(IsEnabled(AVX));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(dst, src1, src2, kL128, k66, k0F, kWIG);
  emit(op);
  emit_sse_operand(dst, src2);
}


3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564 3565 3566 3567 3568 3569 3570 3571 3572 3573 3574 3575 3576
void Assembler::vucomiss(XMMRegister dst, XMMRegister src) {
  DCHECK(IsEnabled(AVX));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG);
  emit(0x2e);
  emit_sse_operand(dst, src);
}


void Assembler::vucomiss(XMMRegister dst, const Operand& src) {
  DCHECK(IsEnabled(AVX));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(dst, xmm0, src, kLIG, kNone, k0F, kWIG);
  emit(0x2e);
  emit_sse_operand(dst, src);
}


void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1,
                    XMMRegister src2) {
  DCHECK(IsEnabled(AVX));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
  emit(op);
  emit_sse_operand(dst, src2);
}


void Assembler::vss(byte op, XMMRegister dst, XMMRegister src1,
                    const Operand& src2) {
  DCHECK(IsEnabled(AVX));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(dst, src1, src2, kLIG, kF3, k0F, kWIG);
  emit(op);
  emit_sse_operand(dst, src2);
}


3577 3578 3579 3580 3581 3582 3583 3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659 3660 3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686 3687 3688 3689 3690 3691 3692 3693 3694 3695 3696 3697 3698 3699 3700 3701 3702 3703 3704 3705 3706 3707 3708 3709 3710 3711 3712 3713 3714 3715 3716 3717 3718 3719 3720 3721 3722 3723 3724 3725 3726 3727 3728 3729 3730 3731 3732 3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753 3754 3755 3756 3757 3758 3759 3760 3761 3762 3763 3764 3765 3766 3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778 3779 3780 3781 3782 3783 3784 3785 3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816 3817 3818 3819 3820 3821 3822 3823 3824 3825 3826 3827 3828 3829 3830 3831 3832
void Assembler::bmi1q(byte op, Register reg, Register vreg, Register rm) {
  DCHECK(IsEnabled(BMI1));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1);
  emit(op);
  emit_modrm(reg, rm);
}


void Assembler::bmi1q(byte op, Register reg, Register vreg, const Operand& rm) {
  DCHECK(IsEnabled(BMI1));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW1);
  emit(op);
  emit_operand(reg, rm);
}


void Assembler::bmi1l(byte op, Register reg, Register vreg, Register rm) {
  DCHECK(IsEnabled(BMI1));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0);
  emit(op);
  emit_modrm(reg, rm);
}


void Assembler::bmi1l(byte op, Register reg, Register vreg, const Operand& rm) {
  DCHECK(IsEnabled(BMI1));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(reg, vreg, rm, kLZ, kNone, k0F38, kW0);
  emit(op);
  emit_operand(reg, rm);
}


void Assembler::tzcntq(Register dst, Register src) {
  DCHECK(IsEnabled(BMI1));
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_rex_64(dst, src);
  emit(0x0F);
  emit(0xBC);
  emit_modrm(dst, src);
}


void Assembler::tzcntq(Register dst, const Operand& src) {
  DCHECK(IsEnabled(BMI1));
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_rex_64(dst, src);
  emit(0x0F);
  emit(0xBC);
  emit_operand(dst, src);
}


void Assembler::tzcntl(Register dst, Register src) {
  DCHECK(IsEnabled(BMI1));
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0xBC);
  emit_modrm(dst, src);
}


void Assembler::tzcntl(Register dst, const Operand& src) {
  DCHECK(IsEnabled(BMI1));
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0xBC);
  emit_operand(dst, src);
}


void Assembler::lzcntq(Register dst, Register src) {
  DCHECK(IsEnabled(LZCNT));
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_rex_64(dst, src);
  emit(0x0F);
  emit(0xBD);
  emit_modrm(dst, src);
}


void Assembler::lzcntq(Register dst, const Operand& src) {
  DCHECK(IsEnabled(LZCNT));
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_rex_64(dst, src);
  emit(0x0F);
  emit(0xBD);
  emit_operand(dst, src);
}


void Assembler::lzcntl(Register dst, Register src) {
  DCHECK(IsEnabled(LZCNT));
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0xBD);
  emit_modrm(dst, src);
}


void Assembler::lzcntl(Register dst, const Operand& src) {
  DCHECK(IsEnabled(LZCNT));
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0xBD);
  emit_operand(dst, src);
}


void Assembler::popcntq(Register dst, Register src) {
  DCHECK(IsEnabled(POPCNT));
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_rex_64(dst, src);
  emit(0x0F);
  emit(0xB8);
  emit_modrm(dst, src);
}


void Assembler::popcntq(Register dst, const Operand& src) {
  DCHECK(IsEnabled(POPCNT));
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_rex_64(dst, src);
  emit(0x0F);
  emit(0xB8);
  emit_operand(dst, src);
}


void Assembler::popcntl(Register dst, Register src) {
  DCHECK(IsEnabled(POPCNT));
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0xB8);
  emit_modrm(dst, src);
}


void Assembler::popcntl(Register dst, const Operand& src) {
  DCHECK(IsEnabled(POPCNT));
  EnsureSpace ensure_space(this);
  emit(0xF3);
  emit_optional_rex_32(dst, src);
  emit(0x0F);
  emit(0xB8);
  emit_operand(dst, src);
}


void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
                      Register rm) {
  DCHECK(IsEnabled(BMI2));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
  emit(op);
  emit_modrm(reg, rm);
}


void Assembler::bmi2q(SIMDPrefix pp, byte op, Register reg, Register vreg,
                      const Operand& rm) {
  DCHECK(IsEnabled(BMI2));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW1);
  emit(op);
  emit_operand(reg, rm);
}


void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
                      Register rm) {
  DCHECK(IsEnabled(BMI2));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
  emit(op);
  emit_modrm(reg, rm);
}


void Assembler::bmi2l(SIMDPrefix pp, byte op, Register reg, Register vreg,
                      const Operand& rm) {
  DCHECK(IsEnabled(BMI2));
  EnsureSpace ensure_space(this);
  emit_vex_prefix(reg, vreg, rm, kLZ, pp, k0F38, kW0);
  emit(op);
  emit_operand(reg, rm);
}


void Assembler::rorxq(Register dst, Register src, byte imm8) {
  DCHECK(IsEnabled(BMI2));
  DCHECK(is_uint8(imm8));
  Register vreg = {0};  // VEX.vvvv unused
  EnsureSpace ensure_space(this);
  emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
  emit(0xF0);
  emit_modrm(dst, src);
  emit(imm8);
}


void Assembler::rorxq(Register dst, const Operand& src, byte imm8) {
  DCHECK(IsEnabled(BMI2));
  DCHECK(is_uint8(imm8));
  Register vreg = {0};  // VEX.vvvv unused
  EnsureSpace ensure_space(this);
  emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW1);
  emit(0xF0);
  emit_operand(dst, src);
  emit(imm8);
}


void Assembler::rorxl(Register dst, Register src, byte imm8) {
  DCHECK(IsEnabled(BMI2));
  DCHECK(is_uint8(imm8));
  Register vreg = {0};  // VEX.vvvv unused
  EnsureSpace ensure_space(this);
  emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
  emit(0xF0);
  emit_modrm(dst, src);
  emit(imm8);
}


void Assembler::rorxl(Register dst, const Operand& src, byte imm8) {
  DCHECK(IsEnabled(BMI2));
  DCHECK(is_uint8(imm8));
  Register vreg = {0};  // VEX.vvvv unused
  EnsureSpace ensure_space(this);
  emit_vex_prefix(dst, vreg, src, kLZ, kF2, k0F3A, kW0);
  emit(0xF0);
  emit_operand(dst, src);
  emit(imm8);
}


3833 3834 3835 3836 3837 3838
void Assembler::emit_sse_operand(XMMRegister reg, const Operand& adr) {
  Register ireg = { reg.code() };
  emit_operand(ireg, adr);
}


3839 3840 3841 3842 3843 3844
void Assembler::emit_sse_operand(Register reg, const Operand& adr) {
  Register ireg = {reg.code()};
  emit_operand(ireg, adr);
}


3845
void Assembler::emit_sse_operand(XMMRegister dst, XMMRegister src) {
3846
  emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
3847 3848
}

3849

3850
void Assembler::emit_sse_operand(XMMRegister dst, Register src) {
3851
  emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
3852 3853
}

3854

3855 3856 3857 3858
void Assembler::emit_sse_operand(Register dst, XMMRegister src) {
  emit(0xC0 | (dst.low_bits() << 3) | src.low_bits());
}

3859

3860 3861 3862 3863 3864 3865
void Assembler::db(uint8_t data) {
  EnsureSpace ensure_space(this);
  emit(data);
}


3866 3867 3868 3869 3870 3871
void Assembler::dd(uint32_t data) {
  EnsureSpace ensure_space(this);
  emitl(data);
}


3872 3873 3874 3875 3876 3877
void Assembler::dq(uint64_t data) {
  EnsureSpace ensure_space(this);
  emitq(data);
}


3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898
void Assembler::dq(Label* label) {
  EnsureSpace ensure_space(this);
  if (label->is_bound()) {
    internal_reference_positions_.push_back(pc_offset());
    emitp(buffer_ + label->pos(), RelocInfo::INTERNAL_REFERENCE);
  } else {
    RecordRelocInfo(RelocInfo::INTERNAL_REFERENCE);
    emitl(0);  // Zero for the first 32bit marks it as 64bit absolute address.
    if (label->is_linked()) {
      emitl(label->pos());
      label->link_to(pc_offset() - sizeof(int32_t));
    } else {
      DCHECK(label->is_unused());
      int32_t current = pc_offset();
      emitl(current);
      label->link_to(current);
    }
  }
}


3899
// Relocation information implementations.
3900 3901

void Assembler::RecordRelocInfo(RelocInfo::Mode rmode, intptr_t data) {
3902
  DCHECK(!RelocInfo::IsNone(rmode));
3903 3904 3905 3906
  // Don't record external references unless the heap will be serialized.
  if (rmode == RelocInfo::EXTERNAL_REFERENCE &&
      !serializer_enabled() && !emit_debug_code()) {
    return;
3907 3908 3909
  } else if (rmode == RelocInfo::CODE_AGE_SEQUENCE) {
    // Don't record psuedo relocation info for code age sequence mode.
    return;
3910
  }
3911
  RelocInfo rinfo(pc_, rmode, data, NULL);
3912 3913 3914
  reloc_info_writer.Write(&rinfo);
}

3915

3916
const int RelocInfo::kApplyMask = RelocInfo::kCodeTargetMask |
3917
    1 << RelocInfo::RUNTIME_ENTRY |
3918 3919
    1 << RelocInfo::INTERNAL_REFERENCE |
    1 << RelocInfo::CODE_AGE_SEQUENCE;
3920

3921 3922 3923 3924 3925 3926 3927 3928

bool RelocInfo::IsCodedSpecially() {
  // The deserializer needs to know whether a pointer is specially coded.  Being
  // specially coded on x64 means that it is a relative 32 bit address, as used
  // by branch instructions.
  return (1 << rmode_) & kApplyMask;
}

3929 3930 3931 3932 3933 3934

bool RelocInfo::IsInConstantPool() {
  return false;
}


3935 3936
}  // namespace internal
}  // namespace v8
3937 3938

#endif  // V8_TARGET_ARCH_X64