instruction-selector-ia32.cc 117 KB
Newer Older
1 2 3 4
// Copyright 2014 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5 6 7 8 9 10 11 12
#include <stddef.h>
#include <stdint.h>

#include <limits>
#include <type_traits>
#include <vector>

#include "src/base/flags.h"
13
#include "src/base/iterator.h"
14 15
#include "src/base/logging.h"
#include "src/base/macros.h"
16
#include "src/base/platform/wrappers.h"
17 18 19 20 21 22 23
#include "src/codegen/cpu-features.h"
#include "src/codegen/ia32/assembler-ia32.h"
#include "src/codegen/ia32/register-ia32.h"
#include "src/codegen/machine-type.h"
#include "src/codegen/turbo-assembler.h"
#include "src/common/globals.h"
#include "src/compiler/backend/instruction-codes.h"
24
#include "src/compiler/backend/instruction-selector-impl.h"
25 26 27 28 29 30 31
#include "src/compiler/backend/instruction-selector.h"
#include "src/compiler/backend/instruction.h"
#include "src/compiler/common-operator.h"
#include "src/compiler/frame.h"
#include "src/compiler/globals.h"
#include "src/compiler/linkage.h"
#include "src/compiler/machine-operator.h"
32
#include "src/compiler/node-matchers.h"
33
#include "src/compiler/node-properties.h"
34 35 36 37 38 39
#include "src/compiler/node.h"
#include "src/compiler/opcodes.h"
#include "src/compiler/operator.h"
#include "src/compiler/write-barrier-kind.h"
#include "src/flags/flags.h"
#include "src/utils/utils.h"
40
#include "src/zone/zone-containers.h"
41

42 43 44 45
#if V8_ENABLE_WEBASSEMBLY
#include "src/wasm/simd-shuffle.h"
#endif  // V8_ENABLE_WEBASSEMBLY

46 47 48 49 50
namespace v8 {
namespace internal {
namespace compiler {

// Adds IA32-specific methods for generating operands.
51
class IA32OperandGenerator final : public OperandGenerator {
52 53 54 55
 public:
  explicit IA32OperandGenerator(InstructionSelector* selector)
      : OperandGenerator(selector) {}

56
  InstructionOperand UseByteRegister(Node* node) {
57
    // TODO(titzer): encode byte register use constraints.
58 59 60
    return UseFixed(node, edx);
  }

61 62 63 64 65
  InstructionOperand DefineAsByteRegister(Node* node) {
    // TODO(titzer): encode byte register def constraints.
    return DefineAsRegister(node);
  }

66 67
  bool CanBeMemoryOperand(InstructionCode opcode, Node* node, Node* input,
                          int effect_level) {
68 69
    if ((input->opcode() != IrOpcode::kLoad &&
         input->opcode() != IrOpcode::kLoadImmutable) ||
70 71 72
        !selector()->CanCover(node, input)) {
      return false;
    }
73 74 75
    if (effect_level != selector()->GetEffectLevel(input)) {
      return false;
    }
76 77 78
    MachineRepresentation rep =
        LoadRepresentationOf(input->op()).representation();
    switch (opcode) {
79 80 81 82 83
      case kIA32And:
      case kIA32Or:
      case kIA32Xor:
      case kIA32Add:
      case kIA32Sub:
84 85
      case kIA32Cmp:
      case kIA32Test:
86
        return rep == MachineRepresentation::kWord32 || IsAnyTagged(rep);
87 88 89 90 91 92 93 94 95 96 97 98
      case kIA32Cmp16:
      case kIA32Test16:
        return rep == MachineRepresentation::kWord16;
      case kIA32Cmp8:
      case kIA32Test8:
        return rep == MachineRepresentation::kWord8;
      default:
        break;
    }
    return false;
  }

99 100 101 102 103
  bool CanBeImmediate(Node* node) {
    switch (node->opcode()) {
      case IrOpcode::kInt32Constant:
      case IrOpcode::kNumberConstant:
      case IrOpcode::kExternalConstant:
104 105
      case IrOpcode::kRelocatableInt32Constant:
      case IrOpcode::kRelocatableInt64Constant:
106 107
        return true;
      case IrOpcode::kHeapConstant: {
108 109 110 111
// TODO(bmeurer): We must not dereference handles concurrently. If we
// really have to this here, then we need to find a way to put this
// information on the HeapConstant node already.
#if 0
112 113 114
        // Constants in young generation cannot be used as immediates in V8
        // because the GC does not scan code objects when collecting the young
        // generation.
115
        Handle<HeapObject> value = HeapConstantOf(node->op());
116
        return !Heap::InYoungGeneration(*value);
117 118
#else
        return false;
119
#endif
120 121 122 123 124
      }
      default:
        return false;
    }
  }
125

126
  AddressingMode GenerateMemoryOperandInputs(
127
      Node* index, int scale, Node* base, int32_t displacement,
128 129
      DisplacementMode displacement_mode, InstructionOperand inputs[],
      size_t* input_count, RegisterMode register_mode = kRegister) {
130
    AddressingMode mode = kMode_MRI;
131 132 133
    if (displacement_mode == kNegativeDisplacement) {
      displacement = -displacement;
    }
134
    if (base != nullptr) {
135
      if (base->opcode() == IrOpcode::kInt32Constant) {
136
        displacement += OpParameter<int32_t>(base->op());
137
        base = nullptr;
138 139
      }
    }
140
    if (base != nullptr) {
141
      inputs[(*input_count)++] = UseRegisterWithMode(base, register_mode);
142
      if (index != nullptr) {
143
        DCHECK(scale >= 0 && scale <= 3);
144
        inputs[(*input_count)++] = UseRegisterWithMode(index, register_mode);
145 146 147 148 149 150 151 152 153 154
        if (displacement != 0) {
          inputs[(*input_count)++] = TempImmediate(displacement);
          static const AddressingMode kMRnI_modes[] = {kMode_MR1I, kMode_MR2I,
                                                       kMode_MR4I, kMode_MR8I};
          mode = kMRnI_modes[scale];
        } else {
          static const AddressingMode kMRn_modes[] = {kMode_MR1, kMode_MR2,
                                                      kMode_MR4, kMode_MR8};
          mode = kMRn_modes[scale];
        }
155 156
      } else {
        if (displacement == 0) {
157
          mode = kMode_MR;
158
        } else {
159 160
          inputs[(*input_count)++] = TempImmediate(displacement);
          mode = kMode_MRI;
161 162 163
        }
      }
    } else {
164
      DCHECK(scale >= 0 && scale <= 3);
165
      if (index != nullptr) {
166
        inputs[(*input_count)++] = UseRegisterWithMode(index, register_mode);
167 168 169 170 171
        if (displacement != 0) {
          inputs[(*input_count)++] = TempImmediate(displacement);
          static const AddressingMode kMnI_modes[] = {kMode_MRI, kMode_M2I,
                                                      kMode_M4I, kMode_M8I};
          mode = kMnI_modes[scale];
172
        } else {
173 174 175
          static const AddressingMode kMn_modes[] = {kMode_MR, kMode_M2,
                                                     kMode_M4, kMode_M8};
          mode = kMn_modes[scale];
176 177
        }
      } else {
178 179
        inputs[(*input_count)++] = TempImmediate(displacement);
        return kMode_MI;
180 181
      }
    }
182
    return mode;
183 184
  }

185 186 187 188 189 190 191 192 193 194 195 196
  AddressingMode GenerateMemoryOperandInputs(
      Node* index, int scale, Node* base, Node* displacement_node,
      DisplacementMode displacement_mode, InstructionOperand inputs[],
      size_t* input_count, RegisterMode register_mode = kRegister) {
    int32_t displacement = (displacement_node == nullptr)
                               ? 0
                               : OpParameter<int32_t>(displacement_node->op());
    return GenerateMemoryOperandInputs(index, scale, base, displacement,
                                       displacement_mode, inputs, input_count,
                                       register_mode);
  }

197 198 199
  AddressingMode GetEffectiveAddressMemoryOperand(
      Node* node, InstructionOperand inputs[], size_t* input_count,
      RegisterMode register_mode = kRegister) {
200 201
    {
      LoadMatcher<ExternalReferenceMatcher> m(node);
202 203 204
      if (m.index().HasResolvedValue() && m.object().HasResolvedValue() &&
          selector()->CanAddressRelativeToRootsRegister(
              m.object().ResolvedValue())) {
205
        ptrdiff_t const delta =
206
            m.index().ResolvedValue() +
207
            TurboAssemblerBase::RootRegisterOffsetForExternalReference(
208
                selector()->isolate(), m.object().ResolvedValue());
209 210 211 212 213 214 215
        if (is_int32(delta)) {
          inputs[(*input_count)++] = TempImmediate(static_cast<int32_t>(delta));
          return kMode_Root;
        }
      }
    }

216
    BaseWithIndexAndDisplacement32Matcher m(node, AddressOption::kAllowAll);
217
    DCHECK(m.matches());
218
    if ((m.displacement() == nullptr || CanBeImmediate(m.displacement()))) {
219 220
      return GenerateMemoryOperandInputs(
          m.index(), m.scale(), m.base(), m.displacement(),
221
          m.displacement_mode(), inputs, input_count, register_mode);
222
    } else {
223 224 225 226
      inputs[(*input_count)++] =
          UseRegisterWithMode(node->InputAt(0), register_mode);
      inputs[(*input_count)++] =
          UseRegisterWithMode(node->InputAt(1), register_mode);
227
      return kMode_MR1;
228 229 230
    }
  }

231 232 233 234 235 236 237 238 239 240 241
  InstructionOperand GetEffectiveIndexOperand(Node* index,
                                              AddressingMode* mode) {
    if (CanBeImmediate(index)) {
      *mode = kMode_MRI;
      return UseImmediate(index);
    } else {
      *mode = kMode_MR1;
      return UseUniqueRegister(index);
    }
  }

242 243 244
  bool CanBeBetterLeftOperand(Node* node) const {
    return !selector()->IsLive(node);
  }
245 246
};

247 248
namespace {

249
void VisitRO(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
250
  IA32OperandGenerator g(selector);
251 252 253 254
  Node* input = node->InputAt(0);
  // We have to use a byte register as input to movsxb.
  InstructionOperand input_op =
      opcode == kIA32Movsxbl ? g.UseFixed(input, eax) : g.Use(input);
255 256 257 258 259 260 261 262 263
  selector->Emit(opcode, g.DefineAsRegister(node), input_op);
}

void VisitROWithTemp(InstructionSelector* selector, Node* node,
                     ArchOpcode opcode) {
  IA32OperandGenerator g(selector);
  InstructionOperand temps[] = {g.TempRegister()};
  selector->Emit(opcode, g.DefineAsRegister(node), g.Use(node->InputAt(0)),
                 arraysize(temps), temps);
264 265
}

266 267 268 269 270 271 272 273 274
void VisitROWithTempSimd(InstructionSelector* selector, Node* node,
                         ArchOpcode opcode) {
  IA32OperandGenerator g(selector);
  InstructionOperand temps[] = {g.TempSimd128Register()};
  selector->Emit(opcode, g.DefineAsRegister(node),
                 g.UseUniqueRegister(node->InputAt(0)), arraysize(temps),
                 temps);
}

275 276
void VisitRR(InstructionSelector* selector, Node* node,
             InstructionCode opcode) {
277 278 279 280 281
  IA32OperandGenerator g(selector);
  selector->Emit(opcode, g.DefineAsRegister(node),
                 g.UseRegister(node->InputAt(0)));
}

282 283 284 285 286 287
void VisitRROFloat(InstructionSelector* selector, Node* node,
                   ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
  IA32OperandGenerator g(selector);
  InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
  InstructionOperand operand1 = g.Use(node->InputAt(1));
  if (selector->IsSupported(AVX)) {
288
    selector->Emit(avx_opcode, g.DefineAsRegister(node), operand0, operand1);
289 290 291 292 293
  } else {
    selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0, operand1);
  }
}

294 295 296
void VisitFloatUnop(InstructionSelector* selector, Node* node, Node* input,
                    ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
  IA32OperandGenerator g(selector);
297
  InstructionOperand temps[] = {g.TempSimd128Register()};
298
  if (selector->IsSupported(AVX)) {
299 300
    selector->Emit(avx_opcode, g.DefineAsRegister(node), g.UseUnique(input),
                   arraysize(temps), temps);
301
  } else {
302 303
    selector->Emit(sse_opcode, g.DefineSameAsFirst(node),
                   g.UseUniqueRegister(input), arraysize(temps), temps);
304 305 306
  }
}

307 308 309 310 311 312 313 314 315 316 317
void VisitRRSimd(InstructionSelector* selector, Node* node,
                 ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
  IA32OperandGenerator g(selector);
  InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
  if (selector->IsSupported(AVX)) {
    selector->Emit(avx_opcode, g.DefineAsRegister(node), operand0);
  } else {
    selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0);
  }
}

318 319 320 321
void VisitRRSimd(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
  VisitRRSimd(selector, node, opcode, opcode);
}

322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337
// TODO(v8:9198): Like VisitRROFloat, but for SIMD. SSE requires operand1 to be
// a register as we don't have memory alignment yet. For AVX, memory operands
// are fine, but can have performance issues if not aligned to 16/32 bytes
// (based on load size), see SDM Vol 1, chapter 14.9
void VisitRROSimd(InstructionSelector* selector, Node* node,
                  ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
  IA32OperandGenerator g(selector);
  InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
  if (selector->IsSupported(AVX)) {
    selector->Emit(avx_opcode, g.DefineAsRegister(node), operand0,
                   g.Use(node->InputAt(1)));
  } else {
    selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0,
                   g.UseRegister(node->InputAt(1)));
  }
}
338 339 340 341 342 343 344 345 346 347
void VisitRRRSimd(InstructionSelector* selector, Node* node,
                  ArchOpcode opcode) {
  IA32OperandGenerator g(selector);
  InstructionOperand dst = selector->IsSupported(AVX)
                               ? g.DefineAsRegister(node)
                               : g.DefineSameAsFirst(node);
  InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
  InstructionOperand operand1 = g.UseRegister(node->InputAt(1));
  selector->Emit(opcode, dst, operand0, operand1);
}
348

349 350 351 352 353 354
void VisitRRISimd(InstructionSelector* selector, Node* node,
                  ArchOpcode opcode) {
  IA32OperandGenerator g(selector);
  InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
  InstructionOperand operand1 =
      g.UseImmediate(OpParameter<int32_t>(node->op()));
355 356
  // 8x16 uses movsx_b on dest to extract a byte, which only works
  // if dest is a byte register.
357
  InstructionOperand dest = opcode == kIA32I8x16ExtractLaneS
358 359 360
                                ? g.DefineAsFixed(node, eax)
                                : g.DefineAsRegister(node);
  selector->Emit(opcode, dest, operand0, operand1);
361 362
}

363 364 365 366
void VisitRRISimd(InstructionSelector* selector, Node* node,
                  ArchOpcode avx_opcode, ArchOpcode sse_opcode) {
  IA32OperandGenerator g(selector);
  InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
367 368
  InstructionOperand operand1 =
      g.UseImmediate(OpParameter<int32_t>(node->op()));
369
  if (selector->IsSupported(AVX)) {
370
    selector->Emit(avx_opcode, g.DefineAsRegister(node), operand0, operand1);
371
  } else {
372
    selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0, operand1);
373 374
  }
}
375

376
void VisitRROSimdShift(InstructionSelector* selector, Node* node,
377
                       ArchOpcode opcode) {
378
  IA32OperandGenerator g(selector);
379 380 381 382 383 384 385
  if (g.CanBeImmediate(node->InputAt(1))) {
    selector->Emit(opcode, g.DefineSameAsFirst(node),
                   g.UseRegister(node->InputAt(0)),
                   g.UseImmediate(node->InputAt(1)));
  } else {
    InstructionOperand operand0 = g.UseUniqueRegister(node->InputAt(0));
    InstructionOperand operand1 = g.UseUniqueRegister(node->InputAt(1));
386
    InstructionOperand temps[] = {g.TempSimd128Register(), g.TempRegister()};
387 388 389
    selector->Emit(opcode, g.DefineSameAsFirst(node), operand0, operand1,
                   arraysize(temps), temps);
  }
390 391
}

392 393
void VisitRROI8x16SimdShift(InstructionSelector* selector, Node* node,
                            ArchOpcode opcode) {
394 395 396 397 398 399 400
  IA32OperandGenerator g(selector);
  InstructionOperand operand0 = g.UseUniqueRegister(node->InputAt(0));
  InstructionOperand operand1 = g.UseUniqueRegister(node->InputAt(1));
  InstructionOperand temps[] = {g.TempRegister(), g.TempSimd128Register()};
  selector->Emit(opcode, g.DefineSameAsFirst(node), operand0, operand1,
                 arraysize(temps), temps);
}
401 402
}  // namespace

403 404
void InstructionSelector::VisitStackSlot(Node* node) {
  StackSlotRepresentation rep = StackSlotRepresentationOf(node->op());
405
  int slot = frame_->AllocateSpillSlot(rep.size(), rep.alignment());
406 407 408 409 410
  OperandGenerator g(this);

  Emit(kArchStackSlot, g.DefineAsRegister(node),
       sequence()->AddImmediate(Constant(slot)), 0, nullptr);
}
411

412 413 414
void InstructionSelector::VisitAbortCSAAssert(Node* node) {
  IA32OperandGenerator g(this);
  Emit(kArchAbortCSAAssert, g.NoOutput(), g.UseFixed(node->InputAt(0), edx));
415 416
}

417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438
void InstructionSelector::VisitLoadLane(Node* node) {
  LoadLaneParameters params = LoadLaneParametersOf(node->op());
  InstructionCode opcode = kArchNop;
  if (params.rep == MachineType::Int8()) {
    opcode = kIA32Pinsrb;
  } else if (params.rep == MachineType::Int16()) {
    opcode = kIA32Pinsrw;
  } else if (params.rep == MachineType::Int32()) {
    opcode = kIA32Pinsrd;
  } else if (params.rep == MachineType::Int64()) {
    // pinsrq not available on IA32.
    if (params.laneidx == 0) {
      opcode = kIA32Movlps;
    } else {
      DCHECK_EQ(1, params.laneidx);
      opcode = kIA32Movhps;
    }
  } else {
    UNREACHABLE();
  }

  IA32OperandGenerator g(this);
439 440
  InstructionOperand outputs[] = {IsSupported(AVX) ? g.DefineAsRegister(node)
                                                   : g.DefineSameAsFirst(node)};
441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463
  // Input 0 is value node, 1 is lane idx, and GetEffectiveAddressMemoryOperand
  // uses up to 3 inputs. This ordering is consistent with other operations that
  // use the same opcode.
  InstructionOperand inputs[5];
  size_t input_count = 0;

  inputs[input_count++] = g.UseRegister(node->InputAt(2));
  inputs[input_count++] = g.UseImmediate(params.laneidx);

  AddressingMode mode =
      g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
  opcode |= AddressingModeField::encode(mode);

  DCHECK_GE(5, input_count);

  // IA32 supports unaligned loads.
  DCHECK_NE(params.kind, MemoryAccessKind::kUnaligned);
  // Trap handler is not supported on IA32.
  DCHECK_NE(params.kind, MemoryAccessKind::kProtected);

  Emit(opcode, 1, outputs, input_count, inputs);
}

464 465
void InstructionSelector::VisitLoadTransform(Node* node) {
  LoadTransformParameters params = LoadTransformParametersOf(node->op());
466
  InstructionCode opcode;
467
  switch (params.transformation) {
468 469
    case LoadTransformation::kS128Load8Splat:
      opcode = kIA32S128Load8Splat;
470
      break;
471 472
    case LoadTransformation::kS128Load16Splat:
      opcode = kIA32S128Load16Splat;
473
      break;
474 475
    case LoadTransformation::kS128Load32Splat:
      opcode = kIA32S128Load32Splat;
476
      break;
477 478
    case LoadTransformation::kS128Load64Splat:
      opcode = kIA32S128Load64Splat;
479
      break;
480 481
    case LoadTransformation::kS128Load8x8S:
      opcode = kIA32S128Load8x8S;
482
      break;
483 484
    case LoadTransformation::kS128Load8x8U:
      opcode = kIA32S128Load8x8U;
485
      break;
486 487
    case LoadTransformation::kS128Load16x4S:
      opcode = kIA32S128Load16x4S;
488
      break;
489 490
    case LoadTransformation::kS128Load16x4U:
      opcode = kIA32S128Load16x4U;
491
      break;
492 493
    case LoadTransformation::kS128Load32x2S:
      opcode = kIA32S128Load32x2S;
494
      break;
495 496
    case LoadTransformation::kS128Load32x2U:
      opcode = kIA32S128Load32x2U;
497
      break;
498 499 500 501 502 503
    case LoadTransformation::kS128Load32Zero:
      opcode = kIA32Movss;
      break;
    case LoadTransformation::kS128Load64Zero:
      opcode = kIA32Movsd;
      break;
504 505 506 507 508
    default:
      UNREACHABLE();
  }

  // IA32 supports unaligned loads.
509
  DCHECK_NE(params.kind, MemoryAccessKind::kUnaligned);
510
  // Trap handler is not supported on IA32.
511
  DCHECK_NE(params.kind, MemoryAccessKind::kProtected);
512 513 514 515 516 517 518 519 520 521 522 523

  IA32OperandGenerator g(this);
  InstructionOperand outputs[1];
  outputs[0] = g.DefineAsRegister(node);
  InstructionOperand inputs[3];
  size_t input_count = 0;
  AddressingMode mode =
      g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
  InstructionCode code = opcode | AddressingModeField::encode(mode);
  Emit(code, 1, outputs, input_count, inputs);
}

524
void InstructionSelector::VisitLoad(Node* node) {
525
  LoadRepresentation load_rep = LoadRepresentationOf(node->op());
526

527
  ArchOpcode opcode;
528 529
  switch (load_rep.representation()) {
    case MachineRepresentation::kFloat32:
530 531
      opcode = kIA32Movss;
      break;
532
    case MachineRepresentation::kFloat64:
533
      opcode = kIA32Movsd;
534
      break;
535 536 537
    case MachineRepresentation::kBit:  // Fall through.
    case MachineRepresentation::kWord8:
      opcode = load_rep.IsSigned() ? kIA32Movsxbl : kIA32Movzxbl;
538
      break;
539 540
    case MachineRepresentation::kWord16:
      opcode = load_rep.IsSigned() ? kIA32Movsxwl : kIA32Movzxwl;
541
      break;
542 543 544
    case MachineRepresentation::kTaggedSigned:   // Fall through.
    case MachineRepresentation::kTaggedPointer:  // Fall through.
    case MachineRepresentation::kTagged:         // Fall through.
545
    case MachineRepresentation::kWord32:
546
      opcode = kIA32Movl;
547
      break;
548 549 550
    case MachineRepresentation::kSimd128:
      opcode = kIA32Movdqu;
      break;
551 552 553
    case MachineRepresentation::kCompressedPointer:  // Fall through.
    case MachineRepresentation::kCompressed:         // Fall through.
    case MachineRepresentation::kWord64:             // Fall through.
554
    case MachineRepresentation::kNone:
555
    case MachineRepresentation::kMapWord:
556 557
      UNREACHABLE();
  }
558 559

  IA32OperandGenerator g(this);
560
  InstructionOperand outputs[1];
561
  outputs[0] = g.DefineAsRegister(node);
562
  InstructionOperand inputs[3];
563 564 565 566
  size_t input_count = 0;
  AddressingMode mode =
      g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
  InstructionCode code = opcode | AddressingModeField::encode(mode);
567
  if (node->opcode() == IrOpcode::kPoisonedLoad) {
568
    CHECK_NE(poisoning_level_, PoisoningMitigationLevel::kDontPoison);
569
    code |= AccessModeField::encode(kMemoryAccessPoisoned);
570
  }
571
  Emit(code, 1, outputs, input_count, inputs);
572 573
}

574 575
void InstructionSelector::VisitPoisonedLoad(Node* node) { VisitLoad(node); }

576 577 578 579
void InstructionSelector::VisitProtectedLoad(Node* node) {
  // TODO(eholk)
  UNIMPLEMENTED();
}
580 581 582 583 584 585 586

void InstructionSelector::VisitStore(Node* node) {
  IA32OperandGenerator g(this);
  Node* base = node->InputAt(0);
  Node* index = node->InputAt(1);
  Node* value = node->InputAt(2);

587
  StoreRepresentation store_rep = StoreRepresentationOf(node->op());
588
  WriteBarrierKind write_barrier_kind = store_rep.write_barrier_kind();
589
  MachineRepresentation rep = store_rep.representation();
590

591 592 593 594
  if (FLAG_enable_unconditional_write_barriers && CanBeTaggedPointer(rep)) {
    write_barrier_kind = kFullWriteBarrier;
  }

595
  if (write_barrier_kind != kNoWriteBarrier && !FLAG_disable_write_barriers) {
596
    DCHECK(CanBeTaggedPointer(rep));
597
    AddressingMode addressing_mode;
598 599 600 601
    InstructionOperand inputs[] = {
        g.UseUniqueRegister(base),
        g.GetEffectiveIndexOperand(index, &addressing_mode),
        g.UseUniqueRegister(value)};
602 603
    RecordWriteMode record_write_mode =
        WriteBarrierKindToRecordWriteMode(write_barrier_kind);
604 605 606 607 608
    InstructionOperand temps[] = {g.TempRegister(), g.TempRegister()};
    size_t const temp_count = arraysize(temps);
    InstructionCode code = kArchStoreWithWriteBarrier;
    code |= AddressingModeField::encode(addressing_mode);
    code |= MiscField::encode(static_cast<int>(record_write_mode));
609
    Emit(code, 0, nullptr, arraysize(inputs), inputs, temp_count, temps);
610
  } else {
611
    ArchOpcode opcode;
612
    switch (rep) {
613
      case MachineRepresentation::kFloat32:
614 615
        opcode = kIA32Movss;
        break;
616
      case MachineRepresentation::kFloat64:
617 618
        opcode = kIA32Movsd;
        break;
619 620
      case MachineRepresentation::kBit:  // Fall through.
      case MachineRepresentation::kWord8:
621 622
        opcode = kIA32Movb;
        break;
623
      case MachineRepresentation::kWord16:
624 625
        opcode = kIA32Movw;
        break;
626 627 628
      case MachineRepresentation::kTaggedSigned:   // Fall through.
      case MachineRepresentation::kTaggedPointer:  // Fall through.
      case MachineRepresentation::kTagged:         // Fall through.
629
      case MachineRepresentation::kWord32:
630 631
        opcode = kIA32Movl;
        break;
632 633 634
      case MachineRepresentation::kSimd128:
        opcode = kIA32Movdqu;
        break;
635 636 637
      case MachineRepresentation::kCompressedPointer:  // Fall through.
      case MachineRepresentation::kCompressed:         // Fall through.
      case MachineRepresentation::kWord64:             // Fall through.
638
      case MachineRepresentation::kMapWord:            // Fall through.
639
      case MachineRepresentation::kNone:
640
        UNREACHABLE();
641
    }
642

643 644 645
    InstructionOperand val;
    if (g.CanBeImmediate(value)) {
      val = g.UseImmediate(value);
646 647
    } else if (rep == MachineRepresentation::kWord8 ||
               rep == MachineRepresentation::kBit) {
648 649 650 651
      val = g.UseByteRegister(value);
    } else {
      val = g.UseRegister(value);
    }
652

653 654 655 656 657 658 659
    InstructionOperand inputs[4];
    size_t input_count = 0;
    AddressingMode addressing_mode =
        g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
    InstructionCode code =
        opcode | AddressingModeField::encode(addressing_mode);
    inputs[input_count++] = val;
660 661
    Emit(code, 0, static_cast<InstructionOperand*>(nullptr), input_count,
         inputs);
662 663 664
  }
}

665 666 667 668 669
void InstructionSelector::VisitProtectedStore(Node* node) {
  // TODO(eholk)
  UNIMPLEMENTED();
}

670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704
void InstructionSelector::VisitStoreLane(Node* node) {
  IA32OperandGenerator g(this);

  StoreLaneParameters params = StoreLaneParametersOf(node->op());
  InstructionCode opcode = kArchNop;
  if (params.rep == MachineRepresentation::kWord8) {
    opcode = kIA32Pextrb;
  } else if (params.rep == MachineRepresentation::kWord16) {
    opcode = kIA32Pextrw;
  } else if (params.rep == MachineRepresentation::kWord32) {
    opcode = kIA32S128Store32Lane;
  } else if (params.rep == MachineRepresentation::kWord64) {
    if (params.laneidx == 0) {
      opcode = kIA32Movlps;
    } else {
      DCHECK_EQ(1, params.laneidx);
      opcode = kIA32Movhps;
    }
  } else {
    UNREACHABLE();
  }

  InstructionOperand inputs[4];
  size_t input_count = 0;
  AddressingMode addressing_mode =
      g.GetEffectiveAddressMemoryOperand(node, inputs, &input_count);
  opcode |= AddressingModeField::encode(addressing_mode);

  InstructionOperand value_operand = g.UseRegister(node->InputAt(2));
  inputs[input_count++] = value_operand;
  inputs[input_count++] = g.UseImmediate(params.laneidx);
  DCHECK_GE(4, input_count);
  Emit(opcode, 0, nullptr, input_count, inputs);
}

705 706 707 708 709
// Architecture supports unaligned access, therefore VisitLoad is used instead
void InstructionSelector::VisitUnalignedLoad(Node* node) { UNREACHABLE(); }

// Architecture supports unaligned access, therefore VisitStore is used instead
void InstructionSelector::VisitUnalignedStore(Node* node) { UNREACHABLE(); }
710

711
namespace {
712

713
// Shared routine for multiple binary operations.
714 715
void VisitBinop(InstructionSelector* selector, Node* node,
                InstructionCode opcode, FlagsContinuation* cont) {
716
  IA32OperandGenerator g(selector);
717
  Int32BinopMatcher m(node);
718 719
  Node* left = m.left().node();
  Node* right = m.right().node();
720
  InstructionOperand inputs[6];
721
  size_t input_count = 0;
722
  InstructionOperand outputs[1];
723 724 725
  size_t output_count = 0;

  // TODO(turbofan): match complex addressing modes.
726 727 728 729 730 731 732 733
  if (left == right) {
    // If both inputs refer to the same operand, enforce allocating a register
    // for both of them to ensure that we don't end up generating code like
    // this:
    //
    //   mov eax, [ebp-0x10]
    //   add eax, [ebp-0x10]
    //   jo label
734
    InstructionOperand const input = g.UseRegister(left);
735 736 737
    inputs[input_count++] = input;
    inputs[input_count++] = input;
  } else if (g.CanBeImmediate(right)) {
738
    inputs[input_count++] = g.UseRegister(left);
739
    inputs[input_count++] = g.UseImmediate(right);
740
  } else {
741
    int effect_level = selector->GetEffectLevel(node, cont);
742
    if (node->op()->HasProperty(Operator::kCommutative) &&
743 744 745
        g.CanBeBetterLeftOperand(right) &&
        (!g.CanBeBetterLeftOperand(left) ||
         !g.CanBeMemoryOperand(opcode, node, right, effect_level))) {
746 747
      std::swap(left, right);
    }
748 749 750 751 752 753 754 755 756
    if (g.CanBeMemoryOperand(opcode, node, right, effect_level)) {
      inputs[input_count++] = g.UseRegister(left);
      AddressingMode addressing_mode =
          g.GetEffectiveAddressMemoryOperand(right, inputs, &input_count);
      opcode |= AddressingModeField::encode(addressing_mode);
    } else {
      inputs[input_count++] = g.UseRegister(left);
      inputs[input_count++] = g.Use(right);
    }
757 758
  }

759
  outputs[output_count++] = g.DefineSameAsFirst(node);
760

761
  DCHECK_NE(0u, input_count);
762
  DCHECK_EQ(1u, output_count);
763 764
  DCHECK_GE(arraysize(inputs), input_count);
  DCHECK_GE(arraysize(outputs), output_count);
765

766 767
  selector->EmitWithContinuation(opcode, output_count, outputs, input_count,
                                 inputs, cont);
768 769 770
}

// Shared routine for multiple binary operations.
771 772
void VisitBinop(InstructionSelector* selector, Node* node,
                InstructionCode opcode) {
773 774
  FlagsContinuation cont;
  VisitBinop(selector, node, opcode, &cont);
775 776
}

777
}  // namespace
778 779 780 781 782 783 784 785 786 787 788 789 790

void InstructionSelector::VisitWord32And(Node* node) {
  VisitBinop(this, node, kIA32And);
}

void InstructionSelector::VisitWord32Or(Node* node) {
  VisitBinop(this, node, kIA32Or);
}

void InstructionSelector::VisitWord32Xor(Node* node) {
  IA32OperandGenerator g(this);
  Int32BinopMatcher m(node);
  if (m.right().Is(-1)) {
791
    Emit(kIA32Not, g.DefineSameAsFirst(node), g.UseRegister(m.left().node()));
792 793 794 795 796
  } else {
    VisitBinop(this, node, kIA32Xor);
  }
}

797 798
void InstructionSelector::VisitStackPointerGreaterThan(
    Node* node, FlagsContinuation* cont) {
799 800 801
  StackCheckKind kind = StackCheckKindOf(node->op());
  InstructionCode opcode =
      kArchStackPointerGreaterThan | MiscField::encode(static_cast<int>(kind));
802

803
  int effect_level = GetEffectLevel(node, cont);
804 805

  IA32OperandGenerator g(this);
806 807 808 809 810 811 812 813 814 815 816 817 818 819 820

  // No outputs.
  InstructionOperand* const outputs = nullptr;
  const int output_count = 0;

  // Applying an offset to this stack check requires a temp register. Offsets
  // are only applied to the first stack check. If applying an offset, we must
  // ensure the input and temp registers do not alias, thus kUniqueRegister.
  InstructionOperand temps[] = {g.TempRegister()};
  const int temp_count = (kind == StackCheckKind::kJSFunctionEntry) ? 1 : 0;
  const auto register_mode = (kind == StackCheckKind::kJSFunctionEntry)
                                 ? OperandGenerator::kUniqueRegister
                                 : OperandGenerator::kRegister;

  Node* const value = node->InputAt(0);
821
  if (g.CanBeMemoryOperand(kIA32Cmp, node, value, effect_level)) {
822 823
    DCHECK(value->opcode() == IrOpcode::kLoad ||
           value->opcode() == IrOpcode::kLoadImmutable);
824 825 826 827 828 829

    // GetEffectiveAddressMemoryOperand can create at most 3 inputs.
    static constexpr int kMaxInputCount = 3;

    size_t input_count = 0;
    InstructionOperand inputs[kMaxInputCount];
830 831
    AddressingMode addressing_mode = g.GetEffectiveAddressMemoryOperand(
        value, inputs, &input_count, register_mode);
832 833 834
    opcode |= AddressingModeField::encode(addressing_mode);
    DCHECK_LE(input_count, kMaxInputCount);

835 836
    EmitWithContinuation(opcode, output_count, outputs, input_count, inputs,
                         temp_count, temps, cont);
837
  } else {
838 839 840 841
    InstructionOperand inputs[] = {g.UseRegisterWithMode(value, register_mode)};
    static constexpr int input_count = arraysize(inputs);
    EmitWithContinuation(opcode, output_count, outputs, input_count, inputs,
                         temp_count, temps, cont);
842 843 844
  }
}

845 846 847 848 849 850 851 852
// Shared routine for multiple shift operations.
static inline void VisitShift(InstructionSelector* selector, Node* node,
                              ArchOpcode opcode) {
  IA32OperandGenerator g(selector);
  Node* left = node->InputAt(0);
  Node* right = node->InputAt(1);

  if (g.CanBeImmediate(right)) {
853
    selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
854 855
                   g.UseImmediate(right));
  } else {
856
    selector->Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(left),
857 858 859 860
                   g.UseFixed(right, ecx));
  }
}

861 862 863 864 865
namespace {

void VisitMulHigh(InstructionSelector* selector, Node* node,
                  ArchOpcode opcode) {
  IA32OperandGenerator g(selector);
866 867 868 869
  InstructionOperand temps[] = {g.TempRegister(eax)};
  selector->Emit(
      opcode, g.DefineAsFixed(node, edx), g.UseFixed(node->InputAt(0), eax),
      g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
870 871 872 873
}

void VisitDiv(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
  IA32OperandGenerator g(selector);
874
  InstructionOperand temps[] = {g.TempRegister(edx)};
875 876 877 878 879 880 881
  selector->Emit(opcode, g.DefineAsFixed(node, eax),
                 g.UseFixed(node->InputAt(0), eax),
                 g.UseUnique(node->InputAt(1)), arraysize(temps), temps);
}

void VisitMod(InstructionSelector* selector, Node* node, ArchOpcode opcode) {
  IA32OperandGenerator g(selector);
882
  InstructionOperand temps[] = {g.TempRegister(eax)};
883 884
  selector->Emit(opcode, g.DefineAsFixed(node, edx),
                 g.UseFixed(node->InputAt(0), eax),
885
                 g.UseUnique(node->InputAt(1)), arraysize(temps), temps);
886 887 888
}

void EmitLea(InstructionSelector* selector, Node* result, Node* index,
889 890
             int scale, Node* base, Node* displacement,
             DisplacementMode displacement_mode) {
891
  IA32OperandGenerator g(selector);
892
  InstructionOperand inputs[4];
893
  size_t input_count = 0;
894 895 896
  AddressingMode mode =
      g.GenerateMemoryOperandInputs(index, scale, base, displacement,
                                    displacement_mode, inputs, &input_count);
897

898
  DCHECK_NE(0u, input_count);
899 900
  DCHECK_GE(arraysize(inputs), input_count);

901
  InstructionOperand outputs[1];
902 903 904 905 906 907 908 909 910
  outputs[0] = g.DefineAsRegister(result);

  InstructionCode opcode = AddressingModeField::encode(mode) | kIA32Lea;

  selector->Emit(opcode, 1, outputs, input_count, inputs);
}

}  // namespace

911
void InstructionSelector::VisitWord32Shl(Node* node) {
912 913 914
  Int32ScaleMatcher m(node, true);
  if (m.matches()) {
    Node* index = node->InputAt(0);
915
    Node* base = m.power_of_two_plus_one() ? index : nullptr;
916
    EmitLea(this, node, index, m.scale(), base, nullptr, kPositiveDisplacement);
917 918
    return;
  }
919 920 921 922 923 924 925 926 927 928 929
  VisitShift(this, node, kIA32Shl);
}

void InstructionSelector::VisitWord32Shr(Node* node) {
  VisitShift(this, node, kIA32Shr);
}

void InstructionSelector::VisitWord32Sar(Node* node) {
  VisitShift(this, node, kIA32Sar);
}

930 931 932
void InstructionSelector::VisitInt32PairAdd(Node* node) {
  IA32OperandGenerator g(this);

933 934 935 936 937
  Node* projection1 = NodeProperties::FindProjection(node, 1);
  if (projection1) {
    // We use UseUniqueRegister here to avoid register sharing with the temp
    // register.
    InstructionOperand inputs[] = {
938 939
        g.UseRegister(node->InputAt(0)),
        g.UseUniqueRegisterOrSlotOrConstant(node->InputAt(1)),
940
        g.UseRegister(node->InputAt(2)), g.UseUniqueRegister(node->InputAt(3))};
941

942 943
    InstructionOperand outputs[] = {g.DefineSameAsFirst(node),
                                    g.DefineAsRegister(projection1)};
944

945
    InstructionOperand temps[] = {g.TempRegister()};
946

947 948 949 950 951 952 953
    Emit(kIA32AddPair, 2, outputs, 4, inputs, 1, temps);
  } else {
    // The high word of the result is not used, so we emit the standard 32 bit
    // instruction.
    Emit(kIA32Add, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)),
         g.Use(node->InputAt(2)));
  }
954 955 956 957 958
}

void InstructionSelector::VisitInt32PairSub(Node* node) {
  IA32OperandGenerator g(this);

959 960 961 962 963
  Node* projection1 = NodeProperties::FindProjection(node, 1);
  if (projection1) {
    // We use UseUniqueRegister here to avoid register sharing with the temp
    // register.
    InstructionOperand inputs[] = {
964 965
        g.UseRegister(node->InputAt(0)),
        g.UseUniqueRegisterOrSlotOrConstant(node->InputAt(1)),
966
        g.UseRegister(node->InputAt(2)), g.UseUniqueRegister(node->InputAt(3))};
967

968 969
    InstructionOperand outputs[] = {g.DefineSameAsFirst(node),
                                    g.DefineAsRegister(projection1)};
970

971
    InstructionOperand temps[] = {g.TempRegister()};
972

973 974 975 976 977 978 979
    Emit(kIA32SubPair, 2, outputs, 4, inputs, 1, temps);
  } else {
    // The high word of the result is not used, so we emit the standard 32 bit
    // instruction.
    Emit(kIA32Sub, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)),
         g.Use(node->InputAt(2)));
  }
980 981 982 983 984
}

void InstructionSelector::VisitInt32PairMul(Node* node) {
  IA32OperandGenerator g(this);

985 986 987 988
  Node* projection1 = NodeProperties::FindProjection(node, 1);
  if (projection1) {
    // InputAt(3) explicitly shares ecx with OutputRegister(1) to save one
    // register and one mov instruction.
989 990 991 992 993
    InstructionOperand inputs[] = {
        g.UseUnique(node->InputAt(0)),
        g.UseUniqueRegisterOrSlotOrConstant(node->InputAt(1)),
        g.UseUniqueRegister(node->InputAt(2)),
        g.UseFixed(node->InputAt(3), ecx)};
994

995 996 997
    InstructionOperand outputs[] = {
        g.DefineAsFixed(node, eax),
        g.DefineAsFixed(NodeProperties::FindProjection(node, 1), ecx)};
998

999
    InstructionOperand temps[] = {g.TempRegister(edx)};
1000

1001 1002 1003 1004 1005 1006 1007
    Emit(kIA32MulPair, 2, outputs, 4, inputs, 1, temps);
  } else {
    // The high word of the result is not used, so we emit the standard 32 bit
    // instruction.
    Emit(kIA32Imul, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)),
         g.Use(node->InputAt(2)));
  }
1008 1009
}

1010 1011 1012
void VisitWord32PairShift(InstructionSelector* selector, InstructionCode opcode,
                          Node* node) {
  IA32OperandGenerator g(selector);
1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024

  Node* shift = node->InputAt(2);
  InstructionOperand shift_operand;
  if (g.CanBeImmediate(shift)) {
    shift_operand = g.UseImmediate(shift);
  } else {
    shift_operand = g.UseFixed(shift, ecx);
  }
  InstructionOperand inputs[] = {g.UseFixed(node->InputAt(0), eax),
                                 g.UseFixed(node->InputAt(1), edx),
                                 shift_operand};

1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035
  InstructionOperand outputs[2];
  InstructionOperand temps[1];
  int32_t output_count = 0;
  int32_t temp_count = 0;
  outputs[output_count++] = g.DefineAsFixed(node, eax);
  Node* projection1 = NodeProperties::FindProjection(node, 1);
  if (projection1) {
    outputs[output_count++] = g.DefineAsFixed(projection1, edx);
  } else {
    temps[temp_count++] = g.TempRegister(edx);
  }
1036

1037
  selector->Emit(opcode, output_count, outputs, 3, inputs, temp_count, temps);
1038 1039
}

1040 1041 1042
void InstructionSelector::VisitWord32PairShl(Node* node) {
  VisitWord32PairShift(this, kIA32ShlPair, node);
}
1043

1044 1045
void InstructionSelector::VisitWord32PairShr(Node* node) {
  VisitWord32PairShift(this, kIA32ShrPair, node);
1046 1047 1048
}

void InstructionSelector::VisitWord32PairSar(Node* node) {
1049
  VisitWord32PairShift(this, kIA32SarPair, node);
1050
}
1051

1052 1053 1054 1055
void InstructionSelector::VisitWord32Rol(Node* node) {
  VisitShift(this, node, kIA32Rol);
}

1056 1057 1058 1059
void InstructionSelector::VisitWord32Ror(Node* node) {
  VisitShift(this, node, kIA32Ror);
}

1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077
#define RO_OP_LIST(V)                                       \
  V(Word32Clz, kIA32Lzcnt)                                  \
  V(Word32Ctz, kIA32Tzcnt)                                  \
  V(Word32Popcnt, kIA32Popcnt)                              \
  V(ChangeFloat32ToFloat64, kSSEFloat32ToFloat64)           \
  V(RoundInt32ToFloat32, kSSEInt32ToFloat32)                \
  V(ChangeInt32ToFloat64, kSSEInt32ToFloat64)               \
  V(TruncateFloat32ToInt32, kSSEFloat32ToInt32)             \
  V(ChangeFloat64ToInt32, kSSEFloat64ToInt32)               \
  V(TruncateFloat64ToFloat32, kSSEFloat64ToFloat32)         \
  V(RoundFloat64ToInt32, kSSEFloat64ToInt32)                \
  V(BitcastFloat32ToInt32, kIA32BitcastFI)                  \
  V(BitcastInt32ToFloat32, kIA32BitcastIF)                  \
  V(Float32Sqrt, kSSEFloat32Sqrt)                           \
  V(Float64Sqrt, kSSEFloat64Sqrt)                           \
  V(Float64ExtractLowWord32, kSSEFloat64ExtractLowWord32)   \
  V(Float64ExtractHighWord32, kSSEFloat64ExtractHighWord32) \
  V(SignExtendWord8ToInt32, kIA32Movsxbl)                   \
1078 1079
  V(SignExtendWord16ToInt32, kIA32Movsxwl)                  \
  V(F64x2Sqrt, kIA32F64x2Sqrt)
1080

1081 1082
#define RO_WITH_TEMP_OP_LIST(V) V(ChangeUint32ToFloat64, kSSEUint32ToFloat64)

1083 1084 1085 1086 1087
#define RO_WITH_TEMP_SIMD_OP_LIST(V)              \
  V(TruncateFloat32ToUint32, kSSEFloat32ToUint32) \
  V(ChangeFloat64ToUint32, kSSEFloat64ToUint32)   \
  V(TruncateFloat64ToUint32, kSSEFloat64ToUint32)

1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
#define RR_OP_LIST(V)                                                         \
  V(TruncateFloat64ToWord32, kArchTruncateDoubleToI)                          \
  V(Float32RoundDown, kSSEFloat32Round | MiscField::encode(kRoundDown))       \
  V(Float64RoundDown, kSSEFloat64Round | MiscField::encode(kRoundDown))       \
  V(Float32RoundUp, kSSEFloat32Round | MiscField::encode(kRoundUp))           \
  V(Float64RoundUp, kSSEFloat64Round | MiscField::encode(kRoundUp))           \
  V(Float32RoundTruncate, kSSEFloat32Round | MiscField::encode(kRoundToZero)) \
  V(Float64RoundTruncate, kSSEFloat64Round | MiscField::encode(kRoundToZero)) \
  V(Float32RoundTiesEven,                                                     \
    kSSEFloat32Round | MiscField::encode(kRoundToNearest))                    \
1098 1099 1100 1101 1102 1103 1104 1105 1106 1107
  V(Float64RoundTiesEven,                                                     \
    kSSEFloat64Round | MiscField::encode(kRoundToNearest))                    \
  V(F32x4Ceil, kIA32F32x4Round | MiscField::encode(kRoundUp))                 \
  V(F32x4Floor, kIA32F32x4Round | MiscField::encode(kRoundDown))              \
  V(F32x4Trunc, kIA32F32x4Round | MiscField::encode(kRoundToZero))            \
  V(F32x4NearestInt, kIA32F32x4Round | MiscField::encode(kRoundToNearest))    \
  V(F64x2Ceil, kIA32F64x2Round | MiscField::encode(kRoundUp))                 \
  V(F64x2Floor, kIA32F64x2Round | MiscField::encode(kRoundDown))              \
  V(F64x2Trunc, kIA32F64x2Round | MiscField::encode(kRoundToZero))            \
  V(F64x2NearestInt, kIA32F64x2Round | MiscField::encode(kRoundToNearest))
1108 1109 1110 1111 1112 1113 1114 1115 1116

#define RRO_FLOAT_OP_LIST(V)                    \
  V(Float32Add, kAVXFloat32Add, kSSEFloat32Add) \
  V(Float64Add, kAVXFloat64Add, kSSEFloat64Add) \
  V(Float32Sub, kAVXFloat32Sub, kSSEFloat32Sub) \
  V(Float64Sub, kAVXFloat64Sub, kSSEFloat64Sub) \
  V(Float32Mul, kAVXFloat32Mul, kSSEFloat32Mul) \
  V(Float64Mul, kAVXFloat64Mul, kSSEFloat64Mul) \
  V(Float32Div, kAVXFloat32Div, kSSEFloat32Div) \
1117 1118 1119 1120
  V(Float64Div, kAVXFloat64Div, kSSEFloat64Div) \
  V(F64x2Add, kIA32F64x2Add, kIA32F64x2Add)     \
  V(F64x2Sub, kIA32F64x2Sub, kIA32F64x2Sub)     \
  V(F64x2Mul, kIA32F64x2Mul, kIA32F64x2Mul)     \
1121 1122 1123 1124 1125
  V(F64x2Div, kIA32F64x2Div, kIA32F64x2Div)     \
  V(F64x2Eq, kIA32F64x2Eq, kIA32F64x2Eq)        \
  V(F64x2Ne, kIA32F64x2Ne, kIA32F64x2Ne)        \
  V(F64x2Lt, kIA32F64x2Lt, kIA32F64x2Lt)        \
  V(F64x2Le, kIA32F64x2Le, kIA32F64x2Le)
1126 1127 1128 1129 1130

#define FLOAT_UNOP_LIST(V)                      \
  V(Float32Abs, kAVXFloat32Abs, kSSEFloat32Abs) \
  V(Float64Abs, kAVXFloat64Abs, kSSEFloat64Abs) \
  V(Float32Neg, kAVXFloat32Neg, kSSEFloat32Neg) \
1131 1132 1133
  V(Float64Neg, kAVXFloat64Neg, kSSEFloat64Neg) \
  V(F64x2Abs, kAVXFloat64Abs, kSSEFloat64Abs)   \
  V(F64x2Neg, kAVXFloat64Neg, kSSEFloat64Neg)
1134 1135 1136 1137 1138 1139 1140

#define RO_VISITOR(Name, opcode)                      \
  void InstructionSelector::Visit##Name(Node* node) { \
    VisitRO(this, node, opcode);                      \
  }
RO_OP_LIST(RO_VISITOR)
#undef RO_VISITOR
1141
#undef RO_OP_LIST
1142

1143 1144 1145 1146 1147 1148 1149 1150
#define RO_WITH_TEMP_VISITOR(Name, opcode)            \
  void InstructionSelector::Visit##Name(Node* node) { \
    VisitROWithTemp(this, node, opcode);              \
  }
RO_WITH_TEMP_OP_LIST(RO_WITH_TEMP_VISITOR)
#undef RO_WITH_TEMP_VISITOR
#undef RO_WITH_TEMP_OP_LIST

1151 1152 1153 1154 1155 1156 1157 1158
#define RO_WITH_TEMP_SIMD_VISITOR(Name, opcode)       \
  void InstructionSelector::Visit##Name(Node* node) { \
    VisitROWithTempSimd(this, node, opcode);          \
  }
RO_WITH_TEMP_SIMD_OP_LIST(RO_WITH_TEMP_SIMD_VISITOR)
#undef RO_WITH_TEMP_SIMD_VISITOR
#undef RO_WITH_TEMP_SIMD_OP_LIST

1159 1160 1161 1162 1163 1164
#define RR_VISITOR(Name, opcode)                      \
  void InstructionSelector::Visit##Name(Node* node) { \
    VisitRR(this, node, opcode);                      \
  }
RR_OP_LIST(RR_VISITOR)
#undef RR_VISITOR
1165
#undef RR_OP_LIST
1166

1167 1168 1169 1170 1171 1172
#define RRO_FLOAT_VISITOR(Name, avx, sse)             \
  void InstructionSelector::Visit##Name(Node* node) { \
    VisitRROFloat(this, node, avx, sse);              \
  }
RRO_FLOAT_OP_LIST(RRO_FLOAT_VISITOR)
#undef RRO_FLOAT_VISITOR
1173
#undef RRO_FLOAT_OP_LIST
1174

1175 1176 1177 1178 1179 1180
#define FLOAT_UNOP_VISITOR(Name, avx, sse)                  \
  void InstructionSelector::Visit##Name(Node* node) {       \
    VisitFloatUnop(this, node, node->InputAt(0), avx, sse); \
  }
FLOAT_UNOP_LIST(FLOAT_UNOP_VISITOR)
#undef FLOAT_UNOP_VISITOR
1181
#undef FLOAT_UNOP_LIST
1182

1183 1184
void InstructionSelector::VisitWord32ReverseBits(Node* node) { UNREACHABLE(); }

1185 1186
void InstructionSelector::VisitWord64ReverseBytes(Node* node) { UNREACHABLE(); }

1187 1188 1189 1190
void InstructionSelector::VisitWord32ReverseBytes(Node* node) {
  IA32OperandGenerator g(this);
  Emit(kIA32Bswap, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)));
}
1191

1192 1193 1194 1195
void InstructionSelector::VisitSimd128ReverseBytes(Node* node) {
  UNREACHABLE();
}

1196 1197
void InstructionSelector::VisitInt32Add(Node* node) {
  IA32OperandGenerator g(this);
1198

1199 1200 1201
  // Try to match the Add to a lea pattern
  BaseWithIndexAndDisplacement32Matcher m(node);
  if (m.matches() &&
1202
      (m.displacement() == nullptr || g.CanBeImmediate(m.displacement()))) {
1203
    InstructionOperand inputs[4];
1204 1205
    size_t input_count = 0;
    AddressingMode mode = g.GenerateMemoryOperandInputs(
1206 1207
        m.index(), m.scale(), m.base(), m.displacement(), m.displacement_mode(),
        inputs, &input_count);
1208

1209
    DCHECK_NE(0u, input_count);
1210 1211
    DCHECK_GE(arraysize(inputs), input_count);

1212
    InstructionOperand outputs[1];
1213 1214 1215 1216 1217 1218 1219 1220
    outputs[0] = g.DefineAsRegister(node);

    InstructionCode opcode = AddressingModeField::encode(mode) | kIA32Lea;
    Emit(opcode, 1, outputs, input_count, inputs);
    return;
  }

  // No lea pattern match, use add
1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233
  VisitBinop(this, node, kIA32Add);
}

void InstructionSelector::VisitInt32Sub(Node* node) {
  IA32OperandGenerator g(this);
  Int32BinopMatcher m(node);
  if (m.left().Is(0)) {
    Emit(kIA32Neg, g.DefineSameAsFirst(node), g.Use(m.right().node()));
  } else {
    VisitBinop(this, node, kIA32Sub);
  }
}

1234
void InstructionSelector::VisitInt32Mul(Node* node) {
1235 1236 1237
  Int32ScaleMatcher m(node, true);
  if (m.matches()) {
    Node* index = node->InputAt(0);
1238
    Node* base = m.power_of_two_plus_one() ? index : nullptr;
1239
    EmitLea(this, node, index, m.scale(), base, nullptr, kPositiveDisplacement);
1240 1241
    return;
  }
1242
  IA32OperandGenerator g(this);
1243 1244
  Node* left = node->InputAt(0);
  Node* right = node->InputAt(1);
1245 1246 1247 1248 1249 1250
  if (g.CanBeImmediate(right)) {
    Emit(kIA32Imul, g.DefineAsRegister(node), g.Use(left),
         g.UseImmediate(right));
  } else {
    if (g.CanBeBetterLeftOperand(right)) {
      std::swap(left, right);
1251
    }
1252 1253
    Emit(kIA32Imul, g.DefineSameAsFirst(node), g.UseRegister(left),
         g.Use(right));
1254 1255 1256
  }
}

1257 1258 1259 1260 1261 1262
void InstructionSelector::VisitInt32MulHigh(Node* node) {
  VisitMulHigh(this, node, kIA32ImulHigh);
}

void InstructionSelector::VisitUint32MulHigh(Node* node) {
  VisitMulHigh(this, node, kIA32UmulHigh);
1263 1264 1265 1266 1267 1268
}

void InstructionSelector::VisitInt32Div(Node* node) {
  VisitDiv(this, node, kIA32Idiv);
}

1269
void InstructionSelector::VisitUint32Div(Node* node) {
1270 1271 1272 1273 1274 1275 1276
  VisitDiv(this, node, kIA32Udiv);
}

void InstructionSelector::VisitInt32Mod(Node* node) {
  VisitMod(this, node, kIA32Idiv);
}

1277
void InstructionSelector::VisitUint32Mod(Node* node) {
1278 1279 1280
  VisitMod(this, node, kIA32Udiv);
}

1281 1282
void InstructionSelector::VisitRoundUint32ToFloat32(Node* node) {
  IA32OperandGenerator g(this);
1283
  InstructionOperand temps[] = {g.TempRegister()};
1284 1285 1286 1287
  Emit(kSSEUint32ToFloat32, g.DefineAsRegister(node), g.Use(node->InputAt(0)),
       arraysize(temps), temps);
}

1288 1289
void InstructionSelector::VisitFloat64Mod(Node* node) {
  IA32OperandGenerator g(this);
1290
  InstructionOperand temps[] = {g.TempRegister(eax), g.TempRegister()};
1291
  Emit(kSSEFloat64Mod, g.DefineSameAsFirst(node),
1292 1293
       g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)),
       arraysize(temps), temps);
1294 1295
}

1296 1297 1298 1299 1300 1301 1302
void InstructionSelector::VisitFloat32Max(Node* node) {
  IA32OperandGenerator g(this);
  InstructionOperand temps[] = {g.TempRegister()};
  Emit(kSSEFloat32Max, g.DefineSameAsFirst(node),
       g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)),
       arraysize(temps), temps);
}
1303

1304
void InstructionSelector::VisitFloat64Max(Node* node) {
1305 1306 1307 1308 1309
  IA32OperandGenerator g(this);
  InstructionOperand temps[] = {g.TempRegister()};
  Emit(kSSEFloat64Max, g.DefineSameAsFirst(node),
       g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)),
       arraysize(temps), temps);
1310 1311
}

1312 1313 1314 1315 1316 1317 1318
void InstructionSelector::VisitFloat32Min(Node* node) {
  IA32OperandGenerator g(this);
  InstructionOperand temps[] = {g.TempRegister()};
  Emit(kSSEFloat32Min, g.DefineSameAsFirst(node),
       g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)),
       arraysize(temps), temps);
}
1319 1320

void InstructionSelector::VisitFloat64Min(Node* node) {
1321 1322 1323 1324 1325
  IA32OperandGenerator g(this);
  InstructionOperand temps[] = {g.TempRegister()};
  Emit(kSSEFloat64Min, g.DefineSameAsFirst(node),
       g.UseRegister(node->InputAt(0)), g.Use(node->InputAt(1)),
       arraysize(temps), temps);
1326 1327
}

1328 1329 1330 1331
void InstructionSelector::VisitFloat64RoundTiesAway(Node* node) {
  UNREACHABLE();
}

1332 1333 1334 1335 1336 1337 1338 1339
void InstructionSelector::VisitFloat64Ieee754Binop(Node* node,
                                                   InstructionCode opcode) {
  IA32OperandGenerator g(this);
  Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)),
       g.UseRegister(node->InputAt(1)))
      ->MarkAsCall();
}

1340 1341 1342 1343 1344 1345 1346
void InstructionSelector::VisitFloat64Ieee754Unop(Node* node,
                                                  InstructionCode opcode) {
  IA32OperandGenerator g(this);
  Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)))
      ->MarkAsCall();
}

1347
void InstructionSelector::EmitPrepareArguments(
1348
    ZoneVector<PushParameter>* arguments, const CallDescriptor* call_descriptor,
1349
    Node* node) {
1350 1351
  IA32OperandGenerator g(this);

1352
  // Prepare for C function call.
1353
  if (call_descriptor->IsCFunctionCall()) {
1354 1355
    InstructionOperand temps[] = {g.TempRegister()};
    size_t const temp_count = arraysize(temps);
1356 1357
    Emit(kArchPrepareCallCFunction | MiscField::encode(static_cast<int>(
                                         call_descriptor->ParameterCount())),
1358 1359 1360
         0, nullptr, 0, nullptr, temp_count, temps);

    // Poke any stack arguments.
1361
    for (size_t n = 0; n < arguments->size(); ++n) {
1362
      PushParameter input = (*arguments)[n];
1363
      if (input.node) {
1364
        int const slot = static_cast<int>(n);
1365
        InstructionOperand value = g.CanBeImmediate(node)
1366 1367
                                       ? g.UseImmediate(input.node)
                                       : g.UseRegister(input.node);
1368 1369 1370 1371 1372
        Emit(kIA32Poke | MiscField::encode(slot), g.NoOutput(), value);
      }
    }
  } else {
    // Push any stack arguments.
1373
    int effect_level = GetEffectLevel(node);
1374
    int stack_decrement = 0;
1375
    for (PushParameter input : base::Reversed(*arguments)) {
1376 1377 1378
      stack_decrement += kSystemPointerSize;
      // Skip holes in the param array. These represent both extra slots for
      // multi-slot values and padding slots for alignment.
1379
      if (input.node == nullptr) continue;
1380 1381
      InstructionOperand decrement = g.UseImmediate(stack_decrement);
      stack_decrement = 0;
1382 1383 1384 1385 1386 1387 1388 1389 1390
      if (g.CanBeImmediate(input.node)) {
        Emit(kIA32Push, g.NoOutput(), decrement, g.UseImmediate(input.node));
      } else if (IsSupported(ATOM) ||
                 sequence()->IsFP(GetVirtualRegister(input.node))) {
        // TODO(bbudge): IA32Push cannot handle stack->stack double moves
        // because there is no way to encode fixed double slots.
        Emit(kIA32Push, g.NoOutput(), decrement, g.UseRegister(input.node));
      } else if (g.CanBeMemoryOperand(kIA32Push, node, input.node,
                                      effect_level)) {
1391
        InstructionOperand outputs[1];
1392
        InstructionOperand inputs[5];
1393
        size_t input_count = 0;
1394
        inputs[input_count++] = decrement;
1395
        AddressingMode mode = g.GetEffectiveAddressMemoryOperand(
1396
            input.node, inputs, &input_count);
1397
        InstructionCode opcode = kIA32Push | AddressingModeField::encode(mode);
1398
        Emit(opcode, 0, outputs, input_count, inputs);
1399
      } else {
1400
        Emit(kIA32Push, g.NoOutput(), decrement, g.UseAny(input.node));
1401
      }
1402
    }
1403 1404 1405
  }
}

1406 1407 1408
void InstructionSelector::EmitPrepareResults(
    ZoneVector<PushParameter>* results, const CallDescriptor* call_descriptor,
    Node* node) {
1409 1410 1411 1412 1413
  IA32OperandGenerator g(this);

  for (PushParameter output : *results) {
    if (!output.location.IsCallerFrameSlot()) continue;
    // Skip any alignment holes in nodes.
1414
    if (output.node != nullptr) {
1415
      DCHECK(!call_descriptor->IsCFunctionCall());
1416 1417 1418 1419
      if (output.location.GetType() == MachineType::Float32()) {
        MarkAsFloat32(output.node);
      } else if (output.location.GetType() == MachineType::Float64()) {
        MarkAsFloat64(output.node);
1420 1421
      } else if (output.location.GetType() == MachineType::Simd128()) {
        MarkAsSimd128(output.node);
1422
      }
1423 1424
      int offset = call_descriptor->GetOffsetToReturns();
      int reverse_slot = -output.location.GetLocation() - offset;
1425 1426
      Emit(kIA32Peek, g.DefineAsRegister(output.node),
           g.UseImmediate(reverse_slot));
1427
    }
1428
  }
1429 1430
}

1431
bool InstructionSelector::IsTailCallAddressImmediate() { return true; }
1432

1433 1434
namespace {

1435 1436 1437 1438
void VisitCompareWithMemoryOperand(InstructionSelector* selector,
                                   InstructionCode opcode, Node* left,
                                   InstructionOperand right,
                                   FlagsContinuation* cont) {
1439 1440
  DCHECK(left->opcode() == IrOpcode::kLoad ||
         left->opcode() == IrOpcode::kLoadImmutable);
1441 1442
  IA32OperandGenerator g(selector);
  size_t input_count = 0;
1443
  InstructionOperand inputs[4];
1444 1445 1446 1447 1448
  AddressingMode addressing_mode =
      g.GetEffectiveAddressMemoryOperand(left, inputs, &input_count);
  opcode |= AddressingModeField::encode(addressing_mode);
  inputs[input_count++] = right;

1449
  selector->EmitWithContinuation(opcode, 0, nullptr, input_count, inputs, cont);
1450 1451
}

1452
// Shared routine for multiple compare operations.
1453
void VisitCompare(InstructionSelector* selector, InstructionCode opcode,
1454
                  InstructionOperand left, InstructionOperand right,
1455
                  FlagsContinuation* cont) {
1456
  selector->EmitWithContinuation(opcode, left, right, cont);
1457 1458
}

1459
// Shared routine for multiple compare operations.
1460 1461 1462
void VisitCompare(InstructionSelector* selector, InstructionCode opcode,
                  Node* left, Node* right, FlagsContinuation* cont,
                  bool commutative) {
1463 1464 1465 1466 1467 1468 1469
  IA32OperandGenerator g(selector);
  if (commutative && g.CanBeBetterLeftOperand(right)) {
    std::swap(left, right);
  }
  VisitCompare(selector, opcode, g.UseRegister(left), g.Use(right), cont);
}

1470
MachineType MachineTypeForNarrow(Node* node, Node* hint_node) {
1471 1472
  if (hint_node->opcode() == IrOpcode::kLoad ||
      hint_node->opcode() == IrOpcode::kLoadImmutable) {
1473 1474 1475 1476
    MachineType hint = LoadRepresentationOf(hint_node->op());
    if (node->opcode() == IrOpcode::kInt32Constant ||
        node->opcode() == IrOpcode::kInt64Constant) {
      int64_t constant = node->opcode() == IrOpcode::kInt32Constant
1477 1478
                             ? OpParameter<int32_t>(node->op())
                             : OpParameter<int64_t>(node->op());
1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505
      if (hint == MachineType::Int8()) {
        if (constant >= std::numeric_limits<int8_t>::min() &&
            constant <= std::numeric_limits<int8_t>::max()) {
          return hint;
        }
      } else if (hint == MachineType::Uint8()) {
        if (constant >= std::numeric_limits<uint8_t>::min() &&
            constant <= std::numeric_limits<uint8_t>::max()) {
          return hint;
        }
      } else if (hint == MachineType::Int16()) {
        if (constant >= std::numeric_limits<int16_t>::min() &&
            constant <= std::numeric_limits<int16_t>::max()) {
          return hint;
        }
      } else if (hint == MachineType::Uint16()) {
        if (constant >= std::numeric_limits<uint16_t>::min() &&
            constant <= std::numeric_limits<uint16_t>::max()) {
          return hint;
        }
      } else if (hint == MachineType::Int32()) {
        return hint;
      } else if (hint == MachineType::Uint32()) {
        if (constant >= 0) return hint;
      }
    }
  }
1506 1507 1508 1509
  return node->opcode() == IrOpcode::kLoad ||
                 node->opcode() == IrOpcode::kLoadImmutable
             ? LoadRepresentationOf(node->op())
             : MachineType::None();
1510 1511
}

1512 1513 1514
// Tries to match the size of the given opcode to that of the operands, if
// possible.
InstructionCode TryNarrowOpcodeSize(InstructionCode opcode, Node* left,
1515
                                    Node* right, FlagsContinuation* cont) {
1516
  // TODO(epertoso): we can probably get some size information out of phi nodes.
1517
  // If the load representations don't match, both operands will be
1518
  // zero/sign-extended to 32bit.
1519 1520
  MachineType left_type = MachineTypeForNarrow(left, right);
  MachineType right_type = MachineTypeForNarrow(right, left);
1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549
  if (left_type == right_type) {
    switch (left_type.representation()) {
      case MachineRepresentation::kBit:
      case MachineRepresentation::kWord8: {
        if (opcode == kIA32Test) return kIA32Test8;
        if (opcode == kIA32Cmp) {
          if (left_type.semantic() == MachineSemantic::kUint32) {
            cont->OverwriteUnsignedIfSigned();
          } else {
            CHECK_EQ(MachineSemantic::kInt32, left_type.semantic());
          }
          return kIA32Cmp8;
        }
        break;
      }
      case MachineRepresentation::kWord16:
        if (opcode == kIA32Test) return kIA32Test16;
        if (opcode == kIA32Cmp) {
          if (left_type.semantic() == MachineSemantic::kUint32) {
            cont->OverwriteUnsignedIfSigned();
          } else {
            CHECK_EQ(MachineSemantic::kInt32, left_type.semantic());
          }
          return kIA32Cmp16;
        }
        break;
      default:
        break;
    }
1550
  }
1551
  return opcode;
1552
}
1553

1554 1555 1556 1557 1558 1559 1560 1561
// Shared routine for multiple float32 compare operations (inputs commuted).
void VisitFloat32Compare(InstructionSelector* selector, Node* node,
                         FlagsContinuation* cont) {
  Node* const left = node->InputAt(0);
  Node* const right = node->InputAt(1);
  VisitCompare(selector, kSSEFloat32Cmp, right, left, cont, false);
}

1562
// Shared routine for multiple float64 compare operations (inputs commuted).
1563 1564
void VisitFloat64Compare(InstructionSelector* selector, Node* node,
                         FlagsContinuation* cont) {
1565 1566 1567
  Node* const left = node->InputAt(0);
  Node* const right = node->InputAt(1);
  VisitCompare(selector, kSSEFloat64Cmp, right, left, cont, false);
1568 1569
}

1570
// Shared routine for multiple word compare operations.
1571 1572
void VisitWordCompare(InstructionSelector* selector, Node* node,
                      InstructionCode opcode, FlagsContinuation* cont) {
1573
  IA32OperandGenerator g(selector);
1574 1575 1576
  Node* left = node->InputAt(0);
  Node* right = node->InputAt(1);

1577 1578
  InstructionCode narrowed_opcode =
      TryNarrowOpcodeSize(opcode, left, right, cont);
1579

1580
  int effect_level = selector->GetEffectLevel(node, cont);
1581

1582 1583 1584
  // If one of the two inputs is an immediate, make sure it's on the right, or
  // if one of the two inputs is a memory operand, make sure it's on the left.
  if ((!g.CanBeImmediate(right) && g.CanBeImmediate(left)) ||
1585 1586
      (g.CanBeMemoryOperand(narrowed_opcode, node, right, effect_level) &&
       !g.CanBeMemoryOperand(narrowed_opcode, node, left, effect_level))) {
1587 1588 1589
    if (!node->op()->HasProperty(Operator::kCommutative)) cont->Commute();
    std::swap(left, right);
  }
1590

1591
  // Match immediates on right side of comparison.
1592
  if (g.CanBeImmediate(right)) {
1593 1594
    if (g.CanBeMemoryOperand(narrowed_opcode, node, left, effect_level)) {
      return VisitCompareWithMemoryOperand(selector, narrowed_opcode, left,
1595 1596 1597 1598 1599 1600
                                           g.UseImmediate(right), cont);
    }
    return VisitCompare(selector, opcode, g.Use(left), g.UseImmediate(right),
                        cont);
  }

1601
  // Match memory operands on left side of comparison.
1602
  if (g.CanBeMemoryOperand(narrowed_opcode, node, left, effect_level)) {
1603 1604 1605 1606 1607 1608 1609 1610
    bool needs_byte_register =
        narrowed_opcode == kIA32Test8 || narrowed_opcode == kIA32Cmp8;
    return VisitCompareWithMemoryOperand(
        selector, narrowed_opcode, left,
        needs_byte_register ? g.UseByteRegister(right) : g.UseRegister(right),
        cont);
  }

1611 1612 1613
  return VisitCompare(selector, opcode, left, right, cont,
                      node->op()->HasProperty(Operator::kCommutative));
}
1614

1615 1616 1617
void VisitWordCompare(InstructionSelector* selector, Node* node,
                      FlagsContinuation* cont) {
  VisitWordCompare(selector, node, kIA32Cmp, cont);
1618 1619
}

1620 1621 1622 1623 1624 1625 1626 1627
void VisitAtomicExchange(InstructionSelector* selector, Node* node,
                         ArchOpcode opcode, MachineRepresentation rep) {
  IA32OperandGenerator g(selector);
  Node* base = node->InputAt(0);
  Node* index = node->InputAt(1);
  Node* value = node->InputAt(2);

  AddressingMode addressing_mode;
1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638
  InstructionOperand value_operand = (rep == MachineRepresentation::kWord8)
                                         ? g.UseFixed(value, edx)
                                         : g.UseUniqueRegister(value);
  InstructionOperand inputs[] = {
      value_operand, g.UseUniqueRegister(base),
      g.GetEffectiveIndexOperand(index, &addressing_mode)};
  InstructionOperand outputs[] = {
      (rep == MachineRepresentation::kWord8)
          // Using DefineSameAsFirst requires the register to be unallocated.
          ? g.DefineAsFixed(node, edx)
          : g.DefineSameAsFirst(node)};
1639
  InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
1640
  selector->Emit(code, 1, outputs, arraysize(inputs), inputs);
1641 1642
}

1643
void VisitAtomicBinOp(InstructionSelector* selector, Node* node,
1644
                      ArchOpcode opcode, MachineRepresentation rep) {
1645 1646 1647 1648 1649
  AddressingMode addressing_mode;
  IA32OperandGenerator g(selector);
  Node* base = node->InputAt(0);
  Node* index = node->InputAt(1);
  Node* value = node->InputAt(2);
1650 1651 1652
  InstructionOperand inputs[] = {
      g.UseUniqueRegister(value), g.UseUniqueRegister(base),
      g.GetEffectiveIndexOperand(index, &addressing_mode)};
1653
  InstructionOperand outputs[] = {g.DefineAsFixed(node, eax)};
1654 1655 1656
  InstructionOperand temp[] = {(rep == MachineRepresentation::kWord8)
                                   ? g.UseByteRegister(node)
                                   : g.TempRegister()};
1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674
  InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
  selector->Emit(code, arraysize(outputs), outputs, arraysize(inputs), inputs,
                 arraysize(temp), temp);
}

void VisitPairAtomicBinOp(InstructionSelector* selector, Node* node,
                          ArchOpcode opcode) {
  IA32OperandGenerator g(selector);
  Node* base = node->InputAt(0);
  Node* index = node->InputAt(1);
  Node* value = node->InputAt(2);
  // For Word64 operations, the value input is split into the a high node,
  // and a low node in the int64-lowering phase.
  Node* value_high = node->InputAt(3);

  // Wasm lives in 32-bit address space, so we do not need to worry about
  // base/index lowering. This will need to be fixed for Wasm64.
  AddressingMode addressing_mode;
1675
  InstructionOperand inputs[] = {
1676
      g.UseUniqueRegisterOrSlotOrConstant(value), g.UseFixed(value_high, ecx),
1677 1678
      g.UseUniqueRegister(base),
      g.GetEffectiveIndexOperand(index, &addressing_mode)};
1679
  InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
1680 1681
  Node* projection0 = NodeProperties::FindProjection(node, 0);
  Node* projection1 = NodeProperties::FindProjection(node, 1);
1682 1683 1684 1685 1686 1687 1688 1689 1690
  InstructionOperand outputs[2];
  size_t output_count = 0;
  InstructionOperand temps[2];
  size_t temp_count = 0;
  if (projection0) {
    outputs[output_count++] = g.DefineAsFixed(projection0, eax);
  } else {
    temps[temp_count++] = g.TempRegister(eax);
  }
1691
  if (projection1) {
1692
    outputs[output_count++] = g.DefineAsFixed(projection1, edx);
1693
  } else {
1694
    temps[temp_count++] = g.TempRegister(edx);
1695
  }
1696 1697
  selector->Emit(code, output_count, outputs, arraysize(inputs), inputs,
                 temp_count, temps);
1698 1699
}

1700
}  // namespace
1701

1702
// Shared routine for word comparison with zero.
1703 1704
void InstructionSelector::VisitWordCompareZero(Node* user, Node* value,
                                               FlagsContinuation* cont) {
1705 1706 1707 1708 1709 1710 1711 1712 1713 1714
  // Try to combine with comparisons against 0 by simply inverting the branch.
  while (value->opcode() == IrOpcode::kWord32Equal && CanCover(user, value)) {
    Int32BinopMatcher m(value);
    if (!m.right().Is(0)) break;

    user = value;
    value = m.left().node();
    cont->Negate();
  }

1715
  if (CanCover(user, value)) {
1716
    switch (value->opcode()) {
1717
      case IrOpcode::kWord32Equal:
1718
        cont->OverwriteAndNegateIfEqual(kEqual);
1719
        return VisitWordCompare(this, value, cont);
1720
      case IrOpcode::kInt32LessThan:
1721
        cont->OverwriteAndNegateIfEqual(kSignedLessThan);
1722
        return VisitWordCompare(this, value, cont);
1723
      case IrOpcode::kInt32LessThanOrEqual:
1724
        cont->OverwriteAndNegateIfEqual(kSignedLessThanOrEqual);
1725
        return VisitWordCompare(this, value, cont);
1726
      case IrOpcode::kUint32LessThan:
1727
        cont->OverwriteAndNegateIfEqual(kUnsignedLessThan);
1728
        return VisitWordCompare(this, value, cont);
1729
      case IrOpcode::kUint32LessThanOrEqual:
1730
        cont->OverwriteAndNegateIfEqual(kUnsignedLessThanOrEqual);
1731
        return VisitWordCompare(this, value, cont);
1732 1733
      case IrOpcode::kFloat32Equal:
        cont->OverwriteAndNegateIfEqual(kUnorderedEqual);
1734
        return VisitFloat32Compare(this, value, cont);
1735 1736
      case IrOpcode::kFloat32LessThan:
        cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThan);
1737
        return VisitFloat32Compare(this, value, cont);
1738 1739
      case IrOpcode::kFloat32LessThanOrEqual:
        cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThanOrEqual);
1740
        return VisitFloat32Compare(this, value, cont);
1741
      case IrOpcode::kFloat64Equal:
1742
        cont->OverwriteAndNegateIfEqual(kUnorderedEqual);
1743
        return VisitFloat64Compare(this, value, cont);
1744
      case IrOpcode::kFloat64LessThan:
1745
        cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThan);
1746
        return VisitFloat64Compare(this, value, cont);
1747
      case IrOpcode::kFloat64LessThanOrEqual:
1748
        cont->OverwriteAndNegateIfEqual(kUnsignedGreaterThanOrEqual);
1749
        return VisitFloat64Compare(this, value, cont);
1750 1751 1752
      case IrOpcode::kProjection:
        // Check if this is the overflow output projection of an
        // <Operation>WithOverflow node.
1753
        if (ProjectionIndexOf(value->op()) == 1u) {
1754 1755
          // We cannot combine the <Operation>WithOverflow with this branch
          // unless the 0th projection (the use of the actual value of the
1756
          // <Operation> is either nullptr, which means there's no use of the
1757 1758
          // actual value, or was already defined, which means it is scheduled
          // *AFTER* this branch).
1759 1760
          Node* const node = value->InputAt(0);
          Node* const result = NodeProperties::FindProjection(node, 0);
1761
          if (result == nullptr || IsDefined(result)) {
1762 1763
            switch (node->opcode()) {
              case IrOpcode::kInt32AddWithOverflow:
1764
                cont->OverwriteAndNegateIfEqual(kOverflow);
1765
                return VisitBinop(this, node, kIA32Add, cont);
1766
              case IrOpcode::kInt32SubWithOverflow:
1767
                cont->OverwriteAndNegateIfEqual(kOverflow);
1768
                return VisitBinop(this, node, kIA32Sub, cont);
1769 1770
              case IrOpcode::kInt32MulWithOverflow:
                cont->OverwriteAndNegateIfEqual(kOverflow);
1771
                return VisitBinop(this, node, kIA32Imul, cont);
1772 1773 1774 1775 1776 1777 1778
              default:
                break;
            }
          }
        }
        break;
      case IrOpcode::kInt32Sub:
1779
        return VisitWordCompare(this, value, cont);
1780
      case IrOpcode::kWord32And:
1781
        return VisitWordCompare(this, value, kIA32Test, cont);
1782 1783 1784
      case IrOpcode::kStackPointerGreaterThan:
        cont->OverwriteAndNegateIfEqual(kStackPointerGreaterThanCondition);
        return VisitStackPointerGreaterThan(value, cont);
1785 1786 1787 1788 1789
      default:
        break;
    }
  }

1790
  // Continuation could not be combined with a compare, emit compare against 0.
1791 1792
  IA32OperandGenerator g(this);
  VisitCompare(this, kIA32Cmp, g.Use(value), g.TempImmediate(0), cont);
1793
}
1794

1795
void InstructionSelector::VisitSwitch(Node* node, const SwitchInfo& sw) {
1796 1797 1798
  IA32OperandGenerator g(this);
  InstructionOperand value_operand = g.UseRegister(node->InputAt(0));

1799
  // Emit either ArchTableSwitch or ArchBinarySearchSwitch.
1800 1801
  if (enable_switch_jump_table_ == kEnableSwitchJumpTable) {
    static const size_t kMaxTableSwitchValueRange = 2 << 16;
1802
    size_t table_space_cost = 4 + sw.value_range();
1803
    size_t table_time_cost = 3;
1804 1805 1806
    size_t lookup_space_cost = 3 + 2 * sw.case_count();
    size_t lookup_time_cost = sw.case_count();
    if (sw.case_count() > 4 &&
1807 1808
        table_space_cost + 3 * table_time_cost <=
            lookup_space_cost + 3 * lookup_time_cost &&
1809 1810
        sw.min_value() > std::numeric_limits<int32_t>::min() &&
        sw.value_range() <= kMaxTableSwitchValueRange) {
1811
      InstructionOperand index_operand = value_operand;
1812
      if (sw.min_value()) {
1813 1814
        index_operand = g.TempRegister();
        Emit(kIA32Lea | AddressingModeField::encode(kMode_MRI), index_operand,
1815
             value_operand, g.TempImmediate(-sw.min_value()));
1816 1817 1818
      }
      // Generate a table lookup.
      return EmitTableSwitch(sw, index_operand);
1819 1820 1821
    }
  }

1822 1823
  // Generate a tree of conditional jumps.
  return EmitBinarySearchSwitch(sw, value_operand);
1824 1825
}

1826
void InstructionSelector::VisitWord32Equal(Node* const node) {
1827
  FlagsContinuation cont = FlagsContinuation::ForSet(kEqual, node);
1828
  Int32BinopMatcher m(node);
1829
  if (m.right().Is(0)) {
1830
    return VisitWordCompareZero(m.node(), m.left().node(), &cont);
1831
  }
1832
  VisitWordCompare(this, node, &cont);
1833 1834
}

1835
void InstructionSelector::VisitInt32LessThan(Node* node) {
1836
  FlagsContinuation cont = FlagsContinuation::ForSet(kSignedLessThan, node);
1837
  VisitWordCompare(this, node, &cont);
1838 1839
}

1840
void InstructionSelector::VisitInt32LessThanOrEqual(Node* node) {
1841 1842
  FlagsContinuation cont =
      FlagsContinuation::ForSet(kSignedLessThanOrEqual, node);
1843
  VisitWordCompare(this, node, &cont);
1844 1845
}

1846
void InstructionSelector::VisitUint32LessThan(Node* node) {
1847
  FlagsContinuation cont = FlagsContinuation::ForSet(kUnsignedLessThan, node);
1848
  VisitWordCompare(this, node, &cont);
1849
}
1850

1851
void InstructionSelector::VisitUint32LessThanOrEqual(Node* node) {
1852 1853
  FlagsContinuation cont =
      FlagsContinuation::ForSet(kUnsignedLessThanOrEqual, node);
1854
  VisitWordCompare(this, node, &cont);
1855
}
1856

1857
void InstructionSelector::VisitInt32AddWithOverflow(Node* node) {
1858
  if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
1859
    FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
1860
    return VisitBinop(this, node, kIA32Add, &cont);
1861
  }
1862 1863 1864
  FlagsContinuation cont;
  VisitBinop(this, node, kIA32Add, &cont);
}
1865

1866
void InstructionSelector::VisitInt32SubWithOverflow(Node* node) {
1867
  if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
1868
    FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
1869
    return VisitBinop(this, node, kIA32Sub, &cont);
1870
  }
1871 1872 1873
  FlagsContinuation cont;
  VisitBinop(this, node, kIA32Sub, &cont);
}
1874

1875 1876 1877 1878 1879 1880 1881 1882
void InstructionSelector::VisitInt32MulWithOverflow(Node* node) {
  if (Node* ovf = NodeProperties::FindProjection(node, 1)) {
    FlagsContinuation cont = FlagsContinuation::ForSet(kOverflow, ovf);
    return VisitBinop(this, node, kIA32Imul, &cont);
  }
  FlagsContinuation cont;
  VisitBinop(this, node, kIA32Imul, &cont);
}
1883

1884
void InstructionSelector::VisitFloat32Equal(Node* node) {
1885
  FlagsContinuation cont = FlagsContinuation::ForSet(kUnorderedEqual, node);
1886 1887 1888 1889
  VisitFloat32Compare(this, node, &cont);
}

void InstructionSelector::VisitFloat32LessThan(Node* node) {
1890 1891
  FlagsContinuation cont =
      FlagsContinuation::ForSet(kUnsignedGreaterThan, node);
1892 1893 1894 1895
  VisitFloat32Compare(this, node, &cont);
}

void InstructionSelector::VisitFloat32LessThanOrEqual(Node* node) {
1896 1897
  FlagsContinuation cont =
      FlagsContinuation::ForSet(kUnsignedGreaterThanOrEqual, node);
1898 1899 1900
  VisitFloat32Compare(this, node, &cont);
}

1901
void InstructionSelector::VisitFloat64Equal(Node* node) {
1902
  FlagsContinuation cont = FlagsContinuation::ForSet(kUnorderedEqual, node);
1903 1904 1905 1906
  VisitFloat64Compare(this, node, &cont);
}

void InstructionSelector::VisitFloat64LessThan(Node* node) {
1907 1908
  FlagsContinuation cont =
      FlagsContinuation::ForSet(kUnsignedGreaterThan, node);
1909 1910 1911 1912
  VisitFloat64Compare(this, node, &cont);
}

void InstructionSelector::VisitFloat64LessThanOrEqual(Node* node) {
1913 1914
  FlagsContinuation cont =
      FlagsContinuation::ForSet(kUnsignedGreaterThanOrEqual, node);
1915
  VisitFloat64Compare(this, node, &cont);
1916 1917
}

1918 1919 1920 1921 1922
void InstructionSelector::VisitFloat64InsertLowWord32(Node* node) {
  IA32OperandGenerator g(this);
  Node* left = node->InputAt(0);
  Node* right = node->InputAt(1);
  Float64Matcher mleft(left);
1923 1924
  if (mleft.HasResolvedValue() &&
      (bit_cast<uint64_t>(mleft.ResolvedValue()) >> 32) == 0u) {
1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939
    Emit(kSSEFloat64LoadLowWord32, g.DefineAsRegister(node), g.Use(right));
    return;
  }
  Emit(kSSEFloat64InsertLowWord32, g.DefineSameAsFirst(node),
       g.UseRegister(left), g.Use(right));
}

void InstructionSelector::VisitFloat64InsertHighWord32(Node* node) {
  IA32OperandGenerator g(this);
  Node* left = node->InputAt(0);
  Node* right = node->InputAt(1);
  Emit(kSSEFloat64InsertHighWord32, g.DefineSameAsFirst(node),
       g.UseRegister(left), g.Use(right));
}

1940 1941 1942 1943 1944 1945
void InstructionSelector::VisitFloat64SilenceNaN(Node* node) {
  IA32OperandGenerator g(this);
  Emit(kSSEFloat64SilenceNaN, g.DefineSameAsFirst(node),
       g.UseRegister(node->InputAt(0)));
}

1946 1947 1948 1949 1950
void InstructionSelector::VisitMemoryBarrier(Node* node) {
  IA32OperandGenerator g(this);
  Emit(kIA32MFence, g.NoOutput());
}

1951
void InstructionSelector::VisitWord32AtomicLoad(Node* node) {
1952 1953 1954 1955 1956 1957 1958
  LoadRepresentation load_rep = LoadRepresentationOf(node->op());
  DCHECK(load_rep.representation() == MachineRepresentation::kWord8 ||
         load_rep.representation() == MachineRepresentation::kWord16 ||
         load_rep.representation() == MachineRepresentation::kWord32);
  USE(load_rep);
  VisitLoad(node);
}
1959

1960
void InstructionSelector::VisitWord32AtomicStore(Node* node) {
1961 1962
  IA32OperandGenerator g(this);
  MachineRepresentation rep = AtomicStoreRepresentationOf(node->op());
1963
  ArchOpcode opcode;
1964 1965
  switch (rep) {
    case MachineRepresentation::kWord8:
1966
      opcode = kWord32AtomicExchangeInt8;
1967 1968
      break;
    case MachineRepresentation::kWord16:
1969
      opcode = kWord32AtomicExchangeInt16;
1970 1971
      break;
    case MachineRepresentation::kWord32:
1972
      opcode = kWord32AtomicExchangeWord32;
1973 1974 1975 1976
      break;
    default:
      UNREACHABLE();
  }
1977
  VisitAtomicExchange(this, node, opcode, rep);
1978 1979
}

1980
void InstructionSelector::VisitWord32AtomicExchange(Node* node) {
1981
  IA32OperandGenerator g(this);
1982
  MachineType type = AtomicOpType(node->op());
1983
  ArchOpcode opcode;
1984
  if (type == MachineType::Int8()) {
1985
    opcode = kWord32AtomicExchangeInt8;
1986
  } else if (type == MachineType::Uint8()) {
1987
    opcode = kWord32AtomicExchangeUint8;
1988
  } else if (type == MachineType::Int16()) {
1989
    opcode = kWord32AtomicExchangeInt16;
1990
  } else if (type == MachineType::Uint16()) {
1991
    opcode = kWord32AtomicExchangeUint16;
1992
  } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
1993
    opcode = kWord32AtomicExchangeWord32;
1994 1995 1996
  } else {
    UNREACHABLE();
  }
1997
  VisitAtomicExchange(this, node, opcode, type.representation());
1998 1999
}

2000
void InstructionSelector::VisitWord32AtomicCompareExchange(Node* node) {
2001 2002 2003 2004 2005 2006
  IA32OperandGenerator g(this);
  Node* base = node->InputAt(0);
  Node* index = node->InputAt(1);
  Node* old_value = node->InputAt(2);
  Node* new_value = node->InputAt(3);

2007
  MachineType type = AtomicOpType(node->op());
2008
  ArchOpcode opcode;
2009
  if (type == MachineType::Int8()) {
2010
    opcode = kWord32AtomicCompareExchangeInt8;
2011
  } else if (type == MachineType::Uint8()) {
2012
    opcode = kWord32AtomicCompareExchangeUint8;
2013
  } else if (type == MachineType::Int16()) {
2014
    opcode = kWord32AtomicCompareExchangeInt16;
2015
  } else if (type == MachineType::Uint16()) {
2016
    opcode = kWord32AtomicCompareExchangeUint16;
2017
  } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
2018
    opcode = kWord32AtomicCompareExchangeWord32;
2019 2020 2021 2022
  } else {
    UNREACHABLE();
  }
  AddressingMode addressing_mode;
2023 2024 2025 2026 2027 2028 2029 2030
  InstructionOperand new_val_operand =
      (type.representation() == MachineRepresentation::kWord8)
          ? g.UseByteRegister(new_value)
          : g.UseUniqueRegister(new_value);
  InstructionOperand inputs[] = {
      g.UseFixed(old_value, eax), new_val_operand, g.UseUniqueRegister(base),
      g.GetEffectiveIndexOperand(index, &addressing_mode)};
  InstructionOperand outputs[] = {g.DefineAsFixed(node, eax)};
2031
  InstructionCode code = opcode | AddressingModeField::encode(addressing_mode);
2032
  Emit(code, 1, outputs, arraysize(inputs), inputs);
2033 2034
}

2035
void InstructionSelector::VisitWord32AtomicBinaryOperation(
2036 2037
    Node* node, ArchOpcode int8_op, ArchOpcode uint8_op, ArchOpcode int16_op,
    ArchOpcode uint16_op, ArchOpcode word32_op) {
2038
  MachineType type = AtomicOpType(node->op());
2039
  ArchOpcode opcode;
2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052
  if (type == MachineType::Int8()) {
    opcode = int8_op;
  } else if (type == MachineType::Uint8()) {
    opcode = uint8_op;
  } else if (type == MachineType::Int16()) {
    opcode = int16_op;
  } else if (type == MachineType::Uint16()) {
    opcode = uint16_op;
  } else if (type == MachineType::Int32() || type == MachineType::Uint32()) {
    opcode = word32_op;
  } else {
    UNREACHABLE();
  }
2053
  VisitAtomicBinOp(this, node, opcode, type.representation());
2054 2055
}

2056 2057
#define VISIT_ATOMIC_BINOP(op)                                   \
  void InstructionSelector::VisitWord32Atomic##op(Node* node) {  \
2058
    VisitWord32AtomicBinaryOperation(                            \
2059 2060 2061
        node, kWord32Atomic##op##Int8, kWord32Atomic##op##Uint8, \
        kWord32Atomic##op##Int16, kWord32Atomic##op##Uint16,     \
        kWord32Atomic##op##Word32);                              \
2062 2063 2064 2065 2066 2067
  }
VISIT_ATOMIC_BINOP(Add)
VISIT_ATOMIC_BINOP(Sub)
VISIT_ATOMIC_BINOP(And)
VISIT_ATOMIC_BINOP(Or)
VISIT_ATOMIC_BINOP(Xor)
2068 2069
#undef VISIT_ATOMIC_BINOP

2070 2071 2072 2073 2074
void InstructionSelector::VisitWord32AtomicPairLoad(Node* node) {
  IA32OperandGenerator g(this);
  AddressingMode mode;
  Node* base = node->InputAt(0);
  Node* index = node->InputAt(1);
2075 2076
  Node* projection0 = NodeProperties::FindProjection(node, 0);
  Node* projection1 = NodeProperties::FindProjection(node, 1);
2077 2078 2079 2080 2081
  if (projection0 && projection1) {
    InstructionOperand inputs[] = {g.UseUniqueRegister(base),
                                   g.GetEffectiveIndexOperand(index, &mode)};
    InstructionCode code =
        kIA32Word32AtomicPairLoad | AddressingModeField::encode(mode);
2082 2083 2084
    InstructionOperand temps[] = {g.TempDoubleRegister()};
    InstructionOperand outputs[] = {g.DefineAsRegister(projection0),
                                    g.DefineAsRegister(projection1)};
2085 2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101
    Emit(code, 2, outputs, 2, inputs, 1, temps);
  } else if (projection0 || projection1) {
    // Only one word is needed, so it's enough to load just that.
    ArchOpcode opcode = kIA32Movl;

    InstructionOperand outputs[] = {
        g.DefineAsRegister(projection0 ? projection0 : projection1)};
    InstructionOperand inputs[3];
    size_t input_count = 0;
    // TODO(ahaas): Introduce an enum for {scale} instead of an integer.
    // {scale = 0} means *1 in the generated code.
    int scale = 0;
    AddressingMode mode = g.GenerateMemoryOperandInputs(
        index, scale, base, projection0 ? 0 : 4, kPositiveDisplacement, inputs,
        &input_count);
    InstructionCode code = opcode | AddressingModeField::encode(mode);
    Emit(code, 1, outputs, input_count, inputs);
2102
  }
2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113
}

void InstructionSelector::VisitWord32AtomicPairStore(Node* node) {
  IA32OperandGenerator g(this);
  Node* base = node->InputAt(0);
  Node* index = node->InputAt(1);
  Node* value = node->InputAt(2);
  Node* value_high = node->InputAt(3);

  AddressingMode addressing_mode;
  InstructionOperand inputs[] = {
2114
      g.UseUniqueRegisterOrSlotOrConstant(value), g.UseFixed(value_high, ecx),
2115 2116 2117 2118 2119
      g.UseUniqueRegister(base),
      g.GetEffectiveIndexOperand(index, &addressing_mode)};
  // Allocating temp registers here as stores are performed using an atomic
  // exchange, the output of which is stored in edx:eax, which should be saved
  // and restored at the end of the instruction.
2120 2121
  InstructionOperand temps[] = {g.TempRegister(eax), g.TempRegister(edx)};
  const int num_temps = arraysize(temps);
2122 2123
  InstructionCode code =
      kIA32Word32AtomicPairStore | AddressingModeField::encode(addressing_mode);
2124
  Emit(code, 0, nullptr, arraysize(inputs), inputs, num_temps, temps);
2125 2126
}

2127 2128 2129 2130 2131 2132 2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146
void InstructionSelector::VisitWord32AtomicPairAdd(Node* node) {
  VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairAdd);
}

void InstructionSelector::VisitWord32AtomicPairSub(Node* node) {
  VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairSub);
}

void InstructionSelector::VisitWord32AtomicPairAnd(Node* node) {
  VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairAnd);
}

void InstructionSelector::VisitWord32AtomicPairOr(Node* node) {
  VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairOr);
}

void InstructionSelector::VisitWord32AtomicPairXor(Node* node) {
  VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairXor);
}

2147 2148 2149 2150 2151 2152 2153 2154
void InstructionSelector::VisitWord32AtomicPairExchange(Node* node) {
  VisitPairAtomicBinOp(this, node, kIA32Word32AtomicPairExchange);
}

void InstructionSelector::VisitWord32AtomicPairCompareExchange(Node* node) {
  IA32OperandGenerator g(this);
  Node* index = node->InputAt(1);
  AddressingMode addressing_mode;
2155

2156 2157 2158 2159
  InstructionOperand inputs[] = {
      // High, Low values of old value
      g.UseFixed(node->InputAt(2), eax), g.UseFixed(node->InputAt(3), edx),
      // High, Low values of new value
2160 2161
      g.UseUniqueRegisterOrSlotOrConstant(node->InputAt(4)),
      g.UseFixed(node->InputAt(5), ecx),
2162
      // InputAt(0) => base
2163 2164
      g.UseUniqueRegister(node->InputAt(0)),
      g.GetEffectiveIndexOperand(index, &addressing_mode)};
2165 2166
  Node* projection0 = NodeProperties::FindProjection(node, 0);
  Node* projection1 = NodeProperties::FindProjection(node, 1);
2167 2168
  InstructionCode code = kIA32Word32AtomicPairCompareExchange |
                         AddressingModeField::encode(addressing_mode);
2169

2170 2171 2172 2173 2174 2175 2176 2177 2178
  InstructionOperand outputs[2];
  size_t output_count = 0;
  InstructionOperand temps[2];
  size_t temp_count = 0;
  if (projection0) {
    outputs[output_count++] = g.DefineAsFixed(projection0, eax);
  } else {
    temps[temp_count++] = g.TempRegister(eax);
  }
2179
  if (projection1) {
2180
    outputs[output_count++] = g.DefineAsFixed(projection1, edx);
2181
  } else {
2182
    temps[temp_count++] = g.TempRegister(edx);
2183
  }
2184 2185
  Emit(code, output_count, outputs, arraysize(inputs), inputs, temp_count,
       temps);
2186 2187
}

2188 2189 2190
#define SIMD_INT_TYPES(V) \
  V(I32x4)                \
  V(I16x8)                \
2191 2192 2193
  V(I8x16)

#define SIMD_BINOP_LIST(V) \
2194 2195
  V(F32x4Min)              \
  V(F32x4Max)              \
2196 2197 2198 2199
  V(F32x4Eq)               \
  V(F32x4Ne)               \
  V(F32x4Lt)               \
  V(F32x4Le)               \
2200
  V(I32x4Add)              \
2201 2202 2203 2204
  V(I32x4Sub)              \
  V(I32x4Mul)              \
  V(I32x4MinS)             \
  V(I32x4MaxS)             \
2205 2206 2207 2208
  V(I32x4Eq)               \
  V(I32x4Ne)               \
  V(I32x4GtS)              \
  V(I32x4GeS)              \
2209
  V(I32x4MinU)             \
2210 2211
  V(I32x4MaxU)             \
  V(I32x4GtU)              \
2212
  V(I32x4GeU)              \
2213
  V(I16x8SConvertI32x4)    \
2214
  V(I16x8Add)              \
2215
  V(I16x8AddSatS)          \
2216
  V(I16x8Sub)              \
2217
  V(I16x8SubSatS)          \
2218 2219 2220 2221 2222
  V(I16x8Mul)              \
  V(I16x8MinS)             \
  V(I16x8MaxS)             \
  V(I16x8Eq)               \
  V(I16x8Ne)               \
2223 2224
  V(I16x8GtS)              \
  V(I16x8GeS)              \
2225 2226
  V(I16x8AddSatU)          \
  V(I16x8SubSatU)          \
2227
  V(I16x8MinU)             \
2228 2229
  V(I16x8MaxU)             \
  V(I16x8GtU)              \
2230
  V(I16x8GeU)              \
2231
  V(I8x16SConvertI16x8)    \
2232 2233 2234
  V(I8x16Ne)               \
  V(I8x16GeS)              \
  V(I8x16GtU)              \
2235 2236 2237 2238
  V(I8x16GeU)              \
  V(S128And)               \
  V(S128Or)                \
  V(S128Xor)
2239

2240
#define SIMD_BINOP_UNIFIED_SSE_AVX_LIST(V) \
2241 2242 2243 2244
  V(F32x4Add)                              \
  V(F32x4Sub)                              \
  V(F32x4Mul)                              \
  V(F32x4Div)                              \
2245
  V(I64x2Add)                              \
2246
  V(I64x2Sub)                              \
2247
  V(I64x2Eq)                               \
2248
  V(I64x2Ne)                               \
2249
  V(I32x4DotI16x8S)                        \
2250
  V(I16x8RoundingAverageU)                 \
2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262
  V(I8x16Add)                              \
  V(I8x16AddSatS)                          \
  V(I8x16Sub)                              \
  V(I8x16SubSatS)                          \
  V(I8x16MinS)                             \
  V(I8x16MaxS)                             \
  V(I8x16Eq)                               \
  V(I8x16GtS)                              \
  V(I8x16AddSatU)                          \
  V(I8x16SubSatU)                          \
  V(I8x16MinU)                             \
  V(I8x16MaxU)                             \
2263
  V(I8x16RoundingAverageU)
2264

2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278
// These opcodes require all inputs to be registers because the codegen is
// simpler with all registers.
#define SIMD_BINOP_RRR(V)  \
  V(I64x2ExtMulLowI32x4S)  \
  V(I64x2ExtMulHighI32x4S) \
  V(I64x2ExtMulLowI32x4U)  \
  V(I64x2ExtMulHighI32x4U) \
  V(I32x4ExtMulLowI16x8S)  \
  V(I32x4ExtMulHighI16x8S) \
  V(I32x4ExtMulLowI16x8U)  \
  V(I32x4ExtMulHighI16x8U) \
  V(I16x8ExtMulLowI8x16S)  \
  V(I16x8ExtMulHighI8x16S) \
  V(I16x8ExtMulLowI8x16U)  \
2279 2280
  V(I16x8ExtMulHighI8x16U) \
  V(I16x8Q15MulRSatS)
2281

2282
#define SIMD_UNOP_LIST(V)   \
2283 2284 2285
  V(F64x2ConvertLowI32x4S)  \
  V(F64x2PromoteLowF32x4)   \
  V(F32x4DemoteF64x2Zero)   \
2286 2287
  V(F32x4Abs)               \
  V(F32x4Neg)               \
2288
  V(F32x4Sqrt)              \
2289 2290 2291
  V(F32x4SConvertI32x4)     \
  V(F32x4RecipApprox)       \
  V(F32x4RecipSqrtApprox)   \
2292
  V(I64x2BitMask)           \
2293 2294 2295 2296
  V(I64x2SConvertI32x4Low)  \
  V(I64x2SConvertI32x4High) \
  V(I64x2UConvertI32x4Low)  \
  V(I64x2UConvertI32x4High) \
2297 2298 2299 2300 2301
  V(I32x4SConvertI16x8Low)  \
  V(I32x4SConvertI16x8High) \
  V(I32x4Neg)               \
  V(I32x4UConvertI16x8Low)  \
  V(I32x4UConvertI16x8High) \
2302
  V(I32x4Abs)               \
2303
  V(I32x4BitMask)           \
2304 2305 2306 2307 2308
  V(I16x8SConvertI8x16Low)  \
  V(I16x8SConvertI8x16High) \
  V(I16x8Neg)               \
  V(I16x8UConvertI8x16Low)  \
  V(I16x8UConvertI8x16High) \
2309 2310
  V(I16x8Abs)               \
  V(I8x16Neg)               \
2311
  V(I8x16Abs)               \
2312
  V(I8x16BitMask)           \
2313 2314
  V(S128Not)

2315
#define SIMD_ALLTRUE_LIST(V) \
2316 2317 2318 2319
  V(I64x2AllTrue)            \
  V(I32x4AllTrue)            \
  V(I16x8AllTrue)            \
  V(I8x16AllTrue)
2320

2321 2322
#define SIMD_SHIFT_OPCODES_UNIFED_SSE_AVX(V) \
  V(I64x2Shl)                                \
2323 2324 2325
  V(I64x2ShrU)                               \
  V(I32x4Shl)                                \
  V(I32x4ShrS)                               \
2326 2327 2328 2329
  V(I32x4ShrU)                               \
  V(I16x8Shl)                                \
  V(I16x8ShrS)                               \
  V(I16x8ShrU)
2330

2331 2332 2333 2334
void InstructionSelector::VisitS128Const(Node* node) {
  IA32OperandGenerator g(this);
  static const int kUint32Immediates = kSimd128Size / sizeof(uint32_t);
  uint32_t val[kUint32Immediates];
2335
  memcpy(val, S128ImmediateParameterOf(node->op()).data(), kSimd128Size);
2336
  // If all bytes are zeros or ones, avoid emitting code for generic constants
2337
  bool all_zeros = !(val[0] || val[1] || val[2] || val[3]);
2338 2339 2340 2341 2342 2343 2344 2345 2346 2347 2348 2349 2350 2351 2352 2353 2354
  bool all_ones = val[0] == UINT32_MAX && val[1] == UINT32_MAX &&
                  val[2] == UINT32_MAX && val[3] == UINT32_MAX;
  InstructionOperand dst = g.DefineAsRegister(node);
  if (all_zeros) {
    Emit(kIA32S128Zero, dst);
  } else if (all_ones) {
    Emit(kIA32S128AllOnes, dst);
  } else {
    InstructionOperand inputs[kUint32Immediates];
    for (int i = 0; i < kUint32Immediates; ++i) {
      inputs[i] = g.UseImmediate(val[i]);
    }
    InstructionOperand temp(g.TempRegister());
    Emit(kIA32S128Const, 1, &dst, kUint32Immediates, inputs, 1, &temp);
  }
}

2355 2356
void InstructionSelector::VisitF64x2Min(Node* node) {
  IA32OperandGenerator g(this);
2357 2358
  InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
  InstructionOperand operand1 = g.UseRegister(node->InputAt(1));
2359 2360

  if (IsSupported(AVX)) {
2361
    Emit(kIA32F64x2Min, g.DefineAsRegister(node), operand0, operand1);
2362
  } else {
2363
    Emit(kIA32F64x2Min, g.DefineSameAsFirst(node), operand0, operand1);
2364 2365 2366 2367 2368
  }
}

void InstructionSelector::VisitF64x2Max(Node* node) {
  IA32OperandGenerator g(this);
2369 2370
  InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
  InstructionOperand operand1 = g.UseRegister(node->InputAt(1));
2371
  if (IsSupported(AVX)) {
2372
    Emit(kIA32F64x2Max, g.DefineAsRegister(node), operand0, operand1);
2373
  } else {
2374
    Emit(kIA32F64x2Max, g.DefineSameAsFirst(node), operand0, operand1);
2375 2376 2377
  }
}

2378
void InstructionSelector::VisitF64x2Splat(Node* node) {
2379
  VisitRRSimd(this, node, kIA32F64x2Splat);
2380 2381 2382
}

void InstructionSelector::VisitF64x2ExtractLane(Node* node) {
2383
  VisitRRISimd(this, node, kF64x2ExtractLane, kF64x2ExtractLane);
2384 2385
}

2386 2387
void InstructionSelector::VisitI64x2SplatI32Pair(Node* node) {
  IA32OperandGenerator g(this);
2388 2389 2390 2391 2392 2393 2394 2395 2396
  Int32Matcher match_left(node->InputAt(0));
  Int32Matcher match_right(node->InputAt(1));
  if (match_left.Is(0) && match_right.Is(0)) {
    Emit(kIA32S128Zero, g.DefineAsRegister(node));
  } else {
    InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
    InstructionOperand operand1 = g.Use(node->InputAt(1));
    Emit(kIA32I64x2SplatI32Pair, g.DefineAsRegister(node), operand0, operand1);
  }
2397 2398 2399 2400 2401 2402 2403 2404
}

void InstructionSelector::VisitI64x2ReplaceLaneI32Pair(Node* node) {
  IA32OperandGenerator g(this);
  InstructionOperand operand = g.UseRegister(node->InputAt(0));
  InstructionOperand lane = g.UseImmediate(OpParameter<int32_t>(node->op()));
  InstructionOperand low = g.Use(node->InputAt(1));
  InstructionOperand high = g.Use(node->InputAt(2));
2405 2406
  Emit(kIA32I64x2ReplaceLaneI32Pair, g.DefineSameAsFirst(node), operand, lane,
       low, high);
2407 2408
}

2409 2410
void InstructionSelector::VisitI64x2Neg(Node* node) {
  IA32OperandGenerator g(this);
2411 2412 2413 2414
  // If AVX unsupported, make sure dst != src to avoid a move.
  InstructionOperand operand0 = IsSupported(AVX)
                                    ? g.UseRegister(node->InputAt(0))
                                    : g.UseUnique(node->InputAt(0));
2415 2416 2417
  Emit(kIA32I64x2Neg, g.DefineAsRegister(node), operand0);
}

2418 2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432
void InstructionSelector::VisitI64x2ShrS(Node* node) {
  IA32OperandGenerator g(this);
  InstructionOperand temps[] = {g.TempSimd128Register(),
                                g.TempSimd128Register()};
  if (IsSupported(AVX)) {
    Emit(kIA32I64x2ShrS, g.DefineAsRegister(node),
         g.UseUniqueRegister(node->InputAt(0)), g.Use(node->InputAt(1)),
         arraysize(temps), temps);
  } else {
    Emit(kIA32I64x2ShrS, g.DefineSameAsFirst(node),
         g.UseUniqueRegister(node->InputAt(0)), g.Use(node->InputAt(1)),
         arraysize(temps), temps);
  }
}

2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447
void InstructionSelector::VisitI64x2Mul(Node* node) {
  IA32OperandGenerator g(this);
  InstructionOperand temps[] = {g.TempSimd128Register(),
                                g.TempSimd128Register()};
  if (IsSupported(AVX)) {
    Emit(kIA32I64x2Mul, g.DefineAsRegister(node),
         g.UseUniqueRegister(node->InputAt(0)),
         g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
  } else {
    Emit(kIA32I64x2Mul, g.DefineSameAsFirst(node),
         g.UseUniqueRegister(node->InputAt(0)),
         g.UseUniqueRegister(node->InputAt(1)), arraysize(temps), temps);
  }
}

2448
void InstructionSelector::VisitF32x4Splat(Node* node) {
2449
  VisitRRSimd(this, node, kIA32F32x4Splat);
2450 2451 2452
}

void InstructionSelector::VisitF32x4ExtractLane(Node* node) {
2453 2454 2455 2456 2457
  IA32OperandGenerator g(this);
  InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
  InstructionOperand operand1 =
      g.UseImmediate(OpParameter<int32_t>(node->op()));
  Emit(kIA32F32x4ExtractLane, g.DefineAsRegister(node), operand0, operand1);
2458 2459
}

2460
void InstructionSelector::VisitF32x4UConvertI32x4(Node* node) {
2461
  VisitRRSimd(this, node, kIA32F32x4UConvertI32x4);
2462 2463
}

2464
void InstructionSelector::VisitI32x4SConvertF32x4(Node* node) {
2465
  VisitRRSimd(this, node, kIA32I32x4SConvertF32x4);
2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476
}

void InstructionSelector::VisitI32x4UConvertF32x4(Node* node) {
  IA32OperandGenerator g(this);
  InstructionOperand temps[] = {g.TempSimd128Register()};
  InstructionCode opcode =
      IsSupported(AVX) ? kAVXI32x4UConvertF32x4 : kSSEI32x4UConvertF32x4;
  Emit(opcode, g.DefineSameAsFirst(node), g.UseRegister(node->InputAt(0)),
       arraysize(temps), temps);
}

2477 2478 2479 2480 2481
void InstructionSelector::VisitS128Zero(Node* node) {
  IA32OperandGenerator g(this);
  Emit(kIA32S128Zero, g.DefineAsRegister(node));
}

jing.bao's avatar
jing.bao committed
2482 2483
void InstructionSelector::VisitS128Select(Node* node) {
  IA32OperandGenerator g(this);
2484 2485 2486 2487
  InstructionOperand dst =
      IsSupported(AVX) ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node);
  Emit(kIA32S128Select, dst, g.UseRegister(node->InputAt(0)),
       g.UseRegister(node->InputAt(1)), g.UseRegister(node->InputAt(2)));
jing.bao's avatar
jing.bao committed
2488 2489
}

2490 2491 2492
void InstructionSelector::VisitS128AndNot(Node* node) {
  IA32OperandGenerator g(this);
  // andnps a b does ~a & b, but we want a & !b, so flip the input.
2493 2494 2495 2496
  InstructionOperand dst =
      IsSupported(AVX) ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node);
  Emit(kIA32S128AndNot, dst, g.UseRegister(node->InputAt(1)),
       g.UseRegister(node->InputAt(0)));
2497 2498
}

2499 2500
#define VISIT_SIMD_SPLAT(Type)                               \
  void InstructionSelector::Visit##Type##Splat(Node* node) { \
2501 2502 2503 2504 2505 2506 2507
    Int32Matcher int32_matcher(node->InputAt(0));            \
    if (int32_matcher.Is(0)) {                               \
      IA32OperandGenerator g(this);                          \
      Emit(kIA32S128Zero, g.DefineAsRegister(node));         \
    } else {                                                 \
      VisitRO(this, node, kIA32##Type##Splat);               \
    }                                                        \
2508
  }
2509
SIMD_INT_TYPES(VISIT_SIMD_SPLAT)
2510
#undef SIMD_INT_TYPES
2511 2512
#undef VISIT_SIMD_SPLAT

2513 2514 2515
void InstructionSelector::VisitI8x16ExtractLaneU(Node* node) {
  VisitRRISimd(this, node, kIA32Pextrb);
}
2516

2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548
void InstructionSelector::VisitI8x16ExtractLaneS(Node* node) {
  VisitRRISimd(this, node, kIA32I8x16ExtractLaneS);
}

void InstructionSelector::VisitI16x8ExtractLaneU(Node* node) {
  VisitRRISimd(this, node, kIA32Pextrw);
}

void InstructionSelector::VisitI16x8ExtractLaneS(Node* node) {
  VisitRRISimd(this, node, kIA32I16x8ExtractLaneS);
}

void InstructionSelector::VisitI32x4ExtractLane(Node* node) {
  VisitRRISimd(this, node, kIA32I32x4ExtractLane);
}

#define SIMD_REPLACE_LANE_TYPE_OP(V) \
  V(I32x4, kIA32Pinsrd)              \
  V(I16x8, kIA32Pinsrw)              \
  V(I8x16, kIA32Pinsrb)              \
  V(F32x4, kIA32Insertps)

#define VISIT_SIMD_REPLACE_LANE(TYPE, OPCODE)                              \
  void InstructionSelector::Visit##TYPE##ReplaceLane(Node* node) {         \
    IA32OperandGenerator g(this);                                          \
    InstructionOperand operand0 = g.UseRegister(node->InputAt(0));         \
    InstructionOperand operand1 =                                          \
        g.UseImmediate(OpParameter<int32_t>(node->op()));                  \
    InstructionOperand operand2 = g.Use(node->InputAt(1));                 \
    InstructionOperand dst = IsSupported(AVX) ? g.DefineAsRegister(node)   \
                                              : g.DefineSameAsFirst(node); \
    Emit(OPCODE, dst, operand0, operand1, operand2);                       \
2549
  }
2550
SIMD_REPLACE_LANE_TYPE_OP(VISIT_SIMD_REPLACE_LANE)
2551
#undef VISIT_SIMD_REPLACE_LANE
2552
#undef SIMD_REPLACE_LANE_TYPE_OP
2553

2554 2555 2556 2557 2558 2559 2560 2561 2562
void InstructionSelector::VisitF64x2ReplaceLane(Node* node) {
  IA32OperandGenerator g(this);
  int32_t lane = OpParameter<int32_t>(node->op());
  // When no-AVX, define dst == src to save a move.
  InstructionOperand dst =
      IsSupported(AVX) ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node);
  Emit(kF64x2ReplaceLane, dst, g.UseRegister(node->InputAt(0)),
       g.UseImmediate(lane), g.UseRegister(node->InputAt(1)));
}
2563

2564 2565 2566
#define VISIT_SIMD_SHIFT_UNIFIED_SSE_AVX(Opcode)        \
  void InstructionSelector::Visit##Opcode(Node* node) { \
    VisitRROSimdShift(this, node, kIA32##Opcode);       \
2567 2568 2569 2570 2571
  }
SIMD_SHIFT_OPCODES_UNIFED_SSE_AVX(VISIT_SIMD_SHIFT_UNIFIED_SSE_AVX)
#undef VISIT_SIMD_SHIFT_UNIFIED_SSE_AVX
#undef SIMD_SHIFT_OPCODES_UNIFED_SSE_AVX

2572 2573 2574 2575 2576 2577 2578 2579 2580
// TODO(v8:9198): SSE requires operand0 to be a register as we don't have memory
// alignment yet. For AVX, memory operands are fine, but can have performance
// issues if not aligned to 16/32 bytes (based on load size), see SDM Vol 1,
// chapter 14.9
#define VISIT_SIMD_UNOP(Opcode)                         \
  void InstructionSelector::Visit##Opcode(Node* node) { \
    IA32OperandGenerator g(this);                       \
    Emit(kIA32##Opcode, g.DefineAsRegister(node),       \
         g.UseRegister(node->InputAt(0)));              \
2581
  }
2582 2583 2584
SIMD_UNOP_LIST(VISIT_SIMD_UNOP)
#undef VISIT_SIMD_UNOP
#undef SIMD_UNOP_LIST
2585

2586 2587 2588 2589 2590 2591
void InstructionSelector::VisitV128AnyTrue(Node* node) {
  IA32OperandGenerator g(this);
  InstructionOperand temps[] = {g.TempRegister()};
  Emit(kIA32S128AnyTrue, g.DefineAsRegister(node),
       g.UseRegister(node->InputAt(0)), arraysize(temps), temps);
}
2592

2593 2594 2595 2596 2597
#define VISIT_SIMD_ALLTRUE(Opcode)                                            \
  void InstructionSelector::Visit##Opcode(Node* node) {                       \
    IA32OperandGenerator g(this);                                             \
    InstructionOperand temps[] = {g.TempRegister(), g.TempSimd128Register()}; \
    Emit(kIA32##Opcode, g.DefineAsRegister(node),                             \
2598
         g.UseUniqueRegister(node->InputAt(0)), arraysize(temps), temps);     \
2599 2600 2601 2602 2603
  }
SIMD_ALLTRUE_LIST(VISIT_SIMD_ALLTRUE)
#undef VISIT_SIMD_ALLTRUE
#undef SIMD_ALLTRUE_LIST

2604 2605 2606
#define VISIT_SIMD_BINOP(Opcode)                          \
  void InstructionSelector::Visit##Opcode(Node* node) {   \
    VisitRROSimd(this, node, kAVX##Opcode, kSSE##Opcode); \
2607 2608 2609
  }
SIMD_BINOP_LIST(VISIT_SIMD_BINOP)
#undef VISIT_SIMD_BINOP
2610
#undef SIMD_BINOP_LIST
2611

2612 2613 2614
#define VISIT_SIMD_BINOP_UNIFIED_SSE_AVX(Opcode)            \
  void InstructionSelector::Visit##Opcode(Node* node) {     \
    VisitRROSimd(this, node, kIA32##Opcode, kIA32##Opcode); \
2615 2616 2617 2618 2619
  }
SIMD_BINOP_UNIFIED_SSE_AVX_LIST(VISIT_SIMD_BINOP_UNIFIED_SSE_AVX)
#undef VISIT_SIMD_BINOP_UNIFIED_SSE_AVX
#undef SIMD_BINOP_UNIFIED_SSE_AVX_LIST

2620 2621 2622 2623 2624 2625 2626 2627
#define VISIT_SIMD_BINOP_RRR(OPCODE)                    \
  void InstructionSelector::Visit##OPCODE(Node* node) { \
    VisitRRRSimd(this, node, kIA32##OPCODE);            \
  }
SIMD_BINOP_RRR(VISIT_SIMD_BINOP_RRR)
#undef VISIT_SIMD_BINOP_RRR
#undef SIMD_BINOP_RRR

2628 2629 2630 2631
// TODO(v8:9198): SSE requires operand1 to be a register as we don't have memory
// alignment yet. For AVX, memory operands are fine, but can have performance
// issues if not aligned to 16/32 bytes (based on load size), see SDM Vol 1,
// chapter 14.9
2632 2633 2634 2635
void VisitPack(InstructionSelector* selector, Node* node, ArchOpcode avx_opcode,
               ArchOpcode sse_opcode) {
  IA32OperandGenerator g(selector);
  InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
2636
  InstructionOperand operand1 = g.UseRegister(node->InputAt(1));
2637 2638 2639 2640 2641 2642 2643 2644 2645 2646 2647
  if (selector->IsSupported(AVX)) {
    selector->Emit(avx_opcode, g.DefineSameAsFirst(node), operand0, operand1);
  } else {
    selector->Emit(sse_opcode, g.DefineSameAsFirst(node), operand0, operand1);
  }
}

void InstructionSelector::VisitI16x8UConvertI32x4(Node* node) {
  VisitPack(this, node, kAVXI16x8UConvertI32x4, kSSEI16x8UConvertI32x4);
}

2648 2649 2650 2651 2652 2653 2654
void InstructionSelector::VisitI16x8BitMask(Node* node) {
  IA32OperandGenerator g(this);
  InstructionOperand temps[] = {g.TempSimd128Register()};
  Emit(kIA32I16x8BitMask, g.DefineAsRegister(node),
       g.UseUniqueRegister(node->InputAt(0)), arraysize(temps), temps);
}

2655 2656 2657 2658
void InstructionSelector::VisitI8x16UConvertI16x8(Node* node) {
  VisitPack(this, node, kAVXI8x16UConvertI16x8, kSSEI8x16UConvertI16x8);
}

2659 2660
void InstructionSelector::VisitI8x16Shl(Node* node) {
  IA32OperandGenerator g(this);
2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691
  if (g.CanBeImmediate(node->InputAt(1))) {
    InstructionOperand temps[] = {g.TempRegister(), g.TempSimd128Register()};
    this->Emit(kIA32I8x16Shl, g.DefineSameAsFirst(node),
               g.UseRegister(node->InputAt(0)),
               g.UseImmediate(node->InputAt(1)), arraysize(temps), temps);
  } else {
    VisitRROI8x16SimdShift(this, node, kIA32I8x16Shl);
  }
}

void InstructionSelector::VisitI8x16ShrS(Node* node) {
  IA32OperandGenerator g(this);
  if (g.CanBeImmediate(node->InputAt(1))) {
    this->Emit(kIA32I8x16ShrS, g.DefineSameAsFirst(node),
               g.UseRegister(node->InputAt(0)),
               g.UseImmediate(node->InputAt(1)));
  } else {
    VisitRROI8x16SimdShift(this, node, kIA32I8x16ShrS);
  }
}

void InstructionSelector::VisitI8x16ShrU(Node* node) {
  IA32OperandGenerator g(this);
  if (g.CanBeImmediate(node->InputAt(1))) {
    InstructionOperand temps[] = {g.TempRegister(), g.TempSimd128Register()};
    this->Emit(kIA32I8x16ShrU, g.DefineSameAsFirst(node),
               g.UseRegister(node->InputAt(0)),
               g.UseImmediate(node->InputAt(1)), arraysize(temps), temps);
  } else {
    VisitRROI8x16SimdShift(this, node, kIA32I8x16ShrU);
  }
2692 2693
}

2694 2695 2696 2697 2698 2699 2700 2701
void InstructionSelector::VisitInt32AbsWithOverflow(Node* node) {
  UNREACHABLE();
}

void InstructionSelector::VisitInt64AbsWithOverflow(Node* node) {
  UNREACHABLE();
}

2702
#if V8_ENABLE_WEBASSEMBLY
2703 2704
namespace {

2705 2706
// Returns true if shuffle can be decomposed into two 16x4 half shuffles
// followed by a 16x8 blend.
2707
// E.g. [3 2 1 0 15 14 13 12].
2708
bool TryMatch16x8HalfShuffle(uint8_t* shuffle16x8, uint8_t* blend_mask) {
2709 2710 2711
  *blend_mask = 0;
  for (int i = 0; i < 8; i++) {
    if ((shuffle16x8[i] & 0x4) != (i & 0x4)) return false;
2712
    *blend_mask |= (shuffle16x8[i] > 7 ? 1 : 0) << i;
2713 2714 2715 2716
  }
  return true;
}

2717 2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798 2799 2800 2801
struct ShuffleEntry {
  uint8_t shuffle[kSimd128Size];
  ArchOpcode opcode;
  ArchOpcode avx_opcode;
  bool src0_needs_reg;
  bool src1_needs_reg;
};

// Shuffles that map to architecture-specific instruction sequences. These are
// matched very early, so we shouldn't include shuffles that match better in
// later tests, like 32x4 and 16x8 shuffles. In general, these patterns should
// map to either a single instruction, or be finer grained, such as zip/unzip or
// transpose patterns.
static const ShuffleEntry arch_shuffles[] = {
    {{0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23},
     kIA32S64x2UnpackLow,
     kIA32S64x2UnpackLow,
     true,
     false},
    {{8, 9, 10, 11, 12, 13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31},
     kIA32S64x2UnpackHigh,
     kIA32S64x2UnpackHigh,
     true,
     false},
    {{0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23},
     kIA32S32x4UnpackLow,
     kIA32S32x4UnpackLow,
     true,
     false},
    {{8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31},
     kIA32S32x4UnpackHigh,
     kIA32S32x4UnpackHigh,
     true,
     false},
    {{0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23},
     kIA32S16x8UnpackLow,
     kIA32S16x8UnpackLow,
     true,
     false},
    {{8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31},
     kIA32S16x8UnpackHigh,
     kIA32S16x8UnpackHigh,
     true,
     false},
    {{0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23},
     kIA32S8x16UnpackLow,
     kIA32S8x16UnpackLow,
     true,
     false},
    {{8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31},
     kIA32S8x16UnpackHigh,
     kIA32S8x16UnpackHigh,
     true,
     false},

    {{0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29},
     kSSES16x8UnzipLow,
     kAVXS16x8UnzipLow,
     true,
     false},
    {{2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31},
     kSSES16x8UnzipHigh,
     kAVXS16x8UnzipHigh,
     true,
     true},
    {{0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30},
     kSSES8x16UnzipLow,
     kAVXS8x16UnzipLow,
     true,
     true},
    {{1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31},
     kSSES8x16UnzipHigh,
     kAVXS8x16UnzipHigh,
     true,
     true},

    {{0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30},
     kSSES8x16TransposeLow,
     kAVXS8x16TransposeLow,
     true,
     true},
    {{1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31},
     kSSES8x16TransposeHigh,
     kAVXS8x16TransposeHigh,
     true,
2802 2803 2804 2805
     true},
    {{7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8},
     kSSES8x8Reverse,
     kAVXS8x8Reverse,
2806 2807
     true,
     true},
2808 2809 2810
    {{3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12},
     kSSES8x4Reverse,
     kAVXS8x4Reverse,
2811 2812
     true,
     true},
2813 2814 2815 2816
    {{1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14},
     kSSES8x2Reverse,
     kAVXS8x2Reverse,
     true,
2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838
     true}};

bool TryMatchArchShuffle(const uint8_t* shuffle, const ShuffleEntry* table,
                         size_t num_entries, bool is_swizzle,
                         const ShuffleEntry** arch_shuffle) {
  uint8_t mask = is_swizzle ? kSimd128Size - 1 : 2 * kSimd128Size - 1;
  for (size_t i = 0; i < num_entries; ++i) {
    const ShuffleEntry& entry = table[i];
    int j = 0;
    for (; j < kSimd128Size; ++j) {
      if ((entry.shuffle[j] & mask) != (shuffle[j] & mask)) {
        break;
      }
    }
    if (j == kSimd128Size) {
      *arch_shuffle = &entry;
      return true;
    }
  }
  return false;
}

2839 2840
}  // namespace

2841
void InstructionSelector::VisitI8x16Shuffle(Node* node) {
2842
  uint8_t shuffle[kSimd128Size];
2843 2844
  bool is_swizzle;
  CanonicalizeShuffle(node, shuffle, &is_swizzle);
2845 2846 2847 2848 2849 2850 2851 2852

  int imm_count = 0;
  static const int kMaxImms = 6;
  uint32_t imms[kMaxImms];
  int temp_count = 0;
  static const int kMaxTemps = 2;
  InstructionOperand temps[kMaxTemps];

2853
  IA32OperandGenerator g(this);
2854
  bool use_avx = CpuFeatures::IsSupported(AVX);
2855 2856
  // AVX and swizzles don't generally need DefineSameAsFirst to avoid a move.
  bool no_same_as_first = use_avx || is_swizzle;
2857
  // We generally need UseRegister for input0, Use for input1.
2858 2859 2860
  // TODO(v8:9198): We don't have 16-byte alignment for SIMD operands yet, but
  // we retain this logic (continue setting these in the various shuffle match
  // clauses), but ignore it when selecting registers or slots.
2861 2862
  bool src0_needs_reg = true;
  bool src1_needs_reg = false;
2863
  ArchOpcode opcode = kIA32I8x16Shuffle;  // general shuffle is the default
2864 2865 2866 2867

  uint8_t offset;
  uint8_t shuffle32x4[4];
  uint8_t shuffle16x8[8];
2868
  int index;
2869
  const ShuffleEntry* arch_shuffle;
2870
  if (wasm::SimdShuffle::TryMatchConcat(shuffle, &offset)) {
2871 2872 2873 2874 2875 2876 2877 2878 2879 2880 2881 2882 2883 2884
    if (wasm::SimdShuffle::TryMatch32x4Rotate(shuffle, shuffle32x4,
                                              is_swizzle)) {
      uint8_t shuffle_mask = wasm::SimdShuffle::PackShuffle4(shuffle32x4);
      opcode = kIA32S32x4Rotate;
      imms[imm_count++] = shuffle_mask;
    } else {
      // Swap inputs from the normal order for (v)palignr.
      SwapShuffleInputs(node);
      is_swizzle = false;  // It's simpler to just handle the general case.
      no_same_as_first = use_avx;  // SSE requires same-as-first.
      opcode = kIA32S8x16Alignr;
      // palignr takes a single imm8 offset.
      imms[imm_count++] = offset;
    }
2885 2886 2887 2888
  } else if (TryMatchArchShuffle(shuffle, arch_shuffles,
                                 arraysize(arch_shuffles), is_swizzle,
                                 &arch_shuffle)) {
    opcode = use_avx ? arch_shuffle->avx_opcode : arch_shuffle->opcode;
2889
    src0_needs_reg = !use_avx || arch_shuffle->src0_needs_reg;
2890 2891 2892 2893
    // SSE can't take advantage of both operands in registers and needs
    // same-as-first.
    src1_needs_reg = use_avx && arch_shuffle->src1_needs_reg;
    no_same_as_first = use_avx;
2894
  } else if (wasm::SimdShuffle::TryMatch32x4Shuffle(shuffle, shuffle32x4)) {
2895
    uint8_t shuffle_mask = wasm::SimdShuffle::PackShuffle4(shuffle32x4);
2896
    if (is_swizzle) {
2897
      if (wasm::SimdShuffle::TryMatchIdentity(shuffle)) {
2898 2899 2900 2901 2902 2903 2904
        // Bypass normal shuffle code generation in this case.
        EmitIdentity(node);
        return;
      } else {
        // pshufd takes a single imm8 shuffle mask.
        opcode = kIA32S32x4Swizzle;
        no_same_as_first = true;
2905 2906 2907 2908
        // TODO(v8:9198): This doesn't strictly require a register, forcing the
        // swizzles to always use registers until generation of incorrect memory
        // operands can be fixed.
        src0_needs_reg = true;
2909 2910
        imms[imm_count++] = shuffle_mask;
      }
2911 2912 2913
    } else {
      // 2 operand shuffle
      // A blend is more efficient than a general 32x4 shuffle; try it first.
2914
      if (wasm::SimdShuffle::TryMatchBlend(shuffle)) {
2915
        opcode = kIA32S16x8Blend;
2916
        uint8_t blend_mask = wasm::SimdShuffle::PackBlend4(shuffle32x4);
2917 2918 2919
        imms[imm_count++] = blend_mask;
      } else {
        opcode = kIA32S32x4Shuffle;
2920
        no_same_as_first = true;
2921 2922 2923 2924 2925
        // TODO(v8:9198): src0 and src1 is used by pshufd in codegen, which
        // requires memory to be 16-byte aligned, since we cannot guarantee that
        // yet, force using a register here.
        src0_needs_reg = true;
        src1_needs_reg = true;
2926
        imms[imm_count++] = shuffle_mask;
2927
        int8_t blend_mask = wasm::SimdShuffle::PackBlend4(shuffle32x4);
2928 2929
        imms[imm_count++] = blend_mask;
      }
2930
    }
2931
  } else if (wasm::SimdShuffle::TryMatch16x8Shuffle(shuffle, shuffle16x8)) {
2932
    uint8_t blend_mask;
2933
    if (wasm::SimdShuffle::TryMatchBlend(shuffle)) {
2934
      opcode = kIA32S16x8Blend;
2935
      blend_mask = wasm::SimdShuffle::PackBlend8(shuffle16x8);
2936
      imms[imm_count++] = blend_mask;
2937
    } else if (wasm::SimdShuffle::TryMatchSplat<8>(shuffle, &index)) {
2938 2939 2940
      opcode = kIA32S16x8Dup;
      src0_needs_reg = false;
      imms[imm_count++] = index;
2941
    } else if (TryMatch16x8HalfShuffle(shuffle16x8, &blend_mask)) {
2942 2943 2944
      opcode = is_swizzle ? kIA32S16x8HalfShuffle1 : kIA32S16x8HalfShuffle2;
      // Half-shuffles don't need DefineSameAsFirst or UseRegister(src0).
      no_same_as_first = true;
2945
      src0_needs_reg = false;
2946 2947
      uint8_t mask_lo = wasm::SimdShuffle::PackShuffle4(shuffle16x8);
      uint8_t mask_hi = wasm::SimdShuffle::PackShuffle4(shuffle16x8 + 4);
2948 2949
      imms[imm_count++] = mask_lo;
      imms[imm_count++] = mask_hi;
2950
      if (!is_swizzle) imms[imm_count++] = blend_mask;
2951
    }
2952
  } else if (wasm::SimdShuffle::TryMatchSplat<16>(shuffle, &index)) {
2953 2954 2955 2956
    opcode = kIA32S8x16Dup;
    no_same_as_first = use_avx;
    src0_needs_reg = true;
    imms[imm_count++] = index;
2957
  }
2958
  if (opcode == kIA32I8x16Shuffle) {
2959 2960
    // Use same-as-first for general swizzle, but not shuffle.
    no_same_as_first = !is_swizzle;
2961
    src0_needs_reg = !no_same_as_first;
2962 2963 2964 2965
    imms[imm_count++] = wasm::SimdShuffle::Pack4Lanes(shuffle);
    imms[imm_count++] = wasm::SimdShuffle::Pack4Lanes(shuffle + 4);
    imms[imm_count++] = wasm::SimdShuffle::Pack4Lanes(shuffle + 8);
    imms[imm_count++] = wasm::SimdShuffle::Pack4Lanes(shuffle + 12);
2966 2967 2968
    temps[temp_count++] = g.TempRegister();
  }

2969 2970 2971 2972 2973
  // Use DefineAsRegister(node) and Use(src0) if we can without forcing an extra
  // move instruction in the CodeGenerator.
  Node* input0 = node->InputAt(0);
  InstructionOperand dst =
      no_same_as_first ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node);
2974 2975 2976
  // TODO(v8:9198): Use src0_needs_reg when we have memory alignment for SIMD.
  InstructionOperand src0 = g.UseRegister(input0);
  USE(src0_needs_reg);
2977 2978 2979 2980

  int input_count = 0;
  InstructionOperand inputs[2 + kMaxImms + kMaxTemps];
  inputs[input_count++] = src0;
2981
  if (!is_swizzle) {
2982
    Node* input1 = node->InputAt(1);
2983 2984 2985
    // TODO(v8:9198): Use src1_needs_reg when we have memory alignment for SIMD.
    inputs[input_count++] = g.UseRegister(input1);
    USE(src1_needs_reg);
2986 2987 2988 2989
  }
  for (int i = 0; i < imm_count; ++i) {
    inputs[input_count++] = g.UseImmediate(imms[i]);
  }
2990
  Emit(opcode, 1, &dst, input_count, inputs, temp_count, temps);
2991 2992
}

2993
void InstructionSelector::VisitI8x16Swizzle(Node* node) {
2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004
  InstructionCode op = kIA32I8x16Swizzle;

  auto m = V128ConstMatcher(node->InputAt(1));
  if (m.HasResolvedValue()) {
    // If the indices vector is a const, check if they are in range, or if the
    // top bit is set, then we can avoid the paddusb in the codegen and simply
    // emit a pshufb.
    auto imms = m.ResolvedValue().immediate();
    op |= MiscField::encode(wasm::SimdSwizzle::AllInRangeOrTopBitSet(imms));
  }

3005
  IA32OperandGenerator g(this);
3006
  InstructionOperand temps[] = {g.TempRegister()};
3007
  Emit(op,
3008 3009
       IsSupported(AVX) ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node),
       g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)),
3010 3011
       arraysize(temps), temps);
}
3012 3013 3014 3015
#else
void InstructionSelector::VisitI8x16Shuffle(Node* node) { UNREACHABLE(); }
void InstructionSelector::VisitI8x16Swizzle(Node* node) { UNREACHABLE(); }
#endif  // V8_ENABLE_WEBASSEMBLY
3016

3017 3018 3019 3020 3021 3022
namespace {
void VisitPminOrPmax(InstructionSelector* selector, Node* node,
                     ArchOpcode opcode) {
  // Due to the way minps/minpd work, we want the dst to be same as the second
  // input: b = pmin(a, b) directly maps to minps b a.
  IA32OperandGenerator g(selector);
3023 3024 3025 3026
  InstructionOperand dst = selector->IsSupported(AVX)
                               ? g.DefineAsRegister(node)
                               : g.DefineSameAsFirst(node);
  selector->Emit(opcode, dst, g.UseRegister(node->InputAt(1)),
3027 3028 3029 3030 3031 3032 3033 3034 3035 3036 3037 3038 3039 3040 3041 3042 3043 3044 3045 3046
                 g.UseRegister(node->InputAt(0)));
}
}  // namespace

void InstructionSelector::VisitF32x4Pmin(Node* node) {
  VisitPminOrPmax(this, node, kIA32F32x4Pmin);
}

void InstructionSelector::VisitF32x4Pmax(Node* node) {
  VisitPminOrPmax(this, node, kIA32F32x4Pmax);
}

void InstructionSelector::VisitF64x2Pmin(Node* node) {
  VisitPminOrPmax(this, node, kIA32F64x2Pmin);
}

void InstructionSelector::VisitF64x2Pmax(Node* node) {
  VisitPminOrPmax(this, node, kIA32F64x2Pmax);
}

3047 3048 3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060 3061 3062 3063
namespace {
void VisitExtAddPairwise(InstructionSelector* selector, Node* node,
                         ArchOpcode opcode, bool need_temp) {
  IA32OperandGenerator g(selector);
  InstructionOperand operand0 = g.UseRegister(node->InputAt(0));
  InstructionOperand dst = (selector->IsSupported(AVX))
                               ? g.DefineAsRegister(node)
                               : g.DefineSameAsFirst(node);
  if (need_temp) {
    InstructionOperand temps[] = {g.TempRegister()};
    selector->Emit(opcode, dst, operand0, arraysize(temps), temps);
  } else {
    selector->Emit(opcode, dst, operand0);
  }
}
}  // namespace

3064
void InstructionSelector::VisitI32x4ExtAddPairwiseI16x8S(Node* node) {
3065
  VisitExtAddPairwise(this, node, kIA32I32x4ExtAddPairwiseI16x8S, true);
3066 3067 3068
}

void InstructionSelector::VisitI32x4ExtAddPairwiseI16x8U(Node* node) {
3069
  VisitExtAddPairwise(this, node, kIA32I32x4ExtAddPairwiseI16x8U, false);
3070 3071 3072
}

void InstructionSelector::VisitI16x8ExtAddPairwiseI8x16S(Node* node) {
3073
  VisitExtAddPairwise(this, node, kIA32I16x8ExtAddPairwiseI8x16S, true);
3074 3075 3076
}

void InstructionSelector::VisitI16x8ExtAddPairwiseI8x16U(Node* node) {
3077
  VisitExtAddPairwise(this, node, kIA32I16x8ExtAddPairwiseI8x16U, true);
3078 3079
}

3080 3081 3082 3083 3084 3085 3086 3087 3088 3089
void InstructionSelector::VisitI8x16Popcnt(Node* node) {
  IA32OperandGenerator g(this);
  InstructionOperand dst = CpuFeatures::IsSupported(AVX)
                               ? g.DefineAsRegister(node)
                               : g.DefineAsRegister(node);
  InstructionOperand temps[] = {g.TempSimd128Register(), g.TempRegister()};
  Emit(kIA32I8x16Popcnt, dst, g.UseUniqueRegister(node->InputAt(0)),
       arraysize(temps), temps);
}

3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120
void InstructionSelector::VisitF64x2ConvertLowI32x4U(Node* node) {
  IA32OperandGenerator g(this);
  InstructionOperand temps[] = {g.TempRegister()};
  InstructionOperand dst =
      IsSupported(AVX) ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node);
  Emit(kIA32F64x2ConvertLowI32x4U, dst, g.UseRegister(node->InputAt(0)),
       arraysize(temps), temps);
}

void InstructionSelector::VisitI32x4TruncSatF64x2SZero(Node* node) {
  IA32OperandGenerator g(this);
  InstructionOperand temps[] = {g.TempRegister()};
  if (IsSupported(AVX)) {
    // Requires dst != src.
    Emit(kIA32I32x4TruncSatF64x2SZero, g.DefineAsRegister(node),
         g.UseUniqueRegister(node->InputAt(0)), arraysize(temps), temps);
  } else {
    Emit(kIA32I32x4TruncSatF64x2SZero, g.DefineSameAsFirst(node),
         g.UseRegister(node->InputAt(0)), arraysize(temps), temps);
  }
}

void InstructionSelector::VisitI32x4TruncSatF64x2UZero(Node* node) {
  IA32OperandGenerator g(this);
  InstructionOperand temps[] = {g.TempRegister()};
  InstructionOperand dst =
      IsSupported(AVX) ? g.DefineAsRegister(node) : g.DefineSameAsFirst(node);
  Emit(kIA32I32x4TruncSatF64x2UZero, dst, g.UseRegister(node->InputAt(0)),
       arraysize(temps), temps);
}

3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151
void InstructionSelector::VisitI64x2GtS(Node* node) {
  IA32OperandGenerator g(this);
  if (CpuFeatures::IsSupported(AVX)) {
    Emit(kIA32I64x2GtS, g.DefineAsRegister(node),
         g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)));
  } else if (CpuFeatures::IsSupported(SSE4_2)) {
    Emit(kIA32I64x2GtS, g.DefineSameAsFirst(node),
         g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)));
  } else {
    Emit(kIA32I64x2GtS, g.DefineAsRegister(node),
         g.UseUniqueRegister(node->InputAt(0)),
         g.UseUniqueRegister(node->InputAt(1)));
  }
}

void InstructionSelector::VisitI64x2GeS(Node* node) {
  IA32OperandGenerator g(this);
  if (CpuFeatures::IsSupported(AVX)) {
    Emit(kIA32I64x2GeS, g.DefineAsRegister(node),
         g.UseRegister(node->InputAt(0)), g.UseRegister(node->InputAt(1)));
  } else if (CpuFeatures::IsSupported(SSE4_2)) {
    Emit(kIA32I64x2GeS, g.DefineAsRegister(node),
         g.UseUniqueRegister(node->InputAt(0)),
         g.UseRegister(node->InputAt(1)));
  } else {
    Emit(kIA32I64x2GeS, g.DefineAsRegister(node),
         g.UseUniqueRegister(node->InputAt(0)),
         g.UseUniqueRegister(node->InputAt(1)));
  }
}

3152
void InstructionSelector::VisitI64x2Abs(Node* node) {
3153
  VisitRRSimd(this, node, kIA32I64x2Abs, kIA32I64x2Abs);
3154 3155
}

3156 3157 3158 3159 3160 3161
void InstructionSelector::AddOutputToSelectContinuation(OperandGenerator* g,
                                                        int first_input_index,
                                                        Node* node) {
  UNREACHABLE();
}

3162 3163 3164
// static
MachineOperatorBuilder::Flags
InstructionSelector::SupportedMachineOperatorFlags() {
3165
  MachineOperatorBuilder::Flags flags =
3166
      MachineOperatorBuilder::kWord32ShiftIsSafe |
3167
      MachineOperatorBuilder::kWord32Ctz | MachineOperatorBuilder::kWord32Rol;
3168 3169 3170
  if (CpuFeatures::IsSupported(POPCNT)) {
    flags |= MachineOperatorBuilder::kWord32Popcnt;
  }
3171 3172 3173 3174 3175 3176 3177 3178 3179 3180
  if (CpuFeatures::IsSupported(SSE4_1)) {
    flags |= MachineOperatorBuilder::kFloat32RoundDown |
             MachineOperatorBuilder::kFloat64RoundDown |
             MachineOperatorBuilder::kFloat32RoundUp |
             MachineOperatorBuilder::kFloat64RoundUp |
             MachineOperatorBuilder::kFloat32RoundTruncate |
             MachineOperatorBuilder::kFloat64RoundTruncate |
             MachineOperatorBuilder::kFloat32RoundTiesEven |
             MachineOperatorBuilder::kFloat64RoundTiesEven;
  }
3181
  return flags;
3182
}
3183

3184 3185 3186 3187 3188 3189 3190
// static
MachineOperatorBuilder::AlignmentRequirements
InstructionSelector::AlignmentRequirements() {
  return MachineOperatorBuilder::AlignmentRequirements::
      FullUnalignedAccessSupport();
}

3191 3192 3193
}  // namespace compiler
}  // namespace internal
}  // namespace v8