code-assembler.cc 56.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12
// Copyright 2015 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/compiler/code-assembler.h"

#include <ostream>

#include "src/code-factory.h"
#include "src/compiler/graph.h"
#include "src/compiler/instruction-selector.h"
#include "src/compiler/linkage.h"
13
#include "src/compiler/node-matchers.h"
14 15 16 17 18 19 20 21
#include "src/compiler/pipeline.h"
#include "src/compiler/raw-machine-assembler.h"
#include "src/compiler/schedule.h"
#include "src/frames.h"
#include "src/interface-descriptors.h"
#include "src/interpreter/bytecodes.h"
#include "src/machine-type.h"
#include "src/macro-assembler.h"
22
#include "src/objects-inl.h"
23
#include "src/utils.h"
24
#include "src/zone/zone.h"
25

26 27 28 29 30
#define REPEAT_1_TO_2(V, T) V(T) V(T, T)
#define REPEAT_1_TO_3(V, T) REPEAT_1_TO_2(V, T) V(T, T, T)
#define REPEAT_1_TO_4(V, T) REPEAT_1_TO_3(V, T) V(T, T, T, T)
#define REPEAT_1_TO_5(V, T) REPEAT_1_TO_4(V, T) V(T, T, T, T, T)
#define REPEAT_1_TO_6(V, T) REPEAT_1_TO_5(V, T) V(T, T, T, T, T, T)
31 32 33
#define REPEAT_1_TO_7(V, T) REPEAT_1_TO_6(V, T) V(T, T, T, T, T, T, T)
#define REPEAT_1_TO_8(V, T) REPEAT_1_TO_7(V, T) V(T, T, T, T, T, T, T, T)
#define REPEAT_1_TO_9(V, T) REPEAT_1_TO_8(V, T) V(T, T, T, T, T, T, T, T, T)
34 35 36
#define REPEAT_1_TO_10(V, T) REPEAT_1_TO_9(V, T) V(T, T, T, T, T, T, T, T, T, T)
#define REPEAT_1_TO_11(V, T) \
  REPEAT_1_TO_10(V, T) V(T, T, T, T, T, T, T, T, T, T, T)
37 38
#define REPEAT_1_TO_12(V, T) \
  REPEAT_1_TO_11(V, T) V(T, T, T, T, T, T, T, T, T, T, T, T)
39

40 41
namespace v8 {
namespace internal {
42 43 44 45

constexpr MachineType MachineTypeOf<Smi>::value;
constexpr MachineType MachineTypeOf<Object>::value;

46 47
namespace compiler {

48 49 50 51 52 53 54 55 56
static_assert(std::is_convertible<TNode<Number>, TNode<Object>>::value,
              "test subtyping");
static_assert(std::is_convertible<TNode<UnionT<Smi, HeapNumber>>,
                                  TNode<UnionT<Smi, HeapObject>>>::value,
              "test subtyping");
static_assert(
    !std::is_convertible<TNode<UnionT<Smi, HeapObject>>, TNode<Number>>::value,
    "test subtyping");

57 58
CodeAssemblerState::CodeAssemblerState(
    Isolate* isolate, Zone* zone, const CallInterfaceDescriptor& descriptor,
59 60
    Code::Kind kind, const char* name, size_t result_size, uint32_t stub_key,
    int32_t builtin_index)
61 62
    // TODO(rmcilroy): Should we use Linkage::GetBytecodeDispatchDescriptor for
    // bytecode handlers?
63
    : CodeAssemblerState(
64 65 66 67 68
          isolate, zone,
          Linkage::GetStubCallDescriptor(
              isolate, zone, descriptor, descriptor.GetStackParameterCount(),
              CallDescriptor::kNoFlags, Operator::kNoProperties,
              MachineType::AnyTagged(), result_size),
69
          kind, name, stub_key, builtin_index) {}
70

71
CodeAssemblerState::CodeAssemblerState(Isolate* isolate, Zone* zone,
72
                                       int parameter_count, Code::Kind kind,
73
                                       const char* name, int32_t builtin_index)
74 75 76 77 78 79
    : CodeAssemblerState(
          isolate, zone,
          Linkage::GetJSCallDescriptor(zone, false, parameter_count,
                                       kind == Code::BUILTIN
                                           ? CallDescriptor::kPushArgumentCount
                                           : CallDescriptor::kNoFlags),
80
          kind, name, 0, builtin_index) {}
81 82 83

CodeAssemblerState::CodeAssemblerState(Isolate* isolate, Zone* zone,
                                       CallDescriptor* call_descriptor,
84
                                       Code::Kind kind, const char* name,
85
                                       uint32_t stub_key, int32_t builtin_index)
86 87 88
    : raw_assembler_(new RawMachineAssembler(
          isolate, new (zone) Graph(zone), call_descriptor,
          MachineType::PointerRepresentation(),
89 90
          InstructionSelector::SupportedMachineOperatorFlags(),
          InstructionSelector::AlignmentRequirements())),
91
      kind_(kind),
92
      name_(name),
93
      stub_key_(stub_key),
94
      builtin_index_(builtin_index),
95 96 97
      code_generated_(false),
      variables_(zone) {}

98 99
CodeAssemblerState::~CodeAssemblerState() {}

100 101 102 103
int CodeAssemblerState::parameter_count() const {
  return static_cast<int>(raw_assembler_->call_descriptor()->ParameterCount());
}

104 105
CodeAssembler::~CodeAssembler() {}

106 107 108 109
#if DEBUG
void CodeAssemblerState::PrintCurrentBlock(std::ostream& os) {
  raw_assembler_->PrintCurrentBlock(os);
}
110 111

bool CodeAssemblerState::InsideBlock() { return raw_assembler_->InsideBlock(); }
112 113 114 115 116 117 118 119 120 121 122
#endif

void CodeAssemblerState::SetInitialDebugInformation(const char* msg,
                                                    const char* file,
                                                    int line) {
#if DEBUG
  AssemblerDebugInfo debug_info = {msg, file, line};
  raw_assembler_->SetInitialDebugInformation(debug_info);
#endif  // DEBUG
}

123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144
class BreakOnNodeDecorator final : public GraphDecorator {
 public:
  explicit BreakOnNodeDecorator(NodeId node_id) : node_id_(node_id) {}

  void Decorate(Node* node) final {
    if (node->id() == node_id_) {
      base::OS::DebugBreak();
    }
  }

 private:
  NodeId node_id_;
};

void CodeAssembler::BreakOnNode(int node_id) {
  Graph* graph = raw_assembler()->graph();
  Zone* zone = graph->zone();
  GraphDecorator* decorator =
      new (zone) BreakOnNodeDecorator(static_cast<NodeId>(node_id));
  graph->AddDecorator(decorator);
}

145 146 147 148 149 150 151 152 153 154 155 156 157 158
void CodeAssembler::RegisterCallGenerationCallbacks(
    const CodeAssemblerCallback& call_prologue,
    const CodeAssemblerCallback& call_epilogue) {
  // The callback can be registered only once.
  DCHECK(!state_->call_prologue_);
  DCHECK(!state_->call_epilogue_);
  state_->call_prologue_ = call_prologue;
  state_->call_epilogue_ = call_epilogue;
}

void CodeAssembler::UnregisterCallGenerationCallbacks() {
  state_->call_prologue_ = nullptr;
  state_->call_epilogue_ = nullptr;
}
159

160 161 162 163 164 165 166 167 168 169 170
void CodeAssembler::CallPrologue() {
  if (state_->call_prologue_) {
    state_->call_prologue_();
  }
}

void CodeAssembler::CallEpilogue() {
  if (state_->call_epilogue_) {
    state_->call_epilogue_();
  }
}
171

172 173 174 175
bool CodeAssembler::Word32ShiftIsSafe() const {
  return raw_assembler()->machine()->Word32ShiftIsSafe();
}

176
// static
177
Handle<Code> CodeAssembler::GenerateCode(CodeAssemblerState* state) {
178
  DCHECK(!state->code_generated_);
179

180 181
  RawMachineAssembler* rasm = state->raw_assembler_.get();
  Schedule* schedule = rasm->Export();
182

183
  JumpOptimizationInfo jump_opt;
184 185
  bool should_optimize_jumps =
      rasm->isolate()->serializer_enabled() && FLAG_turbo_rewrite_far_jumps;
186

187
  Handle<Code> code = Pipeline::GenerateCodeForCodeStub(
188
      rasm->isolate(), rasm->call_descriptor(), rasm->graph(), schedule,
189
      state->kind_, state->name_, state->stub_key_, state->builtin_index_,
190
      should_optimize_jumps ? &jump_opt : nullptr);
191 192 193 194 195 196 197

  if (jump_opt.is_optimizable()) {
    jump_opt.set_optimizing();

    // Regenerate machine code
    code = Pipeline::GenerateCodeForCodeStub(
        rasm->isolate(), rasm->call_descriptor(), rasm->graph(), schedule,
198 199
        state->kind_, state->name_, state->stub_key_, state->builtin_index_,
        &jump_opt);
200
  }
201

202
  state->code_generated_ = true;
203 204 205
  return code;
}

206
bool CodeAssembler::Is64() const { return raw_assembler()->machine()->Is64(); }
207 208

bool CodeAssembler::IsFloat64RoundUpSupported() const {
209
  return raw_assembler()->machine()->Float64RoundUp().IsSupported();
210 211 212
}

bool CodeAssembler::IsFloat64RoundDownSupported() const {
213
  return raw_assembler()->machine()->Float64RoundDown().IsSupported();
214 215
}

216 217 218 219
bool CodeAssembler::IsFloat64RoundTiesEvenSupported() const {
  return raw_assembler()->machine()->Float64RoundTiesEven().IsSupported();
}

220
bool CodeAssembler::IsFloat64RoundTruncateSupported() const {
221
  return raw_assembler()->machine()->Float64RoundTruncate().IsSupported();
222 223
}

224 225 226 227 228 229 230 231 232 233 234 235 236
bool CodeAssembler::IsInt32AbsWithOverflowSupported() const {
  return raw_assembler()->machine()->Int32AbsWithOverflow().IsSupported();
}

bool CodeAssembler::IsInt64AbsWithOverflowSupported() const {
  return raw_assembler()->machine()->Int64AbsWithOverflow().IsSupported();
}

bool CodeAssembler::IsIntPtrAbsWithOverflowSupported() const {
  return Is64() ? IsInt64AbsWithOverflowSupported()
                : IsInt32AbsWithOverflowSupported();
}

237 238
TNode<Int32T> CodeAssembler::Int32Constant(int32_t value) {
  return UncheckedCast<Int32T>(raw_assembler()->Int32Constant(value));
239 240
}

241 242
TNode<Int64T> CodeAssembler::Int64Constant(int64_t value) {
  return UncheckedCast<Int64T>(raw_assembler()->Int64Constant(value));
243 244
}

245 246
TNode<IntPtrT> CodeAssembler::IntPtrConstant(intptr_t value) {
  return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrConstant(value));
247 248
}

249
TNode<Number> CodeAssembler::NumberConstant(double value) {
250 251 252 253 254 255
  int smi_value;
  if (DoubleToSmiInteger(value, &smi_value)) {
    return UncheckedCast<Number>(SmiConstant(smi_value));
  } else {
    return UncheckedCast<Number>(raw_assembler()->NumberConstant(value));
  }
256 257
}

258 259 260
TNode<Smi> CodeAssembler::SmiConstant(Smi* value) {
  return UncheckedCast<Smi>(
      BitcastWordToTaggedSigned(IntPtrConstant(bit_cast<intptr_t>(value))));
261 262
}

263
TNode<Smi> CodeAssembler::SmiConstant(int value) {
264 265 266
  return SmiConstant(Smi::FromInt(value));
}

267 268 269
TNode<HeapObject> CodeAssembler::UntypedHeapConstant(
    Handle<HeapObject> object) {
  return UncheckedCast<HeapObject>(raw_assembler()->HeapConstant(object));
270 271
}

272 273 274
TNode<String> CodeAssembler::StringConstant(const char* str) {
  return UncheckedCast<String>(
      HeapConstant(factory()->NewStringFromAsciiChecked(str, TENURED)));
275 276
}

277 278
TNode<Oddball> CodeAssembler::BooleanConstant(bool value) {
  return UncheckedCast<Oddball>(raw_assembler()->BooleanConstant(value));
279 280
}

281 282 283 284
TNode<ExternalReference> CodeAssembler::ExternalConstant(
    ExternalReference address) {
  return UncheckedCast<ExternalReference>(
      raw_assembler()->ExternalConstant(address));
285 286
}

287 288
TNode<Float64T> CodeAssembler::Float64Constant(double value) {
  return UncheckedCast<Float64T>(raw_assembler()->Float64Constant(value));
289 290
}

291 292
TNode<HeapNumber> CodeAssembler::NaNConstant() {
  return UncheckedCast<HeapNumber>(LoadRoot(Heap::kNanValueRootIndex));
293 294
}

295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312
bool CodeAssembler::ToInt32Constant(Node* node, int32_t& out_value) {
  Int64Matcher m(node);
  if (m.HasValue() &&
      m.IsInRange(std::numeric_limits<int32_t>::min(),
                  std::numeric_limits<int32_t>::max())) {
    out_value = static_cast<int32_t>(m.Value());
    return true;
  }

  return false;
}

bool CodeAssembler::ToInt64Constant(Node* node, int64_t& out_value) {
  Int64Matcher m(node);
  if (m.HasValue()) out_value = m.Value();
  return m.HasValue();
}

313 314 315 316 317 318
bool CodeAssembler::ToSmiConstant(Node* node, Smi*& out_value) {
  if (node->opcode() == IrOpcode::kBitcastWordToTaggedSigned) {
    node = node->InputAt(0);
  }
  IntPtrMatcher m(node);
  if (m.HasValue()) {
319 320 321 322
    intptr_t value = m.Value();
    // Make sure that the value is actually a smi
    CHECK_EQ(0, value & ((static_cast<intptr_t>(1) << kSmiShiftSize) - 1));
    out_value = Smi::cast(bit_cast<Object*>(value));
323 324 325 326 327
    return true;
  }
  return false;
}

328
bool CodeAssembler::ToIntPtrConstant(Node* node, intptr_t& out_value) {
329 330 331 332
  if (node->opcode() == IrOpcode::kBitcastWordToTaggedSigned ||
      node->opcode() == IrOpcode::kBitcastWordToTagged) {
    node = node->InputAt(0);
  }
333 334 335 336 337
  IntPtrMatcher m(node);
  if (m.HasValue()) out_value = m.Value();
  return m.HasValue();
}

338
Node* CodeAssembler::Parameter(int value) {
339
  return raw_assembler()->Parameter(value);
340 341
}

342
TNode<Context> CodeAssembler::GetJSContextParameter() {
343 344
  auto call_descriptor = raw_assembler()->call_descriptor();
  DCHECK(call_descriptor->IsJSFunctionCall());
345
  return CAST(Parameter(Linkage::GetJSCallContextParamIndex(
346
      static_cast<int>(call_descriptor->JSParameterCount()))));
347 348
}

349
void CodeAssembler::Return(SloppyTNode<Object> value) {
350
  return raw_assembler()->Return(value);
351 352
}

353 354
void CodeAssembler::Return(SloppyTNode<Object> value1,
                           SloppyTNode<Object> value2) {
355 356 357
  return raw_assembler()->Return(value1, value2);
}

358 359 360
void CodeAssembler::Return(SloppyTNode<Object> value1,
                           SloppyTNode<Object> value2,
                           SloppyTNode<Object> value3) {
361 362 363
  return raw_assembler()->Return(value1, value2, value3);
}

364
void CodeAssembler::PopAndReturn(Node* pop, Node* value) {
365
  return raw_assembler()->PopAndReturn(pop, value);
366 367
}

368 369 370 371 372 373 374 375
void CodeAssembler::ReturnIf(Node* condition, Node* value) {
  Label if_return(this), if_continue(this);
  Branch(condition, &if_return, &if_continue);
  Bind(&if_return);
  Return(value);
  Bind(&if_continue);
}

376 377 378 379
void CodeAssembler::DebugAbort(Node* message) {
  raw_assembler()->DebugAbort(message);
}

380
void CodeAssembler::DebugBreak() { raw_assembler()->DebugBreak(); }
381

382 383 384 385 386
void CodeAssembler::Unreachable() {
  DebugBreak();
  raw_assembler()->Unreachable();
}

387 388 389 390 391 392 393 394 395 396 397
void CodeAssembler::Comment(const char* format, ...) {
  if (!FLAG_code_comments) return;
  char buffer[4 * KB];
  StringBuilder builder(buffer, arraysize(buffer));
  va_list arguments;
  va_start(arguments, format);
  builder.AddFormattedList(format, arguments);
  va_end(arguments);

  // Copy the string before recording it in the assembler to avoid
  // issues when the stack allocated buffer goes out of scope.
398 399 400 401
  const int prefix_len = 2;
  int length = builder.position() + 1;
  char* copy = reinterpret_cast<char*>(malloc(length + prefix_len));
  MemCopy(copy + prefix_len, builder.Finalize(), length);
402 403
  copy[0] = ';';
  copy[1] = ' ';
404
  raw_assembler()->Comment(copy);
405 406
}

407
void CodeAssembler::Bind(Label* label) { return label->Bind(); }
408

409 410 411 412 413 414
#if DEBUG
void CodeAssembler::Bind(Label* label, AssemblerDebugInfo debug_info) {
  return label->Bind(debug_info);
}
#endif  // DEBUG

415
Node* CodeAssembler::LoadFramePointer() {
416
  return raw_assembler()->LoadFramePointer();
417 418 419
}

Node* CodeAssembler::LoadParentFramePointer() {
420
  return raw_assembler()->LoadParentFramePointer();
421 422 423
}

Node* CodeAssembler::LoadStackPointer() {
424
  return raw_assembler()->LoadStackPointer();
425 426
}

427 428 429 430
Node* CodeAssembler::SpeculationPoison() {
  return raw_assembler()->SpeculationPoison();
}

431 432 433 434
#define DEFINE_CODE_ASSEMBLER_BINARY_OP(name, ResType, Arg1Type, Arg2Type) \
  TNode<ResType> CodeAssembler::name(SloppyTNode<Arg1Type> a,              \
                                     SloppyTNode<Arg2Type> b) {            \
    return UncheckedCast<ResType>(raw_assembler()->name(a, b));            \
435 436 437 438
  }
CODE_ASSEMBLER_BINARY_OP_LIST(DEFINE_CODE_ASSEMBLER_BINARY_OP)
#undef DEFINE_CODE_ASSEMBLER_BINARY_OP

439 440
TNode<WordT> CodeAssembler::IntPtrAdd(SloppyTNode<WordT> left,
                                      SloppyTNode<WordT> right) {
441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(left_constant + right_constant);
    }
    if (left_constant == 0) {
      return right;
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
457
  return UncheckedCast<WordT>(raw_assembler()->IntPtrAdd(left, right));
458 459
}

460 461
TNode<WordT> CodeAssembler::IntPtrSub(SloppyTNode<WordT> left,
                                      SloppyTNode<WordT> right) {
462 463 464 465 466 467 468 469 470 471 472 473 474
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(left_constant - right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
475
  return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrSub(left, right));
476 477
}

478 479 480 481 482 483 484 485 486 487
TNode<WordT> CodeAssembler::IntPtrMul(SloppyTNode<WordT> left,
                                      SloppyTNode<WordT> right) {
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(left_constant * right_constant);
    }
488 489
    if (base::bits::IsPowerOfTwo(left_constant)) {
      return WordShl(right, WhichPowerOf2(left_constant));
490 491
    }
  } else if (is_right_constant) {
492 493
    if (base::bits::IsPowerOfTwo(right_constant)) {
      return WordShl(left, WhichPowerOf2(right_constant));
494 495 496 497 498
    }
  }
  return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrMul(left, right));
}

499 500
TNode<WordT> CodeAssembler::WordShl(SloppyTNode<WordT> value, int shift) {
  return (shift != 0) ? WordShl(value, IntPtrConstant(shift)) : value;
501 502
}

503 504
TNode<WordT> CodeAssembler::WordShr(SloppyTNode<WordT> value, int shift) {
  return (shift != 0) ? WordShr(value, IntPtrConstant(shift)) : value;
oth's avatar
oth committed
505 506
}

507 508
TNode<Word32T> CodeAssembler::Word32Shr(SloppyTNode<Word32T> value, int shift) {
  return (shift != 0) ? Word32Shr(value, Int32Constant(shift)) : value;
509 510
}

511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822
TNode<WordT> CodeAssembler::WordOr(SloppyTNode<WordT> left,
                                   SloppyTNode<WordT> right) {
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(left_constant | right_constant);
    }
    if (left_constant == 0) {
      return right;
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<WordT>(raw_assembler()->WordOr(left, right));
}

TNode<WordT> CodeAssembler::WordAnd(SloppyTNode<WordT> left,
                                    SloppyTNode<WordT> right) {
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(left_constant & right_constant);
    }
  }
  return UncheckedCast<WordT>(raw_assembler()->WordAnd(left, right));
}

TNode<WordT> CodeAssembler::WordXor(SloppyTNode<WordT> left,
                                    SloppyTNode<WordT> right) {
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(left_constant ^ right_constant);
    }
  }
  return UncheckedCast<WordT>(raw_assembler()->WordXor(left, right));
}

TNode<WordT> CodeAssembler::WordShl(SloppyTNode<WordT> left,
                                    SloppyTNode<IntegralT> right) {
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(left_constant << right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<WordT>(raw_assembler()->WordShl(left, right));
}

TNode<WordT> CodeAssembler::WordShr(SloppyTNode<WordT> left,
                                    SloppyTNode<IntegralT> right) {
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(static_cast<uintptr_t>(left_constant) >>
                            right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<WordT>(raw_assembler()->WordShr(left, right));
}

TNode<WordT> CodeAssembler::WordSar(SloppyTNode<WordT> left,
                                    SloppyTNode<IntegralT> right) {
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(left_constant >> right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<WordT>(raw_assembler()->WordSar(left, right));
}

TNode<Word32T> CodeAssembler::Word32Or(SloppyTNode<Word32T> left,
                                       SloppyTNode<Word32T> right) {
  int32_t left_constant;
  bool is_left_constant = ToInt32Constant(left, left_constant);
  int32_t right_constant;
  bool is_right_constant = ToInt32Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int32Constant(left_constant | right_constant);
    }
    if (left_constant == 0) {
      return right;
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<Word32T>(raw_assembler()->Word32Or(left, right));
}

TNode<Word32T> CodeAssembler::Word32And(SloppyTNode<Word32T> left,
                                        SloppyTNode<Word32T> right) {
  int32_t left_constant;
  bool is_left_constant = ToInt32Constant(left, left_constant);
  int32_t right_constant;
  bool is_right_constant = ToInt32Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int32Constant(left_constant & right_constant);
    }
  }
  return UncheckedCast<Word32T>(raw_assembler()->Word32And(left, right));
}

TNode<Word32T> CodeAssembler::Word32Xor(SloppyTNode<Word32T> left,
                                        SloppyTNode<Word32T> right) {
  int32_t left_constant;
  bool is_left_constant = ToInt32Constant(left, left_constant);
  int32_t right_constant;
  bool is_right_constant = ToInt32Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int32Constant(left_constant ^ right_constant);
    }
  }
  return UncheckedCast<Word32T>(raw_assembler()->Word32Xor(left, right));
}

TNode<Word32T> CodeAssembler::Word32Shl(SloppyTNode<Word32T> left,
                                        SloppyTNode<Word32T> right) {
  int32_t left_constant;
  bool is_left_constant = ToInt32Constant(left, left_constant);
  int32_t right_constant;
  bool is_right_constant = ToInt32Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int32Constant(left_constant << right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<Word32T>(raw_assembler()->Word32Shl(left, right));
}

TNode<Word32T> CodeAssembler::Word32Shr(SloppyTNode<Word32T> left,
                                        SloppyTNode<Word32T> right) {
  int32_t left_constant;
  bool is_left_constant = ToInt32Constant(left, left_constant);
  int32_t right_constant;
  bool is_right_constant = ToInt32Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int32Constant(static_cast<uint32_t>(left_constant) >>
                           right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<Word32T>(raw_assembler()->Word32Shr(left, right));
}

TNode<Word32T> CodeAssembler::Word32Sar(SloppyTNode<Word32T> left,
                                        SloppyTNode<Word32T> right) {
  int32_t left_constant;
  bool is_left_constant = ToInt32Constant(left, left_constant);
  int32_t right_constant;
  bool is_right_constant = ToInt32Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int32Constant(left_constant >> right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<Word32T>(raw_assembler()->Word32Sar(left, right));
}

TNode<Word64T> CodeAssembler::Word64Or(SloppyTNode<Word64T> left,
                                       SloppyTNode<Word64T> right) {
  int64_t left_constant;
  bool is_left_constant = ToInt64Constant(left, left_constant);
  int64_t right_constant;
  bool is_right_constant = ToInt64Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int64Constant(left_constant | right_constant);
    }
    if (left_constant == 0) {
      return right;
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<Word64T>(raw_assembler()->Word64Or(left, right));
}

TNode<Word64T> CodeAssembler::Word64And(SloppyTNode<Word64T> left,
                                        SloppyTNode<Word64T> right) {
  int64_t left_constant;
  bool is_left_constant = ToInt64Constant(left, left_constant);
  int64_t right_constant;
  bool is_right_constant = ToInt64Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int64Constant(left_constant & right_constant);
    }
  }
  return UncheckedCast<Word64T>(raw_assembler()->Word64And(left, right));
}

TNode<Word64T> CodeAssembler::Word64Xor(SloppyTNode<Word64T> left,
                                        SloppyTNode<Word64T> right) {
  int64_t left_constant;
  bool is_left_constant = ToInt64Constant(left, left_constant);
  int64_t right_constant;
  bool is_right_constant = ToInt64Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int64Constant(left_constant ^ right_constant);
    }
  }
  return UncheckedCast<Word64T>(raw_assembler()->Word64Xor(left, right));
}

TNode<Word64T> CodeAssembler::Word64Shl(SloppyTNode<Word64T> left,
                                        SloppyTNode<Word64T> right) {
  int64_t left_constant;
  bool is_left_constant = ToInt64Constant(left, left_constant);
  int64_t right_constant;
  bool is_right_constant = ToInt64Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int64Constant(left_constant << right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<Word64T>(raw_assembler()->Word64Shl(left, right));
}

TNode<Word64T> CodeAssembler::Word64Shr(SloppyTNode<Word64T> left,
                                        SloppyTNode<Word64T> right) {
  int64_t left_constant;
  bool is_left_constant = ToInt64Constant(left, left_constant);
  int64_t right_constant;
  bool is_right_constant = ToInt64Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int64Constant(static_cast<uint64_t>(left_constant) >>
                           right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<Word64T>(raw_assembler()->Word64Shr(left, right));
}

TNode<Word64T> CodeAssembler::Word64Sar(SloppyTNode<Word64T> left,
                                        SloppyTNode<Word64T> right) {
  int64_t left_constant;
  bool is_left_constant = ToInt64Constant(left, left_constant);
  int64_t right_constant;
  bool is_right_constant = ToInt64Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int64Constant(left_constant >> right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<Word64T>(raw_assembler()->Word64Sar(left, right));
}

823
TNode<UintPtrT> CodeAssembler::ChangeUint32ToWord(SloppyTNode<Word32T> value) {
824
  if (raw_assembler()->machine()->Is64()) {
825 826
    return UncheckedCast<UintPtrT>(
        raw_assembler()->ChangeUint32ToUint64(value));
827
  }
828
  return ReinterpretCast<UintPtrT>(value);
829 830
}

831
TNode<IntPtrT> CodeAssembler::ChangeInt32ToIntPtr(SloppyTNode<Word32T> value) {
832
  if (raw_assembler()->machine()->Is64()) {
833
    return ReinterpretCast<IntPtrT>(raw_assembler()->ChangeInt32ToInt64(value));
834
  }
835
  return ReinterpretCast<IntPtrT>(value);
836 837
}

838 839
TNode<UintPtrT> CodeAssembler::ChangeFloat64ToUintPtr(
    SloppyTNode<Float64T> value) {
840
  if (raw_assembler()->machine()->Is64()) {
841 842
    return ReinterpretCast<UintPtrT>(
        raw_assembler()->ChangeFloat64ToUint64(value));
843
  }
844 845
  return ReinterpretCast<UintPtrT>(
      raw_assembler()->ChangeFloat64ToUint32(value));
846 847
}

848
Node* CodeAssembler::RoundIntPtrToFloat64(Node* value) {
849 850
  if (raw_assembler()->machine()->Is64()) {
    return raw_assembler()->RoundInt64ToFloat64(value);
851
  }
852
  return raw_assembler()->ChangeInt32ToFloat64(value);
853 854
}

855 856 857 858
#define DEFINE_CODE_ASSEMBLER_UNARY_OP(name, ResType, ArgType) \
  TNode<ResType> CodeAssembler::name(SloppyTNode<ArgType> a) { \
    return UncheckedCast<ResType>(raw_assembler()->name(a));   \
  }
859 860 861
CODE_ASSEMBLER_UNARY_OP_LIST(DEFINE_CODE_ASSEMBLER_UNARY_OP)
#undef DEFINE_CODE_ASSEMBLER_UNARY_OP

862
Node* CodeAssembler::Load(MachineType rep, Node* base) {
863
  return raw_assembler()->Load(rep, base);
864 865
}

866 867
Node* CodeAssembler::Load(MachineType rep, Node* base, Node* offset) {
  return raw_assembler()->Load(rep, base, offset);
868 869
}

870 871
Node* CodeAssembler::AtomicLoad(MachineType rep, Node* base, Node* offset) {
  return raw_assembler()->AtomicLoad(rep, base, offset);
872 873
}

874
TNode<Object> CodeAssembler::LoadRoot(Heap::RootListIndex root_index) {
875 876 877 878 879 880 881 882 883
  if (isolate()->heap()->RootCanBeTreatedAsConstant(root_index)) {
    Handle<Object> root = isolate()->heap()->root_handle(root_index);
    if (root->IsSmi()) {
      return SmiConstant(Smi::cast(*root));
    } else {
      return HeapConstant(Handle<HeapObject>::cast(root));
    }
  }

884
  Node* roots_array_start =
885
      ExternalConstant(ExternalReference::roots_array_start(isolate()));
886 887
  return UncheckedCast<Object>(Load(MachineType::AnyTagged(), roots_array_start,
                                    IntPtrConstant(root_index * kPointerSize)));
888 889
}

890 891 892
Node* CodeAssembler::Store(Node* base, Node* value) {
  return raw_assembler()->Store(MachineRepresentation::kTagged, base, value,
                                kFullWriteBarrier);
893 894
}

895 896 897
Node* CodeAssembler::Store(Node* base, Node* offset, Node* value) {
  return raw_assembler()->Store(MachineRepresentation::kTagged, base, offset,
                                value, kFullWriteBarrier);
898 899
}

900 901 902 903 904 905
Node* CodeAssembler::StoreWithMapWriteBarrier(Node* base, Node* offset,
                                              Node* value) {
  return raw_assembler()->Store(MachineRepresentation::kTagged, base, offset,
                                value, kMapWriteBarrier);
}

906 907
Node* CodeAssembler::StoreNoWriteBarrier(MachineRepresentation rep, Node* base,
                                         Node* value) {
908
  return raw_assembler()->Store(rep, base, value, kNoWriteBarrier);
909 910 911
}

Node* CodeAssembler::StoreNoWriteBarrier(MachineRepresentation rep, Node* base,
912 913
                                         Node* offset, Node* value) {
  return raw_assembler()->Store(rep, base, offset, value, kNoWriteBarrier);
914 915
}

916
Node* CodeAssembler::AtomicStore(MachineRepresentation rep, Node* base,
917 918
                                 Node* offset, Node* value) {
  return raw_assembler()->AtomicStore(rep, base, offset, value);
919 920
}

921 922 923 924 925 926 927 928 929 930 931 932
#define ATOMIC_FUNCTION(name)                                        \
  Node* CodeAssembler::Atomic##name(MachineType type, Node* base,    \
                                    Node* offset, Node* value) {     \
    return raw_assembler()->Atomic##name(type, base, offset, value); \
  }
ATOMIC_FUNCTION(Exchange);
ATOMIC_FUNCTION(Add);
ATOMIC_FUNCTION(Sub);
ATOMIC_FUNCTION(And);
ATOMIC_FUNCTION(Or);
ATOMIC_FUNCTION(Xor);
#undef ATOMIC_FUNCTION
933

934 935 936 937 938 939 940
Node* CodeAssembler::AtomicCompareExchange(MachineType type, Node* base,
                                           Node* offset, Node* old_value,
                                           Node* new_value) {
  return raw_assembler()->AtomicCompareExchange(type, base, offset, old_value,
                                                new_value);
}

941 942 943 944 945 946 947 948
Node* CodeAssembler::StoreRoot(Heap::RootListIndex root_index, Node* value) {
  DCHECK(Heap::RootCanBeWrittenAfterInitialization(root_index));
  Node* roots_array_start =
      ExternalConstant(ExternalReference::roots_array_start(isolate()));
  return StoreNoWriteBarrier(MachineRepresentation::kTagged, roots_array_start,
                             IntPtrConstant(root_index * kPointerSize), value);
}

949
Node* CodeAssembler::Retain(Node* value) {
950
  return raw_assembler()->Retain(value);
951 952
}

953
Node* CodeAssembler::Projection(int index, Node* value) {
954
  return raw_assembler()->Projection(index, value);
955 956
}

957 958
void CodeAssembler::GotoIfException(Node* node, Label* if_exception,
                                    Variable* exception_var) {
959 960 961 962 963 964 965
  DCHECK(!node->op()->HasProperty(Operator::kNoThrow));

  if (if_exception == nullptr) {
    // If no handler is supplied, don't add continuations
    return;
  }

966 967 968 969
  Label success(this), exception(this, Label::kDeferred);
  success.MergeVariables();
  exception.MergeVariables();

970
  raw_assembler()->Continuations(node, success.label_, exception.label_);
971 972

  Bind(&exception);
973 974
  const Operator* op = raw_assembler()->common()->IfException();
  Node* exception_value = raw_assembler()->AddNode(op, node, node);
975 976 977 978 979 980 981 982
  if (exception_var != nullptr) {
    exception_var->Bind(exception_value);
  }
  Goto(if_exception);

  Bind(&success);
}

983
template <class... TArgs>
984 985 986
TNode<Object> CodeAssembler::CallRuntimeImpl(Runtime::FunctionId function,
                                             SloppyTNode<Object> context,
                                             TArgs... args) {
987
  int argc = static_cast<int>(sizeof...(args));
988
  auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
989 990
      zone(), function, argc, Operator::kNoProperties,
      CallDescriptor::kNoFlags);
991
  int return_count = static_cast<int>(call_descriptor->ReturnCount());
992 993 994 995 996 997 998 999

  Node* centry =
      HeapConstant(CodeFactory::RuntimeCEntry(isolate(), return_count));
  Node* ref = ExternalConstant(ExternalReference(function, isolate()));
  Node* arity = Int32Constant(argc);

  Node* nodes[] = {centry, args..., ref, arity, context};

1000
  CallPrologue();
1001 1002
  Node* return_value =
      raw_assembler()->CallN(call_descriptor, arraysize(nodes), nodes);
1003
  CallEpilogue();
1004
  return UncheckedCast<Object>(return_value);
1005 1006
}

1007
// Instantiate CallRuntime() for argument counts used by CSA-generated code
1008 1009
#define INSTANTIATE(...)                                                   \
  template V8_EXPORT_PRIVATE TNode<Object> CodeAssembler::CallRuntimeImpl( \
1010
      Runtime::FunctionId, __VA_ARGS__);
1011
REPEAT_1_TO_7(INSTANTIATE, SloppyTNode<Object>)
1012
#undef INSTANTIATE
1013

1014
template <class... TArgs>
1015 1016 1017
TNode<Object> CodeAssembler::TailCallRuntimeImpl(Runtime::FunctionId function,
                                                 SloppyTNode<Object> context,
                                                 TArgs... args) {
1018
  int argc = static_cast<int>(sizeof...(args));
1019
  auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
1020
      zone(), function, argc, Operator::kNoProperties,
1021
      CallDescriptor::kNoFlags);
1022
  int return_count = static_cast<int>(call_descriptor->ReturnCount());
1023

1024 1025 1026 1027
  Node* centry =
      HeapConstant(CodeFactory::RuntimeCEntry(isolate(), return_count));
  Node* ref = ExternalConstant(ExternalReference(function, isolate()));
  Node* arity = Int32Constant(argc);
1028

1029
  Node* nodes[] = {centry, args..., ref, arity, context};
1030

1031
  return UncheckedCast<Object>(
1032
      raw_assembler()->TailCallN(call_descriptor, arraysize(nodes), nodes));
1033 1034
}

1035
// Instantiate TailCallRuntime() for argument counts used by CSA-generated code
1036 1037
#define INSTANTIATE(...)                                                       \
  template V8_EXPORT_PRIVATE TNode<Object> CodeAssembler::TailCallRuntimeImpl( \
1038
      Runtime::FunctionId, __VA_ARGS__);
1039
REPEAT_1_TO_7(INSTANTIATE, SloppyTNode<Object>)
1040
#undef INSTANTIATE
1041

1042 1043 1044 1045 1046
template <class... TArgs>
Node* CodeAssembler::CallStubR(const CallInterfaceDescriptor& descriptor,
                               size_t result_size, Node* target, Node* context,
                               TArgs... args) {
  Node* nodes[] = {target, args..., context};
1047 1048 1049 1050
  int input_count = arraysize(nodes);
  if (context == nullptr) --input_count;
  return CallStubN(descriptor, result_size, input_count, nodes,
                   context != nullptr);
1051 1052
}

1053
// Instantiate CallStubR() for argument counts used by CSA-generated code.
1054 1055 1056
#define INSTANTIATE(...)                                     \
  template V8_EXPORT_PRIVATE Node* CodeAssembler::CallStubR( \
      const CallInterfaceDescriptor& descriptor, size_t, Node*, __VA_ARGS__);
1057
REPEAT_1_TO_11(INSTANTIATE, Node*)
1058
#undef INSTANTIATE
1059

1060 1061
Node* CodeAssembler::CallStubN(const CallInterfaceDescriptor& descriptor,
                               size_t result_size, int input_count,
1062 1063 1064 1065 1066
                               Node* const* inputs, bool pass_context) {
  // implicit nodes are target and optionally context.
  int implicit_nodes = pass_context ? 2 : 1;
  DCHECK_LE(implicit_nodes, input_count);
  int argc = input_count - implicit_nodes;
1067 1068 1069 1070
  DCHECK_LE(descriptor.GetParameterCount(), argc);
  // Extra arguments not mentioned in the descriptor are passed on the stack.
  int stack_parameter_count = argc - descriptor.GetRegisterParameterCount();
  DCHECK_LE(descriptor.GetStackParameterCount(), stack_parameter_count);
1071
  auto call_descriptor = Linkage::GetStubCallDescriptor(
1072
      isolate(), zone(), descriptor, stack_parameter_count,
1073
      CallDescriptor::kNoFlags, Operator::kNoProperties,
1074 1075
      MachineType::AnyTagged(), result_size,
      pass_context ? Linkage::kPassContext : Linkage::kNoContext);
1076

1077
  CallPrologue();
1078 1079
  Node* return_value =
      raw_assembler()->CallN(call_descriptor, input_count, inputs);
1080 1081
  CallEpilogue();
  return return_value;
1082 1083
}

1084
template <class... TArgs>
1085 1086 1087
Node* CodeAssembler::TailCallStubImpl(const CallInterfaceDescriptor& descriptor,
                                      Node* target, Node* context,
                                      TArgs... args) {
1088 1089
  DCHECK_EQ(descriptor.GetParameterCount(), sizeof...(args));
  size_t result_size = 1;
1090
  auto call_descriptor = Linkage::GetStubCallDescriptor(
1091
      isolate(), zone(), descriptor, descriptor.GetStackParameterCount(),
1092
      CallDescriptor::kNoFlags, Operator::kNoProperties,
1093 1094
      MachineType::AnyTagged(), result_size);

1095
  Node* nodes[] = {target, args..., context};
1096
  CHECK_EQ(descriptor.GetParameterCount() + 2, arraysize(nodes));
1097
  return raw_assembler()->TailCallN(call_descriptor, arraysize(nodes), nodes);
1098 1099
}

1100
// Instantiate TailCallStub() for argument counts used by CSA-generated code
1101 1102
#define INSTANTIATE(...)                                            \
  template V8_EXPORT_PRIVATE Node* CodeAssembler::TailCallStubImpl( \
1103
      const CallInterfaceDescriptor& descriptor, Node*, __VA_ARGS__);
1104
REPEAT_1_TO_12(INSTANTIATE, Node*)
1105
#undef INSTANTIATE
1106

1107 1108 1109 1110 1111 1112 1113 1114 1115
template <class... TArgs>
Node* CodeAssembler::TailCallStubThenBytecodeDispatch(
    const CallInterfaceDescriptor& descriptor, Node* target, Node* context,
    TArgs... args) {
  DCHECK_LE(descriptor.GetParameterCount(), sizeof...(args));
  // Extra arguments not mentioned in the descriptor are passed on the stack.
  int stack_parameter_count =
      sizeof...(args) - descriptor.GetRegisterParameterCount();
  DCHECK_LE(descriptor.GetStackParameterCount(), stack_parameter_count);
1116
  auto call_descriptor = Linkage::GetStubCallDescriptor(
1117
      isolate(), zone(), descriptor, stack_parameter_count,
1118
      CallDescriptor::kNoFlags, Operator::kNoProperties,
1119 1120 1121
      MachineType::AnyTagged(), 0);

  Node* nodes[] = {target, args..., context};
1122
  return raw_assembler()->TailCallN(call_descriptor, arraysize(nodes), nodes);
1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133
}

// Instantiate TailCallJSAndBytecodeDispatch() for argument counts used by
// CSA-generated code
#define INSTANTIATE(...)                           \
  template V8_EXPORT_PRIVATE Node*                 \
  CodeAssembler::TailCallStubThenBytecodeDispatch( \
      const CallInterfaceDescriptor&, Node*, Node*, Node*, __VA_ARGS__);
REPEAT_1_TO_7(INSTANTIATE, Node*)
#undef INSTANTIATE

1134
template <class... TArgs>
1135
Node* CodeAssembler::TailCallBytecodeDispatch(
1136 1137
    const CallInterfaceDescriptor& descriptor, Node* target, TArgs... args) {
  DCHECK_EQ(descriptor.GetParameterCount(), sizeof...(args));
1138
  auto call_descriptor = Linkage::GetBytecodeDispatchCallDescriptor(
1139 1140 1141
      isolate(), zone(), descriptor, descriptor.GetStackParameterCount());

  Node* nodes[] = {target, args...};
1142
  CHECK_EQ(descriptor.GetParameterCount() + 1, arraysize(nodes));
1143
  return raw_assembler()->TailCallN(call_descriptor, arraysize(nodes), nodes);
1144 1145
}

1146 1147
// Instantiate TailCallBytecodeDispatch() for argument counts used by
// CSA-generated code
1148 1149
template V8_EXPORT_PRIVATE Node* CodeAssembler::TailCallBytecodeDispatch(
    const CallInterfaceDescriptor& descriptor, Node* target, Node*, Node*,
1150
    Node*, Node*);
1151

1152 1153
Node* CodeAssembler::CallCFunctionN(Signature<MachineType>* signature,
                                    int input_count, Node* const* inputs) {
1154 1155
  auto call_descriptor = Linkage::GetSimplifiedCDescriptor(zone(), signature);
  return raw_assembler()->CallN(call_descriptor, input_count, inputs);
1156 1157
}

1158 1159 1160 1161 1162 1163 1164
Node* CodeAssembler::CallCFunction1(MachineType return_type,
                                    MachineType arg0_type, Node* function,
                                    Node* arg0) {
  return raw_assembler()->CallCFunction1(return_type, arg0_type, function,
                                         arg0);
}

1165
Node* CodeAssembler::CallCFunction1WithCallerSavedRegisters(
1166 1167
    MachineType return_type, MachineType arg0_type, Node* function, Node* arg0,
    SaveFPRegsMode mode) {
1168 1169
  DCHECK(return_type.LessThanOrEqualPointerSize());
  return raw_assembler()->CallCFunction1WithCallerSavedRegisters(
1170
      return_type, arg0_type, function, arg0, mode);
1171 1172
}

1173 1174 1175 1176
Node* CodeAssembler::CallCFunction2(MachineType return_type,
                                    MachineType arg0_type,
                                    MachineType arg1_type, Node* function,
                                    Node* arg0, Node* arg1) {
1177 1178
  return raw_assembler()->CallCFunction2(return_type, arg0_type, arg1_type,
                                         function, arg0, arg1);
1179 1180
}

1181 1182 1183 1184 1185 1186 1187 1188 1189
Node* CodeAssembler::CallCFunction3(MachineType return_type,
                                    MachineType arg0_type,
                                    MachineType arg1_type,
                                    MachineType arg2_type, Node* function,
                                    Node* arg0, Node* arg1, Node* arg2) {
  return raw_assembler()->CallCFunction3(return_type, arg0_type, arg1_type,
                                         arg2_type, function, arg0, arg1, arg2);
}

1190 1191
Node* CodeAssembler::CallCFunction3WithCallerSavedRegisters(
    MachineType return_type, MachineType arg0_type, MachineType arg1_type,
1192 1193
    MachineType arg2_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
    SaveFPRegsMode mode) {
1194 1195
  DCHECK(return_type.LessThanOrEqualPointerSize());
  return raw_assembler()->CallCFunction3WithCallerSavedRegisters(
1196 1197
      return_type, arg0_type, arg1_type, arg2_type, function, arg0, arg1, arg2,
      mode);
1198 1199
}

1200 1201 1202 1203 1204 1205 1206 1207 1208
Node* CodeAssembler::CallCFunction4(
    MachineType return_type, MachineType arg0_type, MachineType arg1_type,
    MachineType arg2_type, MachineType arg3_type, Node* function, Node* arg0,
    Node* arg1, Node* arg2, Node* arg3) {
  return raw_assembler()->CallCFunction4(return_type, arg0_type, arg1_type,
                                         arg2_type, arg3_type, function, arg0,
                                         arg1, arg2, arg3);
}

1209 1210 1211 1212 1213 1214 1215 1216 1217 1218
Node* CodeAssembler::CallCFunction5(
    MachineType return_type, MachineType arg0_type, MachineType arg1_type,
    MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
    Node* function, Node* arg0, Node* arg1, Node* arg2, Node* arg3,
    Node* arg4) {
  return raw_assembler()->CallCFunction5(
      return_type, arg0_type, arg1_type, arg2_type, arg3_type, arg4_type,
      function, arg0, arg1, arg2, arg3, arg4);
}

1219 1220 1221 1222 1223 1224 1225 1226 1227 1228
Node* CodeAssembler::CallCFunction6(
    MachineType return_type, MachineType arg0_type, MachineType arg1_type,
    MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
    MachineType arg5_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
    Node* arg3, Node* arg4, Node* arg5) {
  return raw_assembler()->CallCFunction6(
      return_type, arg0_type, arg1_type, arg2_type, arg3_type, arg4_type,
      arg5_type, function, arg0, arg1, arg2, arg3, arg4, arg5);
}

1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240
Node* CodeAssembler::CallCFunction9(
    MachineType return_type, MachineType arg0_type, MachineType arg1_type,
    MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
    MachineType arg5_type, MachineType arg6_type, MachineType arg7_type,
    MachineType arg8_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
    Node* arg3, Node* arg4, Node* arg5, Node* arg6, Node* arg7, Node* arg8) {
  return raw_assembler()->CallCFunction9(
      return_type, arg0_type, arg1_type, arg2_type, arg3_type, arg4_type,
      arg5_type, arg6_type, arg7_type, arg8_type, function, arg0, arg1, arg2,
      arg3, arg4, arg5, arg6, arg7, arg8);
}

1241
void CodeAssembler::Goto(Label* label) {
1242
  label->MergeVariables();
1243
  raw_assembler()->Goto(label->label_);
1244 1245
}

1246 1247
void CodeAssembler::GotoIf(SloppyTNode<IntegralT> condition,
                           Label* true_label) {
1248 1249 1250 1251 1252
  Label false_label(this);
  Branch(condition, true_label, &false_label);
  Bind(&false_label);
}

1253 1254
void CodeAssembler::GotoIfNot(SloppyTNode<IntegralT> condition,
                              Label* false_label) {
1255 1256 1257 1258 1259
  Label true_label(this);
  Branch(condition, &true_label, false_label);
  Bind(&true_label);
}

1260
void CodeAssembler::Branch(SloppyTNode<IntegralT> condition, Label* true_label,
1261
                           Label* false_label) {
1262 1263
  true_label->MergeVariables();
  false_label->MergeVariables();
1264 1265
  return raw_assembler()->Branch(condition, true_label->label_,
                                 false_label->label_);
1266 1267 1268
}

void CodeAssembler::Switch(Node* index, Label* default_label,
1269
                           const int32_t* case_values, Label** case_labels,
1270 1271 1272 1273 1274 1275 1276 1277
                           size_t case_count) {
  RawMachineLabel** labels =
      new (zone()->New(sizeof(RawMachineLabel*) * case_count))
          RawMachineLabel*[case_count];
  for (size_t i = 0; i < case_count; ++i) {
    labels[i] = case_labels[i]->label_;
    case_labels[i]->MergeVariables();
  }
1278
  default_label->MergeVariables();
1279 1280
  return raw_assembler()->Switch(index, default_label->label_, case_values,
                                 labels, case_count);
1281 1282
}

1283 1284
bool CodeAssembler::UnalignedLoadSupported(MachineRepresentation rep) const {
  return raw_assembler()->machine()->UnalignedLoadSupported(rep);
1285
}
1286 1287
bool CodeAssembler::UnalignedStoreSupported(MachineRepresentation rep) const {
  return raw_assembler()->machine()->UnalignedStoreSupported(rep);
1288 1289
}

1290
// RawMachineAssembler delegate helpers:
1291
Isolate* CodeAssembler::isolate() const { return raw_assembler()->isolate(); }
1292 1293 1294

Factory* CodeAssembler::factory() const { return isolate()->factory(); }

1295 1296 1297 1298 1299
Zone* CodeAssembler::zone() const { return raw_assembler()->zone(); }

RawMachineAssembler* CodeAssembler::raw_assembler() const {
  return state_->raw_assembler_.get();
}
1300 1301 1302 1303 1304

// The core implementation of Variable is stored through an indirection so
// that it can outlive the often block-scoped Variable declarations. This is
// needed to ensure that variable binding and merging through phis can
// properly be verified.
1305
class CodeAssemblerVariable::Impl : public ZoneObject {
1306
 public:
1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323
  explicit Impl(MachineRepresentation rep)
      :
#if DEBUG
        debug_info_(AssemblerDebugInfo(nullptr, nullptr, -1)),
#endif
        value_(nullptr),
        rep_(rep) {
  }

#if DEBUG
  AssemblerDebugInfo debug_info() const { return debug_info_; }
  void set_debug_info(AssemblerDebugInfo debug_info) {
    debug_info_ = debug_info;
  }

  AssemblerDebugInfo debug_info_;
#endif  // DEBUG
1324 1325 1326 1327
  Node* value_;
  MachineRepresentation rep_;
};

1328 1329 1330
CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
                                             MachineRepresentation rep)
    : impl_(new (assembler->zone()) Impl(rep)), state_(assembler->state()) {
1331
  state_->variables_.insert(impl_);
1332 1333
}

1334 1335 1336 1337 1338 1339 1340
CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
                                             MachineRepresentation rep,
                                             Node* initial_value)
    : CodeAssemblerVariable(assembler, rep) {
  Bind(initial_value);
}

1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359
#if DEBUG
CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
                                             AssemblerDebugInfo debug_info,
                                             MachineRepresentation rep)
    : impl_(new (assembler->zone()) Impl(rep)), state_(assembler->state()) {
  impl_->set_debug_info(debug_info);
  state_->variables_.insert(impl_);
}

CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
                                             AssemblerDebugInfo debug_info,
                                             MachineRepresentation rep,
                                             Node* initial_value)
    : CodeAssemblerVariable(assembler, debug_info, rep) {
  impl_->set_debug_info(debug_info);
  Bind(initial_value);
}
#endif  // DEBUG

1360 1361 1362
CodeAssemblerVariable::~CodeAssemblerVariable() {
  state_->variables_.erase(impl_);
}
1363

1364
void CodeAssemblerVariable::Bind(Node* value) { impl_->value_ = value; }
1365

1366
Node* CodeAssemblerVariable::value() const {
1367 1368 1369 1370
#if DEBUG
  if (!IsBound()) {
    std::stringstream str;
    str << "#Use of unbound variable:"
1371 1372
        << "#\n    Variable:      " << *this << "#\n    Current Block: ";
    state_->PrintCurrentBlock(str);
1373
    FATAL("%s", str.str().c_str());
1374 1375 1376 1377
  }
  if (!state_->InsideBlock()) {
    std::stringstream str;
    str << "#Accessing variable value outside a block:"
1378
        << "#\n    Variable:      " << *this;
1379
    FATAL("%s", str.str().c_str());
1380 1381
  }
#endif  // DEBUG
1382 1383 1384
  return impl_->value_;
}

1385
MachineRepresentation CodeAssemblerVariable::rep() const { return impl_->rep_; }
1386

1387
bool CodeAssemblerVariable::IsBound() const { return impl_->value_ != nullptr; }
1388

1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403
std::ostream& operator<<(std::ostream& os,
                         const CodeAssemblerVariable::Impl& impl) {
#if DEBUG
  AssemblerDebugInfo info = impl.debug_info();
  if (info.name) os << "V" << info;
#endif  // DEBUG
  return os;
}

std::ostream& operator<<(std::ostream& os,
                         const CodeAssemblerVariable& variable) {
  os << *variable.impl_;
  return os;
}

1404 1405
CodeAssemblerLabel::CodeAssemblerLabel(CodeAssembler* assembler,
                                       size_t vars_count,
1406
                                       CodeAssemblerVariable* const* vars,
1407
                                       CodeAssemblerLabel::Type type)
1408 1409
    : bound_(false),
      merge_count_(0),
1410
      state_(assembler->state()),
1411
      label_(nullptr) {
1412 1413 1414 1415
  void* buffer = assembler->zone()->New(sizeof(RawMachineLabel));
  label_ = new (buffer)
      RawMachineLabel(type == kDeferred ? RawMachineLabel::kDeferred
                                        : RawMachineLabel::kNonDeferred);
1416 1417
  for (size_t i = 0; i < vars_count; ++i) {
    variable_phis_[vars[i]->impl_] = nullptr;
1418 1419 1420
  }
}

1421 1422
CodeAssemblerLabel::~CodeAssemblerLabel() { label_->~RawMachineLabel(); }

1423
void CodeAssemblerLabel::MergeVariables() {
1424
  ++merge_count_;
1425
  for (CodeAssemblerVariable::Impl* var : state_->variables_) {
1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449
    size_t count = 0;
    Node* node = var->value_;
    if (node != nullptr) {
      auto i = variable_merges_.find(var);
      if (i != variable_merges_.end()) {
        i->second.push_back(node);
        count = i->second.size();
      } else {
        count = 1;
        variable_merges_[var] = std::vector<Node*>(1, node);
      }
    }
    // If the following asserts, then you've jumped to a label without a bound
    // variable along that path that expects to merge its value into a phi.
    DCHECK(variable_phis_.find(var) == variable_phis_.end() ||
           count == merge_count_);
    USE(count);

    // If the label is already bound, we already know the set of variables to
    // merge and phi nodes have already been created.
    if (bound_) {
      auto phi = variable_phis_.find(var);
      if (phi != variable_phis_.end()) {
        DCHECK_NOT_NULL(phi->second);
1450
        state_->raw_assembler_->AppendPhiInput(phi->second, node);
1451 1452 1453 1454 1455 1456 1457 1458 1459
      } else {
        auto i = variable_merges_.find(var);
        if (i != variable_merges_.end()) {
          // If the following assert fires, then you've declared a variable that
          // has the same bound value along all paths up until the point you
          // bound this label, but then later merged a path with a new value for
          // the variable after the label bind (it's not possible to add phis to
          // the bound label after the fact, just make sure to list the variable
          // in the label's constructor's list of merged variables).
1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471
#if DEBUG
          if (find_if(i->second.begin(), i->second.end(),
                      [node](Node* e) -> bool { return node != e; }) !=
              i->second.end()) {
            std::stringstream str;
            str << "Unmerged variable found when jumping to block. \n"
                << "#    Variable:      " << *var;
            if (bound_) {
              str << "\n#    Target block:  " << *label_->block();
            }
            str << "\n#    Current Block: ";
            state_->PrintCurrentBlock(str);
1472
            FATAL("%s", str.str().c_str());
1473 1474
          }
#endif  // DEBUG
1475 1476 1477 1478 1479 1480
        }
      }
    }
  }
}

1481 1482
#if DEBUG
void CodeAssemblerLabel::Bind(AssemblerDebugInfo debug_info) {
1483 1484 1485 1486 1487
  if (bound_) {
    std::stringstream str;
    str << "Cannot bind the same label twice:"
        << "\n#    current:  " << debug_info
        << "\n#    previous: " << *label_->block();
1488
    FATAL("%s", str.str().c_str());
1489
  }
1490 1491 1492 1493 1494
  state_->raw_assembler_->Bind(label_, debug_info);
  UpdateVariablesAfterBind();
}
#endif  // DEBUG

1495
void CodeAssemblerLabel::Bind() {
1496
  DCHECK(!bound_);
1497
  state_->raw_assembler_->Bind(label_);
1498 1499
  UpdateVariablesAfterBind();
}
1500

1501
void CodeAssemblerLabel::UpdateVariablesAfterBind() {
1502 1503
  // Make sure that all variables that have changed along any path up to this
  // point are marked as merge variables.
1504
  for (auto var : state_->variables_) {
1505 1506 1507 1508
    Node* shared_value = nullptr;
    auto i = variable_merges_.find(var);
    if (i != variable_merges_.end()) {
      for (auto value : i->second) {
1509
        DCHECK_NOT_NULL(value);
1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521
        if (value != shared_value) {
          if (shared_value == nullptr) {
            shared_value = value;
          } else {
            variable_phis_[var] = nullptr;
          }
        }
      }
    }
  }

  for (auto var : variable_phis_) {
1522
    CodeAssemblerVariable::Impl* var_impl = var.first;
1523
    auto i = variable_merges_.find(var_impl);
1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539
#if DEBUG
    bool not_found = i == variable_merges_.end();
    if (not_found || i->second.size() != merge_count_) {
      std::stringstream str;
      str << "A variable that has been marked as beeing merged at the label"
          << "\n# doesn't have a bound value along all of the paths that "
          << "\n# have been merged into the label up to this point."
          << "\n#"
          << "\n# This can happen in the following cases:"
          << "\n# - By explicitly marking it so in the label constructor"
          << "\n# - By having seen different bound values at branches"
          << "\n#"
          << "\n# Merge count:     expected=" << merge_count_
          << " vs. found=" << (not_found ? 0 : i->second.size())
          << "\n# Variable:      " << *var_impl
          << "\n# Current Block: " << *label_->block();
1540
      FATAL("%s", str.str().c_str());
1541 1542
    }
#endif  // DEBUG
1543
    Node* phi = state_->raw_assembler_->Phi(
1544 1545 1546 1547 1548 1549
        var.first->rep_, static_cast<int>(merge_count_), &(i->second[0]));
    variable_phis_[var_impl] = phi;
  }

  // Bind all variables to a merge phi, the common value along all paths or
  // null.
1550
  for (auto var : state_->variables_) {
1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567
    auto i = variable_phis_.find(var);
    if (i != variable_phis_.end()) {
      var->value_ = i->second;
    } else {
      auto j = variable_merges_.find(var);
      if (j != variable_merges_.end() && j->second.size() == merge_count_) {
        var->value_ = j->second.back();
      } else {
        var->value_ = nullptr;
      }
    }
  }

  bound_ = true;
}

}  // namespace compiler
1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601

Smi* CheckObjectType(Object* value, Smi* type, String* location) {
#ifdef DEBUG
  const char* expected;
  switch (static_cast<ObjectType>(type->value())) {
#define TYPE_CASE(Name)                            \
  case ObjectType::k##Name:                        \
    if (value->Is##Name()) return Smi::FromInt(0); \
    expected = #Name;                              \
    break;
#define TYPE_STRUCT_CASE(NAME, Name, name)         \
  case ObjectType::k##Name:                        \
    if (value->Is##Name()) return Smi::FromInt(0); \
    expected = #Name;                              \
    break;

    TYPE_CASE(Object)
    OBJECT_TYPE_LIST(TYPE_CASE)
    HEAP_OBJECT_TYPE_LIST(TYPE_CASE)
    STRUCT_LIST(TYPE_STRUCT_CASE)
#undef TYPE_CASE
#undef TYPE_STRUCT_CASE
  }
  std::stringstream value_description;
  value->Print(value_description);
  V8_Fatal(__FILE__, __LINE__,
           "Type cast failed in %s\n"
           "  Expected %s but found %s",
           location->ToAsciiArray(), expected, value_description.str().c_str());
#else
  UNREACHABLE();
#endif
}

1602 1603
}  // namespace internal
}  // namespace v8