code-assembler.cc 69 KB
Newer Older
1 2 3 4 5 6 7 8 9
// Copyright 2015 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/compiler/code-assembler.h"

#include <ostream>

#include "src/code-factory.h"
10
#include "src/compiler/backend/instruction-selector.h"
11 12
#include "src/compiler/graph.h"
#include "src/compiler/linkage.h"
13
#include "src/compiler/node-matchers.h"
14 15 16 17 18 19 20 21
#include "src/compiler/pipeline.h"
#include "src/compiler/raw-machine-assembler.h"
#include "src/compiler/schedule.h"
#include "src/frames.h"
#include "src/interface-descriptors.h"
#include "src/interpreter/bytecodes.h"
#include "src/machine-type.h"
#include "src/macro-assembler.h"
22
#include "src/memcopy.h"
23
#include "src/objects-inl.h"
24
#include "src/objects/smi.h"
25
#include "src/zone/zone.h"
26 27 28

namespace v8 {
namespace internal {
29 30 31 32

constexpr MachineType MachineTypeOf<Smi>::value;
constexpr MachineType MachineTypeOf<Object>::value;

33 34
namespace compiler {

35 36 37 38 39 40 41 42 43
static_assert(std::is_convertible<TNode<Number>, TNode<Object>>::value,
              "test subtyping");
static_assert(std::is_convertible<TNode<UnionT<Smi, HeapNumber>>,
                                  TNode<UnionT<Smi, HeapObject>>>::value,
              "test subtyping");
static_assert(
    !std::is_convertible<TNode<UnionT<Smi, HeapObject>>, TNode<Number>>::value,
    "test subtyping");

44 45
CodeAssemblerState::CodeAssemblerState(
    Isolate* isolate, Zone* zone, const CallInterfaceDescriptor& descriptor,
46
    Code::Kind kind, const char* name, PoisoningMitigationLevel poisoning_level,
47
    int32_t builtin_index)
48 49
    // TODO(rmcilroy): Should we use Linkage::GetBytecodeDispatchDescriptor for
    // bytecode handlers?
50
    : CodeAssemblerState(
51 52
          isolate, zone,
          Linkage::GetStubCallDescriptor(
53
              zone, descriptor, descriptor.GetStackParameterCount(),
54
              CallDescriptor::kNoFlags, Operator::kNoProperties),
55
          kind, name, poisoning_level, builtin_index) {}
56

57 58 59 60 61
CodeAssemblerState::CodeAssemblerState(Isolate* isolate, Zone* zone,
                                       int parameter_count, Code::Kind kind,
                                       const char* name,
                                       PoisoningMitigationLevel poisoning_level,
                                       int32_t builtin_index)
62 63
    : CodeAssemblerState(
          isolate, zone,
64 65 66 67 68
          Linkage::GetJSCallDescriptor(
              zone, false, parameter_count,
              (kind == Code::BUILTIN ? CallDescriptor::kPushArgumentCount
                                     : CallDescriptor::kNoFlags) |
                  CallDescriptor::kCanUseRoots),
69
          kind, name, poisoning_level, builtin_index) {}
70

71 72 73 74
CodeAssemblerState::CodeAssemblerState(Isolate* isolate, Zone* zone,
                                       CallDescriptor* call_descriptor,
                                       Code::Kind kind, const char* name,
                                       PoisoningMitigationLevel poisoning_level,
75
                                       int32_t builtin_index)
76 77 78
    : raw_assembler_(new RawMachineAssembler(
          isolate, new (zone) Graph(zone), call_descriptor,
          MachineType::PointerRepresentation(),
79
          InstructionSelector::SupportedMachineOperatorFlags(),
80
          InstructionSelector::AlignmentRequirements(), poisoning_level)),
81
      kind_(kind),
82
      name_(name),
83
      builtin_index_(builtin_index),
84 85 86
      code_generated_(false),
      variables_(zone) {}

87
CodeAssemblerState::~CodeAssemblerState() = default;
88

89 90 91 92
int CodeAssemblerState::parameter_count() const {
  return static_cast<int>(raw_assembler_->call_descriptor()->ParameterCount());
}

93
CodeAssembler::~CodeAssembler() = default;
94

95 96 97 98
#if DEBUG
void CodeAssemblerState::PrintCurrentBlock(std::ostream& os) {
  raw_assembler_->PrintCurrentBlock(os);
}
99
#endif
100 101

bool CodeAssemblerState::InsideBlock() { return raw_assembler_->InsideBlock(); }
102 103 104 105 106 107

void CodeAssemblerState::SetInitialDebugInformation(const char* msg,
                                                    const char* file,
                                                    int line) {
#if DEBUG
  AssemblerDebugInfo debug_info = {msg, file, line};
108
  raw_assembler_->SetSourcePosition(file, line);
109 110 111 112
  raw_assembler_->SetInitialDebugInformation(debug_info);
#endif  // DEBUG
}

113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134
class BreakOnNodeDecorator final : public GraphDecorator {
 public:
  explicit BreakOnNodeDecorator(NodeId node_id) : node_id_(node_id) {}

  void Decorate(Node* node) final {
    if (node->id() == node_id_) {
      base::OS::DebugBreak();
    }
  }

 private:
  NodeId node_id_;
};

void CodeAssembler::BreakOnNode(int node_id) {
  Graph* graph = raw_assembler()->graph();
  Zone* zone = graph->zone();
  GraphDecorator* decorator =
      new (zone) BreakOnNodeDecorator(static_cast<NodeId>(node_id));
  graph->AddDecorator(decorator);
}

135 136 137 138 139 140 141 142 143 144 145 146 147 148
void CodeAssembler::RegisterCallGenerationCallbacks(
    const CodeAssemblerCallback& call_prologue,
    const CodeAssemblerCallback& call_epilogue) {
  // The callback can be registered only once.
  DCHECK(!state_->call_prologue_);
  DCHECK(!state_->call_epilogue_);
  state_->call_prologue_ = call_prologue;
  state_->call_epilogue_ = call_epilogue;
}

void CodeAssembler::UnregisterCallGenerationCallbacks() {
  state_->call_prologue_ = nullptr;
  state_->call_epilogue_ = nullptr;
}
149

150 151 152 153 154 155 156 157 158 159 160
void CodeAssembler::CallPrologue() {
  if (state_->call_prologue_) {
    state_->call_prologue_();
  }
}

void CodeAssembler::CallEpilogue() {
  if (state_->call_epilogue_) {
    state_->call_epilogue_();
  }
}
161

162 163 164 165
bool CodeAssembler::Word32ShiftIsSafe() const {
  return raw_assembler()->machine()->Word32ShiftIsSafe();
}

166 167
PoisoningMitigationLevel CodeAssembler::poisoning_level() const {
  return raw_assembler()->poisoning_level();
168 169
}

170
// static
171 172
Handle<Code> CodeAssembler::GenerateCode(CodeAssemblerState* state,
                                         const AssemblerOptions& options) {
173
  DCHECK(!state->code_generated_);
174

175
  RawMachineAssembler* rasm = state->raw_assembler_.get();
176

177
  Handle<Code> code;
178 179
  Graph* graph = rasm->ExportForOptimization();

180 181 182 183 184
  code = Pipeline::GenerateCodeForCodeStub(
             rasm->isolate(), rasm->call_descriptor(), graph,
             rasm->source_positions(), state->kind_, state->name_,
             state->builtin_index_, rasm->poisoning_level(), options)
             .ToHandleChecked();
185

186
  state->code_generated_ = true;
187 188 189
  return code;
}

190
bool CodeAssembler::Is64() const { return raw_assembler()->machine()->Is64(); }
191 192

bool CodeAssembler::IsFloat64RoundUpSupported() const {
193
  return raw_assembler()->machine()->Float64RoundUp().IsSupported();
194 195 196
}

bool CodeAssembler::IsFloat64RoundDownSupported() const {
197
  return raw_assembler()->machine()->Float64RoundDown().IsSupported();
198 199
}

200 201 202 203
bool CodeAssembler::IsFloat64RoundTiesEvenSupported() const {
  return raw_assembler()->machine()->Float64RoundTiesEven().IsSupported();
}

204
bool CodeAssembler::IsFloat64RoundTruncateSupported() const {
205
  return raw_assembler()->machine()->Float64RoundTruncate().IsSupported();
206 207
}

208 209 210 211 212 213 214 215 216 217 218 219 220
bool CodeAssembler::IsInt32AbsWithOverflowSupported() const {
  return raw_assembler()->machine()->Int32AbsWithOverflow().IsSupported();
}

bool CodeAssembler::IsInt64AbsWithOverflowSupported() const {
  return raw_assembler()->machine()->Int64AbsWithOverflow().IsSupported();
}

bool CodeAssembler::IsIntPtrAbsWithOverflowSupported() const {
  return Is64() ? IsInt64AbsWithOverflowSupported()
                : IsInt32AbsWithOverflowSupported();
}

221 222 223 224 225 226 227 228 229 230 231 232 233 234 235
#ifdef DEBUG
void CodeAssembler::GenerateCheckMaybeObjectIsObject(Node* node,
                                                     const char* location) {
  Label ok(this);
  GotoIf(WordNotEqual(WordAnd(BitcastMaybeObjectToWord(node),
                              IntPtrConstant(kHeapObjectTagMask)),
                      IntPtrConstant(kWeakHeapObjectTag)),
         &ok);
  Node* message_node = StringConstant(location);
  DebugAbort(message_node);
  Unreachable();
  Bind(&ok);
}
#endif

236 237
TNode<Int32T> CodeAssembler::Int32Constant(int32_t value) {
  return UncheckedCast<Int32T>(raw_assembler()->Int32Constant(value));
238 239
}

240 241
TNode<Int64T> CodeAssembler::Int64Constant(int64_t value) {
  return UncheckedCast<Int64T>(raw_assembler()->Int64Constant(value));
242 243
}

244 245
TNode<IntPtrT> CodeAssembler::IntPtrConstant(intptr_t value) {
  return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrConstant(value));
246 247
}

248
TNode<Number> CodeAssembler::NumberConstant(double value) {
249 250 251 252
  int smi_value;
  if (DoubleToSmiInteger(value, &smi_value)) {
    return UncheckedCast<Number>(SmiConstant(smi_value));
  } else {
253 254 255 256
    // We allocate the heap number constant eagerly at this point instead of
    // deferring allocation to code generation
    // (see AllocateAndInstallRequestedHeapObjects) since that makes it easier
    // to generate constant lookups for embedded builtins.
257 258
    return UncheckedCast<Number>(
        HeapConstant(isolate()->factory()->NewHeapNumber(value, TENURED)));
259
  }
260 261
}

262 263 264
TNode<Smi> CodeAssembler::SmiConstant(Smi value) {
  return UncheckedCast<Smi>(BitcastWordToTaggedSigned(
      IntPtrConstant(static_cast<intptr_t>(value.ptr()))));
265 266
}

267
TNode<Smi> CodeAssembler::SmiConstant(int value) {
268 269 270
  return SmiConstant(Smi::FromInt(value));
}

271 272 273
TNode<HeapObject> CodeAssembler::UntypedHeapConstant(
    Handle<HeapObject> object) {
  return UncheckedCast<HeapObject>(raw_assembler()->HeapConstant(object));
274 275
}

276
TNode<String> CodeAssembler::StringConstant(const char* str) {
277 278 279
  Handle<String> internalized_string =
      factory()->InternalizeOneByteString(OneByteVector(str));
  return UncheckedCast<String>(HeapConstant(internalized_string));
280 281
}

282
TNode<Oddball> CodeAssembler::BooleanConstant(bool value) {
283 284 285
  Handle<Object> object = isolate()->factory()->ToBoolean(value);
  return UncheckedCast<Oddball>(
      raw_assembler()->HeapConstant(Handle<HeapObject>::cast(object)));
286 287
}

288 289 290 291
TNode<ExternalReference> CodeAssembler::ExternalConstant(
    ExternalReference address) {
  return UncheckedCast<ExternalReference>(
      raw_assembler()->ExternalConstant(address));
292 293
}

294 295
TNode<Float64T> CodeAssembler::Float64Constant(double value) {
  return UncheckedCast<Float64T>(raw_assembler()->Float64Constant(value));
296 297
}

298
TNode<HeapNumber> CodeAssembler::NaNConstant() {
299
  return UncheckedCast<HeapNumber>(LoadRoot(RootIndex::kNanValue));
300 301
}

302
bool CodeAssembler::ToInt32Constant(Node* node, int32_t& out_value) {
303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
  {
    Int64Matcher m(node);
    if (m.HasValue() && m.IsInRange(std::numeric_limits<int32_t>::min(),
                                    std::numeric_limits<int32_t>::max())) {
      out_value = static_cast<int32_t>(m.Value());
      return true;
    }
  }

  {
    Int32Matcher m(node);
    if (m.HasValue()) {
      out_value = m.Value();
      return true;
    }
318 319 320 321 322 323 324 325 326 327 328
  }

  return false;
}

bool CodeAssembler::ToInt64Constant(Node* node, int64_t& out_value) {
  Int64Matcher m(node);
  if (m.HasValue()) out_value = m.Value();
  return m.HasValue();
}

329
bool CodeAssembler::ToSmiConstant(Node* node, Smi* out_value) {
330 331 332 333 334
  if (node->opcode() == IrOpcode::kBitcastWordToTaggedSigned) {
    node = node->InputAt(0);
  }
  IntPtrMatcher m(node);
  if (m.HasValue()) {
335 336 337
    intptr_t value = m.Value();
    // Make sure that the value is actually a smi
    CHECK_EQ(0, value & ((static_cast<intptr_t>(1) << kSmiShiftSize) - 1));
338
    *out_value = Smi(static_cast<Address>(value));
339 340 341 342 343
    return true;
  }
  return false;
}

344
bool CodeAssembler::ToIntPtrConstant(Node* node, intptr_t& out_value) {
345 346 347 348
  if (node->opcode() == IrOpcode::kBitcastWordToTaggedSigned ||
      node->opcode() == IrOpcode::kBitcastWordToTagged) {
    node = node->InputAt(0);
  }
349 350 351 352 353
  IntPtrMatcher m(node);
  if (m.HasValue()) out_value = m.Value();
  return m.HasValue();
}

354 355 356 357 358 359 360 361 362 363
bool CodeAssembler::IsUndefinedConstant(TNode<Object> node) {
  compiler::HeapObjectMatcher m(node);
  return m.Is(isolate()->factory()->undefined_value());
}

bool CodeAssembler::IsNullConstant(TNode<Object> node) {
  compiler::HeapObjectMatcher m(node);
  return m.Is(isolate()->factory()->null_value());
}

364 365 366 367 368 369 370 371
Node* CodeAssembler::Parameter(int index) {
  if (index == kTargetParameterIndex) return raw_assembler()->TargetParameter();
  return raw_assembler()->Parameter(index);
}

bool CodeAssembler::IsJSFunctionCall() const {
  auto call_descriptor = raw_assembler()->call_descriptor();
  return call_descriptor->IsJSFunctionCall();
372 373
}

374
TNode<Context> CodeAssembler::GetJSContextParameter() {
375 376
  auto call_descriptor = raw_assembler()->call_descriptor();
  DCHECK(call_descriptor->IsJSFunctionCall());
377
  return CAST(Parameter(Linkage::GetJSCallContextParamIndex(
378
      static_cast<int>(call_descriptor->JSParameterCount()))));
379 380
}

381
void CodeAssembler::Return(SloppyTNode<Object> value) {
382
  return raw_assembler()->Return(value);
383 384
}

385 386
void CodeAssembler::Return(SloppyTNode<Object> value1,
                           SloppyTNode<Object> value2) {
387 388 389
  return raw_assembler()->Return(value1, value2);
}

390 391 392
void CodeAssembler::Return(SloppyTNode<Object> value1,
                           SloppyTNode<Object> value2,
                           SloppyTNode<Object> value3) {
393 394 395
  return raw_assembler()->Return(value1, value2, value3);
}

396
void CodeAssembler::PopAndReturn(Node* pop, Node* value) {
397
  return raw_assembler()->PopAndReturn(pop, value);
398 399
}

400 401 402 403 404 405 406 407
void CodeAssembler::ReturnIf(Node* condition, Node* value) {
  Label if_return(this), if_continue(this);
  Branch(condition, &if_return, &if_continue);
  Bind(&if_return);
  Return(value);
  Bind(&if_continue);
}

408 409 410 411
void CodeAssembler::ReturnRaw(Node* value) {
  return raw_assembler()->Return(value);
}

412 413 414 415
void CodeAssembler::DebugAbort(Node* message) {
  raw_assembler()->DebugAbort(message);
}

416
void CodeAssembler::DebugBreak() { raw_assembler()->DebugBreak(); }
417

418 419 420 421 422
void CodeAssembler::Unreachable() {
  DebugBreak();
  raw_assembler()->Unreachable();
}

423
void CodeAssembler::Comment(std::string str) {
424
  if (!FLAG_code_comments) return;
425
  raw_assembler()->Comment(str);
426 427
}

428 429 430 431
void CodeAssembler::SetSourcePosition(const char* file, int line) {
  raw_assembler()->SetSourcePosition(file, line);
}

432
void CodeAssembler::Bind(Label* label) { return label->Bind(); }
433

434 435 436 437 438 439
#if DEBUG
void CodeAssembler::Bind(Label* label, AssemblerDebugInfo debug_info) {
  return label->Bind(debug_info);
}
#endif  // DEBUG

440
Node* CodeAssembler::LoadFramePointer() {
441
  return raw_assembler()->LoadFramePointer();
442 443 444
}

Node* CodeAssembler::LoadParentFramePointer() {
445
  return raw_assembler()->LoadParentFramePointer();
446 447 448
}

Node* CodeAssembler::LoadStackPointer() {
449
  return raw_assembler()->LoadStackPointer();
450 451
}

452
TNode<Object> CodeAssembler::TaggedPoisonOnSpeculation(
453 454
    SloppyTNode<Object> value) {
  return UncheckedCast<Object>(
455
      raw_assembler()->TaggedPoisonOnSpeculation(value));
456 457
}

458 459
TNode<WordT> CodeAssembler::WordPoisonOnSpeculation(SloppyTNode<WordT> value) {
  return UncheckedCast<WordT>(raw_assembler()->WordPoisonOnSpeculation(value));
460 461
}

462 463 464 465
#define DEFINE_CODE_ASSEMBLER_BINARY_OP(name, ResType, Arg1Type, Arg2Type) \
  TNode<ResType> CodeAssembler::name(SloppyTNode<Arg1Type> a,              \
                                     SloppyTNode<Arg2Type> b) {            \
    return UncheckedCast<ResType>(raw_assembler()->name(a, b));            \
466 467 468 469
  }
CODE_ASSEMBLER_BINARY_OP_LIST(DEFINE_CODE_ASSEMBLER_BINARY_OP)
#undef DEFINE_CODE_ASSEMBLER_BINARY_OP

470 471
TNode<WordT> CodeAssembler::IntPtrAdd(SloppyTNode<WordT> left,
                                      SloppyTNode<WordT> right) {
472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(left_constant + right_constant);
    }
    if (left_constant == 0) {
      return right;
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
488
  return UncheckedCast<WordT>(raw_assembler()->IntPtrAdd(left, right));
489 490
}

491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507
TNode<IntPtrT> CodeAssembler::IntPtrDiv(TNode<IntPtrT> left,
                                        TNode<IntPtrT> right) {
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_right_constant) {
    if (is_left_constant) {
      return IntPtrConstant(left_constant / right_constant);
    }
    if (base::bits::IsPowerOfTwo(right_constant)) {
      return WordSar(left, WhichPowerOf2(right_constant));
    }
  }
  return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrDiv(left, right));
}

508 509
TNode<WordT> CodeAssembler::IntPtrSub(SloppyTNode<WordT> left,
                                      SloppyTNode<WordT> right) {
510 511 512 513 514 515 516 517 518 519 520 521 522
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(left_constant - right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
523
  return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrSub(left, right));
524 525
}

526 527 528 529 530 531 532 533 534 535
TNode<WordT> CodeAssembler::IntPtrMul(SloppyTNode<WordT> left,
                                      SloppyTNode<WordT> right) {
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(left_constant * right_constant);
    }
536 537
    if (base::bits::IsPowerOfTwo(left_constant)) {
      return WordShl(right, WhichPowerOf2(left_constant));
538 539
    }
  } else if (is_right_constant) {
540 541
    if (base::bits::IsPowerOfTwo(right_constant)) {
      return WordShl(left, WhichPowerOf2(right_constant));
542 543 544 545 546
    }
  }
  return UncheckedCast<IntPtrT>(raw_assembler()->IntPtrMul(left, right));
}

547 548
TNode<WordT> CodeAssembler::WordShl(SloppyTNode<WordT> value, int shift) {
  return (shift != 0) ? WordShl(value, IntPtrConstant(shift)) : value;
549 550
}

551 552
TNode<WordT> CodeAssembler::WordShr(SloppyTNode<WordT> value, int shift) {
  return (shift != 0) ? WordShr(value, IntPtrConstant(shift)) : value;
oth's avatar
oth committed
553 554
}

555 556 557 558
TNode<WordT> CodeAssembler::WordSar(SloppyTNode<WordT> value, int shift) {
  return (shift != 0) ? WordSar(value, IntPtrConstant(shift)) : value;
}

559 560
TNode<Word32T> CodeAssembler::Word32Shr(SloppyTNode<Word32T> value, int shift) {
  return (shift != 0) ? Word32Shr(value, Int32Constant(shift)) : value;
561 562
}

563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874
TNode<WordT> CodeAssembler::WordOr(SloppyTNode<WordT> left,
                                   SloppyTNode<WordT> right) {
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(left_constant | right_constant);
    }
    if (left_constant == 0) {
      return right;
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<WordT>(raw_assembler()->WordOr(left, right));
}

TNode<WordT> CodeAssembler::WordAnd(SloppyTNode<WordT> left,
                                    SloppyTNode<WordT> right) {
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(left_constant & right_constant);
    }
  }
  return UncheckedCast<WordT>(raw_assembler()->WordAnd(left, right));
}

TNode<WordT> CodeAssembler::WordXor(SloppyTNode<WordT> left,
                                    SloppyTNode<WordT> right) {
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(left_constant ^ right_constant);
    }
  }
  return UncheckedCast<WordT>(raw_assembler()->WordXor(left, right));
}

TNode<WordT> CodeAssembler::WordShl(SloppyTNode<WordT> left,
                                    SloppyTNode<IntegralT> right) {
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(left_constant << right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<WordT>(raw_assembler()->WordShl(left, right));
}

TNode<WordT> CodeAssembler::WordShr(SloppyTNode<WordT> left,
                                    SloppyTNode<IntegralT> right) {
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(static_cast<uintptr_t>(left_constant) >>
                            right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<WordT>(raw_assembler()->WordShr(left, right));
}

TNode<WordT> CodeAssembler::WordSar(SloppyTNode<WordT> left,
                                    SloppyTNode<IntegralT> right) {
  intptr_t left_constant;
  bool is_left_constant = ToIntPtrConstant(left, left_constant);
  intptr_t right_constant;
  bool is_right_constant = ToIntPtrConstant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return IntPtrConstant(left_constant >> right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<WordT>(raw_assembler()->WordSar(left, right));
}

TNode<Word32T> CodeAssembler::Word32Or(SloppyTNode<Word32T> left,
                                       SloppyTNode<Word32T> right) {
  int32_t left_constant;
  bool is_left_constant = ToInt32Constant(left, left_constant);
  int32_t right_constant;
  bool is_right_constant = ToInt32Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int32Constant(left_constant | right_constant);
    }
    if (left_constant == 0) {
      return right;
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<Word32T>(raw_assembler()->Word32Or(left, right));
}

TNode<Word32T> CodeAssembler::Word32And(SloppyTNode<Word32T> left,
                                        SloppyTNode<Word32T> right) {
  int32_t left_constant;
  bool is_left_constant = ToInt32Constant(left, left_constant);
  int32_t right_constant;
  bool is_right_constant = ToInt32Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int32Constant(left_constant & right_constant);
    }
  }
  return UncheckedCast<Word32T>(raw_assembler()->Word32And(left, right));
}

TNode<Word32T> CodeAssembler::Word32Xor(SloppyTNode<Word32T> left,
                                        SloppyTNode<Word32T> right) {
  int32_t left_constant;
  bool is_left_constant = ToInt32Constant(left, left_constant);
  int32_t right_constant;
  bool is_right_constant = ToInt32Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int32Constant(left_constant ^ right_constant);
    }
  }
  return UncheckedCast<Word32T>(raw_assembler()->Word32Xor(left, right));
}

TNode<Word32T> CodeAssembler::Word32Shl(SloppyTNode<Word32T> left,
                                        SloppyTNode<Word32T> right) {
  int32_t left_constant;
  bool is_left_constant = ToInt32Constant(left, left_constant);
  int32_t right_constant;
  bool is_right_constant = ToInt32Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int32Constant(left_constant << right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<Word32T>(raw_assembler()->Word32Shl(left, right));
}

TNode<Word32T> CodeAssembler::Word32Shr(SloppyTNode<Word32T> left,
                                        SloppyTNode<Word32T> right) {
  int32_t left_constant;
  bool is_left_constant = ToInt32Constant(left, left_constant);
  int32_t right_constant;
  bool is_right_constant = ToInt32Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int32Constant(static_cast<uint32_t>(left_constant) >>
                           right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<Word32T>(raw_assembler()->Word32Shr(left, right));
}

TNode<Word32T> CodeAssembler::Word32Sar(SloppyTNode<Word32T> left,
                                        SloppyTNode<Word32T> right) {
  int32_t left_constant;
  bool is_left_constant = ToInt32Constant(left, left_constant);
  int32_t right_constant;
  bool is_right_constant = ToInt32Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int32Constant(left_constant >> right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<Word32T>(raw_assembler()->Word32Sar(left, right));
}

TNode<Word64T> CodeAssembler::Word64Or(SloppyTNode<Word64T> left,
                                       SloppyTNode<Word64T> right) {
  int64_t left_constant;
  bool is_left_constant = ToInt64Constant(left, left_constant);
  int64_t right_constant;
  bool is_right_constant = ToInt64Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int64Constant(left_constant | right_constant);
    }
    if (left_constant == 0) {
      return right;
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<Word64T>(raw_assembler()->Word64Or(left, right));
}

TNode<Word64T> CodeAssembler::Word64And(SloppyTNode<Word64T> left,
                                        SloppyTNode<Word64T> right) {
  int64_t left_constant;
  bool is_left_constant = ToInt64Constant(left, left_constant);
  int64_t right_constant;
  bool is_right_constant = ToInt64Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int64Constant(left_constant & right_constant);
    }
  }
  return UncheckedCast<Word64T>(raw_assembler()->Word64And(left, right));
}

TNode<Word64T> CodeAssembler::Word64Xor(SloppyTNode<Word64T> left,
                                        SloppyTNode<Word64T> right) {
  int64_t left_constant;
  bool is_left_constant = ToInt64Constant(left, left_constant);
  int64_t right_constant;
  bool is_right_constant = ToInt64Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int64Constant(left_constant ^ right_constant);
    }
  }
  return UncheckedCast<Word64T>(raw_assembler()->Word64Xor(left, right));
}

TNode<Word64T> CodeAssembler::Word64Shl(SloppyTNode<Word64T> left,
                                        SloppyTNode<Word64T> right) {
  int64_t left_constant;
  bool is_left_constant = ToInt64Constant(left, left_constant);
  int64_t right_constant;
  bool is_right_constant = ToInt64Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int64Constant(left_constant << right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<Word64T>(raw_assembler()->Word64Shl(left, right));
}

TNode<Word64T> CodeAssembler::Word64Shr(SloppyTNode<Word64T> left,
                                        SloppyTNode<Word64T> right) {
  int64_t left_constant;
  bool is_left_constant = ToInt64Constant(left, left_constant);
  int64_t right_constant;
  bool is_right_constant = ToInt64Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int64Constant(static_cast<uint64_t>(left_constant) >>
                           right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<Word64T>(raw_assembler()->Word64Shr(left, right));
}

TNode<Word64T> CodeAssembler::Word64Sar(SloppyTNode<Word64T> left,
                                        SloppyTNode<Word64T> right) {
  int64_t left_constant;
  bool is_left_constant = ToInt64Constant(left, left_constant);
  int64_t right_constant;
  bool is_right_constant = ToInt64Constant(right, right_constant);
  if (is_left_constant) {
    if (is_right_constant) {
      return Int64Constant(left_constant >> right_constant);
    }
  } else if (is_right_constant) {
    if (right_constant == 0) {
      return left;
    }
  }
  return UncheckedCast<Word64T>(raw_assembler()->Word64Sar(left, right));
}

875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893
#define CODE_ASSEMBLER_COMPARE(Name, ArgT, VarT, ToConstant, op)     \
  TNode<BoolT> CodeAssembler::Name(SloppyTNode<ArgT> left,           \
                                   SloppyTNode<ArgT> right) {        \
    VarT lhs, rhs;                                                   \
    if (ToConstant(left, lhs) && ToConstant(right, rhs)) {           \
      return BoolConstant(lhs op rhs);                               \
    }                                                                \
    return UncheckedCast<BoolT>(raw_assembler()->Name(left, right)); \
  }

CODE_ASSEMBLER_COMPARE(IntPtrEqual, WordT, intptr_t, ToIntPtrConstant, ==)
CODE_ASSEMBLER_COMPARE(WordEqual, WordT, intptr_t, ToIntPtrConstant, ==)
CODE_ASSEMBLER_COMPARE(WordNotEqual, WordT, intptr_t, ToIntPtrConstant, !=)
CODE_ASSEMBLER_COMPARE(Word32Equal, Word32T, int32_t, ToInt32Constant, ==)
CODE_ASSEMBLER_COMPARE(Word32NotEqual, Word32T, int32_t, ToInt32Constant, !=)
CODE_ASSEMBLER_COMPARE(Word64Equal, Word64T, int64_t, ToInt64Constant, ==)
CODE_ASSEMBLER_COMPARE(Word64NotEqual, Word64T, int64_t, ToInt64Constant, !=)
#undef CODE_ASSEMBLER_COMPARE

894
TNode<UintPtrT> CodeAssembler::ChangeUint32ToWord(SloppyTNode<Word32T> value) {
895
  if (raw_assembler()->machine()->Is64()) {
896 897
    return UncheckedCast<UintPtrT>(
        raw_assembler()->ChangeUint32ToUint64(value));
898
  }
899
  return ReinterpretCast<UintPtrT>(value);
900 901
}

902
TNode<IntPtrT> CodeAssembler::ChangeInt32ToIntPtr(SloppyTNode<Word32T> value) {
903
  if (raw_assembler()->machine()->Is64()) {
904
    return ReinterpretCast<IntPtrT>(raw_assembler()->ChangeInt32ToInt64(value));
905
  }
906
  return ReinterpretCast<IntPtrT>(value);
907 908
}

909 910
TNode<UintPtrT> CodeAssembler::ChangeFloat64ToUintPtr(
    SloppyTNode<Float64T> value) {
911
  if (raw_assembler()->machine()->Is64()) {
912 913
    return ReinterpretCast<UintPtrT>(
        raw_assembler()->ChangeFloat64ToUint64(value));
914
  }
915 916
  return ReinterpretCast<UintPtrT>(
      raw_assembler()->ChangeFloat64ToUint32(value));
917 918
}

919 920 921 922 923 924 925 926 927 928 929
TNode<Float64T> CodeAssembler::ChangeUintPtrToFloat64(TNode<UintPtrT> value) {
  if (raw_assembler()->machine()->Is64()) {
    // TODO(turbofan): Maybe we should introduce a ChangeUint64ToFloat64
    // machine operator to TurboFan here?
    return ReinterpretCast<Float64T>(
        raw_assembler()->RoundUint64ToFloat64(value));
  }
  return ReinterpretCast<Float64T>(
      raw_assembler()->ChangeUint32ToFloat64(value));
}

930
Node* CodeAssembler::RoundIntPtrToFloat64(Node* value) {
931 932
  if (raw_assembler()->machine()->Is64()) {
    return raw_assembler()->RoundInt64ToFloat64(value);
933
  }
934
  return raw_assembler()->ChangeInt32ToFloat64(value);
935 936
}

937 938 939 940
#define DEFINE_CODE_ASSEMBLER_UNARY_OP(name, ResType, ArgType) \
  TNode<ResType> CodeAssembler::name(SloppyTNode<ArgType> a) { \
    return UncheckedCast<ResType>(raw_assembler()->name(a));   \
  }
941 942 943
CODE_ASSEMBLER_UNARY_OP_LIST(DEFINE_CODE_ASSEMBLER_UNARY_OP)
#undef DEFINE_CODE_ASSEMBLER_UNARY_OP

944 945 946
Node* CodeAssembler::Load(MachineType rep, Node* base,
                          LoadSensitivity needs_poisoning) {
  return raw_assembler()->Load(rep, base, needs_poisoning);
947 948
}

949 950 951
Node* CodeAssembler::Load(MachineType rep, Node* base, Node* offset,
                          LoadSensitivity needs_poisoning) {
  return raw_assembler()->Load(rep, base, offset, needs_poisoning);
952 953
}

954 955 956 957 958 959 960 961 962 963 964 965
Node* CodeAssembler::LoadFullTagged(Node* base,
                                    LoadSensitivity needs_poisoning) {
  return BitcastWordToTagged(
      Load(MachineType::Pointer(), base, needs_poisoning));
}

Node* CodeAssembler::LoadFullTagged(Node* base, Node* offset,
                                    LoadSensitivity needs_poisoning) {
  return BitcastWordToTagged(
      Load(MachineType::Pointer(), base, offset, needs_poisoning));
}

966 967
Node* CodeAssembler::AtomicLoad(MachineType rep, Node* base, Node* offset) {
  return raw_assembler()->AtomicLoad(rep, base, offset);
968 969
}

970
TNode<Object> CodeAssembler::LoadRoot(RootIndex root_index) {
971
  if (RootsTable::IsImmortalImmovable(root_index)) {
972
    Handle<Object> root = isolate()->root_handle(root_index);
973 974 975 976 977 978 979
    if (root->IsSmi()) {
      return SmiConstant(Smi::cast(*root));
    } else {
      return HeapConstant(Handle<HeapObject>::cast(root));
    }
  }

980 981 982
  // TODO(jgruber): In theory we could generate better code for this by
  // letting the macro assembler decide how to load from the roots list. In most
  // cases, it would boil down to loading from a fixed kRootRegister offset.
983 984 985 986
  Node* isolate_root =
      ExternalConstant(ExternalReference::isolate_root(isolate()));
  int offset = IsolateData::root_slot_offset(root_index);
  return UncheckedCast<Object>(
987
      LoadFullTagged(isolate_root, IntPtrConstant(offset)));
988 989
}

990 991 992
Node* CodeAssembler::Store(Node* base, Node* value) {
  return raw_assembler()->Store(MachineRepresentation::kTagged, base, value,
                                kFullWriteBarrier);
993 994
}

995 996 997 998 999 1000 1001 1002 1003 1004
void CodeAssembler::OptimizedStoreField(MachineRepresentation rep,
                                        TNode<HeapObject> object, int offset,
                                        Node* value,
                                        WriteBarrierKind write_barrier) {
  raw_assembler()->OptimizedStoreField(rep, object, offset, value,
                                       write_barrier);
}
void CodeAssembler::OptimizedStoreMap(TNode<HeapObject> object,
                                      TNode<Map> map) {
  raw_assembler()->OptimizedStoreMap(object, map);
1005 1006
}

1007
Node* CodeAssembler::Store(Node* base, Node* offset, Node* value) {
1008
  return raw_assembler()->Store(MachineRepresentation::kTagged, base, offset,
1009
                                value, kFullWriteBarrier);
1010 1011
}

1012 1013
Node* CodeAssembler::StoreNoWriteBarrier(MachineRepresentation rep, Node* base,
                                         Node* value) {
1014
  return raw_assembler()->Store(rep, base, value, kNoWriteBarrier);
1015 1016 1017
}

Node* CodeAssembler::StoreNoWriteBarrier(MachineRepresentation rep, Node* base,
1018 1019
                                         Node* offset, Node* value) {
  return raw_assembler()->Store(rep, base, offset, value, kNoWriteBarrier);
1020 1021
}

1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033
Node* CodeAssembler::StoreFullTaggedNoWriteBarrier(Node* base,
                                                   Node* tagged_value) {
  return StoreNoWriteBarrier(MachineType::PointerRepresentation(), base,
                             BitcastTaggedToWord(tagged_value));
}

Node* CodeAssembler::StoreFullTaggedNoWriteBarrier(Node* base, Node* offset,
                                                   Node* tagged_value) {
  return StoreNoWriteBarrier(MachineType::PointerRepresentation(), base, offset,
                             BitcastTaggedToWord(tagged_value));
}

1034
Node* CodeAssembler::AtomicStore(MachineRepresentation rep, Node* base,
1035 1036
                                 Node* offset, Node* value, Node* value_high) {
  return raw_assembler()->AtomicStore(rep, base, offset, value, value_high);
1037 1038
}

1039 1040 1041 1042 1043 1044
#define ATOMIC_FUNCTION(name)                                       \
  Node* CodeAssembler::Atomic##name(MachineType type, Node* base,   \
                                    Node* offset, Node* value,      \
                                    Node* value_high) {             \
    return raw_assembler()->Atomic##name(type, base, offset, value, \
                                         value_high);               \
1045
  }
1046 1047 1048 1049 1050 1051
ATOMIC_FUNCTION(Exchange)
ATOMIC_FUNCTION(Add)
ATOMIC_FUNCTION(Sub)
ATOMIC_FUNCTION(And)
ATOMIC_FUNCTION(Or)
ATOMIC_FUNCTION(Xor)
1052
#undef ATOMIC_FUNCTION
1053

1054 1055
Node* CodeAssembler::AtomicCompareExchange(MachineType type, Node* base,
                                           Node* offset, Node* old_value,
1056 1057 1058 1059 1060
                                           Node* new_value,
                                           Node* old_value_high,
                                           Node* new_value_high) {
  return raw_assembler()->AtomicCompareExchange(
      type, base, offset, old_value, old_value_high, new_value, new_value_high);
1061 1062
}

1063
Node* CodeAssembler::StoreRoot(RootIndex root_index, Node* value) {
1064
  DCHECK(!RootsTable::IsImmortalImmovable(root_index));
1065 1066 1067
  Node* isolate_root =
      ExternalConstant(ExternalReference::isolate_root(isolate()));
  int offset = IsolateData::root_slot_offset(root_index);
1068 1069
  return StoreFullTaggedNoWriteBarrier(isolate_root, IntPtrConstant(offset),
                                       value);
1070 1071
}

1072
Node* CodeAssembler::Retain(Node* value) {
1073
  return raw_assembler()->Retain(value);
1074 1075
}

1076
Node* CodeAssembler::Projection(int index, Node* value) {
1077
  DCHECK_LT(index, value->op()->ValueOutputCount());
1078
  return raw_assembler()->Projection(index, value);
1079 1080
}

1081 1082
void CodeAssembler::GotoIfException(Node* node, Label* if_exception,
                                    Variable* exception_var) {
1083 1084 1085 1086 1087
  if (if_exception == nullptr) {
    // If no handler is supplied, don't add continuations
    return;
  }

1088 1089
  // No catch handlers should be active if we're using catch labels
  DCHECK_EQ(state()->exception_handler_labels_.size(), 0);
1090 1091
  DCHECK(!node->op()->HasProperty(Operator::kNoThrow));

1092 1093 1094 1095
  Label success(this), exception(this, Label::kDeferred);
  success.MergeVariables();
  exception.MergeVariables();

1096
  raw_assembler()->Continuations(node, success.label_, exception.label_);
1097 1098

  Bind(&exception);
1099 1100
  const Operator* op = raw_assembler()->common()->IfException();
  Node* exception_value = raw_assembler()->AddNode(op, node, node);
1101 1102 1103 1104 1105 1106
  if (exception_var != nullptr) {
    exception_var->Bind(exception_value);
  }
  Goto(if_exception);

  Bind(&success);
1107
  raw_assembler()->AddNode(raw_assembler()->common()->IfSuccess(), node);
1108 1109
}

1110 1111 1112 1113 1114 1115
TNode<HeapObject> CodeAssembler::OptimizedAllocate(TNode<IntPtrT> size,
                                                   PretenureFlag pretenure) {
  return UncheckedCast<HeapObject>(
      raw_assembler()->OptimizedAllocate(size, pretenure));
}

1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135
void CodeAssembler::HandleException(Node* node) {
  if (state_->exception_handler_labels_.size() == 0) return;
  CodeAssemblerExceptionHandlerLabel* label =
      state_->exception_handler_labels_.back();

  if (node->op()->HasProperty(Operator::kNoThrow)) {
    return;
  }

  Label success(this), exception(this, Label::kDeferred);
  success.MergeVariables();
  exception.MergeVariables();

  raw_assembler()->Continuations(node, success.label_, exception.label_);

  Bind(&exception);
  const Operator* op = raw_assembler()->common()->IfException();
  Node* exception_value = raw_assembler()->AddNode(op, node, node);
  label->AddInputs({UncheckedCast<Object>(exception_value)});
  Goto(label->plain_label());
1136

1137
  Bind(&success);
1138
  raw_assembler()->AddNode(raw_assembler()->common()->IfSuccess(), node);
1139 1140
}

1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161
namespace {
template <size_t kMaxSize>
class NodeArray {
 public:
  void Add(Node* node) {
    DCHECK_GT(kMaxSize, size());
    *ptr_++ = node;
  }

  Node* const* data() const { return arr_; }
  int size() const { return static_cast<int>(ptr_ - arr_); }

 private:
  Node* arr_[kMaxSize];
  Node** ptr_ = arr_;
};
}  // namespace

TNode<Object> CodeAssembler::CallRuntimeImpl(
    Runtime::FunctionId function, TNode<Object> context,
    std::initializer_list<TNode<Object>> args) {
1162 1163 1164 1165 1166 1167 1168 1169 1170
  int result_size = Runtime::FunctionForId(function)->result_size;
  TNode<Code> centry =
      HeapConstant(CodeFactory::RuntimeCEntry(isolate(), result_size));
  return CallRuntimeWithCEntryImpl(function, centry, context, args);
}

TNode<Object> CodeAssembler::CallRuntimeWithCEntryImpl(
    Runtime::FunctionId function, TNode<Code> centry, TNode<Object> context,
    std::initializer_list<TNode<Object>> args) {
1171 1172 1173
  constexpr size_t kMaxNumArgs = 6;
  DCHECK_GE(kMaxNumArgs, args.size());
  int argc = static_cast<int>(args.size());
1174
  auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
1175 1176 1177
      zone(), function, argc, Operator::kNoProperties,
      CallDescriptor::kNoFlags);

1178
  Node* ref = ExternalConstant(ExternalReference::Create(function));
1179 1180
  Node* arity = Int32Constant(argc);

1181 1182 1183 1184 1185 1186
  NodeArray<kMaxNumArgs + 4> inputs;
  inputs.Add(centry);
  for (auto arg : args) inputs.Add(arg);
  inputs.Add(ref);
  inputs.Add(arity);
  inputs.Add(context);
1187

1188
  CallPrologue();
1189
  Node* return_value =
1190
      raw_assembler()->CallN(call_descriptor, inputs.size(), inputs.data());
1191
  HandleException(return_value);
1192
  CallEpilogue();
1193
  return UncheckedCast<Object>(return_value);
1194 1195
}

1196 1197 1198
void CodeAssembler::TailCallRuntimeImpl(
    Runtime::FunctionId function, TNode<Int32T> arity, TNode<Object> context,
    std::initializer_list<TNode<Object>> args) {
1199
  int result_size = Runtime::FunctionForId(function)->result_size;
1200
  TNode<Code> centry =
1201
      HeapConstant(CodeFactory::RuntimeCEntry(isolate(), result_size));
1202
  return TailCallRuntimeWithCEntryImpl(function, arity, centry, context, args);
1203 1204
}

1205 1206 1207 1208 1209 1210
void CodeAssembler::TailCallRuntimeWithCEntryImpl(
    Runtime::FunctionId function, TNode<Int32T> arity, TNode<Code> centry,
    TNode<Object> context, std::initializer_list<TNode<Object>> args) {
  constexpr size_t kMaxNumArgs = 6;
  DCHECK_GE(kMaxNumArgs, args.size());
  int argc = static_cast<int>(args.size());
1211
  auto call_descriptor = Linkage::GetRuntimeCallDescriptor(
1212
      zone(), function, argc, Operator::kNoProperties,
1213
      CallDescriptor::kNoFlags);
1214

1215
  Node* ref = ExternalConstant(ExternalReference::Create(function));
1216

1217 1218 1219 1220 1221 1222
  NodeArray<kMaxNumArgs + 4> inputs;
  inputs.Add(centry);
  for (auto arg : args) inputs.Add(arg);
  inputs.Add(ref);
  inputs.Add(arity);
  inputs.Add(context);
1223

1224
  raw_assembler()->TailCallN(call_descriptor, inputs.size(), inputs.data());
1225 1226
}

1227 1228
Node* CodeAssembler::CallStubN(StubCallMode call_mode,
                               const CallInterfaceDescriptor& descriptor,
1229
                               size_t result_size, int input_count,
1230
                               Node* const* inputs) {
1231
  DCHECK(call_mode == StubCallMode::kCallCodeObject ||
1232 1233
         call_mode == StubCallMode::kCallBuiltinPointer);

1234
  // implicit nodes are target and optionally context.
1235
  int implicit_nodes = descriptor.HasContextParameter() ? 2 : 1;
1236 1237
  DCHECK_LE(implicit_nodes, input_count);
  int argc = input_count - implicit_nodes;
1238 1239 1240 1241
  DCHECK_LE(descriptor.GetParameterCount(), argc);
  // Extra arguments not mentioned in the descriptor are passed on the stack.
  int stack_parameter_count = argc - descriptor.GetRegisterParameterCount();
  DCHECK_LE(descriptor.GetStackParameterCount(), stack_parameter_count);
1242
  DCHECK_EQ(result_size, descriptor.GetReturnCount());
1243

1244
  auto call_descriptor = Linkage::GetStubCallDescriptor(
1245
      zone(), descriptor, stack_parameter_count, CallDescriptor::kNoFlags,
1246
      Operator::kNoProperties, call_mode);
1247

1248
  CallPrologue();
1249 1250
  Node* return_value =
      raw_assembler()->CallN(call_descriptor, input_count, inputs);
1251
  HandleException(return_value);
1252 1253
  CallEpilogue();
  return return_value;
1254 1255
}

1256 1257
void CodeAssembler::TailCallStubImpl(const CallInterfaceDescriptor& descriptor,
                                     TNode<Code> target, TNode<Object> context,
1258 1259 1260 1261
                                     std::initializer_list<Node*> args) {
  constexpr size_t kMaxNumArgs = 11;
  DCHECK_GE(kMaxNumArgs, args.size());
  DCHECK_EQ(descriptor.GetParameterCount(), args.size());
1262
  auto call_descriptor = Linkage::GetStubCallDescriptor(
1263
      zone(), descriptor, descriptor.GetStackParameterCount(),
1264
      CallDescriptor::kNoFlags, Operator::kNoProperties);
1265

1266 1267 1268
  NodeArray<kMaxNumArgs + 2> inputs;
  inputs.Add(target);
  for (auto arg : args) inputs.Add(arg);
1269 1270 1271
  if (descriptor.HasContextParameter()) {
    inputs.Add(context);
  }
1272 1273

  raw_assembler()->TailCallN(call_descriptor, inputs.size(), inputs.data());
1274 1275
}

1276 1277 1278
Node* CodeAssembler::CallStubRImpl(StubCallMode call_mode,
                                   const CallInterfaceDescriptor& descriptor,
                                   size_t result_size, Node* target,
1279 1280
                                   SloppyTNode<Object> context,
                                   std::initializer_list<Node*> args) {
1281
  DCHECK(call_mode == StubCallMode::kCallCodeObject ||
1282 1283
         call_mode == StubCallMode::kCallBuiltinPointer);

1284 1285
  constexpr size_t kMaxNumArgs = 10;
  DCHECK_GE(kMaxNumArgs, args.size());
1286

1287 1288 1289
  NodeArray<kMaxNumArgs + 2> inputs;
  inputs.Add(target);
  for (auto arg : args) inputs.Add(arg);
1290 1291 1292
  if (descriptor.HasContextParameter()) {
    inputs.Add(context);
  }
1293

1294 1295
  return CallStubN(call_mode, descriptor, result_size, inputs.size(),
                   inputs.data());
1296 1297 1298
}

Node* CodeAssembler::TailCallStubThenBytecodeDispatchImpl(
1299
    const CallInterfaceDescriptor& descriptor, Node* target, Node* context,
1300 1301 1302 1303 1304 1305
    std::initializer_list<Node*> args) {
  constexpr size_t kMaxNumArgs = 6;
  DCHECK_GE(kMaxNumArgs, args.size());

  DCHECK_LE(descriptor.GetParameterCount(), args.size());
  int argc = static_cast<int>(args.size());
1306
  // Extra arguments not mentioned in the descriptor are passed on the stack.
1307
  int stack_parameter_count = argc - descriptor.GetRegisterParameterCount();
1308
  DCHECK_LE(descriptor.GetStackParameterCount(), stack_parameter_count);
1309
  auto call_descriptor = Linkage::GetStubCallDescriptor(
1310
      zone(), descriptor, stack_parameter_count, CallDescriptor::kNoFlags,
1311
      Operator::kNoProperties);
1312

1313 1314 1315 1316
  NodeArray<kMaxNumArgs + 2> inputs;
  inputs.Add(target);
  for (auto arg : args) inputs.Add(arg);
  inputs.Add(context);
1317

1318 1319 1320
  return raw_assembler()->TailCallN(call_descriptor, inputs.size(),
                                    inputs.data());
}
1321

1322
template <class... TArgs>
1323
Node* CodeAssembler::TailCallBytecodeDispatch(
1324 1325
    const CallInterfaceDescriptor& descriptor, Node* target, TArgs... args) {
  DCHECK_EQ(descriptor.GetParameterCount(), sizeof...(args));
1326
  auto call_descriptor = Linkage::GetBytecodeDispatchCallDescriptor(
1327
      zone(), descriptor, descriptor.GetStackParameterCount());
1328 1329

  Node* nodes[] = {target, args...};
1330
  CHECK_EQ(descriptor.GetParameterCount() + 1, arraysize(nodes));
1331
  return raw_assembler()->TailCallN(call_descriptor, arraysize(nodes), nodes);
1332 1333
}

1334 1335
// Instantiate TailCallBytecodeDispatch() for argument counts used by
// CSA-generated code
1336 1337
template V8_EXPORT_PRIVATE Node* CodeAssembler::TailCallBytecodeDispatch(
    const CallInterfaceDescriptor& descriptor, Node* target, Node*, Node*,
1338
    Node*, Node*);
1339

1340 1341 1342 1343 1344
TNode<Object> CodeAssembler::TailCallJSCode(TNode<Code> code,
                                            TNode<Context> context,
                                            TNode<JSFunction> function,
                                            TNode<Object> new_target,
                                            TNode<Int32T> arg_count) {
1345
  JSTrampolineDescriptor descriptor;
1346
  auto call_descriptor = Linkage::GetStubCallDescriptor(
1347
      zone(), descriptor, descriptor.GetStackParameterCount(),
1348
      CallDescriptor::kFixedTargetRegister, Operator::kNoProperties);
1349 1350 1351 1352 1353 1354 1355

  Node* nodes[] = {code, function, new_target, arg_count, context};
  CHECK_EQ(descriptor.GetParameterCount() + 2, arraysize(nodes));
  return UncheckedCast<Object>(
      raw_assembler()->TailCallN(call_descriptor, arraysize(nodes), nodes));
}

1356 1357
Node* CodeAssembler::CallCFunctionN(Signature<MachineType>* signature,
                                    int input_count, Node* const* inputs) {
1358 1359
  auto call_descriptor = Linkage::GetSimplifiedCDescriptor(zone(), signature);
  return raw_assembler()->CallN(call_descriptor, input_count, inputs);
1360 1361
}

1362 1363 1364 1365 1366 1367 1368
Node* CodeAssembler::CallCFunction1(MachineType return_type,
                                    MachineType arg0_type, Node* function,
                                    Node* arg0) {
  return raw_assembler()->CallCFunction1(return_type, arg0_type, function,
                                         arg0);
}

1369
Node* CodeAssembler::CallCFunction1WithCallerSavedRegisters(
1370 1371
    MachineType return_type, MachineType arg0_type, Node* function, Node* arg0,
    SaveFPRegsMode mode) {
1372 1373
  DCHECK(return_type.LessThanOrEqualPointerSize());
  return raw_assembler()->CallCFunction1WithCallerSavedRegisters(
1374
      return_type, arg0_type, function, arg0, mode);
1375 1376
}

1377 1378 1379 1380
Node* CodeAssembler::CallCFunction2(MachineType return_type,
                                    MachineType arg0_type,
                                    MachineType arg1_type, Node* function,
                                    Node* arg0, Node* arg1) {
1381 1382
  return raw_assembler()->CallCFunction2(return_type, arg0_type, arg1_type,
                                         function, arg0, arg1);
1383 1384
}

1385 1386 1387 1388 1389 1390 1391 1392 1393
Node* CodeAssembler::CallCFunction3(MachineType return_type,
                                    MachineType arg0_type,
                                    MachineType arg1_type,
                                    MachineType arg2_type, Node* function,
                                    Node* arg0, Node* arg1, Node* arg2) {
  return raw_assembler()->CallCFunction3(return_type, arg0_type, arg1_type,
                                         arg2_type, function, arg0, arg1, arg2);
}

1394 1395
Node* CodeAssembler::CallCFunction3WithCallerSavedRegisters(
    MachineType return_type, MachineType arg0_type, MachineType arg1_type,
1396 1397
    MachineType arg2_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
    SaveFPRegsMode mode) {
1398 1399
  DCHECK(return_type.LessThanOrEqualPointerSize());
  return raw_assembler()->CallCFunction3WithCallerSavedRegisters(
1400 1401
      return_type, arg0_type, arg1_type, arg2_type, function, arg0, arg1, arg2,
      mode);
1402 1403
}

1404 1405 1406 1407 1408 1409 1410 1411 1412
Node* CodeAssembler::CallCFunction4(
    MachineType return_type, MachineType arg0_type, MachineType arg1_type,
    MachineType arg2_type, MachineType arg3_type, Node* function, Node* arg0,
    Node* arg1, Node* arg2, Node* arg3) {
  return raw_assembler()->CallCFunction4(return_type, arg0_type, arg1_type,
                                         arg2_type, arg3_type, function, arg0,
                                         arg1, arg2, arg3);
}

1413 1414 1415 1416 1417 1418 1419 1420 1421 1422
Node* CodeAssembler::CallCFunction5(
    MachineType return_type, MachineType arg0_type, MachineType arg1_type,
    MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
    Node* function, Node* arg0, Node* arg1, Node* arg2, Node* arg3,
    Node* arg4) {
  return raw_assembler()->CallCFunction5(
      return_type, arg0_type, arg1_type, arg2_type, arg3_type, arg4_type,
      function, arg0, arg1, arg2, arg3, arg4);
}

1423 1424 1425 1426 1427 1428 1429 1430 1431 1432
Node* CodeAssembler::CallCFunction6(
    MachineType return_type, MachineType arg0_type, MachineType arg1_type,
    MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
    MachineType arg5_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
    Node* arg3, Node* arg4, Node* arg5) {
  return raw_assembler()->CallCFunction6(
      return_type, arg0_type, arg1_type, arg2_type, arg3_type, arg4_type,
      arg5_type, function, arg0, arg1, arg2, arg3, arg4, arg5);
}

1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444
Node* CodeAssembler::CallCFunction9(
    MachineType return_type, MachineType arg0_type, MachineType arg1_type,
    MachineType arg2_type, MachineType arg3_type, MachineType arg4_type,
    MachineType arg5_type, MachineType arg6_type, MachineType arg7_type,
    MachineType arg8_type, Node* function, Node* arg0, Node* arg1, Node* arg2,
    Node* arg3, Node* arg4, Node* arg5, Node* arg6, Node* arg7, Node* arg8) {
  return raw_assembler()->CallCFunction9(
      return_type, arg0_type, arg1_type, arg2_type, arg3_type, arg4_type,
      arg5_type, arg6_type, arg7_type, arg8_type, function, arg0, arg1, arg2,
      arg3, arg4, arg5, arg6, arg7, arg8);
}

1445
void CodeAssembler::Goto(Label* label) {
1446
  label->MergeVariables();
1447
  raw_assembler()->Goto(label->label_);
1448 1449
}

1450 1451
void CodeAssembler::GotoIf(SloppyTNode<IntegralT> condition,
                           Label* true_label) {
1452 1453 1454 1455 1456
  Label false_label(this);
  Branch(condition, true_label, &false_label);
  Bind(&false_label);
}

1457 1458
void CodeAssembler::GotoIfNot(SloppyTNode<IntegralT> condition,
                              Label* false_label) {
1459 1460 1461 1462 1463
  Label true_label(this);
  Branch(condition, &true_label, false_label);
  Bind(&true_label);
}

1464
void CodeAssembler::Branch(SloppyTNode<IntegralT> condition, Label* true_label,
1465
                           Label* false_label) {
1466 1467 1468 1469 1470 1471 1472
  int32_t constant;
  if (ToInt32Constant(condition, constant)) {
    if ((true_label->is_used() || true_label->is_bound()) &&
        (false_label->is_used() || false_label->is_bound())) {
      return Goto(constant ? true_label : false_label);
    }
  }
1473 1474
  true_label->MergeVariables();
  false_label->MergeVariables();
1475 1476
  return raw_assembler()->Branch(condition, true_label->label_,
                                 false_label->label_);
1477 1478
}

1479
void CodeAssembler::Branch(TNode<BoolT> condition,
1480 1481
                           const std::function<void()>& true_body,
                           const std::function<void()>& false_body) {
1482 1483 1484 1485 1486
  int32_t constant;
  if (ToInt32Constant(condition, constant)) {
    return constant ? true_body() : false_body();
  }

1487
  Label vtrue(this), vfalse(this);
1488
  Branch(condition, &vtrue, &vfalse);
1489

1490
  Bind(&vtrue);
1491 1492 1493 1494 1495 1496 1497
  true_body();

  Bind(&vfalse);
  false_body();
}

void CodeAssembler::Branch(TNode<BoolT> condition, Label* true_label,
1498
                           const std::function<void()>& false_body) {
1499 1500 1501
  int32_t constant;
  if (ToInt32Constant(condition, constant)) {
    return constant ? Goto(true_label) : false_body();
1502
  }
1503 1504 1505

  Label vfalse(this);
  Branch(condition, true_label, &vfalse);
1506
  Bind(&vfalse);
1507 1508 1509 1510
  false_body();
}

void CodeAssembler::Branch(TNode<BoolT> condition,
1511
                           const std::function<void()>& true_body,
1512 1513 1514 1515
                           Label* false_label) {
  int32_t constant;
  if (ToInt32Constant(condition, constant)) {
    return constant ? true_body() : Goto(false_label);
1516
  }
1517 1518 1519 1520 1521

  Label vtrue(this);
  Branch(condition, &vtrue, false_label);
  Bind(&vtrue);
  true_body();
1522 1523
}

1524
void CodeAssembler::Switch(Node* index, Label* default_label,
1525
                           const int32_t* case_values, Label** case_labels,
1526 1527 1528 1529 1530 1531 1532 1533
                           size_t case_count) {
  RawMachineLabel** labels =
      new (zone()->New(sizeof(RawMachineLabel*) * case_count))
          RawMachineLabel*[case_count];
  for (size_t i = 0; i < case_count; ++i) {
    labels[i] = case_labels[i]->label_;
    case_labels[i]->MergeVariables();
  }
1534
  default_label->MergeVariables();
1535 1536
  return raw_assembler()->Switch(index, default_label->label_, case_values,
                                 labels, case_count);
1537 1538
}

1539 1540
bool CodeAssembler::UnalignedLoadSupported(MachineRepresentation rep) const {
  return raw_assembler()->machine()->UnalignedLoadSupported(rep);
1541
}
1542 1543
bool CodeAssembler::UnalignedStoreSupported(MachineRepresentation rep) const {
  return raw_assembler()->machine()->UnalignedStoreSupported(rep);
1544 1545
}

1546
// RawMachineAssembler delegate helpers:
1547
Isolate* CodeAssembler::isolate() const { return raw_assembler()->isolate(); }
1548 1549 1550

Factory* CodeAssembler::factory() const { return isolate()->factory(); }

1551 1552
Zone* CodeAssembler::zone() const { return raw_assembler()->zone(); }

1553 1554 1555 1556
bool CodeAssembler::IsExceptionHandlerActive() const {
  return state_->exception_handler_labels_.size() != 0;
}

1557 1558 1559
RawMachineAssembler* CodeAssembler::raw_assembler() const {
  return state_->raw_assembler_.get();
}
1560 1561 1562 1563 1564

// The core implementation of Variable is stored through an indirection so
// that it can outlive the often block-scoped Variable declarations. This is
// needed to ensure that variable binding and merging through phis can
// properly be verified.
1565
class CodeAssemblerVariable::Impl : public ZoneObject {
1566
 public:
1567
  explicit Impl(MachineRepresentation rep, CodeAssemblerState::VariableId id)
1568 1569 1570 1571 1572
      :
#if DEBUG
        debug_info_(AssemblerDebugInfo(nullptr, nullptr, -1)),
#endif
        value_(nullptr),
1573 1574
        rep_(rep),
        var_id_(id) {
1575 1576 1577 1578 1579 1580 1581 1582 1583 1584
  }

#if DEBUG
  AssemblerDebugInfo debug_info() const { return debug_info_; }
  void set_debug_info(AssemblerDebugInfo debug_info) {
    debug_info_ = debug_info;
  }

  AssemblerDebugInfo debug_info_;
#endif  // DEBUG
1585 1586 1587
  bool operator<(const CodeAssemblerVariable::Impl& other) const {
    return var_id_ < other.var_id_;
  }
1588 1589
  Node* value_;
  MachineRepresentation rep_;
1590
  CodeAssemblerState::VariableId var_id_;
1591 1592
};

1593 1594 1595 1596 1597 1598
bool CodeAssemblerVariable::ImplComparator::operator()(
    const CodeAssemblerVariable::Impl* a,
    const CodeAssemblerVariable::Impl* b) const {
  return *a < *b;
}

1599 1600
CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
                                             MachineRepresentation rep)
1601 1602 1603
    : impl_(new (assembler->zone())
                Impl(rep, assembler->state()->NextVariableId())),
      state_(assembler->state()) {
1604
  state_->variables_.insert(impl_);
1605 1606
}

1607 1608 1609 1610 1611 1612 1613
CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
                                             MachineRepresentation rep,
                                             Node* initial_value)
    : CodeAssemblerVariable(assembler, rep) {
  Bind(initial_value);
}

1614 1615 1616 1617
#if DEBUG
CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
                                             AssemblerDebugInfo debug_info,
                                             MachineRepresentation rep)
1618 1619 1620
    : impl_(new (assembler->zone())
                Impl(rep, assembler->state()->NextVariableId())),
      state_(assembler->state()) {
1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634
  impl_->set_debug_info(debug_info);
  state_->variables_.insert(impl_);
}

CodeAssemblerVariable::CodeAssemblerVariable(CodeAssembler* assembler,
                                             AssemblerDebugInfo debug_info,
                                             MachineRepresentation rep,
                                             Node* initial_value)
    : CodeAssemblerVariable(assembler, debug_info, rep) {
  impl_->set_debug_info(debug_info);
  Bind(initial_value);
}
#endif  // DEBUG

1635 1636 1637
CodeAssemblerVariable::~CodeAssemblerVariable() {
  state_->variables_.erase(impl_);
}
1638

1639
void CodeAssemblerVariable::Bind(Node* value) { impl_->value_ = value; }
1640

1641
Node* CodeAssemblerVariable::value() const {
1642 1643 1644 1645
#if DEBUG
  if (!IsBound()) {
    std::stringstream str;
    str << "#Use of unbound variable:"
1646 1647
        << "#\n    Variable:      " << *this << "#\n    Current Block: ";
    state_->PrintCurrentBlock(str);
1648
    FATAL("%s", str.str().c_str());
1649 1650 1651 1652
  }
  if (!state_->InsideBlock()) {
    std::stringstream str;
    str << "#Accessing variable value outside a block:"
1653
        << "#\n    Variable:      " << *this;
1654
    FATAL("%s", str.str().c_str());
1655 1656
  }
#endif  // DEBUG
1657 1658 1659
  return impl_->value_;
}

1660
MachineRepresentation CodeAssemblerVariable::rep() const { return impl_->rep_; }
1661

1662
bool CodeAssemblerVariable::IsBound() const { return impl_->value_ != nullptr; }
1663

1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678
std::ostream& operator<<(std::ostream& os,
                         const CodeAssemblerVariable::Impl& impl) {
#if DEBUG
  AssemblerDebugInfo info = impl.debug_info();
  if (info.name) os << "V" << info;
#endif  // DEBUG
  return os;
}

std::ostream& operator<<(std::ostream& os,
                         const CodeAssemblerVariable& variable) {
  os << *variable.impl_;
  return os;
}

1679 1680
CodeAssemblerLabel::CodeAssemblerLabel(CodeAssembler* assembler,
                                       size_t vars_count,
1681
                                       CodeAssemblerVariable* const* vars,
1682
                                       CodeAssemblerLabel::Type type)
1683 1684
    : bound_(false),
      merge_count_(0),
1685
      state_(assembler->state()),
1686
      label_(nullptr) {
1687 1688 1689 1690
  void* buffer = assembler->zone()->New(sizeof(RawMachineLabel));
  label_ = new (buffer)
      RawMachineLabel(type == kDeferred ? RawMachineLabel::kDeferred
                                        : RawMachineLabel::kNonDeferred);
1691 1692
  for (size_t i = 0; i < vars_count; ++i) {
    variable_phis_[vars[i]->impl_] = nullptr;
1693 1694 1695
  }
}

1696 1697
CodeAssemblerLabel::~CodeAssemblerLabel() { label_->~RawMachineLabel(); }

1698
void CodeAssemblerLabel::MergeVariables() {
1699
  ++merge_count_;
1700
  for (CodeAssemblerVariable::Impl* var : state_->variables_) {
1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724
    size_t count = 0;
    Node* node = var->value_;
    if (node != nullptr) {
      auto i = variable_merges_.find(var);
      if (i != variable_merges_.end()) {
        i->second.push_back(node);
        count = i->second.size();
      } else {
        count = 1;
        variable_merges_[var] = std::vector<Node*>(1, node);
      }
    }
    // If the following asserts, then you've jumped to a label without a bound
    // variable along that path that expects to merge its value into a phi.
    DCHECK(variable_phis_.find(var) == variable_phis_.end() ||
           count == merge_count_);
    USE(count);

    // If the label is already bound, we already know the set of variables to
    // merge and phi nodes have already been created.
    if (bound_) {
      auto phi = variable_phis_.find(var);
      if (phi != variable_phis_.end()) {
        DCHECK_NOT_NULL(phi->second);
1725
        state_->raw_assembler_->AppendPhiInput(phi->second, node);
1726 1727 1728 1729 1730 1731 1732 1733 1734
      } else {
        auto i = variable_merges_.find(var);
        if (i != variable_merges_.end()) {
          // If the following assert fires, then you've declared a variable that
          // has the same bound value along all paths up until the point you
          // bound this label, but then later merged a path with a new value for
          // the variable after the label bind (it's not possible to add phis to
          // the bound label after the fact, just make sure to list the variable
          // in the label's constructor's list of merged variables).
1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746
#if DEBUG
          if (find_if(i->second.begin(), i->second.end(),
                      [node](Node* e) -> bool { return node != e; }) !=
              i->second.end()) {
            std::stringstream str;
            str << "Unmerged variable found when jumping to block. \n"
                << "#    Variable:      " << *var;
            if (bound_) {
              str << "\n#    Target block:  " << *label_->block();
            }
            str << "\n#    Current Block: ";
            state_->PrintCurrentBlock(str);
1747
            FATAL("%s", str.str().c_str());
1748 1749
          }
#endif  // DEBUG
1750 1751 1752 1753 1754 1755
        }
      }
    }
  }
}

1756 1757
#if DEBUG
void CodeAssemblerLabel::Bind(AssemblerDebugInfo debug_info) {
1758 1759 1760 1761 1762
  if (bound_) {
    std::stringstream str;
    str << "Cannot bind the same label twice:"
        << "\n#    current:  " << debug_info
        << "\n#    previous: " << *label_->block();
1763
    FATAL("%s", str.str().c_str());
1764
  }
1765 1766 1767
  if (FLAG_enable_source_at_csa_bind) {
    state_->raw_assembler_->SetSourcePosition(debug_info.file, debug_info.line);
  }
1768 1769 1770 1771 1772
  state_->raw_assembler_->Bind(label_, debug_info);
  UpdateVariablesAfterBind();
}
#endif  // DEBUG

1773
void CodeAssemblerLabel::Bind() {
1774
  DCHECK(!bound_);
1775
  state_->raw_assembler_->Bind(label_);
1776 1777
  UpdateVariablesAfterBind();
}
1778

1779
void CodeAssemblerLabel::UpdateVariablesAfterBind() {
1780 1781
  // Make sure that all variables that have changed along any path up to this
  // point are marked as merge variables.
1782
  for (auto var : state_->variables_) {
1783 1784 1785 1786
    Node* shared_value = nullptr;
    auto i = variable_merges_.find(var);
    if (i != variable_merges_.end()) {
      for (auto value : i->second) {
1787
        DCHECK_NOT_NULL(value);
1788 1789 1790 1791 1792 1793 1794 1795 1796 1797 1798 1799
        if (value != shared_value) {
          if (shared_value == nullptr) {
            shared_value = value;
          } else {
            variable_phis_[var] = nullptr;
          }
        }
      }
    }
  }

  for (auto var : variable_phis_) {
1800
    CodeAssemblerVariable::Impl* var_impl = var.first;
1801
    auto i = variable_merges_.find(var_impl);
1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817
#if DEBUG
    bool not_found = i == variable_merges_.end();
    if (not_found || i->second.size() != merge_count_) {
      std::stringstream str;
      str << "A variable that has been marked as beeing merged at the label"
          << "\n# doesn't have a bound value along all of the paths that "
          << "\n# have been merged into the label up to this point."
          << "\n#"
          << "\n# This can happen in the following cases:"
          << "\n# - By explicitly marking it so in the label constructor"
          << "\n# - By having seen different bound values at branches"
          << "\n#"
          << "\n# Merge count:     expected=" << merge_count_
          << " vs. found=" << (not_found ? 0 : i->second.size())
          << "\n# Variable:      " << *var_impl
          << "\n# Current Block: " << *label_->block();
1818
      FATAL("%s", str.str().c_str());
1819 1820
    }
#endif  // DEBUG
1821
    Node* phi = state_->raw_assembler_->Phi(
1822 1823 1824 1825 1826 1827
        var.first->rep_, static_cast<int>(merge_count_), &(i->second[0]));
    variable_phis_[var_impl] = phi;
  }

  // Bind all variables to a merge phi, the common value along all paths or
  // null.
1828
  for (auto var : state_->variables_) {
1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844
    auto i = variable_phis_.find(var);
    if (i != variable_phis_.end()) {
      var->value_ = i->second;
    } else {
      auto j = variable_merges_.find(var);
      if (j != variable_merges_.end() && j->second.size() == merge_count_) {
        var->value_ = j->second.back();
      } else {
        var->value_ = nullptr;
      }
    }
  }

  bound_ = true;
}

1845 1846 1847 1848
void CodeAssemblerParameterizedLabelBase::AddInputs(std::vector<Node*> inputs) {
  if (!phi_nodes_.empty()) {
    DCHECK_EQ(inputs.size(), phi_nodes_.size());
    for (size_t i = 0; i < inputs.size(); ++i) {
1849 1850
      // We use {nullptr} as a sentinel for an uninitialized value.
      if (phi_nodes_[i] == nullptr) continue;
1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883
      state_->raw_assembler_->AppendPhiInput(phi_nodes_[i], inputs[i]);
    }
  } else {
    DCHECK_EQ(inputs.size(), phi_inputs_.size());
    for (size_t i = 0; i < inputs.size(); ++i) {
      phi_inputs_[i].push_back(inputs[i]);
    }
  }
}

Node* CodeAssemblerParameterizedLabelBase::CreatePhi(
    MachineRepresentation rep, const std::vector<Node*>& inputs) {
  for (Node* input : inputs) {
    // We use {nullptr} as a sentinel for an uninitialized value. We must not
    // create phi nodes for these.
    if (input == nullptr) return nullptr;
  }
  return state_->raw_assembler_->Phi(rep, static_cast<int>(inputs.size()),
                                     &inputs.front());
}

const std::vector<Node*>& CodeAssemblerParameterizedLabelBase::CreatePhis(
    std::vector<MachineRepresentation> representations) {
  DCHECK(is_used());
  DCHECK(phi_nodes_.empty());
  phi_nodes_.reserve(phi_inputs_.size());
  DCHECK_EQ(representations.size(), phi_inputs_.size());
  for (size_t i = 0; i < phi_inputs_.size(); ++i) {
    phi_nodes_.push_back(CreatePhi(representations[i], phi_inputs_[i]));
  }
  return phi_nodes_;
}

1884 1885 1886 1887 1888 1889 1890 1891 1892
void CodeAssemblerState::PushExceptionHandler(
    CodeAssemblerExceptionHandlerLabel* label) {
  exception_handler_labels_.push_back(label);
}

void CodeAssemblerState::PopExceptionHandler() {
  exception_handler_labels_.pop_back();
}

1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937
CodeAssemblerScopedExceptionHandler::CodeAssemblerScopedExceptionHandler(
    CodeAssembler* assembler, CodeAssemblerExceptionHandlerLabel* label)
    : has_handler_(label != nullptr),
      assembler_(assembler),
      compatibility_label_(nullptr),
      exception_(nullptr) {
  if (has_handler_) {
    assembler_->state()->PushExceptionHandler(label);
  }
}

CodeAssemblerScopedExceptionHandler::CodeAssemblerScopedExceptionHandler(
    CodeAssembler* assembler, CodeAssemblerLabel* label,
    TypedCodeAssemblerVariable<Object>* exception)
    : has_handler_(label != nullptr),
      assembler_(assembler),
      compatibility_label_(label),
      exception_(exception) {
  if (has_handler_) {
    label_ = base::make_unique<CodeAssemblerExceptionHandlerLabel>(
        assembler, CodeAssemblerLabel::kDeferred);
    assembler_->state()->PushExceptionHandler(label_.get());
  }
}

CodeAssemblerScopedExceptionHandler::~CodeAssemblerScopedExceptionHandler() {
  if (has_handler_) {
    assembler_->state()->PopExceptionHandler();
  }
  if (label_ && label_->is_used()) {
    CodeAssembler::Label skip(assembler_);
    bool inside_block = assembler_->state()->InsideBlock();
    if (inside_block) {
      assembler_->Goto(&skip);
    }
    TNode<Object> e;
    assembler_->Bind(label_.get(), &e);
    *exception_ = e;
    assembler_->Goto(compatibility_label_);
    if (inside_block) {
      assembler_->Bind(&skip);
    }
  }
}

1938
}  // namespace compiler
1939

1940 1941
Address CheckObjectType(Address raw_value, Address raw_type,
                        Address raw_location) {
1942
#ifdef DEBUG
1943
  Object value(raw_value);
1944
  Smi type(raw_type);
1945
  String location = String::cast(Object(raw_location));
1946 1947
  const char* expected;
  switch (static_cast<ObjectType>(type->value())) {
1948 1949 1950 1951
#define TYPE_CASE(Name)                                  \
  case ObjectType::k##Name:                              \
    if (value->Is##Name()) return Smi::FromInt(0).ptr(); \
    expected = #Name;                                    \
1952
    break;
1953 1954 1955 1956
#define TYPE_STRUCT_CASE(NAME, Name, name)               \
  case ObjectType::k##Name:                              \
    if (value->Is##Name()) return Smi::FromInt(0).ptr(); \
    expected = #Name;                                    \
1957 1958 1959 1960 1961 1962 1963 1964 1965 1966
    break;

    TYPE_CASE(Object)
    OBJECT_TYPE_LIST(TYPE_CASE)
    HEAP_OBJECT_TYPE_LIST(TYPE_CASE)
    STRUCT_LIST(TYPE_STRUCT_CASE)
#undef TYPE_CASE
#undef TYPE_STRUCT_CASE
  }
  std::stringstream value_description;
1967
  value->Print(value_description);
1968 1969 1970 1971 1972 1973 1974 1975 1976
  V8_Fatal(__FILE__, __LINE__,
           "Type cast failed in %s\n"
           "  Expected %s but found %s",
           location->ToAsciiArray(), expected, value_description.str().c_str());
#else
  UNREACHABLE();
#endif
}

1977 1978
}  // namespace internal
}  // namespace v8