code-assembler.h 68.1 KB
Newer Older
1 2 3 4 5 6 7 8
// Copyright 2015 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef V8_COMPILER_CODE_ASSEMBLER_H_
#define V8_COMPILER_CODE_ASSEMBLER_H_

#include <map>
9
#include <memory>
10
#include <initializer_list>
11 12 13

// Clients of this interface shouldn't depend on lots of compiler internals.
// Do not include anything from src/compiler here!
14
#include "src/base/macros.h"
15
#include "src/base/type-traits.h"
16
#include "src/builtins/builtins.h"
17 18 19
#include "src/codegen/code-factory.h"
#include "src/codegen/machine-type.h"
#include "src/codegen/source-position.h"
20
#include "src/heap/heap.h"
21
#include "src/objects/arguments.h"
22
#include "src/objects/data-handler.h"
23
#include "src/objects/heap-number.h"
24
#include "src/objects/js-array-buffer.h"
25
#include "src/objects/js-collection.h"
26
#include "src/objects/js-proxy.h"
27
#include "src/objects/map.h"
28
#include "src/objects/maybe-object.h"
29
#include "src/objects/objects.h"
30
#include "src/objects/oddball.h"
31
#include "src/runtime/runtime.h"
32
#include "src/utils/allocation.h"
33
#include "src/zone/zone-containers.h"
34 35 36 37

namespace v8 {
namespace internal {

38
// Forward declarations.
39
class AsmWasmData;
40
class AsyncGeneratorRequest;
41
struct AssemblerOptions;
42
class BigInt;
43
class CallInterfaceDescriptor;
44 45
class Callable;
class Factory;
46
class FinalizationGroupCleanupJobTask;
47
class InterpreterData;
48
class Isolate;
49
class JSAsyncFunctionObject;
50 51
class JSAsyncGeneratorObject;
class JSCollator;
52
class JSCollection;
53 54 55 56 57
class JSDateTimeFormat;
class JSListFormat;
class JSLocale;
class JSNumberFormat;
class JSPluralRules;
58
class JSRegExpStringIterator;
59
class JSRelativeTimeFormat;
60
class JSSegmentIterator;
61
class JSSegmenter;
62
class JSV8BreakIterator;
63
class JSWeakCollection;
64 65
class JSFinalizationGroup;
class JSFinalizationGroupCleanupIterator;
66
class JSWeakMap;
67
class JSWeakRef;
68
class JSWeakSet;
69 70 71 72 73
class PromiseCapability;
class PromiseFulfillReactionJobTask;
class PromiseReaction;
class PromiseReactionJobTask;
class PromiseRejectReactionJobTask;
74
class WasmDebugInfo;
75 76
class Zone;

77 78 79
template <typename T>
class Signature;

80 81 82 83 84 85
struct UntaggedT {};

struct IntegralT : UntaggedT {};

struct WordT : IntegralT {
  static const MachineRepresentation kMachineRepresentation =
86 87
      (kSystemPointerSize == 4) ? MachineRepresentation::kWord32
                                : MachineRepresentation::kWord64;
88 89
};

90 91 92
struct RawPtrT : WordT {
  static constexpr MachineType kMachineType = MachineType::Pointer();
};
93 94 95 96 97 98 99 100

template <class To>
struct RawPtr : RawPtrT {};

struct Word32T : IntegralT {
  static const MachineRepresentation kMachineRepresentation =
      MachineRepresentation::kWord32;
};
101 102 103 104 105 106
struct Int32T : Word32T {
  static constexpr MachineType kMachineType = MachineType::Int32();
};
struct Uint32T : Word32T {
  static constexpr MachineType kMachineType = MachineType::Uint32();
};
107 108 109 110 111 112 113 114 115 116 117 118
struct Int16T : Int32T {
  static constexpr MachineType kMachineType = MachineType::Int16();
};
struct Uint16T : Uint32T {
  static constexpr MachineType kMachineType = MachineType::Uint16();
};
struct Int8T : Int16T {
  static constexpr MachineType kMachineType = MachineType::Int8();
};
struct Uint8T : Uint16T {
  static constexpr MachineType kMachineType = MachineType::Uint8();
};
119

120 121 122 123 124 125 126 127 128 129
struct Word64T : IntegralT {
  static const MachineRepresentation kMachineRepresentation =
      MachineRepresentation::kWord64;
};
struct Int64T : Word64T {
  static constexpr MachineType kMachineType = MachineType::Int64();
};
struct Uint64T : Word64T {
  static constexpr MachineType kMachineType = MachineType::Uint64();
};
130

131 132 133 134 135 136
struct IntPtrT : WordT {
  static constexpr MachineType kMachineType = MachineType::IntPtr();
};
struct UintPtrT : WordT {
  static constexpr MachineType kMachineType = MachineType::UintPtr();
};
137 138 139 140

struct Float32T : UntaggedT {
  static const MachineRepresentation kMachineRepresentation =
      MachineRepresentation::kFloat32;
141
  static constexpr MachineType kMachineType = MachineType::Float32();
142 143 144 145 146
};

struct Float64T : UntaggedT {
  static const MachineRepresentation kMachineRepresentation =
      MachineRepresentation::kFloat64;
147
  static constexpr MachineType kMachineType = MachineType::Float64();
148 149 150 151 152
};

// Result of a comparison operation.
struct BoolT : Word32T {};

153 154 155 156 157 158 159 160 161 162 163 164
// Value type of a Turbofan node with two results.
template <class T1, class T2>
struct PairT {};

inline constexpr MachineType CommonMachineType(MachineType type1,
                                               MachineType type2) {
  return (type1 == type2) ? type1
                          : ((type1.IsTagged() && type2.IsTagged())
                                 ? MachineType::AnyTagged()
                                 : MachineType::None());
}

165
template <class Type, class Enable = void>
166 167
struct MachineTypeOf {
  static constexpr MachineType value = Type::kMachineType;
168
};
169 170 171 172

template <class Type, class Enable>
constexpr MachineType MachineTypeOf<Type, Enable>::value;

173
template <>
174 175
struct MachineTypeOf<Object> {
  static constexpr MachineType value = MachineType::AnyTagged();
176 177
};
template <>
178 179 180 181
struct MachineTypeOf<MaybeObject> {
  static constexpr MachineType value = MachineType::AnyTagged();
};
template <>
182 183
struct MachineTypeOf<Smi> {
  static constexpr MachineType value = MachineType::TaggedSigned();
184 185
};
template <class HeapObjectSubtype>
186 187 188
struct MachineTypeOf<HeapObjectSubtype,
                     typename std::enable_if<std::is_base_of<
                         HeapObject, HeapObjectSubtype>::value>::type> {
189 190 191 192 193
  static constexpr MachineType value = MachineType::TaggedPointer();
};

template <class HeapObjectSubtype>
constexpr MachineType MachineTypeOf<
194 195
    HeapObjectSubtype, typename std::enable_if<std::is_base_of<
                           HeapObject, HeapObjectSubtype>::value>::type>::value;
196 197 198 199 200 201 202 203

template <class Type, class Enable = void>
struct MachineRepresentationOf {
  static const MachineRepresentation value = Type::kMachineRepresentation;
};
template <class T>
struct MachineRepresentationOf<
    T, typename std::enable_if<std::is_base_of<Object, T>::value>::type> {
204
  static const MachineRepresentation value =
205 206
      MachineTypeOf<T>::value.representation();
};
207 208 209 210 211 212
template <class T>
struct MachineRepresentationOf<
    T, typename std::enable_if<std::is_base_of<MaybeObject, T>::value>::type> {
  static const MachineRepresentation value =
      MachineTypeOf<T>::value.representation();
};
213 214 215 216 217

template <class T>
struct is_valid_type_tag {
  static const bool value = std::is_base_of<Object, T>::value ||
                            std::is_base_of<UntaggedT, T>::value ||
218
                            std::is_base_of<MaybeObject, T>::value ||
219
                            std::is_same<ExternalReference, T>::value;
220 221
  static const bool is_tagged = std::is_base_of<Object, T>::value ||
                                std::is_base_of<MaybeObject, T>::value;
222 223 224 225 226 227 228
};

template <class T1, class T2>
struct is_valid_type_tag<PairT<T1, T2>> {
  static const bool value =
      is_valid_type_tag<T1>::value && is_valid_type_tag<T2>::value;
  static const bool is_tagged = false;
229 230
};

231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
template <class T1, class T2>
struct UnionT;

template <class T1, class T2>
struct is_valid_type_tag<UnionT<T1, T2>> {
  static const bool is_tagged =
      is_valid_type_tag<T1>::is_tagged && is_valid_type_tag<T2>::is_tagged;
  static const bool value = is_tagged;
};

template <class T1, class T2>
struct UnionT {
  static constexpr MachineType kMachineType =
      CommonMachineType(MachineTypeOf<T1>::value, MachineTypeOf<T2>::value);
  static const MachineRepresentation kMachineRepresentation =
      kMachineType.representation();
  static_assert(kMachineRepresentation != MachineRepresentation::kNone,
                "no common representation");
  static_assert(is_valid_type_tag<T1>::is_tagged &&
                    is_valid_type_tag<T2>::is_tagged,
                "union types are only possible for tagged values");
};

using Number = UnionT<Smi, HeapNumber>;
using Numeric = UnionT<Number, BigInt>;

257
// A pointer to a builtin function, used by Torque's function pointers.
258
using BuiltinPtr = Smi;
259

260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277
class int31_t {
 public:
  int31_t() : value_(0) {}
  int31_t(int value) : value_(value) {  // NOLINT(runtime/explicit)
    DCHECK_EQ((value & 0x80000000) != 0, (value & 0x40000000) != 0);
  }
  int31_t& operator=(int value) {
    DCHECK_EQ((value & 0x80000000) != 0, (value & 0x40000000) != 0);
    value_ = value;
    return *this;
  }
  int32_t value() const { return value_; }
  operator int32_t() const { return value_; }

 private:
  int32_t value_;
};

278 279 280
#define ENUM_ELEMENT(Name) k##Name,
#define ENUM_STRUCT_ELEMENT(NAME, Name, name) k##Name,
enum class ObjectType {
281 282 283 284 285 286
  ENUM_ELEMENT(Object)                 //
  ENUM_ELEMENT(Smi)                    //
  ENUM_ELEMENT(HeapObject)             //
  OBJECT_TYPE_LIST(ENUM_ELEMENT)       //
  HEAP_OBJECT_TYPE_LIST(ENUM_ELEMENT)  //
  STRUCT_LIST(ENUM_STRUCT_ELEMENT)     //
287 288 289 290
};
#undef ENUM_ELEMENT
#undef ENUM_STRUCT_ELEMENT

291 292 293 294 295 296 297 298 299 300
enum class CheckBounds { kAlways, kDebugOnly };
inline bool NeedsBoundsCheck(CheckBounds check_bounds) {
  switch (check_bounds) {
    case CheckBounds::kAlways:
      return true;
    case CheckBounds::kDebugOnly:
      return DEBUG_BOOL;
  }
}

301 302
enum class StoreToObjectWriteBarrier { kNone, kMap, kFull };

303
class AccessCheckNeeded;
304
class BigIntWrapper;
305
class ClassBoilerplate;
306
class BooleanWrapper;
307 308 309
class CompilationCacheTable;
class Constructor;
class Filler;
310
class FunctionTemplateRareData;
311
class InternalizedString;
312
class JSArgumentsObject;
313
class JSArrayBufferView;
314 315
class JSContextExtensionObject;
class JSError;
316
class JSSloppyArgumentsObject;
317 318 319
class MapCache;
class MutableHeapNumber;
class NativeContext;
320 321
class NumberWrapper;
class ScriptWrapper;
322
class SloppyArgumentsElements;
323
class StringWrapper;
324
class SymbolWrapper;
325 326
class Undetectable;
class UniqueName;
327
class WasmCapiFunctionData;
328
class WasmExceptionObject;
329
class WasmExceptionTag;
330
class WasmExportedFunctionData;
331
class WasmGlobalObject;
332
class WasmJSFunctionData;
333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355
class WasmMemoryObject;
class WasmModuleObject;
class WasmTableObject;

template <class T>
struct ObjectTypeOf {};

#define OBJECT_TYPE_CASE(Name)                           \
  template <>                                            \
  struct ObjectTypeOf<Name> {                            \
    static const ObjectType value = ObjectType::k##Name; \
  };
#define OBJECT_TYPE_STRUCT_CASE(NAME, Name, name)        \
  template <>                                            \
  struct ObjectTypeOf<Name> {                            \
    static const ObjectType value = ObjectType::k##Name; \
  };
#define OBJECT_TYPE_TEMPLATE_CASE(Name)                  \
  template <class... Args>                               \
  struct ObjectTypeOf<Name<Args...>> {                   \
    static const ObjectType value = ObjectType::k##Name; \
  };
OBJECT_TYPE_CASE(Object)
356 357
OBJECT_TYPE_CASE(Smi)
OBJECT_TYPE_CASE(HeapObject)
358 359 360 361 362 363 364 365
OBJECT_TYPE_LIST(OBJECT_TYPE_CASE)
HEAP_OBJECT_ORDINARY_TYPE_LIST(OBJECT_TYPE_CASE)
STRUCT_LIST(OBJECT_TYPE_STRUCT_CASE)
HEAP_OBJECT_TEMPLATE_TYPE_LIST(OBJECT_TYPE_TEMPLATE_CASE)
#undef OBJECT_TYPE_CASE
#undef OBJECT_TYPE_STRUCT_CASE
#undef OBJECT_TYPE_TEMPLATE_CASE

366
// {raw_value} must be a tagged Object.
367 368 369
// {raw_type} must be a tagged Smi.
// {raw_location} must be a tagged String.
// Returns a tagged Smi.
370 371
Address CheckObjectType(Address raw_value, Address raw_type,
                        Address raw_location);
372

373 374 375
namespace compiler {

class CallDescriptor;
376 377
class CodeAssemblerLabel;
class CodeAssemblerVariable;
378 379
template <class T>
class TypedCodeAssemblerVariable;
380
class CodeAssemblerState;
381 382 383
class Node;
class RawMachineAssembler;
class RawMachineLabel;
384
class SourcePositionTable;
385

386
using CodeAssemblerVariableList = ZoneVector<CodeAssemblerVariable*>;
387

388
using CodeAssemblerCallback = std::function<void()>;
389

390 391
template <class T, class U>
struct is_subtype {
392
  static const bool value = std::is_base_of<U, T>::value;
393
};
394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457
template <class T1, class T2, class U>
struct is_subtype<UnionT<T1, T2>, U> {
  static const bool value =
      is_subtype<T1, U>::value && is_subtype<T2, U>::value;
};
template <class T, class U1, class U2>
struct is_subtype<T, UnionT<U1, U2>> {
  static const bool value =
      is_subtype<T, U1>::value || is_subtype<T, U2>::value;
};
template <class T1, class T2, class U1, class U2>
struct is_subtype<UnionT<T1, T2>, UnionT<U1, U2>> {
  static const bool value =
      (is_subtype<T1, U1>::value || is_subtype<T1, U2>::value) &&
      (is_subtype<T2, U1>::value || is_subtype<T2, U2>::value);
};

template <class T, class U>
struct types_have_common_values {
  static const bool value = is_subtype<T, U>::value || is_subtype<U, T>::value;
};
template <class U>
struct types_have_common_values<Uint32T, U> {
  static const bool value = types_have_common_values<Word32T, U>::value;
};
template <class U>
struct types_have_common_values<Int32T, U> {
  static const bool value = types_have_common_values<Word32T, U>::value;
};
template <class U>
struct types_have_common_values<Uint64T, U> {
  static const bool value = types_have_common_values<Word64T, U>::value;
};
template <class U>
struct types_have_common_values<Int64T, U> {
  static const bool value = types_have_common_values<Word64T, U>::value;
};
template <class U>
struct types_have_common_values<IntPtrT, U> {
  static const bool value = types_have_common_values<WordT, U>::value;
};
template <class U>
struct types_have_common_values<UintPtrT, U> {
  static const bool value = types_have_common_values<WordT, U>::value;
};
template <class T1, class T2, class U>
struct types_have_common_values<UnionT<T1, T2>, U> {
  static const bool value = types_have_common_values<T1, U>::value ||
                            types_have_common_values<T2, U>::value;
};

template <class T, class U1, class U2>
struct types_have_common_values<T, UnionT<U1, U2>> {
  static const bool value = types_have_common_values<T, U1>::value ||
                            types_have_common_values<T, U2>::value;
};
template <class T1, class T2, class U1, class U2>
struct types_have_common_values<UnionT<T1, T2>, UnionT<U1, U2>> {
  static const bool value = types_have_common_values<T1, U1>::value ||
                            types_have_common_values<T1, U2>::value ||
                            types_have_common_values<T2, U1>::value ||
                            types_have_common_values<T2, U2>::value;
};

458 459 460 461 462 463 464 465 466 467
template <class T>
struct types_have_common_values<T, MaybeObject> {
  static const bool value = types_have_common_values<T, Object>::value;
};

template <class T>
struct types_have_common_values<MaybeObject, T> {
  static const bool value = types_have_common_values<Object, T>::value;
};

468 469 470 471 472 473 474 475 476
// TNode<T> is an SSA value with the static type tag T, which is one of the
// following:
//   - a subclass of internal::Object represents a tagged type
//   - a subclass of internal::UntaggedT represents an untagged type
//   - ExternalReference
//   - PairT<T1, T2> for an operation returning two values, with types T1
//     and T2
//   - UnionT<T1, T2> represents either a value of type T1 or of type T2.
template <class T>
477 478
class TNode {
 public:
479 480
  template <class U,
            typename std::enable_if<is_subtype<U, T>::value, int>::type = 0>
481 482 483 484
  TNode(const TNode<U>& other) : node_(other) {
    LazyTemplateChecks();
  }
  TNode() : TNode(nullptr) {}
485 486

  TNode operator=(TNode other) {
487
    DCHECK_NOT_NULL(other.node_);
488 489 490 491 492 493 494 495 496
    node_ = other.node_;
    return *this;
  }

  operator compiler::Node*() const { return node_; }

  static TNode UncheckedCast(compiler::Node* node) { return TNode(node); }

 protected:
497
  explicit TNode(compiler::Node* node) : node_(node) { LazyTemplateChecks(); }
498 499

 private:
500 501 502 503 504
  // These checks shouldn't be checked before TNode is actually used.
  void LazyTemplateChecks() {
    static_assert(is_valid_type_tag<T>::value, "invalid type tag");
  }

505 506 507
  compiler::Node* node_;
};

508
// SloppyTNode<T> is a variant of TNode<T> and allows implicit casts from
509 510 511
// Node*. It is intended for function arguments as long as some call sites
// still use untyped Node* arguments.
// TODO(tebbi): Delete this class once transition is finished.
512 513
template <class T>
class SloppyTNode : public TNode<T> {
514 515
 public:
  SloppyTNode(compiler::Node* node)  // NOLINT(runtime/explicit)
516 517
      : TNode<T>(node) {}
  template <class U, typename std::enable_if<is_subtype<U, T>::value,
518
                                             int>::type = 0>
519 520
  SloppyTNode(const TNode<U>& other)  // NOLINT(runtime/explicit)
      : TNode<T>(other) {}
521 522
};

523 524 525
template <class... Types>
class CodeAssemblerParameterizedLabel;

526 527 528
// This macro alias allows to use PairT<T1, T2> as a macro argument.
#define PAIR_TYPE(T1, T2) PairT<T1, T2>

529 530 531 532 533 534 535
#define CODE_ASSEMBLER_COMPARE_BINARY_OP_LIST(V)          \
  V(Float32Equal, BoolT, Float32T, Float32T)              \
  V(Float32LessThan, BoolT, Float32T, Float32T)           \
  V(Float32LessThanOrEqual, BoolT, Float32T, Float32T)    \
  V(Float32GreaterThan, BoolT, Float32T, Float32T)        \
  V(Float32GreaterThanOrEqual, BoolT, Float32T, Float32T) \
  V(Float64Equal, BoolT, Float64T, Float64T)              \
536
  V(Float64NotEqual, BoolT, Float64T, Float64T)           \
537 538 539 540
  V(Float64LessThan, BoolT, Float64T, Float64T)           \
  V(Float64LessThanOrEqual, BoolT, Float64T, Float64T)    \
  V(Float64GreaterThan, BoolT, Float64T, Float64T)        \
  V(Float64GreaterThanOrEqual, BoolT, Float64T, Float64T) \
541
  /* Use Word32Equal if you need Int32Equal */            \
542 543 544 545
  V(Int32GreaterThan, BoolT, Word32T, Word32T)            \
  V(Int32GreaterThanOrEqual, BoolT, Word32T, Word32T)     \
  V(Int32LessThan, BoolT, Word32T, Word32T)               \
  V(Int32LessThanOrEqual, BoolT, Word32T, Word32T)        \
546
  /* Use WordEqual if you need IntPtrEqual */             \
547 548 549 550
  V(IntPtrLessThan, BoolT, WordT, WordT)                  \
  V(IntPtrLessThanOrEqual, BoolT, WordT, WordT)           \
  V(IntPtrGreaterThan, BoolT, WordT, WordT)               \
  V(IntPtrGreaterThanOrEqual, BoolT, WordT, WordT)        \
551
  /* Use Word32Equal if you need Uint32Equal */           \
552 553
  V(Uint32LessThan, BoolT, Word32T, Word32T)              \
  V(Uint32LessThanOrEqual, BoolT, Word32T, Word32T)       \
554
  V(Uint32GreaterThan, BoolT, Word32T, Word32T)           \
555
  V(Uint32GreaterThanOrEqual, BoolT, Word32T, Word32T)    \
556
  /* Use WordEqual if you need UintPtrEqual */            \
557 558 559
  V(UintPtrLessThan, BoolT, WordT, WordT)                 \
  V(UintPtrLessThanOrEqual, BoolT, WordT, WordT)          \
  V(UintPtrGreaterThan, BoolT, WordT, WordT)              \
560
  V(UintPtrGreaterThanOrEqual, BoolT, WordT, WordT)
561

562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579
#define CODE_ASSEMBLER_BINARY_OP_LIST(V)                                \
  CODE_ASSEMBLER_COMPARE_BINARY_OP_LIST(V)                              \
  V(Float64Add, Float64T, Float64T, Float64T)                           \
  V(Float64Sub, Float64T, Float64T, Float64T)                           \
  V(Float64Mul, Float64T, Float64T, Float64T)                           \
  V(Float64Div, Float64T, Float64T, Float64T)                           \
  V(Float64Mod, Float64T, Float64T, Float64T)                           \
  V(Float64Atan2, Float64T, Float64T, Float64T)                         \
  V(Float64Pow, Float64T, Float64T, Float64T)                           \
  V(Float64Max, Float64T, Float64T, Float64T)                           \
  V(Float64Min, Float64T, Float64T, Float64T)                           \
  V(Float64InsertLowWord32, Float64T, Float64T, Word32T)                \
  V(Float64InsertHighWord32, Float64T, Float64T, Word32T)               \
  V(IntPtrAddWithOverflow, PAIR_TYPE(IntPtrT, BoolT), IntPtrT, IntPtrT) \
  V(IntPtrSubWithOverflow, PAIR_TYPE(IntPtrT, BoolT), IntPtrT, IntPtrT) \
  V(Int32Add, Word32T, Word32T, Word32T)                                \
  V(Int32AddWithOverflow, PAIR_TYPE(Int32T, BoolT), Int32T, Int32T)     \
  V(Int32Sub, Word32T, Word32T, Word32T)                                \
580
  V(Int32SubWithOverflow, PAIR_TYPE(Int32T, BoolT), Int32T, Int32T)     \
581 582 583 584 585 586
  V(Int32Mul, Word32T, Word32T, Word32T)                                \
  V(Int32MulWithOverflow, PAIR_TYPE(Int32T, BoolT), Int32T, Int32T)     \
  V(Int32Div, Int32T, Int32T, Int32T)                                   \
  V(Int32Mod, Int32T, Int32T, Int32T)                                   \
  V(WordRor, WordT, WordT, IntegralT)                                   \
  V(Word32Ror, Word32T, Word32T, Word32T)                               \
587 588 589 590
  V(Word64Ror, Word64T, Word64T, Word64T)

TNode<Float64T> Float64Add(TNode<Float64T> a, TNode<Float64T> b);

591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616
#define CODE_ASSEMBLER_UNARY_OP_LIST(V)                        \
  V(Float64Abs, Float64T, Float64T)                            \
  V(Float64Acos, Float64T, Float64T)                           \
  V(Float64Acosh, Float64T, Float64T)                          \
  V(Float64Asin, Float64T, Float64T)                           \
  V(Float64Asinh, Float64T, Float64T)                          \
  V(Float64Atan, Float64T, Float64T)                           \
  V(Float64Atanh, Float64T, Float64T)                          \
  V(Float64Cos, Float64T, Float64T)                            \
  V(Float64Cosh, Float64T, Float64T)                           \
  V(Float64Exp, Float64T, Float64T)                            \
  V(Float64Expm1, Float64T, Float64T)                          \
  V(Float64Log, Float64T, Float64T)                            \
  V(Float64Log1p, Float64T, Float64T)                          \
  V(Float64Log2, Float64T, Float64T)                           \
  V(Float64Log10, Float64T, Float64T)                          \
  V(Float64Cbrt, Float64T, Float64T)                           \
  V(Float64Neg, Float64T, Float64T)                            \
  V(Float64Sin, Float64T, Float64T)                            \
  V(Float64Sinh, Float64T, Float64T)                           \
  V(Float64Sqrt, Float64T, Float64T)                           \
  V(Float64Tan, Float64T, Float64T)                            \
  V(Float64Tanh, Float64T, Float64T)                           \
  V(Float64ExtractLowWord32, Word32T, Float64T)                \
  V(Float64ExtractHighWord32, Word32T, Float64T)               \
  V(BitcastTaggedToWord, IntPtrT, Object)                      \
617
  V(BitcastTaggedSignedToWord, IntPtrT, Smi)                   \
618
  V(BitcastMaybeObjectToWord, IntPtrT, MaybeObject)            \
619 620 621 622 623 624 625 626 627 628 629 630
  V(BitcastWordToTagged, Object, WordT)                        \
  V(BitcastWordToTaggedSigned, Smi, WordT)                     \
  V(TruncateFloat64ToFloat32, Float32T, Float64T)              \
  V(TruncateFloat64ToWord32, Word32T, Float64T)                \
  V(TruncateInt64ToInt32, Int32T, Int64T)                      \
  V(ChangeFloat32ToFloat64, Float64T, Float32T)                \
  V(ChangeFloat64ToUint32, Uint32T, Float64T)                  \
  V(ChangeFloat64ToUint64, Uint64T, Float64T)                  \
  V(ChangeInt32ToFloat64, Float64T, Int32T)                    \
  V(ChangeInt32ToInt64, Int64T, Int32T)                        \
  V(ChangeUint32ToFloat64, Float64T, Word32T)                  \
  V(ChangeUint32ToUint64, Uint64T, Word32T)                    \
631
  V(BitcastInt32ToFloat32, Float32T, Word32T)                  \
632
  V(BitcastFloat32ToInt32, Word32T, Float32T)                  \
633 634 635 636 637 638 639 640
  V(RoundFloat64ToInt32, Int32T, Float64T)                     \
  V(RoundInt32ToFloat32, Int32T, Float32T)                     \
  V(Float64SilenceNaN, Float64T, Float64T)                     \
  V(Float64RoundDown, Float64T, Float64T)                      \
  V(Float64RoundUp, Float64T, Float64T)                        \
  V(Float64RoundTiesEven, Float64T, Float64T)                  \
  V(Float64RoundTruncate, Float64T, Float64T)                  \
  V(Word32Clz, Int32T, Word32T)                                \
641
  V(Word32BitwiseNot, Word32T, Word32T)                        \
642
  V(WordNot, WordT, WordT)                                     \
643 644 645
  V(Int32AbsWithOverflow, PAIR_TYPE(Int32T, BoolT), Int32T)    \
  V(Int64AbsWithOverflow, PAIR_TYPE(Int64T, BoolT), Int64T)    \
  V(IntPtrAbsWithOverflow, PAIR_TYPE(IntPtrT, BoolT), IntPtrT) \
646
  V(Word32BinaryNot, BoolT, Word32T)
647 648

// A "public" interface used by components outside of compiler directory to
649 650 651 652
// create code objects with TurboFan's backend. This class is mostly a thin
// shim around the RawMachineAssembler, and its primary job is to ensure that
// the innards of the RawMachineAssembler and other compiler implementation
// details don't leak outside of the the compiler directory..
653 654
//
// V8 components that need to generate low-level code using this interface
655 656 657 658
// should include this header--and this header only--from the compiler
// directory (this is actually enforced). Since all interesting data
// structures are forward declared, it's not possible for clients to peek
// inside the compiler internals.
659 660 661 662 663
//
// In addition to providing isolation between TurboFan and code generation
// clients, CodeAssembler also provides an abstraction for creating variables
// and enhanced Label functionality to merge variable values along paths where
// they have differing values, including loops.
664 665 666 667
//
// The CodeAssembler itself is stateless (and instances are expected to be
// temporary-scoped and short-lived); all its state is encapsulated into
// a CodeAssemblerState instance.
668
class V8_EXPORT_PRIVATE CodeAssembler {
669
 public:
670
  explicit CodeAssembler(CodeAssemblerState* state) : state_(state) {}
671
  ~CodeAssembler();
672

673 674
  static Handle<Code> GenerateCode(CodeAssemblerState* state,
                                   const AssemblerOptions& options);
675 676 677 678

  bool Is64() const;
  bool IsFloat64RoundUpSupported() const;
  bool IsFloat64RoundDownSupported() const;
679
  bool IsFloat64RoundTiesEvenSupported() const;
680
  bool IsFloat64RoundTruncateSupported() const;
681 682 683
  bool IsInt32AbsWithOverflowSupported() const;
  bool IsInt64AbsWithOverflowSupported() const;
  bool IsIntPtrAbsWithOverflowSupported() const;
684

685
  // Shortened aliases for use in CodeAssembler subclasses.
686 687 688 689 690
  using Label = CodeAssemblerLabel;
  using Variable = CodeAssemblerVariable;
  template <class T>
  using TVariable = TypedCodeAssemblerVariable<T>;
  using VariableList = CodeAssemblerVariableList;
691

692 693 694 695
  // ===========================================================================
  // Base Assembler
  // ===========================================================================

696
  template <class PreviousType, bool FromTyped>
697 698 699 700 701 702 703 704 705 706 707 708
  class CheckedNode {
   public:
#ifdef DEBUG
    CheckedNode(Node* node, CodeAssembler* code_assembler, const char* location)
        : node_(node), code_assembler_(code_assembler), location_(location) {}
#else
    CheckedNode(compiler::Node* node, CodeAssembler*, const char*)
        : node_(node) {}
#endif

    template <class A>
    operator TNode<A>() {
709 710 711 712
      static_assert(
          !std::is_same<A, MaybeObject>::value,
          "Can't cast to MaybeObject, use explicit conversion functions. ");

713 714
      static_assert(types_have_common_values<A, PreviousType>::value,
                    "Incompatible types: this cast can never succeed.");
715
      static_assert(std::is_convertible<TNode<A>, TNode<Object>>::value,
716 717
                    "Coercion to untagged values cannot be "
                    "checked.");
718 719 720 721
      static_assert(
          !FromTyped ||
              !std::is_convertible<TNode<PreviousType>, TNode<A>>::value,
          "Unnecessary CAST: types are convertible.");
722 723
#ifdef DEBUG
      if (FLAG_debug_code) {
724 725 726
        if (std::is_same<PreviousType, MaybeObject>::value) {
          code_assembler_->GenerateCheckMaybeObjectIsObject(node_, location_);
        }
727
        Node* function = code_assembler_->ExternalConstant(
728
            ExternalReference::check_object_type());
729 730 731 732 733 734 735 736
        code_assembler_->CallCFunction(
            function, MachineType::AnyTagged(),
            std::make_pair(MachineType::AnyTagged(), node_),
            std::make_pair(MachineType::TaggedSigned(),
                           code_assembler_->SmiConstant(
                               static_cast<int>(ObjectTypeOf<A>::value))),
            std::make_pair(MachineType::AnyTagged(),
                           code_assembler_->StringConstant(location_)));
737 738 739 740 741 742 743
      }
#endif
      return TNode<A>::UncheckedCast(node_);
    }

    template <class A>
    operator SloppyTNode<A>() {
744
      return implicit_cast<TNode<A>>(*this);
745 746 747 748 749 750 751 752 753 754 755 756
    }

    Node* node() const { return node_; }

   private:
    Node* node_;
#ifdef DEBUG
    CodeAssembler* code_assembler_;
    const char* location_;
#endif
  };

757 758 759
  template <class T>
  TNode<T> UncheckedCast(Node* value) {
    return TNode<T>::UncheckedCast(value);
760
  }
761 762 763 764 765 766 767 768 769 770 771 772 773 774
  template <class T, class U>
  TNode<T> UncheckedCast(TNode<U> value) {
    static_assert(types_have_common_values<T, U>::value,
                  "Incompatible types: this cast can never succeed.");
    return TNode<T>::UncheckedCast(value);
  }

  // ReinterpretCast<T>(v) has the power to cast even when the type of v is
  // unrelated to T. Use with care.
  template <class T>
  TNode<T> ReinterpretCast(Node* value) {
    return TNode<T>::UncheckedCast(value);
  }

775
  CheckedNode<Object, false> Cast(Node* value, const char* location = "") {
776
    return {value, this, location};
777 778 779
  }

  template <class T>
780
  CheckedNode<T, true> Cast(TNode<T> value, const char* location = "") {
781
    return {value, this, location};
782 783 784 785 786 787 788
  }

#ifdef DEBUG
#define STRINGIFY(x) #x
#define TO_STRING_LITERAL(x) STRINGIFY(x)
#define CAST(x) \
  Cast(x, "CAST(" #x ") at " __FILE__ ":" TO_STRING_LITERAL(__LINE__))
789 790
#define TORQUE_CAST(x) \
  ca_.Cast(x, "CAST(" #x ") at " __FILE__ ":" TO_STRING_LITERAL(__LINE__))
791
#else
792
#define CAST(x) Cast(x)
793
#define TORQUE_CAST(x) ca_.Cast(x)
794 795
#endif

796 797 798 799
#ifdef DEBUG
  void GenerateCheckMaybeObjectIsObject(Node* node, const char* location);
#endif

800
  // Constants.
801 802 803
  TNode<Int32T> Int32Constant(int32_t value);
  TNode<Int64T> Int64Constant(int64_t value);
  TNode<IntPtrT> IntPtrConstant(intptr_t value);
804 805 806
  TNode<Uint32T> Uint32Constant(uint32_t value) {
    return Unsigned(Int32Constant(bit_cast<int32_t>(value)));
  }
807 808 809
  TNode<UintPtrT> UintPtrConstant(uintptr_t value) {
    return Unsigned(IntPtrConstant(bit_cast<intptr_t>(value)));
  }
810 811 812
  TNode<RawPtrT> PointerConstant(void* value) {
    return ReinterpretCast<RawPtrT>(IntPtrConstant(bit_cast<intptr_t>(value)));
  }
813
  TNode<Number> NumberConstant(double value);
814
  TNode<Smi> SmiConstant(Smi value);
815
  TNode<Smi> SmiConstant(int value);
816 817 818 819 820 821
  template <typename E,
            typename = typename std::enable_if<std::is_enum<E>::value>::type>
  TNode<Smi> SmiConstant(E value) {
    STATIC_ASSERT(sizeof(E) <= sizeof(int));
    return SmiConstant(static_cast<int>(value));
  }
822 823 824 825 826 827 828 829 830 831
  TNode<HeapObject> UntypedHeapConstant(Handle<HeapObject> object);
  template <class Type>
  TNode<Type> HeapConstant(Handle<Type> object) {
    return UncheckedCast<Type>(UntypedHeapConstant(object));
  }
  TNode<String> StringConstant(const char* str);
  TNode<Oddball> BooleanConstant(bool value);
  TNode<ExternalReference> ExternalConstant(ExternalReference address);
  TNode<Float64T> Float64Constant(double value);
  TNode<HeapNumber> NaNConstant();
832 833 834 835 836 837
  TNode<BoolT> Int32TrueConstant() {
    return ReinterpretCast<BoolT>(Int32Constant(1));
  }
  TNode<BoolT> Int32FalseConstant() {
    return ReinterpretCast<BoolT>(Int32Constant(0));
  }
838 839 840
  TNode<BoolT> BoolConstant(bool value) {
    return value ? Int32TrueConstant() : Int32FalseConstant();
  }
841

842 843
  // TODO(jkummerow): The style guide wants pointers for output parameters.
  // https://google.github.io/styleguide/cppguide.html#Output_Parameters
844 845
  bool ToInt32Constant(Node* node, int32_t& out_value);
  bool ToInt64Constant(Node* node, int64_t& out_value);
846
  bool ToSmiConstant(Node* node, Smi* out_value);
847 848
  bool ToIntPtrConstant(Node* node, intptr_t& out_value);

849 850 851
  bool IsUndefinedConstant(TNode<Object> node);
  bool IsNullConstant(TNode<Object> node);

852 853 854 855 856 857 858 859 860
  TNode<Int32T> Signed(TNode<Word32T> x) { return UncheckedCast<Int32T>(x); }
  TNode<IntPtrT> Signed(TNode<WordT> x) { return UncheckedCast<IntPtrT>(x); }
  TNode<Uint32T> Unsigned(TNode<Word32T> x) {
    return UncheckedCast<Uint32T>(x);
  }
  TNode<UintPtrT> Unsigned(TNode<WordT> x) {
    return UncheckedCast<UintPtrT>(x);
  }

861 862
  static constexpr int kTargetParameterIndex = -1;

863
  Node* Parameter(int value);
864 865 866 867 868 869

  TNode<Context> GetJSContextParameter();
  void Return(SloppyTNode<Object> value);
  void Return(SloppyTNode<Object> value1, SloppyTNode<Object> value2);
  void Return(SloppyTNode<Object> value1, SloppyTNode<Object> value2,
              SloppyTNode<Object> value3);
870
  void PopAndReturn(Node* pop, Node* value);
871

872 873
  void ReturnIf(Node* condition, Node* value);

874 875
  void ReturnRaw(Node* value);

876
  void DebugAbort(Node* message);
877
  void DebugBreak();
878
  void Unreachable();
879 880 881 882 883 884 885 886 887 888 889 890
  void Comment(const char* msg) {
    if (!FLAG_code_comments) return;
    Comment(std::string(msg));
  }
  void Comment(std::string msg);
  template <class... Args>
  void Comment(Args&&... args) {
    if (!FLAG_code_comments) return;
    std::ostringstream s;
    USE((s << std::forward<Args>(args))...);
    Comment(s.str());
  }
891

892 893
  void StaticAssert(TNode<BoolT> value);

894 895
  void SetSourcePosition(const char* file, int line);

896
  void Bind(Label* label);
897 898 899
#if DEBUG
  void Bind(Label* label, AssemblerDebugInfo debug_info);
#endif  // DEBUG
900
  void Goto(Label* label);
901 902 903 904
  void GotoIf(SloppyTNode<IntegralT> condition, Label* true_label);
  void GotoIfNot(SloppyTNode<IntegralT> condition, Label* false_label);
  void Branch(SloppyTNode<IntegralT> condition, Label* true_label,
              Label* false_label);
905

906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923
  template <class T>
  TNode<T> Uninitialized() {
    return {};
  }

  template <class... T>
  void Bind(CodeAssemblerParameterizedLabel<T...>* label, TNode<T>*... phis) {
    Bind(label->plain_label());
    label->CreatePhis(phis...);
  }
  template <class... T, class... Args>
  void Branch(TNode<BoolT> condition,
              CodeAssemblerParameterizedLabel<T...>* if_true,
              CodeAssemblerParameterizedLabel<T...>* if_false, Args... args) {
    if_true->AddInputs(args...);
    if_false->AddInputs(args...);
    Branch(condition, if_true->plain_label(), if_false->plain_label());
  }
924 925

  template <class... T, class... Args>
926
  void Goto(CodeAssemblerParameterizedLabel<T...>* label, Args... args) {
927 928 929 930
    label->AddInputs(args...);
    Goto(label->plain_label());
  }

931 932
  void Branch(TNode<BoolT> condition, const std::function<void()>& true_body,
              const std::function<void()>& false_body);
933
  void Branch(TNode<BoolT> condition, Label* true_label,
934 935
              const std::function<void()>& false_body);
  void Branch(TNode<BoolT> condition, const std::function<void()>& true_body,
936
              Label* false_label);
937

938
  void Switch(Node* index, Label* default_label, const int32_t* case_values,
939 940 941 942 943 944 945 946 947
              Label** case_labels, size_t case_count);

  // Access to the frame pointer
  Node* LoadFramePointer();
  Node* LoadParentFramePointer();

  // Access to the stack pointer
  Node* LoadStackPointer();

948
  // Poison |value| on speculative paths.
949 950
  TNode<Object> TaggedPoisonOnSpeculation(SloppyTNode<Object> value);
  TNode<WordT> WordPoisonOnSpeculation(SloppyTNode<WordT> value);
951

952
  // Load raw memory location.
953
  Node* Load(MachineType type, Node* base,
954
             LoadSensitivity needs_poisoning = LoadSensitivity::kSafe);
955
  template <class Type>
956
  TNode<Type> Load(MachineType type, TNode<RawPtr<Type>> base) {
957
    DCHECK(
958 959
        IsSubtype(type.representation(), MachineRepresentationOf<Type>::value));
    return UncheckedCast<Type>(Load(type, static_cast<Node*>(base)));
960
  }
961
  Node* Load(MachineType type, Node* base, Node* offset,
962
             LoadSensitivity needs_poisoning = LoadSensitivity::kSafe);
963
  Node* AtomicLoad(MachineType type, Node* base, Node* offset);
964 965 966 967 968 969 970
  // Load uncompressed tagged value from (most likely off JS heap) memory
  // location.
  Node* LoadFullTagged(
      Node* base, LoadSensitivity needs_poisoning = LoadSensitivity::kSafe);
  Node* LoadFullTagged(
      Node* base, Node* offset,
      LoadSensitivity needs_poisoning = LoadSensitivity::kSafe);
971

972 973 974
  Node* LoadFromObject(MachineType type, TNode<HeapObject> object,
                       TNode<IntPtrT> offset);

975
  // Load a value from the root array.
976
  TNode<Object> LoadRoot(RootIndex root_index);
977

978
  // Store value to raw memory location.
979 980
  Node* Store(Node* base, Node* value);
  Node* Store(Node* base, Node* offset, Node* value);
981
  Node* StoreEphemeronKey(Node* base, Node* offset, Node* value);
982
  Node* StoreNoWriteBarrier(MachineRepresentation rep, Node* base, Node* value);
983
  Node* StoreNoWriteBarrier(MachineRepresentation rep, Node* base, Node* offset,
984
                            Node* value);
985 986 987 988 989
  Node* UnsafeStoreNoWriteBarrier(MachineRepresentation rep, Node* base,
                                  Node* value);
  Node* UnsafeStoreNoWriteBarrier(MachineRepresentation rep, Node* base,
                                  Node* offset, Node* value);

990 991 992 993 994 995
  // Stores uncompressed tagged value to (most likely off JS heap) memory
  // location without write barrier.
  Node* StoreFullTaggedNoWriteBarrier(Node* base, Node* tagged_value);
  Node* StoreFullTaggedNoWriteBarrier(Node* base, Node* offset,
                                      Node* tagged_value);

996 997
  // Optimized memory operations that map to Turbofan simplified nodes.
  TNode<HeapObject> OptimizedAllocate(TNode<IntPtrT> size,
998 999
                                      AllocationType allocation,
                                      AllowLargeObjects allow_large_objects);
1000 1001 1002
  void StoreToObject(MachineRepresentation rep, TNode<HeapObject> object,
                     TNode<IntPtrT> offset, Node* value,
                     StoreToObjectWriteBarrier write_barrier);
1003
  void OptimizedStoreField(MachineRepresentation rep, TNode<HeapObject> object,
1004
                           int offset, Node* value);
1005 1006 1007 1008 1009 1010
  void OptimizedStoreFieldAssertNoWriteBarrier(MachineRepresentation rep,
                                               TNode<HeapObject> object,
                                               int offset, Node* value);
  void OptimizedStoreFieldUnsafeNoWriteBarrier(MachineRepresentation rep,
                                               TNode<HeapObject> object,
                                               int offset, Node* value);
1011
  void OptimizedStoreMap(TNode<HeapObject> object, TNode<Map>);
1012 1013
  // {value_high} is used for 64-bit stores on 32-bit platforms, must be
  // nullptr in other cases.
1014
  Node* AtomicStore(MachineRepresentation rep, Node* base, Node* offset,
1015
                    Node* value, Node* value_high = nullptr);
1016

1017
  // Exchange value at raw memory location
1018 1019
  Node* AtomicExchange(MachineType type, Node* base, Node* offset, Node* value,
                       Node* value_high = nullptr);
1020

1021 1022
  // Compare and Exchange value at raw memory location
  Node* AtomicCompareExchange(MachineType type, Node* base, Node* offset,
1023 1024 1025
                              Node* old_value, Node* new_value,
                              Node* old_value_high = nullptr,
                              Node* new_value_high = nullptr);
1026

1027 1028
  Node* AtomicAdd(MachineType type, Node* base, Node* offset, Node* value,
                  Node* value_high = nullptr);
1029

1030 1031
  Node* AtomicSub(MachineType type, Node* base, Node* offset, Node* value,
                  Node* value_high = nullptr);
1032

1033 1034
  Node* AtomicAnd(MachineType type, Node* base, Node* offset, Node* value,
                  Node* value_high = nullptr);
1035

1036 1037
  Node* AtomicOr(MachineType type, Node* base, Node* offset, Node* value,
                 Node* value_high = nullptr);
1038

1039 1040
  Node* AtomicXor(MachineType type, Node* base, Node* offset, Node* value,
                  Node* value_high = nullptr);
1041

1042
  // Store a value to the root array.
1043
  Node* StoreRoot(RootIndex root_index, Node* value);
1044

1045
// Basic arithmetic operations.
1046 1047
#define DECLARE_CODE_ASSEMBLER_BINARY_OP(name, ResType, Arg1Type, Arg2Type) \
  TNode<ResType> name(SloppyTNode<Arg1Type> a, SloppyTNode<Arg2Type> b);
1048 1049 1050
  CODE_ASSEMBLER_BINARY_OP_LIST(DECLARE_CODE_ASSEMBLER_BINARY_OP)
#undef DECLARE_CODE_ASSEMBLER_BINARY_OP

1051 1052 1053 1054
  TNode<IntPtrT> WordShr(TNode<IntPtrT> left, TNode<IntegralT> right) {
    return UncheckedCast<IntPtrT>(
        WordShr(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
1055 1056 1057 1058
  TNode<IntPtrT> WordSar(TNode<IntPtrT> left, TNode<IntegralT> right) {
    return UncheckedCast<IntPtrT>(
        WordSar(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
1059

1060 1061 1062 1063 1064 1065 1066
  TNode<IntPtrT> WordAnd(TNode<IntPtrT> left, TNode<IntPtrT> right) {
    return UncheckedCast<IntPtrT>(
        WordAnd(static_cast<Node*>(left), static_cast<Node*>(right)));
  }

  template <class Left, class Right,
            class = typename std::enable_if<
1067 1068
                std::is_base_of<Object, Left>::value &&
                std::is_base_of<Object, Right>::value>::type>
1069
  TNode<BoolT> WordEqual(TNode<Left> left, TNode<Right> right) {
1070 1071
    return WordEqual(ReinterpretCast<WordT>(left),
                     ReinterpretCast<WordT>(right));
1072 1073
  }
  TNode<BoolT> WordEqual(TNode<Object> left, Node* right) {
1074 1075
    return WordEqual(ReinterpretCast<WordT>(left),
                     ReinterpretCast<WordT>(right));
1076 1077
  }
  TNode<BoolT> WordEqual(Node* left, TNode<Object> right) {
1078 1079
    return WordEqual(ReinterpretCast<WordT>(left),
                     ReinterpretCast<WordT>(right));
1080 1081 1082
  }
  template <class Left, class Right,
            class = typename std::enable_if<
1083 1084
                std::is_base_of<Object, Left>::value &&
                std::is_base_of<Object, Right>::value>::type>
1085
  TNode<BoolT> WordNotEqual(TNode<Left> left, TNode<Right> right) {
1086 1087
    return WordNotEqual(ReinterpretCast<WordT>(left),
                        ReinterpretCast<WordT>(right));
1088 1089
  }
  TNode<BoolT> WordNotEqual(TNode<Object> left, Node* right) {
1090 1091
    return WordNotEqual(ReinterpretCast<WordT>(left),
                        ReinterpretCast<WordT>(right));
1092 1093
  }
  TNode<BoolT> WordNotEqual(Node* left, TNode<Object> right) {
1094 1095
    return WordNotEqual(ReinterpretCast<WordT>(left),
                        ReinterpretCast<WordT>(right));
1096 1097
  }

1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
  TNode<BoolT> IntPtrEqual(SloppyTNode<WordT> left, SloppyTNode<WordT> right);
  TNode<BoolT> WordEqual(SloppyTNode<WordT> left, SloppyTNode<WordT> right);
  TNode<BoolT> WordNotEqual(SloppyTNode<WordT> left, SloppyTNode<WordT> right);
  TNode<BoolT> Word32Equal(SloppyTNode<Word32T> left,
                           SloppyTNode<Word32T> right);
  TNode<BoolT> Word32NotEqual(SloppyTNode<Word32T> left,
                              SloppyTNode<Word32T> right);
  TNode<BoolT> Word64Equal(SloppyTNode<Word64T> left,
                           SloppyTNode<Word64T> right);
  TNode<BoolT> Word64NotEqual(SloppyTNode<Word64T> left,
                              SloppyTNode<Word64T> right);

1110 1111 1112 1113 1114
  TNode<Int32T> Int32Add(TNode<Int32T> left, TNode<Int32T> right) {
    return Signed(
        Int32Add(static_cast<Node*>(left), static_cast<Node*>(right)));
  }

1115 1116 1117 1118 1119
  TNode<Uint32T> Uint32Add(TNode<Uint32T> left, TNode<Uint32T> right) {
    return Unsigned(
        Int32Add(static_cast<Node*>(left), static_cast<Node*>(right)));
  }

1120
  TNode<WordT> IntPtrAdd(SloppyTNode<WordT> left, SloppyTNode<WordT> right);
1121
  TNode<IntPtrT> IntPtrDiv(TNode<IntPtrT> left, TNode<IntPtrT> right);
1122
  TNode<WordT> IntPtrSub(SloppyTNode<WordT> left, SloppyTNode<WordT> right);
1123
  TNode<WordT> IntPtrMul(SloppyTNode<WordT> left, SloppyTNode<WordT> right);
1124 1125 1126 1127 1128 1129 1130 1131
  TNode<IntPtrT> IntPtrAdd(TNode<IntPtrT> left, TNode<IntPtrT> right) {
    return Signed(
        IntPtrAdd(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
  TNode<IntPtrT> IntPtrSub(TNode<IntPtrT> left, TNode<IntPtrT> right) {
    return Signed(
        IntPtrSub(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
1132 1133 1134 1135
  TNode<IntPtrT> IntPtrMul(TNode<IntPtrT> left, TNode<IntPtrT> right) {
    return Signed(
        IntPtrMul(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
1136 1137 1138 1139 1140 1141 1142 1143
  TNode<UintPtrT> UintPtrAdd(TNode<UintPtrT> left, TNode<UintPtrT> right) {
    return Unsigned(
        IntPtrAdd(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
  TNode<UintPtrT> UintPtrSub(TNode<UintPtrT> left, TNode<UintPtrT> right) {
    return Unsigned(
        IntPtrSub(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
1144 1145 1146 1147 1148 1149
  TNode<RawPtrT> RawPtrAdd(TNode<RawPtrT> left, TNode<IntPtrT> right) {
    return ReinterpretCast<RawPtrT>(IntPtrAdd(left, right));
  }
  TNode<RawPtrT> RawPtrAdd(TNode<IntPtrT> left, TNode<RawPtrT> right) {
    return ReinterpretCast<RawPtrT>(IntPtrAdd(left, right));
  }
1150 1151 1152

  TNode<WordT> WordShl(SloppyTNode<WordT> value, int shift);
  TNode<WordT> WordShr(SloppyTNode<WordT> value, int shift);
1153
  TNode<WordT> WordSar(SloppyTNode<WordT> value, int shift);
1154 1155 1156
  TNode<IntPtrT> WordShr(TNode<IntPtrT> value, int shift) {
    return UncheckedCast<IntPtrT>(WordShr(static_cast<Node*>(value), shift));
  }
1157 1158 1159
  TNode<IntPtrT> WordSar(TNode<IntPtrT> value, int shift) {
    return UncheckedCast<IntPtrT>(WordSar(static_cast<Node*>(value), shift));
  }
1160
  TNode<Word32T> Word32Shr(SloppyTNode<Word32T> value, int shift);
1161

1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192
  TNode<WordT> WordOr(SloppyTNode<WordT> left, SloppyTNode<WordT> right);
  TNode<WordT> WordAnd(SloppyTNode<WordT> left, SloppyTNode<WordT> right);
  TNode<WordT> WordXor(SloppyTNode<WordT> left, SloppyTNode<WordT> right);
  TNode<WordT> WordShl(SloppyTNode<WordT> left, SloppyTNode<IntegralT> right);
  TNode<WordT> WordShr(SloppyTNode<WordT> left, SloppyTNode<IntegralT> right);
  TNode<WordT> WordSar(SloppyTNode<WordT> left, SloppyTNode<IntegralT> right);
  TNode<Word32T> Word32Or(SloppyTNode<Word32T> left,
                          SloppyTNode<Word32T> right);
  TNode<Word32T> Word32And(SloppyTNode<Word32T> left,
                           SloppyTNode<Word32T> right);
  TNode<Word32T> Word32Xor(SloppyTNode<Word32T> left,
                           SloppyTNode<Word32T> right);
  TNode<Word32T> Word32Shl(SloppyTNode<Word32T> left,
                           SloppyTNode<Word32T> right);
  TNode<Word32T> Word32Shr(SloppyTNode<Word32T> left,
                           SloppyTNode<Word32T> right);
  TNode<Word32T> Word32Sar(SloppyTNode<Word32T> left,
                           SloppyTNode<Word32T> right);
  TNode<Word64T> Word64Or(SloppyTNode<Word64T> left,
                          SloppyTNode<Word64T> right);
  TNode<Word64T> Word64And(SloppyTNode<Word64T> left,
                           SloppyTNode<Word64T> right);
  TNode<Word64T> Word64Xor(SloppyTNode<Word64T> left,
                           SloppyTNode<Word64T> right);
  TNode<Word64T> Word64Shl(SloppyTNode<Word64T> left,
                           SloppyTNode<Word64T> right);
  TNode<Word64T> Word64Shr(SloppyTNode<Word64T> left,
                           SloppyTNode<Word64T> right);
  TNode<Word64T> Word64Sar(SloppyTNode<Word64T> left,
                           SloppyTNode<Word64T> right);

1193
// Unary
1194 1195
#define DECLARE_CODE_ASSEMBLER_UNARY_OP(name, ResType, ArgType) \
  TNode<ResType> name(SloppyTNode<ArgType> a);
1196 1197 1198
  CODE_ASSEMBLER_UNARY_OP_LIST(DECLARE_CODE_ASSEMBLER_UNARY_OP)
#undef DECLARE_CODE_ASSEMBLER_UNARY_OP

1199 1200 1201 1202 1203 1204
  template <class Dummy = void>
  TNode<IntPtrT> BitcastTaggedToWord(TNode<Smi> node) {
    static_assert(sizeof(Dummy) < 0,
                  "Should use BitcastTaggedSignedToWord instead.");
  }

1205 1206
  // Changes a double to an inptr_t for pointer arithmetic outside of Smi range.
  // Assumes that the double can be exactly represented as an int.
1207
  TNode<UintPtrT> ChangeFloat64ToUintPtr(SloppyTNode<Float64T> value);
1208 1209
  // Same in the opposite direction.
  TNode<Float64T> ChangeUintPtrToFloat64(TNode<UintPtrT> value);
1210

1211 1212 1213 1214
  // Changes an intptr_t to a double, e.g. for storing an element index
  // outside Smi range in a HeapNumber. Lossless on 32-bit,
  // rounds on 64-bit (which doesn't affect valid element indices).
  Node* RoundIntPtrToFloat64(Node* value);
1215
  // No-op on 32-bit, otherwise zero extend.
1216
  TNode<UintPtrT> ChangeUint32ToWord(SloppyTNode<Word32T> value);
1217
  // No-op on 32-bit, otherwise sign extend.
1218
  TNode<IntPtrT> ChangeInt32ToIntPtr(SloppyTNode<Word32T> value);
1219

1220 1221 1222 1223
  // No-op that guarantees that the value is kept alive till this point even
  // if GC happens.
  Node* Retain(Node* value);

1224 1225 1226
  // Projections
  Node* Projection(int index, Node* value);

1227 1228 1229 1230 1231 1232 1233 1234
  template <int index, class T1, class T2>
  TNode<typename std::tuple_element<index, std::tuple<T1, T2>>::type>
  Projection(TNode<PairT<T1, T2>> value) {
    return UncheckedCast<
        typename std::tuple_element<index, std::tuple<T1, T2>>::type>(
        Projection(index, value));
  }

1235
  // Calls
1236
  template <class... TArgs>
1237 1238 1239
  TNode<Object> CallRuntime(Runtime::FunctionId function,
                            SloppyTNode<Object> context, TArgs... args) {
    return CallRuntimeImpl(function, context,
1240
                           {implicit_cast<SloppyTNode<Object>>(args)...});
1241
  }
1242

1243 1244 1245 1246 1247 1248 1249 1250
  template <class... TArgs>
  TNode<Object> CallRuntimeWithCEntry(Runtime::FunctionId function,
                                      TNode<Code> centry,
                                      SloppyTNode<Object> context,
                                      TArgs... args) {
    return CallRuntimeWithCEntryImpl(function, centry, context, {args...});
  }

1251
  template <class... TArgs>
1252 1253 1254 1255 1256
  void TailCallRuntime(Runtime::FunctionId function,
                       SloppyTNode<Object> context, TArgs... args) {
    int argc = static_cast<int>(sizeof...(args));
    TNode<Int32T> arity = Int32Constant(argc);
    return TailCallRuntimeImpl(function, arity, context,
1257
                               {implicit_cast<SloppyTNode<Object>>(args)...});
1258
  }
1259

1260
  template <class... TArgs>
1261 1262 1263
  void TailCallRuntime(Runtime::FunctionId function, TNode<Int32T> arity,
                       SloppyTNode<Object> context, TArgs... args) {
    return TailCallRuntimeImpl(function, arity, context,
1264
                               {implicit_cast<SloppyTNode<Object>>(args)...});
1265 1266
  }

1267
  template <class... TArgs>
1268 1269 1270
  void TailCallRuntimeWithCEntry(Runtime::FunctionId function,
                                 TNode<Code> centry, TNode<Object> context,
                                 TArgs... args) {
1271
    int argc = sizeof...(args);
1272 1273 1274
    TNode<Int32T> arity = Int32Constant(argc);
    return TailCallRuntimeWithCEntryImpl(
        function, arity, centry, context,
1275
        {implicit_cast<SloppyTNode<Object>>(args)...});
1276 1277
  }

1278 1279 1280 1281
  //
  // If context passed to CallStub is nullptr, it won't be passed to the stub.
  //

1282 1283 1284 1285
  template <class T = Object, class... TArgs>
  TNode<T> CallStub(Callable const& callable, SloppyTNode<Object> context,
                    TArgs... args) {
    TNode<Code> target = HeapConstant(callable.code());
1286
    return CallStub<T>(callable.descriptor(), target, context, args...);
1287
  }
1288

1289 1290 1291 1292
  template <class T = Object, class... TArgs>
  TNode<T> CallStub(const CallInterfaceDescriptor& descriptor,
                    SloppyTNode<Code> target, SloppyTNode<Object> context,
                    TArgs... args) {
1293 1294
    return UncheckedCast<T>(CallStubR(StubCallMode::kCallCodeObject, descriptor,
                                      1, target, context, args...));
1295 1296 1297
  }

  template <class... TArgs>
1298 1299
  Node* CallStubR(StubCallMode call_mode,
                  const CallInterfaceDescriptor& descriptor, size_t result_size,
1300
                  SloppyTNode<Object> target, SloppyTNode<Object> context,
1301
                  TArgs... args) {
1302 1303
    return CallStubRImpl(call_mode, descriptor, result_size, target, context,
                         {args...});
1304
  }
1305

1306 1307
  Node* CallStubN(StubCallMode call_mode,
                  const CallInterfaceDescriptor& descriptor, size_t result_size,
1308
                  int input_count, Node* const* inputs);
1309

1310 1311 1312 1313 1314 1315 1316 1317
  template <class T = Object, class... TArgs>
  TNode<T> CallBuiltinPointer(const CallInterfaceDescriptor& descriptor,
                              TNode<BuiltinPtr> target, TNode<Object> context,
                              TArgs... args) {
    return UncheckedCast<T>(CallStubR(StubCallMode::kCallBuiltinPointer,
                                      descriptor, 1, target, context, args...));
  }

1318
  template <class... TArgs>
1319 1320 1321
  void TailCallStub(Callable const& callable, SloppyTNode<Object> context,
                    TArgs... args) {
    TNode<Code> target = HeapConstant(callable.code());
1322 1323
    return TailCallStub(callable.descriptor(), target, context, args...);
  }
1324

1325
  template <class... TArgs>
1326 1327 1328
  void TailCallStub(const CallInterfaceDescriptor& descriptor,
                    SloppyTNode<Code> target, SloppyTNode<Object> context,
                    TArgs... args) {
1329
    return TailCallStubImpl(descriptor, target, context, {args...});
1330
  }
1331

1332
  template <class... TArgs>
1333
  Node* TailCallBytecodeDispatch(const CallInterfaceDescriptor& descriptor,
1334
                                 Node* target, TArgs... args);
1335

1336 1337
  template <class... TArgs>
  Node* TailCallStubThenBytecodeDispatch(
1338 1339 1340 1341 1342
      const CallInterfaceDescriptor& descriptor, Node* target, Node* context,
      TArgs... args) {
    return TailCallStubThenBytecodeDispatchImpl(descriptor, target, context,
                                                {args...});
  }
1343

1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355
  // Tailcalls to the given code object with JSCall linkage. The JS arguments
  // (including receiver) are supposed to be already on the stack.
  // This is a building block for implementing trampoline stubs that are
  // installed instead of code objects with JSCall linkage.
  // Note that no arguments adaption is going on here - all the JavaScript
  // arguments are left on the stack unmodified. Therefore, this tail call can
  // only be used after arguments adaptation has been performed already.
  TNode<Object> TailCallJSCode(TNode<Code> code, TNode<Context> context,
                               TNode<JSFunction> function,
                               TNode<Object> new_target,
                               TNode<Int32T> arg_count);

1356
  template <class... TArgs>
1357
  Node* CallJS(Callable const& callable, Node* context, Node* function,
1358 1359 1360 1361 1362
               Node* receiver, TArgs... args) {
    int argc = static_cast<int>(sizeof...(args));
    Node* arity = Int32Constant(argc);
    return CallStub(callable, context, function, arity, receiver, args...);
  }
1363

1364
  template <class... TArgs>
1365 1366
  Node* ConstructJSWithTarget(Callable const& callable, Node* context,
                              Node* target, Node* new_target, TArgs... args) {
1367 1368
    int argc = static_cast<int>(sizeof...(args));
    Node* arity = Int32Constant(argc);
1369
    Node* receiver = LoadRoot(RootIndex::kUndefinedValue);
1370 1371

    // Construct(target, new_target, arity, receiver, arguments...)
1372
    return CallStub(callable, context, target, new_target, arity, receiver,
1373 1374
                    args...);
  }
1375 1376 1377 1378 1379 1380
  template <class... TArgs>
  Node* ConstructJS(Callable const& callable, Node* context, Node* new_target,
                    TArgs... args) {
    return ConstructJSWithTarget(callable, context, new_target, new_target,
                                 args...);
  }
1381

1382 1383 1384
  Node* CallCFunctionN(Signature<MachineType>* signature, int input_count,
                       Node* const* inputs);

1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408
  // Type representing C function argument with type info.
  using CFunctionArg = std::pair<MachineType, Node*>;

  // Call to a C function.
  template <class... CArgs>
  Node* CallCFunction(Node* function, MachineType return_type, CArgs... cargs) {
    static_assert(v8::internal::conjunction<
                      std::is_convertible<CArgs, CFunctionArg>...>::value,
                  "invalid argument types");
    return CallCFunction(function, return_type, {cargs...});
  }

  // Call to a C function, while saving/restoring caller registers.
  template <class... CArgs>
  Node* CallCFunctionWithCallerSavedRegisters(Node* function,
                                              MachineType return_type,
                                              SaveFPRegsMode mode,
                                              CArgs... cargs) {
    static_assert(v8::internal::conjunction<
                      std::is_convertible<CArgs, CFunctionArg>...>::value,
                  "invalid argument types");
    return CallCFunctionWithCallerSavedRegisters(function, return_type, mode,
                                                 {cargs...});
  }
1409

1410 1411 1412 1413
  // Exception handling support.
  void GotoIfException(Node* node, Label* if_exception,
                       Variable* exception_var = nullptr);

1414 1415 1416 1417 1418
  // Helpers which delegate to RawMachineAssembler.
  Factory* factory() const;
  Isolate* isolate() const;
  Zone* zone() const;

1419 1420
  CodeAssemblerState* state() { return state_; }

1421 1422
  void BreakOnNode(int node_id);

1423 1424
  bool UnalignedLoadSupported(MachineRepresentation rep) const;
  bool UnalignedStoreSupported(MachineRepresentation rep) const;
1425

1426 1427
  bool IsExceptionHandlerActive() const;

1428
 protected:
1429 1430 1431 1432
  void RegisterCallGenerationCallbacks(
      const CodeAssemblerCallback& call_prologue,
      const CodeAssemblerCallback& call_epilogue);
  void UnregisterCallGenerationCallbacks();
1433

1434
  bool Word32ShiftIsSafe() const;
1435
  PoisoningMitigationLevel poisoning_level() const;
1436

1437 1438
  bool IsJSFunctionCall() const;

1439
 private:
1440 1441
  void HandleException(Node* result);

1442 1443 1444 1445 1446 1447 1448
  Node* CallCFunction(Node* function, MachineType return_type,
                      std::initializer_list<CFunctionArg> args);

  Node* CallCFunctionWithCallerSavedRegisters(
      Node* function, MachineType return_type, SaveFPRegsMode mode,
      std::initializer_list<CFunctionArg> args);

1449
  TNode<Object> CallRuntimeImpl(Runtime::FunctionId function,
1450 1451
                                TNode<Object> context,
                                std::initializer_list<TNode<Object>> args);
1452

1453 1454 1455 1456
  TNode<Object> CallRuntimeWithCEntryImpl(
      Runtime::FunctionId function, TNode<Code> centry, TNode<Object> context,
      std::initializer_list<TNode<Object>> args);

1457
  void TailCallRuntimeImpl(Runtime::FunctionId function, TNode<Int32T> arity,
1458 1459
                           TNode<Object> context,
                           std::initializer_list<TNode<Object>> args);
1460 1461 1462

  void TailCallRuntimeWithCEntryImpl(Runtime::FunctionId function,
                                     TNode<Int32T> arity, TNode<Code> centry,
1463 1464
                                     TNode<Object> context,
                                     std::initializer_list<TNode<Object>> args);
1465 1466 1467

  void TailCallStubImpl(const CallInterfaceDescriptor& descriptor,
                        TNode<Code> target, TNode<Object> context,
1468 1469 1470 1471 1472 1473
                        std::initializer_list<Node*> args);

  Node* TailCallStubThenBytecodeDispatchImpl(
      const CallInterfaceDescriptor& descriptor, Node* target, Node* context,
      std::initializer_list<Node*> args);

1474 1475 1476
  Node* CallStubRImpl(StubCallMode call_mode,
                      const CallInterfaceDescriptor& descriptor,
                      size_t result_size, Node* target,
1477 1478 1479
                      SloppyTNode<Object> context,
                      std::initializer_list<Node*> args);

1480 1481 1482 1483 1484
  // These two don't have definitions and are here only for catching use cases
  // where the cast is not necessary.
  TNode<Int32T> Signed(TNode<Int32T> x);
  TNode<Uint32T> Unsigned(TNode<Uint32T> x);

1485 1486
  RawMachineAssembler* raw_assembler() const;

1487 1488 1489 1490
  // Calls respective callback registered in the state.
  void CallPrologue();
  void CallEpilogue();

1491
  CodeAssemblerState* state_;
1492 1493 1494 1495

  DISALLOW_COPY_AND_ASSIGN(CodeAssembler);
};

1496
class V8_EXPORT_PRIVATE CodeAssemblerVariable {
1497 1498 1499
 public:
  explicit CodeAssemblerVariable(CodeAssembler* assembler,
                                 MachineRepresentation rep);
1500 1501
  CodeAssemblerVariable(CodeAssembler* assembler, MachineRepresentation rep,
                        Node* initial_value);
1502 1503 1504 1505 1506 1507 1508
#if DEBUG
  CodeAssemblerVariable(CodeAssembler* assembler, AssemblerDebugInfo debug_info,
                        MachineRepresentation rep);
  CodeAssemblerVariable(CodeAssembler* assembler, AssemblerDebugInfo debug_info,
                        MachineRepresentation rep, Node* initial_value);
#endif  // DEBUG

1509 1510 1511 1512 1513 1514 1515
  ~CodeAssemblerVariable();
  void Bind(Node* value);
  Node* value() const;
  MachineRepresentation rep() const;
  bool IsBound() const;

 private:
1516
  class Impl;
1517 1518
  friend class CodeAssemblerLabel;
  friend class CodeAssemblerState;
1519 1520
  friend std::ostream& operator<<(std::ostream&, const Impl&);
  friend std::ostream& operator<<(std::ostream&, const CodeAssemblerVariable&);
1521 1522 1523 1524
  struct ImplComparator {
    bool operator()(const CodeAssemblerVariable::Impl* a,
                    const CodeAssemblerVariable::Impl* b) const;
  };
1525 1526
  Impl* impl_;
  CodeAssemblerState* state_;
1527
  DISALLOW_COPY_AND_ASSIGN(CodeAssemblerVariable);
1528 1529
};

1530 1531 1532
std::ostream& operator<<(std::ostream&, const CodeAssemblerVariable&);
std::ostream& operator<<(std::ostream&, const CodeAssemblerVariable::Impl&);

1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552
template <class T>
class TypedCodeAssemblerVariable : public CodeAssemblerVariable {
 public:
  TypedCodeAssemblerVariable(TNode<T> initial_value, CodeAssembler* assembler)
      : CodeAssemblerVariable(assembler, MachineRepresentationOf<T>::value,
                              initial_value) {}
  explicit TypedCodeAssemblerVariable(CodeAssembler* assembler)
      : CodeAssemblerVariable(assembler, MachineRepresentationOf<T>::value) {}
#if DEBUG
  TypedCodeAssemblerVariable(AssemblerDebugInfo debug_info,
                             CodeAssembler* assembler)
      : CodeAssemblerVariable(assembler, debug_info,
                              MachineRepresentationOf<T>::value) {}
  TypedCodeAssemblerVariable(AssemblerDebugInfo debug_info,
                             TNode<T> initial_value, CodeAssembler* assembler)
      : CodeAssemblerVariable(assembler, debug_info,
                              MachineRepresentationOf<T>::value,
                              initial_value) {}
#endif  // DEBUG

1553 1554
  TNode<T> value() const {
    return TNode<T>::UncheckedCast(CodeAssemblerVariable::value());
1555 1556 1557
  }

  void operator=(TNode<T> value) { Bind(value); }
1558 1559 1560
  void operator=(const TypedCodeAssemblerVariable<T>& variable) {
    Bind(variable.value());
  }
1561 1562 1563 1564 1565

 private:
  using CodeAssemblerVariable::Bind;
};

1566
class V8_EXPORT_PRIVATE CodeAssemblerLabel {
1567 1568 1569
 public:
  enum Type { kDeferred, kNonDeferred };

1570 1571 1572 1573 1574
  explicit CodeAssemblerLabel(
      CodeAssembler* assembler,
      CodeAssemblerLabel::Type type = CodeAssemblerLabel::kNonDeferred)
      : CodeAssemblerLabel(assembler, 0, nullptr, type) {}
  CodeAssemblerLabel(
1575
      CodeAssembler* assembler,
1576 1577
      const CodeAssemblerVariableList& merged_variables,
      CodeAssemblerLabel::Type type = CodeAssemblerLabel::kNonDeferred)
1578
      : CodeAssemblerLabel(assembler, merged_variables.size(),
1579 1580
                           &(merged_variables[0]), type) {}
  CodeAssemblerLabel(
1581 1582
      CodeAssembler* assembler, size_t count,
      CodeAssemblerVariable* const* vars,
1583
      CodeAssemblerLabel::Type type = CodeAssemblerLabel::kNonDeferred);
1584 1585 1586 1587 1588
  CodeAssemblerLabel(
      CodeAssembler* assembler,
      std::initializer_list<CodeAssemblerVariable*> vars,
      CodeAssemblerLabel::Type type = CodeAssemblerLabel::kNonDeferred)
      : CodeAssemblerLabel(assembler, vars.size(), vars.begin(), type) {}
1589 1590 1591 1592
  CodeAssemblerLabel(
      CodeAssembler* assembler, CodeAssemblerVariable* merged_variable,
      CodeAssemblerLabel::Type type = CodeAssemblerLabel::kNonDeferred)
      : CodeAssemblerLabel(assembler, 1, &merged_variable, type) {}
1593
  ~CodeAssemblerLabel();
1594

1595
  inline bool is_bound() const { return bound_; }
1596
  inline bool is_used() const { return merge_count_ != 0; }
1597

1598 1599 1600 1601
 private:
  friend class CodeAssembler;

  void Bind();
1602 1603 1604 1605
#if DEBUG
  void Bind(AssemblerDebugInfo debug_info);
#endif  // DEBUG
  void UpdateVariablesAfterBind();
1606 1607 1608 1609
  void MergeVariables();

  bool bound_;
  size_t merge_count_;
1610
  CodeAssemblerState* state_;
1611 1612 1613
  RawMachineLabel* label_;
  // Map of variables that need to be merged to their phi nodes (or placeholders
  // for those phis).
1614 1615 1616
  std::map<CodeAssemblerVariable::Impl*, Node*,
           CodeAssemblerVariable::ImplComparator>
      variable_phis_;
1617 1618
  // Map of variables to the list of value nodes that have been added from each
  // merge path in their order of merging.
1619 1620 1621
  std::map<CodeAssemblerVariable::Impl*, std::vector<Node*>,
           CodeAssemblerVariable::ImplComparator>
      variable_merges_;
1622 1623 1624 1625

  // Cannot be copied because the destructor explicitly call the destructor of
  // the underlying {RawMachineLabel}, hence only one pointer can point to it.
  DISALLOW_COPY_AND_ASSIGN(CodeAssemblerLabel);
1626 1627
};

1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661
class CodeAssemblerParameterizedLabelBase {
 public:
  bool is_used() const { return plain_label_.is_used(); }
  explicit CodeAssemblerParameterizedLabelBase(CodeAssembler* assembler,
                                               size_t arity,
                                               CodeAssemblerLabel::Type type)
      : state_(assembler->state()),
        phi_inputs_(arity),
        plain_label_(assembler, type) {}

 protected:
  CodeAssemblerLabel* plain_label() { return &plain_label_; }
  void AddInputs(std::vector<Node*> inputs);
  Node* CreatePhi(MachineRepresentation rep, const std::vector<Node*>& inputs);
  const std::vector<Node*>& CreatePhis(
      std::vector<MachineRepresentation> representations);

 private:
  CodeAssemblerState* state_;
  std::vector<std::vector<Node*>> phi_inputs_;
  std::vector<Node*> phi_nodes_;
  CodeAssemblerLabel plain_label_;
};

template <class... Types>
class CodeAssemblerParameterizedLabel
    : public CodeAssemblerParameterizedLabelBase {
 public:
  static constexpr size_t kArity = sizeof...(Types);
  explicit CodeAssemblerParameterizedLabel(CodeAssembler* assembler,
                                           CodeAssemblerLabel::Type type)
      : CodeAssemblerParameterizedLabelBase(assembler, kArity, type) {}

 private:
1662
  friend class CodeAssembler;
1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681

  void AddInputs(TNode<Types>... inputs) {
    CodeAssemblerParameterizedLabelBase::AddInputs(
        std::vector<Node*>{inputs...});
  }
  void CreatePhis(TNode<Types>*... results) {
    const std::vector<Node*>& phi_nodes =
        CodeAssemblerParameterizedLabelBase::CreatePhis(
            {MachineRepresentationOf<Types>::value...});
    auto it = phi_nodes.begin();
    USE(it);
    ITERATE_PACK(AssignPhi(results, *(it++)));
  }
  template <class T>
  static void AssignPhi(TNode<T>* result, Node* phi) {
    if (phi != nullptr) *result = TNode<T>::UncheckedCast(phi);
  }
};

1682 1683
using CodeAssemblerExceptionHandlerLabel =
    CodeAssemblerParameterizedLabel<Object>;
1684

1685 1686 1687 1688 1689 1690
class V8_EXPORT_PRIVATE CodeAssemblerState {
 public:
  // Create with CallStub linkage.
  // |result_size| specifies the number of results returned by the stub.
  // TODO(rmcilroy): move result_size to the CallInterfaceDescriptor.
  CodeAssemblerState(Isolate* isolate, Zone* zone,
1691
                     const CallInterfaceDescriptor& descriptor, Code::Kind kind,
1692
                     const char* name, PoisoningMitigationLevel poisoning_level,
1693
                     int32_t builtin_index = Builtins::kNoBuiltinId);
1694 1695 1696

  // Create with JSCall linkage.
  CodeAssemblerState(Isolate* isolate, Zone* zone, int parameter_count,
1697
                     Code::Kind kind, const char* name,
1698
                     PoisoningMitigationLevel poisoning_level,
1699
                     int32_t builtin_index = Builtins::kNoBuiltinId);
1700 1701 1702

  ~CodeAssemblerState();

1703
  const char* name() const { return name_; }
1704
  int parameter_count() const;
1705

1706 1707 1708
#if DEBUG
  void PrintCurrentBlock(std::ostream& os);
#endif  // DEBUG
1709
  bool InsideBlock();
1710 1711
  void SetInitialDebugInformation(const char* msg, const char* file, int line);

1712 1713
 private:
  friend class CodeAssembler;
1714 1715
  friend class CodeAssemblerLabel;
  friend class CodeAssemblerVariable;
1716
  friend class CodeAssemblerTester;
1717
  friend class CodeAssemblerParameterizedLabelBase;
1718
  friend class CodeAssemblerScopedExceptionHandler;
1719 1720

  CodeAssemblerState(Isolate* isolate, Zone* zone,
1721
                     CallDescriptor* call_descriptor, Code::Kind kind,
1722
                     const char* name, PoisoningMitigationLevel poisoning_level,
1723
                     int32_t builtin_index);
1724

1725 1726 1727
  void PushExceptionHandler(CodeAssemblerExceptionHandlerLabel* label);
  void PopExceptionHandler();

1728
  std::unique_ptr<RawMachineAssembler> raw_assembler_;
1729
  Code::Kind kind_;
1730
  const char* name_;
1731
  int32_t builtin_index_;
1732
  bool code_generated_;
1733 1734
  ZoneSet<CodeAssemblerVariable::Impl*, CodeAssemblerVariable::ImplComparator>
      variables_;
1735 1736
  CodeAssemblerCallback call_prologue_;
  CodeAssemblerCallback call_epilogue_;
1737
  std::vector<CodeAssemblerExceptionHandlerLabel*> exception_handler_labels_;
1738
  using VariableId = uint32_t;
1739
  VariableId next_variable_id_ = 0;
1740

1741
  VariableId NextVariableId() { return next_variable_id_++; }
1742 1743 1744 1745

  DISALLOW_COPY_AND_ASSIGN(CodeAssemblerState);
};

1746
class V8_EXPORT_PRIVATE CodeAssemblerScopedExceptionHandler {
1747
 public:
1748 1749
  CodeAssemblerScopedExceptionHandler(
      CodeAssembler* assembler, CodeAssemblerExceptionHandlerLabel* label);
1750

1751 1752 1753 1754 1755 1756 1757
  // Use this constructor for compatability/ports of old CSA code only. New code
  // should use the CodeAssemblerExceptionHandlerLabel version.
  CodeAssemblerScopedExceptionHandler(
      CodeAssembler* assembler, CodeAssemblerLabel* label,
      TypedCodeAssemblerVariable<Object>* exception);

  ~CodeAssemblerScopedExceptionHandler();
1758 1759

 private:
1760
  bool has_handler_;
1761
  CodeAssembler* assembler_;
1762 1763 1764
  CodeAssemblerLabel* compatibility_label_;
  std::unique_ptr<CodeAssemblerExceptionHandlerLabel> label_;
  TypedCodeAssemblerVariable<Object>* exception_;
1765 1766
};

1767
}  // namespace compiler
1768 1769

#if defined(V8_HOST_ARCH_32_BIT)
1770
using BInt = Smi;
1771
#elif defined(V8_HOST_ARCH_64_BIT)
1772
using BInt = IntPtrT;
1773 1774 1775 1776
#else
#error Unknown architecture.
#endif

1777 1778 1779 1780
}  // namespace internal
}  // namespace v8

#endif  // V8_COMPILER_CODE_ASSEMBLER_H_