code-assembler.h 59.2 KB
Newer Older
1 2 3 4 5 6 7
// Copyright 2015 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef V8_COMPILER_CODE_ASSEMBLER_H_
#define V8_COMPILER_CODE_ASSEMBLER_H_

8
#include <initializer_list>
9
#include <map>
10
#include <memory>
11 12 13

// Clients of this interface shouldn't depend on lots of compiler internals.
// Do not include anything from src/compiler here!
14
#include "src/base/macros.h"
15
#include "src/base/type-traits.h"
16
#include "src/builtins/builtins.h"
17 18 19
#include "src/codegen/code-factory.h"
#include "src/codegen/machine-type.h"
#include "src/codegen/source-position.h"
20
#include "src/codegen/tnode.h"
21
#include "src/heap/heap.h"
22
#include "src/objects/arguments.h"
23
#include "src/objects/data-handler.h"
24
#include "src/objects/heap-number.h"
25
#include "src/objects/js-array-buffer.h"
26
#include "src/objects/js-collection.h"
27
#include "src/objects/js-proxy.h"
28
#include "src/objects/map.h"
29
#include "src/objects/maybe-object.h"
30
#include "src/objects/objects.h"
31
#include "src/objects/oddball.h"
32 33
#include "src/objects/smi.h"
#include "src/objects/tagged-index.h"
34
#include "src/runtime/runtime.h"
35
#include "src/utils/allocation.h"
36
#include "src/zone/zone-containers.h"
37 38 39 40

namespace v8 {
namespace internal {

41
// Forward declarations.
42
class AsmWasmData;
43
class AsyncGeneratorRequest;
44
struct AssemblerOptions;
45
class BigInt;
46
class CallInterfaceDescriptor;
47 48 49
class Callable;
class Factory;
class InterpreterData;
50
class Isolate;
51
class JSAsyncFunctionObject;
52 53
class JSAsyncGeneratorObject;
class JSCollator;
54
class JSCollection;
55
class JSDateTimeFormat;
56
class JSDisplayNames;
57 58 59 60
class JSListFormat;
class JSLocale;
class JSNumberFormat;
class JSPluralRules;
61
class JSRegExpStringIterator;
62
class JSRelativeTimeFormat;
63
class JSSegmentIterator;
64
class JSSegmenter;
65
class JSV8BreakIterator;
66
class JSWeakCollection;
67
class JSFinalizationRegistry;
68
class JSWeakMap;
69
class JSWeakRef;
70
class JSWeakSet;
71 72 73 74 75
class PromiseCapability;
class PromiseFulfillReactionJobTask;
class PromiseReaction;
class PromiseReactionJobTask;
class PromiseRejectReactionJobTask;
76
class WasmDebugInfo;
77
class Zone;
78 79
#define MAKE_FORWARD_DECLARATION(Name) class Name;
TORQUE_INTERNAL_CLASS_LIST(MAKE_FORWARD_DECLARATION)
80
#undef MAKE_FORWARD_DECLARATION
81

82 83 84
template <typename T>
class Signature;

85 86 87
#define ENUM_ELEMENT(Name) k##Name,
#define ENUM_STRUCT_ELEMENT(NAME, Name, name) k##Name,
enum class ObjectType {
88 89
  ENUM_ELEMENT(Object)                 //
  ENUM_ELEMENT(Smi)                    //
90
  ENUM_ELEMENT(TaggedIndex)            //
91 92 93 94
  ENUM_ELEMENT(HeapObject)             //
  OBJECT_TYPE_LIST(ENUM_ELEMENT)       //
  HEAP_OBJECT_TYPE_LIST(ENUM_ELEMENT)  //
  STRUCT_LIST(ENUM_STRUCT_ELEMENT)     //
95 96 97 98
};
#undef ENUM_ELEMENT
#undef ENUM_STRUCT_ELEMENT

99 100 101 102 103 104 105 106 107 108
enum class CheckBounds { kAlways, kDebugOnly };
inline bool NeedsBoundsCheck(CheckBounds check_bounds) {
  switch (check_bounds) {
    case CheckBounds::kAlways:
      return true;
    case CheckBounds::kDebugOnly:
      return DEBUG_BOOL;
  }
}

109 110
enum class StoreToObjectWriteBarrier { kNone, kMap, kFull };

111
class AccessCheckNeeded;
112
class BigIntBase;
113
class BigIntWrapper;
114
class ClassBoilerplate;
115
class BooleanWrapper;
116 117 118
class CompilationCacheTable;
class Constructor;
class Filler;
119
class FunctionTemplateRareData;
120
class HeapNumber;
121
class InternalizedString;
122
class JSArgumentsObject;
123
class JSArrayBufferView;
124 125
class JSContextExtensionObject;
class JSError;
126
class JSSloppyArgumentsObject;
127 128
class MapCache;
class NativeContext;
129 130
class NumberWrapper;
class ScriptWrapper;
131
class SloppyArgumentsElements;
132
class StringWrapper;
133
class SymbolWrapper;
134 135
class Undetectable;
class UniqueName;
136
class WasmCapiFunctionData;
137
class WasmExceptionObject;
138
class WasmExceptionPackage;
139
class WasmExceptionTag;
140
class WasmExportedFunctionData;
141
class WasmGlobalObject;
142
class WasmIndirectFunctionTable;
143
class WasmJSFunctionData;
144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166
class WasmMemoryObject;
class WasmModuleObject;
class WasmTableObject;

template <class T>
struct ObjectTypeOf {};

#define OBJECT_TYPE_CASE(Name)                           \
  template <>                                            \
  struct ObjectTypeOf<Name> {                            \
    static const ObjectType value = ObjectType::k##Name; \
  };
#define OBJECT_TYPE_STRUCT_CASE(NAME, Name, name)        \
  template <>                                            \
  struct ObjectTypeOf<Name> {                            \
    static const ObjectType value = ObjectType::k##Name; \
  };
#define OBJECT_TYPE_TEMPLATE_CASE(Name)                  \
  template <class... Args>                               \
  struct ObjectTypeOf<Name<Args...>> {                   \
    static const ObjectType value = ObjectType::k##Name; \
  };
OBJECT_TYPE_CASE(Object)
167
OBJECT_TYPE_CASE(Smi)
168
OBJECT_TYPE_CASE(TaggedIndex)
169
OBJECT_TYPE_CASE(HeapObject)
170 171 172 173 174 175 176 177
OBJECT_TYPE_LIST(OBJECT_TYPE_CASE)
HEAP_OBJECT_ORDINARY_TYPE_LIST(OBJECT_TYPE_CASE)
STRUCT_LIST(OBJECT_TYPE_STRUCT_CASE)
HEAP_OBJECT_TEMPLATE_TYPE_LIST(OBJECT_TYPE_TEMPLATE_CASE)
#undef OBJECT_TYPE_CASE
#undef OBJECT_TYPE_STRUCT_CASE
#undef OBJECT_TYPE_TEMPLATE_CASE

178
// {raw_value} must be a tagged Object.
179 180 181
// {raw_type} must be a tagged Smi.
// {raw_location} must be a tagged String.
// Returns a tagged Smi.
182 183
Address CheckObjectType(Address raw_value, Address raw_type,
                        Address raw_location);
184

185 186 187
namespace compiler {

class CallDescriptor;
188 189
class CodeAssemblerLabel;
class CodeAssemblerVariable;
190 191
template <class T>
class TypedCodeAssemblerVariable;
192
class CodeAssemblerState;
193
class JSGraph;
194 195 196
class Node;
class RawMachineAssembler;
class RawMachineLabel;
197
class SourcePositionTable;
198

199
using CodeAssemblerVariableList = ZoneVector<CodeAssemblerVariable*>;
200

201
using CodeAssemblerCallback = std::function<void()>;
202

203 204 205
template <class... Types>
class CodeAssemblerParameterizedLabel;

206 207 208
// This macro alias allows to use PairT<T1, T2> as a macro argument.
#define PAIR_TYPE(T1, T2) PairT<T1, T2>

209 210 211 212 213 214 215
#define CODE_ASSEMBLER_COMPARE_BINARY_OP_LIST(V)          \
  V(Float32Equal, BoolT, Float32T, Float32T)              \
  V(Float32LessThan, BoolT, Float32T, Float32T)           \
  V(Float32LessThanOrEqual, BoolT, Float32T, Float32T)    \
  V(Float32GreaterThan, BoolT, Float32T, Float32T)        \
  V(Float32GreaterThanOrEqual, BoolT, Float32T, Float32T) \
  V(Float64Equal, BoolT, Float64T, Float64T)              \
216
  V(Float64NotEqual, BoolT, Float64T, Float64T)           \
217 218 219 220
  V(Float64LessThan, BoolT, Float64T, Float64T)           \
  V(Float64LessThanOrEqual, BoolT, Float64T, Float64T)    \
  V(Float64GreaterThan, BoolT, Float64T, Float64T)        \
  V(Float64GreaterThanOrEqual, BoolT, Float64T, Float64T) \
221
  /* Use Word32Equal if you need Int32Equal */            \
222 223 224 225
  V(Int32GreaterThan, BoolT, Word32T, Word32T)            \
  V(Int32GreaterThanOrEqual, BoolT, Word32T, Word32T)     \
  V(Int32LessThan, BoolT, Word32T, Word32T)               \
  V(Int32LessThanOrEqual, BoolT, Word32T, Word32T)        \
226
  /* Use WordEqual if you need IntPtrEqual */             \
227 228 229 230
  V(IntPtrLessThan, BoolT, WordT, WordT)                  \
  V(IntPtrLessThanOrEqual, BoolT, WordT, WordT)           \
  V(IntPtrGreaterThan, BoolT, WordT, WordT)               \
  V(IntPtrGreaterThanOrEqual, BoolT, WordT, WordT)        \
231
  /* Use Word32Equal if you need Uint32Equal */           \
232 233
  V(Uint32LessThan, BoolT, Word32T, Word32T)              \
  V(Uint32LessThanOrEqual, BoolT, Word32T, Word32T)       \
234
  V(Uint32GreaterThan, BoolT, Word32T, Word32T)           \
235
  V(Uint32GreaterThanOrEqual, BoolT, Word32T, Word32T)    \
236
  /* Use WordEqual if you need UintPtrEqual */            \
237 238 239
  V(UintPtrLessThan, BoolT, WordT, WordT)                 \
  V(UintPtrLessThanOrEqual, BoolT, WordT, WordT)          \
  V(UintPtrGreaterThan, BoolT, WordT, WordT)              \
240
  V(UintPtrGreaterThanOrEqual, BoolT, WordT, WordT)
241

242 243 244 245 246 247 248 249 250 251 252 253 254
#define CODE_ASSEMBLER_BINARY_OP_LIST(V)                                \
  CODE_ASSEMBLER_COMPARE_BINARY_OP_LIST(V)                              \
  V(Float64Add, Float64T, Float64T, Float64T)                           \
  V(Float64Sub, Float64T, Float64T, Float64T)                           \
  V(Float64Mul, Float64T, Float64T, Float64T)                           \
  V(Float64Div, Float64T, Float64T, Float64T)                           \
  V(Float64Mod, Float64T, Float64T, Float64T)                           \
  V(Float64Atan2, Float64T, Float64T, Float64T)                         \
  V(Float64Pow, Float64T, Float64T, Float64T)                           \
  V(Float64Max, Float64T, Float64T, Float64T)                           \
  V(Float64Min, Float64T, Float64T, Float64T)                           \
  V(Float64InsertLowWord32, Float64T, Float64T, Word32T)                \
  V(Float64InsertHighWord32, Float64T, Float64T, Word32T)               \
255 256 257 258
  V(IntPtrAdd, WordT, WordT, WordT)                                     \
  V(IntPtrSub, WordT, WordT, WordT)                                     \
  V(IntPtrMul, WordT, WordT, WordT)                                     \
  V(IntPtrDiv, IntPtrT, IntPtrT, IntPtrT)                               \
259 260 261 262 263
  V(IntPtrAddWithOverflow, PAIR_TYPE(IntPtrT, BoolT), IntPtrT, IntPtrT) \
  V(IntPtrSubWithOverflow, PAIR_TYPE(IntPtrT, BoolT), IntPtrT, IntPtrT) \
  V(Int32Add, Word32T, Word32T, Word32T)                                \
  V(Int32AddWithOverflow, PAIR_TYPE(Int32T, BoolT), Int32T, Int32T)     \
  V(Int32Sub, Word32T, Word32T, Word32T)                                \
264
  V(Int32SubWithOverflow, PAIR_TYPE(Int32T, BoolT), Int32T, Int32T)     \
265 266 267 268
  V(Int32Mul, Word32T, Word32T, Word32T)                                \
  V(Int32MulWithOverflow, PAIR_TYPE(Int32T, BoolT), Int32T, Int32T)     \
  V(Int32Div, Int32T, Int32T, Int32T)                                   \
  V(Int32Mod, Int32T, Int32T, Int32T)                                   \
269 270 271
  V(WordOr, WordT, WordT, WordT)                                        \
  V(WordAnd, WordT, WordT, WordT)                                       \
  V(WordXor, WordT, WordT, WordT)                                       \
272
  V(WordRor, WordT, WordT, IntegralT)                                   \
273 274 275
  V(WordShl, WordT, WordT, IntegralT)                                   \
  V(WordShr, WordT, WordT, IntegralT)                                   \
  V(WordSar, WordT, WordT, IntegralT)                                   \
276
  V(WordSarShiftOutZeros, WordT, WordT, IntegralT)                      \
277 278 279
  V(Word32Or, Word32T, Word32T, Word32T)                                \
  V(Word32And, Word32T, Word32T, Word32T)                               \
  V(Word32Xor, Word32T, Word32T, Word32T)                               \
280
  V(Word32Ror, Word32T, Word32T, Word32T)                               \
281 282 283
  V(Word32Shl, Word32T, Word32T, Word32T)                               \
  V(Word32Shr, Word32T, Word32T, Word32T)                               \
  V(Word32Sar, Word32T, Word32T, Word32T)                               \
284
  V(Word32SarShiftOutZeros, Word32T, Word32T, Word32T)                  \
285 286 287 288 289 290 291
  V(Word64And, Word64T, Word64T, Word64T)                               \
  V(Word64Or, Word64T, Word64T, Word64T)                                \
  V(Word64Xor, Word64T, Word64T, Word64T)                               \
  V(Word64Ror, Word64T, Word64T, Word64T)                               \
  V(Word64Shl, Word64T, Word64T, Word64T)                               \
  V(Word64Shr, Word64T, Word64T, Word64T)                               \
  V(Word64Sar, Word64T, Word64T, Word64T)
292 293 294

TNode<Float64T> Float64Add(TNode<Float64T> a, TNode<Float64T> b);

295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317
#define CODE_ASSEMBLER_UNARY_OP_LIST(V)                        \
  V(Float64Abs, Float64T, Float64T)                            \
  V(Float64Acos, Float64T, Float64T)                           \
  V(Float64Acosh, Float64T, Float64T)                          \
  V(Float64Asin, Float64T, Float64T)                           \
  V(Float64Asinh, Float64T, Float64T)                          \
  V(Float64Atan, Float64T, Float64T)                           \
  V(Float64Atanh, Float64T, Float64T)                          \
  V(Float64Cos, Float64T, Float64T)                            \
  V(Float64Cosh, Float64T, Float64T)                           \
  V(Float64Exp, Float64T, Float64T)                            \
  V(Float64Expm1, Float64T, Float64T)                          \
  V(Float64Log, Float64T, Float64T)                            \
  V(Float64Log1p, Float64T, Float64T)                          \
  V(Float64Log2, Float64T, Float64T)                           \
  V(Float64Log10, Float64T, Float64T)                          \
  V(Float64Cbrt, Float64T, Float64T)                           \
  V(Float64Neg, Float64T, Float64T)                            \
  V(Float64Sin, Float64T, Float64T)                            \
  V(Float64Sinh, Float64T, Float64T)                           \
  V(Float64Sqrt, Float64T, Float64T)                           \
  V(Float64Tan, Float64T, Float64T)                            \
  V(Float64Tanh, Float64T, Float64T)                           \
318 319
  V(Float64ExtractLowWord32, Uint32T, Float64T)                \
  V(Float64ExtractHighWord32, Uint32T, Float64T)               \
320
  V(BitcastTaggedToWord, IntPtrT, Object)                      \
321
  V(BitcastTaggedToWordForTagAndSmiBits, IntPtrT, AnyTaggedT)  \
322
  V(BitcastMaybeObjectToWord, IntPtrT, MaybeObject)            \
323 324
  V(BitcastWordToTagged, Object, WordT)                        \
  V(BitcastWordToTaggedSigned, Smi, WordT)                     \
325
  V(TruncateFloat32ToInt32, Int32T, Float32T)                  \
326
  V(TruncateFloat64ToFloat32, Float32T, Float64T)              \
327
  V(TruncateFloat64ToWord32, Uint32T, Float64T)                \
328 329 330 331 332 333 334 335
  V(TruncateInt64ToInt32, Int32T, Int64T)                      \
  V(ChangeFloat32ToFloat64, Float64T, Float32T)                \
  V(ChangeFloat64ToUint32, Uint32T, Float64T)                  \
  V(ChangeFloat64ToUint64, Uint64T, Float64T)                  \
  V(ChangeInt32ToFloat64, Float64T, Int32T)                    \
  V(ChangeInt32ToInt64, Int64T, Int32T)                        \
  V(ChangeUint32ToFloat64, Float64T, Word32T)                  \
  V(ChangeUint32ToUint64, Uint64T, Word32T)                    \
336
  V(BitcastInt32ToFloat32, Float32T, Word32T)                  \
337
  V(BitcastFloat32ToInt32, Uint32T, Float32T)                  \
338
  V(RoundFloat64ToInt32, Int32T, Float64T)                     \
339
  V(RoundInt32ToFloat32, Float32T, Int32T)                     \
340 341 342 343 344 345
  V(Float64SilenceNaN, Float64T, Float64T)                     \
  V(Float64RoundDown, Float64T, Float64T)                      \
  V(Float64RoundUp, Float64T, Float64T)                        \
  V(Float64RoundTiesEven, Float64T, Float64T)                  \
  V(Float64RoundTruncate, Float64T, Float64T)                  \
  V(Word32Clz, Int32T, Word32T)                                \
346
  V(Word32BitwiseNot, Word32T, Word32T)                        \
347
  V(WordNot, WordT, WordT)                                     \
348 349 350
  V(Int32AbsWithOverflow, PAIR_TYPE(Int32T, BoolT), Int32T)    \
  V(Int64AbsWithOverflow, PAIR_TYPE(Int64T, BoolT), Int64T)    \
  V(IntPtrAbsWithOverflow, PAIR_TYPE(IntPtrT, BoolT), IntPtrT) \
351 352
  V(Word32BinaryNot, BoolT, Word32T)                           \
  V(StackPointerGreaterThan, BoolT, WordT)
353 354

// A "public" interface used by components outside of compiler directory to
355 356 357 358
// create code objects with TurboFan's backend. This class is mostly a thin
// shim around the RawMachineAssembler, and its primary job is to ensure that
// the innards of the RawMachineAssembler and other compiler implementation
// details don't leak outside of the the compiler directory..
359 360
//
// V8 components that need to generate low-level code using this interface
361 362 363 364
// should include this header--and this header only--from the compiler
// directory (this is actually enforced). Since all interesting data
// structures are forward declared, it's not possible for clients to peek
// inside the compiler internals.
365 366 367 368 369
//
// In addition to providing isolation between TurboFan and code generation
// clients, CodeAssembler also provides an abstraction for creating variables
// and enhanced Label functionality to merge variable values along paths where
// they have differing values, including loops.
370 371 372 373
//
// The CodeAssembler itself is stateless (and instances are expected to be
// temporary-scoped and short-lived); all its state is encapsulated into
// a CodeAssemblerState instance.
374
class V8_EXPORT_PRIVATE CodeAssembler {
375
 public:
376
  explicit CodeAssembler(CodeAssemblerState* state) : state_(state) {}
377
  ~CodeAssembler();
378

379 380
  static Handle<Code> GenerateCode(CodeAssemblerState* state,
                                   const AssemblerOptions& options);
381 382

  bool Is64() const;
383
  bool Is32() const;
384 385
  bool IsFloat64RoundUpSupported() const;
  bool IsFloat64RoundDownSupported() const;
386
  bool IsFloat64RoundTiesEvenSupported() const;
387
  bool IsFloat64RoundTruncateSupported() const;
388 389 390
  bool IsInt32AbsWithOverflowSupported() const;
  bool IsInt64AbsWithOverflowSupported() const;
  bool IsIntPtrAbsWithOverflowSupported() const;
391

392
  // Shortened aliases for use in CodeAssembler subclasses.
393 394 395 396 397
  using Label = CodeAssemblerLabel;
  using Variable = CodeAssemblerVariable;
  template <class T>
  using TVariable = TypedCodeAssemblerVariable<T>;
  using VariableList = CodeAssemblerVariableList;
398

399 400 401 402
  // ===========================================================================
  // Base Assembler
  // ===========================================================================

403
  template <class PreviousType, bool FromTyped>
404 405 406 407 408 409 410 411 412 413 414 415
  class CheckedNode {
   public:
#ifdef DEBUG
    CheckedNode(Node* node, CodeAssembler* code_assembler, const char* location)
        : node_(node), code_assembler_(code_assembler), location_(location) {}
#else
    CheckedNode(compiler::Node* node, CodeAssembler*, const char*)
        : node_(node) {}
#endif

    template <class A>
    operator TNode<A>() {
416 417 418 419
      static_assert(
          !std::is_same<A, MaybeObject>::value,
          "Can't cast to MaybeObject, use explicit conversion functions. ");

420 421
      static_assert(types_have_common_values<A, PreviousType>::value,
                    "Incompatible types: this cast can never succeed.");
422
      static_assert(std::is_convertible<TNode<A>, TNode<Object>>::value,
423 424
                    "Coercion to untagged values cannot be "
                    "checked.");
425 426 427 428
      static_assert(
          !FromTyped ||
              !std::is_convertible<TNode<PreviousType>, TNode<A>>::value,
          "Unnecessary CAST: types are convertible.");
429 430
#ifdef DEBUG
      if (FLAG_debug_code) {
431 432 433
        if (std::is_same<PreviousType, MaybeObject>::value) {
          code_assembler_->GenerateCheckMaybeObjectIsObject(node_, location_);
        }
434
        TNode<ExternalReference> function = code_assembler_->ExternalConstant(
435
            ExternalReference::check_object_type());
436 437 438 439 440 441 442
        code_assembler_->CallCFunction(
            function, MachineType::AnyTagged(),
            std::make_pair(MachineType::AnyTagged(), node_),
            std::make_pair(MachineType::TaggedSigned(),
                           code_assembler_->SmiConstant(
                               static_cast<int>(ObjectTypeOf<A>::value))),
            std::make_pair(MachineType::AnyTagged(),
443
                           code_assembler_->StringConstant(location_)));
444 445 446 447 448 449 450
      }
#endif
      return TNode<A>::UncheckedCast(node_);
    }

    template <class A>
    operator SloppyTNode<A>() {
451
      return implicit_cast<TNode<A>>(*this);
452 453 454 455 456 457 458 459 460 461 462 463
    }

    Node* node() const { return node_; }

   private:
    Node* node_;
#ifdef DEBUG
    CodeAssembler* code_assembler_;
    const char* location_;
#endif
  };

464 465 466
  template <class T>
  TNode<T> UncheckedCast(Node* value) {
    return TNode<T>::UncheckedCast(value);
467
  }
468 469 470 471 472 473 474 475 476 477 478 479 480 481
  template <class T, class U>
  TNode<T> UncheckedCast(TNode<U> value) {
    static_assert(types_have_common_values<T, U>::value,
                  "Incompatible types: this cast can never succeed.");
    return TNode<T>::UncheckedCast(value);
  }

  // ReinterpretCast<T>(v) has the power to cast even when the type of v is
  // unrelated to T. Use with care.
  template <class T>
  TNode<T> ReinterpretCast(Node* value) {
    return TNode<T>::UncheckedCast(value);
  }

482
  CheckedNode<Object, false> Cast(Node* value, const char* location = "") {
483
    return {value, this, location};
484 485 486
  }

  template <class T>
487
  CheckedNode<T, true> Cast(TNode<T> value, const char* location = "") {
488
    return {value, this, location};
489 490 491 492 493 494 495
  }

#ifdef DEBUG
#define STRINGIFY(x) #x
#define TO_STRING_LITERAL(x) STRINGIFY(x)
#define CAST(x) \
  Cast(x, "CAST(" #x ") at " __FILE__ ":" TO_STRING_LITERAL(__LINE__))
496 497
#define TORQUE_CAST(x) \
  ca_.Cast(x, "CAST(" #x ") at " __FILE__ ":" TO_STRING_LITERAL(__LINE__))
498
#else
499
#define CAST(x) Cast(x)
500
#define TORQUE_CAST(x) ca_.Cast(x)
501 502
#endif

503 504 505 506
#ifdef DEBUG
  void GenerateCheckMaybeObjectIsObject(Node* node, const char* location);
#endif

507
  // Constants.
508 509 510
  TNode<Int32T> Int32Constant(int32_t value);
  TNode<Int64T> Int64Constant(int64_t value);
  TNode<IntPtrT> IntPtrConstant(intptr_t value);
511 512 513
  TNode<Uint32T> Uint32Constant(uint32_t value) {
    return Unsigned(Int32Constant(bit_cast<int32_t>(value)));
  }
514 515 516
  TNode<UintPtrT> UintPtrConstant(uintptr_t value) {
    return Unsigned(IntPtrConstant(bit_cast<intptr_t>(value)));
  }
517
  TNode<TaggedIndex> TaggedIndexConstant(intptr_t value);
518 519 520
  TNode<RawPtrT> PointerConstant(void* value) {
    return ReinterpretCast<RawPtrT>(IntPtrConstant(bit_cast<intptr_t>(value)));
  }
521
  TNode<Number> NumberConstant(double value);
522
  TNode<Smi> SmiConstant(Smi value);
523
  TNode<Smi> SmiConstant(int value);
524 525 526 527 528 529
  template <typename E,
            typename = typename std::enable_if<std::is_enum<E>::value>::type>
  TNode<Smi> SmiConstant(E value) {
    STATIC_ASSERT(sizeof(E) <= sizeof(int));
    return SmiConstant(static_cast<int>(value));
  }
530 531 532 533 534 535 536 537
  TNode<HeapObject> UntypedHeapConstant(Handle<HeapObject> object);
  template <class Type>
  TNode<Type> HeapConstant(Handle<Type> object) {
    return UncheckedCast<Type>(UntypedHeapConstant(object));
  }
  TNode<String> StringConstant(const char* str);
  TNode<Oddball> BooleanConstant(bool value);
  TNode<ExternalReference> ExternalConstant(ExternalReference address);
538
  TNode<Float32T> Float32Constant(double value);
539
  TNode<Float64T> Float64Constant(double value);
540 541 542 543 544 545
  TNode<BoolT> Int32TrueConstant() {
    return ReinterpretCast<BoolT>(Int32Constant(1));
  }
  TNode<BoolT> Int32FalseConstant() {
    return ReinterpretCast<BoolT>(Int32Constant(0));
  }
546 547 548
  TNode<BoolT> BoolConstant(bool value) {
    return value ? Int32TrueConstant() : Int32FalseConstant();
  }
549

550 551 552
  bool ToInt32Constant(Node* node, int32_t* out_value);
  bool ToInt64Constant(Node* node, int64_t* out_value);
  bool ToIntPtrConstant(Node* node, intptr_t* out_value);
553
  bool ToSmiConstant(Node* node, Smi* out_value);
554

555 556 557
  bool IsUndefinedConstant(TNode<Object> node);
  bool IsNullConstant(TNode<Object> node);

558 559 560 561 562 563 564 565 566
  TNode<Int32T> Signed(TNode<Word32T> x) { return UncheckedCast<Int32T>(x); }
  TNode<IntPtrT> Signed(TNode<WordT> x) { return UncheckedCast<IntPtrT>(x); }
  TNode<Uint32T> Unsigned(TNode<Word32T> x) {
    return UncheckedCast<Uint32T>(x);
  }
  TNode<UintPtrT> Unsigned(TNode<WordT> x) {
    return UncheckedCast<UintPtrT>(x);
  }

567 568
  static constexpr int kTargetParameterIndex = -1;

569
  Node* Parameter(int value);
570 571

  TNode<Context> GetJSContextParameter();
572 573 574
  void Return(TNode<Object> value);
  void Return(TNode<Object> value1, TNode<Object> value2);
  void Return(TNode<Object> value1, TNode<Object> value2, TNode<Object> value3);
575 576
  void Return(TNode<Int32T> value);
  void Return(TNode<Uint32T> value);
577
  void Return(TNode<WordT> value);
578
  void Return(TNode<Float32T> value);
579
  void Return(TNode<Float64T> value);
580
  void Return(TNode<WordT> value1, TNode<WordT> value2);
581
  void PopAndReturn(Node* pop, Node* value);
582

583
  void ReturnIf(TNode<BoolT> condition, TNode<Object> value);
584

585
  void AbortCSAAssert(Node* message);
586
  void DebugBreak();
587
  void Unreachable();
588 589 590 591 592 593 594 595 596 597 598 599
  void Comment(const char* msg) {
    if (!FLAG_code_comments) return;
    Comment(std::string(msg));
  }
  void Comment(std::string msg);
  template <class... Args>
  void Comment(Args&&... args) {
    if (!FLAG_code_comments) return;
    std::ostringstream s;
    USE((s << std::forward<Args>(args))...);
    Comment(s.str());
  }
600

601 602
  void StaticAssert(TNode<BoolT> value);

603 604
  void SetSourcePosition(const char* file, int line);

605
  void Bind(Label* label);
606 607 608
#if DEBUG
  void Bind(Label* label, AssemblerDebugInfo debug_info);
#endif  // DEBUG
609
  void Goto(Label* label);
610 611 612
  void GotoIf(TNode<IntegralT> condition, Label* true_label);
  void GotoIfNot(TNode<IntegralT> condition, Label* false_label);
  void Branch(TNode<IntegralT> condition, Label* true_label,
613
              Label* false_label);
614

615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632
  template <class T>
  TNode<T> Uninitialized() {
    return {};
  }

  template <class... T>
  void Bind(CodeAssemblerParameterizedLabel<T...>* label, TNode<T>*... phis) {
    Bind(label->plain_label());
    label->CreatePhis(phis...);
  }
  template <class... T, class... Args>
  void Branch(TNode<BoolT> condition,
              CodeAssemblerParameterizedLabel<T...>* if_true,
              CodeAssemblerParameterizedLabel<T...>* if_false, Args... args) {
    if_true->AddInputs(args...);
    if_false->AddInputs(args...);
    Branch(condition, if_true->plain_label(), if_false->plain_label());
  }
633 634 635 636 637 638 639 640 641 642
  template <class... T, class... U>
  void Branch(TNode<BoolT> condition,
              CodeAssemblerParameterizedLabel<T...>* if_true,
              std::vector<Node*> args_true,
              CodeAssemblerParameterizedLabel<U...>* if_false,
              std::vector<Node*> args_false) {
    if_true->AddInputsVector(std::move(args_true));
    if_false->AddInputsVector(std::move(args_false));
    Branch(condition, if_true->plain_label(), if_false->plain_label());
  }
643 644

  template <class... T, class... Args>
645
  void Goto(CodeAssemblerParameterizedLabel<T...>* label, Args... args) {
646 647 648 649
    label->AddInputs(args...);
    Goto(label->plain_label());
  }

650 651
  void Branch(TNode<BoolT> condition, const std::function<void()>& true_body,
              const std::function<void()>& false_body);
652
  void Branch(TNode<BoolT> condition, Label* true_label,
653 654
              const std::function<void()>& false_body);
  void Branch(TNode<BoolT> condition, const std::function<void()>& true_body,
655
              Label* false_label);
656

657
  void Switch(Node* index, Label* default_label, const int32_t* case_values,
658 659 660
              Label** case_labels, size_t case_count);

  // Access to the frame pointer
661 662
  TNode<RawPtrT> LoadFramePointer();
  TNode<RawPtrT> LoadParentFramePointer();
663

664
  // Poison |value| on speculative paths.
665 666
  TNode<Object> TaggedPoisonOnSpeculation(TNode<Object> value);
  TNode<WordT> WordPoisonOnSpeculation(TNode<WordT> value);
667

668
  // Load raw memory location.
669
  Node* Load(MachineType type, Node* base,
670
             LoadSensitivity needs_poisoning = LoadSensitivity::kSafe);
671
  template <class Type>
672
  TNode<Type> Load(MachineType type, TNode<RawPtr<Type>> base) {
673
    DCHECK(
674 675
        IsSubtype(type.representation(), MachineRepresentationOf<Type>::value));
    return UncheckedCast<Type>(Load(type, static_cast<Node*>(base)));
676
  }
677
  Node* Load(MachineType type, Node* base, Node* offset,
678
             LoadSensitivity needs_poisoning = LoadSensitivity::kSafe);
679 680 681 682 683 684 685 686 687 688 689 690
  template <class Type>
  TNode<Type> Load(Node* base,
                   LoadSensitivity needs_poisoning = LoadSensitivity::kSafe) {
    return UncheckedCast<Type>(
        Load(MachineTypeOf<Type>::value, base, needs_poisoning));
  }
  template <class Type>
  TNode<Type> Load(Node* base, SloppyTNode<WordT> offset,
                   LoadSensitivity needs_poisoning = LoadSensitivity::kSafe) {
    return UncheckedCast<Type>(
        Load(MachineTypeOf<Type>::value, base, offset, needs_poisoning));
  }
691
  Node* AtomicLoad(MachineType type, Node* base, Node* offset);
692 693
  // Load uncompressed tagged value from (most likely off JS heap) memory
  // location.
694
  TNode<Object> LoadFullTagged(
695
      Node* base, LoadSensitivity needs_poisoning = LoadSensitivity::kSafe);
696
  TNode<Object> LoadFullTagged(
697 698
      Node* base, Node* offset,
      LoadSensitivity needs_poisoning = LoadSensitivity::kSafe);
699

700 701 702
  Node* LoadFromObject(MachineType type, TNode<HeapObject> object,
                       TNode<IntPtrT> offset);

703
  // Load a value from the root array.
704
  TNode<Object> LoadRoot(RootIndex root_index);
705

706
  // Store value to raw memory location.
707 708
  Node* Store(Node* base, Node* value);
  Node* Store(Node* base, Node* offset, Node* value);
709
  Node* StoreEphemeronKey(Node* base, Node* offset, Node* value);
710
  Node* StoreNoWriteBarrier(MachineRepresentation rep, Node* base, Node* value);
711
  Node* StoreNoWriteBarrier(MachineRepresentation rep, Node* base, Node* offset,
712
                            Node* value);
713 714 715 716 717
  Node* UnsafeStoreNoWriteBarrier(MachineRepresentation rep, Node* base,
                                  Node* value);
  Node* UnsafeStoreNoWriteBarrier(MachineRepresentation rep, Node* base,
                                  Node* offset, Node* value);

718 719 720 721 722 723
  // Stores uncompressed tagged value to (most likely off JS heap) memory
  // location without write barrier.
  Node* StoreFullTaggedNoWriteBarrier(Node* base, Node* tagged_value);
  Node* StoreFullTaggedNoWriteBarrier(Node* base, Node* offset,
                                      Node* tagged_value);

724 725
  // Optimized memory operations that map to Turbofan simplified nodes.
  TNode<HeapObject> OptimizedAllocate(TNode<IntPtrT> size,
726 727
                                      AllocationType allocation,
                                      AllowLargeObjects allow_large_objects);
728 729 730
  void StoreToObject(MachineRepresentation rep, TNode<HeapObject> object,
                     TNode<IntPtrT> offset, Node* value,
                     StoreToObjectWriteBarrier write_barrier);
731
  void OptimizedStoreField(MachineRepresentation rep, TNode<HeapObject> object,
732
                           int offset, Node* value);
733 734 735 736 737 738
  void OptimizedStoreFieldAssertNoWriteBarrier(MachineRepresentation rep,
                                               TNode<HeapObject> object,
                                               int offset, Node* value);
  void OptimizedStoreFieldUnsafeNoWriteBarrier(MachineRepresentation rep,
                                               TNode<HeapObject> object,
                                               int offset, Node* value);
739
  void OptimizedStoreMap(TNode<HeapObject> object, TNode<Map>);
740 741
  // {value_high} is used for 64-bit stores on 32-bit platforms, must be
  // nullptr in other cases.
742
  Node* AtomicStore(MachineRepresentation rep, Node* base, Node* offset,
743
                    Node* value, Node* value_high = nullptr);
744

745
  // Exchange value at raw memory location
746 747
  Node* AtomicExchange(MachineType type, Node* base, Node* offset, Node* value,
                       Node* value_high = nullptr);
748

749 750
  // Compare and Exchange value at raw memory location
  Node* AtomicCompareExchange(MachineType type, Node* base, Node* offset,
751 752 753
                              Node* old_value, Node* new_value,
                              Node* old_value_high = nullptr,
                              Node* new_value_high = nullptr);
754

755 756
  Node* AtomicAdd(MachineType type, Node* base, Node* offset, Node* value,
                  Node* value_high = nullptr);
757

758 759
  Node* AtomicSub(MachineType type, Node* base, Node* offset, Node* value,
                  Node* value_high = nullptr);
760

761 762
  Node* AtomicAnd(MachineType type, Node* base, Node* offset, Node* value,
                  Node* value_high = nullptr);
763

764 765
  Node* AtomicOr(MachineType type, Node* base, Node* offset, Node* value,
                 Node* value_high = nullptr);
766

767 768
  Node* AtomicXor(MachineType type, Node* base, Node* offset, Node* value,
                  Node* value_high = nullptr);
769

770
  // Store a value to the root array.
771
  Node* StoreRoot(RootIndex root_index, Node* value);
772

773
// Basic arithmetic operations.
774 775
#define DECLARE_CODE_ASSEMBLER_BINARY_OP(name, ResType, Arg1Type, Arg2Type) \
  TNode<ResType> name(SloppyTNode<Arg1Type> a, SloppyTNode<Arg2Type> b);
776 777 778
  CODE_ASSEMBLER_BINARY_OP_LIST(DECLARE_CODE_ASSEMBLER_BINARY_OP)
#undef DECLARE_CODE_ASSEMBLER_BINARY_OP

779 780
  TNode<UintPtrT> WordShr(TNode<UintPtrT> left, TNode<IntegralT> right) {
    return Unsigned(
781 782
        WordShr(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
783
  TNode<IntPtrT> WordSar(TNode<IntPtrT> left, TNode<IntegralT> right) {
784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804
    return Signed(WordSar(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
  TNode<IntPtrT> WordShl(TNode<IntPtrT> left, TNode<IntegralT> right) {
    return Signed(WordShl(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
  TNode<UintPtrT> WordShl(TNode<UintPtrT> left, TNode<IntegralT> right) {
    return Unsigned(
        WordShl(static_cast<Node*>(left), static_cast<Node*>(right)));
  }

  TNode<Int32T> Word32Shl(TNode<Int32T> left, TNode<Int32T> right) {
    return Signed(
        Word32Shl(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
  TNode<Uint32T> Word32Shl(TNode<Uint32T> left, TNode<Uint32T> right) {
    return Unsigned(
        Word32Shl(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
  TNode<Uint32T> Word32Shr(TNode<Uint32T> left, TNode<Uint32T> right) {
    return Unsigned(
        Word32Shr(static_cast<Node*>(left), static_cast<Node*>(right)));
805
  }
806

807
  TNode<IntPtrT> WordAnd(TNode<IntPtrT> left, TNode<IntPtrT> right) {
808 809 810 811
    return Signed(WordAnd(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
  TNode<UintPtrT> WordAnd(TNode<UintPtrT> left, TNode<UintPtrT> right) {
    return Unsigned(
812 813 814
        WordAnd(static_cast<Node*>(left), static_cast<Node*>(right)));
  }

815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832
  TNode<Int32T> Word32And(TNode<Int32T> left, TNode<Int32T> right) {
    return Signed(
        Word32And(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
  TNode<Uint32T> Word32And(TNode<Uint32T> left, TNode<Uint32T> right) {
    return Unsigned(
        Word32And(static_cast<Node*>(left), static_cast<Node*>(right)));
  }

  TNode<Int32T> Word32Or(TNode<Int32T> left, TNode<Int32T> right) {
    return Signed(
        Word32Or(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
  TNode<Uint32T> Word32Or(TNode<Uint32T> left, TNode<Uint32T> right) {
    return Unsigned(
        Word32Or(static_cast<Node*>(left), static_cast<Node*>(right)));
  }

833 834 835 836 837 838 839
  TNode<BoolT> IntPtrEqual(TNode<WordT> left, TNode<WordT> right);
  TNode<BoolT> WordEqual(TNode<WordT> left, TNode<WordT> right);
  TNode<BoolT> WordNotEqual(TNode<WordT> left, TNode<WordT> right);
  TNode<BoolT> Word32Equal(TNode<Word32T> left, TNode<Word32T> right);
  TNode<BoolT> Word32NotEqual(TNode<Word32T> left, TNode<Word32T> right);
  TNode<BoolT> Word64Equal(TNode<Word64T> left, TNode<Word64T> right);
  TNode<BoolT> Word64NotEqual(TNode<Word64T> left, TNode<Word64T> right);
840

841 842 843 844 845 846 847 848 849
  TNode<BoolT> Word32Or(TNode<BoolT> left, TNode<BoolT> right) {
    return UncheckedCast<BoolT>(
        Word32Or(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
  TNode<BoolT> Word32And(TNode<BoolT> left, TNode<BoolT> right) {
    return UncheckedCast<BoolT>(
        Word32And(static_cast<Node*>(left), static_cast<Node*>(right)));
  }

850 851 852 853 854
  TNode<Int32T> Int32Add(TNode<Int32T> left, TNode<Int32T> right) {
    return Signed(
        Int32Add(static_cast<Node*>(left), static_cast<Node*>(right)));
  }

855 856 857 858 859
  TNode<Uint32T> Uint32Add(TNode<Uint32T> left, TNode<Uint32T> right) {
    return Unsigned(
        Int32Add(static_cast<Node*>(left), static_cast<Node*>(right)));
  }

860 861 862 863 864 865 866 867 868 869
  TNode<Int32T> Int32Sub(TNode<Int32T> left, TNode<Int32T> right) {
    return Signed(
        Int32Sub(static_cast<Node*>(left), static_cast<Node*>(right)));
  }

  TNode<Int32T> Int32Mul(TNode<Int32T> left, TNode<Int32T> right) {
    return Signed(
        Int32Mul(static_cast<Node*>(left), static_cast<Node*>(right)));
  }

870 871 872 873 874 875 876 877
  TNode<IntPtrT> IntPtrAdd(TNode<IntPtrT> left, TNode<IntPtrT> right) {
    return Signed(
        IntPtrAdd(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
  TNode<IntPtrT> IntPtrSub(TNode<IntPtrT> left, TNode<IntPtrT> right) {
    return Signed(
        IntPtrSub(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
878 879 880 881
  TNode<IntPtrT> IntPtrMul(TNode<IntPtrT> left, TNode<IntPtrT> right) {
    return Signed(
        IntPtrMul(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
882 883 884 885 886 887 888 889
  TNode<UintPtrT> UintPtrAdd(TNode<UintPtrT> left, TNode<UintPtrT> right) {
    return Unsigned(
        IntPtrAdd(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
  TNode<UintPtrT> UintPtrSub(TNode<UintPtrT> left, TNode<UintPtrT> right) {
    return Unsigned(
        IntPtrSub(static_cast<Node*>(left), static_cast<Node*>(right)));
  }
890 891 892
  TNode<RawPtrT> RawPtrAdd(TNode<RawPtrT> left, TNode<IntPtrT> right) {
    return ReinterpretCast<RawPtrT>(IntPtrAdd(left, right));
  }
893 894 895 896 897 898
  TNode<RawPtrT> RawPtrSub(TNode<RawPtrT> left, TNode<IntPtrT> right) {
    return ReinterpretCast<RawPtrT>(IntPtrSub(left, right));
  }
  TNode<IntPtrT> RawPtrSub(TNode<RawPtrT> left, TNode<RawPtrT> right) {
    return Signed(
        IntPtrSub(static_cast<Node*>(left), static_cast<Node*>(right)));
899
  }
900 901 902

  TNode<WordT> WordShl(SloppyTNode<WordT> value, int shift);
  TNode<WordT> WordShr(SloppyTNode<WordT> value, int shift);
903
  TNode<WordT> WordSar(SloppyTNode<WordT> value, int shift);
904 905 906
  TNode<IntPtrT> WordShr(TNode<IntPtrT> value, int shift) {
    return UncheckedCast<IntPtrT>(WordShr(static_cast<Node*>(value), shift));
  }
907 908 909
  TNode<IntPtrT> WordSar(TNode<IntPtrT> value, int shift) {
    return UncheckedCast<IntPtrT>(WordSar(static_cast<Node*>(value), shift));
  }
910
  TNode<Word32T> Word32Shr(SloppyTNode<Word32T> value, int shift);
911
  TNode<Word32T> Word32Sar(SloppyTNode<Word32T> value, int shift);
912 913

// Unary
914 915
#define DECLARE_CODE_ASSEMBLER_UNARY_OP(name, ResType, ArgType) \
  TNode<ResType> name(SloppyTNode<ArgType> a);
916 917 918
  CODE_ASSEMBLER_UNARY_OP_LIST(DECLARE_CODE_ASSEMBLER_UNARY_OP)
#undef DECLARE_CODE_ASSEMBLER_UNARY_OP

919 920 921
  template <class Dummy = void>
  TNode<IntPtrT> BitcastTaggedToWord(TNode<Smi> node) {
    static_assert(sizeof(Dummy) < 0,
922
                  "Should use BitcastTaggedToWordForTagAndSmiBits instead.");
923 924
  }

925 926
  // Changes a double to an inptr_t for pointer arithmetic outside of Smi range.
  // Assumes that the double can be exactly represented as an int.
927
  TNode<IntPtrT> ChangeFloat64ToIntPtr(TNode<Float64T> value);
928
  TNode<UintPtrT> ChangeFloat64ToUintPtr(TNode<Float64T> value);
929 930
  // Same in the opposite direction.
  TNode<Float64T> ChangeUintPtrToFloat64(TNode<UintPtrT> value);
931

932 933 934
  // Changes an intptr_t to a double, e.g. for storing an element index
  // outside Smi range in a HeapNumber. Lossless on 32-bit,
  // rounds on 64-bit (which doesn't affect valid element indices).
935
  TNode<Float64T> RoundIntPtrToFloat64(Node* value);
936
  // No-op on 32-bit, otherwise zero extend.
937
  TNode<UintPtrT> ChangeUint32ToWord(TNode<Word32T> value);
938
  // No-op on 32-bit, otherwise sign extend.
939
  TNode<IntPtrT> ChangeInt32ToIntPtr(TNode<Word32T> value);
940

941 942 943 944
  // No-op that guarantees that the value is kept alive till this point even
  // if GC happens.
  Node* Retain(Node* value);

945 946 947
  // Projections
  Node* Projection(int index, Node* value);

948 949 950 951 952 953 954 955
  template <int index, class T1, class T2>
  TNode<typename std::tuple_element<index, std::tuple<T1, T2>>::type>
  Projection(TNode<PairT<T1, T2>> value) {
    return UncheckedCast<
        typename std::tuple_element<index, std::tuple<T1, T2>>::type>(
        Projection(index, value));
  }

956
  // Calls
957
  template <class... TArgs>
958 959
  TNode<Object> CallRuntime(Runtime::FunctionId function, TNode<Object> context,
                            TArgs... args) {
960
    return CallRuntimeImpl(function, context,
961
                           {implicit_cast<TNode<Object>>(args)...});
962
  }
963

964
  template <class... TArgs>
965 966
  void TailCallRuntime(Runtime::FunctionId function, TNode<Object> context,
                       TArgs... args) {
967 968 969
    int argc = static_cast<int>(sizeof...(args));
    TNode<Int32T> arity = Int32Constant(argc);
    return TailCallRuntimeImpl(function, arity, context,
970
                               {implicit_cast<TNode<Object>>(args)...});
971
  }
972

973
  template <class... TArgs>
974
  void TailCallRuntime(Runtime::FunctionId function, TNode<Int32T> arity,
975
                       TNode<Object> context, TArgs... args) {
976
    return TailCallRuntimeImpl(function, arity, context,
977
                               {implicit_cast<TNode<Object>>(args)...});
978 979
  }

980 981 982 983
  //
  // If context passed to CallStub is nullptr, it won't be passed to the stub.
  //

984
  template <class T = Object, class... TArgs>
985
  TNode<T> CallStub(Callable const& callable, TNode<Object> context,
986 987
                    TArgs... args) {
    TNode<Code> target = HeapConstant(callable.code());
988
    return CallStub<T>(callable.descriptor(), target, context, args...);
989
  }
990

991 992
  template <class T = Object, class... TArgs>
  TNode<T> CallStub(const CallInterfaceDescriptor& descriptor,
993
                    TNode<Code> target, TNode<Object> context, TArgs... args) {
994 995
    return UncheckedCast<T>(CallStubR(StubCallMode::kCallCodeObject, descriptor,
                                      1, target, context, args...));
996 997 998
  }

  template <class... TArgs>
999 1000
  Node* CallStubR(StubCallMode call_mode,
                  const CallInterfaceDescriptor& descriptor, size_t result_size,
1001
                  TNode<Object> target, TNode<Object> context, TArgs... args) {
1002 1003
    return CallStubRImpl(call_mode, descriptor, result_size, target, context,
                         {args...});
1004
  }
1005

1006 1007
  Node* CallStubN(StubCallMode call_mode,
                  const CallInterfaceDescriptor& descriptor, size_t result_size,
1008
                  int input_count, Node* const* inputs);
1009

1010 1011 1012 1013 1014 1015 1016 1017
  template <class T = Object, class... TArgs>
  TNode<T> CallBuiltinPointer(const CallInterfaceDescriptor& descriptor,
                              TNode<BuiltinPtr> target, TNode<Object> context,
                              TArgs... args) {
    return UncheckedCast<T>(CallStubR(StubCallMode::kCallBuiltinPointer,
                                      descriptor, 1, target, context, args...));
  }

1018
  template <class... TArgs>
1019
  void TailCallStub(Callable const& callable, TNode<Object> context,
1020 1021
                    TArgs... args) {
    TNode<Code> target = HeapConstant(callable.code());
1022
    TailCallStub(callable.descriptor(), target, context, args...);
1023
  }
1024

1025
  template <class... TArgs>
1026
  void TailCallStub(const CallInterfaceDescriptor& descriptor,
1027
                    TNode<Code> target, TNode<Object> context, TArgs... args) {
1028
    TailCallStubImpl(descriptor, target, context, {args...});
1029
  }
1030

1031
  template <class... TArgs>
1032 1033
  void TailCallBytecodeDispatch(const CallInterfaceDescriptor& descriptor,
                                TNode<RawPtrT> target, TArgs... args);
1034

1035
  template <class... TArgs>
1036
  void TailCallStubThenBytecodeDispatch(
1037 1038
      const CallInterfaceDescriptor& descriptor, Node* target, Node* context,
      TArgs... args) {
1039 1040
    TailCallStubThenBytecodeDispatchImpl(descriptor, target, context,
                                         {args...});
1041
  }
1042

1043 1044 1045 1046 1047 1048 1049
  // Tailcalls to the given code object with JSCall linkage. The JS arguments
  // (including receiver) are supposed to be already on the stack.
  // This is a building block for implementing trampoline stubs that are
  // installed instead of code objects with JSCall linkage.
  // Note that no arguments adaption is going on here - all the JavaScript
  // arguments are left on the stack unmodified. Therefore, this tail call can
  // only be used after arguments adaptation has been performed already.
1050 1051 1052
  void TailCallJSCode(TNode<Code> code, TNode<Context> context,
                      TNode<JSFunction> function, TNode<Object> new_target,
                      TNode<Int32T> arg_count);
1053

1054
  template <class... TArgs>
1055 1056
  TNode<Object> CallJS(Callable const& callable, Node* context, Node* function,
                       Node* receiver, TArgs... args) {
1057
    int argc = static_cast<int>(sizeof...(args));
1058
    TNode<Int32T> arity = Int32Constant(argc);
1059 1060 1061 1062
    TNode<Code> target = HeapConstant(callable.code());
    return CAST(CallJSStubImpl(callable.descriptor(), target, CAST(context),
                               CAST(function), TNode<Object>(), arity,
                               {receiver, args...}));
1063
  }
1064

1065
  template <class... TArgs>
1066
  Node* ConstructJSWithTarget(Callable const& callable, Node* context,
1067
                              Node* function, Node* new_target, TArgs... args) {
1068
    int argc = static_cast<int>(sizeof...(args));
1069 1070
    TNode<Int32T> arity = Int32Constant(argc);
    TNode<Object> receiver = LoadRoot(RootIndex::kUndefinedValue);
1071 1072 1073 1074
    TNode<Code> target = HeapConstant(callable.code());
    return CallJSStubImpl(callable.descriptor(), target, CAST(context),
                          CAST(function), CAST(new_target), arity,
                          {receiver, args...});
1075
  }
1076 1077 1078 1079 1080 1081
  template <class... TArgs>
  Node* ConstructJS(Callable const& callable, Node* context, Node* new_target,
                    TArgs... args) {
    return ConstructJSWithTarget(callable, context, new_target, new_target,
                                 args...);
  }
1082

1083 1084 1085
  Node* CallCFunctionN(Signature<MachineType>* signature, int input_count,
                       Node* const* inputs);

1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097
  // Type representing C function argument with type info.
  using CFunctionArg = std::pair<MachineType, Node*>;

  // Call to a C function.
  template <class... CArgs>
  Node* CallCFunction(Node* function, MachineType return_type, CArgs... cargs) {
    static_assert(v8::internal::conjunction<
                      std::is_convertible<CArgs, CFunctionArg>...>::value,
                  "invalid argument types");
    return CallCFunction(function, return_type, {cargs...});
  }

1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109
  // Call to a C function without a function discriptor on AIX.
  template <class... CArgs>
  Node* CallCFunctionWithoutFunctionDescriptor(Node* function,
                                               MachineType return_type,
                                               CArgs... cargs) {
    static_assert(v8::internal::conjunction<
                      std::is_convertible<CArgs, CFunctionArg>...>::value,
                  "invalid argument types");
    return CallCFunctionWithoutFunctionDescriptor(function, return_type,
                                                  {cargs...});
  }

1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
  // Call to a C function, while saving/restoring caller registers.
  template <class... CArgs>
  Node* CallCFunctionWithCallerSavedRegisters(Node* function,
                                              MachineType return_type,
                                              SaveFPRegsMode mode,
                                              CArgs... cargs) {
    static_assert(v8::internal::conjunction<
                      std::is_convertible<CArgs, CFunctionArg>...>::value,
                  "invalid argument types");
    return CallCFunctionWithCallerSavedRegisters(function, return_type, mode,
                                                 {cargs...});
  }
1122

1123 1124 1125 1126 1127
  // Helpers which delegate to RawMachineAssembler.
  Factory* factory() const;
  Isolate* isolate() const;
  Zone* zone() const;

1128 1129
  CodeAssemblerState* state() { return state_; }

1130 1131
  void BreakOnNode(int node_id);

1132 1133
  bool UnalignedLoadSupported(MachineRepresentation rep) const;
  bool UnalignedStoreSupported(MachineRepresentation rep) const;
1134

1135 1136
  bool IsExceptionHandlerActive() const;

1137
 protected:
1138 1139 1140 1141
  void RegisterCallGenerationCallbacks(
      const CodeAssemblerCallback& call_prologue,
      const CodeAssemblerCallback& call_epilogue);
  void UnregisterCallGenerationCallbacks();
1142

1143
  bool Word32ShiftIsSafe() const;
1144
  PoisoningMitigationLevel poisoning_level() const;
1145

1146 1147
  bool IsJSFunctionCall() const;

1148
 private:
1149 1150
  void HandleException(Node* result);

1151 1152 1153
  Node* CallCFunction(Node* function, MachineType return_type,
                      std::initializer_list<CFunctionArg> args);

1154 1155 1156 1157
  Node* CallCFunctionWithoutFunctionDescriptor(
      Node* function, MachineType return_type,
      std::initializer_list<CFunctionArg> args);

1158 1159 1160 1161
  Node* CallCFunctionWithCallerSavedRegisters(
      Node* function, MachineType return_type, SaveFPRegsMode mode,
      std::initializer_list<CFunctionArg> args);

1162
  TNode<Object> CallRuntimeImpl(Runtime::FunctionId function,
1163 1164
                                TNode<Object> context,
                                std::initializer_list<TNode<Object>> args);
1165 1166

  void TailCallRuntimeImpl(Runtime::FunctionId function, TNode<Int32T> arity,
1167 1168
                           TNode<Object> context,
                           std::initializer_list<TNode<Object>> args);
1169 1170 1171

  void TailCallStubImpl(const CallInterfaceDescriptor& descriptor,
                        TNode<Code> target, TNode<Object> context,
1172 1173
                        std::initializer_list<Node*> args);

1174
  void TailCallStubThenBytecodeDispatchImpl(
1175 1176 1177
      const CallInterfaceDescriptor& descriptor, Node* target, Node* context,
      std::initializer_list<Node*> args);

1178 1179
  Node* CallStubRImpl(StubCallMode call_mode,
                      const CallInterfaceDescriptor& descriptor,
1180 1181
                      size_t result_size, TNode<Object> target,
                      TNode<Object> context, std::initializer_list<Node*> args);
1182

1183 1184 1185 1186 1187
  Node* CallJSStubImpl(const CallInterfaceDescriptor& descriptor,
                       TNode<Object> target, TNode<Object> context,
                       TNode<Object> function, TNode<Object> new_target,
                       TNode<Int32T> arity, std::initializer_list<Node*> args);

1188 1189 1190 1191 1192
  // These two don't have definitions and are here only for catching use cases
  // where the cast is not necessary.
  TNode<Int32T> Signed(TNode<Int32T> x);
  TNode<Uint32T> Unsigned(TNode<Uint32T> x);

1193
  RawMachineAssembler* raw_assembler() const;
1194
  JSGraph* jsgraph() const;
1195

1196 1197 1198 1199
  // Calls respective callback registered in the state.
  void CallPrologue();
  void CallEpilogue();

1200
  CodeAssemblerState* state_;
1201 1202 1203 1204

  DISALLOW_COPY_AND_ASSIGN(CodeAssembler);
};

1205
class V8_EXPORT_PRIVATE CodeAssemblerVariable {
1206 1207 1208
 public:
  explicit CodeAssemblerVariable(CodeAssembler* assembler,
                                 MachineRepresentation rep);
1209 1210
  CodeAssemblerVariable(CodeAssembler* assembler, MachineRepresentation rep,
                        Node* initial_value);
1211 1212 1213 1214 1215 1216 1217
#if DEBUG
  CodeAssemblerVariable(CodeAssembler* assembler, AssemblerDebugInfo debug_info,
                        MachineRepresentation rep);
  CodeAssemblerVariable(CodeAssembler* assembler, AssemblerDebugInfo debug_info,
                        MachineRepresentation rep, Node* initial_value);
#endif  // DEBUG

1218 1219 1220 1221 1222 1223 1224
  ~CodeAssemblerVariable();
  void Bind(Node* value);
  Node* value() const;
  MachineRepresentation rep() const;
  bool IsBound() const;

 private:
1225
  class Impl;
1226 1227
  friend class CodeAssemblerLabel;
  friend class CodeAssemblerState;
1228 1229
  friend std::ostream& operator<<(std::ostream&, const Impl&);
  friend std::ostream& operator<<(std::ostream&, const CodeAssemblerVariable&);
1230 1231 1232 1233
  struct ImplComparator {
    bool operator()(const CodeAssemblerVariable::Impl* a,
                    const CodeAssemblerVariable::Impl* b) const;
  };
1234 1235
  Impl* impl_;
  CodeAssemblerState* state_;
1236
  DISALLOW_COPY_AND_ASSIGN(CodeAssemblerVariable);
1237 1238
};

1239 1240 1241
std::ostream& operator<<(std::ostream&, const CodeAssemblerVariable&);
std::ostream& operator<<(std::ostream&, const CodeAssemblerVariable::Impl&);

1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261
template <class T>
class TypedCodeAssemblerVariable : public CodeAssemblerVariable {
 public:
  TypedCodeAssemblerVariable(TNode<T> initial_value, CodeAssembler* assembler)
      : CodeAssemblerVariable(assembler, MachineRepresentationOf<T>::value,
                              initial_value) {}
  explicit TypedCodeAssemblerVariable(CodeAssembler* assembler)
      : CodeAssemblerVariable(assembler, MachineRepresentationOf<T>::value) {}
#if DEBUG
  TypedCodeAssemblerVariable(AssemblerDebugInfo debug_info,
                             CodeAssembler* assembler)
      : CodeAssemblerVariable(assembler, debug_info,
                              MachineRepresentationOf<T>::value) {}
  TypedCodeAssemblerVariable(AssemblerDebugInfo debug_info,
                             TNode<T> initial_value, CodeAssembler* assembler)
      : CodeAssemblerVariable(assembler, debug_info,
                              MachineRepresentationOf<T>::value,
                              initial_value) {}
#endif  // DEBUG

1262 1263
  TNode<T> value() const {
    return TNode<T>::UncheckedCast(CodeAssemblerVariable::value());
1264 1265 1266
  }

  void operator=(TNode<T> value) { Bind(value); }
1267 1268 1269
  void operator=(const TypedCodeAssemblerVariable<T>& variable) {
    Bind(variable.value());
  }
1270 1271 1272 1273 1274

 private:
  using CodeAssemblerVariable::Bind;
};

1275
class V8_EXPORT_PRIVATE CodeAssemblerLabel {
1276 1277 1278
 public:
  enum Type { kDeferred, kNonDeferred };

1279 1280 1281 1282 1283
  explicit CodeAssemblerLabel(
      CodeAssembler* assembler,
      CodeAssemblerLabel::Type type = CodeAssemblerLabel::kNonDeferred)
      : CodeAssemblerLabel(assembler, 0, nullptr, type) {}
  CodeAssemblerLabel(
1284
      CodeAssembler* assembler,
1285 1286
      const CodeAssemblerVariableList& merged_variables,
      CodeAssemblerLabel::Type type = CodeAssemblerLabel::kNonDeferred)
1287
      : CodeAssemblerLabel(assembler, merged_variables.size(),
1288 1289
                           &(merged_variables[0]), type) {}
  CodeAssemblerLabel(
1290 1291
      CodeAssembler* assembler, size_t count,
      CodeAssemblerVariable* const* vars,
1292
      CodeAssemblerLabel::Type type = CodeAssemblerLabel::kNonDeferred);
1293 1294 1295 1296 1297
  CodeAssemblerLabel(
      CodeAssembler* assembler,
      std::initializer_list<CodeAssemblerVariable*> vars,
      CodeAssemblerLabel::Type type = CodeAssemblerLabel::kNonDeferred)
      : CodeAssemblerLabel(assembler, vars.size(), vars.begin(), type) {}
1298 1299 1300 1301
  CodeAssemblerLabel(
      CodeAssembler* assembler, CodeAssemblerVariable* merged_variable,
      CodeAssemblerLabel::Type type = CodeAssemblerLabel::kNonDeferred)
      : CodeAssemblerLabel(assembler, 1, &merged_variable, type) {}
1302
  ~CodeAssemblerLabel();
1303

1304
  inline bool is_bound() const { return bound_; }
1305
  inline bool is_used() const { return merge_count_ != 0; }
1306

1307 1308 1309 1310
 private:
  friend class CodeAssembler;

  void Bind();
1311 1312 1313 1314
#if DEBUG
  void Bind(AssemblerDebugInfo debug_info);
#endif  // DEBUG
  void UpdateVariablesAfterBind();
1315 1316 1317 1318
  void MergeVariables();

  bool bound_;
  size_t merge_count_;
1319
  CodeAssemblerState* state_;
1320 1321 1322
  RawMachineLabel* label_;
  // Map of variables that need to be merged to their phi nodes (or placeholders
  // for those phis).
1323 1324 1325
  std::map<CodeAssemblerVariable::Impl*, Node*,
           CodeAssemblerVariable::ImplComparator>
      variable_phis_;
1326 1327
  // Map of variables to the list of value nodes that have been added from each
  // merge path in their order of merging.
1328 1329 1330
  std::map<CodeAssemblerVariable::Impl*, std::vector<Node*>,
           CodeAssemblerVariable::ImplComparator>
      variable_merges_;
1331 1332 1333 1334

  // Cannot be copied because the destructor explicitly call the destructor of
  // the underlying {RawMachineLabel}, hence only one pointer can point to it.
  DISALLOW_COPY_AND_ASSIGN(CodeAssemblerLabel);
1335 1336
};

1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370
class CodeAssemblerParameterizedLabelBase {
 public:
  bool is_used() const { return plain_label_.is_used(); }
  explicit CodeAssemblerParameterizedLabelBase(CodeAssembler* assembler,
                                               size_t arity,
                                               CodeAssemblerLabel::Type type)
      : state_(assembler->state()),
        phi_inputs_(arity),
        plain_label_(assembler, type) {}

 protected:
  CodeAssemblerLabel* plain_label() { return &plain_label_; }
  void AddInputs(std::vector<Node*> inputs);
  Node* CreatePhi(MachineRepresentation rep, const std::vector<Node*>& inputs);
  const std::vector<Node*>& CreatePhis(
      std::vector<MachineRepresentation> representations);

 private:
  CodeAssemblerState* state_;
  std::vector<std::vector<Node*>> phi_inputs_;
  std::vector<Node*> phi_nodes_;
  CodeAssemblerLabel plain_label_;
};

template <class... Types>
class CodeAssemblerParameterizedLabel
    : public CodeAssemblerParameterizedLabelBase {
 public:
  static constexpr size_t kArity = sizeof...(Types);
  explicit CodeAssemblerParameterizedLabel(CodeAssembler* assembler,
                                           CodeAssemblerLabel::Type type)
      : CodeAssemblerParameterizedLabelBase(assembler, kArity, type) {}

 private:
1371
  friend class CodeAssembler;
1372

1373 1374 1375
  void AddInputsVector(std::vector<Node*> inputs) {
    CodeAssemblerParameterizedLabelBase::AddInputs(std::move(inputs));
  }
1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393
  void AddInputs(TNode<Types>... inputs) {
    CodeAssemblerParameterizedLabelBase::AddInputs(
        std::vector<Node*>{inputs...});
  }
  void CreatePhis(TNode<Types>*... results) {
    const std::vector<Node*>& phi_nodes =
        CodeAssemblerParameterizedLabelBase::CreatePhis(
            {MachineRepresentationOf<Types>::value...});
    auto it = phi_nodes.begin();
    USE(it);
    ITERATE_PACK(AssignPhi(results, *(it++)));
  }
  template <class T>
  static void AssignPhi(TNode<T>* result, Node* phi) {
    if (phi != nullptr) *result = TNode<T>::UncheckedCast(phi);
  }
};

1394 1395
using CodeAssemblerExceptionHandlerLabel =
    CodeAssemblerParameterizedLabel<Object>;
1396

1397 1398 1399 1400 1401 1402
class V8_EXPORT_PRIVATE CodeAssemblerState {
 public:
  // Create with CallStub linkage.
  // |result_size| specifies the number of results returned by the stub.
  // TODO(rmcilroy): move result_size to the CallInterfaceDescriptor.
  CodeAssemblerState(Isolate* isolate, Zone* zone,
1403
                     const CallInterfaceDescriptor& descriptor, Code::Kind kind,
1404
                     const char* name, PoisoningMitigationLevel poisoning_level,
1405
                     int32_t builtin_index = Builtins::kNoBuiltinId);
1406 1407 1408

  // Create with JSCall linkage.
  CodeAssemblerState(Isolate* isolate, Zone* zone, int parameter_count,
1409
                     Code::Kind kind, const char* name,
1410
                     PoisoningMitigationLevel poisoning_level,
1411
                     int32_t builtin_index = Builtins::kNoBuiltinId);
1412 1413 1414

  ~CodeAssemblerState();

1415
  const char* name() const { return name_; }
1416
  int parameter_count() const;
1417

1418 1419 1420
#if DEBUG
  void PrintCurrentBlock(std::ostream& os);
#endif  // DEBUG
1421
  bool InsideBlock();
1422 1423
  void SetInitialDebugInformation(const char* msg, const char* file, int line);

1424 1425
 private:
  friend class CodeAssembler;
1426 1427
  friend class CodeAssemblerLabel;
  friend class CodeAssemblerVariable;
1428
  friend class CodeAssemblerTester;
1429
  friend class CodeAssemblerParameterizedLabelBase;
1430
  friend class ScopedExceptionHandler;
1431 1432

  CodeAssemblerState(Isolate* isolate, Zone* zone,
1433
                     CallDescriptor* call_descriptor, Code::Kind kind,
1434
                     const char* name, PoisoningMitigationLevel poisoning_level,
1435
                     int32_t builtin_index);
1436

1437 1438 1439
  void PushExceptionHandler(CodeAssemblerExceptionHandlerLabel* label);
  void PopExceptionHandler();

1440
  std::unique_ptr<RawMachineAssembler> raw_assembler_;
1441
  Code::Kind kind_;
1442
  const char* name_;
1443
  int32_t builtin_index_;
1444
  bool code_generated_;
1445 1446
  ZoneSet<CodeAssemblerVariable::Impl*, CodeAssemblerVariable::ImplComparator>
      variables_;
1447 1448
  CodeAssemblerCallback call_prologue_;
  CodeAssemblerCallback call_epilogue_;
1449
  std::vector<CodeAssemblerExceptionHandlerLabel*> exception_handler_labels_;
1450
  using VariableId = uint32_t;
1451
  VariableId next_variable_id_ = 0;
1452
  JSGraph* jsgraph_;
1453

1454
  VariableId NextVariableId() { return next_variable_id_++; }
1455 1456 1457 1458

  DISALLOW_COPY_AND_ASSIGN(CodeAssemblerState);
};

1459
class V8_EXPORT_PRIVATE ScopedExceptionHandler {
1460
 public:
1461 1462
  ScopedExceptionHandler(CodeAssembler* assembler,
                         CodeAssemblerExceptionHandlerLabel* label);
1463

1464 1465
  // Use this constructor for compatability/ports of old CSA code only. New code
  // should use the CodeAssemblerExceptionHandlerLabel version.
1466 1467
  ScopedExceptionHandler(CodeAssembler* assembler, CodeAssemblerLabel* label,
                         TypedCodeAssemblerVariable<Object>* exception);
1468

1469
  ~ScopedExceptionHandler();
1470 1471

 private:
1472
  bool has_handler_;
1473
  CodeAssembler* assembler_;
1474 1475 1476
  CodeAssemblerLabel* compatibility_label_;
  std::unique_ptr<CodeAssemblerExceptionHandlerLabel> label_;
  TypedCodeAssemblerVariable<Object>* exception_;
1477 1478
};

1479
}  // namespace compiler
1480

1481
#if defined(V8_HOST_ARCH_32_BIT)
1482
#define BINT_IS_SMI
1483
using BInt = Smi;
1484
#elif defined(V8_HOST_ARCH_64_BIT)
1485
#define BINT_IS_INTPTR
1486
using BInt = IntPtrT;
1487 1488 1489 1490
#else
#error Unknown architecture.
#endif

1491 1492 1493 1494
}  // namespace internal
}  // namespace v8

#endif  // V8_COMPILER_CODE_ASSEMBLER_H_