bytecode-graph-builder.cc 182 KB
Newer Older
1 2 3 4 5 6
// Copyright 2015 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include "src/compiler/bytecode-graph-builder.h"

7
#include "src/ast/ast.h"
8
#include "src/base/platform/wrappers.h"
9
#include "src/codegen/source-position-table.h"
10
#include "src/codegen/tick-counter.h"
11
#include "src/common/assert-scope.h"
12
#include "src/compiler/access-builder.h"
13
#include "src/compiler/bytecode-analysis.h"
14
#include "src/compiler/compiler-source-position-table.h"
15
#include "src/compiler/js-heap-broker.h"
16
#include "src/compiler/linkage.h"
17
#include "src/compiler/node-matchers.h"
18
#include "src/compiler/node-observer.h"
19
#include "src/compiler/operator-properties.h"
20
#include "src/compiler/simplified-operator.h"
21 22 23
#include "src/compiler/state-values-utils.h"
#include "src/interpreter/bytecode-array-iterator.h"
#include "src/interpreter/bytecode-flags.h"
24
#include "src/interpreter/bytecodes.h"
25
#include "src/objects/js-array-inl.h"
26
#include "src/objects/js-generator.h"
27
#include "src/objects/literal-objects-inl.h"
28
#include "src/objects/objects-inl.h"
29
#include "src/objects/smi.h"
30
#include "src/objects/template-objects.h"
31 32 33 34 35

namespace v8 {
namespace internal {
namespace compiler {

36 37
class BytecodeGraphBuilder {
 public:
38
  BytecodeGraphBuilder(JSHeapBroker* broker, Zone* local_zone,
39 40
                       NativeContextRef const& native_context,
                       SharedFunctionInfoRef const& shared_info,
41
                       FeedbackCellRef const& feedback_cell,
42
                       BytecodeOffset osr_offset, JSGraph* jsgraph,
43
                       CallFrequency const& invocation_frequency,
44
                       SourcePositionTable* source_positions, int inlining_id,
45
                       CodeKind code_kind, BytecodeGraphBuilderFlags flags,
46 47
                       TickCounter* tick_counter,
                       ObserveNodeInfo const& observe_node_info);
48

49 50 51
  BytecodeGraphBuilder(const BytecodeGraphBuilder&) = delete;
  BytecodeGraphBuilder& operator=(const BytecodeGraphBuilder&) = delete;

52 53 54 55 56 57 58 59 60
  // Creates a graph by visiting bytecodes.
  void CreateGraph();

 private:
  class Environment;
  class OsrIteratorState;
  struct SubEnvironment;

  void RemoveMergeEnvironmentsBeforeOffset(int limit_offset);
61
  void AdvanceToOsrEntryAndPeelLoops();
62

63 64 65 66
  // Advance {bytecode_iterator} to the given offset. If possible, also advance
  // {source_position_iterator} while updating the source position table.
  void AdvanceIteratorsTo(int bytecode_offset);

67
  void VisitSingleBytecode();
68 69 70 71 72
  void VisitBytecodes();

  // Get or create the node that represents the outer function closure.
  Node* GetFunctionClosure();

73
  CodeKind code_kind() const { return code_kind_; }
74
  bool native_context_independent() const {
75
    return CodeKindIsNativeContextIndependentJSFunction(code_kind_);
76
  }
77
  bool is_turboprop() const { return code_kind_ == CodeKind::TURBOPROP; }
78 79 80 81 82 83 84
  bool generate_full_feedback_collection() const {
    // NCI code currently collects full feedback.
    DCHECK_IMPLIES(native_context_independent(),
                   CollectFeedbackInGenericLowering());
    return native_context_independent();
  }

85 86 87
  static JSTypeHintLowering::LoweringResult NoChange() {
    return JSTypeHintLowering::LoweringResult::NoChange();
  }
88 89 90 91 92 93 94
  bool CanApplyTypeHintLowering(IrOpcode::Value opcode) const {
    return !generate_full_feedback_collection() ||
           !IrOpcode::IsFeedbackCollectingOpcode(opcode);
  }
  bool CanApplyTypeHintLowering(const Operator* op) const {
    return CanApplyTypeHintLowering(static_cast<IrOpcode::Value>(op->opcode()));
  }
95 96 97 98 99 100 101 102 103 104 105 106 107 108

  // The node representing the current feedback vector is generated once prior
  // to visiting bytecodes, and is later passed as input to other nodes that
  // may need it.
  // TODO(jgruber): Remove feedback_vector() and rename feedback_vector_node()
  // to feedback_vector() once all uses of the direct heap object reference
  // have been replaced with a Node* reference.
  void CreateFeedbackVectorNode();
  Node* BuildLoadFeedbackVector();
  Node* feedback_vector_node() const {
    DCHECK_NOT_NULL(feedback_vector_node_);
    return feedback_vector_node_;
  }

109 110 111 112 113 114 115
  void CreateFeedbackCellNode();
  Node* BuildLoadFeedbackCell();
  Node* feedback_cell_node() const {
    DCHECK_NOT_NULL(feedback_cell_node_);
    return feedback_cell_node_;
  }

116 117 118 119 120 121 122 123
  // Same as above for the feedback vector node.
  void CreateNativeContextNode();
  Node* BuildLoadNativeContext();
  Node* native_context_node() const {
    DCHECK_NOT_NULL(native_context_node_);
    return native_context_node_;
  }

124 125
  Node* BuildLoadFeedbackCell(int index);

126 127
  // Checks the optimization marker and potentially triggers compilation or
  // installs the finished code object.
128
  // Only relevant for specific code kinds (see CodeKindCanTierUp).
129 130
  void MaybeBuildTierUpCheck();

131 132 133 134 135 136 137
  // Like bytecode, NCI code must collect call feedback to preserve proper
  // behavior of inlining heuristics when tiering up to Turbofan in the future.
  // The invocation count (how often a particular JSFunction has been called)
  // is tracked by the callee. For bytecode, this happens in the
  // InterpreterEntryTrampoline, for NCI code it happens here in the prologue.
  void MaybeBuildIncrementInvocationCount();

138 139 140
  // Builder for loading the a native context field.
  Node* BuildLoadNativeContextField(int index);

141 142 143
  // Helper function for creating a feedback source containing type feedback
  // vector and a feedback slot.
  FeedbackSource CreateFeedbackSource(int slot_id);
144
  FeedbackSource CreateFeedbackSource(FeedbackSlot slot);
145 146 147 148 149 150 151 152 153 154

  void set_environment(Environment* env) { environment_ = env; }
  const Environment* environment() const { return environment_; }
  Environment* environment() { return environment_; }

  // Node creation helpers
  Node* NewNode(const Operator* op, bool incomplete = false) {
    return MakeNode(op, 0, static_cast<Node**>(nullptr), incomplete);
  }

155 156 157
  template <class... Args>
  Node* NewNode(const Operator* op, Node* n0, Args... nodes) {
    Node* buffer[] = {n0, nodes...};
158
    return MakeNode(op, arraysize(buffer), buffer);
159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187
  }

  // Helpers to create new control nodes.
  Node* NewIfTrue() { return NewNode(common()->IfTrue()); }
  Node* NewIfFalse() { return NewNode(common()->IfFalse()); }
  Node* NewIfValue(int32_t value) { return NewNode(common()->IfValue(value)); }
  Node* NewIfDefault() { return NewNode(common()->IfDefault()); }
  Node* NewMerge() { return NewNode(common()->Merge(1), true); }
  Node* NewLoop() { return NewNode(common()->Loop(1), true); }
  Node* NewBranch(Node* condition, BranchHint hint = BranchHint::kNone,
                  IsSafetyCheck is_safety_check = IsSafetyCheck::kSafetyCheck) {
    return NewNode(common()->Branch(hint, is_safety_check), condition);
  }
  Node* NewSwitch(Node* condition, int control_output_count) {
    return NewNode(common()->Switch(control_output_count), condition);
  }

  // Creates a new Phi node having {count} input values.
  Node* NewPhi(int count, Node* input, Node* control);
  Node* NewEffectPhi(int count, Node* input, Node* control);

  // Helpers for merging control, effect or value dependencies.
  Node* MergeControl(Node* control, Node* other);
  Node* MergeEffect(Node* effect, Node* other_effect, Node* control);
  Node* MergeValue(Node* value, Node* other_value, Node* control);

  // The main node creation chokepoint. Adds context, frame state, effect,
  // and control dependencies depending on the operator.
  Node* MakeNode(const Operator* op, int value_input_count,
188
                 Node* const* value_inputs, bool incomplete = false);
189 190 191 192

  Node** EnsureInputBufferSize(int size);

  Node* const* GetCallArgumentsFromRegisters(Node* callee, Node* receiver,
193 194
                                              interpreter::Register first_arg,
                                              int arg_count);
195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211
  Node* const* ProcessCallVarArgs(ConvertReceiverMode receiver_mode,
                                  Node* callee, interpreter::Register first_reg,
                                  int arg_count);
  Node* const* GetConstructArgumentsFromRegister(
      Node* target, Node* new_target, interpreter::Register first_arg,
      int arg_count);
  Node* ProcessCallRuntimeArguments(const Operator* call_runtime_op,
                                    interpreter::Register receiver,
                                    size_t reg_count);

  // Prepare information for eager deoptimization. This information is carried
  // by dedicated {Checkpoint} nodes that are wired into the effect chain.
  // Conceptually this frame state is "before" a given operation.
  void PrepareEagerCheckpoint();

  // Prepare information for lazy deoptimization. This information is attached
  // to the given node and the output value produced by the node is combined.
212 213
  //
  // The low-level chokepoint - use the variants below instead.
214
  void PrepareFrameState(Node* node, OutputFrameStateCombine combine,
215
                         BytecodeOffset bailout_id,
216 217 218 219 220 221 222
                         const BytecodeLivenessState* liveness);

  // In the common case, frame states are conceptually "after" a given
  // operation and at the current bytecode offset.
  void PrepareFrameState(Node* node, OutputFrameStateCombine combine) {
    if (!OperatorProperties::HasFrameStateInput(node->op())) return;
    const int offset = bytecode_iterator().current_offset();
223
    return PrepareFrameState(node, combine, BytecodeOffset(offset),
224 225 226 227 228 229 230 231 232 233 234 235 236
                             bytecode_analysis().GetOutLivenessFor(offset));
  }

  // For function-entry stack checks, they're conceptually "before" the first
  // bytecode and at a special marker bytecode offset.
  // In the case of FE stack checks, the current bytecode is also the first
  // bytecode, so we use a special marker bytecode offset to signify a virtual
  // bytecode before the first physical bytecode.
  void PrepareFrameStateForFunctionEntryStackCheck(Node* node) {
    DCHECK_EQ(bytecode_iterator().current_offset(), 0);
    DCHECK(OperatorProperties::HasFrameStateInput(node->op()));
    DCHECK(node->opcode() == IrOpcode::kJSStackCheck);
    return PrepareFrameState(node, OutputFrameStateCombine::Ignore(),
237
                             BytecodeOffset(kFunctionEntryBytecodeOffset),
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
                             bytecode_analysis().GetInLivenessFor(0));
  }

  // For OSR-entry stack checks, they're conceptually "before" the first
  // bytecode of the current loop. We implement this in a similar manner to
  // function-entry (FE) stack checks above, i.e. we deopt at the predecessor
  // of the current bytecode.
  // In the case of OSR-entry stack checks, a physical predecessor bytecode
  // exists: the JumpLoop bytecode. We attach to JumpLoop by using
  // `bytecode_analysis().osr_bailout_id()` instead of current_offset (the
  // former points at JumpLoop, the latter at the loop header, i.e. the target
  // of JumpLoop).
  void PrepareFrameStateForOSREntryStackCheck(Node* node) {
    DCHECK_EQ(bytecode_iterator().current_offset(),
              bytecode_analysis().osr_entry_point());
    DCHECK(OperatorProperties::HasFrameStateInput(node->op()));
    DCHECK(node->opcode() == IrOpcode::kJSStackCheck);
    const int offset = bytecode_analysis().osr_bailout_id().ToInt();
    return PrepareFrameState(node, OutputFrameStateCombine::Ignore(),
257
                             BytecodeOffset(offset),
258 259
                             bytecode_analysis().GetOutLivenessFor(offset));
  }
260 261

  void BuildCreateArguments(CreateArgumentsType type);
262
  Node* BuildLoadGlobal(NameRef name, uint32_t feedback_slot_index,
263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310
                        TypeofMode typeof_mode);

  enum class StoreMode {
    // Check the prototype chain before storing.
    kNormal,
    // Store value to the receiver without checking the prototype chain.
    kOwn,
  };
  void BuildNamedStore(StoreMode store_mode);
  void BuildLdaLookupSlot(TypeofMode typeof_mode);
  void BuildLdaLookupContextSlot(TypeofMode typeof_mode);
  void BuildLdaLookupGlobalSlot(TypeofMode typeof_mode);
  void BuildCallVarArgs(ConvertReceiverMode receiver_mode);
  void BuildCall(ConvertReceiverMode receiver_mode, Node* const* args,
                 size_t arg_count, int slot_id);
  void BuildCall(ConvertReceiverMode receiver_mode,
                 std::initializer_list<Node*> args, int slot_id) {
    BuildCall(receiver_mode, args.begin(), args.size(), slot_id);
  }
  void BuildUnaryOp(const Operator* op);
  void BuildBinaryOp(const Operator* op);
  void BuildBinaryOpWithImmediate(const Operator* op);
  void BuildCompareOp(const Operator* op);
  void BuildDelete(LanguageMode language_mode);
  void BuildCastOperator(const Operator* op);
  void BuildHoleCheckAndThrow(Node* condition, Runtime::FunctionId runtime_id,
                              Node* name = nullptr);

  // Optional early lowering to the simplified operator level.  Note that
  // the result has already been wired into the environment just like
  // any other invocation of {NewNode} would do.
  JSTypeHintLowering::LoweringResult TryBuildSimplifiedUnaryOp(
      const Operator* op, Node* operand, FeedbackSlot slot);
  JSTypeHintLowering::LoweringResult TryBuildSimplifiedBinaryOp(
      const Operator* op, Node* left, Node* right, FeedbackSlot slot);
  JSTypeHintLowering::LoweringResult TryBuildSimplifiedForInNext(
      Node* receiver, Node* cache_array, Node* cache_type, Node* index,
      FeedbackSlot slot);
  JSTypeHintLowering::LoweringResult TryBuildSimplifiedForInPrepare(
      Node* receiver, FeedbackSlot slot);
  JSTypeHintLowering::LoweringResult TryBuildSimplifiedToNumber(
      Node* input, FeedbackSlot slot);
  JSTypeHintLowering::LoweringResult TryBuildSimplifiedCall(const Operator* op,
                                                            Node* const* args,
                                                            int arg_count,
                                                            FeedbackSlot slot);
  JSTypeHintLowering::LoweringResult TryBuildSimplifiedConstruct(
      const Operator* op, Node* const* args, int arg_count, FeedbackSlot slot);
311 312 313
  JSTypeHintLowering::LoweringResult TryBuildSimplifiedGetIterator(
      const Operator* op, Node* receiver, FeedbackSlot load_slot,
      FeedbackSlot call_slot);
314
  JSTypeHintLowering::LoweringResult TryBuildSimplifiedLoadNamed(
315
      const Operator* op, FeedbackSlot slot);
316 317 318 319 320 321 322 323 324 325 326 327 328
  JSTypeHintLowering::LoweringResult TryBuildSimplifiedLoadKeyed(
      const Operator* op, Node* receiver, Node* key, FeedbackSlot slot);
  JSTypeHintLowering::LoweringResult TryBuildSimplifiedStoreNamed(
      const Operator* op, Node* receiver, Node* value, FeedbackSlot slot);
  JSTypeHintLowering::LoweringResult TryBuildSimplifiedStoreKeyed(
      const Operator* op, Node* receiver, Node* key, Node* value,
      FeedbackSlot slot);

  // Applies the given early reduction onto the current environment.
  void ApplyEarlyReduction(JSTypeHintLowering::LoweringResult reduction);

  // Check the context chain for extensions, for lookup fast paths.
  Environment* CheckContextExtensions(uint32_t depth);
329 330 331 332 333 334 335
  // Slow path taken when we cannot figure out the current scope info.
  Environment* CheckContextExtensionsSlowPath(uint32_t depth);
  // Helper function that tries to get the current scope info.
  base::Optional<ScopeInfoRef> TryGetScopeInfo();
  // Helper function to create a context extension check.
  Environment* CheckContextExtensionAtDepth(Environment* slow_environment,
                                            uint32_t depth);
336 337

  // Helper function to create for-in mode from the recorded type feedback.
338
  ForInMode GetForInMode(FeedbackSlot slot);
339 340

  // Helper function to compute call frequency from the recorded type
341 342
  // feedback. Returns unknown if invocation count is unknown. Returns 0 if
  // feedback is insufficient.
343 344 345
  CallFrequency ComputeCallFrequency(int slot_id) const;

  // Helper function to extract the speculation mode from the recorded type
346
  // feedback. Returns kDisallowSpeculation if feedback is insufficient.
347 348
  SpeculationMode GetSpeculationMode(int slot_id) const;

349 350
  // Helpers for building the implicit FunctionEntry and IterationBody
  // StackChecks.
351
  void BuildFunctionEntryStackCheck();
352
  void BuildIterationBodyStackCheck();
353
  void MaybeBuildOSREntryStackCheck();
354

355 356 357 358 359 360 361 362 363 364 365 366 367
  // Control flow plumbing.
  void BuildJump();
  void BuildJumpIf(Node* condition);
  void BuildJumpIfNot(Node* condition);
  void BuildJumpIfEqual(Node* comperand);
  void BuildJumpIfNotEqual(Node* comperand);
  void BuildJumpIfTrue();
  void BuildJumpIfFalse();
  void BuildJumpIfToBooleanTrue();
  void BuildJumpIfToBooleanFalse();
  void BuildJumpIfNotHole();
  void BuildJumpIfJSReceiver();

368 369
  void BuildUpdateInterruptBudget(int delta);

370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397
  void BuildSwitchOnSmi(Node* condition);
  void BuildSwitchOnGeneratorState(
      const ZoneVector<ResumeJumpTarget>& resume_jump_targets,
      bool allow_fallthrough_on_executing);

  // Simulates control flow by forward-propagating environments.
  void MergeIntoSuccessorEnvironment(int target_offset);
  void BuildLoopHeaderEnvironment(int current_offset);
  void SwitchToMergeEnvironment(int current_offset);

  // Simulates control flow that exits the function body.
  void MergeControlToLeaveFunction(Node* exit);

  // Builds loop exit nodes for every exited loop between the current bytecode
  // offset and {target_offset}.
  void BuildLoopExitsForBranch(int target_offset);
  void BuildLoopExitsForFunctionExit(const BytecodeLivenessState* liveness);
  void BuildLoopExitsUntilLoop(int loop_offset,
                               const BytecodeLivenessState* liveness);

  // Helper for building a return (from an actual return or a suspend).
  void BuildReturn(const BytecodeLivenessState* liveness);

  // Simulates entry and exit of exception handlers.
  void ExitThenEnterExceptionHandlers(int current_offset);

  // Update the current position of the {SourcePositionTable} to that of the
  // bytecode at {offset}, if any.
398
  void UpdateSourcePosition(int offset);
399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414

  // Growth increment for the temporary buffer used to construct input lists to
  // new nodes.
  static const int kInputBufferSizeIncrement = 64;

  // An abstract representation for an exception handler that is being
  // entered and exited while the graph builder is iterating over the
  // underlying bytecode. The exception handlers within the bytecode are
  // well scoped, hence will form a stack during iteration.
  struct ExceptionHandler {
    int start_offset_;      // Start offset of the handled area in the bytecode.
    int end_offset_;        // End offset of the handled area in the bytecode.
    int handler_offset_;    // Handler entry offset within the bytecode.
    int context_register_;  // Index of register holding handler context.
  };

415 416 417 418 419 420
  Handle<Object> GetConstantForIndexOperand(int operand_index) const {
    return broker_->CanonicalPersistentHandle(
        bytecode_iterator().GetConstantForIndexOperand(operand_index,
                                                       local_isolate_));
  }

421 422 423 424 425 426 427 428 429 430
  Graph* graph() const { return jsgraph_->graph(); }
  CommonOperatorBuilder* common() const { return jsgraph_->common(); }
  Zone* graph_zone() const { return graph()->zone(); }
  JSGraph* jsgraph() const { return jsgraph_; }
  Isolate* isolate() const { return jsgraph_->isolate(); }
  JSOperatorBuilder* javascript() const { return jsgraph_->javascript(); }
  SimplifiedOperatorBuilder* simplified() const {
    return jsgraph_->simplified();
  }
  Zone* local_zone() const { return local_zone_; }
431 432 433 434
  BytecodeArrayRef bytecode_array() const {
    return shared_info().GetBytecodeArray();
  }
  FeedbackVectorRef const& feedback_vector() const { return feedback_vector_; }
435 436 437 438 439 440
  const JSTypeHintLowering& type_hint_lowering() const {
    return type_hint_lowering_;
  }
  const FrameStateFunctionInfo* frame_state_function_info() const {
    return frame_state_function_info_;
  }
441
  SourcePositionTableIterator& source_position_iterator() {
442
    return *source_position_iterator_.get();
443
  }
444 445 446
  interpreter::BytecodeArrayIterator const& bytecode_iterator() const {
    return bytecode_iterator_;
  }
447 448
  interpreter::BytecodeArrayIterator& bytecode_iterator() {
    return bytecode_iterator_;
449
  }
450
  BytecodeAnalysis const& bytecode_analysis() const {
451 452 453 454 455 456 457 458
    return bytecode_analysis_;
  }
  int currently_peeled_loop_offset() const {
    return currently_peeled_loop_offset_;
  }
  void set_currently_peeled_loop_offset(int offset) {
    currently_peeled_loop_offset_ = offset;
  }
459
  bool skip_first_stack_check() const { return skip_first_stack_check_; }
460
  int current_exception_handler() const { return current_exception_handler_; }
461 462 463 464 465 466 467
  void set_current_exception_handler(int index) {
    current_exception_handler_ = index;
  }
  bool needs_eager_checkpoint() const { return needs_eager_checkpoint_; }
  void mark_as_needing_eager_checkpoint(bool value) {
    needs_eager_checkpoint_ = value;
  }
468
  JSHeapBroker* broker() const { return broker_; }
469 470
  NativeContextRef native_context() const { return native_context_; }
  SharedFunctionInfoRef shared_info() const { return shared_info_; }
471

472 473 474 475
#define DECLARE_VISIT_BYTECODE(name, ...) void Visit##name();
  BYTECODE_LIST(DECLARE_VISIT_BYTECODE)
#undef DECLARE_VISIT_BYTECODE

476
  JSHeapBroker* const broker_;
477
  LocalIsolate* const local_isolate_;
478 479
  Zone* const local_zone_;
  JSGraph* const jsgraph_;
480 481 482
  // The native context for which we optimize.
  NativeContextRef const native_context_;
  SharedFunctionInfoRef const shared_info_;
483
  FeedbackCellRef const feedback_cell_;
484
  FeedbackVectorRef const feedback_vector_;
485
  CallFrequency const invocation_frequency_;
486 487
  JSTypeHintLowering const type_hint_lowering_;
  const FrameStateFunctionInfo* const frame_state_function_info_;
488
  std::unique_ptr<SourcePositionTableIterator> source_position_iterator_;
489
  interpreter::BytecodeArrayIterator bytecode_iterator_;
490
  BytecodeAnalysis const bytecode_analysis_;
491
  Environment* environment_;
492
  bool const osr_;
493
  int currently_peeled_loop_offset_;
494
  bool is_osr_entry_stack_check_pending_;
495 496

  const bool skip_first_stack_check_;
497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519

  // Merge environments are snapshots of the environment at points where the
  // control flow merges. This models a forward data flow propagation of all
  // values from all predecessors of the merge in question. They are indexed by
  // the bytecode offset
  ZoneMap<int, Environment*> merge_environments_;

  // Generator merge environments are snapshots of the current resume
  // environment, tracing back through loop headers to the resume switch of a
  // generator. They allow us to model a single resume jump as several switch
  // statements across loop headers, keeping those loop headers reducible,
  // without having to merge the "executing" environments of the generator into
  // the "resuming" ones. They are indexed by the suspend id of the resume.
  ZoneMap<int, Environment*> generator_merge_environments_;

  // Exception handlers currently entered by the iteration.
  ZoneStack<ExceptionHandler> exception_handlers_;
  int current_exception_handler_;

  // Temporary storage for building node input lists.
  int input_buffer_size_;
  Node** input_buffer_;

520
  const CodeKind code_kind_;
521
  Node* feedback_cell_node_;
522
  Node* feedback_vector_node_;
523
  Node* native_context_node_;
524

525 526 527 528 529 530 531 532 533 534 535 536 537 538
  // Optimization to only create checkpoints when the current position in the
  // control-flow is not effect-dominated by another checkpoint already. All
  // operations that do not have observable side-effects can be re-evaluated.
  bool needs_eager_checkpoint_;

  // Nodes representing values in the activation record.
  SetOncePointer<Node> function_closure_;

  // Control nodes that exit the function body.
  ZoneVector<Node*> exit_controls_;

  StateValuesCache state_values_cache_;

  // The source position table, to be populated.
539
  SourcePositionTable* const source_positions_;
540 541 542

  SourcePosition const start_position_;

543 544
  TickCounter* const tick_counter_;

545 546
  ObserveNodeInfo const observe_node_info_;

547 548 549 550 551
  static constexpr int kBinaryOperationHintIndex = 1;
  static constexpr int kBinaryOperationSmiHintIndex = 1;
  static constexpr int kCompareOperationHintIndex = 1;
  static constexpr int kCountOperationHintIndex = 0;
  static constexpr int kUnaryOperationHintIndex = 0;
552 553
};

554 555 556 557 558 559
// The abstract execution environment simulates the content of the interpreter
// register file. The environment performs SSA-renaming of all tracked nodes at
// split and merge points in the control flow.
class BytecodeGraphBuilder::Environment : public ZoneObject {
 public:
  Environment(BytecodeGraphBuilder* builder, int register_count,
560 561 562
              int parameter_count,
              interpreter::Register incoming_new_target_or_generator,
              Node* control_dependency);
563

564 565 566 567 568
  // Specifies whether environment binding methods should attach frame state
  // inputs to nodes representing the value being bound. This is done because
  // the {OutputFrameStateCombine} is closely related to the binding method.
  enum FrameStateAttachmentMode { kAttachFrameState, kDontAttachFrameState };

569 570 571 572 573
  int parameter_count() const { return parameter_count_; }
  int register_count() const { return register_count_; }

  Node* LookupAccumulator() const;
  Node* LookupRegister(interpreter::Register the_register) const;
574
  Node* LookupGeneratorState() const;
575

576 577
  void BindAccumulator(Node* node,
                       FrameStateAttachmentMode mode = kDontAttachFrameState);
578
  void BindRegister(interpreter::Register the_register, Node* node,
579 580 581 582
                    FrameStateAttachmentMode mode = kDontAttachFrameState);
  void BindRegistersToProjections(
      interpreter::Register first_reg, Node* node,
      FrameStateAttachmentMode mode = kDontAttachFrameState);
583
  void BindGeneratorState(Node* node);
584 585
  void RecordAfterState(Node* node,
                        FrameStateAttachmentMode mode = kDontAttachFrameState);
586 587 588 589 590 591 592 593 594

  // Effect dependency tracked by this environment.
  Node* GetEffectDependency() { return effect_dependency_; }
  void UpdateEffectDependency(Node* dependency) {
    effect_dependency_ = dependency;
  }

  // Preserve a checkpoint of the environment for the IR graph. Any
  // further mutation of the environment will not affect checkpoints.
595 596
  Node* Checkpoint(BytecodeOffset bytecode_offset,
                   OutputFrameStateCombine combine,
597
                   const BytecodeLivenessState* liveness);
598 599 600 601 602 603 604 605 606 607

  // Control dependency tracked by this environment.
  Node* GetControlDependency() const { return control_dependency_; }
  void UpdateControlDependency(Node* dependency) {
    control_dependency_ = dependency;
  }

  Node* Context() const { return context_; }
  void SetContext(Node* new_context) { context_ = new_context; }

608
  Environment* Copy();
609
  void Merge(Environment* other, const BytecodeLivenessState* liveness);
610

611
  void FillWithOsrValues();
612 613
  void PrepareForLoop(const BytecodeLoopAssignments& assignments,
                      const BytecodeLivenessState* liveness);
614
  void PrepareForLoopExit(Node* loop,
615 616
                          const BytecodeLoopAssignments& assignments,
                          const BytecodeLivenessState* liveness);
617

618
 private:
619 620
  friend Zone;

621
  explicit Environment(const Environment* copy);
622

623 624
  bool StateValuesRequireUpdate(Node** state_values, Node** values, int count);
  void UpdateStateValues(Node** state_values, Node** values, int count);
625 626
  Node* GetStateValuesFromCache(Node** values, int count,
                                const BitVector* liveness, int liveness_offset);
627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646

  int RegisterToValuesIndex(interpreter::Register the_register) const;

  Zone* zone() const { return builder_->local_zone(); }
  Graph* graph() const { return builder_->graph(); }
  CommonOperatorBuilder* common() const { return builder_->common(); }
  BytecodeGraphBuilder* builder() const { return builder_; }
  const NodeVector* values() const { return &values_; }
  NodeVector* values() { return &values_; }
  int register_base() const { return register_base_; }
  int accumulator_base() const { return accumulator_base_; }

  BytecodeGraphBuilder* builder_;
  int register_count_;
  int parameter_count_;
  Node* context_;
  Node* control_dependency_;
  Node* effect_dependency_;
  NodeVector values_;
  Node* parameters_state_values_;
647
  Node* generator_state_;
648 649 650 651
  int register_base_;
  int accumulator_base_;
};

652 653 654 655 656 657 658 659 660 661 662 663
// A helper for creating a temporary sub-environment for simple branches.
struct BytecodeGraphBuilder::SubEnvironment final {
 public:
  explicit SubEnvironment(BytecodeGraphBuilder* builder)
      : builder_(builder), parent_(builder->environment()->Copy()) {}

  ~SubEnvironment() { builder_->set_environment(parent_); }

 private:
  BytecodeGraphBuilder* builder_;
  BytecodeGraphBuilder::Environment* parent_;
};
664

665 666 667
// Issues:
// - Scopes - intimately tied to AST. Need to eval what is needed.
// - Need to resolve closure parameter treatment.
668 669 670 671
BytecodeGraphBuilder::Environment::Environment(
    BytecodeGraphBuilder* builder, int register_count, int parameter_count,
    interpreter::Register incoming_new_target_or_generator,
    Node* control_dependency)
672 673 674 675 676
    : builder_(builder),
      register_count_(register_count),
      parameter_count_(parameter_count),
      control_dependency_(control_dependency),
      effect_dependency_(control_dependency),
677
      values_(builder->local_zone()),
678 679
      parameters_state_values_(nullptr),
      generator_state_(nullptr) {
680 681
  // The layout of values_ is:
  //
682
  // [receiver] [parameters] [registers] [accumulator]
683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701
  //
  // parameter[0] is the receiver (this), parameters 1..N are the
  // parameters supplied to the method (arg0..argN-1). The accumulator
  // is stored separately.

  // Parameters including the receiver
  for (int i = 0; i < parameter_count; i++) {
    const char* debug_name = (i == 0) ? "%this" : nullptr;
    const Operator* op = common()->Parameter(i, debug_name);
    Node* parameter = builder->graph()->NewNode(op, graph()->start());
    values()->push_back(parameter);
  }

  // Registers
  register_base_ = static_cast<int>(values()->size());
  Node* undefined_constant = builder->jsgraph()->UndefinedConstant();
  values()->insert(values()->end(), register_count, undefined_constant);

  // Accumulator
702 703
  accumulator_base_ = static_cast<int>(values()->size());
  values()->push_back(undefined_constant);
704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719

  // Context
  int context_index = Linkage::GetJSCallContextParamIndex(parameter_count);
  const Operator* op = common()->Parameter(context_index, "%context");
  context_ = builder->graph()->NewNode(op, graph()->start());

  // Incoming new.target or generator register
  if (incoming_new_target_or_generator.is_valid()) {
    int new_target_index =
        Linkage::GetJSCallNewTargetParamIndex(parameter_count);
    const Operator* op = common()->Parameter(new_target_index, "%new.target");
    Node* new_target_node = builder->graph()->NewNode(op, graph()->start());

    int values_index = RegisterToValuesIndex(incoming_new_target_or_generator);
    values()->at(values_index) = new_target_node;
  }
720 721
}

722
BytecodeGraphBuilder::Environment::Environment(
723
    const BytecodeGraphBuilder::Environment* other)
724 725 726 727 728 729 730
    : builder_(other->builder_),
      register_count_(other->register_count_),
      parameter_count_(other->parameter_count_),
      context_(other->context_),
      control_dependency_(other->control_dependency_),
      effect_dependency_(other->effect_dependency_),
      values_(other->zone()),
731
      parameters_state_values_(other->parameters_state_values_),
732
      generator_state_(other->generator_state_),
733
      register_base_(other->register_base_),
734
      accumulator_base_(other->accumulator_base_) {
735 736 737 738
  values_ = other->values_;
}


739 740 741 742 743 744 745 746 747
int BytecodeGraphBuilder::Environment::RegisterToValuesIndex(
    interpreter::Register the_register) const {
  if (the_register.is_parameter()) {
    return the_register.ToParameterIndex(parameter_count());
  } else {
    return the_register.index() + register_base();
  }
}

748 749
Node* BytecodeGraphBuilder::Environment::LookupAccumulator() const {
  return values()->at(accumulator_base_);
750 751
}

752 753 754 755
Node* BytecodeGraphBuilder::Environment::LookupGeneratorState() const {
  DCHECK_NOT_NULL(generator_state_);
  return generator_state_;
}
756 757 758

Node* BytecodeGraphBuilder::Environment::LookupRegister(
    interpreter::Register the_register) const {
759 760
  if (the_register.is_current_context()) {
    return Context();
761 762 763 764 765 766
  } else if (the_register.is_function_closure()) {
    return builder()->GetFunctionClosure();
  } else {
    int values_index = RegisterToValuesIndex(the_register);
    return values()->at(values_index);
  }
767 768
}

769
void BytecodeGraphBuilder::Environment::BindAccumulator(
770 771 772
    Node* node, FrameStateAttachmentMode mode) {
  if (mode == FrameStateAttachmentMode::kAttachFrameState) {
    builder()->PrepareFrameState(node, OutputFrameStateCombine::PokeAt(0));
773 774 775 776
  }
  values()->at(accumulator_base_) = node;
}

777 778 779 780
void BytecodeGraphBuilder::Environment::BindGeneratorState(Node* node) {
  generator_state_ = node;
}

781 782
void BytecodeGraphBuilder::Environment::BindRegister(
    interpreter::Register the_register, Node* node,
783
    FrameStateAttachmentMode mode) {
784
  int values_index = RegisterToValuesIndex(the_register);
785 786 787
  if (mode == FrameStateAttachmentMode::kAttachFrameState) {
    builder()->PrepareFrameState(node, OutputFrameStateCombine::PokeAt(
                                           accumulator_base_ - values_index));
788 789
  }
  values()->at(values_index) = node;
790 791
}

792 793
void BytecodeGraphBuilder::Environment::BindRegistersToProjections(
    interpreter::Register first_reg, Node* node,
794
    FrameStateAttachmentMode mode) {
795
  int values_index = RegisterToValuesIndex(first_reg);
796 797 798
  if (mode == FrameStateAttachmentMode::kAttachFrameState) {
    builder()->PrepareFrameState(node, OutputFrameStateCombine::PokeAt(
                                           accumulator_base_ - values_index));
799 800 801 802 803
  }
  for (int i = 0; i < node->op()->ValueOutputCount(); i++) {
    values()->at(values_index + i) =
        builder()->NewNode(common()->Projection(i), node);
  }
804 805
}

806
void BytecodeGraphBuilder::Environment::RecordAfterState(
807 808 809 810
    Node* node, FrameStateAttachmentMode mode) {
  if (mode == FrameStateAttachmentMode::kAttachFrameState) {
    builder()->PrepareFrameState(node, OutputFrameStateCombine::Ignore());
  }
811 812
}

813
BytecodeGraphBuilder::Environment* BytecodeGraphBuilder::Environment::Copy() {
814
  return zone()->New<Environment>(this);
815 816 817
}

void BytecodeGraphBuilder::Environment::Merge(
818 819
    BytecodeGraphBuilder::Environment* other,
    const BytecodeLivenessState* liveness) {
820 821 822 823 824 825 826 827 828 829 830 831
  // Create a merge of the control dependencies of both environments and update
  // the current environment's control dependency accordingly.
  Node* control = builder()->MergeControl(GetControlDependency(),
                                          other->GetControlDependency());
  UpdateControlDependency(control);

  // Create a merge of the effect dependencies of both environments and update
  // the current environment's effect dependency accordingly.
  Node* effect = builder()->MergeEffect(GetEffectDependency(),
                                        other->GetEffectDependency(), control);
  UpdateEffectDependency(effect);

832 833
  // Introduce Phi nodes for values that are live and have differing inputs at
  // the merge point, potentially extending an existing Phi node if possible.
834
  context_ = builder()->MergeValue(context_, other->context_, control);
835
  for (int i = 0; i < parameter_count(); i++) {
836 837
    values_[i] = builder()->MergeValue(values_[i], other->values_[i], control);
  }
838 839 840
  for (int i = 0; i < register_count(); i++) {
    int index = register_base() + i;
    if (liveness == nullptr || liveness->RegisterIsLive(i)) {
841 842 843 844 845 846 847 848 849 850 851 852
#if DEBUG
      // We only do these DCHECKs when we are not in the resume path of a
      // generator -- this is, when either there is no generator state at all,
      // or the generator state is not the constant "executing" value.
      if (generator_state_ == nullptr ||
          NumberMatcher(generator_state_)
              .Is(JSGeneratorObject::kGeneratorExecuting)) {
        DCHECK_NE(values_[index], builder()->jsgraph()->OptimizedOutConstant());
        DCHECK_NE(other->values_[index],
                  builder()->jsgraph()->OptimizedOutConstant());
      }
#endif
853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873

      values_[index] =
          builder()->MergeValue(values_[index], other->values_[index], control);

    } else {
      values_[index] = builder()->jsgraph()->OptimizedOutConstant();
    }
  }

  if (liveness == nullptr || liveness->AccumulatorIsLive()) {
    DCHECK_NE(values_[accumulator_base()],
              builder()->jsgraph()->OptimizedOutConstant());
    DCHECK_NE(other->values_[accumulator_base()],
              builder()->jsgraph()->OptimizedOutConstant());

    values_[accumulator_base()] =
        builder()->MergeValue(values_[accumulator_base()],
                              other->values_[accumulator_base()], control);
  } else {
    values_[accumulator_base()] = builder()->jsgraph()->OptimizedOutConstant();
  }
874 875 876 877 878 879

  if (generator_state_ != nullptr) {
    DCHECK_NOT_NULL(other->generator_state_);
    generator_state_ = builder()->MergeValue(generator_state_,
                                             other->generator_state_, control);
  }
880 881
}

882
void BytecodeGraphBuilder::Environment::PrepareForLoop(
883 884
    const BytecodeLoopAssignments& assignments,
    const BytecodeLivenessState* liveness) {
885 886 887 888 889 890 891
  // Create a control node for the loop header.
  Node* control = builder()->NewLoop();

  // Create a Phi for external effects.
  Node* effect = builder()->NewEffectPhi(1, GetEffectDependency(), control);
  UpdateEffectDependency(effect);

892 893
  // Create Phis for any values that are live on entry to the loop and may be
  // updated by the end of the loop.
894
  context_ = builder()->NewPhi(1, context_, control);
895 896 897 898 899 900
  for (int i = 0; i < parameter_count(); i++) {
    if (assignments.ContainsParameter(i)) {
      values_[i] = builder()->NewPhi(1, values_[i], control);
    }
  }
  for (int i = 0; i < register_count(); i++) {
901 902
    if (assignments.ContainsLocal(i) &&
        (liveness == nullptr || liveness->RegisterIsLive(i))) {
903 904 905 906
      int index = register_base() + i;
      values_[index] = builder()->NewPhi(1, values_[index], control);
    }
  }
907 908
  // The accumulator should not be live on entry.
  DCHECK_IMPLIES(liveness != nullptr, !liveness->AccumulatorIsLive());
909

910 911 912 913
  if (generator_state_ != nullptr) {
    generator_state_ = builder()->NewPhi(1, generator_state_, control);
  }

914 915 916 917 918 919
  // Connect to the loop end.
  Node* terminate = builder()->graph()->NewNode(
      builder()->common()->Terminate(), effect, control);
  builder()->exit_controls_.push_back(terminate);
}

920
void BytecodeGraphBuilder::Environment::FillWithOsrValues() {
921 922
  Node* start = graph()->start();

923 924
  // Create OSR values for each environment value.
  SetContext(graph()->NewNode(
925
      common()->OsrValue(Linkage::kOsrContextSpillSlotIndex), start));
926 927 928 929 930
  int size = static_cast<int>(values()->size());
  for (int i = 0; i < size; i++) {
    int idx = i;  // Indexing scheme follows {StandardFrame}, adapt accordingly.
    if (i >= register_base()) idx += InterpreterFrameConstants::kExtraSlotCount;
    if (i >= accumulator_base()) idx = Linkage::kOsrAccumulatorRegisterIndex;
931
    values()->at(i) = graph()->NewNode(common()->OsrValue(idx), start);
932 933
  }
}
934

935
bool BytecodeGraphBuilder::Environment::StateValuesRequireUpdate(
936
    Node** state_values, Node** values, int count) {
937 938
  if (*state_values == nullptr) {
    return true;
939
  }
940
  Node::Inputs inputs = (*state_values)->inputs();
941
  if (inputs.count() != count) return true;
942
  for (int i = 0; i < count; i++) {
943
    if (inputs[i] != values[i]) {
944
      return true;
945 946 947 948 949
    }
  }
  return false;
}

950
void BytecodeGraphBuilder::Environment::PrepareForLoopExit(
951 952
    Node* loop, const BytecodeLoopAssignments& assignments,
    const BytecodeLivenessState* liveness) {
953 954 955 956 957 958 959 960 961 962 963 964 965
  DCHECK_EQ(loop->opcode(), IrOpcode::kLoop);

  Node* control = GetControlDependency();

  // Create the loop exit node.
  Node* loop_exit = graph()->NewNode(common()->LoopExit(), control, loop);
  UpdateControlDependency(loop_exit);

  // Rename the effect.
  Node* effect_rename = graph()->NewNode(common()->LoopExitEffect(),
                                         GetEffectDependency(), loop_exit);
  UpdateEffectDependency(effect_rename);

966
  // TODO(jarin) We should also rename context here. However, unconditional
967 968
  // renaming confuses global object and native context specialization.
  // We should only rename if the context is assigned in the loop.
969

970 971
  // Rename the environment values if they were assigned in the loop and are
  // live after exiting the loop.
972 973
  for (int i = 0; i < parameter_count(); i++) {
    if (assignments.ContainsParameter(i)) {
974 975 976
      Node* rename = graph()->NewNode(
          common()->LoopExitValue(MachineRepresentation::kTagged), values_[i],
          loop_exit);
977 978 979 980
      values_[i] = rename;
    }
  }
  for (int i = 0; i < register_count(); i++) {
981 982
    if (assignments.ContainsLocal(i) &&
        (liveness == nullptr || liveness->RegisterIsLive(i))) {
983 984 985
      Node* rename = graph()->NewNode(
          common()->LoopExitValue(MachineRepresentation::kTagged),
          values_[register_base() + i], loop_exit);
986 987 988
      values_[register_base() + i] = rename;
    }
  }
989
  if (liveness == nullptr || liveness->AccumulatorIsLive()) {
990 991 992
    Node* rename = graph()->NewNode(
        common()->LoopExitValue(MachineRepresentation::kTagged),
        values_[accumulator_base()], loop_exit);
993 994
    values_[accumulator_base()] = rename;
  }
995 996

  if (generator_state_ != nullptr) {
997 998 999
    generator_state_ = graph()->NewNode(
        common()->LoopExitValue(MachineRepresentation::kTagged),
        generator_state_, loop_exit);
1000
  }
1001
}
1002 1003

void BytecodeGraphBuilder::Environment::UpdateStateValues(Node** state_values,
1004
                                                          Node** values,
1005
                                                          int count) {
1006
  if (StateValuesRequireUpdate(state_values, values, count)) {
1007
    const Operator* op = common()->StateValues(count, SparseInputMask::Dense());
1008
    (*state_values) = graph()->NewNode(op, count, values);
1009 1010 1011
  }
}

1012 1013 1014
Node* BytecodeGraphBuilder::Environment::GetStateValuesFromCache(
    Node** values, int count, const BitVector* liveness, int liveness_offset) {
  return builder_->state_values_cache_.GetNodeForValues(
1015
      values, static_cast<size_t>(count), liveness, liveness_offset);
1016 1017
}

1018
Node* BytecodeGraphBuilder::Environment::Checkpoint(
1019
    BytecodeOffset bailout_id, OutputFrameStateCombine combine,
1020
    const BytecodeLivenessState* liveness) {
1021 1022 1023
  if (parameter_count() == register_count()) {
    // Re-use the state-value cache if the number of local registers happens
    // to match the parameter count.
1024 1025
    parameters_state_values_ = GetStateValuesFromCache(
        &values()->at(0), parameter_count(), nullptr, 0);
1026 1027 1028 1029
  } else {
    UpdateStateValues(&parameters_state_values_, &values()->at(0),
                      parameter_count());
  }
1030

1031 1032 1033
  Node* registers_state_values =
      GetStateValuesFromCache(&values()->at(register_base()), register_count(),
                              liveness ? &liveness->bit_vector() : nullptr, 0);
1034 1035

  bool accumulator_is_live = !liveness || liveness->AccumulatorIsLive();
1036
  Node* accumulator_state_value =
1037 1038 1039
      accumulator_is_live && combine != OutputFrameStateCombine::PokeAt(0)
          ? values()->at(accumulator_base())
          : builder()->jsgraph()->OptimizedOutConstant();
1040 1041 1042 1043

  const Operator* op = common()->FrameState(
      bailout_id, combine, builder()->frame_state_function_info());
  Node* result = graph()->NewNode(
1044
      op, parameters_state_values_, registers_state_values,
1045
      accumulator_state_value, Context(), builder()->GetFunctionClosure(),
1046 1047 1048 1049 1050
      builder()->graph()->start());

  return result;
}

1051
BytecodeGraphBuilder::BytecodeGraphBuilder(
1052 1053 1054
    JSHeapBroker* broker, Zone* local_zone,
    NativeContextRef const& native_context,
    SharedFunctionInfoRef const& shared_info,
1055
    FeedbackCellRef const& feedback_cell, BytecodeOffset osr_offset,
1056
    JSGraph* jsgraph, CallFrequency const& invocation_frequency,
1057
    SourcePositionTable* source_positions, int inlining_id, CodeKind code_kind,
1058 1059
    BytecodeGraphBuilderFlags flags, TickCounter* tick_counter,
    ObserveNodeInfo const& observe_node_info)
1060
    : broker_(broker),
1061 1062 1063
      local_isolate_(broker_->local_isolate()
                         ? broker_->local_isolate()
                         : broker_->isolate()->AsLocalIsolate()),
1064
      local_zone_(local_zone),
1065
      jsgraph_(jsgraph),
1066 1067
      native_context_(native_context),
      shared_info_(shared_info),
1068
      feedback_cell_(feedback_cell),
1069
      feedback_vector_(feedback_cell.value()->AsFeedbackVector()),
1070
      invocation_frequency_(invocation_frequency),
1071
      type_hint_lowering_(
1072
          broker, jsgraph, feedback_vector_,
1073 1074 1075
          (flags & BytecodeGraphBuilderFlag::kBailoutOnUninitialized)
              ? JSTypeHintLowering::kBailoutOnUninitialized
              : JSTypeHintLowering::kNoFlags),
1076
      frame_state_function_info_(common()->CreateFrameStateFunctionInfo(
1077
          FrameStateType::kUnoptimizedFunction,
1078
          bytecode_array().parameter_count(), bytecode_array().register_count(),
1079
          shared_info.object())),
1080
      source_position_iterator_(std::make_unique<SourcePositionTableIterator>(
1081
          bytecode_array().SourcePositionTable())),
1082
      bytecode_iterator_(bytecode_array().object()),
1083 1084 1085
      bytecode_analysis_(
          bytecode_array().object(), local_zone, osr_offset,
          flags & BytecodeGraphBuilderFlag::kAnalyzeEnvironmentLiveness),
1086
      environment_(nullptr),
1087
      osr_(!osr_offset.IsNone()),
1088
      currently_peeled_loop_offset_(-1),
1089
      is_osr_entry_stack_check_pending_(osr_),
1090 1091
      skip_first_stack_check_(flags &
                              BytecodeGraphBuilderFlag::kSkipFirstStackCheck),
1092
      merge_environments_(local_zone),
1093
      generator_merge_environments_(local_zone),
1094 1095
      exception_handlers_(local_zone),
      current_exception_handler_(0),
1096 1097
      input_buffer_size_(0),
      input_buffer_(nullptr),
1098
      code_kind_(code_kind),
1099
      feedback_cell_node_(nullptr),
1100
      feedback_vector_node_(nullptr),
1101
      native_context_node_(nullptr),
1102
      needs_eager_checkpoint_(true),
1103 1104
      exit_controls_(local_zone),
      state_values_cache_(jsgraph),
1105
      source_positions_(source_positions),
1106
      start_position_(shared_info.StartPosition(), inlining_id),
1107 1108
      tick_counter_(tick_counter),
      observe_node_info_(observe_node_info) {}
1109

1110 1111
Node* BytecodeGraphBuilder::GetFunctionClosure() {
  if (!function_closure_.is_set()) {
1112 1113
    int index = Linkage::kJSCallClosureParamIndex;
    const Operator* op = common()->Parameter(index, "%closure");
1114 1115 1116 1117 1118 1119
    Node* node = NewNode(op, graph()->start());
    function_closure_.set(node);
  }
  return function_closure_.get();
}

1120 1121 1122 1123
void BytecodeGraphBuilder::CreateFeedbackCellNode() {
  DCHECK_NULL(feedback_cell_node_);
  if (native_context_independent()) {
    feedback_cell_node_ = BuildLoadFeedbackCell();
1124 1125
  } else if (is_turboprop()) {
    feedback_cell_node_ = jsgraph()->Constant(feedback_cell_);
1126 1127 1128 1129 1130 1131
  }
}

Node* BytecodeGraphBuilder::BuildLoadFeedbackCell() {
  DCHECK(native_context_independent());
  DCHECK_NULL(feedback_cell_node_);
1132
  return NewNode(
1133
      simplified()->LoadField(AccessBuilder::ForJSFunctionFeedbackCell()),
1134
      GetFunctionClosure());
1135 1136
}

1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150
void BytecodeGraphBuilder::CreateFeedbackVectorNode() {
  DCHECK_NULL(feedback_vector_node_);
  feedback_vector_node_ = native_context_independent()
                              ? BuildLoadFeedbackVector()
                              : jsgraph()->Constant(feedback_vector());
}

Node* BytecodeGraphBuilder::BuildLoadFeedbackVector() {
  DCHECK(native_context_independent());
  DCHECK_NULL(feedback_vector_node_);

  // The feedback vector must exist and remain live while the generated code
  // lives. Specifically that means it must be created when NCI code is
  // installed, and must not be flushed.
1151 1152
  return NewNode(simplified()->LoadField(AccessBuilder::ForFeedbackCellValue()),
                 feedback_cell_node());
1153 1154
}

1155 1156 1157 1158
Node* BytecodeGraphBuilder::BuildLoadFeedbackCell(int index) {
  if (native_context_independent()) {
    // TODO(jgruber,v8:8888): Assumes that the feedback vector has been
    // allocated.
1159 1160 1161 1162
    Node* closure_feedback_cell_array =
        NewNode(simplified()->LoadField(
                    AccessBuilder::ForFeedbackVectorClosureFeedbackCellArray()),
                feedback_vector_node());
1163

1164
    return NewNode(
1165
        simplified()->LoadField(AccessBuilder::ForFixedArraySlot(index)),
1166
        closure_feedback_cell_array);
1167 1168 1169 1170 1171
  } else {
    return jsgraph()->Constant(feedback_vector().GetClosureFeedbackCell(index));
  }
}

1172 1173 1174 1175 1176 1177 1178 1179 1180 1181
void BytecodeGraphBuilder::CreateNativeContextNode() {
  DCHECK_NULL(native_context_node_);
  native_context_node_ = native_context_independent()
                             ? BuildLoadNativeContext()
                             : jsgraph()->Constant(native_context());
}

Node* BytecodeGraphBuilder::BuildLoadNativeContext() {
  DCHECK(native_context_independent());
  DCHECK_NULL(native_context_node_);
1182 1183 1184 1185
  Node* context_map = NewNode(simplified()->LoadField(AccessBuilder::ForMap()),
                              environment()->Context());
  return NewNode(simplified()->LoadField(AccessBuilder::ForMapNativeContext()),
                 context_map);
1186 1187
}

1188
void BytecodeGraphBuilder::MaybeBuildTierUpCheck() {
1189 1190 1191 1192
  // For OSR we don't tier up, so we don't need to build this check. Also
  // tiering up currently tail calls to IET which tail calls aren't supported
  // with OSR. See AdjustStackPointerForTailCall.
  if (!CodeKindCanTierUp(code_kind()) || osr_) return;
1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209

  int parameter_count = bytecode_array().parameter_count();
  Node* target = GetFunctionClosure();
  Node* new_target = graph()->NewNode(
      common()->Parameter(
          Linkage::GetJSCallNewTargetParamIndex(parameter_count),
          "%new.target"),
      graph()->start());
  Node* argc = graph()->NewNode(
      common()->Parameter(Linkage::GetJSCallArgCountParamIndex(parameter_count),
                          "%argc"),
      graph()->start());
  DCHECK_EQ(environment()->Context()->opcode(), IrOpcode::kParameter);
  Node* context = environment()->Context();

  NewNode(simplified()->TierUpCheck(), feedback_vector_node(), target,
          new_target, argc, context);
1210 1211
}

1212 1213 1214
void BytecodeGraphBuilder::MaybeBuildIncrementInvocationCount() {
  if (!generate_full_feedback_collection()) return;

1215 1216 1217 1218
  Node* current_invocation_count =
      NewNode(simplified()->LoadField(
                  AccessBuilder::ForFeedbackVectorInvocationCount()),
              feedback_vector_node());
1219
  Node* next_invocation_count =
1220 1221 1222 1223 1224
      NewNode(simplified()->NumberAdd(), current_invocation_count,
              jsgraph()->SmiConstant(1));
  NewNode(simplified()->StoreField(
              AccessBuilder::ForFeedbackVectorInvocationCount()),
          feedback_vector_node(), next_invocation_count);
1225 1226
}

1227
Node* BytecodeGraphBuilder::BuildLoadNativeContextField(int index) {
1228
  Node* result = NewNode(javascript()->LoadContext(0, index, true));
1229
  NodeProperties::ReplaceContextInput(result, native_context_node());
1230
  return result;
1231 1232
}

1233
FeedbackSource BytecodeGraphBuilder::CreateFeedbackSource(int slot_id) {
1234 1235 1236 1237
  return CreateFeedbackSource(FeedbackVector::ToSlot(slot_id));
}

FeedbackSource BytecodeGraphBuilder::CreateFeedbackSource(FeedbackSlot slot) {
1238
  return FeedbackSource(feedback_vector(), slot);
1239 1240
}

1241
void BytecodeGraphBuilder::CreateGraph() {
1242 1243
  SourcePositionTable::Scope pos_scope(source_positions_, start_position_);

1244
  // Set up the basic structure of the graph. Outputs for {Start} are the formal
1245 1246
  // parameters (including the receiver) plus new target, number of arguments,
  // context and closure.
1247
  int start_output_arity = StartNode::OutputArityForFormalParameterCount(
1248
      bytecode_array().parameter_count());
1249
  graph()->SetStart(graph()->NewNode(common()->Start(start_output_arity)));
1250

1251 1252 1253 1254
  Environment env(this, bytecode_array().register_count(),
                  bytecode_array().parameter_count(),
                  bytecode_array().incoming_new_target_or_generator_register(),
                  graph()->start());
1255 1256
  set_environment(&env);

1257
  CreateFeedbackCellNode();
1258
  CreateFeedbackVectorNode();
1259
  MaybeBuildTierUpCheck();
1260
  MaybeBuildIncrementInvocationCount();
1261
  CreateNativeContextNode();
1262

1263
  VisitBytecodes();
1264 1265 1266 1267 1268 1269 1270 1271 1272

  // Finish the basic structure of the graph.
  DCHECK_NE(0u, exit_controls_.size());
  int const input_count = static_cast<int>(exit_controls_.size());
  Node** const inputs = &exit_controls_.front();
  Node* end = graph()->NewNode(common()->End(input_count), input_count, inputs);
  graph()->SetEnd(end);
}

1273
void BytecodeGraphBuilder::PrepareEagerCheckpoint() {
1274
  if (needs_eager_checkpoint()) {
1275 1276
    // Create an explicit checkpoint node for before the operation. This only
    // needs to happen if we aren't effect-dominated by a {Checkpoint} already.
1277
    mark_as_needing_eager_checkpoint(false);
1278 1279 1280 1281
    Node* node = NewNode(common()->Checkpoint());
    DCHECK_EQ(1, OperatorProperties::GetFrameStateInputCount(node->op()));
    DCHECK_EQ(IrOpcode::kDead,
              NodeProperties::GetFrameStateInput(node)->opcode());
1282
    BytecodeOffset bailout_id(bytecode_iterator().current_offset());
1283

1284
    const BytecodeLivenessState* liveness_before =
1285
        bytecode_analysis().GetInLivenessFor(
1286
            bytecode_iterator().current_offset());
1287

1288
    Node* frame_state_before = environment()->Checkpoint(
1289
        bailout_id, OutputFrameStateCombine::Ignore(), liveness_before);
1290
    NodeProperties::ReplaceFrameStateInput(node, frame_state_before);
1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303
#ifdef DEBUG
  } else {
    // In case we skipped checkpoint creation above, we must be able to find an
    // existing checkpoint that effect-dominates the nodes about to be created.
    // Starting a search from the current effect-dependency has to succeed.
    Node* effect = environment()->GetEffectDependency();
    while (effect->opcode() != IrOpcode::kCheckpoint) {
      DCHECK(effect->op()->HasProperty(Operator::kNoWrite));
      DCHECK_EQ(1, effect->op()->EffectInputCount());
      effect = NodeProperties::GetEffectInput(effect);
    }
  }
#else
1304
  }
1305
#endif  // DEBUG
1306 1307
}

1308
void BytecodeGraphBuilder::PrepareFrameState(
1309
    Node* node, OutputFrameStateCombine combine, BytecodeOffset bailout_id,
1310
    const BytecodeLivenessState* liveness) {
1311 1312 1313 1314 1315 1316
  if (OperatorProperties::HasFrameStateInput(node->op())) {
    // Add the frame state for after the operation. The node in question has
    // already been created and had a {Dead} frame state input up until now.
    DCHECK_EQ(1, OperatorProperties::GetFrameStateInputCount(node->op()));
    DCHECK_EQ(IrOpcode::kDead,
              NodeProperties::GetFrameStateInput(node)->opcode());
1317

1318
    Node* frame_state_after =
1319
        environment()->Checkpoint(bailout_id, combine, liveness);
1320
    NodeProperties::ReplaceFrameStateInput(node, frame_state_after);
1321 1322 1323
  }
}

1324 1325 1326 1327 1328 1329 1330
void BytecodeGraphBuilder::AdvanceIteratorsTo(int bytecode_offset) {
  for (; bytecode_iterator().current_offset() != bytecode_offset;
       bytecode_iterator().Advance()) {
    UpdateSourcePosition(bytecode_iterator().current_offset());
  }
}

1331 1332 1333 1334 1335 1336 1337 1338
// Stores the state of the SourcePosition iterator, and the index to the
// current exception handlers stack. We need, during the OSR graph generation,
// to backup the states of these iterators at the LoopHeader offset of each
// outer loop which contains the OSR loop. The iterators are then restored when
// peeling the loops, so that both exception handling and synchronisation with
// the source position can be achieved.
class BytecodeGraphBuilder::OsrIteratorState {
 public:
1339 1340
  explicit OsrIteratorState(BytecodeGraphBuilder* graph_builder)
      : graph_builder_(graph_builder),
1341 1342 1343 1344
        saved_states_(graph_builder->local_zone()) {}

  void ProcessOsrPrelude() {
    ZoneVector<int> outer_loop_offsets(graph_builder_->local_zone());
1345
    int osr_entry = graph_builder_->bytecode_analysis().osr_entry_point();
1346 1347

    // We find here the outermost loop which contains the OSR loop.
1348 1349 1350 1351
    int outermost_loop_offset = osr_entry;
    while ((outermost_loop_offset = graph_builder_->bytecode_analysis()
                                        .GetLoopInfoFor(outermost_loop_offset)
                                        .parent_offset()) != -1) {
1352 1353 1354
      outer_loop_offsets.push_back(outermost_loop_offset);
    }
    outermost_loop_offset =
1355
        outer_loop_offsets.empty() ? osr_entry : outer_loop_offsets.back();
1356
    graph_builder_->AdvanceIteratorsTo(outermost_loop_offset);
1357 1358 1359 1360 1361 1362 1363

    // We save some iterators states at the offsets of the loop headers of the
    // outer loops (the ones containing the OSR loop). They will be used for
    // jumping back in the bytecode.
    for (ZoneVector<int>::const_reverse_iterator it =
             outer_loop_offsets.crbegin();
         it != outer_loop_offsets.crend(); ++it) {
1364
      graph_builder_->AdvanceIteratorsTo(*it);
1365
      graph_builder_->ExitThenEnterExceptionHandlers(
1366 1367 1368 1369
          graph_builder_->bytecode_iterator().current_offset());
      saved_states_.push(IteratorsStates(
          graph_builder_->current_exception_handler(),
          graph_builder_->source_position_iterator().GetState()));
1370 1371 1372
    }

    // Finishing by advancing to the OSR entry
1373
    graph_builder_->AdvanceIteratorsTo(osr_entry);
1374 1375 1376 1377

    // Enters all remaining exception handler which end before the OSR loop
    // so that on next call of VisitSingleBytecode they will get popped from
    // the exception handlers stack.
1378
    graph_builder_->ExitThenEnterExceptionHandlers(osr_entry);
1379
    graph_builder_->set_currently_peeled_loop_offset(
1380 1381 1382
        graph_builder_->bytecode_analysis()
            .GetLoopInfoFor(osr_entry)
            .parent_offset());
1383 1384 1385
  }

  void RestoreState(int target_offset, int new_parent_offset) {
1386
    graph_builder_->bytecode_iterator().SetOffset(target_offset);
1387 1388 1389 1390
    // In case of a return, we must not build loop exits for
    // not-yet-built outer loops.
    graph_builder_->set_currently_peeled_loop_offset(new_parent_offset);
    IteratorsStates saved_state = saved_states_.top();
1391 1392
    graph_builder_->source_position_iterator().RestoreState(
        saved_state.source_iterator_state_);
1393 1394 1395 1396 1397 1398 1399 1400
    graph_builder_->set_current_exception_handler(
        saved_state.exception_handler_index_);
    saved_states_.pop();
  }

 private:
  struct IteratorsStates {
    int exception_handler_index_;
1401
    SourcePositionTableIterator::IndexAndPositionState source_iterator_state_;
1402

1403 1404 1405
    IteratorsStates(int exception_handler_index,
                    SourcePositionTableIterator::IndexAndPositionState
                        source_iterator_state)
1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424
        : exception_handler_index_(exception_handler_index),
          source_iterator_state_(source_iterator_state) {}
  };

  BytecodeGraphBuilder* graph_builder_;
  ZoneStack<IteratorsStates> saved_states_;
};

void BytecodeGraphBuilder::RemoveMergeEnvironmentsBeforeOffset(
    int limit_offset) {
  if (!merge_environments_.empty()) {
    ZoneMap<int, Environment*>::iterator it = merge_environments_.begin();
    ZoneMap<int, Environment*>::iterator stop_it = merge_environments_.end();
    while (it != stop_it && it->first <= limit_offset) {
      it = merge_environments_.erase(it);
    }
  }
}

1425 1426 1427 1428
void BytecodeGraphBuilder::BuildFunctionEntryStackCheck() {
  if (!skip_first_stack_check()) {
    Node* node =
        NewNode(javascript()->StackCheck(StackCheckKind::kJSFunctionEntry));
1429
    PrepareFrameStateForFunctionEntryStackCheck(node);
1430 1431 1432
  }
}

1433 1434 1435 1436 1437 1438
void BytecodeGraphBuilder::BuildIterationBodyStackCheck() {
  Node* node =
      NewNode(javascript()->StackCheck(StackCheckKind::kJSIterationBody));
  environment()->RecordAfterState(node, Environment::kAttachFrameState);
}

1439 1440 1441 1442 1443 1444 1445 1446 1447
void BytecodeGraphBuilder::MaybeBuildOSREntryStackCheck() {
  if (V8_UNLIKELY(is_osr_entry_stack_check_pending_)) {
    is_osr_entry_stack_check_pending_ = false;
    Node* node =
        NewNode(javascript()->StackCheck(StackCheckKind::kJSFunctionEntry));
    PrepareFrameStateForOSREntryStackCheck(node);
  }
}

1448 1449 1450
// We will iterate through the OSR loop, then its parent, and so on
// until we have reached the outmost loop containing the OSR loop. We do
// not generate nodes for anything before the outermost loop.
1451 1452
void BytecodeGraphBuilder::AdvanceToOsrEntryAndPeelLoops() {
  OsrIteratorState iterator_states(this);
1453
  iterator_states.ProcessOsrPrelude();
1454 1455
  int osr_entry = bytecode_analysis().osr_entry_point();
  DCHECK_EQ(bytecode_iterator().current_offset(), osr_entry);
1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472

  environment()->FillWithOsrValues();

  // Suppose we have n nested loops, loop_0 being the outermost one, and
  // loop_n being the OSR loop. We start iterating the bytecode at the header
  // of loop_n (the OSR loop), and then we peel the part of the the body of
  // loop_{n-1} following the end of loop_n. We then rewind the iterator to
  // the header of loop_{n-1}, and so on until we have partly peeled loop 0.
  // The full loop_0 body will be generating with the rest of the function,
  // outside the OSR generation.

  // To do so, if we are visiting a loop, we continue to visit what's left
  // of its parent, and then when reaching the parent's JumpLoop, we do not
  // create any jump for that but rewind the bytecode iterator to visit the
  // parent loop entirely, and so on.

  int current_parent_offset =
1473
      bytecode_analysis().GetLoopInfoFor(osr_entry).parent_offset();
1474
  while (current_parent_offset != -1) {
1475
    const LoopInfo& current_parent_loop =
1476
        bytecode_analysis().GetLoopInfoFor(current_parent_offset);
1477 1478
    // We iterate until the back edge of the parent loop, which we detect by
    // the offset that the JumpLoop targets.
1479 1480 1481 1482
    for (; !bytecode_iterator().done(); bytecode_iterator().Advance()) {
      if (bytecode_iterator().current_bytecode() ==
              interpreter::Bytecode::kJumpLoop &&
          bytecode_iterator().GetJumpTargetOffset() == current_parent_offset) {
1483 1484 1485
        // Reached the end of the current parent loop.
        break;
      }
1486
      VisitSingleBytecode();
1487
    }
1488 1489
    DCHECK(!bytecode_iterator()
                .done());  // Should have found the loop's jump target.
1490 1491 1492 1493

    // We also need to take care of the merge environments and exceptions
    // handlers here because the omitted JumpLoop bytecode can still be the
    // target of jumps or the first bytecode after a try block.
1494 1495
    ExitThenEnterExceptionHandlers(bytecode_iterator().current_offset());
    SwitchToMergeEnvironment(bytecode_iterator().current_offset());
1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508

    // This jump is the jump of our parent loop, which is not yet created.
    // So we do not build the jump nodes, but restore the bytecode and the
    // SourcePosition iterators to the values they had when we were visiting
    // the offset pointed at by the JumpLoop we've just reached.
    // We have already built nodes for inner loops, but now we will
    // iterate again over them and build new nodes corresponding to the same
    // bytecode offsets. Any jump or reference to this inner loops must now
    // point to the new nodes we will build, hence we clear the relevant part
    // of the environment.
    // Completely clearing the environment is not possible because merge
    // environments for forward jumps out of the loop need to be preserved
    // (e.g. a return or a labeled break in the middle of a loop).
1509
    RemoveMergeEnvironmentsBeforeOffset(bytecode_iterator().current_offset());
1510 1511 1512 1513 1514 1515
    iterator_states.RestoreState(current_parent_offset,
                                 current_parent_loop.parent_offset());
    current_parent_offset = current_parent_loop.parent_offset();
  }
}

1516
void BytecodeGraphBuilder::VisitSingleBytecode() {
1517
  tick_counter_->TickAndMaybeEnterSafepoint();
1518 1519
  int current_offset = bytecode_iterator().current_offset();
  UpdateSourcePosition(current_offset);
1520 1521 1522 1523 1524 1525 1526 1527
  ExitThenEnterExceptionHandlers(current_offset);
  DCHECK_GE(exception_handlers_.empty() ? current_offset
                                        : exception_handlers_.top().end_offset_,
            current_offset);
  SwitchToMergeEnvironment(current_offset);

  if (environment() != nullptr) {
    BuildLoopHeaderEnvironment(current_offset);
1528 1529 1530 1531 1532 1533 1534

    // The OSR-entry stack check must be emitted during the first call to
    // VisitSingleBytecode in an OSR'd function. We don't know if that call
    // will be made from AdvanceToOsrEntryAndPeelLoops or from VisitBytecodes,
    // therefore we insert the logic here inside VisitSingleBytecode itself.
    MaybeBuildOSREntryStackCheck();

1535
    switch (bytecode_iterator().current_bytecode()) {
1536 1537 1538 1539 1540
#define BYTECODE_CASE(name, ...)       \
  case interpreter::Bytecode::k##name: \
    Visit##name();                     \
    break;
      BYTECODE_LIST(BYTECODE_CASE)
1541
#undef BYTECODE_CASE
1542 1543 1544 1545 1546
    }
  }
}

void BytecodeGraphBuilder::VisitBytecodes() {
1547
  if (!bytecode_analysis().resume_jump_targets().empty()) {
1548 1549 1550 1551
    environment()->BindGeneratorState(
        jsgraph()->SmiConstant(JSGeneratorObject::kGeneratorExecuting));
  }

1552
  if (osr_) {
1553 1554 1555 1556
    // We peel the OSR loop and any outer loop containing it except that we
    // leave the nodes corresponding to the whole outermost loop (including
    // the last copies of the loops it contains) to be generated by the normal
    // bytecode iteration below.
1557
    AdvanceToOsrEntryAndPeelLoops();
1558 1559
  } else {
    BuildFunctionEntryStackCheck();
1560
  }
1561

1562
  bool has_one_shot_bytecode = false;
1563
  for (; !bytecode_iterator().done(); bytecode_iterator().Advance()) {
1564
    if (interpreter::Bytecodes::IsOneShotBytecode(
1565
            bytecode_iterator().current_bytecode())) {
1566 1567
      has_one_shot_bytecode = true;
    }
1568
    VisitSingleBytecode();
1569
  }
1570

1571 1572
  // TODO(leszeks): Increment usage counter on BG thread.
  if (!FLAG_concurrent_inlining && has_one_shot_bytecode) {
1573
    // (For concurrent inlining this is done in the serializer instead.)
1574 1575 1576 1577
    isolate()->CountUsage(
        v8::Isolate::UseCounterFeature::kOptimizedFunctionWithOneShotBytecode);
  }

1578
  DCHECK(exception_handlers_.empty());
1579 1580
}

1581
void BytecodeGraphBuilder::VisitLdaZero() {
1582 1583 1584 1585
  Node* node = jsgraph()->ZeroConstant();
  environment()->BindAccumulator(node);
}

1586
void BytecodeGraphBuilder::VisitLdaSmi() {
1587
  Node* node = jsgraph()->Constant(bytecode_iterator().GetImmediateOperand(0));
1588 1589 1590
  environment()->BindAccumulator(node);
}

1591
void BytecodeGraphBuilder::VisitLdaConstant() {
1592
  ObjectRef object(broker(), GetConstantForIndexOperand(0),
1593
                   ObjectRef::BackgroundSerialization::kAllowed);
1594
  Node* node = jsgraph()->Constant(object);
1595 1596 1597
  environment()->BindAccumulator(node);
}

1598
void BytecodeGraphBuilder::VisitLdaUndefined() {
1599 1600 1601 1602
  Node* node = jsgraph()->UndefinedConstant();
  environment()->BindAccumulator(node);
}

1603
void BytecodeGraphBuilder::VisitLdaNull() {
1604 1605 1606 1607
  Node* node = jsgraph()->NullConstant();
  environment()->BindAccumulator(node);
}

1608
void BytecodeGraphBuilder::VisitLdaTheHole() {
1609 1610 1611 1612
  Node* node = jsgraph()->TheHoleConstant();
  environment()->BindAccumulator(node);
}

1613
void BytecodeGraphBuilder::VisitLdaTrue() {
1614 1615 1616 1617
  Node* node = jsgraph()->TrueConstant();
  environment()->BindAccumulator(node);
}

1618
void BytecodeGraphBuilder::VisitLdaFalse() {
1619 1620 1621 1622
  Node* node = jsgraph()->FalseConstant();
  environment()->BindAccumulator(node);
}

1623 1624 1625
void BytecodeGraphBuilder::VisitLdar() {
  Node* value =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
1626 1627 1628
  environment()->BindAccumulator(value);
}

1629
void BytecodeGraphBuilder::VisitStar() {
1630
  Node* value = environment()->LookupAccumulator();
1631
  environment()->BindRegister(bytecode_iterator().GetRegisterOperand(0), value);
1632 1633
}

1634 1635 1636 1637 1638 1639 1640 1641 1642 1643
#define SHORT_STAR_VISITOR(Name, ...)                                         \
  void BytecodeGraphBuilder::Visit##Name() {                                  \
    Node* value = environment()->LookupAccumulator();                         \
    environment()->BindRegister(                                              \
        interpreter::Register::FromShortStar(interpreter::Bytecode::k##Name), \
        value);                                                               \
  }
SHORT_STAR_BYTECODE_LIST(SHORT_STAR_VISITOR)
#undef SHORT_STAR_VISITOR

1644 1645 1646 1647
void BytecodeGraphBuilder::VisitMov() {
  Node* value =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
  environment()->BindRegister(bytecode_iterator().GetRegisterOperand(1), value);
1648 1649
}

1650
Node* BytecodeGraphBuilder::BuildLoadGlobal(NameRef name,
1651
                                            uint32_t feedback_slot_index,
1652
                                            TypeofMode typeof_mode) {
1653 1654
  FeedbackSource feedback = CreateFeedbackSource(feedback_slot_index);
  DCHECK(IsLoadGlobalICKind(broker()->GetFeedbackSlotKind(feedback)));
1655 1656
  const Operator* op =
      javascript()->LoadGlobal(name.object(), feedback, typeof_mode);
1657
  DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
1658
  return NewNode(op, feedback_vector_node());
1659 1660
}

1661
void BytecodeGraphBuilder::VisitLdaGlobal() {
1662
  PrepareEagerCheckpoint();
1663
  NameRef name(broker(), GetConstantForIndexOperand(0));
1664 1665 1666
  uint32_t feedback_slot_index = bytecode_iterator().GetIndexOperand(1);
  Node* node =
      BuildLoadGlobal(name, feedback_slot_index, TypeofMode::NOT_INSIDE_TYPEOF);
1667
  environment()->BindAccumulator(node, Environment::kAttachFrameState);
1668 1669
}

1670
void BytecodeGraphBuilder::VisitLdaGlobalInsideTypeof() {
1671
  PrepareEagerCheckpoint();
1672
  NameRef name(broker(), GetConstantForIndexOperand(0));
1673 1674 1675
  uint32_t feedback_slot_index = bytecode_iterator().GetIndexOperand(1);
  Node* node =
      BuildLoadGlobal(name, feedback_slot_index, TypeofMode::INSIDE_TYPEOF);
1676
  environment()->BindAccumulator(node, Environment::kAttachFrameState);
1677 1678
}

1679
void BytecodeGraphBuilder::VisitStaGlobal() {
1680
  PrepareEagerCheckpoint();
1681
  NameRef name(broker(), GetConstantForIndexOperand(0));
1682 1683
  FeedbackSource feedback =
      CreateFeedbackSource(bytecode_iterator().GetIndexOperand(1));
1684 1685
  Node* value = environment()->LookupAccumulator();

1686
  LanguageMode language_mode =
1687
      GetLanguageModeFromSlotKind(broker()->GetFeedbackSlotKind(feedback));
1688 1689
  const Operator* op =
      javascript()->StoreGlobal(language_mode, name.object(), feedback);
1690
  DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
1691
  Node* node = NewNode(op, value, feedback_vector_node());
1692
  environment()->RecordAfterState(node, Environment::kAttachFrameState);
1693 1694
}

1695 1696 1697 1698 1699 1700 1701
void BytecodeGraphBuilder::VisitStaInArrayLiteral() {
  PrepareEagerCheckpoint();
  Node* value = environment()->LookupAccumulator();
  Node* array =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
  Node* index =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(1));
1702 1703
  FeedbackSource feedback =
      CreateFeedbackSource(bytecode_iterator().GetIndexOperand(2));
1704 1705 1706
  const Operator* op = javascript()->StoreInArrayLiteral(feedback);

  JSTypeHintLowering::LoweringResult lowering =
1707
      TryBuildSimplifiedStoreKeyed(op, array, index, value, feedback.slot);
1708 1709 1710 1711 1712 1713 1714
  if (lowering.IsExit()) return;

  Node* node = nullptr;
  if (lowering.IsSideEffectFree()) {
    node = lowering.value();
  } else {
    DCHECK(!lowering.Changed());
1715
    DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
1716
    node = NewNode(op, array, index, value, feedback_vector_node());
1717 1718 1719 1720 1721
  }

  environment()->RecordAfterState(node, Environment::kAttachFrameState);
}

1722
void BytecodeGraphBuilder::VisitStaDataPropertyInLiteral() {
1723 1724
  PrepareEagerCheckpoint();

1725 1726 1727 1728
  Node* object =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
  Node* name =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(1));
1729 1730
  Node* value = environment()->LookupAccumulator();
  int flags = bytecode_iterator().GetFlagOperand(2);
1731 1732
  FeedbackSource feedback =
      CreateFeedbackSource(bytecode_iterator().GetIndexOperand(3));
1733
  const Operator* op = javascript()->StoreDataPropertyInLiteral(feedback);
1734 1735

  JSTypeHintLowering::LoweringResult lowering =
1736
      TryBuildSimplifiedStoreKeyed(op, object, name, value, feedback.slot);
1737 1738 1739 1740 1741 1742 1743
  if (lowering.IsExit()) return;

  Node* node = nullptr;
  if (lowering.IsSideEffectFree()) {
    node = lowering.value();
  } else {
    DCHECK(!lowering.Changed());
1744
    DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
1745 1746
    node = NewNode(op, object, name, value, jsgraph()->Constant(flags),
                   feedback_vector_node());
1747 1748
  }

1749
  environment()->RecordAfterState(node, Environment::kAttachFrameState);
1750 1751 1752 1753 1754
}

void BytecodeGraphBuilder::VisitCollectTypeProfile() {
  PrepareEagerCheckpoint();

1755 1756
  Node* position =
      jsgraph()->Constant(bytecode_iterator().GetImmediateOperand(0));
1757 1758 1759 1760 1761
  Node* value = environment()->LookupAccumulator();
  Node* vector = jsgraph()->Constant(feedback_vector());

  const Operator* op = javascript()->CallRuntime(Runtime::kCollectTypeProfile);

1762
  Node* node = NewNode(op, position, value, vector);
1763
  environment()->RecordAfterState(node, Environment::kAttachFrameState);
1764 1765
}

1766
void BytecodeGraphBuilder::VisitLdaContextSlot() {
1767 1768 1769
  const Operator* op = javascript()->LoadContext(
      bytecode_iterator().GetUnsignedImmediateOperand(2),
      bytecode_iterator().GetIndexOperand(1), false);
1770
  Node* node = NewNode(op);
1771 1772
  Node* context =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
1773
  NodeProperties::ReplaceContextInput(node, context);
1774
  environment()->BindAccumulator(node);
1775 1776
}

1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787
void BytecodeGraphBuilder::VisitLdaImmutableContextSlot() {
  const Operator* op = javascript()->LoadContext(
      bytecode_iterator().GetUnsignedImmediateOperand(2),
      bytecode_iterator().GetIndexOperand(1), true);
  Node* node = NewNode(op);
  Node* context =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
  NodeProperties::ReplaceContextInput(node, context);
  environment()->BindAccumulator(node);
}

1788
void BytecodeGraphBuilder::VisitLdaCurrentContextSlot() {
1789 1790
  const Operator* op = javascript()->LoadContext(
      0, bytecode_iterator().GetIndexOperand(0), false);
1791
  Node* node = NewNode(op);
1792
  environment()->BindAccumulator(node);
1793 1794
}

1795 1796 1797 1798 1799 1800 1801
void BytecodeGraphBuilder::VisitLdaImmutableCurrentContextSlot() {
  const Operator* op = javascript()->LoadContext(
      0, bytecode_iterator().GetIndexOperand(0), true);
  Node* node = NewNode(op);
  environment()->BindAccumulator(node);
}

1802
void BytecodeGraphBuilder::VisitStaContextSlot() {
1803 1804 1805
  const Operator* op = javascript()->StoreContext(
      bytecode_iterator().GetUnsignedImmediateOperand(2),
      bytecode_iterator().GetIndexOperand(1));
1806 1807
  Node* value = environment()->LookupAccumulator();
  Node* node = NewNode(op, value);
1808 1809
  Node* context =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
1810
  NodeProperties::ReplaceContextInput(node, context);
1811 1812
}

1813 1814 1815 1816
void BytecodeGraphBuilder::VisitStaCurrentContextSlot() {
  const Operator* op =
      javascript()->StoreContext(0, bytecode_iterator().GetIndexOperand(0));
  Node* value = environment()->LookupAccumulator();
1817
  NewNode(op, value);
1818 1819
}

1820
void BytecodeGraphBuilder::BuildLdaLookupSlot(TypeofMode typeof_mode) {
1821
  PrepareEagerCheckpoint();
1822 1823
  Node* name =
      jsgraph()->Constant(ObjectRef(broker(), GetConstantForIndexOperand(0)));
1824 1825 1826 1827 1828
  const Operator* op =
      javascript()->CallRuntime(typeof_mode == TypeofMode::NOT_INSIDE_TYPEOF
                                    ? Runtime::kLoadLookupSlot
                                    : Runtime::kLoadLookupSlotInsideTypeof);
  Node* value = NewNode(op, name);
1829
  environment()->BindAccumulator(value, Environment::kAttachFrameState);
1830 1831
}

1832 1833
void BytecodeGraphBuilder::VisitLdaLookupSlot() {
  BuildLdaLookupSlot(TypeofMode::NOT_INSIDE_TYPEOF);
1834 1835
}

1836 1837
void BytecodeGraphBuilder::VisitLdaLookupSlotInsideTypeof() {
  BuildLdaLookupSlot(TypeofMode::INSIDE_TYPEOF);
1838 1839
}

1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884 1885 1886 1887
BytecodeGraphBuilder::Environment*
BytecodeGraphBuilder::CheckContextExtensionAtDepth(
    Environment* slow_environment, uint32_t depth) {
  Node* extension_slot = NewNode(
      javascript()->LoadContext(depth, Context::EXTENSION_INDEX, false));
  Node* check_no_extension =
      NewNode(simplified()->ReferenceEqual(), extension_slot,
              jsgraph()->UndefinedConstant());
  NewBranch(check_no_extension);
  {
    SubEnvironment sub_environment(this);
    NewIfFalse();
    // If there is an extension, merge into the slow path.
    if (slow_environment == nullptr) {
      slow_environment = environment();
      NewMerge();
    } else {
      slow_environment->Merge(environment(),
                              bytecode_analysis().GetInLivenessFor(
                                  bytecode_iterator().current_offset()));
    }
  }
  NewIfTrue();
  // Do nothing on if there is no extension, eventually falling through to
  // the fast path.
  DCHECK_NOT_NULL(slow_environment);
  return slow_environment;
}

base::Optional<ScopeInfoRef> BytecodeGraphBuilder::TryGetScopeInfo() {
  Node* context = environment()->Context();
  switch (context->opcode()) {
    case IrOpcode::kJSCreateFunctionContext:
      return ScopeInfoRef(
          broker(),
          CreateFunctionContextParametersOf(context->op()).scope_info());
    case IrOpcode::kJSCreateBlockContext:
    case IrOpcode::kJSCreateCatchContext:
    case IrOpcode::kJSCreateWithContext:
      return ScopeInfoRef(broker(), ScopeInfoOf(context->op()));
    case IrOpcode::kParameter: {
      ScopeInfoRef scope_info = shared_info_.scope_info();
      if (scope_info.HasOuterScopeInfo()) {
        scope_info = scope_info.OuterScopeInfo();
      }
      return scope_info;
    }
    default:
1888
      return base::nullopt;
1889 1890 1891
  }
}

1892 1893
BytecodeGraphBuilder::Environment* BytecodeGraphBuilder::CheckContextExtensions(
    uint32_t depth) {
1894 1895 1896 1897 1898 1899 1900 1901 1902 1903
  base::Optional<ScopeInfoRef> maybe_scope_info = TryGetScopeInfo();
  if (!maybe_scope_info.has_value()) {
    return CheckContextExtensionsSlowPath(depth);
  }

  ScopeInfoRef scope_info = maybe_scope_info.value();
  // We only need to check up to the last-but-one depth, because an eval
  // in the same scope as the variable itself has no way of shadowing it.
  Environment* slow_environment = nullptr;
  for (uint32_t d = 0; d < depth; d++) {
1904
    if (scope_info.HasContextExtensionSlot()) {
1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920
      slow_environment = CheckContextExtensionAtDepth(slow_environment, d);
    }
    DCHECK_IMPLIES(!scope_info.HasOuterScopeInfo(), d + 1 == depth);
    if (scope_info.HasOuterScopeInfo()) {
      scope_info = scope_info.OuterScopeInfo();
    }
  }

  // The depth can be zero, in which case no slow-path checks are built, and
  // the slow path environment can be null.
  DCHECK_IMPLIES(slow_environment == nullptr, depth == 0);
  return slow_environment;
}

BytecodeGraphBuilder::Environment*
BytecodeGraphBuilder::CheckContextExtensionsSlowPath(uint32_t depth) {
1921
  // Output environment where the context has an extension
1922 1923
  Environment* slow_environment = nullptr;

1924
  // We only need to check up to the last-but-one depth, because an eval
1925
  // in the same scope as the variable itself has no way of shadowing it.
1926
  for (uint32_t d = 0; d < depth; d++) {
1927
    Node* has_extension = NewNode(javascript()->HasContextExtension(d));
1928

1929 1930
    Environment* undefined_extension_env;
    NewBranch(has_extension);
1931
    {
1932
      SubEnvironment sub_environment(this);
1933
      NewIfTrue();
1934
      slow_environment = CheckContextExtensionAtDepth(slow_environment, d);
1935
      undefined_extension_env = environment();
1936
    }
1937 1938 1939 1940 1941
    NewIfFalse();
    environment()->Merge(undefined_extension_env,
                         bytecode_analysis().GetInLivenessFor(
                             bytecode_iterator().current_offset()));
    mark_as_needing_eager_checkpoint(true);
1942 1943
    // Do nothing on if there is no extension, eventually falling through to
    // the fast path.
1944 1945
  }

1946 1947
  // The depth can be zero, in which case no slow-path checks are built, and
  // the slow path environment can be null.
1948
  DCHECK_IMPLIES(slow_environment == nullptr, depth == 0);
1949 1950 1951 1952 1953 1954 1955 1956 1957
  return slow_environment;
}

void BytecodeGraphBuilder::BuildLdaLookupContextSlot(TypeofMode typeof_mode) {
  uint32_t depth = bytecode_iterator().GetUnsignedImmediateOperand(2);

  // Check if any context in the depth has an extension.
  Environment* slow_environment = CheckContextExtensions(depth);

1958 1959 1960 1961 1962
  // Fast path, do a context load.
  {
    uint32_t slot_index = bytecode_iterator().GetIndexOperand(1);

    const Operator* op = javascript()->LoadContext(depth, slot_index, false);
1963
    environment()->BindAccumulator(NewNode(op));
1964
  }
1965

1966 1967 1968 1969 1970
  // Only build the slow path if there were any slow-path checks.
  if (slow_environment != nullptr) {
    // Add a merge to the fast environment.
    NewMerge();
    Environment* fast_environment = environment();
1971

1972 1973 1974
    // Slow path, do a runtime load lookup.
    set_environment(slow_environment);
    {
1975 1976
      Node* name = jsgraph()->Constant(
          ObjectRef(broker(), GetConstantForIndexOperand(0)));
1977 1978 1979 1980 1981 1982

      const Operator* op =
          javascript()->CallRuntime(typeof_mode == TypeofMode::NOT_INSIDE_TYPEOF
                                        ? Runtime::kLoadLookupSlot
                                        : Runtime::kLoadLookupSlotInsideTypeof);
      Node* value = NewNode(op, name);
1983
      environment()->BindAccumulator(value, Environment::kAttachFrameState);
1984
    }
1985

1986
    fast_environment->Merge(environment(),
1987
                            bytecode_analysis().GetOutLivenessFor(
1988
                                bytecode_iterator().current_offset()));
1989
    set_environment(fast_environment);
1990
    mark_as_needing_eager_checkpoint(true);
1991
  }
1992 1993 1994 1995 1996 1997 1998 1999 2000 2001
}

void BytecodeGraphBuilder::VisitLdaLookupContextSlot() {
  BuildLdaLookupContextSlot(TypeofMode::NOT_INSIDE_TYPEOF);
}

void BytecodeGraphBuilder::VisitLdaLookupContextSlotInsideTypeof() {
  BuildLdaLookupContextSlot(TypeofMode::INSIDE_TYPEOF);
}

2002
void BytecodeGraphBuilder::BuildLdaLookupGlobalSlot(TypeofMode typeof_mode) {
2003 2004 2005 2006 2007 2008 2009
  uint32_t depth = bytecode_iterator().GetUnsignedImmediateOperand(2);

  // Check if any context in the depth has an extension.
  Environment* slow_environment = CheckContextExtensions(depth);

  // Fast path, do a global load.
  {
2010
    PrepareEagerCheckpoint();
2011
    NameRef name(broker(), GetConstantForIndexOperand(0));
2012 2013
    uint32_t feedback_slot_index = bytecode_iterator().GetIndexOperand(1);
    Node* node = BuildLoadGlobal(name, feedback_slot_index, typeof_mode);
2014
    environment()->BindAccumulator(node, Environment::kAttachFrameState);
2015
  }
2016

2017 2018 2019
  // Only build the slow path if there were any slow-path checks.
  if (slow_environment != nullptr) {
    // Add a merge to the fast environment.
2020
    NewMerge();
2021
    Environment* fast_environment = environment();
2022

2023 2024 2025
    // Slow path, do a runtime load lookup.
    set_environment(slow_environment);
    {
2026 2027
      Node* name =
          jsgraph()->Constant(NameRef(broker(), GetConstantForIndexOperand(0)));
2028

2029 2030 2031 2032 2033
      const Operator* op =
          javascript()->CallRuntime(typeof_mode == TypeofMode::NOT_INSIDE_TYPEOF
                                        ? Runtime::kLoadLookupSlot
                                        : Runtime::kLoadLookupSlotInsideTypeof);
      Node* value = NewNode(op, name);
2034
      environment()->BindAccumulator(value, Environment::kAttachFrameState);
2035
    }
2036

2037
    fast_environment->Merge(environment(),
2038
                            bytecode_analysis().GetOutLivenessFor(
2039
                                bytecode_iterator().current_offset()));
2040
    set_environment(fast_environment);
2041
    mark_as_needing_eager_checkpoint(true);
2042
  }
2043 2044 2045 2046 2047 2048 2049 2050 2051 2052
}

void BytecodeGraphBuilder::VisitLdaLookupGlobalSlot() {
  BuildLdaLookupGlobalSlot(TypeofMode::NOT_INSIDE_TYPEOF);
}

void BytecodeGraphBuilder::VisitLdaLookupGlobalSlotInsideTypeof() {
  BuildLdaLookupGlobalSlot(TypeofMode::INSIDE_TYPEOF);
}

2053
void BytecodeGraphBuilder::VisitStaLookupSlot() {
2054
  PrepareEagerCheckpoint();
2055
  Node* value = environment()->LookupAccumulator();
2056 2057
  Node* name =
      jsgraph()->Constant(ObjectRef(broker(), GetConstantForIndexOperand(0)));
2058 2059 2060 2061 2062 2063 2064 2065 2066
  int bytecode_flags = bytecode_iterator().GetFlagOperand(1);
  LanguageMode language_mode = static_cast<LanguageMode>(
      interpreter::StoreLookupSlotFlags::LanguageModeBit::decode(
          bytecode_flags));
  LookupHoistingMode lookup_hoisting_mode = static_cast<LookupHoistingMode>(
      interpreter::StoreLookupSlotFlags::LookupHoistingModeBit::decode(
          bytecode_flags));
  DCHECK_IMPLIES(lookup_hoisting_mode == LookupHoistingMode::kLegacySloppy,
                 is_sloppy(language_mode));
2067
  const Operator* op = javascript()->CallRuntime(
2068 2069 2070 2071 2072
      is_strict(language_mode)
          ? Runtime::kStoreLookupSlot_Strict
          : lookup_hoisting_mode == LookupHoistingMode::kLegacySloppy
                ? Runtime::kStoreLookupSlot_SloppyHoisting
                : Runtime::kStoreLookupSlot_Sloppy);
2073
  Node* store = NewNode(op, name, value);
2074
  environment()->BindAccumulator(store, Environment::kAttachFrameState);
2075 2076
}

2077 2078
void BytecodeGraphBuilder::VisitLdaNamedProperty() {
  PrepareEagerCheckpoint();
2079 2080
  Node* object =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
2081
  NameRef name(broker(), GetConstantForIndexOperand(1));
2082 2083
  FeedbackSource feedback =
      CreateFeedbackSource(bytecode_iterator().GetIndexOperand(2));
2084
  const Operator* op = javascript()->LoadNamed(name.object(), feedback);
2085

2086
  JSTypeHintLowering::LoweringResult lowering =
2087
      TryBuildSimplifiedLoadNamed(op, feedback.slot);
2088 2089
  if (lowering.IsExit()) return;

2090
  Node* node = nullptr;
2091 2092
  if (lowering.IsSideEffectFree()) {
    node = lowering.value();
2093
  } else {
2094
    DCHECK(!lowering.Changed());
2095
    DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
2096
    node = NewNode(op, object, feedback_vector_node());
2097
  }
2098
  environment()->BindAccumulator(node, Environment::kAttachFrameState);
2099
}
2100

2101 2102 2103 2104
void BytecodeGraphBuilder::VisitLdaNamedPropertyNoFeedback() {
  PrepareEagerCheckpoint();
  Node* object =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
2105
  NameRef name(broker(), GetConstantForIndexOperand(1));
2106
  const Operator* op = javascript()->LoadNamed(name.object(), FeedbackSource());
2107
  DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
2108
  Node* node = NewNode(op, object, feedback_vector_node());
2109 2110 2111
  environment()->BindAccumulator(node, Environment::kAttachFrameState);
}

2112 2113 2114 2115 2116
void BytecodeGraphBuilder::VisitLdaNamedPropertyFromSuper() {
  PrepareEagerCheckpoint();
  Node* receiver =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
  Node* home_object = environment()->LookupAccumulator();
2117
  NameRef name(broker(), GetConstantForIndexOperand(1));
2118

2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132 2133 2134 2135
  FeedbackSource feedback =
      CreateFeedbackSource(bytecode_iterator().GetIndexOperand(2));
  const Operator* op =
      javascript()->LoadNamedFromSuper(name.object(), feedback);

  JSTypeHintLowering::LoweringResult lowering =
      TryBuildSimplifiedLoadNamed(op, feedback.slot);
  if (lowering.IsExit()) return;

  Node* node = nullptr;
  if (lowering.IsSideEffectFree()) {
    node = lowering.value();
  } else {
    DCHECK(!lowering.Changed());
    DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
    node = NewNode(op, receiver, home_object, feedback_vector_node());
  }
2136 2137 2138
  environment()->BindAccumulator(node, Environment::kAttachFrameState);
}

2139
void BytecodeGraphBuilder::VisitLdaKeyedProperty() {
2140
  PrepareEagerCheckpoint();
2141
  Node* key = environment()->LookupAccumulator();
2142 2143
  Node* object =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
2144 2145
  FeedbackSource feedback =
      CreateFeedbackSource(bytecode_iterator().GetIndexOperand(1));
2146
  const Operator* op = javascript()->LoadProperty(feedback);
2147

2148
  JSTypeHintLowering::LoweringResult lowering =
2149
      TryBuildSimplifiedLoadKeyed(op, object, key, feedback.slot);
2150 2151
  if (lowering.IsExit()) return;

2152
  Node* node = nullptr;
2153 2154
  if (lowering.IsSideEffectFree()) {
    node = lowering.value();
2155
  } else {
2156
    DCHECK(!lowering.Changed());
2157 2158 2159
    STATIC_ASSERT(JSLoadPropertyNode::ObjectIndex() == 0);
    STATIC_ASSERT(JSLoadPropertyNode::KeyIndex() == 1);
    STATIC_ASSERT(JSLoadPropertyNode::FeedbackVectorIndex() == 2);
2160
    DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
2161
    node = NewNode(op, object, key, feedback_vector_node());
2162
  }
2163
  environment()->BindAccumulator(node, Environment::kAttachFrameState);
2164 2165
}

2166
void BytecodeGraphBuilder::BuildNamedStore(StoreMode store_mode) {
2167
  PrepareEagerCheckpoint();
2168
  Node* value = environment()->LookupAccumulator();
2169 2170
  Node* object =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
2171
  NameRef name(broker(), GetConstantForIndexOperand(1));
2172 2173
  FeedbackSource feedback =
      CreateFeedbackSource(bytecode_iterator().GetIndexOperand(2));
2174

2175 2176 2177
  const Operator* op;
  if (store_mode == StoreMode::kOwn) {
    DCHECK_EQ(FeedbackSlotKind::kStoreOwnNamed,
2178 2179
              broker()->GetFeedbackSlotKind(feedback));

2180
    op = javascript()->StoreNamedOwn(name.object(), feedback);
2181
  } else {
2182
    DCHECK_EQ(StoreMode::kNormal, store_mode);
2183
    LanguageMode language_mode =
2184
        GetLanguageModeFromSlotKind(broker()->GetFeedbackSlotKind(feedback));
2185
    op = javascript()->StoreNamed(language_mode, name.object(), feedback);
2186
  }
2187

2188
  JSTypeHintLowering::LoweringResult lowering =
2189
      TryBuildSimplifiedStoreNamed(op, object, value, feedback.slot);
2190 2191
  if (lowering.IsExit()) return;

2192
  Node* node = nullptr;
2193 2194
  if (lowering.IsSideEffectFree()) {
    node = lowering.value();
2195
  } else {
2196
    DCHECK(!lowering.Changed());
2197
    DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
2198
    node = NewNode(op, object, value, feedback_vector_node());
2199
  }
2200
  environment()->RecordAfterState(node, Environment::kAttachFrameState);
2201 2202
}

2203 2204
void BytecodeGraphBuilder::VisitStaNamedProperty() {
  BuildNamedStore(StoreMode::kNormal);
2205 2206
}

2207 2208 2209 2210 2211
void BytecodeGraphBuilder::VisitStaNamedPropertyNoFeedback() {
  PrepareEagerCheckpoint();
  Node* value = environment()->LookupAccumulator();
  Node* object =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
2212
  NameRef name(broker(), GetConstantForIndexOperand(1));
2213 2214 2215
  LanguageMode language_mode =
      static_cast<LanguageMode>(bytecode_iterator().GetFlagOperand(2));
  const Operator* op =
2216
      javascript()->StoreNamed(language_mode, name.object(), FeedbackSource());
2217
  DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
2218
  Node* node = NewNode(op, object, value, feedback_vector_node());
2219 2220 2221
  environment()->RecordAfterState(node, Environment::kAttachFrameState);
}

2222
void BytecodeGraphBuilder::VisitStaNamedOwnProperty() {
2223
  BuildNamedStore(StoreMode::kOwn);
2224 2225
}

2226
void BytecodeGraphBuilder::VisitStaKeyedProperty() {
2227
  PrepareEagerCheckpoint();
2228
  Node* value = environment()->LookupAccumulator();
2229 2230 2231 2232
  Node* object =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
  Node* key =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(1));
2233 2234
  FeedbackSource source =
      CreateFeedbackSource(bytecode_iterator().GetIndexOperand(2));
2235
  LanguageMode language_mode =
2236 2237
      GetLanguageModeFromSlotKind(broker()->GetFeedbackSlotKind(source));
  const Operator* op = javascript()->StoreProperty(language_mode, source);
2238

2239
  JSTypeHintLowering::LoweringResult lowering =
2240
      TryBuildSimplifiedStoreKeyed(op, object, key, value, source.slot);
2241 2242
  if (lowering.IsExit()) return;

2243
  Node* node = nullptr;
2244 2245
  if (lowering.IsSideEffectFree()) {
    node = lowering.value();
2246
  } else {
2247
    DCHECK(!lowering.Changed());
2248 2249 2250 2251
    STATIC_ASSERT(JSStorePropertyNode::ObjectIndex() == 0);
    STATIC_ASSERT(JSStorePropertyNode::KeyIndex() == 1);
    STATIC_ASSERT(JSStorePropertyNode::ValueIndex() == 2);
    STATIC_ASSERT(JSStorePropertyNode::FeedbackVectorIndex() == 3);
2252
    DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
2253
    node = NewNode(op, object, key, value, feedback_vector_node());
2254 2255
  }

2256
  environment()->RecordAfterState(node, Environment::kAttachFrameState);
2257 2258
}

2259
void BytecodeGraphBuilder::VisitLdaModuleVariable() {
2260 2261
  int32_t cell_index = bytecode_iterator().GetImmediateOperand(0);
  uint32_t depth = bytecode_iterator().GetUnsignedImmediateOperand(1);
2262 2263
  Node* module =
      NewNode(javascript()->LoadContext(depth, Context::EXTENSION_INDEX, true));
2264 2265
  Node* value = NewNode(javascript()->LoadModule(cell_index), module);
  environment()->BindAccumulator(value);
2266 2267 2268
}

void BytecodeGraphBuilder::VisitStaModuleVariable() {
2269 2270
  int32_t cell_index = bytecode_iterator().GetImmediateOperand(0);
  uint32_t depth = bytecode_iterator().GetUnsignedImmediateOperand(1);
2271 2272
  Node* module =
      NewNode(javascript()->LoadContext(depth, Context::EXTENSION_INDEX, true));
2273
  Node* value = environment()->LookupAccumulator();
2274
  NewNode(javascript()->StoreModule(cell_index), module, value);
2275 2276
}

2277
void BytecodeGraphBuilder::VisitPushContext() {
2278
  Node* new_context = environment()->LookupAccumulator();
2279
  environment()->BindRegister(bytecode_iterator().GetRegisterOperand(0),
2280 2281
                              environment()->Context());
  environment()->SetContext(new_context);
2282 2283
}

2284 2285 2286
void BytecodeGraphBuilder::VisitPopContext() {
  Node* context =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
2287
  environment()->SetContext(context);
2288 2289
}

2290
void BytecodeGraphBuilder::VisitCreateClosure() {
2291
  SharedFunctionInfoRef shared_info(broker(), GetConstantForIndexOperand(0));
2292
  AllocationType allocation =
2293
      interpreter::CreateClosureFlags::PretenuredBit::decode(
2294
          bytecode_iterator().GetFlagOperand(2))
2295 2296
          ? AllocationType::kOld
          : AllocationType::kYoung;
2297

2298
  const Operator* op = javascript()->CreateClosure(
2299
      shared_info.object(),
2300
      jsgraph()->isolate()->builtins()->builtin_handle(Builtins::kCompileLazy),
2301
      allocation);
2302 2303
  Node* closure = NewNode(
      op, BuildLoadFeedbackCell(bytecode_iterator().GetIndexOperand(1)));
2304 2305 2306
  environment()->BindAccumulator(closure);
}

2307
void BytecodeGraphBuilder::VisitCreateBlockContext() {
2308
  ScopeInfoRef scope_info(broker(), GetConstantForIndexOperand(0));
2309
  const Operator* op = javascript()->CreateBlockContext(scope_info.object());
2310
  Node* context = NewNode(op);
2311 2312 2313
  environment()->BindAccumulator(context);
}

2314
void BytecodeGraphBuilder::VisitCreateFunctionContext() {
2315
  ScopeInfoRef scope_info(broker(), GetConstantForIndexOperand(0));
2316
  uint32_t slots = bytecode_iterator().GetUnsignedImmediateOperand(1);
2317 2318
  const Operator* op = javascript()->CreateFunctionContext(
      scope_info.object(), slots, FUNCTION_SCOPE);
2319
  Node* context = NewNode(op);
2320 2321 2322 2323
  environment()->BindAccumulator(context);
}

void BytecodeGraphBuilder::VisitCreateEvalContext() {
2324
  ScopeInfoRef scope_info(broker(), GetConstantForIndexOperand(0));
2325
  uint32_t slots = bytecode_iterator().GetUnsignedImmediateOperand(1);
2326 2327
  const Operator* op = javascript()->CreateFunctionContext(scope_info.object(),
                                                           slots, EVAL_SCOPE);
2328
  Node* context = NewNode(op);
2329 2330 2331
  environment()->BindAccumulator(context);
}

2332 2333 2334
void BytecodeGraphBuilder::VisitCreateCatchContext() {
  interpreter::Register reg = bytecode_iterator().GetRegisterOperand(0);
  Node* exception = environment()->LookupRegister(reg);
2335
  ScopeInfoRef scope_info(broker(), GetConstantForIndexOperand(1));
2336

2337
  const Operator* op = javascript()->CreateCatchContext(scope_info.object());
2338
  Node* context = NewNode(op, exception);
2339 2340 2341
  environment()->BindAccumulator(context);
}

2342 2343 2344
void BytecodeGraphBuilder::VisitCreateWithContext() {
  Node* object =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
2345
  ScopeInfoRef scope_info(broker(), GetConstantForIndexOperand(1));
2346

2347
  const Operator* op = javascript()->CreateWithContext(scope_info.object());
2348
  Node* context = NewNode(op, object);
2349 2350 2351
  environment()->BindAccumulator(context);
}

2352 2353
void BytecodeGraphBuilder::BuildCreateArguments(CreateArgumentsType type) {
  const Operator* op = javascript()->CreateArguments(type);
2354
  Node* object = NewNode(op, GetFunctionClosure());
2355
  environment()->BindAccumulator(object, Environment::kAttachFrameState);
2356 2357
}

2358
void BytecodeGraphBuilder::VisitCreateMappedArguments() {
2359
  BuildCreateArguments(CreateArgumentsType::kMappedArguments);
2360 2361
}

2362
void BytecodeGraphBuilder::VisitCreateUnmappedArguments() {
2363
  BuildCreateArguments(CreateArgumentsType::kUnmappedArguments);
2364 2365
}

2366 2367
void BytecodeGraphBuilder::VisitCreateRestParameter() {
  BuildCreateArguments(CreateArgumentsType::kRestParameter);
2368 2369
}

2370
void BytecodeGraphBuilder::VisitCreateRegExpLiteral() {
2371
  StringRef constant_pattern(broker(), GetConstantForIndexOperand(0));
2372
  int const slot_id = bytecode_iterator().GetIndexOperand(1);
2373
  FeedbackSource pair = CreateFeedbackSource(slot_id);
2374
  int literal_flags = bytecode_iterator().GetFlagOperand(2);
2375
  STATIC_ASSERT(JSCreateLiteralRegExpNode::FeedbackVectorIndex() == 0);
2376 2377 2378 2379
  const Operator* op = javascript()->CreateLiteralRegExp(
      constant_pattern.object(), pair, literal_flags);
  DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
  Node* literal = NewNode(op, feedback_vector_node());
2380
  environment()->BindAccumulator(literal, Environment::kAttachFrameState);
2381 2382
}

2383
void BytecodeGraphBuilder::VisitCreateArrayLiteral() {
2384
  ArrayBoilerplateDescriptionRef array_boilerplate_description(
2385
      broker(), GetConstantForIndexOperand(0));
2386
  int const slot_id = bytecode_iterator().GetIndexOperand(1);
2387
  FeedbackSource pair = CreateFeedbackSource(slot_id);
2388 2389 2390
  int bytecode_flags = bytecode_iterator().GetFlagOperand(2);
  int literal_flags =
      interpreter::CreateArrayLiteralFlags::FlagsBits::decode(bytecode_flags);
2391 2392 2393 2394 2395
  // Disable allocation site mementos. Only unoptimized code will collect
  // feedback about allocation site. Once the code is optimized we expect the
  // data to converge. So, we disable allocation site mementos in optimized
  // code. We can revisit this when we have data to the contrary.
  literal_flags |= ArrayLiteral::kDisableMementos;
2396
  int number_of_elements =
2397
      array_boilerplate_description.constants_elements_length();
2398
  STATIC_ASSERT(JSCreateLiteralArrayNode::FeedbackVectorIndex() == 0);
2399
  const Operator* op =
2400
      javascript()->CreateLiteralArray(array_boilerplate_description.object(),
2401 2402 2403
                                       pair, literal_flags, number_of_elements);
  DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
  Node* literal = NewNode(op, feedback_vector_node());
2404
  environment()->BindAccumulator(literal, Environment::kAttachFrameState);
2405 2406
}

2407
void BytecodeGraphBuilder::VisitCreateEmptyArrayLiteral() {
2408
  int const slot_id = bytecode_iterator().GetIndexOperand(0);
2409
  FeedbackSource pair = CreateFeedbackSource(slot_id);
2410 2411 2412
  const Operator* op = javascript()->CreateEmptyLiteralArray(pair);
  DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
  Node* literal = NewNode(op, feedback_vector_node());
2413 2414 2415
  environment()->BindAccumulator(literal);
}

2416 2417 2418 2419 2420 2421
void BytecodeGraphBuilder::VisitCreateArrayFromIterable() {
  Node* iterable = NewNode(javascript()->CreateArrayFromIterable(),
                           environment()->LookupAccumulator());
  environment()->BindAccumulator(iterable, Environment::kAttachFrameState);
}

2422
void BytecodeGraphBuilder::VisitCreateObjectLiteral() {
2423
  ObjectBoilerplateDescriptionRef constant_properties(
2424
      broker(), GetConstantForIndexOperand(0));
2425
  int const slot_id = bytecode_iterator().GetIndexOperand(1);
2426
  FeedbackSource pair = CreateFeedbackSource(slot_id);
2427 2428 2429
  int bytecode_flags = bytecode_iterator().GetFlagOperand(2);
  int literal_flags =
      interpreter::CreateObjectLiteralFlags::FlagsBits::decode(bytecode_flags);
2430
  int number_of_properties = constant_properties.size();
2431
  STATIC_ASSERT(JSCreateLiteralObjectNode::FeedbackVectorIndex() == 0);
2432 2433 2434 2435
  const Operator* op = javascript()->CreateLiteralObject(
      constant_properties.object(), pair, literal_flags, number_of_properties);
  DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
  Node* literal = NewNode(op, feedback_vector_node());
2436
  environment()->BindAccumulator(literal, Environment::kAttachFrameState);
2437 2438
}

2439
void BytecodeGraphBuilder::VisitCreateEmptyObjectLiteral() {
2440
  Node* literal = NewNode(javascript()->CreateEmptyLiteralObject());
2441 2442 2443
  environment()->BindAccumulator(literal);
}

2444 2445 2446 2447 2448 2449 2450
void BytecodeGraphBuilder::VisitCloneObject() {
  PrepareEagerCheckpoint();
  Node* source =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
  int flags = bytecode_iterator().GetFlagOperand(1);
  int slot = bytecode_iterator().GetIndexOperand(2);
  const Operator* op =
2451
      javascript()->CloneObject(CreateFeedbackSource(slot), flags);
2452 2453
  STATIC_ASSERT(JSCloneObjectNode::SourceIndex() == 0);
  STATIC_ASSERT(JSCloneObjectNode::FeedbackVectorIndex() == 1);
2454
  DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
2455
  Node* value = NewNode(op, source, feedback_vector_node());
2456 2457 2458
  environment()->BindAccumulator(value, Environment::kAttachFrameState);
}

2459
void BytecodeGraphBuilder::VisitGetTemplateObject() {
2460 2461
  FeedbackSource source =
      CreateFeedbackSource(bytecode_iterator().GetIndexOperand(1));
2462 2463
  TemplateObjectDescriptionRef description(broker(),
                                           GetConstantForIndexOperand(0));
2464
  STATIC_ASSERT(JSGetTemplateObjectNode::FeedbackVectorIndex() == 0);
2465 2466 2467 2468
  const Operator* op = javascript()->GetTemplateObject(
      description.object(), shared_info().object(), source);
  DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
  Node* template_object = NewNode(op, feedback_vector_node());
2469
  environment()->BindAccumulator(template_object);
2470 2471
}

2472
Node* const* BytecodeGraphBuilder::GetCallArgumentsFromRegisters(
2473 2474
    Node* callee, Node* receiver, interpreter::Register first_arg,
    int arg_count) {
2475 2476 2477
  const int arity = JSCallNode::ArityForArgc(arg_count);
  Node** all = local_zone()->NewArray<Node*>(static_cast<size_t>(arity));
  int cursor = 0;
2478

2479 2480 2481 2482 2483
  STATIC_ASSERT(JSCallNode::TargetIndex() == 0);
  STATIC_ASSERT(JSCallNode::ReceiverIndex() == 1);
  STATIC_ASSERT(JSCallNode::FirstArgumentIndex() == 2);
  STATIC_ASSERT(JSCallNode::kFeedbackVectorIsLastInput);

2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494
  all[cursor++] = callee;
  all[cursor++] = receiver;

  // The function arguments are in consecutive registers.
  const int arg_base = first_arg.index();
  for (int i = 0; i < arg_count; ++i) {
    all[cursor++] =
        environment()->LookupRegister(interpreter::Register(arg_base + i));
  }

  all[cursor++] = feedback_vector_node();
2495

2496 2497
  DCHECK_EQ(cursor, arity);
  return all;
2498 2499
}

2500
void BytecodeGraphBuilder::BuildCall(ConvertReceiverMode receiver_mode,
2501 2502
                                     Node* const* args, size_t arg_count,
                                     int slot_id) {
2503 2504 2505
  DCHECK_EQ(interpreter::Bytecodes::GetReceiverMode(
                bytecode_iterator().current_bytecode()),
            receiver_mode);
2506
  PrepareEagerCheckpoint();
2507

2508
  FeedbackSource feedback = CreateFeedbackSource(slot_id);
2509
  CallFrequency frequency = ComputeCallFrequency(slot_id);
2510
  SpeculationMode speculation_mode = GetSpeculationMode(slot_id);
2511 2512 2513
  const Operator* op =
      javascript()->Call(arg_count, frequency, feedback, receiver_mode,
                         speculation_mode, CallFeedbackRelation::kRelated);
2514
  DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
2515

2516
  JSTypeHintLowering::LoweringResult lowering = TryBuildSimplifiedCall(
2517
      op, args, static_cast<int>(arg_count), feedback.slot);
2518 2519
  if (lowering.IsExit()) return;

2520
  Node* node = nullptr;
2521 2522
  if (lowering.IsSideEffectFree()) {
    node = lowering.value();
2523
  } else {
2524
    DCHECK(!lowering.Changed());
2525
    node = MakeNode(op, static_cast<int>(arg_count), args);
2526 2527
  }
  environment()->BindAccumulator(node, Environment::kAttachFrameState);
2528 2529
}

2530 2531 2532 2533
Node* const* BytecodeGraphBuilder::ProcessCallVarArgs(
    ConvertReceiverMode receiver_mode, Node* callee,
    interpreter::Register first_reg, int arg_count) {
  DCHECK_GE(arg_count, 0);
2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548
  Node* receiver_node;
  interpreter::Register first_arg;

  if (receiver_mode == ConvertReceiverMode::kNullOrUndefined) {
    // The receiver is implicit (and undefined), the arguments are in
    // consecutive registers.
    receiver_node = jsgraph()->UndefinedConstant();
    first_arg = first_reg;
  } else {
    // The receiver is the first register, followed by the arguments in the
    // consecutive registers.
    receiver_node = environment()->LookupRegister(first_reg);
    first_arg = interpreter::Register(first_reg.index() + 1);
  }

2549 2550
  Node* const* call_args = GetCallArgumentsFromRegisters(callee, receiver_node,
                                                         first_arg, arg_count);
2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563 2564 2565 2566
  return call_args;
}

void BytecodeGraphBuilder::BuildCallVarArgs(ConvertReceiverMode receiver_mode) {
  DCHECK_EQ(interpreter::Bytecodes::GetReceiverMode(
                bytecode_iterator().current_bytecode()),
            receiver_mode);
  Node* callee =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
  interpreter::Register first_reg = bytecode_iterator().GetRegisterOperand(1);
  size_t reg_count = bytecode_iterator().GetRegisterCountOperand(2);
  int const slot_id = bytecode_iterator().GetIndexOperand(3);

  int arg_count = receiver_mode == ConvertReceiverMode::kNullOrUndefined
                      ? static_cast<int>(reg_count)
                      : static_cast<int>(reg_count) - 1;
2567
  Node* const* call_args =
2568
      ProcessCallVarArgs(receiver_mode, callee, first_reg, arg_count);
2569 2570
  BuildCall(receiver_mode, call_args, JSCallNode::ArityForArgc(arg_count),
            slot_id);
2571 2572
}

2573
void BytecodeGraphBuilder::VisitCallAnyReceiver() {
2574
  BuildCallVarArgs(ConvertReceiverMode::kAny);
2575 2576
}

2577
void BytecodeGraphBuilder::VisitCallNoFeedback() {
2578 2579 2580 2581
  DCHECK_EQ(interpreter::Bytecodes::GetReceiverMode(
                bytecode_iterator().current_bytecode()),
            ConvertReceiverMode::kAny);

2582
  PrepareEagerCheckpoint();
2583 2584 2585 2586 2587 2588 2589 2590 2591
  Node* callee =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));

  interpreter::Register first_reg = bytecode_iterator().GetRegisterOperand(1);
  size_t reg_count = bytecode_iterator().GetRegisterCountOperand(2);

  // The receiver is the first register, followed by the arguments in the
  // consecutive registers.
  int arg_count = static_cast<int>(reg_count) - 1;
2592
  int arity = JSCallNode::ArityForArgc(arg_count);
2593 2594 2595 2596 2597 2598 2599 2600

  // Setting call frequency to a value less than min_inlining frequency to
  // prevent inlining of one-shot call node.
  DCHECK(CallFrequency::kNoFeedbackCallFrequency < FLAG_min_inlining_frequency);
  const Operator* call = javascript()->Call(
      arity, CallFrequency(CallFrequency::kNoFeedbackCallFrequency));
  Node* const* call_args = ProcessCallVarArgs(ConvertReceiverMode::kAny, callee,
                                              first_reg, arg_count);
2601
  Node* value = MakeNode(call, arity, call_args);
2602
  environment()->BindAccumulator(value, Environment::kAttachFrameState);
2603 2604
}

2605
void BytecodeGraphBuilder::VisitCallProperty() {
2606
  BuildCallVarArgs(ConvertReceiverMode::kNotNullOrUndefined);
2607 2608 2609
}

void BytecodeGraphBuilder::VisitCallProperty0() {
2610 2611 2612 2613 2614
  Node* callee =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
  Node* receiver =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(1));
  int const slot_id = bytecode_iterator().GetIndexOperand(2);
2615 2616
  BuildCall(ConvertReceiverMode::kNotNullOrUndefined,
            {callee, receiver, feedback_vector_node()}, slot_id);
2617 2618
}

2619
void BytecodeGraphBuilder::VisitCallProperty1() {
2620 2621 2622 2623 2624 2625 2626
  Node* callee =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
  Node* receiver =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(1));
  Node* arg0 =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(2));
  int const slot_id = bytecode_iterator().GetIndexOperand(3);
2627 2628
  BuildCall(ConvertReceiverMode::kNotNullOrUndefined,
            {callee, receiver, arg0, feedback_vector_node()}, slot_id);
2629 2630
}

2631
void BytecodeGraphBuilder::VisitCallProperty2() {
2632 2633 2634 2635 2636 2637 2638 2639 2640
  Node* callee =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
  Node* receiver =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(1));
  Node* arg0 =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(2));
  Node* arg1 =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(3));
  int const slot_id = bytecode_iterator().GetIndexOperand(4);
2641
  BuildCall(ConvertReceiverMode::kNotNullOrUndefined,
2642
            {callee, receiver, arg0, arg1, feedback_vector_node()}, slot_id);
2643 2644
}

2645
void BytecodeGraphBuilder::VisitCallUndefinedReceiver() {
2646
  BuildCallVarArgs(ConvertReceiverMode::kNullOrUndefined);
2647 2648
}

2649
void BytecodeGraphBuilder::VisitCallUndefinedReceiver0() {
2650 2651
  Node* callee =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
2652 2653
  Node* receiver = jsgraph()->UndefinedConstant();
  int const slot_id = bytecode_iterator().GetIndexOperand(1);
2654 2655
  BuildCall(ConvertReceiverMode::kNullOrUndefined,
            {callee, receiver, feedback_vector_node()}, slot_id);
2656 2657
}

2658
void BytecodeGraphBuilder::VisitCallUndefinedReceiver1() {
2659 2660
  Node* callee =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
2661
  Node* receiver = jsgraph()->UndefinedConstant();
2662
  Node* arg0 =
2663 2664
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(1));
  int const slot_id = bytecode_iterator().GetIndexOperand(2);
2665 2666
  BuildCall(ConvertReceiverMode::kNullOrUndefined,
            {callee, receiver, arg0, feedback_vector_node()}, slot_id);
2667 2668
}

2669
void BytecodeGraphBuilder::VisitCallUndefinedReceiver2() {
2670 2671
  Node* callee =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
2672
  Node* receiver = jsgraph()->UndefinedConstant();
2673
  Node* arg0 =
2674
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(1));
2675
  Node* arg1 =
2676 2677
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(2));
  int const slot_id = bytecode_iterator().GetIndexOperand(3);
2678
  BuildCall(ConvertReceiverMode::kNullOrUndefined,
2679
            {callee, receiver, arg0, arg1, feedback_vector_node()}, slot_id);
2680 2681
}

2682 2683
void BytecodeGraphBuilder::VisitCallWithSpread() {
  PrepareEagerCheckpoint();
2684 2685 2686
  Node* callee =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
  interpreter::Register receiver = bytecode_iterator().GetRegisterOperand(1);
2687
  Node* receiver_node = environment()->LookupRegister(receiver);
2688
  size_t reg_count = bytecode_iterator().GetRegisterCountOperand(2);
2689 2690
  interpreter::Register first_arg = interpreter::Register(receiver.index() + 1);
  int arg_count = static_cast<int>(reg_count) - 1;
2691 2692
  Node* const* args = GetCallArgumentsFromRegisters(callee, receiver_node,
                                                    first_arg, arg_count);
2693
  int const slot_id = bytecode_iterator().GetIndexOperand(3);
2694
  FeedbackSource feedback = CreateFeedbackSource(slot_id);
2695
  CallFrequency frequency = ComputeCallFrequency(slot_id);
2696
  SpeculationMode speculation_mode = GetSpeculationMode(slot_id);
2697
  const Operator* op = javascript()->CallWithSpread(
2698 2699
      JSCallWithSpreadNode::ArityForArgc(arg_count), frequency, feedback,
      speculation_mode);
2700
  DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
2701 2702

  JSTypeHintLowering::LoweringResult lowering = TryBuildSimplifiedCall(
2703
      op, args, static_cast<int>(arg_count), feedback.slot);
2704 2705
  if (lowering.IsExit()) return;

2706
  Node* node = nullptr;
2707 2708
  if (lowering.IsSideEffectFree()) {
    node = lowering.value();
2709
  } else {
2710
    DCHECK(!lowering.Changed());
2711
    node = MakeNode(op, JSCallWithSpreadNode::ArityForArgc(arg_count), args);
2712 2713
  }
  environment()->BindAccumulator(node, Environment::kAttachFrameState);
2714 2715
}

2716
void BytecodeGraphBuilder::VisitCallJSRuntime() {
2717
  PrepareEagerCheckpoint();
2718 2719
  Node* callee = BuildLoadNativeContextField(
      bytecode_iterator().GetNativeContextIndexOperand(0));
2720
  interpreter::Register first_reg = bytecode_iterator().GetRegisterOperand(1);
2721
  size_t reg_count = bytecode_iterator().GetRegisterCountOperand(2);
2722
  int arg_count = static_cast<int>(reg_count);
2723
  int arity = JSCallNode::ArityForArgc(arg_count);
2724

2725
  const Operator* call = javascript()->Call(arity);
2726 2727
  Node* const* call_args = ProcessCallVarArgs(
      ConvertReceiverMode::kNullOrUndefined, callee, first_reg, arg_count);
2728
  Node* value = MakeNode(call, arity, call_args);
2729
  environment()->BindAccumulator(value, Environment::kAttachFrameState);
2730 2731
}

2732
Node* BytecodeGraphBuilder::ProcessCallRuntimeArguments(
2733 2734 2735 2736 2737 2738 2739 2740
    const Operator* call_runtime_op, interpreter::Register receiver,
    size_t reg_count) {
  int arg_count = static_cast<int>(reg_count);
  // arity is args.
  int arity = arg_count;
  Node** all = local_zone()->NewArray<Node*>(static_cast<size_t>(arity));
  int first_arg_index = receiver.index();
  for (int i = 0; i < static_cast<int>(reg_count); ++i) {
2741 2742 2743
    all[i] = environment()->LookupRegister(
        interpreter::Register(first_arg_index + i));
  }
2744
  Node* value = MakeNode(call_runtime_op, arity, all);
2745 2746 2747
  return value;
}

2748
void BytecodeGraphBuilder::VisitCallRuntime() {
2749
  PrepareEagerCheckpoint();
2750
  Runtime::FunctionId function_id = bytecode_iterator().GetRuntimeIdOperand(0);
2751 2752
  interpreter::Register receiver = bytecode_iterator().GetRegisterOperand(1);
  size_t reg_count = bytecode_iterator().GetRegisterCountOperand(2);
2753

2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772
  // Handle %ObserveNode here (rather than in JSIntrinsicLowering) to observe
  // the node as early as possible.
  if (function_id == Runtime::FunctionId::kObserveNode) {
    DCHECK_EQ(1, reg_count);
    Node* value = environment()->LookupRegister(receiver);
    observe_node_info_.StartObserving(value);
    environment()->BindAccumulator(value);
  } else {
    // Create node to perform the runtime call.
    const Operator* call = javascript()->CallRuntime(function_id, reg_count);
    Node* value = ProcessCallRuntimeArguments(call, receiver, reg_count);
    environment()->BindAccumulator(value, Environment::kAttachFrameState);

    // Connect to the end if {function_id} is non-returning.
    if (Runtime::IsNonReturning(function_id)) {
      // TODO(7099): Investigate if we need LoopExit node here.
      Node* control = NewNode(common()->Throw());
      MergeControlToLeaveFunction(control);
    }
2773
  }
2774 2775
}

2776
void BytecodeGraphBuilder::VisitCallRuntimeForPair() {
2777
  PrepareEagerCheckpoint();
2778
  Runtime::FunctionId functionId = bytecode_iterator().GetRuntimeIdOperand(0);
2779 2780
  interpreter::Register receiver = bytecode_iterator().GetRegisterOperand(1);
  size_t reg_count = bytecode_iterator().GetRegisterCountOperand(2);
2781 2782
  interpreter::Register first_return =
      bytecode_iterator().GetRegisterOperand(3);
2783 2784

  // Create node to perform the runtime call.
2785 2786
  const Operator* call = javascript()->CallRuntime(functionId, reg_count);
  Node* return_pair = ProcessCallRuntimeArguments(call, receiver, reg_count);
2787 2788
  environment()->BindRegistersToProjections(first_return, return_pair,
                                            Environment::kAttachFrameState);
2789 2790
}

2791 2792 2793
Node* const* BytecodeGraphBuilder::GetConstructArgumentsFromRegister(
    Node* target, Node* new_target, interpreter::Register first_arg,
    int arg_count) {
2794
  const int arity = JSConstructNode::ArityForArgc(arg_count);
2795
  Node** all = local_zone()->NewArray<Node*>(static_cast<size_t>(arity));
2796 2797 2798
  int cursor = 0;

  STATIC_ASSERT(JSConstructNode::TargetIndex() == 0);
2799 2800 2801
  STATIC_ASSERT(JSConstructNode::NewTargetIndex() == 1);
  STATIC_ASSERT(JSConstructNode::FirstArgumentIndex() == 2);
  STATIC_ASSERT(JSConstructNode::kFeedbackVectorIsLastInput);
2802 2803

  all[cursor++] = target;
2804
  all[cursor++] = new_target;
2805 2806 2807

  // The function arguments are in consecutive registers.
  int arg_base = first_arg.index();
2808
  for (int i = 0; i < arg_count; ++i) {
2809 2810
    all[cursor++] =
        environment()->LookupRegister(interpreter::Register(arg_base + i));
2811
  }
2812

2813
  all[cursor++] = feedback_vector_node();
2814 2815

  DCHECK_EQ(cursor, arity);
2816 2817 2818 2819 2820 2821 2822 2823 2824
  return all;
}

void BytecodeGraphBuilder::VisitConstruct() {
  PrepareEagerCheckpoint();
  interpreter::Register callee_reg = bytecode_iterator().GetRegisterOperand(0);
  interpreter::Register first_reg = bytecode_iterator().GetRegisterOperand(1);
  size_t reg_count = bytecode_iterator().GetRegisterCountOperand(2);
  int const slot_id = bytecode_iterator().GetIndexOperand(3);
2825
  FeedbackSource feedback = CreateFeedbackSource(slot_id);
2826 2827 2828 2829 2830

  Node* new_target = environment()->LookupAccumulator();
  Node* callee = environment()->LookupRegister(callee_reg);

  CallFrequency frequency = ComputeCallFrequency(slot_id);
2831
  const uint32_t arg_count = static_cast<uint32_t>(reg_count);
2832 2833
  const uint32_t arity = JSConstructNode::ArityForArgc(arg_count);
  const Operator* op = javascript()->Construct(arity, frequency, feedback);
2834
  DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
2835 2836
  Node* const* args = GetConstructArgumentsFromRegister(callee, new_target,
                                                        first_reg, arg_count);
2837
  JSTypeHintLowering::LoweringResult lowering = TryBuildSimplifiedConstruct(
2838
      op, args, static_cast<int>(arg_count), feedback.slot);
2839 2840
  if (lowering.IsExit()) return;

2841
  Node* node = nullptr;
2842 2843
  if (lowering.IsSideEffectFree()) {
    node = lowering.value();
2844
  } else {
2845
    DCHECK(!lowering.Changed());
2846
    node = MakeNode(op, arity, args);
2847 2848
  }
  environment()->BindAccumulator(node, Environment::kAttachFrameState);
2849 2850
}

2851
void BytecodeGraphBuilder::VisitConstructWithSpread() {
2852
  PrepareEagerCheckpoint();
2853
  interpreter::Register callee_reg = bytecode_iterator().GetRegisterOperand(0);
2854
  interpreter::Register first_reg = bytecode_iterator().GetRegisterOperand(1);
2855
  size_t reg_count = bytecode_iterator().GetRegisterCountOperand(2);
2856
  int const slot_id = bytecode_iterator().GetIndexOperand(3);
2857
  FeedbackSource feedback = CreateFeedbackSource(slot_id);
2858 2859 2860

  Node* new_target = environment()->LookupAccumulator();
  Node* callee = environment()->LookupRegister(callee_reg);
2861

2862
  CallFrequency frequency = ComputeCallFrequency(slot_id);
2863
  const uint32_t arg_count = static_cast<uint32_t>(reg_count);
2864 2865 2866
  const uint32_t arity = JSConstructNode::ArityForArgc(arg_count);
  const Operator* op =
      javascript()->ConstructWithSpread(arity, frequency, feedback);
2867
  DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
2868 2869
  Node* const* args = GetConstructArgumentsFromRegister(callee, new_target,
                                                        first_reg, arg_count);
2870
  JSTypeHintLowering::LoweringResult lowering = TryBuildSimplifiedConstruct(
2871
      op, args, static_cast<int>(arg_count), feedback.slot);
2872 2873
  if (lowering.IsExit()) return;

2874
  Node* node = nullptr;
2875 2876
  if (lowering.IsSideEffectFree()) {
    node = lowering.value();
2877
  } else {
2878
    DCHECK(!lowering.Changed());
2879
    node = MakeNode(op, arity, args);
2880 2881
  }
  environment()->BindAccumulator(node, Environment::kAttachFrameState);
2882 2883
}

2884
void BytecodeGraphBuilder::VisitInvokeIntrinsic() {
2885
  PrepareEagerCheckpoint();
2886
  Runtime::FunctionId functionId = bytecode_iterator().GetIntrinsicIdOperand(0);
2887 2888
  interpreter::Register receiver = bytecode_iterator().GetRegisterOperand(1);
  size_t reg_count = bytecode_iterator().GetRegisterCountOperand(2);
2889 2890 2891

  // Create node to perform the runtime call. Turbofan will take care of the
  // lowering.
2892 2893
  const Operator* call = javascript()->CallRuntime(functionId, reg_count);
  Node* value = ProcessCallRuntimeArguments(call, receiver, reg_count);
2894
  environment()->BindAccumulator(value, Environment::kAttachFrameState);
2895 2896
}

2897
void BytecodeGraphBuilder::VisitThrow() {
2898
  BuildLoopExitsForFunctionExit(bytecode_analysis().GetInLivenessFor(
2899
      bytecode_iterator().current_offset()));
2900
  Node* value = environment()->LookupAccumulator();
2901
  Node* call = NewNode(javascript()->CallRuntime(Runtime::kThrow), value);
2902
  environment()->BindAccumulator(call, Environment::kAttachFrameState);
2903
  Node* control = NewNode(common()->Throw());
2904
  MergeControlToLeaveFunction(control);
2905 2906
}

2907
void BytecodeGraphBuilder::VisitAbort() {
2908
  BuildLoopExitsForFunctionExit(bytecode_analysis().GetInLivenessFor(
2909
      bytecode_iterator().current_offset()));
2910 2911
  AbortReason reason =
      static_cast<AbortReason>(bytecode_iterator().GetIndexOperand(0));
2912 2913 2914 2915 2916
  NewNode(simplified()->RuntimeAbort(reason));
  Node* control = NewNode(common()->Throw());
  MergeControlToLeaveFunction(control);
}

2917
void BytecodeGraphBuilder::VisitReThrow() {
2918
  BuildLoopExitsForFunctionExit(bytecode_analysis().GetInLivenessFor(
2919
      bytecode_iterator().current_offset()));
2920
  Node* value = environment()->LookupAccumulator();
2921 2922
  NewNode(javascript()->CallRuntime(Runtime::kReThrow), value);
  Node* control = NewNode(common()->Throw());
2923
  MergeControlToLeaveFunction(control);
2924 2925
}

2926 2927 2928 2929 2930 2931 2932 2933
void BytecodeGraphBuilder::BuildHoleCheckAndThrow(
    Node* condition, Runtime::FunctionId runtime_id, Node* name) {
  Node* accumulator = environment()->LookupAccumulator();
  NewBranch(condition, BranchHint::kFalse);
  {
    SubEnvironment sub_environment(this);

    NewIfTrue();
2934
    BuildLoopExitsForFunctionExit(bytecode_analysis().GetInLivenessFor(
2935
        bytecode_iterator().current_offset()));
2936 2937
    Node* node;
    const Operator* op = javascript()->CallRuntime(runtime_id);
2938
    if (runtime_id == Runtime::kThrowAccessedUninitializedVariable) {
2939
      DCHECK_NOT_NULL(name);
2940 2941 2942 2943 2944 2945 2946 2947 2948 2949 2950 2951 2952 2953 2954
      node = NewNode(op, name);
    } else {
      DCHECK(runtime_id == Runtime::kThrowSuperAlreadyCalledError ||
             runtime_id == Runtime::kThrowSuperNotCalled);
      node = NewNode(op);
    }
    environment()->RecordAfterState(node, Environment::kAttachFrameState);
    Node* control = NewNode(common()->Throw());
    MergeControlToLeaveFunction(control);
  }
  NewIfFalse();
  environment()->BindAccumulator(accumulator);
}

void BytecodeGraphBuilder::VisitThrowReferenceErrorIfHole() {
2955 2956 2957
  Node* accumulator = environment()->LookupAccumulator();
  Node* check_for_hole = NewNode(simplified()->ReferenceEqual(), accumulator,
                                 jsgraph()->TheHoleConstant());
2958 2959
  Node* name =
      jsgraph()->Constant(ObjectRef(broker(), GetConstantForIndexOperand(0)));
2960 2961
  BuildHoleCheckAndThrow(check_for_hole,
                         Runtime::kThrowAccessedUninitializedVariable, name);
2962 2963 2964
}

void BytecodeGraphBuilder::VisitThrowSuperNotCalledIfHole() {
2965 2966 2967 2968
  Node* accumulator = environment()->LookupAccumulator();
  Node* check_for_hole = NewNode(simplified()->ReferenceEqual(), accumulator,
                                 jsgraph()->TheHoleConstant());
  BuildHoleCheckAndThrow(check_for_hole, Runtime::kThrowSuperNotCalled);
2969 2970 2971
}

void BytecodeGraphBuilder::VisitThrowSuperAlreadyCalledIfNotHole() {
2972 2973 2974 2975 2976 2977 2978
  Node* accumulator = environment()->LookupAccumulator();
  Node* check_for_hole = NewNode(simplified()->ReferenceEqual(), accumulator,
                                 jsgraph()->TheHoleConstant());
  Node* check_for_not_hole =
      NewNode(simplified()->BooleanNot(), check_for_hole);
  BuildHoleCheckAndThrow(check_for_not_hole,
                         Runtime::kThrowSuperAlreadyCalledError);
2979 2980
}

2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005
void BytecodeGraphBuilder::VisitThrowIfNotSuperConstructor() {
  Node* constructor =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
  Node* check_is_constructor =
      NewNode(simplified()->ObjectIsConstructor(), constructor);
  NewBranch(check_is_constructor, BranchHint::kTrue);
  {
    SubEnvironment sub_environment(this);
    NewIfFalse();
    BuildLoopExitsForFunctionExit(bytecode_analysis().GetInLivenessFor(
        bytecode_iterator().current_offset()));
    Node* node =
        NewNode(javascript()->CallRuntime(Runtime::kThrowNotSuperConstructor),
                constructor, GetFunctionClosure());
    environment()->RecordAfterState(node, Environment::kAttachFrameState);
    Node* control = NewNode(common()->Throw());
    MergeControlToLeaveFunction(control);
  }
  NewIfTrue();

  constructor = NewNode(common()->TypeGuard(Type::Callable()), constructor);
  environment()->BindRegister(bytecode_iterator().GetRegisterOperand(0),
                              constructor);
}

3006
void BytecodeGraphBuilder::BuildUnaryOp(const Operator* op) {
3007
  DCHECK(JSOperator::IsUnaryWithFeedback(op->opcode()));
3008 3009 3010
  PrepareEagerCheckpoint();
  Node* operand = environment()->LookupAccumulator();

3011 3012
  FeedbackSlot slot =
      bytecode_iterator().GetSlotOperand(kUnaryOperationHintIndex);
3013 3014 3015 3016 3017 3018 3019 3020 3021
  JSTypeHintLowering::LoweringResult lowering =
      TryBuildSimplifiedUnaryOp(op, operand, slot);
  if (lowering.IsExit()) return;

  Node* node = nullptr;
  if (lowering.IsSideEffectFree()) {
    node = lowering.value();
  } else {
    DCHECK(!lowering.Changed());
3022
    DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
3023
    node = NewNode(op, operand, feedback_vector_node());
3024 3025 3026 3027 3028
  }

  environment()->BindAccumulator(node, Environment::kAttachFrameState);
}

3029
void BytecodeGraphBuilder::BuildBinaryOp(const Operator* op) {
3030
  DCHECK(JSOperator::IsBinaryWithFeedback(op->opcode()));
3031
  PrepareEagerCheckpoint();
3032 3033
  Node* left =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
3034
  Node* right = environment()->LookupAccumulator();
3035

3036 3037
  FeedbackSlot slot =
      bytecode_iterator().GetSlotOperand(kBinaryOperationHintIndex);
3038 3039
  JSTypeHintLowering::LoweringResult lowering =
      TryBuildSimplifiedBinaryOp(op, left, right, slot);
3040
  if (lowering.IsExit()) return;
3041 3042 3043 3044

  Node* node = nullptr;
  if (lowering.IsSideEffectFree()) {
    node = lowering.value();
3045
  } else {
3046
    DCHECK(!lowering.Changed());
3047
    DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
3048
    node = NewNode(op, left, right, feedback_vector_node());
3049 3050
  }

3051
  environment()->BindAccumulator(node, Environment::kAttachFrameState);
3052 3053
}

3054
// Helper function to create for-in mode from the recorded type feedback.
3055
ForInMode BytecodeGraphBuilder::GetForInMode(FeedbackSlot slot) {
3056 3057
  FeedbackSource source(feedback_vector(), slot);
  switch (broker()->GetFeedbackForForIn(source)) {
3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068
    case ForInHint::kNone:
    case ForInHint::kEnumCacheKeysAndIndices:
      return ForInMode::kUseEnumCacheKeysAndIndices;
    case ForInHint::kEnumCacheKeys:
      return ForInMode::kUseEnumCacheKeys;
    case ForInHint::kAny:
      return ForInMode::kGeneric;
  }
  UNREACHABLE();
}

3069 3070
CallFrequency BytecodeGraphBuilder::ComputeCallFrequency(int slot_id) const {
  if (invocation_frequency_.IsUnknown()) return CallFrequency();
3071 3072 3073 3074 3075 3076
  FeedbackSlot slot = FeedbackVector::ToSlot(slot_id);
  FeedbackSource source(feedback_vector(), slot);
  ProcessedFeedback const& feedback = broker()->GetFeedbackForCall(source);
  float feedback_frequency =
      feedback.IsInsufficient() ? 0.0f : feedback.AsCall().frequency();
  if (feedback_frequency == 0.0f) {  // Prevent multiplying zero and infinity.
3077 3078 3079 3080
    return CallFrequency(0.0f);
  } else {
    return CallFrequency(feedback_frequency * invocation_frequency_.value());
  }
3081 3082
}

3083
SpeculationMode BytecodeGraphBuilder::GetSpeculationMode(int slot_id) const {
3084 3085 3086 3087 3088
  FeedbackSlot slot = FeedbackVector::ToSlot(slot_id);
  FeedbackSource source(feedback_vector(), slot);
  ProcessedFeedback const& feedback = broker()->GetFeedbackForCall(source);
  return feedback.IsInsufficient() ? SpeculationMode::kDisallowSpeculation
                                   : feedback.AsCall().speculation_mode();
3089 3090
}

3091
void BytecodeGraphBuilder::VisitBitwiseNot() {
3092
  FeedbackSource feedback = CreateFeedbackSource(
3093
      bytecode_iterator().GetSlotOperand(kUnaryOperationHintIndex));
3094
  BuildUnaryOp(javascript()->BitwiseNot(feedback));
3095
}
3096

3097
void BytecodeGraphBuilder::VisitDec() {
3098
  FeedbackSource feedback = CreateFeedbackSource(
3099
      bytecode_iterator().GetSlotOperand(kUnaryOperationHintIndex));
3100
  BuildUnaryOp(javascript()->Decrement(feedback));
3101
}
3102

3103
void BytecodeGraphBuilder::VisitInc() {
3104
  FeedbackSource feedback = CreateFeedbackSource(
3105
      bytecode_iterator().GetSlotOperand(kUnaryOperationHintIndex));
3106
  BuildUnaryOp(javascript()->Increment(feedback));
3107 3108
}

3109
void BytecodeGraphBuilder::VisitNegate() {
3110
  FeedbackSource feedback = CreateFeedbackSource(
3111
      bytecode_iterator().GetSlotOperand(kUnaryOperationHintIndex));
3112
  BuildUnaryOp(javascript()->Negate(feedback));
3113 3114
}

3115
void BytecodeGraphBuilder::VisitAdd() {
3116 3117 3118
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationHintIndex));
  BuildBinaryOp(javascript()->Add(feedback));
3119 3120
}

3121
void BytecodeGraphBuilder::VisitSub() {
3122 3123 3124
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationHintIndex));
  BuildBinaryOp(javascript()->Subtract(feedback));
3125 3126
}

3127
void BytecodeGraphBuilder::VisitMul() {
3128 3129 3130
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationHintIndex));
  BuildBinaryOp(javascript()->Multiply(feedback));
3131 3132
}

3133 3134 3135 3136 3137
void BytecodeGraphBuilder::VisitDiv() {
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationHintIndex));
  BuildBinaryOp(javascript()->Divide(feedback));
}
3138

3139
void BytecodeGraphBuilder::VisitMod() {
3140 3141 3142
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationHintIndex));
  BuildBinaryOp(javascript()->Modulus(feedback));
3143 3144
}

3145
void BytecodeGraphBuilder::VisitExp() {
3146 3147 3148
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationHintIndex));
  BuildBinaryOp(javascript()->Exponentiate(feedback));
3149 3150
}

3151
void BytecodeGraphBuilder::VisitBitwiseOr() {
3152 3153 3154
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationHintIndex));
  BuildBinaryOp(javascript()->BitwiseOr(feedback));
3155 3156
}

3157
void BytecodeGraphBuilder::VisitBitwiseXor() {
3158 3159 3160
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationHintIndex));
  BuildBinaryOp(javascript()->BitwiseXor(feedback));
3161 3162
}

3163
void BytecodeGraphBuilder::VisitBitwiseAnd() {
3164 3165 3166
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationHintIndex));
  BuildBinaryOp(javascript()->BitwiseAnd(feedback));
3167 3168
}

3169
void BytecodeGraphBuilder::VisitShiftLeft() {
3170 3171 3172
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationHintIndex));
  BuildBinaryOp(javascript()->ShiftLeft(feedback));
3173 3174
}

3175
void BytecodeGraphBuilder::VisitShiftRight() {
3176 3177 3178
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationHintIndex));
  BuildBinaryOp(javascript()->ShiftRight(feedback));
3179 3180
}

3181
void BytecodeGraphBuilder::VisitShiftRightLogical() {
3182 3183 3184
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationHintIndex));
  BuildBinaryOp(javascript()->ShiftRightLogical(feedback));
3185 3186
}

3187
void BytecodeGraphBuilder::BuildBinaryOpWithImmediate(const Operator* op) {
3188
  DCHECK(JSOperator::IsBinaryWithFeedback(op->opcode()));
3189
  PrepareEagerCheckpoint();
3190
  Node* left = environment()->LookupAccumulator();
3191
  Node* right = jsgraph()->Constant(bytecode_iterator().GetImmediateOperand(0));
3192

3193 3194
  FeedbackSlot slot =
      bytecode_iterator().GetSlotOperand(kBinaryOperationSmiHintIndex);
3195 3196
  JSTypeHintLowering::LoweringResult lowering =
      TryBuildSimplifiedBinaryOp(op, left, right, slot);
3197 3198
  if (lowering.IsExit()) return;

3199 3200 3201
  Node* node = nullptr;
  if (lowering.IsSideEffectFree()) {
    node = lowering.value();
3202
  } else {
3203
    DCHECK(!lowering.Changed());
3204
    DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
3205
    node = NewNode(op, left, right, feedback_vector_node());
3206
  }
3207
  environment()->BindAccumulator(node, Environment::kAttachFrameState);
3208 3209 3210
}

void BytecodeGraphBuilder::VisitAddSmi() {
3211 3212 3213
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationSmiHintIndex));
  BuildBinaryOpWithImmediate(javascript()->Add(feedback));
3214 3215 3216
}

void BytecodeGraphBuilder::VisitSubSmi() {
3217 3218 3219
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationSmiHintIndex));
  BuildBinaryOpWithImmediate(javascript()->Subtract(feedback));
3220 3221
}

3222
void BytecodeGraphBuilder::VisitMulSmi() {
3223 3224 3225
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationSmiHintIndex));
  BuildBinaryOpWithImmediate(javascript()->Multiply(feedback));
3226 3227 3228
}

void BytecodeGraphBuilder::VisitDivSmi() {
3229 3230 3231
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationSmiHintIndex));
  BuildBinaryOpWithImmediate(javascript()->Divide(feedback));
3232 3233 3234
}

void BytecodeGraphBuilder::VisitModSmi() {
3235 3236 3237
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationSmiHintIndex));
  BuildBinaryOpWithImmediate(javascript()->Modulus(feedback));
3238 3239
}

3240
void BytecodeGraphBuilder::VisitExpSmi() {
3241 3242 3243
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationSmiHintIndex));
  BuildBinaryOpWithImmediate(javascript()->Exponentiate(feedback));
3244 3245
}

3246
void BytecodeGraphBuilder::VisitBitwiseOrSmi() {
3247 3248 3249
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationSmiHintIndex));
  BuildBinaryOpWithImmediate(javascript()->BitwiseOr(feedback));
3250 3251
}

3252
void BytecodeGraphBuilder::VisitBitwiseXorSmi() {
3253 3254 3255
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationSmiHintIndex));
  BuildBinaryOpWithImmediate(javascript()->BitwiseXor(feedback));
3256 3257
}

3258
void BytecodeGraphBuilder::VisitBitwiseAndSmi() {
3259 3260 3261
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationSmiHintIndex));
  BuildBinaryOpWithImmediate(javascript()->BitwiseAnd(feedback));
3262 3263 3264
}

void BytecodeGraphBuilder::VisitShiftLeftSmi() {
3265 3266 3267
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationSmiHintIndex));
  BuildBinaryOpWithImmediate(javascript()->ShiftLeft(feedback));
3268 3269 3270
}

void BytecodeGraphBuilder::VisitShiftRightSmi() {
3271 3272 3273
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationSmiHintIndex));
  BuildBinaryOpWithImmediate(javascript()->ShiftRight(feedback));
3274 3275
}

3276
void BytecodeGraphBuilder::VisitShiftRightLogicalSmi() {
3277 3278 3279
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kBinaryOperationSmiHintIndex));
  BuildBinaryOpWithImmediate(javascript()->ShiftRightLogical(feedback));
3280 3281
}

3282
void BytecodeGraphBuilder::VisitLogicalNot() {
3283
  Node* value = environment()->LookupAccumulator();
3284
  Node* node = NewNode(simplified()->BooleanNot(), value);
3285 3286 3287 3288
  environment()->BindAccumulator(node);
}

void BytecodeGraphBuilder::VisitToBooleanLogicalNot() {
3289 3290
  Node* value =
      NewNode(simplified()->ToBoolean(), environment()->LookupAccumulator());
3291
  Node* node = NewNode(simplified()->BooleanNot(), value);
3292
  environment()->BindAccumulator(node);
3293 3294
}

3295
void BytecodeGraphBuilder::VisitTypeOf() {
3296
  Node* node =
3297
      NewNode(simplified()->TypeOf(), environment()->LookupAccumulator());
3298 3299 3300
  environment()->BindAccumulator(node);
}

3301
void BytecodeGraphBuilder::BuildDelete(LanguageMode language_mode) {
3302
  PrepareEagerCheckpoint();
3303
  Node* key = environment()->LookupAccumulator();
3304 3305
  Node* object =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
3306 3307
  Node* mode = jsgraph()->Constant(static_cast<int32_t>(language_mode));
  Node* node = NewNode(javascript()->DeleteProperty(), object, key, mode);
3308
  environment()->BindAccumulator(node, Environment::kAttachFrameState);
3309 3310
}

3311
void BytecodeGraphBuilder::VisitDeletePropertyStrict() {
3312
  BuildDelete(LanguageMode::kStrict);
3313 3314
}

3315
void BytecodeGraphBuilder::VisitDeletePropertySloppy() {
3316
  BuildDelete(LanguageMode::kSloppy);
3317 3318
}

3319 3320 3321 3322 3323 3324 3325
void BytecodeGraphBuilder::VisitGetSuperConstructor() {
  Node* node = NewNode(javascript()->GetSuperConstructor(),
                       environment()->LookupAccumulator());
  environment()->BindRegister(bytecode_iterator().GetRegisterOperand(0), node,
                              Environment::kAttachFrameState);
}

3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342
void BytecodeGraphBuilder::BuildCompareOp(const Operator* op) {
  DCHECK(JSOperator::IsBinaryWithFeedback(op->opcode()));
  PrepareEagerCheckpoint();
  Node* left =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
  Node* right = environment()->LookupAccumulator();

  FeedbackSlot slot = bytecode_iterator().GetSlotOperand(1);
  JSTypeHintLowering::LoweringResult lowering =
      TryBuildSimplifiedBinaryOp(op, left, right, slot);
  if (lowering.IsExit()) return;

  Node* node = nullptr;
  if (lowering.IsSideEffectFree()) {
    node = lowering.value();
  } else {
    DCHECK(!lowering.Changed());
3343
    DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
3344 3345 3346 3347 3348
    node = NewNode(op, left, right, feedback_vector_node());
  }
  environment()->BindAccumulator(node, Environment::kAttachFrameState);
}

3349
void BytecodeGraphBuilder::VisitTestEqual() {
3350
  FeedbackSource feedback = CreateFeedbackSource(
3351
      bytecode_iterator().GetSlotOperand(kCompareOperationHintIndex));
3352
  BuildCompareOp(javascript()->Equal(feedback));
3353 3354
}

3355
void BytecodeGraphBuilder::VisitTestEqualStrict() {
3356
  FeedbackSource feedback = CreateFeedbackSource(
3357
      bytecode_iterator().GetSlotOperand(kCompareOperationHintIndex));
3358
  BuildCompareOp(javascript()->StrictEqual(feedback));
3359 3360
}

3361
void BytecodeGraphBuilder::VisitTestLessThan() {
3362
  FeedbackSource feedback = CreateFeedbackSource(
3363
      bytecode_iterator().GetSlotOperand(kCompareOperationHintIndex));
3364
  BuildCompareOp(javascript()->LessThan(feedback));
3365 3366
}

3367
void BytecodeGraphBuilder::VisitTestGreaterThan() {
3368
  FeedbackSource feedback = CreateFeedbackSource(
3369
      bytecode_iterator().GetSlotOperand(kCompareOperationHintIndex));
3370
  BuildCompareOp(javascript()->GreaterThan(feedback));
3371 3372
}

3373
void BytecodeGraphBuilder::VisitTestLessThanOrEqual() {
3374
  FeedbackSource feedback = CreateFeedbackSource(
3375
      bytecode_iterator().GetSlotOperand(kCompareOperationHintIndex));
3376
  BuildCompareOp(javascript()->LessThanOrEqual(feedback));
3377 3378
}

3379
void BytecodeGraphBuilder::VisitTestGreaterThanOrEqual() {
3380
  FeedbackSource feedback = CreateFeedbackSource(
3381
      bytecode_iterator().GetSlotOperand(kCompareOperationHintIndex));
3382
  BuildCompareOp(javascript()->GreaterThanOrEqual(feedback));
3383 3384
}

3385
void BytecodeGraphBuilder::VisitTestReferenceEqual() {
3386 3387 3388
  Node* left =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
  Node* right = environment()->LookupAccumulator();
3389 3390
  Node* result = NewNode(simplified()->ReferenceEqual(), left, right);
  environment()->BindAccumulator(result);
3391 3392
}

3393
void BytecodeGraphBuilder::VisitTestIn() {
3394
  PrepareEagerCheckpoint();
3395 3396
  Node* object = environment()->LookupAccumulator();
  Node* key =
3397
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
3398 3399
  FeedbackSource feedback =
      CreateFeedbackSource(bytecode_iterator().GetIndexOperand(1));
3400 3401 3402
  STATIC_ASSERT(JSHasPropertyNode::ObjectIndex() == 0);
  STATIC_ASSERT(JSHasPropertyNode::KeyIndex() == 1);
  STATIC_ASSERT(JSHasPropertyNode::FeedbackVectorIndex() == 2);
3403 3404 3405
  const Operator* op = javascript()->HasProperty(feedback);
  DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
  Node* node = NewNode(op, object, key, feedback_vector_node());
3406 3407 3408
  environment()->BindAccumulator(node, Environment::kAttachFrameState);
}

3409
void BytecodeGraphBuilder::VisitTestInstanceOf() {
3410 3411 3412
  FeedbackSource feedback = CreateFeedbackSource(
      bytecode_iterator().GetSlotOperand(kCompareOperationHintIndex));
  BuildCompareOp(javascript()->InstanceOf(feedback));
3413 3414
}

3415
void BytecodeGraphBuilder::VisitTestUndetectable() {
3416
  Node* object = environment()->LookupAccumulator();
3417 3418 3419 3420
  Node* node = NewNode(jsgraph()->simplified()->ObjectIsUndetectable(), object);
  environment()->BindAccumulator(node);
}

3421
void BytecodeGraphBuilder::VisitTestNull() {
3422
  Node* object = environment()->LookupAccumulator();
3423 3424
  Node* result = NewNode(simplified()->ReferenceEqual(), object,
                         jsgraph()->NullConstant());
3425 3426 3427 3428
  environment()->BindAccumulator(result);
}

void BytecodeGraphBuilder::VisitTestUndefined() {
3429
  Node* object = environment()->LookupAccumulator();
3430 3431
  Node* result = NewNode(simplified()->ReferenceEqual(), object,
                         jsgraph()->UndefinedConstant());
3432 3433 3434
  environment()->BindAccumulator(result);
}

3435 3436 3437 3438 3439 3440 3441 3442 3443 3444 3445 3446 3447 3448 3449
void BytecodeGraphBuilder::VisitTestTypeOf() {
  Node* object = environment()->LookupAccumulator();
  auto literal_flag = interpreter::TestTypeOfFlags::Decode(
      bytecode_iterator().GetFlagOperand(0));
  Node* result;
  switch (literal_flag) {
    case interpreter::TestTypeOfFlags::LiteralFlag::kNumber:
      result = NewNode(simplified()->ObjectIsNumber(), object);
      break;
    case interpreter::TestTypeOfFlags::LiteralFlag::kString:
      result = NewNode(simplified()->ObjectIsString(), object);
      break;
    case interpreter::TestTypeOfFlags::LiteralFlag::kSymbol:
      result = NewNode(simplified()->ObjectIsSymbol(), object);
      break;
3450 3451 3452
    case interpreter::TestTypeOfFlags::LiteralFlag::kBigInt:
      result = NewNode(simplified()->ObjectIsBigInt(), object);
      break;
3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465 3466 3467 3468 3469 3470 3471 3472 3473 3474 3475 3476 3477 3478 3479 3480 3481 3482 3483 3484 3485 3486 3487
    case interpreter::TestTypeOfFlags::LiteralFlag::kBoolean:
      result = NewNode(common()->Select(MachineRepresentation::kTagged),
                       NewNode(simplified()->ReferenceEqual(), object,
                               jsgraph()->TrueConstant()),
                       jsgraph()->TrueConstant(),
                       NewNode(simplified()->ReferenceEqual(), object,
                               jsgraph()->FalseConstant()));
      break;
    case interpreter::TestTypeOfFlags::LiteralFlag::kUndefined:
      result = graph()->NewNode(
          common()->Select(MachineRepresentation::kTagged),
          graph()->NewNode(simplified()->ReferenceEqual(), object,
                           jsgraph()->NullConstant()),
          jsgraph()->FalseConstant(),
          graph()->NewNode(simplified()->ObjectIsUndetectable(), object));
      break;
    case interpreter::TestTypeOfFlags::LiteralFlag::kFunction:
      result =
          graph()->NewNode(simplified()->ObjectIsDetectableCallable(), object);
      break;
    case interpreter::TestTypeOfFlags::LiteralFlag::kObject:
      result = graph()->NewNode(
          common()->Select(MachineRepresentation::kTagged),
          graph()->NewNode(simplified()->ObjectIsNonCallable(), object),
          jsgraph()->TrueConstant(),
          graph()->NewNode(simplified()->ReferenceEqual(), object,
                           jsgraph()->NullConstant()));
      break;
    case interpreter::TestTypeOfFlags::LiteralFlag::kOther:
      UNREACHABLE();  // Should never be emitted.
      break;
  }
  environment()->BindAccumulator(result);
}

3488 3489
void BytecodeGraphBuilder::BuildCastOperator(const Operator* js_op) {
  Node* value = NewNode(js_op, environment()->LookupAccumulator());
3490
  environment()->BindRegister(bytecode_iterator().GetRegisterOperand(0), value,
3491
                              Environment::kAttachFrameState);
3492 3493
}

3494 3495 3496 3497
void BytecodeGraphBuilder::VisitToName() {
  BuildCastOperator(javascript()->ToName());
}

3498
void BytecodeGraphBuilder::VisitToObject() {
3499
  BuildCastOperator(javascript()->ToObject());
3500 3501
}

3502 3503 3504 3505 3506 3507
void BytecodeGraphBuilder::VisitToString() {
  Node* value =
      NewNode(javascript()->ToString(), environment()->LookupAccumulator());
  environment()->BindAccumulator(value, Environment::kAttachFrameState);
}

3508
void BytecodeGraphBuilder::VisitToNumber() {
3509 3510 3511
  PrepareEagerCheckpoint();
  Node* object = environment()->LookupAccumulator();

3512
  FeedbackSlot slot = bytecode_iterator().GetSlotOperand(0);
3513 3514 3515 3516 3517 3518
  JSTypeHintLowering::LoweringResult lowering =
      TryBuildSimplifiedToNumber(object, slot);

  Node* node = nullptr;
  if (lowering.IsSideEffectFree()) {
    node = lowering.value();
3519
  } else {
3520
    DCHECK(!lowering.Changed());
3521 3522 3523
    node = NewNode(javascript()->ToNumber(), object);
  }

3524
  environment()->BindAccumulator(node, Environment::kAttachFrameState);
3525 3526
}

3527
void BytecodeGraphBuilder::VisitToNumeric() {
3528 3529 3530 3531 3532
  PrepareEagerCheckpoint();
  Node* object = environment()->LookupAccumulator();

  // If we have some kind of Number feedback, we do the same lowering as for
  // ToNumber.
3533
  FeedbackSlot slot = bytecode_iterator().GetSlotOperand(0);
3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545
  JSTypeHintLowering::LoweringResult lowering =
      TryBuildSimplifiedToNumber(object, slot);

  Node* node = nullptr;
  if (lowering.IsSideEffectFree()) {
    node = lowering.value();
  } else {
    DCHECK(!lowering.Changed());
    node = NewNode(javascript()->ToNumeric(), object);
  }

  environment()->BindAccumulator(node, Environment::kAttachFrameState);
3546 3547
}

3548
void BytecodeGraphBuilder::VisitJump() { BuildJump(); }
3549

3550
void BytecodeGraphBuilder::VisitJumpConstant() { BuildJump(); }
3551

3552
void BytecodeGraphBuilder::VisitJumpIfTrue() { BuildJumpIfTrue(); }
3553

3554
void BytecodeGraphBuilder::VisitJumpIfTrueConstant() { BuildJumpIfTrue(); }
3555

3556
void BytecodeGraphBuilder::VisitJumpIfFalse() { BuildJumpIfFalse(); }
3557

3558
void BytecodeGraphBuilder::VisitJumpIfFalseConstant() { BuildJumpIfFalse(); }
3559

3560
void BytecodeGraphBuilder::VisitJumpIfToBooleanTrue() {
3561
  BuildJumpIfToBooleanTrue();
3562 3563
}

3564
void BytecodeGraphBuilder::VisitJumpIfToBooleanTrueConstant() {
3565
  BuildJumpIfToBooleanTrue();
3566 3567
}

3568
void BytecodeGraphBuilder::VisitJumpIfToBooleanFalse() {
3569
  BuildJumpIfToBooleanFalse();
3570 3571
}

3572
void BytecodeGraphBuilder::VisitJumpIfToBooleanFalseConstant() {
3573
  BuildJumpIfToBooleanFalse();
3574
}
3575

3576 3577 3578 3579 3580 3581
void BytecodeGraphBuilder::VisitJumpIfJSReceiver() { BuildJumpIfJSReceiver(); }

void BytecodeGraphBuilder::VisitJumpIfJSReceiverConstant() {
  BuildJumpIfJSReceiver();
}

3582
void BytecodeGraphBuilder::VisitJumpIfNull() {
3583
  BuildJumpIfEqual(jsgraph()->NullConstant());
3584 3585
}

3586
void BytecodeGraphBuilder::VisitJumpIfNullConstant() {
3587 3588 3589
  BuildJumpIfEqual(jsgraph()->NullConstant());
}

3590 3591 3592 3593 3594 3595 3596 3597
void BytecodeGraphBuilder::VisitJumpIfNotNull() {
  BuildJumpIfNotEqual(jsgraph()->NullConstant());
}

void BytecodeGraphBuilder::VisitJumpIfNotNullConstant() {
  BuildJumpIfNotEqual(jsgraph()->NullConstant());
}

3598
void BytecodeGraphBuilder::VisitJumpIfUndefined() {
3599
  BuildJumpIfEqual(jsgraph()->UndefinedConstant());
3600 3601
}

3602
void BytecodeGraphBuilder::VisitJumpIfUndefinedConstant() {
3603 3604 3605
  BuildJumpIfEqual(jsgraph()->UndefinedConstant());
}

3606 3607 3608 3609 3610 3611 3612 3613
void BytecodeGraphBuilder::VisitJumpIfNotUndefined() {
  BuildJumpIfNotEqual(jsgraph()->UndefinedConstant());
}

void BytecodeGraphBuilder::VisitJumpIfNotUndefinedConstant() {
  BuildJumpIfNotEqual(jsgraph()->UndefinedConstant());
}

3614 3615 3616 3617 3618 3619 3620 3621 3622 3623
void BytecodeGraphBuilder::VisitJumpIfUndefinedOrNull() {
  BuildJumpIfEqual(jsgraph()->UndefinedConstant());
  BuildJumpIfEqual(jsgraph()->NullConstant());
}

void BytecodeGraphBuilder::VisitJumpIfUndefinedOrNullConstant() {
  BuildJumpIfEqual(jsgraph()->UndefinedConstant());
  BuildJumpIfEqual(jsgraph()->NullConstant());
}

3624 3625 3626 3627
void BytecodeGraphBuilder::VisitJumpLoop() {
  BuildIterationBodyStackCheck();
  BuildJump();
}
3628

3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641
void BytecodeGraphBuilder::BuildSwitchOnSmi(Node* condition) {
  interpreter::JumpTableTargetOffsets offsets =
      bytecode_iterator().GetJumpTableTargetOffsets();

  NewSwitch(condition, offsets.size() + 1);
  for (const auto& entry : offsets) {
    SubEnvironment sub_environment(this);
    NewIfValue(entry.case_value);
    MergeIntoSuccessorEnvironment(entry.target_offset);
  }
  NewIfDefault();
}

3642 3643 3644 3645
void BytecodeGraphBuilder::VisitSwitchOnSmiNoFeedback() {
  PrepareEagerCheckpoint();

  Node* acc = environment()->LookupAccumulator();
3646
  Node* acc_smi = NewNode(simplified()->CheckSmi(FeedbackSource()), acc);
3647
  BuildSwitchOnSmi(acc_smi);
3648 3649
}

3650 3651 3652 3653 3654 3655
void BytecodeGraphBuilder::VisitSetPendingMessage() {
  Node* previous_message = NewNode(javascript()->LoadMessage());
  NewNode(javascript()->StoreMessage(), environment()->LookupAccumulator());
  environment()->BindAccumulator(previous_message);
}

3656 3657
void BytecodeGraphBuilder::BuildReturn(const BytecodeLivenessState* liveness) {
  BuildLoopExitsForFunctionExit(liveness);
3658 3659 3660
  // Note: Negated offset since a return acts like a backwards jump, and should
  // decrement the budget.
  BuildUpdateInterruptBudget(-bytecode_iterator().current_offset());
3661
  Node* pop_node = jsgraph()->ZeroConstant();
3662
  Node* control =
3663
      NewNode(common()->Return(), pop_node, environment()->LookupAccumulator());
3664
  MergeControlToLeaveFunction(control);
3665 3666
}

3667
void BytecodeGraphBuilder::VisitReturn() {
3668
  BuildReturn(bytecode_analysis().GetInLivenessFor(
3669 3670 3671
      bytecode_iterator().current_offset()));
}

3672
void BytecodeGraphBuilder::VisitDebugger() {
3673
  PrepareEagerCheckpoint();
3674 3675
  Node* call = NewNode(javascript()->Debugger());
  environment()->RecordAfterState(call, Environment::kAttachFrameState);
3676 3677
}

3678 3679 3680
// We cannot create a graph from the debugger copy of the bytecode array.
#define DEBUG_BREAK(Name, ...) \
  void BytecodeGraphBuilder::Visit##Name() { UNREACHABLE(); }
3681
DEBUG_BREAK_BYTECODE_LIST(DEBUG_BREAK)
3682 3683
#undef DEBUG_BREAK

3684 3685 3686 3687 3688
void BytecodeGraphBuilder::VisitIncBlockCounter() {
  Node* closure = GetFunctionClosure();
  Node* coverage_array_slot =
      jsgraph()->Constant(bytecode_iterator().GetIndexOperand(0));

3689 3690 3691
  // Lowered by js-intrinsic-lowering to call Builtins::kIncBlockCounter.
  const Operator* op =
      javascript()->CallRuntime(Runtime::kInlineIncBlockCounter);
3692 3693 3694 3695

  NewNode(op, closure, coverage_array_slot);
}

3696
void BytecodeGraphBuilder::VisitForInEnumerate() {
3697 3698
  Node* receiver =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
3699 3700 3701 3702 3703 3704 3705 3706
  Node* enumerator = NewNode(javascript()->ForInEnumerate(), receiver);
  environment()->BindAccumulator(enumerator, Environment::kAttachFrameState);
}

void BytecodeGraphBuilder::VisitForInPrepare() {
  PrepareEagerCheckpoint();
  Node* enumerator = environment()->LookupAccumulator();

3707
  FeedbackSlot slot = bytecode_iterator().GetSlotOperand(1);
3708 3709 3710 3711
  JSTypeHintLowering::LoweringResult lowering =
      TryBuildSimplifiedForInPrepare(enumerator, slot);
  if (lowering.IsExit()) return;
  DCHECK(!lowering.Changed());
3712 3713 3714
  FeedbackSource feedback = CreateFeedbackSource(slot);
  Node* node = NewNode(javascript()->ForInPrepare(GetForInMode(slot), feedback),
                       enumerator, feedback_vector_node());
3715
  environment()->BindRegistersToProjections(
3716
      bytecode_iterator().GetRegisterOperand(0), node);
3717 3718
}

3719
void BytecodeGraphBuilder::VisitForInContinue() {
3720
  PrepareEagerCheckpoint();
3721 3722
  Node* index =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
3723
  Node* cache_length =
3724
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(1));
3725 3726 3727
  Node* exit_cond = NewNode(simplified()->SpeculativeNumberLessThan(
                                NumberOperationHint::kSignedSmall),
                            index, cache_length);
3728
  environment()->BindAccumulator(exit_cond);
3729 3730
}

3731
void BytecodeGraphBuilder::VisitForInNext() {
3732
  PrepareEagerCheckpoint();
3733
  Node* receiver =
3734 3735 3736 3737
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
  Node* index =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(1));
  int catch_reg_pair_index = bytecode_iterator().GetRegisterOperand(2).index();
3738 3739 3740 3741 3742
  Node* cache_type = environment()->LookupRegister(
      interpreter::Register(catch_reg_pair_index));
  Node* cache_array = environment()->LookupRegister(
      interpreter::Register(catch_reg_pair_index + 1));

3743
  // We need to rename the {index} here, as in case of OSR we lose the
3744
  // information that the {index} is always a valid unsigned Smi value.
3745
  index = NewNode(common()->TypeGuard(Type::UnsignedSmall()), index);
3746

3747
  FeedbackSlot slot = bytecode_iterator().GetSlotOperand(3);
3748 3749 3750 3751
  JSTypeHintLowering::LoweringResult lowering = TryBuildSimplifiedForInNext(
      receiver, cache_array, cache_type, index, slot);
  if (lowering.IsExit()) return;

3752
  DCHECK(!lowering.Changed());
3753 3754 3755 3756
  FeedbackSource feedback = CreateFeedbackSource(slot);
  Node* node =
      NewNode(javascript()->ForInNext(GetForInMode(slot), feedback), receiver,
              cache_array, cache_type, index, feedback_vector_node());
3757
  environment()->BindAccumulator(node, Environment::kAttachFrameState);
3758 3759
}

3760
void BytecodeGraphBuilder::VisitForInStep() {
3761
  PrepareEagerCheckpoint();
3762 3763
  Node* index =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
3764 3765 3766
  index = NewNode(simplified()->SpeculativeSafeIntegerAdd(
                      NumberOperationHint::kSignedSmall),
                  index, jsgraph()->OneConstant());
3767
  environment()->BindAccumulator(index, Environment::kAttachFrameState);
3768 3769
}

3770 3771
void BytecodeGraphBuilder::VisitGetIterator() {
  PrepareEagerCheckpoint();
3772
  Node* receiver =
3773
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
3774
  FeedbackSource load_feedback =
3775
      CreateFeedbackSource(bytecode_iterator().GetIndexOperand(1));
3776 3777 3778
  FeedbackSource call_feedback =
      CreateFeedbackSource(bytecode_iterator().GetIndexOperand(2));
  const Operator* op = javascript()->GetIterator(load_feedback, call_feedback);
3779

3780 3781
  JSTypeHintLowering::LoweringResult lowering = TryBuildSimplifiedGetIterator(
      op, receiver, load_feedback.slot, call_feedback.slot);
3782 3783 3784
  if (lowering.IsExit()) return;

  DCHECK(!lowering.Changed());
3785 3786
  STATIC_ASSERT(JSGetIteratorNode::ReceiverIndex() == 0);
  STATIC_ASSERT(JSGetIteratorNode::FeedbackVectorIndex() == 1);
3787
  DCHECK(IrOpcode::IsFeedbackCollectingOpcode(op->opcode()));
3788
  Node* iterator = NewNode(op, receiver, feedback_vector_node());
3789
  environment()->BindAccumulator(iterator, Environment::kAttachFrameState);
3790 3791
}

3792
void BytecodeGraphBuilder::VisitSuspendGenerator() {
3793 3794
  Node* generator = environment()->LookupRegister(
      bytecode_iterator().GetRegisterOperand(0));
3795 3796 3797 3798 3799
  interpreter::Register first_reg = bytecode_iterator().GetRegisterOperand(1);
  // We assume we are storing a range starting from index 0.
  CHECK_EQ(0, first_reg.index());
  int register_count =
      static_cast<int>(bytecode_iterator().GetRegisterCountOperand(2));
3800
  int parameter_count_without_receiver = bytecode_array().parameter_count() - 1;
3801

3802 3803
  Node* suspend_id = jsgraph()->SmiConstant(
      bytecode_iterator().GetUnsignedImmediateOperand(3));
3804

3805 3806 3807 3808 3809
  // The offsets used by the bytecode iterator are relative to a different base
  // than what is used in the interpreter, hence the addition.
  Node* offset =
      jsgraph()->Constant(bytecode_iterator().current_offset() +
                          (BytecodeArray::kHeaderSize - kHeapObjectTag));
3810

3811
  const BytecodeLivenessState* liveness = bytecode_analysis().GetInLivenessFor(
3812 3813 3814 3815 3816 3817
      bytecode_iterator().current_offset());

  // Maybe overallocate the value list since we don't know how many registers
  // are live.
  // TODO(leszeks): We could get this count from liveness rather than the
  // register list.
3818
  int value_input_count = 3 + parameter_count_without_receiver + register_count;
3819 3820 3821

  Node** value_inputs = local_zone()->NewArray<Node*>(value_input_count);
  value_inputs[0] = generator;
3822
  value_inputs[1] = suspend_id;
3823
  value_inputs[2] = offset;
3824 3825

  int count_written = 0;
3826 3827 3828
  // Store the parameters.
  for (int i = 0; i < parameter_count_without_receiver; i++) {
    value_inputs[3 + count_written++] =
3829
        environment()->LookupRegister(bytecode_iterator().GetParameter(i));
3830 3831 3832
  }

  // Store the registers.
3833
  for (int i = 0; i < register_count; ++i) {
3834
    if (liveness == nullptr || liveness->RegisterIsLive(i)) {
3835 3836 3837
      int index_in_parameters_and_registers =
          parameter_count_without_receiver + i;
      while (count_written < index_in_parameters_and_registers) {
3838 3839 3840 3841
        value_inputs[3 + count_written++] = jsgraph()->OptimizedOutConstant();
      }
      value_inputs[3 + count_written++] =
          environment()->LookupRegister(interpreter::Register(i));
3842
      DCHECK_EQ(count_written, index_in_parameters_and_registers + 1);
3843
    }
3844 3845
  }

3846 3847 3848
  // Use the actual written count rather than the register count to create the
  // node.
  MakeNode(javascript()->GeneratorStore(count_written), 3 + count_written,
3849
           value_inputs, false);
3850 3851 3852

  // TODO(leszeks): This over-approximates the liveness at exit, only the
  // accumulator should be live by this point.
3853
  BuildReturn(bytecode_analysis().GetInLivenessFor(
3854
      bytecode_iterator().current_offset()));
3855 3856
}

3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884
void BytecodeGraphBuilder::BuildSwitchOnGeneratorState(
    const ZoneVector<ResumeJumpTarget>& resume_jump_targets,
    bool allow_fallthrough_on_executing) {
  Node* generator_state = environment()->LookupGeneratorState();

  int extra_cases = allow_fallthrough_on_executing ? 2 : 1;
  NewSwitch(generator_state,
            static_cast<int>(resume_jump_targets.size() + extra_cases));
  for (const ResumeJumpTarget& target : resume_jump_targets) {
    SubEnvironment sub_environment(this);
    NewIfValue(target.suspend_id());
    if (target.is_leaf()) {
      // Mark that we are resuming executing.
      environment()->BindGeneratorState(
          jsgraph()->SmiConstant(JSGeneratorObject::kGeneratorExecuting));
    }
    // Jump to the target offset, whether it's a loop header or the resume.
    MergeIntoSuccessorEnvironment(target.target_offset());
  }

  {
    SubEnvironment sub_environment(this);
    // We should never hit the default case (assuming generator state cannot be
    // corrupted), so abort if we do.
    // TODO(leszeks): Maybe only check this in debug mode, and otherwise use
    // the default to represent one of the cases above/fallthrough below?
    NewIfDefault();
    NewNode(simplified()->RuntimeAbort(AbortReason::kInvalidJumpTableIndex));
3885
    // TODO(7099): Investigate if we need LoopExit here.
3886 3887 3888 3889 3890 3891 3892 3893 3894 3895 3896 3897 3898 3899 3900 3901 3902
    Node* control = NewNode(common()->Throw());
    MergeControlToLeaveFunction(control);
  }

  if (allow_fallthrough_on_executing) {
    // If we are executing (rather than resuming), and we allow it, just fall
    // through to the actual loop body.
    NewIfValue(JSGeneratorObject::kGeneratorExecuting);
  } else {
    // Otherwise, this environment is dead.
    set_environment(nullptr);
  }
}

void BytecodeGraphBuilder::VisitSwitchOnGeneratorState() {
  Node* generator =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
3903

3904 3905 3906
  Node* generator_is_undefined =
      NewNode(simplified()->ReferenceEqual(), generator,
              jsgraph()->UndefinedConstant());
3907

3908 3909 3910 3911 3912 3913 3914 3915 3916 3917 3918 3919 3920
  NewBranch(generator_is_undefined);
  {
    SubEnvironment resume_env(this);
    NewIfFalse();

    Node* generator_state =
        NewNode(javascript()->GeneratorRestoreContinuation(), generator);
    environment()->BindGeneratorState(generator_state);

    Node* generator_context =
        NewNode(javascript()->GeneratorRestoreContext(), generator);
    environment()->SetContext(generator_context);

3921
    BuildSwitchOnGeneratorState(bytecode_analysis().resume_jump_targets(),
3922 3923
                                false);
  }
3924

3925 3926
  // Fallthrough for the first-call case.
  NewIfTrue();
3927 3928
}

3929
void BytecodeGraphBuilder::VisitResumeGenerator() {
3930 3931
  Node* generator =
      environment()->LookupRegister(bytecode_iterator().GetRegisterOperand(0));
3932
  interpreter::Register first_reg = bytecode_iterator().GetRegisterOperand(1);
3933 3934
  // We assume we are restoring registers starting fromm index 0.
  CHECK_EQ(0, first_reg.index());
3935

3936 3937
  const BytecodeLivenessState* liveness = bytecode_analysis().GetOutLivenessFor(
      bytecode_iterator().current_offset());
3938

3939
  int parameter_count_without_receiver = bytecode_array().parameter_count() - 1;
3940 3941 3942

  // Mapping between registers and array indices must match that used in
  // InterpreterAssembler::ExportParametersAndRegisterFile.
3943 3944
  for (int i = 0; i < environment()->register_count(); ++i) {
    if (liveness == nullptr || liveness->RegisterIsLive(i)) {
3945 3946 3947
      Node* value = NewNode(javascript()->GeneratorRestoreRegister(
                                parameter_count_without_receiver + i),
                            generator);
3948 3949
      environment()->BindRegister(interpreter::Register(i), value);
    }
3950
  }
3951 3952 3953 3954 3955

  // Update the accumulator with the generator's input_or_debug_pos.
  Node* input_or_debug_pos =
      NewNode(javascript()->GeneratorRestoreInputOrDebugPos(), generator);
  environment()->BindAccumulator(input_or_debug_pos);
3956 3957
}

3958 3959 3960 3961 3962 3963 3964 3965 3966 3967 3968
void BytecodeGraphBuilder::VisitWide() {
  // Consumed by the BytecodeArrayIterator.
  UNREACHABLE();
}

void BytecodeGraphBuilder::VisitExtraWide() {
  // Consumed by the BytecodeArrayIterator.
  UNREACHABLE();
}

void BytecodeGraphBuilder::VisitIllegal() {
3969 3970
  // Not emitted in valid bytecode.
  UNREACHABLE();
3971 3972
}

3973
void BytecodeGraphBuilder::SwitchToMergeEnvironment(int current_offset) {
3974 3975
  auto it = merge_environments_.find(current_offset);
  if (it != merge_environments_.end()) {
3976
    mark_as_needing_eager_checkpoint(true);
3977
    if (environment() != nullptr) {
3978
      it->second->Merge(environment(),
3979
                        bytecode_analysis().GetInLivenessFor(current_offset));
3980
    }
3981
    set_environment(it->second);
3982 3983 3984
  }
}

3985
void BytecodeGraphBuilder::BuildLoopHeaderEnvironment(int current_offset) {
3986
  if (bytecode_analysis().IsLoopHeader(current_offset)) {
3987
    mark_as_needing_eager_checkpoint(true);
3988
    const LoopInfo& loop_info =
3989
        bytecode_analysis().GetLoopInfoFor(current_offset);
3990
    const BytecodeLivenessState* liveness =
3991
        bytecode_analysis().GetInLivenessFor(current_offset);
3992

3993 3994 3995
    const auto& resume_jump_targets = loop_info.resume_jump_targets();
    bool generate_suspend_switch = !resume_jump_targets.empty();

3996
    // Add loop header.
3997
    environment()->PrepareForLoop(loop_info.assignments(), liveness);
3998 3999 4000 4001

    // Store a copy of the environment so we can connect merged back edge inputs
    // to the loop header.
    merge_environments_[current_offset] = environment()->Copy();
4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014 4015

    // If this loop contains resumes, create a new switch just after the loop
    // for those resumes.
    if (generate_suspend_switch) {
      BuildSwitchOnGeneratorState(loop_info.resume_jump_targets(), true);

      // TODO(leszeks): At this point we know we are executing rather than
      // resuming, so we should be able to prune off the phis in the environment
      // related to the resume path.

      // Set the generator state to a known constant.
      environment()->BindGeneratorState(
          jsgraph()->SmiConstant(JSGeneratorObject::kGeneratorExecuting));
    }
4016 4017 4018
  }
}

4019
void BytecodeGraphBuilder::MergeIntoSuccessorEnvironment(int target_offset) {
4020
  BuildLoopExitsForBranch(target_offset);
4021
  Environment*& merge_environment = merge_environments_[target_offset];
4022

4023
  if (merge_environment == nullptr) {
4024 4025 4026 4027
    // Append merge nodes to the environment. We may merge here with another
    // environment. So add a place holder for merge nodes. We may add redundant
    // but will be eliminated in a later pass.
    NewMerge();
4028
    merge_environment = environment();
4029
  } else {
4030 4031
    // Merge any values which are live coming into the successor.
    merge_environment->Merge(
4032
        environment(), bytecode_analysis().GetInLivenessFor(target_offset));
4033 4034 4035 4036
  }
  set_environment(nullptr);
}

4037 4038 4039 4040
void BytecodeGraphBuilder::MergeControlToLeaveFunction(Node* exit) {
  exit_controls_.push_back(exit);
  set_environment(nullptr);
}
4041

4042 4043 4044 4045
void BytecodeGraphBuilder::BuildLoopExitsForBranch(int target_offset) {
  int origin_offset = bytecode_iterator().current_offset();
  // Only build loop exits for forward edges.
  if (target_offset > origin_offset) {
4046
    BuildLoopExitsUntilLoop(
4047 4048
        bytecode_analysis().GetLoopOffsetFor(target_offset),
        bytecode_analysis().GetInLivenessFor(target_offset));
4049 4050 4051
  }
}

4052 4053
void BytecodeGraphBuilder::BuildLoopExitsUntilLoop(
    int loop_offset, const BytecodeLivenessState* liveness) {
4054
  int origin_offset = bytecode_iterator().current_offset();
4055
  int current_loop = bytecode_analysis().GetLoopOffsetFor(origin_offset);
4056 4057 4058 4059
  // The limit_offset is the stop offset for building loop exists, used for OSR.
  // It prevents the creations of loopexits for loops which do not exist.
  loop_offset = std::max(loop_offset, currently_peeled_loop_offset_);

4060 4061
  while (loop_offset < current_loop) {
    Node* loop_node = merge_environments_[current_loop]->GetControlDependency();
4062
    const LoopInfo& loop_info =
4063
        bytecode_analysis().GetLoopInfoFor(current_loop);
4064 4065
    environment()->PrepareForLoopExit(loop_node, loop_info.assignments(),
                                      liveness);
4066
    current_loop = loop_info.parent_offset();
4067 4068 4069
  }
}

4070 4071 4072
void BytecodeGraphBuilder::BuildLoopExitsForFunctionExit(
    const BytecodeLivenessState* liveness) {
  BuildLoopExitsUntilLoop(-1, liveness);
4073 4074
}

4075
void BytecodeGraphBuilder::BuildJump() {
4076
  BuildUpdateInterruptBudget(bytecode_iterator().GetRelativeJumpTargetOffset());
4077
  MergeIntoSuccessorEnvironment(bytecode_iterator().GetJumpTargetOffset());
4078 4079
}

4080
void BytecodeGraphBuilder::BuildJumpIf(Node* condition) {
4081
  NewBranch(condition, BranchHint::kNone, IsSafetyCheck::kNoSafetyCheck);
4082 4083 4084
  {
    SubEnvironment sub_environment(this);
    NewIfTrue();
4085 4086
    BuildUpdateInterruptBudget(
        bytecode_iterator().GetRelativeJumpTargetOffset());
4087 4088
    MergeIntoSuccessorEnvironment(bytecode_iterator().GetJumpTargetOffset());
  }
4089 4090 4091
  NewIfFalse();
}

4092
void BytecodeGraphBuilder::BuildJumpIfNot(Node* condition) {
4093
  NewBranch(condition, BranchHint::kNone, IsSafetyCheck::kNoSafetyCheck);
4094 4095 4096
  {
    SubEnvironment sub_environment(this);
    NewIfFalse();
4097 4098
    BuildUpdateInterruptBudget(
        bytecode_iterator().GetRelativeJumpTargetOffset());
4099 4100
    MergeIntoSuccessorEnvironment(bytecode_iterator().GetJumpTargetOffset());
  }
4101 4102
  NewIfTrue();
}
4103

4104
void BytecodeGraphBuilder::BuildJumpIfEqual(Node* comperand) {
4105
  Node* accumulator = environment()->LookupAccumulator();
4106
  Node* condition =
4107
      NewNode(simplified()->ReferenceEqual(), accumulator, comperand);
4108
  BuildJumpIf(condition);
4109 4110
}

4111 4112 4113 4114 4115 4116 4117
void BytecodeGraphBuilder::BuildJumpIfNotEqual(Node* comperand) {
  Node* accumulator = environment()->LookupAccumulator();
  Node* condition =
      NewNode(simplified()->ReferenceEqual(), accumulator, comperand);
  BuildJumpIfNot(condition);
}

4118
void BytecodeGraphBuilder::BuildJumpIfFalse() {
4119
  NewBranch(environment()->LookupAccumulator(), BranchHint::kNone,
4120
            IsSafetyCheck::kNoSafetyCheck);
4121 4122 4123
  {
    SubEnvironment sub_environment(this);
    NewIfFalse();
4124 4125
    BuildUpdateInterruptBudget(
        bytecode_iterator().GetRelativeJumpTargetOffset());
4126 4127 4128 4129 4130
    environment()->BindAccumulator(jsgraph()->FalseConstant());
    MergeIntoSuccessorEnvironment(bytecode_iterator().GetJumpTargetOffset());
  }
  NewIfTrue();
  environment()->BindAccumulator(jsgraph()->TrueConstant());
4131
}
4132

4133
void BytecodeGraphBuilder::BuildJumpIfTrue() {
4134
  NewBranch(environment()->LookupAccumulator(), BranchHint::kNone,
4135
            IsSafetyCheck::kNoSafetyCheck);
4136 4137 4138 4139
  {
    SubEnvironment sub_environment(this);
    NewIfTrue();
    environment()->BindAccumulator(jsgraph()->TrueConstant());
4140 4141
    BuildUpdateInterruptBudget(
        bytecode_iterator().GetRelativeJumpTargetOffset());
4142 4143 4144 4145
    MergeIntoSuccessorEnvironment(bytecode_iterator().GetJumpTargetOffset());
  }
  NewIfFalse();
  environment()->BindAccumulator(jsgraph()->FalseConstant());
4146 4147 4148
}

void BytecodeGraphBuilder::BuildJumpIfToBooleanTrue() {
4149
  Node* accumulator = environment()->LookupAccumulator();
4150
  Node* condition = NewNode(simplified()->ToBoolean(), accumulator);
4151 4152 4153 4154 4155
  BuildJumpIf(condition);
}

void BytecodeGraphBuilder::BuildJumpIfToBooleanFalse() {
  Node* accumulator = environment()->LookupAccumulator();
4156
  Node* condition = NewNode(simplified()->ToBoolean(), accumulator);
4157
  BuildJumpIfNot(condition);
4158 4159
}

4160 4161
void BytecodeGraphBuilder::BuildJumpIfNotHole() {
  Node* accumulator = environment()->LookupAccumulator();
4162 4163
  Node* condition = NewNode(simplified()->ReferenceEqual(), accumulator,
                            jsgraph()->TheHoleConstant());
4164
  BuildJumpIfNot(condition);
4165
}
4166

4167 4168 4169 4170 4171 4172
void BytecodeGraphBuilder::BuildJumpIfJSReceiver() {
  Node* accumulator = environment()->LookupAccumulator();
  Node* condition = NewNode(simplified()->ObjectIsReceiver(), accumulator);
  BuildJumpIf(condition);
}

4173
void BytecodeGraphBuilder::BuildUpdateInterruptBudget(int delta) {
4174 4175 4176 4177 4178 4179 4180
  if (!CodeKindCanTierUp(code_kind())) return;

  // Keep uses of this in sync with Ignition's UpdateInterruptBudget.
  int delta_with_current_bytecode =
      delta - bytecode_iterator().current_bytecode_size();
  NewNode(simplified()->UpdateInterruptBudget(delta_with_current_bytecode),
          feedback_cell_node());
4181 4182
}

4183 4184 4185 4186
JSTypeHintLowering::LoweringResult
BytecodeGraphBuilder::TryBuildSimplifiedUnaryOp(const Operator* op,
                                                Node* operand,
                                                FeedbackSlot slot) {
4187
  if (!CanApplyTypeHintLowering(op)) return NoChange();
4188 4189 4190 4191 4192 4193 4194 4195 4196
  Node* effect = environment()->GetEffectDependency();
  Node* control = environment()->GetControlDependency();
  JSTypeHintLowering::LoweringResult result =
      type_hint_lowering().ReduceUnaryOperation(op, operand, effect, control,
                                                slot);
  ApplyEarlyReduction(result);
  return result;
}

4197 4198 4199 4200
JSTypeHintLowering::LoweringResult
BytecodeGraphBuilder::TryBuildSimplifiedBinaryOp(const Operator* op, Node* left,
                                                 Node* right,
                                                 FeedbackSlot slot) {
4201
  if (!CanApplyTypeHintLowering(op)) return NoChange();
4202 4203
  Node* effect = environment()->GetEffectDependency();
  Node* control = environment()->GetControlDependency();
4204 4205 4206 4207 4208
  JSTypeHintLowering::LoweringResult result =
      type_hint_lowering().ReduceBinaryOperation(op, left, right, effect,
                                                 control, slot);
  ApplyEarlyReduction(result);
  return result;
4209 4210
}

4211 4212 4213 4214 4215
JSTypeHintLowering::LoweringResult
BytecodeGraphBuilder::TryBuildSimplifiedForInNext(Node* receiver,
                                                  Node* cache_array,
                                                  Node* cache_type, Node* index,
                                                  FeedbackSlot slot) {
4216
  if (!CanApplyTypeHintLowering(IrOpcode::kJSForInNext)) return NoChange();
4217 4218
  Node* effect = environment()->GetEffectDependency();
  Node* control = environment()->GetControlDependency();
4219 4220 4221
  JSTypeHintLowering::LoweringResult result =
      type_hint_lowering().ReduceForInNextOperation(
          receiver, cache_array, cache_type, index, effect, control, slot);
4222 4223 4224 4225 4226 4227 4228
  ApplyEarlyReduction(result);
  return result;
}

JSTypeHintLowering::LoweringResult
BytecodeGraphBuilder::TryBuildSimplifiedForInPrepare(Node* enumerator,
                                                     FeedbackSlot slot) {
4229
  if (!CanApplyTypeHintLowering(IrOpcode::kJSForInPrepare)) return NoChange();
4230 4231 4232 4233 4234
  Node* effect = environment()->GetEffectDependency();
  Node* control = environment()->GetControlDependency();
  JSTypeHintLowering::LoweringResult result =
      type_hint_lowering().ReduceForInPrepareOperation(enumerator, effect,
                                                       control, slot);
4235 4236
  ApplyEarlyReduction(result);
  return result;
4237 4238
}

4239 4240 4241
JSTypeHintLowering::LoweringResult
BytecodeGraphBuilder::TryBuildSimplifiedToNumber(Node* value,
                                                 FeedbackSlot slot) {
4242
  if (!CanApplyTypeHintLowering(IrOpcode::kJSToNumber)) return NoChange();
4243 4244
  Node* effect = environment()->GetEffectDependency();
  Node* control = environment()->GetControlDependency();
4245 4246 4247 4248 4249
  JSTypeHintLowering::LoweringResult result =
      type_hint_lowering().ReduceToNumberOperation(value, effect, control,
                                                   slot);
  ApplyEarlyReduction(result);
  return result;
4250 4251
}

4252 4253
JSTypeHintLowering::LoweringResult BytecodeGraphBuilder::TryBuildSimplifiedCall(
    const Operator* op, Node* const* args, int arg_count, FeedbackSlot slot) {
4254
  if (!CanApplyTypeHintLowering(op)) return NoChange();
4255 4256
  Node* effect = environment()->GetEffectDependency();
  Node* control = environment()->GetControlDependency();
4257 4258 4259 4260 4261
  JSTypeHintLowering::LoweringResult result =
      type_hint_lowering().ReduceCallOperation(op, args, arg_count, effect,
                                               control, slot);
  ApplyEarlyReduction(result);
  return result;
4262 4263
}

4264 4265 4266 4267 4268
JSTypeHintLowering::LoweringResult
BytecodeGraphBuilder::TryBuildSimplifiedConstruct(const Operator* op,
                                                  Node* const* args,
                                                  int arg_count,
                                                  FeedbackSlot slot) {
4269
  if (!CanApplyTypeHintLowering(op)) return NoChange();
4270 4271
  Node* effect = environment()->GetEffectDependency();
  Node* control = environment()->GetControlDependency();
4272 4273 4274 4275 4276
  JSTypeHintLowering::LoweringResult result =
      type_hint_lowering().ReduceConstructOperation(op, args, arg_count, effect,
                                                    control, slot);
  ApplyEarlyReduction(result);
  return result;
4277 4278
}

4279 4280 4281 4282 4283
JSTypeHintLowering::LoweringResult
BytecodeGraphBuilder::TryBuildSimplifiedGetIterator(const Operator* op,
                                                    Node* receiver,
                                                    FeedbackSlot load_slot,
                                                    FeedbackSlot call_slot) {
4284
  if (!CanApplyTypeHintLowering(op)) return NoChange();
4285 4286 4287 4288 4289 4290 4291 4292 4293
  Node* effect = environment()->GetEffectDependency();
  Node* control = environment()->GetControlDependency();
  JSTypeHintLowering::LoweringResult early_reduction =
      type_hint_lowering().ReduceGetIteratorOperation(
          op, receiver, effect, control, load_slot, call_slot);
  ApplyEarlyReduction(early_reduction);
  return early_reduction;
}

4294 4295 4296
JSTypeHintLowering::LoweringResult
BytecodeGraphBuilder::TryBuildSimplifiedLoadNamed(const Operator* op,
                                                  FeedbackSlot slot) {
4297
  if (!CanApplyTypeHintLowering(op)) return NoChange();
4298 4299
  Node* effect = environment()->GetEffectDependency();
  Node* control = environment()->GetControlDependency();
4300
  JSTypeHintLowering::LoweringResult early_reduction =
4301
      type_hint_lowering().ReduceLoadNamedOperation(op, effect, control, slot);
4302 4303
  ApplyEarlyReduction(early_reduction);
  return early_reduction;
4304 4305
}

4306 4307 4308 4309
JSTypeHintLowering::LoweringResult
BytecodeGraphBuilder::TryBuildSimplifiedLoadKeyed(const Operator* op,
                                                  Node* receiver, Node* key,
                                                  FeedbackSlot slot) {
4310
  if (!CanApplyTypeHintLowering(op)) return NoChange();
4311 4312
  Node* effect = environment()->GetEffectDependency();
  Node* control = environment()->GetControlDependency();
4313 4314 4315 4316 4317
  JSTypeHintLowering::LoweringResult result =
      type_hint_lowering().ReduceLoadKeyedOperation(op, receiver, key, effect,
                                                    control, slot);
  ApplyEarlyReduction(result);
  return result;
4318 4319
}

4320 4321 4322 4323
JSTypeHintLowering::LoweringResult
BytecodeGraphBuilder::TryBuildSimplifiedStoreNamed(const Operator* op,
                                                   Node* receiver, Node* value,
                                                   FeedbackSlot slot) {
4324
  if (!CanApplyTypeHintLowering(op)) return NoChange();
4325 4326
  Node* effect = environment()->GetEffectDependency();
  Node* control = environment()->GetControlDependency();
4327 4328 4329 4330 4331
  JSTypeHintLowering::LoweringResult result =
      type_hint_lowering().ReduceStoreNamedOperation(op, receiver, value,
                                                     effect, control, slot);
  ApplyEarlyReduction(result);
  return result;
4332 4333
}

4334 4335 4336 4337 4338
JSTypeHintLowering::LoweringResult
BytecodeGraphBuilder::TryBuildSimplifiedStoreKeyed(const Operator* op,
                                                   Node* receiver, Node* key,
                                                   Node* value,
                                                   FeedbackSlot slot) {
4339
  if (!CanApplyTypeHintLowering(op)) return NoChange();
4340 4341
  Node* effect = environment()->GetEffectDependency();
  Node* control = environment()->GetControlDependency();
4342 4343 4344 4345 4346
  JSTypeHintLowering::LoweringResult result =
      type_hint_lowering().ReduceStoreKeyedOperation(op, receiver, key, value,
                                                     effect, control, slot);
  ApplyEarlyReduction(result);
  return result;
4347 4348
}

4349 4350 4351 4352 4353 4354 4355 4356 4357 4358 4359 4360
void BytecodeGraphBuilder::ApplyEarlyReduction(
    JSTypeHintLowering::LoweringResult reduction) {
  if (reduction.IsExit()) {
    MergeControlToLeaveFunction(reduction.control());
  } else if (reduction.IsSideEffectFree()) {
    environment()->UpdateEffectDependency(reduction.effect());
    environment()->UpdateControlDependency(reduction.control());
  } else {
    DCHECK(!reduction.Changed());
    // At the moment, we assume side-effect free reduction. To support
    // side-effects, we would have to invalidate the eager checkpoint,
    // so that deoptimization does not repeat the side effect.
4361 4362 4363
  }
}

4364 4365 4366 4367 4368 4369 4370 4371 4372
Node** BytecodeGraphBuilder::EnsureInputBufferSize(int size) {
  if (size > input_buffer_size_) {
    size = size + kInputBufferSizeIncrement + input_buffer_size_;
    input_buffer_ = local_zone()->NewArray<Node*>(size);
    input_buffer_size_ = size;
  }
  return input_buffer_;
}

4373
void BytecodeGraphBuilder::ExitThenEnterExceptionHandlers(int current_offset) {
4374
  DisallowGarbageCollection no_gc;
4375 4376 4377
  HandlerTable table(bytecode_array().handler_table_address(),
                     bytecode_array().handler_table_size(),
                     HandlerTable::kRangeBasedEncoding);
4378 4379 4380 4381 4382 4383 4384 4385 4386

  // Potentially exit exception handlers.
  while (!exception_handlers_.empty()) {
    int current_end = exception_handlers_.top().end_offset_;
    if (current_offset < current_end) break;  // Still covered by range.
    exception_handlers_.pop();
  }

  // Potentially enter exception handlers.
4387
  int num_entries = table.NumberOfRangeEntries();
4388
  while (current_exception_handler_ < num_entries) {
4389
    int next_start = table.GetRangeStart(current_exception_handler_);
4390
    if (current_offset < next_start) break;  // Not yet covered by range.
4391 4392 4393
    int next_end = table.GetRangeEnd(current_exception_handler_);
    int next_handler = table.GetRangeHandler(current_exception_handler_);
    int context_register = table.GetRangeData(current_exception_handler_);
4394
    exception_handlers_.push(
4395
        {next_start, next_end, next_handler, context_register});
4396 4397 4398
    current_exception_handler_++;
  }
}
4399 4400

Node* BytecodeGraphBuilder::MakeNode(const Operator* op, int value_input_count,
4401 4402
                                     Node* const* value_inputs,
                                     bool incomplete) {
4403 4404 4405
  DCHECK_EQ(op->ValueInputCount(), value_input_count);

  bool has_context = OperatorProperties::HasContextInput(op);
4406
  bool has_frame_state = OperatorProperties::HasFrameStateInput(op);
4407 4408 4409 4410 4411 4412
  bool has_control = op->ControlInputCount() == 1;
  bool has_effect = op->EffectInputCount() == 1;

  DCHECK_LT(op->ControlInputCount(), 2);
  DCHECK_LT(op->EffectInputCount(), 2);

4413
  Node* result = nullptr;
4414
  if (!has_context && !has_frame_state && !has_control && !has_effect) {
4415 4416
    result = graph()->NewNode(op, value_input_count, value_inputs, incomplete);
  } else {
4417
    bool inside_handler = !exception_handlers_.empty();
4418 4419
    int input_count_with_deps = value_input_count;
    if (has_context) ++input_count_with_deps;
4420
    if (has_frame_state) ++input_count_with_deps;
4421 4422 4423
    if (has_control) ++input_count_with_deps;
    if (has_effect) ++input_count_with_deps;
    Node** buffer = EnsureInputBufferSize(input_count_with_deps);
4424
    if (value_input_count > 0) {
4425 4426
      base::Memcpy(buffer, value_inputs,
                   kSystemPointerSize * value_input_count);
4427
    }
4428 4429
    Node** current_input = buffer + value_input_count;
    if (has_context) {
4430 4431
      *current_input++ = OperatorProperties::NeedsExactContext(op)
                             ? environment()->Context()
4432
                             : native_context_node();
4433
    }
4434
    if (has_frame_state) {
4435 4436 4437
      // The frame state will be inserted later. Here we misuse the {Dead} node
      // as a sentinel to be later overwritten with the real frame state by the
      // calls to {PrepareFrameState} within individual visitor methods.
4438
      *current_input++ = jsgraph()->Dead();
4439 4440 4441 4442 4443 4444 4445 4446
    }
    if (has_effect) {
      *current_input++ = environment()->GetEffectDependency();
    }
    if (has_control) {
      *current_input++ = environment()->GetControlDependency();
    }
    result = graph()->NewNode(op, input_count_with_deps, buffer, incomplete);
4447
    // Update the current control dependency for control-producing nodes.
4448
    if (result->op()->ControlOutputCount() > 0) {
4449 4450 4451 4452 4453 4454 4455 4456 4457
      environment()->UpdateControlDependency(result);
    }
    // Update the current effect dependency for effect-producing nodes.
    if (result->op()->EffectOutputCount() > 0) {
      environment()->UpdateEffectDependency(result);
    }
    // Add implicit exception continuation for throwing nodes.
    if (!result->op()->HasProperty(Operator::kNoThrow) && inside_handler) {
      int handler_offset = exception_handlers_.top().handler_offset_;
4458 4459
      int context_index = exception_handlers_.top().context_register_;
      interpreter::Register context_register(context_index);
4460
      Environment* success_env = environment()->Copy();
4461
      const Operator* op = common()->IfException();
4462 4463
      Node* effect = environment()->GetEffectDependency();
      Node* on_exception = graph()->NewNode(op, effect, result);
4464
      Node* context = environment()->LookupRegister(context_register);
4465 4466 4467
      environment()->UpdateControlDependency(on_exception);
      environment()->UpdateEffectDependency(on_exception);
      environment()->BindAccumulator(on_exception);
4468
      environment()->SetContext(context);
4469 4470 4471 4472
      MergeIntoSuccessorEnvironment(handler_offset);
      set_environment(success_env);
    }
    // Add implicit success continuation for throwing nodes.
4473
    if (!result->op()->HasProperty(Operator::kNoThrow) && inside_handler) {
4474 4475 4476
      const Operator* if_success = common()->IfSuccess();
      Node* on_success = graph()->NewNode(if_success, result);
      environment()->UpdateControlDependency(on_success);
4477
    }
4478 4479 4480 4481
    // Ensure checkpoints are created after operations with side-effects.
    if (has_effect && !result->op()->HasProperty(Operator::kNoWrite)) {
      mark_as_needing_eager_checkpoint(true);
    }
4482 4483 4484 4485 4486 4487
  }

  return result;
}


4488 4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500 4501 4502 4503 4504 4505
Node* BytecodeGraphBuilder::NewPhi(int count, Node* input, Node* control) {
  const Operator* phi_op = common()->Phi(MachineRepresentation::kTagged, count);
  Node** buffer = EnsureInputBufferSize(count + 1);
  MemsetPointer(buffer, input, count);
  buffer[count] = control;
  return graph()->NewNode(phi_op, count + 1, buffer, true);
}

Node* BytecodeGraphBuilder::NewEffectPhi(int count, Node* input,
                                         Node* control) {
  const Operator* phi_op = common()->EffectPhi(count);
  Node** buffer = EnsureInputBufferSize(count + 1);
  MemsetPointer(buffer, input, count);
  buffer[count] = control;
  return graph()->NewNode(phi_op, count + 1, buffer, true);
}


4506 4507 4508 4509 4510 4511
Node* BytecodeGraphBuilder::MergeControl(Node* control, Node* other) {
  int inputs = control->op()->ControlInputCount() + 1;
  if (control->opcode() == IrOpcode::kLoop) {
    // Control node for loop exists, add input.
    const Operator* op = common()->Loop(inputs);
    control->AppendInput(graph_zone(), other);
4512
    NodeProperties::ChangeOp(control, op);
4513 4514 4515 4516
  } else if (control->opcode() == IrOpcode::kMerge) {
    // Control node for merge exists, add input.
    const Operator* op = common()->Merge(inputs);
    control->AppendInput(graph_zone(), other);
4517
    NodeProperties::ChangeOp(control, op);
4518 4519 4520
  } else {
    // Control node is a singleton, introduce a merge.
    const Operator* op = common()->Merge(inputs);
4521 4522
    Node* merge_inputs[] = {control, other};
    control = graph()->NewNode(op, arraysize(merge_inputs), merge_inputs, true);
4523 4524 4525 4526
  }
  return control;
}

4527 4528 4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543 4544 4545 4546 4547 4548 4549 4550 4551 4552 4553 4554 4555 4556 4557 4558 4559
Node* BytecodeGraphBuilder::MergeEffect(Node* value, Node* other,
                                        Node* control) {
  int inputs = control->op()->ControlInputCount();
  if (value->opcode() == IrOpcode::kEffectPhi &&
      NodeProperties::GetControlInput(value) == control) {
    // Phi already exists, add input.
    value->InsertInput(graph_zone(), inputs - 1, other);
    NodeProperties::ChangeOp(value, common()->EffectPhi(inputs));
  } else if (value != other) {
    // Phi does not exist yet, introduce one.
    value = NewEffectPhi(inputs, value, control);
    value->ReplaceInput(inputs - 1, other);
  }
  return value;
}

Node* BytecodeGraphBuilder::MergeValue(Node* value, Node* other,
                                       Node* control) {
  int inputs = control->op()->ControlInputCount();
  if (value->opcode() == IrOpcode::kPhi &&
      NodeProperties::GetControlInput(value) == control) {
    // Phi already exists, add input.
    value->InsertInput(graph_zone(), inputs - 1, other);
    NodeProperties::ChangeOp(
        value, common()->Phi(MachineRepresentation::kTagged, inputs));
  } else if (value != other) {
    // Phi does not exist yet, introduce one.
    value = NewPhi(inputs, value, control);
    value->ReplaceInput(inputs - 1, other);
  }
  return value;
}

4560 4561 4562
void BytecodeGraphBuilder::UpdateSourcePosition(int offset) {
  if (source_position_iterator().done()) return;
  if (source_position_iterator().code_offset() == offset) {
4563
    source_positions_->SetCurrentPosition(SourcePosition(
4564 4565 4566
        source_position_iterator().source_position().ScriptOffset(),
        start_position_.InliningId()));
    source_position_iterator().Advance();
4567
  } else {
4568
    DCHECK_GT(source_position_iterator().code_offset(), offset);
4569 4570 4571
  }
}

4572
void BuildGraphFromBytecode(JSHeapBroker* broker, Zone* local_zone,
4573
                            SharedFunctionInfoRef const& shared_info,
4574
                            FeedbackCellRef const& feedback_cell,
4575
                            BytecodeOffset osr_offset, JSGraph* jsgraph,
4576
                            CallFrequency const& invocation_frequency,
4577
                            SourcePositionTable* source_positions,
4578 4579
                            int inlining_id, CodeKind code_kind,
                            BytecodeGraphBuilderFlags flags,
4580 4581
                            TickCounter* tick_counter,
                            ObserveNodeInfo const& observe_node_info) {
4582
  DCHECK(broker->IsSerializedForCompilation(
4583 4584
      shared_info, feedback_cell.value()->AsFeedbackVector()));
  DCHECK(feedback_cell.value()->AsFeedbackVector().serialized());
4585
  BytecodeGraphBuilder builder(
4586
      broker, local_zone, broker->target_native_context(), shared_info,
4587
      feedback_cell, osr_offset, jsgraph, invocation_frequency,
4588 4589
      source_positions, inlining_id, code_kind, flags, tick_counter,
      observe_node_info);
4590 4591 4592
  builder.CreateGraph();
}

4593 4594 4595
}  // namespace compiler
}  // namespace internal
}  // namespace v8