[turbofan] Introduce explicit loop exits markers.

This CL introduces explicit LoopExit control nodes at loop exits.
We also attach explicit value renames (LoopExitMarker) and effect
rename (LoopExitEffect) to each loop exit. This is in preparation
to loop peeling, which will replace LoopExit, LoopExitMarker and
LoopExitEffect with Merge, Phi and EffectPhi respectively.

At the moment, we insert loop exit at every return, break,
continue and locally caught throw. We do not yet handle
uncaught throws (including error throws, such as ReferenceError).

Review-Url: https://codereview.chromium.org/2140673007
Cr-Commit-Position: refs/heads/master@{#37769}

src/compiler/ast-graph-builder.cc

@@ -178,14 +178,14 @@ class AstGraphBuilder::ControlScope BASE_EMBEDDED {
 
   // Interface to execute a given command in this scope. Returning {true} here
   // indicates successful execution whereas {false} requests to skip scope.
-  virtual bool Execute(Command cmd, Statement* target, Node* value) {
+  virtual bool Execute(Command cmd, Statement* target, Node** value) {
     // For function-level control.
     switch (cmd) {
       case CMD_THROW:
-        builder()->BuildThrow(value);
+        builder()->BuildThrow(*value);
         return true;
       case CMD_RETURN:
-        builder()->BuildReturn(value);
+        builder()->BuildReturn(*value);
         return true;
       case CMD_BREAK:
       case CMD_CONTINUE:
@@ -303,7 +303,7 @@ class AstGraphBuilder::ControlScopeForBreakable : public ControlScope {
       : ControlScope(owner), target_(target), control_(control) {}
 
  protected:
-  bool Execute(Command cmd, Statement* target, Node* value) override {
+  bool Execute(Command cmd, Statement* target, Node** value) override {
     if (target != target_) return false;  // We are not the command target.
     switch (cmd) {
       case CMD_BREAK:
@@ -331,8 +331,11 @@ class AstGraphBuilder::ControlScopeForIteration : public ControlScope {
       : ControlScope(owner), target_(target), control_(control) {}
 
  protected:
-  bool Execute(Command cmd, Statement* target, Node* value) override {
-    if (target != target_) return false;  // We are not the command target.
+  bool Execute(Command cmd, Statement* target, Node** value) override {
+    if (target != target_) {
+      control_->ExitLoop(value);
+      return false;
+    }
     switch (cmd) {
       case CMD_BREAK:
         control_->Break();
@@ -370,10 +373,10 @@ class AstGraphBuilder::ControlScopeForCatch : public ControlScope {
   }
 
  protected:
-  bool Execute(Command cmd, Statement* target, Node* value) override {
+  bool Execute(Command cmd, Statement* target, Node** value) override {
     switch (cmd) {
       case CMD_THROW:
-        control_->Throw(value);
+        control_->Throw(*value);
         return true;
       case CMD_BREAK:
       case CMD_CONTINUE:
@@ -407,9 +410,9 @@ class AstGraphBuilder::ControlScopeForFinally : public ControlScope {
   }
 
  protected:
-  bool Execute(Command cmd, Statement* target, Node* value) override {
-    Node* token = commands_->RecordCommand(cmd, target, value);
-    control_->LeaveTry(token, value);
+  bool Execute(Command cmd, Statement* target, Node** value) override {
+    Node* token = commands_->RecordCommand(cmd, target, *value);
+    control_->LeaveTry(token, *value);
     return true;
   }
 
@@ -935,6 +938,34 @@ Node* AstGraphBuilder::Environment::Checkpoint(BailoutId ast_id,
   return result;
 }
 
+void AstGraphBuilder::Environment::PrepareForLoopExit(
+    Node* loop, BitVector* assigned_variables) {
+  if (IsMarkedAsUnreachable()) return;
+
+  DCHECK_EQ(loop->opcode(), IrOpcode::kLoop);
+
+  Node* control = GetControlDependency();
+
+  // Create the loop exit node.
+  Node* loop_exit = graph()->NewNode(common()->LoopExit(), control, loop);
+  UpdateControlDependency(loop_exit);
+
+  // Rename the environmnent values.
+  for (size_t i = 0; i < values()->size(); i++) {
+    if (assigned_variables == nullptr ||
+        static_cast<int>(i) >= assigned_variables->length() ||
+        assigned_variables->Contains(static_cast<int>(i))) {
+      Node* rename = graph()->NewNode(common()->LoopExitValue(), (*values())[i],
+                                      loop_exit);
+      (*values())[i] = rename;
+    }
+  }
+
+  // Rename the effect.
+  Node* effect_rename = graph()->NewNode(common()->LoopExitEffect(),
+                                         GetEffectDependency(), loop_exit);
+  UpdateEffectDependency(effect_rename);
+}
 
 bool AstGraphBuilder::Environment::IsLivenessAnalysisEnabled() {
   return FLAG_analyze_environment_liveness &&
@@ -1027,7 +1058,7 @@ void AstGraphBuilder::ControlScope::PerformCommand(Command command,
   while (current != nullptr) {
     environment()->TrimStack(current->stack_height());
     environment()->TrimContextChain(current->context_length());
-    if (current->Execute(command, target, value)) break;
+    if (current->Execute(command, target, &value)) break;
     current = current->outer_;
   }
   builder()->set_environment(env);

src/compiler/ast-graph-builder.h

@@ -540,6 +540,10 @@ class AstGraphBuilder::Environment : public ZoneObject {
                                          OutputFrameStateCombine::Ignore(),
                    bool node_has_exception = false);
 
+  // Inserts a loop exit control node and renames the environment.
+  // This is useful for loop peeling to insert phis at loop exits.
+  void PrepareForLoopExit(Node* loop, BitVector* assigned_variables);
+
   // Control dependency tracked by this environment.
   Node* GetControlDependency() { return control_dependency_; }
   void UpdateControlDependency(Node* dependency) {

src/compiler/common-operator.cc

@@ -213,6 +213,9 @@ std::ostream& operator<<(std::ostream& os,
   V(Terminate, Operator::kKontrol, 0, 1, 1, 0, 0, 1)         \
   V(OsrNormalEntry, Operator::kFoldable, 0, 1, 1, 0, 1, 1)   \
   V(OsrLoopEntry, Operator::kFoldable, 0, 1, 1, 0, 1, 1)     \
+  V(LoopExit, Operator::kKontrol, 0, 0, 2, 0, 0, 1)          \
+  V(LoopExitValue, Operator::kPure, 1, 0, 1, 1, 0, 0)        \
+  V(LoopExitEffect, Operator::kNoThrow, 0, 1, 1, 0, 1, 0)    \
   V(Checkpoint, Operator::kKontrol, 0, 1, 1, 0, 1, 0)        \
   V(FinishRegion, Operator::kKontrol, 1, 1, 0, 1, 1, 0)
 

src/compiler/common-operator.h

@@ -202,6 +202,9 @@ class CommonOperatorBuilder final : public ZoneObject {
   const Operator* Phi(MachineRepresentation representation,
                       int value_input_count);
   const Operator* EffectPhi(int effect_input_count);
+  const Operator* LoopExit();
+  const Operator* LoopExitValue();
+  const Operator* LoopExitEffect();
   const Operator* Checkpoint();
   const Operator* BeginRegion(RegionObservability);
   const Operator* FinishRegion();

src/compiler/control-builders.cc

@@ -36,6 +36,7 @@ void LoopBuilder::BeginLoop(BitVector* assigned, bool is_osr) {
   loop_environment_ = environment()->CopyForLoop(assigned, is_osr);
   continue_environment_ = environment()->CopyAsUnreachable();
   break_environment_ = environment()->CopyAsUnreachable();
+  assigned_ = assigned;
 }
 
 
@@ -60,6 +61,7 @@ void LoopBuilder::EndBody() {
 void LoopBuilder::EndLoop() {
   loop_environment_->Merge(environment());
   set_environment(break_environment_);
+  ExitLoop();
 }
 
 
@@ -82,6 +84,16 @@ void LoopBuilder::BreakWhen(Node* condition) {
   control_if.End();
 }
 
+void LoopBuilder::ExitLoop(Node** extra_value_to_rename) {
+  if (extra_value_to_rename) {
+    environment()->Push(*extra_value_to_rename);
+  }
+  environment()->PrepareForLoopExit(loop_environment_->GetControlDependency(),
+                                    assigned_);
+  if (extra_value_to_rename) {
+    environment()->Pop();
+  }
+}
 
 void SwitchBuilder::BeginSwitch() {
   body_environment_ = environment()->CopyAsUnreachable();

src/compiler/control-builders.h

@@ -63,7 +63,8 @@ class LoopBuilder final : public ControlBuilder {
       : ControlBuilder(builder),
         loop_environment_(nullptr),
         continue_environment_(nullptr),
-        break_environment_(nullptr) {}
+        break_environment_(nullptr),
+        assigned_(nullptr) {}
 
   // Primitive control commands.
   void BeginLoop(BitVector* assigned, bool is_osr = false);
@@ -74,6 +75,10 @@ class LoopBuilder final : public ControlBuilder {
   // Primitive support for break.
   void Break() final;
 
+  // Loop exit support. Used to introduce explicit loop exit control
+  // node and variable markers.
+  void ExitLoop(Node** extra_value_to_rename = nullptr);
+
   // Compound control commands for conditional break.
   void BreakUnless(Node* condition);
   void BreakWhen(Node* condition);
@@ -82,6 +87,7 @@ class LoopBuilder final : public ControlBuilder {
   Environment* loop_environment_;      // Environment of the loop header.
   Environment* continue_environment_;  // Environment after the loop body.
   Environment* break_environment_;     // Environment after the loop exits.
+  BitVector* assigned_;                // Assigned values in the environment.
 };
 
 

src/compiler/dead-code-elimination.cc

@@ -28,6 +28,8 @@ Reduction DeadCodeElimination::Reduce(Node* node) {
     case IrOpcode::kLoop:
     case IrOpcode::kMerge:
       return ReduceLoopOrMerge(node);
+    case IrOpcode::kLoopExit:
+      return ReduceLoopExit(node);
     default:
       return ReduceNode(node);
   }
@@ -96,6 +98,9 @@ Reduction DeadCodeElimination::ReduceLoopOrMerge(Node* node) {
     for (Node* const use : node->uses()) {
       if (NodeProperties::IsPhi(use)) {
         Replace(use, use->InputAt(0));
+      } else if (use->opcode() == IrOpcode::kLoopExit &&
+                 use->InputAt(1) == node) {
+        RemoveLoopExit(use);
       } else if (use->opcode() == IrOpcode::kTerminate) {
         DCHECK_EQ(IrOpcode::kLoop, node->opcode());
         Replace(use, dead());
@@ -121,6 +126,18 @@ Reduction DeadCodeElimination::ReduceLoopOrMerge(Node* node) {
   return NoChange();
 }
 
+Reduction DeadCodeElimination::RemoveLoopExit(Node* node) {
+  DCHECK_EQ(IrOpcode::kLoopExit, node->opcode());
+  for (Node* const use : node->uses()) {
+    if (use->opcode() == IrOpcode::kLoopExitValue ||
+        use->opcode() == IrOpcode::kLoopExitEffect) {
+      Replace(use, use->InputAt(0));
+    }
+  }
+  Node* control = NodeProperties::GetControlInput(node, 0);
+  Replace(node, control);
+  return Replace(control);
+}
 
 Reduction DeadCodeElimination::ReduceNode(Node* node) {
   // If {node} has exactly one control input and this is {Dead},
@@ -133,6 +150,15 @@ Reduction DeadCodeElimination::ReduceNode(Node* node) {
   return NoChange();
 }
 
+Reduction DeadCodeElimination::ReduceLoopExit(Node* node) {
+  Node* control = NodeProperties::GetControlInput(node, 0);
+  Node* loop = NodeProperties::GetControlInput(node, 1);
+  if (control->opcode() == IrOpcode::kDead ||
+      loop->opcode() == IrOpcode::kDead) {
+    return RemoveLoopExit(node);
+  }
+  return NoChange();
+}
 
 void DeadCodeElimination::TrimMergeOrPhi(Node* node, int size) {
   const Operator* const op = common()->ResizeMergeOrPhi(node->op(), size);

src/compiler/dead-code-elimination.h

@@ -30,8 +30,11 @@ class DeadCodeElimination final : public AdvancedReducer {
  private:
   Reduction ReduceEnd(Node* node);
   Reduction ReduceLoopOrMerge(Node* node);
+  Reduction ReduceLoopExit(Node* node);
   Reduction ReduceNode(Node* node);
 
+  Reduction RemoveLoopExit(Node* node);
+
   void TrimMergeOrPhi(Node* node, int size);
 
   Graph* graph() const { return graph_; }

src/compiler/loop-peeling.cc

@@ -329,6 +329,60 @@ PeeledIteration* LoopPeeler::Peel(Graph* graph, CommonOperatorBuilder* common,
   return iter;
 }
 
+namespace {
+
+void EliminateLoopExit(Node* node) {
+  DCHECK_EQ(IrOpcode::kLoopExit, node->opcode());
+  // The exit markers take the loop exit as input. We iterate over uses
+  // and remove all the markers from the graph.
+  for (Edge edge : node->use_edges()) {
+    if (NodeProperties::IsControlEdge(edge)) {
+      Node* marker = edge.from();
+      if (marker->opcode() == IrOpcode::kLoopExitValue) {
+        NodeProperties::ReplaceUses(marker, marker->InputAt(0));
+        marker->Kill();
+      } else if (marker->opcode() == IrOpcode::kLoopExitEffect) {
+        NodeProperties::ReplaceUses(marker, nullptr,
+                                    NodeProperties::GetEffectInput(marker));
+        marker->Kill();
+      }
+    }
+  }
+  NodeProperties::ReplaceUses(node, nullptr, nullptr,
+                              NodeProperties::GetControlInput(node, 0));
+  node->Kill();
+}
+
+}  // namespace
+
+// static
+void LoopPeeler::EliminateLoopExits(Graph* graph, Zone* temp_zone) {
+  ZoneQueue<Node*> queue(temp_zone);
+  ZoneVector<bool> visited(graph->NodeCount(), false, temp_zone);
+  queue.push(graph->end());
+  while (!queue.empty()) {
+    Node* node = queue.front();
+    queue.pop();
+
+    if (node->opcode() == IrOpcode::kLoopExit) {
+      Node* control = NodeProperties::GetControlInput(node);
+      EliminateLoopExit(node);
+      if (!visited[control->id()]) {
+        visited[control->id()] = true;
+        queue.push(control);
+      }
+    } else {
+      for (int i = 0; i < node->op()->ControlInputCount(); i++) {
+        Node* control = NodeProperties::GetControlInput(node, i);
+        if (!visited[control->id()]) {
+          visited[control->id()] = true;
+          queue.push(control);
+        }
+      }
+    }
+  }
+}
+
 }  // namespace compiler
 }  // namespace internal
 }  // namespace v8

src/compiler/loop-peeling.h

@@ -33,6 +33,7 @@ class LoopPeeler {
   static PeeledIteration* Peel(Graph* graph, CommonOperatorBuilder* common,
                                LoopTree* loop_tree, LoopTree::Loop* loop,
                                Zone* tmp_zone);
+  static void EliminateLoopExits(Graph* graph, Zone* temp_zone);
 };
 
 

src/compiler/opcodes.h

@@ -57,6 +57,9 @@
   V(Call)                \
   V(Parameter)           \
   V(OsrValue)            \
+  V(LoopExit)            \
+  V(LoopExitValue)       \
+  V(LoopExitEffect)      \
   V(Projection)
 
 #define COMMON_OP_LIST(V) \

src/compiler/pipeline.cc

@@ -942,6 +942,14 @@ struct RepresentationSelectionPhase {
   }
 };
 
+struct LoopExitEliminationPhase {
+  static const char* phase_name() { return "loop exit elimination"; }
+
+  void Run(PipelineData* data, Zone* temp_zone) {
+    LoopPeeler::EliminateLoopExits(data->graph(), temp_zone);
+  }
+};
+
 struct EarlyOptimizationPhase {
   static const char* phase_name() { return "early optimization"; }
 
@@ -1430,6 +1438,10 @@ bool PipelineImpl::CreateGraph() {
     Run<TypedLoweringPhase>();
     RunPrintAndVerify("Lowered typed");
 
+    // Eventually, loop peeling will be done here.
+    Run<LoopExitEliminationPhase>();
+    RunPrintAndVerify("Loop exits eliminated", true);
+
     if (FLAG_turbo_stress_loop_peeling) {
       Run<StressLoopPeelingPhase>();
       RunPrintAndVerify("Loop peeled");

src/compiler/simplified-lowering.cc

@@ -2275,6 +2275,9 @@ class RepresentationSelector {
       case IrOpcode::kCheckedTaggedToFloat64:
       case IrOpcode::kPlainPrimitiveToWord32:
       case IrOpcode::kPlainPrimitiveToFloat64:
+      case IrOpcode::kLoopExit:
+      case IrOpcode::kLoopExitValue:
+      case IrOpcode::kLoopExitEffect:
         FATAL("Representation inference: unsupported opcodes.");
         break;
 

src/compiler/typer.cc

@@ -713,6 +713,18 @@ Type* Typer::Visitor::TypeEffectPhi(Node* node) {
   return nullptr;
 }
 
+Type* Typer::Visitor::TypeLoopExit(Node* node) {
+  UNREACHABLE();
+  return nullptr;
+}
+
+Type* Typer::Visitor::TypeLoopExitValue(Node* node) { return Operand(node, 0); }
+
+Type* Typer::Visitor::TypeLoopExitEffect(Node* node) {
+  UNREACHABLE();
+  return nullptr;
+}
+
 Type* Typer::Visitor::TypeCheckpoint(Node* node) {
   UNREACHABLE();
   return nullptr;

src/compiler/verifier.cc

@@ -395,6 +395,24 @@ void Verifier::Visitor::Check(Node* node) {
       CHECK_EQ(input_count, 1 + effect_count);
       break;
     }
+    case IrOpcode::kLoopExit: {
+      CHECK_EQ(2, control_count);
+      Node* loop = NodeProperties::GetControlInput(node, 1);
+      CHECK_EQ(IrOpcode::kLoop, loop->opcode());
+      break;
+    }
+    case IrOpcode::kLoopExitValue: {
+      CHECK_EQ(1, control_count);
+      Node* loop_exit = NodeProperties::GetControlInput(node, 0);
+      CHECK_EQ(IrOpcode::kLoopExit, loop_exit->opcode());
+      break;
+    }
+    case IrOpcode::kLoopExitEffect: {
+      CHECK_EQ(1, control_count);
+      Node* loop_exit = NodeProperties::GetControlInput(node, 0);
+      CHECK_EQ(IrOpcode::kLoopExit, loop_exit->opcode());
+      break;
+    }
     case IrOpcode::kCheckpoint:
       // Type is empty.
       CheckNotTyped(node);