[turbofan] Loop peeling with explicit loop exits.

Review-Url: https://codereview.chromium.org/2143163002
Cr-Commit-Position: refs/heads/master@{#37791}

src/compiler/loop-analysis.cc

@@ -29,6 +29,7 @@ struct NodeInfo {
 struct LoopInfo {
   Node* header;
   NodeInfo* header_list;
+  NodeInfo* exit_list;
   NodeInfo* body_list;
   LoopTree::Loop* loop;
 };
@@ -81,9 +82,9 @@ class LoopFinderImpl {
         if (marked_forward && marked_backward) {
           PrintF("X");
         } else if (marked_forward) {
-          PrintF("/");
+          PrintF(">");
         } else if (marked_backward) {
-          PrintF("\\");
+          PrintF("<");
         } else {
           PrintF(" ");
         }
@@ -198,12 +199,22 @@ class LoopFinderImpl {
         if (merge->opcode() == IrOpcode::kLoop) {
           loop_num = CreateLoopInfo(merge);
         }
+      } else if (node->opcode() == IrOpcode::kLoopExit) {
+        // Intentionally ignore return value. Loop exit node marks
+        // are propagated normally.
+        CreateLoopInfo(node->InputAt(1));
+      } else if (node->opcode() == IrOpcode::kLoopExitValue ||
+                 node->opcode() == IrOpcode::kLoopExitEffect) {
+        Node* loop_exit = NodeProperties::GetControlInput(node);
+        // Intentionally ignore return value. Loop exit node marks
+        // are propagated normally.
+        CreateLoopInfo(loop_exit->InputAt(1));
       }
 
       // Propagate marks backwards from this node.
       for (int i = 0; i < node->InputCount(); i++) {
         Node* input = node->InputAt(i);
-        if (loop_num > 0 && i != kAssumedLoopEntryIndex) {
+        if (IsBackedge(node, i)) {
           // Only propagate the loop mark on backedges.
           if (SetBackwardMark(input, loop_num)) Queue(input);
         } else {
@@ -216,6 +227,7 @@ class LoopFinderImpl {
 
   // Make a new loop if necessary for the given node.
   int CreateLoopInfo(Node* node) {
+    DCHECK_EQ(IrOpcode::kLoop, node->opcode());
     int loop_num = LoopNum(node);
     if (loop_num > 0) return loop_num;
 
@@ -223,21 +235,39 @@ class LoopFinderImpl {
     if (INDEX(loop_num) >= width_) ResizeBackwardMarks();
 
     // Create a new loop.
-    loops_.push_back({node, nullptr, nullptr, nullptr});
+    loops_.push_back({node, nullptr, nullptr, nullptr, nullptr});
     loop_tree_->NewLoop();
+    SetLoopMarkForLoopHeader(node, loop_num);
+    return loop_num;
+  }
+
+  void SetLoopMark(Node* node, int loop_num) {
+    info(node);  // create the NodeInfo
     SetBackwardMark(node, loop_num);
     loop_tree_->node_to_loop_num_[node->id()] = loop_num;
+  }
 
-    // Setup loop mark for phis attached to loop header.
+  void SetLoopMarkForLoopHeader(Node* node, int loop_num) {
+    DCHECK_EQ(IrOpcode::kLoop, node->opcode());
+    SetLoopMark(node, loop_num);
     for (Node* use : node->uses()) {
       if (NodeProperties::IsPhi(use)) {
-        info(use);  // create the NodeInfo
-        SetBackwardMark(use, loop_num);
-        loop_tree_->node_to_loop_num_[use->id()] = loop_num;
+        SetLoopMark(use, loop_num);
       }
-    }
 
-    return loop_num;
+      // Do not keep the loop alive if it does not have any backedges.
+      if (node->InputCount() <= 1) continue;
+
+      if (use->opcode() == IrOpcode::kLoopExit) {
+        SetLoopMark(use, loop_num);
+        for (Node* exit_use : use->uses()) {
+          if (exit_use->opcode() == IrOpcode::kLoopExitValue ||
+              exit_use->opcode() == IrOpcode::kLoopExitEffect) {
+            SetLoopMark(exit_use, loop_num);
+          }
+        }
+      }
+    }
   }
 
   void ResizeBackwardMarks() {
@@ -276,20 +306,33 @@ class LoopFinderImpl {
       queued_.Set(node, false);
       for (Edge edge : node->use_edges()) {
         Node* use = edge.from();
-        if (!IsBackedge(use, edge)) {
+        if (!IsBackedge(use, edge.index())) {
           if (PropagateForwardMarks(node, use)) Queue(use);
         }
       }
     }
   }
 
-  bool IsBackedge(Node* use, Edge& edge) {
+  bool IsLoopHeaderNode(Node* node) {
+    return node->opcode() == IrOpcode::kLoop || NodeProperties::IsPhi(node);
+  }
+
+  bool IsLoopExitNode(Node* node) {
+    return node->opcode() == IrOpcode::kLoopExit ||
+           node->opcode() == IrOpcode::kLoopExitValue ||
+           node->opcode() == IrOpcode::kLoopExitEffect;
+  }
+
+  bool IsBackedge(Node* use, int index) {
     if (LoopNum(use) <= 0) return false;
-    if (edge.index() == kAssumedLoopEntryIndex) return false;
     if (NodeProperties::IsPhi(use)) {
-      return !NodeProperties::IsControlEdge(edge);
+      return index != NodeProperties::FirstControlIndex(use) &&
+             index != kAssumedLoopEntryIndex;
+    } else if (use->opcode() == IrOpcode::kLoop) {
+      return index != kAssumedLoopEntryIndex;
     }
-    return true;
+    DCHECK(IsLoopExitNode(use));
+    return false;
   }
 
   int LoopNum(Node* node) { return loop_tree_->node_to_loop_num_[node->id()]; }
@@ -307,6 +350,22 @@ class LoopFinderImpl {
     }
   }
 
+  void AddNodeToLoop(NodeInfo* node_info, LoopInfo* loop, int loop_num) {
+    if (LoopNum(node_info->node) == loop_num) {
+      if (IsLoopHeaderNode(node_info->node)) {
+        node_info->next = loop->header_list;
+        loop->header_list = node_info;
+      } else {
+        DCHECK(IsLoopExitNode(node_info->node));
+        node_info->next = loop->exit_list;
+        loop->exit_list = node_info;
+      }
+    } else {
+      node_info->next = loop->body_list;
+      loop->body_list = node_info;
+    }
+  }
+
   void FinishLoopTree() {
     DCHECK(loops_found_ == static_cast<int>(loops_.size()));
     DCHECK(loops_found_ == static_cast<int>(loop_tree_->all_loops_.size()));
@@ -342,13 +401,7 @@ class LoopFinderImpl {
         }
       }
       if (innermost == nullptr) continue;
-      if (LoopNum(ni.node) == innermost_index) {
-        ni.next = innermost->header_list;
-        innermost->header_list = &ni;
-      } else {
-        ni.next = innermost->body_list;
-        innermost->body_list = &ni;
-      }
+      AddNodeToLoop(&ni, innermost, innermost_index);
       count++;
     }
 
@@ -368,13 +421,7 @@ class LoopFinderImpl {
     size_t count = 0;
     for (NodeInfo& ni : info_) {
       if (ni.node == nullptr || !IsInLoop(ni.node, 1)) continue;
-      if (LoopNum(ni.node) == 1) {
-        ni.next = li->header_list;
-        li->header_list = &ni;
-      } else {
-        ni.next = li->body_list;
-        li->body_list = &ni;
-      }
+      AddNodeToLoop(&ni, li, 1);
       count++;
     }
 
@@ -406,7 +453,14 @@ class LoopFinderImpl {
     // Serialize nested loops.
     for (LoopTree::Loop* child : loop->children_) SerializeLoop(child);
 
-    loop->body_end_ = static_cast<int>(loop_tree_->loop_nodes_.size());
+    // Serialize the exits.
+    loop->exits_start_ = static_cast<int>(loop_tree_->loop_nodes_.size());
+    for (NodeInfo* ni = li.exit_list; ni != nullptr; ni = ni->next) {
+      loop_tree_->loop_nodes_.push_back(ni->node);
+      loop_tree_->node_to_loop_num_[ni->node->id()] = loop_num;
+    }
+
+    loop->exits_end_ = static_cast<int>(loop_tree_->loop_nodes_.size());
   }
 
   // Connect the LoopTree loops to their parents recursively.
@@ -438,9 +492,12 @@ class LoopFinderImpl {
     while (i < loop->body_start_) {
       PrintF(" H#%d", loop_tree_->loop_nodes_[i++]->id());
     }
-    while (i < loop->body_end_) {
+    while (i < loop->exits_start_) {
       PrintF(" B#%d", loop_tree_->loop_nodes_[i++]->id());
     }
+    while (i < loop->exits_end_) {
+      PrintF(" E#%d", loop_tree_->loop_nodes_[i++]->id());
+    }
     PrintF("\n");
     for (LoopTree::Loop* child : loop->children_) PrintLoop(child);
   }

src/compiler/loop-analysis.h

@@ -38,8 +38,9 @@ class LoopTree : public ZoneObject {
     Loop* parent() const { return parent_; }
     const ZoneVector<Loop*>& children() const { return children_; }
     size_t HeaderSize() const { return body_start_ - header_start_; }
-    size_t BodySize() const { return body_end_ - body_start_; }
-    size_t TotalSize() const { return body_end_ - header_start_; }
+    size_t BodySize() const { return exits_start_ - body_start_; }
+    size_t ExitsSize() const { return exits_end_ - exits_start_; }
+    size_t TotalSize() const { return exits_end_ - header_start_; }
     size_t depth() const { return static_cast<size_t>(depth_); }
 
    private:
@@ -52,13 +53,15 @@ class LoopTree : public ZoneObject {
           children_(zone),
           header_start_(-1),
           body_start_(-1),
-          body_end_(-1) {}
+          exits_start_(-1),
+          exits_end_(-1) {}
     Loop* parent_;
     int depth_;
     ZoneVector<Loop*> children_;
     int header_start_;
     int body_start_;
-    int body_end_;
+    int exits_start_;
+    int exits_end_;
   };
 
   // Return the innermost nested loop, if any, that contains {node}.
@@ -97,13 +100,19 @@ class LoopTree : public ZoneObject {
   // Return a range which can iterate over the body nodes of {loop}.
   NodeRange BodyNodes(Loop* loop) {
     return NodeRange(&loop_nodes_[0] + loop->body_start_,
-                     &loop_nodes_[0] + loop->body_end_);
+                     &loop_nodes_[0] + loop->exits_start_);
+  }
+
+  // Return a range which can iterate over the body nodes of {loop}.
+  NodeRange ExitNodes(Loop* loop) {
+    return NodeRange(&loop_nodes_[0] + loop->exits_start_,
+                     &loop_nodes_[0] + loop->exits_end_);
   }
 
   // Return a range which can iterate over the nodes of {loop}.
   NodeRange LoopNodes(Loop* loop) {
     return NodeRange(&loop_nodes_[0] + loop->header_start_,
-                     &loop_nodes_[0] + loop->body_end_);
+                     &loop_nodes_[0] + loop->exits_end_);
   }
 
   // Return the node that represents the control, i.e. the loop node itself.

src/compiler/loop-peeling.cc

@@ -126,8 +126,14 @@ struct Peeling {
     // Copy all the nodes first.
     for (Node* node : nodes) {
       inputs.clear();
-      for (Node* input : node->inputs()) inputs.push_back(map(input));
-      Insert(node, graph->NewNode(node->op(), node->InputCount(), &inputs[0]));
+      for (Node* input : node->inputs()) {
+        inputs.push_back(map(input));
+      }
+      Node* copy = graph->NewNode(node->op(), node->InputCount(), &inputs[0]);
+      if (NodeProperties::IsTyped(node)) {
+        NodeProperties::SetType(copy, NodeProperties::GetType(node));
+      }
+      Insert(node, copy);
     }
 
     // Fix remaining inputs of the copies.
@@ -160,56 +166,54 @@ Node* PeeledIteration::map(Node* node) {
   return node;
 }
 
-
-static void FindLoopExits(LoopTree* loop_tree, LoopTree::Loop* loop,
-                          NodeVector& exits, NodeVector& rets) {
+bool LoopPeeler::CanPeel(LoopTree* loop_tree, LoopTree::Loop* loop) {
   // Look for returns and if projections that are outside the loop but whose
   // control input is inside the loop.
+  Node* loop_node = loop_tree->GetLoopControl(loop);
   for (Node* node : loop_tree->LoopNodes(loop)) {
     for (Node* use : node->uses()) {
       if (!loop_tree->Contains(loop, use)) {
-        if (IrOpcode::IsIfProjectionOpcode(use->opcode())) {
-          // This is a branch from inside the loop to outside the loop.
-          exits.push_back(use);
-        } else if (use->opcode() == IrOpcode::kReturn &&
-                   loop_tree->Contains(loop,
-                                       NodeProperties::GetControlInput(use))) {
-          // This is a return from inside the loop.
-          rets.push_back(use);
+        bool unmarked_exit;
+        switch (node->opcode()) {
+          case IrOpcode::kLoopExit:
+            unmarked_exit = (node->InputAt(1) != loop_node);
+            break;
+          case IrOpcode::kLoopExitValue:
+          case IrOpcode::kLoopExitEffect:
+            unmarked_exit = (node->InputAt(1)->InputAt(1) != loop_node);
+            break;
+          default:
+            unmarked_exit = (use->opcode() != IrOpcode::kTerminate);
+        }
+        if (unmarked_exit) {
+          if (FLAG_trace_turbo_graph) {
+            Node* loop_node = loop_tree->GetLoopControl(loop);
+            PrintF(
+                "Cannot peel loop %i. Loop exit without explicit mark: Node %i "
+                "(%s) is inside "
+                "loop, but its use %i (%s) is outside.\n",
+                loop_node->id(), node->id(), node->op()->mnemonic(), use->id(),
+                use->op()->mnemonic());
+          }
+          return false;
         }
       }
     }
   }
-}
-
-
-bool LoopPeeler::CanPeel(LoopTree* loop_tree, LoopTree::Loop* loop) {
-  Zone zone(loop_tree->zone()->allocator());
-  NodeVector exits(&zone);
-  NodeVector rets(&zone);
-  FindLoopExits(loop_tree, loop, exits, rets);
-  return exits.size() <= 1u;
+  return true;
 }
 
 
 PeeledIteration* LoopPeeler::Peel(Graph* graph, CommonOperatorBuilder* common,
                                   LoopTree* loop_tree, LoopTree::Loop* loop,
                                   Zone* tmp_zone) {
-  //============================================================================
-  // Find the loop exit region to determine if this loop can be peeled.
-  //============================================================================
-  NodeVector exits(tmp_zone);
-  NodeVector rets(tmp_zone);
-  FindLoopExits(loop_tree, loop, exits, rets);
-
-  if (exits.size() != 1) return nullptr;  // not peelable currently.
+  if (!CanPeel(loop_tree, loop)) return nullptr;
 
   //============================================================================
   // Construct the peeled iteration.
   //============================================================================
   PeeledIterationImpl* iter = new (tmp_zone) PeeledIterationImpl(tmp_zone);
-  size_t estimated_peeled_size =
-      5 + (loop->TotalSize() + exits.size() + rets.size()) * 2;
+  size_t estimated_peeled_size = 5 + (loop->TotalSize()) * 2;
   Peeling peeling(graph, tmp_zone, estimated_peeled_size, &iter->node_pairs_);
 
   Node* dead = graph->NewNode(common->Dead());
@@ -260,77 +264,64 @@ PeeledIteration* LoopPeeler::Peel(Graph* graph, CommonOperatorBuilder* common,
     // Only one backedge, simply replace the input to loop with output of
     // peeling.
     for (Node* node : loop_tree->HeaderNodes(loop)) {
-      node->ReplaceInput(0, peeling.map(node->InputAt(0)));
+      node->ReplaceInput(0, peeling.map(node->InputAt(1)));
     }
     new_entry = peeling.map(loop_node->InputAt(1));
   }
   loop_node->ReplaceInput(0, new_entry);
 
   //============================================================================
-  // Duplicate the loop exit region and add a merge.
+  // Change the exit and exit markers to merge/phi/effect-phi.
   //============================================================================
-
-  // Currently we are limited to peeling loops with a single exit. The exit is
-  // the postdominator of the loop (ignoring returns).
-  Node* postdom = exits[0];
-  for (Node* node : rets) exits.push_back(node);
-  for (Node* use : postdom->uses()) {
-    if (NodeProperties::IsPhi(use)) exits.push_back(use);
+  for (Node* exit : loop_tree->ExitNodes(loop)) {
+    switch (exit->opcode()) {
+      case IrOpcode::kLoopExit:
+        // Change the loop exit node to a merge node.
+        exit->ReplaceInput(1, peeling.map(exit->InputAt(0)));
+        NodeProperties::ChangeOp(exit, common->Merge(2));
+        break;
+      case IrOpcode::kLoopExitValue:
+        // Change exit marker to phi.
+        exit->InsertInput(graph->zone(), 1, peeling.map(exit->InputAt(0)));
+        NodeProperties::ChangeOp(
+            exit, common->Phi(MachineRepresentation::kTagged, 2));
+        break;
+      case IrOpcode::kLoopExitEffect:
+        // Change effect exit marker to effect phi.
+        exit->InsertInput(graph->zone(), 1, peeling.map(exit->InputAt(0)));
+        NodeProperties::ChangeOp(exit, common->EffectPhi(2));
+        break;
+      default:
+        break;
+    }
   }
+  return iter;
+}
 
-  NodeRange exit_range(&exits[0], &exits[0] + exits.size());
-  peeling.CopyNodes(graph, tmp_zone, dead, exit_range);
-
-  Node* merge = graph->NewNode(common->Merge(2), postdom, peeling.map(postdom));
-  postdom->ReplaceUses(merge);
-  merge->ReplaceInput(0, postdom);  // input 0 overwritten by above line.
-
-  // Find and update all the edges into either the loop or exit region.
-  for (int i = 0; i < 2; i++) {
-    NodeRange range = i == 0 ? loop_tree->LoopNodes(loop) : exit_range;
-    ZoneVector<Edge> value_edges(tmp_zone);
-    ZoneVector<Edge> effect_edges(tmp_zone);
-
-    for (Node* node : range) {
-      // Gather value and effect edges from outside the region.
-      for (Edge edge : node->use_edges()) {
-        if (!peeling.Marked(edge.from())) {
-          // Edge from outside the loop into the region.
-          if (NodeProperties::IsValueEdge(edge) ||
-              NodeProperties::IsContextEdge(edge)) {
-            value_edges.push_back(edge);
-          } else if (NodeProperties::IsEffectEdge(edge)) {
-            effect_edges.push_back(edge);
-          } else {
-            // don't do anything for control edges.
-            // TODO(titzer): should update control edges to peeled?
-          }
-        }
-      }
+namespace {
 
-      // Update all the value and effect edges at once.
-      if (!value_edges.empty()) {
-        // TODO(titzer): machine type is wrong here.
-        Node* phi =
-            graph->NewNode(common->Phi(MachineRepresentation::kTagged, 2), node,
-                           peeling.map(node), merge);
-        for (Edge edge : value_edges) edge.UpdateTo(phi);
-        value_edges.clear();
-      }
-      if (!effect_edges.empty()) {
-        Node* effect_phi = graph->NewNode(common->EffectPhi(2), node,
-                                          peeling.map(node), merge);
-        for (Edge edge : effect_edges) edge.UpdateTo(effect_phi);
-        effect_edges.clear();
-      }
+void PeelInnerLoops(Graph* graph, CommonOperatorBuilder* common,
+                    LoopTree* loop_tree, LoopTree::Loop* loop,
+                    Zone* temp_zone) {
+  // If the loop has nested loops, peel inside those.
+  if (!loop->children().empty()) {
+    for (LoopTree::Loop* inner_loop : loop->children()) {
+      PeelInnerLoops(graph, common, loop_tree, inner_loop, temp_zone);
+    }
+    return;
+  }
+  // Only peel small-enough loops.
+  if (loop->TotalSize() > LoopPeeler::kMaxPeeledNodes) return;
+  if (FLAG_trace_turbo_graph) {
+    PrintF("Peeling loop with header: ");
+    for (Node* node : loop_tree->HeaderNodes(loop)) {
+      PrintF("%i ", node->id());
     }
   }
 
-  return iter;
+  LoopPeeler::Peel(graph, common, loop_tree, loop, temp_zone);
 }
 
-namespace {
-
 void EliminateLoopExit(Node* node) {
   DCHECK_EQ(IrOpcode::kLoopExit, node->opcode());
   // The exit markers take the loop exit as input. We iterate over uses
@@ -355,6 +346,17 @@ void EliminateLoopExit(Node* node) {
 
 }  // namespace
 
+// static
+void LoopPeeler::PeelInnerLoopsOfTree(Graph* graph,
+                                      CommonOperatorBuilder* common,
+                                      LoopTree* loop_tree, Zone* temp_zone) {
+  for (LoopTree::Loop* loop : loop_tree->outer_loops()) {
+    PeelInnerLoops(graph, common, loop_tree, loop, temp_zone);
+  }
+
+  EliminateLoopExits(graph, temp_zone);
+}
+
 // static
 void LoopPeeler::EliminateLoopExits(Graph* graph, Zone* temp_zone) {
   ZoneQueue<Node*> queue(temp_zone);

src/compiler/loop-peeling.h

@@ -33,7 +33,11 @@ class LoopPeeler {
   static PeeledIteration* Peel(Graph* graph, CommonOperatorBuilder* common,
                                LoopTree* loop_tree, LoopTree::Loop* loop,
                                Zone* tmp_zone);
+  static void PeelInnerLoopsOfTree(Graph* graph, CommonOperatorBuilder* common,
+                                   LoopTree* loop_tree, Zone* tmp_zone);
+
   static void EliminateLoopExits(Graph* graph, Zone* temp_zone);
+  static const size_t kMaxPeeledNodes = 1000;
 };
 
 

src/compiler/pipeline.cc

@@ -942,6 +942,22 @@ struct RepresentationSelectionPhase {
   }
 };
 
+struct LoopPeelingPhase {
+  static const char* phase_name() { return "loop peeling"; }
+
+  void Run(PipelineData* data, Zone* temp_zone) {
+    GraphTrimmer trimmer(temp_zone, data->graph());
+    NodeVector roots(temp_zone);
+    data->jsgraph()->GetCachedNodes(&roots);
+    trimmer.TrimGraph(roots.begin(), roots.end());
+
+    LoopTree* loop_tree =
+        LoopFinder::BuildLoopTree(data->jsgraph()->graph(), temp_zone);
+    LoopPeeler::PeelInnerLoopsOfTree(data->graph(), data->common(), loop_tree,
+                                     temp_zone);
+  }
+};
+
 struct LoopExitEliminationPhase {
   static const char* phase_name() { return "loop exit elimination"; }
 
@@ -1438,9 +1454,13 @@ bool PipelineImpl::CreateGraph() {
     Run<TypedLoweringPhase>();
     RunPrintAndVerify("Lowered typed");
 
-    // Eventually, loop peeling will be done here.
-    Run<LoopExitEliminationPhase>();
-    RunPrintAndVerify("Loop exits eliminated", true);
+    if (FLAG_turbo_loop_peeling) {
+      Run<LoopPeelingPhase>();
+      RunPrintAndVerify("Loops peeled", true);
+    } else {
+      Run<LoopExitEliminationPhase>();
+      RunPrintAndVerify("Loop exits eliminated", true);
+    }
 
     if (FLAG_turbo_stress_loop_peeling) {
       Run<StressLoopPeelingPhase>();

src/flag-definitions.h

@@ -473,6 +473,7 @@ DEFINE_BOOL(turbo_move_optimization, true, "optimize gap moves in TurboFan")
 DEFINE_BOOL(turbo_jt, true, "enable jump threading in TurboFan")
 DEFINE_BOOL(turbo_stress_loop_peeling, false,
             "stress loop peeling optimization")
+DEFINE_BOOL(turbo_loop_peeling, false, "Turbofan loop peeling")
 DEFINE_BOOL(turbo_cf_optimization, true, "optimize control flow in TurboFan")
 DEFINE_BOOL(turbo_frame_elision, true, "elide frames in TurboFan")
 DEFINE_BOOL(turbo_cache_shared_code, true, "cache context-independent code")