[turbofan] Refactor loop peeling/analysis infrastructure for unrolling

In preparation of loop unrolling, we move some loop analysis
infrastructure out of loop-peeling.{h, cc}, and implement some
additional required functionality.

Changes:
- Implement inner_loops() in loop-analysis.h. Change some parameters
  in other functions from Loop* to (const Loop*) to accommodate this
  change.
- Move Peeling class into loop-analysis, rename it to NodeCopier.
- Simplify NodeCopier::CopyNodes().
- Allow NodeCopier to produce multiple copies of the targeted Nodes.
- Introduce LoopFinder::HasMarkedExits(). Move the implementation of
  LoopPeeling::CanPeel() there. CanPeel() is now an alias for
  HasMarkedExits().

Bug: v8:11298
Change-Id: I245b2e937393e4a78ce4d355e1290aaf6e617114
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2672019
Commit-Queue: Manos Koukoutos <manoskouk@chromium.org>
Reviewed-by: Georg Neis <neis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#72555}

src/compiler/loop-analysis.cc

@@ -542,7 +542,44 @@ LoopTree* LoopFinder::BuildLoopTree(Graph* graph, TickCounter* tick_counter,
   return loop_tree;
 }
 
-Node* LoopTree::HeaderNode(Loop* loop) {
+bool LoopFinder::HasMarkedExits(LoopTree* loop_tree,
+                                const LoopTree::Loop* loop) {
+  // Look for returns and if projections that are outside the loop but whose
+  // control input is inside the loop.
+  Node* loop_node = loop_tree->GetLoopControl(loop);
+  for (Node* node : loop_tree->LoopNodes(loop)) {
+    for (Node* use : node->uses()) {
+      if (!loop_tree->Contains(loop, use)) {
+        bool unmarked_exit;
+        switch (node->opcode()) {
+          case IrOpcode::kLoopExit:
+            unmarked_exit = (node->InputAt(1) != loop_node);
+            break;
+          case IrOpcode::kLoopExitValue:
+          case IrOpcode::kLoopExitEffect:
+            unmarked_exit = (node->InputAt(1)->InputAt(1) != loop_node);
+            break;
+          default:
+            unmarked_exit = (use->opcode() != IrOpcode::kTerminate);
+        }
+        if (unmarked_exit) {
+          if (FLAG_trace_turbo_loop) {
+            Node* loop_node = loop_tree->GetLoopControl(loop);
+            PrintF(
+                "Cannot peel loop %i. Loop exit without explicit mark: Node %i "
+                "(%s) is inside loop, but its use %i (%s) is outside.\n",
+                loop_node->id(), node->id(), node->op()->mnemonic(), use->id(),
+                use->op()->mnemonic());
+          }
+          return false;
+        }
+      }
+    }
+  }
+  return true;
+}
+
+Node* LoopTree::HeaderNode(const Loop* loop) {
   Node* first = *HeaderNodes(loop).begin();
   if (first->opcode() == IrOpcode::kLoop) return first;
   DCHECK(IrOpcode::IsPhiOpcode(first->opcode()));
@@ -551,6 +588,54 @@ Node* LoopTree::HeaderNode(Loop* loop) {
   return header;
 }
 
+Node* NodeCopier::map(Node* node, uint32_t copy_index) {
+  DCHECK_LT(copy_index, copy_count_);
+  if (node_map_.Get(node) == 0) return node;
+  return copies_->at(node_map_.Get(node) + copy_index);
+}
+
+void NodeCopier::Insert(Node* original, const NodeVector& new_copies) {
+  DCHECK_EQ(new_copies.size(), copy_count_);
+  node_map_.Set(original, copies_->size() + 1);
+  copies_->push_back(original);
+  copies_->insert(copies_->end(), new_copies.begin(), new_copies.end());
+}
+
+void NodeCopier::Insert(Node* original, Node* copy) {
+  DCHECK_EQ(copy_count_, 1);
+  node_map_.Set(original, copies_->size() + 1);
+  copies_->push_back(original);
+  copies_->push_back(copy);
+}
+
+void NodeCopier::CopyNodes(Graph* graph, Zone* tmp_zone_, Node* dead,
+                           NodeRange nodes,
+                           SourcePositionTable* source_positions,
+                           NodeOriginTable* node_origins) {
+  // Copy all the nodes first.
+  for (Node* original : nodes) {
+    SourcePositionTable::Scope position(
+        source_positions, source_positions->GetSourcePosition(original));
+    NodeOriginTable::Scope origin_scope(node_origins, "copy nodes", original);
+    node_map_.Set(original, copies_->size() + 1);
+    copies_->push_back(original);
+    for (uint32_t copy_index = 0; copy_index < copy_count_; copy_index++) {
+      Node* copy = graph->CloneNode(original);
+      copies_->push_back(copy);
+    }
+  }
+
+  // Fix inputs of the copies.
+  for (Node* original : nodes) {
+    for (uint32_t copy_index = 0; copy_index < copy_count_; copy_index++) {
+      Node* copy = map(original, copy_index);
+      for (int i = 0; i < copy->InputCount(); i++) {
+        copy->ReplaceInput(i, map(original->InputAt(i), copy_index));
+      }
+    }
+  }
+}
+
 }  // namespace compiler
 }  // namespace internal
 }  // namespace v8

src/compiler/loop-analysis.h

@@ -7,7 +7,11 @@
 
 #include "src/base/iterator.h"
 #include "src/common/globals.h"
+#include "src/compiler/compiler-source-position-table.h"
 #include "src/compiler/graph.h"
+#include "src/compiler/node-marker.h"
+#include "src/compiler/node-origin-table.h"
+#include "src/compiler/node-properties.h"
 #include "src/compiler/node.h"
 #include "src/zone/zone-containers.h"
 
@@ -41,11 +45,11 @@ class LoopTree : public ZoneObject {
    public:
     Loop* parent() const { return parent_; }
     const ZoneVector<Loop*>& children() const { return children_; }
-    size_t HeaderSize() const { return body_start_ - header_start_; }
-    size_t BodySize() const { return exits_start_ - body_start_; }
-    size_t ExitsSize() const { return exits_end_ - exits_start_; }
-    size_t TotalSize() const { return exits_end_ - header_start_; }
-    size_t depth() const { return static_cast<size_t>(depth_); }
+    uint32_t HeaderSize() const { return body_start_ - header_start_; }
+    uint32_t BodySize() const { return exits_start_ - body_start_; }
+    uint32_t ExitsSize() const { return exits_end_ - exits_start_; }
+    uint32_t TotalSize() const { return exits_end_ - header_start_; }
+    uint32_t depth() const { return depth_; }
 
    private:
     friend class LoopTree;
@@ -77,7 +81,7 @@ class LoopTree : public ZoneObject {
 
   // Check if the {loop} contains the {node}, either directly or by containing
   // a nested loop that contains {node}.
-  bool Contains(Loop* loop, Node* node) {
+  bool Contains(const Loop* loop, Node* node) {
     for (Loop* c = ContainingLoop(node); c != nullptr; c = c->parent_) {
       if (c == loop) return true;
     }
@@ -87,40 +91,51 @@ class LoopTree : public ZoneObject {
   // Return the list of outer loops.
   const ZoneVector<Loop*>& outer_loops() const { return outer_loops_; }
 
+  // Return a new vector containing the inner loops.
+  ZoneVector<const Loop*> inner_loops() const {
+    ZoneVector<const Loop*> inner_loops(zone_);
+    for (const Loop& loop : all_loops_) {
+      if (loop.children().empty()) {
+        inner_loops.push_back(&loop);
+      }
+    }
+    return inner_loops;
+  }
+
   // Return the unique loop number for a given loop. Loop numbers start at {1}.
-  int LoopNum(Loop* loop) const {
+  int LoopNum(const Loop* loop) const {
     return 1 + static_cast<int>(loop - &all_loops_[0]);
   }
 
   // Return a range which can iterate over the header nodes of {loop}.
-  NodeRange HeaderNodes(Loop* loop) {
+  NodeRange HeaderNodes(const Loop* loop) {
     return NodeRange(&loop_nodes_[0] + loop->header_start_,
                      &loop_nodes_[0] + loop->body_start_);
   }
 
   // Return the header control node for a loop.
-  Node* HeaderNode(Loop* loop);
+  Node* HeaderNode(const Loop* loop);
 
   // Return a range which can iterate over the body nodes of {loop}.
-  NodeRange BodyNodes(Loop* loop) {
+  NodeRange BodyNodes(const Loop* loop) {
     return NodeRange(&loop_nodes_[0] + loop->body_start_,
                      &loop_nodes_[0] + loop->exits_start_);
   }
 
   // Return a range which can iterate over the body nodes of {loop}.
-  NodeRange ExitNodes(Loop* loop) {
+  NodeRange ExitNodes(const Loop* loop) {
     return NodeRange(&loop_nodes_[0] + loop->exits_start_,
                      &loop_nodes_[0] + loop->exits_end_);
   }
 
   // Return a range which can iterate over the nodes of {loop}.
-  NodeRange LoopNodes(Loop* loop) {
+  NodeRange LoopNodes(const Loop* loop) {
     return NodeRange(&loop_nodes_[0] + loop->header_start_,
                      &loop_nodes_[0] + loop->exits_end_);
   }
 
   // Return the node that represents the control, i.e. the loop node itself.
-  Node* GetLoopControl(Loop* loop) {
+  Node* GetLoopControl(const Loop* loop) {
     // TODO(turbofan): make the loop control node always first?
     for (Node* node : HeaderNodes(loop)) {
       if (node->opcode() == IrOpcode::kLoop) return node;
@@ -161,8 +176,49 @@ class V8_EXPORT_PRIVATE LoopFinder {
   // Build a loop tree for the entire graph.
   static LoopTree* BuildLoopTree(Graph* graph, TickCounter* tick_counter,
                                  Zone* temp_zone);
+
+  static bool HasMarkedExits(LoopTree* loop_tree_, const LoopTree::Loop* loop);
 };
 
+// Copies a range of nodes any number of times.
+class NodeCopier {
+ public:
+  // {max}: The maximum number of nodes that this copier will track, including
+  //        The original nodes and all copies.
+  // {p}: A vector that holds the original nodes and all copies.
+  // {copy_count}: How many times the nodes should be copied.
+  NodeCopier(Graph* graph, uint32_t max, NodeVector* p, uint32_t copy_count)
+      : node_map_(graph, max), copies_(p), copy_count_(copy_count) {
+    DCHECK_GT(copy_count, 0);
+  }
+
+  // Returns the mapping of {node} in the {copy_index}'th copy, or {node} itself
+  // if it is not present in the mapping. The copies are 0-indexed.
+  Node* map(Node* node, uint32_t copy_index);
+
+  // Helper version of {map} for one copy.
+  V8_INLINE Node* map(Node* node) { return map(node, 0); }
+
+  // Insert a new mapping from {original} to {new_copies} into the copier.
+  void Insert(Node* original, const NodeVector& new_copies);
+
+  // Helper version of {Insert} for one copy.
+  void Insert(Node* original, Node* copy);
+
+  void CopyNodes(Graph* graph, Zone* tmp_zone_, Node* dead, NodeRange nodes,
+                 SourcePositionTable* source_positions,
+                 NodeOriginTable* node_origins);
+
+  bool Marked(Node* node) { return node_map_.Get(node) > 0; }
+
+ private:
+  // Maps a node to its index in the {copies_} vector.
+  NodeMarker<size_t> node_map_;
+  // The vector which contains the mapped nodes.
+  NodeVector* copies_;
+  // How many copies of the nodes should be generated.
+  const uint32_t copy_count_;
+};
 
 }  // namespace compiler
 }  // namespace internal

src/compiler/loop-peeling.cc

@@ -3,9 +3,11 @@
 // found in the LICENSE file.
 
 #include "src/compiler/loop-peeling.h"
+
 #include "src/compiler/common-operator.h"
 #include "src/compiler/compiler-source-position-table.h"
 #include "src/compiler/graph.h"
+#include "src/compiler/loop-analysis.h"
 #include "src/compiler/node-marker.h"
 #include "src/compiler/node-origin-table.h"
 #include "src/compiler/node-properties.h"
@@ -103,59 +105,6 @@ namespace v8 {
 namespace internal {
 namespace compiler {
 
-struct Peeling {
-  // Maps a node to its index in the {pairs} vector.
-  NodeMarker<size_t> node_map;
-  // The vector which contains the mapped nodes.
-  NodeVector* pairs;
-
-  Peeling(Graph* graph, size_t max, NodeVector* p)
-      : node_map(graph, static_cast<uint32_t>(max)), pairs(p) {}
-
-  Node* map(Node* node) {
-    if (node_map.Get(node) == 0) return node;
-    return pairs->at(node_map.Get(node));
-  }
-
-  void Insert(Node* original, Node* copy) {
-    node_map.Set(original, 1 + pairs->size());
-    pairs->push_back(original);
-    pairs->push_back(copy);
-  }
-
-  void CopyNodes(Graph* graph, Zone* tmp_zone_, Node* dead, NodeRange nodes,
-                 SourcePositionTable* source_positions,
-                 NodeOriginTable* node_origins) {
-    NodeVector inputs(tmp_zone_);
-    // Copy all the nodes first.
-    for (Node* node : nodes) {
-      SourcePositionTable::Scope position(
-          source_positions, source_positions->GetSourcePosition(node));
-      NodeOriginTable::Scope origin_scope(node_origins, "copy nodes", node);
-      inputs.clear();
-      for (Node* input : node->inputs()) {
-        inputs.push_back(map(input));
-      }
-      Node* copy = graph->NewNode(node->op(), node->InputCount(), &inputs[0]);
-      if (NodeProperties::IsTyped(node)) {
-        NodeProperties::SetType(copy, NodeProperties::GetType(node));
-      }
-      Insert(node, copy);
-    }
-
-    // Fix remaining inputs of the copies.
-    for (Node* original : nodes) {
-      Node* copy = pairs->at(node_map.Get(original));
-      for (int i = 0; i < copy->InputCount(); i++) {
-        copy->ReplaceInput(i, map(original->InputAt(i)));
-      }
-    }
-  }
-
-  bool Marked(Node* node) { return node_map.Get(node) > 0; }
-};
-
-
 class PeeledIterationImpl : public PeeledIteration {
  public:
   NodeVector node_pairs_;
@@ -173,43 +122,6 @@ Node* PeeledIteration::map(Node* node) {
   return node;
 }
 
-bool LoopPeeler::CanPeel(LoopTree::Loop* loop) {
-  // Look for returns and if projections that are outside the loop but whose
-  // control input is inside the loop.
-  Node* loop_node = loop_tree_->GetLoopControl(loop);
-  for (Node* node : loop_tree_->LoopNodes(loop)) {
-    for (Node* use : node->uses()) {
-      if (!loop_tree_->Contains(loop, use)) {
-        bool unmarked_exit;
-        switch (node->opcode()) {
-          case IrOpcode::kLoopExit:
-            unmarked_exit = (node->InputAt(1) != loop_node);
-            break;
-          case IrOpcode::kLoopExitValue:
-          case IrOpcode::kLoopExitEffect:
-            unmarked_exit = (node->InputAt(1)->InputAt(1) != loop_node);
-            break;
-          default:
-            unmarked_exit = (use->opcode() != IrOpcode::kTerminate);
-        }
-        if (unmarked_exit) {
-          if (FLAG_trace_turbo_loop) {
-            Node* loop_node = loop_tree_->GetLoopControl(loop);
-            PrintF(
-                "Cannot peel loop %i. Loop exit without explicit mark: Node %i "
-                "(%s) is inside "
-                "loop, but its use %i (%s) is outside.\n",
-                loop_node->id(), node->id(), node->op()->mnemonic(), use->id(),
-                use->op()->mnemonic());
-          }
-          return false;
-        }
-      }
-    }
-  }
-  return true;
-}
-
 PeeledIteration* LoopPeeler::Peel(LoopTree::Loop* loop) {
   if (!CanPeel(loop)) return nullptr;
 
@@ -217,19 +129,19 @@ PeeledIteration* LoopPeeler::Peel(LoopTree::Loop* loop) {
   // Construct the peeled iteration.
   //============================================================================
   PeeledIterationImpl* iter = tmp_zone_->New<PeeledIterationImpl>(tmp_zone_);
-  size_t estimated_peeled_size = 5 + (loop->TotalSize()) * 2;
-  Peeling peeling(graph_, estimated_peeled_size, &iter->node_pairs_);
+  uint32_t estimated_peeled_size = 5 + loop->TotalSize() * 2;
+  NodeCopier copier(graph_, estimated_peeled_size, &iter->node_pairs_, 1);
 
   Node* dead = graph_->NewNode(common_->Dead());
 
   // Map the loop header nodes to their entry values.
   for (Node* node : loop_tree_->HeaderNodes(loop)) {
-    peeling.Insert(node, node->InputAt(kAssumedLoopEntryIndex));
+    copier.Insert(node, node->InputAt(kAssumedLoopEntryIndex));
   }
 
   // Copy all the nodes of loop body for the peeled iteration.
-  peeling.CopyNodes(graph_, tmp_zone_, dead, loop_tree_->BodyNodes(loop),
-                    source_positions_, node_origins_);
+  copier.CopyNodes(graph_, tmp_zone_, dead, loop_tree_->BodyNodes(loop),
+                   source_positions_, node_origins_);
 
   //============================================================================
   // Replace the entry to the loop with the output of the peeled iteration.
@@ -242,7 +154,7 @@ PeeledIteration* LoopPeeler::Peel(LoopTree::Loop* loop) {
     // from the peeled iteration.
     NodeVector inputs(tmp_zone_);
     for (int i = 1; i < loop_node->InputCount(); i++) {
-      inputs.push_back(peeling.map(loop_node->InputAt(i)));
+      inputs.push_back(copier.map(loop_node->InputAt(i)));
     }
     Node* merge =
         graph_->NewNode(common_->Merge(backedges), backedges, &inputs[0]);
@@ -252,7 +164,7 @@ PeeledIteration* LoopPeeler::Peel(LoopTree::Loop* loop) {
       if (node->opcode() == IrOpcode::kLoop) continue;  // already done.
       inputs.clear();
       for (int i = 0; i < backedges; i++) {
-        inputs.push_back(peeling.map(node->InputAt(1 + i)));
+        inputs.push_back(copier.map(node->InputAt(1 + i)));
       }
       for (Node* input : inputs) {
         if (input != inputs[0]) {  // Non-redundant phi.
@@ -269,9 +181,9 @@ PeeledIteration* LoopPeeler::Peel(LoopTree::Loop* loop) {
     // Only one backedge, simply replace the input to loop with output of
     // peeling.
     for (Node* node : loop_tree_->HeaderNodes(loop)) {
-      node->ReplaceInput(0, peeling.map(node->InputAt(1)));
+      node->ReplaceInput(0, copier.map(node->InputAt(1)));
     }
-    new_entry = peeling.map(loop_node->InputAt(1));
+    new_entry = copier.map(loop_node->InputAt(1));
   }
   loop_node->ReplaceInput(0, new_entry);
 
@@ -282,18 +194,18 @@ PeeledIteration* LoopPeeler::Peel(LoopTree::Loop* loop) {
     switch (exit->opcode()) {
       case IrOpcode::kLoopExit:
         // Change the loop exit node to a merge node.
-        exit->ReplaceInput(1, peeling.map(exit->InputAt(0)));
+        exit->ReplaceInput(1, copier.map(exit->InputAt(0)));
         NodeProperties::ChangeOp(exit, common_->Merge(2));
         break;
       case IrOpcode::kLoopExitValue:
         // Change exit marker to phi.
-        exit->InsertInput(graph_->zone(), 1, peeling.map(exit->InputAt(0)));
+        exit->InsertInput(graph_->zone(), 1, copier.map(exit->InputAt(0)));
         NodeProperties::ChangeOp(
             exit, common_->Phi(LoopExitValueRepresentationOf(exit->op()), 2));
         break;
       case IrOpcode::kLoopExitEffect:
         // Change effect exit marker to effect phi.
-        exit->InsertInput(graph_->zone(), 1, peeling.map(exit->InputAt(0)));
+        exit->InsertInput(graph_->zone(), 1, copier.map(exit->InputAt(0)));
         NodeProperties::ChangeOp(exit, common_->EffectPhi(2));
         break;
       default:

src/compiler/loop-peeling.h

@@ -43,7 +43,9 @@ class V8_EXPORT_PRIVATE LoopPeeler {
         tmp_zone_(tmp_zone),
         source_positions_(source_positions),
         node_origins_(node_origins) {}
-  bool CanPeel(LoopTree::Loop* loop);
+  bool CanPeel(LoopTree::Loop* loop) {
+    return LoopFinder::HasMarkedExits(loop_tree_, loop);
+  }
   PeeledIteration* Peel(LoopTree::Loop* loop);
   void PeelInnerLoopsOfTree();