[TurboProp] Add reference map population to fast reg alloc.

Adds support for populating reference maps to the fast
register allocator. In order to calculate whether a stack slot
is live at a given instruction, we use the dominator tree to
build a bitmap of blocks which are dominated by each block.
A variable's spill operand is classed as alive for any blocks that are
dominated by the block it was defined in, until the instruction index
of the spill operand's last use. As such, it may be classified as live
down a branch where the spill operand is never used, however it is safe
since the spill slot won't be re-allocated until after it's last-use
instruction index in any case.

BUG=v8:9684

Change-Id: I772374599ef916f57d82d468f66429e32c712ddf
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2298008
Commit-Queue: Ross McIlroy <rmcilroy@chromium.org>
Reviewed-by: Georg Neis <neis@chromium.org>
Cr-Commit-Position: refs/heads/master@{#69108}

src/compiler/backend/instruction.cc

@@ -583,7 +583,8 @@ void PhiInstruction::RenameInput(size_t offset, int virtual_register) {
 
 InstructionBlock::InstructionBlock(Zone* zone, RpoNumber rpo_number,
                                    RpoNumber loop_header, RpoNumber loop_end,
-                                   bool deferred, bool handler)
+                                   RpoNumber dominator, bool deferred,
+                                   bool handler)
     : successors_(zone),
       predecessors_(zone),
       phis_(zone),
@@ -591,6 +592,7 @@ InstructionBlock::InstructionBlock(Zone* zone, RpoNumber rpo_number,
       rpo_number_(rpo_number),
       loop_header_(loop_header),
       loop_end_(loop_end),
+      dominator_(dominator),
       deferred_(deferred),
       handler_(handler) {}
 
@@ -619,7 +621,7 @@ static InstructionBlock* InstructionBlockFor(Zone* zone,
       !block->empty() && block->front()->opcode() == IrOpcode::kIfException;
   InstructionBlock* instr_block = zone->New<InstructionBlock>(
       zone, GetRpo(block), GetRpo(block->loop_header()), GetLoopEndRpo(block),
-      block->deferred(), is_handler);
+      GetRpo(block->dominator()), block->deferred(), is_handler);
   // Map successors and precessors
   instr_block->successors().reserve(block->SuccessorCount());
   for (BasicBlock* successor : block->successors()) {

src/compiler/backend/instruction.h

@@ -1025,7 +1025,7 @@ class RpoNumber final {
   int32_t index_;
 };
 
-std::ostream& operator<<(std::ostream&, const RpoNumber&);
+V8_EXPORT_PRIVATE std::ostream& operator<<(std::ostream&, const RpoNumber&);
 
 class V8_EXPORT_PRIVATE Constant final {
  public:
@@ -1388,7 +1388,8 @@ class V8_EXPORT_PRIVATE InstructionBlock final
     : public NON_EXPORTED_BASE(ZoneObject) {
  public:
   InstructionBlock(Zone* zone, RpoNumber rpo_number, RpoNumber loop_header,
-                   RpoNumber loop_end, bool deferred, bool handler);
+                   RpoNumber loop_end, RpoNumber dominator, bool deferred,
+                   bool handler);
 
   // Instruction indexes (used by the register allocator).
   int first_instruction_index() const {
@@ -1437,6 +1438,9 @@ class V8_EXPORT_PRIVATE InstructionBlock final
   const Successors& successors() const { return successors_; }
   size_t SuccessorCount() const { return successors_.size(); }
 
+  RpoNumber dominator() const { return dominator_; }
+  void set_dominator(RpoNumber dominator) { dominator_ = dominator; }
+
   using PhiInstructions = ZoneVector<PhiInstruction*>;
   const PhiInstructions& phis() const { return phis_; }
   PhiInstruction* PhiAt(size_t i) const { return phis_[i]; }
@@ -1465,6 +1469,7 @@ class V8_EXPORT_PRIVATE InstructionBlock final
   const RpoNumber rpo_number_;
   const RpoNumber loop_header_;
   const RpoNumber loop_end_;
+  RpoNumber dominator_;
   int32_t code_start_;   // start index of arch-specific code.
   int32_t code_end_ = -1;     // end index of arch-specific code.
   const bool deferred_;       // Block contains deferred code.

src/compiler/backend/mid-tier-register-allocator.cc

@@ -24,6 +24,20 @@ namespace compiler {
 
 class RegisterState;
 
+// BlockState stores details associated with a particular basic block.
+class BlockState final {
+ public:
+  BlockState(int block_count, Zone* zone)
+      : dominated_blocks_(block_count, zone) {}
+
+  // Returns a bitvector representing all the basic blocks that are dominated
+  // by this basic block.
+  BitVector* dominated_blocks() { return &dominated_blocks_; }
+
+ private:
+  BitVector dominated_blocks_;
+};
+
 MidTierRegisterAllocationData::MidTierRegisterAllocationData(
     const RegisterConfiguration* config, Zone* zone, Frame* frame,
     InstructionSequence* code, TickCounter* tick_counter,
@@ -35,10 +49,17 @@ MidTierRegisterAllocationData::MidTierRegisterAllocationData(
       debug_name_(debug_name),
       config_(config),
       virtual_register_data_(code->VirtualRegisterCount(), allocation_zone()),
+      block_states_(allocation_zone()),
       reference_map_instructions_(allocation_zone()),
       spilled_virtual_registers_(code->VirtualRegisterCount(),
                                  allocation_zone()),
-      tick_counter_(tick_counter) {}
+      tick_counter_(tick_counter) {
+  int basic_block_count = code->InstructionBlockCount();
+  block_states_.reserve(basic_block_count);
+  for (int i = 0; i < basic_block_count; i++) {
+    block_states_.emplace_back(basic_block_count, allocation_zone());
+  }
+}
 
 MoveOperands* MidTierRegisterAllocationData::AddGapMove(
     int instr_index, Instruction::GapPosition position,
@@ -63,6 +84,10 @@ MachineRepresentation MidTierRegisterAllocationData::RepresentationFor(
   }
 }
 
+BlockState& MidTierRegisterAllocationData::block_state(RpoNumber rpo_number) {
+  return block_states_[rpo_number.ToInt()];
+}
+
 const InstructionBlock* MidTierRegisterAllocationData::GetBlock(
     RpoNumber rpo_number) {
   return code()->InstructionBlockAt(rpo_number);
@@ -73,6 +98,12 @@ const InstructionBlock* MidTierRegisterAllocationData::GetBlock(
   return code()->InstructionAt(instr_index)->block();
 }
 
+const BitVector* MidTierRegisterAllocationData::GetBlocksDominatedBy(
+    int instr_index) {
+  const InstructionBlock* block = GetBlock(instr_index);
+  return block_state(block->rpo_number()).dominated_blocks();
+}
+
 // RegisterIndex represents a particular register of a given kind (depending
 // on the RegisterKind of the allocator).
 class RegisterIndex final {
@@ -204,13 +235,16 @@ class VirtualRegisterData final {
    public:
     // Defines a spill range for an output operand.
     SpillRange(int definition_instr_index, MidTierRegisterAllocationData* data)
-        : live_range_(definition_instr_index, definition_instr_index) {}
+        : live_range_(definition_instr_index, definition_instr_index),
+          live_blocks_(data->GetBlocksDominatedBy(definition_instr_index)) {}
 
     // Defines a spill range for a Phi variable.
     SpillRange(const InstructionBlock* phi_block,
                MidTierRegisterAllocationData* data)
         : live_range_(phi_block->first_instruction_index(),
-                      phi_block->first_instruction_index()) {
+                      phi_block->first_instruction_index()),
+          live_blocks_(data->GetBlocksDominatedBy(
+              phi_block->first_instruction_index())) {
       // For phis, add the gap move instructions in the predecssor blocks to
       // the live range.
       for (RpoNumber pred_rpo : phi_block->predecessors()) {
@@ -220,8 +254,8 @@ class VirtualRegisterData final {
     }
 
     bool IsLiveAt(int instr_index, InstructionBlock* block) {
-      // TODO(rmcilroy): Only include basic blocks dominated by the variable.
-      return live_range_.Contains(instr_index);
+      return live_range_.Contains(instr_index) &&
+             live_blocks_->Contains(block->rpo_number().ToInt());
     }
 
     void ExtendRangeTo(int instr_index) { live_range_.AddInstr(instr_index); }
@@ -230,6 +264,7 @@ class VirtualRegisterData final {
 
    private:
     Range live_range_;
+    const BitVector* live_blocks_;
 
     DISALLOW_COPY_AND_ASSIGN(SpillRange);
   };
@@ -1458,10 +1493,28 @@ void MidTierRegisterAllocator::DefineOutputs() {
        base::Reversed(code()->instruction_blocks())) {
     data_->tick_counter()->DoTick();
 
+    InitializeBlockState(block);
     DefineOutputs(block);
   }
 }
 
+void MidTierRegisterAllocator::InitializeBlockState(
+    const InstructionBlock* block) {
+  // Mark this block as dominating itself.
+  BlockState& block_state = data()->block_state(block->rpo_number());
+  block_state.dominated_blocks()->Add(block->rpo_number().ToInt());
+
+  if (block->dominator().IsValid()) {
+    // Add all the blocks this block dominates to its dominator.
+    BlockState& dominator_block_state = data()->block_state(block->dominator());
+    dominator_block_state.dominated_blocks()->Union(
+        *block_state.dominated_blocks());
+  } else {
+    // Only the first block shouldn't have a dominator.
+    DCHECK_EQ(block, code()->instruction_blocks().front());
+  }
+}
+
 void MidTierRegisterAllocator::DefineOutputs(const InstructionBlock* block) {
   int block_start = block->first_instruction_index();
   for (int index = block->last_instruction_index(); index >= block_start;
@@ -1707,6 +1760,36 @@ void MidTierRegisterAllocator::UpdateSpillRangesForLoops() {
   }
 }
 
+// Spill slot allocator for mid-tier register allocation.
+class MidTierSpillSlotAllocator final {
+ public:
+  explicit MidTierSpillSlotAllocator(MidTierRegisterAllocationData* data);
+
+  void Allocate(VirtualRegisterData* virtual_register);
+
+ private:
+  class SpillSlot;
+
+  void AdvanceTo(int instr_index);
+  SpillSlot* GetFreeSpillSlot(int byte_width);
+
+  MidTierRegisterAllocationData* data() const { return data_; }
+  InstructionSequence* code() const { return data()->code(); }
+  Frame* frame() const { return data()->frame(); }
+  Zone* zone() const { return data()->allocation_zone(); }
+
+  struct OrderByLastUse {
+    bool operator()(const SpillSlot* a, const SpillSlot* b) const;
+  };
+
+  MidTierRegisterAllocationData* data_;
+  ZonePriorityQueue<SpillSlot*, OrderByLastUse> allocated_slots_;
+  ZoneLinkedList<SpillSlot*> free_slots_;
+  int position_;
+
+  DISALLOW_COPY_AND_ASSIGN(MidTierSpillSlotAllocator);
+};
+
 class MidTierSpillSlotAllocator::SpillSlot : public ZoneObject {
  public:
   SpillSlot(int stack_slot, int byte_width)
@@ -1791,12 +1874,12 @@ void MidTierSpillSlotAllocator::Allocate(
   allocated_slots_.push(slot);
 }
 
-void MidTierSpillSlotAllocator::AllocateSpillSlots() {
-  ZoneVector<VirtualRegisterData*> spilled(zone());
-  BitVector::Iterator iterator(&data()->spilled_virtual_registers());
+void AllocateSpillSlots(MidTierRegisterAllocationData* data) {
+  ZoneVector<VirtualRegisterData*> spilled(data->allocation_zone());
+  BitVector::Iterator iterator(&data->spilled_virtual_registers());
   for (; !iterator.Done(); iterator.Advance()) {
     VirtualRegisterData& vreg_data =
-        data()->VirtualRegisterDataFor(iterator.Current());
+        data->VirtualRegisterDataFor(iterator.Current());
     if (vreg_data.HasPendingSpillOperand()) {
       spilled.push_back(&vreg_data);
     }
@@ -1811,8 +1894,59 @@ void MidTierSpillSlotAllocator::AllocateSpillSlots() {
             });
 
   // Allocate a spill slot for each virtual register with a spill range.
+  MidTierSpillSlotAllocator allocator(data);
   for (VirtualRegisterData* spill : spilled) {
-    Allocate(spill);
+    allocator.Allocate(spill);
+  }
+}
+
+// Populates reference maps for mid-tier register allocation.
+class MidTierReferenceMapPopulator final {
+ public:
+  explicit MidTierReferenceMapPopulator(MidTierRegisterAllocationData* data);
+
+  void RecordReferences(const VirtualRegisterData& virtual_register);
+
+ private:
+  MidTierRegisterAllocationData* data() const { return data_; }
+  InstructionSequence* code() const { return data()->code(); }
+
+  MidTierRegisterAllocationData* data_;
+
+  DISALLOW_COPY_AND_ASSIGN(MidTierReferenceMapPopulator);
+};
+
+MidTierReferenceMapPopulator::MidTierReferenceMapPopulator(
+    MidTierRegisterAllocationData* data)
+    : data_(data) {}
+
+void MidTierReferenceMapPopulator::RecordReferences(
+    const VirtualRegisterData& virtual_register) {
+  if (!virtual_register.HasAllocatedSpillOperand()) return;
+  if (!code()->IsReference(virtual_register.vreg())) return;
+
+  VirtualRegisterData::SpillRange* spill_range = virtual_register.spill_range();
+  Range& live_range = spill_range->live_range();
+  AllocatedOperand allocated =
+      *AllocatedOperand::cast(virtual_register.spill_operand());
+  for (int instr_index : data()->reference_map_instructions()) {
+    if (instr_index > live_range.end() || instr_index < live_range.start())
+      continue;
+    Instruction* instr = data()->code()->InstructionAt(instr_index);
+    DCHECK(instr->HasReferenceMap());
+
+    if (spill_range->IsLiveAt(instr_index, instr->block())) {
+      instr->reference_map()->RecordReference(allocated);
+    }
+  }
+}
+
+void PopulateReferenceMaps(MidTierRegisterAllocationData* data) {
+  MidTierReferenceMapPopulator populator(data);
+  BitVector::Iterator iterator(&data->spilled_virtual_registers());
+  for (; !iterator.Done(); iterator.Advance()) {
+    populator.RecordReferences(
+        data->VirtualRegisterDataFor(iterator.Current()));
   }
 }
 

src/compiler/backend/mid-tier-register-allocator.h

@@ -21,6 +21,7 @@ class TickCounter;
 
 namespace compiler {
 
+class BlockState;
 class SinglePassRegisterAllocator;
 class VirtualRegisterData;
 
@@ -56,6 +57,10 @@ class MidTierRegisterAllocationData final : public RegisterAllocationData {
   const InstructionBlock* GetBlock(const RpoNumber rpo_number);
   const InstructionBlock* GetBlock(int instr_index);
 
+  // Returns a bitvector representing all the blocks that are dominated by the
+  // output of the instruction at |instr_index|.
+  const BitVector* GetBlocksDominatedBy(int instr_index);
+
   // List of all instruction indexs that require a reference map.
   ZoneVector<int>& reference_map_instructions() {
     return reference_map_instructions_;
@@ -72,6 +77,8 @@ class MidTierRegisterAllocationData final : public RegisterAllocationData {
   // allocation.
   Zone* code_zone() const { return code()->zone(); }
 
+  BlockState& block_state(RpoNumber rpo_number);
+
   InstructionSequence* code() const { return code_; }
   Frame* frame() const { return frame_; }
   const char* debug_name() const { return debug_name_; }
@@ -86,6 +93,7 @@ class MidTierRegisterAllocationData final : public RegisterAllocationData {
   const RegisterConfiguration* const config_;
 
   ZoneVector<VirtualRegisterData> virtual_register_data_;
+  ZoneVector<BlockState> block_states_;
   ZoneVector<int> reference_map_instructions_;
   BitVector spilled_virtual_registers_;
 
@@ -146,38 +154,11 @@ class MidTierRegisterAllocator final {
   DISALLOW_COPY_AND_ASSIGN(MidTierRegisterAllocator);
 };
 
-// Spill slot allocator for mid-tier register allocation.
-class MidTierSpillSlotAllocator final {
- public:
-  explicit MidTierSpillSlotAllocator(MidTierRegisterAllocationData* data);
-
-  // Phase 3: assign spilled operands to specific spill slots.
-  void AllocateSpillSlots();
-
- private:
-  class SpillSlot;
-
-  void Allocate(VirtualRegisterData* virtual_register);
-
-  void AdvanceTo(int instr_index);
-  SpillSlot* GetFreeSpillSlot(int byte_width);
+// Phase 3: assign spilled operands to specific spill slots.
+void AllocateSpillSlots(MidTierRegisterAllocationData* data);
 
-  MidTierRegisterAllocationData* data() const { return data_; }
-  InstructionSequence* code() const { return data()->code(); }
-  Frame* frame() const { return data()->frame(); }
-  Zone* zone() const { return data()->allocation_zone(); }
-
-  struct OrderByLastUse {
-    bool operator()(const SpillSlot* a, const SpillSlot* b) const;
-  };
-
-  MidTierRegisterAllocationData* data_;
-  ZonePriorityQueue<SpillSlot*, OrderByLastUse> allocated_slots_;
-  ZoneLinkedList<SpillSlot*> free_slots_;
-  int position_;
-
-  DISALLOW_COPY_AND_ASSIGN(MidTierSpillSlotAllocator);
-};
+// Phase 4: Populate reference maps for spilled references.
+void PopulateReferenceMaps(MidTierRegisterAllocationData* data);
 
 }  // namespace compiler
 }  // namespace internal

src/compiler/pipeline.cc

@@ -1957,7 +1957,7 @@ struct ScheduledMachineLoweringPhase {
 
     // TODO(rmcilroy) Avoid having to rebuild rpo_order on schedule each time.
     Scheduler::ComputeSpecialRPO(temp_zone, data->schedule());
-    if (FLAG_turbo_verify) Scheduler::GenerateDominatorTree(data->schedule());
+    Scheduler::GenerateDominatorTree(data->schedule());
     TraceScheduleAndVerify(data->info(), data, data->schedule(),
                            "machine lowered schedule");
   }
@@ -2295,9 +2295,15 @@ struct MidTierSpillSlotAllocatorPhase {
   DECL_PIPELINE_PHASE_CONSTANTS(MidTierSpillSlotAllocator)
 
   void Run(PipelineData* data, Zone* temp_zone) {
-    MidTierSpillSlotAllocator spill_allocator(
-        data->mid_tier_register_allocator_data());
-    spill_allocator.AllocateSpillSlots();
+    AllocateSpillSlots(data->mid_tier_register_allocator_data());
+  }
+};
+
+struct MidTierPopulateReferenceMapsPhase {
+  DECL_PIPELINE_PHASE_CONSTANTS(MidTierPopulateReferenceMaps)
+
+  void Run(PipelineData* data, Zone* temp_zone) {
+    PopulateReferenceMaps(data->mid_tier_register_allocator_data());
   }
 };
 
@@ -3671,7 +3677,7 @@ void PipelineImpl::AllocateRegistersForMidTier(
 
   Run<MidTierSpillSlotAllocatorPhase>();
 
-  // TODO(rmcilroy): Run reference map population phase.
+  Run<MidTierPopulateReferenceMapsPhase>();
 
   TraceSequence(info(), data, "after register allocation");
 

src/utils/bit-vector.h

@@ -280,6 +280,8 @@ class V8_EXPORT_PRIVATE BitVector : public ZoneObject {
   void Print();
 #endif
 
+  MOVE_ONLY_NO_DEFAULT_CONSTRUCTOR(BitVector);
+
  private:
   int length_;
   int data_length_;
@@ -307,8 +309,6 @@ class V8_EXPORT_PRIVATE BitVector : public ZoneObject {
       }
     }
   }
-
-  DISALLOW_COPY_AND_ASSIGN(BitVector);
 };
 
 class GrowableBitVector {

test/cctest/compiler/test-code-generator.cc

@@ -918,6 +918,7 @@ class TestEnvironment : public HandleAndZoneScope {
 
   static InstructionBlock* NewBlock(Zone* zone, RpoNumber rpo) {
     return zone->New<InstructionBlock>(zone, rpo, RpoNumber::Invalid(),
+                                       RpoNumber::Invalid(),
                                        RpoNumber::Invalid(), false, false);
   }
 

test/cctest/compiler/test-instruction-scheduler.cc

@@ -16,7 +16,7 @@ namespace compiler {
 InstructionBlocks* CreateSingleBlock(Zone* zone) {
   InstructionBlock* block = zone->New<InstructionBlock>(
       zone, RpoNumber::FromInt(0), RpoNumber::Invalid(), RpoNumber::Invalid(),
-      false, false);
+      RpoNumber::Invalid(), false, false);
   InstructionBlocks* blocks = zone->NewArray<InstructionBlocks>(1);
   new (blocks) InstructionBlocks(1, block, zone);
   return blocks;

test/cctest/compiler/test-jump-threading.cc

@@ -92,7 +92,7 @@ class TestCode : public HandleAndZoneScope {
     if (current_ == nullptr) {
       current_ = main_zone()->New<InstructionBlock>(
           main_zone(), rpo_number_, RpoNumber::Invalid(), RpoNumber::Invalid(),
-          deferred, false);
+          RpoNumber::Invalid(), deferred, false);
       blocks_.push_back(current_);
       sequence_.StartBlock(rpo_number_);
     }

test/unittests/compiler/backend/instruction-sequence-unittest.cc

@@ -442,7 +442,7 @@ InstructionBlock* InstructionSequenceTest::NewBlock(bool deferred) {
   }
   // Construct instruction block.
   auto instruction_block = zone()->New<InstructionBlock>(
-      zone(), rpo, loop_header, loop_end, deferred, false);
+      zone(), rpo, loop_header, loop_end, Rpo::Invalid(), deferred, false);
   instruction_blocks_.push_back(instruction_block);
   current_block_ = instruction_block;
   sequence()->StartBlock(rpo);
@@ -478,6 +478,7 @@ void InstructionSequenceTest::WireBlocks() {
     }
     ++offset;
   }
+  CalculateDominators();
 }
 
 void InstructionSequenceTest::WireBlock(size_t block_offset, int jump_offset) {
@@ -490,6 +491,42 @@ void InstructionSequenceTest::WireBlock(size_t block_offset, int jump_offset) {
   target->predecessors().push_back(block->rpo_number());
 }
 
+void InstructionSequenceTest::CalculateDominators() {
+  CHECK_GT(instruction_blocks_.size(), 0);
+  ZoneVector<int> dominator_depth(instruction_blocks_.size(), -1, zone());
+
+  CHECK_EQ(instruction_blocks_[0]->rpo_number(), RpoNumber::FromInt(0));
+  dominator_depth[0] = 0;
+  instruction_blocks_[0]->set_dominator(RpoNumber::FromInt(0));
+
+  for (size_t i = 1; i < instruction_blocks_.size(); i++) {
+    InstructionBlock* block = instruction_blocks_[i];
+    auto pred = block->predecessors().begin();
+    auto end = block->predecessors().end();
+    DCHECK(pred != end);  // All blocks except start have predecessors.
+    RpoNumber dominator = *pred;
+    // For multiple predecessors, walk up the dominator tree until a common
+    // dominator is found. Visitation order guarantees that all predecessors
+    // except for backwards edges have been visited.
+    for (++pred; pred != end; ++pred) {
+      // Don't examine backwards edges.
+      if (dominator_depth[pred->ToInt()] < 0) continue;
+
+      RpoNumber other = *pred;
+      while (dominator != other) {
+        if (dominator_depth[dominator.ToInt()] <
+            dominator_depth[other.ToInt()]) {
+          other = instruction_blocks_[other.ToInt()]->dominator();
+        } else {
+          dominator = instruction_blocks_[dominator.ToInt()]->dominator();
+        }
+      }
+    }
+    block->set_dominator(dominator);
+    dominator_depth[i] = dominator_depth[dominator.ToInt()] + 1;
+  }
+}
+
 Instruction* InstructionSequenceTest::Emit(
     InstructionCode code, size_t outputs_size, InstructionOperand* outputs,
     size_t inputs_size, InstructionOperand* inputs, size_t temps_size,

test/unittests/compiler/backend/instruction-sequence-unittest.h

@@ -246,6 +246,7 @@ class InstructionSequenceTest : public TestWithIsolateAndZone {
   InstructionOperand ConvertOutputOp(VReg vreg, TestOperand op);
   InstructionBlock* NewBlock(bool deferred = false);
   void WireBlock(size_t block_offset, int jump_offset);
+  void CalculateDominators();
 
   Instruction* Emit(InstructionCode code, size_t outputs_size = 0,
                     InstructionOperand* outputs = nullptr,