[compiler] Share liveness across straight-line bytecode

Straight-line bytecode with exactly one "next" bytecode (i.e. everything
that can't affect control flow) will always have the same "out" liveness
as the next bytecode's "in" liveness. For those cases, we can save a bit
of time and memory by aliasing the pointers between the bytecode's out
liveness and the next bytecode's in liveness, and skipping copying
between them.

This is done by specializing the current liveness update on whether this
is the first pass (which will allocate and initialize the liveness
bitvectors) or an update pass (which will revisit loops to collect
liveness crossing over the back-edge, and propagate this liveness
through the loop bodies). On the first pass, we can delay allocation of
the out liveness until we know it needs to be union of multiple in
livenesses, and on the update pass we can skip it if it is an alias.

As a drive-by, tweak BitVector::CopyFrom to require copying from a
vector with the same size (same as Union or Intersect), and move the
only different sized vector use (in Resize) to be inline.

Change-Id: Iad1b2e1b927a37ad925ef68e2a224152aaa2ba18
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3350452
Auto-Submit: Leszek Swirski <leszeks@chromium.org>
Reviewed-by: Maya Lekova <mslekova@chromium.org>
Commit-Queue: Maya Lekova <mslekova@chromium.org>
Cr-Commit-Position: refs/heads/main@{#78425}

src/compiler/bytecode-analysis.cc

@@ -4,6 +4,7 @@
 
 #include "src/compiler/bytecode-analysis.h"
 
+#include "src/compiler/bytecode-liveness-map.h"
 #include "src/interpreter/bytecode-array-iterator.h"
 #include "src/interpreter/bytecode-array-random-iterator.h"
 #include "src/objects/objects-inl.h"
@@ -210,57 +211,105 @@ void UpdateInLiveness(Bytecode bytecode, BytecodeLivenessState* in_liveness,
   }
 }
 
-void UpdateOutLiveness(Bytecode bytecode, BytecodeLivenessState* out_liveness,
+template <bool IsFirstUpdate = false>
+void EnsureOutLivenessIsNotAlias(
+    BytecodeLiveness& liveness,
+    BytecodeLivenessState* next_bytecode_in_liveness, Zone* zone) {
+  if (!IsFirstUpdate) {
+    // We should have copied the next bytecode's in liveness already in the
+    // first pass, so on subsequent passes this should already not be an alias.
+    DCHECK_NE(liveness.out, next_bytecode_in_liveness);
+    return;
+  }
+  if (liveness.out == next_bytecode_in_liveness) {
+    // If the out-liveness is aliasing the next bytecode's in-liveness,
+    // reallocate it and copy the data to the newly allocated state.
+    liveness.out =
+        zone->New<BytecodeLivenessState>(*next_bytecode_in_liveness, zone);
+  }
+}
+
+template <bool IsFirstUpdate = false>
+void UpdateOutLiveness(Bytecode bytecode, BytecodeLiveness& liveness,
                        BytecodeLivenessState* next_bytecode_in_liveness,
                        const interpreter::BytecodeArrayIterator& iterator,
                        Handle<BytecodeArray> bytecode_array,
-                       const BytecodeLivenessMap& liveness_map) {
-  int current_offset = iterator.current_offset();
+                       const BytecodeLivenessMap& liveness_map, Zone* zone) {
+  // On subsequent updates, only update out-liveness manually if it isn't
+  // already aliasing the next bytecode's in-liveness.
+  if (!IsFirstUpdate && liveness.out == next_bytecode_in_liveness) return;
 
   // Special case Suspend and Resume to just pass through liveness.
   if (bytecode == Bytecode::kSuspendGenerator ||
       bytecode == Bytecode::kResumeGenerator) {
-    out_liveness->Union(*next_bytecode_in_liveness);
+    DCHECK_NOT_NULL(next_bytecode_in_liveness);
+    if (IsFirstUpdate) {
+      liveness.out = next_bytecode_in_liveness;
+    } else {
+      liveness.out->Union(*next_bytecode_in_liveness);
+    }
     return;
   }
 
+  // Update from next bytecode (unless there isn't one or this is an
+  // unconditional jump).
+  if (next_bytecode_in_liveness != nullptr &&
+      !Bytecodes::IsUnconditionalJump(bytecode)) {
+    if (IsFirstUpdate) {
+      // On first update, we can assume that this out-liveness is the same as
+      // the next liveness, and can directly alias it -- we'll allocate a new
+      // one using EnsureOutLivenessIsNotAlias if it needs to be mutated.
+      DCHECK_NULL(liveness.out);
+      liveness.out = next_bytecode_in_liveness;
+    } else {
+      liveness.out->Union(*next_bytecode_in_liveness);
+    }
+  } else if (IsFirstUpdate) {
+    // Otherwise, on the first allocation we need to make sure that there is an
+    // allocated out liveness.
+    DCHECK_NULL(liveness.out);
+    liveness.out = zone->New<BytecodeLivenessState>(
+        bytecode_array->register_count(), zone);
+  }
+
+  DCHECK_NOT_NULL(liveness.out);
+
   // Update from jump target (if any). Skip loops, we update these manually in
   // the liveness iterations.
   if (Bytecodes::IsForwardJump(bytecode)) {
     int target_offset = iterator.GetJumpTargetOffset();
-    out_liveness->Union(*liveness_map.GetInLiveness(target_offset));
+    EnsureOutLivenessIsNotAlias<IsFirstUpdate>(liveness,
+                                               next_bytecode_in_liveness, zone);
+    liveness.out->Union(*liveness_map.GetInLiveness(target_offset));
   } else if (Bytecodes::IsSwitch(bytecode)) {
+    EnsureOutLivenessIsNotAlias<IsFirstUpdate>(liveness,
+                                               next_bytecode_in_liveness, zone);
     for (interpreter::JumpTableTargetOffset entry :
          iterator.GetJumpTableTargetOffsets()) {
-      out_liveness->Union(*liveness_map.GetInLiveness(entry.target_offset));
+      liveness.out->Union(*liveness_map.GetInLiveness(entry.target_offset));
     }
   }
 
-  // Update from next bytecode (unless there isn't one or this is an
-  // unconditional jump).
-  if (next_bytecode_in_liveness != nullptr &&
-      !Bytecodes::IsUnconditionalJump(bytecode)) {
-    out_liveness->Union(*next_bytecode_in_liveness);
-  }
-
   // Update from exception handler (if any).
   if (!interpreter::Bytecodes::IsWithoutExternalSideEffects(bytecode)) {
     int handler_context;
     // TODO(leszeks): We should look up this range only once per entry.
     HandlerTable table(*bytecode_array);
     int handler_offset =
-        table.LookupRange(current_offset, &handler_context, nullptr);
+        table.LookupRange(iterator.current_offset(), &handler_context, nullptr);
 
     if (handler_offset != -1) {
-      bool was_accumulator_live = out_liveness->AccumulatorIsLive();
-      out_liveness->Union(*liveness_map.GetInLiveness(handler_offset));
-      out_liveness->MarkRegisterLive(handler_context);
+      EnsureOutLivenessIsNotAlias<IsFirstUpdate>(
+          liveness, next_bytecode_in_liveness, zone);
+      bool was_accumulator_live = liveness.out->AccumulatorIsLive();
+      liveness.out->Union(*liveness_map.GetInLiveness(handler_offset));
+      liveness.out->MarkRegisterLive(handler_context);
       if (!was_accumulator_live) {
         // The accumulator is reset to the exception on entry into a handler,
         // and so shouldn't be considered live coming out of this bytecode just
         // because it's live coming into the handler. So, kill the accumulator
         // if the handler is the only thing that made it live.
-        out_liveness->MarkAccumulatorDead();
+        liveness.out->MarkAccumulatorDead();
 
         // TODO(leszeks): Ideally the accumulator wouldn't be considered live at
         // the start of the handler, but looking up if the current bytecode is
@@ -271,14 +320,26 @@ void UpdateOutLiveness(Bytecode bytecode, BytecodeLivenessState* out_liveness,
   }
 }
 
-void UpdateLiveness(Bytecode bytecode, BytecodeLiveness const& liveness,
+template <bool IsFirstUpdate = false>
+void UpdateLiveness(Bytecode bytecode, BytecodeLiveness& liveness,
                     BytecodeLivenessState** next_bytecode_in_liveness,
                     const interpreter::BytecodeArrayIterator& iterator,
                     Handle<BytecodeArray> bytecode_array,
-                    const BytecodeLivenessMap& liveness_map) {
-  UpdateOutLiveness(bytecode, liveness.out, *next_bytecode_in_liveness,
-                    iterator, bytecode_array, liveness_map);
+                    const BytecodeLivenessMap& liveness_map, Zone* zone) {
+  UpdateOutLiveness<IsFirstUpdate>(bytecode, liveness,
+                                   *next_bytecode_in_liveness, iterator,
+                                   bytecode_array, liveness_map, zone);
+  if (IsFirstUpdate) {
+    // On the first update, allocate the in-liveness as a copy of the
+    // out-liveness.
+    DCHECK_NULL(liveness.in);
+    liveness.in = zone->New<BytecodeLivenessState>(*liveness.out, zone);
+  } else {
+    // On subsequent updates, copy liveness from the out vector.
+    // TODO(leszeks): If this copy doesn't change liveness, we could
+    // opportunistically terminate early.
     liveness.in->CopyFrom(*liveness.out);
+  }
   UpdateInLiveness(bytecode, liveness.in, iterator);
 
   *next_bytecode_in_liveness = liveness.in;
@@ -443,10 +504,10 @@ void BytecodeAnalysis::Analyze() {
     }
 
     if (analyze_liveness_) {
-      BytecodeLiveness const& liveness = liveness_map().InitializeLiveness(
-          current_offset, bytecode_array()->register_count(), zone());
-      UpdateLiveness(bytecode, liveness, &next_bytecode_in_liveness, iterator,
-                     bytecode_array(), liveness_map());
+      BytecodeLiveness& liveness =
+          liveness_map().InsertNewLiveness(current_offset);
+      UpdateLiveness<true>(bytecode, liveness, &next_bytecode_in_liveness,
+                           iterator, bytecode_array(), liveness_map(), zone());
     }
   }
 
@@ -506,16 +567,15 @@ void BytecodeAnalysis::Analyze() {
     for (; iterator.current_offset() > header_offset; --iterator) {
       Bytecode bytecode = iterator.current_bytecode();
       int current_offset = iterator.current_offset();
-      BytecodeLiveness const& liveness =
-          liveness_map().GetLiveness(current_offset);
+      BytecodeLiveness& liveness = liveness_map().GetLiveness(current_offset);
       UpdateLiveness(bytecode, liveness, &next_bytecode_in_liveness, iterator,
-                     bytecode_array(), liveness_map());
+                     bytecode_array(), liveness_map(), zone());
     }
     // Now we are at the loop header. Since the in-liveness of the header
     // can't change, we need only to update the out-liveness.
-    UpdateOutLiveness(iterator.current_bytecode(), header_liveness.out,
+    UpdateOutLiveness(iterator.current_bytecode(), header_liveness,
                       next_bytecode_in_liveness, iterator, bytecode_array(),
-                      liveness_map());
+                      liveness_map(), zone());
   }
 
   // Process the generator switch statement separately, once the loops are done.
@@ -548,14 +608,14 @@ void BytecodeAnalysis::Analyze() {
       next_bytecode_in_liveness = switch_liveness.in;
       for (--iterator; iterator.IsValid(); --iterator) {
         Bytecode bytecode = iterator.current_bytecode();
-        BytecodeLiveness const& liveness =
+        BytecodeLiveness& liveness =
             liveness_map().GetLiveness(iterator.current_offset());
 
         // There shouldn't be any more loops.
         DCHECK_NE(bytecode, Bytecode::kJumpLoop);
 
         UpdateLiveness(bytecode, liveness, &next_bytecode_in_liveness, iterator,
-                       bytecode_array(), liveness_map());
+                       bytecode_array(), liveness_map(), zone());
       }
     }
   }
@@ -827,6 +887,8 @@ bool BytecodeAnalysis::LivenessIsValid() {
 
   int invalid_offset = -1;
   int which_invalid = -1;
+  BytecodeLivenessState invalid_liveness(bytecode_array()->register_count(),
+                                         zone());
 
   BytecodeLivenessState* next_bytecode_in_liveness = nullptr;
 
@@ -840,8 +902,8 @@ bool BytecodeAnalysis::LivenessIsValid() {
 
     previous_liveness.CopyFrom(*liveness.out);
 
-    UpdateOutLiveness(bytecode, liveness.out, next_bytecode_in_liveness,
-                      iterator, bytecode_array(), liveness_map());
+    UpdateOutLiveness(bytecode, liveness, next_bytecode_in_liveness, iterator,
+                      bytecode_array(), liveness_map(), zone());
     // UpdateOutLiveness skips kJumpLoop, so we update it manually.
     if (bytecode == Bytecode::kJumpLoop) {
       int target_offset = iterator.GetJumpTargetOffset();
@@ -849,6 +911,7 @@ bool BytecodeAnalysis::LivenessIsValid() {
     }
 
     if (!liveness.out->Equals(previous_liveness)) {
+      invalid_liveness.CopyFrom(*liveness.out);
       // Reset the invalid liveness.
       liveness.out->CopyFrom(previous_liveness);
       invalid_offset = current_offset;
@@ -862,6 +925,7 @@ bool BytecodeAnalysis::LivenessIsValid() {
     UpdateInLiveness(bytecode, liveness.in, iterator);
 
     if (!liveness.in->Equals(previous_liveness)) {
+      invalid_liveness.CopyFrom(*liveness.in);
       // Reset the invalid liveness.
       liveness.in->CopyFrom(previous_liveness);
       invalid_offset = current_offset;
@@ -975,6 +1039,31 @@ bool BytecodeAnalysis::LivenessIsValid() {
         }
 
         of << std::endl;
+
+        // Print the invalid liveness.
+        if (which_invalid == 0) {
+          for (int i = 0; i < in_liveness.length(); ++i) {
+            of << (invalid_liveness.bit_vector().Contains(i) ? 'L' : '.');
+          }
+          for (int i = 0; i < out_liveness.length() + 3; ++i) {
+            of << ' ';
+          }
+        } else {
+          for (int i = 0; i < in_liveness.length() + 3; ++i) {
+            of << ' ';
+          }
+          for (int i = 0; i < out_liveness.length(); ++i) {
+            of << (invalid_liveness.bit_vector().Contains(i) ? 'L' : '.');
+          }
+        }
+
+        // Make sure to draw the loop indentation marks on this additional line.
+        of << " : " << current_offset << " : ";
+        for (int i = 0; i < loop_indent; ++i) {
+          of << "| ";
+        }
+
+        of << std::endl;
       }
     }
   }

src/compiler/bytecode-liveness-map.cc

@@ -8,10 +8,6 @@ namespace v8 {
 namespace internal {
 namespace compiler {
 
-BytecodeLiveness::BytecodeLiveness(int register_count, Zone* zone)
-    : in(zone->New<BytecodeLivenessState>(register_count, zone)),
-      out(zone->New<BytecodeLivenessState>(register_count, zone)) {}
-
 BytecodeLivenessMap::BytecodeLivenessMap(int bytecode_size, Zone* zone)
     : liveness_map_(base::bits::RoundUpToPowerOfTwo32(bytecode_size / 4 + 1),
                     base::KeyEqualityMatcher<int>(),
@@ -19,13 +15,8 @@ BytecodeLivenessMap::BytecodeLivenessMap(int bytecode_size, Zone* zone)
 
 uint32_t OffsetHash(int offset) { return offset; }
 
-BytecodeLiveness& BytecodeLivenessMap::InitializeLiveness(int offset,
-                                                          int register_count,
-                                                          Zone* zone) {
-  return liveness_map_
-      .LookupOrInsert(offset, OffsetHash(offset),
-                      [&]() { return BytecodeLiveness(register_count, zone); })
-      ->value;
+BytecodeLiveness& BytecodeLivenessMap::InsertNewLiveness(int offset) {
+  return liveness_map_.LookupOrInsert(offset, OffsetHash(offset))->value;
 }
 
 BytecodeLiveness& BytecodeLivenessMap::GetLiveness(int offset) {

src/compiler/bytecode-liveness-map.h

@@ -23,6 +23,9 @@ class BytecodeLivenessState : public ZoneObject {
   BytecodeLivenessState(const BytecodeLivenessState&) = delete;
   BytecodeLivenessState& operator=(const BytecodeLivenessState&) = delete;
 
+  BytecodeLivenessState(const BytecodeLivenessState& other, Zone* zone)
+      : bit_vector_(other.bit_vector_, zone) {}
+
   const BitVector& bit_vector() const { return bit_vector_; }
 
   BitVector& bit_vector() { return bit_vector_; }
@@ -78,16 +81,13 @@ class BytecodeLivenessState : public ZoneObject {
 struct BytecodeLiveness {
   BytecodeLivenessState* in;
   BytecodeLivenessState* out;
-
-  BytecodeLiveness(int register_count, Zone* zone);
 };
 
 class V8_EXPORT_PRIVATE BytecodeLivenessMap {
  public:
   BytecodeLivenessMap(int size, Zone* zone);
 
-  BytecodeLiveness& InitializeLiveness(int offset, int register_count,
-                                       Zone* zone);
+  BytecodeLiveness& InsertNewLiveness(int offset);
 
   BytecodeLiveness& GetLiveness(int offset);
   const BytecodeLiveness& GetLiveness(int offset) const;

src/utils/bit-vector.h

@@ -18,7 +18,7 @@ class V8_EXPORT_PRIVATE BitVector : public ZoneObject {
     uintptr_t* ptr_;    // valid if data_length_ > 1
     uintptr_t inline_;  // valid if data_length_ == 1
 
-    DataStorage(uintptr_t value) : inline_(value) {}
+    explicit DataStorage(uintptr_t value) : inline_(value) {}
   };
 
   // Iterator for the elements of this BitVector.
@@ -111,8 +111,15 @@ class V8_EXPORT_PRIVATE BitVector : public ZoneObject {
   }
 
   void CopyFrom(const BitVector& other) {
-    DCHECK_LE(other.length(), length());
-    CopyFrom(other.data_, other.data_length_);
+    DCHECK_EQ(other.length(), length());
+    if (is_inline()) {
+      DCHECK(other.is_inline());
+      data_.inline_ = other.data_.inline_;
+    } else {
+      for (int i = 0; i < data_length_; i++) {
+        data_.ptr_[i] = other.data_.ptr_[i];
+      }
+    }
   }
 
   void Resize(int new_length, Zone* zone) {
@@ -126,7 +133,19 @@ class V8_EXPORT_PRIVATE BitVector : public ZoneObject {
       DCHECK_GT(new_data_length, kDataLengthForInline);
       data_.ptr_ = zone->NewArray<uintptr_t>(new_data_length);
       data_length_ = new_data_length;
-      CopyFrom(old_data, old_data_length);
+
+      // Copy over the data.
+      if (old_data_length == kDataLengthForInline) {
+        data_.ptr_[0] = old_data.inline_;
+      } else {
+        for (int i = 0; i < old_data_length; i++) {
+          data_.ptr_[i] = old_data.ptr_[i];
+        }
+      }
+      // Zero out the rest of the data.
+      for (int i = old_data_length; i < data_length_; i++) {
+        data_.ptr_[i] = 0;
+      }
     }
     length_ = new_length;
   }
@@ -288,27 +307,6 @@ class V8_EXPORT_PRIVATE BitVector : public ZoneObject {
   DataStorage data_;
 
   bool is_inline() const { return data_length_ == kDataLengthForInline; }
-
-  void CopyFrom(DataStorage other_data, int other_data_length) {
-    DCHECK_LE(other_data_length, data_length_);
-
-    if (is_inline()) {
-      DCHECK_EQ(other_data_length, kDataLengthForInline);
-      data_.inline_ = other_data.inline_;
-    } else if (other_data_length == kDataLengthForInline) {
-      data_.ptr_[0] = other_data.inline_;
-      for (int i = 1; i < data_length_; i++) {
-        data_.ptr_[i] = 0;
-      }
-    } else {
-      for (int i = 0; i < other_data_length; i++) {
-        data_.ptr_[i] = other_data.ptr_[i];
-      }
-      for (int i = other_data_length; i < data_length_; i++) {
-        data_.ptr_[i] = 0;
-      }
-    }
-  }
 };
 
 class GrowableBitVector {