[base] Allow BitVector to store data inline

If the data to be stored in a BitVector is small enough (less than the
pointer size), it can be stored directly on the BitVector instead of the
pointer. This patch makes the data field of the BitVector a union
between a pointer and uintptr_t, and uses the latter is the data length
is 0.

Change-Id: I24c1920f2c16373c883cf69b123bf59812fef28e
Reviewed-on: https://chromium-review.googlesource.com/541307
Commit-Queue: Leszek Swirski <leszeks@chromium.org>
Reviewed-by: Ross McIlroy <rmcilroy@chromium.org>
Cr-Commit-Position: refs/heads/master@{#46209}

src/bit-vector.cc

@@ -32,7 +32,8 @@ void BitVector::Iterator::Advance() {
   while (val == 0) {
     current_index_++;
     if (Done()) return;
-    val = target_->data_[current_index_];
+    DCHECK(!target_->is_inline());
+    val = target_->data_.ptr_[current_index_];
     current_ = current_index_ << kDataBitShift;
   }
   val = SkipZeroBytes(val);
@@ -42,11 +43,15 @@ void BitVector::Iterator::Advance() {
 
 
 int BitVector::Count() const {
-  int count = 0;
-  for (int i = 0; i < data_length_; i++) {
-    count += base::bits::CountPopulation(data_[i]);
+  if (data_length_ == 0) {
+    return base::bits::CountPopulation(data_.inline_);
+  } else {
+    int count = 0;
+    for (int i = 0; i < data_length_; i++) {
+      count += base::bits::CountPopulation(data_.ptr_[i]);
+    }
+    return count;
   }
-  return count;
 }
 
 }  // namespace internal

src/bit-vector.h

@@ -13,15 +13,22 @@ namespace internal {
 
 class BitVector : public ZoneObject {
  public:
+  union DataStorage {
+    uintptr_t* ptr_;    // valid if data_length_ > 1
+    uintptr_t inline_;  // valid if data_length_ == 1
+
+    DataStorage(uintptr_t value) : inline_(value) {}
+  };
+
   // Iterator for the elements of this BitVector.
   class Iterator BASE_EMBEDDED {
    public:
     explicit Iterator(BitVector* target)
         : target_(target),
           current_index_(0),
-          current_value_(target->data_[0]),
+          current_value_(target->is_inline() ? target->data_.inline_
+                                             : target->data_.ptr_[0]),
           current_(-1) {
-      DCHECK(target->data_length_ > 0);
       Advance();
     }
     ~Iterator() {}
@@ -58,28 +65,43 @@ class BitVector : public ZoneObject {
     friend class BitVector;
   };
 
+  static const int kDataLengthForInline = 1;
   static const int kDataBits = kPointerSize * 8;
   static const int kDataBitShift = kPointerSize == 8 ? 6 : 5;
   static const uintptr_t kOne = 1;  // This saves some static_casts.
 
+  BitVector() : length_(0), data_length_(kDataLengthForInline), data_(0) {}
+
   BitVector(int length, Zone* zone)
-      : length_(length),
-        data_length_(SizeFor(length)),
-        data_(zone->NewArray<uintptr_t>(data_length_)) {
+      : length_(length), data_length_(SizeFor(length)), data_(0) {
     DCHECK_LE(0, length);
-    Clear();
+    if (!is_inline()) {
+      data_.ptr_ = zone->NewArray<uintptr_t>(data_length_);
+      Clear();
+    }
+    // Otherwise, clearing is implicit
   }
 
   BitVector(const BitVector& other, Zone* zone)
-      : length_(other.length()),
-        data_length_(SizeFor(length_)),
-        data_(zone->NewArray<uintptr_t>(data_length_)) {
-    CopyFrom(other);
+      : length_(other.length_),
+        data_length_(other.data_length_),
+        data_(other.data_.inline_) {
+    if (!is_inline()) {
+      data_.ptr_ = zone->NewArray<uintptr_t>(data_length_);
+      for (int i = 0; i < other.data_length_; i++) {
+        data_.ptr_[i] = other.data_.ptr_[i];
+      }
+    }
   }
 
   static int SizeFor(int length) {
-    if (length == 0) return 1;
-    return 1 + ((length - 1) / kDataBits);
+    if (length <= kDataBits) {
+      return kDataLengthForInline;
+    }
+
+    int data_length = 1 + ((length - 1) / kDataBits);
+    DCHECK_GT(data_length, kDataLengthForInline);
+    return data_length;
   }
 
   void CopyFrom(const BitVector& other) {
@@ -91,10 +113,12 @@ class BitVector : public ZoneObject {
     DCHECK_GT(new_length, length());
     int new_data_length = SizeFor(new_length);
     if (new_data_length > data_length_) {
-      uintptr_t* old_data = data_;
+      DataStorage old_data = data_;
       int old_data_length = data_length_;
 
-      data_ = zone->NewArray<uintptr_t>(new_data_length);
+      // Make sure the new data length is large enough to need allocation.
+      DCHECK_GT(new_data_length, kDataLengthForInline);
+      data_.ptr_ = zone->NewArray<uintptr_t>(new_data_length);
       data_length_ = new_data_length;
       CopyFrom(old_data, old_data_length);
     }
@@ -103,83 +127,143 @@ class BitVector : public ZoneObject {
 
   bool Contains(int i) const {
     DCHECK(i >= 0 && i < length());
-    uintptr_t block = data_[i / kDataBits];
+    uintptr_t block = is_inline() ? data_.inline_ : data_.ptr_[i / kDataBits];
     return (block & (kOne << (i % kDataBits))) != 0;
   }
 
   void Add(int i) {
     DCHECK(i >= 0 && i < length());
-    data_[i / kDataBits] |= (kOne << (i % kDataBits));
+    if (is_inline()) {
+      data_.inline_ |= (kOne << i);
+    } else {
+      data_.ptr_[i / kDataBits] |= (kOne << (i % kDataBits));
+    }
   }
 
-  void AddAll() { memset(data_, -1, sizeof(uintptr_t) * data_length_); }
+  void AddAll() {
+    // TODO(leszeks): This sets bits outside of the length of this bit-vector,
+    // which is observable if we resize it or copy from it. If this is a
+    // problem, we should clear the high bits either on add, or on resize/copy.
+    if (is_inline()) {
+      data_.inline_ = -1;
+    } else {
+      memset(data_.ptr_, -1, sizeof(uintptr_t) * data_length_);
+    }
+  }
 
   void Remove(int i) {
     DCHECK(i >= 0 && i < length());
-    data_[i / kDataBits] &= ~(kOne << (i % kDataBits));
+    if (is_inline()) {
+      data_.inline_ &= ~(kOne << i);
+    } else {
+      data_.ptr_[i / kDataBits] &= ~(kOne << (i % kDataBits));
+    }
   }
 
   void Union(const BitVector& other) {
     DCHECK(other.length() == length());
-    for (int i = 0; i < data_length_; i++) {
-      data_[i] |= other.data_[i];
+    if (is_inline()) {
+      DCHECK(other.is_inline());
+      data_.inline_ |= other.data_.inline_;
+    } else {
+      for (int i = 0; i < data_length_; i++) {
+        data_.ptr_[i] |= other.data_.ptr_[i];
+      }
     }
   }
 
   bool UnionIsChanged(const BitVector& other) {
     DCHECK(other.length() == length());
-    bool changed = false;
-    for (int i = 0; i < data_length_; i++) {
-      uintptr_t old_data = data_[i];
-      data_[i] |= other.data_[i];
-      if (data_[i] != old_data) changed = true;
+    if (is_inline()) {
+      DCHECK(other.is_inline());
+      uintptr_t old_data = data_.inline_;
+      data_.inline_ |= other.data_.inline_;
+      return data_.inline_ != old_data;
+    } else {
+      bool changed = false;
+      for (int i = 0; i < data_length_; i++) {
+        uintptr_t old_data = data_.ptr_[i];
+        data_.ptr_[i] |= other.data_.ptr_[i];
+        if (data_.ptr_[i] != old_data) changed = true;
+      }
+      return changed;
     }
-    return changed;
   }
 
   void Intersect(const BitVector& other) {
     DCHECK(other.length() == length());
-    for (int i = 0; i < data_length_; i++) {
-      data_[i] &= other.data_[i];
+    if (is_inline()) {
+      DCHECK(other.is_inline());
+      data_.inline_ &= other.data_.inline_;
+    } else {
+      for (int i = 0; i < data_length_; i++) {
+        data_.ptr_[i] &= other.data_.ptr_[i];
+      }
     }
   }
 
   bool IntersectIsChanged(const BitVector& other) {
     DCHECK(other.length() == length());
-    bool changed = false;
-    for (int i = 0; i < data_length_; i++) {
-      uintptr_t old_data = data_[i];
-      data_[i] &= other.data_[i];
-      if (data_[i] != old_data) changed = true;
+    if (is_inline()) {
+      DCHECK(other.is_inline());
+      uintptr_t old_data = data_.inline_;
+      data_.inline_ &= other.data_.inline_;
+      return data_.inline_ != old_data;
+    } else {
+      bool changed = false;
+      for (int i = 0; i < data_length_; i++) {
+        uintptr_t old_data = data_.ptr_[i];
+        data_.ptr_[i] &= other.data_.ptr_[i];
+        if (data_.ptr_[i] != old_data) changed = true;
+      }
+      return changed;
     }
-    return changed;
   }
 
   void Subtract(const BitVector& other) {
     DCHECK(other.length() == length());
-    for (int i = 0; i < data_length_; i++) {
-      data_[i] &= ~other.data_[i];
+    if (is_inline()) {
+      DCHECK(other.is_inline());
+      data_.inline_ &= ~other.data_.inline_;
+    } else {
+      for (int i = 0; i < data_length_; i++) {
+        data_.ptr_[i] &= ~other.data_.ptr_[i];
+      }
     }
   }
 
   void Clear() {
-    for (int i = 0; i < data_length_; i++) {
-      data_[i] = 0;
+    if (is_inline()) {
+      data_.inline_ = 0;
+    } else {
+      for (int i = 0; i < data_length_; i++) {
+        data_.ptr_[i] = 0;
+      }
     }
   }
 
   bool IsEmpty() const {
-    for (int i = 0; i < data_length_; i++) {
-      if (data_[i] != 0) return false;
+    if (is_inline()) {
+      return data_.inline_ == 0;
+    } else {
+      for (int i = 0; i < data_length_; i++) {
+        if (data_.ptr_[i] != 0) return false;
+      }
+      return true;
     }
-    return true;
   }
 
   bool Equals(const BitVector& other) const {
-    for (int i = 0; i < data_length_; i++) {
-      if (data_[i] != other.data_[i]) return false;
+    DCHECK(other.length() == length());
+    if (is_inline()) {
+      DCHECK(other.is_inline());
+      return data_.inline_ == other.data_.inline_;
+    } else {
+      for (int i = 0; i < data_length_; i++) {
+        if (data_.ptr_[i] != other.data_.ptr_[i]) return false;
+      }
+      return true;
     }
-    return true;
   }
 
   int Count() const;
@@ -193,15 +277,28 @@ class BitVector : public ZoneObject {
  private:
   int length_;
   int data_length_;
-  uintptr_t* data_;
+  DataStorage data_;
 
-  void CopyFrom(uintptr_t* other_data, int other_data_length) {
+  bool is_inline() const { return data_length_ == kDataLengthForInline; }
+
+  void CopyFrom(DataStorage other_data, int other_data_length) {
     DCHECK_LE(other_data_length, data_length_);
-    for (int i = 0; i < other_data_length; i++) {
-      data_[i] = other_data[i];
-    }
-    for (int i = other_data_length; i < data_length_; i++) {
-      data_[i] = 0;
+
+    if (is_inline()) {
+      DCHECK_EQ(other_data_length, kDataLengthForInline);
+      data_.inline_ = other_data.inline_;
+    } else if (other_data_length == kDataLengthForInline) {
+      data_.ptr_[0] = other_data.inline_;
+      for (int i = 1; i < data_length_; i++) {
+        data_.ptr_[i] = 0;
+      }
+    } else {
+      for (int i = 0; i < other_data_length; i++) {
+        data_.ptr_[i] = other_data.ptr_[i];
+      }
+      for (int i = other_data_length; i < data_length_; i++) {
+        data_.ptr_[i] = 0;
+      }
     }
   }