[base] Optimize VLQ

Templatize the VLQ methods to avoid std::function overheads, and add a
few optimisations (small value fast path, split writing and tagging).

Change-Id: I840d60c972916d1a6023c8ea2a67bbd540f2d159
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2839554
Commit-Queue: Leszek Swirski <leszeks@chromium.org>
Auto-Submit: Leszek Swirski <leszeks@chromium.org>
Reviewed-by: Igor Sheludko <ishell@chromium.org>
Reviewed-by: Patrick Thier <pthier@chromium.org>
Cr-Commit-Position: refs/heads/master@{#74081}

src/base/vlq.h

@@ -14,60 +14,95 @@ namespace v8 {
 namespace base {
 
 static constexpr uint32_t kContinueShift = 7;
-static constexpr uint32_t kContinueMask = 1 << kContinueShift;
-static constexpr uint32_t kDataMask = kContinueMask - 1;
+static constexpr uint32_t kContinueBit = 1 << kContinueShift;
+static constexpr uint32_t kDataMask = kContinueBit - 1;
 
 // Encodes an unsigned value using variable-length encoding and stores it using
-// the passed process_byte function.
-inline void VLQEncodeUnsigned(const std::function<void(byte)>& process_byte,
-                              uint32_t value) {
-  bool has_next;
+// the passed process_byte function. The function should return a pointer to
+// the byte that was written, so that VLQEncodeUnsigned can mutate it after
+// writing it.
+template <typename Function>
+inline typename std::enable_if<
+    std::is_same<decltype(std::declval<Function>()(0)), byte*>::value,
+    void>::type
+VLQEncodeUnsigned(Function&& process_byte, uint32_t value) {
+  byte* written_byte = process_byte(value);
+  if (value <= kDataMask) {
+    // Value fits in first byte, early return.
+    return;
+  }
   do {
-    byte cur_byte = value & kDataMask;
+    // Turn on continuation bit in the byte we just wrote.
+    *written_byte |= kContinueBit;
     value >>= kContinueShift;
-    has_next = value != 0;
-    // The most significant bit is set when we are not done with the value yet.
-    cur_byte |= static_cast<uint32_t>(has_next) << kContinueShift;
-    process_byte(cur_byte);
-  } while (has_next);
+    written_byte = process_byte(value);
+  } while (value > kDataMask);
 }
 
 // Encodes value using variable-length encoding and stores it using the passed
 // process_byte function.
-inline void VLQEncode(const std::function<void(byte)>& process_byte,
-                      int32_t value) {
+template <typename Function>
+inline typename std::enable_if<
+    std::is_same<decltype(std::declval<Function>()(0)), byte*>::value,
+    void>::type
+VLQEncode(Function&& process_byte, int32_t value) {
   // This wouldn't handle kMinInt correctly if it ever encountered it.
   DCHECK_NE(value, std::numeric_limits<int32_t>::min());
   bool is_negative = value < 0;
   // Encode sign in least significant bit.
   uint32_t bits = static_cast<uint32_t>((is_negative ? -value : value) << 1) |
                   static_cast<uint32_t>(is_negative);
-  VLQEncodeUnsigned(process_byte, bits);
+  VLQEncodeUnsigned(std::forward<Function>(process_byte), bits);
 }
 
 // Wrapper of VLQEncode for std::vector backed storage containers.
 template <typename A>
 inline void VLQEncode(std::vector<byte, A>* data, int32_t value) {
-  VLQEncode([data](byte value) { data->push_back(value); }, value);
+  VLQEncode(
+      [data](byte value) {
+        data->push_back(value);
+        return &data->back();
+      },
+      value);
 }
 
 // Wrapper of VLQEncodeUnsigned for std::vector backed storage containers.
 template <typename A>
 inline void VLQEncodeUnsigned(std::vector<byte, A>* data, uint32_t value) {
-  VLQEncodeUnsigned([data](byte value) { data->push_back(value); }, value);
+  VLQEncodeUnsigned(
+      [data](byte value) {
+        data->push_back(value);
+        return &data->back();
+      },
+      value);
+}
+
+// Decodes a variable-length encoded unsigned value from bytes returned by
+// successive calls to the given function.
+template <typename GetNextFunction>
+inline typename std::enable_if<
+    std::is_same<decltype(std::declval<GetNextFunction>()()), byte>::value,
+    uint32_t>::type
+VLQDecodeUnsigned(GetNextFunction&& get_next) {
+  byte cur_byte = get_next();
+  // Single byte fast path; no need to mask.
+  if (cur_byte <= kDataMask) {
+    return cur_byte;
+  }
+  uint32_t bits = cur_byte & kDataMask;
+  for (int shift = kContinueShift; shift <= 32; shift += kContinueShift) {
+    byte cur_byte = get_next();
+    bits |= (cur_byte & kDataMask) << shift;
+    if (cur_byte <= kDataMask) break;
+  }
+  return bits;
 }
 
 // Decodes a variable-length encoded unsigned value stored in contiguous memory
 // starting at data_start + index, updating index to where the next encoded
 // value starts.
 inline uint32_t VLQDecodeUnsigned(byte* data_start, int* index) {
-  uint32_t bits = 0;
-  for (int shift = 0; true; shift += kContinueShift) {
-    byte cur_byte = data_start[(*index)++];
-    bits += (cur_byte & kDataMask) << shift;
-    if ((cur_byte & kContinueMask) == 0) break;
-  }
-  return bits;
+  return VLQDecodeUnsigned([&] { return data_start[(*index)++]; });
 }
 
 // Decodes a variable-length encoded value stored in contiguous memory starting