[string] Refactor UTF-8 and WTF-8 decoders to share code

This will allow us to more easily add a strict UTF-8 decoder, for use in
stringrefs.

Bug: v8:12868
Change-Id: I6835dca619417f4d2994d8283728cf8ebe599bd7
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3714660Reviewed-by: Marja Hölttä <marja@chromium.org>
Commit-Queue: Andy Wingo <wingo@igalia.com>
Reviewed-by: Jakob Kummerow <jkummerow@chromium.org>
Cr-Commit-Position: refs/heads/main@{#81333}

src/strings/unicode-decoder.cc

@@ -14,95 +14,42 @@
 namespace v8 {
 namespace internal {
 
-Utf8Decoder::Utf8Decoder(const base::Vector<const uint8_t>& chars)
-    : encoding_(Encoding::kAscii),
-      non_ascii_start_(NonAsciiStart(chars.begin(), chars.length())),
-      utf16_length_(non_ascii_start_) {
-  if (non_ascii_start_ == chars.length()) return;
-
-  const uint8_t* cursor = chars.begin() + non_ascii_start_;
-  const uint8_t* end = chars.begin() + chars.length();
-
-  bool is_one_byte = true;
-  uint32_t incomplete_char = 0;
-  unibrow::Utf8::State state = unibrow::Utf8::State::kAccept;
-
-  while (cursor < end) {
-    unibrow::uchar t =
-        unibrow::Utf8::ValueOfIncremental(&cursor, &state, &incomplete_char);
-    if (t != unibrow::Utf8::kIncomplete) {
-      is_one_byte = is_one_byte && t <= unibrow::Latin1::kMaxChar;
-      utf16_length_++;
-      if (t > unibrow::Utf16::kMaxNonSurrogateCharCode) utf16_length_++;
-    }
-  }
-
-  unibrow::uchar t = unibrow::Utf8::ValueOfIncrementalFinish(&state);
-  if (t != unibrow::Utf8::kBufferEmpty) {
-    is_one_byte = false;
-    utf16_length_++;
-  }
-
-  encoding_ = is_one_byte ? Encoding::kLatin1 : Encoding::kUtf16;
-}
-
-template <typename Char>
-void Utf8Decoder::Decode(Char* out, const base::Vector<const uint8_t>& data) {
-  CopyChars(out, data.begin(), non_ascii_start_);
-
-  out += non_ascii_start_;
-
-  uint32_t incomplete_char = 0;
-  unibrow::Utf8::State state = unibrow::Utf8::State::kAccept;
-
-  const uint8_t* cursor = data.begin() + non_ascii_start_;
-  const uint8_t* end = data.begin() + data.length();
-
-  while (cursor < end) {
-    unibrow::uchar t =
-        unibrow::Utf8::ValueOfIncremental(&cursor, &state, &incomplete_char);
-    if (t != unibrow::Utf8::kIncomplete) {
-      if (sizeof(Char) == 1 || t <= unibrow::Utf16::kMaxNonSurrogateCharCode) {
-        *(out++) = static_cast<Char>(t);
-      } else {
-        *(out++) = unibrow::Utf16::LeadSurrogate(t);
-        *(out++) = unibrow::Utf16::TrailSurrogate(t);
-      }
-    }
-  }
-
-  unibrow::uchar t = unibrow::Utf8::ValueOfIncrementalFinish(&state);
-  if (t != unibrow::Utf8::kBufferEmpty) *out = static_cast<Char>(t);
-}
-
-template V8_EXPORT_PRIVATE void Utf8Decoder::Decode(
-    uint8_t* out, const base::Vector<const uint8_t>& data);
-
-template V8_EXPORT_PRIVATE void Utf8Decoder::Decode(
-    uint16_t* out, const base::Vector<const uint8_t>& data);
-
-#if V8_ENABLE_WEBASSEMBLY
-Wtf8Decoder::Wtf8Decoder(const base::Vector<const uint8_t>& data)
+template <class Decoder>
+Utf8DecoderBase<Decoder>::Utf8DecoderBase(
+    const base::Vector<const uint8_t>& data)
     : encoding_(Encoding::kAscii),
       non_ascii_start_(NonAsciiStart(data.begin(), data.length())),
       utf16_length_(non_ascii_start_) {
   if (non_ascii_start_ == data.length()) return;
 
   bool is_one_byte = true;
-  auto state = GeneralizedUtf8DfaDecoder::kAccept;
+  auto state = Decoder::DfaDecoder::kAccept;
   uint32_t current = 0;
   uint32_t previous = 0;
+  const uint8_t* cursor = data.begin() + non_ascii_start_;
+  const uint8_t* end = data.begin() + data.length();
 
-  for (size_t i = non_ascii_start_; i < data.size(); i++) {
-    GeneralizedUtf8DfaDecoder::Decode(data[i], &state, &current);
-    if (state < GeneralizedUtf8DfaDecoder::kAccept) {
-      DCHECK_EQ(state, GeneralizedUtf8DfaDecoder::kReject);
+  while (cursor < end) {
+    auto previous_state = state;
+    Decoder::DfaDecoder::Decode(*cursor, &state, &current);
+    if (state < Decoder::DfaDecoder::kAccept) {
+      DCHECK_EQ(state, Decoder::DfaDecoder::kReject);
+      if (Decoder::kAllowIncompleteSequences) {
+        state = Decoder::DfaDecoder::kAccept;
+        static_assert(unibrow::Utf8::kBadChar > unibrow::Latin1::kMaxChar);
+        is_one_byte = false;
+        utf16_length_++;
+        previous = unibrow::Utf8::kBadChar;
+        current = 0;
+        // If we were trying to continue a multibyte sequence, try this byte
+        // again.
+        if (previous_state != Decoder::DfaDecoder::kAccept) continue;
+      } else {
         encoding_ = Encoding::kInvalid;
         return;
       }
-    if (state == GeneralizedUtf8DfaDecoder::kAccept) {
-      if (unibrow::Utf16::IsTrailSurrogate(current) &&
-          unibrow::Utf16::IsLeadSurrogate(previous)) {
+    } else if (state == Decoder::DfaDecoder::kAccept) {
+      if (Decoder::InvalidCodePointSequence(current, previous)) {
         encoding_ = Encoding::kInvalid;
         return;
       }
@@ -112,44 +59,84 @@ Wtf8Decoder::Wtf8Decoder(const base::Vector<const uint8_t>& data)
       previous = current;
       current = 0;
     }
+    cursor++;
   }
 
-  if (state == GeneralizedUtf8DfaDecoder::kAccept) {
+  if (state == Decoder::DfaDecoder::kAccept) {
     encoding_ = is_one_byte ? Encoding::kLatin1 : Encoding::kUtf16;
+  } else if (Decoder::kAllowIncompleteSequences) {
+    static_assert(unibrow::Utf8::kBadChar > unibrow::Latin1::kMaxChar);
+    encoding_ = Encoding::kUtf16;
+    utf16_length_++;
   } else {
     encoding_ = Encoding::kInvalid;
   }
 }
 
+template <class Decoder>
 template <typename Char>
-void Wtf8Decoder::Decode(Char* out, const base::Vector<const uint8_t>& data) {
+void Utf8DecoderBase<Decoder>::Decode(Char* out,
+                                      const base::Vector<const uint8_t>& data) {
   DCHECK(!is_invalid());
   CopyChars(out, data.begin(), non_ascii_start_);
 
   out += non_ascii_start_;
 
-  auto state = GeneralizedUtf8DfaDecoder::kAccept;
-  uint32_t t = 0;
-  for (size_t i = non_ascii_start_; i < data.size(); i++) {
-    GeneralizedUtf8DfaDecoder::Decode(data[i], &state, &t);
-    if (state == GeneralizedUtf8DfaDecoder::kAccept) {
-      if (sizeof(Char) == 1 || t <= unibrow::Utf16::kMaxNonSurrogateCharCode) {
-        *(out++) = static_cast<Char>(t);
+  auto state = Decoder::DfaDecoder::kAccept;
+  uint32_t current = 0;
+  const uint8_t* cursor = data.begin() + non_ascii_start_;
+  const uint8_t* end = data.begin() + data.length();
+
+  while (cursor < end) {
+    auto previous_state = state;
+    Decoder::DfaDecoder::Decode(*cursor, &state, &current);
+    if (Decoder::kAllowIncompleteSequences &&
+        state < Decoder::DfaDecoder::kAccept) {
+      state = Decoder::DfaDecoder::kAccept;
+      *(out++) = static_cast<Char>(unibrow::Utf8::kBadChar);
+      current = 0;
+      // If we were trying to continue a multibyte sequence, try this byte
+      // again.
+      if (previous_state != Decoder::DfaDecoder::kAccept) continue;
+    } else if (state == Decoder::DfaDecoder::kAccept) {
+      if (sizeof(Char) == 1 ||
+          current <= unibrow::Utf16::kMaxNonSurrogateCharCode) {
+        *(out++) = static_cast<Char>(current);
       } else {
-        *(out++) = unibrow::Utf16::LeadSurrogate(t);
-        *(out++) = unibrow::Utf16::TrailSurrogate(t);
+        *(out++) = unibrow::Utf16::LeadSurrogate(current);
+        *(out++) = unibrow::Utf16::TrailSurrogate(current);
+      }
+      current = 0;
     }
-      t = 0;
+    cursor++;
   }
+
+  if (Decoder::kAllowIncompleteSequences &&
+      state != Decoder::DfaDecoder::kAccept) {
+    *out = static_cast<Char>(unibrow::Utf8::kBadChar);
+  } else {
+    DCHECK_EQ(state, Decoder::DfaDecoder::kAccept);
   }
-  DCHECK_EQ(state, GeneralizedUtf8DfaDecoder::kAccept);
 }
 
-template void Wtf8Decoder::Decode(uint8_t* out,
+template V8_EXPORT_PRIVATE Utf8DecoderBase<Utf8Decoder>::Utf8DecoderBase(
     const base::Vector<const uint8_t>& data);
 
-template void Wtf8Decoder::Decode(uint16_t* out,
+template V8_EXPORT_PRIVATE void Utf8DecoderBase<Utf8Decoder>::Decode(
+    uint8_t* out, const base::Vector<const uint8_t>& data);
+
+template V8_EXPORT_PRIVATE void Utf8DecoderBase<Utf8Decoder>::Decode(
+    uint16_t* out, const base::Vector<const uint8_t>& data);
+
+#if V8_ENABLE_WEBASSEMBLY
+template Utf8DecoderBase<Wtf8Decoder>::Utf8DecoderBase(
     const base::Vector<const uint8_t>& data);
+
+template void Utf8DecoderBase<Wtf8Decoder>::Decode(
+    uint8_t* out, const base::Vector<const uint8_t>& data);
+
+template void Utf8DecoderBase<Wtf8Decoder>::Decode(
+    uint16_t* out, const base::Vector<const uint8_t>& data);
 #endif  // V8_ENABLE_WEBASSEMBLY
 
 }  // namespace internal

src/strings/unicode-decoder.h

@@ -8,6 +8,10 @@
 #include "src/base/vector.h"
 #include "src/strings/unicode.h"
 
+#if V8_ENABLE_WEBASSEMBLY
+#include "src/third_party/utf8-decoder/generalized-utf8-decoder.h"
+#endif
+
 namespace v8 {
 namespace internal {
 
@@ -48,40 +52,14 @@ inline int NonAsciiStart(const uint8_t* chars, int length) {
   return static_cast<int>(chars - start);
 }
 
-class V8_EXPORT_PRIVATE Utf8Decoder final {
- public:
-  enum class Encoding : uint8_t { kAscii, kLatin1, kUtf16 };
-
-  explicit Utf8Decoder(const base::Vector<const uint8_t>& chars);
-
-  // This decoder never fails; an invalid byte sequence decodes to U+FFFD and
-  // then the decode continues.
-  bool is_invalid() const { return false; }
-  bool is_ascii() const { return encoding_ == Encoding::kAscii; }
-  bool is_one_byte() const { return encoding_ <= Encoding::kLatin1; }
-  int utf16_length() const { return utf16_length_; }
-  int non_ascii_start() const { return non_ascii_start_; }
-
-  template <typename Char>
-  V8_EXPORT_PRIVATE void Decode(Char* out,
-                                const base::Vector<const uint8_t>& data);
-
- private:
-  Encoding encoding_;
-  int non_ascii_start_;
-  int utf16_length_;
-};
-
-#if V8_ENABLE_WEBASSEMBLY
-// Like Utf8Decoder above, except that instead of replacing invalid sequences
-// with U+FFFD, we have a separate Encoding::kInvalid state.
-class Wtf8Decoder {
+template <class Decoder>
+class Utf8DecoderBase {
  public:
   enum class Encoding : uint8_t { kAscii, kLatin1, kUtf16, kInvalid };
 
-  explicit Wtf8Decoder(const base::Vector<const uint8_t>& data);
-
-  bool is_invalid() const { return encoding_ == Encoding::kInvalid; }
+  bool is_invalid() const {
+    return static_cast<const Decoder&>(*this).is_invalid();
+  }
   bool is_ascii() const { return encoding_ == Encoding::kAscii; }
   bool is_one_byte() const { return encoding_ <= Encoding::kLatin1; }
   int utf16_length() const {
@@ -94,14 +72,55 @@ class Wtf8Decoder {
   }
 
   template <typename Char>
-  V8_EXPORT_PRIVATE void Decode(Char* out,
-                                const base::Vector<const uint8_t>& data);
+  void Decode(Char* out, const base::Vector<const uint8_t>& data);
 
- private:
+ protected:
+  explicit Utf8DecoderBase(const base::Vector<const uint8_t>& data);
   Encoding encoding_;
   int non_ascii_start_;
   int utf16_length_;
 };
+
+class V8_EXPORT_PRIVATE Utf8Decoder final
+    : public Utf8DecoderBase<Utf8Decoder> {
+ public:
+  static bool InvalidCodePointSequence(uint32_t current, uint32_t previous) {
+    // The DfaDecoder will only ever decode Unicode scalar values, and all
+    // sequences of USVs are valid.
+    DCHECK(!unibrow::Utf16::IsLeadSurrogate(current));
+    DCHECK(!unibrow::Utf16::IsTrailSurrogate(current));
+    return false;
+  }
+  static const bool kAllowIncompleteSequences = true;
+  using DfaDecoder = Utf8DfaDecoder;
+
+  explicit Utf8Decoder(const base::Vector<const uint8_t>& data)
+      : Utf8DecoderBase(data) {}
+
+  // This decoder never fails; an invalid byte sequence decodes to U+FFFD and
+  // then the decode continues.
+  bool is_invalid() const {
+    DCHECK_NE(encoding_, Encoding::kInvalid);
+    return false;
+  }
+};
+
+#if V8_ENABLE_WEBASSEMBLY
+// Like Utf8Decoder above, except that instead of replacing invalid sequences
+// with U+FFFD, we have a separate Encoding::kInvalid state.
+class Wtf8Decoder : public Utf8DecoderBase<Wtf8Decoder> {
+ public:
+  static bool InvalidCodePointSequence(uint32_t current, uint32_t previous) {
+    return unibrow::Utf16::IsSurrogatePair(current, previous);
+  }
+  static const bool kAllowIncompleteSequences = false;
+  using DfaDecoder = GeneralizedUtf8DfaDecoder;
+
+  explicit Wtf8Decoder(const base::Vector<const uint8_t>& data)
+      : Utf8DecoderBase(data) {}
+
+  bool is_invalid() const { return encoding_ == Encoding::kInvalid; }
+};
 #endif  // V8_ENABLE_WEBASSEMBLY
 
 }  // namespace internal

src/strings/unicode.h

@@ -103,10 +103,10 @@ class Utf16 {
     return IsLeadSurrogate(lead) && IsTrailSurrogate(trail);
   }
   static inline bool IsLeadSurrogate(int code) {
-    return (code & 0xfc00) == 0xd800;
+    return (code & 0x1ffc00) == 0xd800;
   }
   static inline bool IsTrailSurrogate(int code) {
-    return (code & 0xfc00) == 0xdc00;
+    return (code & 0x1ffc00) == 0xdc00;
   }
 
   static inline int CombineSurrogatePair(uchar lead, uchar trail) {

src/third_party/utf8-decoder/generalized-utf8-decoder.h

@@ -9,9 +9,8 @@
 #ifndef __GENERALIZED_UTF8_DFA_DECODER_H
 #define __GENERALIZED_UTF8_DFA_DECODER_H
 
-namespace GeneralizedUtf8DfaDecoder {
-
-enum State : uint8_t {
+struct GeneralizedUtf8DfaDecoder {
+  enum State : uint8_t {
     kReject = 0,
     kAccept = 11,
     kTwoByte = 22,
@@ -20,17 +19,17 @@ enum State : uint8_t {
     kFourByteLow = 55,
     kThreeByteHigh = 66,
     kFourByteMidHigh = 77,
-};
+  };
 
-static inline void Decode(uint8_t byte, State* state, uint32_t* buffer) {
+  static inline void Decode(uint8_t byte, State* state, uint32_t* buffer) {
     // This first table maps bytes to character to a transition.
     //
     // The transition value takes a state to a new state, but it also determines
     // the set of bits from the current byte that contribute to the decoded
     // codepoint:
     //
-  //   Transition | Bits from current byte that contribute to decoded codepoint
-  //   ------------------------------------------------------------------------
+    //   Transition | Current byte bits that contribute to decoded codepoint
+    //   -------------------------------------------------------------------
     //    0, 1      | 0b01111111
     //    2, 3      | 0b00111111
     //    4, 5      | 0b00011111
@@ -39,19 +38,20 @@ static inline void Decode(uint8_t byte, State* state, uint32_t* buffer) {
     //    10        | 0b00000011
     //
     // Given the WTF-8 encoding, we therefore have the following constraints:
-  //   1. The transition value for 1-byte encodings should have the value 0 or 1
-  //      so that we preserve all of the low 7 bits.
+
+    //   1. The transition value for 1-byte encodings should have the value 0 or
+    //      1 so that we preserve all of the low 7 bits.
     //   2. Continuation bytes (0x80 to 0xBF) are of the form 0b10xxxxxx, and
     //      therefore should have transition value between 0 and 3.
     //   3. Leading bytes for 2-byte encodings are of the form 0b110yyyyy, and
     //      therefore the transition value can be between 2 and 5.
-  //   4. Leading bytes for 3-byte encodings (0b1110zzzz) need transition value
-  //      between 4 and 7.
-  //   5. Leading bytes for 4-byte encodings (0b11110uuu) need transition value
-  //      between 6 and 9.
-  //   6. We need more states to impose irregular constraints.  Sometimes we can
-  //      use the knowldege that e.g. some high significant bits of the xxxx in
-  //      0b1110xxxx are 0, then we can use a higher transition value.
+    //   4. Leading bytes for 3-byte encodings (0b1110zzzz) need transition
+    //      value between 4 and 7.
+    //   5. Leading bytes for 4-byte encodings (0b11110uuu) need transition
+    //      value between 6 and 9.
+    //   6. We need more states to impose irregular constraints.  Sometimes we
+    //      can use the knowldege that e.g. some high significant bits of the
+    //      xxxx in 0b1110xxxx are 0, then we can use a higher transition value.
     //   7. Transitions to invalid states can use any transition value.
     static constexpr uint8_t transitions[] = {
         0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // 00-0F
@@ -98,8 +98,7 @@ static inline void Decode(uint8_t byte, State* state, uint32_t* buffer) {
     uint8_t type = transitions[byte];
     *state = static_cast<State>(states[*state + type]);
     *buffer = (*buffer << 6) | (byte & (0x7F >> (type >> 1)));
-}
-
-}  // namespace GeneralizedUtf8DfaDecoder
+  }
+};
 
 #endif  // __GENERALIZED_UTF8_DFA_DECODER_H

src/third_party/utf8-decoder/utf8-decoder.h

@@ -7,9 +7,8 @@
 #ifndef __UTF8_DFA_DECODER_H
 #define __UTF8_DFA_DECODER_H
 
-namespace Utf8DfaDecoder {
-
-enum State : uint8_t {
+struct Utf8DfaDecoder {
+  enum State : uint8_t {
     kReject = 0,
     kAccept = 12,
     kTwoByte = 24,
@@ -19,9 +18,9 @@ enum State : uint8_t {
     kFourByteLow = 72,
     kThreeByteHigh = 84,
     kFourByteMidHigh = 96,
-};
+  };
 
-static inline void Decode(uint8_t byte, State* state, uint32_t* buffer) {
+  static inline void Decode(uint8_t byte, State* state, uint32_t* buffer) {
     // This first table maps bytes to character to a transition.
     static constexpr uint8_t transitions[] = {
         0,  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,  // 00-0F
@@ -70,8 +69,7 @@ static inline void Decode(uint8_t byte, State* state, uint32_t* buffer) {
     uint8_t type = transitions[byte];
     *state = static_cast<State>(states[*state + type]);
     *buffer = (*buffer << 6) | (byte & (0x7F >> (type >> 1)));
-}
-
-}  // namespace Utf8DfaDecoder
+  }
+};
 
 #endif /* __UTF8_DFA_DECODER_H */