unicode-decoder.cc 2.57 KB
Newer Older
1 2 3 4
// Copyright 2014 the V8 project authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

5
#include "src/strings/unicode-decoder.h"
6

7
#include "src/strings/unicode-inl.h"
8
#include "src/utils/memcopy.h"
9

10 11 12
namespace v8 {
namespace internal {

13
Utf8Decoder::Utf8Decoder(const base::Vector<const uint8_t>& chars)
14
    : encoding_(Encoding::kAscii),
15
      non_ascii_start_(NonAsciiStart(chars.begin(), chars.length())),
16 17 18
      utf16_length_(non_ascii_start_) {
  if (non_ascii_start_ == chars.length()) return;

19 20
  const uint8_t* cursor = chars.begin() + non_ascii_start_;
  const uint8_t* end = chars.begin() + chars.length();
21 22 23 24 25 26 27 28 29 30 31 32 33

  bool is_one_byte = true;
  uint32_t incomplete_char = 0;
  unibrow::Utf8::State state = unibrow::Utf8::State::kAccept;

  while (cursor < end) {
    unibrow::uchar t =
        unibrow::Utf8::ValueOfIncremental(&cursor, &state, &incomplete_char);
    if (t != unibrow::Utf8::kIncomplete) {
      is_one_byte = is_one_byte && t <= unibrow::Latin1::kMaxChar;
      utf16_length_++;
      if (t > unibrow::Utf16::kMaxNonSurrogateCharCode) utf16_length_++;
    }
34 35
  }

36 37 38 39
  unibrow::uchar t = unibrow::Utf8::ValueOfIncrementalFinish(&state);
  if (t != unibrow::Utf8::kBufferEmpty) {
    is_one_byte = false;
    utf16_length_++;
40 41
  }

42
  encoding_ = is_one_byte ? Encoding::kLatin1 : Encoding::kUtf16;
43 44
}

45
template <typename Char>
46
void Utf8Decoder::Decode(Char* out, const base::Vector<const uint8_t>& data) {
47
  CopyChars(out, data.begin(), non_ascii_start_);
48 49 50 51 52 53

  out += non_ascii_start_;

  uint32_t incomplete_char = 0;
  unibrow::Utf8::State state = unibrow::Utf8::State::kAccept;

54 55
  const uint8_t* cursor = data.begin() + non_ascii_start_;
  const uint8_t* end = data.begin() + data.length();
56 57 58 59 60 61 62 63 64 65 66 67 68

  while (cursor < end) {
    unibrow::uchar t =
        unibrow::Utf8::ValueOfIncremental(&cursor, &state, &incomplete_char);
    if (t != unibrow::Utf8::kIncomplete) {
      if (sizeof(Char) == 1 || t <= unibrow::Utf16::kMaxNonSurrogateCharCode) {
        *(out++) = static_cast<Char>(t);
      } else {
        *(out++) = unibrow::Utf16::LeadSurrogate(t);
        *(out++) = unibrow::Utf16::TrailSurrogate(t);
      }
    }
  }
69

70 71
  unibrow::uchar t = unibrow::Utf8::ValueOfIncrementalFinish(&state);
  if (t != unibrow::Utf8::kBufferEmpty) *out = static_cast<Char>(t);
72 73
}

74
template V8_EXPORT_PRIVATE void Utf8Decoder::Decode(
75
    uint8_t* out, const base::Vector<const uint8_t>& data);
76

77
template V8_EXPORT_PRIVATE void Utf8Decoder::Decode(
78
    uint16_t* out, const base::Vector<const uint8_t>& data);
79

80 81
}  // namespace internal
}  // namespace v8