unicode.h 8.82 KB
Newer Older
1
// Copyright 2011 the V8 project authors. All rights reserved.
2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
//       notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
//       copyright notice, this list of conditions and the following
//       disclaimer in the documentation and/or other materials provided
//       with the distribution.
//     * Neither the name of Google Inc. nor the names of its
//       contributors may be used to endorse or promote products derived
//       from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

28 29
#ifndef V8_UNICODE_H_
#define V8_UNICODE_H_
30 31

#include <sys/types.h>
32
#include <globals.h>
33 34 35 36 37 38 39 40 41 42 43 44 45 46
/**
 * \file
 * Definitions and convenience functions for working with unicode.
 */

namespace unibrow {

typedef unsigned int uchar;
typedef unsigned char byte;

/**
 * The max length of the result of converting the case of a single
 * character.
 */
47
const int kMaxMappingSize = 4;
48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82

template <class T, int size = 256>
class Predicate {
 public:
  inline Predicate() { }
  inline bool get(uchar c);
 private:
  friend class Test;
  bool CalculateValue(uchar c);
  struct CacheEntry {
    inline CacheEntry() : code_point_(0), value_(0) { }
    inline CacheEntry(uchar code_point, bool value)
      : code_point_(code_point),
        value_(value) { }
    uchar code_point_ : 21;
    bool value_ : 1;
  };
  static const int kSize = size;
  static const int kMask = kSize - 1;
  CacheEntry entries_[kSize];
};

// A cache used in case conversion.  It caches the value for characters
// that either have no mapping or map to a single character independent
// of context.  Characters that map to more than one character or that
// map differently depending on context are always looked up.
template <class T, int size = 256>
class Mapping {
 public:
  inline Mapping() { }
  inline int get(uchar c, uchar n, uchar* result);
 private:
  friend class Test;
  int CalculateValue(uchar c, uchar n, uchar* result);
  struct CacheEntry {
83
    inline CacheEntry() : code_point_(kNoChar), offset_(0) { }
84 85 86
    inline CacheEntry(uchar code_point, signed offset)
      : code_point_(code_point),
        offset_(offset) { }
87 88 89
    uchar code_point_;
    signed offset_;
    static const int kNoChar = (1 << 21) - 1;
90 91 92 93 94 95 96 97 98 99
  };
  static const int kSize = size;
  static const int kMask = kSize - 1;
  CacheEntry entries_[kSize];
};

class UnicodeData {
 private:
  friend class Test;
  static int GetByteCount();
100
  static const uchar kMaxCodePoint;
101 102
};

103 104
class Utf16 {
 public:
105
  static inline bool IsLeadSurrogate(int code) {
106 107 108
    if (code == kNoPreviousCharacter) return false;
    return (code & 0xfc00) == 0xd800;
  }
109
  static inline bool IsTrailSurrogate(int code) {
110 111 112 113
    if (code == kNoPreviousCharacter) return false;
    return (code & 0xfc00) == 0xdc00;
  }

114
  static inline int CombineSurrogatePair(uchar lead, uchar trail) {
115 116
    return 0x10000 + ((lead & 0x3ff) << 10) + (trail & 0x3ff);
  }
117
  static const int kNoPreviousCharacter = -1;
118 119 120 121 122 123 124 125 126 127
  static const uchar kMaxNonSurrogateCharCode = 0xffff;
  // Encoding a single UTF-16 code unit will produce 1, 2 or 3 bytes
  // of UTF-8 data.  The special case where the unit is a surrogate
  // trail produces 1 byte net, because the encoding of the pair is
  // 4 bytes and the 3 bytes that were used to encode the lead surrogate
  // can be reclaimed.
  static const int kMaxExtraUtf8BytesForOneUtf16CodeUnit = 3;
  // One UTF-16 surrogate is endoded (illegally) as 3 UTF-8 bytes.
  // The illegality stems from the surrogate not being part of a pair.
  static const int kUtf8BytesToCodeASurrogate = 3;
128
  static inline uint16_t LeadSurrogate(uint32_t char_code) {
129 130
    return 0xd800 + (((char_code - 0x10000) >> 10) & 0x3ff);
  }
131
  static inline uint16_t TrailSurrogate(uint32_t char_code) {
132 133 134 135
    return 0xdc00 + (char_code & 0x3ff);
  }
};

136 137 138
class Latin1 {
 public:
  static const unsigned kMaxChar = 0xff;
139 140 141 142
  // Returns 0 if character does not convert to single latin-1 character
  // or if the character doesn't not convert back to latin-1 via inverse
  // operation (upper to lower, etc).
  static inline uint16_t ConvertNonLatin1ToLatin1(uint16_t);
143
};
144

145 146
class Utf8 {
 public:
147
  static inline uchar Length(uchar chr, int previous);
148
  static inline unsigned EncodeOneByte(char* out, uint8_t c);
149 150
  static inline unsigned Encode(
      char* out, uchar c, int previous);
151 152 153
  static uchar CalculateValue(const byte* str,
                              unsigned length,
                              unsigned* cursor);
154 155 156 157 158 159 160
  static const uchar kBadChar = 0xFFFD;
  static const unsigned kMaxEncodedSize   = 4;
  static const unsigned kMaxOneByteChar   = 0x7f;
  static const unsigned kMaxTwoByteChar   = 0x7ff;
  static const unsigned kMaxThreeByteChar = 0xffff;
  static const unsigned kMaxFourByteChar  = 0x1fffff;

161 162 163 164
  // A single surrogate is coded as a 3 byte UTF-8 sequence, but two together
  // that match are coded as a 4 byte UTF-8 sequence.
  static const unsigned kBytesSavedByCombiningSurrogates = 2;
  static const unsigned kSizeOfUnmatchedSurrogate = 3;
165 166 167 168 169 170
  static inline uchar ValueOf(const byte* str,
                              unsigned length,
                              unsigned* cursor);
};


171
class Utf8DecoderBase {
172
 public:
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
  // Initialization done in subclass.
  inline Utf8DecoderBase();
  inline Utf8DecoderBase(uint16_t* buffer,
                         unsigned buffer_length,
                         const uint8_t* stream,
                         unsigned stream_length);
  inline unsigned Utf16Length() const { return utf16_length_; }
 protected:
  // This reads all characters and sets the utf16_length_.
  // The first buffer_length utf16 chars are cached in the buffer.
  void Reset(uint16_t* buffer,
             unsigned buffer_length,
             const uint8_t* stream,
             unsigned stream_length);
  static void WriteUtf16Slow(const uint8_t* stream,
                             uint16_t* data,
                             unsigned length);
  const uint8_t* unbuffered_start_;
  unsigned utf16_length_;
  bool last_byte_of_buffer_unused_;
 private:
  DISALLOW_COPY_AND_ASSIGN(Utf8DecoderBase);
};

template <unsigned kBufferSize>
class Utf8Decoder : public Utf8DecoderBase {
 public:
  inline Utf8Decoder() {}
  inline Utf8Decoder(const char* stream, unsigned length);
  inline void Reset(const char* stream, unsigned length);
  inline unsigned WriteUtf16(uint16_t* data, unsigned length) const;
 private:
  uint16_t buffer_[kBufferSize];
206 207
};

208

209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236
struct Uppercase {
  static bool Is(uchar c);
};
struct Lowercase {
  static bool Is(uchar c);
};
struct Letter {
  static bool Is(uchar c);
};
struct Space {
  static bool Is(uchar c);
};
struct Number {
  static bool Is(uchar c);
};
struct WhiteSpace {
  static bool Is(uchar c);
};
struct LineTerminator {
  static bool Is(uchar c);
};
struct CombiningMark {
  static bool Is(uchar c);
};
struct ConnectorPunctuation {
  static bool Is(uchar c);
};
struct ToLowercase {
237
  static const int kMaxWidth = 3;
238 239 240 241 242 243
  static int Convert(uchar c,
                     uchar n,
                     uchar* result,
                     bool* allow_caching_ptr);
};
struct ToUppercase {
244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265
  static const int kMaxWidth = 3;
  static int Convert(uchar c,
                     uchar n,
                     uchar* result,
                     bool* allow_caching_ptr);
};
struct Ecma262Canonicalize {
  static const int kMaxWidth = 1;
  static int Convert(uchar c,
                     uchar n,
                     uchar* result,
                     bool* allow_caching_ptr);
};
struct Ecma262UnCanonicalize {
  static const int kMaxWidth = 4;
  static int Convert(uchar c,
                     uchar n,
                     uchar* result,
                     bool* allow_caching_ptr);
};
struct CanonicalizationRange {
  static const int kMaxWidth = 1;
266 267 268 269 270 271 272 273
  static int Convert(uchar c,
                     uchar n,
                     uchar* result,
                     bool* allow_caching_ptr);
};

}  // namespace unibrow

274
#endif  // V8_UNICODE_H_