scanner-character-streams.h 5.72 KB
Newer Older
1
// Copyright 2011 the V8 project authors. All rights reserved.
2 3
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
4

5 6
#ifndef V8_PARSING_SCANNER_CHARACTER_STREAMS_H_
#define V8_PARSING_SCANNER_CHARACTER_STREAMS_H_
7

8
#include "src/handles.h"
9
#include "src/parsing/scanner.h"
10
#include "src/vector.h"
11 12 13 14

namespace v8 {
namespace internal {

15 16 17
// Forward declarations.
class ExternalTwoByteString;

18 19 20
// A buffered character stream based on a random access character
// source (ReadBlock can be called with pos_ pointing to any position,
// even positions before the current).
21
class BufferedUtf16CharacterStream: public Utf16CharacterStream {
22
 public:
23
  BufferedUtf16CharacterStream();
24
  ~BufferedUtf16CharacterStream() override;
25

26
  void PushBack(uc32 character) override;
27 28

 protected:
29 30
  static const size_t kBufferSize = 512;
  static const size_t kPushBackStepSize = 16;
31

32 33
  size_t SlowSeekForward(size_t delta) override;
  bool ReadBlock() override;
34 35
  virtual void SlowPushBack(uc16 character);

36 37
  virtual size_t BufferSeekForward(size_t delta) = 0;
  virtual size_t FillBuffer(size_t position) = 0;
38 39 40 41 42 43 44

  const uc16* pushback_limit_;
  uc16 buffer_[kBufferSize];
};


// Generic string stream.
45
class GenericStringUtf16CharacterStream: public BufferedUtf16CharacterStream {
46
 public:
47 48
  GenericStringUtf16CharacterStream(Handle<String> data, size_t start_position,
                                    size_t end_position);
49
  ~GenericStringUtf16CharacterStream() override;
50

51 52
  bool SetBookmark() override;
  void ResetToBookmark() override;
53

54
 protected:
55 56
  static const size_t kNoBookmark = -1;

57 58
  size_t BufferSeekForward(size_t delta) override;
  size_t FillBuffer(size_t position) override;
59 60

  Handle<String> string_;
61
  size_t length_;
62
  size_t bookmark_;
63 64 65
};


66 67
// Utf16 stream based on a literal UTF-8 string.
class Utf8ToUtf16CharacterStream: public BufferedUtf16CharacterStream {
68
 public:
69
  Utf8ToUtf16CharacterStream(const byte* data, size_t length);
70
  ~Utf8ToUtf16CharacterStream() override;
71

72 73
  static size_t CopyChars(uint16_t* dest, size_t length, const byte* src,
                          size_t* src_pos, size_t src_length);
74

75
 protected:
76 77
  size_t BufferSeekForward(size_t delta) override;
  size_t FillBuffer(size_t char_position) override;
78
  void SetRawPosition(size_t char_position);
79 80

  const byte* raw_data_;
81 82
  size_t raw_data_length_;  // Measured in bytes, not characters.
  size_t raw_data_pos_;
83 84
  // The character position of the character at raw_data[raw_data_pos_].
  // Not necessarily the same as pos_.
85
  size_t raw_character_position_;
86 87 88
};


89 90 91 92 93 94 95 96 97 98 99
// ExternalStreamingStream is a wrapper around an ExternalSourceStream (see
// include/v8.h) subclass implemented by the embedder.
class ExternalStreamingStream : public BufferedUtf16CharacterStream {
 public:
  ExternalStreamingStream(ScriptCompiler::ExternalSourceStream* source_stream,
                          v8::ScriptCompiler::StreamedSource::Encoding encoding)
      : source_stream_(source_stream),
        encoding_(encoding),
        current_data_(NULL),
        current_data_offset_(0),
        current_data_length_(0),
100
        utf8_split_char_buffer_length_(0),
vogelheim's avatar
vogelheim committed
101
        bookmark_(0),
102 103
        bookmark_data_is_from_current_data_(false),
        bookmark_data_offset_(0),
vogelheim's avatar
vogelheim committed
104
        bookmark_utf8_split_char_buffer_length_(0) {}
105

106
  ~ExternalStreamingStream() override {
107 108 109 110
    delete[] current_data_;
    bookmark_buffer_.Dispose();
    bookmark_data_.Dispose();
  }
111

112
  size_t BufferSeekForward(size_t delta) override {
113 114 115 116 117 118 119 120
    // We never need to seek forward when streaming scripts. We only seek
    // forward when we want to parse a function whose location we already know,
    // and when streaming, we don't know the locations of anything we haven't
    // seen yet.
    UNREACHABLE();
    return 0;
  }

121
  size_t FillBuffer(size_t position) override;
122

123 124
  bool SetBookmark() override;
  void ResetToBookmark() override;
125

126
 private:
127
  void HandleUtf8SplitCharacters(size_t* data_in_buffer);
128
  void FlushCurrent();
129 130 131 132

  ScriptCompiler::ExternalSourceStream* source_stream_;
  v8::ScriptCompiler::StreamedSource::Encoding encoding_;
  const uint8_t* current_data_;
133 134
  size_t current_data_offset_;
  size_t current_data_length_;
135 136
  // For converting UTF-8 characters which are split across two data chunks.
  uint8_t utf8_split_char_buffer_[4];
137
  size_t utf8_split_char_buffer_length_;
138 139 140 141 142 143

  // Bookmark support. See comments in ExternalStreamingStream::SetBookmark
  // for additional details.
  size_t bookmark_;
  Vector<uint16_t> bookmark_buffer_;
  Vector<uint8_t> bookmark_data_;
144 145
  bool bookmark_data_is_from_current_data_;
  size_t bookmark_data_offset_;
vogelheim's avatar
vogelheim committed
146 147
  uint8_t bookmark_utf8_split_char_buffer_[4];
  size_t bookmark_utf8_split_char_buffer_length_;
148 149 150
};


151
// UTF16 buffer to read characters from an external string.
152
class ExternalTwoByteStringUtf16CharacterStream: public Utf16CharacterStream {
153
 public:
154 155 156
  ExternalTwoByteStringUtf16CharacterStream(Handle<ExternalTwoByteString> data,
                                            int start_position,
                                            int end_position);
157
  ~ExternalTwoByteStringUtf16CharacterStream() override;
158

159
  void PushBack(uc32 character) override {
160
    DCHECK(buffer_cursor_ > raw_data_);
161 162 163 164
    buffer_cursor_--;
    pos_--;
  }

165 166
  bool SetBookmark() override;
  void ResetToBookmark() override;
167

168
 protected:
169
  size_t SlowSeekForward(size_t delta) override {
170 171 172
    // Fast case always handles seeking.
    return 0;
  }
173
  bool ReadBlock() override {
174 175 176 177 178
    // Entire string is read at start.
    return false;
  }
  Handle<ExternalTwoByteString> source_;
  const uc16* raw_data_;  // Pointer to the actual array of characters.
179 180 181 182 183

 private:
  static const size_t kNoBookmark = -1;

  size_t bookmark_;
184 185
};

186 187
}  // namespace internal
}  // namespace v8
188

189
#endif  // V8_PARSING_SCANNER_CHARACTER_STREAMS_H_