Commit d2936564 authored by marja's avatar marja Committed by Commit bot

[scanner] Fix bom handling

The bug used to show up when
- we were streaming a script starting with 0xef 0xbb 0xbf
- we aborted preparsing a function (and reset to a bookmark)

BUG=chromium:685618

Review-Url: https://codereview.chromium.org/2663773002
Cr-Commit-Position: refs/heads/master@{#42790}
parent 6c84b93a
......@@ -15,6 +15,10 @@
namespace v8 {
namespace internal {
namespace {
const unibrow::uchar kUtf8Bom = 0xfeff;
} // namespace
// ----------------------------------------------------------------------------
// BufferedUtf16CharacterStreams
//
......@@ -263,7 +267,9 @@ bool Utf8ExternalStreamingStream::SkipToPosition(size_t position) {
while (it < chunk.length && chars < position) {
unibrow::uchar t =
unibrow::Utf8::ValueOfIncremental(chunk.data[it], &incomplete_char);
if (t != unibrow::Utf8::kIncomplete) {
if (t == kUtf8Bom && current_.pos.chars == 0) {
// BOM detected at beginning of the stream. Don't copy it.
} else if (t != unibrow::Utf8::kIncomplete) {
chars++;
if (t > unibrow::Utf16::kMaxNonSurrogateCharCode) chars++;
}
......@@ -304,8 +310,6 @@ void Utf8ExternalStreamingStream::FillBufferFromCurrentChunk() {
return;
}
static const unibrow::uchar kUtf8Bom = 0xfeff;
unibrow::Utf8::Utf8IncrementalBuffer incomplete_char =
current_.pos.incomplete_char;
size_t it;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment