Commit d2936564 authored by marja's avatar marja Committed by Commit bot

[scanner] Fix bom handling

The bug used to show up when
- we were streaming a script starting with 0xef 0xbb 0xbf
- we aborted preparsing a function (and reset to a bookmark)

BUG=chromium:685618

Review-Url: https://codereview.chromium.org/2663773002
Cr-Commit-Position: refs/heads/master@{#42790}
parent 6c84b93a
...@@ -15,6 +15,10 @@ ...@@ -15,6 +15,10 @@
namespace v8 { namespace v8 {
namespace internal { namespace internal {
namespace {
const unibrow::uchar kUtf8Bom = 0xfeff;
} // namespace
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
// BufferedUtf16CharacterStreams // BufferedUtf16CharacterStreams
// //
...@@ -263,7 +267,9 @@ bool Utf8ExternalStreamingStream::SkipToPosition(size_t position) { ...@@ -263,7 +267,9 @@ bool Utf8ExternalStreamingStream::SkipToPosition(size_t position) {
while (it < chunk.length && chars < position) { while (it < chunk.length && chars < position) {
unibrow::uchar t = unibrow::uchar t =
unibrow::Utf8::ValueOfIncremental(chunk.data[it], &incomplete_char); unibrow::Utf8::ValueOfIncremental(chunk.data[it], &incomplete_char);
if (t != unibrow::Utf8::kIncomplete) { if (t == kUtf8Bom && current_.pos.chars == 0) {
// BOM detected at beginning of the stream. Don't copy it.
} else if (t != unibrow::Utf8::kIncomplete) {
chars++; chars++;
if (t > unibrow::Utf16::kMaxNonSurrogateCharCode) chars++; if (t > unibrow::Utf16::kMaxNonSurrogateCharCode) chars++;
} }
...@@ -304,8 +310,6 @@ void Utf8ExternalStreamingStream::FillBufferFromCurrentChunk() { ...@@ -304,8 +310,6 @@ void Utf8ExternalStreamingStream::FillBufferFromCurrentChunk() {
return; return;
} }
static const unibrow::uchar kUtf8Bom = 0xfeff;
unibrow::Utf8::Utf8IncrementalBuffer incomplete_char = unibrow::Utf8::Utf8IncrementalBuffer incomplete_char =
current_.pos.incomplete_char; current_.pos.incomplete_char;
size_t it; size_t it;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment