[scanner] Templatize scan functions by encoding

This way we can avoid reencoding everything to utf16 (buffered) and avoid the
overhead of needing to check the encoding for each character individually.

This may result in a minor asm.js scanning regression due to one-byte tokens
possibly being more common.

Change-Id: I90b51c256d56d4f4fa2d235d7e1e58fc01e43f31
Reviewed-on: https://chromium-review.googlesource.com/1172437
Commit-Queue: Toon Verwaest <verwaest@chromium.org>
Reviewed-by: Andreas Haas <ahaas@chromium.org>
Reviewed-by: Leszek Swirski <leszeks@chromium.org>
Cr-Commit-Position: refs/heads/master@{#55217}

src/asmjs/asm-scanner.cc

@@ -20,7 +20,11 @@ namespace {
 static const int kMaxIdentifierCount = 0xF000000;
 };
 
-AsmJsScanner::AsmJsScanner(CharacterStream<uint16_t>* stream, int start)
+#define SPECIALIZE(Call, ...)                           \
+  (stream_->is_two_byte() ? Call<uint16_t>(__VA_ARGS__) \
+                          : Call<uint8_t>(__VA_ARGS__))
+
+AsmJsScanner::AsmJsScanner(ScannerStream* stream, int start)
     : stream_(stream),
       token_(kUninitialized),
       preceding_token_(kUninitialized),
@@ -34,7 +38,7 @@ AsmJsScanner::AsmJsScanner(CharacterStream<uint16_t>* stream, int start)
       double_value_(0.0),
       unsigned_value_(0),
       preceded_by_newline_(false) {
-  stream->Seek(start);
+  SPECIALIZE(DoSeek, start);
 #define V(name, _junk1, _junk2, _junk3) property_names_[#name] = kToken_##name;
   STDLIB_MATH_FUNCTION_LIST(V)
   STDLIB_ARRAY_TYPE_LIST(V)
@@ -51,7 +55,10 @@ AsmJsScanner::AsmJsScanner(CharacterStream<uint16_t>* stream, int start)
   Next();
 }
 
-void AsmJsScanner::Next() {
+void AsmJsScanner::Next() { SPECIALIZE(Scan); }
+
+template <typename Char>
+void AsmJsScanner::Scan() {
   if (rewind_) {
     preceding_token_ = token_;
     preceding_position_ = position_;
@@ -85,8 +92,8 @@ void AsmJsScanner::Next() {
   preceding_position_ = position_;
 
   for (;;) {
-    position_ = stream_->pos();
-    uc32 ch = stream_->Advance();
+    position_ = Source<Char>()->pos();
+    uc32 ch = Advance<Char>();
     switch (ch) {
       case ' ':
       case '\t':
@@ -106,20 +113,20 @@ void AsmJsScanner::Next() {
 
       case '\'':
       case '"':
-        ConsumeString(ch);
+        ConsumeString<Char>(ch);
         return;
 
       case '/':
-        ch = stream_->Advance();
+        ch = Advance<Char>();
         if (ch == '/') {
-          ConsumeCPPComment();
+          ConsumeCPPComment<Char>();
         } else if (ch == '*') {
-          if (!ConsumeCComment()) {
+          if (!ConsumeCComment<Char>()) {
             token_ = kParseError;
             return;
           }
         } else {
-          stream_->Back();
+          Back<Char>();
           token_ = '/';
           return;
         }
@@ -131,7 +138,7 @@ void AsmJsScanner::Next() {
       case '>':
       case '=':
       case '!':
-        ConsumeCompareOrShift(ch);
+        ConsumeCompareOrShift<Char>(ch);
         return;
 
 #define V(single_char_token) case single_char_token:
@@ -143,9 +150,9 @@ void AsmJsScanner::Next() {
 
       default:
         if (IsIdentifierStart(ch)) {
-          ConsumeIdentifier(ch);
+          ConsumeIdentifier<Char>(ch);
         } else if (IsNumberStart(ch)) {
-          ConsumeNumber(ch);
+          ConsumeNumber<Char>(ch);
         } else {
           // TODO(bradnelson): Support unicode (probably via UnicodeCache).
           token_ = kParseError;
@@ -213,7 +220,7 @@ std::string AsmJsScanner::Name(token_t token) const {
 #endif
 
 void AsmJsScanner::Seek(size_t pos) {
-  stream_->Seek(pos);
+  SPECIALIZE(DoSeek, pos);
   preceding_token_ = kUninitialized;
   token_ = kUninitialized;
   next_token_ = kUninitialized;
@@ -224,15 +231,16 @@ void AsmJsScanner::Seek(size_t pos) {
   Next();
 }
 
+template <typename Char>
 void AsmJsScanner::ConsumeIdentifier(uc32 ch) {
   // Consume characters while still part of the identifier.
   identifier_string_.clear();
   while (IsIdentifierPart(ch)) {
     identifier_string_ += ch;
-    ch = stream_->Advance();
+    ch = Advance<Char>();
   }
   // Go back one for next time.
-  stream_->Back();
+  Back<Char>();
 
   // Decode what the identifier means.
   if (preceding_token_ == '.') {
@@ -272,13 +280,14 @@ void AsmJsScanner::ConsumeIdentifier(uc32 ch) {
   }
 }
 
+template <typename Char>
 void AsmJsScanner::ConsumeNumber(uc32 ch) {
   std::string number;
   number = ch;
   bool has_dot = ch == '.';
   bool has_prefix = false;
   for (;;) {
-    ch = stream_->Advance();
+    ch = Advance<Char>();
     if ((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') ||
         (ch >= 'A' && ch <= 'F') || ch == '.' || ch == 'b' || ch == 'o' ||
         ch == 'x' ||
@@ -297,7 +306,7 @@ void AsmJsScanner::ConsumeNumber(uc32 ch) {
       break;
     }
   }
-  stream_->Back();
+  Back<Char>();
   // Special case the most common number.
   if (number.size() == 1 && number[0] == '0') {
     unsigned_value_ = 0;
@@ -324,7 +333,7 @@ void AsmJsScanner::ConsumeNumber(uc32 ch) {
     // problem.
     if (number[0] == '.') {
       for (size_t k = 1; k < number.size(); ++k) {
-        stream_->Back();
+        Back<Char>();
       }
       token_ = '.';
       return;
@@ -346,11 +355,12 @@ void AsmJsScanner::ConsumeNumber(uc32 ch) {
   }
 }
 
+template <typename Char>
 bool AsmJsScanner::ConsumeCComment() {
   for (;;) {
-    uc32 ch = stream_->Advance();
+    uc32 ch = Advance<Char>();
     while (ch == '*') {
-      ch = stream_->Advance();
+      ch = Advance<Char>();
       if (ch == '/') {
         return true;
       }
@@ -361,33 +371,36 @@ bool AsmJsScanner::ConsumeCComment() {
   }
 }
 
+template <typename Char>
 void AsmJsScanner::ConsumeCPPComment() {
   for (;;) {
-    uc32 ch = stream_->Advance();
+    uc32 ch = Advance<Char>();
     if (ch == '\n' || ch == kEndOfInput) {
       return;
     }
   }
 }
 
+template <typename Char>
 void AsmJsScanner::ConsumeString(uc32 quote) {
   // Only string allowed is 'use asm' / "use asm".
   const char* expected = "use asm";
   for (; *expected != '\0'; ++expected) {
-    if (stream_->Advance() != *expected) {
+    if (Advance<Char>() != *expected) {
       token_ = kParseError;
       return;
     }
   }
-  if (stream_->Advance() != quote) {
+  if (Advance<Char>() != quote) {
     token_ = kParseError;
     return;
   }
   token_ = kToken_UseAsm;
 }
 
+template <typename Char>
 void AsmJsScanner::ConsumeCompareOrShift(uc32 ch) {
-  uc32 next_ch = stream_->Advance();
+  uc32 next_ch = Advance<Char>();
   if (next_ch == '=') {
     switch (ch) {
       case '<':
@@ -408,14 +421,14 @@ void AsmJsScanner::ConsumeCompareOrShift(uc32 ch) {
   } else if (ch == '<' && next_ch == '<') {
     token_ = kToken_SHL;
   } else if (ch == '>' && next_ch == '>') {
-    if (stream_->Advance() == '>') {
+    if (Advance<Char>() == '>') {
       token_ = kToken_SHR;
     } else {
       token_ = kToken_SAR;
-      stream_->Back();
+      Back<Char>();
     }
   } else {
-    stream_->Back();
+    Back<Char>();
     token_ = ch;
   }
 }
@@ -430,5 +443,7 @@ bool AsmJsScanner::IsNumberStart(uc32 ch) {
   return ch == '.' || IsDecimalDigit(ch);
 }
 
+#undef SPECIALIZE
+
 }  // namespace internal
 }  // namespace v8

src/asmjs/asm-scanner.h

@@ -16,6 +16,7 @@
 namespace v8 {
 namespace internal {
 
+class ScannerStream;
 template <typename Char>
 class CharacterStream;
 
@@ -32,7 +33,7 @@ class V8_EXPORT_PRIVATE AsmJsScanner {
  public:
   typedef int32_t token_t;
 
-  AsmJsScanner(CharacterStream<uint16_t>* stream, int start);
+  AsmJsScanner(ScannerStream* stream, int start);
 
   // Get current token.
   token_t Token() const { return token_; }
@@ -137,7 +138,7 @@ class V8_EXPORT_PRIVATE AsmJsScanner {
   // clang-format on
 
  private:
-  CharacterStream<uint16_t>* stream_;
+  ScannerStream* const stream_;
   token_t token_;
   token_t preceding_token_;
   token_t next_token_;         // Only set when in {rewind} state.
@@ -155,12 +156,37 @@ class V8_EXPORT_PRIVATE AsmJsScanner {
   uint32_t unsigned_value_;
   bool preceded_by_newline_;
 
+  template <typename Char>
+  void Scan();
+  template <typename Char>
+  inline CharacterStream<Char>* Source() {
+    return static_cast<CharacterStream<Char>*>(stream_);
+  }
+  template <typename Char>
+  inline uc32 Advance() {
+    return Source<Char>()->Advance();
+  }
+  template <typename Char>
+  inline void Back() {
+    return Source<Char>()->Back();
+  }
+  template <typename Char>
+  void DoSeek(size_t pos) {
+    Source<Char>()->Seek(pos);
+  }
+
   // Consume multiple characters.
+  template <typename Char>
   void ConsumeIdentifier(uc32 ch);
+  template <typename Char>
   void ConsumeNumber(uc32 ch);
+  template <typename Char>
   bool ConsumeCComment();
+  template <typename Char>
   void ConsumeCPPComment();
+  template <typename Char>
   void ConsumeString(uc32 quote);
+  template <typename Char>
   void ConsumeCompareOrShift(uc32 ch);
 
   // Classify character categories.

src/parsing/parser.cc

@@ -412,7 +412,8 @@ Parser::Parser(ParseInfo* info)
                          info->runtime_call_stats(), info->logger(),
                          info->script().is_null() ? -1 : info->script()->id(),
                          info->is_module(), true),
-      scanner_(info->unicode_cache()),
+      scanner_(info->unicode_cache(), info->character_stream(),
+               info->is_module()),
       reusable_preparser_(nullptr),
       mode_(PARSE_EAGERLY),  // Lazy mode must be set explicitly.
       source_range_map_(info->source_range_map()),
@@ -507,9 +508,8 @@ FunctionLiteral* Parser::ParseProgram(Isolate* isolate, ParseInfo* info) {
   // Initialize parser state.
   DeserializeScopeChain(isolate, info, info->maybe_outer_scope_info());
 
-  auto stream =
-      static_cast<CharacterStream<uint16_t>*>(info->character_stream());
-  scanner_.Initialize(stream, info->is_module());
+  scanner_.Initialize();
+
   FunctionLiteral* result = DoParseProgram(isolate, info);
   MaybeResetCharacterStream(info, result);
 
@@ -703,9 +703,8 @@ FunctionLiteral* Parser::ParseFunction(Isolate* isolate, ParseInfo* info,
   // Initialize parser state.
   Handle<String> name(shared_info->Name(), isolate);
   info->set_function_name(ast_value_factory()->GetString(name));
-  auto stream =
-      static_cast<CharacterStream<uint16_t>*>(info->character_stream());
-  scanner_.Initialize(stream, info->is_module());
+
+  scanner_.Initialize();
 
   FunctionLiteral* result =
       DoParseFunction(isolate, info, info->function_name());
@@ -3451,9 +3450,8 @@ void Parser::ParseOnBackground(ParseInfo* info) {
   DCHECK_NULL(info->literal());
   FunctionLiteral* result = nullptr;
 
-  auto stream =
-      static_cast<CharacterStream<uint16_t>*>(info->character_stream());
-  scanner_.Initialize(stream, info->is_module());
+  scanner_.Initialize();
+
   DCHECK(info->maybe_outer_scope_info().is_null());
 
   DCHECK(original_scope_);

src/parsing/scanner-character-streams.cc

@@ -250,82 +250,45 @@ class Utf8ChunkedStream : public ChunkedStream<uint16_t> {
   bool seen_bom_ = false;
 };
 
-// Provides a buffered utf-16 view on the bytes from the underlying ByteStream.
-// Chars are buffered if either the underlying stream isn't utf-16 or the
-// underlying utf-16 stream might move (is on-heap).
-template <template <typename T> class ByteStream>
-class BufferedCharacterStream : public CharacterStream<uint16_t> {
- public:
-  template <class... TArgs>
-  BufferedCharacterStream(size_t pos, TArgs... args) : byte_stream_(args...) {
-    buffer_pos_ = pos;
-  }
-
- protected:
-  bool ReadBlock() final {
-    size_t position = pos();
-    buffer_pos_ = position;
-    buffer_start_ = &buffer_[0];
-    buffer_cursor_ = buffer_start_;
-
-    Range<uint8_t> range = byte_stream_.GetDataAt(position);
-    if (range.length() == 0) {
-      buffer_end_ = buffer_start_;
-      return false;
-    }
-
-    size_t length = Min(kBufferSize, range.length());
-    i::CopyCharsUnsigned(buffer_, range.start, length);
-    buffer_end_ = &buffer_[length];
-    return true;
-  }
-
-  bool can_access_heap() final { return ByteStream<uint8_t>::kCanAccessHeap; }
-
- private:
-  static const size_t kBufferSize = 512;
-  uc16 buffer_[kBufferSize];
-  ByteStream<uint8_t> byte_stream_;
-};
-
 // Provides a unbuffered utf-16 view on the bytes from the underlying
 // ByteStream.
-template <template <typename T> class ByteStream>
-class UnbufferedCharacterStream : public CharacterStream<uint16_t> {
+template <typename Char, template <typename T> class ByteStream>
+class UnbufferedCharacterStream : public CharacterStream<Char> {
  public:
   template <class... TArgs>
   UnbufferedCharacterStream(size_t pos, TArgs... args) : byte_stream_(args...) {
-    buffer_pos_ = pos;
+    this->buffer_pos_ = pos;
   }
 
  protected:
   bool ReadBlock() final {
-    size_t position = pos();
-    buffer_pos_ = position;
-    Range<uint16_t> range = byte_stream_.GetDataAt(position);
-    buffer_start_ = range.start;
-    buffer_end_ = range.end;
-    buffer_cursor_ = buffer_start_;
+    size_t position = this->pos();
+    this->buffer_pos_ = position;
+    Range<Char> range = this->byte_stream_.GetDataAt(position);
+    this->buffer_start_ = range.start;
+    this->buffer_end_ = range.end;
+    this->buffer_cursor_ = range.start;
     if (range.length() == 0) return false;
 
     DCHECK(!range.unaligned_start());
-    DCHECK_LE(buffer_start_, buffer_end_);
+    DCHECK_LE(this->buffer_start_, this->buffer_end_);
     return true;
   }
 
-  bool can_access_heap() final { return ByteStream<uint16_t>::kCanAccessHeap; }
+  bool can_access_heap() final { return ByteStream<Char>::kCanAccessHeap; }
 
-  ByteStream<uint16_t> byte_stream_;
+  ByteStream<Char> byte_stream_;
 };
 
 // Provides a unbuffered utf-16 view on the bytes from the underlying
 // ByteStream.
+template <typename Char>
 class RelocatingCharacterStream
-    : public UnbufferedCharacterStream<OnHeapStream> {
+    : public UnbufferedCharacterStream<Char, OnHeapStream> {
  public:
   template <class... TArgs>
   RelocatingCharacterStream(Isolate* isolate, size_t pos, TArgs... args)
-      : UnbufferedCharacterStream<OnHeapStream>(pos, args...),
+      : UnbufferedCharacterStream<Char, OnHeapStream>(pos, args...),
         isolate_(isolate) {
     isolate->heap()->AddGCEpilogueCallback(UpdateBufferPointersCallback,
                                            v8::kGCTypeAll, this);
@@ -341,16 +304,17 @@ class RelocatingCharacterStream
                                            v8::GCType type,
                                            v8::GCCallbackFlags flags,
                                            void* stream) {
-    reinterpret_cast<RelocatingCharacterStream*>(stream)
+    reinterpret_cast<RelocatingCharacterStream<Char>*>(stream)
         ->UpdateBufferPointers();
   }
 
   void UpdateBufferPointers() {
-    Range<uint16_t> range = byte_stream_.GetDataAt(0);
-    if (range.start != buffer_start_) {
-      buffer_cursor_ = (buffer_cursor_ - buffer_start_) + range.start;
-      buffer_start_ = range.start;
-      buffer_end_ = range.end;
+    Range<Char> range = this->byte_stream_.GetDataAt(0);
+    if (range.start != this->buffer_start_) {
+      this->buffer_cursor_ =
+          (this->buffer_cursor_ - this->buffer_start_) + range.start;
+      this->buffer_start_ = range.start;
+      this->buffer_end_ = range.end;
     }
   }
 
@@ -360,6 +324,21 @@ class RelocatingCharacterStream
 // ----------------------------------------------------------------------------
 // ScannerStream: Create stream instances.
 
+#define SPECIALIZE(Call, ...)                                             \
+  (is_two_byte_                                                           \
+       ? static_cast<CharacterStream<uint16_t>*>(this)->Call(__VA_ARGS__) \
+       : static_cast<CharacterStream<uint8_t>*>(this)->Call(__VA_ARGS__))
+
+uc32 ScannerStream::Advance() { return SPECIALIZE(Advance); }
+
+void ScannerStream::Seek(size_t pos) { SPECIALIZE(Seek, pos); }
+
+size_t ScannerStream::pos() { return SPECIALIZE(pos); }
+
+void ScannerStream::Back() { SPECIALIZE(Back); }
+
+#undef SPECIALIZE
+
 ScannerStream* ScannerStream::For(Isolate* isolate, Handle<String> data) {
   return ScannerStream::For(isolate, data, 0, data->length());
 }
@@ -380,21 +359,22 @@ ScannerStream* ScannerStream::For(Isolate* isolate, Handle<String> data,
     data = String::Flatten(isolate, data);
   }
   if (data->IsExternalOneByteString()) {
-    return new BufferedCharacterStream<ExternalStringStream>(
+    return new UnbufferedCharacterStream<uint8_t, ExternalStringStream>(
         static_cast<size_t>(start_pos),
         ExternalOneByteString::cast(*data)->GetChars() + start_offset,
         static_cast<size_t>(end_pos));
   } else if (data->IsExternalTwoByteString()) {
-    return new UnbufferedCharacterStream<ExternalStringStream>(
+    return new UnbufferedCharacterStream<uint16_t, ExternalStringStream>(
         static_cast<size_t>(start_pos),
         ExternalTwoByteString::cast(*data)->GetChars() + start_offset,
         static_cast<size_t>(end_pos));
   } else if (data->IsSeqOneByteString()) {
-    return new BufferedCharacterStream<OnHeapStream>(
-        static_cast<size_t>(start_pos), Handle<SeqOneByteString>::cast(data),
-        start_offset, static_cast<size_t>(end_pos));
+    return new RelocatingCharacterStream<uint8_t>(
+        isolate, static_cast<size_t>(start_pos),
+        Handle<SeqOneByteString>::cast(data), start_offset,
+        static_cast<size_t>(end_pos));
   } else if (data->IsSeqTwoByteString()) {
-    return new RelocatingCharacterStream(
+    return new RelocatingCharacterStream<uint16_t>(
         isolate, static_cast<size_t>(start_pos),
         Handle<SeqTwoByteString>::cast(data), start_offset,
         static_cast<size_t>(end_pos));
@@ -403,15 +383,15 @@ ScannerStream* ScannerStream::For(Isolate* isolate, Handle<String> data,
   }
 }
 
-std::unique_ptr<CharacterStream<uint16_t>> ScannerStream::ForTesting(
+std::unique_ptr<CharacterStream<uint8_t>> ScannerStream::ForTesting(
     const char* data) {
   return ScannerStream::ForTesting(data, strlen(data));
 }
 
-std::unique_ptr<CharacterStream<uint16_t>> ScannerStream::ForTesting(
+std::unique_ptr<CharacterStream<uint8_t>> ScannerStream::ForTesting(
     const char* data, size_t length) {
-  return std::unique_ptr<CharacterStream<uint16_t>>(
-      new BufferedCharacterStream<ExternalStringStream>(
+  return std::unique_ptr<CharacterStream<uint8_t>>(
+      new UnbufferedCharacterStream<uint8_t, ExternalStringStream>(
           static_cast<size_t>(0), reinterpret_cast<const uint8_t*>(data),
           static_cast<size_t>(length)));
 }
@@ -422,13 +402,13 @@ ScannerStream* ScannerStream::For(
     RuntimeCallStats* stats) {
   switch (encoding) {
     case v8::ScriptCompiler::StreamedSource::TWO_BYTE:
-      return new UnbufferedCharacterStream<ChunkedStream>(
+      return new UnbufferedCharacterStream<uint16_t, ChunkedStream>(
           static_cast<size_t>(0), source_stream, stats);
     case v8::ScriptCompiler::StreamedSource::ONE_BYTE:
-      return new BufferedCharacterStream<ChunkedStream>(static_cast<size_t>(0),
-                                                        source_stream, stats);
+      return new UnbufferedCharacterStream<uint8_t, ChunkedStream>(
+          static_cast<size_t>(0), source_stream, stats);
     case v8::ScriptCompiler::StreamedSource::UTF8:
-      return new UnbufferedCharacterStream<Utf8ChunkedStream>(
+      return new UnbufferedCharacterStream<uint16_t, Utf8ChunkedStream>(
           static_cast<size_t>(0), source_stream, stats);
   }
   UNREACHABLE();

src/parsing/scanner-character-streams.h

@@ -32,19 +32,27 @@ class V8_EXPORT_PRIVATE ScannerStream {
                             RuntimeCallStats* stats);
 
   // For testing:
-  static std::unique_ptr<CharacterStream<uint16_t>> ForTesting(
-      const char* data);
-  static std::unique_ptr<CharacterStream<uint16_t>> ForTesting(const char* data,
-                                                               size_t length);
+  static std::unique_ptr<CharacterStream<uint8_t>> ForTesting(const char* data);
+  static std::unique_ptr<CharacterStream<uint8_t>> ForTesting(const char* data,
+                                                              size_t length);
 
   // Returns true if the stream could access the V8 heap after construction.
   virtual bool can_access_heap() = 0;
-  virtual uc32 Advance() = 0;
-  virtual void Seek(size_t pos) = 0;
-  virtual size_t pos() const = 0;
-  virtual void Back() = 0;
+  uc32 Advance();
+  void Seek(size_t pos);
+  size_t pos();
+  void Back();
+  void Back2();
 
   virtual ~ScannerStream() {}
+
+  bool is_two_byte() const { return is_two_byte_; }
+
+ protected:
+  explicit ScannerStream(bool is_two_byte) : is_two_byte_(is_two_byte) {}
+
+ private:
+  const bool is_two_byte_;
 };
 
 template <typename Char>
@@ -52,7 +60,7 @@ class CharacterStream : public ScannerStream {
  public:
   // Returns and advances past the next UTF-16 code unit in the input
   // stream. If there are no more code units it returns kEndOfInput.
-  inline uc32 Advance() final {
+  inline uc32 Advance() {
     uc32 result = Peek();
     buffer_cursor_++;
     return result;
@@ -95,7 +103,7 @@ class CharacterStream : public ScannerStream {
 
   // Go back one by one character in the input stream.
   // This undoes the most recent Advance().
-  inline void Back() final {
+  inline void Back() {
     // The common case - if the previous character is within
     // buffer_start_ .. buffer_end_ will be handles locally.
     // Otherwise, a new block is requested.
@@ -106,11 +114,11 @@ class CharacterStream : public ScannerStream {
     }
   }
 
-  inline size_t pos() const final {
+  inline size_t pos() const {
     return buffer_pos_ + (buffer_cursor_ - buffer_start_);
   }
 
-  inline void Seek(size_t pos) final {
+  inline void Seek(size_t pos) {
     if (V8_LIKELY(pos >= buffer_pos_ &&
                   pos < (buffer_pos_ + (buffer_end_ - buffer_start_)))) {
       buffer_cursor_ = buffer_start_ + (pos - buffer_pos_);
@@ -119,13 +127,11 @@ class CharacterStream : public ScannerStream {
     }
   }
 
-  // Returns true if the stream could access the V8 heap after construction.
-  virtual bool can_access_heap() = 0;
-
  protected:
-  CharacterStream(const uint16_t* buffer_start, const uint16_t* buffer_cursor,
-                  const uint16_t* buffer_end, size_t buffer_pos)
-      : buffer_start_(buffer_start),
+  CharacterStream(const Char* buffer_start, const Char* buffer_cursor,
+                  const Char* buffer_end, size_t buffer_pos)
+      : ScannerStream(sizeof(Char) == 2),
+        buffer_start_(buffer_start),
         buffer_cursor_(buffer_cursor),
         buffer_end_(buffer_end),
         buffer_pos_(buffer_pos) {}

src/parsing/scanner-inl.h

@@ -11,8 +11,9 @@
 namespace v8 {
 namespace internal {
 
+template <typename Char>
 V8_INLINE Token::Value Scanner::SkipWhiteSpace() {
-  int start_position = source_pos();
+  int start_position = SourcePos<Char>();
 
   while (true) {
     // We won't skip behind the end of input.
@@ -25,11 +26,11 @@ V8_INLINE Token::Value Scanner::SkipWhiteSpace() {
     } else if (!unicode_cache_->IsWhiteSpace(c0_)) {
       break;
     }
-    Advance();
+    Advance<Char>();
   }
 
   // Return whether or not we skipped any characters.
-  if (source_pos() == start_position) {
+  if (SourcePos<Char>() == start_position) {
     DCHECK_NE('0', c0_);
     return Token::ILLEGAL;
   }

test/cctest/parsing/test-scanner-streams.cc

@@ -246,7 +246,7 @@ TEST(AdvanceMatchAdvanceUntil) {
             nullptr));
 
     int32_t au_c0 =
-        static_cast<i::CharacterStream<uint16_t>*>(stream_advance_until.get())
+        static_cast<i::CharacterStream<uint8_t>*>(stream_advance_until.get())
             ->AdvanceUntil(
                 [](int32_t c0) { return unibrow::IsLineTerminator(c0); });
 

test/cctest/parsing/test-scanner.cc

@@ -27,7 +27,7 @@ struct ScannerTestHelper {
         scanner(std::move(other.scanner)) {}
 
   std::unique_ptr<UnicodeCache> unicode_cache;
-  std::unique_ptr<CharacterStream<uint16_t>> stream;
+  std::unique_ptr<CharacterStream<uint8_t>> stream;
   std::unique_ptr<Scanner> scanner;
 
   Scanner* operator->() const { return scanner.get(); }
@@ -38,9 +38,9 @@ ScannerTestHelper make_scanner(const char* src) {
   ScannerTestHelper helper;
   helper.unicode_cache = std::unique_ptr<UnicodeCache>(new UnicodeCache);
   helper.stream = ScannerStream::ForTesting(src);
-  helper.scanner =
-      std::unique_ptr<Scanner>(new Scanner(helper.unicode_cache.get()));
-  helper.scanner->Initialize(helper.stream.get(), false);
+  helper.scanner = std::unique_ptr<Scanner>(
+      new Scanner(helper.unicode_cache.get(), helper.stream.get(), false));
+  helper.scanner->Initialize();
   return helper;
 }
 

test/cctest/test-parsing.cc

@@ -92,16 +92,16 @@ TEST(ScanKeywords) {
     CHECK(static_cast<int>(sizeof(buffer)) >= length);
     {
       auto stream = i::ScannerStream::ForTesting(keyword, length);
-      i::Scanner scanner(&unicode_cache);
-      scanner.Initialize(stream.get(), false);
+      i::Scanner scanner(&unicode_cache, stream.get(), false);
+      scanner.Initialize();
       CHECK_EQ(key_token.token, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
     }
     // Removing characters will make keyword matching fail.
     {
       auto stream = i::ScannerStream::ForTesting(keyword, length - 1);
-      i::Scanner scanner(&unicode_cache);
-      scanner.Initialize(stream.get(), false);
+      i::Scanner scanner(&unicode_cache, stream.get(), false);
+      scanner.Initialize();
       CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
     }
@@ -111,8 +111,8 @@ TEST(ScanKeywords) {
       i::MemMove(buffer, keyword, length);
       buffer[length] = chars_to_append[j];
       auto stream = i::ScannerStream::ForTesting(buffer, length + 1);
-      i::Scanner scanner(&unicode_cache);
-      scanner.Initialize(stream.get(), false);
+      i::Scanner scanner(&unicode_cache, stream.get(), false);
+      scanner.Initialize();
       CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
     }
@@ -121,8 +121,8 @@ TEST(ScanKeywords) {
       i::MemMove(buffer, keyword, length);
       buffer[length - 1] = '_';
       auto stream = i::ScannerStream::ForTesting(buffer, length);
-      i::Scanner scanner(&unicode_cache);
-      scanner.Initialize(stream.get(), false);
+      i::Scanner scanner(&unicode_cache, stream.get(), false);
+      scanner.Initialize();
       CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
     }
@@ -188,8 +188,8 @@ TEST(ScanHTMLEndComments) {
   for (int i = 0; tests[i]; i++) {
     const char* source = tests[i];
     auto stream = i::ScannerStream::ForTesting(source);
-    i::Scanner scanner(i_isolate->unicode_cache());
-    scanner.Initialize(stream.get(), false);
+    i::Scanner scanner(i_isolate->unicode_cache(), stream.get(), false);
+    scanner.Initialize();
     i::Zone zone(i_isolate->allocator(), ZONE_NAME);
     i::AstValueFactory ast_value_factory(&zone,
                                          i_isolate->ast_string_constants(),
@@ -207,8 +207,8 @@ TEST(ScanHTMLEndComments) {
   for (int i = 0; fail_tests[i]; i++) {
     const char* source = fail_tests[i];
     auto stream = i::ScannerStream::ForTesting(source);
-    i::Scanner scanner(i_isolate->unicode_cache());
-    scanner.Initialize(stream.get(), false);
+    i::Scanner scanner(i_isolate->unicode_cache(), stream.get(), false);
+    scanner.Initialize();
     i::Zone zone(i_isolate->allocator(), ZONE_NAME);
     i::AstValueFactory ast_value_factory(&zone,
                                          i_isolate->ast_string_constants(),
@@ -232,8 +232,8 @@ TEST(ScanHtmlComments) {
   // Disallow HTML comments.
   {
     auto stream = i::ScannerStream::ForTesting(src);
-    i::Scanner scanner(&unicode_cache);
-    scanner.Initialize(stream.get(), true);
+    i::Scanner scanner(&unicode_cache, stream.get(), true);
+    scanner.Initialize();
     CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
     CHECK_EQ(i::Token::ILLEGAL, scanner.Next());
   }
@@ -241,8 +241,8 @@ TEST(ScanHtmlComments) {
   // Skip HTML comments:
   {
     auto stream = i::ScannerStream::ForTesting(src);
-    i::Scanner scanner(&unicode_cache);
-    scanner.Initialize(stream.get(), false);
+    i::Scanner scanner(&unicode_cache, stream.get(), false);
+    scanner.Initialize();
     CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
     CHECK_EQ(i::Token::EOS, scanner.Next());
   }
@@ -280,8 +280,8 @@ TEST(StandAlonePreParser) {
   uintptr_t stack_limit = i_isolate->stack_guard()->real_climit();
   for (int i = 0; programs[i]; i++) {
     auto stream = i::ScannerStream::ForTesting(programs[i]);
-    i::Scanner scanner(i_isolate->unicode_cache());
-    scanner.Initialize(stream.get(), false);
+    i::Scanner scanner(i_isolate->unicode_cache(), stream.get(), false);
+    scanner.Initialize();
 
     i::Zone zone(i_isolate->allocator(), ZONE_NAME);
     i::AstValueFactory ast_value_factory(&zone,
@@ -313,8 +313,8 @@ TEST(StandAlonePreParserNoNatives) {
   uintptr_t stack_limit = isolate->stack_guard()->real_climit();
   for (int i = 0; programs[i]; i++) {
     auto stream = i::ScannerStream::ForTesting(programs[i]);
-    i::Scanner scanner(isolate->unicode_cache());
-    scanner.Initialize(stream.get(), false);
+    i::Scanner scanner(isolate->unicode_cache(), stream.get(), false);
+    scanner.Initialize();
 
     // Preparser defaults to disallowing natives syntax.
     i::Zone zone(CcTest::i_isolate()->allocator(), ZONE_NAME);
@@ -348,8 +348,8 @@ TEST(RegressChromium62639) {
   // failed in debug mode, and sometimes crashed in release mode.
 
   auto stream = i::ScannerStream::ForTesting(program);
-  i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
-  scanner.Initialize(stream.get(), false);
+  i::Scanner scanner(CcTest::i_isolate()->unicode_cache(), stream.get(), false);
+  scanner.Initialize();
   i::Zone zone(CcTest::i_isolate()->allocator(), ZONE_NAME);
   i::AstValueFactory ast_value_factory(
       &zone, CcTest::i_isolate()->ast_string_constants(),
@@ -381,8 +381,8 @@ TEST(PreParseOverflow) {
   uintptr_t stack_limit = isolate->stack_guard()->real_climit();
 
   auto stream = i::ScannerStream::ForTesting(program.get(), kProgramSize);
-  i::Scanner scanner(isolate->unicode_cache());
-  scanner.Initialize(stream.get(), false);
+  i::Scanner scanner(isolate->unicode_cache(), stream.get(), false);
+  scanner.Initialize();
 
   i::Zone zone(CcTest::i_isolate()->allocator(), ZONE_NAME);
   i::AstValueFactory ast_value_factory(
@@ -396,12 +396,12 @@ TEST(PreParseOverflow) {
   CHECK_EQ(i::PreParser::kPreParseStackOverflow, result);
 }
 
-void TestStreamScanner(i::CharacterStream<uint16_t>* stream,
+void TestStreamScanner(i::ScannerStream* stream,
                        i::Token::Value* expected_tokens,
                        int skip_pos = 0,  // Zero means not skipping.
                        int skip_to = 0) {
-  i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
-  scanner.Initialize(stream, false);
+  i::Scanner scanner(CcTest::i_isolate()->unicode_cache(), stream, false);
+  scanner.Initialize();
 
   int i = 0;
   do {
@@ -475,8 +475,8 @@ TEST(StreamScanner) {
 void TestScanRegExp(const char* re_source, const char* expected) {
   auto stream = i::ScannerStream::ForTesting(re_source);
   i::HandleScope scope(CcTest::i_isolate());
-  i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
-  scanner.Initialize(stream.get(), false);
+  i::Scanner scanner(CcTest::i_isolate()->unicode_cache(), stream.get(), false);
+  scanner.Initialize();
 
   i::Token::Value start = scanner.peek();
   CHECK(start == i::Token::DIV || start == i::Token::ASSIGN_DIV);
@@ -1141,8 +1141,6 @@ void SetParserFlags(i::PreParser* parser, i::EnumSet<ParserFlag> flags) {
   parser->set_allow_natives(flags.Contains(kAllowNatives));
   parser->set_allow_harmony_public_fields(
       flags.Contains(kAllowHarmonyPublicFields));
-  parser->set_allow_harmony_private_fields(
-      flags.Contains(kAllowHarmonyPrivateFields));
   parser->set_allow_harmony_static_fields(
       flags.Contains(kAllowHarmonyStaticFields));
   parser->set_allow_harmony_dynamic_import(
@@ -1151,11 +1149,13 @@ void SetParserFlags(i::PreParser* parser, i::EnumSet<ParserFlag> flags) {
       flags.Contains(kAllowHarmonyImportMeta));
   parser->set_allow_harmony_do_expressions(
       flags.Contains(kAllowHarmonyDoExpressions));
+
+  parser->set_allow_harmony_private_fields(
+      flags.Contains(kAllowHarmonyPrivateFields));
   parser->set_allow_harmony_numeric_separator(
       flags.Contains(kAllowHarmonyNumericSeparator));
 }
 
-template <typename Char>
 void TestParserSyncWithFlags(i::Handle<i::String> source,
                              i::EnumSet<ParserFlag> flags,
                              ParserSyncTestResult result,
@@ -1169,9 +1169,9 @@ void TestParserSyncWithFlags(i::Handle<i::String> source,
   // Preparse the data.
   i::PendingCompilationErrorHandler pending_error_handler;
   if (test_preparser) {
-    i::Scanner scanner(isolate->unicode_cache());
     std::unique_ptr<i::ScannerStream> stream(
         i::ScannerStream::For(isolate, source));
+    i::Scanner scanner(isolate->unicode_cache(), stream.get(), is_module);
     i::Zone zone(CcTest::i_isolate()->allocator(), ZONE_NAME);
     i::AstValueFactory ast_value_factory(
         &zone, CcTest::i_isolate()->ast_string_constants(),
@@ -1181,8 +1181,8 @@ void TestParserSyncWithFlags(i::Handle<i::String> source,
                            isolate->counters()->runtime_call_stats(),
                            isolate->logger(), -1, is_module);
     SetParserFlags(&preparser, flags);
-    scanner.Initialize(static_cast<CharacterStream<Char>*>(stream.get()),
-                       is_module);
+    scanner.Initialize();
+    // Make sure we can the first token with the correct flags.
     i::PreParser::PreParseResult result = preparser.PreParseProgram();
     CHECK_EQ(i::PreParser::kPreParseSuccess, result);
   }
@@ -1292,15 +1292,8 @@ void TestParserSync(const char* source, const ParserFlag* varying_flags,
          ++flag_index) {
       flags.Remove(always_false_flags[flag_index]);
     }
-    if (str->IsSeqOneByteString()) {
-      // TODO(verwaest): Switch to uint8_t.
-      TestParserSyncWithFlags<uint16_t>(str, flags, result, is_module,
-                                        test_preparser, ignore_error_msg);
-    } else {
-      DCHECK(str->IsSeqTwoByteString());
-      TestParserSyncWithFlags<uint16_t>(str, flags, result, is_module,
-                                        test_preparser, ignore_error_msg);
-    }
+    TestParserSyncWithFlags(str, flags, result, is_module, test_preparser,
+                            ignore_error_msg);
   }
 }