[scanner] Go back to untemplatized scanning with buffering

This reverts the following 3 CLs:

Revert "[scanner] Templatize scan functions by encoding"
Revert "[asm] Remove invalid static cast of character stream"
Revert "[scanner] Prepare CharacterStreams for specializing scanner and parser by character type"

The original idea behind this work was to avoid copying, converting and
buffering characters to be scanned by specializing the scanner functions. The
additional benefit was for scanner functions to have a bigger window over the
input. Even though we can get a pretty nice speedup from having a larger
window, in practice this rarely helps. The cost is a larger binary.

Since we can't eagerly convert utf8 to utf16 due to memory overhead, we'd also
need to have a specialized version of the scanner just for utf8. That's pretty
complex, and likely won't be better than simply bulk converting and buffering
utf8 as utf16.

Change-Id: Ic3564683932a0097e3f9f51cd88f62c6ac879dcb
Reviewed-on: https://chromium-review.googlesource.com/1183190Reviewed-by: Andreas Haas <ahaas@chromium.org>
Reviewed-by: Marja Hölttä <marja@chromium.org>
Commit-Queue: Toon Verwaest <verwaest@chromium.org>
Cr-Commit-Position: refs/heads/master@{#55258}

src/asmjs/asm-js.cc

@@ -235,13 +235,13 @@ UnoptimizedCompilationJob::Status AsmJsCompilationJob::ExecuteJobImpl() {
   Zone* compile_zone = compilation_info()->zone();
   Zone translate_zone(allocator_, ZONE_NAME);
 
-  ScannerStream* stream = parse_info()->character_stream();
+  Utf16CharacterStream* stream = parse_info()->character_stream();
   base::Optional<AllowHandleDereference> allow_deref;
   if (stream->can_access_heap()) {
     allow_deref.emplace();
   }
-  wasm::AsmJsParser parser(&translate_zone, stack_limit(), stream,
-                           compilation_info()->literal()->start_position());
+  stream->Seek(compilation_info()->literal()->start_position());
+  wasm::AsmJsParser parser(&translate_zone, stack_limit(), stream);
   if (!parser.Run()) {
     if (!FLAG_suppress_asm_messages) {
       ReportCompilationFailure(parse_info(), parser.failure_location(),

src/asmjs/asm-parser.cc

@@ -69,9 +69,9 @@ namespace wasm {
 #define TOK(name) AsmJsScanner::kToken_##name
 
 AsmJsParser::AsmJsParser(Zone* zone, uintptr_t stack_limit,
-                         ScannerStream* stream, int start)
+                         Utf16CharacterStream* stream)
     : zone_(zone),
-      scanner_(stream, start),
+      scanner_(stream),
       module_builder_(new (zone) WasmModuleBuilder(zone)),
       return_type_(nullptr),
       stack_limit_(stack_limit),

src/asmjs/asm-parser.h

@@ -16,7 +16,7 @@
 namespace v8 {
 namespace internal {
 
-class ScannerStream;
+class Utf16CharacterStream;
 
 namespace wasm {
 
@@ -49,8 +49,8 @@ class AsmJsParser {
 
   typedef EnumSet<StandardMember, uint64_t> StdlibSet;
 
-  explicit AsmJsParser(Zone* zone, uintptr_t stack_limit, ScannerStream* stream,
-                       int start);
+  explicit AsmJsParser(Zone* zone, uintptr_t stack_limit,
+                       Utf16CharacterStream* stream);
   bool Run();
   const char* failure_message() const { return failure_message_; }
   int failure_location() const { return failure_location_; }

src/asmjs/asm-scanner.cc

@@ -7,7 +7,6 @@
 #include "src/char-predicates-inl.h"
 #include "src/conversions.h"
 #include "src/flags.h"
-#include "src/parsing/scanner-character-streams.h"
 #include "src/parsing/scanner.h"
 #include "src/unicode-cache.h"
 
@@ -20,11 +19,7 @@ namespace {
 static const int kMaxIdentifierCount = 0xF000000;
 };
 
-#define SPECIALIZE(Call, ...)                           \
-  (stream_->is_two_byte() ? Call<uint16_t>(__VA_ARGS__) \
-                          : Call<uint8_t>(__VA_ARGS__))
-
-AsmJsScanner::AsmJsScanner(ScannerStream* stream, int start)
+AsmJsScanner::AsmJsScanner(Utf16CharacterStream* stream)
     : stream_(stream),
       token_(kUninitialized),
       preceding_token_(kUninitialized),
@@ -38,7 +33,6 @@ AsmJsScanner::AsmJsScanner(ScannerStream* stream, int start)
       double_value_(0.0),
       unsigned_value_(0),
       preceded_by_newline_(false) {
-  SPECIALIZE(DoSeek, start);
 #define V(name, _junk1, _junk2, _junk3) property_names_[#name] = kToken_##name;
   STDLIB_MATH_FUNCTION_LIST(V)
   STDLIB_ARRAY_TYPE_LIST(V)
@@ -55,10 +49,7 @@ AsmJsScanner::AsmJsScanner(ScannerStream* stream, int start)
   Next();
 }
 
-void AsmJsScanner::Next() { SPECIALIZE(Scan); }
-
-template <typename Char>
-void AsmJsScanner::Scan() {
+void AsmJsScanner::Next() {
   if (rewind_) {
     preceding_token_ = token_;
     preceding_position_ = position_;
@@ -92,8 +83,8 @@ void AsmJsScanner::Scan() {
   preceding_position_ = position_;
 
   for (;;) {
-    position_ = Source<Char>()->pos();
-    uc32 ch = Advance<Char>();
+    position_ = stream_->pos();
+    uc32 ch = stream_->Advance();
     switch (ch) {
       case ' ':
       case '\t':
@@ -113,20 +104,20 @@ void AsmJsScanner::Scan() {
 
       case '\'':
       case '"':
-        ConsumeString<Char>(ch);
+        ConsumeString(ch);
         return;
 
       case '/':
-        ch = Advance<Char>();
+        ch = stream_->Advance();
         if (ch == '/') {
-          ConsumeCPPComment<Char>();
+          ConsumeCPPComment();
         } else if (ch == '*') {
-          if (!ConsumeCComment<Char>()) {
+          if (!ConsumeCComment()) {
             token_ = kParseError;
             return;
           }
         } else {
-          Back<Char>();
+          stream_->Back();
           token_ = '/';
           return;
         }
@@ -138,7 +129,7 @@ void AsmJsScanner::Scan() {
       case '>':
       case '=':
       case '!':
-        ConsumeCompareOrShift<Char>(ch);
+        ConsumeCompareOrShift(ch);
         return;
 
 #define V(single_char_token) case single_char_token:
@@ -150,9 +141,9 @@ void AsmJsScanner::Scan() {
 
       default:
         if (IsIdentifierStart(ch)) {
-          ConsumeIdentifier<Char>(ch);
+          ConsumeIdentifier(ch);
         } else if (IsNumberStart(ch)) {
-          ConsumeNumber<Char>(ch);
+          ConsumeNumber(ch);
         } else {
           // TODO(bradnelson): Support unicode (probably via UnicodeCache).
           token_ = kParseError;
@@ -220,7 +211,7 @@ std::string AsmJsScanner::Name(token_t token) const {
 #endif
 
 void AsmJsScanner::Seek(size_t pos) {
-  SPECIALIZE(DoSeek, pos);
+  stream_->Seek(pos);
   preceding_token_ = kUninitialized;
   token_ = kUninitialized;
   next_token_ = kUninitialized;
@@ -231,16 +222,15 @@ void AsmJsScanner::Seek(size_t pos) {
   Next();
 }
 
-template <typename Char>
 void AsmJsScanner::ConsumeIdentifier(uc32 ch) {
   // Consume characters while still part of the identifier.
   identifier_string_.clear();
   while (IsIdentifierPart(ch)) {
     identifier_string_ += ch;
-    ch = Advance<Char>();
+    ch = stream_->Advance();
   }
   // Go back one for next time.
-  Back<Char>();
+  stream_->Back();
 
   // Decode what the identifier means.
   if (preceding_token_ == '.') {
@@ -280,14 +270,13 @@ void AsmJsScanner::ConsumeIdentifier(uc32 ch) {
   }
 }
 
-template <typename Char>
 void AsmJsScanner::ConsumeNumber(uc32 ch) {
   std::string number;
   number = ch;
   bool has_dot = ch == '.';
   bool has_prefix = false;
   for (;;) {
-    ch = Advance<Char>();
+    ch = stream_->Advance();
     if ((ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f') ||
         (ch >= 'A' && ch <= 'F') || ch == '.' || ch == 'b' || ch == 'o' ||
         ch == 'x' ||
@@ -306,7 +295,7 @@ void AsmJsScanner::ConsumeNumber(uc32 ch) {
       break;
     }
   }
-  Back<Char>();
+  stream_->Back();
   // Special case the most common number.
   if (number.size() == 1 && number[0] == '0') {
     unsigned_value_ = 0;
@@ -333,7 +322,7 @@ void AsmJsScanner::ConsumeNumber(uc32 ch) {
     // problem.
     if (number[0] == '.') {
       for (size_t k = 1; k < number.size(); ++k) {
-        Back<Char>();
+        stream_->Back();
       }
       token_ = '.';
       return;
@@ -355,12 +344,11 @@ void AsmJsScanner::ConsumeNumber(uc32 ch) {
   }
 }
 
-template <typename Char>
 bool AsmJsScanner::ConsumeCComment() {
   for (;;) {
-    uc32 ch = Advance<Char>();
+    uc32 ch = stream_->Advance();
     while (ch == '*') {
-      ch = Advance<Char>();
+      ch = stream_->Advance();
       if (ch == '/') {
         return true;
       }
@@ -371,36 +359,33 @@ bool AsmJsScanner::ConsumeCComment() {
   }
 }
 
-template <typename Char>
 void AsmJsScanner::ConsumeCPPComment() {
   for (;;) {
-    uc32 ch = Advance<Char>();
+    uc32 ch = stream_->Advance();
     if (ch == '\n' || ch == kEndOfInput) {
       return;
     }
   }
 }
 
-template <typename Char>
 void AsmJsScanner::ConsumeString(uc32 quote) {
   // Only string allowed is 'use asm' / "use asm".
   const char* expected = "use asm";
   for (; *expected != '\0'; ++expected) {
-    if (Advance<Char>() != *expected) {
+    if (stream_->Advance() != *expected) {
       token_ = kParseError;
       return;
     }
   }
-  if (Advance<Char>() != quote) {
+  if (stream_->Advance() != quote) {
     token_ = kParseError;
     return;
   }
   token_ = kToken_UseAsm;
 }
 
-template <typename Char>
 void AsmJsScanner::ConsumeCompareOrShift(uc32 ch) {
-  uc32 next_ch = Advance<Char>();
+  uc32 next_ch = stream_->Advance();
   if (next_ch == '=') {
     switch (ch) {
       case '<':
@@ -421,14 +406,14 @@ void AsmJsScanner::ConsumeCompareOrShift(uc32 ch) {
   } else if (ch == '<' && next_ch == '<') {
     token_ = kToken_SHL;
   } else if (ch == '>' && next_ch == '>') {
-    if (Advance<Char>() == '>') {
+    if (stream_->Advance() == '>') {
       token_ = kToken_SHR;
     } else {
       token_ = kToken_SAR;
-      Back<Char>();
+      stream_->Back();
     }
   } else {
-    Back<Char>();
+    stream_->Back();
     token_ = ch;
   }
 }
@@ -443,7 +428,5 @@ bool AsmJsScanner::IsNumberStart(uc32 ch) {
   return ch == '.' || IsDecimalDigit(ch);
 }
 
-#undef SPECIALIZE
-
 }  // namespace internal
 }  // namespace v8

src/asmjs/asm-scanner.h

@@ -16,9 +16,7 @@
 namespace v8 {
 namespace internal {
 
-class ScannerStream;
-template <typename Char>
-class CharacterStream;
+class Utf16CharacterStream;
 
 // A custom scanner to extract the token stream needed to parse valid
 // asm.js: http://asmjs.org/spec/latest/
@@ -33,7 +31,7 @@ class V8_EXPORT_PRIVATE AsmJsScanner {
  public:
   typedef int32_t token_t;
 
-  AsmJsScanner(ScannerStream* stream, int start);
+  explicit AsmJsScanner(Utf16CharacterStream* stream);
 
   // Get current token.
   token_t Token() const { return token_; }
@@ -138,7 +136,7 @@ class V8_EXPORT_PRIVATE AsmJsScanner {
   // clang-format on
 
  private:
-  ScannerStream* const stream_;
+  Utf16CharacterStream* stream_;
   token_t token_;
   token_t preceding_token_;
   token_t next_token_;         // Only set when in {rewind} state.
@@ -156,37 +154,12 @@ class V8_EXPORT_PRIVATE AsmJsScanner {
   uint32_t unsigned_value_;
   bool preceded_by_newline_;
 
-  template <typename Char>
-  void Scan();
-  template <typename Char>
-  inline CharacterStream<Char>* Source() {
-    return static_cast<CharacterStream<Char>*>(stream_);
-  }
-  template <typename Char>
-  inline uc32 Advance() {
-    return Source<Char>()->Advance();
-  }
-  template <typename Char>
-  inline void Back() {
-    return Source<Char>()->Back();
-  }
-  template <typename Char>
-  void DoSeek(size_t pos) {
-    Source<Char>()->Seek(pos);
-  }
-
   // Consume multiple characters.
-  template <typename Char>
   void ConsumeIdentifier(uc32 ch);
-  template <typename Char>
   void ConsumeNumber(uc32 ch);
-  template <typename Char>
   bool ConsumeCComment();
-  template <typename Char>
   void ConsumeCPPComment();
-  template <typename Char>
   void ConsumeString(uc32 quote);
-  template <typename Char>
   void ConsumeCompareOrShift(uc32 ch);
 
   // Classify character categories.

src/compiler-dispatcher/unoptimized-compile-job.cc

@@ -133,7 +133,7 @@ void UnoptimizedCompileJob::PrepareOnMainThread(Isolate* isolate) {
   DCHECK(script->type() != Script::TYPE_NATIVE);
   Handle<String> source(String::cast(script->source()), isolate);
   if (source->IsExternalTwoByteString() || source->IsExternalOneByteString()) {
-    std::unique_ptr<ScannerStream> stream(ScannerStream::For(
+    std::unique_ptr<Utf16CharacterStream> stream(ScannerStream::For(
         isolate, source, shared_->StartPosition(), shared_->EndPosition()));
     parse_info_->set_character_stream(std::move(stream));
   } else {
@@ -191,7 +191,7 @@ void UnoptimizedCompileJob::PrepareOnMainThread(Isolate* isolate) {
                     .ToHandleChecked();
     }
     wrapper_ = isolate->global_handles()->Create(*wrapper);
-    std::unique_ptr<ScannerStream> stream(
+    std::unique_ptr<Utf16CharacterStream> stream(
         ScannerStream::For(isolate, wrapper_, shared_->StartPosition() - offset,
                            shared_->EndPosition() - offset));
     parse_info_->set_character_stream(std::move(stream));

src/compiler.cc

@@ -966,7 +966,7 @@ BackgroundCompileTask::BackgroundCompileTask(ScriptStreamingData* source,
     info->set_runtime_call_stats(nullptr);
   }
   info->set_toplevel();
-  std::unique_ptr<ScannerStream> stream(
+  std::unique_ptr<Utf16CharacterStream> stream(
       ScannerStream::For(source->source_stream.get(), source->encoding,
                          info->runtime_call_stats()));
   info->set_character_stream(std::move(stream));

src/parsing/parse-info.cc

@@ -188,7 +188,7 @@ void ParseInfo::AllocateSourceRangeMap() {
 void ParseInfo::ResetCharacterStream() { character_stream_.reset(); }
 
 void ParseInfo::set_character_stream(
-    std::unique_ptr<ScannerStream> character_stream) {
+    std::unique_ptr<Utf16CharacterStream> character_stream) {
   DCHECK_NULL(character_stream_);
   character_stream_.swap(character_stream);
 }

src/parsing/parse-info.h

@@ -31,7 +31,7 @@ class RuntimeCallStats;
 class Logger;
 class SourceRangeMap;
 class UnicodeCache;
-class ScannerStream;
+class Utf16CharacterStream;
 class Zone;
 
 // A container for the inputs, configuration options, and outputs of parsing.
@@ -101,8 +101,11 @@ class V8_EXPORT_PRIVATE ParseInfo {
                                       : NO_PARSE_RESTRICTION;
   }
 
-  ScannerStream* character_stream() const { return character_stream_.get(); }
-  void set_character_stream(std::unique_ptr<ScannerStream> character_stream);
+  Utf16CharacterStream* character_stream() const {
+    return character_stream_.get();
+  }
+  void set_character_stream(
+      std::unique_ptr<Utf16CharacterStream> character_stream);
   void ResetCharacterStream();
 
   v8::Extension* extension() const { return extension_; }
@@ -270,7 +273,7 @@ class V8_EXPORT_PRIVATE ParseInfo {
   MaybeHandle<ScopeInfo> maybe_outer_scope_info_;
 
   //----------- Inputs+Outputs of parsing and scope analysis -----------------
-  std::unique_ptr<ScannerStream> character_stream_;
+  std::unique_ptr<Utf16CharacterStream> character_stream_;
   ConsumedPreParsedScopeData consumed_preparsed_scope_data_;
   std::shared_ptr<AstValueFactory> ast_value_factory_;
   const class AstStringConstants* ast_string_constants_;

src/parsing/parser.cc

@@ -412,8 +412,7 @@ Parser::Parser(ParseInfo* info)
                          info->runtime_call_stats(), info->logger(),
                          info->script().is_null() ? -1 : info->script()->id(),
                          info->is_module(), true),
-      scanner_(info->unicode_cache(), info->character_stream(),
-               info->is_module()),
+      scanner_(info->unicode_cache()),
       reusable_preparser_(nullptr),
       mode_(PARSE_EAGERLY),  // Lazy mode must be set explicitly.
       source_range_map_(info->source_range_map()),
@@ -508,8 +507,7 @@ FunctionLiteral* Parser::ParseProgram(Isolate* isolate, ParseInfo* info) {
   // Initialize parser state.
   DeserializeScopeChain(isolate, info, info->maybe_outer_scope_info());
 
-  scanner_.Initialize();
-
+  scanner_.Initialize(info->character_stream(), info->is_module());
   FunctionLiteral* result = DoParseProgram(isolate, info);
   MaybeResetCharacterStream(info, result);
 
@@ -703,8 +701,7 @@ FunctionLiteral* Parser::ParseFunction(Isolate* isolate, ParseInfo* info,
   // Initialize parser state.
   Handle<String> name(shared_info->Name(), isolate);
   info->set_function_name(ast_value_factory()->GetString(name));
-
-  scanner_.Initialize();
+  scanner_.Initialize(info->character_stream(), info->is_module());
 
   FunctionLiteral* result =
       DoParseFunction(isolate, info, info->function_name());
@@ -3450,8 +3447,7 @@ void Parser::ParseOnBackground(ParseInfo* info) {
   DCHECK_NULL(info->literal());
   FunctionLiteral* result = nullptr;
 
-  scanner_.Initialize();
-
+  scanner_.Initialize(info->character_stream(), info->is_module());
   DCHECK(info->maybe_outer_scope_info().is_null());
 
   DCHECK(original_scope_);

src/parsing/parsing.cc

@@ -26,7 +26,8 @@ bool ParseProgram(ParseInfo* info, Isolate* isolate) {
   // Create a character stream for the parser.
   Handle<String> source(String::cast(info->script()->source()), isolate);
   isolate->counters()->total_parse_size()->Increment(source->length());
-  std::unique_ptr<ScannerStream> stream(ScannerStream::For(isolate, source));
+  std::unique_ptr<Utf16CharacterStream> stream(
+      ScannerStream::For(isolate, source));
   info->set_character_stream(std::move(stream));
 
   Parser parser(info);
@@ -60,7 +61,7 @@ bool ParseFunction(ParseInfo* info, Handle<SharedFunctionInfo> shared_info,
   // Create a character stream for the parser.
   Handle<String> source(String::cast(info->script()->source()), isolate);
   isolate->counters()->total_parse_size()->Increment(source->length());
-  std::unique_ptr<ScannerStream> stream(
+  std::unique_ptr<Utf16CharacterStream> stream(
       ScannerStream::For(isolate, source, shared_info->StartPosition(),
                          shared_info->EndPosition()));
   info->set_character_stream(std::move(stream));

src/parsing/scanner-character-streams.h

@@ -5,8 +5,6 @@
 #ifndef V8_PARSING_SCANNER_CHARACTER_STREAMS_H_
 #define V8_PARSING_SCANNER_CHARACTER_STREAMS_H_
 
-#include <algorithm>
-
 #include "include/v8.h"  // for v8::ScriptCompiler
 #include "src/globals.h"
 
@@ -15,178 +13,23 @@ namespace internal {
 
 template <typename T>
 class Handle;
-template <typename Char>
-class CharacterStream;
+class Utf16CharacterStream;
 class RuntimeCallStats;
 class String;
 
 class V8_EXPORT_PRIVATE ScannerStream {
  public:
-  static const uc32 kEndOfInput = -1;
-
-  static ScannerStream* For(Isolate* isolate, Handle<String> data);
-  static ScannerStream* For(Isolate* isolate, Handle<String> data,
-                            int start_pos, int end_pos);
-  static ScannerStream* For(ScriptCompiler::ExternalSourceStream* source_stream,
-                            ScriptCompiler::StreamedSource::Encoding encoding,
-                            RuntimeCallStats* stats);
-
-  // For testing:
-  static std::unique_ptr<CharacterStream<uint8_t>> ForTesting(const char* data);
-  static std::unique_ptr<CharacterStream<uint8_t>> ForTesting(const char* data,
-                                                              size_t length);
-
-  // Returns true if the stream could access the V8 heap after construction.
-  virtual bool can_access_heap() = 0;
-  uc32 Advance();
-  void Seek(size_t pos);
-  size_t pos();
-  void Back();
-  void Back2();
-
-  virtual ~ScannerStream() {}
-
-  bool is_two_byte() const { return is_two_byte_; }
-
- protected:
-  explicit ScannerStream(bool is_two_byte) : is_two_byte_(is_two_byte) {}
-
- private:
-  const bool is_two_byte_;
-};
-
-template <typename Char>
-class CharacterStream : public ScannerStream {
- public:
-  // Returns and advances past the next UTF-16 code unit in the input
-  // stream. If there are no more code units it returns kEndOfInput.
-  inline uc32 Advance() {
-    uc32 result = Peek();
-    buffer_cursor_++;
-    return result;
-  }
-
-  inline uc32 Peek() {
-    if (V8_LIKELY(buffer_cursor_ < buffer_end_)) {
-      return static_cast<uc32>(*buffer_cursor_);
-    } else if (ReadBlockChecked()) {
-      return static_cast<uc32>(*buffer_cursor_);
-    } else {
-      return kEndOfInput;
-    }
-  }
-
-  // Returns and advances past the next UTF-16 code unit in the input stream
-  // that meets the checks requirement. If there are no more code units it
-  // returns kEndOfInput.
-  template <typename FunctionType>
-  V8_INLINE uc32 AdvanceUntil(FunctionType check) {
-    while (true) {
-      auto next_cursor_pos =
-          std::find_if(buffer_cursor_, buffer_end_, [&check](Char raw_c0) {
-            uc32 c0 = static_cast<uc32>(raw_c0);
-            return check(c0);
-          });
-
-      if (next_cursor_pos == buffer_end_) {
-        buffer_cursor_ = buffer_end_;
-        if (!ReadBlockChecked()) {
-          buffer_cursor_++;
-          return kEndOfInput;
-        }
-      } else {
-        buffer_cursor_ = next_cursor_pos + 1;
-        return static_cast<uc32>(*next_cursor_pos);
-      }
-    }
-  }
-
-  // Go back one by one character in the input stream.
-  // This undoes the most recent Advance().
-  inline void Back() {
-    // The common case - if the previous character is within
-    // buffer_start_ .. buffer_end_ will be handles locally.
-    // Otherwise, a new block is requested.
-    if (V8_LIKELY(buffer_cursor_ > buffer_start_)) {
-      buffer_cursor_--;
-    } else {
-      ReadBlockAt(pos() - 1);
-    }
-  }
-
-  inline size_t pos() const {
-    return buffer_pos_ + (buffer_cursor_ - buffer_start_);
-  }
-
-  inline void Seek(size_t pos) {
-    if (V8_LIKELY(pos >= buffer_pos_ &&
-                  pos < (buffer_pos_ + (buffer_end_ - buffer_start_)))) {
-      buffer_cursor_ = buffer_start_ + (pos - buffer_pos_);
-    } else {
-      ReadBlockAt(pos);
-    }
-  }
-
- protected:
-  CharacterStream(const Char* buffer_start, const Char* buffer_cursor,
-                  const Char* buffer_end, size_t buffer_pos)
-      : ScannerStream(sizeof(Char) == 2),
-        buffer_start_(buffer_start),
-        buffer_cursor_(buffer_cursor),
-        buffer_end_(buffer_end),
-        buffer_pos_(buffer_pos) {}
-  CharacterStream() : CharacterStream(nullptr, nullptr, nullptr, 0) {}
-
-  bool ReadBlockChecked() {
-    size_t position = pos();
-    USE(position);
-    bool success = ReadBlock();
-
-    // Post-conditions: 1, We should always be at the right position.
-    //                  2, Cursor should be inside the buffer.
-    //                  3, We should have more characters available iff success.
-    DCHECK_EQ(pos(), position);
-    DCHECK_LE(buffer_cursor_, buffer_end_);
-    DCHECK_LE(buffer_start_, buffer_cursor_);
-    DCHECK_EQ(success, buffer_cursor_ < buffer_end_);
-    return success;
-  }
-
-  void ReadBlockAt(size_t new_pos) {
-    // The callers of this method (Back/Seek) should handle the easy
-    // case (seeking within the current buffer), and we should only get here
-    // if we actually require new data.
-    // (This is really an efficiency check, not a correctness invariant.)
-    DCHECK(new_pos < buffer_pos_ ||
-           new_pos >= buffer_pos_ + (buffer_end_ - buffer_start_));
-
-    // Change pos() to point to new_pos.
-    buffer_pos_ = new_pos;
-    buffer_cursor_ = buffer_start_;
-    DCHECK_EQ(pos(), new_pos);
-    ReadBlockChecked();
-  }
-
-  // Read more data, and update buffer_*_ to point to it.
-  // Returns true if more data was available.
-  //
-  // ReadBlock() may modify any of the buffer_*_ members, but must sure that
-  // the result of pos() remains unaffected.
-  //
-  // Examples:
-  // - a stream could either fill a separate buffer. Then buffer_start_ and
-  //   buffer_cursor_ would point to the beginning of the buffer, and
-  //   buffer_pos would be the old pos().
-  // - a stream with existing buffer chunks would set buffer_start_ and
-  //   buffer_end_ to cover the full chunk, and then buffer_cursor_ would
-  //   point into the middle of the buffer, while buffer_pos_ would describe
-  //   the start of the buffer.
-  virtual bool ReadBlock() = 0;
-
-  const Char* buffer_start_;
-  const Char* buffer_cursor_;
-  const Char* buffer_end_;
-  size_t buffer_pos_;
+  static Utf16CharacterStream* For(Isolate* isolate, Handle<String> data);
+  static Utf16CharacterStream* For(Isolate* isolate, Handle<String> data,
+                                   int start_pos, int end_pos);
+  static Utf16CharacterStream* For(
+      ScriptCompiler::ExternalSourceStream* source_stream,
+      ScriptCompiler::StreamedSource::Encoding encoding,
+      RuntimeCallStats* stats);
+
+  static std::unique_ptr<Utf16CharacterStream> ForTesting(const char* data);
+  static std::unique_ptr<Utf16CharacterStream> ForTesting(const char* data,
+                                                          size_t length);
 };
 
 }  // namespace internal

src/parsing/scanner-inl.h

@@ -11,9 +11,8 @@
 namespace v8 {
 namespace internal {
 
-template <typename Char>
 V8_INLINE Token::Value Scanner::SkipWhiteSpace() {
-  int start_position = SourcePos<Char>();
+  int start_position = source_pos();
 
   while (true) {
     // We won't skip behind the end of input.
@@ -26,11 +25,11 @@ V8_INLINE Token::Value Scanner::SkipWhiteSpace() {
     } else if (!unicode_cache_->IsWhiteSpace(c0_)) {
       break;
     }
-    Advance<Char>();
+    Advance();
   }
 
   // Return whether or not we skipped any characters.
-  if (SourcePos<Char>() == start_position) {
+  if (source_pos() == start_position) {
     DCHECK_NE('0', c0_);
     return Token::ILLEGAL;
   }

test/cctest/parsing/test-scanner.cc

@@ -27,7 +27,7 @@ struct ScannerTestHelper {
         scanner(std::move(other.scanner)) {}
 
   std::unique_ptr<UnicodeCache> unicode_cache;
-  std::unique_ptr<CharacterStream<uint8_t>> stream;
+  std::unique_ptr<Utf16CharacterStream> stream;
   std::unique_ptr<Scanner> scanner;
 
   Scanner* operator->() const { return scanner.get(); }
@@ -38,9 +38,9 @@ ScannerTestHelper make_scanner(const char* src) {
   ScannerTestHelper helper;
   helper.unicode_cache = std::unique_ptr<UnicodeCache>(new UnicodeCache);
   helper.stream = ScannerStream::ForTesting(src);
-  helper.scanner = std::unique_ptr<Scanner>(
-      new Scanner(helper.unicode_cache.get(), helper.stream.get(), false));
-  helper.scanner->Initialize();
+  helper.scanner =
+      std::unique_ptr<Scanner>(new Scanner(helper.unicode_cache.get()));
+  helper.scanner->Initialize(helper.stream.get(), false);
   return helper;
 }
 

test/unittests/asmjs/asm-scanner-unittest.cc

@@ -17,7 +17,7 @@ class AsmJsScannerTest : public ::testing::Test {
  protected:
   void SetupScanner(const char* source) {
     stream = ScannerStream::ForTesting(source);
-    scanner.reset(new AsmJsScanner(stream.get(), 0));
+    scanner.reset(new AsmJsScanner(stream.get()));
   }
 
   void Skip(AsmJsScanner::token_t t) {
@@ -41,7 +41,7 @@ class AsmJsScannerTest : public ::testing::Test {
     CHECK_EQ(scanner->Token(), AsmJsScanner::kParseError);
   }
 
-  std::unique_ptr<ScannerStream> stream;
+  std::unique_ptr<Utf16CharacterStream> stream;
   std::unique_ptr<AsmJsScanner> scanner;
 };