Optimized scanner to avoid virtual calls for every character read.

Review URL: http://codereview.chromium.org/5545006

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@5935 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/api.cc

@@ -1165,14 +1165,22 @@ void ObjectTemplate::SetInternalFieldCount(int value) {
 
 
 ScriptData* ScriptData::PreCompile(const char* input, int length) {
-  unibrow::Utf8InputBuffer<> buf(input, length);
-  return i::ParserApi::PreParse(i::Handle<i::String>(), &buf, NULL);
+  i::Utf8ToUC16CharacterStream stream(
+      reinterpret_cast<const unsigned char*>(input), length);
+  return i::ParserApi::PreParse(&stream, NULL);
 }
 
 
 ScriptData* ScriptData::PreCompile(v8::Handle<String> source) {
   i::Handle<i::String> str = Utils::OpenHandle(*source);
-  return i::ParserApi::PreParse(str, NULL, NULL);
+  if (str->IsExternalTwoByteString()) {
+    i::ExternalTwoByteStringUC16CharacterStream stream(
+      i::Handle<i::ExternalTwoByteString>::cast(str), 0, str->length());
+    return i::ParserApi::PreParse(&stream, NULL);
+  } else {
+    i::GenericStringUC16CharacterStream stream(str, 0, str->length());
+    return i::ParserApi::PreParse(&stream, NULL);
+  }
 }
 
 

src/checks.h

@@ -231,6 +231,8 @@ static inline void CheckNonEqualsHelper(const char* file,
 
 #define CHECK_GT(a, b) CHECK((a) > (b))
 #define CHECK_GE(a, b) CHECK((a) >= (b))
+#define CHECK_LT(a, b) CHECK((a) < (b))
+#define CHECK_LE(a, b) CHECK((a) <= (b))
 
 
 // This is inspired by the static assertion facility in boost.  This

src/compiler.cc

@@ -461,7 +461,14 @@ Handle<SharedFunctionInfo> Compiler::Compile(Handle<String> source,
     ScriptDataImpl* pre_data = input_pre_data;
     if (pre_data == NULL
         && source_length >= FLAG_min_preparse_length) {
-      pre_data = ParserApi::PartialPreParse(source, NULL, extension);
+      if (source->IsExternalTwoByteString()) {
+        ExternalTwoByteStringUC16CharacterStream stream(
+            Handle<ExternalTwoByteString>::cast(source), 0, source->length());
+        pre_data = ParserApi::PartialPreParse(&stream, extension);
+      } else {
+        GenericStringUC16CharacterStream stream(source, 0, source->length());
+        pre_data = ParserApi::PartialPreParse(&stream, extension);
+      }
     }
 
     // Create a script object describing the script to be compiled.

src/parser.h

@@ -169,14 +169,12 @@ class ParserApi {
   static bool Parse(CompilationInfo* info);
 
   // Generic preparser generating full preparse data.
-  static ScriptDataImpl* PreParse(Handle<String> source,
-                                  unibrow::CharacterStream* stream,
+  static ScriptDataImpl* PreParse(UC16CharacterStream* source,
                                   v8::Extension* extension);
 
   // Preparser that only does preprocessing that makes sense if only used
   // immediately after.
-  static ScriptDataImpl* PartialPreParse(Handle<String> source,
-                                         unibrow::CharacterStream* stream,
+  static ScriptDataImpl* PartialPreParse(UC16CharacterStream* source,
                                          v8::Extension* extension);
 };
 
@@ -435,18 +433,26 @@ class Parser {
                        Vector<const char*> args);
 
  protected:
+  FunctionLiteral* ParseLazy(Handle<SharedFunctionInfo> info,
+                             UC16CharacterStream* source,
+                             ZoneScope* zone_scope);
   enum Mode {
     PARSE_LAZILY,
     PARSE_EAGERLY
   };
 
+  // Called by ParseProgram after setting up the scanner.
+  FunctionLiteral* DoParseProgram(Handle<String> source,
+                                  bool in_global_context,
+                                  ZoneScope* zone_scope);
+
   // Report syntax error
   void ReportUnexpectedToken(Token::Value token);
   void ReportInvalidPreparseData(Handle<String> name, bool* ok);
   void ReportMessage(const char* message, Vector<const char*> args);
 
   bool inside_with() const { return with_nesting_level_ > 0; }
-  Scanner& scanner()  { return scanner_; }
+  V8JavaScriptScanner& scanner()  { return scanner_; }
   Mode mode() const { return mode_; }
   ScriptDataImpl* pre_data() const { return pre_data_; }
 
@@ -548,7 +554,7 @@ class Parser {
 
   INLINE(Token::Value peek()) {
     if (stack_overflow_) return Token::ILLEGAL;
-    return scanner_.peek();
+    return scanner().peek();
   }
 
   INLINE(Token::Value Next()) {
@@ -560,9 +566,11 @@ class Parser {
     }
     if (StackLimitCheck().HasOverflowed()) {
       // Any further calls to Next or peek will return the illegal token.
+      // The current call must return the next token, which might already
+      // have been peek'ed.
       stack_overflow_ = true;
     }
-    return scanner_.Next();
+    return scanner().Next();
   }
 
   INLINE(void Consume(Token::Value token));
@@ -702,7 +710,14 @@ class JsonParser BASE_EMBEDDED {
   // Parse JSON input as a single JSON value.
   // Returns null handle and sets exception if parsing failed.
   static Handle<Object> Parse(Handle<String> source) {
-    return JsonParser().ParseJson(source);
+    if (source->IsExternalTwoByteString()) {
+      ExternalTwoByteStringUC16CharacterStream stream(
+          Handle<ExternalTwoByteString>::cast(source), 0, source->length());
+      return JsonParser().ParseJson(source, &stream);
+    } else {
+      GenericStringUC16CharacterStream stream(source, 0, source->length());
+      return JsonParser().ParseJson(source, &stream);
+    }
   }
 
  private:
@@ -710,7 +725,7 @@ class JsonParser BASE_EMBEDDED {
   ~JsonParser() { }
 
   // Parse a string containing a single JSON value.
-  Handle<Object> ParseJson(Handle<String>);
+  Handle<Object> ParseJson(Handle<String> script, UC16CharacterStream* source);
   // Parse a single JSON value from input (grammar production JSONValue).
   // A JSON value is either a (double-quoted) string literal, a number literal,
   // one of "true", "false", or "null", or an object or array literal.

src/preparser-api.cc

@@ -39,39 +39,82 @@ namespace v8 {
 namespace internal {
 
 // UTF16Buffer based on a v8::UnicodeInputStream.
-class InputStreamUTF16Buffer : public UTF16Buffer {
+class InputStreamUTF16Buffer : public UC16CharacterStream {
  public:
-  explicit InputStreamUTF16Buffer(UnicodeInputStream* stream)
-      : UTF16Buffer(),
-        stream_(stream) { }
+  explicit InputStreamUTF16Buffer(v8::UnicodeInputStream* stream)
+      : UC16CharacterStream(),
+        stream_(stream),
+        pushback_active_(false) {
+    buffer_cursor_ = buffer_end_ = buffer_ + kPushBackSize;
+  }
 
   virtual ~InputStreamUTF16Buffer() { }
 
-  virtual void PushBack(uc32 ch) {
+  virtual void PushBack(uc16 ch) {
+    ASSERT(pos_ > 0);
+    if (buffer_cursor_ > buffer_) {
+      // While we can stay within the buffer, just do so.
+      *--buffer_cursor_ = ch;
+      pos_--;
+      return;
+    }
+    if (!pushback_active_) {
+      // Push back the entire buffer to the stream and let the
+      // stream handle pushbacks from now.
+      // We leave buffer_cursor_ == buffer_end_, so the next read
+      // will fill the buffer from the current position.
+      // This should happen exceedingly rarely.
+      while (buffer_end_ > buffer_) {
+        stream_->PushBack(*--buffer_end_);
+      }
+      buffer_cursor_ = buffer_end_;
+      pushback_active_ = true;
+    }
     stream_->PushBack(ch);
     pos_--;
   }
 
-  virtual uc32 Advance() {
-    uc32 result = stream_->Next();
-    if (result >= 0) pos_++;
-    return result;
+ protected:
+  virtual bool ReadBlock() {
+    // Copy the top of the buffer into the pushback area.
+    pushback_active_ = false;
+    int32_t value;
+    uc16* buffer_start = buffer_ + kPushBackSize;
+    buffer_cursor_ = buffer_end_ = buffer_start;
+    while ((value = stream_->Next()) >= 0) {
+      if (value > static_cast<int32_t>(unibrow::Utf8::kMaxThreeByteChar)) {
+        value = unibrow::Utf8::kBadChar;
+      }
+      // buffer_end_ is a const pointer, but buffer_ is writable.
+      buffer_start[buffer_end_++ - buffer_start] = static_cast<uc16>(value);
+      if (buffer_end_ == buffer_ + kPushBackSize + kBufferSize) break;
+    }
+    return buffer_end_ > buffer_start;
   }
 
-  virtual void SeekForward(int pos) {
+  virtual unsigned SlowSeekForward(unsigned pos) {
     // Seeking in the input is not used by preparsing.
     // It's only used by the real parser based on preparser data.
     UNIMPLEMENTED();
+    return 0;
   }
 
  private:
+  static const unsigned kBufferSize = 512;
+  static const unsigned kPushBackSize = 16;
   v8::UnicodeInputStream* const stream_;
+  // Buffer holding first kPushBackSize characters of pushback buffer,
+  // then kBufferSize chars of read-ahead.
+  // The pushback buffer is only used if pushing back characters past
+  // the start of a block.
+  uc16 buffer_[kBufferSize + kPushBackSize];
+  bool pushback_active_;
 };
 
 
 class StandAloneJavaScriptScanner : public JavaScriptScanner {
  public:
-  void Initialize(UTF16Buffer* source) {
+  void Initialize(UC16CharacterStream* source) {
     source_ = source;
     literal_flags_ = kLiteralString | kLiteralIdentifier;
     Init();

src/preparser.cc

@@ -1078,6 +1078,7 @@ PreParser::Expression PreParser::ParseFunctionLiteral(bool* ok) {
 
     Expect(i::Token::RBRACE, CHECK_OK);
 
+    // Position right after terminal '}'.
     int end_pos = scanner_->location().end_pos;
     log_->LogFunction(function_block_pos, end_pos,
                       function_scope.materialized_literal_count(),

src/scanner-base.cc

@@ -34,12 +34,6 @@
 namespace v8 {
 namespace internal {
 
-// ----------------------------------------------------------------------------
-// UTF16Buffer
-
-UTF16Buffer::UTF16Buffer()
-    : pos_(0), end_(kNoEndPosition) { }
-
 // ----------------------------------------------------------------------------
 // LiteralCollector
 
@@ -92,7 +86,7 @@ bool ScannerConstants::IsIdentifier(unibrow::CharacterStream* buffer) {
 // ----------------------------------------------------------------------------
 // Scanner
 
-Scanner::Scanner() : source_(NULL) {}
+Scanner::Scanner() { }
 
 
 uc32 Scanner::ScanHexEscape(uc32 c, int length) {
@@ -142,8 +136,7 @@ uc32 Scanner::ScanOctalEscape(uc32 c, int length) {
 // ----------------------------------------------------------------------------
 // JavaScriptScanner
 
-JavaScriptScanner::JavaScriptScanner()
-    : has_line_terminator_before_next_(false) {}
+JavaScriptScanner::JavaScriptScanner() : Scanner() {}
 
 
 Token::Value JavaScriptScanner::Next() {
@@ -503,13 +496,23 @@ void JavaScriptScanner::Scan() {
 
 
 void JavaScriptScanner::SeekForward(int pos) {
-  source_->SeekForward(pos - 1);
-  Advance();
-  // This function is only called to seek to the location
-  // of the end of a function (at the "}" token). It doesn't matter
-  // whether there was a line terminator in the part we skip.
-  has_line_terminator_before_next_ = false;
+  // After this call, we will have the token at the given position as
+  // the "next" token. The "current" token will be invalid.
+  if (pos == next_.location.beg_pos) return;
+  int current_pos = source_pos();
+  ASSERT_EQ(next_.location.end_pos, current_pos);
+  // Positions inside the lookahead token aren't supported.
+  ASSERT(pos >= current_pos);
+  if (pos != current_pos) {
+    source_->SeekForward(pos - source_->pos());
+    Advance();
+    // This function is only called to seek to the location
+    // of the end of a function (at the "}" token). It doesn't matter
+    // whether there was a line terminator in the part we skip.
+    has_line_terminator_before_next_ = false;
+  }
   Scan();
+  ASSERT_EQ(Token::RBRACE, next_.token);
 }
 
 

src/scanner-base.h

@@ -52,31 +52,75 @@ inline int HexValue(uc32 c) {
   return -1;
 }
 
-// ----------------------------------------------------------------------------
-// UTF16Buffer - scanner input source with pushback.
 
-class UTF16Buffer {
+// ---------------------------------------------------------------------
+// Buffered stream of characters, using an internal UC16 buffer.
+
+class UC16CharacterStream {
  public:
-  UTF16Buffer();
-  virtual ~UTF16Buffer() {}
+  UC16CharacterStream() : pos_(0) { }
+  virtual ~UC16CharacterStream() { }
+
+  // Returns and advances past the next UC16 character in the input
+  // stream. If there are no more characters, it returns a negative
+  // value.
+  inline int32_t Advance() {
+    if (buffer_cursor_ < buffer_end_ || ReadBlock()) {
+      pos_++;
+      return *(buffer_cursor_++);
+    }
+    // Note: currently the following increment is necessary to avoid a
+    // parser problem! The scanner treats the final kEndOfInput as
+    // a character with a position, and does math relative to that
+    // position.
+    pos_++;
 
-  virtual void PushBack(uc32 ch) = 0;
-  // Returns a value < 0 when the buffer end is reached.
-  virtual uc32 Advance() = 0;
-  virtual void SeekForward(int pos) = 0;
+    return kEndOfInput;
+  }
 
-  int pos() const { return pos_; }
+  // Return the current position in the character stream.
+  // Starts at zero.
+  inline unsigned pos() const { return pos_; }
+
+  // Skips forward past the next character_count UC16 characters
+  // in the input, or until the end of input if that comes sooner.
+  // Returns the number of characters actually skipped. If less
+  // than character_count,
+  inline unsigned SeekForward(unsigned character_count) {
+    unsigned buffered_chars =
+        static_cast<unsigned>(buffer_end_ - buffer_cursor_);
+    if (character_count <= buffered_chars) {
+      buffer_cursor_ += character_count;
+      pos_ += character_count;
+      return character_count;
+    }
+    return SlowSeekForward(character_count);
+  }
 
-  static const int kNoEndPosition = 1;
+  // Pushes back the most recently read UC16 character, i.e.,
+  // the value returned by the most recent call to Advance.
+  // Must not be used right after calling SeekForward.
+  virtual void PushBack(uc16 character) = 0;
 
  protected:
-  // Initial value of end_ before the input stream is initialized.
-
-  int pos_;  // Current position in the buffer.
-  int end_;  // Position where scanning should stop (EOF).
+  static const int32_t kEndOfInput = -1;
+
+  // Ensures that the buffer_cursor_ points to the character at
+  // position pos_ of the input, if possible. If the position
+  // is at or after the end of the input, return false. If there
+  // are more characters available, return true.
+  virtual bool ReadBlock() = 0;
+  virtual unsigned SlowSeekForward(unsigned character_count) = 0;
+
+  const uc16* buffer_cursor_;
+  const uc16* buffer_end_;
+  unsigned pos_;
 };
 
 
+// ---------------------------------------------------------------------
+// Constants used by scanners.
+
 class ScannerConstants : AllStatic {
  public:
   typedef unibrow::Utf8InputBuffer<1024> Utf8Decoder;
@@ -277,7 +321,7 @@ class Scanner {
   // Low-level scanning support.
   void Advance() { c0_ = source_->Advance(); }
   void PushBack(uc32 ch) {
-    source_->PushBack(ch);
+    source_->PushBack(c0_);
     c0_ = ch;
   }
 
@@ -307,8 +351,8 @@ class Scanner {
   TokenDesc current_;  // desc for current token (as returned by Next())
   TokenDesc next_;     // desc for next token (one token look-ahead)
 
-  // Input stream. Must be initialized to an UTF16Buffer.
-  UTF16Buffer* source_;
+  // Input stream. Must be initialized to an UC16CharacterStream.
+  UC16CharacterStream* source_;
 
   // Buffer to hold literal values (identifiers, strings, numbers)
   // using '\x00'-terminated UTF-8 encoding. Handles allocation internally.

src/scanner.h

@@ -35,67 +35,97 @@
 namespace v8 {
 namespace internal {
 
-// UTF16 buffer to read characters from a character stream.
-class CharacterStreamUTF16Buffer: public UTF16Buffer {
+// A buffered character stream based on a random access character
+// source (ReadBlock can be called with pos_ pointing to any position,
+// even positions before the current).
+class BufferedUC16CharacterStream: public UC16CharacterStream {
  public:
-  CharacterStreamUTF16Buffer();
-  virtual ~CharacterStreamUTF16Buffer() {}
-  void Initialize(Handle<String> data,
-                  unibrow::CharacterStream* stream,
-                  int start_position,
-                  int end_position);
-  virtual void PushBack(uc32 ch);
-  virtual uc32 Advance();
-  virtual void SeekForward(int pos);
-
- private:
-  List<uc32> pushback_buffer_;
-  uc32 last_;
-  unibrow::CharacterStream* stream_;
-
-  List<uc32>* pushback_buffer() { return &pushback_buffer_; }
+  BufferedUC16CharacterStream();
+  virtual ~BufferedUC16CharacterStream();
+
+  virtual void PushBack(uc16 character);
+
+ protected:
+  static const unsigned kBufferSize = 512;
+  static const unsigned kPushBackStepSize = 16;
+
+  virtual unsigned SlowSeekForward(unsigned delta);
+  virtual bool ReadBlock();
+  virtual void SlowPushBack(uc16 character);
+
+  virtual unsigned BufferSeekForward(unsigned delta) = 0;
+  virtual unsigned FillBuffer(unsigned position, unsigned length) = 0;
+
+  const uc16* pushback_limit_;
+  uc16 buffer_[kBufferSize];
 };
 
 
-// UTF16 buffer to read characters from an external string.
-template <typename StringType, typename CharType>
-class ExternalStringUTF16Buffer: public UTF16Buffer {
+// Generic string stream.
+class GenericStringUC16CharacterStream: public BufferedUC16CharacterStream {
  public:
-  ExternalStringUTF16Buffer();
-  virtual ~ExternalStringUTF16Buffer() {}
-  void Initialize(Handle<StringType> data,
-                  int start_position,
-                  int end_position);
-  virtual void PushBack(uc32 ch);
-  virtual uc32 Advance();
-  virtual void SeekForward(int pos);
-
- private:
-  const CharType* raw_data_;  // Pointer to the actual array of characters.
+  GenericStringUC16CharacterStream(Handle<String> data,
+                                   unsigned start_position,
+                                   unsigned end_position);
+  virtual ~GenericStringUC16CharacterStream();
+
+ protected:
+  virtual unsigned BufferSeekForward(unsigned delta);
+  virtual unsigned FillBuffer(unsigned position, unsigned length);
+
+  Handle<String> string_;
+  unsigned start_position_;
+  unsigned length_;
 };
 
 
-// Initializes a UTF16Buffer as input stream, using one of a number
-// of strategies depending on the available character sources.
-class StreamInitializer {
+// UC16 stream based on a literal UTF-8 string.
+class Utf8ToUC16CharacterStream: public BufferedUC16CharacterStream {
  public:
-  UTF16Buffer* Init(Handle<String> source,
-                    unibrow::CharacterStream* stream,
-                    int start_position,
-                    int end_position);
- private:
-  // Different UTF16 buffers used to pull characters from. Based on input one of
-  // these will be initialized as the actual data source.
-  CharacterStreamUTF16Buffer char_stream_buffer_;
-  ExternalStringUTF16Buffer<ExternalTwoByteString, uint16_t>
-      two_byte_string_buffer_;
-  ExternalStringUTF16Buffer<ExternalAsciiString, char> ascii_string_buffer_;
-
-  // Used to convert the source string into a character stream when a stream
-  // is not passed to the scanner.
-  SafeStringInputBuffer safe_string_input_buffer_;
+  Utf8ToUC16CharacterStream(const byte* data, unsigned length);
+  virtual ~Utf8ToUC16CharacterStream();
+
+ protected:
+  virtual unsigned BufferSeekForward(unsigned delta);
+  virtual unsigned FillBuffer(unsigned char_position, unsigned length);
+  void SetRawPosition(unsigned char_position);
+
+  const byte* raw_data_;
+  unsigned raw_data_length_;  // Measured in bytes, not characters.
+  unsigned raw_data_pos_;
+  // The character position of the character at raw_data[raw_data_pos_].
+  // Not necessarily the same as pos_.
+  unsigned raw_character_position_;
+};
+
+
+// UTF16 buffer to read characters from an external string.
+class ExternalTwoByteStringUC16CharacterStream: public UC16CharacterStream {
+ public:
+  ExternalTwoByteStringUC16CharacterStream(Handle<ExternalTwoByteString> data,
+                                           int start_position,
+                                           int end_position);
+  virtual ~ExternalTwoByteStringUC16CharacterStream();
+
+  virtual void PushBack(uc16 character) {
+    ASSERT(buffer_cursor_ > raw_data_);
+    buffer_cursor_--;
+    pos_--;
+  }
+ protected:
+  virtual unsigned SlowSeekForward(unsigned delta) {
+    // Fast case always handles seeking.
+    return 0;
+  }
+  virtual bool ReadBlock() {
+    // Entire string is read at start.
+    return false;
+  }
+  Handle<ExternalTwoByteString> source_;
+  const uc16* raw_data_;  // Pointer to the actual array of characters.
 };
 
+
 // ----------------------------------------------------------------------------
 // V8JavaScriptScanner
 // JavaScript scanner getting its input from either a V8 String or a unicode
@@ -103,19 +133,9 @@ class StreamInitializer {
 
 class V8JavaScriptScanner : public JavaScriptScanner {
  public:
-  V8JavaScriptScanner() {}
-
-  // Initialize the Scanner to scan source.
-  void Initialize(Handle<String> source, int literal_flags = kAllLiterals);
-  void Initialize(Handle<String> source,
-                  unibrow::CharacterStream* stream,
-                  int literal_flags = kAllLiterals);
-  void Initialize(Handle<String> source,
-                  int start_position, int end_position,
+  V8JavaScriptScanner();
+  void Initialize(UC16CharacterStream* source,
                   int literal_flags = kAllLiterals);
-
- protected:
-  StreamInitializer stream_initializer_;
 };
 
 
@@ -123,8 +143,7 @@ class JsonScanner : public Scanner {
  public:
   JsonScanner();
 
-  // Initialize the Scanner to scan source.
-  void Initialize(Handle<String> source);
+  void Initialize(UC16CharacterStream* source);
 
   // Returns the next token.
   Token::Value Next();
@@ -138,7 +157,7 @@ class JsonScanner : public Scanner {
   // Recognizes all of the single-character tokens directly, or calls a function
   // to scan a number, string or identifier literal.
   // The only allowed whitespace characters between tokens are tab,
-  // carrige-return, newline and space.
+  // carriage-return, newline and space.
   void ScanJson();
 
   // A JSON number (production JSONNumber) is a subset of the valid JavaScript
@@ -159,60 +178,8 @@ class JsonScanner : public Scanner {
   // are the only valid JSON identifiers (productions JSONBooleanLiteral,
   // JSONNullLiteral).
   Token::Value ScanJsonIdentifier(const char* text, Token::Value token);
-
-  StreamInitializer stream_initializer_;
 };
 
-
-// ExternalStringUTF16Buffer
-template <typename StringType, typename CharType>
-ExternalStringUTF16Buffer<StringType, CharType>::ExternalStringUTF16Buffer()
-    : raw_data_(NULL) { }
-
-
-template <typename StringType, typename CharType>
-void ExternalStringUTF16Buffer<StringType, CharType>::Initialize(
-     Handle<StringType> data,
-     int start_position,
-     int end_position) {
-  ASSERT(!data.is_null());
-  raw_data_ = data->resource()->data();
-
-  ASSERT(end_position <= data->length());
-  if (start_position > 0) {
-    SeekForward(start_position);
-  }
-  end_ =
-      end_position != kNoEndPosition ? end_position : data->length();
-}
-
-
-template <typename StringType, typename CharType>
-uc32 ExternalStringUTF16Buffer<StringType, CharType>::Advance() {
-  if (pos_ < end_) {
-    return raw_data_[pos_++];
-  } else {
-    // note: currently the following increment is necessary to avoid a
-    // test-parser problem!
-    pos_++;
-    return static_cast<uc32>(-1);
-  }
-}
-
-
-template <typename StringType, typename CharType>
-void ExternalStringUTF16Buffer<StringType, CharType>::PushBack(uc32 ch) {
-  pos_--;
-  ASSERT(pos_ >= Scanner::kCharacterLookaheadBufferSize);
-  ASSERT(raw_data_[pos_ - Scanner::kCharacterLookaheadBufferSize] == ch);
-}
-
-
-template <typename StringType, typename CharType>
-void ExternalStringUTF16Buffer<StringType, CharType>::SeekForward(int pos) {
-  pos_ = pos;
-}
-
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_H_