Optimize the lexical scanner by selective inlining, and

by dealing with whitespace as part of the token scanning
instead of as a separate step before it.
Review URL: http://codereview.chromium.org/113336

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@1934 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/scanner.h

@@ -39,16 +39,33 @@ class UTF8Buffer {
   UTF8Buffer();
   ~UTF8Buffer();
 
-  void Initialize(char* src, int length);
-  void AddChar(uc32 c);
-  void Reset() { pos_ = 0; }
-  int pos() const { return pos_; }
+  void AddChar(uc32 c) {
+    if (cursor_ <= limit_ &&
+        static_cast<unsigned>(c) <= unibrow::Utf8::kMaxOneByteChar) {
+      *cursor_++ = static_cast<char>(c);
+    } else {
+      AddCharSlow(c);
+    }
+  }
+
+  void Reset() { cursor_ = data_; }
+  int pos() const { return cursor_ - data_; }
   char* data() const { return data_; }
 
  private:
   char* data_;
-  int size_;
-  int pos_;
+  char* cursor_;
+  char* limit_;
+
+  int Capacity() const {
+    return (limit_ - data_) + unibrow::Utf8::kMaxEncodedSize;
+  }
+
+  static char* ComputeLimit(char* data, int capacity) {
+    return (data + capacity) - unibrow::Utf8::kMaxEncodedSize;
+  }
+
+  void AddCharSlow(uc32 c);
 };
 
 
@@ -204,7 +221,7 @@ class Scanner {
   void Advance();
   void PushBack(uc32 ch);
 
-  void SkipWhiteSpace(bool initial);
+  bool SkipWhiteSpace();
   Token::Value SkipSingleLineComment();
   Token::Value SkipMultiLineComment();
 
@@ -212,7 +229,6 @@ class Scanner {
   inline Token::Value Select(uc32 next, Token::Value then, Token::Value else_);
 
   void Scan();
-  Token::Value ScanToken();
   void ScanDecimalDigits();
   Token::Value ScanNumber(bool seen_period);
   Token::Value ScanIdentifier();

src/token.h

@@ -197,7 +197,7 @@ namespace v8 { namespace internal {
   T(ILLEGAL, "ILLEGAL", 0)                                              \
                                                                         \
   /* Scanner-internal use only. */                                      \
-  T(COMMENT, NULL, 0)
+  T(WHITESPACE, NULL, 0)
 
 
 class Token {