Merged Scanner and JavaScriptScanner.

JavaScriptScanner had become the only concrete subclass of Scanner, so there
was no longer a need for the distinction.

Also fixed up comments.

Review URL: http://codereview.chromium.org/8384003

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@9854 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/parser.cc

@@ -5302,7 +5302,7 @@ static ScriptDataImpl* DoPreParse(UC16CharacterStream* source,
                                   int flags,
                                   ParserRecorder* recorder) {
   Isolate* isolate = Isolate::Current();
-  JavaScriptScanner scanner(isolate->unicode_cache());
+  Scanner scanner(isolate->unicode_cache());
   scanner.SetHarmonyScoping((flags & kHarmonyScoping) != 0);
   scanner.Initialize(source);
   intptr_t stack_limit = isolate->stack_guard()->real_climit();

src/parser.h

@@ -486,7 +486,7 @@ class Parser {
   void ReportMessage(const char* message, Vector<const char*> args);
 
   bool inside_with() const { return top_scope_->inside_with(); }
-  JavaScriptScanner& scanner()  { return scanner_; }
+  Scanner& scanner()  { return scanner_; }
   Mode mode() const { return mode_; }
   ScriptDataImpl* pre_data() const { return pre_data_; }
 
@@ -726,7 +726,7 @@ class Parser {
   ZoneList<Handle<String> > symbol_cache_;
 
   Handle<Script> script_;
-  JavaScriptScanner scanner_;
+  Scanner scanner_;
 
   Scope* top_scope_;
 

src/preparser-api.cc

@@ -182,7 +182,7 @@ PreParserData Preparse(UnicodeInputStream* input, size_t max_stack) {
   internal::InputStreamUTF16Buffer buffer(input);
   uintptr_t stack_limit = reinterpret_cast<uintptr_t>(&buffer) - max_stack;
   internal::UnicodeCache unicode_cache;
-  internal::JavaScriptScanner scanner(&unicode_cache);
+  internal::Scanner scanner(&unicode_cache);
   scanner.Initialize(&buffer);
   internal::CompleteParserRecorder recorder;
   preparser::PreParser::PreParseResult result =

src/preparser.cc

@@ -72,7 +72,7 @@ void PreParser::ReportUnexpectedToken(i::Token::Value token) {
   if (token == i::Token::ILLEGAL && stack_overflow_) {
     return;
   }
-  i::JavaScriptScanner::Location source_location = scanner_->location();
+  i::Scanner::Location source_location = scanner_->location();
 
   // Four of the tokens are treated specially
   switch (token) {
@@ -647,7 +647,7 @@ PreParser::Statement PreParser::ParseThrowStatement(bool* ok) {
 
   Expect(i::Token::THROW, CHECK_OK);
   if (scanner_->HasAnyLineTerminatorBeforeNext()) {
-    i::JavaScriptScanner::Location pos = scanner_->location();
+    i::Scanner::Location pos = scanner_->location();
     ReportMessageAt(pos, "newline_after_throw", NULL);
     *ok = false;
     return Statement::Default();

src/preparser.h

@@ -116,7 +116,7 @@ class PreParser {
   // success (even if parsing failed, the pre-parse data successfully
   // captured the syntax error), and false if a stack-overflow happened
   // during parsing.
-  static PreParseResult PreParseProgram(i::JavaScriptScanner* scanner,
+  static PreParseResult PreParseProgram(i::Scanner* scanner,
                                         i::ParserRecorder* log,
                                         int flags,
                                         uintptr_t stack_limit) {
@@ -449,7 +449,7 @@ class PreParser {
   };
 
   // Private constructor only used in PreParseProgram.
-  PreParser(i::JavaScriptScanner* scanner,
+  PreParser(i::Scanner* scanner,
             i::ParserRecorder* log,
             uintptr_t stack_limit,
             bool allow_lazy,
@@ -619,7 +619,7 @@ class PreParser {
                                      Identifier identifier,
                                      bool* ok);
 
-  i::JavaScriptScanner* scanner_;
+  i::Scanner* scanner_;
   i::ParserRecorder* log_;
   Scope* scope_;
   uintptr_t stack_limit_;

src/scanner.cc

@@ -36,30 +36,26 @@ namespace v8 {
 namespace internal {
 
 // ----------------------------------------------------------------------------
-// Scanner::LiteralScope
-
-Scanner::LiteralScope::LiteralScope(Scanner* self)
-    : scanner_(self), complete_(false) {
-  self->StartLiteral();
-}
-
+// Scanner
 
-Scanner::LiteralScope::~LiteralScope() {
-  if (!complete_) scanner_->DropLiteral();
-}
+Scanner::Scanner(UnicodeCache* unicode_cache)
+    : unicode_cache_(unicode_cache),
+      octal_pos_(Location::invalid()),
+      harmony_scoping_(false) { }
 
 
-void Scanner::LiteralScope::Complete() {
-  scanner_->TerminateLiteral();
-  complete_ = true;
+void Scanner::Initialize(UC16CharacterStream* source) {
+  source_ = source;
+  // Need to capture identifiers in order to recognize "get" and "set"
+  // in object literals.
+  Init();
+  // Skip initial whitespace allowing HTML comment ends just like
+  // after a newline and scan first token.
+  has_line_terminator_before_next_ = true;
+  SkipWhiteSpace();
+  Scan();
 }
 
-// ----------------------------------------------------------------------------
-// Scanner
-
-Scanner::Scanner(UnicodeCache* unicode_cache)
-    : unicode_cache_(unicode_cache) { }
-
 
 uc32 Scanner::ScanHexNumber(int expected_length) {
   ASSERT(expected_length <= 4);  // prevent overflow
@@ -88,29 +84,6 @@ uc32 Scanner::ScanHexNumber(int expected_length) {
 }
 
 
-
-// ----------------------------------------------------------------------------
-// JavaScriptScanner
-
-JavaScriptScanner::JavaScriptScanner(UnicodeCache* scanner_contants)
-    : Scanner(scanner_contants),
-      octal_pos_(Location::invalid()),
-      harmony_scoping_(false) { }
-
-
-void JavaScriptScanner::Initialize(UC16CharacterStream* source) {
-  source_ = source;
-  // Need to capture identifiers in order to recognize "get" and "set"
-  // in object literals.
-  Init();
-  // Skip initial whitespace allowing HTML comment ends just like
-  // after a newline and scan first token.
-  has_line_terminator_before_next_ = true;
-  SkipWhiteSpace();
-  Scan();
-}
-
-
 // Ensure that tokens can be stored in a byte.
 STATIC_ASSERT(Token::NUM_TOKENS <= 0x100);
 
@@ -247,7 +220,7 @@ static const byte one_char_tokens[] = {
 };
 
 
-Token::Value JavaScriptScanner::Next() {
+Token::Value Scanner::Next() {
   current_ = next_;
   has_line_terminator_before_next_ = false;
   has_multiline_comment_before_next_ = false;
@@ -279,7 +252,7 @@ static inline bool IsByteOrderMark(uc32 c) {
 }
 
 
-bool JavaScriptScanner::SkipWhiteSpace() {
+bool Scanner::SkipWhiteSpace() {
   int start_position = source_pos();
 
   while (true) {
@@ -319,7 +292,7 @@ bool JavaScriptScanner::SkipWhiteSpace() {
 }
 
 
-Token::Value JavaScriptScanner::SkipSingleLineComment() {
+Token::Value Scanner::SkipSingleLineComment() {
   Advance();
 
   // The line terminator at the end of the line is not considered
@@ -335,7 +308,7 @@ Token::Value JavaScriptScanner::SkipSingleLineComment() {
 }
 
 
-Token::Value JavaScriptScanner::SkipMultiLineComment() {
+Token::Value Scanner::SkipMultiLineComment() {
   ASSERT(c0_ == '*');
   Advance();
 
@@ -361,7 +334,7 @@ Token::Value JavaScriptScanner::SkipMultiLineComment() {
 }
 
 
-Token::Value JavaScriptScanner::ScanHtmlComment() {
+Token::Value Scanner::ScanHtmlComment() {
   // Check for <!-- comments.
   ASSERT(c0_ == '!');
   Advance();
@@ -376,7 +349,7 @@ Token::Value JavaScriptScanner::ScanHtmlComment() {
 }
 
 
-void JavaScriptScanner::Scan() {
+void Scanner::Scan() {
   next_.literal_chars = NULL;
   Token::Value token;
   do {
@@ -616,7 +589,7 @@ void JavaScriptScanner::Scan() {
 }
 
 
-void JavaScriptScanner::SeekForward(int pos) {
+void Scanner::SeekForward(int pos) {
   // After this call, we will have the token at the given position as
   // the "next" token. The "current" token will be invalid.
   if (pos == next_.location.beg_pos) return;
@@ -637,7 +610,7 @@ void JavaScriptScanner::SeekForward(int pos) {
 }
 
 
-void JavaScriptScanner::ScanEscape() {
+void Scanner::ScanEscape() {
   uc32 c = c0_;
   Advance();
 
@@ -689,7 +662,7 @@ void JavaScriptScanner::ScanEscape() {
 
 // Octal escapes of the forms '\0xx' and '\xxx' are not a part of
 // ECMA-262. Other JS VMs support them.
-uc32 JavaScriptScanner::ScanOctalEscape(uc32 c, int length) {
+uc32 Scanner::ScanOctalEscape(uc32 c, int length) {
   uc32 x = c - '0';
   int i = 0;
   for (; i < length; i++) {
@@ -712,7 +685,7 @@ uc32 JavaScriptScanner::ScanOctalEscape(uc32 c, int length) {
 }
 
 
-Token::Value JavaScriptScanner::ScanString() {
+Token::Value Scanner::ScanString() {
   uc32 quote = c0_;
   Advance();  // consume quote
 
@@ -736,13 +709,13 @@ Token::Value JavaScriptScanner::ScanString() {
 }
 
 
-void JavaScriptScanner::ScanDecimalDigits() {
+void Scanner::ScanDecimalDigits() {
   while (IsDecimalDigit(c0_))
     AddLiteralCharAdvance();
 }
 
 
-Token::Value JavaScriptScanner::ScanNumber(bool seen_period) {
+Token::Value Scanner::ScanNumber(bool seen_period) {
   ASSERT(IsDecimalDigit(c0_));  // the first digit of the number or the fraction
 
   enum { DECIMAL, HEX, OCTAL } kind = DECIMAL;
@@ -827,7 +800,7 @@ Token::Value JavaScriptScanner::ScanNumber(bool seen_period) {
 }
 
 
-uc32 JavaScriptScanner::ScanIdentifierUnicodeEscape() {
+uc32 Scanner::ScanIdentifierUnicodeEscape() {
   Advance();
   if (c0_ != 'u') return -1;
   Advance();
@@ -944,7 +917,7 @@ static Token::Value KeywordOrIdentifierToken(const char* input,
 }
 
 
-Token::Value JavaScriptScanner::ScanIdentifierOrKeyword() {
+Token::Value Scanner::ScanIdentifierOrKeyword() {
   ASSERT(unicode_cache_->IsIdentifierStart(c0_));
   LiteralScope literal(this);
   // Scan identifier start character.
@@ -989,7 +962,7 @@ Token::Value JavaScriptScanner::ScanIdentifierOrKeyword() {
 }
 
 
-Token::Value JavaScriptScanner::ScanIdentifierSuffix(LiteralScope* literal) {
+Token::Value Scanner::ScanIdentifierSuffix(LiteralScope* literal) {
   // Scan the rest of the identifier characters.
   while (unicode_cache_->IsIdentifierPart(c0_)) {
     if (c0_ == '\\') {
@@ -1012,7 +985,7 @@ Token::Value JavaScriptScanner::ScanIdentifierSuffix(LiteralScope* literal) {
 }
 
 
-bool JavaScriptScanner::ScanRegExpPattern(bool seen_equal) {
+bool Scanner::ScanRegExpPattern(bool seen_equal) {
   // Scan: ('/' | '/=') RegularExpressionBody '/' RegularExpressionFlags
   bool in_character_class = false;
 
@@ -1059,7 +1032,7 @@ bool JavaScriptScanner::ScanRegExpPattern(bool seen_equal) {
 }
 
 
-bool JavaScriptScanner::ScanLiteralUnicodeEscape() {
+bool Scanner::ScanLiteralUnicodeEscape() {
   ASSERT(c0_ == '\\');
   uc32 chars_read[6] = {'\\', 'u', 0, 0, 0, 0};
   Advance();
@@ -1089,7 +1062,7 @@ bool JavaScriptScanner::ScanLiteralUnicodeEscape() {
 }
 
 
-bool JavaScriptScanner::ScanRegExpFlags() {
+bool Scanner::ScanRegExpFlags() {
   // Scan regular expression flags.
   LiteralScope literal(this);
   while (unicode_cache_->IsIdentifierPart(c0_)) {

test/cctest/test-parsing.cc

@@ -63,7 +63,7 @@ TEST(ScanKeywords) {
     CHECK(static_cast<int>(sizeof(buffer)) >= length);
     {
       i::Utf8ToUC16CharacterStream stream(keyword, length);
-      i::JavaScriptScanner scanner(&unicode_cache);
+      i::Scanner scanner(&unicode_cache);
       // The scanner should parse 'let' as Token::LET for this test.
       scanner.SetHarmonyScoping(true);
       scanner.Initialize(&stream);
@@ -73,7 +73,7 @@ TEST(ScanKeywords) {
     // Removing characters will make keyword matching fail.
     {
       i::Utf8ToUC16CharacterStream stream(keyword, length - 1);
-      i::JavaScriptScanner scanner(&unicode_cache);
+      i::Scanner scanner(&unicode_cache);
       scanner.Initialize(&stream);
       CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
@@ -84,7 +84,7 @@ TEST(ScanKeywords) {
       memmove(buffer, keyword, length);
       buffer[length] = chars_to_append[j];
       i::Utf8ToUC16CharacterStream stream(buffer, length + 1);
-      i::JavaScriptScanner scanner(&unicode_cache);
+      i::Scanner scanner(&unicode_cache);
       scanner.Initialize(&stream);
       CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
@@ -94,7 +94,7 @@ TEST(ScanKeywords) {
       memmove(buffer, keyword, length);
       buffer[length - 1] = '_';
       i::Utf8ToUC16CharacterStream stream(buffer, length);
-      i::JavaScriptScanner scanner(&unicode_cache);
+      i::Scanner scanner(&unicode_cache);
       scanner.Initialize(&stream);
       CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());
@@ -257,7 +257,7 @@ TEST(StandAlonePreParser) {
         reinterpret_cast<const i::byte*>(program),
         static_cast<unsigned>(strlen(program)));
     i::CompleteParserRecorder log;
-    i::JavaScriptScanner scanner(i::Isolate::Current()->unicode_cache());
+    i::Scanner scanner(i::Isolate::Current()->unicode_cache());
     scanner.Initialize(&stream);
 
     int flags = i::kAllowLazy | i::kAllowNativesSyntax;
@@ -293,7 +293,7 @@ TEST(StandAlonePreParserNoNatives) {
         reinterpret_cast<const i::byte*>(program),
         static_cast<unsigned>(strlen(program)));
     i::CompleteParserRecorder log;
-    i::JavaScriptScanner scanner(i::Isolate::Current()->unicode_cache());
+    i::Scanner scanner(i::Isolate::Current()->unicode_cache());
     scanner.Initialize(&stream);
 
     // Flags don't allow natives syntax.
@@ -394,7 +394,7 @@ TEST(PreParseOverflow) {
       reinterpret_cast<const i::byte*>(*program),
       static_cast<unsigned>(kProgramSize));
   i::CompleteParserRecorder log;
-  i::JavaScriptScanner scanner(i::Isolate::Current()->unicode_cache());
+  i::Scanner scanner(i::Isolate::Current()->unicode_cache());
   scanner.Initialize(&stream);
 
 
@@ -612,7 +612,7 @@ void TestStreamScanner(i::UC16CharacterStream* stream,
                        i::Token::Value* expected_tokens,
                        int skip_pos = 0,  // Zero means not skipping.
                        int skip_to = 0) {
-  i::JavaScriptScanner scanner(i::Isolate::Current()->unicode_cache());
+  i::Scanner scanner(i::Isolate::Current()->unicode_cache());
   scanner.Initialize(stream);
 
   int i = 0;
@@ -693,7 +693,7 @@ void TestScanRegExp(const char* re_source, const char* expected) {
   i::Utf8ToUC16CharacterStream stream(
        reinterpret_cast<const i::byte*>(re_source),
        static_cast<unsigned>(strlen(re_source)));
-  i::JavaScriptScanner scanner(i::Isolate::Current()->unicode_cache());
+  i::Scanner scanner(i::Isolate::Current()->unicode_cache());
   scanner.Initialize(&stream);
 
   i::Token::Value start = scanner.peek();