Combined identical classes V8JavaScriptScanner and StandAloneJavaScriptScanner.

Now only uses the common superclass of the two scanner classes.
Updated comment on KeywordMatcher.

Review URL: http://codereview.chromium.org/7211013

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@8362 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/parser.cc

@@ -4914,7 +4914,7 @@ static ScriptDataImpl* DoPreParse(UC16CharacterStream* source,
                                   bool allow_lazy,
                                   ParserRecorder* recorder) {
   Isolate* isolate = Isolate::Current();
-  V8JavaScriptScanner scanner(isolate->unicode_cache());
+  JavaScriptScanner scanner(isolate->unicode_cache());
   scanner.Initialize(source);
   intptr_t stack_limit = isolate->stack_guard()->real_climit();
   if (!preparser::PreParser::PreParseProgram(&scanner,

src/parser.h

@@ -466,7 +466,7 @@ class Parser {
   void ReportMessage(const char* message, Vector<const char*> args);
 
   bool inside_with() const { return with_nesting_level_ > 0; }
-  V8JavaScriptScanner& scanner()  { return scanner_; }
+  JavaScriptScanner& scanner()  { return scanner_; }
   Mode mode() const { return mode_; }
   ScriptDataImpl* pre_data() const { return pre_data_; }
 
@@ -695,7 +695,7 @@ class Parser {
   ZoneList<Handle<String> > symbol_cache_;
 
   Handle<Script> script_;
-  V8JavaScriptScanner scanner_;
+  JavaScriptScanner scanner_;
 
   Scope* top_scope_;
   int with_nesting_level_;

src/preparser-api.cc

@@ -158,24 +158,6 @@ class InputStreamUTF16Buffer : public UC16CharacterStream {
 };
 
 
-class StandAloneJavaScriptScanner : public JavaScriptScanner {
- public:
-  explicit StandAloneJavaScriptScanner(UnicodeCache* unicode_cache)
-      : JavaScriptScanner(unicode_cache) { }
-
-  void Initialize(UC16CharacterStream* source) {
-    source_ = source;
-    Init();
-    // Skip initial whitespace allowing HTML comment ends just like
-    // after a newline and scan first token.
-    has_line_terminator_before_next_ = true;
-    has_multiline_comment_before_next_ = false;
-    SkipWhiteSpace();
-    Scan();
-  }
-};
-
-
 // Functions declared by allocation.h and implemented in both api.cc (for v8)
 // or here (for a stand-alone preparser).
 
@@ -195,7 +177,7 @@ PreParserData Preparse(UnicodeInputStream* input, size_t max_stack) {
   internal::InputStreamUTF16Buffer buffer(input);
   uintptr_t stack_limit = reinterpret_cast<uintptr_t>(&buffer) - max_stack;
   internal::UnicodeCache unicode_cache;
-  internal::StandAloneJavaScriptScanner scanner(&unicode_cache);
+  internal::JavaScriptScanner scanner(&unicode_cache);
   scanner.Initialize(&buffer);
   internal::CompleteParserRecorder recorder;
   preparser::PreParser::PreParseResult result =

src/scanner-base.cc

@@ -77,6 +77,18 @@ JavaScriptScanner::JavaScriptScanner(UnicodeCache* scanner_contants)
   : Scanner(scanner_contants), octal_pos_(Location::invalid()) { }
 
 
+void JavaScriptScanner::Initialize(UC16CharacterStream* source) {
+  source_ = source;
+  // Need to capture identifiers in order to recognize "get" and "set"
+  // in object literals.
+  Init();
+  // Skip initial whitespace allowing HTML comment ends just like
+  // after a newline and scan first token.
+  has_line_terminator_before_next_ = true;
+  SkipWhiteSpace();
+  Scan();
+}
+
 Token::Value JavaScriptScanner::Next() {
   current_ = next_;
   has_line_terminator_before_next_ = false;

src/scanner-base.h

@@ -471,6 +471,8 @@ class JavaScriptScanner : public Scanner {
 
   explicit JavaScriptScanner(UnicodeCache* scanner_contants);
 
+  void Initialize(UC16CharacterStream* source);
+
   // Returns the next token.
   Token::Value Next();
 
@@ -547,14 +549,20 @@ class JavaScriptScanner : public Scanner {
 class KeywordMatcher {
 //  Incrementally recognize keywords.
 //
-//  Recognized keywords:
-//      break case catch const* continue debugger* default delete do else
-//      finally false for function if in instanceof native* new null
-//      return switch this throw true try typeof var void while with
+//  Recognized as keywords:
+//      break, case, catch, const*, continue, debugger, default, delete, do,
+//      else, finally, false, for, function, if, in, instanceof, new, null,
+//      return, switch, this, throw, true, try, typeof, var, void, while, with.
+//
+//  Recognized as Future Reserved Keywords (normal and strict mode):
+//      class, enum, export, extends, implements, import, interface,
+//      let, package, private, private, protected, public, public,
+//      static, yield.
+//
+//  *: Actually a "future reserved keyword". It's the only one we are
+//     recognizing outside of ES5 strict mode, the remaining are allowed
+//     as identifiers.
 //
-//  *: Actually "future reserved keywords". These are the only ones we
-//     recognize, the remaining are allowed as identifiers.
-//     In ES5 strict mode, we should disallow all reserved keywords.
  public:
   KeywordMatcher()
       : state_(INITIAL),

src/scanner.cc

@@ -324,23 +324,4 @@ void Scanner::LiteralScope::Complete() {
   complete_ = true;
 }
 
-
-// ----------------------------------------------------------------------------
-// V8JavaScriptScanner
-
-
-void V8JavaScriptScanner::Initialize(UC16CharacterStream* source) {
-  source_ = source;
-  // Need to capture identifiers in order to recognize "get" and "set"
-  // in object literals.
-  Init();
-  // Skip initial whitespace allowing HTML comment ends just like
-  // after a newline and scan first token.
-  has_line_terminator_before_next_ = true;
-  has_multiline_comment_before_next_ = false;
-  SkipWhiteSpace();
-  Scan();
-}
-
-
 } }  // namespace v8::internal

src/scanner.h

@@ -126,21 +126,6 @@ class ExternalTwoByteStringUC16CharacterStream: public UC16CharacterStream {
   const uc16* raw_data_;  // Pointer to the actual array of characters.
 };
 
-
-// ----------------------------------------------------------------------------
-// V8JavaScriptScanner
-// JavaScript scanner getting its input from either a V8 String or a unicode
-// CharacterStream.
-
-class V8JavaScriptScanner : public JavaScriptScanner {
- public:
-  explicit V8JavaScriptScanner(UnicodeCache* unicode_cache)
-      : JavaScriptScanner(unicode_cache) {}
-
-  void Initialize(UC16CharacterStream* source);
-};
-
-
 } }  // namespace v8::internal
 
 #endif  // V8_SCANNER_H_

test/cctest/test-parsing.cc

@@ -283,7 +283,7 @@ TEST(StandAlonePreParser) {
         reinterpret_cast<const i::byte*>(program),
         static_cast<unsigned>(strlen(program)));
     i::CompleteParserRecorder log;
-    i::V8JavaScriptScanner scanner(i::Isolate::Current()->unicode_cache());
+    i::JavaScriptScanner scanner(i::Isolate::Current()->unicode_cache());
     scanner.Initialize(&stream);
 
     v8::preparser::PreParser::PreParseResult result =
@@ -376,7 +376,7 @@ TEST(PreParseOverflow) {
       reinterpret_cast<const i::byte*>(*program),
       static_cast<unsigned>(kProgramSize));
   i::CompleteParserRecorder log;
-  i::V8JavaScriptScanner scanner(i::Isolate::Current()->unicode_cache());
+  i::JavaScriptScanner scanner(i::Isolate::Current()->unicode_cache());
   scanner.Initialize(&stream);
 
 
@@ -594,7 +594,7 @@ void TestStreamScanner(i::UC16CharacterStream* stream,
                        i::Token::Value* expected_tokens,
                        int skip_pos = 0,  // Zero means not skipping.
                        int skip_to = 0) {
-  i::V8JavaScriptScanner scanner(i::Isolate::Current()->unicode_cache());
+  i::JavaScriptScanner scanner(i::Isolate::Current()->unicode_cache());
   scanner.Initialize(stream);
 
   int i = 0;
@@ -673,7 +673,7 @@ void TestScanRegExp(const char* re_source, const char* expected) {
   i::Utf8ToUC16CharacterStream stream(
        reinterpret_cast<const i::byte*>(re_source),
        static_cast<unsigned>(strlen(re_source)));
-  i::V8JavaScriptScanner scanner(i::Isolate::Current()->unicode_cache());
+  i::JavaScriptScanner scanner(i::Isolate::Current()->unicode_cache());
   scanner.Initialize(&stream);
 
   i::Token::Value start = scanner.peek();