Rename scanner.* to scanner-character-streams.*. and scanner-base.* to scanner.*

R=lrn@chromium.org
Signed-off-by: Thiago Farina <tfarina@chromium.org>

Review URL: http://codereview.chromium.org/7739020

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@9195 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/SConscript

@@ -111,8 +111,8 @@ SOURCES = {
     runtime.cc
     runtime-profiler.cc
     safepoint-table.cc
-    scanner-base.cc
     scanner.cc
+    scanner-character-streams.cc
     scopeinfo.cc
     scopes.cc
     serialize.cc
@@ -245,7 +245,7 @@ PREPARSER_SOURCES = {
     preparse-data.cc
     preparser.cc
     preparser-api.cc
-    scanner-base.cc
+    scanner.cc
     strtod.cc
     token.cc
     unicode.cc

src/api.cc

@@ -44,6 +44,7 @@
 #include "platform.h"
 #include "profile-generator-inl.h"
 #include "runtime-profiler.h"
+#include "scanner-character-streams.h"
 #include "serialize.h"
 #include "snapshot.h"
 #include "v8threads.h"

src/compiler.cc

@@ -41,6 +41,7 @@
 #include "parser.h"
 #include "rewriter.h"
 #include "runtime-profiler.h"
+#include "scanner-character-streams.h"
 #include "scopeinfo.h"
 #include "scopes.h"
 #include "vm-state-inl.h"

src/conversions-inl.h

@@ -38,9 +38,10 @@
 // Extra POSIX/ANSI functions for Win32/MSVC.
 
 #include "conversions.h"
-#include "strtod.h"
-#include "platform.h"
 #include "double.h"
+#include "platform.h"
+#include "scanner.h"
+#include "strtod.h"
 
 namespace v8 {
 namespace internal {

src/conversions.cc

@@ -31,7 +31,6 @@
 
 #include "conversions-inl.h"
 #include "dtoa.h"
-#include "scanner-base.h"
 #include "strtod.h"
 #include "utils.h"
 

src/conversions.h

@@ -30,11 +30,13 @@
 
 #include <limits>
 
-#include "scanner-base.h"
+#include "utils.h"
 
 namespace v8 {
 namespace internal {
 
+class UnicodeCache;
+
 // Maximum number of significant digits in decimal representation.
 // The longest possible double in decimal representation is
 // (2^53 - 1) * 2 ^ -1074 that is (2 ^ 53 - 1) * 5 ^ 1074 / 10 ^ 1074

src/dateparser.h

@@ -30,7 +30,6 @@
 
 #include "allocation.h"
 #include "char-predicates-inl.h"
-#include "scanner-base.h"
 
 namespace v8 {
 namespace internal {

src/heap.cc

@@ -41,7 +41,6 @@
 #include "natives.h"
 #include "objects-visiting.h"
 #include "runtime-profiler.h"
-#include "scanner-base.h"
 #include "scopeinfo.h"
 #include "snapshot.h"
 #include "v8threads.h"

src/isolate.cc

@@ -43,7 +43,6 @@
 #include "messages.h"
 #include "regexp-stack.h"
 #include "runtime-profiler.h"
-#include "scanner.h"
 #include "scopeinfo.h"
 #include "serialize.h"
 #include "simulator.h"

src/objects.cc

@@ -41,7 +41,6 @@
 #include "objects-visiting.h"
 #include "macro-assembler.h"
 #include "safepoint-table.h"
-#include "scanner-base.h"
 #include "string-stream.h"
 #include "utils.h"
 #include "vm-state-inl.h"

src/parser.cc

@@ -39,6 +39,7 @@
 #include "platform.h"
 #include "preparser.h"
 #include "runtime.h"
+#include "scanner-character-streams.h"
 #include "scopeinfo.h"
 #include "string-stream.h"
 

src/parser.h

@@ -30,10 +30,9 @@
 
 #include "allocation.h"
 #include "ast.h"
-#include "scanner.h"
-#include "scopes.h"
 #include "preparse-data-format.h"
 #include "preparse-data.h"
+#include "scopes.h"
 
 namespace v8 {
 namespace internal {

src/preparser-api.cc

@@ -38,7 +38,6 @@
 #include "utils.h"
 #include "list.h"
 #include "hashmap.h"
-#include "scanner-base.h"
 #include "preparse-data-format.h"
 #include "preparse-data.h"
 #include "preparser.h"

src/preparser.cc

@@ -28,21 +28,19 @@
 #include <math.h>
 
 #include "../include/v8stdint.h"
-#include "unicode.h"
-#include "globals.h"
-#include "checks.h"
+
 #include "allocation.h"
-#include "utils.h"
-#include "list.h"
+#include "checks.h"
 #include "conversions.h"
+#include "conversions-inl.h"
+#include "globals.h"
 #include "hashmap.h"
-
-#include "scanner-base.h"
+#include "list.h"
 #include "preparse-data-format.h"
 #include "preparse-data.h"
 #include "preparser.h"
-
-#include "conversions-inl.h"
+#include "unicode.h"
+#include "utils.h"
 
 namespace v8 {
 

src/preparser.h

@@ -28,7 +28,15 @@
 #ifndef V8_PREPARSER_H
 #define V8_PREPARSER_H
 
+#include "token.h"
+#include "scanner.h"
+
 namespace v8 {
+
+namespace internal {
+class UnicodeCache;
+}
+
 namespace preparser {
 
 typedef uint8_t byte;
@@ -106,7 +114,7 @@ class PreParser {
     kPreParseSuccess
   };
 
-  ~PreParser() { }
+  ~PreParser() {}
 
   // Pre-parse the program from the character stream; returns true on
   // success (even if parsing failed, the pre-parse data successfully

src/scanner-character-streams.h

+// Copyright 2011 the V8 project authors. All rights reserved.
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+//       copyright notice, this list of conditions and the following
+//       disclaimer in the documentation and/or other materials provided
+//       with the distribution.
+//     * Neither the name of Google Inc. nor the names of its
+//       contributors may be used to endorse or promote products derived
+//       from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef V8_SCANNER_CHARACTER_STREAMS_H_
+#define V8_SCANNER_CHARACTER_STREAMS_H_
+
+#include "scanner.h"
+
+namespace v8 {
+namespace internal {
+
+// A buffered character stream based on a random access character
+// source (ReadBlock can be called with pos_ pointing to any position,
+// even positions before the current).
+class BufferedUC16CharacterStream: public UC16CharacterStream {
+ public:
+  BufferedUC16CharacterStream();
+  virtual ~BufferedUC16CharacterStream();
+
+  virtual void PushBack(uc32 character);
+
+ protected:
+  static const unsigned kBufferSize = 512;
+  static const unsigned kPushBackStepSize = 16;
+
+  virtual unsigned SlowSeekForward(unsigned delta);
+  virtual bool ReadBlock();
+  virtual void SlowPushBack(uc16 character);
+
+  virtual unsigned BufferSeekForward(unsigned delta) = 0;
+  virtual unsigned FillBuffer(unsigned position, unsigned length) = 0;
+
+  const uc16* pushback_limit_;
+  uc16 buffer_[kBufferSize];
+};
+
+
+// Generic string stream.
+class GenericStringUC16CharacterStream: public BufferedUC16CharacterStream {
+ public:
+  GenericStringUC16CharacterStream(Handle<String> data,
+                                   unsigned start_position,
+                                   unsigned end_position);
+  virtual ~GenericStringUC16CharacterStream();
+
+ protected:
+  virtual unsigned BufferSeekForward(unsigned delta);
+  virtual unsigned FillBuffer(unsigned position, unsigned length);
+
+  Handle<String> string_;
+  unsigned start_position_;
+  unsigned length_;
+};
+
+
+// UC16 stream based on a literal UTF-8 string.
+class Utf8ToUC16CharacterStream: public BufferedUC16CharacterStream {
+ public:
+  Utf8ToUC16CharacterStream(const byte* data, unsigned length);
+  virtual ~Utf8ToUC16CharacterStream();
+
+ protected:
+  virtual unsigned BufferSeekForward(unsigned delta);
+  virtual unsigned FillBuffer(unsigned char_position, unsigned length);
+  void SetRawPosition(unsigned char_position);
+
+  const byte* raw_data_;
+  unsigned raw_data_length_;  // Measured in bytes, not characters.
+  unsigned raw_data_pos_;
+  // The character position of the character at raw_data[raw_data_pos_].
+  // Not necessarily the same as pos_.
+  unsigned raw_character_position_;
+};
+
+
+// UTF16 buffer to read characters from an external string.
+class ExternalTwoByteStringUC16CharacterStream: public UC16CharacterStream {
+ public:
+  ExternalTwoByteStringUC16CharacterStream(Handle<ExternalTwoByteString> data,
+                                           int start_position,
+                                           int end_position);
+  virtual ~ExternalTwoByteStringUC16CharacterStream();
+
+  virtual void PushBack(uc32 character) {
+    ASSERT(buffer_cursor_ > raw_data_);
+    buffer_cursor_--;
+    pos_--;
+  }
+
+ protected:
+  virtual unsigned SlowSeekForward(unsigned delta) {
+    // Fast case always handles seeking.
+    return 0;
+  }
+  virtual bool ReadBlock() {
+    // Entire string is read at start.
+    return false;
+  }
+  Handle<ExternalTwoByteString> source_;
+  const uc16* raw_data_;  // Pointer to the actual array of characters.
+};
+
+} }  // namespace v8::internal
+
+#endif  // V8_SCANNER_CHARACTER_STREAMS_H_

src/v8conversions.cc

@@ -34,7 +34,6 @@
 #include "v8conversions.h"
 #include "dtoa.h"
 #include "factory.h"
-#include "scanner-base.h"
 #include "strtod.h"
 
 namespace v8 {

test/cctest/test-parsing.cc

@@ -31,14 +31,14 @@
 
 #include "v8.h"
 
+#include "cctest.h"
+#include "execution.h"
 #include "isolate.h"
-#include "token.h"
-#include "scanner.h"
 #include "parser.h"
-#include "utils.h"
-#include "execution.h"
 #include "preparser.h"
-#include "cctest.h"
+#include "scanner-character-streams.h"
+#include "token.h"
+#include "utils.h"
 
 TEST(ScanKeywords) {
   struct KeywordToken {

tools/gyp/v8.gyp

@@ -407,10 +407,10 @@
             '../../src/runtime-profiler.h',
             '../../src/safepoint-table.cc',
             '../../src/safepoint-table.h',
-            '../../src/scanner-base.cc',
-            '../../src/scanner-base.h',
             '../../src/scanner.cc',
             '../../src/scanner.h',
+            '../../src/scanner-character-streams.cc',
+            '../../src/scanner-character-streams.h',
             '../../src/scopeinfo.cc',
             '../../src/scopeinfo.h',
             '../../src/scopes.cc',
@@ -825,8 +825,8 @@
             '../../src/preparser.cc',
             '../../src/preparser.h',
             '../../src/preparser-api.cc',
-            '../../src/scanner-base.cc',
-            '../../src/scanner-base.h',
+            '../../src/scanner.cc',
+            '../../src/scanner.h',
             '../../src/strtod.cc',
             '../../src/strtod.h',
             '../../src/token.cc',