[scanner] Rewrite character streams by separating underlying bytestreams from buffering.

Additionally now we only scan over flat heap strings.

Change-Id: Ia73c538a3c7923ec66089e16efa529ef3cea2d06
Reviewed-on: https://chromium-review.googlesource.com/1126938
Commit-Queue: Toon Verwaest <verwaest@chromium.org>
Reviewed-by: Marja Hölttä <marja@chromium.org>
Cr-Commit-Position: refs/heads/master@{#54258}

src/compiler-dispatcher/unoptimized-compile-job.cc

@@ -134,7 +134,7 @@ void UnoptimizedCompileJob::PrepareOnMainThread(Isolate* isolate) {
   Handle<String> source(String::cast(script->source()), isolate);
   if (source->IsExternalTwoByteString() || source->IsExternalOneByteString()) {
     std::unique_ptr<Utf16CharacterStream> stream(ScannerStream::For(
-        source, shared_->StartPosition(), shared_->EndPosition()));
+        isolate, source, shared_->StartPosition(), shared_->EndPosition()));
     parse_info_->set_character_stream(std::move(stream));
   } else {
     source = String::Flatten(isolate, source);
@@ -192,7 +192,7 @@ void UnoptimizedCompileJob::PrepareOnMainThread(Isolate* isolate) {
     }
     wrapper_ = isolate->global_handles()->Create(*wrapper);
     std::unique_ptr<Utf16CharacterStream> stream(
-        ScannerStream::For(wrapper_, shared_->StartPosition() - offset,
+        ScannerStream::For(isolate, wrapper_, shared_->StartPosition() - offset,
                            shared_->EndPosition() - offset));
     parse_info_->set_character_stream(std::move(stream));
   }

src/parsing/parsing.cc

@@ -25,9 +25,9 @@ bool ParseProgram(ParseInfo* info, Isolate* isolate) {
 
   // Create a character stream for the parser.
   Handle<String> source(String::cast(info->script()->source()), isolate);
-  source = String::Flatten(isolate, source);
   isolate->counters()->total_parse_size()->Increment(source->length());
-  std::unique_ptr<Utf16CharacterStream> stream(ScannerStream::For(source));
+  std::unique_ptr<Utf16CharacterStream> stream(
+      ScannerStream::For(isolate, source));
   info->set_character_stream(std::move(stream));
 
   Parser parser(info);
@@ -60,10 +60,10 @@ bool ParseFunction(ParseInfo* info, Handle<SharedFunctionInfo> shared_info,
 
   // Create a character stream for the parser.
   Handle<String> source(String::cast(info->script()->source()), isolate);
-  source = String::Flatten(isolate, source);
   isolate->counters()->total_parse_size()->Increment(source->length());
-  std::unique_ptr<Utf16CharacterStream> stream(ScannerStream::For(
-      source, shared_info->StartPosition(), shared_info->EndPosition()));
+  std::unique_ptr<Utf16CharacterStream> stream(
+      ScannerStream::For(isolate, source, shared_info->StartPosition(),
+                         shared_info->EndPosition()));
   info->set_character_stream(std::move(stream));
 
   VMState<PARSER> state(isolate);

src/parsing/scanner-character-streams.h

@@ -19,9 +19,9 @@ class String;
 
 class V8_EXPORT_PRIVATE ScannerStream {
  public:
-  static Utf16CharacterStream* For(Handle<String> data);
-  static Utf16CharacterStream* For(Handle<String> data, int start_pos,
-                                   int end_pos);
+  static Utf16CharacterStream* For(Isolate* isolate, Handle<String> data);
+  static Utf16CharacterStream* For(Isolate* isolate, Handle<String> data,
+                                   int start_pos, int end_pos);
   static Utf16CharacterStream* For(
       ScriptCompiler::ExternalSourceStream* source_stream,
       ScriptCompiler::StreamedSource::Encoding encoding,

test/cctest/parsing/test-scanner-streams.cc

@@ -315,7 +315,7 @@ void TestCharacterStreams(const char* one_byte_source, unsigned length,
     i::Handle<i::String> uc16_string(
         factory->NewExternalStringFromTwoByte(&resource).ToHandleChecked());
     std::unique_ptr<i::Utf16CharacterStream> uc16_stream(
-        i::ScannerStream::For(uc16_string, start, end));
+        i::ScannerStream::For(isolate, uc16_string, start, end));
     TestCharacterStream(one_byte_source, uc16_stream.get(), length, start, end);
 
     // This avoids the GC from trying to free a stack allocated resource.
@@ -335,7 +335,7 @@ void TestCharacterStreams(const char* one_byte_source, unsigned length,
         factory->NewExternalStringFromOneByte(&one_byte_resource)
             .ToHandleChecked());
     std::unique_ptr<i::Utf16CharacterStream> one_byte_stream(
-        i::ScannerStream::For(ext_one_byte_string, start, end));
+        i::ScannerStream::For(isolate, ext_one_byte_string, start, end));
     TestCharacterStream(one_byte_source, one_byte_stream.get(), length, start,
                         end);
     // This avoids the GC from trying to free a stack allocated resource.
@@ -347,7 +347,7 @@ void TestCharacterStreams(const char* one_byte_source, unsigned length,
   // 1-byte generic i::String
   {
     std::unique_ptr<i::Utf16CharacterStream> string_stream(
-        i::ScannerStream::For(one_byte_string, start, end));
+        i::ScannerStream::For(isolate, one_byte_string, start, end));
     TestCharacterStream(one_byte_source, string_stream.get(), length, start,
                         end);
   }
@@ -357,7 +357,7 @@ void TestCharacterStreams(const char* one_byte_source, unsigned length,
     i::Handle<i::String> two_byte_string =
         factory->NewStringFromTwoByte(two_byte_vector).ToHandleChecked();
     std::unique_ptr<i::Utf16CharacterStream> two_byte_string_stream(
-        i::ScannerStream::For(two_byte_string, start, end));
+        i::ScannerStream::For(isolate, two_byte_string, start, end));
     TestCharacterStream(one_byte_source, two_byte_string_stream.get(), length,
                         start, end);
   }

test/cctest/test-parsing.cc

@@ -1174,7 +1174,7 @@ void TestParserSyncWithFlags(i::Handle<i::String> source,
   if (test_preparser) {
     i::Scanner scanner(isolate->unicode_cache());
     std::unique_ptr<i::Utf16CharacterStream> stream(
-        i::ScannerStream::For(source));
+        i::ScannerStream::For(isolate, source));
     i::Zone zone(CcTest::i_isolate()->allocator(), ZONE_NAME);
     i::AstValueFactory ast_value_factory(
         &zone, CcTest::i_isolate()->ast_string_constants(),