Revert "[Compile] Refactor BackgroundCompileTask to enable its use by CompilerDispatcher"

This reverts commit 31228a69.

Reason for revert: Introduces new pretty consistent flakes on Linux sanitizers, one of which block LKGR. Please see https://crbug.com/v8/8219

Original change's description:
> [Compile] Refactor BackgroundCompileTask to enable its use by CompilerDispatcher
> 
> Splits background compilation data out of ScriptStreamingData and into
> BackgroundCompileTask. Also makes BackgroundCompileTask no longer a sub-class
> of ScriptStreamingTask, and instead have ScriptStreamingTask delegate to a
> BackgroundCompileTask.
> 
> As part of this change, we now create the CharacterStream on the main thread,
> and therefore have to set the (thread-local) runtime_call_stats on the already
> created CharacterStream when the BackgroundCompileTask is run on the background
> thread. As such, changes to CharacterStream were needed to feed the
> runtime_call_stats through appropriately.
> 
> Deprecates Source::GetCachedData and StreamedSource::GetCachedData since they are
> no longer used, and the streamed source never has cached data (streaming is
> suppressed if cached data is available). Also removes Utf8ChunkedStream which
> is dead code.
> 
> BUG=v8:8041, v8:8015
> TBR=yangguo@chromium.org
> 
> Cq-Include-Trybots: luci.chromium.try:linux_chromium_rel_ng
> Change-Id: Ifcc723ebf930a1dc01135fcb70929d6168471cb3
> Reviewed-on: https://chromium-review.googlesource.com/1236353
> Commit-Queue: Ross McIlroy <rmcilroy@chromium.org>
> Reviewed-by: Toon Verwaest <verwaest@chromium.org>
> Cr-Commit-Position: refs/heads/master@{#56182}

TBR=rmcilroy@chromium.org,yangguo@chromium.org,verwaest@chromium.org

Change-Id: Ib05bcbde2e9a588bd4008d2155f75cdac5cc47f5
No-Presubmit: true
No-Tree-Checks: true
No-Try: true
Bug: v8:8041, v8:8015
Cq-Include-Trybots: luci.chromium.try:linux_chromium_rel_ng
Reviewed-on: https://chromium-review.googlesource.com/1241958Reviewed-by: Maya Lekova <mslekova@chromium.org>
Commit-Queue: Maya Lekova <mslekova@chromium.org>
Cr-Commit-Position: refs/heads/master@{#56187}

include/v8.h

@@ -1380,11 +1380,12 @@ class V8_EXPORT ScriptCompiler {
     virtual void ResetToBookmark();
   };
 
+
   /**
    * Source code which can be streamed into V8 in pieces. It will be parsed
-   * while streaming and compiled after parsing has completed. StreamedSource
-   * must be kept alive while the streaming task is run (see ScriptStreamingTask
-   * below).
+   * while streaming. It can be compiled after the streaming is complete.
+   * StreamedSource must be kept alive while the streaming task is ran (see
+   * ScriptStreamingTask below).
    */
   class V8_EXPORT StreamedSource {
    public:
@@ -1393,35 +1394,29 @@ class V8_EXPORT ScriptCompiler {
     StreamedSource(ExternalSourceStream* source_stream, Encoding encoding);
     ~StreamedSource();
 
-    V8_DEPRECATED("No longer used", const CachedData* GetCachedData() const) {
-      return nullptr;
-    }
+    // Ownership of the CachedData or its buffers is *not* transferred to the
+    // caller. The CachedData object is alive as long as the StreamedSource
+    // object is alive.
+    const CachedData* GetCachedData() const;
 
-    internal::ScriptStreamingData* impl() const { return impl_.get(); }
+    internal::ScriptStreamingData* impl() const { return impl_; }
 
     // Prevent copying.
     StreamedSource(const StreamedSource&) = delete;
     StreamedSource& operator=(const StreamedSource&) = delete;
 
    private:
-    std::unique_ptr<internal::ScriptStreamingData> impl_;
+    internal::ScriptStreamingData* impl_;
   };
 
   /**
    * A streaming task which the embedder must run on a background thread to
    * stream scripts into V8. Returned by ScriptCompiler::StartStreamingScript.
    */
-  class V8_EXPORT ScriptStreamingTask final {
+  class ScriptStreamingTask {
    public:
-    void Run();
-
-   private:
-    friend class ScriptCompiler;
-
-    explicit ScriptStreamingTask(internal::ScriptStreamingData* data)
-        : data_(data) {}
-
-    internal::ScriptStreamingData* data_;
+    virtual ~ScriptStreamingTask() = default;
+    virtual void Run() = 0;
   };
 
   enum CompileOptions {

src/api.cc

@@ -59,7 +59,6 @@
 #include "src/objects/module-inl.h"
 #include "src/objects/ordered-hash-table-inl.h"
 #include "src/objects/templates.h"
-#include "src/parsing/parse-info.h"
 #include "src/parsing/parser.h"
 #include "src/parsing/scanner-character-streams.h"
 #include "src/pending-compilation-error-handler.h"
@@ -2010,15 +2009,24 @@ ScriptCompiler::CachedData::~CachedData() {
   }
 }
 
+
 bool ScriptCompiler::ExternalSourceStream::SetBookmark() { return false; }
 
+
 void ScriptCompiler::ExternalSourceStream::ResetToBookmark() { UNREACHABLE(); }
 
 ScriptCompiler::StreamedSource::StreamedSource(ExternalSourceStream* stream,
                                                Encoding encoding)
     : impl_(new i::ScriptStreamingData(stream, encoding)) {}
 
-ScriptCompiler::StreamedSource::~StreamedSource() = default;
+ScriptCompiler::StreamedSource::~StreamedSource() { delete impl_; }
+
+
+const ScriptCompiler::CachedData*
+ScriptCompiler::StreamedSource::GetCachedData() const {
+  return impl_->cached_data.get();
+}
+
 
 Local<Script> UnboundScript::BindToCurrentContext() {
   auto function_info =
@@ -2030,6 +2038,7 @@ Local<Script> UnboundScript::BindToCurrentContext() {
   return ToApiHandle<Script>(function);
 }
 
+
 int UnboundScript::GetId() {
   auto function_info =
       i::Handle<i::SharedFunctionInfo>::cast(Utils::OpenHandle(this));
@@ -2525,7 +2534,6 @@ MaybeLocal<Function> ScriptCompiler::CompileFunctionInContext(
   RETURN_ESCAPED(Utils::CallableToLocal(result));
 }
 
-void ScriptCompiler::ScriptStreamingTask::Run() { data_->task->Run(); }
 
 ScriptCompiler::ScriptStreamingTask* ScriptCompiler::StartStreamingScript(
     Isolate* v8_isolate, StreamedSource* source, CompileOptions options) {
@@ -2536,13 +2544,10 @@ ScriptCompiler::ScriptStreamingTask* ScriptCompiler::StartStreamingScript(
   // TODO(rmcilroy): remove CompileOptions from the API.
   CHECK(options == ScriptCompiler::kNoCompileOptions);
   i::Isolate* isolate = reinterpret_cast<i::Isolate*>(v8_isolate);
-  i::ScriptStreamingData* data = source->impl();
-  std::unique_ptr<i::BackgroundCompileTask> task =
-      base::make_unique<i::BackgroundCompileTask>(data, isolate);
-  data->task = std::move(task);
-  return new ScriptCompiler::ScriptStreamingTask(data);
+  return i::Compiler::NewBackgroundCompileTask(source->impl(), isolate);
 }
 
+
 MaybeLocal<Script> ScriptCompiler::Compile(Local<Context> context,
                                            StreamedSource* v8_source,
                                            Local<String> full_source_string,
@@ -2557,11 +2562,11 @@ MaybeLocal<Script> ScriptCompiler::Compile(Local<Context> context,
       isolate, origin.ResourceName(), origin.ResourceLineOffset(),
       origin.ResourceColumnOffset(), origin.SourceMapUrl(),
       origin.HostDefinedOptions());
-  i::ScriptStreamingData* data = v8_source->impl();
+  i::ScriptStreamingData* streaming_data = v8_source->impl();
 
   i::MaybeHandle<i::SharedFunctionInfo> maybe_function_info =
       i::Compiler::GetSharedFunctionInfoForStreamedScript(
-          isolate, str, script_details, origin.Options(), data);
+          isolate, str, script_details, origin.Options(), streaming_data);
 
   i::Handle<i::SharedFunctionInfo> result;
   has_pending_exception = !maybe_function_info.ToHandle(&result);

src/compiler.cc

@@ -941,75 +941,93 @@ std::unique_ptr<UnoptimizedCompilationJob> CompileTopLevelOnBackgroundThread(
   return outer_function_job;
 }
 
-}  // namespace
+class BackgroundCompileTask : public ScriptCompiler::ScriptStreamingTask {
+ public:
+  BackgroundCompileTask(ScriptStreamingData* source, Isolate* isolate);
+
+  void Run() override;
+
+ private:
+  ScriptStreamingData* source_;  // Not owned.
+  int stack_size_;
+  WorkerThreadRuntimeCallStats* worker_thread_runtime_call_stats_;
+  AccountingAllocator* allocator_;
+  TimedHistogram* timer_;
+
+  DISALLOW_COPY_AND_ASSIGN(BackgroundCompileTask);
+};
 
-BackgroundCompileTask::BackgroundCompileTask(ScriptStreamingData* streamed_data,
+BackgroundCompileTask::BackgroundCompileTask(ScriptStreamingData* source,
                                              Isolate* isolate)
-    : info_(new ParseInfo(isolate)),
+    : source_(source),
       stack_size_(i::FLAG_stack_size),
       worker_thread_runtime_call_stats_(
           isolate->counters()->worker_thread_runtime_call_stats()),
-      allocator_(isolate->allocator()),
       timer_(isolate->counters()->compile_script_on_background()) {
   VMState<PARSER> state(isolate);
 
   // Prepare the data for the internalization phase and compilation phase, which
   // will happen in the main thread after parsing.
+  ParseInfo* info = new ParseInfo(isolate);
   LOG(isolate, ScriptEvent(Logger::ScriptEventType::kStreamingCompile,
-                           info_->script_id()));
-  info_->set_toplevel();
-  info_->set_unicode_cache(&unicode_cache_);
-  info_->set_allow_lazy_parsing();
-  if (V8_UNLIKELY(info_->block_coverage_enabled())) {
-    info_->AllocateSourceRangeMap();
+                           info->script_id()));
+  info->set_toplevel();
+  info->set_unicode_cache(&source_->unicode_cache);
+  info->set_allow_lazy_parsing();
+  if (V8_UNLIKELY(info->block_coverage_enabled())) {
+    info->AllocateSourceRangeMap();
   }
   LanguageMode language_mode = construct_language_mode(FLAG_use_strict);
-  info_->set_language_mode(
-      stricter_language_mode(info_->language_mode(), language_mode));
+  info->set_language_mode(
+      stricter_language_mode(info->language_mode(), language_mode));
 
-  std::unique_ptr<Utf16CharacterStream> stream(ScannerStream::For(
-      streamed_data->source_stream.get(), streamed_data->encoding));
-  info_->set_character_stream(std::move(stream));
+  source->info.reset(info);
+  allocator_ = isolate->allocator();
 }
 
 void BackgroundCompileTask::Run() {
   TimedHistogramScope timer(timer_);
   DisallowHeapAccess no_heap_access;
 
-  info_->set_on_background_thread(true);
+  source_->info->set_on_background_thread(true);
 
   // Get a runtime call stats table associated with the current worker thread.
   WorkerThreadRuntimeCallStatsScope runtime_call_stats_scope(
       worker_thread_runtime_call_stats_);
-  RuntimeCallStats* old_runtime_call_stats = info_->runtime_call_stats();
-  info_->set_runtime_call_stats(runtime_call_stats_scope.Get());
-  info_->character_stream()->set_runtime_call_stats(
-      runtime_call_stats_scope.Get());
+  RuntimeCallStats* old_runtime_call_stats =
+      source_->info->runtime_call_stats();
+  source_->info->set_runtime_call_stats(runtime_call_stats_scope.Get());
 
   // Reset the stack limit of the parser to reflect correctly that we're on a
   // background thread.
-  uintptr_t old_stack_limit = info_->stack_limit();
+  uintptr_t old_stack_limit = source_->info->stack_limit();
   uintptr_t stack_limit = GetCurrentStackPosition() - stack_size_ * KB;
-  info_->set_stack_limit(stack_limit);
+  source_->info->set_stack_limit(stack_limit);
+
+  std::unique_ptr<Utf16CharacterStream> stream(
+      ScannerStream::For(source_->source_stream.get(), source_->encoding,
+                         source_->info->runtime_call_stats()));
+  source_->info->set_character_stream(std::move(stream));
 
   // Parser needs to stay alive for finalizing the parsing on the main
   // thread.
-  parser_.reset(new Parser(info_.get()));
-  parser_->set_stack_limit(stack_limit);
-  parser_->InitializeEmptyScopeChain(info_.get());
+  source_->parser.reset(new Parser(source_->info.get()));
+  source_->parser->set_stack_limit(stack_limit);
+  source_->parser->InitializeEmptyScopeChain(source_->info.get());
 
-  parser_->ParseOnBackground(info_.get());
-  if (info_->literal() != nullptr) {
+  source_->parser->ParseOnBackground(source_->info.get());
+  if (source_->info->literal() != nullptr) {
     // Parsing has succeeded, compile.
-    outer_function_job_ = CompileTopLevelOnBackgroundThread(
-        info_.get(), allocator_, &inner_function_jobs_);
+    source_->outer_function_job = CompileTopLevelOnBackgroundThread(
+        source_->info.get(), allocator_, &source_->inner_function_jobs);
   }
 
-  info_->set_stack_limit(old_stack_limit);
-  info_->set_runtime_call_stats(old_runtime_call_stats);
-  info_->set_on_background_thread(false);
+  source_->info->set_stack_limit(old_stack_limit);
+  source_->info->set_runtime_call_stats(old_runtime_call_stats);
+  source_->info->set_on_background_thread(false);
 }
 
+}  // namespace
 
 // ----------------------------------------------------------------------------
 // Implementation of Compiler
@@ -1743,6 +1761,11 @@ MaybeHandle<JSFunction> Compiler::GetWrappedFunction(
                                                             NOT_TENURED);
 }
 
+ScriptCompiler::ScriptStreamingTask* Compiler::NewBackgroundCompileTask(
+    ScriptStreamingData* source, Isolate* isolate) {
+  return new BackgroundCompileTask(source, isolate);
+}
+
 MaybeHandle<SharedFunctionInfo>
 Compiler::GetSharedFunctionInfoForStreamedScript(
     Isolate* isolate, Handle<String> source,
@@ -1756,9 +1779,7 @@ Compiler::GetSharedFunctionInfoForStreamedScript(
   isolate->counters()->total_load_size()->Increment(source_length);
   isolate->counters()->total_compile_size()->Increment(source_length);
 
-  BackgroundCompileTask* task = streaming_data->task.get();
-  ParseInfo* parse_info = task->info();
-  DCHECK(parse_info->is_toplevel());
+  ParseInfo* parse_info = streaming_data->info.get();
   // Check if compile cache already holds the SFI, if so no need to finalize
   // the code compiled on the background thread.
   CompilationCache* compilation_cache = isolate->compilation_cache();
@@ -1777,8 +1798,8 @@ Compiler::GetSharedFunctionInfoForStreamedScript(
     Handle<Script> script =
         NewScript(isolate, parse_info, source, script_details, origin_options,
                   NOT_NATIVES_CODE);
-    task->parser()->UpdateStatistics(isolate, script);
-    task->parser()->HandleSourceURLComments(isolate, script);
+    streaming_data->parser->UpdateStatistics(isolate, script);
+    streaming_data->parser->HandleSourceURLComments(isolate, script);
 
     if (parse_info->literal() == nullptr) {
       // Parsing has failed - report error messages.
@@ -1786,10 +1807,10 @@ Compiler::GetSharedFunctionInfoForStreamedScript(
           isolate, script, parse_info->ast_value_factory());
     } else {
       // Parsing has succeeded - finalize compilation.
-      if (task->outer_function_job()) {
-        maybe_result =
-            FinalizeTopLevel(parse_info, isolate, task->outer_function_job(),
-                             task->inner_function_jobs());
+      if (streaming_data->outer_function_job) {
+        maybe_result = FinalizeTopLevel(
+            parse_info, isolate, streaming_data->outer_function_job.get(),
+            &streaming_data->inner_function_jobs);
       } else {
         // Compilation failed on background thread - throw an exception.
         FailWithPendingException(isolate, parse_info,
@@ -1906,7 +1927,12 @@ ScriptStreamingData::ScriptStreamingData(
 
 ScriptStreamingData::~ScriptStreamingData() = default;
 
-void ScriptStreamingData::Release() { task.reset(); }
+void ScriptStreamingData::Release() {
+  parser.reset();
+  info.reset();
+  outer_function_job.reset();
+  inner_function_jobs.clear();
+}
 
 }  // namespace internal
 }  // namespace v8

src/compiler.h

@@ -20,7 +20,6 @@ namespace v8 {
 namespace internal {
 
 // Forward declarations.
-class BackgroundCompileTask;
 class JavaScriptFrame;
 class OptimizedCompilationInfo;
 class OptimizedCompilationJob;
@@ -28,10 +27,8 @@ class ParseInfo;
 class Parser;
 class ScriptData;
 struct ScriptStreamingData;
-class TimedHistogram;
 class UnoptimizedCompilationInfo;
 class UnoptimizedCompilationJob;
-class WorkerThreadRuntimeCallStats;
 
 typedef std::forward_list<std::unique_ptr<UnoptimizedCompilationJob>>
     UnoptimizedCompilationJobList;
@@ -64,6 +61,13 @@ class V8_EXPORT_PRIVATE Compiler : public AllStatic {
   V8_WARN_UNUSED_RESULT static MaybeHandle<SharedFunctionInfo>
   CompileForLiveEdit(ParseInfo* parse_info, Isolate* isolate);
 
+  // Creates a new task that when run will parse and compile the streamed
+  // script associated with |streaming_data| and can be finalized with
+  // Compiler::GetSharedFunctionInfoForStreamedScript.
+  // Note: does not take ownership of streaming_data.
+  static ScriptCompiler::ScriptStreamingTask* NewBackgroundCompileTask(
+      ScriptStreamingData* streaming_data, Isolate* isolate);
+
   // Generate and install code from previously queued compilation job.
   static bool FinalizeCompilationJob(UnoptimizedCompilationJob* job,
                                      Handle<SharedFunctionInfo> shared_info,
@@ -313,47 +317,6 @@ class OptimizedCompilationJob : public CompilationJob {
   const char* compiler_name_;
 };
 
-class BackgroundCompileTask {
- public:
-  // Creates a new task that when run will parse and compile the streamed
-  // script associated with |data| and can be finalized with
-  // Compiler::GetSharedFunctionInfoForStreamedScript.
-  // Note: does not take ownership of |data|.
-  BackgroundCompileTask(ScriptStreamingData* data, Isolate* isolate);
-
-  void Run();
-
-  ParseInfo* info() { return info_.get(); }
-  Parser* parser() { return parser_.get(); }
-  UnoptimizedCompilationJob* outer_function_job() {
-    return outer_function_job_.get();
-  }
-  UnoptimizedCompilationJobList* inner_function_jobs() {
-    return &inner_function_jobs_;
-  }
-
- private:
-  // Data needed for parsing, and data needed to to be passed between thread
-  // between parsing and compilation. These need to be initialized before the
-  // compilation starts.
-  std::unique_ptr<ParseInfo> info_;
-  std::unique_ptr<Parser> parser_;
-  // TODO(rmcilroy): Consider having thread-local unicode-caches rather than
-  // creating a new one each time.
-  UnicodeCache unicode_cache_;
-
-  // Data needed for finalizing compilation after background compilation.
-  std::unique_ptr<UnoptimizedCompilationJob> outer_function_job_;
-  UnoptimizedCompilationJobList inner_function_jobs_;
-
-  int stack_size_;
-  WorkerThreadRuntimeCallStats* worker_thread_runtime_call_stats_;
-  AccountingAllocator* allocator_;
-  TimedHistogram* timer_;
-
-  DISALLOW_COPY_AND_ASSIGN(BackgroundCompileTask);
-};
-
 // Contains all data which needs to be transmitted between threads for
 // background parsing and compiling and finalizing it on the main thread.
 struct ScriptStreamingData {
@@ -366,9 +329,18 @@ struct ScriptStreamingData {
   // Internal implementation of v8::ScriptCompiler::StreamedSource.
   std::unique_ptr<ScriptCompiler::ExternalSourceStream> source_stream;
   ScriptCompiler::StreamedSource::Encoding encoding;
+  std::unique_ptr<ScriptCompiler::CachedData> cached_data;
+
+  // Data needed for parsing, and data needed to to be passed between thread
+  // between parsing and compilation. These need to be initialized before the
+  // compilation starts.
+  UnicodeCache unicode_cache;
+  std::unique_ptr<ParseInfo> info;
+  std::unique_ptr<Parser> parser;
 
-  // Task that performs background parsing and compilation.
-  std::unique_ptr<BackgroundCompileTask> task;
+  // Data needed for finalizing compilation after background compilation.
+  std::unique_ptr<UnoptimizedCompilationJob> outer_function_job;
+  UnoptimizedCompilationJobList inner_function_jobs;
 
   DISALLOW_COPY_AND_ASSIGN(ScriptStreamingData);
 };

src/parsing/scanner-character-streams.cc

@@ -88,7 +88,7 @@ class OnHeapStream {
     UNREACHABLE();
   }
 
-  Range<Char> GetDataAt(size_t pos, RuntimeCallStats* stats) {
+  Range<Char> GetDataAt(size_t pos) {
     return {&string_->GetChars()[start_offset_ + Min(length_, pos)],
             &string_->GetChars()[start_offset_ + length_]};
   }
@@ -118,7 +118,7 @@ class ExternalStringStream {
   ExternalStringStream(const ExternalStringStream& other)
       : lock_(other.lock_), data_(other.data_), length_(other.length_) {}
 
-  Range<Char> GetDataAt(size_t pos, RuntimeCallStats* stats) {
+  Range<Char> GetDataAt(size_t pos) {
     return {&data_[Min(length_, pos)], &data_[length_]};
   }
 
@@ -137,7 +137,7 @@ class TestingStream {
  public:
   TestingStream(const Char* data, size_t length)
       : data_(data), length_(length) {}
-  Range<Char> GetDataAt(size_t pos, RuntimeCallStats* stats) {
+  Range<Char> GetDataAt(size_t pos) {
     return {&data_[Min(length_, pos)], &data_[length_]};
   }
 
@@ -153,16 +153,17 @@ class TestingStream {
 template <typename Char>
 class ChunkedStream {
  public:
-  explicit ChunkedStream(ScriptCompiler::ExternalSourceStream* source)
-      : source_(source) {}
+  ChunkedStream(ScriptCompiler::ExternalSourceStream* source,
+                RuntimeCallStats* stats)
+      : source_(source), stats_(stats) {}
 
   ChunkedStream(const ChunkedStream& other) {
     // TODO(rmcilroy): Implement cloning for chunked streams.
     UNREACHABLE();
   }
 
-  Range<Char> GetDataAt(size_t pos, RuntimeCallStats* stats) {
-    Chunk chunk = FindChunk(pos, stats);
+  Range<Char> GetDataAt(size_t pos) {
+    Chunk chunk = FindChunk(pos);
     size_t buffer_end = chunk.length;
     size_t buffer_pos = Min(buffer_end, pos - chunk.position);
     return {&chunk.data[buffer_pos], &chunk.data[buffer_end]};
@@ -186,13 +187,13 @@ class ChunkedStream {
     size_t end_position() const { return position + length; }
   };
 
-  Chunk FindChunk(size_t position, RuntimeCallStats* stats) {
-    while (V8_UNLIKELY(chunks_.empty())) FetchChunk(size_t{0}, stats);
+  Chunk FindChunk(size_t position) {
+    while (V8_UNLIKELY(chunks_.empty())) FetchChunk(size_t{0});
 
     // Walk forwards while the position is in front of the current chunk.
     while (position >= chunks_.back().end_position() &&
            chunks_.back().length > 0) {
-      FetchChunk(chunks_.back().end_position(), stats);
+      FetchChunk(chunks_.back().end_position());
     }
 
     // Walk backwards.
@@ -212,11 +213,11 @@ class ChunkedStream {
                          length / sizeof(Char));
   }
 
-  void FetchChunk(size_t position, RuntimeCallStats* stats) {
+  void FetchChunk(size_t position) {
     const uint8_t* data = nullptr;
     size_t length;
     {
-      RuntimeCallTimerScope scope(stats,
+      RuntimeCallTimerScope scope(stats_,
                                   RuntimeCallCounterId::kGetMoreDataCallback);
       length = source_->GetMoreData(&data);
     }
@@ -224,11 +225,102 @@ class ChunkedStream {
   }
 
   ScriptCompiler::ExternalSourceStream* source_;
+  RuntimeCallStats* stats_;
 
  protected:
   std::vector<struct Chunk> chunks_;
 };
 
+template <typename Char>
+class Utf8ChunkedStream : public ChunkedStream<uint16_t> {
+ public:
+  Utf8ChunkedStream(ScriptCompiler::ExternalSourceStream* source,
+                    RuntimeCallStats* stats)
+      : ChunkedStream<uint16_t>(source, stats) {}
+
+  STATIC_ASSERT(sizeof(Char) == sizeof(uint16_t));
+  void ProcessChunk(const uint8_t* data, size_t position, size_t length) final {
+    if (length == 0) {
+      unibrow::uchar t = unibrow::Utf8::ValueOfIncrementalFinish(&state_);
+      if (t != unibrow::Utf8::kBufferEmpty) {
+        DCHECK_EQ(t, unibrow::Utf8::kBadChar);
+        incomplete_char_ = 0;
+        uint16_t* result = new uint16_t[1];
+        result[0] = unibrow::Utf8::kBadChar;
+        chunks_.emplace_back(result, position, 1);
+        position++;
+      }
+      chunks_.emplace_back(nullptr, position, 0);
+      delete[] data;
+      return;
+    }
+
+    // First count the number of complete characters that can be produced.
+
+    unibrow::Utf8::State state = state_;
+    uint32_t incomplete_char = incomplete_char_;
+    bool seen_bom = seen_bom_;
+
+    size_t i = 0;
+    size_t chars = 0;
+    while (i < length) {
+      unibrow::uchar t = unibrow::Utf8::ValueOfIncremental(data[i], &i, &state,
+                                                           &incomplete_char);
+      if (!seen_bom && t == kUtf8Bom && position + chars == 0) {
+        seen_bom = true;
+        // BOM detected at beginning of the stream. Don't copy it.
+      } else if (t != unibrow::Utf8::kIncomplete) {
+        chars++;
+        if (t > unibrow::Utf16::kMaxNonSurrogateCharCode) chars++;
+      }
+    }
+
+    // Process the data.
+
+    // If there aren't any complete characters, update the state without
+    // producing a chunk.
+    if (chars == 0) {
+      state_ = state;
+      incomplete_char_ = incomplete_char;
+      seen_bom_ = seen_bom;
+      delete[] data;
+      return;
+    }
+
+    // Update the state and produce a chunk with complete characters.
+    uint16_t* result = new uint16_t[chars];
+    uint16_t* cursor = result;
+    i = 0;
+
+    while (i < length) {
+      unibrow::uchar t = unibrow::Utf8::ValueOfIncremental(data[i], &i, &state_,
+                                                           &incomplete_char_);
+      if (V8_LIKELY(t < kUtf8Bom)) {
+        *(cursor++) = static_cast<uc16>(t);  // The by most frequent case.
+      } else if (t == unibrow::Utf8::kIncomplete) {
+        continue;
+      } else if (!seen_bom_ && t == kUtf8Bom && position == 0 &&
+                 cursor == result) {
+        // BOM detected at beginning of the stream. Don't copy it.
+        seen_bom_ = true;
+      } else if (t <= unibrow::Utf16::kMaxNonSurrogateCharCode) {
+        *(cursor++) = static_cast<uc16>(t);
+      } else {
+        *(cursor++) = unibrow::Utf16::LeadSurrogate(t);
+        *(cursor++) = unibrow::Utf16::TrailSurrogate(t);
+      }
+    }
+
+    chunks_.emplace_back(result, position, chars);
+    delete[] data;
+  }
+
+ private:
+  uint32_t incomplete_char_ = 0;
+  unibrow::Utf8::State state_ = unibrow::Utf8::State::kAccept;
+  bool seen_bom_ = false;
+};
+
 // Provides a buffered utf-16 view on the bytes from the underlying ByteStream.
 // Chars are buffered if either the underlying stream isn't utf-16 or the
 // underlying utf-16 stream might move (is on-heap).
@@ -257,8 +349,7 @@ class BufferedCharacterStream : public Utf16CharacterStream {
     buffer_start_ = &buffer_[0];
     buffer_cursor_ = buffer_start_;
 
-    Range<uint8_t> range =
-        byte_stream_.GetDataAt(position, runtime_call_stats());
+    Range<uint8_t> range = byte_stream_.GetDataAt(position);
     if (range.length() == 0) {
       buffer_end_ = buffer_start_;
       return false;
@@ -310,8 +401,7 @@ class UnbufferedCharacterStream : public Utf16CharacterStream {
   bool ReadBlock() final {
     size_t position = pos();
     buffer_pos_ = position;
-    Range<uint16_t> range =
-        byte_stream_.GetDataAt(position, runtime_call_stats());
+    Range<uint16_t> range = byte_stream_.GetDataAt(position);
     buffer_start_ = range.start;
     buffer_end_ = range.end;
     buffer_cursor_ = buffer_start_;
@@ -356,7 +446,7 @@ class RelocatingCharacterStream
   }
 
   void UpdateBufferPointers() {
-    Range<uint16_t> range = byte_stream_.GetDataAt(0, runtime_call_stats());
+    Range<uint16_t> range = byte_stream_.GetDataAt(0);
     if (range.start != buffer_start_) {
       buffer_cursor_ = (buffer_cursor_ - buffer_start_) + range.start;
       buffer_start_ = range.start;
@@ -422,9 +512,11 @@ bool BufferedUtf16CharacterStream::ReadBlock() {
 class Utf8ExternalStreamingStream : public BufferedUtf16CharacterStream {
  public:
   Utf8ExternalStreamingStream(
-      ScriptCompiler::ExternalSourceStream* source_stream)
+      ScriptCompiler::ExternalSourceStream* source_stream,
+      RuntimeCallStats* stats)
       : current_({0, {0, 0, 0, unibrow::Utf8::State::kAccept}}),
-        source_stream_(source_stream) {}
+        source_stream_(source_stream),
+        stats_(stats) {}
   ~Utf8ExternalStreamingStream() final {
     for (size_t i = 0; i < chunks_.size(); i++) delete[] chunks_[i].data;
   }
@@ -482,6 +574,7 @@ class Utf8ExternalStreamingStream : public BufferedUtf16CharacterStream {
   std::vector<Chunk> chunks_;
   Position current_;
   ScriptCompiler::ExternalSourceStream* source_stream_;
+  RuntimeCallStats* stats_;
 };
 
 bool Utf8ExternalStreamingStream::SkipToPosition(size_t position) {
@@ -575,7 +668,7 @@ void Utf8ExternalStreamingStream::FillBufferFromCurrentChunk() {
 }
 
 bool Utf8ExternalStreamingStream::FetchChunk() {
-  RuntimeCallTimerScope scope(runtime_call_stats(),
+  RuntimeCallTimerScope scope(stats_,
                               RuntimeCallCounterId::kGetMoreDataCallback);
   DCHECK_EQ(current_.chunk_no, chunks_.size());
   DCHECK(chunks_.empty() || chunks_.back().length != 0);
@@ -752,16 +845,17 @@ std::unique_ptr<Utf16CharacterStream> ScannerStream::ForTesting(
 
 Utf16CharacterStream* ScannerStream::For(
     ScriptCompiler::ExternalSourceStream* source_stream,
-    v8::ScriptCompiler::StreamedSource::Encoding encoding) {
+    v8::ScriptCompiler::StreamedSource::Encoding encoding,
+    RuntimeCallStats* stats) {
   switch (encoding) {
     case v8::ScriptCompiler::StreamedSource::TWO_BYTE:
       return new UnbufferedCharacterStream<ChunkedStream>(
-          static_cast<size_t>(0), source_stream);
+          static_cast<size_t>(0), source_stream, stats);
     case v8::ScriptCompiler::StreamedSource::ONE_BYTE:
       return new BufferedCharacterStream<ChunkedStream>(static_cast<size_t>(0),
-                                                        source_stream);
+                                                        source_stream, stats);
     case v8::ScriptCompiler::StreamedSource::UTF8:
-      return new Utf8ExternalStreamingStream(source_stream);
+      return new Utf8ExternalStreamingStream(source_stream, stats);
   }
   UNREACHABLE();
 }

src/parsing/scanner-character-streams.h

@@ -24,7 +24,8 @@ class V8_EXPORT_PRIVATE ScannerStream {
                                    int start_pos, int end_pos);
   static Utf16CharacterStream* For(
       ScriptCompiler::ExternalSourceStream* source_stream,
-      ScriptCompiler::StreamedSource::Encoding encoding);
+      ScriptCompiler::StreamedSource::Encoding encoding,
+      RuntimeCallStats* stats);
 
   static std::unique_ptr<Utf16CharacterStream> ForTesting(const char* data);
   static std::unique_ptr<Utf16CharacterStream> ForTesting(const char* data,

src/parsing/scanner.h

@@ -21,13 +21,13 @@
 namespace v8 {
 namespace internal {
 
+
 class AstRawString;
 class AstValueFactory;
 class DuplicateFinder;
 class ExternalOneByteString;
 class ExternalTwoByteString;
 class ParserRecorder;
-class RuntimeCallStats;
 class UnicodeCache;
 
 // ---------------------------------------------------------------------
@@ -120,11 +120,6 @@ class Utf16CharacterStream {
   // Returns true if the stream could access the V8 heap after construction.
   virtual bool can_access_heap() const = 0;
 
-  RuntimeCallStats* runtime_call_stats() const { return runtime_call_stats_; }
-  void set_runtime_call_stats(RuntimeCallStats* runtime_call_stats) {
-    runtime_call_stats_ = runtime_call_stats;
-  }
-
  protected:
   Utf16CharacterStream(const uint16_t* buffer_start,
                        const uint16_t* buffer_cursor,
@@ -185,7 +180,6 @@ class Utf16CharacterStream {
   const uint16_t* buffer_cursor_;
   const uint16_t* buffer_end_;
   size_t buffer_pos_;
-  RuntimeCallStats* runtime_call_stats_;
 };
 
 // ----------------------------------------------------------------------------

test/cctest/parsing/test-scanner-streams.cc

@@ -155,7 +155,7 @@ TEST(Utf8StreamAsciiOnly) {
   ChunkSource chunk_source(chunks);
   std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
       v8::internal::ScannerStream::For(
-          &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
+          &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
 
   // Read the data without dying.
   v8::internal::uc32 c;
@@ -173,7 +173,7 @@ TEST(Utf8StreamBOM) {
   ChunkSource chunk_source(chunks);
   std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
       v8::internal::ScannerStream::For(
-          &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
+          &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
 
   // Read the data without tripping over the BOM.
   for (size_t i = 0; unicode_ucs2[i]; i++) {
@@ -207,7 +207,7 @@ TEST(Utf8SplitBOM) {
     ChunkSource chunk_source(chunks);
     std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
         v8::internal::ScannerStream::For(
-            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
+            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
 
     // Read the data without tripping over the BOM.
     for (size_t i = 0; unicode_ucs2[i]; i++) {
@@ -223,7 +223,7 @@ TEST(Utf8SplitBOM) {
     ChunkSource chunk_source(chunks);
     std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
         v8::internal::ScannerStream::For(
-            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
+            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
 
     // Read the data without tripping over the BOM.
     for (size_t i = 0; unicode_ucs2[i]; i++) {
@@ -238,7 +238,7 @@ TEST(Utf8SplitMultiBOM) {
   ChunkSource chunk_source(chunks);
   std::unique_ptr<i::Utf16CharacterStream> stream(
       v8::internal::ScannerStream::For(
-          &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
+          &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
 
   // Read the data, ensuring we get exactly one of the two BOMs back.
   CHECK_EQ(0xFEFF, stream->Advance());
@@ -260,7 +260,7 @@ TEST(Utf8AdvanceUntil) {
     ChunkSource chunk_source(chunks);
     std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
         v8::internal::ScannerStream::For(
-            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
+            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
 
     int32_t res = stream->AdvanceUntil(
         [](int32_t c0_) { return unibrow::IsLineTerminator(c0_); });
@@ -279,12 +279,14 @@ TEST(AdvanceMatchAdvanceUntil) {
 
     std::unique_ptr<v8::internal::Utf16CharacterStream> stream_advance(
         v8::internal::ScannerStream::For(
-            &chunk_source_a, v8::ScriptCompiler::StreamedSource::UTF8));
+            &chunk_source_a, v8::ScriptCompiler::StreamedSource::UTF8,
+            nullptr));
 
     ChunkSource chunk_source_au(chunks);
     std::unique_ptr<v8::internal::Utf16CharacterStream> stream_advance_until(
         v8::internal::ScannerStream::For(
-            &chunk_source_au, v8::ScriptCompiler::StreamedSource::UTF8));
+            &chunk_source_au, v8::ScriptCompiler::StreamedSource::UTF8,
+            nullptr));
 
     int32_t au_c0_ = stream_advance_until->AdvanceUntil(
         [](int32_t c0_) { return unibrow::IsLineTerminator(c0_); });
@@ -326,7 +328,7 @@ TEST(Utf8AdvanceUntilOverChunkBoundaries) {
     ChunkSource chunk_source(chunks);
     std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
         v8::internal::ScannerStream::For(
-            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
+            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
 
     int32_t res = stream->AdvanceUntil(
         [](int32_t c0_) { return unibrow::IsLineTerminator(c0_); });
@@ -354,7 +356,7 @@ TEST(Utf8ChunkBoundaries) {
     ChunkSource chunk_source(chunks);
     std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
         v8::internal::ScannerStream::For(
-            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
+            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
 
     for (size_t i = 0; unicode_ucs2[i]; i++) {
       CHECK_EQ(unicode_ucs2[i], stream->Advance());
@@ -383,7 +385,7 @@ TEST(Utf8SingleByteChunks) {
     ChunkSource chunk_source(chunks);
     std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
         v8::internal::ScannerStream::For(
-            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
+            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
 
     for (size_t j = 0; unicode_ucs2[j]; j++) {
       CHECK_EQ(unicode_ucs2[j], stream->Advance());
@@ -545,13 +547,14 @@ void TestCharacterStreams(const char* one_byte_source, unsigned length,
     ChunkSource single_chunk(data, 1, data_end - data, false);
     std::unique_ptr<i::Utf16CharacterStream> one_byte_streaming_stream(
         i::ScannerStream::For(&single_chunk,
-                              v8::ScriptCompiler::StreamedSource::ONE_BYTE));
+                              v8::ScriptCompiler::StreamedSource::ONE_BYTE,
+                              nullptr));
     TestCharacterStream(one_byte_source, one_byte_streaming_stream.get(),
                         length, start, end);
 
     ChunkSource many_chunks(data, 1, data_end - data, true);
     one_byte_streaming_stream.reset(i::ScannerStream::For(
-        &many_chunks, v8::ScriptCompiler::StreamedSource::ONE_BYTE));
+        &many_chunks, v8::ScriptCompiler::StreamedSource::ONE_BYTE, nullptr));
     TestCharacterStream(one_byte_source, one_byte_streaming_stream.get(),
                         length, start, end);
   }
@@ -562,14 +565,14 @@ void TestCharacterStreams(const char* one_byte_source, unsigned length,
     const uint8_t* data_end = one_byte_vector.end();
     ChunkSource chunks(data, 1, data_end - data, false);
     std::unique_ptr<i::Utf16CharacterStream> utf8_streaming_stream(
-        i::ScannerStream::For(&chunks,
-                              v8::ScriptCompiler::StreamedSource::UTF8));
+        i::ScannerStream::For(&chunks, v8::ScriptCompiler::StreamedSource::UTF8,
+                              nullptr));
     TestCharacterStream(one_byte_source, utf8_streaming_stream.get(), length,
                         start, end);
 
     ChunkSource many_chunks(data, 1, data_end - data, true);
     utf8_streaming_stream.reset(i::ScannerStream::For(
-        &many_chunks, v8::ScriptCompiler::StreamedSource::UTF8));
+        &many_chunks, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
     TestCharacterStream(one_byte_source, utf8_streaming_stream.get(), length,
                         start, end);
   }
@@ -582,14 +585,14 @@ void TestCharacterStreams(const char* one_byte_source, unsigned length,
         reinterpret_cast<const uint8_t*>(two_byte_vector.end());
     ChunkSource chunks(data, 2, data_end - data, false);
     std::unique_ptr<i::Utf16CharacterStream> two_byte_streaming_stream(
-        i::ScannerStream::For(&chunks,
-                              v8::ScriptCompiler::StreamedSource::TWO_BYTE));
+        i::ScannerStream::For(
+            &chunks, v8::ScriptCompiler::StreamedSource::TWO_BYTE, nullptr));
     TestCharacterStream(one_byte_source, two_byte_streaming_stream.get(),
                         length, start, end);
 
     ChunkSource many_chunks(data, 2, data_end - data, true);
     two_byte_streaming_stream.reset(i::ScannerStream::For(
-        &many_chunks, v8::ScriptCompiler::StreamedSource::TWO_BYTE));
+        &many_chunks, v8::ScriptCompiler::StreamedSource::TWO_BYTE, nullptr));
     TestCharacterStream(one_byte_source, two_byte_streaming_stream.get(),
                         length, start, end);
   }
@@ -631,7 +634,7 @@ TEST(Regress651333) {
     // 65533) instead of the incorrectly coded Latin1 char.
     ChunkSource chunks(bytes, 1, len, false);
     std::unique_ptr<i::Utf16CharacterStream> stream(i::ScannerStream::For(
-        &chunks, v8::ScriptCompiler::StreamedSource::UTF8));
+        &chunks, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
     for (size_t i = 0; i < len; i++) {
       CHECK_EQ(unicode[i], stream->Advance());
     }
@@ -645,7 +648,7 @@ void TestChunkStreamAgainstReference(
   for (size_t c = 0; c < unicode_expected.size(); ++c) {
     ChunkSource chunk_source(cases[c]);
     std::unique_ptr<i::Utf16CharacterStream> stream(i::ScannerStream::For(
-        &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
+        &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
     for (size_t i = 0; i < unicode_expected[c].size(); i++) {
       CHECK_EQ(unicode_expected[c][i], stream->Advance());
     }
@@ -840,17 +843,19 @@ TEST(CloneCharacterStreams) {
     ChunkSource chunk_source(chunks);
     std::unique_ptr<i::Utf16CharacterStream> one_byte_streaming_stream(
         i::ScannerStream::For(&chunk_source,
-                              v8::ScriptCompiler::StreamedSource::ONE_BYTE));
+                              v8::ScriptCompiler::StreamedSource::ONE_BYTE,
+                              nullptr));
     CHECK(!one_byte_streaming_stream->can_be_cloned());
 
     std::unique_ptr<i::Utf16CharacterStream> utf8_streaming_stream(
-        i::ScannerStream::For(&chunk_source,
-                              v8::ScriptCompiler::StreamedSource::UTF8));
+        i::ScannerStream::For(
+            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
     CHECK(!utf8_streaming_stream->can_be_cloned());
 
     std::unique_ptr<i::Utf16CharacterStream> two_byte_streaming_stream(
         i::ScannerStream::For(&chunk_source,
-                              v8::ScriptCompiler::StreamedSource::TWO_BYTE));
+                              v8::ScriptCompiler::StreamedSource::TWO_BYTE,
+                              nullptr));
     CHECK(!two_byte_streaming_stream->can_be_cloned());
   }
 }