[Compile] Refactor BackgroundCompileTask to enable its use by CompilerDispatcher

Splits background compilation data out of ScriptStreamingData and into
BackgroundCompileTask. Also makes BackgroundCompileTask no longer a sub-class
of ScriptStreamingTask, and instead have ScriptStreamingTask delegate to a
BackgroundCompileTask.

As part of this change, we now create the CharacterStream on the main thread,
and therefore have to set the (thread-local) runtime_call_stats on the already
created CharacterStream when the BackgroundCompileTask is run on the background
thread. As such, changes to CharacterStream were needed to feed the
runtime_call_stats through appropriately.

Deprecates Source::GetCachedData and StreamedSource::GetCachedData since they are
no longer used, and the streamed source never has cached data (streaming is
suppressed if cached data is available). Also removes Utf8ChunkedStream which
is dead code.

BUG=v8:8041, v8:8015
TBR=yangguo@chromium.org

Cq-Include-Trybots: luci.chromium.try:linux_chromium_rel_ng
Change-Id: Ifcc723ebf930a1dc01135fcb70929d6168471cb3
Reviewed-on: https://chromium-review.googlesource.com/1236353
Commit-Queue: Ross McIlroy <rmcilroy@chromium.org>
Reviewed-by: Toon Verwaest <verwaest@chromium.org>
Cr-Commit-Position: refs/heads/master@{#56182}

include/v8.h

@@ -1380,12 +1380,11 @@ class V8_EXPORT ScriptCompiler {
     virtual void ResetToBookmark();
   };
 
-
   /**
    * Source code which can be streamed into V8 in pieces. It will be parsed
-   * while streaming. It can be compiled after the streaming is complete.
-   * StreamedSource must be kept alive while the streaming task is ran (see
-   * ScriptStreamingTask below).
+   * while streaming and compiled after parsing has completed. StreamedSource
+   * must be kept alive while the streaming task is run (see ScriptStreamingTask
+   * below).
    */
   class V8_EXPORT StreamedSource {
    public:
@@ -1394,29 +1393,35 @@ class V8_EXPORT ScriptCompiler {
     StreamedSource(ExternalSourceStream* source_stream, Encoding encoding);
     ~StreamedSource();
 
-    // Ownership of the CachedData or its buffers is *not* transferred to the
-    // caller. The CachedData object is alive as long as the StreamedSource
-    // object is alive.
-    const CachedData* GetCachedData() const;
+    V8_DEPRECATED("No longer used", const CachedData* GetCachedData() const) {
+      return nullptr;
+    }
 
-    internal::ScriptStreamingData* impl() const { return impl_; }
+    internal::ScriptStreamingData* impl() const { return impl_.get(); }
 
     // Prevent copying.
     StreamedSource(const StreamedSource&) = delete;
     StreamedSource& operator=(const StreamedSource&) = delete;
 
    private:
-    internal::ScriptStreamingData* impl_;
+    std::unique_ptr<internal::ScriptStreamingData> impl_;
   };
 
   /**
    * A streaming task which the embedder must run on a background thread to
    * stream scripts into V8. Returned by ScriptCompiler::StartStreamingScript.
    */
-  class ScriptStreamingTask {
+  class V8_EXPORT ScriptStreamingTask final {
    public:
-    virtual ~ScriptStreamingTask() = default;
-    virtual void Run() = 0;
+    void Run();
+
+   private:
+    friend class ScriptCompiler;
+
+    explicit ScriptStreamingTask(internal::ScriptStreamingData* data)
+        : data_(data) {}
+
+    internal::ScriptStreamingData* data_;
   };
 
   enum CompileOptions {

src/api.cc

@@ -59,6 +59,7 @@
 #include "src/objects/module-inl.h"
 #include "src/objects/ordered-hash-table-inl.h"
 #include "src/objects/templates.h"
+#include "src/parsing/parse-info.h"
 #include "src/parsing/parser.h"
 #include "src/parsing/scanner-character-streams.h"
 #include "src/pending-compilation-error-handler.h"
@@ -2009,24 +2010,15 @@ ScriptCompiler::CachedData::~CachedData() {
   }
 }
 
-
 bool ScriptCompiler::ExternalSourceStream::SetBookmark() { return false; }
 
-
 void ScriptCompiler::ExternalSourceStream::ResetToBookmark() { UNREACHABLE(); }
 
 ScriptCompiler::StreamedSource::StreamedSource(ExternalSourceStream* stream,
                                                Encoding encoding)
     : impl_(new i::ScriptStreamingData(stream, encoding)) {}
 
-ScriptCompiler::StreamedSource::~StreamedSource() { delete impl_; }
-
-
-const ScriptCompiler::CachedData*
-ScriptCompiler::StreamedSource::GetCachedData() const {
-  return impl_->cached_data.get();
-}
-
+ScriptCompiler::StreamedSource::~StreamedSource() = default;
 
 Local<Script> UnboundScript::BindToCurrentContext() {
   auto function_info =
@@ -2038,7 +2030,6 @@ Local<Script> UnboundScript::BindToCurrentContext() {
   return ToApiHandle<Script>(function);
 }
 
-
 int UnboundScript::GetId() {
   auto function_info =
       i::Handle<i::SharedFunctionInfo>::cast(Utils::OpenHandle(this));
@@ -2534,6 +2525,7 @@ MaybeLocal<Function> ScriptCompiler::CompileFunctionInContext(
   RETURN_ESCAPED(Utils::CallableToLocal(result));
 }
 
+void ScriptCompiler::ScriptStreamingTask::Run() { data_->task->Run(); }
 
 ScriptCompiler::ScriptStreamingTask* ScriptCompiler::StartStreamingScript(
     Isolate* v8_isolate, StreamedSource* source, CompileOptions options) {
@@ -2544,10 +2536,13 @@ ScriptCompiler::ScriptStreamingTask* ScriptCompiler::StartStreamingScript(
   // TODO(rmcilroy): remove CompileOptions from the API.
   CHECK(options == ScriptCompiler::kNoCompileOptions);
   i::Isolate* isolate = reinterpret_cast<i::Isolate*>(v8_isolate);
-  return i::Compiler::NewBackgroundCompileTask(source->impl(), isolate);
+  i::ScriptStreamingData* data = source->impl();
+  std::unique_ptr<i::BackgroundCompileTask> task =
+      base::make_unique<i::BackgroundCompileTask>(data, isolate);
+  data->task = std::move(task);
+  return new ScriptCompiler::ScriptStreamingTask(data);
 }
 
-
 MaybeLocal<Script> ScriptCompiler::Compile(Local<Context> context,
                                            StreamedSource* v8_source,
                                            Local<String> full_source_string,
@@ -2562,11 +2557,11 @@ MaybeLocal<Script> ScriptCompiler::Compile(Local<Context> context,
       isolate, origin.ResourceName(), origin.ResourceLineOffset(),
       origin.ResourceColumnOffset(), origin.SourceMapUrl(),
       origin.HostDefinedOptions());
-  i::ScriptStreamingData* streaming_data = v8_source->impl();
+  i::ScriptStreamingData* data = v8_source->impl();
 
   i::MaybeHandle<i::SharedFunctionInfo> maybe_function_info =
       i::Compiler::GetSharedFunctionInfoForStreamedScript(
-          isolate, str, script_details, origin.Options(), streaming_data);
+          isolate, str, script_details, origin.Options(), data);
 
   i::Handle<i::SharedFunctionInfo> result;
   has_pending_exception = !maybe_function_info.ToHandle(&result);

src/compiler.cc

@@ -941,93 +941,75 @@ std::unique_ptr<UnoptimizedCompilationJob> CompileTopLevelOnBackgroundThread(
   return outer_function_job;
 }
 
-class BackgroundCompileTask : public ScriptCompiler::ScriptStreamingTask {
- public:
-  BackgroundCompileTask(ScriptStreamingData* source, Isolate* isolate);
-
-  void Run() override;
-
- private:
-  ScriptStreamingData* source_;  // Not owned.
-  int stack_size_;
-  WorkerThreadRuntimeCallStats* worker_thread_runtime_call_stats_;
-  AccountingAllocator* allocator_;
-  TimedHistogram* timer_;
-
-  DISALLOW_COPY_AND_ASSIGN(BackgroundCompileTask);
-};
+}  // namespace
 
-BackgroundCompileTask::BackgroundCompileTask(ScriptStreamingData* source,
+BackgroundCompileTask::BackgroundCompileTask(ScriptStreamingData* streamed_data,
                                              Isolate* isolate)
-    : source_(source),
+    : info_(new ParseInfo(isolate)),
       stack_size_(i::FLAG_stack_size),
       worker_thread_runtime_call_stats_(
           isolate->counters()->worker_thread_runtime_call_stats()),
+      allocator_(isolate->allocator()),
       timer_(isolate->counters()->compile_script_on_background()) {
   VMState<PARSER> state(isolate);
 
   // Prepare the data for the internalization phase and compilation phase, which
   // will happen in the main thread after parsing.
-  ParseInfo* info = new ParseInfo(isolate);
   LOG(isolate, ScriptEvent(Logger::ScriptEventType::kStreamingCompile,
-                           info->script_id()));
-  info->set_toplevel();
-  info->set_unicode_cache(&source_->unicode_cache);
-  info->set_allow_lazy_parsing();
-  if (V8_UNLIKELY(info->block_coverage_enabled())) {
-    info->AllocateSourceRangeMap();
+                           info_->script_id()));
+  info_->set_toplevel();
+  info_->set_unicode_cache(&unicode_cache_);
+  info_->set_allow_lazy_parsing();
+  if (V8_UNLIKELY(info_->block_coverage_enabled())) {
+    info_->AllocateSourceRangeMap();
   }
   LanguageMode language_mode = construct_language_mode(FLAG_use_strict);
-  info->set_language_mode(
-      stricter_language_mode(info->language_mode(), language_mode));
+  info_->set_language_mode(
+      stricter_language_mode(info_->language_mode(), language_mode));
 
-  source->info.reset(info);
-  allocator_ = isolate->allocator();
+  std::unique_ptr<Utf16CharacterStream> stream(ScannerStream::For(
+      streamed_data->source_stream.get(), streamed_data->encoding));
+  info_->set_character_stream(std::move(stream));
 }
 
 void BackgroundCompileTask::Run() {
   TimedHistogramScope timer(timer_);
   DisallowHeapAccess no_heap_access;
 
-  source_->info->set_on_background_thread(true);
+  info_->set_on_background_thread(true);
 
   // Get a runtime call stats table associated with the current worker thread.
   WorkerThreadRuntimeCallStatsScope runtime_call_stats_scope(
       worker_thread_runtime_call_stats_);
-  RuntimeCallStats* old_runtime_call_stats =
-      source_->info->runtime_call_stats();
-  source_->info->set_runtime_call_stats(runtime_call_stats_scope.Get());
+  RuntimeCallStats* old_runtime_call_stats = info_->runtime_call_stats();
+  info_->set_runtime_call_stats(runtime_call_stats_scope.Get());
+  info_->character_stream()->set_runtime_call_stats(
+      runtime_call_stats_scope.Get());
 
   // Reset the stack limit of the parser to reflect correctly that we're on a
   // background thread.
-  uintptr_t old_stack_limit = source_->info->stack_limit();
+  uintptr_t old_stack_limit = info_->stack_limit();
   uintptr_t stack_limit = GetCurrentStackPosition() - stack_size_ * KB;
-  source_->info->set_stack_limit(stack_limit);
-
-  std::unique_ptr<Utf16CharacterStream> stream(
-      ScannerStream::For(source_->source_stream.get(), source_->encoding,
-                         source_->info->runtime_call_stats()));
-  source_->info->set_character_stream(std::move(stream));
+  info_->set_stack_limit(stack_limit);
 
   // Parser needs to stay alive for finalizing the parsing on the main
   // thread.
-  source_->parser.reset(new Parser(source_->info.get()));
-  source_->parser->set_stack_limit(stack_limit);
-  source_->parser->InitializeEmptyScopeChain(source_->info.get());
+  parser_.reset(new Parser(info_.get()));
+  parser_->set_stack_limit(stack_limit);
+  parser_->InitializeEmptyScopeChain(info_.get());
 
-  source_->parser->ParseOnBackground(source_->info.get());
-  if (source_->info->literal() != nullptr) {
+  parser_->ParseOnBackground(info_.get());
+  if (info_->literal() != nullptr) {
     // Parsing has succeeded, compile.
-    source_->outer_function_job = CompileTopLevelOnBackgroundThread(
-        source_->info.get(), allocator_, &source_->inner_function_jobs);
+    outer_function_job_ = CompileTopLevelOnBackgroundThread(
+        info_.get(), allocator_, &inner_function_jobs_);
   }
 
-  source_->info->set_stack_limit(old_stack_limit);
-  source_->info->set_runtime_call_stats(old_runtime_call_stats);
-  source_->info->set_on_background_thread(false);
+  info_->set_stack_limit(old_stack_limit);
+  info_->set_runtime_call_stats(old_runtime_call_stats);
+  info_->set_on_background_thread(false);
 }
 
-}  // namespace
 
 // ----------------------------------------------------------------------------
 // Implementation of Compiler
@@ -1761,11 +1743,6 @@ MaybeHandle<JSFunction> Compiler::GetWrappedFunction(
                                                             NOT_TENURED);
 }
 
-ScriptCompiler::ScriptStreamingTask* Compiler::NewBackgroundCompileTask(
-    ScriptStreamingData* source, Isolate* isolate) {
-  return new BackgroundCompileTask(source, isolate);
-}
-
 MaybeHandle<SharedFunctionInfo>
 Compiler::GetSharedFunctionInfoForStreamedScript(
     Isolate* isolate, Handle<String> source,
@@ -1779,7 +1756,9 @@ Compiler::GetSharedFunctionInfoForStreamedScript(
   isolate->counters()->total_load_size()->Increment(source_length);
   isolate->counters()->total_compile_size()->Increment(source_length);
 
-  ParseInfo* parse_info = streaming_data->info.get();
+  BackgroundCompileTask* task = streaming_data->task.get();
+  ParseInfo* parse_info = task->info();
+  DCHECK(parse_info->is_toplevel());
   // Check if compile cache already holds the SFI, if so no need to finalize
   // the code compiled on the background thread.
   CompilationCache* compilation_cache = isolate->compilation_cache();
@@ -1798,8 +1777,8 @@ Compiler::GetSharedFunctionInfoForStreamedScript(
     Handle<Script> script =
         NewScript(isolate, parse_info, source, script_details, origin_options,
                   NOT_NATIVES_CODE);
-    streaming_data->parser->UpdateStatistics(isolate, script);
-    streaming_data->parser->HandleSourceURLComments(isolate, script);
+    task->parser()->UpdateStatistics(isolate, script);
+    task->parser()->HandleSourceURLComments(isolate, script);
 
     if (parse_info->literal() == nullptr) {
       // Parsing has failed - report error messages.
@@ -1807,10 +1786,10 @@ Compiler::GetSharedFunctionInfoForStreamedScript(
           isolate, script, parse_info->ast_value_factory());
     } else {
       // Parsing has succeeded - finalize compilation.
-      if (streaming_data->outer_function_job) {
-        maybe_result = FinalizeTopLevel(
-            parse_info, isolate, streaming_data->outer_function_job.get(),
-            &streaming_data->inner_function_jobs);
+      if (task->outer_function_job()) {
+        maybe_result =
+            FinalizeTopLevel(parse_info, isolate, task->outer_function_job(),
+                             task->inner_function_jobs());
       } else {
         // Compilation failed on background thread - throw an exception.
         FailWithPendingException(isolate, parse_info,
@@ -1927,12 +1906,7 @@ ScriptStreamingData::ScriptStreamingData(
 
 ScriptStreamingData::~ScriptStreamingData() = default;
 
-void ScriptStreamingData::Release() {
-  parser.reset();
-  info.reset();
-  outer_function_job.reset();
-  inner_function_jobs.clear();
-}
+void ScriptStreamingData::Release() { task.reset(); }
 
 }  // namespace internal
 }  // namespace v8

src/compiler.h

@@ -20,6 +20,7 @@ namespace v8 {
 namespace internal {
 
 // Forward declarations.
+class BackgroundCompileTask;
 class JavaScriptFrame;
 class OptimizedCompilationInfo;
 class OptimizedCompilationJob;
@@ -27,8 +28,10 @@ class ParseInfo;
 class Parser;
 class ScriptData;
 struct ScriptStreamingData;
+class TimedHistogram;
 class UnoptimizedCompilationInfo;
 class UnoptimizedCompilationJob;
+class WorkerThreadRuntimeCallStats;
 
 typedef std::forward_list<std::unique_ptr<UnoptimizedCompilationJob>>
     UnoptimizedCompilationJobList;
@@ -61,13 +64,6 @@ class V8_EXPORT_PRIVATE Compiler : public AllStatic {
   V8_WARN_UNUSED_RESULT static MaybeHandle<SharedFunctionInfo>
   CompileForLiveEdit(ParseInfo* parse_info, Isolate* isolate);
 
-  // Creates a new task that when run will parse and compile the streamed
-  // script associated with |streaming_data| and can be finalized with
-  // Compiler::GetSharedFunctionInfoForStreamedScript.
-  // Note: does not take ownership of streaming_data.
-  static ScriptCompiler::ScriptStreamingTask* NewBackgroundCompileTask(
-      ScriptStreamingData* streaming_data, Isolate* isolate);
-
   // Generate and install code from previously queued compilation job.
   static bool FinalizeCompilationJob(UnoptimizedCompilationJob* job,
                                      Handle<SharedFunctionInfo> shared_info,
@@ -317,6 +313,47 @@ class OptimizedCompilationJob : public CompilationJob {
   const char* compiler_name_;
 };
 
+class BackgroundCompileTask {
+ public:
+  // Creates a new task that when run will parse and compile the streamed
+  // script associated with |data| and can be finalized with
+  // Compiler::GetSharedFunctionInfoForStreamedScript.
+  // Note: does not take ownership of |data|.
+  BackgroundCompileTask(ScriptStreamingData* data, Isolate* isolate);
+
+  void Run();
+
+  ParseInfo* info() { return info_.get(); }
+  Parser* parser() { return parser_.get(); }
+  UnoptimizedCompilationJob* outer_function_job() {
+    return outer_function_job_.get();
+  }
+  UnoptimizedCompilationJobList* inner_function_jobs() {
+    return &inner_function_jobs_;
+  }
+
+ private:
+  // Data needed for parsing, and data needed to to be passed between thread
+  // between parsing and compilation. These need to be initialized before the
+  // compilation starts.
+  std::unique_ptr<ParseInfo> info_;
+  std::unique_ptr<Parser> parser_;
+  // TODO(rmcilroy): Consider having thread-local unicode-caches rather than
+  // creating a new one each time.
+  UnicodeCache unicode_cache_;
+
+  // Data needed for finalizing compilation after background compilation.
+  std::unique_ptr<UnoptimizedCompilationJob> outer_function_job_;
+  UnoptimizedCompilationJobList inner_function_jobs_;
+
+  int stack_size_;
+  WorkerThreadRuntimeCallStats* worker_thread_runtime_call_stats_;
+  AccountingAllocator* allocator_;
+  TimedHistogram* timer_;
+
+  DISALLOW_COPY_AND_ASSIGN(BackgroundCompileTask);
+};
+
 // Contains all data which needs to be transmitted between threads for
 // background parsing and compiling and finalizing it on the main thread.
 struct ScriptStreamingData {
@@ -329,18 +366,9 @@ struct ScriptStreamingData {
   // Internal implementation of v8::ScriptCompiler::StreamedSource.
   std::unique_ptr<ScriptCompiler::ExternalSourceStream> source_stream;
   ScriptCompiler::StreamedSource::Encoding encoding;
-  std::unique_ptr<ScriptCompiler::CachedData> cached_data;
-
-  // Data needed for parsing, and data needed to to be passed between thread
-  // between parsing and compilation. These need to be initialized before the
-  // compilation starts.
-  UnicodeCache unicode_cache;
-  std::unique_ptr<ParseInfo> info;
-  std::unique_ptr<Parser> parser;
 
-  // Data needed for finalizing compilation after background compilation.
-  std::unique_ptr<UnoptimizedCompilationJob> outer_function_job;
-  UnoptimizedCompilationJobList inner_function_jobs;
+  // Task that performs background parsing and compilation.
+  std::unique_ptr<BackgroundCompileTask> task;
 
   DISALLOW_COPY_AND_ASSIGN(ScriptStreamingData);
 };

src/parsing/scanner-character-streams.cc

@@ -88,7 +88,7 @@ class OnHeapStream {
     UNREACHABLE();
   }
 
-  Range<Char> GetDataAt(size_t pos) {
+  Range<Char> GetDataAt(size_t pos, RuntimeCallStats* stats) {
     return {&string_->GetChars()[start_offset_ + Min(length_, pos)],
             &string_->GetChars()[start_offset_ + length_]};
   }
@@ -118,7 +118,7 @@ class ExternalStringStream {
   ExternalStringStream(const ExternalStringStream& other)
       : lock_(other.lock_), data_(other.data_), length_(other.length_) {}
 
-  Range<Char> GetDataAt(size_t pos) {
+  Range<Char> GetDataAt(size_t pos, RuntimeCallStats* stats) {
     return {&data_[Min(length_, pos)], &data_[length_]};
   }
 
@@ -137,7 +137,7 @@ class TestingStream {
  public:
   TestingStream(const Char* data, size_t length)
       : data_(data), length_(length) {}
-  Range<Char> GetDataAt(size_t pos) {
+  Range<Char> GetDataAt(size_t pos, RuntimeCallStats* stats) {
     return {&data_[Min(length_, pos)], &data_[length_]};
   }
 
@@ -153,17 +153,16 @@ class TestingStream {
 template <typename Char>
 class ChunkedStream {
  public:
-  ChunkedStream(ScriptCompiler::ExternalSourceStream* source,
-                RuntimeCallStats* stats)
-      : source_(source), stats_(stats) {}
+  explicit ChunkedStream(ScriptCompiler::ExternalSourceStream* source)
+      : source_(source) {}
 
   ChunkedStream(const ChunkedStream& other) {
     // TODO(rmcilroy): Implement cloning for chunked streams.
     UNREACHABLE();
   }
 
-  Range<Char> GetDataAt(size_t pos) {
-    Chunk chunk = FindChunk(pos);
+  Range<Char> GetDataAt(size_t pos, RuntimeCallStats* stats) {
+    Chunk chunk = FindChunk(pos, stats);
     size_t buffer_end = chunk.length;
     size_t buffer_pos = Min(buffer_end, pos - chunk.position);
     return {&chunk.data[buffer_pos], &chunk.data[buffer_end]};
@@ -187,13 +186,13 @@ class ChunkedStream {
     size_t end_position() const { return position + length; }
   };
 
-  Chunk FindChunk(size_t position) {
-    while (V8_UNLIKELY(chunks_.empty())) FetchChunk(size_t{0});
+  Chunk FindChunk(size_t position, RuntimeCallStats* stats) {
+    while (V8_UNLIKELY(chunks_.empty())) FetchChunk(size_t{0}, stats);
 
     // Walk forwards while the position is in front of the current chunk.
     while (position >= chunks_.back().end_position() &&
            chunks_.back().length > 0) {
-      FetchChunk(chunks_.back().end_position());
+      FetchChunk(chunks_.back().end_position(), stats);
     }
 
     // Walk backwards.
@@ -213,11 +212,11 @@ class ChunkedStream {
                          length / sizeof(Char));
   }
 
-  void FetchChunk(size_t position) {
+  void FetchChunk(size_t position, RuntimeCallStats* stats) {
     const uint8_t* data = nullptr;
     size_t length;
     {
-      RuntimeCallTimerScope scope(stats_,
+      RuntimeCallTimerScope scope(stats,
                                   RuntimeCallCounterId::kGetMoreDataCallback);
       length = source_->GetMoreData(&data);
     }
@@ -225,102 +224,11 @@ class ChunkedStream {
   }
 
   ScriptCompiler::ExternalSourceStream* source_;
-  RuntimeCallStats* stats_;
 
  protected:
   std::vector<struct Chunk> chunks_;
 };
 
-template <typename Char>
-class Utf8ChunkedStream : public ChunkedStream<uint16_t> {
- public:
-  Utf8ChunkedStream(ScriptCompiler::ExternalSourceStream* source,
-                    RuntimeCallStats* stats)
-      : ChunkedStream<uint16_t>(source, stats) {}
-
-  STATIC_ASSERT(sizeof(Char) == sizeof(uint16_t));
-  void ProcessChunk(const uint8_t* data, size_t position, size_t length) final {
-    if (length == 0) {
-      unibrow::uchar t = unibrow::Utf8::ValueOfIncrementalFinish(&state_);
-      if (t != unibrow::Utf8::kBufferEmpty) {
-        DCHECK_EQ(t, unibrow::Utf8::kBadChar);
-        incomplete_char_ = 0;
-        uint16_t* result = new uint16_t[1];
-        result[0] = unibrow::Utf8::kBadChar;
-        chunks_.emplace_back(result, position, 1);
-        position++;
-      }
-      chunks_.emplace_back(nullptr, position, 0);
-      delete[] data;
-      return;
-    }
-
-    // First count the number of complete characters that can be produced.
-
-    unibrow::Utf8::State state = state_;
-    uint32_t incomplete_char = incomplete_char_;
-    bool seen_bom = seen_bom_;
-
-    size_t i = 0;
-    size_t chars = 0;
-    while (i < length) {
-      unibrow::uchar t = unibrow::Utf8::ValueOfIncremental(data[i], &i, &state,
-                                                           &incomplete_char);
-      if (!seen_bom && t == kUtf8Bom && position + chars == 0) {
-        seen_bom = true;
-        // BOM detected at beginning of the stream. Don't copy it.
-      } else if (t != unibrow::Utf8::kIncomplete) {
-        chars++;
-        if (t > unibrow::Utf16::kMaxNonSurrogateCharCode) chars++;
-      }
-    }
-
-    // Process the data.
-
-    // If there aren't any complete characters, update the state without
-    // producing a chunk.
-    if (chars == 0) {
-      state_ = state;
-      incomplete_char_ = incomplete_char;
-      seen_bom_ = seen_bom;
-      delete[] data;
-      return;
-    }
-
-    // Update the state and produce a chunk with complete characters.
-    uint16_t* result = new uint16_t[chars];
-    uint16_t* cursor = result;
-    i = 0;
-
-    while (i < length) {
-      unibrow::uchar t = unibrow::Utf8::ValueOfIncremental(data[i], &i, &state_,
-                                                           &incomplete_char_);
-      if (V8_LIKELY(t < kUtf8Bom)) {
-        *(cursor++) = static_cast<uc16>(t);  // The by most frequent case.
-      } else if (t == unibrow::Utf8::kIncomplete) {
-        continue;
-      } else if (!seen_bom_ && t == kUtf8Bom && position == 0 &&
-                 cursor == result) {
-        // BOM detected at beginning of the stream. Don't copy it.
-        seen_bom_ = true;
-      } else if (t <= unibrow::Utf16::kMaxNonSurrogateCharCode) {
-        *(cursor++) = static_cast<uc16>(t);
-      } else {
-        *(cursor++) = unibrow::Utf16::LeadSurrogate(t);
-        *(cursor++) = unibrow::Utf16::TrailSurrogate(t);
-      }
-    }
-
-    chunks_.emplace_back(result, position, chars);
-    delete[] data;
-  }
-
- private:
-  uint32_t incomplete_char_ = 0;
-  unibrow::Utf8::State state_ = unibrow::Utf8::State::kAccept;
-  bool seen_bom_ = false;
-};
-
 // Provides a buffered utf-16 view on the bytes from the underlying ByteStream.
 // Chars are buffered if either the underlying stream isn't utf-16 or the
 // underlying utf-16 stream might move (is on-heap).
@@ -349,7 +257,8 @@ class BufferedCharacterStream : public Utf16CharacterStream {
     buffer_start_ = &buffer_[0];
     buffer_cursor_ = buffer_start_;
 
-    Range<uint8_t> range = byte_stream_.GetDataAt(position);
+    Range<uint8_t> range =
+        byte_stream_.GetDataAt(position, runtime_call_stats());
     if (range.length() == 0) {
       buffer_end_ = buffer_start_;
       return false;
@@ -401,7 +310,8 @@ class UnbufferedCharacterStream : public Utf16CharacterStream {
   bool ReadBlock() final {
     size_t position = pos();
     buffer_pos_ = position;
-    Range<uint16_t> range = byte_stream_.GetDataAt(position);
+    Range<uint16_t> range =
+        byte_stream_.GetDataAt(position, runtime_call_stats());
     buffer_start_ = range.start;
     buffer_end_ = range.end;
     buffer_cursor_ = buffer_start_;
@@ -446,7 +356,7 @@ class RelocatingCharacterStream
   }
 
   void UpdateBufferPointers() {
-    Range<uint16_t> range = byte_stream_.GetDataAt(0);
+    Range<uint16_t> range = byte_stream_.GetDataAt(0, runtime_call_stats());
     if (range.start != buffer_start_) {
       buffer_cursor_ = (buffer_cursor_ - buffer_start_) + range.start;
       buffer_start_ = range.start;
@@ -512,11 +422,9 @@ bool BufferedUtf16CharacterStream::ReadBlock() {
 class Utf8ExternalStreamingStream : public BufferedUtf16CharacterStream {
  public:
   Utf8ExternalStreamingStream(
-      ScriptCompiler::ExternalSourceStream* source_stream,
-      RuntimeCallStats* stats)
+      ScriptCompiler::ExternalSourceStream* source_stream)
       : current_({0, {0, 0, 0, unibrow::Utf8::State::kAccept}}),
-        source_stream_(source_stream),
-        stats_(stats) {}
+        source_stream_(source_stream) {}
   ~Utf8ExternalStreamingStream() final {
     for (size_t i = 0; i < chunks_.size(); i++) delete[] chunks_[i].data;
   }
@@ -574,7 +482,6 @@ class Utf8ExternalStreamingStream : public BufferedUtf16CharacterStream {
   std::vector<Chunk> chunks_;
   Position current_;
   ScriptCompiler::ExternalSourceStream* source_stream_;
-  RuntimeCallStats* stats_;
 };
 
 bool Utf8ExternalStreamingStream::SkipToPosition(size_t position) {
@@ -668,7 +575,7 @@ void Utf8ExternalStreamingStream::FillBufferFromCurrentChunk() {
 }
 
 bool Utf8ExternalStreamingStream::FetchChunk() {
-  RuntimeCallTimerScope scope(stats_,
+  RuntimeCallTimerScope scope(runtime_call_stats(),
                               RuntimeCallCounterId::kGetMoreDataCallback);
   DCHECK_EQ(current_.chunk_no, chunks_.size());
   DCHECK(chunks_.empty() || chunks_.back().length != 0);
@@ -845,17 +752,16 @@ std::unique_ptr<Utf16CharacterStream> ScannerStream::ForTesting(
 
 Utf16CharacterStream* ScannerStream::For(
     ScriptCompiler::ExternalSourceStream* source_stream,
-    v8::ScriptCompiler::StreamedSource::Encoding encoding,
-    RuntimeCallStats* stats) {
+    v8::ScriptCompiler::StreamedSource::Encoding encoding) {
   switch (encoding) {
     case v8::ScriptCompiler::StreamedSource::TWO_BYTE:
       return new UnbufferedCharacterStream<ChunkedStream>(
-          static_cast<size_t>(0), source_stream, stats);
+          static_cast<size_t>(0), source_stream);
     case v8::ScriptCompiler::StreamedSource::ONE_BYTE:
       return new BufferedCharacterStream<ChunkedStream>(static_cast<size_t>(0),
-                                                        source_stream, stats);
+                                                        source_stream);
     case v8::ScriptCompiler::StreamedSource::UTF8:
-      return new Utf8ExternalStreamingStream(source_stream, stats);
+      return new Utf8ExternalStreamingStream(source_stream);
   }
   UNREACHABLE();
 }

src/parsing/scanner-character-streams.h

@@ -24,8 +24,7 @@ class V8_EXPORT_PRIVATE ScannerStream {
                                    int start_pos, int end_pos);
   static Utf16CharacterStream* For(
       ScriptCompiler::ExternalSourceStream* source_stream,
-      ScriptCompiler::StreamedSource::Encoding encoding,
-      RuntimeCallStats* stats);
+      ScriptCompiler::StreamedSource::Encoding encoding);
 
   static std::unique_ptr<Utf16CharacterStream> ForTesting(const char* data);
   static std::unique_ptr<Utf16CharacterStream> ForTesting(const char* data,

src/parsing/scanner.h

@@ -21,13 +21,13 @@
 namespace v8 {
 namespace internal {
 
-
 class AstRawString;
 class AstValueFactory;
 class DuplicateFinder;
 class ExternalOneByteString;
 class ExternalTwoByteString;
 class ParserRecorder;
+class RuntimeCallStats;
 class UnicodeCache;
 
 // ---------------------------------------------------------------------
@@ -120,6 +120,11 @@ class Utf16CharacterStream {
   // Returns true if the stream could access the V8 heap after construction.
   virtual bool can_access_heap() const = 0;
 
+  RuntimeCallStats* runtime_call_stats() const { return runtime_call_stats_; }
+  void set_runtime_call_stats(RuntimeCallStats* runtime_call_stats) {
+    runtime_call_stats_ = runtime_call_stats;
+  }
+
  protected:
   Utf16CharacterStream(const uint16_t* buffer_start,
                        const uint16_t* buffer_cursor,
@@ -180,6 +185,7 @@ class Utf16CharacterStream {
   const uint16_t* buffer_cursor_;
   const uint16_t* buffer_end_;
   size_t buffer_pos_;
+  RuntimeCallStats* runtime_call_stats_;
 };
 
 // ----------------------------------------------------------------------------

test/cctest/parsing/test-scanner-streams.cc

@@ -155,7 +155,7 @@ TEST(Utf8StreamAsciiOnly) {
   ChunkSource chunk_source(chunks);
   std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
       v8::internal::ScannerStream::For(
-          &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
+          &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
 
   // Read the data without dying.
   v8::internal::uc32 c;
@@ -173,7 +173,7 @@ TEST(Utf8StreamBOM) {
   ChunkSource chunk_source(chunks);
   std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
       v8::internal::ScannerStream::For(
-          &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
+          &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
 
   // Read the data without tripping over the BOM.
   for (size_t i = 0; unicode_ucs2[i]; i++) {
@@ -207,7 +207,7 @@ TEST(Utf8SplitBOM) {
     ChunkSource chunk_source(chunks);
     std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
         v8::internal::ScannerStream::For(
-            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
+            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
 
     // Read the data without tripping over the BOM.
     for (size_t i = 0; unicode_ucs2[i]; i++) {
@@ -223,7 +223,7 @@ TEST(Utf8SplitBOM) {
     ChunkSource chunk_source(chunks);
     std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
         v8::internal::ScannerStream::For(
-            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
+            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
 
     // Read the data without tripping over the BOM.
     for (size_t i = 0; unicode_ucs2[i]; i++) {
@@ -238,7 +238,7 @@ TEST(Utf8SplitMultiBOM) {
   ChunkSource chunk_source(chunks);
   std::unique_ptr<i::Utf16CharacterStream> stream(
       v8::internal::ScannerStream::For(
-          &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
+          &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
 
   // Read the data, ensuring we get exactly one of the two BOMs back.
   CHECK_EQ(0xFEFF, stream->Advance());
@@ -260,7 +260,7 @@ TEST(Utf8AdvanceUntil) {
     ChunkSource chunk_source(chunks);
     std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
         v8::internal::ScannerStream::For(
-            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
+            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
 
     int32_t res = stream->AdvanceUntil(
         [](int32_t c0_) { return unibrow::IsLineTerminator(c0_); });
@@ -279,14 +279,12 @@ TEST(AdvanceMatchAdvanceUntil) {
 
     std::unique_ptr<v8::internal::Utf16CharacterStream> stream_advance(
         v8::internal::ScannerStream::For(
-            &chunk_source_a, v8::ScriptCompiler::StreamedSource::UTF8,
-            nullptr));
+            &chunk_source_a, v8::ScriptCompiler::StreamedSource::UTF8));
 
     ChunkSource chunk_source_au(chunks);
     std::unique_ptr<v8::internal::Utf16CharacterStream> stream_advance_until(
         v8::internal::ScannerStream::For(
-            &chunk_source_au, v8::ScriptCompiler::StreamedSource::UTF8,
-            nullptr));
+            &chunk_source_au, v8::ScriptCompiler::StreamedSource::UTF8));
 
     int32_t au_c0_ = stream_advance_until->AdvanceUntil(
         [](int32_t c0_) { return unibrow::IsLineTerminator(c0_); });
@@ -328,7 +326,7 @@ TEST(Utf8AdvanceUntilOverChunkBoundaries) {
     ChunkSource chunk_source(chunks);
     std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
         v8::internal::ScannerStream::For(
-            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
+            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
 
     int32_t res = stream->AdvanceUntil(
         [](int32_t c0_) { return unibrow::IsLineTerminator(c0_); });
@@ -356,7 +354,7 @@ TEST(Utf8ChunkBoundaries) {
     ChunkSource chunk_source(chunks);
     std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
         v8::internal::ScannerStream::For(
-            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
+            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
 
     for (size_t i = 0; unicode_ucs2[i]; i++) {
       CHECK_EQ(unicode_ucs2[i], stream->Advance());
@@ -385,7 +383,7 @@ TEST(Utf8SingleByteChunks) {
     ChunkSource chunk_source(chunks);
     std::unique_ptr<v8::internal::Utf16CharacterStream> stream(
         v8::internal::ScannerStream::For(
-            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
+            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
 
     for (size_t j = 0; unicode_ucs2[j]; j++) {
       CHECK_EQ(unicode_ucs2[j], stream->Advance());
@@ -547,14 +545,13 @@ void TestCharacterStreams(const char* one_byte_source, unsigned length,
     ChunkSource single_chunk(data, 1, data_end - data, false);
     std::unique_ptr<i::Utf16CharacterStream> one_byte_streaming_stream(
         i::ScannerStream::For(&single_chunk,
-                              v8::ScriptCompiler::StreamedSource::ONE_BYTE,
-                              nullptr));
+                              v8::ScriptCompiler::StreamedSource::ONE_BYTE));
     TestCharacterStream(one_byte_source, one_byte_streaming_stream.get(),
                         length, start, end);
 
     ChunkSource many_chunks(data, 1, data_end - data, true);
     one_byte_streaming_stream.reset(i::ScannerStream::For(
-        &many_chunks, v8::ScriptCompiler::StreamedSource::ONE_BYTE, nullptr));
+        &many_chunks, v8::ScriptCompiler::StreamedSource::ONE_BYTE));
     TestCharacterStream(one_byte_source, one_byte_streaming_stream.get(),
                         length, start, end);
   }
@@ -565,14 +562,14 @@ void TestCharacterStreams(const char* one_byte_source, unsigned length,
     const uint8_t* data_end = one_byte_vector.end();
     ChunkSource chunks(data, 1, data_end - data, false);
     std::unique_ptr<i::Utf16CharacterStream> utf8_streaming_stream(
-        i::ScannerStream::For(&chunks, v8::ScriptCompiler::StreamedSource::UTF8,
-                              nullptr));
+        i::ScannerStream::For(&chunks,
+                              v8::ScriptCompiler::StreamedSource::UTF8));
     TestCharacterStream(one_byte_source, utf8_streaming_stream.get(), length,
                         start, end);
 
     ChunkSource many_chunks(data, 1, data_end - data, true);
     utf8_streaming_stream.reset(i::ScannerStream::For(
-        &many_chunks, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
+        &many_chunks, v8::ScriptCompiler::StreamedSource::UTF8));
     TestCharacterStream(one_byte_source, utf8_streaming_stream.get(), length,
                         start, end);
   }
@@ -585,14 +582,14 @@ void TestCharacterStreams(const char* one_byte_source, unsigned length,
         reinterpret_cast<const uint8_t*>(two_byte_vector.end());
     ChunkSource chunks(data, 2, data_end - data, false);
     std::unique_ptr<i::Utf16CharacterStream> two_byte_streaming_stream(
-        i::ScannerStream::For(
-            &chunks, v8::ScriptCompiler::StreamedSource::TWO_BYTE, nullptr));
+        i::ScannerStream::For(&chunks,
+                              v8::ScriptCompiler::StreamedSource::TWO_BYTE));
     TestCharacterStream(one_byte_source, two_byte_streaming_stream.get(),
                         length, start, end);
 
     ChunkSource many_chunks(data, 2, data_end - data, true);
     two_byte_streaming_stream.reset(i::ScannerStream::For(
-        &many_chunks, v8::ScriptCompiler::StreamedSource::TWO_BYTE, nullptr));
+        &many_chunks, v8::ScriptCompiler::StreamedSource::TWO_BYTE));
     TestCharacterStream(one_byte_source, two_byte_streaming_stream.get(),
                         length, start, end);
   }
@@ -634,7 +631,7 @@ TEST(Regress651333) {
     // 65533) instead of the incorrectly coded Latin1 char.
     ChunkSource chunks(bytes, 1, len, false);
     std::unique_ptr<i::Utf16CharacterStream> stream(i::ScannerStream::For(
-        &chunks, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
+        &chunks, v8::ScriptCompiler::StreamedSource::UTF8));
     for (size_t i = 0; i < len; i++) {
       CHECK_EQ(unicode[i], stream->Advance());
     }
@@ -648,7 +645,7 @@ void TestChunkStreamAgainstReference(
   for (size_t c = 0; c < unicode_expected.size(); ++c) {
     ChunkSource chunk_source(cases[c]);
     std::unique_ptr<i::Utf16CharacterStream> stream(i::ScannerStream::For(
-        &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
+        &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8));
     for (size_t i = 0; i < unicode_expected[c].size(); i++) {
       CHECK_EQ(unicode_expected[c][i], stream->Advance());
     }
@@ -843,19 +840,17 @@ TEST(CloneCharacterStreams) {
     ChunkSource chunk_source(chunks);
     std::unique_ptr<i::Utf16CharacterStream> one_byte_streaming_stream(
         i::ScannerStream::For(&chunk_source,
-                              v8::ScriptCompiler::StreamedSource::ONE_BYTE,
-                              nullptr));
+                              v8::ScriptCompiler::StreamedSource::ONE_BYTE));
     CHECK(!one_byte_streaming_stream->can_be_cloned());
 
     std::unique_ptr<i::Utf16CharacterStream> utf8_streaming_stream(
-        i::ScannerStream::For(
-            &chunk_source, v8::ScriptCompiler::StreamedSource::UTF8, nullptr));
+        i::ScannerStream::For(&chunk_source,
+                              v8::ScriptCompiler::StreamedSource::UTF8));
     CHECK(!utf8_streaming_stream->can_be_cloned());
 
     std::unique_ptr<i::Utf16CharacterStream> two_byte_streaming_stream(
         i::ScannerStream::For(&chunk_source,
-                              v8::ScriptCompiler::StreamedSource::TWO_BYTE,
-                              nullptr));
+                              v8::ScriptCompiler::StreamedSource::TWO_BYTE));
     CHECK(!two_byte_streaming_stream->can_be_cloned());
   }
 }