[compiler-dispatcher] Just parse functions without SFI.

This is inital work in order to utilize CompilerDispatcher in parallel
parsing.

BUG=v8:6093

Change-Id: I6aae4f32ddb2314585d09039c1c5d7e658dc896f
Reviewed-on: https://chromium-review.googlesource.com/469709Reviewed-by: Marja Hölttä <marja@chromium.org>
Reviewed-by: Jochen Eisinger <jochen@chromium.org>
Commit-Queue: Wiktor Garbacz <wiktorg@google.com>
Cr-Commit-Position: refs/heads/master@{#44509}

src/compiler-dispatcher/compiler-dispatcher-job.cc

@@ -61,6 +61,51 @@ class TwoByteWrapper : public v8::String::ExternalStringResource {
 
 }  // namespace
 
+CompilerDispatcherJob::CompilerDispatcherJob(
+    CompilerDispatcherTracer* tracer, size_t max_stack_size,
+    Handle<String> source, int start_position, int end_position,
+    LanguageMode language_mode, int function_literal_id, bool native,
+    bool module, bool is_named_expression, bool calls_eval, uint32_t hash_seed,
+    AccountingAllocator* zone_allocator, int compiler_hints,
+    const AstStringConstants* ast_string_constants,
+    FinishCallback* finish_callback)
+    : status_(CompileJobStatus::kReadyToParse),
+      isolate_(nullptr),
+      tracer_(tracer),
+      max_stack_size_(max_stack_size),
+      finish_callback_(finish_callback),
+      trace_compiler_dispatcher_jobs_(FLAG_trace_compiler_dispatcher_jobs) {
+  parse_info_.reset(new ParseInfo(zone_allocator));
+  DCHECK(source->IsExternalTwoByteString() ||
+         source->IsExternalOneByteString());
+  character_stream_.reset(
+      ScannerStream::For(source, start_position, end_position));
+  parse_info_->set_character_stream(character_stream_.get());
+  parse_info_->set_hash_seed(hash_seed);
+  parse_info_->set_compiler_hints(compiler_hints);
+  parse_info_->set_start_position(start_position);
+  parse_info_->set_end_position(end_position);
+  unicode_cache_.reset(new UnicodeCache());
+  parse_info_->set_unicode_cache(unicode_cache_.get());
+  parse_info_->set_language_mode(language_mode);
+  parse_info_->set_function_literal_id(function_literal_id);
+  parse_info_->set_ast_string_constants(ast_string_constants);
+
+  parse_info_->set_native(native);
+  parse_info_->set_module(module);
+  parse_info_->set_is_named_expression(is_named_expression);
+  parse_info_->set_calls_eval(calls_eval);
+
+  parser_.reset(new Parser(parse_info_.get()));
+  parser_->DeserializeScopeChain(parse_info_.get(), MaybeHandle<ScopeInfo>());
+
+  if (trace_compiler_dispatcher_jobs_) {
+    PrintF("CompilerDispatcherJob[%p] created for ", static_cast<void*>(this));
+    ShortPrint();
+    PrintF(" in ready to parse state.\n");
+  }
+}
+
 CompilerDispatcherJob::CompilerDispatcherJob(Isolate* isolate,
                                              CompilerDispatcherTracer* tracer,
                                              Handle<SharedFunctionInfo> shared,
@@ -78,7 +123,7 @@ CompilerDispatcherJob::CompilerDispatcherJob(Isolate* isolate,
   Handle<String> source(String::cast(script->source()), isolate_);
   if (trace_compiler_dispatcher_jobs_) {
     PrintF("CompilerDispatcherJob[%p] created for ", static_cast<void*>(this));
-    shared_->ShortPrint();
+    ShortPrint();
     PrintF(" in initial state.\n");
   }
 }
@@ -107,17 +152,23 @@ CompilerDispatcherJob::CompilerDispatcherJob(
 
   if (trace_compiler_dispatcher_jobs_) {
     PrintF("CompilerDispatcherJob[%p] created for ", static_cast<void*>(this));
-    shared_->ShortPrint();
+    ShortPrint();
     PrintF(" in Analyzed state.\n");
   }
 }
 
 CompilerDispatcherJob::~CompilerDispatcherJob() {
-  DCHECK(ThreadId::Current().Equals(isolate_->thread_id()));
   DCHECK(status_ == CompileJobStatus::kInitial ||
+         (status_ == CompileJobStatus::kReadyToParse && finish_callback_) ||
          status_ == CompileJobStatus::kDone);
-  i::GlobalHandles::Destroy(Handle<Object>::cast(shared_).location());
-  i::GlobalHandles::Destroy(Handle<Object>::cast(context_).location());
+  if (!shared_.is_null()) {
+    DCHECK(ThreadId::Current().Equals(isolate_->thread_id()));
+    i::GlobalHandles::Destroy(Handle<Object>::cast(shared_).location());
+  }
+  if (!context_.is_null()) {
+    DCHECK(ThreadId::Current().Equals(isolate_->thread_id()));
+    i::GlobalHandles::Destroy(Handle<Object>::cast(context_).location());
+  }
 }
 
 bool CompilerDispatcherJob::IsAssociatedWith(
@@ -249,7 +300,12 @@ void CompilerDispatcherJob::Parse() {
   parser_->set_stack_limit(stack_limit);
   parser_->ParseOnBackground(parse_info_.get());
 
-  status_ = CompileJobStatus::kParsed;
+  if (finish_callback_) {
+    finish_callback_->ParseFinished(std::move(parse_info_));
+    status_ = CompileJobStatus::kDone;
+  } else {
+    status_ = CompileJobStatus::kParsed;
+  }
 }
 
 bool CompilerDispatcherJob::FinalizeParsingOnMainThread() {
@@ -402,8 +458,6 @@ bool CompilerDispatcherJob::FinalizeCompilingOnMainThread() {
 }
 
 void CompilerDispatcherJob::ResetOnMainThread() {
-  DCHECK(ThreadId::Current().Equals(isolate_->thread_id()));
-
   if (trace_compiler_dispatcher_jobs_) {
     PrintF("CompilerDispatcherJob[%p]: Resetting\n", static_cast<void*>(this));
   }
@@ -415,12 +469,15 @@ void CompilerDispatcherJob::ResetOnMainThread() {
   unicode_cache_.reset();
   character_stream_.reset();
   parse_info_.reset();
+  finish_callback_ = nullptr;
 
   if (!source_.is_null()) {
+    DCHECK(ThreadId::Current().Equals(isolate_->thread_id()));
     i::GlobalHandles::Destroy(Handle<Object>::cast(source_).location());
     source_ = Handle<String>::null();
   }
   if (!wrapper_.is_null()) {
+    DCHECK(ThreadId::Current().Equals(isolate_->thread_id()));
     i::GlobalHandles::Destroy(Handle<Object>::cast(wrapper_).location());
     wrapper_ = Handle<String>::null();
   }
@@ -463,8 +520,18 @@ double CompilerDispatcherJob::EstimateRuntimeOfNextStepInMs() const {
 }
 
 void CompilerDispatcherJob::ShortPrint() {
-  DCHECK(ThreadId::Current().Equals(isolate_->thread_id()));
-  shared_->ShortPrint();
+  if (isolate_) {
+    DCHECK(ThreadId::Current().Equals(isolate_->thread_id()));
+    DCHECK(!shared_.is_null());
+    shared_->ShortPrint();
+  } else {
+    // TODO(wiktorg) more useful info in those cases
+    if (parse_info_) {
+      PrintF("function at %d", parse_info_->start_position());
+    } else {
+      PrintF("parsed function");
+    }
+  }
 }
 
 }  // namespace internal

src/compiler-dispatcher/compiler-dispatcher-job.h

@@ -17,6 +17,7 @@ namespace v8 {
 namespace internal {
 
 class AstValueFactory;
+class AstStringConstants;
 class CompilerDispatcherTracer;
 class CompilationInfo;
 class CompilationJob;
@@ -44,10 +45,27 @@ enum class CompileJobStatus {
 
 class V8_EXPORT_PRIVATE CompilerDispatcherJob {
  public:
+  class FinishCallback {
+   public:
+    virtual ~FinishCallback() {}
+    virtual void ParseFinished(std::unique_ptr<ParseInfo> parse_info) = 0;
+  };
+
   // Creates a CompilerDispatcherJob in the initial state.
   CompilerDispatcherJob(Isolate* isolate, CompilerDispatcherTracer* tracer,
                         Handle<SharedFunctionInfo> shared,
                         size_t max_stack_size);
+  // TODO(wiktorg) document it better once I know how it relates to whole stuff
+  // Creates a CompilerDispatcherJob in ready to parse top-level function state.
+  CompilerDispatcherJob(CompilerDispatcherTracer* tracer, size_t max_stack_size,
+                        Handle<String> source, int start_position,
+                        int end_position, LanguageMode language_mode,
+                        int function_literal_id, bool native, bool module,
+                        bool is_named_expression, bool calls_eval,
+                        uint32_t hash_seed, AccountingAllocator* zone_allocator,
+                        int compiler_hints,
+                        const AstStringConstants* ast_string_constants,
+                        FinishCallback* finish_callback);
   // Creates a CompilerDispatcherJob in the analyzed state.
   CompilerDispatcherJob(Isolate* isolate, CompilerDispatcherTracer* tracer,
                         Handle<Script> script,
@@ -72,7 +90,8 @@ class V8_EXPORT_PRIVATE CompilerDispatcherJob {
   // Transition from kInitial to kReadyToParse.
   void PrepareToParseOnMainThread();
 
-  // Transition from kReadyToParse to kParsed.
+  // Transition from kReadyToParse to kParsed (or kDone if there is
+  // finish_callback).
   void Parse();
 
   // Transition from kParsed to kReadyToAnalyze (or kFailed). Returns false
@@ -116,6 +135,7 @@ class V8_EXPORT_PRIVATE CompilerDispatcherJob {
   Handle<String> wrapper_;       // Global handle.
   std::unique_ptr<v8::String::ExternalStringResourceBase> source_wrapper_;
   size_t max_stack_size_;
+  FinishCallback* finish_callback_ = nullptr;
 
   // Members required for parsing.
   std::unique_ptr<UnicodeCache> unicode_cache_;

src/parsing/parser.cc

@@ -805,8 +805,8 @@ FunctionLiteral* Parser::ParseFunction(Isolate* isolate, ParseInfo* info) {
     std::unique_ptr<Utf16CharacterStream> stream(ScannerStream::For(
         source, shared_info->start_position(), shared_info->end_position()));
     Handle<String> name(String::cast(shared_info->name()));
-    result = DoParseFunction(info, ast_value_factory()->GetString(name),
-                             stream.get());
+    scanner_.Initialize(stream.get());
+    result = DoParseFunction(info, ast_value_factory()->GetString(name));
     if (result != nullptr) {
       Handle<String> inferred_name(shared_info->inferred_name());
       result->set_inferred_name(inferred_name);
@@ -836,9 +836,8 @@ static FunctionLiteral::FunctionType ComputeFunctionType(ParseInfo* info) {
 }
 
 FunctionLiteral* Parser::DoParseFunction(ParseInfo* info,
-                                         const AstRawString* raw_name,
-                                         Utf16CharacterStream* source) {
-  scanner_.Initialize(source);
+                                         const AstRawString* raw_name) {
+  DCHECK_NOT_NULL(raw_name);
   DCHECK_NULL(scope_);
   DCHECK_NULL(target_stack_);
 
@@ -3534,6 +3533,7 @@ void Parser::ParseOnBackground(ParseInfo* info) {
                                     runtime_call_stats_));
     stream_ptr = stream.get();
   }
+  scanner_.Initialize(stream_ptr);
   DCHECK(info->maybe_outer_scope_info().is_null());
 
   DCHECK(original_scope_);
@@ -3546,10 +3546,14 @@ void Parser::ParseOnBackground(ParseInfo* info) {
   // scopes) and set their end position after we know the script length.
   if (info->is_toplevel()) {
     fni_ = new (zone()) FuncNameInferrer(ast_value_factory(), zone());
-    scanner_.Initialize(stream_ptr);
     result = DoParseProgram(info);
   } else {
-    result = DoParseFunction(info, info->function_name(), stream_ptr);
+    const AstRawString* function_name = info->function_name();
+    if (!function_name) {
+      // FIXME(wiktorg) solve fni in parse tasks
+      function_name = ast_value_factory()->empty_string();
+    }
+    result = DoParseFunction(info, function_name);
   }
 
   info->set_literal(result);

src/parsing/parser.h

@@ -275,8 +275,7 @@ class V8_EXPORT_PRIVATE Parser : public NON_EXPORTED_BASE(ParserBase<Parser>) {
 
   FunctionLiteral* ParseFunction(Isolate* isolate, ParseInfo* info);
   FunctionLiteral* DoParseFunction(ParseInfo* info,
-                                   const AstRawString* raw_name,
-                                   Utf16CharacterStream* source);
+                                   const AstRawString* raw_name);
 
   // Called by ParseProgram after setting up the scanner.
   FunctionLiteral* DoParseProgram(ParseInfo* info);

test/unittests/compiler-dispatcher/compiler-dispatcher-job-unittest.cc

@@ -72,17 +72,20 @@ class ScriptResource : public v8::String::ExternalOneByteStringResource {
   DISALLOW_COPY_AND_ASSIGN(ScriptResource);
 };
 
+Handle<String> CreateSource(Isolate* isolate,
+                            ExternalOneByteString::Resource* maybe_resource) {
+  if (maybe_resource) {
+    return isolate->factory()
+        ->NewExternalStringFromOneByte(maybe_resource)
+        .ToHandleChecked();
+  }
+  return isolate->factory()->NewStringFromAsciiChecked(test_script);
+}
+
 Handle<SharedFunctionInfo> CreateSharedFunctionInfo(
     Isolate* isolate, ExternalOneByteString::Resource* maybe_resource) {
   HandleScope scope(isolate);
-  Handle<String> source;
-  if (maybe_resource) {
-    source = isolate->factory()
-                 ->NewExternalStringFromOneByte(maybe_resource)
-                 .ToHandleChecked();
-  } else {
-    source = isolate->factory()->NewStringFromAsciiChecked(test_script);
-  }
+  Handle<String> source = CreateSource(isolate, maybe_resource);
   Handle<Script> script = isolate->factory()->NewScript(source);
   Handle<FixedArray> infos = isolate->factory()->NewFixedArray(3);
   script->set_shared_function_infos(*infos);
@@ -96,6 +99,17 @@ Handle<SharedFunctionInfo> CreateSharedFunctionInfo(
   return scope.CloseAndEscape(shared);
 }
 
+class FinishCallback : public CompilerDispatcherJob::FinishCallback {
+ public:
+  void ParseFinished(std::unique_ptr<ParseInfo> result) override {
+    result_ = std::move(result);
+  }
+  ParseInfo* result() const { return result_.get(); }
+
+ private:
+  std::unique_ptr<ParseInfo> result_;
+};
+
 }  // namespace
 
 TEST_F(CompilerDispatcherJobTest, Construct) {
@@ -104,6 +118,18 @@ TEST_F(CompilerDispatcherJobTest, Construct) {
       FLAG_stack_size));
 }
 
+TEST_F(CompilerDispatcherJobTest, ConstructWithoutSFI) {
+  std::unique_ptr<FinishCallback> callback(new FinishCallback());
+  std::unique_ptr<ScriptResource> resource(
+      new ScriptResource(test_script, strlen(test_script)));
+  std::unique_ptr<CompilerDispatcherJob> job(new CompilerDispatcherJob(
+      tracer(), FLAG_stack_size, CreateSource(i_isolate(), resource.get()), 0,
+      static_cast<int>(resource->length()), SLOPPY, 1, false, false, false,
+      false, i_isolate()->heap()->HashSeed(), i_isolate()->allocator(),
+      ScriptCompiler::kNoCompileOptions, i_isolate()->ast_string_constants(),
+      callback.get()));
+}
+
 TEST_F(CompilerDispatcherJobTest, StateTransitions) {
   std::unique_ptr<CompilerDispatcherJob> job(new CompilerDispatcherJob(
       i_isolate(), tracer(), CreateSharedFunctionInfo(i_isolate(), nullptr),
@@ -128,6 +154,24 @@ TEST_F(CompilerDispatcherJobTest, StateTransitions) {
   ASSERT_TRUE(job->status() == CompileJobStatus::kInitial);
 }
 
+TEST_F(CompilerDispatcherJobTest, StateTransitionsParseWithCallback) {
+  std::unique_ptr<FinishCallback> callback(new FinishCallback());
+  std::unique_ptr<ScriptResource> resource(
+      new ScriptResource(test_script, strlen(test_script)));
+  std::unique_ptr<CompilerDispatcherJob> job(new CompilerDispatcherJob(
+      tracer(), FLAG_stack_size, CreateSource(i_isolate(), resource.get()), 0,
+      static_cast<int>(resource->length()), SLOPPY, 1, false, false, false,
+      false, i_isolate()->heap()->HashSeed(), i_isolate()->allocator(),
+      ScriptCompiler::kNoCompileOptions, i_isolate()->ast_string_constants(),
+      callback.get()));
+  ASSERT_TRUE(job->status() == CompileJobStatus::kReadyToParse);
+  job->Parse();
+  ASSERT_TRUE(job->status() == CompileJobStatus::kDone);
+  job->ResetOnMainThread();
+  ASSERT_TRUE(job->status() == CompileJobStatus::kInitial);
+  ASSERT_TRUE(callback->result() != nullptr);
+}
+
 TEST_F(CompilerDispatcherJobTest, SyntaxError) {
   ScriptResource script("^^^", strlen("^^^"));
   std::unique_ptr<CompilerDispatcherJob> job(new CompilerDispatcherJob(