[parser] Inital parallel parse tasks implementation.

While parsing top-level code eager functions are skipped just like lazy
ones, but also a parse task is created for each.

The parse tasks are run by the compiler dispatcher and can be executed
either on background thread or in idle time.
After parsing of top-level code finishes it waits for all unfinished
parser tasks - possibly picking up and executing them on current thread.
Afterwards parse task results are stitched together with top-level AST,
in case of failures eager functions are treated just like lazy -
parsing/compilation is retriggered for them in the runtime and proper
errors are generated (performance is not optimized for error case at
all).

BUG=v8:6093

Change-Id: Ie6508211a04b90becfe44139cce1c8ecec386b6e
Reviewed-on: https://chromium-review.googlesource.com/486725Reviewed-by: Jochen Eisinger <jochen@chromium.org>
Reviewed-by: Marja Hölttä <marja@chromium.org>
Reviewed-by: Daniel Vogelheim <vogelheim@chromium.org>
Commit-Queue: Wiktor Garbacz <wiktorg@google.com>
Cr-Commit-Position: refs/heads/master@{#45016}

src/ast/ast.cc

@@ -350,6 +350,23 @@ bool FunctionLiteral::NeedsHomeObject(Expression* expr) {
   return expr->AsFunctionLiteral()->scope()->NeedsHomeObject();
 }
 
+void FunctionLiteral::ReplaceBodyAndScope(FunctionLiteral* other) {
+  DCHECK_NULL(body_);
+  DCHECK_NOT_NULL(scope_);
+  DCHECK_NOT_NULL(other->scope());
+
+  Scope* outer_scope = scope_->outer_scope();
+
+  body_ = other->body();
+  scope_ = other->scope();
+  scope_->ReplaceOuterScope(outer_scope);
+#ifdef DEBUG
+  scope_->set_replaced_from_parse_task(true);
+#endif
+
+  function_length_ = other->function_length_;
+}
+
 ObjectLiteralProperty::ObjectLiteralProperty(Expression* key, Expression* value,
                                              Kind kind, bool is_computed_name)
     : LiteralProperty(key, value, is_computed_name),

src/ast/ast.h

@@ -2765,6 +2765,8 @@ class FunctionLiteral final : public Expression {
     function_literal_id_ = function_literal_id;
   }
 
+  void ReplaceBodyAndScope(FunctionLiteral* other);
+
  private:
   friend class AstNodeFactory;
 

src/ast/scopes.cc

@@ -1799,7 +1799,9 @@ void Scope::CheckZones() {
       DCHECK_NULL(scope->inner_scope_);
       continue;
     }
-    CHECK_EQ(scope->zone(), zone());
+    if (!scope->replaced_from_parse_task()) {
+      CHECK_EQ(scope->zone(), zone());
+    }
     scope->CheckZones();
   }
 }

src/ast/scopes.h

@@ -456,6 +456,11 @@ class V8_EXPORT_PRIVATE Scope : public NON_EXPORTED_BASE(ZoneObject) {
 
   // Check that all Scopes in the scope tree use the same Zone.
   void CheckZones();
+
+  bool replaced_from_parse_task() const { return replaced_from_parse_task_; }
+  void set_replaced_from_parse_task(bool replaced_from_parse_task) {
+    replaced_from_parse_task_ = replaced_from_parse_task;
+  }
 #endif
 
   // Retrieve `IsSimpleParameterList` of current or outer function.
@@ -535,6 +540,10 @@ class V8_EXPORT_PRIVATE Scope : public NON_EXPORTED_BASE(ZoneObject) {
   // True if this scope may contain objects from a temp zone that needs to be
   // fixed up.
   bool needs_migration_;
+
+  // True if scope comes from other zone - as a result of being created in a
+  // parse tasks.
+  bool replaced_from_parse_task_ = false;
 #endif
 
   // Source positions.

src/compiler-dispatcher/compiler-dispatcher-job.h

@@ -43,7 +43,7 @@ enum class CompileJobStatus {
   kDone,
 };
 
-class CompileJobFinishCallback {
+class V8_EXPORT_PRIVATE CompileJobFinishCallback {
  public:
   virtual ~CompileJobFinishCallback() {}
   virtual void ParseFinished(std::unique_ptr<ParseInfo> parse_info) = 0;

src/compiler-dispatcher/compiler-dispatcher.cc

@@ -248,8 +248,6 @@ CompilerDispatcher::~CompilerDispatcher() {
 bool CompilerDispatcher::CanEnqueue() {
   if (!IsEnabled()) return false;
 
-  DCHECK(FLAG_ignition);
-
   if (memory_pressure_level_.Value() != MemoryPressureLevel::kNone) {
     return false;
   }
@@ -263,6 +261,8 @@ bool CompilerDispatcher::CanEnqueue() {
 }
 
 bool CompilerDispatcher::CanEnqueue(Handle<SharedFunctionInfo> function) {
+  DCHECK_IMPLIES(IsEnabled(), FLAG_ignition);
+
   if (!CanEnqueue()) return false;
 
   // We only handle functions (no eval / top-level code / wasm) that are
@@ -598,7 +598,7 @@ CompilerDispatcher::JobMap::const_iterator CompilerDispatcher::GetJobFor(
 
 void CompilerDispatcher::ScheduleIdleTaskFromAnyThread() {
   v8::Isolate* v8_isolate = reinterpret_cast<v8::Isolate*>(isolate_);
-  DCHECK(platform_->IdleTasksEnabled(v8_isolate));
+  if (!platform_->IdleTasksEnabled(v8_isolate)) return;
   {
     base::LockGuard<base::Mutex> lock(&mutex_);
     if (idle_task_scheduled_) return;

src/parsing/parse-info.cc

@@ -178,6 +178,24 @@ void ParseInfo::UpdateStatisticsAfterBackgroundParse(Isolate* isolate) {
   set_runtime_call_stats(main_call_stats);
 }
 
+void ParseInfo::ParseFinished(std::unique_ptr<ParseInfo> info) {
+  if (info->literal()) {
+    base::LockGuard<base::Mutex> access_child_infos(&child_infos_mutex_);
+    child_infos_.emplace_back(std::move(info));
+  }
+}
+
+std::map<int, ParseInfo*> ParseInfo::child_infos() const {
+  base::LockGuard<base::Mutex> access_child_infos(&child_infos_mutex_);
+  std::map<int, ParseInfo*> rv;
+  for (const auto& child_info : child_infos_) {
+    DCHECK_NOT_NULL(child_info->literal());
+    int start_position = child_info->literal()->start_position();
+    rv.insert(std::make_pair(start_position, child_info.get()));
+  }
+  return rv;
+}
+
 #ifdef DEBUG
 bool ParseInfo::script_is_native() const {
   return script_->type() == Script::TYPE_NATIVE;

src/parsing/parse-info.h

@@ -5,9 +5,12 @@
 #ifndef V8_PARSING_PARSE_INFO_H_
 #define V8_PARSING_PARSE_INFO_H_
 
+#include <map>
 #include <memory>
+#include <vector>
 
 #include "include/v8.h"
+#include "src/compiler-dispatcher/compiler-dispatcher-job.h"
 #include "src/globals.h"
 #include "src/handles.h"
 #include "src/parsing/preparsed-scope-data.h"
@@ -33,7 +36,7 @@ class Utf16CharacterStream;
 class Zone;
 
 // A container for the inputs, configuration options, and outputs of parsing.
-class V8_EXPORT_PRIVATE ParseInfo {
+class V8_EXPORT_PRIVATE ParseInfo : public CompileJobFinishCallback {
  public:
   explicit ParseInfo(AccountingAllocator* zone_allocator);
   ParseInfo(Handle<Script> script);
@@ -245,6 +248,11 @@ class V8_EXPORT_PRIVATE ParseInfo {
 
   void UpdateStatisticsAfterBackgroundParse(Isolate* isolate);
 
+  // The key of the map is the FunctionLiteral's start_position
+  std::map<int, ParseInfo*> child_infos() const;
+
+  void ParseFinished(std::unique_ptr<ParseInfo> info) override;
+
 #ifdef DEBUG
   bool script_is_native() const;
 #endif  // DEBUG
@@ -305,6 +313,9 @@ class V8_EXPORT_PRIVATE ParseInfo {
   FunctionLiteral* literal_;
   std::shared_ptr<DeferredHandles> deferred_handles_;
 
+  std::vector<std::unique_ptr<ParseInfo>> child_infos_;
+  mutable base::Mutex child_infos_mutex_;
+
   void SetFlag(Flag f) { flags_ |= f; }
   void SetFlag(Flag f, bool v) { flags_ = v ? flags_ | f : flags_ & ~f; }
   bool GetFlag(Flag f) const { return (flags_ & f) != 0; }

src/parsing/parser.h

@@ -273,6 +273,8 @@ class V8_EXPORT_PRIVATE Parser : public NON_EXPORTED_BASE(ParserBase<Parser>) {
 
   void SetCachedData(ParseInfo* info);
 
+  void StitchAst(ParseInfo* top_level_parse_info, Isolate* isolate);
+
   ScriptCompiler::CompileOptions compile_options() const {
     return compile_options_;
   }
@@ -1152,6 +1154,11 @@ class V8_EXPORT_PRIVATE Parser : public NON_EXPORTED_BASE(ParserBase<Parser>) {
   PreParser* reusable_preparser_;
   Mode mode_;
 
+  std::vector<FunctionLiteral*> literals_to_stitch_;
+  Handle<String> source_;
+  CompilerDispatcher* compiler_dispatcher_ = nullptr;
+  ParseInfo* main_parse_info_ = nullptr;
+
   friend class ParserTarget;
   friend class ParserTargetScope;
   ParserTarget* target_stack_;  // for break, continue statements

src/parsing/preparser.cc

@@ -101,10 +101,8 @@ PreParserIdentifier PreParser::GetSymbol() const {
   return symbol;
 }
 
-PreParser::PreParseResult PreParser::PreParseProgram(bool is_module,
-                                                     int* use_counts) {
+PreParser::PreParseResult PreParser::PreParseProgram(bool is_module) {
   DCHECK_NULL(scope_);
-  use_counts_ = use_counts;
   DeclarationScope* scope = NewScriptScope();
 #ifdef DEBUG
   scope->set_is_being_lazily_parsed(true);
@@ -123,7 +121,6 @@ PreParser::PreParseResult PreParser::PreParseProgram(bool is_module,
   PreParserStatementList body;
   ParseStatementList(body, Token::EOS, &ok);
   original_scope_ = nullptr;
-  use_counts_ = nullptr;
   if (stack_overflow()) return kPreParseStackOverflow;
   if (!ok) {
     ReportUnexpectedToken(scanner()->current_token());
@@ -289,9 +286,6 @@ PreParser::Expression PreParser::ParseFunctionLiteral(
       runtime_call_stats_,
       counters[track_unresolved_variables_][parsing_on_main_thread_]);
 
-  bool is_top_level =
-      scope()->AllowsLazyParsingWithoutUnresolvedVariables(original_scope_);
-
   DeclarationScope* function_scope = NewFunctionScope(kind);
   function_scope->SetLanguageMode(language_mode);
   FunctionState function_state(&function_state_, &scope_, function_scope);
@@ -339,22 +333,6 @@ PreParser::Expression PreParser::ParseFunctionLiteral(
     CheckStrictOctalLiteral(start_position, end_position, CHECK_OK);
   }
 
-  if (FLAG_use_parse_tasks && is_top_level && preparse_data_) {
-    preparse_data_->AddFunctionData(
-        start_position, PreParseData::FunctionData(
-                            end_position, formals.num_parameters(),
-                            GetLastFunctionLiteralId() - func_id, language_mode,
-                            function_scope->uses_super_property()));
-    // TODO(wiktorg) spin-off a parse task
-    if (FLAG_trace_parse_tasks) {
-      PrintF("Saved function at %d to %d with:\n", start_position,
-             end_position);
-      PrintF("\t- %d params\n", formals.num_parameters());
-      PrintF("\t- %d function length\n", formals.function_length);
-      PrintF("\t- %d inner-funcs\n", GetLastFunctionLiteralId() - func_id);
-    }
-  }
-
   if (FLAG_experimental_preparser_scope_analysis &&
       track_unresolved_variables_ && preparsed_scope_data_ != nullptr) {
     preparsed_scope_data_->AddSkippableFunction(

src/parsing/preparser.h

@@ -901,7 +901,6 @@ class PreParser : public ParserBase<PreParser> {
                               ast_value_factory, runtime_call_stats,
                               preparsed_scope_data, parsing_on_main_thread),
         use_counts_(nullptr),
-        preparse_data_(FLAG_use_parse_tasks ? new PreParseData() : nullptr),
         track_unresolved_variables_(false),
         pending_error_handler_(pending_error_handler) {}
 
@@ -913,8 +912,7 @@ class PreParser : public ParserBase<PreParser> {
   // success (even if parsing failed, the pre-parse data successfully
   // captured the syntax error), and false if a stack-overflow happened
   // during parsing.
-  PreParseResult PreParseProgram(bool is_module = false,
-                                 int* use_counts = nullptr);
+  PreParseResult PreParseProgram(bool is_module = false);
 
   // Parses a single function literal, from the opening parentheses before
   // parameters to the closing brace after the body.
@@ -930,8 +928,6 @@ class PreParser : public ParserBase<PreParser> {
                                   bool track_unresolved_variables,
                                   bool may_abort, int* use_counts);
 
-  const PreParseData* preparse_data() const { return preparse_data_.get(); }
-
  private:
   // These types form an algebra over syntactic categories that is just
   // rich enough to let us recognize and propagate the constructs that
@@ -1669,7 +1665,6 @@ class PreParser : public ParserBase<PreParser> {
   // Preparser's private field members.
 
   int* use_counts_;
-  std::unique_ptr<PreParseData> preparse_data_;
   bool track_unresolved_variables_;
   PreParserLogger log_;
   PendingCompilationErrorHandler* pending_error_handler_;

test/mjsunit/parse-tasks.js

+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --compiler-dispatcher --use-parse-tasks --use-external-strings
+
+(function(a) {
+ assertEquals(a, "IIFE");
+})("IIFE");
+
+(function(a, ...rest) {
+ assertEquals(a, 1);
+ assertEquals(rest.length, 2);
+ assertEquals(rest[0], 2);
+ assertEquals(rest[1], 3);
+})(1,2,3);
+
+var outer_var = 42;
+
+function lazy_outer() {
+ return 42;
+}
+
+var eager_outer = (function() { return 42; });
+
+(function() {
+ assertEquals(outer_var, 42);
+ assertEquals(lazy_outer(), 42);
+ assertEquals(eager_outer(), 42);
+})();
+
+var gen = (function*() {
+ yield 1;
+ yield 2;
+})();
+
+assertEquals(gen.next().value, 1);
+assertEquals(gen.next().value, 2);
+
+var result = (function recursive(a=0) {
+ if (a == 1) {
+  return 42;
+ }
+ return recursive(1);
+})();
+
+assertEquals(result, 42);