[parser]: Skipping inner funcs / initial implemetation of storing scope...

[parser]: Skipping inner funcs / initial implemetation of storing scope analysis data from preparsed scopes.

The data produced at the moment only contains information about scope type +
positions, and only the most trivial tests pass.

Upcoming CLs will extend the data to contain information about variables (once
PreParser can produce it) and add more test cases.

BUG=v8:5516

Review-Url: https://codereview.chromium.org/2650703003
Cr-Commit-Position: refs/heads/master@{#42656}

BUILD.gn

@@ -93,9 +93,7 @@ declare_args() {
   # TODO(brettw) http://crbug.com/684096 Remove the define condition when the
   # build_override conversion is complete.
   if (!defined(v8_extra_library_files)) {
-    v8_extra_library_files = [
-      "//test/cctest/test-extra.js"
-    ]
+    v8_extra_library_files = [ "//test/cctest/test-extra.js" ]
   }
 
   # Like v8_extra_library_files but for experimental features.
@@ -105,17 +103,16 @@ declare_args() {
   # TODO(brettw) http://crbug.com/684096 Remove the define condition when the
   # build_override conversion is complete.
   if (!defined(v8_experimental_extra_library_files)) {
-    v8_experimental_extra_library_files = [
-      "//test/cctest/test-experimental-extra.js",
-    ]
+    v8_experimental_extra_library_files =
+        [ "//test/cctest/test-experimental-extra.js" ]
   }
 
   if (defined(v8_enable_gdbjit_default)) {
     v8_enable_gdbjit = v8_enable_gdbjit_default
   } else {
     v8_enable_gdbjit = ((v8_current_cpu == "x86" || v8_current_cpu == "x64" ||
-        v8_current_cpu == "x87") && (is_linux || is_mac)) ||
-      (v8_current_cpu == "ppc64" && is_linux)
+                         v8_current_cpu == "x87") && (is_linux || is_mac)) ||
+                       (v8_current_cpu == "ppc64" && is_linux)
   }
 }
 
@@ -1620,6 +1617,8 @@ v8_source_set("v8_base") {
     "src/parsing/preparse-data-format.h",
     "src/parsing/preparse-data.cc",
     "src/parsing/preparse-data.h",
+    "src/parsing/preparsed-scope-data.cc",
+    "src/parsing/preparsed-scope-data.h",
     "src/parsing/preparser.cc",
     "src/parsing/preparser.h",
     "src/parsing/rewriter.cc",

src/ast/scopes.cc

@@ -15,6 +15,7 @@
 #include "src/objects/module-info.h"
 #include "src/objects/scope-info.h"
 #include "src/parsing/parse-info.h"
+#include "src/parsing/preparsed-scope-data.h"
 
 namespace v8 {
 namespace internal {
@@ -1377,7 +1378,9 @@ void DeclarationScope::ResetAfterPreparsing(AstValueFactory* ast_value_factory,
   was_lazily_parsed_ = !aborted;
 }
 
-void DeclarationScope::AnalyzePartially(AstNodeFactory* ast_node_factory) {
+void DeclarationScope::AnalyzePartially(
+    AstNodeFactory* ast_node_factory,
+    PreParsedScopeData* preparsed_scope_data) {
   DCHECK(!force_eager_compilation_);
   VariableProxy* unresolved = nullptr;
 
@@ -1398,6 +1401,13 @@ void DeclarationScope::AnalyzePartially(AstNodeFactory* ast_node_factory) {
         !(MustAllocate(arguments_) && !has_arguments_parameter_)) {
       arguments_ = nullptr;
     }
+
+    if (FLAG_preparser_scope_analysis) {
+      // Decide context allocation for the locals and parameters and store the
+      // info away.
+      AllocateVariablesRecursively();
+      CollectVariableData(preparsed_scope_data);
+    }
   }
 
   ResetAfterPreparsing(ast_node_factory->ast_value_factory(), false);
@@ -2151,6 +2161,16 @@ void Scope::AllocateDebuggerScopeInfos(Isolate* isolate,
   }
 }
 
+void Scope::CollectVariableData(PreParsedScopeData* data) {
+  PreParsedScopeData::ScopeScope scope_scope(data, scope_type(),
+                                             start_position(), end_position());
+  // TODO(marja): Add data about the variables.
+
+  for (Scope* inner = inner_scope_; inner != nullptr; inner = inner->sibling_) {
+    inner->CollectVariableData(data);
+  }
+}
+
 int Scope::StackLocalCount() const {
   Variable* function =
       is_function_scope() ? AsDeclarationScope()->function_var() : nullptr;

src/ast/scopes.h

@@ -19,6 +19,7 @@ class AstValueFactory;
 class AstRawString;
 class Declaration;
 class ParseInfo;
+class PreParsedScopeData;
 class SloppyBlockFunctionStatement;
 class Statement;
 class StringSet;
@@ -585,6 +586,8 @@ class V8_EXPORT_PRIVATE Scope : public NON_EXPORTED_BASE(ZoneObject) {
   void AllocateDebuggerScopeInfos(Isolate* isolate,
                                   MaybeHandle<ScopeInfo> outer_scope);
 
+  void CollectVariableData(PreParsedScopeData* data);
+
   // Construct a scope based on the scope info.
   Scope(Zone* zone, ScopeType type, Handle<ScopeInfo> scope_info);
 
@@ -788,7 +791,8 @@ class DeclarationScope : public Scope {
   // records variables which cannot be resolved inside the Scope (we don't yet
   // know what they will resolve to since the outer Scopes are incomplete) and
   // migrates them into migrate_to.
-  void AnalyzePartially(AstNodeFactory* ast_node_factory);
+  void AnalyzePartially(AstNodeFactory* ast_node_factory,
+                        PreParsedScopeData* preparsed_scope_data);
 
   Handle<StringSet> CollectNonLocals(ParseInfo* info,
                                      Handle<StringSet> non_locals);

src/flag-definitions.h

@@ -889,6 +889,9 @@ DEFINE_BOOL(lazy_inner_functions, true, "enable lazy parsing inner functions")
 DEFINE_BOOL(aggressive_lazy_inner_functions, false,
             "even lazier inner function parsing")
 DEFINE_IMPLICATION(aggressive_lazy_inner_functions, lazy_inner_functions)
+DEFINE_BOOL(preparser_scope_analysis, false,
+            "perform scope analysis for preparsed inner functions")
+DEFINE_IMPLICATION(preparser_scope_analysis, lazy_inner_functions)
 
 // simulator-arm.cc, simulator-arm64.cc and simulator-mips.cc
 DEFINE_BOOL(trace_sim, false, "Trace simulator execution")

src/parsing/parse-info.h

@@ -8,6 +8,7 @@
 #include "include/v8.h"
 #include "src/globals.h"
 #include "src/handles.h"
+#include "src/parsing/preparsed-scope-data.h"
 
 namespace v8 {
 
@@ -93,6 +94,8 @@ class V8_EXPORT_PRIVATE ParseInfo {
   ScriptData** cached_data() const { return cached_data_; }
   void set_cached_data(ScriptData** cached_data) { cached_data_ = cached_data; }
 
+  PreParsedScopeData* preparsed_scope_data() { return &preparsed_scope_data_; }
+
   ScriptCompiler::CompileOptions compile_options() const {
     return compile_options_;
   }
@@ -249,6 +252,7 @@ class V8_EXPORT_PRIVATE ParseInfo {
 
   //----------- Inputs+Outputs of parsing and scope analysis -----------------
   ScriptData** cached_data_;  // used if available, populated if requested.
+  PreParsedScopeData preparsed_scope_data_;
   AstValueFactory* ast_value_factory_;  // used if available, otherwise new.
   const AstRawString* function_name_;
 

src/parsing/parser.cc

@@ -519,7 +519,8 @@ Parser::Parser(ParseInfo* info)
       cached_parse_data_(nullptr),
       total_preparse_skipped_(0),
       temp_zoned_(false),
-      log_(nullptr) {
+      log_(nullptr),
+      preparsed_scope_data_(info->preparsed_scope_data()) {
   // Even though we were passed ParseInfo, we should not store it in
   // Parser - this makes sure that Isolate is not accidentally accessed via
   // ParseInfo during background parsing.
@@ -2685,7 +2686,8 @@ FunctionLiteral* Parser::ParseFunctionLiteral(
       // If the preconditions are correct the function body should never be
       // accessed, but do this anyway for better behaviour if they're wrong.
       body = nullptr;
-      scope->AnalyzePartially(&previous_zone_ast_node_factory);
+      scope->AnalyzePartially(&previous_zone_ast_node_factory,
+                              preparsed_scope_data_);
     }
 
     DCHECK_IMPLIES(use_temp_zone, temp_zoned_);

src/parsing/parser.h

@@ -27,6 +27,7 @@ class ParseInfo;
 class ScriptData;
 class ParserTarget;
 class ParserTargetScope;
+class PreParsedScopeData;
 
 class FunctionEntry BASE_EMBEDDED {
  public:
@@ -1136,6 +1137,8 @@ class V8_EXPORT_PRIVATE Parser : public NON_EXPORTED_BASE(ParserBase<Parser>) {
   bool allow_lazy_;
   bool temp_zoned_;
   ParserLogger* log_;
+
+  PreParsedScopeData* preparsed_scope_data_;
 };
 
 // ----------------------------------------------------------------------------

src/parsing/preparsed-scope-data.cc

+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "src/parsing/preparsed-scope-data.h"
+
+namespace v8 {
+namespace internal {
+
+PreParsedScopeData::ScopeScope::ScopeScope(PreParsedScopeData* data,
+                                           ScopeType scope_type,
+                                           int start_position, int end_position)
+    : data_(data),
+      previous_scope_(data->current_scope_),
+      inner_scope_count_(0),
+      variable_count_(0) {
+  data->current_scope_ = this;
+  if (previous_scope_ != nullptr) {
+    ++previous_scope_->inner_scope_count_;
+  }
+
+  data->backing_store_.push_back(scope_type);
+  data->backing_store_.push_back(start_position);
+  data->backing_store_.push_back(end_position);
+  // Reserve space for variable and inner scope count (we don't know yet how
+  // many will be added).
+  index_in_data_ = data->backing_store_.size();
+  data->backing_store_.push_back(-1);
+  data->backing_store_.push_back(-1);
+}
+
+PreParsedScopeData::ScopeScope::~ScopeScope() {
+  data_->current_scope_ = previous_scope_;
+  data_->backing_store_[index_in_data_] = inner_scope_count_;
+  data_->backing_store_[index_in_data_ + 1] = variable_count_;
+}
+
+}  // namespace internal
+}  // namespace v8

src/parsing/preparsed-scope-data.h

+// Copyright 2017 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef V8_PARSING_PREPARSED_SCOPE_DATA_H_
+#define V8_PARSING_PREPARSED_SCOPE_DATA_H_
+
+#include <vector>
+
+#include "src/globals.h"
+
+namespace v8 {
+namespace internal {
+
+class PreParsedScopeData {
+ public:
+  PreParsedScopeData() {}
+  ~PreParsedScopeData() {}
+
+  class ScopeScope {
+   public:
+    ScopeScope(PreParsedScopeData* data, ScopeType scope_type,
+               int start_position, int end_position);
+    ~ScopeScope();
+
+    // TODO(marja): The functions for adding information about local variables
+    // will appear here.
+
+   private:
+    PreParsedScopeData* data_;
+    size_t index_in_data_;
+    ScopeScope* previous_scope_;
+
+    int inner_scope_count_;
+    int variable_count_;
+    DISALLOW_COPY_AND_ASSIGN(ScopeScope);
+  };
+
+ private:
+  friend class ScopeTestHelper;
+
+  // TODO(marja): Make the backing store more efficient once we know exactly
+  // what data is needed.
+  std::vector<int> backing_store_;
+  ScopeScope* current_scope_ = nullptr;
+
+  DISALLOW_COPY_AND_ASSIGN(PreParsedScopeData);
+};
+
+}  // namespace internal
+}  // namespace v8
+
+#endif  // V8_PARSING_PREPARSED_SCOPE_DATA_H_

src/v8.gyp

@@ -1106,6 +1106,8 @@
         'parsing/preparse-data-format.h',
         'parsing/preparse-data.cc',
         'parsing/preparse-data.h',
+        'parsing/preparsed-scope-data.cc',
+        'parsing/preparsed-scope-data.h',
         'parsing/preparser.cc',
         'parsing/preparser.h',
         'parsing/rewriter.cc',

test/cctest/test-parsing.cc

@@ -8697,6 +8697,28 @@ class ScopeTestHelper {
   static bool MustAllocateInContext(Variable* var) {
     return var->scope()->MustAllocateInContext(var);
   }
+
+  static void CompareScopeToData(Scope* scope, const PreParsedScopeData* data,
+                                 size_t& index) {
+    CHECK_EQ(data->backing_store_[index++], scope->scope_type());
+    CHECK_EQ(data->backing_store_[index++], scope->start_position());
+    CHECK_EQ(data->backing_store_[index++], scope->end_position());
+
+    int inner_scope_count = 0;
+    for (Scope* inner = scope->inner_scope(); inner != nullptr;
+         inner = inner->sibling()) {
+      ++inner_scope_count;
+    }
+    CHECK_EQ(data->backing_store_[index++], inner_scope_count);
+
+    // Variable count is 0. TODO(marja): implement.
+    CHECK_EQ(data->backing_store_[index++], 0);
+
+    for (Scope* inner = scope->inner_scope(); inner != nullptr;
+         inner = inner->sibling()) {
+      CompareScopeToData(inner, data, index);
+    }
+  }
 };
 }  // namespace internal
 }  // namespace v8
@@ -8975,3 +8997,97 @@ TEST(NoPessimisticContextAllocation) {
     }
   }
 }
+
+TEST(PreParserScopeAnalysis) {
+  i::FLAG_lazy_inner_functions = true;
+  i::FLAG_preparser_scope_analysis = true;
+  i::Isolate* isolate = CcTest::i_isolate();
+  i::Factory* factory = isolate->factory();
+  i::HandleScope scope(isolate);
+  LocalContext env;
+
+  const char* prefix = "(function outer() { ";
+  const char* suffix = " })();";
+  int prefix_len = Utf8LengthHelper(prefix);
+  int suffix_len = Utf8LengthHelper(suffix);
+
+  // The scope start positions must match; note the extra space in lazy_inner.
+  const char* lazy_inner = " function inner(%s) { %s }";
+  const char* eager_inner = "(function inner(%s) { %s })()";
+
+  struct {
+    const char* params;
+    const char* source;
+  } inners[] = {
+      {"", "var1"},
+      {"", "if (true) {}"},
+      {"", "function f1() {}"},
+      {"", "if (true) { function f1() {} }"},
+  };
+
+  for (unsigned i = 0; i < arraysize(inners); ++i) {
+    // First compile with the lazy inner function and extract the scope data.
+    const char* inner_function = lazy_inner;
+    int inner_function_len = Utf8LengthHelper(inner_function) - 4;
+
+    int params_len = Utf8LengthHelper(inners[i].params);
+    int source_len = Utf8LengthHelper(inners[i].source);
+    int len =
+        prefix_len + inner_function_len + params_len + source_len + suffix_len;
+
+    i::ScopedVector<char> lazy_program(len + 1);
+    i::SNPrintF(lazy_program, "%s", prefix);
+    i::SNPrintF(lazy_program + prefix_len, inner_function, inners[i].params,
+                inners[i].source);
+    i::SNPrintF(lazy_program + prefix_len + inner_function_len + params_len +
+                    source_len,
+                "%s", suffix);
+
+    i::Handle<i::String> source =
+        factory->InternalizeUtf8String(lazy_program.start());
+    source->PrintOn(stdout);
+    printf("\n");
+
+    i::Handle<i::Script> script = factory->NewScript(source);
+    i::Zone zone(CcTest::i_isolate()->allocator(), ZONE_NAME);
+    i::ParseInfo lazy_info(&zone, script);
+
+    // No need to run scope analysis; preparser scope data is produced when
+    // parsing.
+    CHECK(i::parsing::ParseProgram(&lazy_info));
+
+    // Then parse eagerly and check against the scope data.
+    inner_function = eager_inner;
+    inner_function_len = Utf8LengthHelper(inner_function) - 4;
+    len =
+        prefix_len + inner_function_len + params_len + source_len + suffix_len;
+
+    i::ScopedVector<char> eager_program(len + 1);
+    i::SNPrintF(eager_program, "%s", prefix);
+    i::SNPrintF(eager_program + prefix_len, inner_function, inners[i].params,
+                inners[i].source);
+    i::SNPrintF(eager_program + prefix_len + inner_function_len + params_len +
+                    source_len,
+                "%s", suffix);
+
+    source = factory->InternalizeUtf8String(eager_program.start());
+    source->PrintOn(stdout);
+    printf("\n");
+
+    script = factory->NewScript(source);
+    i::ParseInfo eager_info(&zone, script);
+
+    CHECK(i::parsing::ParseProgram(&eager_info));
+    CHECK(i::Compiler::Analyze(&eager_info));
+
+    i::Scope* scope =
+        eager_info.literal()->scope()->inner_scope()->inner_scope();
+    DCHECK_NOT_NULL(scope);
+    DCHECK_NULL(scope->sibling());
+    DCHECK(scope->is_function_scope());
+
+    size_t index = 0;
+    i::ScopeTestHelper::CompareScopeToData(
+        scope, lazy_info.preparsed_scope_data(), index);
+  }
+}