Don't preparse large files to find boundaries of lazy functions.

Instead use the preparser inline to parse only the lazy function
bodies.

This is still disabled for small files.
More measurements are needed to determine if lazy-compiling small
sources is worth it.

Review URL: http://codereview.chromium.org/8662037

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@10066 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/compiler.cc

@@ -377,8 +377,14 @@ static Handle<SharedFunctionInfo> MakeFunctionInfo(CompilationInfo* info) {
 
   // Only allow non-global compiles for eval.
   ASSERT(info->is_eval() || info->is_global());
-
-  if (!ParserApi::Parse(info)) return Handle<SharedFunctionInfo>::null();
+  ParsingFlags flags = kNoParsingFlags;
+  if (info->pre_parse_data() != NULL ||
+      String::cast(script->source())->length() > FLAG_min_preparse_length) {
+    flags = kAllowLazy;
+  }
+  if (!ParserApi::Parse(info, flags)) {
+    return Handle<SharedFunctionInfo>::null();
+  }
 
   // Measure how long it takes to do the compilation; only take the
   // rest of the function into account to avoid overlap with the
@@ -453,7 +459,7 @@ Handle<SharedFunctionInfo> Compiler::Compile(Handle<String> source,
                                              int line_offset,
                                              int column_offset,
                                              v8::Extension* extension,
-                                             ScriptDataImpl* input_pre_data,
+                                             ScriptDataImpl* pre_data,
                                              Handle<Object> script_data,
                                              NativesFlag natives) {
   Isolate* isolate = source->GetIsolate();
@@ -484,24 +490,12 @@ Handle<SharedFunctionInfo> Compiler::Compile(Handle<String> source,
     // for small sources, odds are that there aren't many functions
     // that would be compiled lazily anyway, so we skip the preparse step
     // in that case too.
-    ScriptDataImpl* pre_data = input_pre_data;
     int flags = kNoParsingFlags;
     if ((natives == NATIVES_CODE) || FLAG_allow_natives_syntax) {
       flags |= kAllowNativesSyntax;
     }
     if (natives != NATIVES_CODE && FLAG_harmony_scoping) {
-      flags |= kHarmonyScoping;
-    }
-    if (pre_data == NULL
-        && source_length >= FLAG_min_preparse_length) {
-      if (source->IsExternalTwoByteString()) {
-        ExternalTwoByteStringUC16CharacterStream stream(
-            Handle<ExternalTwoByteString>::cast(source), 0, source->length());
-        pre_data = ParserApi::PartialPreParse(&stream, extension, flags);
-      } else {
-        GenericStringUC16CharacterStream stream(source, 0, source->length());
-        pre_data = ParserApi::PartialPreParse(&stream, extension, flags);
-      }
+      flags |= EXTENDED_MODE;
     }
 
     // Create a script object describing the script to be compiled.
@@ -527,11 +521,6 @@ Handle<SharedFunctionInfo> Compiler::Compile(Handle<String> source,
     if (extension == NULL && !result.is_null()) {
       compilation_cache->PutScript(source, result);
     }
-
-    // Get rid of the pre-parsing data (if necessary).
-    if (input_pre_data == NULL && pre_data != NULL) {
-      delete pre_data;
-    }
   }
 
   if (result.is_null()) isolate->ReportPendingMessages();
@@ -604,7 +593,7 @@ bool Compiler::CompileLazy(CompilationInfo* info) {
   isolate->counters()->total_compile_size()->Increment(compiled_size);
 
   // Generate the AST for the lazily compiled function.
-  if (ParserApi::Parse(info)) {
+  if (ParserApi::Parse(info, kNoParsingFlags)) {
     // Measure how long it takes to do the lazy compilation; only take the
     // rest of the function into account to avoid overlap with the lazy
     // parsing statistics.

src/hydrogen.cc

@@ -4763,7 +4763,7 @@ bool HGraphBuilder::TryInline(Call* expr, bool drop_extra) {
 
   // Parse and allocate variables.
   CompilationInfo target_info(target);
-  if (!ParserApi::Parse(&target_info) ||
+  if (!ParserApi::Parse(&target_info, kNoParsingFlags) ||
       !Scope::Analyze(&target_info)) {
     if (target_info.isolate()->has_pending_exception()) {
       // Parse or scope error, never optimize this function.

src/liveedit.cc

@@ -602,7 +602,8 @@ static void CompileScriptForTracker(Isolate* isolate, Handle<Script> script) {
   // Build AST.
   CompilationInfo info(script);
   info.MarkAsGlobal();
-  if (ParserApi::Parse(&info)) {
+  // Parse and don't allow skipping lazy functions.
+  if (ParserApi::Parse(&info, kNoParsingFlags)) {
     // Compile the code.
     LiveEditFunctionTracker tracker(info.isolate(), info.function());
     if (Compiler::MakeCodeForLiveEdit(&info)) {

src/parser.h

@@ -76,8 +76,10 @@ class FunctionEntry BASE_EMBEDDED {
     kSize
   };
 
-  explicit FunctionEntry(Vector<unsigned> backing) : backing_(backing) { }
-  FunctionEntry() { }
+  explicit FunctionEntry(Vector<unsigned> backing)
+    : backing_(backing) { }
+
+  FunctionEntry() : backing_() { }
 
   int start_pos() { return backing_[kStartPositionIndex]; }
   int end_pos() { return backing_[kEndPositionIndex]; }
@@ -94,6 +96,7 @@ class FunctionEntry BASE_EMBEDDED {
 
  private:
   Vector<unsigned> backing_;
+  bool owns_data_;
 };
 
 
@@ -166,7 +169,7 @@ class ParserApi {
   // Parses the source code represented by the compilation info and sets its
   // function literal.  Returns false (and deallocates any allocated AST
   // nodes) if parsing failed.
-  static bool Parse(CompilationInfo* info);
+  static bool Parse(CompilationInfo* info, int flags);
 
   // Generic preparser generating full preparse data.
   static ScriptDataImpl* PreParse(UC16CharacterStream* source,
@@ -422,13 +425,20 @@ class RegExpParser {
 // ----------------------------------------------------------------------------
 // JAVASCRIPT PARSING
 
+// Forward declaration.
+class SingletonLogger;
+
 class Parser {
  public:
   Parser(Handle<Script> script,
-         bool allow_natives_syntax,
+         int parsing_flags,  // Combination of ParsingFlags
          v8::Extension* extension,
          ScriptDataImpl* pre_data);
-  virtual ~Parser() { }
+  virtual ~Parser() {
+    if (reusable_preparser_ != NULL) {
+      delete reusable_preparser_;
+    }
+  }
 
   // Returns NULL if parsing failed.
   FunctionLiteral* ParseProgram(CompilationInfo* info);
@@ -728,12 +738,15 @@ class Parser {
                             Handle<String> type,
                             Vector< Handle<Object> > arguments);
 
+  preparser::PreParser::PreParseResult LazyParseFunctionLiteral(
+       SingletonLogger* logger);
+
   Isolate* isolate_;
   ZoneList<Handle<String> > symbol_cache_;
 
   Handle<Script> script_;
   Scanner scanner_;
-
+  preparser::PreParser* reusable_preparser_;
   Scope* top_scope_;
   FunctionState* current_function_state_;
   Target* target_stack_;  // for break, continue statements
@@ -743,6 +756,7 @@ class Parser {
 
   Mode mode_;
   bool allow_natives_syntax_;
+  bool allow_lazy_;
   bool stack_overflow_;
   // If true, the next (and immediately following) function literal is
   // preceded by a parenthesis.

src/preparser.cc

@@ -52,6 +52,34 @@ int isfinite(double value);
 
 namespace preparser {
 
+PreParser::PreParseResult PreParser::PreParseLazyFunction(
+    i::LanguageMode mode, i::ParserRecorder* log) {
+  log_ = log;
+  // Lazy functions always have trivial outer scopes (no with/catch scopes).
+  Scope top_scope(&scope_, kTopLevelScope);
+  set_language_mode(mode);
+  Scope function_scope(&scope_, kFunctionScope);
+  ASSERT_EQ(i::Token::LBRACE, scanner_->current_token());
+  bool ok = true;
+  int start_position = scanner_->peek_location().beg_pos;
+  ParseLazyFunctionLiteralBody(&ok);
+  if (stack_overflow_) return kPreParseStackOverflow;
+  if (!ok) {
+    ReportUnexpectedToken(scanner_->current_token());
+  } else {
+    ASSERT_EQ(i::Token::RBRACE, scanner_->peek());
+    if (!is_classic_mode()) {
+      int end_pos = scanner_->location().end_pos;
+      CheckOctalLiteral(start_position, end_pos, &ok);
+      if (ok) {
+        CheckDelayedStrictModeViolation(start_position, end_pos, &ok);
+      }
+    }
+  }
+  return kPreParseSuccess;
+}
+
+
 // Preparsing checks a JavaScript program and emits preparse-data that helps
 // a later parsing to be faster.
 // See preparser-data.h for the data.
@@ -1350,9 +1378,6 @@ PreParser::Expression PreParser::ParseFunctionLiteral(bool* ok) {
   }
   Expect(i::Token::RPAREN, CHECK_OK);
 
-  Expect(i::Token::LBRACE, CHECK_OK);
-  int function_block_pos = scanner_->location().beg_pos;
-
   // Determine if the function will be lazily compiled.
   // Currently only happens to top-level functions.
   // Optimistically assume that all top-level functions are lazily compiled.
@@ -1361,24 +1386,13 @@ PreParser::Expression PreParser::ParseFunctionLiteral(bool* ok) {
                              !parenthesized_function_);
   parenthesized_function_ = false;
 
+  Expect(i::Token::LBRACE, CHECK_OK);
   if (is_lazily_compiled) {
-    log_->PauseRecording();
-    ParseSourceElements(i::Token::RBRACE, ok);
-    log_->ResumeRecording();
-    if (!*ok) Expression::Default();
-
-    Expect(i::Token::RBRACE, CHECK_OK);
-
-    // Position right after terminal '}'.
-    int end_pos = scanner_->location().end_pos;
-    log_->LogFunction(function_block_pos, end_pos,
-                      function_scope.materialized_literal_count(),
-                      function_scope.expected_properties(),
-                      language_mode());
+    ParseLazyFunctionLiteralBody(CHECK_OK);
   } else {
-    ParseSourceElements(i::Token::RBRACE, CHECK_OK);
-    Expect(i::Token::RBRACE, CHECK_OK);
+    ParseSourceElements(i::Token::RBRACE, ok);
   }
+  Expect(i::Token::RBRACE, CHECK_OK);
 
   if (!is_classic_mode()) {
     int end_position = scanner_->location().end_pos;
@@ -1391,6 +1405,23 @@ PreParser::Expression PreParser::ParseFunctionLiteral(bool* ok) {
 }
 
 
+void PreParser::ParseLazyFunctionLiteralBody(bool* ok) {
+  int body_start = scanner_->location().beg_pos;
+  log_->PauseRecording();
+  ParseSourceElements(i::Token::RBRACE, ok);
+  log_->ResumeRecording();
+  if (!*ok) return;
+
+  // Position right after terminal '}'.
+  ASSERT_EQ(i::Token::RBRACE, scanner_->peek());
+  int body_end = scanner_->peek_location().end_pos;
+  log_->LogFunction(body_start, body_end,
+                    scope_->materialized_literal_count(),
+                    scope_->expected_properties(),
+                    language_mode());
+}
+
+
 PreParser::Expression PreParser::ParseV8Intrinsic(bool* ok) {
   // CallRuntime ::
   //   '%' Identifier Arguments

src/preparser.h

@@ -110,6 +110,24 @@ class PreParser {
     kPreParseSuccess
   };
 
+
+  PreParser(i::Scanner* scanner,
+            i::ParserRecorder* log,
+            uintptr_t stack_limit,
+            bool allow_lazy,
+            bool allow_natives_syntax)
+      : scanner_(scanner),
+        log_(log),
+        scope_(NULL),
+        stack_limit_(stack_limit),
+        strict_mode_violation_location_(i::Scanner::Location::invalid()),
+        strict_mode_violation_type_(NULL),
+        stack_overflow_(false),
+        allow_lazy_(allow_lazy),
+        allow_natives_syntax_(allow_natives_syntax),
+        parenthesized_function_(false),
+        harmony_scoping_(scanner->HarmonyScoping()) { }
+
   ~PreParser() {}
 
   // Pre-parse the program from the character stream; returns true on
@@ -126,6 +144,17 @@ class PreParser {
                      allow_lazy, allow_natives_syntax).PreParse();
   }
 
+  // Parses a single function literal, from the opening parentheses before
+  // parameters to the closing brace after the body.
+  // Returns a FunctionEntry describing the body of the funciton in enough
+  // detail that it can be lazily compiled.
+  // The scanner is expected to have matched the "function" keyword and
+  // parameters, and have consumed the initial '{'.
+  // At return, unless an error occured, the scanner is positioned before the
+  // the final '}'.
+  PreParseResult PreParseLazyFunction(i::LanguageMode mode,
+                                      i::ParserRecorder* log);
+
  private:
   // Used to detect duplicates in object literals. Each of the values
   // kGetterProperty, kSetterProperty and kValueProperty represents
@@ -450,24 +479,6 @@ class PreParser {
     i::LanguageMode language_mode_;
   };
 
-  // Private constructor only used in PreParseProgram.
-  PreParser(i::Scanner* scanner,
-            i::ParserRecorder* log,
-            uintptr_t stack_limit,
-            bool allow_lazy,
-            bool allow_natives_syntax)
-      : scanner_(scanner),
-        log_(log),
-        scope_(NULL),
-        stack_limit_(stack_limit),
-        strict_mode_violation_location_(i::Scanner::Location::invalid()),
-        strict_mode_violation_type_(NULL),
-        stack_overflow_(false),
-        allow_lazy_(allow_lazy),
-        allow_natives_syntax_(allow_natives_syntax),
-        parenthesized_function_(false),
-        harmony_scoping_(scanner->HarmonyScoping()) { }
-
   // Preparse the program. Only called in PreParseProgram after creating
   // the instance.
   PreParseResult PreParse() {
@@ -547,6 +558,7 @@ class PreParser {
 
   Arguments ParseArguments(bool* ok);
   Expression ParseFunctionLiteral(bool* ok);
+  void ParseLazyFunctionLiteralBody(bool* ok);
 
   Identifier ParseIdentifier(bool* ok);
   Identifier ParseIdentifierName(bool* ok);

src/scanner.h

@@ -42,15 +42,23 @@ namespace v8 {
 namespace internal {
 
 
-// General collection of bit-flags that can be passed to scanners and
+// General collection of (multi-)bit-flags that can be passed to scanners and
 // parsers to signify their (initial) mode of operation.
 enum ParsingFlags {
   kNoParsingFlags = 0,
-  kAllowLazy = 1,
-  kAllowNativesSyntax = 2,
-  kHarmonyScoping = 4
+  // Embed LanguageMode values in parsing flags, i.e., equivalent to:
+  // CLASSIC_MODE = 0,
+  // STRICT_MODE,
+  // EXTENDED_MODE,
+  kLanguageModeMask = 0x03,
+  kAllowLazy = 4,
+  kAllowNativesSyntax = 8
 };
 
+STATIC_ASSERT((kLanguageModeMask & CLASSIC_MODE) == CLASSIC_MODE);
+STATIC_ASSERT((kLanguageModeMask & STRICT_MODE) == STRICT_MODE);
+STATIC_ASSERT((kLanguageModeMask & EXTENDED_MODE) == EXTENDED_MODE);
+
 
 // Returns the value (0 .. 15) of a hexadecimal character c.
 // If c is not a legal hexadecimal character, returns a value < 0.

test/cctest/test-parsing.cc

@@ -862,7 +862,7 @@ TEST(ScopePositions) {
     i::Handle<i::String> source(
         FACTORY->NewStringFromAscii(i::CStrVector(program.start())));
     i::Handle<i::Script> script = FACTORY->NewScript(source);
-    i::Parser parser(script, false, NULL, NULL);
+    i::Parser parser(script, i::kAllowLazy | i::EXTENDED_MODE, NULL, NULL);
     i::CompilationInfo info(script);
     info.MarkAsGlobal();
     info.SetLanguageMode(source_data[i].language_mode);

test/mjsunit/regress/regress-1110.js

@@ -28,10 +28,9 @@
 // Test that the illegal continue is thrown at parse time.
 
 try {
-  function Crash() { continue;if (Crash) {
-    } }
+  eval("function Crash() { assertUnreachable(); continue;if (Crash) {  } }");
   Crash();
-  assertTrue(false);
+  assertUnreachable();
 } catch (e) {
   assertTrue(e instanceof SyntaxError);
   assertTrue(/continue/.test(e.message));