Commit be957002 authored by vogelheim's avatar vogelheim Committed by Commit bot

Implement a 'trial parse' step, that will abort pre-parsing excessively

long and trivial functions, so that they can be eagerly compiled after all.
This essentially allows the parser to renege on its earlier decision to
lazy-parse, if additional information suggests it was a bad decision.

BUG=chromium:470930
LOG=Y

Review URL: https://codereview.chromium.org/1102523003

Cr-Commit-Position: refs/heads/master@{#28252}
parent 7b33409b
...@@ -2421,6 +2421,8 @@ class FunctionLiteral final : public Expression { ...@@ -2421,6 +2421,8 @@ class FunctionLiteral final : public Expression {
enum EagerCompileHint { kShouldEagerCompile, kShouldLazyCompile }; enum EagerCompileHint { kShouldEagerCompile, kShouldLazyCompile };
enum ShouldBeUsedOnceHint { kShouldBeUsedOnce, kDontKnowIfShouldBeUsedOnce };
enum ArityRestriction { enum ArityRestriction {
NORMAL_ARITY, NORMAL_ARITY,
GETTER_ARITY, GETTER_ARITY,
...@@ -2516,6 +2518,15 @@ class FunctionLiteral final : public Expression { ...@@ -2516,6 +2518,15 @@ class FunctionLiteral final : public Expression {
bitfield_ = EagerCompileHintBit::update(bitfield_, kShouldEagerCompile); bitfield_ = EagerCompileHintBit::update(bitfield_, kShouldEagerCompile);
} }
// A hint that we expect this function to be called (exactly) once,
// i.e. we suspect it's an initialization function.
bool should_be_used_once_hint() const {
return ShouldBeUsedOnceHintBit::decode(bitfield_) == kShouldBeUsedOnce;
}
void set_should_be_used_once_hint() {
bitfield_ = ShouldBeUsedOnceHintBit::update(bitfield_, kShouldBeUsedOnce);
}
FunctionKind kind() { return FunctionKindBits::decode(bitfield_); } FunctionKind kind() { return FunctionKindBits::decode(bitfield_); }
int ast_node_count() { return ast_properties_.node_count(); } int ast_node_count() { return ast_properties_.node_count(); }
...@@ -2560,7 +2571,8 @@ class FunctionLiteral final : public Expression { ...@@ -2560,7 +2571,8 @@ class FunctionLiteral final : public Expression {
HasDuplicateParameters::encode(has_duplicate_parameters) | HasDuplicateParameters::encode(has_duplicate_parameters) |
IsFunction::encode(is_function) | IsFunction::encode(is_function) |
EagerCompileHintBit::encode(eager_compile_hint) | EagerCompileHintBit::encode(eager_compile_hint) |
FunctionKindBits::encode(kind); FunctionKindBits::encode(kind) |
ShouldBeUsedOnceHintBit::encode(kDontKnowIfShouldBeUsedOnce);
DCHECK(IsValidFunctionKind(kind)); DCHECK(IsValidFunctionKind(kind));
} }
...@@ -2589,6 +2601,8 @@ class FunctionLiteral final : public Expression { ...@@ -2589,6 +2601,8 @@ class FunctionLiteral final : public Expression {
class IsFunction : public BitField<IsFunctionFlag, 4, 1> {}; class IsFunction : public BitField<IsFunctionFlag, 4, 1> {};
class EagerCompileHintBit : public BitField<EagerCompileHint, 5, 1> {}; class EagerCompileHintBit : public BitField<EagerCompileHint, 5, 1> {};
class FunctionKindBits : public BitField<FunctionKind, 6, 8> {}; class FunctionKindBits : public BitField<FunctionKind, 6, 8> {};
class ShouldBeUsedOnceHintBit : public BitField<ShouldBeUsedOnceHint, 15, 1> {
};
}; };
......
...@@ -1371,6 +1371,10 @@ Handle<SharedFunctionInfo> Compiler::BuildFunctionInfo( ...@@ -1371,6 +1371,10 @@ Handle<SharedFunctionInfo> Compiler::BuildFunctionInfo(
// appropriately sized. // appropriately sized.
DCHECK(!info.code().is_null()); DCHECK(!info.code().is_null());
scope_info = ScopeInfo::Create(info.isolate(), info.zone(), info.scope()); scope_info = ScopeInfo::Create(info.isolate(), info.zone(), info.scope());
if (literal->should_eager_compile() &&
literal->should_be_used_once_hint()) {
info.code()->MarkToBeExecutedOnce(isolate);
}
} else { } else {
return Handle<SharedFunctionInfo>::null(); return Handle<SharedFunctionInfo>::null();
} }
......
...@@ -4036,6 +4036,7 @@ FunctionLiteral* Parser::ParseFunctionLiteral( ...@@ -4036,6 +4036,7 @@ FunctionLiteral* Parser::ParseFunctionLiteral(
FunctionLiteral::EagerCompileHint eager_compile_hint = FunctionLiteral::EagerCompileHint eager_compile_hint =
parenthesized_function_ ? FunctionLiteral::kShouldEagerCompile parenthesized_function_ ? FunctionLiteral::kShouldEagerCompile
: FunctionLiteral::kShouldLazyCompile; : FunctionLiteral::kShouldLazyCompile;
bool should_be_used_once_hint = false;
// Parse function body. // Parse function body.
{ {
AstNodeFactory function_factory(ast_value_factory()); AstNodeFactory function_factory(ast_value_factory());
...@@ -4133,14 +4134,36 @@ FunctionLiteral* Parser::ParseFunctionLiteral( ...@@ -4133,14 +4134,36 @@ FunctionLiteral* Parser::ParseFunctionLiteral(
!parenthesized_function_); !parenthesized_function_);
parenthesized_function_ = false; // The bit was set for this function only. parenthesized_function_ = false; // The bit was set for this function only.
// Eager or lazy parse?
// If is_lazily_parsed, we'll parse lazy. If we can set a bookmark, we'll
// pass it to SkipLazyFunctionBody, which may use it to abort lazy
// parsing if it suspect that wasn't a good idea. If so, or if we didn't
// try to lazy parse in the first place, we'll have to parse eagerly.
Scanner::BookmarkScope bookmark(scanner());
if (is_lazily_parsed) { if (is_lazily_parsed) {
for (Scope* s = scope_->outer_scope(); for (Scope* s = scope_->outer_scope();
s != nullptr && (s != s->DeclarationScope()); s = s->outer_scope()) { s != nullptr && (s != s->DeclarationScope()); s = s->outer_scope()) {
s->ForceContextAllocation(); s->ForceContextAllocation();
} }
Scanner::BookmarkScope* maybe_bookmark =
bookmark.Set() ? &bookmark : nullptr;
SkipLazyFunctionBody(&materialized_literal_count, SkipLazyFunctionBody(&materialized_literal_count,
&expected_property_count, CHECK_OK); &expected_property_count, /*CHECK_OK*/ ok,
} else { maybe_bookmark);
if (bookmark.HasBeenReset()) {
// Trigger eager (re-)parsing, just below this block.
is_lazily_parsed = false;
// This is probably an initialization function. Inform the compiler it
// should also eager-compile this function, and that we expect it to be
// used once.
eager_compile_hint = FunctionLiteral::kShouldEagerCompile;
should_be_used_once_hint = true;
}
}
if (!is_lazily_parsed) {
body = ParseEagerFunctionBody(function_name, pos, fvar, fvar_init_op, body = ParseEagerFunctionBody(function_name, pos, fvar, fvar_init_op,
kind, CHECK_OK); kind, CHECK_OK);
materialized_literal_count = function_state.materialized_literal_count(); materialized_literal_count = function_state.materialized_literal_count();
...@@ -4183,6 +4206,8 @@ FunctionLiteral* Parser::ParseFunctionLiteral( ...@@ -4183,6 +4206,8 @@ FunctionLiteral* Parser::ParseFunctionLiteral(
num_parameters, duplicate_parameters, function_type, num_parameters, duplicate_parameters, function_type,
FunctionLiteral::kIsFunction, eager_compile_hint, kind, pos); FunctionLiteral::kIsFunction, eager_compile_hint, kind, pos);
function_literal->set_function_token_position(function_token_pos); function_literal->set_function_token_position(function_token_pos);
if (should_be_used_once_hint)
function_literal->set_should_be_used_once_hint();
if (scope->has_rest_parameter()) { if (scope->has_rest_parameter()) {
// TODO(caitp): enable optimization of functions with rest params // TODO(caitp): enable optimization of functions with rest params
...@@ -4195,8 +4220,9 @@ FunctionLiteral* Parser::ParseFunctionLiteral( ...@@ -4195,8 +4220,9 @@ FunctionLiteral* Parser::ParseFunctionLiteral(
void Parser::SkipLazyFunctionBody(int* materialized_literal_count, void Parser::SkipLazyFunctionBody(int* materialized_literal_count,
int* expected_property_count, int* expected_property_count, bool* ok,
bool* ok) { Scanner::BookmarkScope* bookmark) {
DCHECK_IMPLIES(bookmark, bookmark->HasBeenSet());
if (produce_cached_parse_data()) CHECK(log_); if (produce_cached_parse_data()) CHECK(log_);
int function_block_pos = position(); int function_block_pos = position();
...@@ -4229,7 +4255,10 @@ void Parser::SkipLazyFunctionBody(int* materialized_literal_count, ...@@ -4229,7 +4255,10 @@ void Parser::SkipLazyFunctionBody(int* materialized_literal_count,
// AST. This gathers the data needed to build a lazy function. // AST. This gathers the data needed to build a lazy function.
SingletonLogger logger; SingletonLogger logger;
PreParser::PreParseResult result = PreParser::PreParseResult result =
ParseLazyFunctionBodyWithPreParser(&logger); ParseLazyFunctionBodyWithPreParser(&logger, bookmark);
if (bookmark && bookmark->HasBeenReset()) {
return; // Return immediately if pre-parser devided to abort parsing.
}
if (result == PreParser::kPreParseStackOverflow) { if (result == PreParser::kPreParseStackOverflow) {
// Propagate stack overflow. // Propagate stack overflow.
set_stack_overflow(); set_stack_overflow();
...@@ -4358,7 +4387,7 @@ ZoneList<Statement*>* Parser::ParseEagerFunctionBody( ...@@ -4358,7 +4387,7 @@ ZoneList<Statement*>* Parser::ParseEagerFunctionBody(
PreParser::PreParseResult Parser::ParseLazyFunctionBodyWithPreParser( PreParser::PreParseResult Parser::ParseLazyFunctionBodyWithPreParser(
SingletonLogger* logger) { SingletonLogger* logger, Scanner::BookmarkScope* bookmark) {
// This function may be called on a background thread too; record only the // This function may be called on a background thread too; record only the
// main thread preparse times. // main thread preparse times.
if (pre_parse_timer_ != NULL) { if (pre_parse_timer_ != NULL) {
...@@ -4390,7 +4419,7 @@ PreParser::PreParseResult Parser::ParseLazyFunctionBodyWithPreParser( ...@@ -4390,7 +4419,7 @@ PreParser::PreParseResult Parser::ParseLazyFunctionBodyWithPreParser(
reusable_preparser_->set_allow_strong_mode(allow_strong_mode()); reusable_preparser_->set_allow_strong_mode(allow_strong_mode());
} }
PreParser::PreParseResult result = reusable_preparser_->PreParseLazyFunction( PreParser::PreParseResult result = reusable_preparser_->PreParseLazyFunction(
language_mode(), function_state_->kind(), logger); language_mode(), function_state_->kind(), logger, bookmark);
if (pre_parse_timer_ != NULL) { if (pre_parse_timer_ != NULL) {
pre_parse_timer_->Stop(); pre_parse_timer_->Stop();
} }
......
...@@ -773,8 +773,9 @@ class ParserTraits { ...@@ -773,8 +773,9 @@ class ParserTraits {
bool name_is_strict_reserved, FunctionKind kind, bool name_is_strict_reserved, FunctionKind kind,
int function_token_position, FunctionLiteral::FunctionType type, int function_token_position, FunctionLiteral::FunctionType type,
FunctionLiteral::ArityRestriction arity_restriction, bool* ok); FunctionLiteral::ArityRestriction arity_restriction, bool* ok);
V8_INLINE void SkipLazyFunctionBody(int* materialized_literal_count, V8_INLINE void SkipLazyFunctionBody(
int* expected_property_count, bool* ok); int* materialized_literal_count, int* expected_property_count, bool* ok,
Scanner::BookmarkScope* bookmark = nullptr);
V8_INLINE ZoneList<Statement*>* ParseEagerFunctionBody( V8_INLINE ZoneList<Statement*>* ParseEagerFunctionBody(
const AstRawString* name, int pos, Variable* fvar, const AstRawString* name, int pos, Variable* fvar,
Token::Value fvar_init_op, FunctionKind kind, bool* ok); Token::Value fvar_init_op, FunctionKind kind, bool* ok);
...@@ -1023,12 +1024,16 @@ class Parser : public ParserBase<ParserTraits> { ...@@ -1023,12 +1024,16 @@ class Parser : public ParserBase<ParserTraits> {
// Skip over a lazy function, either using cached data if we have it, or // Skip over a lazy function, either using cached data if we have it, or
// by parsing the function with PreParser. Consumes the ending }. // by parsing the function with PreParser. Consumes the ending }.
//
// If bookmark is set, the (pre-)parser may decide to abort skipping
// in order to force the function to be eagerly parsed, after all.
// In this case, it'll reset the scanner using the bookmark.
void SkipLazyFunctionBody(int* materialized_literal_count, void SkipLazyFunctionBody(int* materialized_literal_count,
int* expected_property_count, int* expected_property_count, bool* ok,
bool* ok); Scanner::BookmarkScope* bookmark = nullptr);
PreParser::PreParseResult ParseLazyFunctionBodyWithPreParser( PreParser::PreParseResult ParseLazyFunctionBodyWithPreParser(
SingletonLogger* logger); SingletonLogger* logger, Scanner::BookmarkScope* bookmark = nullptr);
// Consumes the ending }. // Consumes the ending }.
ZoneList<Statement*>* ParseEagerFunctionBody( ZoneList<Statement*>* ParseEagerFunctionBody(
...@@ -1089,10 +1094,10 @@ const AstRawString* ParserTraits::EmptyIdentifierString() { ...@@ -1089,10 +1094,10 @@ const AstRawString* ParserTraits::EmptyIdentifierString() {
void ParserTraits::SkipLazyFunctionBody(int* materialized_literal_count, void ParserTraits::SkipLazyFunctionBody(int* materialized_literal_count,
int* expected_property_count, int* expected_property_count, bool* ok,
bool* ok) { Scanner::BookmarkScope* bookmark) {
return parser_->SkipLazyFunctionBody( return parser_->SkipLazyFunctionBody(materialized_literal_count,
materialized_literal_count, expected_property_count, ok); expected_property_count, ok, bookmark);
} }
......
...@@ -99,7 +99,8 @@ PreParserExpression PreParserTraits::ParseFunctionLiteral( ...@@ -99,7 +99,8 @@ PreParserExpression PreParserTraits::ParseFunctionLiteral(
PreParser::PreParseResult PreParser::PreParseLazyFunction( PreParser::PreParseResult PreParser::PreParseLazyFunction(
LanguageMode language_mode, FunctionKind kind, ParserRecorder* log) { LanguageMode language_mode, FunctionKind kind, ParserRecorder* log,
Scanner::BookmarkScope* bookmark) {
log_ = log; log_ = log;
// Lazy functions always have trivial outer scopes (no with/catch scopes). // Lazy functions always have trivial outer scopes (no with/catch scopes).
Scope* top_scope = NewScope(scope_, SCRIPT_SCOPE); Scope* top_scope = NewScope(scope_, SCRIPT_SCOPE);
...@@ -114,9 +115,12 @@ PreParser::PreParseResult PreParser::PreParseLazyFunction( ...@@ -114,9 +115,12 @@ PreParser::PreParseResult PreParser::PreParseLazyFunction(
DCHECK_EQ(Token::LBRACE, scanner()->current_token()); DCHECK_EQ(Token::LBRACE, scanner()->current_token());
bool ok = true; bool ok = true;
int start_position = peek_position(); int start_position = peek_position();
ParseLazyFunctionLiteralBody(&ok); ParseLazyFunctionLiteralBody(&ok, bookmark);
if (stack_overflow()) return kPreParseStackOverflow; if (bookmark && bookmark->HasBeenReset()) {
if (!ok) { ; // Do nothing, as we've just aborted scanning this function.
} else if (stack_overflow()) {
return kPreParseStackOverflow;
} else if (!ok) {
ReportUnexpectedToken(scanner()->current_token()); ReportUnexpectedToken(scanner()->current_token());
} else { } else {
DCHECK_EQ(Token::RBRACE, scanner()->peek()); DCHECK_EQ(Token::RBRACE, scanner()->peek());
...@@ -196,15 +200,22 @@ PreParser::Statement PreParser::ParseStatementListItem(bool* ok) { ...@@ -196,15 +200,22 @@ PreParser::Statement PreParser::ParseStatementListItem(bool* ok) {
} }
void PreParser::ParseStatementList(int end_token, bool* ok) { void PreParser::ParseStatementList(int end_token, bool* ok,
Scanner::BookmarkScope* bookmark) {
// SourceElements :: // SourceElements ::
// (Statement)* <end_token> // (Statement)* <end_token>
// Bookkeeping for trial parse if bookmark is set:
DCHECK_IMPLIES(bookmark, bookmark->HasBeenSet());
bool maybe_reset = bookmark != nullptr;
int count_statements = 0;
bool directive_prologue = true; bool directive_prologue = true;
while (peek() != end_token) { while (peek() != end_token) {
if (directive_prologue && peek() != Token::STRING) { if (directive_prologue && peek() != Token::STRING) {
directive_prologue = false; directive_prologue = false;
} }
bool starts_with_identifier = peek() == Token::IDENTIFIER;
Scanner::Location token_loc = scanner()->peek_location(); Scanner::Location token_loc = scanner()->peek_location();
Scanner::Location old_this_loc = function_state_->this_location(); Scanner::Location old_this_loc = function_state_->this_location();
Scanner::Location old_super_loc = function_state_->super_location(); Scanner::Location old_super_loc = function_state_->super_location();
...@@ -241,6 +252,20 @@ void PreParser::ParseStatementList(int end_token, bool* ok) { ...@@ -241,6 +252,20 @@ void PreParser::ParseStatementList(int end_token, bool* ok) {
directive_prologue = false; directive_prologue = false;
} }
} }
// If we're allowed to reset to a bookmark, we will do so when we see a long
// and trivial function.
// Our current definition of 'long and trivial' is:
// - over 200 statements
// - all starting with an identifier (i.e., no if, for, while, etc.)
if (maybe_reset && (!starts_with_identifier ||
++count_statements > kLazyParseTrialLimit)) {
if (count_statements > kLazyParseTrialLimit) {
bookmark->Reset();
return;
}
maybe_reset = false;
}
} }
} }
...@@ -1057,10 +1082,12 @@ PreParser::Expression PreParser::ParseFunctionLiteral( ...@@ -1057,10 +1082,12 @@ PreParser::Expression PreParser::ParseFunctionLiteral(
} }
void PreParser::ParseLazyFunctionLiteralBody(bool* ok) { void PreParser::ParseLazyFunctionLiteralBody(bool* ok,
Scanner::BookmarkScope* bookmark) {
int body_start = position(); int body_start = position();
ParseStatementList(Token::RBRACE, ok); ParseStatementList(Token::RBRACE, ok, bookmark);
if (!*ok) return; if (!*ok) return;
if (bookmark && bookmark->HasBeenReset()) return;
// Position right after terminal '}'. // Position right after terminal '}'.
DCHECK_EQ(Token::RBRACE, scanner()->peek()); DCHECK_EQ(Token::RBRACE, scanner()->peek());
......
...@@ -1821,12 +1821,15 @@ class PreParser : public ParserBase<PreParserTraits> { ...@@ -1821,12 +1821,15 @@ class PreParser : public ParserBase<PreParserTraits> {
// keyword and parameters, and have consumed the initial '{'. // keyword and parameters, and have consumed the initial '{'.
// At return, unless an error occurred, the scanner is positioned before the // At return, unless an error occurred, the scanner is positioned before the
// the final '}'. // the final '}'.
PreParseResult PreParseLazyFunction(LanguageMode language_mode, PreParseResult PreParseLazyFunction(
FunctionKind kind, ParserRecorder* log); LanguageMode language_mode, FunctionKind kind, ParserRecorder* log,
Scanner::BookmarkScope* bookmark = nullptr);
private: private:
friend class PreParserTraits; friend class PreParserTraits;
static const int kLazyParseTrialLimit = 200;
// These types form an algebra over syntactic categories that is just // These types form an algebra over syntactic categories that is just
// rich enough to let us recognize and propagate the constructs that // rich enough to let us recognize and propagate the constructs that
// are either being counted in the preparser data, or is important // are either being counted in the preparser data, or is important
...@@ -1837,7 +1840,8 @@ class PreParser : public ParserBase<PreParserTraits> { ...@@ -1837,7 +1840,8 @@ class PreParser : public ParserBase<PreParserTraits> {
// By making the 'exception handling' explicit, we are forced to check // By making the 'exception handling' explicit, we are forced to check
// for failure at the call sites. // for failure at the call sites.
Statement ParseStatementListItem(bool* ok); Statement ParseStatementListItem(bool* ok);
void ParseStatementList(int end_token, bool* ok); void ParseStatementList(int end_token, bool* ok,
Scanner::BookmarkScope* bookmark = nullptr);
Statement ParseStatement(bool* ok); Statement ParseStatement(bool* ok);
Statement ParseSubStatement(bool* ok); Statement ParseSubStatement(bool* ok);
Statement ParseFunctionDeclaration(bool* ok); Statement ParseFunctionDeclaration(bool* ok);
...@@ -1879,7 +1883,8 @@ class PreParser : public ParserBase<PreParserTraits> { ...@@ -1879,7 +1883,8 @@ class PreParser : public ParserBase<PreParserTraits> {
bool name_is_strict_reserved, FunctionKind kind, int function_token_pos, bool name_is_strict_reserved, FunctionKind kind, int function_token_pos,
FunctionLiteral::FunctionType function_type, FunctionLiteral::FunctionType function_type,
FunctionLiteral::ArityRestriction arity_restriction, bool* ok); FunctionLiteral::ArityRestriction arity_restriction, bool* ok);
void ParseLazyFunctionLiteralBody(bool* ok); void ParseLazyFunctionLiteralBody(bool* ok,
Scanner::BookmarkScope* bookmark = nullptr);
PreParserExpression ParseClassLiteral(PreParserIdentifier name, PreParserExpression ParseClassLiteral(PreParserIdentifier name,
Scanner::Location class_name_location, Scanner::Location class_name_location,
......
...@@ -130,7 +130,7 @@ size_t BufferedUtf16CharacterStream::SlowSeekForward(size_t delta) { ...@@ -130,7 +130,7 @@ size_t BufferedUtf16CharacterStream::SlowSeekForward(size_t delta) {
GenericStringUtf16CharacterStream::GenericStringUtf16CharacterStream( GenericStringUtf16CharacterStream::GenericStringUtf16CharacterStream(
Handle<String> data, size_t start_position, size_t end_position) Handle<String> data, size_t start_position, size_t end_position)
: string_(data), length_(end_position) { : string_(data), length_(end_position), bookmark_(kNoBookmark) {
DCHECK(end_position >= start_position); DCHECK(end_position >= start_position);
pos_ = start_position; pos_ = start_position;
} }
...@@ -139,6 +139,20 @@ GenericStringUtf16CharacterStream::GenericStringUtf16CharacterStream( ...@@ -139,6 +139,20 @@ GenericStringUtf16CharacterStream::GenericStringUtf16CharacterStream(
GenericStringUtf16CharacterStream::~GenericStringUtf16CharacterStream() { } GenericStringUtf16CharacterStream::~GenericStringUtf16CharacterStream() { }
bool GenericStringUtf16CharacterStream::SetBookmark() {
bookmark_ = pos_;
return true;
}
void GenericStringUtf16CharacterStream::ResetToBookmark() {
DCHECK(bookmark_ != kNoBookmark);
pos_ = bookmark_;
buffer_cursor_ = buffer_;
buffer_end_ = buffer_ + FillBuffer(pos_);
}
size_t GenericStringUtf16CharacterStream::BufferSeekForward(size_t delta) { size_t GenericStringUtf16CharacterStream::BufferSeekForward(size_t delta) {
size_t old_pos = pos_; size_t old_pos = pos_;
pos_ = Min(pos_ + delta, length_); pos_ = Min(pos_ + delta, length_);
...@@ -447,17 +461,29 @@ ExternalTwoByteStringUtf16CharacterStream:: ...@@ -447,17 +461,29 @@ ExternalTwoByteStringUtf16CharacterStream::
~ExternalTwoByteStringUtf16CharacterStream() { } ~ExternalTwoByteStringUtf16CharacterStream() { }
ExternalTwoByteStringUtf16CharacterStream ExternalTwoByteStringUtf16CharacterStream::
::ExternalTwoByteStringUtf16CharacterStream( ExternalTwoByteStringUtf16CharacterStream(
Handle<ExternalTwoByteString> data, Handle<ExternalTwoByteString> data, int start_position,
int start_position,
int end_position) int end_position)
: Utf16CharacterStream(), : Utf16CharacterStream(),
source_(data), source_(data),
raw_data_(data->GetTwoByteData(start_position)) { raw_data_(data->GetTwoByteData(start_position)),
bookmark_(kNoBookmark) {
buffer_cursor_ = raw_data_, buffer_cursor_ = raw_data_,
buffer_end_ = raw_data_ + (end_position - start_position); buffer_end_ = raw_data_ + (end_position - start_position);
pos_ = start_position; pos_ = start_position;
} }
bool ExternalTwoByteStringUtf16CharacterStream::SetBookmark() {
bookmark_ = pos_;
return true;
}
void ExternalTwoByteStringUtf16CharacterStream::ResetToBookmark() {
DCHECK(bookmark_ != kNoBookmark);
pos_ = bookmark_;
buffer_cursor_ = raw_data_ + bookmark_;
}
} } // namespace v8::internal } } // namespace v8::internal
...@@ -43,12 +43,18 @@ class GenericStringUtf16CharacterStream: public BufferedUtf16CharacterStream { ...@@ -43,12 +43,18 @@ class GenericStringUtf16CharacterStream: public BufferedUtf16CharacterStream {
size_t end_position); size_t end_position);
virtual ~GenericStringUtf16CharacterStream(); virtual ~GenericStringUtf16CharacterStream();
virtual bool SetBookmark();
virtual void ResetToBookmark();
protected: protected:
static const size_t kNoBookmark = -1;
virtual size_t BufferSeekForward(size_t delta); virtual size_t BufferSeekForward(size_t delta);
virtual size_t FillBuffer(size_t position); virtual size_t FillBuffer(size_t position);
Handle<String> string_; Handle<String> string_;
size_t length_; size_t length_;
size_t bookmark_;
}; };
...@@ -129,6 +135,9 @@ class ExternalTwoByteStringUtf16CharacterStream: public Utf16CharacterStream { ...@@ -129,6 +135,9 @@ class ExternalTwoByteStringUtf16CharacterStream: public Utf16CharacterStream {
pos_--; pos_--;
} }
virtual bool SetBookmark();
virtual void ResetToBookmark();
protected: protected:
virtual size_t SlowSeekForward(size_t delta) { virtual size_t SlowSeekForward(size_t delta) {
// Fast case always handles seeking. // Fast case always handles seeking.
...@@ -140,6 +149,11 @@ class ExternalTwoByteStringUtf16CharacterStream: public Utf16CharacterStream { ...@@ -140,6 +149,11 @@ class ExternalTwoByteStringUtf16CharacterStream: public Utf16CharacterStream {
} }
Handle<ExternalTwoByteString> source_; Handle<ExternalTwoByteString> source_;
const uc16* raw_data_; // Pointer to the actual array of characters. const uc16* raw_data_; // Pointer to the actual array of characters.
private:
static const size_t kNoBookmark = -1;
size_t bookmark_;
}; };
} } // namespace v8::internal } } // namespace v8::internal
......
...@@ -29,15 +29,26 @@ Handle<String> LiteralBuffer::Internalize(Isolate* isolate) const { ...@@ -29,15 +29,26 @@ Handle<String> LiteralBuffer::Internalize(Isolate* isolate) const {
} }
// Default implementation for streams that do not support bookmarks.
bool Utf16CharacterStream::SetBookmark() { return false; }
void Utf16CharacterStream::ResetToBookmark() { UNREACHABLE(); }
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
// Scanner // Scanner
Scanner::Scanner(UnicodeCache* unicode_cache) Scanner::Scanner(UnicodeCache* unicode_cache)
: unicode_cache_(unicode_cache), : unicode_cache_(unicode_cache),
bookmark_c0_(kNoBookmark),
octal_pos_(Location::invalid()), octal_pos_(Location::invalid()),
harmony_modules_(false), harmony_modules_(false),
harmony_classes_(false), harmony_classes_(false),
harmony_unicode_(false) {} harmony_unicode_(false) {
bookmark_current_.literal_chars = &bookmark_current_literal_;
bookmark_current_.raw_literal_chars = &bookmark_current_raw_literal_;
bookmark_next_.literal_chars = &bookmark_next_literal_;
bookmark_next_.raw_literal_chars = &bookmark_next_raw_literal_;
}
void Scanner::Initialize(Utf16CharacterStream* source) { void Scanner::Initialize(Utf16CharacterStream* source) {
...@@ -1426,6 +1437,56 @@ int Scanner::FindSymbol(DuplicateFinder* finder, int value) { ...@@ -1426,6 +1437,56 @@ int Scanner::FindSymbol(DuplicateFinder* finder, int value) {
} }
bool Scanner::SetBookmark() {
if (c0_ != kNoBookmark && bookmark_c0_ == kNoBookmark &&
source_->SetBookmark()) {
bookmark_c0_ = c0_;
CopyTokenDesc(&bookmark_current_, &current_);
CopyTokenDesc(&bookmark_next_, &next_);
return true;
}
return false;
}
void Scanner::ResetToBookmark() {
DCHECK(BookmarkHasBeenSet()); // Caller hasn't called SetBookmark.
source_->ResetToBookmark();
c0_ = bookmark_c0_;
StartLiteral();
StartRawLiteral();
CopyTokenDesc(&next_, &bookmark_current_);
current_ = next_;
StartLiteral();
StartRawLiteral();
CopyTokenDesc(&next_, &bookmark_next_);
bookmark_c0_ = kBookmarkWasApplied;
}
bool Scanner::BookmarkHasBeenSet() { return bookmark_c0_ >= 0; }
bool Scanner::BookmarkHasBeenReset() {
return bookmark_c0_ == kBookmarkWasApplied;
}
void Scanner::DropBookmark() { bookmark_c0_ = kNoBookmark; }
void Scanner::CopyTokenDesc(TokenDesc* to, TokenDesc* from) {
DCHECK_NOT_NULL(to);
DCHECK_NOT_NULL(from);
to->token = from->token;
to->location = from->location;
to->literal_chars->CopyFrom(from->literal_chars);
to->raw_literal_chars->CopyFrom(from->raw_literal_chars);
}
int DuplicateFinder::AddOneByteSymbol(Vector<const uint8_t> key, int value) { int DuplicateFinder::AddOneByteSymbol(Vector<const uint8_t> key, int value) {
return AddSymbol(key, true, value); return AddSymbol(key, true, value);
} }
......
...@@ -89,6 +89,9 @@ class Utf16CharacterStream { ...@@ -89,6 +89,9 @@ class Utf16CharacterStream {
// Must not be used right after calling SeekForward. // Must not be used right after calling SeekForward.
virtual void PushBack(int32_t code_unit) = 0; virtual void PushBack(int32_t code_unit) = 0;
virtual bool SetBookmark();
virtual void ResetToBookmark();
protected: protected:
static const uc32 kEndOfInput = -1; static const uc32 kEndOfInput = -1;
...@@ -268,6 +271,17 @@ class LiteralBuffer { ...@@ -268,6 +271,17 @@ class LiteralBuffer {
Handle<String> Internalize(Isolate* isolate) const; Handle<String> Internalize(Isolate* isolate) const;
void CopyFrom(const LiteralBuffer* other) {
if (other == nullptr) {
Reset();
} else {
is_one_byte_ = other->is_one_byte_;
position_ = other->position_;
backing_store_.Dispose();
backing_store_ = other->backing_store_.Clone();
}
}
private: private:
static const int kInitialCapacity = 16; static const int kInitialCapacity = 16;
static const int kGrowthFactory = 4; static const int kGrowthFactory = 4;
...@@ -342,6 +356,25 @@ class Scanner { ...@@ -342,6 +356,25 @@ class Scanner {
bool complete_; bool complete_;
}; };
// Scoped helper for a re-settable bookmark.
class BookmarkScope {
public:
explicit BookmarkScope(Scanner* scanner) : scanner_(scanner) {
DCHECK_NOT_NULL(scanner_);
}
~BookmarkScope() { scanner_->DropBookmark(); }
bool Set() { return scanner_->SetBookmark(); }
void Reset() { scanner_->ResetToBookmark(); }
bool HasBeenSet() { return scanner_->BookmarkHasBeenSet(); }
bool HasBeenReset() { return scanner_->BookmarkHasBeenReset(); }
private:
Scanner* scanner_;
DISALLOW_COPY_AND_ASSIGN(BookmarkScope);
};
// Representation of an interval of source positions. // Representation of an interval of source positions.
struct Location { struct Location {
Location(int b, int e) : beg_pos(b), end_pos(e) { } Location(int b, int e) : beg_pos(b), end_pos(e) { }
...@@ -510,6 +543,14 @@ class Scanner { ...@@ -510,6 +543,14 @@ class Scanner {
current_.raw_literal_chars = NULL; current_.raw_literal_chars = NULL;
} }
// Support BookmarkScope functionality.
bool SetBookmark();
void ResetToBookmark();
bool BookmarkHasBeenSet();
bool BookmarkHasBeenReset();
void DropBookmark();
static void CopyTokenDesc(TokenDesc* to, TokenDesc* from);
// Literal buffer support // Literal buffer support
inline void StartLiteral() { inline void StartLiteral() {
LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ? LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ?
...@@ -712,6 +753,37 @@ class Scanner { ...@@ -712,6 +753,37 @@ class Scanner {
TokenDesc current_; // desc for current token (as returned by Next()) TokenDesc current_; // desc for current token (as returned by Next())
TokenDesc next_; // desc for next token (one token look-ahead) TokenDesc next_; // desc for next token (one token look-ahead)
// Variables for Scanner::BookmarkScope and the *Bookmark implementation.
// These variables contain the scanner state when a bookmark is set.
//
// We will use bookmark_c0_ as a 'control' variable, where:
// - bookmark_c0_ >= 0: A bookmark has been set and this contains c0_.
// - bookmark_c0_ == -1: No bookmark has been set.
// - bookmark_c0_ == -2: The bookmark has been applied (ResetToBookmark).
//
// Which state is being bookmarked? The parser state is distributed over
// several variables, roughly like this:
// ... 1234 + 5678 ..... [character stream]
// [current_] [next_] c0_ | [scanner state]
// So when the scanner is logically at the beginning of an expression
// like "1234 + 4567", then:
// - current_ contains "1234"
// - next_ contains "+"
// - c0_ contains ' ' (the space between "+" and "5678",
// - the source_ character stream points to the beginning of "5678".
// To be able to restore this state, we will keep copies of current_, next_,
// and c0_; we'll ask the stream to bookmark itself, and we'll copy the
// contents of current_'s and next_'s literal buffers to bookmark_*_literal_.
static const uc32 kNoBookmark = -1;
static const uc32 kBookmarkWasApplied = -2;
uc32 bookmark_c0_;
TokenDesc bookmark_current_;
TokenDesc bookmark_next_;
LiteralBuffer bookmark_current_literal_;
LiteralBuffer bookmark_current_raw_literal_;
LiteralBuffer bookmark_next_literal_;
LiteralBuffer bookmark_next_raw_literal_;
// Input stream. Must be initialized to an Utf16CharacterStream. // Input stream. Must be initialized to an Utf16CharacterStream.
Utf16CharacterStream* source_; Utf16CharacterStream* source_;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment