Skip to content

V
V8
Commit f386f974 authored by lrn@chromium.org's avatar lrn@chromium.org
Browse files
Options

Move part of scanner.* into scanner-base.* for reuse in preparser scanner. 

Make checks.h not depend on flags.h or global.h (or anything else except
include/v8stdint.h). Only checks.cc has the dependencies (so another
implementation of checks.cc can be provided by the preparser).
Now files depending on checks.h (using ASSERT macros) can include it
directly without depending on all of v8.

Review URL: http://codereview.chromium.org/4576001

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@5775 ce2b1a6d-e550-0410-aec6-3dcde31c8c00
parent 8838fdea
Show whitespace changes
Inline Side-by-side
Showing with 603 additions and 484 deletions
src/SConscript
View file @ f386f974
......@@ -95,6 +95,7 @@ SOURCES = {
register-allocator.cc
rewriter.cc
runtime.cc
scanner-base.cc
scanner.cc
scopeinfo.cc
scopes.cc
......
src/checks.cc
View file @ f386f974
......@@ -98,3 +98,12 @@ void API_Fatal(const char* location, const char* format, ...) {
i::OS::PrintError("\n#\n\n");
i::OS::Abort();
}
namespace v8 { namespace internal {
bool EnableSlowAsserts() { return FLAG_enable_slow_asserts; }
intptr_t HeapObjectTagMask() { return kHeapObjectTagMask; }
} } // namespace v8::internal
src/checks.h
View file @ f386f974
......@@ -30,7 +30,7 @@
#include <string.h>
#include "flags.h"
#include "../include/v8stdint.h"
extern "C" void V8_Fatal(const char* file, int line, const char* format, ...);
void API_Fatal(const char* location, const char* format, ...);
......@@ -279,6 +279,12 @@ template <int> class StaticAssertionHelper { };
SEMI_STATIC_JOIN(__StaticAssertTypedef__, __LINE__)
namespace v8 { namespace internal {
bool EnableSlowAsserts();
} } // namespace v8::internal
// The ASSERT macro is equivalent to CHECK except that it only
// generates code in debug builds.
#ifdef DEBUG
......@@ -287,7 +293,7 @@ template <int> class StaticAssertionHelper { };
#define ASSERT_EQ(v1, v2) CHECK_EQ(v1, v2)
#define ASSERT_NE(v1, v2) CHECK_NE(v1, v2)
#define ASSERT_GE(v1, v2) CHECK_GE(v1, v2)
#define SLOW_ASSERT(condition) if (FLAG_enable_slow_asserts) CHECK(condition)
#define SLOW_ASSERT(condition) if (EnableSlowAsserts()) CHECK(condition)
#else
#define ASSERT_RESULT(expr) (expr)
#define ASSERT(condition) ((void) 0)
......@@ -303,11 +309,16 @@ template <int> class StaticAssertionHelper { };
// and release compilation modes behaviour.
#define STATIC_ASSERT(test) STATIC_CHECK(test)
namespace v8 { namespace internal {
intptr_t HeapObjectTagMask();
} } // namespace v8::internal
#define ASSERT_TAG_ALIGNED(address) \
ASSERT((reinterpret_cast<intptr_t>(address) & kHeapObjectTagMask) == 0)
ASSERT((reinterpret_cast<intptr_t>(address) & HeapObjectTagMask()) == 0)
#define ASSERT_SIZE_TAG_ALIGNED(size) ASSERT((size & kHeapObjectTagMask) == 0)
#define ASSERT_SIZE_TAG_ALIGNED(size) ASSERT((size & HeapObjectTagMask()) == 0)
#define ASSERT_NOT_NULL(p) ASSERT_NE(NULL, p)
......
src/preparser.h
View file @ f386f974
......@@ -31,7 +31,6 @@
#include "unicode.h"
namespace v8 {
namespace internal {
namespace preparser {
// Preparsing checks a JavaScript program and emits preparse-data that helps
......@@ -47,6 +46,8 @@ namespace preparser {
// That means that contextual checks (like a label being declared where
// it is used) are generally omitted.
namespace i = v8::internal;
enum StatementType {
kUnknownStatement
};
......@@ -92,7 +93,7 @@ class PreParser {
log_ = log;
Scope top_scope(&scope_, kTopLevelScope);
bool ok = true;
ParseSourceElements(Token::EOS, &ok);
ParseSourceElements(i::Token::EOS, &ok);
bool stack_overflow = scanner_->stack_overflow();
if (!ok && !stack_overflow) {
ReportUnexpectedToken(scanner_->current_token());
......@@ -144,7 +145,7 @@ class PreParser {
// simple this-property assignments.
// Report syntax error
void ReportUnexpectedToken(Token::Value token);
void ReportUnexpectedToken(i::Token::Value token);
void ReportMessageAt(int start_pos,
int end_pos,
const char* type,
......@@ -205,24 +206,24 @@ class PreParser {
Expression GetStringSymbol();
Token::Value peek() { return scanner_->peek(); }
Token::Value Next() {
Token::Value next = scanner_->Next();
i::Token::Value peek() { return scanner_->peek(); }
i::Token::Value Next() {
i::Token::Value next = scanner_->Next();
return next;
}
void Consume(Token::Value token) {
void Consume(i::Token::Value token) {
Next();
}
void Expect(Token::Value token, bool* ok) {
void Expect(i::Token::Value token, bool* ok) {
if (Next() != token) {
*ok = false;
}
}
bool Check(Token::Value token) {
Token::Value next = peek();
bool Check(i::Token::Value token) {
i::Token::Value next = peek();
if (next == token) {
Consume(next);
return true;
......@@ -231,7 +232,7 @@ class PreParser {
}
void ExpectSemicolon(bool* ok);
static int Precedence(Token::Value tok, bool accept_IN);
static int Precedence(i::Token::Value tok, bool accept_IN);
Scanner* scanner_;
PreParserLog* log_;
......@@ -248,31 +249,31 @@ class PreParser {
template <typename Scanner, typename Log>
void PreParser<Scanner, Log>::ReportUnexpectedToken(Token::Value token) {
void PreParser<Scanner, Log>::ReportUnexpectedToken(i::Token::Value token) {
// We don't report stack overflows here, to avoid increasing the
// stack depth even further. Instead we report it after parsing is
// over, in ParseProgram.
if (token == Token::ILLEGAL && scanner_->stack_overflow()) {
if (token == i::Token::ILLEGAL && scanner_->stack_overflow()) {
return;
}
typename Scanner::Location source_location = scanner_->location();
// Four of the tokens are treated specially
switch (token) {
case Token::EOS:
case i::Token::EOS:
return ReportMessageAt(source_location.beg_pos, source_location.end_pos,
"unexpected_eos", NULL);
case Token::NUMBER:
case i::Token::NUMBER:
return ReportMessageAt(source_location.beg_pos, source_location.end_pos,
"unexpected_token_number", NULL);
case Token::STRING:
case i::Token::STRING:
return ReportMessageAt(source_location.beg_pos, source_location.end_pos,
"unexpected_token_string", NULL);
case Token::IDENTIFIER:
case i::Token::IDENTIFIER:
return ReportMessageAt(source_location.beg_pos, source_location.end_pos,
"unexpected_token_identifier", NULL);
default:
const char* name = Token::String(token);
const char* name = i::Token::String(token);
ReportMessageAt(source_location.beg_pos, source_location.end_pos,
"unexpected_token", name);
}
......@@ -320,57 +321,57 @@ Statement PreParser<Scanner, Log>::ParseStatement(bool* ok) {
// Keep the source position of the statement
switch (peek()) {
case Token::LBRACE:
case i::Token::LBRACE:
return ParseBlock(ok);
case Token::CONST:
case Token::VAR:
case i::Token::CONST:
case i::Token::VAR:
return ParseVariableStatement(ok);
case Token::SEMICOLON:
case i::Token::SEMICOLON:
Next();
return kUnknownStatement;
case Token::IF:
case i::Token::IF:
return ParseIfStatement(ok);
case Token::DO:
case i::Token::DO:
return ParseDoWhileStatement(ok);
case Token::WHILE:
case i::Token::WHILE:
return ParseWhileStatement(ok);
case Token::FOR:
case i::Token::FOR:
return ParseForStatement(ok);
case Token::CONTINUE:
case i::Token::CONTINUE:
return ParseContinueStatement(ok);
case Token::BREAK:
case i::Token::BREAK:
return ParseBreakStatement(ok);
case Token::RETURN:
case i::Token::RETURN:
return ParseReturnStatement(ok);
case Token::WITH:
case i::Token::WITH:
return ParseWithStatement(ok);
case Token::SWITCH:
case i::Token::SWITCH:
return ParseSwitchStatement(ok);
case Token::THROW:
case i::Token::THROW:
return ParseThrowStatement(ok);
case Token::TRY:
case i::Token::TRY:
return ParseTryStatement(ok);
case Token::FUNCTION:
case i::Token::FUNCTION:
return ParseFunctionDeclaration(ok);
case Token::NATIVE:
case i::Token::NATIVE:
return ParseNativeDeclaration(ok);
case Token::DEBUGGER:
case i::Token::DEBUGGER:
return ParseDebuggerStatement(ok);
default:
......@@ -383,7 +384,7 @@ template <typename Scanner, typename Log>
Statement PreParser<Scanner, Log>::ParseFunctionDeclaration(bool* ok) {
// FunctionDeclaration ::
// 'function' Identifier '(' FormalParameterListopt ')' '{' FunctionBody '}'
Expect(Token::FUNCTION, CHECK_OK);
Expect(i::Token::FUNCTION, CHECK_OK);
ParseIdentifier(CHECK_OK);
ParseFunctionLiteral(CHECK_OK);
return kUnknownStatement;
......@@ -396,20 +397,20 @@ Statement PreParser<Scanner, Log>::ParseFunctionDeclaration(bool* ok) {
// callback provided by the extension.
template <typename Scanner, typename Log>
Statement PreParser<Scanner, Log>::ParseNativeDeclaration(bool* ok) {
Expect(Token::NATIVE, CHECK_OK);
Expect(Token::FUNCTION, CHECK_OK);
Expect(i::Token::NATIVE, CHECK_OK);
Expect(i::Token::FUNCTION, CHECK_OK);
ParseIdentifier(CHECK_OK);
Expect(Token::LPAREN, CHECK_OK);
bool done = (peek() == Token::RPAREN);
Expect(i::Token::LPAREN, CHECK_OK);
bool done = (peek() == i::Token::RPAREN);
while (!done) {
ParseIdentifier(CHECK_OK);
done = (peek() == Token::RPAREN);
done = (peek() == i::Token::RPAREN);
if (!done) {
Expect(Token::COMMA, CHECK_OK);
Expect(i::Token::COMMA, CHECK_OK);
}
}
Expect(Token::RPAREN, CHECK_OK);
Expect(Token::SEMICOLON, CHECK_OK);
Expect(i::Token::RPAREN, CHECK_OK);
Expect(i::Token::SEMICOLON, CHECK_OK);
return kUnknownStatement;
}
......@@ -422,11 +423,11 @@ Statement PreParser<Scanner, Log>::ParseBlock(bool* ok) {
// Note that a Block does not introduce a new execution scope!
// (ECMA-262, 3rd, 12.2)
//
Expect(Token::LBRACE, CHECK_OK);
while (peek() != Token::RBRACE) {
Expect(i::Token::LBRACE, CHECK_OK);
while (peek() != i::Token::RBRACE) {
ParseStatement(CHECK_OK);
}
Expect(Token::RBRACE, CHECK_OK);
Expect(i::Token::RBRACE, CHECK_OK);
return kUnknownStatement;
}
......@@ -454,10 +455,10 @@ Statement PreParser<Scanner, Log>::ParseVariableDeclarations(bool accept_IN,
// VariableDeclarations ::
// ('var' | 'const') (Identifier ('=' AssignmentExpression)?)+[',']
if (peek() == Token::VAR) {
Consume(Token::VAR);
} else if (peek() == Token::CONST) {
Consume(Token::CONST);
if (peek() == i::Token::VAR) {
Consume(i::Token::VAR);
} else if (peek() == i::Token::CONST) {
Consume(i::Token::CONST);
} else {
*ok = false;
return 0;
......@@ -468,14 +469,14 @@ Statement PreParser<Scanner, Log>::ParseVariableDeclarations(bool accept_IN,
int nvars = 0; // the number of variables declared
do {
// Parse variable name.
if (nvars > 0) Consume(Token::COMMA);
if (nvars > 0) Consume(i::Token::COMMA);
ParseIdentifier(CHECK_OK);
nvars++;
if (peek() == Token::ASSIGN) {
Expect(Token::ASSIGN, CHECK_OK);
if (peek() == i::Token::ASSIGN) {
Expect(i::Token::ASSIGN, CHECK_OK);
ParseAssignmentExpression(accept_IN, CHECK_OK);
}
} while (peek() == Token::COMMA);
} while (peek() == i::Token::COMMA);
if (num_decl != NULL) *num_decl = nvars;
return kUnknownStatement;
......@@ -490,8 +491,8 @@ Statement PreParser<Scanner, Log>::ParseExpressionOrLabelledStatement(
// Identifier ':' Statement
Expression expr = ParseExpression(true, CHECK_OK);
if (peek() == Token::COLON && expr == kIdentifierExpression) {
Consume(Token::COLON);
if (peek() == i::Token::COLON && expr == kIdentifierExpression) {
Consume(i::Token::COLON);
return ParseStatement(ok);
}
// Parsed expression statement.
......@@ -505,12 +506,12 @@ Statement PreParser<Scanner, Log>::ParseIfStatement(bool* ok) {
// IfStatement ::
// 'if' '(' Expression ')' Statement ('else' Statement)?
Expect(Token::IF, CHECK_OK);
Expect(Token::LPAREN, CHECK_OK);
Expect(i::Token::IF, CHECK_OK);
Expect(i::Token::LPAREN, CHECK_OK);
ParseExpression(true, CHECK_OK);
Expect(Token::RPAREN, CHECK_OK);
Expect(i::Token::RPAREN, CHECK_OK);
ParseStatement(CHECK_OK);
if (peek() == Token::ELSE) {
if (peek() == i::Token::ELSE) {
Next();
ParseStatement(CHECK_OK);
}
......@@ -523,12 +524,12 @@ Statement PreParser<Scanner, Log>::ParseContinueStatement(bool* ok) {
// ContinueStatement ::
// 'continue' [no line terminator] Identifier? ';'
Expect(Token::CONTINUE, CHECK_OK);
Token::Value tok = peek();
Expect(i::Token::CONTINUE, CHECK_OK);
i::Token::Value tok = peek();
if (!scanner_->has_line_terminator_before_next() &&
tok != Token::SEMICOLON &&
tok != Token::RBRACE &&
tok != Token::EOS) {
tok != i::Token::SEMICOLON &&
tok != i::Token::RBRACE &&
tok != i::Token::EOS) {
ParseIdentifier(CHECK_OK);
}
ExpectSemicolon(CHECK_OK);
......@@ -541,12 +542,12 @@ Statement PreParser<Scanner, Log>::ParseBreakStatement(bool* ok) {
// BreakStatement ::
// 'break' [no line terminator] Identifier? ';'
Expect(Token::BREAK, CHECK_OK);
Token::Value tok = peek();
Expect(i::Token::BREAK, CHECK_OK);
i::Token::Value tok = peek();
if (!scanner_->has_line_terminator_before_next() &&
tok != Token::SEMICOLON &&
tok != Token::RBRACE &&
tok != Token::EOS) {
tok != i::Token::SEMICOLON &&
tok != i::Token::RBRACE &&
tok != i::Token::EOS) {
ParseIdentifier(CHECK_OK);
}
ExpectSemicolon(CHECK_OK);
......@@ -562,18 +563,18 @@ Statement PreParser<Scanner, Log>::ParseReturnStatement(bool* ok) {
// Consume the return token. It is necessary to do the before
// reporting any errors on it, because of the way errors are
// reported (underlining).
Expect(Token::RETURN, CHECK_OK);
Expect(i::Token::RETURN, CHECK_OK);
// An ECMAScript program is considered syntactically incorrect if it
// contains a return statement that is not within the body of a
// function. See ECMA-262, section 12.9, page 67.
// This is not handled during preparsing.
Token::Value tok = peek();
i::Token::Value tok = peek();
if (!scanner_->has_line_terminator_before_next() &&
tok != Token::SEMICOLON &&
tok != Token::RBRACE &&
tok != Token::EOS) {
tok != i::Token::SEMICOLON &&
tok != i::Token::RBRACE &&
tok != i::Token::EOS) {
ParseExpression(true, CHECK_OK);
}
ExpectSemicolon(CHECK_OK);
......@@ -585,10 +586,10 @@ template <typename Scanner, typename Log>
Statement PreParser<Scanner, Log>::ParseWithStatement(bool* ok) {
// WithStatement ::
// 'with' '(' Expression ')' Statement
Expect(Token::WITH, CHECK_OK);
Expect(Token::LPAREN, CHECK_OK);
Expect(i::Token::WITH, CHECK_OK);
Expect(i::Token::LPAREN, CHECK_OK);
ParseExpression(true, CHECK_OK);
Expect(Token::RPAREN, CHECK_OK);
Expect(i::Token::RPAREN, CHECK_OK);
scope_->EnterWith();
ParseStatement(CHECK_OK);
......@@ -602,27 +603,27 @@ Statement PreParser<Scanner, Log>::ParseSwitchStatement(bool* ok) {
// SwitchStatement ::
// 'switch' '(' Expression ')' '{' CaseClause* '}'
Expect(Token::SWITCH, CHECK_OK);
Expect(Token::LPAREN, CHECK_OK);
Expect(i::Token::SWITCH, CHECK_OK);
Expect(i::Token::LPAREN, CHECK_OK);
ParseExpression(true, CHECK_OK);
Expect(Token::RPAREN, CHECK_OK);
Expect(i::Token::RPAREN, CHECK_OK);
Expect(Token::LBRACE, CHECK_OK);
Token::Value token = peek();
while (token != Token::RBRACE) {
if (token == Token::CASE) {
Expect(Token::CASE, CHECK_OK);
Expect(i::Token::LBRACE, CHECK_OK);
i::Token::Value token = peek();
while (token != i::Token::RBRACE) {
if (token == i::Token::CASE) {
Expect(i::Token::CASE, CHECK_OK);
ParseExpression(true, CHECK_OK);
Expect(Token::COLON, CHECK_OK);
} else if (token == Token::DEFAULT) {
Expect(Token::DEFAULT, CHECK_OK);
Expect(Token::COLON, CHECK_OK);
Expect(i::Token::COLON, CHECK_OK);
} else if (token == i::Token::DEFAULT) {
Expect(i::Token::DEFAULT, CHECK_OK);
Expect(i::Token::COLON, CHECK_OK);
} else {
ParseStatement(CHECK_OK);
}
token = peek();
}
Expect(Token::RBRACE, CHECK_OK);
Expect(i::Token::RBRACE, CHECK_OK);
return kUnknownStatement;
}
......@@ -633,12 +634,12 @@ Statement PreParser<Scanner, Log>::ParseDoWhileStatement(bool* ok) {
// DoStatement ::
// 'do' Statement 'while' '(' Expression ')' ';'
Expect(Token::DO, CHECK_OK);
Expect(i::Token::DO, CHECK_OK);
ParseStatement(CHECK_OK);
Expect(Token::WHILE, CHECK_OK);
Expect(Token::LPAREN, CHECK_OK);
Expect(i::Token::WHILE, CHECK_OK);
Expect(i::Token::LPAREN, CHECK_OK);
ParseExpression(true, CHECK_OK);
Expect(Token::RPAREN, CHECK_OK);
Expect(i::Token::RPAREN, CHECK_OK);
return kUnknownStatement;
}
......@@ -648,10 +649,10 @@ Statement PreParser<Scanner, Log>::ParseWhileStatement(bool* ok) {
// WhileStatement ::
// 'while' '(' Expression ')' Statement
Expect(Token::WHILE, CHECK_OK);
Expect(Token::LPAREN, CHECK_OK);
Expect(i::Token::WHILE, CHECK_OK);
Expect(i::Token::LPAREN, CHECK_OK);
ParseExpression(true, CHECK_OK);
Expect(Token::RPAREN, CHECK_OK);
Expect(i::Token::RPAREN, CHECK_OK);
ParseStatement(CHECK_OK);
return kUnknownStatement;
}
......@@ -662,26 +663,26 @@ Statement PreParser<Scanner, Log>::ParseForStatement(bool* ok) {
// ForStatement ::
// 'for' '(' Expression? ';' Expression? ';' Expression? ')' Statement
Expect(Token::FOR, CHECK_OK);
Expect(Token::LPAREN, CHECK_OK);
if (peek() != Token::SEMICOLON) {
if (peek() == Token::VAR || peek() == Token::CONST) {
Expect(i::Token::FOR, CHECK_OK);
Expect(i::Token::LPAREN, CHECK_OK);
if (peek() != i::Token::SEMICOLON) {
if (peek() == i::Token::VAR || peek() == i::Token::CONST) {
int decl_count;
ParseVariableDeclarations(false, &decl_count, CHECK_OK);
if (peek() == Token::IN && decl_count == 1) {
Expect(Token::IN, CHECK_OK);
if (peek() == i::Token::IN && decl_count == 1) {
Expect(i::Token::IN, CHECK_OK);
ParseExpression(true, CHECK_OK);
Expect(Token::RPAREN, CHECK_OK);
Expect(i::Token::RPAREN, CHECK_OK);
ParseStatement(CHECK_OK);
return kUnknownStatement;
}
} else {
ParseExpression(false, CHECK_OK);
if (peek() == Token::IN) {
Expect(Token::IN, CHECK_OK);
if (peek() == i::Token::IN) {
Expect(i::Token::IN, CHECK_OK);
ParseExpression(true, CHECK_OK);
Expect(Token::RPAREN, CHECK_OK);
Expect(i::Token::RPAREN, CHECK_OK);
ParseStatement(CHECK_OK);
return kUnknownStatement;
......@@ -690,17 +691,17 @@ Statement PreParser<Scanner, Log>::ParseForStatement(bool* ok) {
}
// Parsed initializer at this point.
Expect(Token::SEMICOLON, CHECK_OK);
Expect(i::Token::SEMICOLON, CHECK_OK);
if (peek() != Token::SEMICOLON) {
if (peek() != i::Token::SEMICOLON) {
ParseExpression(true, CHECK_OK);
}
Expect(Token::SEMICOLON, CHECK_OK);
Expect(i::Token::SEMICOLON, CHECK_OK);
if (peek() != Token::RPAREN) {
if (peek() != i::Token::RPAREN) {
ParseExpression(true, CHECK_OK);
}
Expect(Token::RPAREN, CHECK_OK);
Expect(i::Token::RPAREN, CHECK_OK);
ParseStatement(CHECK_OK);
return kUnknownStatement;
......@@ -712,7 +713,7 @@ Statement PreParser<Scanner, Log>::ParseThrowStatement(bool* ok) {
// ThrowStatement ::
// 'throw' [no line terminator] Expression ';'
Expect(Token::THROW, CHECK_OK);
Expect(i::Token::THROW, CHECK_OK);
if (scanner_->has_line_terminator_before_next()) {
typename Scanner::Location pos = scanner_->location();
ReportMessageAt(pos.beg_pos, pos.end_pos,
......@@ -743,21 +744,21 @@ Statement PreParser<Scanner, Log>::ParseTryStatement(bool* ok) {
// In preparsing, allow any number of catch/finally blocks, including zero
// of both.
Expect(Token::TRY, CHECK_OK);
Expect(i::Token::TRY, CHECK_OK);
ParseBlock(CHECK_OK);
bool catch_or_finally_seen = false;
if (peek() == Token::CATCH) {
Expect(Token::CATCH, CHECK_OK);
Expect(Token::LPAREN, CHECK_OK);
if (peek() == i::Token::CATCH) {
Expect(i::Token::CATCH, CHECK_OK);
Expect(i::Token::LPAREN, CHECK_OK);
ParseIdentifier(CHECK_OK);
Expect(Token::RPAREN, CHECK_OK);
Expect(i::Token::RPAREN, CHECK_OK);
ParseBlock(CHECK_OK);
catch_or_finally_seen = true;
}
if (peek() == Token::FINALLY) {
Expect(Token::FINALLY, CHECK_OK);
if (peek() == i::Token::FINALLY) {
Expect(i::Token::FINALLY, CHECK_OK);
ParseBlock(CHECK_OK);
catch_or_finally_seen = true;
}
......@@ -776,7 +777,7 @@ Statement PreParser<Scanner, Log>::ParseDebuggerStatement(bool* ok) {
// DebuggerStatement ::
// 'debugger' ';'
Expect(Token::DEBUGGER, CHECK_OK);
Expect(i::Token::DEBUGGER, CHECK_OK);
ExpectSemicolon(CHECK_OK);
return kUnknownStatement;
}
......@@ -790,8 +791,8 @@ Expression PreParser<Scanner, Log>::ParseExpression(bool accept_IN, bool* ok) {
// Expression ',' AssignmentExpression
Expression result = ParseAssignmentExpression(accept_IN, CHECK_OK);
while (peek() == Token::COMMA) {
Expect(Token::COMMA, CHECK_OK);
while (peek() == i::Token::COMMA) {
Expect(i::Token::COMMA, CHECK_OK);
ParseAssignmentExpression(accept_IN, CHECK_OK);
result = kUnknownExpression;
}
......@@ -809,15 +810,15 @@ Expression PreParser<Scanner, Log>::ParseAssignmentExpression(bool accept_IN,
Expression expression = ParseConditionalExpression(accept_IN, CHECK_OK);
if (!Token::IsAssignmentOp(peek())) {
if (!i::Token::IsAssignmentOp(peek())) {
// Parsed conditional expression only (no assignment).
return expression;
}
Token::Value op = Next(); // Get assignment operator.
i::Token::Value op = Next(); // Get assignment operator.
ParseAssignmentExpression(accept_IN, CHECK_OK);
if ((op == Token::ASSIGN) && (expression == kThisPropertyExpression)) {
if ((op == i::Token::ASSIGN) && (expression == kThisPropertyExpression)) {
scope_->AddProperty();
}
......@@ -835,24 +836,24 @@ Expression PreParser<Scanner, Log>::ParseConditionalExpression(bool accept_IN,
// We start using the binary expression parser for prec >= 4 only!
Expression expression = ParseBinaryExpression(4, accept_IN, CHECK_OK);
if (peek() != Token::CONDITIONAL) return expression;
Consume(Token::CONDITIONAL);
if (peek() != i::Token::CONDITIONAL) return expression;
Consume(i::Token::CONDITIONAL);
// In parsing the first assignment expression in conditional
// expressions we always accept the 'in' keyword; see ECMA-262,
// section 11.12, page 58.
ParseAssignmentExpression(true, CHECK_OK);
Expect(Token::COLON, CHECK_OK);
Expect(i::Token::COLON, CHECK_OK);
ParseAssignmentExpression(accept_IN, CHECK_OK);
return kUnknownExpression;
}
template <typename Scanner, typename Log>
int PreParser<Scanner, Log>::Precedence(Token::Value tok, bool accept_IN) {
if (tok == Token::IN && !accept_IN)
int PreParser<Scanner, Log>::Precedence(i::Token::Value tok, bool accept_IN) {
if (tok == i::Token::IN && !accept_IN)
return 0; // 0 precedence will terminate binary expression parsing
return Token::Precedence(tok);
return i::Token::Precedence(tok);
}
......@@ -888,8 +889,8 @@ Expression PreParser<Scanner, Log>::ParseUnaryExpression(bool* ok) {
// '~' UnaryExpression
// '!' UnaryExpression
Token::Value op = peek();
if (Token::IsUnaryOp(op) || Token::IsCountOp(op)) {
i::Token::Value op = peek();
if (i::Token::IsUnaryOp(op) || i::Token::IsCountOp(op)) {
op = Next();
ParseUnaryExpression(ok);
return kUnknownExpression;
......@@ -906,7 +907,7 @@ Expression PreParser<Scanner, Log>::ParsePostfixExpression(bool* ok) {
Expression expression = ParseLeftHandSideExpression(CHECK_OK);
if (!scanner_->has_line_terminator_before_next() &&
Token::IsCountOp(peek())) {
i::Token::IsCountOp(peek())) {
Next();
return kUnknownExpression;
}
......@@ -920,7 +921,7 @@ Expression PreParser<Scanner, Log>::ParseLeftHandSideExpression(bool* ok) {
// (NewExpression | MemberExpression) ...
Expression result;
if (peek() == Token::NEW) {
if (peek() == i::Token::NEW) {
result = ParseNewExpression(CHECK_OK);
} else {
result = ParseMemberExpression(CHECK_OK);
......@@ -928,10 +929,10 @@ Expression PreParser<Scanner, Log>::ParseLeftHandSideExpression(bool* ok) {
while (true) {
switch (peek()) {
case Token::LBRACK: {
Consume(Token::LBRACK);
case i::Token::LBRACK: {
Consume(i::Token::LBRACK);
ParseExpression(true, CHECK_OK);
Expect(Token::RBRACK, CHECK_OK);
Expect(i::Token::RBRACK, CHECK_OK);
if (result == kThisExpression) {
result = kThisPropertyExpression;
} else {
......@@ -940,14 +941,14 @@ Expression PreParser<Scanner, Log>::ParseLeftHandSideExpression(bool* ok) {
break;
}
case Token::LPAREN: {
case i::Token::LPAREN: {
ParseArguments(CHECK_OK);
result = kUnknownExpression;
break;
}
case Token::PERIOD: {
Consume(Token::PERIOD);
case i::Token::PERIOD: {
Consume(i::Token::PERIOD);
ParseIdentifierName(CHECK_OK);
if (result == kThisExpression) {
result = kThisPropertyExpression;
......@@ -979,9 +980,9 @@ Expression PreParser<Scanner, Log>::ParseNewExpression(bool* ok) {
// lists as long as it has 'new' prefixes left
unsigned new_count = 0;
do {
Consume(Token::NEW);
Consume(i::Token::NEW);
new_count++;
} while (peek() == Token::NEW);
} while (peek() == i::Token::NEW);
return ParseMemberWithNewPrefixesExpression(new_count, ok);
}
......@@ -1002,9 +1003,9 @@ Expression PreParser<Scanner, Log>::ParseMemberWithNewPrefixesExpression(
// Parse the initial primary or function expression.
Expression result = NULL;
if (peek() == Token::FUNCTION) {
Consume(Token::FUNCTION);
if (peek() == Token::IDENTIFIER) {
if (peek() == i::Token::FUNCTION) {
Consume(i::Token::FUNCTION);
if (peek() == i::Token::IDENTIFIER) {
ParseIdentifier(CHECK_OK);
}
result = ParseFunctionLiteral(CHECK_OK);
......@@ -1014,10 +1015,10 @@ Expression PreParser<Scanner, Log>::ParseMemberWithNewPrefixesExpression(
while (true) {
switch (peek()) {
case Token::LBRACK: {
Consume(Token::LBRACK);
case i::Token::LBRACK: {
Consume(i::Token::LBRACK);
ParseExpression(true, CHECK_OK);
Expect(Token::RBRACK, CHECK_OK);
Expect(i::Token::RBRACK, CHECK_OK);
if (result == kThisExpression) {
result = kThisPropertyExpression;
} else {
......@@ -1025,8 +1026,8 @@ Expression PreParser<Scanner, Log>::ParseMemberWithNewPrefixesExpression(
}
break;
}
case Token::PERIOD: {
Consume(Token::PERIOD);
case i::Token::PERIOD: {
Consume(i::Token::PERIOD);
ParseIdentifierName(CHECK_OK);
if (result == kThisExpression) {
result = kThisPropertyExpression;
......@@ -1035,7 +1036,7 @@ Expression PreParser<Scanner, Log>::ParseMemberWithNewPrefixesExpression(
}
break;
}
case Token::LPAREN: {
case i::Token::LPAREN: {
if (new_count == 0) return result;
// Consume one of the new prefixes (already parsed).
ParseArguments(CHECK_OK);
......@@ -1067,55 +1068,55 @@ Expression PreParser<Scanner, Log>::ParsePrimaryExpression(bool* ok) {
Expression result = kUnknownExpression;
switch (peek()) {
case Token::THIS: {
case i::Token::THIS: {
Next();
result = kThisExpression;
break;
}
case Token::IDENTIFIER: {
case i::Token::IDENTIFIER: {
ParseIdentifier(CHECK_OK);
result = kIdentifierExpression;
break;
}
case Token::NULL_LITERAL:
case Token::TRUE_LITERAL:
case Token::FALSE_LITERAL:
case Token::NUMBER: {
case i::Token::NULL_LITERAL:
case i::Token::TRUE_LITERAL:
case i::Token::FALSE_LITERAL:
case i::Token::NUMBER: {
Next();
break;
}
case Token::STRING: {
case i::Token::STRING: {
Next();
result = GetStringSymbol();
break;
}
case Token::ASSIGN_DIV:
case i::Token::ASSIGN_DIV:
result = ParseRegExpLiteral(true, CHECK_OK);
break;
case Token::DIV:
case i::Token::DIV:
result = ParseRegExpLiteral(false, CHECK_OK);
break;
case Token::LBRACK:
case i::Token::LBRACK:
result = ParseArrayLiteral(CHECK_OK);
break;
case Token::LBRACE:
case i::Token::LBRACE:
result = ParseObjectLiteral(CHECK_OK);
break;
case Token::LPAREN:
Consume(Token::LPAREN);
case i::Token::LPAREN:
Consume(i::Token::LPAREN);
result = ParseExpression(true, CHECK_OK);
Expect(Token::RPAREN, CHECK_OK);
Expect(i::Token::RPAREN, CHECK_OK);
if (result == kIdentifierExpression) result = kUnknownExpression;
break;
case Token::MOD:
case i::Token::MOD:
result = ParseV8Intrinsic(CHECK_OK);
break;
......@@ -1134,16 +1135,16 @@ template <typename Scanner, typename Log>
Expression PreParser<Scanner, Log>::ParseArrayLiteral(bool* ok) {
// ArrayLiteral ::
// '[' Expression? (',' Expression?)* ']'
Expect(Token::LBRACK, CHECK_OK);
while (peek() != Token::RBRACK) {
if (peek() != Token::COMMA) {
Expect(i::Token::LBRACK, CHECK_OK);
while (peek() != i::Token::RBRACK) {
if (peek() != i::Token::COMMA) {
ParseAssignmentExpression(true, CHECK_OK);
}
if (peek() != Token::RBRACK) {
Expect(Token::COMMA, CHECK_OK);
if (peek() != i::Token::RBRACK) {
Expect(i::Token::COMMA, CHECK_OK);
}
}
Expect(Token::RBRACK, CHECK_OK);
Expect(i::Token::RBRACK, CHECK_OK);
scope_->NextMaterializedLiteralIndex();
return kUnknownExpression;
......@@ -1158,40 +1159,40 @@ Expression PreParser<Scanner, Log>::ParseObjectLiteral(bool* ok) {
// | (('get' | 'set') (IdentifierName | String | Number) FunctionLiteral)
// )*[','] '}'
Expect(Token::LBRACE, CHECK_OK);
while (peek() != Token::RBRACE) {
Token::Value next = peek();
Expect(i::Token::LBRACE, CHECK_OK);
while (peek() != i::Token::RBRACE) {
i::Token::Value next = peek();
switch (next) {
case Token::IDENTIFIER: {
case i::Token::IDENTIFIER: {
bool is_getter = false;
bool is_setter = false;
ParseIdentifierOrGetOrSet(&is_getter, &is_setter, CHECK_OK);
if ((is_getter || is_setter) && peek() != Token::COLON) {
Token::Value name = Next();
if (name != Token::IDENTIFIER &&
name != Token::NUMBER &&
name != Token::STRING &&
!Token::IsKeyword(name)) {
if ((is_getter || is_setter) && peek() != i::Token::COLON) {
i::Token::Value name = Next();
if (name != i::Token::IDENTIFIER &&
name != i::Token::NUMBER &&
name != i::Token::STRING &&
!i::Token::IsKeyword(name)) {
*ok = false;
return kUnknownExpression;
}
ParseFunctionLiteral(CHECK_OK);
if (peek() != Token::RBRACE) {
Expect(Token::COMMA, CHECK_OK);
if (peek() != i::Token::RBRACE) {
Expect(i::Token::COMMA, CHECK_OK);
}
continue; // restart the while
}
break;
}
case Token::STRING:
case i::Token::STRING:
Consume(next);
GetStringSymbol();
break;
case Token::NUMBER:
case i::Token::NUMBER:
Consume(next);
break;
default:
if (Token::IsKeyword(next)) {
if (i::Token::IsKeyword(next)) {
Consume(next);
} else {
// Unexpected token.
......@@ -1200,13 +1201,13 @@ Expression PreParser<Scanner, Log>::ParseObjectLiteral(bool* ok) {
}
}
Expect(Token::COLON, CHECK_OK);
Expect(i::Token::COLON, CHECK_OK);
ParseAssignmentExpression(true, CHECK_OK);
// TODO(1240767): Consider allowing trailing comma.
if (peek() != Token::RBRACE) Expect(Token::COMMA, CHECK_OK);
if (peek() != i::Token::RBRACE) Expect(i::Token::COMMA, CHECK_OK);
}
Expect(Token::RBRACE, CHECK_OK);
Expect(i::Token::RBRACE, CHECK_OK);
scope_->NextMaterializedLiteralIndex();
return kUnknownExpression;
......@@ -1245,16 +1246,16 @@ Arguments PreParser<Scanner, Log>::ParseArguments(bool* ok) {
// Arguments ::
// '(' (AssignmentExpression)*[','] ')'
Expect(Token::LPAREN, CHECK_OK);
bool done = (peek() == Token::RPAREN);
Expect(i::Token::LPAREN, CHECK_OK);
bool done = (peek() == i::Token::RPAREN);
int argc = 0;
while (!done) {
ParseAssignmentExpression(true, CHECK_OK);
argc++;
done = (peek() == Token::RPAREN);
if (!done) Expect(Token::COMMA, CHECK_OK);
done = (peek() == i::Token::RPAREN);
if (!done) Expect(i::Token::COMMA, CHECK_OK);
}
Expect(Token::RPAREN, CHECK_OK);
Expect(i::Token::RPAREN, CHECK_OK);
return argc;
}
......@@ -1271,18 +1272,18 @@ Expression PreParser<Scanner, Log>::ParseFunctionLiteral(bool* ok) {
// FormalParameterList ::
// '(' (Identifier)*[','] ')'
Expect(Token::LPAREN, CHECK_OK);
bool done = (peek() == Token::RPAREN);
Expect(i::Token::LPAREN, CHECK_OK);
bool done = (peek() == i::Token::RPAREN);
while (!done) {
ParseIdentifier(CHECK_OK);
done = (peek() == Token::RPAREN);
done = (peek() == i::Token::RPAREN);
if (!done) {
Expect(Token::COMMA, CHECK_OK);
Expect(i::Token::COMMA, CHECK_OK);
}
}
Expect(Token::RPAREN, CHECK_OK);
Expect(i::Token::RPAREN, CHECK_OK);
Expect(Token::LBRACE, CHECK_OK);
Expect(i::Token::LBRACE, CHECK_OK);
int function_block_pos = scanner_->location().beg_pos;
// Determine if the function will be lazily compiled.
......@@ -1293,19 +1294,19 @@ Expression PreParser<Scanner, Log>::ParseFunctionLiteral(bool* ok) {
if (is_lazily_compiled) {
log_->PauseRecording();
ParseSourceElements(Token::RBRACE, ok);
ParseSourceElements(i::Token::RBRACE, ok);
log_->ResumeRecording();
if (!*ok) return kUnknownExpression;
Expect(Token::RBRACE, CHECK_OK);
Expect(i::Token::RBRACE, CHECK_OK);
int end_pos = scanner_->location().end_pos;
log_->LogFunction(function_block_pos, end_pos,
function_scope.materialized_literal_count(),
function_scope.expected_properties());
} else {
ParseSourceElements(Token::RBRACE, CHECK_OK);
Expect(Token::RBRACE, CHECK_OK);
ParseSourceElements(i::Token::RBRACE, CHECK_OK);
Expect(i::Token::RBRACE, CHECK_OK);
}
return kUnknownExpression;
}
......@@ -1316,7 +1317,7 @@ Expression PreParser<Scanner, Log>::ParseV8Intrinsic(bool* ok) {
// CallRuntime ::
// '%' Identifier Arguments
Expect(Token::MOD, CHECK_OK);
Expect(i::Token::MOD, CHECK_OK);
ParseIdentifier(CHECK_OK);
ParseArguments(CHECK_OK);
......@@ -1328,17 +1329,17 @@ template <typename Scanner, typename Log>
void PreParser<Scanner, Log>::ExpectSemicolon(bool* ok) {
// Check for automatic semicolon insertion according to
// the rules given in ECMA-262, section 7.9, page 21.
Token::Value tok = peek();
if (tok == Token::SEMICOLON) {
i::Token::Value tok = peek();
if (tok == i::Token::SEMICOLON) {
Next();
return;
}
if (scanner_->has_line_terminator_before_next() ||
tok == Token::RBRACE ||
tok == Token::EOS) {
tok == i::Token::RBRACE ||
tok == i::Token::EOS) {
return;
}
Expect(Token::SEMICOLON, ok);
Expect(i::Token::SEMICOLON, ok);
}
......@@ -1368,21 +1369,21 @@ Expression PreParser<Scanner, Log>::GetStringSymbol() {
template <typename Scanner, typename Log>
Identifier PreParser<Scanner, Log>::ParseIdentifier(bool* ok) {
Expect(Token::IDENTIFIER, ok);
Expect(i::Token::IDENTIFIER, ok);
return GetIdentifierSymbol();
}
template <typename Scanner, typename Log>
Identifier PreParser<Scanner, Log>::ParseIdentifierName(bool* ok) {
Token::Value next = Next();
if (Token::IsKeyword(next)) {
i::Token::Value next = Next();
if (i::Token::IsKeyword(next)) {
int pos = scanner_->location().beg_pos;
const char* keyword = Token::String(next);
const char* keyword = i::Token::String(next);
log_->LogSymbol(pos, keyword, strlen(keyword));
return kUnknownExpression;
}
if (next == Token::IDENTIFIER) {
if (next == i::Token::IDENTIFIER) {
return GetIdentifierSymbol();
}
*ok = false;
......@@ -1398,7 +1399,7 @@ template <typename Scanner, typename Log>
Identifier PreParser<Scanner, Log>::ParseIdentifierOrGetOrSet(bool* is_get,
bool* is_set,
bool* ok) {
Expect(Token::IDENTIFIER, CHECK_OK);
Expect(i::Token::IDENTIFIER, CHECK_OK);
if (scanner_->literal_length() == 3) {
const char* token = scanner_->literal_string();
*is_get = strncmp(token, "get", 3) == 0;
......@@ -1408,6 +1409,6 @@ Identifier PreParser<Scanner, Log>::ParseIdentifierOrGetOrSet(bool* is_get,
}
#undef CHECK_OK
} } } // v8::internal::preparser
} } // v8::preparser
#endif // V8_PREPARSER_H
src/scanner-base.cc 0 → 100644
View file @ f386f974
// Copyright 2010 the V8 project authors. All rights reserved.
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following
// disclaimer in the documentation and/or other materials provided
// with the distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Features shared by parsing and pre-parsing scanners.
#include "scanner-base.h"
namespace v8 {
namespace internal {
// ----------------------------------------------------------------------------
// Keyword Matcher
KeywordMatcher::FirstState KeywordMatcher::first_states_[] = {
{ "break", KEYWORD_PREFIX, Token::BREAK },
{ NULL, C, Token::ILLEGAL },
{ NULL, D, Token::ILLEGAL },
{ "else", KEYWORD_PREFIX, Token::ELSE },
{ NULL, F, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, I, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, N, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ "return", KEYWORD_PREFIX, Token::RETURN },
{ "switch", KEYWORD_PREFIX, Token::SWITCH },
{ NULL, T, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, V, Token::ILLEGAL },
{ NULL, W, Token::ILLEGAL }
};
void KeywordMatcher::Step(unibrow::uchar input) {
switch (state_) {
case INITIAL: {
// matching the first character is the only state with significant fanout.
// Match only lower-case letters in range 'b'..'w'.
unsigned int offset = input - kFirstCharRangeMin;
if (offset < kFirstCharRangeLength) {
state_ = first_states_[offset].state;
if (state_ == KEYWORD_PREFIX) {
keyword_ = first_states_[offset].keyword;
counter_ = 1;
keyword_token_ = first_states_[offset].token;
}
return;
}
break;
}
case KEYWORD_PREFIX:
if (static_cast<unibrow::uchar>(keyword_[counter_]) == input) {
counter_++;
if (keyword_[counter_] == '\0') {
state_ = KEYWORD_MATCHED;
token_ = keyword_token_;
}
return;
}
break;
case KEYWORD_MATCHED:
token_ = Token::IDENTIFIER;
break;
case C:
if (MatchState(input, 'a', CA)) return;
if (MatchState(input, 'o', CO)) return;
break;
case CA:
if (MatchKeywordStart(input, "case", 2, Token::CASE)) return;
if (MatchKeywordStart(input, "catch", 2, Token::CATCH)) return;
break;
case CO:
if (MatchState(input, 'n', CON)) return;
break;
case CON:
if (MatchKeywordStart(input, "const", 3, Token::CONST)) return;
if (MatchKeywordStart(input, "continue", 3, Token::CONTINUE)) return;
break;
case D:
if (MatchState(input, 'e', DE)) return;
if (MatchKeyword(input, 'o', KEYWORD_MATCHED, Token::DO)) return;
break;
case DE:
if (MatchKeywordStart(input, "debugger", 2, Token::DEBUGGER)) return;
if (MatchKeywordStart(input, "default", 2, Token::DEFAULT)) return;
if (MatchKeywordStart(input, "delete", 2, Token::DELETE)) return;
break;
case F:
if (MatchKeywordStart(input, "false", 1, Token::FALSE_LITERAL)) return;
if (MatchKeywordStart(input, "finally", 1, Token::FINALLY)) return;
if (MatchKeywordStart(input, "for", 1, Token::FOR)) return;
if (MatchKeywordStart(input, "function", 1, Token::FUNCTION)) return;
break;
case I:
if (MatchKeyword(input, 'f', KEYWORD_MATCHED, Token::IF)) return;
if (MatchKeyword(input, 'n', IN, Token::IN)) return;
break;
case IN:
token_ = Token::IDENTIFIER;
if (MatchKeywordStart(input, "instanceof", 2, Token::INSTANCEOF)) {
return;
}
break;
case N:
if (MatchKeywordStart(input, "native", 1, Token::NATIVE)) return;
if (MatchKeywordStart(input, "new", 1, Token::NEW)) return;
if (MatchKeywordStart(input, "null", 1, Token::NULL_LITERAL)) return;
break;
case T:
if (MatchState(input, 'h', TH)) return;
if (MatchState(input, 'r', TR)) return;
if (MatchKeywordStart(input, "typeof", 1, Token::TYPEOF)) return;
break;
case TH:
if (MatchKeywordStart(input, "this", 2, Token::THIS)) return;
if (MatchKeywordStart(input, "throw", 2, Token::THROW)) return;
break;
case TR:
if (MatchKeywordStart(input, "true", 2, Token::TRUE_LITERAL)) return;
if (MatchKeyword(input, 'y', KEYWORD_MATCHED, Token::TRY)) return;
break;
case V:
if (MatchKeywordStart(input, "var", 1, Token::VAR)) return;
if (MatchKeywordStart(input, "void", 1, Token::VOID)) return;
break;
case W:
if (MatchKeywordStart(input, "while", 1, Token::WHILE)) return;
if (MatchKeywordStart(input, "with", 1, Token::WITH)) return;
break;
case UNMATCHABLE:
break;
}
// On fallthrough, it's a failure.
state_ = UNMATCHABLE;
}
} } // namespace v8::internal
src/scanner-base.h 0 → 100644
View file @ f386f974
// Copyright 2010 the V8 project authors. All rights reserved.
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following
// disclaimer in the documentation and/or other materials provided
// with the distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived
// from this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Features shared by parsing and pre-parsing scanners.
#ifndef V8_SCANNER_BASE_H_
#define V8_SCANNER_BASE_H_
#include "token.h"
#include "unicode.h"
namespace v8 {
namespace internal {
class KeywordMatcher {
// Incrementally recognize keywords.
//
// Recognized keywords:
// break case catch const* continue debugger* default delete do else
// finally false for function if in instanceof native* new null
// return switch this throw true try typeof var void while with
//
// *: Actually "future reserved keywords". These are the only ones we
// recognized, the remaining are allowed as identifiers.
public:
KeywordMatcher()
: state_(INITIAL),
token_(Token::IDENTIFIER),
keyword_(NULL),
counter_(0),
keyword_token_(Token::ILLEGAL) {}
Token::Value token() { return token_; }
inline void AddChar(unibrow::uchar input) {
if (state_ != UNMATCHABLE) {
Step(input);
}
}
void Fail() {
token_ = Token::IDENTIFIER;
state_ = UNMATCHABLE;
}
private:
enum State {
UNMATCHABLE,
INITIAL,
KEYWORD_PREFIX,
KEYWORD_MATCHED,
C,
CA,
CO,
CON,
D,
DE,
F,
I,
IN,
N,
T,
TH,
TR,
V,
W
};
struct FirstState {
const char* keyword;
State state;
Token::Value token;
};
// Range of possible first characters of a keyword.
static const unsigned int kFirstCharRangeMin = 'b';
static const unsigned int kFirstCharRangeMax = 'w';
static const unsigned int kFirstCharRangeLength =
kFirstCharRangeMax - kFirstCharRangeMin + 1;
// State map for first keyword character range.
static FirstState first_states_[kFirstCharRangeLength];
// If input equals keyword's character at position, continue matching keyword
// from that position.
inline bool MatchKeywordStart(unibrow::uchar input,
const char* keyword,
int position,
Token::Value token_if_match) {
if (input == static_cast<unibrow::uchar>(keyword[position])) {
state_ = KEYWORD_PREFIX;
this->keyword_ = keyword;
this->counter_ = position + 1;
this->keyword_token_ = token_if_match;
return true;
}
return false;
}
// If input equals match character, transition to new state and return true.
inline bool MatchState(unibrow::uchar input, char match, State new_state) {
if (input == static_cast<unibrow::uchar>(match)) {
state_ = new_state;
return true;
}
return false;
}
inline bool MatchKeyword(unibrow::uchar input,
char match,
State new_state,
Token::Value keyword_token) {
if (input != static_cast<unibrow::uchar>(match)) {
return false;
}
state_ = new_state;
token_ = keyword_token;
return true;
}
void Step(unibrow::uchar input);
// Current state.
State state_;
// Token for currently added characters.
Token::Value token_;
// Matching a specific keyword string (there is only one possible valid
// keyword with the current prefix).
const char* keyword_;
int counter_;
Token::Value keyword_token_;
};
} } // namespace v8::internal
#endif // V8_SCANNER_BASE_H_
src/scanner.cc
View file @ f386f974
......@@ -184,142 +184,6 @@ void ExternalStringUTF16Buffer<StringType, CharType>::SeekForward(int pos) {
pos_ = pos;
}
// ----------------------------------------------------------------------------
// Keyword Matcher
KeywordMatcher::FirstState KeywordMatcher::first_states_[] = {
{ "break", KEYWORD_PREFIX, Token::BREAK },
{ NULL, C, Token::ILLEGAL },
{ NULL, D, Token::ILLEGAL },
{ "else", KEYWORD_PREFIX, Token::ELSE },
{ NULL, F, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, I, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, N, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ "return", KEYWORD_PREFIX, Token::RETURN },
{ "switch", KEYWORD_PREFIX, Token::SWITCH },
{ NULL, T, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, V, Token::ILLEGAL },
{ NULL, W, Token::ILLEGAL }
};
void KeywordMatcher::Step(uc32 input) {
switch (state_) {
case INITIAL: {
// matching the first character is the only state with significant fanout.
// Match only lower-case letters in range 'b'..'w'.
unsigned int offset = input - kFirstCharRangeMin;
if (offset < kFirstCharRangeLength) {
state_ = first_states_[offset].state;
if (state_ == KEYWORD_PREFIX) {
keyword_ = first_states_[offset].keyword;
counter_ = 1;
keyword_token_ = first_states_[offset].token;
}
return;
}
break;
}
case KEYWORD_PREFIX:
if (keyword_[counter_] == input) {
ASSERT_NE(input, '\0');
counter_++;
if (keyword_[counter_] == '\0') {
state_ = KEYWORD_MATCHED;
token_ = keyword_token_;
}
return;
}
break;
case KEYWORD_MATCHED:
token_ = Token::IDENTIFIER;
break;
case C:
if (MatchState(input, 'a', CA)) return;
if (MatchState(input, 'o', CO)) return;
break;
case CA:
if (MatchKeywordStart(input, "case", 2, Token::CASE)) return;
if (MatchKeywordStart(input, "catch", 2, Token::CATCH)) return;
break;
case CO:
if (MatchState(input, 'n', CON)) return;
break;
case CON:
if (MatchKeywordStart(input, "const", 3, Token::CONST)) return;
if (MatchKeywordStart(input, "continue", 3, Token::CONTINUE)) return;
break;
case D:
if (MatchState(input, 'e', DE)) return;
if (MatchKeyword(input, 'o', KEYWORD_MATCHED, Token::DO)) return;
break;
case DE:
if (MatchKeywordStart(input, "debugger", 2, Token::DEBUGGER)) return;
if (MatchKeywordStart(input, "default", 2, Token::DEFAULT)) return;
if (MatchKeywordStart(input, "delete", 2, Token::DELETE)) return;
break;
case F:
if (MatchKeywordStart(input, "false", 1, Token::FALSE_LITERAL)) return;
if (MatchKeywordStart(input, "finally", 1, Token::FINALLY)) return;
if (MatchKeywordStart(input, "for", 1, Token::FOR)) return;
if (MatchKeywordStart(input, "function", 1, Token::FUNCTION)) return;
break;
case I:
if (MatchKeyword(input, 'f', KEYWORD_MATCHED, Token::IF)) return;
if (MatchKeyword(input, 'n', IN, Token::IN)) return;
break;
case IN:
token_ = Token::IDENTIFIER;
if (MatchKeywordStart(input, "instanceof", 2, Token::INSTANCEOF)) {
return;
}
break;
case N:
if (MatchKeywordStart(input, "native", 1, Token::NATIVE)) return;
if (MatchKeywordStart(input, "new", 1, Token::NEW)) return;
if (MatchKeywordStart(input, "null", 1, Token::NULL_LITERAL)) return;
break;
case T:
if (MatchState(input, 'h', TH)) return;
if (MatchState(input, 'r', TR)) return;
if (MatchKeywordStart(input, "typeof", 1, Token::TYPEOF)) return;
break;
case TH:
if (MatchKeywordStart(input, "this", 2, Token::THIS)) return;
if (MatchKeywordStart(input, "throw", 2, Token::THROW)) return;
break;
case TR:
if (MatchKeywordStart(input, "true", 2, Token::TRUE_LITERAL)) return;
if (MatchKeyword(input, 'y', KEYWORD_MATCHED, Token::TRY)) return;
break;
case V:
if (MatchKeywordStart(input, "var", 1, Token::VAR)) return;
if (MatchKeywordStart(input, "void", 1, Token::VOID)) return;
break;
case W:
if (MatchKeywordStart(input, "while", 1, Token::WHILE)) return;
if (MatchKeywordStart(input, "with", 1, Token::WITH)) return;
break;
default:
UNREACHABLE();
}
// On fallthrough, it's a failure.
state_ = UNMATCHABLE;
}
// ----------------------------------------------------------------------------
// Scanner::LiteralScope
......
src/scanner.h
View file @ f386f974
......@@ -30,6 +30,7 @@
#include "token.h"
#include "char-predicates-inl.h"
#include "scanner-base.h"
namespace v8 {
namespace internal {
......@@ -142,126 +143,6 @@ class ExternalStringUTF16Buffer: public UTF16Buffer {
};
class KeywordMatcher {
// Incrementally recognize keywords.
//
// Recognized keywords:
// break case catch const* continue debugger* default delete do else
// finally false for function if in instanceof native* new null
// return switch this throw true try typeof var void while with
//
// *: Actually "future reserved keywords". These are the only ones we
// recognized, the remaining are allowed as identifiers.
public:
KeywordMatcher()
: state_(INITIAL),
token_(Token::IDENTIFIER),
keyword_(NULL),
counter_(0),
keyword_token_(Token::ILLEGAL) {}
Token::Value token() { return token_; }
inline void AddChar(uc32 input) {
if (state_ != UNMATCHABLE) {
Step(input);
}
}
void Fail() {
token_ = Token::IDENTIFIER;
state_ = UNMATCHABLE;
}
private:
enum State {
UNMATCHABLE,
INITIAL,
KEYWORD_PREFIX,
KEYWORD_MATCHED,
C,
CA,
CO,
CON,
D,
DE,
F,
I,
IN,
N,
T,
TH,
TR,
V,
W
};
struct FirstState {
const char* keyword;
State state;
Token::Value token;
};
// Range of possible first characters of a keyword.
static const unsigned int kFirstCharRangeMin = 'b';
static const unsigned int kFirstCharRangeMax = 'w';
static const unsigned int kFirstCharRangeLength =
kFirstCharRangeMax - kFirstCharRangeMin + 1;
// State map for first keyword character range.
static FirstState first_states_[kFirstCharRangeLength];
// If input equals keyword's character at position, continue matching keyword
// from that position.
inline bool MatchKeywordStart(uc32 input,
const char* keyword,
int position,
Token::Value token_if_match) {
if (input == keyword[position]) {
state_ = KEYWORD_PREFIX;
this->keyword_ = keyword;
this->counter_ = position + 1;
this->keyword_token_ = token_if_match;
return true;
}
return false;
}
// If input equals match character, transition to new state and return true.
inline bool MatchState(uc32 input, char match, State new_state) {
if (input == match) {
state_ = new_state;
return true;
}
return false;
}
inline bool MatchKeyword(uc32 input,
char match,
State new_state,
Token::Value keyword_token) {
if (input != match) {
return false;
}
state_ = new_state;
token_ = keyword_token;
return true;
}
void Step(uc32 input);
// Current state.
State state_;
// Token for currently added characters.
Token::Value token_;
// Matching a specific keyword string (there is only one possible valid
// keyword with the current prefix).
const char* keyword_;
int counter_;
Token::Value keyword_token_;
};
enum ParserLanguage { JAVASCRIPT, JSON };
......
src/token.h
View file @ f386f974
......@@ -28,6 +28,8 @@
#ifndef V8_TOKEN_H_
#define V8_TOKEN_H_
#include "checks.h"
namespace v8 {
namespace internal {
......
test/cctest/test-parsing.cc
View file @ f386f974
......@@ -263,7 +263,7 @@ TEST(StandAlonePreParser) {
i::CompleteParserRecorder log;
i::Scanner scanner;
scanner.Initialize(i::Handle<i::String>::null(), &stream, i::JAVASCRIPT);
i::preparser::PreParser<i::Scanner, i::CompleteParserRecorder> preparser;
v8::preparser::PreParser<i::Scanner, i::CompleteParserRecorder> preparser;
bool result = preparser.PreParseProgram(&scanner, &log, true);
CHECK(result);
i::ScriptDataImpl data(log.ExtractData());
......
tools/gyp/v8.gyp
View file @ f386f974
......@@ -426,6 +426,8 @@
'../../src/rewriter.h',
'../../src/runtime.cc',
'../../src/runtime.h',
'../../src/scanner-base.cc',
'../../src/scanner-base.h',
'../../src/scanner.cc',
'../../src/scanner.h',
'../../src/scopeinfo.cc',
......
tools/v8.xcodeproj/project.pbxproj
View file @ f386f974
......@@ -115,6 +115,7 @@
89A88E180E71A6960043BA31 /* property.cc in Sources */ = {isa = PBXBuildFile; fileRef = 897FF16D0E719B8F00D62E90 /* property.cc */; };
89A88E190E71A6970043BA31 /* rewriter.cc in Sources */ = {isa = PBXBuildFile; fileRef = 897FF16F0E719B8F00D62E90 /* rewriter.cc */; };
89A88E1A0E71A69B0043BA31 /* runtime.cc in Sources */ = {isa = PBXBuildFile; fileRef = 897FF1710E719B8F00D62E90 /* runtime.cc */; };
89A88E1B0E71A69D0043BA31 /* scanner-base.cc in Sources */ = {isa = PBXBuildFile; fileRef = 897FF1730E719B8F00D62E90 /* scanner-base.cc */; };
89A88E1B0E71A69D0043BA31 /* scanner.cc in Sources */ = {isa = PBXBuildFile; fileRef = 897FF1730E719B8F00D62E90 /* scanner.cc */; };
89A88E1C0E71A69E0043BA31 /* scopeinfo.cc in Sources */ = {isa = PBXBuildFile; fileRef = 897FF1760E719B8F00D62E90 /* scopeinfo.cc */; };
89A88E1D0E71A6A00043BA31 /* scopes.cc in Sources */ = {isa = PBXBuildFile; fileRef = 897FF1780E719B8F00D62E90 /* scopes.cc */; };
......@@ -177,6 +178,7 @@
89F23C6C0E78D5B2006B2466 /* property.cc in Sources */ = {isa = PBXBuildFile; fileRef = 897FF16D0E719B8F00D62E90 /* property.cc */; };
89F23C6D0E78D5B2006B2466 /* rewriter.cc in Sources */ = {isa = PBXBuildFile; fileRef = 897FF16F0E719B8F00D62E90 /* rewriter.cc */; };
89F23C6E0E78D5B2006B2466 /* runtime.cc in Sources */ = {isa = PBXBuildFile; fileRef = 897FF1710E719B8F00D62E90 /* runtime.cc */; };
89F23C6F0E78D5B2006B2466 /* scanner-base.cc in Sources */ = {isa = PBXBuildFile; fileRef = 897FF1730E719B8F00D62E90 /* scanner-base.cc */; };
89F23C6F0E78D5B2006B2466 /* scanner.cc in Sources */ = {isa = PBXBuildFile; fileRef = 897FF1730E719B8F00D62E90 /* scanner.cc */; };
89F23C700E78D5B2006B2466 /* scopeinfo.cc in Sources */ = {isa = PBXBuildFile; fileRef = 897FF1760E719B8F00D62E90 /* scopeinfo.cc */; };
89F23C710E78D5B2006B2466 /* scopes.cc in Sources */ = {isa = PBXBuildFile; fileRef = 897FF1780E719B8F00D62E90 /* scopes.cc */; };
......@@ -481,6 +483,8 @@
897FF1700E719B8F00D62E90 /* rewriter.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = rewriter.h; sourceTree = "<group>"; };
897FF1710E719B8F00D62E90 /* runtime.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = runtime.cc; sourceTree = "<group>"; };
897FF1720E719B8F00D62E90 /* runtime.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = runtime.h; sourceTree = "<group>"; };
897FF1730E719B8F00D62E90 /* scanner-base.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = scanner-base.cc; sourceTree = "<group>"; };
897FF1740E719B8F00D62E90 /* scanner-base.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = scanner-base.h; sourceTree = "<group>"; };
897FF1730E719B8F00D62E90 /* scanner.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = scanner.cc; sourceTree = "<group>"; };
897FF1740E719B8F00D62E90 /* scanner.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = scanner.h; sourceTree = "<group>"; };
897FF1750E719B8F00D62E90 /* SConscript */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = SConscript; sourceTree = "<group>"; };
......@@ -943,6 +947,8 @@
897FF1700E719B8F00D62E90 /* rewriter.h */,
897FF1710E719B8F00D62E90 /* runtime.cc */,
897FF1720E719B8F00D62E90 /* runtime.h */,
897FF1730E719B8F00D62E90 /* scanner-base.cc */,
897FF1740E719B8F00D62E90 /* scanner-base.h */,
897FF1730E719B8F00D62E90 /* scanner.cc */,
897FF1740E719B8F00D62E90 /* scanner.h */,
897FF1760E719B8F00D62E90 /* scopeinfo.cc */,
......@@ -1348,6 +1354,7 @@
58950D630F5551AF00F3E8BA /* register-allocator.cc in Sources */,
89A88E190E71A6970043BA31 /* rewriter.cc in Sources */,
89A88E1A0E71A69B0043BA31 /* runtime.cc in Sources */,
89A88E1B0E71A69D0043BA31 /* scanner-base.cc in Sources */,
89A88E1B0E71A69D0043BA31 /* scanner.cc in Sources */,
89A88E1C0E71A69E0043BA31 /* scopeinfo.cc in Sources */,
89A88E1D0E71A6A00043BA31 /* scopes.cc in Sources */,
......@@ -1472,6 +1479,7 @@
58950D640F5551B500F3E8BA /* register-allocator.cc in Sources */,
89F23C6D0E78D5B2006B2466 /* rewriter.cc in Sources */,
89F23C6E0E78D5B2006B2466 /* runtime.cc in Sources */,
89F23C6F0E78D5B2006B2466 /* scanner-base.cc in Sources */,
89F23C6F0E78D5B2006B2466 /* scanner.cc in Sources */,
89F23C700E78D5B2006B2466 /* scopeinfo.cc in Sources */,
89F23C710E78D5B2006B2466 /* scopes.cc in Sources */,
......
tools/visual_studio/v8_base.vcproj
View file @ f386f974
......@@ -881,6 +881,14 @@
RelativePath="..\..\src\runtime.h"
>
</File>
<File
RelativePath="..\..\src\scanner-base.cc"
>
</File>
<File
RelativePath="..\..\src\scanner-base.h"
>
</File>
<File
RelativePath="..\..\src\scanner.cc"
>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment