Skip to content

V
V8
Commit df91f6b3 authored by vitalyr@chromium.org's avatar vitalyr@chromium.org
Browse files
Options

Simpler (and a bit faster) keyword matcher. 

Replaced the keyword matching state machine with a switch on the first char followed up by inlined char comparisons.

R=lrn@chromium.org
TEST=cctest/test-parsing/ScanKeywords

Review URL: http://codereview.chromium.org/7558017

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@8866 ce2b1a6d-e550-0410-aec6-3dcde31c8c00
parent 7adb10a4
Hide whitespace changes
Inline Side-by-side
Showing with 163 additions and 414 deletions
src/scanner-base.cc
View file @ df91f6b3
......@@ -94,7 +94,7 @@ void JavaScriptScanner::Initialize(UC16CharacterStream* source) {
STATIC_ASSERT(Token::NUM_TOKENS <= 0x100);
// Table of one-character tokens, by character (0x00..0x7f only).
static byte one_char_tokens[] = {
static const byte one_char_tokens[] = {
Token::ILLEGAL,
Token::ILLEGAL,
Token::ILLEGAL,
......@@ -810,10 +810,114 @@ uc32 JavaScriptScanner::ScanIdentifierUnicodeEscape() {
}
// ----------------------------------------------------------------------------
// Keyword Matcher
#define KEYWORDS(KEYWORD_GROUP, KEYWORD) \
KEYWORD_GROUP('b') \
KEYWORD("break", BREAK) \
KEYWORD_GROUP('c') \
KEYWORD("case", CASE) \
KEYWORD("catch", CATCH) \
KEYWORD("class", FUTURE_RESERVED_WORD) \
KEYWORD("const", CONST) \
KEYWORD("continue", CONTINUE) \
KEYWORD_GROUP('d') \
KEYWORD("debugger", DEBUGGER) \
KEYWORD("default", DEFAULT) \
KEYWORD("delete", DELETE) \
KEYWORD("do", DO) \
KEYWORD_GROUP('e') \
KEYWORD("else", ELSE) \
KEYWORD("enum", FUTURE_RESERVED_WORD) \
KEYWORD("export", FUTURE_RESERVED_WORD) \
KEYWORD("extends", FUTURE_RESERVED_WORD) \
KEYWORD_GROUP('f') \
KEYWORD("false", FALSE_LITERAL) \
KEYWORD("finally", FINALLY) \
KEYWORD("for", FOR) \
KEYWORD("function", FUNCTION) \
KEYWORD_GROUP('i') \
KEYWORD("if", IF) \
KEYWORD("implements", FUTURE_STRICT_RESERVED_WORD) \
KEYWORD("import", FUTURE_RESERVED_WORD) \
KEYWORD("in", IN) \
KEYWORD("instanceof", INSTANCEOF) \
KEYWORD("interface", FUTURE_STRICT_RESERVED_WORD) \
KEYWORD_GROUP('l') \
KEYWORD("let", FUTURE_STRICT_RESERVED_WORD) \
KEYWORD_GROUP('n') \
KEYWORD("new", NEW) \
KEYWORD("null", NULL_LITERAL) \
KEYWORD_GROUP('p') \
KEYWORD("package", FUTURE_STRICT_RESERVED_WORD) \
KEYWORD("private", FUTURE_STRICT_RESERVED_WORD) \
KEYWORD("protected", FUTURE_STRICT_RESERVED_WORD) \
KEYWORD("public", FUTURE_STRICT_RESERVED_WORD) \
KEYWORD_GROUP('r') \
KEYWORD("return", RETURN) \
KEYWORD_GROUP('s') \
KEYWORD("static", FUTURE_STRICT_RESERVED_WORD) \
KEYWORD("super", FUTURE_RESERVED_WORD) \
KEYWORD("switch", SWITCH) \
KEYWORD_GROUP('t') \
KEYWORD("this", THIS) \
KEYWORD("throw", THROW) \
KEYWORD("true", TRUE_LITERAL) \
KEYWORD("try", TRY) \
KEYWORD("typeof", TYPEOF) \
KEYWORD_GROUP('v') \
KEYWORD("var", VAR) \
KEYWORD("void", VOID) \
KEYWORD_GROUP('w') \
KEYWORD("while", WHILE) \
KEYWORD("with", WITH) \
KEYWORD_GROUP('y') \
KEYWORD("yield", FUTURE_STRICT_RESERVED_WORD)
static Token::Value KeywordOrIdentifierToken(const char* input,
int input_length) {
ASSERT(input_length >= 1);
const int kMinLength = 2;
const int kMaxLength = 10;
if (input_length < kMinLength || input_length > kMaxLength) {
return Token::IDENTIFIER;
}
switch (input[0]) {
default:
#define KEYWORD_GROUP_CASE(ch) \
break; \
case ch:
#define KEYWORD(keyword, token) \
{ \
/* 'keyword' is a char array, so sizeof(keyword) is */ \
/* strlen(keyword) plus 1 for the NUL char. */ \
const int keyword_length = sizeof(keyword) - 1; \
STATIC_ASSERT(keyword_length >= kMinLength); \
STATIC_ASSERT(keyword_length <= kMaxLength); \
if (input_length == keyword_length && \
input[1] == keyword[1] && \
(keyword_length <= 2 || input[2] == keyword[2]) && \
(keyword_length <= 3 || input[3] == keyword[3]) && \
(keyword_length <= 4 || input[4] == keyword[4]) && \
(keyword_length <= 5 || input[5] == keyword[5]) && \
(keyword_length <= 6 || input[6] == keyword[6]) && \
(keyword_length <= 7 || input[7] == keyword[7]) && \
(keyword_length <= 8 || input[8] == keyword[8]) && \
(keyword_length <= 9 || input[9] == keyword[9])) { \
return Token::token; \
} \
}
KEYWORDS(KEYWORD_GROUP_CASE, KEYWORD)
}
return Token::IDENTIFIER;
}
Token::Value JavaScriptScanner::ScanIdentifierOrKeyword() {
ASSERT(unicode_cache_->IsIdentifierStart(c0_));
LiteralScope literal(this);
KeywordMatcher keyword_match;
// Scan identifier start character.
if (c0_ == '\\') {
uc32 c = ScanIdentifierUnicodeEscape();
......@@ -826,9 +930,6 @@ Token::Value JavaScriptScanner::ScanIdentifierOrKeyword() {
uc32 first_char = c0_;
Advance();
AddLiteralChar(first_char);
if (!keyword_match.AddChar(first_char)) {
return ScanIdentifierSuffix(&literal);
}
// Scan the rest of the identifier characters.
while (unicode_cache_->IsIdentifierPart(c0_)) {
......@@ -836,14 +937,20 @@ Token::Value JavaScriptScanner::ScanIdentifierOrKeyword() {
uc32 next_char = c0_;
Advance();
AddLiteralChar(next_char);
if (keyword_match.AddChar(next_char)) continue;
continue;
}
// Fallthrough if no loner able to complete keyword.
// Fallthrough if no longer able to complete keyword.
return ScanIdentifierSuffix(&literal);
}
literal.Complete();
return keyword_match.token();
if (next_.literal_chars->is_ascii()) {
Vector<const char> chars = next_.literal_chars->ascii_literal();
return KeywordOrIdentifierToken(chars.start(), chars.length());
}
return Token::IDENTIFIER;
}
......@@ -933,182 +1040,4 @@ bool JavaScriptScanner::ScanRegExpFlags() {
return true;
}
// ----------------------------------------------------------------------------
// Keyword Matcher
const KeywordMatcher::FirstState KeywordMatcher::first_states_[] = {
{ "break", KEYWORD_PREFIX, Token::BREAK },
{ NULL, C, Token::ILLEGAL },
{ NULL, D, Token::ILLEGAL },
{ NULL, E, Token::ILLEGAL },
{ NULL, F, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, I, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ "let", KEYWORD_PREFIX, Token::FUTURE_STRICT_RESERVED_WORD },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, N, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, P, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ "return", KEYWORD_PREFIX, Token::RETURN },
{ NULL, S, Token::ILLEGAL },
{ NULL, T, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ NULL, V, Token::ILLEGAL },
{ NULL, W, Token::ILLEGAL },
{ NULL, UNMATCHABLE, Token::ILLEGAL },
{ "yield", KEYWORD_PREFIX, Token::FUTURE_STRICT_RESERVED_WORD }
};
void KeywordMatcher::Step(unibrow::uchar input) {
switch (state_) {
case INITIAL: {
// matching the first character is the only state with significant fanout.
// Match only lower-case letters in range 'b'..'y'.
unsigned int offset = input - kFirstCharRangeMin;
if (offset < kFirstCharRangeLength) {
state_ = first_states_[offset].state;
if (state_ == KEYWORD_PREFIX) {
keyword_ = first_states_[offset].keyword;
counter_ = 1;
keyword_token_ = first_states_[offset].token;
}
return;
}
break;
}
case KEYWORD_PREFIX:
if (static_cast<unibrow::uchar>(keyword_[counter_]) == input) {
counter_++;
if (keyword_[counter_] == '\0') {
state_ = KEYWORD_MATCHED;
token_ = keyword_token_;
}
return;
}
break;
case KEYWORD_MATCHED:
token_ = Token::IDENTIFIER;
break;
case C:
if (MatchState(input, 'a', CA)) return;
if (MatchKeywordStart(input, "class", 1,
Token::FUTURE_RESERVED_WORD)) return;
if (MatchState(input, 'o', CO)) return;
break;
case CA:
if (MatchKeywordStart(input, "case", 2, Token::CASE)) return;
if (MatchKeywordStart(input, "catch", 2, Token::CATCH)) return;
break;
case CO:
if (MatchState(input, 'n', CON)) return;
break;
case CON:
if (MatchKeywordStart(input, "const", 3, Token::CONST)) return;
if (MatchKeywordStart(input, "continue", 3, Token::CONTINUE)) return;
break;
case D:
if (MatchState(input, 'e', DE)) return;
if (MatchKeyword(input, 'o', KEYWORD_MATCHED, Token::DO)) return;
break;
case DE:
if (MatchKeywordStart(input, "debugger", 2, Token::DEBUGGER)) return;
if (MatchKeywordStart(input, "default", 2, Token::DEFAULT)) return;
if (MatchKeywordStart(input, "delete", 2, Token::DELETE)) return;
break;
case E:
if (MatchKeywordStart(input, "else", 1, Token::ELSE)) return;
if (MatchKeywordStart(input, "enum", 1,
Token::FUTURE_RESERVED_WORD)) return;
if (MatchState(input, 'x', EX)) return;
break;
case EX:
if (MatchKeywordStart(input, "export", 2,
Token::FUTURE_RESERVED_WORD)) return;
if (MatchKeywordStart(input, "extends", 2,
Token::FUTURE_RESERVED_WORD)) return;
break;
case F:
if (MatchKeywordStart(input, "false", 1, Token::FALSE_LITERAL)) return;
if (MatchKeywordStart(input, "finally", 1, Token::FINALLY)) return;
if (MatchKeywordStart(input, "for", 1, Token::FOR)) return;
if (MatchKeywordStart(input, "function", 1, Token::FUNCTION)) return;
break;
case I:
if (MatchKeyword(input, 'f', KEYWORD_MATCHED, Token::IF)) return;
if (MatchState(input, 'm', IM)) return;
if (MatchKeyword(input, 'n', IN, Token::IN)) return;
break;
case IM:
if (MatchState(input, 'p', IMP)) return;
break;
case IMP:
if (MatchKeywordStart(input, "implements", 3,
Token::FUTURE_STRICT_RESERVED_WORD )) return;
if (MatchKeywordStart(input, "import", 3,
Token::FUTURE_RESERVED_WORD)) return;
break;
case IN:
token_ = Token::IDENTIFIER;
if (MatchKeywordStart(input, "interface", 2,
Token::FUTURE_STRICT_RESERVED_WORD)) return;
if (MatchKeywordStart(input, "instanceof", 2, Token::INSTANCEOF)) return;
break;
case N:
if (MatchKeywordStart(input, "new", 1, Token::NEW)) return;
if (MatchKeywordStart(input, "null", 1, Token::NULL_LITERAL)) return;
break;
case P:
if (MatchKeywordStart(input, "package", 1,
Token::FUTURE_STRICT_RESERVED_WORD)) return;
if (MatchState(input, 'r', PR)) return;
if (MatchKeywordStart(input, "public", 1,
Token::FUTURE_STRICT_RESERVED_WORD)) return;
break;
case PR:
if (MatchKeywordStart(input, "private", 2,
Token::FUTURE_STRICT_RESERVED_WORD)) return;
if (MatchKeywordStart(input, "protected", 2,
Token::FUTURE_STRICT_RESERVED_WORD)) return;
break;
case S:
if (MatchKeywordStart(input, "static", 1,
Token::FUTURE_STRICT_RESERVED_WORD)) return;
if (MatchKeywordStart(input, "super", 1,
Token::FUTURE_RESERVED_WORD)) return;
if (MatchKeywordStart(input, "switch", 1,
Token::SWITCH)) return;
break;
case T:
if (MatchState(input, 'h', TH)) return;
if (MatchState(input, 'r', TR)) return;
if (MatchKeywordStart(input, "typeof", 1, Token::TYPEOF)) return;
break;
case TH:
if (MatchKeywordStart(input, "this", 2, Token::THIS)) return;
if (MatchKeywordStart(input, "throw", 2, Token::THROW)) return;
break;
case TR:
if (MatchKeywordStart(input, "true", 2, Token::TRUE_LITERAL)) return;
if (MatchKeyword(input, 'y', KEYWORD_MATCHED, Token::TRY)) return;
break;
case V:
if (MatchKeywordStart(input, "var", 1, Token::VAR)) return;
if (MatchKeywordStart(input, "void", 1, Token::VOID)) return;
break;
case W:
if (MatchKeywordStart(input, "while", 1, Token::WHILE)) return;
if (MatchKeywordStart(input, "with", 1, Token::WITH)) return;
break;
case UNMATCHABLE:
break;
}
// On fallthrough, it's a failure.
state_ = UNMATCHABLE;
}
} } // namespace v8::internal
src/scanner-base.h
View file @ df91f6b3
......@@ -542,155 +542,6 @@ class JavaScriptScanner : public Scanner {
bool has_multiline_comment_before_next_;
};
// ----------------------------------------------------------------------------
// Keyword matching state machine.
class KeywordMatcher {
// Incrementally recognize keywords.
//
// We distinguish between normal future reserved words and words that are
// considered to be future reserved words only in strict mode as required by
// ECMA-262 7.6.1.2.
//
// Recognized as keywords:
// break, case, catch, const*, continue, debugger, default, delete, do,
// else, finally, false, for, function, if, in, instanceof, new, null,
// return, switch, this, throw, true, try, typeof, var, void, while, with.
//
// Recognized as Future Reserved Keywords:
// class, enum, export, extends, import, super.
//
// Recognized as Future Reserved Keywords (strict mode only):
// implements, interface, let, package, private, protected, public,
// static, yield.
//
// *: Actually a "future reserved keyword". It's the only one we are
// recognizing outside of ES5 strict mode, the remaining are allowed
// as identifiers.
//
public:
KeywordMatcher()
: state_(INITIAL),
token_(Token::IDENTIFIER),
keyword_(NULL),
counter_(0),
keyword_token_(Token::ILLEGAL) {}
Token::Value token() { return token_; }
inline bool AddChar(unibrow::uchar input) {
if (state_ != UNMATCHABLE) {
Step(input);
}
return state_ != UNMATCHABLE;
}
void Fail() {
token_ = Token::IDENTIFIER;
state_ = UNMATCHABLE;
}
private:
enum State {
UNMATCHABLE,
INITIAL,
KEYWORD_PREFIX,
KEYWORD_MATCHED,
C,
CA,
CO,
CON,
D,
DE,
E,
EX,
F,
I,
IM,
IMP,
IN,
N,
P,
PR,
S,
T,
TH,
TR,
V,
W,
LAST_STATE = W
};
STATIC_ASSERT(LAST_STATE <= 0xFF);
STATIC_ASSERT(Token::NUM_TOKENS <= 0x100);
struct FirstState {
const char* keyword;
State state : 8;
Token::Value token : 8;
};
// Range of possible first characters of a keyword.
static const unsigned int kFirstCharRangeMin = 'b';
static const unsigned int kFirstCharRangeMax = 'y';
static const unsigned int kFirstCharRangeLength =
kFirstCharRangeMax - kFirstCharRangeMin + 1;
// State map for first keyword character range.
static const FirstState first_states_[kFirstCharRangeLength];
// If input equals keyword's character at position, continue matching keyword
// from that position.
inline bool MatchKeywordStart(unibrow::uchar input,
const char* keyword,
int position,
Token::Value token_if_match) {
if (input != static_cast<unibrow::uchar>(keyword[position])) {
return false;
}
state_ = KEYWORD_PREFIX;
this->keyword_ = keyword;
this->counter_ = position + 1;
this->keyword_token_ = token_if_match;
return true;
}
// If input equals match character, transition to new state and return true.
inline bool MatchState(unibrow::uchar input, char match, State new_state) {
if (input != static_cast<unibrow::uchar>(match)) {
return false;
}
state_ = new_state;
return true;
}
inline bool MatchKeyword(unibrow::uchar input,
char match,
State new_state,
Token::Value keyword_token) {
if (input != static_cast<unibrow::uchar>(match)) {
return false;
}
state_ = new_state;
token_ = keyword_token;
return true;
}
void Step(unibrow::uchar input);
// Current state.
State state_;
// Token for currently added characters.
Token::Value token_;
// Matching a specific keyword string (there is only one possible valid
// keyword with the current prefix).
const char* keyword_;
int counter_;
Token::Value keyword_token_;
};
} } // namespace v8::internal
#endif // V8_SCANNER_BASE_H_
src/token.cc
View file @ df91f6b3
......@@ -33,21 +33,21 @@ namespace internal {
#define T(name, string, precedence) #name,
const char* const Token::name_[NUM_TOKENS] = {
TOKEN_LIST(T, T, IGNORE_TOKEN)
TOKEN_LIST(T, T)
};
#undef T
#define T(name, string, precedence) string,
const char* const Token::string_[NUM_TOKENS] = {
TOKEN_LIST(T, T, IGNORE_TOKEN)
TOKEN_LIST(T, T)
};
#undef T
#define T(name, string, precedence) precedence,
const int8_t Token::precedence_[NUM_TOKENS] = {
TOKEN_LIST(T, T, IGNORE_TOKEN)
TOKEN_LIST(T, T)
};
#undef T
......@@ -55,7 +55,7 @@ const int8_t Token::precedence_[NUM_TOKENS] = {
#define KT(a, b, c) 'T',
#define KK(a, b, c) 'K',
const char Token::token_type[] = {
TOKEN_LIST(KT, KK, IGNORE_TOKEN)
TOKEN_LIST(KT, KK)
};
#undef KT
#undef KK
......
src/token.h
View file @ df91f6b3
......@@ -41,7 +41,6 @@ namespace internal {
//
// T: Non-keyword tokens
// K: Keyword tokens
// F: Future (reserved) keyword tokens
// IGNORE_TOKEN is a convenience macro that can be supplied as
// an argument (at any position) for a TOKEN_LIST call. It does
......@@ -49,7 +48,7 @@ namespace internal {
#define IGNORE_TOKEN(name, string, precedence)
#define TOKEN_LIST(T, K, F) \
#define TOKEN_LIST(T, K) \
/* End of source indicator. */ \
T(EOS, "EOS", 0) \
\
......@@ -182,7 +181,7 @@ class Token {
// All token values.
#define T(name, string, precedence) name,
enum Value {
TOKEN_LIST(T, T, IGNORE_TOKEN)
TOKEN_LIST(T, T)
NUM_TOKENS
};
#undef T
......
test/cctest/test-parsing.cc
View file @ df91f6b3
......@@ -42,7 +42,7 @@
namespace i = ::v8::internal;
TEST(KeywordMatcher) {
TEST(ScanKeywords) {
struct KeywordToken {
const char* keyword;
i::Token::Value token;
......@@ -50,86 +50,56 @@ TEST(KeywordMatcher) {
static const KeywordToken keywords[] = {
#define KEYWORD(t, s, d) { s, i::Token::t },
#define IGNORE(t, s, d) /* */
TOKEN_LIST(IGNORE, KEYWORD, IGNORE)
TOKEN_LIST(IGNORE_TOKEN, KEYWORD)
#undef KEYWORD
{ NULL, i::Token::IDENTIFIER }
};
static const char* future_keywords[] = {
#define FUTURE(t, s, d) s,
TOKEN_LIST(IGNORE, IGNORE, FUTURE)
#undef FUTURE
#undef IGNORE
NULL
};
KeywordToken key_token;
i::UnicodeCache unicode_cache;
i::byte buffer[32];
for (int i = 0; (key_token = keywords[i]).keyword != NULL; i++) {
i::KeywordMatcher matcher;
const char* keyword = key_token.keyword;
int length = i::StrLength(keyword);
for (int j = 0; j < length; j++) {
if (key_token.token == i::Token::INSTANCEOF && j == 2) {
// "in" is a prefix of "instanceof". It's the only keyword
// that is a prefix of another.
CHECK_EQ(i::Token::IN, matcher.token());
} else {
CHECK_EQ(i::Token::IDENTIFIER, matcher.token());
}
matcher.AddChar(keyword[j]);
const i::byte* keyword =
reinterpret_cast<const i::byte*>(key_token.keyword);
int length = i::StrLength(key_token.keyword);
CHECK(static_cast<int>(sizeof(buffer)) >= length);
{
i::Utf8ToUC16CharacterStream stream(keyword, length);
i::JavaScriptScanner scanner(&unicode_cache);
scanner.Initialize(&stream);
CHECK_EQ(key_token.token, scanner.Next());
CHECK_EQ(i::Token::EOS, scanner.Next());
}
CHECK_EQ(key_token.token, matcher.token());
// Adding more characters will make keyword matching fail.
matcher.AddChar('z');
CHECK_EQ(i::Token::IDENTIFIER, matcher.token());
// Adding a keyword later will not make it match again.
matcher.AddChar('i');
matcher.AddChar('f');
CHECK_EQ(i::Token::IDENTIFIER, matcher.token());
}
// Future keywords are not recognized.
const char* future_keyword;
for (int i = 0; (future_keyword = future_keywords[i]) != NULL; i++) {
i::KeywordMatcher matcher;
int length = i::StrLength(future_keyword);
for (int j = 0; j < length; j++) {
matcher.AddChar(future_keyword[j]);
// Removing characters will make keyword matching fail.
{
i::Utf8ToUC16CharacterStream stream(keyword, length - 1);
i::JavaScriptScanner scanner(&unicode_cache);
scanner.Initialize(&stream);
CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
CHECK_EQ(i::Token::EOS, scanner.Next());
}
// Adding characters will make keyword matching fail.
static const char chars_to_append[] = { 'z', '0', '_' };
for (int j = 0; j < static_cast<int>(ARRAY_SIZE(chars_to_append)); ++j) {
memmove(buffer, keyword, length);
buffer[length] = chars_to_append[j];
i::Utf8ToUC16CharacterStream stream(buffer, length + 1);
i::JavaScriptScanner scanner(&unicode_cache);
scanner.Initialize(&stream);
CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
CHECK_EQ(i::Token::EOS, scanner.Next());
}
// Replacing characters will make keyword matching fail.
{
memmove(buffer, keyword, length);
buffer[length - 1] = '_';
i::Utf8ToUC16CharacterStream stream(buffer, length);
i::JavaScriptScanner scanner(&unicode_cache);
scanner.Initialize(&stream);
CHECK_EQ(i::Token::IDENTIFIER, scanner.Next());
CHECK_EQ(i::Token::EOS, scanner.Next());
}
CHECK_EQ(i::Token::IDENTIFIER, matcher.token());
}
// Zero isn't ignored at first.
i::KeywordMatcher bad_start;
bad_start.AddChar(0);
CHECK_EQ(i::Token::IDENTIFIER, bad_start.token());
bad_start.AddChar('i');
bad_start.AddChar('f');
CHECK_EQ(i::Token::IDENTIFIER, bad_start.token());
// Zero isn't ignored at end.
i::KeywordMatcher bad_end;
bad_end.AddChar('i');
bad_end.AddChar('f');
CHECK_EQ(i::Token::IF, bad_end.token());
bad_end.AddChar(0);
CHECK_EQ(i::Token::IDENTIFIER, bad_end.token());
// Case isn't ignored.
i::KeywordMatcher bad_case;
bad_case.AddChar('i');
bad_case.AddChar('F');
CHECK_EQ(i::Token::IDENTIFIER, bad_case.token());
// If we mark it as failure, continuing won't help.
i::KeywordMatcher full_stop;
full_stop.AddChar('i');
CHECK_EQ(i::Token::IDENTIFIER, full_stop.token());
full_stop.Fail();
CHECK_EQ(i::Token::IDENTIFIER, full_stop.token());
full_stop.AddChar('f');
CHECK_EQ(i::Token::IDENTIFIER, full_stop.token());
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment