[scanner] Push surrogate pair handling down into identifier scanning

Most scanner logic doesn't need to care about surrogate pairs, so we can just
push it down to identifier scanning.

This CL additionally drops some explicit kEndOfInput checks that are subsumed
by predicates seemlessly returning false for kEndOfInput (-1).

Change-Id: If031a9355ab5fbca0c3b647045e3034f42923979
Reviewed-on: https://chromium-review.googlesource.com/1124447Reviewed-by: Marja Hölttä <marja@chromium.org>
Commit-Queue: Toon Verwaest <verwaest@chromium.org>
Cr-Commit-Position: refs/heads/master@{#54172}

src/parsing/scanner.cc

@@ -438,8 +438,8 @@ Token::Value Scanner::SkipWhiteSpace() {
 
   while (true) {
     while (true) {
-      // Don't skip behind the end of input.
-      if (c0_ == kEndOfInput) break;
+      // We won't skip behind the end of input.
+      DCHECK(!unicode_cache_->IsWhiteSpace(kEndOfInput));
 
       // Advance as long as character is a WhiteSpace or LineTerminator.
       // Remember if the latter is the case.
@@ -521,9 +521,11 @@ Token::Value Scanner::SkipSourceURLComment() {
 void Scanner::TryToParseSourceURLComment() {
   // Magic comments are of the form: //[#@]\s<name>=\s*<value>\s*.* and this
   // function will just return if it cannot parse a magic comment.
-  if (c0_ == kEndOfInput || !unicode_cache_->IsWhiteSpace(c0_)) return;
+  DCHECK(!unicode_cache_->IsWhiteSpaceOrLineTerminator(kEndOfInput));
+  if (!unicode_cache_->IsWhiteSpace(c0_)) return;
   Advance();
   LiteralBuffer name;
+
   while (c0_ != kEndOfInput &&
          !unicode_cache_->IsWhiteSpaceOrLineTerminator(c0_) && c0_ != '=') {
     name.AddChar(c0_);
@@ -543,7 +545,7 @@ void Scanner::TryToParseSourceURLComment() {
     return;
   Advance();
   value->Reset();
-  while (c0_ != kEndOfInput && unicode_cache_->IsWhiteSpace(c0_)) {
+  while (unicode_cache_->IsWhiteSpace(c0_)) {
     Advance();
   }
   while (c0_ != kEndOfInput && !unibrow::IsLineTerminator(c0_)) {
@@ -576,7 +578,8 @@ Token::Value Scanner::SkipMultiLineComment() {
   while (c0_ != kEndOfInput) {
     uc32 ch = c0_;
     Advance();
-    if (c0_ != kEndOfInput && unibrow::IsLineTerminator(ch)) {
+    DCHECK(!unibrow::IsLineTerminator(kEndOfInput));
+    if (unibrow::IsLineTerminator(ch)) {
       // Following ECMA-262, section 7.4, a comment containing
       // a newline will make the comment count as a line-terminator.
       has_multiline_comment_before_next_ = true;
@@ -866,17 +869,17 @@ void Scanner::Scan() {
         break;
 
       default:
-        if (c0_ == kEndOfInput) {
-          token = Token::EOS;
-        } else if (unicode_cache_->IsIdentifierStart(c0_)) {
+        if (unicode_cache_->IsIdentifierStart(c0_) ||
+            (CombineSurrogatePair() &&
+             unicode_cache_->IsIdentifierStart(c0_))) {
           token = ScanIdentifierOrKeyword();
         } else if (IsDecimalDigit(c0_)) {
           token = ScanNumber(false);
+        } else if (c0_ == kEndOfInput) {
+          token = Token::EOS;
         } else {
           token = SkipWhiteSpace();
-          if (token == Token::ILLEGAL) {
-            Advance();
-          }
+          if (token == Token::ILLEGAL) Advance();
         }
         break;
     }
@@ -978,8 +981,8 @@ bool Scanner::ScanEscape() {
   Advance<capture_raw>();
 
   // Skip escaped newlines.
-  if (!in_template_literal && c0_ != kEndOfInput &&
-      unibrow::IsLineTerminator(c)) {
+  DCHECK(!unibrow::IsLineTerminator(kEndOfInput));
+  if (!in_template_literal && unibrow::IsLineTerminator(c)) {
     // Allow escaped CR+LF newlines in multiline string literals.
     if (IsCarriageReturn(c) && IsLineFeed(c0_)) Advance<capture_raw>();
     return true;
@@ -1053,43 +1056,28 @@ uc32 Scanner::ScanOctalEscape(uc32 c, int length, bool in_template_literal) {
 
 Token::Value Scanner::ScanString() {
   uc32 quote = c0_;
-  Advance<false, false>();  // consume quote
+  Advance();  // consume quote
 
   LiteralScope literal(this);
   while (true) {
-    if (c0_ > kMaxAscii) {
-      HandleLeadSurrogate();
-      break;
-    }
-    if (c0_ == kEndOfInput || c0_ == '\n' || c0_ == '\r') return Token::ILLEGAL;
     if (c0_ == quote) {
       literal.Complete();
-      Advance<false, false>();
+      Advance();
       return Token::STRING;
     }
-    char c = static_cast<char>(c0_);
-    if (c == '\\') break;
-    Advance<false, false>();
-    AddLiteralChar(c);
+    if (c0_ == kEndOfInput || unibrow::IsStringLiteralLineTerminator(c0_)) {
+      return Token::ILLEGAL;
     }
-
-  while (c0_ != quote && c0_ != kEndOfInput &&
-         !unibrow::IsStringLiteralLineTerminator(c0_)) {
-    uc32 c = c0_;
+    if (c0_ == '\\') {
       Advance();
-    if (c == '\\') {
+      // TODO(verwaest): Check whether we can remove the additional check.
       if (c0_ == kEndOfInput || !ScanEscape<false, false>()) {
         return Token::ILLEGAL;
       }
-    } else {
-      AddLiteralChar(c);
+      continue;
     }
+    AddLiteralCharAdvance();
   }
-  if (c0_ != quote) return Token::ILLEGAL;
-  literal.Complete();
-
-  Advance();  // consume quote
-  return Token::STRING;
 }
 
 Token::Value Scanner::ScanPrivateName() {
@@ -1102,7 +1090,8 @@ Token::Value Scanner::ScanPrivateName() {
   LiteralScope literal(this);
   DCHECK_EQ(c0_, '#');
   AddLiteralCharAdvance();
-  if (c0_ == kEndOfInput || !unicode_cache_->IsIdentifierStart(c0_)) {
+  DCHECK(!unicode_cache_->IsIdentifierStart(kEndOfInput));
+  if (!unicode_cache_->IsIdentifierStart(c0_)) {
     PushBack(c0_);
     ReportScannerError(source_pos(),
                        MessageTemplate::kInvalidOrUnexpectedToken);
@@ -1150,7 +1139,8 @@ Token::Value Scanner::ScanTemplateSpan() {
       ReduceRawLiteralLength(2);
       break;
     } else if (c == '\\') {
-      if (c0_ != kEndOfInput && unibrow::IsLineTerminator(c0_)) {
+      DCHECK(!unibrow::IsLineTerminator(kEndOfInput));
+      if (unibrow::IsLineTerminator(c0_)) {
         // The TV of LineContinuation :: \ LineTerminatorSequence is the empty
         // code unit sequence.
         uc32 lastChar = c0_;
@@ -1230,7 +1220,7 @@ bool Scanner::ScanDigitsWithNumericSeparators(bool (*predicate)(uc32 ch),
   bool separator_seen = false;
   while (predicate(c0_) || c0_ == '_') {
     if (c0_ == '_') {
-      Advance<false, false>();
+      Advance();
       if (c0_ == '_') {
         ReportScannerError(Location(source_pos(), source_pos() + 1),
                            MessageTemplate::kContinuousNumericSeparator);
@@ -1266,7 +1256,7 @@ bool Scanner::ScanDecimalAsSmiWithNumericSeparators(uint64_t* value) {
   bool separator_seen = false;
   while (IsDecimalDigit(c0_) || c0_ == '_') {
     if (c0_ == '_') {
-      Advance<false, false>();
+      Advance();
       if (c0_ == '_') {
         ReportScannerError(Location(source_pos(), source_pos() + 1),
                            MessageTemplate::kContinuousNumericSeparator);
@@ -1278,7 +1268,7 @@ bool Scanner::ScanDecimalAsSmiWithNumericSeparators(uint64_t* value) {
     separator_seen = false;
     *value = 10 * *value + (c0_ - '0');
     uc32 first_char = c0_;
-    Advance<false, false>();
+    Advance();
     AddLiteralChar(first_char);
   }
 
@@ -1299,7 +1289,7 @@ bool Scanner::ScanDecimalAsSmi(uint64_t* value) {
   while (IsDecimalDigit(c0_)) {
     *value = 10 * *value + (c0_ - '0');
     uc32 first_char = c0_;
-    Advance<false, false>();
+    Advance();
     AddLiteralChar(first_char);
   }
   return true;
@@ -1444,10 +1434,9 @@ Token::Value Scanner::ScanNumber(bool seen_period) {
 
         if (next_.literal_chars->one_byte_literal().length() <= 10 &&
             value <= Smi::kMaxValue && c0_ != '.' &&
-            (c0_ == kEndOfInput || !unicode_cache_->IsIdentifierStart(c0_))) {
+            !unicode_cache_->IsIdentifierStart(c0_)) {
           next_.smi_value_ = static_cast<uint32_t>(value);
           literal.Complete();
-          HandleLeadSurrogate();
 
           if (kind == DECIMAL_WITH_LEADING_ZERO) {
             octal_pos_ = Location(start_pos, source_pos());
@@ -1455,7 +1444,6 @@ Token::Value Scanner::ScanNumber(bool seen_period) {
           }
           return Token::SMI;
         }
-        HandleLeadSurrogate();
       }
 
       if (!ScanDecimalDigits()) return Token::ILLEGAL;
@@ -1503,9 +1491,9 @@ Token::Value Scanner::ScanNumber(bool seen_period) {
   // not be an identifier start or a decimal digit; see ECMA-262
   // section 7.8.3, page 17 (note that we read only one decimal digit
   // if the value is 0).
-  if (IsDecimalDigit(c0_) ||
-      (c0_ != kEndOfInput && unicode_cache_->IsIdentifierStart(c0_)))
+  if (IsDecimalDigit(c0_) || unicode_cache_->IsIdentifierStart(c0_)) {
     return Token::ILLEGAL;
+  }
 
   literal.Complete();
 
@@ -1688,24 +1676,18 @@ Token::Value Scanner::ScanIdentifierOrKeyword() {
 
 Token::Value Scanner::ScanIdentifierOrKeywordInner(LiteralScope* literal) {
   DCHECK(unicode_cache_->IsIdentifierStart(c0_));
+  bool escaped = false;
   if (IsInRange(c0_, 'a', 'z') || c0_ == '_') {
     do {
-      char first_char = static_cast<char>(c0_);
-      Advance<false, false>();
-      AddLiteralChar(first_char);
+      AddLiteralCharAdvance();
     } while (IsInRange(c0_, 'a', 'z') || c0_ == '_');
 
-    if (IsDecimalDigit(c0_) || IsInRange(c0_, 'A', 'Z') || c0_ == '_' ||
-        c0_ == '$') {
-      // Identifier starting with lowercase.
-      char first_char = static_cast<char>(c0_);
-      Advance<false, false>();
-      AddLiteralChar(first_char);
-      while (IsAsciiIdentifier(c0_)) {
-        char first_char = static_cast<char>(c0_);
-        Advance<false, false>();
-        AddLiteralChar(first_char);
-      }
+    if (IsDecimalDigit(c0_) || IsInRange(c0_, 'A', 'Z') || c0_ == '$') {
+      // Identifier starting with lowercase or _.
+      do {
+        AddLiteralCharAdvance();
+      } while (IsAsciiIdentifier(c0_));
+
       if (c0_ <= kMaxAscii && c0_ != '\\') {
         literal->Complete();
         return Token::IDENTIFIER;
@@ -1721,100 +1703,67 @@ Token::Value Scanner::ScanIdentifierOrKeywordInner(LiteralScope* literal) {
         literal->Complete();
       return token;
     }
-
-    HandleLeadSurrogate();
-  } else if (IsInRange(c0_, 'A', 'Z') || c0_ == '_' || c0_ == '$') {
+  } else if (IsInRange(c0_, 'A', 'Z') || c0_ == '$') {
     do {
-      char first_char = static_cast<char>(c0_);
-      Advance<false, false>();
-      AddLiteralChar(first_char);
+      AddLiteralCharAdvance();
     } while (IsAsciiIdentifier(c0_));
 
     if (c0_ <= kMaxAscii && c0_ != '\\') {
       literal->Complete();
       return Token::IDENTIFIER;
     }
-
-    HandleLeadSurrogate();
   } else if (c0_ == '\\') {
-    // Scan identifier start character.
+    escaped = true;
     uc32 c = ScanIdentifierUnicodeEscape();
-    // Only allow legal identifier start characters.
-    if (c < 0 ||
-        c == '\\' ||  // No recursive escapes.
-        !unicode_cache_->IsIdentifierStart(c)) {
+    DCHECK(!unicode_cache_->IsIdentifierStart(-1));
+    if (c == '\\' || !unicode_cache_->IsIdentifierStart(c)) {
       return Token::ILLEGAL;
     }
     AddLiteralChar(c);
-    return ScanIdentifierSuffix(literal, true);
-  } else {
-    uc32 first_char = c0_;
-    Advance();
-    AddLiteralChar(first_char);
   }
 
-  // Scan the rest of the identifier characters.
-  while (c0_ != kEndOfInput && unicode_cache_->IsIdentifierPart(c0_)) {
-    if (c0_ != '\\') {
-      uc32 next_char = c0_;
-      Advance();
-      AddLiteralChar(next_char);
-      continue;
-    }
-    // Fallthrough if no longer able to complete keyword.
-    return ScanIdentifierSuffix(literal, false);
-  }
-
-  if (next_.literal_chars->is_one_byte()) {
-    Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal();
-    Token::Value token =
-        KeywordOrIdentifierToken(chars.start(), chars.length());
-    if (token == Token::IDENTIFIER ||
-        token == Token::FUTURE_STRICT_RESERVED_WORD ||
-        Token::IsContextualKeyword(token))
-      literal->Complete();
-    return token;
-  }
-  literal->Complete();
-  return Token::IDENTIFIER;
-}
-
-
-Token::Value Scanner::ScanIdentifierSuffix(LiteralScope* literal,
-                                           bool escaped) {
-  // Scan the rest of the identifier characters.
-  while (c0_ != kEndOfInput && unicode_cache_->IsIdentifierPart(c0_)) {
+  while (true) {
     if (c0_ == '\\') {
-      uc32 c = ScanIdentifierUnicodeEscape();
       escaped = true;
+      uc32 c = ScanIdentifierUnicodeEscape();
       // Only allow legal identifier part characters.
-      if (c < 0 ||
-          c == '\\' ||
-          !unicode_cache_->IsIdentifierPart(c)) {
+      // TODO(verwaest): Make this true.
+      // DCHECK(!unicode_cache_->IsIdentifierPart('\\'));
+      DCHECK(!unicode_cache_->IsIdentifierPart(-1));
+      if (c == '\\' || !unicode_cache_->IsIdentifierPart(c)) {
         return Token::ILLEGAL;
       }
       AddLiteralChar(c);
+    } else if (unicode_cache_->IsIdentifierPart(c0_) ||
+               (CombineSurrogatePair() &&
+                unicode_cache_->IsIdentifierPart(c0_))) {
+      AddLiteralCharAdvance();
     } else {
-      AddLiteralChar(c0_);
-      Advance();
+      break;
     }
   }
-  literal->Complete();
 
-  if (escaped && next_.literal_chars->is_one_byte()) {
+  if (next_.literal_chars->is_one_byte()) {
     Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal();
     Token::Value token =
         KeywordOrIdentifierToken(chars.start(), chars.length());
     /* TODO(adamk): YIELD should be handled specially. */
     if (token == Token::IDENTIFIER || Token::IsContextualKeyword(token)) {
+      literal->Complete();
       return token;
-    } else if (token == Token::FUTURE_STRICT_RESERVED_WORD ||
-               token == Token::LET || token == Token::STATIC) {
+    }
+
+    if (!escaped) return token;
+
+    literal->Complete();
+    if (token == Token::FUTURE_STRICT_RESERVED_WORD || token == Token::LET ||
+        token == Token::STATIC) {
       return Token::ESCAPED_STRICT_RESERVED_WORD;
-    } else {
-      return Token::ESCAPED_KEYWORD;
     }
+    return Token::ESCAPED_KEYWORD;
   }
+
+  literal->Complete();
   return Token::IDENTIFIER;
 }
 
@@ -1879,7 +1828,7 @@ Maybe<RegExp::Flags> Scanner::ScanRegExpFlags() {
 
   // Scan regular expression flags.
   int flags = 0;
-  while (c0_ != kEndOfInput && unicode_cache_->IsIdentifierPart(c0_)) {
+  while (unicode_cache_->IsIdentifierPart(c0_)) {
     RegExp::Flags flag = RegExp::kNone;
     switch (c0_) {
       case 'g':

src/parsing/scanner.h

@@ -608,26 +608,26 @@ class Scanner {
   }
 
   // Low-level scanning support.
-  template <bool capture_raw = false, bool check_surrogate = true>
+  template <bool capture_raw = false>
   void Advance() {
     if (capture_raw) {
       AddRawLiteralChar(c0_);
     }
     c0_ = source_->Advance();
-    if (check_surrogate) HandleLeadSurrogate();
   }
 
-  void HandleLeadSurrogate() {
+  bool CombineSurrogatePair() {
     DCHECK(!unibrow::Utf16::IsLeadSurrogate(kEndOfInput));
     if (unibrow::Utf16::IsLeadSurrogate(c0_)) {
       uc32 c1 = source_->Advance();
       DCHECK(!unibrow::Utf16::IsTrailSurrogate(kEndOfInput));
-      if (!unibrow::Utf16::IsTrailSurrogate(c1)) {
-        source_->Back();
-      } else {
+      if (unibrow::Utf16::IsTrailSurrogate(c1)) {
         c0_ = unibrow::Utf16::CombineSurrogatePair(c0_, c1);
+        return true;
       }
+      source_->Back();
     }
+    return false;
   }
 
   void PushBack(uc32 ch) {
@@ -752,7 +752,6 @@ class Scanner {
   Token::Value ScanNumber(bool seen_period);
   Token::Value ScanIdentifierOrKeyword();
   Token::Value ScanIdentifierOrKeywordInner(LiteralScope* literal);
-  Token::Value ScanIdentifierSuffix(LiteralScope* literal, bool escaped);
 
   Token::Value ScanString();
   Token::Value ScanPrivateName();

src/unicode.h

@@ -39,8 +39,12 @@ class Predicate {
     inline CacheEntry()
         : bit_field_(CodePointField::encode(0) | ValueField::encode(0)) {}
     inline CacheEntry(uchar code_point, bool value)
-        : bit_field_(CodePointField::encode(code_point) |
-                     ValueField::encode(value)) {}
+        : bit_field_(
+              CodePointField::encode(CodePointField::kMask & code_point) |
+              ValueField::encode(value)) {
+      DCHECK_IMPLIES((CodePointField::kMask & code_point) != code_point,
+                     code_point == static_cast<uchar>(-1));
+    }
 
     uchar code_point() const { return CodePointField::decode(bit_field_); }
     bool value() const { return ValueField::decode(bit_field_); }