Sloppy-mode let parsing

This patch makes 'let' a contextual keyword in both strict and sloppy mode.
It behaves as a keyword when used at the beginning of a StatementListItem
or lexical declaration at the beginning of a for statement, if it is followed
by an identifier, [ or {. Implementing this change requires an extra token
look-ahead by the parser which is only invoked in certain cases (so as to
avoid parsing RegExps as ECMAScript tokens). This might result in a slowdown
of the scanner, but performance testing of this patch hasn't yet found much
of a regression.

BUG=v8:3305
LOG=Y
R=adamk,vogelheim

Review URL: https://codereview.chromium.org/1315673009

Cr-Commit-Position: refs/heads/master@{#30451}

src/parser.cc

@@ -1402,7 +1402,7 @@ Statement* Parser::ParseStatementListItem(bool* ok) {
     case Token::VAR:
       return ParseVariableStatement(kStatementListItem, NULL, ok);
     case Token::LET:
-      if (allow_let()) {
+      if (IsNextLetKeyword()) {
         return ParseVariableStatement(kStatementListItem, NULL, ok);
       }
       break;
@@ -2652,9 +2652,6 @@ Statement* Parser::ParseExpressionOrLabelledStatement(
       }
       break;
 
-    // TODO(arv): Handle `let [`
-    // https://code.google.com/p/v8/issues/detail?id=3847
-
     default:
       break;
   }
@@ -3564,7 +3561,7 @@ Statement* Parser::ParseForStatement(ZoneList<const AstRawString*>* labels,
   DeclarationParsingResult parsing_result;
   if (peek() != Token::SEMICOLON) {
     if (peek() == Token::VAR || (peek() == Token::CONST && allow_const()) ||
-        (peek() == Token::LET && allow_let())) {
+        (peek() == Token::LET && IsNextLetKeyword())) {
       ParseVariableDeclarations(kForStatement, &parsing_result, CHECK_OK);
       is_const = parsing_result.descriptor.mode == CONST;
 

src/preparser.cc

@@ -199,7 +199,7 @@ PreParser::Statement PreParser::ParseStatementListItem(bool* ok) {
       }
       break;
     case Token::LET:
-      if (allow_let()) {
+      if (IsNextLetKeyword()) {
         return ParseVariableStatement(kStatementListItem, ok);
       }
       break;
@@ -893,7 +893,7 @@ PreParser::Statement PreParser::ParseForStatement(bool* ok) {
   if (peek() != Token::SEMICOLON) {
     ForEachStatement::VisitMode mode;
     if (peek() == Token::VAR || (peek() == Token::CONST && allow_const()) ||
-        (peek() == Token::LET && allow_let())) {
+        (peek() == Token::LET && IsNextLetKeyword())) {
       int decl_count;
       Scanner::Location first_initializer_loc = Scanner::Location::invalid();
       Scanner::Location bindings_loc = Scanner::Location::invalid();

src/preparser.h

@@ -338,6 +338,11 @@ class ParserBase : public Traits {
     return scanner()->peek();
   }
 
+  INLINE(Token::Value PeekAhead()) {
+    if (stack_overflow_) return Token::ILLEGAL;
+    return scanner()->PeekAhead();
+  }
+
   INLINE(Token::Value Next()) {
     if (stack_overflow_) return Token::ILLEGAL;
     {
@@ -727,6 +732,8 @@ class ParserBase : public Traits {
       int param_count, FunctionLiteral::ArityRestriction arity_restriction,
       bool has_rest, int formals_start_pos, int formals_end_pos, bool* ok);
 
+  bool IsNextLetKeyword();
+
   // Checks if the expression is a valid reference expression (e.g., on the
   // left-hand side of assignments). Although ruled out by ECMA as early errors,
   // we allow calls for web compatibility and rewrite them to a runtime throw.
@@ -3796,6 +3803,27 @@ void ParserBase<Traits>::CheckArityRestrictions(
 }
 
 
+template <class Traits>
+bool ParserBase<Traits>::IsNextLetKeyword() {
+  DCHECK(peek() == Token::LET);
+  if (!allow_let()) {
+    return false;
+  }
+  Token::Value next_next = PeekAhead();
+  switch (next_next) {
+    case Token::LBRACE:
+    case Token::LBRACK:
+    case Token::IDENTIFIER:
+    case Token::STATIC:
+    case Token::LET:  // Yes, you can do let let = ... in sloppy mode
+    case Token::YIELD:
+      return true;
+    default:
+      return false;
+  }
+}
+
+
 template <class Traits>
 typename ParserBase<Traits>::ExpressionT
 ParserBase<Traits>::ParseArrowFunctionLiteral(

src/scanner.cc

@@ -237,6 +237,11 @@ Token::Value Scanner::Next() {
     next_.location.end_pos = current_.location.end_pos;
   }
   current_ = next_;
+  if (V8_UNLIKELY(next_next_.token != Token::UNINITIALIZED)) {
+    next_ = next_next_;
+    next_next_.token = Token::UNINITIALIZED;
+    return current_.token;
+  }
   has_line_terminator_before_next_ = false;
   has_multiline_comment_before_next_ = false;
   if (static_cast<unsigned>(c0_) <= 0x7f) {
@@ -255,6 +260,20 @@ Token::Value Scanner::Next() {
 }
 
 
+Token::Value Scanner::PeekAhead() {
+  if (next_next_.token != Token::UNINITIALIZED) {
+    return next_next_.token;
+  }
+  TokenDesc prev = current_;
+  Next();
+  Token::Value ret = next_.token;
+  next_next_ = next_;
+  next_ = current_;
+  current_ = prev;
+  return ret;
+}
+
+
 // TODO(yangguo): check whether this is actually necessary.
 static inline bool IsLittleEndianByteOrderMark(uc32 c) {
   // The Unicode value U+FFFE is guaranteed never to be assigned as a
@@ -1432,7 +1451,7 @@ int Scanner::FindSymbol(DuplicateFinder* finder, int value) {
 
 bool Scanner::SetBookmark() {
   if (c0_ != kNoBookmark && bookmark_c0_ == kNoBookmark &&
-      source_->SetBookmark()) {
+      next_next_.token == Token::UNINITIALIZED && source_->SetBookmark()) {
     bookmark_c0_ = c0_;
     CopyTokenDesc(&bookmark_current_, &current_);
     CopyTokenDesc(&bookmark_next_, &next_);

src/scanner.h

@@ -361,6 +361,8 @@ class Scanner {
 
   // Returns the next token and advances input.
   Token::Value Next();
+  // Returns the token following peek()
+  Token::Value PeekAhead();
   // Returns the current token again.
   Token::Value current_token() { return current_.token; }
   // Returns the location information for the current token
@@ -489,6 +491,7 @@ class Scanner {
     // Initialize current_ to not refer to a literal.
     current_.literal_chars = NULL;
     current_.raw_literal_chars = NULL;
+    next_next_.token = Token::UNINITIALIZED;
   }
 
   // Support BookmarkScope functionality.
@@ -501,16 +504,22 @@ class Scanner {
 
   // Literal buffer support
   inline void StartLiteral() {
-    LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ?
-            &literal_buffer2_ : &literal_buffer1_;
+    LiteralBuffer* free_buffer =
+        (current_.literal_chars == &literal_buffer0_)
+            ? &literal_buffer1_
+            : (current_.literal_chars == &literal_buffer1_) ? &literal_buffer2_
+                                                            : &literal_buffer0_;
     free_buffer->Reset();
     next_.literal_chars = free_buffer;
   }
 
   inline void StartRawLiteral() {
     LiteralBuffer* free_buffer =
-        (current_.raw_literal_chars == &raw_literal_buffer1_) ?
-            &raw_literal_buffer2_ : &raw_literal_buffer1_;
+        (current_.raw_literal_chars == &raw_literal_buffer0_)
+            ? &raw_literal_buffer1_
+            : (current_.raw_literal_chars == &raw_literal_buffer1_)
+                  ? &raw_literal_buffer2_
+                  : &raw_literal_buffer0_;
     free_buffer->Reset();
     next_.raw_literal_chars = free_buffer;
   }
@@ -687,6 +696,7 @@ class Scanner {
   UnicodeCache* unicode_cache_;
 
   // Buffers collecting literal strings, numbers, etc.
+  LiteralBuffer literal_buffer0_;
   LiteralBuffer literal_buffer1_;
   LiteralBuffer literal_buffer2_;
 
@@ -695,11 +705,13 @@ class Scanner {
   LiteralBuffer source_mapping_url_;
 
   // Buffer to store raw string values
+  LiteralBuffer raw_literal_buffer0_;
   LiteralBuffer raw_literal_buffer1_;
   LiteralBuffer raw_literal_buffer2_;
 
-  TokenDesc current_;  // desc for current token (as returned by Next())
-  TokenDesc next_;     // desc for next token (one token look-ahead)
+  TokenDesc current_;    // desc for current token (as returned by Next())
+  TokenDesc next_;       // desc for next token (one token look-ahead)
+  TokenDesc next_next_;  // desc for the token after next (after PeakAhead())
 
   // Variables for Scanner::BookmarkScope and the *Bookmark implementation.
   // These variables contain the scanner state when a bookmark is set.

src/token.h

@@ -166,6 +166,7 @@ namespace internal {
                                                                      \
   /* Scanner-internal use only. */                                   \
   T(WHITESPACE, NULL, 0)                                             \
+  T(UNINITIALIZED, NULL, 0)                                          \
                                                                      \
   /* ES6 Template Literals */                                        \
   T(TEMPLATE_SPAN, NULL, 0)                                          \

test/cctest/test-parsing.cc

@@ -7077,3 +7077,32 @@ TEST(LanguageModeDirectivesNonSimpleParameterListErrors) {
   RunParserSyncTest(context_data, data, kError, NULL, 0, always_flags,
                     arraysize(always_flags));
 }
+
+
+TEST(LetSloppyOnly) {
+  // clang-format off
+  const char* context_data[][2] = {
+    {"", ""},
+    {"{", "}"},
+    {NULL, NULL}
+  };
+
+  const char* data[] = {
+    "let let",
+    "let",
+    "let let = 1",
+    "let = 1",
+    "for (let let = 1; let < 1; let++) {}",
+    "for (let = 1; let < 1; let++) {}",
+    "for (let let in {}) {}",
+    "for (let let of []) {}",
+    "for (let in {}) {}",
+    NULL
+  };
+  // clang-format on
+
+  static const ParserFlag always_flags[] = {kAllowHarmonySloppy,
+                                            kAllowHarmonySloppyLet};
+  RunParserSyncTest(context_data, data, kSuccess, NULL, 0, always_flags,
+                    arraysize(always_flags));
+}

test/mjsunit/harmony/block-let-contextual-sloppy.js

+// Copyright 2015 the V8 project authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Flags: --harmony-sloppy --harmony-sloppy-let --harmony-destructuring
+
+
+{
+  assertThrows(function() { return let; }, ReferenceError);
+  let let;
+
+  let = 5;
+  assertEquals(5, let);
+
+  { let let = 1; assertEquals(1, let); }
+  assertEquals(5, let);
+}
+
+assertThrows(function() { return let; }, ReferenceError);
+
+(function() {
+   var let, sum = 0;
+   for (let in [1, 2, 3, 4]) sum += Number(let);
+   assertEquals(6, sum);
+
+   for (let let of [4, 5]) sum += let;
+   assertEquals(15, sum);
+
+   for (let let in [6]) sum += Number([6][let]);
+   assertEquals(21, sum);
+
+   for (let = 7; let < 8; let++) sum += let;
+   assertEquals(28, sum);
+   assertEquals(8, let);
+
+   for (let let = 8; let < 9; let++) sum += let;
+   assertEquals(36, sum);
+   assertEquals(8, let);
+})()
+
+assertThrows(function() { return let; }, ReferenceError);
+
+{
+  let obj = {};
+  let {let} = {let() { return obj; }};
+  let().x = 1;
+  assertEquals(1, obj.x);
+}
+
+{
+  let obj = {};
+  let [let] = [function() { return obj; }];
+  let().x = 1;
+  assertEquals(1, obj.x);
+}
+
+(function() {
+  function let() {
+    return 1;
+  }
+  assertEquals(1, let());
+})()
+
+assertThrows('for (let of []) {}', SyntaxError);