Simplify Scanner bookmarking.

- Eliminates *all* copies in the process.
- Moves (nearly) all functionality into Scanner::BookmarkScope.
- Significant code reduction.

[Needs to be rebased once crrev.com/2347883002 lands. All changes in *parser* are from that CL.]

R=marja@chromium.org
BUG=v8:4947

Review-Url: https://codereview.chromium.org/2341323002
Cr-Commit-Position: refs/heads/master@{#39554}

src/parsing/parser-base.h

@@ -3904,7 +3904,7 @@ ParserBase<Impl>::ParseArrowFunctionLiteral(
         }
 
         if (result == kLazyParsingAborted) {
-          bookmark.Reset();
+          bookmark.Apply();
           // Trigger eager (re-)parsing, just below this block.
           is_lazily_parsed = false;
 

src/parsing/parser.cc

@@ -3043,7 +3043,7 @@ FunctionLiteral* Parser::ParseFunctionLiteral(
                                     function_state.materialized_literal_count();
 
       if (result == kLazyParsingAborted) {
-        bookmark.Reset();
+        bookmark.Apply();
         // Trigger eager (re-)parsing, just below this block.
         is_lazily_parsed = false;
 

src/parsing/scanner.cc

@@ -26,21 +26,60 @@ Handle<String> Scanner::LiteralBuffer::Internalize(Isolate* isolate) const {
   return isolate->factory()->InternalizeTwoByteString(two_byte_literal());
 }
 
+// ----------------------------------------------------------------------------
+// Scanner::BookmarkScope
+
+const size_t Scanner::BookmarkScope::kBookmarkAtFirstPos =
+    std::numeric_limits<size_t>::max() - 2;
+const size_t Scanner::BookmarkScope::kNoBookmark =
+    std::numeric_limits<size_t>::max() - 1;
+const size_t Scanner::BookmarkScope::kBookmarkWasApplied =
+    std::numeric_limits<size_t>::max();
+
+void Scanner::BookmarkScope::Set() {
+  DCHECK_EQ(bookmark_, kNoBookmark);
+  DCHECK_EQ(scanner_->next_next_.token, Token::UNINITIALIZED);
+
+  // The first token is a bit special, since current_ will still be
+  // uninitialized. In this case, store kBookmarkAtFirstPos and special-case it
+  // when
+  // applying the bookmark.
+  DCHECK_IMPLIES(
+      scanner_->current_.token == Token::UNINITIALIZED,
+      scanner_->current_.location.beg_pos == scanner_->next_.location.beg_pos);
+  bookmark_ = (scanner_->current_.token == Token::UNINITIALIZED)
+                  ? kBookmarkAtFirstPos
+                  : scanner_->location().beg_pos;
+}
+
+void Scanner::BookmarkScope::Apply() {
+  DCHECK(HasBeenSet());  // Caller hasn't called SetBookmark.
+  if (bookmark_ == kBookmarkAtFirstPos) {
+    scanner_->SeekNext(0);
+  } else {
+    scanner_->SeekNext(bookmark_);
+    scanner_->Next();
+    DCHECK_EQ(scanner_->location().beg_pos, bookmark_);
+  }
+  bookmark_ = kBookmarkWasApplied;
+}
+
+bool Scanner::BookmarkScope::HasBeenSet() {
+  return bookmark_ != kNoBookmark && bookmark_ != kBookmarkWasApplied;
+}
+
+bool Scanner::BookmarkScope::HasBeenApplied() {
+  return bookmark_ == kBookmarkWasApplied;
+}
 
 // ----------------------------------------------------------------------------
 // Scanner
 
 Scanner::Scanner(UnicodeCache* unicode_cache)
     : unicode_cache_(unicode_cache),
-      bookmark_c0_(kNoBookmark),
-      bookmark_position_(0),
       octal_pos_(Location::invalid()),
       decimal_with_leading_zero_pos_(Location::invalid()),
       found_html_comment_(false) {
-  bookmark_current_.literal_chars = &bookmark_current_literal_;
-  bookmark_current_.raw_literal_chars = &bookmark_current_raw_literal_;
-  bookmark_next_.literal_chars = &bookmark_next_literal_;
-  bookmark_next_.raw_literal_chars = &bookmark_next_raw_literal_;
 }
 
 
@@ -1581,56 +1620,24 @@ int Scanner::FindSymbol(DuplicateFinder* finder, int value) {
   return finder->AddTwoByteSymbol(literal_two_byte_string(), value);
 }
 
-void Scanner::SetBookmark() {
-  DCHECK_EQ(bookmark_c0_, kNoBookmark);
-  DCHECK_EQ(next_next_.token, Token::UNINITIALIZED);
-  bookmark_c0_ = c0_;
-  bookmark_position_ = source_->pos();
-  CopyTokenDesc(&bookmark_current_, &current_);
-  CopyTokenDesc(&bookmark_next_, &next_);
-}
-
+void Scanner::SeekNext(size_t position) {
+  // Use with care: This cleanly resets most, but not all scanner state.
+  // TODO(vogelheim): Fix this, or at least DCHECK the relevant conditions.
 
-void Scanner::ResetToBookmark() {
-  DCHECK(BookmarkHasBeenSet());  // Caller hasn't called SetBookmark.
-
-  source_->Seek(bookmark_position_);
-  c0_ = bookmark_c0_;
-  CopyToNextTokenDesc(&bookmark_current_);
-  current_ = next_;
-  CopyToNextTokenDesc(&bookmark_next_);
-  bookmark_c0_ = kBookmarkWasApplied;
-}
-
-bool Scanner::BookmarkHasBeenSet() {
-  return bookmark_c0_ != kNoBookmark && bookmark_c0_ != kBookmarkWasApplied;
-}
-
-bool Scanner::BookmarkHasBeenReset() {
-  return bookmark_c0_ == kBookmarkWasApplied;
-}
-
-
-void Scanner::DropBookmark() { bookmark_c0_ = kNoBookmark; }
-
-void Scanner::CopyToNextTokenDesc(TokenDesc* from) {
-  StartLiteral();
-  StartRawLiteral();
-  CopyTokenDesc(&next_, from);
-  if (next_.literal_chars->length() == 0) next_.literal_chars = nullptr;
-  if (next_.raw_literal_chars->length() == 0) next_.raw_literal_chars = nullptr;
-}
-
-void Scanner::CopyTokenDesc(TokenDesc* to, TokenDesc* from) {
-  DCHECK_NOT_NULL(to);
-  DCHECK_NOT_NULL(from);
-  to->token = from->token;
-  to->location = from->location;
-  to->literal_chars->CopyFrom(from->literal_chars);
-  to->raw_literal_chars->CopyFrom(from->raw_literal_chars);
+  // To re-scan from a given character position, we need to:
+  // 1, Reset the current_, next_ and next_next_ tokens
+  //    (next_ + next_next_ will be overwrittem by Next(),
+  //     current_ will remain unchanged, so overwrite it fully.)
+  current_ = {{0, 0}, nullptr, nullptr, 0, Token::UNINITIALIZED};
+  next_.token = Token::UNINITIALIZED;
+  next_next_.token = Token::UNINITIALIZED;
+  // 2, reset the source to the desired position,
+  source_->Seek(position);
+  // 3, re-scan, by scanning the look-ahead char + 1 token (next_).
+  c0_ = source_->Advance();
+  Next();
+  DCHECK_EQ(next_.location.beg_pos, position);
 }
 
-
-
 }  // namespace internal
 }  // namespace v8

src/parsing/scanner.h

@@ -154,18 +154,24 @@ class Scanner {
   // Scoped helper for a re-settable bookmark.
   class BookmarkScope {
    public:
-    explicit BookmarkScope(Scanner* scanner) : scanner_(scanner) {
+    explicit BookmarkScope(Scanner* scanner)
+        : scanner_(scanner), bookmark_(kNoBookmark) {
       DCHECK_NOT_NULL(scanner_);
     }
-    ~BookmarkScope() { scanner_->DropBookmark(); }
+    ~BookmarkScope() {}
 
-    void Set() { scanner_->SetBookmark(); }
-    void Reset() { scanner_->ResetToBookmark(); }
-    bool HasBeenSet() { return scanner_->BookmarkHasBeenSet(); }
-    bool HasBeenReset() { return scanner_->BookmarkHasBeenReset(); }
+    void Set();
+    void Apply();
+    bool HasBeenSet();
+    bool HasBeenApplied();
 
    private:
+    static const size_t kNoBookmark;
+    static const size_t kBookmarkWasApplied;
+    static const size_t kBookmarkAtFirstPos;
+
     Scanner* scanner_;
+    size_t bookmark_;
 
     DISALLOW_COPY_AND_ASSIGN(BookmarkScope);
   };
@@ -418,23 +424,6 @@ class Scanner {
 
     Handle<String> Internalize(Isolate* isolate) const;
 
-    void CopyFrom(const LiteralBuffer* other) {
-      if (other == nullptr) {
-        Reset();
-      } else {
-        is_one_byte_ = other->is_one_byte_;
-        position_ = other->position_;
-        if (position_ < backing_store_.length()) {
-          std::copy(other->backing_store_.begin(),
-                    other->backing_store_.begin() + position_,
-                    backing_store_.begin());
-        } else {
-          backing_store_.Dispose();
-          backing_store_ = other->backing_store_.Clone();
-        }
-      }
-    }
-
    private:
     static const int kInitialCapacity = 16;
     static const int kGrowthFactory = 4;
@@ -528,15 +517,6 @@ class Scanner {
     scanner_error_ = MessageTemplate::kNone;
   }
 
-  // Support BookmarkScope functionality.
-  void SetBookmark();
-  void ResetToBookmark();
-  bool BookmarkHasBeenSet();
-  bool BookmarkHasBeenReset();
-  void DropBookmark();
-  void CopyToNextTokenDesc(TokenDesc* from);
-  static void CopyTokenDesc(TokenDesc* to, TokenDesc* from);
-
   void ReportScannerError(const Location& location,
                           MessageTemplate::Template error) {
     if (has_error()) return;
@@ -550,6 +530,9 @@ class Scanner {
     scanner_error_location_ = Location(pos, pos + 1);
   }
 
+  // Seek to the next_ token at the given position.
+  void SeekNext(size_t position);
+
   // Literal buffer support
   inline void StartLiteral() {
     LiteralBuffer* free_buffer =
@@ -789,38 +772,6 @@ class Scanner {
   TokenDesc next_;       // desc for next token (one token look-ahead)
   TokenDesc next_next_;  // desc for the token after next (after PeakAhead())
 
-  // Variables for Scanner::BookmarkScope and the *Bookmark implementation.
-  // These variables contain the scanner state when a bookmark is set.
-  //
-  // We will use bookmark_c0_ as a 'control' variable, where:
-  // - bookmark_c0_ >= 0: A bookmark has been set and this contains c0_.
-  // - bookmark_c0_ == -1: No bookmark has been set.
-  // - bookmark_c0_ == -2: The bookmark has been applied (ResetToBookmark).
-  //
-  // Which state is being bookmarked? The parser state is distributed over
-  // several variables, roughly like this:
-  //   ...    1234        +       5678 ..... [character stream]
-  //       [current_] [next_] c0_ |      [scanner state]
-  // So when the scanner is logically at the beginning of an expression
-  // like "1234 + 4567", then:
-  // - current_ contains "1234"
-  // - next_ contains "+"
-  // - c0_ contains ' ' (the space between "+" and "5678",
-  // - the source_ character stream points to the beginning of "5678".
-  // To be able to restore this state, we will keep copies of current_, next_,
-  // and c0_; we'll ask the stream to bookmark itself, and we'll copy the
-  // contents of current_'s and next_'s literal buffers to bookmark_*_literal_.
-  static const uc32 kNoBookmark = -2;
-  static const uc32 kBookmarkWasApplied = -3;
-  uc32 bookmark_c0_;
-  size_t bookmark_position_;
-  TokenDesc bookmark_current_;
-  TokenDesc bookmark_next_;
-  LiteralBuffer bookmark_current_literal_;
-  LiteralBuffer bookmark_current_raw_literal_;
-  LiteralBuffer bookmark_next_literal_;
-  LiteralBuffer bookmark_next_raw_literal_;
-
   // Input stream. Must be initialized to an Utf16CharacterStream.
   Utf16CharacterStream* source_;
 

test/cctest/parsing/test-scanner.cc

@@ -56,7 +56,7 @@ TEST(Bookmarks) {
       DCHECK_EQ(tokens[i], scanner->Next());
     }
 
-    bookmark.Reset();
+    bookmark.Apply();
     for (size_t i = bookmark_pos; i < tokens.size(); i++) {
       DCHECK_EQ(tokens[i], scanner->Next());
     }