scanner.h 25.7 KB
Newer Older
1
// Copyright 2011 the V8 project authors. All rights reserved.
2 3
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
4

5 6
// Features shared by parsing and pre-parsing scanners.

7 8
#ifndef V8_PARSING_SCANNER_H_
#define V8_PARSING_SCANNER_H_
9

10
#include <algorithm>
11
#include <memory>
12

13
#include "src/base/logging.h"
14
#include "src/base/strings.h"
15
#include "src/common/globals.h"
16
#include "src/common/message-template.h"
17
#include "src/parsing/literal-buffer.h"
18
#include "src/parsing/parse-info.h"
19
#include "src/parsing/token.h"
20
#include "src/regexp/regexp-flags.h"
21 22
#include "src/strings/char-predicates.h"
#include "src/strings/unicode.h"
23 24
#include "src/utils/allocation.h"
#include "src/utils/pointer-with-payload.h"
25

26 27
namespace v8 {
namespace internal {
28

29 30
class AstRawString;
class AstValueFactory;
31 32
class ExternalOneByteString;
class ExternalTwoByteString;
33
class ParserRecorder;
34
class RuntimeCallStats;
35
class Zone;
36

37 38 39 40 41 42
// ---------------------------------------------------------------------
// Buffered stream of UTF-16 code units, using an internal UTF-16 buffer.
// A code unit is a 16 bit value representing either a 16 bit code point
// or one part of a surrogate pair that make a single 21 bit code point.
class Utf16CharacterStream {
 public:
43
  static constexpr base::uc32 kEndOfInput = static_cast<base::uc32>(-1);
44

45
  virtual ~Utf16CharacterStream() = default;
46

47
  V8_INLINE void set_parser_error() {
48 49 50
    buffer_cursor_ = buffer_end_;
    has_parser_error_ = true;
  }
51 52
  V8_INLINE void reset_parser_error_flag() { has_parser_error_ = false; }
  V8_INLINE bool has_parser_error() const { return has_parser_error_; }
53

54
  inline base::uc32 Peek() {
55
    if (V8_LIKELY(buffer_cursor_ < buffer_end_)) {
56
      return static_cast<base::uc32>(*buffer_cursor_);
57
    } else if (ReadBlockChecked(pos())) {
58
      return static_cast<base::uc32>(*buffer_cursor_);
59 60 61 62 63 64 65
    } else {
      return kEndOfInput;
    }
  }

  // Returns and advances past the next UTF-16 code unit in the input
  // stream. If there are no more code units it returns kEndOfInput.
66 67
  inline base::uc32 Advance() {
    base::uc32 result = Peek();
68 69 70 71 72 73 74 75
    buffer_cursor_++;
    return result;
  }

  // Returns and advances past the next UTF-16 code unit in the input stream
  // that meets the checks requirement. If there are no more code units it
  // returns kEndOfInput.
  template <typename FunctionType>
76
  V8_INLINE base::uc32 AdvanceUntil(FunctionType check) {
77
    while (true) {
78 79
      auto next_cursor_pos =
          std::find_if(buffer_cursor_, buffer_end_, [&check](uint16_t raw_c0_) {
80
            base::uc32 c0_ = static_cast<base::uc32>(raw_c0_);
81 82 83 84 85
            return check(c0_);
          });

      if (next_cursor_pos == buffer_end_) {
        buffer_cursor_ = buffer_end_;
86
        if (!ReadBlockChecked(pos())) {
87
          buffer_cursor_++;
88
          return kEndOfInput;
89
        }
90 91
      } else {
        buffer_cursor_ = next_cursor_pos + 1;
92
        return static_cast<base::uc32>(*next_cursor_pos);
93 94 95 96 97 98 99 100 101 102 103 104 105
      }
    }
  }

  // Go back one by one character in the input stream.
  // This undoes the most recent Advance().
  inline void Back() {
    // The common case - if the previous character is within
    // buffer_start_ .. buffer_end_ will be handles locally.
    // Otherwise, a new block is requested.
    if (V8_LIKELY(buffer_cursor_ > buffer_start_)) {
      buffer_cursor_--;
    } else {
106
      ReadBlockChecked(pos() - 1);
107 108 109 110 111 112 113 114 115 116 117 118
    }
  }

  inline size_t pos() const {
    return buffer_pos_ + (buffer_cursor_ - buffer_start_);
  }

  inline void Seek(size_t pos) {
    if (V8_LIKELY(pos >= buffer_pos_ &&
                  pos < (buffer_pos_ + (buffer_end_ - buffer_start_)))) {
      buffer_cursor_ = buffer_start_ + (pos - buffer_pos_);
    } else {
119
      ReadBlockChecked(pos);
120 121 122
    }
  }

123 124 125 126 127
  // Returns true if the stream could access the V8 heap after construction.
  bool can_be_cloned_for_parallel_access() const {
    return can_be_cloned() && !can_access_heap();
  }

128 129 130 131 132 133 134 135
  // Returns true if the stream can be cloned with Clone.
  // TODO(rmcilroy): Remove this once ChunkedStreams can be cloned.
  virtual bool can_be_cloned() const = 0;

  // Clones the character stream to enable another independent scanner to access
  // the same underlying stream.
  virtual std::unique_ptr<Utf16CharacterStream> Clone() const = 0;

136
  // Returns true if the stream could access the V8 heap after construction.
137
  virtual bool can_access_heap() const = 0;
138

139 140 141 142 143
  RuntimeCallStats* runtime_call_stats() const { return runtime_call_stats_; }
  void set_runtime_call_stats(RuntimeCallStats* runtime_call_stats) {
    runtime_call_stats_ = runtime_call_stats;
  }

144 145 146 147 148 149 150 151 152 153
 protected:
  Utf16CharacterStream(const uint16_t* buffer_start,
                       const uint16_t* buffer_cursor,
                       const uint16_t* buffer_end, size_t buffer_pos)
      : buffer_start_(buffer_start),
        buffer_cursor_(buffer_cursor),
        buffer_end_(buffer_end),
        buffer_pos_(buffer_pos) {}
  Utf16CharacterStream() : Utf16CharacterStream(nullptr, nullptr, nullptr, 0) {}

154 155 156 157 158 159 160 161 162
  bool ReadBlockChecked(size_t position) {
    // The callers of this method (Back/Back2/Seek) should handle the easy
    // case (seeking within the current buffer), and we should only get here
    // if we actually require new data.
    // (This is really an efficiency check, not a correctness invariant.)
    DCHECK(position < buffer_pos_ ||
           position >= buffer_pos_ + (buffer_end_ - buffer_start_));

    bool success = !has_parser_error() && ReadBlock(position);
163 164 165 166 167 168 169 170 171 172 173 174 175 176

    // Post-conditions: 1, We should always be at the right position.
    //                  2, Cursor should be inside the buffer.
    //                  3, We should have more characters available iff success.
    DCHECK_EQ(pos(), position);
    DCHECK_LE(buffer_cursor_, buffer_end_);
    DCHECK_LE(buffer_start_, buffer_cursor_);
    DCHECK_EQ(success, buffer_cursor_ < buffer_end_);
    return success;
  }

  // Read more data, and update buffer_*_ to point to it.
  // Returns true if more data was available.
  //
177 178
  // ReadBlock(position) may modify any of the buffer_*_ members, but must make
  // sure that the result of pos() becomes |position|.
179 180 181 182 183 184 185 186 187
  //
  // Examples:
  // - a stream could either fill a separate buffer. Then buffer_start_ and
  //   buffer_cursor_ would point to the beginning of the buffer, and
  //   buffer_pos would be the old pos().
  // - a stream with existing buffer chunks would set buffer_start_ and
  //   buffer_end_ to cover the full chunk, and then buffer_cursor_ would
  //   point into the middle of the buffer, while buffer_pos_ would describe
  //   the start of the buffer.
188
  virtual bool ReadBlock(size_t position) = 0;
189

190 191 192 193 194 195 196 197 198 199 200 201 202
  // Fields describing the location of the current buffer physically in memory,
  // and semantically within the source string.
  //
  //                  0              buffer_pos_   pos()
  //                  |                        |   |
  //                  v________________________v___v_____________
  //                  |                        |        |        |
  //   Source string: |                        | Buffer |        |
  //                  |________________________|________|________|
  //                                           ^   ^    ^
  //                                           |   |    |
  //                   Pointers:   buffer_start_   |    buffer_end_
  //                                         buffer_cursor_
203 204 205 206
  const uint16_t* buffer_start_;
  const uint16_t* buffer_cursor_;
  const uint16_t* buffer_end_;
  size_t buffer_pos_;
207
  RuntimeCallStats* runtime_call_stats_;
208
  bool has_parser_error_ = false;
209 210
};

211
// ----------------------------------------------------------------------------
212
// JavaScript Scanner.
213

214
class V8_EXPORT_PRIVATE Scanner {
215
 public:
216
  // Scoped helper for a re-settable bookmark.
217
  class V8_EXPORT_PRIVATE V8_NODISCARD BookmarkScope {
218
   public:
219
    explicit BookmarkScope(Scanner* scanner)
220 221
        : scanner_(scanner),
          bookmark_(kNoBookmark),
222
          had_parser_error_(scanner->has_parser_error()) {
223 224
      DCHECK_NOT_NULL(scanner_);
    }
225
    ~BookmarkScope() = default;
226 227
    BookmarkScope(const BookmarkScope&) = delete;
    BookmarkScope& operator=(const BookmarkScope&) = delete;
228

229
    void Set(size_t bookmark);
230
    void Apply();
231 232
    bool HasBeenSet() const;
    bool HasBeenApplied() const;
233 234

   private:
235 236 237
    static const size_t kNoBookmark;
    static const size_t kBookmarkWasApplied;

238
    Scanner* scanner_;
239
    size_t bookmark_;
240
    bool had_parser_error_;
241 242
  };

243 244
  // Sets the Scanner into an error state to stop further scanning and terminate
  // the parsing by only returning ILLEGAL tokens after that.
245
  V8_INLINE void set_parser_error() {
246
    if (!has_parser_error()) {
247 248
      c0_ = kEndOfInput;
      source_->set_parser_error();
249
      for (TokenDesc& desc : token_storage_) desc.token = Token::ILLEGAL;
250 251
    }
  }
252 253 254
  V8_INLINE void reset_parser_error_flag() {
    source_->reset_parser_error_flag();
  }
255
  V8_INLINE bool has_parser_error() const {
256 257
    return source_->has_parser_error();
  }
258

259
  // Representation of an interval of source positions.
260 261 262 263
  struct Location {
    Location(int b, int e) : beg_pos(b), end_pos(e) { }
    Location() : beg_pos(0), end_pos(0) { }

264
    int length() const { return end_pos - beg_pos; }
265
    bool IsValid() const { return base::IsInRange(beg_pos, 0, end_pos); }
266

267
    static Location invalid() { return Location(-1, 0); }
268 269 270 271 272

    int beg_pos;
    int end_pos;
  };

273
  // -1 is outside of the range of any real source code.
274 275
  static constexpr base::uc32 kEndOfInput = Utf16CharacterStream::kEndOfInput;
  static constexpr base::uc32 kInvalidSequence = static_cast<base::uc32>(-1);
276

277 278
  static constexpr base::uc32 Invalid() { return Scanner::kInvalidSequence; }
  static bool IsInvalid(base::uc32 c);
279

280
  explicit Scanner(Utf16CharacterStream* source, UnoptimizedCompileFlags flags);
281

282
  void Initialize();
283 284

  // Returns the next token and advances input.
285
  Token::Value Next();
littledan's avatar
littledan committed
286
  // Returns the token following peek()
287
  Token::Value PeekAhead();
288
  // Returns the current token again.
289
  Token::Value current_token() const { return current().token; }
290

291
  // Returns the location information for the current token
292
  // (the token last returned by Next()).
293
  const Location& location() const { return current().location; }
294

295
  // This error is specifically an invalid hex or unicode escape sequence.
296
  bool has_error() const { return scanner_error_ != MessageTemplate::kNone; }
297
  MessageTemplate error() const { return scanner_error_; }
298
  const Location& error_location() const { return scanner_error_location_; }
299

300
  bool has_invalid_template_escape() const {
301
    return current().invalid_template_escape_message != MessageTemplate::kNone;
302
  }
303
  MessageTemplate invalid_template_escape_message() const {
304
    DCHECK(has_invalid_template_escape());
305
    return current().invalid_template_escape_message;
306
  }
307 308 309 310 311 312

  void clear_invalid_template_escape_message() {
    DCHECK(has_invalid_template_escape());
    current_->invalid_template_escape_message = MessageTemplate::kNone;
  }

313 314
  Location invalid_template_escape_location() const {
    DCHECK(has_invalid_template_escape());
315
    return current().invalid_template_escape_location;
316 317
  }

318 319 320
  // Similar functions for the upcoming token.

  // One token look-ahead (past the token returned by Next()).
321
  Token::Value peek() const { return next().token; }
322

323
  const Location& peek_location() const { return next().location; }
324 325

  bool literal_contains_escapes() const {
326
    return LiteralContainsEscapes(current());
327
  }
328

329 330 331 332
  bool next_literal_contains_escapes() const {
    return LiteralContainsEscapes(next());
  }

333
  const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory) const;
334

335 336 337
  const AstRawString* NextSymbol(AstValueFactory* ast_value_factory) const;
  const AstRawString* CurrentRawSymbol(
      AstValueFactory* ast_value_factory) const;
338 339

  double DoubleValue();
340

341 342
  const char* CurrentLiteralAsCString(Zone* zone) const;

343 344
  inline bool CurrentMatches(Token::Value token) const {
    DCHECK(Token::IsKeyword(token));
345
    return current().token == token;
346
  }
347

348
  template <size_t N>
349 350
  bool NextLiteralExactlyEquals(const char (&s)[N]) {
    DCHECK(next().CanAccessLiteral());
351 352 353 354 355 356
    // The length of the token is used to make sure the literal equals without
    // taking escape sequences (e.g., "use \x73trict") or line continuations
    // (e.g., "use \(newline) strict") into account.
    if (!is_next_literal_one_byte()) return false;
    if (peek_location().length() != N + 1) return false;

357
    base::Vector<const uint8_t> next = next_literal_one_byte_string();
358
    const char* chars = reinterpret_cast<const char*>(next.begin());
359 360 361
    return next.length() == N - 1 && strncmp(s, chars, N - 1) == 0;
  }

362 363 364 365 366
  template <size_t N>
  bool CurrentLiteralEquals(const char (&s)[N]) {
    DCHECK(current().CanAccessLiteral());
    if (!is_literal_one_byte()) return false;

367
    base::Vector<const uint8_t> current = literal_one_byte_string();
368
    const char* chars = reinterpret_cast<const char*>(current.begin());
369 370 371
    return current.length() == N - 1 && strncmp(s, chars, N - 1) == 0;
  }

372 373
  // Returns the location of the last seen octal literal.
  Location octal_position() const { return octal_pos_; }
374 375 376 377
  void clear_octal_position() {
    octal_pos_ = Location::invalid();
    octal_message_ = MessageTemplate::kNone;
  }
378
  MessageTemplate octal_message() const { return octal_message_; }
379

verwaest's avatar
verwaest committed
380
  // Returns the value of the last smi that was scanned.
381
  uint32_t smi_value() const { return current().smi_value_; }
verwaest's avatar
verwaest committed
382

383 384 385 386 387 388 389 390
  // Seek forward to the given position.  This operation does not
  // work in general, for instance when there are pushed back
  // characters, but works for seeking forward until simple delimiter
  // tokens, which is what it is used for.
  void SeekForward(int pos);

  // Returns true if there was a line terminator before the peek'ed token,
  // possibly inside a multi-line comment.
391 392
  bool HasLineTerminatorBeforeNext() const {
    return next().after_line_terminator;
393 394
  }

395
  bool HasLineTerminatorAfterNext() {
396 397
    Token::Value ensure_next_next = PeekAhead();
    USE(ensure_next_next);
398
    return next_next().after_line_terminator;
399 400
  }

401 402
  // Scans the input as a regular expression pattern, next token must be /(=).
  // Returns true if a pattern is scanned.
403
  bool ScanRegExpPattern();
404
  // Scans the input as regular expression flags. Returns the flags on success.
405
  base::Optional<RegExpFlags> ScanRegExpFlags();
406

407
  // Scans the input as a template literal
408
  Token::Value ScanTemplateContinuation() {
409
    DCHECK_EQ(next().token, Token::RBRACE);
410
    DCHECK_EQ(source_pos() - 1, next().location.beg_pos);
411
    return ScanTemplateSpan();
412
  }
413

414 415 416 417
  template <typename IsolateT>
  Handle<String> SourceUrl(IsolateT* isolate) const;
  template <typename IsolateT>
  Handle<String> SourceMappingUrl(IsolateT* isolate) const;
418

419 420
  bool FoundHtmlComment() const { return found_html_comment_; }

421 422
  const Utf16CharacterStream* stream() const { return source_; }

423
 private:
424 425 426 427 428
  // Scoped helper for saving & restoring scanner error state.
  // This is used for tagged template literals, in which normally forbidden
  // escape sequences are allowed.
  class ErrorState;

429 430
  // The current and look-ahead token.
  struct TokenDesc {
431 432 433 434
    Location location = {0, 0};
    LiteralBuffer literal_chars;
    LiteralBuffer raw_literal_chars;
    Token::Value token = Token::UNINITIALIZED;
435
    MessageTemplate invalid_template_escape_message = MessageTemplate::kNone;
436
    Location invalid_template_escape_location;
437 438
    uint32_t smi_value_ = 0;
    bool after_line_terminator = false;
439 440 441 442

#ifdef DEBUG
    bool CanAccessLiteral() const {
      return token == Token::PRIVATE_NAME || token == Token::ILLEGAL ||
443 444
             token == Token::ESCAPED_KEYWORD || token == Token::UNINITIALIZED ||
             token == Token::REGEXP_LITERAL ||
445
             base::IsInRange(token, Token::NUMBER, Token::STRING) ||
446
             Token::IsAnyIdentifier(token) || Token::IsKeyword(token) ||
447
             base::IsInRange(token, Token::TEMPLATE_SPAN, Token::TEMPLATE_TAIL);
448 449 450
    }
    bool CanAccessRawLiteral() const {
      return token == Token::ILLEGAL || token == Token::UNINITIALIZED ||
451
             base::IsInRange(token, Token::TEMPLATE_SPAN, Token::TEMPLATE_TAIL);
452 453
    }
#endif  // DEBUG
454 455
  };

456
  enum NumberKind {
457
    IMPLICIT_OCTAL,
458 459 460 461 462 463 464
    BINARY,
    OCTAL,
    HEX,
    DECIMAL,
    DECIMAL_WITH_LEADING_ZERO
  };

465
  inline bool IsValidBigIntKind(NumberKind kind) {
466
    return base::IsInRange(kind, BINARY, DECIMAL);
467 468 469
  }

  inline bool IsDecimalNumberKind(NumberKind kind) {
470
    return base::IsInRange(kind, DECIMAL, DECIMAL_WITH_LEADING_ZERO);
471 472
  }

473
  static const int kCharacterLookaheadBufferSize = 1;
474
  static const int kMaxAscii = 127;
475 476

  // Scans octal escape sequence. Also accepts "\0" decimal escape sequence.
477
  template <bool capture_raw>
478
  base::uc32 ScanOctalEscape(base::uc32 c, int length);
479

480 481 482 483 484
  // Call this after setting source_ to the input.
  void Init() {
    // Set c0_ (one character ahead)
    STATIC_ASSERT(kCharacterLookaheadBufferSize == 1);
    Advance();
485

486 487 488 489
    current_ = &token_storage_[0];
    next_ = &token_storage_[1];
    next_next_ = &token_storage_[2];

490 491 492
    found_html_comment_ = false;
    scanner_error_ = MessageTemplate::kNone;
  }
493

494
  void ReportScannerError(const Location& location, MessageTemplate error) {
495 496 497 498 499
    if (has_error()) return;
    scanner_error_ = error;
    scanner_error_location_ = location;
  }

500
  void ReportScannerError(int pos, MessageTemplate error) {
501 502 503 504 505
    if (has_error()) return;
    scanner_error_ = error;
    scanner_error_location_ = Location(pos, pos + 1);
  }

506 507 508
  // Seek to the next_ token at the given position.
  void SeekNext(size_t position);

509 510 511
  V8_INLINE void AddLiteralChar(base::uc32 c) {
    next().literal_chars.AddChar(c);
  }
512 513

  V8_INLINE void AddLiteralChar(char c) { next().literal_chars.AddChar(c); }
514

515
  V8_INLINE void AddRawLiteralChar(base::uc32 c) {
516
    next().raw_literal_chars.AddChar(c);
517 518
  }

519
  V8_INLINE void AddLiteralCharAdvance() {
520
    AddLiteralChar(c0_);
521
    Advance();
522 523 524
  }

  // Low-level scanning support.
525
  template <bool capture_raw = false>
526
  void Advance() {
527 528 529 530
    if (capture_raw) {
      AddRawLiteralChar(c0_);
    }
    c0_ = source_->Advance();
531 532
  }

533
  template <typename FunctionType>
534
  V8_INLINE void AdvanceUntil(FunctionType check) {
535
    c0_ = source_->AdvanceUntil(check);
536 537
  }

538
  bool CombineSurrogatePair() {
539
    DCHECK(!unibrow::Utf16::IsLeadSurrogate(kEndOfInput));
540
    if (unibrow::Utf16::IsLeadSurrogate(c0_)) {
541
      base::uc32 c1 = source_->Advance();
542
      DCHECK(!unibrow::Utf16::IsTrailSurrogate(kEndOfInput));
543
      if (unibrow::Utf16::IsTrailSurrogate(c1)) {
544
        c0_ = unibrow::Utf16::CombineSurrogatePair(c0_, c1);
545
        return true;
546
      }
547
      source_->Back();
548
    }
549
    return false;
550 551
  }

552
  void PushBack(base::uc32 ch) {
553 554
    DCHECK(IsInvalid(c0_) ||
           base::IsInRange(c0_, 0u, unibrow::Utf16::kMaxNonSurrogateCharCode));
555
    source_->Back();
556 557 558
    c0_ = ch;
  }

559
  base::uc32 Peek() const { return source_->Peek(); }
560

561
  inline Token::Value Select(Token::Value tok) {
562
    Advance();
563 564 565
    return tok;
  }

566 567
  inline Token::Value Select(base::uc32 next, Token::Value then,
                             Token::Value else_) {
568
    Advance();
569
    if (c0_ == next) {
570
      Advance();
571
      return then;
572 573
    } else {
      return else_;
574 575
    }
  }
576 577
  // Returns the literal string, if any, for the current token (the
  // token last returned by Next()). The string is 0-terminated.
578 579 580
  // Literal strings are collected for identifiers, strings, numbers as well
  // as for template literals. For template literals we also collect the raw
  // form.
581 582
  // These functions only give the correct result if the literal was scanned
  // when a LiteralScope object is alive.
583 584 585 586 587 588
  //
  // Current usage of these functions is unfortunately a little undisciplined,
  // and is_literal_one_byte() + is_literal_one_byte_string() is also
  // requested for tokens that do not have a literal. Hence, we treat any
  // token as a one-byte literal. E.g. Token::FUNCTION pretends to have a
  // literal "function".
589
  base::Vector<const uint8_t> literal_one_byte_string() const {
590 591
    DCHECK(current().CanAccessLiteral() || Token::IsKeyword(current().token) ||
           current().token == Token::ESCAPED_KEYWORD);
592
    return current().literal_chars.one_byte_literal();
593
  }
594
  base::Vector<const uint16_t> literal_two_byte_string() const {
595 596
    DCHECK(current().CanAccessLiteral() || Token::IsKeyword(current().token) ||
           current().token == Token::ESCAPED_KEYWORD);
597
    return current().literal_chars.two_byte_literal();
598
  }
599
  bool is_literal_one_byte() const {
600 601
    DCHECK(current().CanAccessLiteral() || Token::IsKeyword(current().token) ||
           current().token == Token::ESCAPED_KEYWORD);
602
    return current().literal_chars.is_one_byte();
603 604 605
  }
  // Returns the literal string for the next token (the token that
  // would be returned if Next() were called).
606
  base::Vector<const uint8_t> next_literal_one_byte_string() const {
607
    DCHECK(next().CanAccessLiteral());
608
    return next().literal_chars.one_byte_literal();
609
  }
610
  base::Vector<const uint16_t> next_literal_two_byte_string() const {
611
    DCHECK(next().CanAccessLiteral());
612
    return next().literal_chars.two_byte_literal();
613
  }
614
  bool is_next_literal_one_byte() const {
615
    DCHECK(next().CanAccessLiteral());
616
    return next().literal_chars.is_one_byte();
617
  }
618
  base::Vector<const uint8_t> raw_literal_one_byte_string() const {
619
    DCHECK(current().CanAccessRawLiteral());
620
    return current().raw_literal_chars.one_byte_literal();
621
  }
622
  base::Vector<const uint16_t> raw_literal_two_byte_string() const {
623
    DCHECK(current().CanAccessRawLiteral());
624
    return current().raw_literal_chars.two_byte_literal();
625
  }
626
  bool is_raw_literal_one_byte() const {
627
    DCHECK(current().CanAccessRawLiteral());
628
    return current().raw_literal_chars.is_one_byte();
629 630
  }

631
  template <bool capture_raw, bool unicode = false>
632
  base::uc32 ScanHexNumber(int expected_length);
marja's avatar
marja committed
633 634 635
  // Scan a number of any length but not bigger than max_value. For example, the
  // number can be 000000001, so it's very long in characters but its value is
  // small.
636
  template <bool capture_raw>
637
  base::uc32 ScanUnlimitedLengthHexNumber(base::uc32 max_value, int beg_pos);
638

639
  // Scans a single JavaScript token.
640
  V8_INLINE Token::Value ScanSingleToken();
641
  V8_INLINE void Scan();
642 643 644 645 646
  // Performance hack: pass through a pre-calculated "next()" value to avoid
  // having to re-calculate it in Scan. You'd think the compiler would be able
  // to hoist the next() calculation out of the inlined Scan method, but seems
  // that pointer aliasing analysis fails show that this is safe.
  V8_INLINE void Scan(TokenDesc* next_desc);
647

648
  V8_INLINE Token::Value SkipWhiteSpace();
649
  Token::Value SkipSingleHTMLComment();
650
  Token::Value SkipSingleLineComment();
651 652
  Token::Value SkipSourceURLComment();
  void TryToParseSourceURLComment();
653
  Token::Value SkipMultiLineComment();
654 655
  // Scans a possible HTML comment -- begins with '<!'.
  Token::Value ScanHtmlComment();
656

657
  bool ScanDigitsWithNumericSeparators(bool (*predicate)(base::uc32 ch),
658
                                       bool is_check_first_digit);
659
  bool ScanDecimalDigits(bool allow_numeric_separator);
660
  // Optimized function to scan decimal number as Smi.
661
  bool ScanDecimalAsSmi(uint64_t* value, bool allow_numeric_separator);
662 663 664 665 666
  bool ScanDecimalAsSmiWithNumericSeparators(uint64_t* value);
  bool ScanHexDigits();
  bool ScanBinaryDigits();
  bool ScanSignedInteger();
  bool ScanOctalDigits();
667
  bool ScanImplicitOctalDigits(int start_pos, NumberKind* kind);
668

669
  Token::Value ScanNumber(bool seen_period);
670
  V8_INLINE Token::Value ScanIdentifierOrKeyword();
671
  V8_INLINE Token::Value ScanIdentifierOrKeywordInner();
672 673
  Token::Value ScanIdentifierOrKeywordInnerSlow(bool escaped,
                                                bool can_be_keyword);
674 675

  Token::Value ScanString();
676
  Token::Value ScanPrivateName();
677

678 679 680
  // Scans an escape-sequence which is part of a string and adds the
  // decoded character to the current literal. Returns true if a pattern
  // is scanned.
681
  template <bool capture_raw>
682
  bool ScanEscape();
683

684
  // Decodes a Unicode escape-sequence which is part of an identifier.
685
  // If the escape sequence cannot be decoded the result is kBadChar.
686
  base::uc32 ScanIdentifierUnicodeEscape();
marja's avatar
marja committed
687
  // Helper for the above functions.
688
  template <bool capture_raw>
689
  base::uc32 ScanUnicodeEscape();
690

691 692
  Token::Value ScanTemplateSpan();

693
  // Return the current source position.
694 695
  int source_pos() {
    return static_cast<int>(source_->pos()) - kCharacterLookaheadBufferSize;
696 697
  }

698 699 700 701 702 703 704
  static bool LiteralContainsEscapes(const TokenDesc& token) {
    Location location = token.location;
    int source_length = (location.end_pos - location.beg_pos);
    if (token.token == Token::STRING) {
      // Subtract delimiters.
      source_length -= 2;
    }
705
    return token.literal_chars.length() != source_length;
706 707
  }

708 709 710 711
#ifdef DEBUG
  void SanityCheckTokenDesc(const TokenDesc&) const;
#endif

712
  TokenDesc& next() { return *next_; }
713

714 715 716
  const TokenDesc& current() const { return *current_; }
  const TokenDesc& next() const { return *next_; }
  const TokenDesc& next_next() const { return *next_next_; }
717

718 719
  UnoptimizedCompileFlags flags_;

720 721 722
  TokenDesc* current_;    // desc for current token (as returned by Next())
  TokenDesc* next_;       // desc for next token (one token look-ahead)
  TokenDesc* next_next_;  // desc for the token after next (after PeakAhead())
723

724
  // Input stream. Must be initialized to an Utf16CharacterStream.
725
  Utf16CharacterStream* const source_;
726 727

  // One Unicode character look-ahead; c0_ < 0 at the end of the input.
728
  base::uc32 c0_;
729

730 731
  TokenDesc token_storage_[3];

732 733
  // Whether this scanner encountered an HTML comment.
  bool found_html_comment_;
734

735 736 737 738 739 740
  // Values parsed from magic comments.
  LiteralBuffer source_url_;
  LiteralBuffer source_mapping_url_;

  // Last-seen positions of potentially problematic tokens.
  Location octal_pos_;
741
  MessageTemplate octal_message_;
742

743
  MessageTemplate scanner_error_;
744
  Location scanner_error_location_;
745 746
};

747 748
}  // namespace internal
}  // namespace v8
749

750
#endif  // V8_PARSING_SCANNER_H_