Extend scanner with new Harmony module keywords (under flag).

R=mstarzinger@chromium.org
BUG=
TEST=

Review URL: https://chromiumcodereview.appspot.com/9352013

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@10638 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/flag-definitions.h

@@ -109,11 +109,13 @@ private:
 // Flags for experimental language features.
 DEFINE_bool(harmony_typeof, false, "enable harmony semantics for typeof")
 DEFINE_bool(harmony_scoping, false, "enable harmony block scoping")
+DEFINE_bool(harmony_modules, false, "enable harmony modules")
 DEFINE_bool(harmony_proxies, false, "enable harmony proxies")
 DEFINE_bool(harmony_collections, false,
             "enable harmony collections (sets, maps, and weak maps)")
 DEFINE_bool(harmony, false, "enable all harmony features (except typeof)")
 DEFINE_implication(harmony, harmony_scoping)
+DEFINE_implication(harmony, harmony_modules)
 DEFINE_implication(harmony, harmony_proxies)
 DEFINE_implication(harmony, harmony_collections)
 

src/parser.cc

@@ -547,12 +547,16 @@ Parser::Parser(Handle<Script> script,
       fni_(NULL),
       allow_natives_syntax_((parser_flags & kAllowNativesSyntax) != 0),
       allow_lazy_((parser_flags & kAllowLazy) != 0),
+      allow_modules_((parser_flags & kAllowModules) != 0),
       stack_overflow_(false),
       parenthesized_function_(false) {
   AstNode::ResetIds();
   if ((parser_flags & kLanguageModeMask) == EXTENDED_MODE) {
     scanner().SetHarmonyScoping(true);
   }
+  if ((parser_flags & kAllowModules) != 0) {
+    scanner().SetHarmonyModules(true);
+  }
 }
 
 
@@ -4271,7 +4275,8 @@ preparser::PreParser::PreParseResult Parser::LazyParseFunctionLiteral(
                                                    NULL,
                                                    stack_limit,
                                                    do_allow_lazy,
-                                                   allow_natives_syntax_);
+                                                   allow_natives_syntax_,
+                                                   allow_modules_);
   }
   preparser::PreParser::PreParseResult result =
       reusable_preparser_->PreParseLazyFunction(top_scope_->language_mode(),
@@ -5579,6 +5584,9 @@ bool ParserApi::Parse(CompilationInfo* info, int parsing_flags) {
     // Harmony scoping is requested.
     parsing_flags |= EXTENDED_MODE;
   }
+  if (!info->is_native() && FLAG_harmony_modules) {
+    parsing_flags |= kAllowModules;
+  }
   if (FLAG_allow_natives_syntax || info->is_native()) {
     // We require %identifier(..) syntax.
     parsing_flags |= kAllowNativesSyntax;

src/parser.h

@@ -807,6 +807,7 @@ class Parser {
   Mode mode_;
   bool allow_natives_syntax_;
   bool allow_lazy_;
+  bool allow_modules_;
   bool stack_overflow_;
   // If true, the next (and immediately following) function literal is
   // preceded by a parenthesis.

src/preparser.h

@@ -115,7 +115,8 @@ class PreParser {
             i::ParserRecorder* log,
             uintptr_t stack_limit,
             bool allow_lazy,
-            bool allow_natives_syntax)
+            bool allow_natives_syntax,
+            bool allow_modules)
       : scanner_(scanner),
         log_(log),
         scope_(NULL),
@@ -124,6 +125,7 @@ class PreParser {
         strict_mode_violation_type_(NULL),
         stack_overflow_(false),
         allow_lazy_(allow_lazy),
+        allow_modules_(allow_modules),
         allow_natives_syntax_(allow_natives_syntax),
         parenthesized_function_(false),
         harmony_scoping_(scanner->HarmonyScoping()) { }
@@ -140,8 +142,9 @@ class PreParser {
                                         uintptr_t stack_limit) {
     bool allow_lazy = (flags & i::kAllowLazy) != 0;
     bool allow_natives_syntax = (flags & i::kAllowNativesSyntax) != 0;
-    return PreParser(scanner, log, stack_limit,
-                     allow_lazy, allow_natives_syntax).PreParse();
+    bool allow_modules = (flags & i::kAllowModules) != 0;
+    return PreParser(scanner, log, stack_limit, allow_lazy,
+                     allow_natives_syntax, allow_modules).PreParse();
   }
 
   // Parses a single function literal, from the opening parentheses before
@@ -647,6 +650,7 @@ class PreParser {
   const char* strict_mode_violation_type_;
   bool stack_overflow_;
   bool allow_lazy_;
+  bool allow_modules_;
   bool allow_natives_syntax_;
   bool parenthesized_function_;
   bool harmony_scoping_;

src/scanner.cc

@@ -41,7 +41,8 @@ namespace internal {
 Scanner::Scanner(UnicodeCache* unicode_cache)
     : unicode_cache_(unicode_cache),
       octal_pos_(Location::invalid()),
-      harmony_scoping_(false) { }
+      harmony_scoping_(false),
+      harmony_modules_(false) { }
 
 
 void Scanner::Initialize(UC16CharacterStream* source) {
@@ -830,7 +831,8 @@ uc32 Scanner::ScanIdentifierUnicodeEscape() {
   KEYWORD_GROUP('e')                                                \
   KEYWORD("else", Token::ELSE)                                      \
   KEYWORD("enum", Token::FUTURE_RESERVED_WORD)                      \
-  KEYWORD("export", Token::FUTURE_RESERVED_WORD)                    \
+  KEYWORD("export", harmony_modules                                 \
+                    ? Token::EXPORT : Token::FUTURE_RESERVED_WORD)  \
   KEYWORD("extends", Token::FUTURE_RESERVED_WORD)                   \
   KEYWORD_GROUP('f')                                                \
   KEYWORD("false", Token::FALSE_LITERAL)                            \
@@ -840,13 +842,17 @@ uc32 Scanner::ScanIdentifierUnicodeEscape() {
   KEYWORD_GROUP('i')                                                \
   KEYWORD("if", Token::IF)                                          \
   KEYWORD("implements", Token::FUTURE_STRICT_RESERVED_WORD)         \
-  KEYWORD("import", Token::FUTURE_RESERVED_WORD)                    \
+  KEYWORD("import", harmony_modules                                 \
+                    ? Token::IMPORT : Token::FUTURE_RESERVED_WORD)  \
   KEYWORD("in", Token::IN)                                          \
   KEYWORD("instanceof", Token::INSTANCEOF)                          \
   KEYWORD("interface", Token::FUTURE_STRICT_RESERVED_WORD)          \
   KEYWORD_GROUP('l')                                                \
   KEYWORD("let", harmony_scoping                                    \
                  ? Token::LET : Token::FUTURE_STRICT_RESERVED_WORD) \
+  KEYWORD_GROUP('m')                                                \
+  KEYWORD("module", harmony_modules                                 \
+                    ? Token::MODULE : Token::IDENTIFIER)            \
   KEYWORD_GROUP('n')                                                \
   KEYWORD("new", Token::NEW)                                        \
   KEYWORD("null", Token::NULL_LITERAL)                              \
@@ -879,7 +885,8 @@ uc32 Scanner::ScanIdentifierUnicodeEscape() {
 
 static Token::Value KeywordOrIdentifierToken(const char* input,
                                              int input_length,
-                                             bool harmony_scoping) {
+                                             bool harmony_scoping,
+                                             bool harmony_modules) {
   ASSERT(input_length >= 1);
   const int kMinLength = 2;
   const int kMaxLength = 10;
@@ -955,7 +962,8 @@ Token::Value Scanner::ScanIdentifierOrKeyword() {
     Vector<const char> chars = next_.literal_chars->ascii_literal();
     return KeywordOrIdentifierToken(chars.start(),
                                     chars.length(),
-                                    harmony_scoping_);
+                                    harmony_scoping_,
+                                    harmony_modules_);
   }
 
   return Token::IDENTIFIER;

src/scanner.h

@@ -51,8 +51,9 @@ enum ParsingFlags {
   // STRICT_MODE,
   // EXTENDED_MODE,
   kLanguageModeMask = 0x03,
-  kAllowLazy = 4,
-  kAllowNativesSyntax = 8
+  kAllowLazy = 0x04,
+  kAllowNativesSyntax = 0x08,
+  kAllowModules = 0x10
 };
 
 STATIC_ASSERT((kLanguageModeMask & CLASSIC_MODE) == CLASSIC_MODE);
@@ -403,8 +404,14 @@ class Scanner {
   bool HarmonyScoping() const {
     return harmony_scoping_;
   }
-  void SetHarmonyScoping(bool block_scoping) {
-    harmony_scoping_ = block_scoping;
+  void SetHarmonyScoping(bool scoping) {
+    harmony_scoping_ = scoping;
+  }
+  bool HarmonyModules() const {
+    return harmony_modules_;
+  }
+  void SetHarmonyModules(bool modules) {
+    harmony_modules_ = modules;
   }
 
 
@@ -552,9 +559,10 @@ class Scanner {
   // Whether there is a multi-line comment that contains a
   // line-terminator after the current token, and before the next.
   bool has_multiline_comment_before_next_;
-  // Whether we scan 'let' as a keyword for harmony block scoped
-  // let bindings.
+  // Whether we scan 'let' as a keyword for harmony block-scoped let bindings.
   bool harmony_scoping_;
+  // Whether we scan 'module', 'import', 'export' as keywords.
+  bool harmony_modules_;
 };
 
 } }  // namespace v8::internal

src/token.h

@@ -170,7 +170,10 @@ namespace internal {
   T(FUTURE_RESERVED_WORD, NULL, 0)                                      \
   T(FUTURE_STRICT_RESERVED_WORD, NULL, 0)                               \
   K(CONST, "const", 0)                                                  \
+  K(EXPORT, "export", 0)                                                \
+  K(IMPORT, "import", 0)                                                \
   K(LET, "let", 0)                                                      \
+  K(MODULE, "module", 0)                                                \
                                                                         \
   /* Illegal token - not able to scan. */                               \
   T(ILLEGAL, "ILLEGAL", 0)                                              \

test/cctest/test-parsing.cc

@@ -65,8 +65,9 @@ TEST(ScanKeywords) {
     {
       i::Utf8ToUC16CharacterStream stream(keyword, length);
       i::Scanner scanner(&unicode_cache);
-      // The scanner should parse 'let' as Token::LET for this test.
+      // The scanner should parse Harmony keywords for this test.
       scanner.SetHarmonyScoping(true);
+      scanner.SetHarmonyModules(true);
       scanner.Initialize(&stream);
       CHECK_EQ(key_token.token, scanner.Next());
       CHECK_EQ(i::Token::EOS, scanner.Next());