Add parser support for generators.

This patchset begins by adding support for "yield", which is unlike other tokens
in JS. In a generator, whether strict or classic, it is a syntactic keyword.
In classic mode it is an identifier. In strict mode it is reserved.

This patch adds YIELD as a token to the scanner, and adapts the preparser and
parser appropriately. It also parses "function*", indicating that a function is
actually a generator, for both eagerly and lazily parsed functions.

Currently "yield" just compiles as "return".

BUG=v8:2355
TEST=mjsunit/harmony/generators-parsing

Review URL: https://codereview.chromium.org/12646003
Patch from Andy Wingo <wingo@igalia.com>.

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@14116 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/ast.cc

@@ -1062,6 +1062,7 @@ DONT_OPTIMIZE_NODE(ModuleVariable)
 DONT_OPTIMIZE_NODE(ModulePath)
 DONT_OPTIMIZE_NODE(ModuleUrl)
 DONT_OPTIMIZE_NODE(ModuleStatement)
+DONT_OPTIMIZE_NODE(Yield)
 DONT_OPTIMIZE_NODE(WithStatement)
 DONT_OPTIMIZE_NODE(TryCatchStatement)
 DONT_OPTIMIZE_NODE(TryFinallyStatement)

src/ast.h

@@ -102,6 +102,7 @@ namespace internal {
   V(ObjectLiteral)                              \
   V(ArrayLiteral)                               \
   V(Assignment)                                 \
+  V(Yield)                                      \
   V(Throw)                                      \
   V(Property)                                   \
   V(Call)                                       \
@@ -1953,6 +1954,31 @@ class Assignment: public Expression {
 };
 
 
+class Yield: public Expression {
+ public:
+  DECLARE_NODE_TYPE(Yield)
+
+  Expression* expression() const { return expression_; }
+  bool is_delegating_yield() const { return is_delegating_yield_; }
+  virtual int position() const { return pos_; }
+
+ protected:
+  Yield(Isolate* isolate,
+        Expression* expression,
+        bool is_delegating_yield,
+        int pos)
+      : Expression(isolate),
+        expression_(expression),
+        is_delegating_yield_(is_delegating_yield),
+        pos_(pos) { }
+
+ private:
+  Expression* expression_;
+  bool is_delegating_yield_;
+  int pos_;
+};
+
+
 class Throw: public Expression {
  public:
   DECLARE_NODE_TYPE(Throw)
@@ -1993,6 +2019,11 @@ class FunctionLiteral: public Expression {
     kNotParenthesized
   };
 
+  enum IsGeneratorFlag {
+    kIsGenerator,
+    kNotGenerator
+  };
+
   DECLARE_NODE_TYPE(FunctionLiteral)
 
   Handle<String> name() const { return name_; }
@@ -2053,6 +2084,10 @@ class FunctionLiteral: public Expression {
     bitfield_ = IsParenthesized::update(bitfield_, kIsParenthesized);
   }
 
+  bool is_generator() {
+    return IsGenerator::decode(bitfield_) == kIsGenerator;
+  }
+
   int ast_node_count() { return ast_properties_.node_count(); }
   AstProperties::Flags* flags() { return ast_properties_.flags(); }
   void set_ast_properties(AstProperties* ast_properties) {
@@ -2073,7 +2108,8 @@ class FunctionLiteral: public Expression {
                   Type type,
                   ParameterFlag has_duplicate_parameters,
                   IsFunctionFlag is_function,
-                  IsParenthesizedFlag is_parenthesized)
+                  IsParenthesizedFlag is_parenthesized,
+                  IsGeneratorFlag is_generator)
       : Expression(isolate),
         name_(name),
         scope_(scope),
@@ -2093,7 +2129,8 @@ class FunctionLiteral: public Expression {
         Pretenure::encode(false) |
         HasDuplicateParameters::encode(has_duplicate_parameters) |
         IsFunction::encode(is_function) |
-        IsParenthesized::encode(is_parenthesized);
+        IsParenthesized::encode(is_parenthesized) |
+        IsGenerator::encode(is_generator);
   }
 
  private:
@@ -2118,6 +2155,7 @@ class FunctionLiteral: public Expression {
   class HasDuplicateParameters: public BitField<ParameterFlag, 4, 1> {};
   class IsFunction: public BitField<IsFunctionFlag, 5, 1> {};
   class IsParenthesized: public BitField<IsParenthesizedFlag, 6, 1> {};
+  class IsGenerator: public BitField<IsGeneratorFlag, 7, 1> {};
 };
 
 
@@ -2916,6 +2954,12 @@ class AstNodeFactory BASE_EMBEDDED {
     VISIT_AND_RETURN(Assignment, assign)
   }
 
+  Yield* NewYield(Expression* expression, bool is_delegating_yield, int pos) {
+    Yield* yield =
+        new(zone_) Yield(isolate_, expression, is_delegating_yield, pos);
+    VISIT_AND_RETURN(Yield, yield)
+  }
+
   Throw* NewThrow(Expression* exception, int pos) {
     Throw* t = new(zone_) Throw(isolate_, exception, pos);
     VISIT_AND_RETURN(Throw, t)
@@ -2934,13 +2978,14 @@ class AstNodeFactory BASE_EMBEDDED {
       FunctionLiteral::ParameterFlag has_duplicate_parameters,
       FunctionLiteral::Type type,
       FunctionLiteral::IsFunctionFlag is_function,
-      FunctionLiteral::IsParenthesizedFlag is_parenthesized) {
+      FunctionLiteral::IsParenthesizedFlag is_parenthesized,
+      FunctionLiteral::IsGeneratorFlag is_generator) {
     FunctionLiteral* lit = new(zone_) FunctionLiteral(
         isolate_, name, scope, body,
         materialized_literal_count, expected_property_count, handler_count,
         has_only_simple_this_property_assignments, this_property_assignments,
         parameter_count, type, has_duplicate_parameters, is_function,
-        is_parenthesized);
+        is_parenthesized, is_generator);
     // Top-level literal doesn't count for the AST's properties.
     if (is_function == FunctionLiteral::kIsFunction) {
       visitor_.VisitFunctionLiteral(lit);

src/compiler.cc

@@ -1121,6 +1121,7 @@ void Compiler::SetFunctionInfo(Handle<SharedFunctionInfo> function_info,
   function_info->set_dont_optimize(lit->flags()->Contains(kDontOptimize));
   function_info->set_dont_inline(lit->flags()->Contains(kDontInline));
   function_info->set_dont_cache(lit->flags()->Contains(kDontCache));
+  function_info->set_is_generator(lit->is_generator());
 }
 
 

src/flag-definitions.h

@@ -150,6 +150,7 @@ DEFINE_bool(harmony_observation, false,
             "enable harmony object observation (implies harmony collections")
 DEFINE_bool(harmony_typed_arrays, false,
             "enable harmony typed arrays")
+DEFINE_bool(harmony_generators, false, "enable harmony generators")
 DEFINE_bool(harmony, false, "enable all harmony features (except typeof)")
 DEFINE_implication(harmony, harmony_scoping)
 DEFINE_implication(harmony, harmony_modules)
@@ -157,6 +158,7 @@ DEFINE_implication(harmony, harmony_symbols)
 DEFINE_implication(harmony, harmony_proxies)
 DEFINE_implication(harmony, harmony_collections)
 DEFINE_implication(harmony, harmony_observation)
+DEFINE_implication(harmony, harmony_generators)
 DEFINE_implication(harmony_modules, harmony_scoping)
 DEFINE_implication(harmony_observation, harmony_collections)
 DEFINE_implication(harmony, harmony_typed_arrays)

src/full-codegen.cc

@@ -232,6 +232,12 @@ void BreakableStatementChecker::VisitAssignment(Assignment* expr) {
 }
 
 
+void BreakableStatementChecker::VisitYield(Yield* expr) {
+  // Yield is breakable if the expression is.
+  Visit(expr->expression());
+}
+
+
 void BreakableStatementChecker::VisitThrow(Throw* expr) {
   // Throw is breakable if the expression is.
   Visit(expr->exception());
@@ -1538,6 +1544,28 @@ void FullCodeGenerator::VisitSharedFunctionInfoLiteral(
 }
 
 
+void FullCodeGenerator::VisitYield(Yield* expr) {
+  if (expr->is_delegating_yield())
+    UNIMPLEMENTED();
+
+  Comment cmnt(masm_, "[ Yield");
+  VisitForAccumulatorValue(expr->expression());
+  // TODO(wingo): Assert that the operand stack depth is 0, at least while
+  // general yield expressions are unimplemented.
+
+  // TODO(wingo): What follows is as in VisitReturnStatement.  Replace it with a
+  // call to a builtin that will resume the generator.
+  NestedStatement* current = nesting_stack_;
+  int stack_depth = 0;
+  int context_length = 0;
+  while (current != NULL) {
+    current = current->Exit(&stack_depth, &context_length);
+  }
+  __ Drop(stack_depth);
+  EmitReturnSequence();
+}
+
+
 void FullCodeGenerator::VisitThrow(Throw* expr) {
   Comment cmnt(masm_, "[ Throw");
   VisitForStackValue(expr->exception());

src/hydrogen.cc

@@ -4122,6 +4122,10 @@ void HOptimizedGraphBuilder::VisitExpressions(
 
 
 bool HOptimizedGraphBuilder::BuildGraph() {
+  if (info()->function()->is_generator()) {
+    Bailout("function is a generator");
+    return false;
+  }
   Scope* scope = info()->scope();
   if (scope->HasIllegalRedeclaration()) {
     Bailout("function with illegal redeclaration");
@@ -7010,6 +7014,12 @@ void HOptimizedGraphBuilder::VisitAssignment(Assignment* expr) {
 }
 
 
+void HOptimizedGraphBuilder::VisitYield(Yield* expr) {
+  // Generators are not optimized, so we should never get here.
+  UNREACHABLE();
+}
+
+
 void HOptimizedGraphBuilder::VisitThrow(Throw* expr) {
   ASSERT(!HasStackOverflow());
   ASSERT(current_block() != NULL);

src/objects-inl.h

@@ -4519,6 +4519,7 @@ BOOL_ACCESSORS(SharedFunctionInfo, compiler_hints, dont_optimize,
                kDontOptimize)
 BOOL_ACCESSORS(SharedFunctionInfo, compiler_hints, dont_inline, kDontInline)
 BOOL_ACCESSORS(SharedFunctionInfo, compiler_hints, dont_cache, kDontCache)
+BOOL_ACCESSORS(SharedFunctionInfo, compiler_hints, is_generator, kIsGenerator)
 
 void SharedFunctionInfo::BeforeVisitingPointers() {
   if (IsInobjectSlackTrackingInProgress()) DetachInitialMap();

src/objects.h

@@ -5948,6 +5948,9 @@ class SharedFunctionInfo: public HeapObject {
   // Indicates that code for this function cannot be cached.
   DECL_BOOLEAN_ACCESSORS(dont_cache)
 
+  // Indicates that this function is a generator.
+  DECL_BOOLEAN_ACCESSORS(is_generator)
+
   // Indicates whether or not the code in the shared function support
   // deoptimization.
   inline bool has_deoptimization_support();
@@ -6174,6 +6177,7 @@ class SharedFunctionInfo: public HeapObject {
     kDontOptimize,
     kDontInline,
     kDontCache,
+    kIsGenerator,
     kCompilerHintsCount  // Pseudo entry
   };
 

src/parser.cc

@@ -486,10 +486,12 @@ class Parser::BlockState BASE_EMBEDDED {
 
 Parser::FunctionState::FunctionState(Parser* parser,
                                      Scope* scope,
+                                     bool is_generator,
                                      Isolate* isolate)
     : next_materialized_literal_index_(JSFunction::kLiteralsPrefixSize),
       next_handler_index_(0),
       expected_property_count_(0),
+      is_generator_(is_generator),
       only_simple_this_property_assignments_(false),
       this_property_assignments_(isolate->factory()->empty_fixed_array()),
       parser_(parser),
@@ -642,7 +644,10 @@ FunctionLiteral* Parser::DoParseProgram(CompilationInfo* info,
     }
     ParsingModeScope parsing_mode(this, mode);
 
-    FunctionState function_state(this, scope, isolate());  // Enters 'scope'.
+    bool is_generator = false;
+    // Enters 'scope'.
+    FunctionState function_state(this, scope, is_generator, isolate());
+
     top_scope_->SetLanguageMode(info->language_mode());
     ZoneList<Statement*>* body = new(zone()) ZoneList<Statement*>(16, zone());
     bool ok = true;
@@ -680,7 +685,8 @@ FunctionLiteral* Parser::DoParseProgram(CompilationInfo* info,
           FunctionLiteral::kNoDuplicateParameters,
           FunctionLiteral::ANONYMOUS_EXPRESSION,
           FunctionLiteral::kGlobalOrEval,
-          FunctionLiteral::kNotParenthesized);
+          FunctionLiteral::kNotParenthesized,
+          FunctionLiteral::kNotGenerator);
       result->set_ast_properties(factory()->visitor()->ast_properties());
     } else if (stack_overflow_) {
       isolate()->StackOverflow();
@@ -754,7 +760,8 @@ FunctionLiteral* Parser::ParseLazy(Utf16CharacterStream* source,
       scope = Scope::DeserializeScopeChain(info()->closure()->context(), scope,
                                            zone());
     }
-    FunctionState function_state(this, scope, isolate());
+    bool is_generator = false;  // Top scope is not a generator.
+    FunctionState function_state(this, scope, is_generator, isolate());
     ASSERT(scope->language_mode() != STRICT_MODE || !info()->is_classic_mode());
     ASSERT(scope->language_mode() != EXTENDED_MODE ||
            info()->is_extended_mode());
@@ -768,6 +775,7 @@ FunctionLiteral* Parser::ParseLazy(Utf16CharacterStream* source,
     bool ok = true;
     result = ParseFunctionLiteral(name,
                                   false,  // Strict mode name already checked.
+                                  shared_info->is_generator(),
                                   RelocInfo::kNoPosition,
                                   type,
                                   &ok);
@@ -1132,6 +1140,7 @@ Statement* Parser::ParseModuleElement(ZoneStringList* labels,
   //    ModuleDeclaration
   //    ImportDeclaration
   //    ExportDeclaration
+  //    GeneratorDeclaration
 
   switch (peek()) {
     case Token::FUNCTION:
@@ -1430,6 +1439,7 @@ Statement* Parser::ParseExportDeclaration(bool* ok) {
   //    'export' Identifier (',' Identifier)* ';'
   //    'export' VariableDeclaration
   //    'export' FunctionDeclaration
+  //    'export' GeneratorDeclaration
   //    'export' ModuleDeclaration
   //
   // TODO(ES6): implement structuring ExportSpecifiers
@@ -1509,6 +1519,7 @@ Statement* Parser::ParseBlockElement(ZoneStringList* labels,
   // BlockElement (aka SourceElement):
   //    LetDeclaration
   //    ConstDeclaration
+  //    GeneratorDeclaration
 
   switch (peek()) {
     case Token::FUNCTION:
@@ -1628,6 +1639,10 @@ Statement* Parser::ParseStatement(ZoneStringList* labels, bool* ok) {
       //    FunctionDeclaration
       // Common language extension is to allow function declaration in place
       // of any statement. This language extension is disabled in strict mode.
+      //
+      // In Harmony mode, this case also handles the extension:
+      // Statement:
+      //    GeneratorDeclaration
       if (!top_scope_->is_classic_mode()) {
         ReportMessageAt(scanner().peek_location(), "strict_function",
                         Vector<const char*>::empty());
@@ -1890,13 +1905,18 @@ Statement* Parser::ParseNativeDeclaration(bool* ok) {
 Statement* Parser::ParseFunctionDeclaration(ZoneStringList* names, bool* ok) {
   // FunctionDeclaration ::
   //   'function' Identifier '(' FormalParameterListopt ')' '{' FunctionBody '}'
+  // GeneratorDeclaration ::
+  //   'function' '*' Identifier '(' FormalParameterListopt ')'
+  //      '{' FunctionBody '}'
   Expect(Token::FUNCTION, CHECK_OK);
   int function_token_position = scanner().location().beg_pos;
+  bool is_generator = FLAG_harmony_generators && Check(Token::MUL);
   bool is_strict_reserved = false;
   Handle<String> name = ParseIdentifierOrStrictReservedWord(
       &is_strict_reserved, CHECK_OK);
   FunctionLiteral* fun = ParseFunctionLiteral(name,
                                               is_strict_reserved,
+                                              is_generator,
                                               function_token_position,
                                               FunctionLiteral::DECLARATION,
                                               CHECK_OK);
@@ -3004,8 +3024,13 @@ Expression* Parser::ParseExpression(bool accept_IN, bool* ok) {
 Expression* Parser::ParseAssignmentExpression(bool accept_IN, bool* ok) {
   // AssignmentExpression ::
   //   ConditionalExpression
+  //   YieldExpression
   //   LeftHandSideExpression AssignmentOperator AssignmentExpression
 
+  if (peek() == Token::YIELD && is_generator()) {
+    return ParseYieldExpression(ok);
+  }
+
   if (fni_ != NULL) fni_->Enter();
   Expression* expression = ParseConditionalExpression(accept_IN, CHECK_OK);
 
@@ -3074,6 +3099,17 @@ Expression* Parser::ParseAssignmentExpression(bool accept_IN, bool* ok) {
 }
 
 
+Expression* Parser::ParseYieldExpression(bool* ok) {
+  // YieldExpression ::
+  //   'yield' '*'? AssignmentExpression
+  int position = scanner().peek_location().beg_pos;
+  Expect(Token::YIELD, CHECK_OK);
+  bool is_yield_star = Check(Token::MUL);
+  Expression* expression = ParseAssignmentExpression(false, CHECK_OK);
+  return factory()->NewYield(expression, is_yield_star, position);
+}
+
+
 // Precedence = 3
 Expression* Parser::ParseConditionalExpression(bool accept_IN, bool* ok) {
   // ConditionalExpression ::
@@ -3450,6 +3486,7 @@ Expression* Parser::ParseMemberWithNewPrefixesExpression(PositionStack* stack,
   if (peek() == Token::FUNCTION) {
     Expect(Token::FUNCTION, CHECK_OK);
     int function_token_position = scanner().location().beg_pos;
+    bool is_generator = FLAG_harmony_generators && Check(Token::MUL);
     Handle<String> name;
     bool is_strict_reserved_name = false;
     if (peek_any_identifier()) {
@@ -3461,6 +3498,7 @@ Expression* Parser::ParseMemberWithNewPrefixesExpression(PositionStack* stack,
         : FunctionLiteral::NAMED_EXPRESSION;
     result = ParseFunctionLiteral(name,
                                   is_strict_reserved_name,
+                                  is_generator,
                                   function_token_position,
                                   type,
                                   CHECK_OK);
@@ -3604,6 +3642,7 @@ Expression* Parser::ParsePrimaryExpression(bool* ok) {
       break;
 
     case Token::IDENTIFIER:
+    case Token::YIELD:
     case Token::FUTURE_STRICT_RESERVED_WORD: {
       Handle<String> name = ParseIdentifier(CHECK_OK);
       if (fni_ != NULL) fni_->PushVariableName(name);
@@ -4009,6 +4048,7 @@ ObjectLiteral::Property* Parser::ParseObjectLiteralGetSet(bool is_getter,
     FunctionLiteral* value =
         ParseFunctionLiteral(name,
                              false,   // reserved words are allowed here
+                             false,   // not a generator
                              RelocInfo::kNoPosition,
                              FunctionLiteral::ANONYMOUS_EXPRESSION,
                              CHECK_OK);
@@ -4310,6 +4350,7 @@ class SingletonLogger : public ParserRecorder {
 
 FunctionLiteral* Parser::ParseFunctionLiteral(Handle<String> function_name,
                                               bool name_is_strict_reserved,
+                                              bool is_generator,
                                               int function_token_position,
                                               FunctionLiteral::Type type,
                                               bool* ok) {
@@ -4344,9 +4385,12 @@ FunctionLiteral* Parser::ParseFunctionLiteral(Handle<String> function_name,
   FunctionLiteral::IsParenthesizedFlag parenthesized = parenthesized_function_
       ? FunctionLiteral::kIsParenthesized
       : FunctionLiteral::kNotParenthesized;
+  FunctionLiteral::IsGeneratorFlag generator = is_generator
+      ? FunctionLiteral::kIsGenerator
+      : FunctionLiteral::kNotGenerator;
   AstProperties ast_properties;
   // Parse function body.
-  { FunctionState function_state(this, scope, isolate());
+  { FunctionState function_state(this, scope, is_generator, isolate());
     top_scope_->SetScopeName(function_name);
 
     //  FormalParameterList ::
@@ -4584,7 +4628,8 @@ FunctionLiteral* Parser::ParseFunctionLiteral(Handle<String> function_name,
                                     duplicate_parameters,
                                     type,
                                     FunctionLiteral::kIsFunction,
-                                    parenthesized);
+                                    parenthesized,
+                                    generator);
   function_literal->set_function_token_position(function_token_position);
   function_literal->set_ast_properties(&ast_properties);
 
@@ -4606,10 +4651,12 @@ preparser::PreParser::PreParseResult Parser::LazyParseFunctionLiteral(
                                                    stack_limit,
                                                    do_allow_lazy,
                                                    allow_natives_syntax_,
-                                                   allow_modules_);
+                                                   allow_modules_,
+                                                   FLAG_harmony_generators);
   }
   preparser::PreParser::PreParseResult result =
       reusable_preparser_->PreParseLazyFunction(top_scope_->language_mode(),
+                                                is_generator(),
                                                 logger);
   return result;
 }
@@ -4672,7 +4719,8 @@ bool Parser::peek_any_identifier() {
   Token::Value next = peek();
   return next == Token::IDENTIFIER ||
          next == Token::FUTURE_RESERVED_WORD ||
-         next == Token::FUTURE_STRICT_RESERVED_WORD;
+         next == Token::FUTURE_STRICT_RESERVED_WORD ||
+         next == Token::YIELD;
 }
 
 
@@ -4744,13 +4792,17 @@ Literal* Parser::GetLiteralTheHole() {
 // Parses an identifier that is valid for the current scope, in particular it
 // fails on strict mode future reserved keywords in a strict scope.
 Handle<String> Parser::ParseIdentifier(bool* ok) {
-  if (!top_scope_->is_classic_mode()) {
-    Expect(Token::IDENTIFIER, ok);
-  } else if (!Check(Token::IDENTIFIER)) {
-    Expect(Token::FUTURE_STRICT_RESERVED_WORD, ok);
+  Token::Value next = Next();
+  if (next == Token::IDENTIFIER ||
+      (top_scope_->is_classic_mode() &&
+       (next == Token::FUTURE_STRICT_RESERVED_WORD ||
+        (next == Token::YIELD && !is_generator())))) {
+    return GetSymbol(ok);
+  } else {
+    ReportUnexpectedToken(next);
+    *ok = false;
+    return Handle<String>();
   }
-  if (!*ok) return Handle<String>();
-  return GetSymbol(ok);
 }
 
 
@@ -4758,12 +4810,17 @@ Handle<String> Parser::ParseIdentifier(bool* ok) {
 // whether it is strict mode future reserved.
 Handle<String> Parser::ParseIdentifierOrStrictReservedWord(
     bool* is_strict_reserved, bool* ok) {
-  *is_strict_reserved = false;
-  if (!Check(Token::IDENTIFIER)) {
-    Expect(Token::FUTURE_STRICT_RESERVED_WORD, ok);
+  Token::Value next = Next();
+  if (next == Token::IDENTIFIER) {
+    *is_strict_reserved = false;
+  } else if (next == Token::FUTURE_STRICT_RESERVED_WORD ||
+             (next == Token::YIELD && !is_generator())) {
     *is_strict_reserved = true;
+  } else {
+    ReportUnexpectedToken(next);
+    *ok = false;
+    return Handle<String>();
   }
-  if (!*ok) return Handle<String>();
   return GetSymbol(ok);
 }
 
@@ -5875,6 +5932,9 @@ ScriptDataImpl* ParserApi::PreParse(Utf16CharacterStream* source,
   if (FLAG_lazy && (extension == NULL)) {
     flags |= kAllowLazy;
   }
+  if (FLAG_harmony_generators) {
+    flags |= kAllowGenerators;
+  }
   CompleteParserRecorder recorder;
   return DoPreParse(source, flags, &recorder);
 }

src/parser.h

@@ -474,6 +474,7 @@ class Parser {
    public:
     FunctionState(Parser* parser,
                   Scope* scope,
+                  bool is_generator,
                   Isolate* isolate);
     ~FunctionState();
 
@@ -504,6 +505,8 @@ class Parser {
     void AddProperty() { expected_property_count_++; }
     int expected_property_count() { return expected_property_count_; }
 
+    bool is_generator() const { return is_generator_; }
+
     AstNodeFactory<AstConstructionVisitor>* factory() { return &factory_; }
 
    private:
@@ -518,6 +521,9 @@ class Parser {
     // Properties count estimation.
     int expected_property_count_;
 
+    // Indicates that this function is a generator.
+    bool is_generator_;
+
     // Keeps track of assignments to properties of this. Used for
     // optimizing constructors.
     bool only_simple_this_property_assignments_;
@@ -631,6 +637,7 @@ class Parser {
 
   Expression* ParseExpression(bool accept_IN, bool* ok);
   Expression* ParseAssignmentExpression(bool accept_IN, bool* ok);
+  Expression* ParseYieldExpression(bool* ok);
   Expression* ParseConditionalExpression(bool accept_IN, bool* ok);
   Expression* ParseBinaryExpression(int prec, bool accept_IN, bool* ok);
   Expression* ParseUnaryExpression(bool* ok);
@@ -674,6 +681,7 @@ class Parser {
   ZoneList<Expression*>* ParseArguments(bool* ok);
   FunctionLiteral* ParseFunctionLiteral(Handle<String> var_name,
                                         bool name_is_reserved,
+                                        bool is_generator,
                                         int function_token_position,
                                         FunctionLiteral::Type type,
                                         bool* ok);
@@ -703,6 +711,8 @@ class Parser {
     return scanner().Next();
   }
 
+  bool is_generator() const { return current_function_state_->is_generator(); }
+
   bool peek_any_identifier();
 
   INLINE(void Consume(Token::Value token));

src/preparser.cc

@@ -53,12 +53,13 @@ int isfinite(double value);
 namespace preparser {
 
 PreParser::PreParseResult PreParser::PreParseLazyFunction(
-    i::LanguageMode mode, i::ParserRecorder* log) {
+    i::LanguageMode mode, bool is_generator, i::ParserRecorder* log) {
   log_ = log;
   // Lazy functions always have trivial outer scopes (no with/catch scopes).
   Scope top_scope(&scope_, kTopLevelScope);
   set_language_mode(mode);
   Scope function_scope(&scope_, kFunctionScope);
+  function_scope.set_is_generator(is_generator);
   ASSERT_EQ(i::Token::LBRACE, scanner_->current_token());
   bool ok = true;
   int start_position = scanner_->peek_location().beg_pos;
@@ -154,6 +155,7 @@ PreParser::Statement PreParser::ParseSourceElement(bool* ok) {
   // SourceElement:
   //    LetDeclaration
   //    ConstDeclaration
+  //    GeneratorDeclaration
 
   switch (peek()) {
     case i::Token::FUNCTION:
@@ -294,19 +296,23 @@ PreParser::Statement PreParser::ParseStatement(bool* ok) {
 PreParser::Statement PreParser::ParseFunctionDeclaration(bool* ok) {
   // FunctionDeclaration ::
   //   'function' Identifier '(' FormalParameterListopt ')' '{' FunctionBody '}'
+  // GeneratorDeclaration ::
+  //   'function' '*' Identifier '(' FormalParameterListopt ')'
+  //      '{' FunctionBody '}'
   Expect(i::Token::FUNCTION, CHECK_OK);
 
+  bool is_generator = allow_generators_ && Check(i::Token::MUL);
   Identifier identifier = ParseIdentifier(CHECK_OK);
   i::Scanner::Location location = scanner_->location();
 
-  Expression function_value = ParseFunctionLiteral(CHECK_OK);
+  Expression function_value = ParseFunctionLiteral(is_generator, CHECK_OK);
 
   if (function_value.IsStrictFunction() &&
       !identifier.IsValidStrictVariable()) {
     // Strict mode violation, using either reserved word or eval/arguments
     // as name of strict function.
     const char* type = "strict_function_name";
-    if (identifier.IsFutureStrictReserved()) {
+    if (identifier.IsFutureStrictReserved() || identifier.IsYield()) {
       type = "strict_reserved_word";
     }
     ReportMessageAt(location, type, NULL);
@@ -475,7 +481,9 @@ PreParser::Statement PreParser::ParseExpressionOrLabelledStatement(bool* ok) {
   Expression expr = ParseExpression(true, CHECK_OK);
   if (expr.IsRawIdentifier()) {
     ASSERT(!expr.AsIdentifier().IsFutureReserved());
-    ASSERT(is_classic_mode() || !expr.AsIdentifier().IsFutureStrictReserved());
+    ASSERT(is_classic_mode() ||
+           (!expr.AsIdentifier().IsFutureStrictReserved() &&
+            !expr.AsIdentifier().IsYield()));
     if (peek() == i::Token::COLON) {
       Consume(i::Token::COLON);
       return ParseStatement(ok);
@@ -810,8 +818,13 @@ PreParser::Expression PreParser::ParseAssignmentExpression(bool accept_IN,
                                                            bool* ok) {
   // AssignmentExpression ::
   //   ConditionalExpression
+  //   YieldExpression
   //   LeftHandSideExpression AssignmentOperator AssignmentExpression
 
+  if (scope_->is_generator() && peek() == i::Token::YIELD) {
+    return ParseYieldExpression(ok);
+  }
+
   i::Scanner::Location before = scanner_->peek_location();
   Expression expression = ParseConditionalExpression(accept_IN, CHECK_OK);
 
@@ -841,6 +854,19 @@ PreParser::Expression PreParser::ParseAssignmentExpression(bool accept_IN,
 }
 
 
+// Precedence = 3
+PreParser::Expression PreParser::ParseYieldExpression(bool* ok) {
+  // YieldExpression ::
+  //   'yield' '*'? AssignmentExpression
+  Consume(i::Token::YIELD);
+  Check(i::Token::MUL);
+
+  ParseAssignmentExpression(false, CHECK_OK);
+
+  return Expression::Default();
+}
+
+
 // Precedence = 3
 PreParser::Expression PreParser::ParseConditionalExpression(bool accept_IN,
                                                             bool* ok) {
@@ -1034,11 +1060,13 @@ PreParser::Expression PreParser::ParseMemberWithNewPrefixesExpression(
   Expression result = Expression::Default();
   if (peek() == i::Token::FUNCTION) {
     Consume(i::Token::FUNCTION);
+
+    bool is_generator = allow_generators_ && Check(i::Token::MUL);
     Identifier identifier = Identifier::Default();
     if (peek_any_identifier()) {
       identifier = ParseIdentifier(CHECK_OK);
     }
-    result = ParseFunctionLiteral(CHECK_OK);
+    result = ParseFunctionLiteral(is_generator, CHECK_OK);
     if (result.IsStrictFunction() && !identifier.IsValidStrictVariable()) {
       StrictModeIdentifierViolation(scanner_->location(),
                                     "strict_function_name",
@@ -1112,6 +1140,7 @@ PreParser::Expression PreParser::ParsePrimaryExpression(bool* ok) {
 
     case i::Token::FUTURE_RESERVED_WORD:
     case i::Token::FUTURE_STRICT_RESERVED_WORD:
+    case i::Token::YIELD:
     case i::Token::IDENTIFIER: {
       Identifier id = ParseIdentifier(CHECK_OK);
       result = Expression::FromIdentifier(id);
@@ -1257,7 +1286,7 @@ PreParser::Expression PreParser::ParseObjectLiteral(bool* ok) {
             }
             PropertyType type = is_getter ? kGetterProperty : kSetterProperty;
             CheckDuplicate(&duplicate_finder, name, type, CHECK_OK);
-            ParseFunctionLiteral(CHECK_OK);
+            ParseFunctionLiteral(false, CHECK_OK);
             if (peek() != i::Token::RBRACE) {
               Expect(i::Token::COMMA, CHECK_OK);
             }
@@ -1344,7 +1373,8 @@ PreParser::Arguments PreParser::ParseArguments(bool* ok) {
 }
 
 
-PreParser::Expression PreParser::ParseFunctionLiteral(bool* ok) {
+PreParser::Expression PreParser::ParseFunctionLiteral(bool is_generator,
+                                                      bool* ok) {
   // Function ::
   //   '(' FormalParameterList? ')' '{' FunctionBody '}'
 
@@ -1352,6 +1382,7 @@ PreParser::Expression PreParser::ParseFunctionLiteral(bool* ok) {
   ScopeType outer_scope_type = scope_->type();
   bool inside_with = scope_->IsInsideWith();
   Scope function_scope(&scope_, kFunctionScope);
+  function_scope.set_is_generator(is_generator);
   //  FormalParameterList ::
   //    '(' (Identifier)*[','] ')'
   Expect(i::Token::LPAREN, CHECK_OK);
@@ -1497,6 +1528,8 @@ PreParser::Identifier PreParser::GetIdentifierSymbol() {
   } else if (scanner_->current_token() ==
              i::Token::FUTURE_STRICT_RESERVED_WORD) {
     return Identifier::FutureStrictReserved();
+  } else if (scanner_->current_token() == i::Token::YIELD) {
+    return Identifier::Yield();
   }
   if (scanner_->is_literal_ascii()) {
     // Detect strict-mode poison words.
@@ -1523,6 +1556,14 @@ PreParser::Identifier PreParser::ParseIdentifier(bool* ok) {
       *ok = false;
       return GetIdentifierSymbol();
     }
+    case i::Token::YIELD:
+      if (scope_->is_generator()) {
+        // 'yield' in a generator is only valid as part of a YieldExpression.
+        ReportMessageAt(scanner_->location(), "unexpected_token", "yield");
+        *ok = false;
+        return Identifier::Yield();
+      }
+      // FALLTHROUGH
     case i::Token::FUTURE_STRICT_RESERVED_WORD:
       if (!is_classic_mode()) {
         i::Scanner::Location location = scanner_->location();
@@ -1580,7 +1621,7 @@ void PreParser::StrictModeIdentifierViolation(i::Scanner::Location location,
   const char* type = eval_args_type;
   if (identifier.IsFutureReserved()) {
     type = "reserved_word";
-  } else if (identifier.IsFutureStrictReserved()) {
+  } else if (identifier.IsFutureStrictReserved() || identifier.IsYield()) {
     type = "strict_reserved_word";
   }
   if (!is_classic_mode()) {
@@ -1634,7 +1675,8 @@ bool PreParser::peek_any_identifier() {
   i::Token::Value next = peek();
   return next == i::Token::IDENTIFIER ||
          next == i::Token::FUTURE_RESERVED_WORD ||
-         next == i::Token::FUTURE_STRICT_RESERVED_WORD;
+         next == i::Token::FUTURE_STRICT_RESERVED_WORD ||
+         next == i::Token::YIELD;
 }
 
 

src/preparser.h

@@ -117,7 +117,8 @@ class PreParser {
             uintptr_t stack_limit,
             bool allow_lazy,
             bool allow_natives_syntax,
-            bool allow_modules)
+            bool allow_modules,
+            bool allow_generators)
       : scanner_(scanner),
         log_(log),
         scope_(NULL),
@@ -128,6 +129,7 @@ class PreParser {
         allow_lazy_(allow_lazy),
         allow_modules_(allow_modules),
         allow_natives_syntax_(allow_natives_syntax),
+        allow_generators_(allow_generators),
         parenthesized_function_(false),
         harmony_scoping_(scanner->HarmonyScoping()) { }
 
@@ -144,19 +146,22 @@ class PreParser {
     bool allow_lazy = (flags & i::kAllowLazy) != 0;
     bool allow_natives_syntax = (flags & i::kAllowNativesSyntax) != 0;
     bool allow_modules = (flags & i::kAllowModules) != 0;
+    bool allow_generators = (flags & i::kAllowGenerators) != 0;
     return PreParser(scanner, log, stack_limit, allow_lazy,
-                     allow_natives_syntax, allow_modules).PreParse();
+                     allow_natives_syntax, allow_modules,
+                     allow_generators).PreParse();
   }
 
   // Parses a single function literal, from the opening parentheses before
   // parameters to the closing brace after the body.
   // Returns a FunctionEntry describing the body of the function in enough
   // detail that it can be lazily compiled.
-  // The scanner is expected to have matched the "function" keyword and
-  // parameters, and have consumed the initial '{'.
+  // The scanner is expected to have matched the "function" or "function*"
+  // keyword and parameters, and have consumed the initial '{'.
   // At return, unless an error occurred, the scanner is positioned before the
   // the final '}'.
   PreParseResult PreParseLazyFunction(i::LanguageMode mode,
+                                      bool is_generator,
                                       i::ParserRecorder* log);
 
  private:
@@ -240,9 +245,13 @@ class PreParser {
     static Identifier FutureStrictReserved()  {
       return Identifier(kFutureStrictReservedIdentifier);
     }
+    static Identifier Yield()  {
+      return Identifier(kYieldIdentifier);
+    }
     bool IsEval() { return type_ == kEvalIdentifier; }
     bool IsArguments() { return type_ == kArgumentsIdentifier; }
     bool IsEvalOrArguments() { return type_ >= kEvalIdentifier; }
+    bool IsYield() { return type_ == kYieldIdentifier; }
     bool IsFutureReserved() { return type_ == kFutureReservedIdentifier; }
     bool IsFutureStrictReserved() {
       return type_ == kFutureStrictReservedIdentifier;
@@ -254,6 +263,7 @@ class PreParser {
       kUnknownIdentifier,
       kFutureReservedIdentifier,
       kFutureStrictReservedIdentifier,
+      kYieldIdentifier,
       kEvalIdentifier,
       kArgumentsIdentifier
     };
@@ -347,7 +357,7 @@ class PreParser {
         // Identifiers and string literals can be parenthesized.
         // They no longer work as labels or directive prologues,
         // but are still recognized in other contexts.
-        return Expression(code_ | kParentesizedExpressionFlag);
+        return Expression(code_ | kParenthesizedExpressionFlag);
       }
       // For other types of expressions, it's not important to remember
       // the parentheses.
@@ -373,7 +383,8 @@ class PreParser {
       kUseStrictString = kStringLiteralFlag | 8,
       kStringLiteralMask = kUseStrictString,
 
-      kParentesizedExpressionFlag = 4,  // Only if identifier or string literal.
+      // Only if identifier or string literal.
+      kParenthesizedExpressionFlag = 4,
 
       // Below here applies if neither identifier nor string literal.
       kThisExpression = 4,
@@ -451,7 +462,8 @@ class PreParser {
           expected_properties_(0),
           with_nesting_count_(0),
           language_mode_(
-              (prev_ != NULL) ? prev_->language_mode() : i::CLASSIC_MODE) {
+              (prev_ != NULL) ? prev_->language_mode() : i::CLASSIC_MODE),
+          is_generator_(false) {
       *variable = this;
     }
     ~Scope() { *variable_ = prev_; }
@@ -461,6 +473,8 @@ class PreParser {
     int expected_properties() { return expected_properties_; }
     int materialized_literal_count() { return materialized_literal_count_; }
     bool IsInsideWith() { return with_nesting_count_ != 0; }
+    bool is_generator() { return is_generator_; }
+    void set_is_generator(bool is_generator) { is_generator_ = is_generator; }
     bool is_classic_mode() {
       return language_mode_ == i::CLASSIC_MODE;
     }
@@ -492,6 +506,7 @@ class PreParser {
     int expected_properties_;
     int with_nesting_count_;
     i::LanguageMode language_mode_;
+    bool is_generator_;
   };
 
   // Preparse the program. Only called in PreParseProgram after creating
@@ -557,6 +572,7 @@ class PreParser {
 
   Expression ParseExpression(bool accept_IN, bool* ok);
   Expression ParseAssignmentExpression(bool accept_IN, bool* ok);
+  Expression ParseYieldExpression(bool* ok);
   Expression ParseConditionalExpression(bool accept_IN, bool* ok);
   Expression ParseBinaryExpression(int prec, bool accept_IN, bool* ok);
   Expression ParseUnaryExpression(bool* ok);
@@ -572,7 +588,7 @@ class PreParser {
   Expression ParseV8Intrinsic(bool* ok);
 
   Arguments ParseArguments(bool* ok);
-  Expression ParseFunctionLiteral(bool* ok);
+  Expression ParseFunctionLiteral(bool is_generator, bool* ok);
   void ParseLazyFunctionLiteralBody(bool* ok);
 
   Identifier ParseIdentifier(bool* ok);
@@ -664,6 +680,7 @@ class PreParser {
   bool allow_lazy_;
   bool allow_modules_;
   bool allow_natives_syntax_;
+  bool allow_generators_;
   bool parenthesized_function_;
   bool harmony_scoping_;
 };

src/prettyprinter.cc

@@ -353,6 +353,12 @@ void PrettyPrinter::VisitAssignment(Assignment* node) {
 }
 
 
+void PrettyPrinter::VisitYield(Yield* node) {
+  Print("yield ");
+  Visit(node->expression());
+}
+
+
 void PrettyPrinter::VisitThrow(Throw* node) {
   Print("throw ");
   Visit(node->exception());
@@ -1059,6 +1065,11 @@ void AstPrinter::VisitAssignment(Assignment* node) {
 }
 
 
+void AstPrinter::VisitYield(Yield* node) {
+  PrintIndentedVisit("YIELD", node->expression());
+}
+
+
 void AstPrinter::VisitThrow(Throw* node) {
   PrintIndentedVisit("THROW", node->exception());
 }

src/scanner.cc

@@ -877,7 +877,7 @@ uc32 Scanner::ScanIdentifierUnicodeEscape() {
   KEYWORD("while", Token::WHILE)                                    \
   KEYWORD("with", Token::WITH)                                      \
   KEYWORD_GROUP('y')                                                \
-  KEYWORD("yield", Token::FUTURE_STRICT_RESERVED_WORD)
+  KEYWORD("yield", Token::YIELD)
 
 
 static Token::Value KeywordOrIdentifierToken(const char* input,

src/scanner.h

@@ -53,7 +53,8 @@ enum ParsingFlags {
   kLanguageModeMask = 0x03,
   kAllowLazy = 0x04,
   kAllowNativesSyntax = 0x08,
-  kAllowModules = 0x10
+  kAllowModules = 0x10,
+  kAllowGenerators = 0x20
 };
 
 STATIC_ASSERT((kLanguageModeMask & CLASSIC_MODE) == CLASSIC_MODE);

src/token.h

@@ -174,6 +174,7 @@ namespace internal {
   K(EXPORT, "export", 0)                                                \
   K(IMPORT, "import", 0)                                                \
   K(LET, "let", 0)                                                      \
+  K(YIELD, "yield", 0)                                                  \
                                                                         \
   /* Illegal token - not able to scan. */                               \
   T(ILLEGAL, "ILLEGAL", 0)                                              \

test/mjsunit/harmony/generators-parsing.js

+// Copyright 2013 the V8 project authors. All rights reserved.
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+//       notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+//       copyright notice, this list of conditions and the following
+//       disclaimer in the documentation and/or other materials provided
+//       with the distribution.
+//     * Neither the name of Google Inc. nor the names of its
+//       contributors may be used to endorse or promote products derived
+//       from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// Flags: --harmony-generators
+
+// Test basic generator syntax.
+
+// Yield statements.
+function* g() { yield 3; yield 4; }
+
+// Yield expressions.
+function* g() { (yield 3) + (yield 4); }
+
+// You can have a generator in strict mode.
+function* g() { "use strict"; yield 3; yield 4; }
+
+// Generator expression.
+(function* () { yield 3; });
+
+// Named generator expression.
+(function* g() { yield 3; });
+
+// A generator without a yield is specified as causing an early error.  This
+// behavior is currently unimplemented.  See
+// https://bugs.ecmascript.org/show_bug.cgi?id=1283.
+function* g() { }
+
+// A YieldExpression in the RHS of a YieldExpression is currently specified as
+// causing an early error.  This behavior is currently unimplemented.  See
+// https://bugs.ecmascript.org/show_bug.cgi?id=1283.
+function* g() { yield yield 1; }
+function* g() { yield 3 + (yield 4); }
+
+// Generator definitions with a name of "yield" are not specifically ruled out
+// by the spec, as the `yield' name is outside the generator itself.  However,
+// in strict-mode, "yield" is an invalid identifier.
+function* yield() { (yield 3) + (yield 4); }
+assertThrows("function* yield() { \"use strict\"; (yield 3) + (yield 4); }",
+             SyntaxError);
+
+// In classic mode, yield is a normal identifier, outside of generators.
+function yield(yield) { yield: yield (yield + yield (0)); }
+
+// Yield is always valid as a key in an object literal.
+({ yield: 1 });
+function* g() { yield ({ yield: 1 }) }
+function* g() { yield ({ get yield() { return 1; }}) }
+
+// Checks that yield is a valid label in classic mode, but not valid in a strict
+// mode or in generators.
+function f() { yield: 1 }
+assertThrows("function f() { \"use strict\"; yield: 1 }", SyntaxError)
+assertThrows("function f*() { yield: 1 }", SyntaxError)
+
+// Yield is only a keyword in the body of the generator, not in nested
+// functions.
+function* g() { function f() { yield (yield + yield (0)); } }
+
+// Yield needs a RHS.
+assertThrows("function* g() { yield; }", SyntaxError);
+
+// Yield in a generator is not an identifier.
+assertThrows("function* g() { yield = 10; }", SyntaxError);
+
+// Yield binds very loosely, so this parses as "yield (3 + yield 4)", which is
+// invalid.
+assertThrows("function* g() { yield 3 + yield 4; }", SyntaxError);
+
+// Yield is still a future-reserved-word in strict mode
+assertThrows("function f() { \"use strict\"; var yield = 13; }", SyntaxError);
+
+// The name of the NFE is let-bound in G, so is invalid.
+assertThrows("function* g() { yield (function yield() {}); }", SyntaxError);
+
+// In generators, yield is invalid as a formal argument name.
+assertThrows("function* g(yield) { yield (10); }", SyntaxError);