Calling a generator function returns a generator object

* src/heap.h:
* src/heap.cc:
* src/objects-debug.cc:
* src/objects-inl.h:
* src/objects-printer.cc:
* src/objects-visiting.cc:
* src/objects.cc:
* src/objects.h: Define a new object type, JSGeneratorObject.

* src/factory.h:
* src/factory.cc (NewFunctionFromSharedFunctionInfo): Generator function
  inital maps construct the new JS_GENERATOR_OBJECT_TYPE objects, not
  generic JSObjects.

* src/runtime.h:
* src/runtime.cc (Runtime_CreateJSGeneratorObject):
* src/arm/full-codegen-arm.cc (Generate):
* src/ia32/full-codegen-ia32.cc (Generate):
* src/x64/full-codegen-x64.cc (Generate): Before visiting generator
  bodies, arrange to construct and return a generator object.

* test/mjsunit/harmony/generators-objects.js: Add tests for the
  properties and prototype of generator objects.

BUG=v8:2355
TEST=mjsunit/harmony/generators-objects

Review URL: https://codereview.chromium.org/13542002

Patch from Andy Wingo <wingo@igalia.com>.

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@14264 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/ast.h

@@ -1964,21 +1964,25 @@ class Yield: public Expression {
  public:
   DECLARE_NODE_TYPE(Yield)
 
+  Expression* generator_object() const { return generator_object_; }
   Expression* expression() const { return expression_; }
   bool is_delegating_yield() const { return is_delegating_yield_; }
   virtual int position() const { return pos_; }
 
  protected:
   Yield(Isolate* isolate,
+        Expression* generator_object,
         Expression* expression,
         bool is_delegating_yield,
         int pos)
       : Expression(isolate),
+        generator_object_(generator_object),
         expression_(expression),
         is_delegating_yield_(is_delegating_yield),
         pos_(pos) { }
 
  private:
+  Expression* generator_object_;
   Expression* expression_;
   bool is_delegating_yield_;
   int pos_;
@@ -2960,9 +2964,12 @@ class AstNodeFactory BASE_EMBEDDED {
     VISIT_AND_RETURN(Assignment, assign)
   }
 
-  Yield* NewYield(Expression* expression, bool is_delegating_yield, int pos) {
-    Yield* yield =
-        new(zone_) Yield(isolate_, expression, is_delegating_yield, pos);
+  Yield* NewYield(Expression *generator_object,
+                  Expression* expression,
+                  bool is_delegating_yield,
+                  int pos) {
+    Yield* yield = new(zone_) Yield(
+        isolate_, generator_object, expression, is_delegating_yield, pos);
     VISIT_AND_RETURN(Yield, yield)
   }
 

src/full-codegen.cc

@@ -1553,6 +1553,8 @@ void FullCodeGenerator::VisitYield(Yield* expr) {
     UNIMPLEMENTED();
 
   Comment cmnt(masm_, "[ Yield");
+  // TODO(wingo): Actually update the iterator state.
+  VisitForEffect(expr->generator_object());
   VisitForAccumulatorValue(expr->expression());
   // TODO(wingo): Assert that the operand stack depth is 0, at least while
   // general yield expressions are unimplemented.

src/heap.cc

@@ -4101,9 +4101,8 @@ MaybeObject* Heap::AllocateInitialMap(JSFunction* fun) {
   int instance_size;
   int in_object_properties;
   if (fun->shared()->is_generator()) {
-    // TODO(wingo): Replace with JS_GENERATOR_OBJECT_TYPE.
-    instance_type = JS_OBJECT_TYPE;
-    instance_size = JSObject::kHeaderSize;
+    instance_type = JS_GENERATOR_OBJECT_TYPE;
+    instance_size = JSGeneratorObject::kSize;
     in_object_properties = 0;
   } else {
     instance_type = JS_OBJECT_TYPE;
@@ -4352,6 +4351,22 @@ MaybeObject* Heap::AllocateJSObjectWithAllocationSite(JSFunction* constructor,
 }
 
 
+MaybeObject* Heap::AllocateJSGeneratorObject(JSFunction *function) {
+  ASSERT(function->shared()->is_generator());
+  Map *map;
+  if (function->has_initial_map()) {
+    map = function->initial_map();
+  } else {
+    // Allocate the initial map if absent.
+    MaybeObject* maybe_map = AllocateInitialMap(function);
+    if (!maybe_map->To(&map)) return maybe_map;
+    function->set_initial_map(map);
+  }
+  ASSERT(map->instance_type() == JS_GENERATOR_OBJECT_TYPE);
+  return AllocateJSObjectFromMap(map);
+}
+
+
 MaybeObject* Heap::AllocateJSModule(Context* context, ScopeInfo* scope_info) {
   // Allocate a fresh map. Modules do not have a prototype.
   Map* map;

src/heap.h

@@ -625,6 +625,9 @@ class Heap {
       JSFunction* constructor,
       Handle<Object> allocation_site_info_payload);
 
+  MUST_USE_RESULT MaybeObject* AllocateJSGeneratorObject(
+      JSFunction* function);
+
   MUST_USE_RESULT MaybeObject* AllocateJSModule(Context* context,
                                                 ScopeInfo* scope_info);
 

src/objects-debug.cc

@@ -139,6 +139,9 @@ void HeapObject::HeapObjectVerify() {
     case JS_CONTEXT_EXTENSION_OBJECT_TYPE:
       JSObject::cast(this)->JSObjectVerify();
       break;
+    case JS_GENERATOR_OBJECT_TYPE:
+      JSGeneratorObject::cast(this)->JSGeneratorObjectVerify();
+      break;
     case JS_MODULE_TYPE:
       JSModule::cast(this)->JSModuleVerify();
       break;
@@ -404,6 +407,17 @@ void FixedDoubleArray::FixedDoubleArrayVerify() {
 }
 
 
+void JSGeneratorObject::JSGeneratorObjectVerify() {
+  // In an expression like "new g()", there can be a point where a generator
+  // object is allocated but its fields are all undefined, as it hasn't yet been
+  // initialized by the generator.  Hence these weak checks.
+  VerifyObjectField(kFunctionOffset);
+  VerifyObjectField(kContextOffset);
+  VerifyObjectField(kOperandStackOffset);
+  VerifyObjectField(kContinuationOffset);
+}
+
+
 void JSModule::JSModuleVerify() {
   VerifyObjectField(kContextOffset);
   VerifyObjectField(kScopeInfoOffset);

src/objects-inl.h

@@ -653,6 +653,7 @@ TYPE_CHECKER(Code, CODE_TYPE)
 TYPE_CHECKER(Oddball, ODDBALL_TYPE)
 TYPE_CHECKER(JSGlobalPropertyCell, JS_GLOBAL_PROPERTY_CELL_TYPE)
 TYPE_CHECKER(SharedFunctionInfo, SHARED_FUNCTION_INFO_TYPE)
+TYPE_CHECKER(JSGeneratorObject, JS_GENERATOR_OBJECT_TYPE)
 TYPE_CHECKER(JSModule, JS_MODULE_TYPE)
 TYPE_CHECKER(JSValue, JS_VALUE_TYPE)
 TYPE_CHECKER(JSDate, JS_DATE_TYPE)
@@ -1584,6 +1585,8 @@ int JSObject::GetHeaderSize() {
   // field operations considerably on average.
   if (type == JS_OBJECT_TYPE) return JSObject::kHeaderSize;
   switch (type) {
+    case JS_GENERATOR_OBJECT_TYPE:
+      return JSGeneratorObject::kSize;
     case JS_MODULE_TYPE:
       return JSModule::kSize;
     case JS_GLOBAL_PROXY_TYPE:
@@ -5010,6 +5013,19 @@ void Foreign::set_foreign_address(Address value) {
 }
 
 
+ACCESSORS(JSGeneratorObject, function, JSFunction, kFunctionOffset)
+ACCESSORS(JSGeneratorObject, context, Object, kContextOffset)
+SMI_ACCESSORS(JSGeneratorObject, continuation, kContinuationOffset)
+ACCESSORS(JSGeneratorObject, operand_stack, FixedArray, kOperandStackOffset)
+
+
+JSGeneratorObject* JSGeneratorObject::cast(Object* obj) {
+  ASSERT(obj->IsJSGeneratorObject());
+  ASSERT(HeapObject::cast(obj)->Size() == JSGeneratorObject::kSize);
+  return reinterpret_cast<JSGeneratorObject*>(obj);
+}
+
+
 ACCESSORS(JSModule, context, Object, kContextOffset)
 ACCESSORS(JSModule, scope_info, ScopeInfo, kScopeInfoOffset)
 

src/objects-printer.cc

@@ -132,6 +132,7 @@ void HeapObject::HeapObjectPrint(FILE* out) {
     case JS_OBJECT_TYPE:  // fall through
     case JS_CONTEXT_EXTENSION_OBJECT_TYPE:
     case JS_ARRAY_TYPE:
+    case JS_GENERATOR_OBJECT_TYPE:
     case JS_REGEXP_TYPE:
       JSObject::cast(this)->JSObjectPrint(out);
       break;
@@ -530,6 +531,7 @@ static const char* TypeToString(InstanceType type) {
     case ODDBALL_TYPE: return "ODDBALL";
     case JS_GLOBAL_PROPERTY_CELL_TYPE: return "JS_GLOBAL_PROPERTY_CELL";
     case SHARED_FUNCTION_INFO_TYPE: return "SHARED_FUNCTION_INFO";
+    case JS_GENERATOR_OBJECT_TYPE: return "JS_GENERATOR_OBJECT";
     case JS_MODULE_TYPE: return "JS_MODULE";
     case JS_FUNCTION_TYPE: return "JS_FUNCTION";
     case CODE_TYPE: return "CODE";

src/objects-visiting.cc

@@ -136,6 +136,7 @@ StaticVisitorBase::VisitorId StaticVisitorBase::GetVisitorId(
 
     case JS_OBJECT_TYPE:
     case JS_CONTEXT_EXTENSION_OBJECT_TYPE:
+    case JS_GENERATOR_OBJECT_TYPE:
     case JS_MODULE_TYPE:
     case JS_VALUE_TYPE:
     case JS_DATE_TYPE:

src/objects.cc

@@ -1290,6 +1290,10 @@ void JSObject::JSObjectShortPrint(StringStream* accumulator) {
       }
       break;
     }
+    case JS_GENERATOR_OBJECT_TYPE: {
+      accumulator->Add("<JS Generator>");
+      break;
+    }
     case JS_MODULE_TYPE: {
       accumulator->Add("<JS Module>");
       break;
@@ -1546,6 +1550,7 @@ void HeapObject::IterateBody(InstanceType type, int object_size,
       break;
     case JS_OBJECT_TYPE:
     case JS_CONTEXT_EXTENSION_OBJECT_TYPE:
+    case JS_GENERATOR_OBJECT_TYPE:
     case JS_MODULE_TYPE:
     case JS_VALUE_TYPE:
     case JS_DATE_TYPE:

src/objects.h

@@ -62,6 +62,7 @@
 //           - JSWeakMap
 //           - JSRegExp
 //           - JSFunction
+//           - JSGeneratorObject
 //           - JSModule
 //           - GlobalObject
 //             - JSGlobalObject
@@ -395,6 +396,7 @@ const int kStubMinorKeyBits = kBitsPerInt - kSmiTagSize - kStubMajorKeyBits;
   V(JS_DATE_TYPE)                                                              \
   V(JS_OBJECT_TYPE)                                                            \
   V(JS_CONTEXT_EXTENSION_OBJECT_TYPE)                                          \
+  V(JS_GENERATOR_OBJECT_TYPE)                                                  \
   V(JS_MODULE_TYPE)                                                            \
   V(JS_GLOBAL_OBJECT_TYPE)                                                     \
   V(JS_BUILTINS_OBJECT_TYPE)                                                   \
@@ -726,6 +728,7 @@ enum InstanceType {
   JS_DATE_TYPE,
   JS_OBJECT_TYPE,
   JS_CONTEXT_EXTENSION_OBJECT_TYPE,
+  JS_GENERATOR_OBJECT_TYPE,
   JS_MODULE_TYPE,
   JS_GLOBAL_OBJECT_TYPE,
   JS_BUILTINS_OBJECT_TYPE,
@@ -953,13 +956,14 @@ class MaybeObject BASE_EMBEDDED {
   V(JSReceiver)                                \
   V(JSObject)                                  \
   V(JSContextExtensionObject)                  \
+  V(JSGeneratorObject)                         \
   V(JSModule)                                  \
   V(Map)                                       \
   V(DescriptorArray)                           \
   V(TransitionArray)                           \
   V(DeoptimizationInputData)                   \
   V(DeoptimizationOutputData)                  \
-  V(DependentCode)                            \
+  V(DependentCode)                             \
   V(TypeFeedbackCells)                         \
   V(FixedArray)                                \
   V(FixedDoubleArray)                          \
@@ -6255,6 +6259,40 @@ class SharedFunctionInfo: public HeapObject {
 };
 
 
+class JSGeneratorObject: public JSObject {
+ public:
+  // [function]: The function corresponding to this generator object.
+  DECL_ACCESSORS(function, JSFunction)
+
+  // [context]: The context of the suspended computation, or undefined.
+  DECL_ACCESSORS(context, Object)
+
+  // [continuation]: Offset into code of continuation.
+  inline int continuation();
+  inline void set_continuation(int continuation);
+
+  // [operands]: Saved operand stack.
+  DECL_ACCESSORS(operand_stack, FixedArray)
+
+  // Casting.
+  static inline JSGeneratorObject* cast(Object* obj);
+
+  // Dispatched behavior.
+  DECLARE_PRINTER(JSGeneratorObject)
+  DECLARE_VERIFIER(JSGeneratorObject)
+
+  // Layout description.
+  static const int kFunctionOffset = JSObject::kHeaderSize;
+  static const int kContextOffset = kFunctionOffset + kPointerSize;
+  static const int kContinuationOffset = kContextOffset + kPointerSize;
+  static const int kOperandStackOffset = kContinuationOffset + kPointerSize;
+  static const int kSize = kOperandStackOffset + kPointerSize;
+
+ private:
+  DISALLOW_IMPLICIT_CONSTRUCTORS(JSGeneratorObject);
+};
+
+
 // Representation for module instance objects.
 class JSModule: public JSObject {
  public:

src/parser.cc

@@ -486,14 +486,13 @@ class Parser::BlockState BASE_EMBEDDED {
 
 Parser::FunctionState::FunctionState(Parser* parser,
                                      Scope* scope,
-                                     bool is_generator,
                                      Isolate* isolate)
     : next_materialized_literal_index_(JSFunction::kLiteralsPrefixSize),
       next_handler_index_(0),
       expected_property_count_(0),
-      is_generator_(is_generator),
       only_simple_this_property_assignments_(false),
       this_property_assignments_(isolate->factory()->empty_fixed_array()),
+      generator_object_variable_(NULL),
       parser_(parser),
       outer_function_state_(parser->current_function_state_),
       outer_scope_(parser->top_scope_),
@@ -642,9 +641,8 @@ FunctionLiteral* Parser::DoParseProgram(CompilationInfo* info,
     }
     ParsingModeScope parsing_mode(this, mode);
 
-    bool is_generator = false;
     // Enters 'scope'.
-    FunctionState function_state(this, scope, is_generator, isolate());
+    FunctionState function_state(this, scope, isolate());
 
     top_scope_->SetLanguageMode(info->language_mode());
     ZoneList<Statement*>* body = new(zone()) ZoneList<Statement*>(16, zone());
@@ -758,8 +756,7 @@ FunctionLiteral* Parser::ParseLazy(Utf16CharacterStream* source,
       scope = Scope::DeserializeScopeChain(info()->closure()->context(), scope,
                                            zone());
     }
-    bool is_generator = false;  // Top scope is not a generator.
-    FunctionState function_state(this, scope, is_generator, isolate());
+    FunctionState function_state(this, scope, isolate());
     ASSERT(scope->language_mode() != STRICT_MODE || !info()->is_classic_mode());
     ASSERT(scope->language_mode() != EXTENDED_MODE ||
            info()->is_extended_mode());
@@ -3103,8 +3100,11 @@ Expression* Parser::ParseYieldExpression(bool* ok) {
   int position = scanner().peek_location().beg_pos;
   Expect(Token::YIELD, CHECK_OK);
   bool is_yield_star = Check(Token::MUL);
+  Expression* generator_object = factory()->NewVariableProxy(
+      current_function_state_->generator_object_variable());
   Expression* expression = ParseAssignmentExpression(false, CHECK_OK);
-  return factory()->NewYield(expression, is_yield_star, position);
+  return factory()->NewYield(generator_object, expression, is_yield_star,
+                             position);
 }
 
 
@@ -4389,11 +4389,24 @@ FunctionLiteral* Parser::ParseFunctionLiteral(Handle<String> function_name,
       : FunctionLiteral::kNotGenerator;
   AstProperties ast_properties;
   // Parse function body.
-  { FunctionState function_state(this, scope, is_generator, isolate());
+  { FunctionState function_state(this, scope, isolate());
     top_scope_->SetScopeName(function_name);
-    // For generators, allocating variables in contexts is currently a win
-    // because it minimizes the work needed to suspend and resume an activation.
-    if (is_generator) top_scope_->ForceContextAllocation();
+
+    if (is_generator) {
+      // For generators, allocating variables in contexts is currently a win
+      // because it minimizes the work needed to suspend and resume an
+      // activation.
+      top_scope_->ForceContextAllocation();
+
+      // Calling a generator returns a generator object.  That object is stored
+      // in a temporary variable, a definition that is used by "yield"
+      // expressions.  Presence of a variable for the generator object in the
+      // FunctionState indicates that this function is a generator.
+      Handle<String> tempname = isolate()->factory()->InternalizeOneByteString(
+          STATIC_ASCII_VECTOR(".generator_object"));
+      Variable* temp = top_scope_->DeclarationScope()->NewTemporary(tempname);
+      function_state.set_generator_object_variable(temp);
+    }
 
     //  FormalParameterList ::
     //    '(' (Identifier)*[','] ')'
@@ -4551,6 +4564,26 @@ FunctionLiteral* Parser::ParseFunctionLiteral(Handle<String> function_name,
                                      RelocInfo::kNoPosition)),
                                      zone());
       }
+
+      // For generators, allocate and yield an iterator on function entry.
+      if (is_generator) {
+        ZoneList<Expression*>* arguments =
+            new(zone()) ZoneList<Expression*>(0, zone());
+        CallRuntime* allocation = factory()->NewCallRuntime(
+            isolate()->factory()->empty_string(),
+            Runtime::FunctionForId(Runtime::kCreateJSGeneratorObject),
+            arguments);
+        VariableProxy* init_proxy = factory()->NewVariableProxy(
+            current_function_state_->generator_object_variable());
+        Assignment* assignment = factory()->NewAssignment(
+            Token::INIT_VAR, init_proxy, allocation, RelocInfo::kNoPosition);
+        VariableProxy* get_proxy = factory()->NewVariableProxy(
+            current_function_state_->generator_object_variable());
+        Yield* yield = factory()->NewYield(
+            get_proxy, assignment, false, RelocInfo::kNoPosition);
+        body->Add(factory()->NewExpressionStatement(yield), zone());
+      }
+
       ParseSourceElements(body, Token::RBRACE, false, false, CHECK_OK);
 
       materialized_literal_count = function_state.materialized_literal_count();

src/parser.h

@@ -488,7 +488,6 @@ class Parser BASE_EMBEDDED {
    public:
     FunctionState(Parser* parser,
                   Scope* scope,
-                  bool is_generator,
                   Isolate* isolate);
     ~FunctionState();
 
@@ -519,7 +518,17 @@ class Parser BASE_EMBEDDED {
     void AddProperty() { expected_property_count_++; }
     int expected_property_count() { return expected_property_count_; }
 
-    bool is_generator() const { return is_generator_; }
+    void set_generator_object_variable(Variable *variable) {
+      ASSERT(variable != NULL);
+      ASSERT(!is_generator());
+      generator_object_variable_ = variable;
+    }
+    Variable* generator_object_variable() const {
+      return generator_object_variable_;
+    }
+    bool is_generator() const {
+      return generator_object_variable_ != NULL;
+    }
 
     AstNodeFactory<AstConstructionVisitor>* factory() { return &factory_; }
 
@@ -535,14 +544,16 @@ class Parser BASE_EMBEDDED {
     // Properties count estimation.
     int expected_property_count_;
 
-    // Indicates that this function is a generator.
-    bool is_generator_;
-
     // Keeps track of assignments to properties of this. Used for
     // optimizing constructors.
     bool only_simple_this_property_assignments_;
     Handle<FixedArray> this_property_assignments_;
 
+    // For generators, the variable that holds the generator object.  This
+    // variable is used by yield expressions and return statements.  NULL
+    // indicates that this function is not a generator.
+    Variable* generator_object_variable_;
+
     Parser* parser_;
     FunctionState* outer_function_state_;
     Scope* outer_scope_;

src/runtime.cc

@@ -2292,6 +2292,31 @@ RUNTIME_FUNCTION(MaybeObject*, Runtime_SetExpectedNumberOfProperties) {
 }
 
 
+RUNTIME_FUNCTION(MaybeObject*, Runtime_CreateJSGeneratorObject) {
+  NoHandleAllocation ha(isolate);
+  ASSERT(args.length() == 0);
+  JavaScriptFrameIterator it(isolate);
+  JavaScriptFrame* frame = it.frame();
+  JSFunction* function = JSFunction::cast(frame->function());
+  RUNTIME_ASSERT(function->shared()->is_generator());
+
+  JSGeneratorObject* generator;
+  if (frame->IsConstructor()) {
+    generator = JSGeneratorObject::cast(frame->receiver());
+  } else {
+    MaybeObject* maybe_generator =
+        isolate->heap()->AllocateJSGeneratorObject(function);
+    if (!maybe_generator->To(&generator)) return maybe_generator;
+  }
+  generator->set_function(function);
+  generator->set_context(isolate->heap()->undefined_value());
+  generator->set_continuation(0);
+  generator->set_operand_stack(isolate->heap()->empty_fixed_array());
+
+  return generator;
+}
+
+
 MUST_USE_RESULT static MaybeObject* CharFromCode(Isolate* isolate,
                                                  Object* char_code) {
   uint32_t code;

src/runtime.h

@@ -295,6 +295,9 @@ namespace internal {
   F(CreateArrayLiteral, 3, 1) \
   F(CreateArrayLiteralShallow, 3, 1) \
   \
+  /* Harmony generators */ \
+  F(CreateJSGeneratorObject, 0, 1) \
+  \
   /* Harmony modules */ \
   F(IsJSModule, 1, 1) \
   \

test/mjsunit/harmony/generators-instantiation.js → test/mjsunit/harmony/generators-objects.js

@@ -45,5 +45,24 @@ function TestContextAllocation() {
   g4();
   g5(["foo"]);
 }
-
 TestContextAllocation();
+
+
+// Test the properties and prototype of a generator object.
+function TestGeneratorObject() {
+  function* g() { yield 1; }
+
+  var iter = g();
+  assertSame(g.prototype, Object.getPrototypeOf(iter));
+  assertTrue(iter instanceof g);
+  assertEquals([], Object.getOwnPropertyNames(iter));
+  assertTrue(iter !== g());
+
+  // g() is the same as new g().
+  iter = new g();
+  assertSame(g.prototype, Object.getPrototypeOf(iter));
+  assertTrue(iter instanceof g);
+  assertEquals([], Object.getOwnPropertyNames(iter));
+  assertTrue(iter !== new g());
+}
+TestGeneratorObject();