Generator objects can suspend

* src/ast.h:
* src/parser.cc: Differentiate between the different kinds of yields, in
  anticipation of boxing return values.  Parse `return' into `yield' in
  a generator.

* src/runtime.h:
* src/runtime.cc (Runtime_SuspendJSGeneratorObject): New horrible
  runtime function: saves continuation, context, and operands into the
  generator object.

* src/arm/full-codegen-arm.cc (VisitYield):
* src/ia32/full-codegen-ia32.cc (VisitYield):
* src/x64/full-codegen-x64.cc (VisitYield): Arrange to call
  SuspendJSGeneratorObject.  If the call returns the hole, we suspend.
  Otherwise we resume.

BUG=v8:2355
TEST=These codepaths are tested when the generator is first invoked, and so
are covered by mjsunit/harmony/generators-objects.js.

Review URL: https://codereview.chromium.org/13704010

Patch from Andy Wingo <wingo@igalia.com>.

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@14353 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/arm/full-codegen-arm.cc

@@ -1922,6 +1922,60 @@ void FullCodeGenerator::VisitAssignment(Assignment* expr) {
 }
 
 
+void FullCodeGenerator::VisitYield(Yield* expr) {
+  Comment cmnt(masm_, "[ Yield");
+  // Evaluate yielded value first; the initial iterator definition depends on
+  // this.  It stays on the stack while we update the iterator.
+  VisitForStackValue(expr->expression());
+
+  switch (expr->yield_kind()) {
+    case Yield::INITIAL:
+    case Yield::SUSPEND: {
+      VisitForStackValue(expr->generator_object());
+      __ CallRuntime(Runtime::kSuspendJSGeneratorObject, 1);
+      __ ldr(context_register(),
+             MemOperand(fp, StandardFrameConstants::kContextOffset));
+
+      Label resume;
+      __ CompareRoot(result_register(), Heap::kTheHoleValueRootIndex);
+      __ b(ne, &resume);
+      __ pop(result_register());
+      if (expr->yield_kind() == Yield::SUSPEND) {
+        // TODO(wingo): Box into { value: VALUE, done: false }.
+      }
+      EmitReturnSequence();
+
+      __ bind(&resume);
+      context()->Plug(result_register());
+      break;
+    }
+
+    case Yield::FINAL: {
+      VisitForAccumulatorValue(expr->generator_object());
+      __ mov(r1, Operand(Smi::FromInt(JSGeneratorObject::kGeneratorClosed)));
+      __ str(r1, FieldMemOperand(result_register(),
+                                 JSGeneratorObject::kContinuationOffset));
+      __ pop(result_register());
+      // TODO(wingo): Box into { value: VALUE, done: true }.
+
+      // Exit all nested statements.
+      NestedStatement* current = nesting_stack_;
+      int stack_depth = 0;
+      int context_length = 0;
+      while (current != NULL) {
+        current = current->Exit(&stack_depth, &context_length);
+      }
+      __ Drop(stack_depth);
+      EmitReturnSequence();
+      break;
+    }
+
+    case Yield::DELEGATING:
+      UNIMPLEMENTED();
+  }
+}
+
+
 void FullCodeGenerator::EmitNamedPropertyLoad(Property* prop) {
   SetSourcePosition(prop->position());
   Literal* key = prop->key()->AsLiteral();

src/ast.h

@@ -1964,27 +1964,34 @@ class Yield: public Expression {
  public:
   DECLARE_NODE_TYPE(Yield)
 
+  enum Kind {
+    INITIAL,     // The initial yield that returns the unboxed generator object.
+    SUSPEND,     // A normal yield: { value: EXPRESSION, done: false }
+    DELEGATING,  // A yield*.
+    FINAL        // A return: { value: EXPRESSION, done: true }
+  };
+
   Expression* generator_object() const { return generator_object_; }
   Expression* expression() const { return expression_; }
-  bool is_delegating_yield() const { return is_delegating_yield_; }
+  Kind yield_kind() const { return yield_kind_; }
   virtual int position() const { return pos_; }
 
  protected:
   Yield(Isolate* isolate,
         Expression* generator_object,
         Expression* expression,
-        bool is_delegating_yield,
+        Kind yield_kind,
         int pos)
       : Expression(isolate),
         generator_object_(generator_object),
         expression_(expression),
-        is_delegating_yield_(is_delegating_yield),
+        yield_kind_(yield_kind),
         pos_(pos) { }
 
  private:
   Expression* generator_object_;
   Expression* expression_;
-  bool is_delegating_yield_;
+  Kind yield_kind_;
   int pos_;
 };
 
@@ -2966,10 +2973,10 @@ class AstNodeFactory BASE_EMBEDDED {
 
   Yield* NewYield(Expression *generator_object,
                   Expression* expression,
-                  bool is_delegating_yield,
+                  Yield::Kind yield_kind,
                   int pos) {
     Yield* yield = new(zone_) Yield(
-        isolate_, generator_object, expression, is_delegating_yield, pos);
+        isolate_, generator_object, expression, yield_kind, pos);
     VISIT_AND_RETURN(Yield, yield)
   }
 

src/full-codegen.cc

@@ -1548,30 +1548,6 @@ void FullCodeGenerator::VisitSharedFunctionInfoLiteral(
 }
 
 
-void FullCodeGenerator::VisitYield(Yield* expr) {
-  if (expr->is_delegating_yield())
-    UNIMPLEMENTED();
-
-  Comment cmnt(masm_, "[ Yield");
-  // TODO(wingo): Actually update the iterator state.
-  VisitForEffect(expr->generator_object());
-  VisitForAccumulatorValue(expr->expression());
-  // TODO(wingo): Assert that the operand stack depth is 0, at least while
-  // general yield expressions are unimplemented.
-
-  // TODO(wingo): What follows is as in VisitReturnStatement.  Replace it with a
-  // call to a builtin that will resume the generator.
-  NestedStatement* current = nesting_stack_;
-  int stack_depth = 0;
-  int context_length = 0;
-  while (current != NULL) {
-    current = current->Exit(&stack_depth, &context_length);
-  }
-  __ Drop(stack_depth);
-  EmitReturnSequence();
-}
-
-
 void FullCodeGenerator::VisitThrow(Throw* expr) {
   Comment cmnt(masm_, "[ Throw");
   VisitForStackValue(expr->exception());

src/ia32/full-codegen-ia32.cc

@@ -1883,6 +1883,60 @@ void FullCodeGenerator::VisitAssignment(Assignment* expr) {
 }
 
 
+void FullCodeGenerator::VisitYield(Yield* expr) {
+  Comment cmnt(masm_, "[ Yield");
+  // Evaluate yielded value first; the initial iterator definition depends on
+  // this.  It stays on the stack while we update the iterator.
+  VisitForStackValue(expr->expression());
+
+  switch (expr->yield_kind()) {
+    case Yield::INITIAL:
+    case Yield::SUSPEND: {
+      VisitForStackValue(expr->generator_object());
+      __ CallRuntime(Runtime::kSuspendJSGeneratorObject, 1);
+      __ mov(context_register(),
+             Operand(ebp, StandardFrameConstants::kContextOffset));
+
+      Label resume;
+      __ CompareRoot(result_register(), Heap::kTheHoleValueRootIndex);
+      __ j(not_equal, &resume);
+      __ pop(result_register());
+      if (expr->yield_kind() == Yield::SUSPEND) {
+        // TODO(wingo): Box into { value: VALUE, done: false }.
+      }
+      EmitReturnSequence();
+
+      __ bind(&resume);
+      context()->Plug(result_register());
+      break;
+    }
+
+    case Yield::FINAL: {
+      VisitForAccumulatorValue(expr->generator_object());
+      __ mov(FieldOperand(result_register(),
+                          JSGeneratorObject::kContinuationOffset),
+             Immediate(Smi::FromInt(JSGeneratorObject::kGeneratorClosed)));
+      __ pop(result_register());
+      // TODO(wingo): Box into { value: VALUE, done: true }.
+
+      // Exit all nested statements.
+      NestedStatement* current = nesting_stack_;
+      int stack_depth = 0;
+      int context_length = 0;
+      while (current != NULL) {
+        current = current->Exit(&stack_depth, &context_length);
+      }
+      __ Drop(stack_depth);
+      EmitReturnSequence();
+      break;
+    }
+
+    case Yield::DELEGATING:
+      UNIMPLEMENTED();
+  }
+}
+
+
 void FullCodeGenerator::EmitNamedPropertyLoad(Property* prop) {
   SetSourcePosition(prop->position());
   Literal* key = prop->key()->AsLiteral();

src/objects-inl.h

@@ -5029,7 +5029,7 @@ void Foreign::set_foreign_address(Address value) {
 
 
 ACCESSORS(JSGeneratorObject, function, JSFunction, kFunctionOffset)
-ACCESSORS(JSGeneratorObject, context, Object, kContextOffset)
+ACCESSORS(JSGeneratorObject, context, Context, kContextOffset)
 SMI_ACCESSORS(JSGeneratorObject, continuation, kContinuationOffset)
 ACCESSORS(JSGeneratorObject, operand_stack, FixedArray, kOperandStackOffset)
 

src/objects.h

@@ -6297,10 +6297,14 @@ class JSGeneratorObject: public JSObject {
   // [function]: The function corresponding to this generator object.
   DECL_ACCESSORS(function, JSFunction)
 
-  // [context]: The context of the suspended computation, or undefined.
-  DECL_ACCESSORS(context, Object)
+  // [context]: The context of the suspended computation.
+  DECL_ACCESSORS(context, Context)
 
   // [continuation]: Offset into code of continuation.
+  //
+  // A positive offset indicates a suspended generator.  The special
+  // kGeneratorExecuting and kGeneratorClosed values indicate that a generator
+  // cannot be resumed.
   inline int continuation();
   inline void set_continuation(int continuation);
 
@@ -6314,6 +6318,10 @@ class JSGeneratorObject: public JSObject {
   DECLARE_PRINTER(JSGeneratorObject)
   DECLARE_VERIFIER(JSGeneratorObject)
 
+  // Magic sentinel values for the continuation.
+  static const int kGeneratorExecuting = -1;
+  static const int kGeneratorClosed = 0;
+
   // Layout description.
   static const int kFunctionOffset = JSObject::kHeaderSize;
   static const int kContextOffset = kFunctionOffset + kPointerSize;

src/parser.cc

@@ -2511,16 +2511,24 @@ Statement* Parser::ParseReturnStatement(bool* ok) {
 
   Token::Value tok = peek();
   Statement* result;
+  Expression* return_value;
   if (scanner().HasAnyLineTerminatorBeforeNext() ||
       tok == Token::SEMICOLON ||
       tok == Token::RBRACE ||
       tok == Token::EOS) {
-    ExpectSemicolon(CHECK_OK);
-    result = factory()->NewReturnStatement(GetLiteralUndefined());
+    return_value = GetLiteralUndefined();
   } else {
-    Expression* expr = ParseExpression(true, CHECK_OK);
-    ExpectSemicolon(CHECK_OK);
-    result = factory()->NewReturnStatement(expr);
+    return_value = ParseExpression(true, CHECK_OK);
+  }
+  ExpectSemicolon(CHECK_OK);
+  if (is_generator()) {
+    Expression* generator = factory()->NewVariableProxy(
+        current_function_state_->generator_object_variable());
+    Expression* yield = factory()->NewYield(
+        generator, return_value, Yield::FINAL, RelocInfo::kNoPosition);
+    result = factory()->NewExpressionStatement(yield);
+  } else {
+    result = factory()->NewReturnStatement(return_value);
   }
 
   // An ECMAScript program is considered syntactically incorrect if it
@@ -3100,12 +3108,12 @@ Expression* Parser::ParseYieldExpression(bool* ok) {
   //   'yield' '*'? AssignmentExpression
   int position = scanner().peek_location().beg_pos;
   Expect(Token::YIELD, CHECK_OK);
-  bool is_yield_star = Check(Token::MUL);
+  Yield::Kind kind =
+      Check(Token::MUL) ? Yield::DELEGATING : Yield::SUSPEND;
   Expression* generator_object = factory()->NewVariableProxy(
       current_function_state_->generator_object_variable());
   Expression* expression = ParseAssignmentExpression(false, CHECK_OK);
-  return factory()->NewYield(generator_object, expression, is_yield_star,
-                             position);
+  return factory()->NewYield(generator_object, expression, kind, position);
 }
 
 
@@ -4591,12 +4599,22 @@ FunctionLiteral* Parser::ParseFunctionLiteral(Handle<String> function_name,
         VariableProxy* get_proxy = factory()->NewVariableProxy(
             current_function_state_->generator_object_variable());
         Yield* yield = factory()->NewYield(
-            get_proxy, assignment, false, RelocInfo::kNoPosition);
+            get_proxy, assignment, Yield::INITIAL, RelocInfo::kNoPosition);
         body->Add(factory()->NewExpressionStatement(yield), zone());
       }
 
       ParseSourceElements(body, Token::RBRACE, false, false, CHECK_OK);
 
+      if (is_generator) {
+        VariableProxy* get_proxy = factory()->NewVariableProxy(
+            current_function_state_->generator_object_variable());
+        Expression *undefined = factory()->NewLiteral(
+            isolate()->factory()->undefined_value());
+        Yield* yield = factory()->NewYield(
+            get_proxy, undefined, Yield::FINAL, RelocInfo::kNoPosition);
+        body->Add(factory()->NewExpressionStatement(yield), zone());
+      }
+
       materialized_literal_count = function_state.materialized_literal_count();
       expected_property_count = function_state.expected_property_count();
       handler_count = function_state.handler_count();

src/runtime.cc

@@ -2397,6 +2397,7 @@ RUNTIME_FUNCTION(MaybeObject*, Runtime_SetExpectedNumberOfProperties) {
 RUNTIME_FUNCTION(MaybeObject*, Runtime_CreateJSGeneratorObject) {
   NoHandleAllocation ha(isolate);
   ASSERT(args.length() == 0);
+
   JavaScriptFrameIterator it(isolate);
   JavaScriptFrame* frame = it.frame();
   JSFunction* function = JSFunction::cast(frame->function());
@@ -2411,7 +2412,7 @@ RUNTIME_FUNCTION(MaybeObject*, Runtime_CreateJSGeneratorObject) {
     if (!maybe_generator->To(&generator)) return maybe_generator;
   }
   generator->set_function(function);
-  generator->set_context(isolate->heap()->undefined_value());
+  generator->set_context(Context::cast(frame->context()));
   generator->set_continuation(0);
   generator->set_operand_stack(isolate->heap()->empty_fixed_array());
 
@@ -2419,6 +2420,61 @@ RUNTIME_FUNCTION(MaybeObject*, Runtime_CreateJSGeneratorObject) {
 }
 
 
+RUNTIME_FUNCTION(MaybeObject*, Runtime_SuspendJSGeneratorObject) {
+  HandleScope scope(isolate);
+  ASSERT(args.length() == 1);
+  CONVERT_ARG_HANDLE_CHECKED(JSGeneratorObject, generator_object, 0);
+
+  JavaScriptFrameIterator stack_iterator(isolate);
+  JavaScriptFrame *frame = stack_iterator.frame();
+  Handle<JSFunction> function(JSFunction::cast(frame->function()));
+  RUNTIME_ASSERT(function->shared()->is_generator());
+
+  intptr_t offset = frame->pc() - function->code()->instruction_start();
+  ASSERT(*function == generator_object->function());
+  ASSERT(offset > 0 && Smi::IsValid(offset));
+  generator_object->set_continuation(offset);
+
+  // Generator functions force context allocation for locals, so Local0 points
+  // to the bottom of the operand stack.  Assume the stack grows down.
+  //
+  // TODO(wingo): Move these magical calculations to frames.h when the
+  // generators implementation has stabilized.
+  intptr_t stack_size_in_bytes =
+      (frame->fp() + JavaScriptFrameConstants::kLocal0Offset) -
+      (frame->sp() - kPointerSize);
+  ASSERT(IsAddressAligned(frame->fp(), kPointerSize));
+  ASSERT(IsAligned(stack_size_in_bytes, kPointerSize));
+  ASSERT(stack_size_in_bytes >= 0);
+  ASSERT(Smi::IsValid(stack_size_in_bytes));
+  unsigned stack_size = stack_size_in_bytes >> kPointerSizeLog2;
+
+  // We expect there to be at least two values on the stack: the return value of
+  // the yield expression, and the argument to this runtime call.  Neither of
+  // those should be saved.
+  ASSERT(stack_size >= 2);
+  stack_size -= 2;
+
+  if (stack_size == 0) {
+    ASSERT_EQ(generator_object->operand_stack(),
+              isolate->heap()->empty_fixed_array());
+    // If there are no operands on the stack, there shouldn't be a handler
+    // active either.  Also, the active context will be the same as the function
+    // itself, so there is no need to save the context.
+    ASSERT_EQ(frame->context(), generator_object->context());
+    ASSERT(!frame->HasHandler());
+  } else {
+    generator_object->set_context(Context::cast(frame->context()));
+    // TODO(wingo): Save the operand stack and/or the stack handlers.
+    UNIMPLEMENTED();
+  }
+
+  // The return value is the hole for a suspend return, and anything else for a
+  // resume return.
+  return isolate->heap()->the_hole_value();
+}
+
+
 MUST_USE_RESULT static MaybeObject* CharFromCode(Isolate* isolate,
                                                  Object* char_code) {
   if (char_code->IsNumber()) {

src/runtime.h

@@ -298,6 +298,7 @@ namespace internal {
   \
   /* Harmony generators */ \
   F(CreateJSGeneratorObject, 0, 1) \
+  F(SuspendJSGeneratorObject, 1, 1) \
   \
   /* Harmony modules */ \
   F(IsJSModule, 1, 1) \

src/x64/full-codegen-x64.cc

@@ -1907,6 +1907,60 @@ void FullCodeGenerator::VisitAssignment(Assignment* expr) {
 }
 
 
+void FullCodeGenerator::VisitYield(Yield* expr) {
+  Comment cmnt(masm_, "[ Yield");
+  // Evaluate yielded value first; the initial iterator definition depends on
+  // this.  It stays on the stack while we update the iterator.
+  VisitForStackValue(expr->expression());
+
+  switch (expr->yield_kind()) {
+    case Yield::INITIAL:
+    case Yield::SUSPEND: {
+      VisitForStackValue(expr->generator_object());
+      __ CallRuntime(Runtime::kSuspendJSGeneratorObject, 1);
+      __ movq(context_register(),
+              Operand(rbp, StandardFrameConstants::kContextOffset));
+
+      Label resume;
+      __ CompareRoot(result_register(), Heap::kTheHoleValueRootIndex);
+      __ j(not_equal, &resume);
+      __ pop(result_register());
+      if (expr->yield_kind() == Yield::SUSPEND) {
+        // TODO(wingo): Box into { value: VALUE, done: false }.
+      }
+      EmitReturnSequence();
+
+      __ bind(&resume);
+      context()->Plug(result_register());
+      break;
+    }
+
+    case Yield::FINAL: {
+      VisitForAccumulatorValue(expr->generator_object());
+      __ Move(FieldOperand(result_register(),
+                           JSGeneratorObject::kContinuationOffset),
+              Smi::FromInt(JSGeneratorObject::kGeneratorClosed));
+      __ pop(result_register());
+      // TODO(wingo): Box into { value: VALUE, done: true }.
+
+      // Exit all nested statements.
+      NestedStatement* current = nesting_stack_;
+      int stack_depth = 0;
+      int context_length = 0;
+      while (current != NULL) {
+        current = current->Exit(&stack_depth, &context_length);
+      }
+      __ Drop(stack_depth);
+      EmitReturnSequence();
+      break;
+    }
+
+    case Yield::DELEGATING:
+      UNIMPLEMENTED();
+  }
+}
+
+
 void FullCodeGenerator::EmitNamedPropertyLoad(Property* prop) {
   SetSourcePosition(prop->position());
   Literal* key = prop->key()->AsLiteral();

test/mjsunit/harmony/generators-parsing.js

@@ -38,6 +38,12 @@ function* g() { (yield 3) + (yield 4); }
 // You can have a generator in strict mode.
 function* g() { "use strict"; yield 3; yield 4; }
 
+// Generators can have return statements also, which internally parse to a kind
+// of yield expression.
+function* g() { yield 1; return; }
+function* g() { yield 1; return 2; }
+function* g() { yield 1; return 2; yield "dead"; }
+
 // Generator expression.
 (function* () { yield 3; });