Added allocation folding support for old space allocations.

BUG=
R=mstarzinger@chromium.org, titzer@chromium.org

Review URL: https://codereview.chromium.org/22378003

git-svn-id: http://v8.googlecode.com/svn/branches/bleeding_edge@16346 ce2b1a6d-e550-0410-aec6-3dcde31c8c00

src/hydrogen-instructions.cc

@@ -2398,6 +2398,24 @@ HConstant::HConstant(Handle<Object> handle,
 }
 
 
+HConstant::HConstant(Handle<Map> handle,
+                     UniqueValueId unique_id)
+  : HTemplateInstruction<0>(HType::Tagged()),
+    handle_(handle),
+    unique_id_(unique_id),
+    has_smi_value_(false),
+    has_int32_value_(false),
+    has_double_value_(false),
+    has_external_reference_value_(false),
+    is_internalized_string_(false),
+    is_not_in_new_space_(true),
+    is_cell_(false),
+    boolean_value_(false) {
+  ASSERT(!handle.is_null());
+  Initialize(Representation::Tagged());
+}
+
+
 HConstant::HConstant(int32_t integer_value,
                      Representation r,
                      bool is_not_in_new_space,
@@ -3212,6 +3230,7 @@ Representation HUnaryMathOperation::RepresentationFromInputs() {
 void HAllocate::HandleSideEffectDominator(GVNFlag side_effect,
                                           HValue* dominator) {
   ASSERT(side_effect == kChangesNewSpacePromotion);
+  Zone* zone = block()->zone();
   if (!FLAG_use_allocation_folding) return;
 
   // Try to fold allocations together with their dominating allocations.
@@ -3223,22 +3242,34 @@ void HAllocate::HandleSideEffectDominator(GVNFlag side_effect,
     return;
   }
 
-  HAllocate* dominator_allocate_instr = HAllocate::cast(dominator);
-  HValue* dominator_size = dominator_allocate_instr->size();
+  HAllocate* dominator_allocate = HAllocate::cast(dominator);
+  HValue* dominator_size = dominator_allocate->size();
   HValue* current_size = size();
-  // We can just fold allocations that are guaranteed in new space.
+
   // TODO(hpayer): Add support for non-constant allocation in dominator.
-  if (!IsNewSpaceAllocation() || !current_size->IsInteger32Constant() ||
-      !dominator_allocate_instr->IsNewSpaceAllocation() ||
+  if (!current_size->IsInteger32Constant() ||
       !dominator_size->IsInteger32Constant()) {
     if (FLAG_trace_allocation_folding) {
-      PrintF("#%d (%s) cannot fold into #%d (%s)\n",
+      PrintF("#%d (%s) cannot fold into #%d (%s), dynamic allocation size\n",
           id(), Mnemonic(), dominator->id(), dominator->Mnemonic());
     }
     return;
   }
 
+  dominator_allocate = GetFoldableDominator(dominator_allocate);
+  if (dominator_allocate == NULL) {
+    return;
+  }
+
+  ASSERT((IsNewSpaceAllocation() &&
+         dominator_allocate->IsNewSpaceAllocation()) ||
+         (IsOldDataSpaceAllocation() &&
+         dominator_allocate->IsOldDataSpaceAllocation()) ||
+         (IsOldPointerSpaceAllocation() &&
+         dominator_allocate->IsOldPointerSpaceAllocation()));
+
   // First update the size of the dominator allocate instruction.
+  dominator_size = dominator_allocate->size();
   int32_t dominator_size_constant =
       HConstant::cast(dominator_size)->GetInteger32Constant();
   int32_t current_size_constant =
@@ -3246,8 +3277,8 @@ void HAllocate::HandleSideEffectDominator(GVNFlag side_effect,
   int32_t new_dominator_size = dominator_size_constant + current_size_constant;
 
   if (MustAllocateDoubleAligned()) {
-    if (!dominator_allocate_instr->MustAllocateDoubleAligned()) {
-      dominator_allocate_instr->MakeDoubleAligned();
+    if (!dominator_allocate->MustAllocateDoubleAligned()) {
+      dominator_allocate->MakeDoubleAligned();
     }
     if ((dominator_size_constant & kDoubleAlignmentMask) != 0) {
       dominator_size_constant += kDoubleSize / 2;
@@ -3258,21 +3289,19 @@ void HAllocate::HandleSideEffectDominator(GVNFlag side_effect,
   if (new_dominator_size > Page::kMaxNonCodeHeapObjectSize) {
     if (FLAG_trace_allocation_folding) {
       PrintF("#%d (%s) cannot fold into #%d (%s) due to size: %d\n",
-          id(), Mnemonic(), dominator->id(), dominator->Mnemonic(),
-          new_dominator_size);
+          id(), Mnemonic(), dominator_allocate->id(),
+          dominator_allocate->Mnemonic(), new_dominator_size);
     }
     return;
   }
-  HBasicBlock* block = dominator->block();
-  Zone* zone = block->zone();
-  HInstruction* new_dominator_size_constant =
-      HConstant::New(zone, context(), new_dominator_size);
-  new_dominator_size_constant->InsertBefore(dominator_allocate_instr);
-  dominator_allocate_instr->UpdateSize(new_dominator_size_constant);
+
+  HInstruction* new_dominator_size_constant = HConstant::CreateAndInsertBefore(
+      zone, context(), new_dominator_size, dominator_allocate);
+  dominator_allocate->UpdateSize(new_dominator_size_constant);
 
 #ifdef VERIFY_HEAP
-  if (FLAG_verify_heap) {
-    dominator_allocate_instr->MakePrefillWithFiller();
+  if (FLAG_verify_heap && dominator_allocate->IsNewSpaceAllocation()) {
+    dominator_allocate->MakePrefillWithFiller();
   }
 #endif
 
@@ -3280,18 +3309,128 @@ void HAllocate::HandleSideEffectDominator(GVNFlag side_effect,
   HInstruction* dominated_allocate_instr =
       HInnerAllocatedObject::New(zone,
                                  context(),
-                                 dominator_allocate_instr,
+                                 dominator_allocate,
                                  dominator_size_constant,
                                  type());
   dominated_allocate_instr->InsertBefore(this);
   DeleteAndReplaceWith(dominated_allocate_instr);
   if (FLAG_trace_allocation_folding) {
     PrintF("#%d (%s) folded into #%d (%s)\n",
-        id(), Mnemonic(), dominator->id(), dominator->Mnemonic());
+        id(), Mnemonic(), dominator_allocate->id(),
+        dominator_allocate->Mnemonic());
   }
 }
 
 
+HAllocate* HAllocate::GetFoldableDominator(HAllocate* dominator) {
+  if (!IsFoldable(dominator)) {
+    // We cannot hoist old space allocations over new space allocations.
+    if (IsNewSpaceAllocation() || dominator->IsNewSpaceAllocation()) {
+      if (FLAG_trace_allocation_folding) {
+        PrintF("#%d (%s) cannot fold into #%d (%s), new space hoisting\n",
+            id(), Mnemonic(), dominator->id(), dominator->Mnemonic());
+      }
+      return NULL;
+    }
+
+    HAllocate* dominator_dominator = dominator->dominating_allocate_;
+
+    // We can hoist old data space allocations over an old pointer space
+    // allocation and vice versa. For that we have to check the dominator
+    // of the dominator allocate instruction.
+    if (dominator_dominator == NULL) {
+      dominating_allocate_ = dominator;
+      if (FLAG_trace_allocation_folding) {
+        PrintF("#%d (%s) cannot fold into #%d (%s), different spaces\n",
+            id(), Mnemonic(), dominator->id(), dominator->Mnemonic());
+      }
+      return NULL;
+    }
+
+    // We can just fold old space allocations that are in the same basic block,
+    // since it is not guaranteed that we fill up the whole allocated old
+    // space memory.
+    // TODO(hpayer): Remove this limitation and add filler maps for each each
+    // allocation as soon as we have store elimination.
+    if (block()->block_id() != dominator_dominator->block()->block_id()) {
+      if (FLAG_trace_allocation_folding) {
+        PrintF("#%d (%s) cannot fold into #%d (%s), different basic blocks\n",
+            id(), Mnemonic(), dominator_dominator->id(),
+            dominator_dominator->Mnemonic());
+      }
+      return NULL;
+    }
+
+    ASSERT((IsOldDataSpaceAllocation() &&
+           dominator_dominator->IsOldDataSpaceAllocation()) ||
+           (IsOldPointerSpaceAllocation() &&
+           dominator_dominator->IsOldPointerSpaceAllocation()));
+
+    int32_t current_size = HConstant::cast(size())->GetInteger32Constant();
+    HStoreNamedField* dominator_free_space_size =
+        dominator->filler_free_space_size_;
+    if (dominator_free_space_size != NULL) {
+      // We already hoisted one old space allocation, i.e., we already installed
+      // a filler map. Hence, we just have to update the free space size.
+      dominator->UpdateFreeSpaceFiller(current_size);
+    } else {
+      // This is the first old space allocation that gets hoisted. We have to
+      // install a filler map since the follwing allocation may cause a GC.
+      dominator->CreateFreeSpaceFiller(current_size);
+    }
+
+    // We can hoist the old space allocation over the actual dominator.
+    return dominator_dominator;
+  }
+  return dominator;
+}
+
+
+void HAllocate::UpdateFreeSpaceFiller(int32_t free_space_size) {
+  ASSERT(filler_free_space_size_ != NULL);
+  Zone* zone = block()->zone();
+  HConstant* new_free_space_size = HConstant::CreateAndInsertBefore(
+      zone,
+      context(),
+      filler_free_space_size_->value()->GetInteger32Constant() +
+          free_space_size,
+      filler_free_space_size_);
+  filler_free_space_size_->UpdateValue(new_free_space_size);
+}
+
+
+void HAllocate::CreateFreeSpaceFiller(int32_t free_space_size) {
+  ASSERT(filler_free_space_size_ == NULL);
+  Zone* zone = block()->zone();
+  int32_t dominator_size =
+      HConstant::cast(dominating_allocate_->size())->GetInteger32Constant();
+  HInstruction* free_space_instr =
+      HInnerAllocatedObject::New(zone, context(), dominating_allocate_,
+      dominator_size, type());
+  free_space_instr->InsertBefore(this);
+  HConstant* filler_map = HConstant::New(
+      zone,
+      context(),
+      isolate()->factory()->free_space_map(),
+      UniqueValueId(isolate()->heap()->free_space_map()));
+  filler_map->InsertAfter(free_space_instr);
+  HInstruction* store_map = HStoreNamedField::New(zone, context(),
+      free_space_instr, HObjectAccess::ForMap(), filler_map);
+  store_map->SetFlag(HValue::kHasNoObservableSideEffects);
+  store_map->InsertAfter(filler_map);
+
+  HConstant* filler_size = HConstant::CreateAndInsertAfter(
+      zone, context(), free_space_size, store_map);
+  HObjectAccess access =
+      HObjectAccess::ForJSObjectOffset(FreeSpace::kSizeOffset);
+  HStoreNamedField* store_size = HStoreNamedField::New(zone, context(),
+      free_space_instr, access, filler_size);
+  store_size->SetFlag(HValue::kHasNoObservableSideEffects);
+  store_size->InsertAfter(filler_size);
+  filler_free_space_size_ = store_size;
+}
+
+
 void HAllocate::PrintDataTo(StringStream* stream) {
   size()->PrintNameTo(stream);
   stream->Add(" (");

src/hydrogen-instructions.h

@@ -49,11 +49,11 @@ class HEnvironment;
 class HInferRepresentationPhase;
 class HInstruction;
 class HLoopInformation;
+class HStoreNamedField;
 class HValue;
 class LInstruction;
 class LChunkBuilder;
 
-
 #define HYDROGEN_ABSTRACT_INSTRUCTION_LIST(V)  \
   V(ArithmeticBinaryOperation)                 \
   V(BinaryOperation)                           \
@@ -3240,8 +3240,27 @@ class HConstant V8_FINAL : public HTemplateInstruction<0> {
   DECLARE_INSTRUCTION_FACTORY_P2(HConstant, int32_t, Representation);
   DECLARE_INSTRUCTION_FACTORY_P1(HConstant, double);
   DECLARE_INSTRUCTION_FACTORY_P1(HConstant, Handle<Object>);
+  DECLARE_INSTRUCTION_FACTORY_P2(HConstant, Handle<Map>, UniqueValueId);
   DECLARE_INSTRUCTION_FACTORY_P1(HConstant, ExternalReference);
 
+  static HConstant* CreateAndInsertAfter(Zone* zone,
+                                         HValue* context,
+                                         int32_t value,
+                                         HInstruction* instruction) {
+    HConstant* new_constant = HConstant::New(zone, context, value);
+    new_constant->InsertAfter(instruction);
+    return new_constant;
+  }
+
+  static HConstant* CreateAndInsertBefore(Zone* zone,
+                                          HValue* context,
+                                          int32_t value,
+                                          HInstruction* instruction) {
+    HConstant* new_constant = HConstant::New(zone, context, value);
+    new_constant->InsertBefore(instruction);
+    return new_constant;
+  }
+
   Handle<Object> handle() {
     if (handle_.is_null()) {
       Factory* factory = Isolate::Current()->factory();
@@ -3444,6 +3463,8 @@ class HConstant V8_FINAL : public HTemplateInstruction<0> {
             bool is_not_in_new_space,
             bool is_cell,
             bool boolean_value);
+  HConstant(Handle<Map> handle,
+            UniqueValueId unique_id);
   explicit HConstant(ExternalReference reference);
 
   void Initialize(Representation r);
@@ -5111,10 +5132,6 @@ class HAllocate V8_FINAL : public HTemplateInstruction<2> {
     flags_ = static_cast<HAllocate::Flags>(flags_ | ALLOCATE_DOUBLE_ALIGNED);
   }
 
-  void UpdateSize(HValue* size) {
-    SetOperandAt(1, size);
-  }
-
   virtual void HandleSideEffectDominator(GVNFlag side_effect,
                                          HValue* dominator) V8_OVERRIDE;
 
@@ -5136,7 +5153,9 @@ class HAllocate V8_FINAL : public HTemplateInstruction<2> {
             HType type,
             PretenureFlag pretenure_flag,
             InstanceType instance_type)
-      : HTemplateInstruction<2>(type) {
+      : HTemplateInstruction<2>(type),
+        dominating_allocate_(NULL),
+        filler_free_space_size_(NULL) {
     SetOperandAt(0, context);
     SetOperandAt(1, size);
     set_representation(Representation::Tagged());
@@ -5153,8 +5172,27 @@ class HAllocate V8_FINAL : public HTemplateInstruction<2> {
     }
   }
 
+  void UpdateSize(HValue* size) {
+    SetOperandAt(1, size);
+  }
+
+  HAllocate* GetFoldableDominator(HAllocate* dominator);
+
+  void UpdateFreeSpaceFiller(int32_t filler_size);
+
+  void CreateFreeSpaceFiller(int32_t filler_size);
+
+  bool IsFoldable(HAllocate* allocate) {
+    return (IsNewSpaceAllocation() && allocate->IsNewSpaceAllocation()) ||
+        (IsOldDataSpaceAllocation() && allocate->IsOldDataSpaceAllocation()) ||
+        (IsOldPointerSpaceAllocation() &&
+            allocate->IsOldPointerSpaceAllocation());
+  }
+
   Flags flags_;
   Handle<Map> known_initial_map_;
+  HAllocate* dominating_allocate_;
+  HStoreNamedField* filler_free_space_size_;
 };
 
 
@@ -6001,6 +6039,10 @@ class HStoreNamedField V8_FINAL : public HTemplateInstruction<3> {
     return access_.representation();
   }
 
+  void UpdateValue(HValue* value) {
+    SetOperandAt(1, value);
+  }
+
  private:
   HStoreNamedField(HValue* obj,
                    HObjectAccess access,

test/cctest/test-heap.cc

@@ -2115,6 +2115,78 @@ TEST(OptimizedAllocationAlwaysInNewSpace) {
 }
 
 
+TEST(OptimizedPretenuringAllocationFolding) {
+  i::FLAG_allow_natives_syntax = true;
+  CcTest::InitializeVM();
+  if (!i::V8::UseCrankshaft() || i::FLAG_always_opt) return;
+  if (i::FLAG_gc_global || i::FLAG_stress_compaction) return;
+  v8::HandleScope scope(CcTest::isolate());
+  HEAP->SetNewSpaceHighPromotionModeActive(true);
+
+  v8::Local<v8::Value> res = CompileRun(
+      "function DataObject() {"
+      "  this.a = 1.1;"
+      "  this.b = [{}];"
+      "  this.c = 1.2;"
+      "  this.d = [{}];"
+      "  this.e = 1.3;"
+      "  this.f = [{}];"
+      "}"
+      "function f() {"
+      "  return new DataObject();"
+      "};"
+      "f(); f(); f();"
+      "%OptimizeFunctionOnNextCall(f);"
+      "f();");
+
+  Handle<JSObject> o =
+      v8::Utils::OpenHandle(*v8::Handle<v8::Object>::Cast(res));
+
+  CHECK(HEAP->InOldDataSpace(o->RawFastPropertyAt(0)));
+  CHECK(HEAP->InOldPointerSpace(o->RawFastPropertyAt(1)));
+  CHECK(HEAP->InOldDataSpace(o->RawFastPropertyAt(2)));
+  CHECK(HEAP->InOldPointerSpace(o->RawFastPropertyAt(3)));
+  CHECK(HEAP->InOldDataSpace(o->RawFastPropertyAt(4)));
+  CHECK(HEAP->InOldPointerSpace(o->RawFastPropertyAt(5)));
+}
+
+
+TEST(OptimizedPretenuringAllocationFoldingBlocks) {
+  i::FLAG_allow_natives_syntax = true;
+  CcTest::InitializeVM();
+  if (!i::V8::UseCrankshaft() || i::FLAG_always_opt) return;
+  if (i::FLAG_gc_global || i::FLAG_stress_compaction) return;
+  v8::HandleScope scope(CcTest::isolate());
+  HEAP->SetNewSpaceHighPromotionModeActive(true);
+
+  v8::Local<v8::Value> res = CompileRun(
+      "function DataObject() {"
+      "  this.a = [{}];"
+      "  this.b = [{}];"
+      "  this.c = 1.1;"
+      "  this.d = 1.2;"
+      "  this.e = [{}];"
+      "  this.f = 1.3;"
+      "}"
+      "function f() {"
+      "  return new DataObject();"
+      "};"
+      "f(); f(); f();"
+      "%OptimizeFunctionOnNextCall(f);"
+      "f();");
+
+  Handle<JSObject> o =
+      v8::Utils::OpenHandle(*v8::Handle<v8::Object>::Cast(res));
+
+  CHECK(HEAP->InOldPointerSpace(o->RawFastPropertyAt(0)));
+  CHECK(HEAP->InOldPointerSpace(o->RawFastPropertyAt(1)));
+  CHECK(HEAP->InOldDataSpace(o->RawFastPropertyAt(2)));
+  CHECK(HEAP->InOldDataSpace(o->RawFastPropertyAt(3)));
+  CHECK(HEAP->InOldPointerSpace(o->RawFastPropertyAt(4)));
+  CHECK(HEAP->InOldDataSpace(o->RawFastPropertyAt(5)));
+}
+
+
 TEST(OptimizedPretenuringObjectArrayLiterals) {
   i::FLAG_allow_natives_syntax = true;
   CcTest::InitializeVM();