[turbofan] Lower FastNew*Elements operators to inline allocations.

Further optimize the

  new Array(n)

expressions by also inlining the backing store allocation into TurboFan
optimized code. Currently these backing store allocations cannot be
eliminated by escape analysis and can also not be optimized further by
the memory optimizer (i.e. no allocation folding and not write barrier
elimination) yet, but this can be done in follow-up CLs.

This yields another ~5% improvement on the ARES6 ML benchmark (steady
state).

Drive-by-fix: Add support for loops to the GraphAssembler. This was
necessary to implement the initialization loops in for the backing
store allocations in the EffectControlLinearizer.

Bug: v8:6399, v8:6901
Change-Id: I759d6802db01eb797e78c7d82d82caaee3463e16
Reviewed-on: https://chromium-review.googlesource.com/705934Reviewed-by: Jaroslav Sevcik <jarin@chromium.org>
Reviewed-by: Benedikt Meurer <bmeurer@chromium.org>
Commit-Queue: Benedikt Meurer <bmeurer@chromium.org>
Cr-Commit-Position: refs/heads/master@{#48352}

src/builtins/builtins-definitions.h

@@ -167,12 +167,6 @@ namespace internal {
   TFC(GrowFastDoubleElements, GrowArrayElements, 1)                            \
   TFC(GrowFastSmiOrObjectElements, GrowArrayElements, 1)                       \
   TFC(NewArgumentsElements, NewArgumentsElements, 1)                           \
-  /* TODO(bmeurer): Remove these four builtins when the operations are */      \
-  /* inlined into TurboFan, which requires Loop support in GraphAssembler. */  \
-  TFS(NewFastDoubleElements_NotTenured, kLength)                               \
-  TFS(NewFastDoubleElements_Tenured, kLength)                                  \
-  TFS(NewFastSmiOrObjectElements_NotTenured, kLength)                          \
-  TFS(NewFastSmiOrObjectElements_Tenured, kLength)                             \
                                                                                \
   /* Debugger */                                                               \
   ASM(FrameDropperTrampoline)                                                  \

src/builtins/builtins-internal-gen.cc

@@ -94,88 +94,6 @@ TF_BUILTIN(GrowFastSmiOrObjectElements, CodeStubAssembler) {
   TailCallRuntime(Runtime::kGrowArrayElements, context, object, key);
 }
 
-// TODO(bmeurer): Temporary builtin, will disappear soon.
-TF_BUILTIN(NewFastDoubleElements_NotTenured, CodeStubAssembler) {
-  Node* length = SmiUntag(Parameter(Descriptor::kLength));
-
-  Node* const zero = IntPtrConstant(0);
-  ElementsKind const elements_kind = PACKED_DOUBLE_ELEMENTS;
-
-  Label if_empty(this, Label::kDeferred), if_nonempty(this);
-  Branch(WordEqual(length, zero), &if_empty, &if_nonempty);
-
-  BIND(&if_empty);
-  Return(EmptyFixedArrayConstant());
-
-  BIND(&if_nonempty);
-  Node* const elements = AllocateFixedArray(elements_kind, length);
-  FillFixedArrayWithValue(elements_kind, elements, zero, length,
-                          Heap::kTheHoleValueRootIndex);
-  Return(elements);
-}
-
-// TODO(bmeurer): Temporary builtin, will disappear soon.
-TF_BUILTIN(NewFastDoubleElements_Tenured, CodeStubAssembler) {
-  Node* length = SmiUntag(Parameter(Descriptor::kLength));
-
-  Node* const zero = IntPtrConstant(0);
-  ElementsKind const elements_kind = PACKED_DOUBLE_ELEMENTS;
-
-  Label if_empty(this, Label::kDeferred), if_nonempty(this);
-  Branch(WordEqual(length, zero), &if_empty, &if_nonempty);
-
-  BIND(&if_empty);
-  Return(EmptyFixedArrayConstant());
-
-  BIND(&if_nonempty);
-  Node* const elements =
-      AllocateFixedArray(elements_kind, length, INTPTR_PARAMETERS, kPretenured);
-  FillFixedArrayWithValue(elements_kind, elements, zero, length,
-                          Heap::kTheHoleValueRootIndex);
-  Return(elements);
-}
-
-// TODO(bmeurer): Temporary builtin, will disappear soon.
-TF_BUILTIN(NewFastSmiOrObjectElements_NotTenured, CodeStubAssembler) {
-  Node* length = SmiUntag(Parameter(Descriptor::kLength));
-
-  Node* const zero = IntPtrConstant(0);
-  ElementsKind const elements_kind = PACKED_ELEMENTS;
-
-  Label if_empty(this, Label::kDeferred), if_nonempty(this);
-  Branch(WordEqual(length, zero), &if_empty, &if_nonempty);
-
-  BIND(&if_empty);
-  Return(EmptyFixedArrayConstant());
-
-  BIND(&if_nonempty);
-  Node* const elements = AllocateFixedArray(elements_kind, length);
-  FillFixedArrayWithValue(elements_kind, elements, zero, length,
-                          Heap::kTheHoleValueRootIndex);
-  Return(elements);
-}
-
-// TODO(bmeurer): Temporary builtin, will disappear soon.
-TF_BUILTIN(NewFastSmiOrObjectElements_Tenured, CodeStubAssembler) {
-  Node* length = SmiUntag(Parameter(Descriptor::kLength));
-
-  Node* const zero = IntPtrConstant(0);
-  ElementsKind const elements_kind = PACKED_ELEMENTS;
-
-  Label if_empty(this, Label::kDeferred), if_nonempty(this);
-  Branch(WordEqual(length, zero), &if_empty, &if_nonempty);
-
-  BIND(&if_empty);
-  Return(EmptyFixedArrayConstant());
-
-  BIND(&if_nonempty);
-  Node* const elements =
-      AllocateFixedArray(elements_kind, length, INTPTR_PARAMETERS, kPretenured);
-  FillFixedArrayWithValue(elements_kind, elements, zero, length,
-                          Heap::kTheHoleValueRootIndex);
-  Return(elements);
-}
-
 TF_BUILTIN(NewArgumentsElements, CodeStubAssembler) {
   Node* frame = Parameter(Descriptor::kFrame);
   Node* length = SmiToWord(Parameter(Descriptor::kLength));

src/compiler/effect-control-linearizer.cc

@@ -2225,32 +2225,87 @@ Node* EffectControlLinearizer::LowerNewFastDoubleElements(Node* node) {
   PretenureFlag const pretenure = PretenureFlagOf(node->op());
   Node* length = node->InputAt(0);
 
-  Callable const callable = Builtins::CallableFor(
-      isolate(), pretenure == NOT_TENURED
-                     ? Builtins::kNewFastDoubleElements_NotTenured
-                     : Builtins::kNewFastDoubleElements_Tenured);
-  Operator::Properties const properties = node->op()->properties();
-  CallDescriptor::Flags const flags = CallDescriptor::kNoFlags;
-  CallDescriptor* desc = Linkage::GetStubCallDescriptor(
-      isolate(), graph()->zone(), callable.descriptor(), 0, flags, properties);
-  return __ Call(desc, __ HeapConstant(callable.code()), length,
-                 __ NoContextConstant());
+  // Compute the effective size of the backing store.
+  Node* size =
+      __ Int32Add(__ Word32Shl(length, __ Int32Constant(kDoubleSizeLog2)),
+                  __ Int32Constant(FixedDoubleArray::kHeaderSize));
+
+  // Allocate the result and initialize the header.
+  Node* result = __ Allocate(pretenure, size);
+  __ StoreField(AccessBuilder::ForMap(), result,
+                __ FixedDoubleArrayMapConstant());
+  __ StoreField(AccessBuilder::ForFixedArrayLength(), result,
+                ChangeInt32ToSmi(length));
+
+  // Initialize the backing store with holes.
+  STATIC_ASSERT(HeapNumber::kValueOffset == Oddball::kToNumberRawOffset);
+  Node* the_hole =
+      __ LoadField(AccessBuilder::ForHeapNumberValue(), __ TheHoleConstant());
+  auto loop = __ MakeLoopLabel(MachineRepresentation::kWord32);
+  auto done_loop = __ MakeLabel();
+  __ Goto(&loop, __ Int32Constant(0));
+  __ Bind(&loop);
+  {
+    // Check if we've initialized everything.
+    Node* index = loop.PhiAt(0);
+    Node* check = __ Int32LessThan(index, length);
+    __ GotoIfNot(check, &done_loop);
+
+    // Storing "the_hole" doesn't need a write barrier.
+    ElementAccess const access = {kTaggedBase, FixedDoubleArray::kHeaderSize,
+                                  Type::NumberOrHole(), MachineType::Float64(),
+                                  kNoWriteBarrier};
+    __ StoreElement(access, result, index, the_hole);
+
+    // Advance the {index}.
+    index = __ Int32Add(index, __ Int32Constant(1));
+    __ Goto(&loop, index);
+  }
+
+  __ Bind(&done_loop);
+  return result;
 }
 
 Node* EffectControlLinearizer::LowerNewFastSmiOrObjectElements(Node* node) {
   PretenureFlag const pretenure = PretenureFlagOf(node->op());
   Node* length = node->InputAt(0);
 
-  Callable const callable = Builtins::CallableFor(
-      isolate(), pretenure == NOT_TENURED
-                     ? Builtins::kNewFastSmiOrObjectElements_NotTenured
-                     : Builtins::kNewFastSmiOrObjectElements_Tenured);
-  Operator::Properties const properties = node->op()->properties();
-  CallDescriptor::Flags const flags = CallDescriptor::kNoFlags;
-  CallDescriptor* desc = Linkage::GetStubCallDescriptor(
-      isolate(), graph()->zone(), callable.descriptor(), 0, flags, properties);
-  return __ Call(desc, __ HeapConstant(callable.code()), length,
-                 __ NoContextConstant());
+  // Compute the effective size of the backing store.
+  Node* size =
+      __ Int32Add(__ Word32Shl(length, __ Int32Constant(kPointerSizeLog2)),
+                  __ Int32Constant(FixedArray::kHeaderSize));
+
+  // Allocate the result and initialize the header.
+  Node* result = __ Allocate(pretenure, size);
+  __ StoreField(AccessBuilder::ForMap(), result, __ FixedArrayMapConstant());
+  __ StoreField(AccessBuilder::ForFixedArrayLength(), result,
+                ChangeInt32ToSmi(length));
+
+  // Initialize the backing store with holes.
+  Node* the_hole = __ TheHoleConstant();
+  auto loop = __ MakeLoopLabel(MachineRepresentation::kWord32);
+  auto done_loop = __ MakeLabel();
+  __ Goto(&loop, __ Int32Constant(0));
+  __ Bind(&loop);
+  {
+    // Check if we've initialized everything.
+    Node* index = loop.PhiAt(0);
+    Node* check = __ Int32LessThan(index, length);
+    __ GotoIfNot(check, &done_loop);
+
+    // Storing "the_hole" doesn't need a write barrier.
+    ElementAccess const access = {kTaggedBase, FixedArray::kHeaderSize,
+                                  Type::Any(), MachineType::AnyTagged(),
+                                  kNoWriteBarrier};
+    __ StoreElement(access, result, index, the_hole);
+
+    // Advance the {index}.
+    index = __ Int32Add(index, __ Int32Constant(1));
+    __ Goto(&loop, index);
+  }
+
+  __ Bind(&done_loop);
+  return result;
 }
 
 Node* EffectControlLinearizer::LowerNewArgumentsElements(Node* node) {

src/compiler/graph-assembler.h

@@ -78,13 +78,14 @@ namespace compiler {
   V(UndefinedConstant)                     \
   V(TheHoleConstant)                       \
   V(FixedArrayMapConstant)                 \
+  V(FixedDoubleArrayMapConstant)           \
   V(ToNumberBuiltinConstant)               \
   V(AllocateInNewSpaceStubConstant)        \
   V(AllocateInOldSpaceStubConstant)
 
 class GraphAssembler;
 
-enum class GraphAssemblerLabelType { kDeferred, kNonDeferred };
+enum class GraphAssemblerLabelType { kDeferred, kNonDeferred, kLoop };
 
 // Label with statically known count of incoming branches and phis.
 template <size_t VarCount>
@@ -93,9 +94,8 @@ class GraphAssemblerLabel {
   Node* PhiAt(size_t index);
 
   template <typename... Reps>
-  explicit GraphAssemblerLabel(GraphAssemblerLabelType is_deferred,
-                               Reps... reps)
-      : is_deferred_(is_deferred == GraphAssemblerLabelType::kDeferred) {
+  explicit GraphAssemblerLabel(GraphAssemblerLabelType type, Reps... reps)
+      : type_(type) {
     STATIC_ASSERT(VarCount == sizeof...(reps));
     MachineRepresentation reps_array[] = {MachineRepresentation::kNone,
                                           reps...};
@@ -114,10 +114,13 @@ class GraphAssemblerLabel {
     is_bound_ = true;
   }
   bool IsBound() const { return is_bound_; }
-  bool IsDeferred() const { return is_deferred_; }
+  bool IsDeferred() const {
+    return type_ == GraphAssemblerLabelType::kDeferred;
+  }
+  bool IsLoop() const { return type_ == GraphAssemblerLabelType::kLoop; }
 
   bool is_bound_ = false;
-  bool is_deferred_;
+  GraphAssemblerLabelType const type_;
   size_t merged_count_ = 0;
   Node* effect_;
   Node* control_;
@@ -134,8 +137,8 @@ class GraphAssembler {
   // Create label.
   template <typename... Reps>
   static GraphAssemblerLabel<sizeof...(Reps)> MakeLabelFor(
-      GraphAssemblerLabelType is_deferred, Reps... reps) {
-    return GraphAssemblerLabel<sizeof...(Reps)>(is_deferred, reps...);
+      GraphAssemblerLabelType type, Reps... reps) {
+    return GraphAssemblerLabel<sizeof...(Reps)>(type, reps...);
   }
 
   // Convenience wrapper for creating non-deferred labels.
@@ -144,6 +147,12 @@ class GraphAssembler {
     return MakeLabelFor(GraphAssemblerLabelType::kNonDeferred, reps...);
   }
 
+  // Convenience wrapper for creating loop labels.
+  template <typename... Reps>
+  static GraphAssemblerLabel<sizeof...(Reps)> MakeLoopLabel(Reps... reps) {
+    return MakeLabelFor(GraphAssemblerLabelType::kLoop, reps...);
+  }
+
   // Convenience wrapper for creating deferred labels.
   template <typename... Reps>
   static GraphAssemblerLabel<sizeof...(Reps)> MakeDeferredLabel(Reps... reps) {
@@ -264,48 +273,71 @@ Node* GraphAssemblerLabel<VarCount>::PhiAt(size_t index) {
 template <typename... Vars>
 void GraphAssembler::MergeState(GraphAssemblerLabel<sizeof...(Vars)>* label,
                                 Vars... vars) {
-  DCHECK(!label->IsBound());
-
   int merged_count = static_cast<int>(label->merged_count_);
   Node* var_array[] = {nullptr, vars...};
-  if (merged_count == 0) {
-    // Just set the control, effect and variables directly.
-    label->control_ = current_control_;
-    label->effect_ = current_effect_;
-    for (size_t i = 0; i < sizeof...(vars); i++) {
-      label->bindings_[i] = var_array[i + 1];
-    }
-  } else if (merged_count == 1) {
-    // Create merge, effect phi and a phi for each variable.
-    label->control_ =
-        graph()->NewNode(common()->Merge(2), label->control_, current_control_);
-    label->effect_ = graph()->NewNode(common()->EffectPhi(2), label->effect_,
-                                      current_effect_, label->control_);
-    for (size_t i = 0; i < sizeof...(vars); i++) {
-      label->bindings_[i] = graph()->NewNode(
-          common()->Phi(label->representations_[i], 2), label->bindings_[i],
-          var_array[i + 1], label->control_);
+  if (label->IsLoop()) {
+    if (merged_count == 0) {
+      DCHECK(!label->IsBound());
+      label->control_ = graph()->NewNode(common()->Loop(2), current_control_,
+                                         current_control_);
+      label->effect_ = graph()->NewNode(common()->EffectPhi(2), current_effect_,
+                                        current_effect_, label->control_);
+      for (size_t i = 0; i < sizeof...(vars); i++) {
+        label->bindings_[i] = graph()->NewNode(
+            common()->Phi(label->representations_[i], 2), var_array[i + 1],
+            var_array[i + 1], label->control_);
+      }
+    } else {
+      DCHECK(label->IsBound());
+      DCHECK_EQ(1, merged_count);
+      label->control_->ReplaceInput(1, current_control_);
+      label->effect_->ReplaceInput(1, current_effect_);
+      for (size_t i = 0; i < sizeof...(vars); i++) {
+        label->bindings_[i]->ReplaceInput(1, var_array[i + 1]);
+      }
     }
   } else {
-    // Append to the merge, effect phi and phis.
-    DCHECK_EQ(IrOpcode::kMerge, label->control_->opcode());
-    label->control_->AppendInput(graph()->zone(), current_control_);
-    NodeProperties::ChangeOp(label->control_,
-                             common()->Merge(merged_count + 1));
-
-    DCHECK_EQ(IrOpcode::kEffectPhi, label->effect_->opcode());
-    label->effect_->ReplaceInput(merged_count, current_effect_);
-    label->effect_->AppendInput(graph()->zone(), label->control_);
-    NodeProperties::ChangeOp(label->effect_,
-                             common()->EffectPhi(merged_count + 1));
-
-    for (size_t i = 0; i < sizeof...(vars); i++) {
-      DCHECK_EQ(IrOpcode::kPhi, label->bindings_[i]->opcode());
-      label->bindings_[i]->ReplaceInput(merged_count, var_array[i + 1]);
-      label->bindings_[i]->AppendInput(graph()->zone(), label->control_);
-      NodeProperties::ChangeOp(
-          label->bindings_[i],
-          common()->Phi(label->representations_[i], merged_count + 1));
+    DCHECK(!label->IsBound());
+    if (merged_count == 0) {
+      // Just set the control, effect and variables directly.
+      DCHECK(!label->IsBound());
+      label->control_ = current_control_;
+      label->effect_ = current_effect_;
+      for (size_t i = 0; i < sizeof...(vars); i++) {
+        label->bindings_[i] = var_array[i + 1];
+      }
+    } else if (merged_count == 1) {
+      // Create merge, effect phi and a phi for each variable.
+      label->control_ = graph()->NewNode(common()->Merge(2), label->control_,
+                                         current_control_);
+      label->effect_ = graph()->NewNode(common()->EffectPhi(2), label->effect_,
+                                        current_effect_, label->control_);
+      for (size_t i = 0; i < sizeof...(vars); i++) {
+        label->bindings_[i] = graph()->NewNode(
+            common()->Phi(label->representations_[i], 2), label->bindings_[i],
+            var_array[i + 1], label->control_);
+      }
+    } else {
+      // Append to the merge, effect phi and phis.
+      DCHECK_EQ(IrOpcode::kMerge, label->control_->opcode());
+      label->control_->AppendInput(graph()->zone(), current_control_);
+      NodeProperties::ChangeOp(label->control_,
+                               common()->Merge(merged_count + 1));
+
+      DCHECK_EQ(IrOpcode::kEffectPhi, label->effect_->opcode());
+      label->effect_->ReplaceInput(merged_count, current_effect_);
+      label->effect_->AppendInput(graph()->zone(), label->control_);
+      NodeProperties::ChangeOp(label->effect_,
+                               common()->EffectPhi(merged_count + 1));
+
+      for (size_t i = 0; i < sizeof...(vars); i++) {
+        DCHECK_EQ(IrOpcode::kPhi, label->bindings_[i]->opcode());
+        label->bindings_[i]->ReplaceInput(merged_count, var_array[i + 1]);
+        label->bindings_[i]->AppendInput(graph()->zone(), label->control_);
+        NodeProperties::ChangeOp(
+            label->bindings_[i],
+            common()->Phi(label->representations_[i], merged_count + 1));
+      }
     }
   }
   label->merged_count_++;

src/compiler/simplified-lowering.cc

@@ -2779,7 +2779,7 @@ class RepresentationSelector {
       }
       case IrOpcode::kNewFastDoubleElements:
       case IrOpcode::kNewFastSmiOrObjectElements: {
-        VisitUnop(node, UseInfo::TaggedSigned(),
+        VisitUnop(node, UseInfo::TruncatingWord32(),
                   MachineRepresentation::kTaggedPointer);
         return;
       }