[turbofan] Add initial support for growing stores.

Introduce a dedicated MaybeGrowFastElements simplified operator, which
tries to grow a fast elements backing store for a given element that
should be added to an array/object. Use that to lower a growing keyed
store to a sequence of

 1) check index is a valid array index,
 2) check stored value,
 3) maybe grow elements backing store (and deoptimize if it would
    normalize), and
 4) store the actual element.

The actual growing is done by two dedicated GrowFastDoubleElements
and GrowFastSmiOrObjectElements builtins, which are very similar to
the GrowArrayElementsStub that is used by Crankshaft.

Drive-by-fix: Turn CopyFixedArray into CopyFastSmiOrObjectElements
builtin, similar to the new growing builtins, so we don't need to
inline the store+write barrier for the elements into all optimized
code objects anymore.

Also fix a bug in the OperationTyper for NumberSilenceNaN, which was
triggered by this change.

BUG=v8:5272

Review-Url: https://codereview.chromium.org/2227493002
Cr-Commit-Position: refs/heads/master@{#38418}

src/builtins/builtins-internal.cc

@@ -51,15 +51,19 @@ void Builtins::Generate_StackCheck(MacroAssembler* masm) {
 }
 
 // -----------------------------------------------------------------------------
-// FixedArray helpers.
+// TurboFan support builtins.
 
-void Builtins::Generate_CopyFixedArray(CodeStubAssembler* assembler) {
+void Builtins::Generate_CopyFastSmiOrObjectElements(
+    CodeStubAssembler* assembler) {
   typedef CodeStubAssembler::Label Label;
   typedef compiler::Node Node;
   typedef CodeStubAssembler::Variable Variable;
-  typedef CopyFixedArrayDescriptor Descriptor;
+  typedef CopyFastSmiOrObjectElementsDescriptor Descriptor;
 
-  Node* source = assembler->Parameter(Descriptor::kSource);
+  Node* object = assembler->Parameter(Descriptor::kObject);
+
+  // Load the {object}s elements.
+  Node* source = assembler->LoadObjectField(object, JSObject::kElementsOffset);
 
   // Load the {source} length.
   Node* source_length_tagged =
@@ -122,8 +126,12 @@ void Builtins::Generate_CopyFixedArray(CodeStubAssembler* assembler) {
     }
 
     assembler->Bind(&done_loop);
+    {
+      // Update the {object}s element to {target}.
+      assembler->StoreObjectField(object, JSObject::kElementsOffset, target);
       assembler->Return(target);
     }
+  }
 
   assembler->Bind(&if_oldspace);
   {
@@ -173,8 +181,53 @@ void Builtins::Generate_CopyFixedArray(CodeStubAssembler* assembler) {
     }
 
     assembler->Bind(&done_loop);
+    {
+      // Update the {object}s element to {target}.
+      assembler->StoreObjectField(object, JSObject::kElementsOffset, target);
       assembler->Return(target);
     }
+  }
+}
+
+void Builtins::Generate_GrowFastDoubleElements(CodeStubAssembler* assembler) {
+  typedef CodeStubAssembler::Label Label;
+  typedef compiler::Node Node;
+  typedef GrowArrayElementsDescriptor Descriptor;
+
+  Node* object = assembler->Parameter(Descriptor::kObject);
+  Node* key = assembler->Parameter(Descriptor::kKey);
+  Node* context = assembler->Parameter(Descriptor::kContext);
+
+  Label runtime(assembler, CodeStubAssembler::Label::kDeferred);
+  Node* elements = assembler->LoadElements(object);
+  elements = assembler->CheckAndGrowElementsCapacity(
+      context, elements, FAST_DOUBLE_ELEMENTS, key, &runtime);
+  assembler->StoreObjectField(object, JSObject::kElementsOffset, elements);
+  assembler->Return(elements);
+
+  assembler->Bind(&runtime);
+  assembler->TailCallRuntime(Runtime::kGrowArrayElements, context, object, key);
+}
+
+void Builtins::Generate_GrowFastSmiOrObjectElements(
+    CodeStubAssembler* assembler) {
+  typedef CodeStubAssembler::Label Label;
+  typedef compiler::Node Node;
+  typedef GrowArrayElementsDescriptor Descriptor;
+
+  Node* object = assembler->Parameter(Descriptor::kObject);
+  Node* key = assembler->Parameter(Descriptor::kKey);
+  Node* context = assembler->Parameter(Descriptor::kContext);
+
+  Label runtime(assembler, CodeStubAssembler::Label::kDeferred);
+  Node* elements = assembler->LoadElements(object);
+  elements = assembler->CheckAndGrowElementsCapacity(
+      context, elements, FAST_ELEMENTS, key, &runtime);
+  assembler->StoreObjectField(object, JSObject::kElementsOffset, elements);
+  assembler->Return(elements);
+
+  assembler->Bind(&runtime);
+  assembler->TailCallRuntime(Runtime::kGrowArrayElements, context, object, key);
 }
 
 }  // namespace internal

src/builtins/builtins.h

@@ -159,8 +159,12 @@ namespace internal {
   ASM(AllocateInNewSpace)                                                    \
   ASM(AllocateInOldSpace)                                                    \
                                                                              \
-  /* FixedArray helpers */                                                   \
-  TFS(CopyFixedArray, BUILTIN, kNoExtraICState, CopyFixedArray)              \
+  /* TurboFan support builtins */                                            \
+  TFS(CopyFastSmiOrObjectElements, BUILTIN, kNoExtraICState,                 \
+      CopyFastSmiOrObjectElements)                                           \
+  TFS(GrowFastDoubleElements, BUILTIN, kNoExtraICState, GrowArrayElements)   \
+  TFS(GrowFastSmiOrObjectElements, BUILTIN, kNoExtraICState,                 \
+      GrowArrayElements)                                                     \
                                                                              \
   /* Debugger */                                                             \
   DBG(FrameDropper_LiveEdit)                                                 \

src/code-factory.cc

@@ -510,9 +510,21 @@ Callable CodeFactory::FastNewStrictArguments(Isolate* isolate,
 }
 
 // static
-Callable CodeFactory::CopyFixedArray(Isolate* isolate) {
-  return Callable(isolate->builtins()->CopyFixedArray(),
-                  CopyFixedArrayDescriptor(isolate));
+Callable CodeFactory::CopyFastSmiOrObjectElements(Isolate* isolate) {
+  return Callable(isolate->builtins()->CopyFastSmiOrObjectElements(),
+                  CopyFastSmiOrObjectElementsDescriptor(isolate));
+}
+
+// static
+Callable CodeFactory::GrowFastDoubleElements(Isolate* isolate) {
+  return Callable(isolate->builtins()->GrowFastDoubleElements(),
+                  GrowArrayElementsDescriptor(isolate));
+}
+
+// static
+Callable CodeFactory::GrowFastSmiOrObjectElements(Isolate* isolate) {
+  return Callable(isolate->builtins()->GrowFastSmiOrObjectElements(),
+                  GrowArrayElementsDescriptor(isolate));
 }
 
 // static

src/code-factory.h

@@ -136,7 +136,9 @@ class CodeFactory final {
   static Callable FastNewStrictArguments(Isolate* isolate,
                                          bool skip_stub_frame = false);
 
-  static Callable CopyFixedArray(Isolate* isolate);
+  static Callable CopyFastSmiOrObjectElements(Isolate* isolate);
+  static Callable GrowFastDoubleElements(Isolate* isolate);
+  static Callable GrowFastSmiOrObjectElements(Isolate* isolate);
 
   static Callable AllocateHeapNumber(Isolate* isolate);
 #define SIMD128_ALLOC(TYPE, Type, type, lane_count, lane_type) \

src/compiler/effect-control-linearizer.cc

@@ -726,6 +726,9 @@ bool EffectControlLinearizer::TryWireInStateEffect(Node* node,
     case IrOpcode::kEnsureWritableFastElements:
       state = LowerEnsureWritableFastElements(node, *effect, *control);
       break;
+    case IrOpcode::kMaybeGrowFastElements:
+      state = LowerMaybeGrowFastElements(node, frame_state, *effect, *control);
+      break;
     case IrOpcode::kTransitionElementsKind:
       state = LowerTransitionElementsKind(node, *effect, *control);
       break;
@@ -2632,19 +2635,14 @@ EffectControlLinearizer::LowerEnsureWritableFastElements(Node* node,
   {
     // We need to create a copy of the {elements} for {object}.
     Operator::Properties properties = Operator::kEliminatable;
-    Callable callable = CodeFactory::CopyFixedArray(isolate());
+    Callable callable = CodeFactory::CopyFastSmiOrObjectElements(isolate());
     CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
     CallDescriptor const* const desc = Linkage::GetStubCallDescriptor(
         isolate(), graph()->zone(), callable.descriptor(), 0, flags,
         properties);
     vfalse = efalse = graph()->NewNode(
-        common()->Call(desc), jsgraph()->HeapConstant(callable.code()),
-        elements, jsgraph()->NoContextConstant(), efalse);
-
-    // Store the new {elements} into {object}.
-    efalse = graph()->NewNode(
-        simplified()->StoreField(AccessBuilder::ForJSObjectElements()), object,
-        vfalse, efalse, if_false);
+        common()->Call(desc), jsgraph()->HeapConstant(callable.code()), object,
+        jsgraph()->NoContextConstant(), efalse);
   }
 
   control = graph()->NewNode(common()->Merge(2), if_true, if_false);
@@ -2655,6 +2653,115 @@ EffectControlLinearizer::LowerEnsureWritableFastElements(Node* node,
   return ValueEffectControl(value, effect, control);
 }
 
+EffectControlLinearizer::ValueEffectControl
+EffectControlLinearizer::LowerMaybeGrowFastElements(Node* node,
+                                                    Node* frame_state,
+                                                    Node* effect,
+                                                    Node* control) {
+  GrowFastElementsFlags flags = GrowFastElementsFlagsOf(node->op());
+  Node* object = node->InputAt(0);
+  Node* elements = node->InputAt(1);
+  Node* index = node->InputAt(2);
+  Node* length = node->InputAt(3);
+
+  Node* check0 = graph()->NewNode((flags & GrowFastElementsFlag::kHoleyElements)
+                                      ? machine()->Uint32LessThanOrEqual()
+                                      : machine()->Word32Equal(),
+                                  length, index);
+  Node* branch0 = graph()->NewNode(common()->Branch(), check0, control);
+
+  Node* if_true0 = graph()->NewNode(common()->IfTrue(), branch0);
+  Node* etrue0 = effect;
+  Node* vtrue0 = elements;
+  {
+    // Load the length of the {elements} backing store.
+    Node* elements_length = etrue0 = graph()->NewNode(
+        simplified()->LoadField(AccessBuilder::ForFixedArrayLength()), elements,
+        etrue0, if_true0);
+    elements_length = ChangeSmiToInt32(elements_length);
+
+    // Check if we need to grow the {elements} backing store.
+    Node* check1 =
+        graph()->NewNode(machine()->Uint32LessThan(), index, elements_length);
+    Node* branch1 =
+        graph()->NewNode(common()->Branch(BranchHint::kTrue), check1, if_true0);
+
+    Node* if_true1 = graph()->NewNode(common()->IfTrue(), branch1);
+    Node* etrue1 = etrue0;
+    Node* vtrue1 = vtrue0;
+
+    Node* if_false1 = graph()->NewNode(common()->IfFalse(), branch1);
+    Node* efalse1 = etrue0;
+    Node* vfalse1 = vtrue0;
+    {
+      // We need to grow the {elements} for {object}.
+      Operator::Properties properties = Operator::kEliminatable;
+      Callable callable =
+          (flags & GrowFastElementsFlag::kDoubleElements)
+              ? CodeFactory::GrowFastDoubleElements(isolate())
+              : CodeFactory::GrowFastSmiOrObjectElements(isolate());
+      CallDescriptor::Flags flags = CallDescriptor::kNoFlags;
+      CallDescriptor const* const desc = Linkage::GetStubCallDescriptor(
+          isolate(), graph()->zone(), callable.descriptor(), 0, flags,
+          properties);
+      vfalse1 = efalse1 = graph()->NewNode(
+          common()->Call(desc), jsgraph()->HeapConstant(callable.code()),
+          object, ChangeInt32ToSmi(index), jsgraph()->NoContextConstant(),
+          efalse1);
+
+      // Ensure that we were able to grow the {elements}.
+      // TODO(turbofan): We use kSmi as reason here similar to Crankshaft,
+      // but maybe we should just introduce a reason that makes sense.
+      efalse1 = if_false1 = graph()->NewNode(
+          common()->DeoptimizeIf(DeoptimizeReason::kSmi), ObjectIsSmi(vfalse1),
+          frame_state, efalse1, if_false1);
+    }
+
+    if_true0 = graph()->NewNode(common()->Merge(2), if_true1, if_false1);
+    etrue0 =
+        graph()->NewNode(common()->EffectPhi(2), etrue1, efalse1, if_true0);
+    vtrue0 = graph()->NewNode(common()->Phi(MachineRepresentation::kTagged, 2),
+                              vtrue1, vfalse1, if_true0);
+
+    // For JSArray {object}s we also need to update the "length".
+    if (flags & GrowFastElementsFlag::kArrayObject) {
+      // Compute the new {length}.
+      Node* object_length = ChangeInt32ToSmi(graph()->NewNode(
+          machine()->Int32Add(), index, jsgraph()->Int32Constant(1)));
+
+      // Update the "length" property of the {object}.
+      etrue0 =
+          graph()->NewNode(simplified()->StoreField(
+                               AccessBuilder::ForJSArrayLength(FAST_ELEMENTS)),
+                           object, object_length, etrue0, if_true0);
+    }
+  }
+
+  Node* if_false0 = graph()->NewNode(common()->IfFalse(), branch0);
+  Node* efalse0 = effect;
+  Node* vfalse0 = elements;
+  {
+    // In case of non-holey {elements}, we need to verify that the {index} is
+    // in-bounds, otherwise for holey {elements}, the check above already
+    // guards the index (and the operator forces {index} to be unsigned).
+    if (!(flags & GrowFastElementsFlag::kHoleyElements)) {
+      Node* check1 =
+          graph()->NewNode(machine()->Uint32LessThan(), index, length);
+      efalse0 = if_false0 = graph()->NewNode(
+          common()->DeoptimizeUnless(DeoptimizeReason::kOutOfBounds), check1,
+          frame_state, efalse0, if_false0);
+    }
+  }
+
+  control = graph()->NewNode(common()->Merge(2), if_true0, if_false0);
+  effect = graph()->NewNode(common()->EffectPhi(2), etrue0, efalse0, control);
+  Node* value =
+      graph()->NewNode(common()->Phi(MachineRepresentation::kTagged, 2), vtrue0,
+                       vfalse0, control);
+
+  return ValueEffectControl(value, effect, control);
+}
+
 EffectControlLinearizer::ValueEffectControl
 EffectControlLinearizer::LowerTransitionElementsKind(Node* node, Node* effect,
                                                      Node* control) {

src/compiler/effect-control-linearizer.h

@@ -138,6 +138,8 @@ class EffectControlLinearizer {
                                                   Node* control);
   ValueEffectControl LowerEnsureWritableFastElements(Node* node, Node* effect,
                                                      Node* control);
+  ValueEffectControl LowerMaybeGrowFastElements(Node* node, Node* frame_state,
+                                                Node* effect, Node* control);
   ValueEffectControl LowerTransitionElementsKind(Node* node, Node* effect,
                                                  Node* control);
   ValueEffectControl LowerLoadTypedElement(Node* node, Node* effect,

src/compiler/js-graph.cc

@@ -57,6 +57,11 @@ Node* JSGraph::FixedArrayMapConstant() {
                 HeapConstant(factory()->fixed_array_map()));
 }
 
+Node* JSGraph::FixedDoubleArrayMapConstant() {
+  return CACHED(kFixedDoubleArrayMapConstant,
+                HeapConstant(factory()->fixed_double_array_map()));
+}
+
 Node* JSGraph::HeapNumberMapConstant() {
   return CACHED(kHeapNumberMapConstant,
                 HeapConstant(factory()->heap_number_map()));

src/compiler/js-graph.h

@@ -47,6 +47,7 @@ class JSGraph : public ZoneObject {
   Node* EmptyLiteralsArrayConstant();
   Node* EmptyStringConstant();
   Node* FixedArrayMapConstant();
+  Node* FixedDoubleArrayMapConstant();
   Node* HeapNumberMapConstant();
   Node* OptimizedOutConstant();
   Node* StaleRegisterConstant();
@@ -158,6 +159,7 @@ class JSGraph : public ZoneObject {
     kEmptyLiteralsArrayConstant,
     kEmptyStringConstant,
     kFixedArrayMapConstant,
+    kFixedDoubleArrayMapConstant,
     kHeapNumberMapConstant,
     kOptimizedOutConstant,
     kStaleRegisterConstant,

src/compiler/js-native-context-specialization.cc

@@ -426,13 +426,6 @@ Reduction JSNativeContextSpecialization::ReduceElementAccess(
   // Not much we can do if deoptimization support is disabled.
   if (!(flags() & kDeoptimizationEnabled)) return NoChange();
 
-  // TODO(bmeurer): Add support for non-standard stores.
-  if (store_mode != STANDARD_STORE &&
-      store_mode != STORE_NO_TRANSITION_HANDLE_COW &&
-      store_mode != STORE_NO_TRANSITION_IGNORE_OUT_OF_BOUNDS) {
-    return NoChange();
-  }
-
   // Retrieve the native context from the given {node}.
   Handle<Context> native_context;
   if (!GetNativeContext(node).ToHandle(&native_context)) return NoChange();
@@ -1068,13 +1061,12 @@ JSNativeContextSpecialization::BuildElementAccess(
       }
     }
   } else {
-    // TODO(turbofan): Add support for additional store modes.
-    DCHECK(store_mode == STANDARD_STORE ||
-           store_mode == STORE_NO_TRANSITION_HANDLE_COW);
+    // Check if the {receiver} is a JSArray.
+    bool receiver_is_jsarray = HasOnlyJSArrayMaps(receiver_maps);
 
     // Load the length of the {receiver}.
     Node* length = effect =
-        HasOnlyJSArrayMaps(receiver_maps)
+        receiver_is_jsarray
             ? graph()->NewNode(
                   simplified()->LoadField(
                       AccessBuilder::ForJSArrayLength(elements_kind)),
@@ -1083,9 +1075,20 @@ JSNativeContextSpecialization::BuildElementAccess(
                   simplified()->LoadField(AccessBuilder::ForFixedArrayLength()),
                   elements, effect, control);
 
+    // Check if we might need to grow the {elements} backing store.
+    if (IsGrowStoreMode(store_mode)) {
+      DCHECK_EQ(AccessMode::kStore, access_mode);
+
+      // Check that the {index} is a valid array index; the actual checking
+      // happens below right before the element store.
+      index = effect = graph()->NewNode(simplified()->CheckBounds(), index,
+                                        jsgraph()->Constant(Smi::kMaxValue),
+                                        effect, control);
+    } else {
       // Check that the {index} is in the valid range for the {receiver}.
       index = effect = graph()->NewNode(simplified()->CheckBounds(), index,
                                         length, effect, control);
+    }
 
     // Compute the element access.
     Type* element_type = Type::Any();
@@ -1157,6 +1160,24 @@ JSNativeContextSpecialization::BuildElementAccess(
         elements = effect =
             graph()->NewNode(simplified()->EnsureWritableFastElements(),
                              receiver, elements, effect, control);
+      } else if (IsGrowStoreMode(store_mode)) {
+        // Grow {elements} backing store if necessary. Also updates the
+        // "length" property for JSArray {receiver}s, hence there must
+        // not be any other check after this operation, as the write
+        // to the "length" property is observable.
+        GrowFastElementsFlags flags = GrowFastElementsFlag::kNone;
+        if (receiver_is_jsarray) {
+          flags |= GrowFastElementsFlag::kArrayObject;
+        }
+        if (IsHoleyElementsKind(elements_kind)) {
+          flags |= GrowFastElementsFlag::kHoleyElements;
+        }
+        if (IsFastDoubleElementsKind(elements_kind)) {
+          flags |= GrowFastElementsFlag::kDoubleElements;
+        }
+        elements = effect = graph()->NewNode(
+            simplified()->MaybeGrowFastElements(flags), receiver, elements,
+            index, length, effect, control);
       }
 
       // Perform the actual element access.

src/compiler/load-elimination.cc

@@ -56,6 +56,8 @@ Reduction LoadElimination::Reduce(Node* node) {
       return ReduceCheckMaps(node);
     case IrOpcode::kEnsureWritableFastElements:
       return ReduceEnsureWritableFastElements(node);
+    case IrOpcode::kMaybeGrowFastElements:
+      return ReduceMaybeGrowFastElements(node);
     case IrOpcode::kTransitionElementsKind:
       return ReduceTransitionElementsKind(node);
     case IrOpcode::kLoadField:
@@ -352,6 +354,32 @@ Reduction LoadElimination::ReduceEnsureWritableFastElements(Node* node) {
   return UpdateState(node, state);
 }
 
+Reduction LoadElimination::ReduceMaybeGrowFastElements(Node* node) {
+  GrowFastElementsFlags flags = GrowFastElementsFlagsOf(node->op());
+  Node* const object = NodeProperties::GetValueInput(node, 0);
+  Node* const effect = NodeProperties::GetEffectInput(node);
+  AbstractState const* state = node_states_.Get(effect);
+  if (state == nullptr) return NoChange();
+  if (flags & GrowFastElementsFlag::kDoubleElements) {
+    // We know that the resulting elements have the fixed double array map.
+    Node* fixed_double_array_map = jsgraph()->FixedDoubleArrayMapConstant();
+    state = state->AddField(node, 0, fixed_double_array_map, zone());
+  } else {
+    // We know that the resulting elements have the fixed array map.
+    Node* fixed_array_map = jsgraph()->FixedArrayMapConstant();
+    state = state->AddField(node, 0, fixed_array_map, zone());
+  }
+  if (flags & GrowFastElementsFlag::kArrayObject) {
+    // Kill the previous Array::length on {object}.
+    state = state->KillField(object, 3, zone());
+  }
+  // Kill the previous elements on {object}.
+  state = state->KillField(object, 2, zone());
+  // Add the new elements on {object}.
+  state = state->AddField(object, 2, node, zone());
+  return UpdateState(node, state);
+}
+
 Reduction LoadElimination::ReduceTransitionElementsKind(Node* node) {
   Node* const object = NodeProperties::GetValueInput(node, 0);
   Node* const source_map = NodeProperties::GetValueInput(node, 1);
@@ -580,11 +608,19 @@ LoadElimination::AbstractState const* LoadElimination::ComputeLoopState(
         switch (current->opcode()) {
           case IrOpcode::kEnsureWritableFastElements: {
             Node* const object = NodeProperties::GetValueInput(current, 0);
-            Node* const elements = NodeProperties::GetValueInput(current, 1);
-            state = state->KillField(elements, 0, zone());
             state = state->KillField(object, 2, zone());
             break;
           }
+          case IrOpcode::kMaybeGrowFastElements: {
+            GrowFastElementsFlags flags =
+                GrowFastElementsFlagsOf(current->op());
+            Node* const object = NodeProperties::GetValueInput(current, 0);
+            state = state->KillField(object, 2, zone());
+            if (flags & GrowFastElementsFlag::kArrayObject) {
+              state = state->KillField(object, 3, zone());
+            }
+            break;
+          }
           case IrOpcode::kTransitionElementsKind: {
             Node* const object = NodeProperties::GetValueInput(current, 0);
             state = state->KillField(object, 0, zone());

src/compiler/load-elimination.h

@@ -152,6 +152,7 @@ class LoadElimination final : public AdvancedReducer {
 
   Reduction ReduceCheckMaps(Node* node);
   Reduction ReduceEnsureWritableFastElements(Node* node);
+  Reduction ReduceMaybeGrowFastElements(Node* node);
   Reduction ReduceTransitionElementsKind(Node* node);
   Reduction ReduceLoadField(Node* node);
   Reduction ReduceStoreField(Node* node);

src/compiler/opcodes.h

@@ -306,6 +306,7 @@
   V(ObjectIsString)                 \
   V(ObjectIsUndetectable)           \
   V(EnsureWritableFastElements)     \
+  V(MaybeGrowFastElements)          \
   V(TransitionElementsKind)
 
 #define SIMPLIFIED_OP_LIST(V)                 \

src/compiler/operation-typer.cc

@@ -486,7 +486,10 @@ Type* OperationTyper::NumberToUint32(Type* type) {
 
 Type* OperationTyper::NumberSilenceNaN(Type* type) {
   DCHECK(type->Is(Type::Number()));
-  // TODO(turbofan): We should have a dedicated type for the signaling NaN.
+  // TODO(jarin): This is a terrible hack; we definitely need a dedicated type
+  // for the hole (tagged and/or double). Otherwise if the input is the hole
+  // NaN constant, we'd just eliminate this node in JSTypedLowering.
+  if (type->Maybe(Type::NaN())) return Type::Number();
   return type;
 }
 

src/compiler/pipeline.cc

@@ -618,7 +618,9 @@ PipelineCompilationJob::Status PipelineCompilationJob::CreateGraphImpl() {
   // TODO(mstarzinger): Hack to ensure that certain call descriptors are
   // initialized on the main thread, since it is needed off-thread by the
   // effect control linearizer.
-  CodeFactory::CopyFixedArray(info()->isolate());
+  CodeFactory::CopyFastSmiOrObjectElements(info()->isolate());
+  CodeFactory::GrowFastDoubleElements(info()->isolate());
+  CodeFactory::GrowFastSmiOrObjectElements(info()->isolate());
   CodeFactory::ToNumber(info()->isolate());
 
   linkage_ = new (&zone_) Linkage(Linkage::ComputeIncoming(&zone_, info()));

src/compiler/simplified-lowering.cc

@@ -2385,6 +2385,15 @@ class RepresentationSelector {
       case IrOpcode::kEnsureWritableFastElements:
         return VisitBinop(node, UseInfo::AnyTagged(),
                           MachineRepresentation::kTagged);
+      case IrOpcode::kMaybeGrowFastElements: {
+        ProcessInput(node, 0, UseInfo::AnyTagged());         // object
+        ProcessInput(node, 1, UseInfo::AnyTagged());         // elements
+        ProcessInput(node, 2, UseInfo::TruncatingWord32());  // index
+        ProcessInput(node, 3, UseInfo::TruncatingWord32());  // length
+        ProcessRemainingInputs(node, 4);
+        SetOutput(node, MachineRepresentation::kTagged);
+        return;
+      }
 
       //------------------------------------------------------------------
       // Machine-level operators.

src/compiler/simplified-operator.cc

@@ -249,6 +249,31 @@ CheckTaggedHoleMode CheckTaggedHoleModeOf(const Operator* op) {
   return OpParameter<CheckTaggedHoleMode>(op);
 }
 
+std::ostream& operator<<(std::ostream& os, GrowFastElementsFlags flags) {
+  bool empty = true;
+  if (flags & GrowFastElementsFlag::kArrayObject) {
+    os << "ArrayObject";
+    empty = false;
+  }
+  if (flags & GrowFastElementsFlag::kDoubleElements) {
+    if (!empty) os << "|";
+    os << "DoubleElements";
+    empty = false;
+  }
+  if (flags & GrowFastElementsFlag::kHoleyElements) {
+    if (!empty) os << "|";
+    os << "HoleyElements";
+    empty = false;
+  }
+  if (empty) os << "None";
+  return os;
+}
+
+GrowFastElementsFlags GrowFastElementsFlagsOf(const Operator* op) {
+  DCHECK_EQ(IrOpcode::kMaybeGrowFastElements, op->opcode());
+  return OpParameter<GrowFastElementsFlags>(op);
+}
+
 size_t hash_value(ElementsTransition transition) {
   return static_cast<uint8_t>(transition);
 }
@@ -654,6 +679,16 @@ const Operator* SimplifiedOperatorBuilder::EnsureWritableFastElements() {
   return &cache_.kEnsureWritableFastElements;
 }
 
+const Operator* SimplifiedOperatorBuilder::MaybeGrowFastElements(
+    GrowFastElementsFlags flags) {
+  return new (zone()) Operator1<GrowFastElementsFlags>(  // --
+      IrOpcode::kMaybeGrowFastElements,                  // opcode
+      Operator::kNoThrow,                                // flags
+      "MaybeGrowFastElements",                           // name
+      4, 1, 1, 1, 1, 0,                                  // counts
+      flags);                                            // parameter
+}
+
 const Operator* SimplifiedOperatorBuilder::TransitionElementsKind(
     ElementsTransition transition) {
   return new (zone()) Operator1<ElementsTransition>(  // --

src/compiler/simplified-operator.h

@@ -141,6 +141,22 @@ std::ostream& operator<<(std::ostream&, CheckForMinusZeroMode);
 
 CheckForMinusZeroMode CheckMinusZeroModeOf(const Operator*) WARN_UNUSED_RESULT;
 
+// A descriptor for growing elements backing stores.
+enum class GrowFastElementsFlag : uint8_t {
+  kNone = 0u,
+  kArrayObject = 1u << 0,     // Update JSArray::length field.
+  kHoleyElements = 1u << 1,   // Backing store is holey.
+  kDoubleElements = 1u << 2,  // Backing store contains doubles.
+};
+typedef base::Flags<GrowFastElementsFlag> GrowFastElementsFlags;
+
+DEFINE_OPERATORS_FOR_FLAGS(GrowFastElementsFlags)
+
+std::ostream& operator<<(std::ostream&, GrowFastElementsFlags);
+
+GrowFastElementsFlags GrowFastElementsFlagsOf(const Operator*)
+    WARN_UNUSED_RESULT;
+
 // A descriptor for elements kind transitions.
 enum class ElementsTransition : uint8_t {
   kFastTransition,  // simple transition, just updating the map.
@@ -323,6 +339,9 @@ class SimplifiedOperatorBuilder final : public ZoneObject {
   // ensure-writable-fast-elements object, elements
   const Operator* EnsureWritableFastElements();
 
+  // maybe-grow-fast-elements object, elements, index, length
+  const Operator* MaybeGrowFastElements(GrowFastElementsFlags flags);
+
   // transition-elements-kind object, from-map, to-map
   const Operator* TransitionElementsKind(ElementsTransition transition);
 

src/compiler/typer.cc

@@ -726,6 +726,10 @@ Type* Typer::Visitor::TypeEnsureWritableFastElements(Node* node) {
   return Operand(node, 1);
 }
 
+Type* Typer::Visitor::TypeMaybeGrowFastElements(Node* node) {
+  return Operand(node, 1);
+}
+
 Type* Typer::Visitor::TypeTransitionElementsKind(Node* node) {
   UNREACHABLE();
   return nullptr;

src/compiler/verifier.cc

@@ -856,6 +856,13 @@ void Verifier::Visitor::Check(Node* node) {
       CheckValueInputIs(node, 1, Type::Internal());
       CheckUpperIs(node, Type::Internal());
       break;
+    case IrOpcode::kMaybeGrowFastElements:
+      CheckValueInputIs(node, 0, Type::Any());
+      CheckValueInputIs(node, 1, Type::Internal());
+      CheckValueInputIs(node, 2, Type::Unsigned31());
+      CheckValueInputIs(node, 3, Type::Unsigned31());
+      CheckUpperIs(node, Type::Internal());
+      break;
     case IrOpcode::kTransitionElementsKind:
       CheckValueInputIs(node, 0, Type::Any());
       CheckValueInputIs(node, 1, Type::Internal());

src/interface-descriptors.h

@@ -55,7 +55,7 @@ class PlatformInterfaceDescriptor;
   V(ConstructStub)                     \
   V(ConstructTrampoline)               \
   V(RegExpConstructResult)             \
-  V(CopyFixedArray)                    \
+  V(CopyFastSmiOrObjectElements)       \
   V(TransitionElementsKind)            \
   V(AllocateHeapNumber)                \
   V(AllocateFloat32x4)                 \
@@ -649,11 +649,11 @@ class StoreGlobalViaContextDescriptor : public CallInterfaceDescriptor {
   static const Register ValueRegister();
 };
 
-class CopyFixedArrayDescriptor : public CallInterfaceDescriptor {
+class CopyFastSmiOrObjectElementsDescriptor : public CallInterfaceDescriptor {
  public:
-  DEFINE_PARAMETERS(kSource)
-  DECLARE_DEFAULT_DESCRIPTOR(CopyFixedArrayDescriptor, CallInterfaceDescriptor,
-                             kParameterCount)
+  DEFINE_PARAMETERS(kObject)
+  DECLARE_DEFAULT_DESCRIPTOR(CopyFastSmiOrObjectElementsDescriptor,
+                             CallInterfaceDescriptor, kParameterCount)
 };
 
 class TransitionElementsKindDescriptor : public CallInterfaceDescriptor {
@@ -876,6 +876,7 @@ class VarArgFunctionDescriptor : public CallInterfaceDescriptor {
                                                CallInterfaceDescriptor)
 };
 
+// TODO(turbofan): We should probably rename this to GrowFastElementsDescriptor.
 class GrowArrayElementsDescriptor : public CallInterfaceDescriptor {
  public:
   DEFINE_PARAMETERS(kObject, kKey)