[Turbofan] Introduce TransitionAndStore[Non]NumberElement

In Array.prototype.map, we have to store the map result in an output array.
If we know we are storing objects, or special objects like boolean, rather
than a number, then we can reduce the amount of checks we have to do to
transition the output array to the appropriate ElementsKind.

Likewise, if we know we've got floating point values, we can specialize 
appropriately to a double array.

Bug: v8:6896
Change-Id: I375daf604562b53638ea749945c1a4c907e33547
Reviewed-on: https://chromium-review.googlesource.com/711845
Commit-Queue: Michael Stanton <mvstanton@chromium.org>
Reviewed-by: Jaroslav Sevcik <jarin@chromium.org>
Cr-Commit-Position: refs/heads/master@{#48579}

src/compiler/effect-control-linearizer.cc

@@ -847,6 +847,12 @@ bool EffectControlLinearizer::TryWireInStateEffect(Node* node,
     case IrOpcode::kFindOrderedHashMapEntryForInt32Key:
       result = LowerFindOrderedHashMapEntryForInt32Key(node);
       break;
+    case IrOpcode::kTransitionAndStoreNumberElement:
+      LowerTransitionAndStoreNumberElement(node);
+      break;
+    case IrOpcode::kTransitionAndStoreNonNumberElement:
+      LowerTransitionAndStoreNonNumberElement(node);
+      break;
     case IrOpcode::kTransitionAndStoreElement:
       LowerTransitionAndStoreElement(node);
       break;
@@ -3275,22 +3281,152 @@ void EffectControlLinearizer::LowerTransitionAndStoreElement(Node* node) {
   __ Bind(&done);
 }
 
-void EffectControlLinearizer::LowerStoreSignedSmallElement(Node* node) {
+void EffectControlLinearizer::LowerTransitionAndStoreNumberElement(Node* node) {
+  Node* array = node->InputAt(0);
+  Node* index = node->InputAt(1);
+  Node* value = node->InputAt(2);  // This is a Float64, not tagged.
+
+  // Possibly transition array based on input and store.
+  //
+  //   -- TRANSITION PHASE -----------------
+  //   kind = ElementsKind(array)
+  //   if kind == HOLEY_SMI_ELEMENTS {
+  //     Transition array to HOLEY_DOUBLE_ELEMENTS
+  //   } else if kind != HOLEY_DOUBLE_ELEMENTS {
+  //     This is UNREACHABLE, execute a debug break.
+  //   }
+  //
+  //   -- STORE PHASE ----------------------
+  //   Store array[index] = value (it's a float)
+  //
+  Node* map = __ LoadField(AccessBuilder::ForMap(), array);
+  Node* kind;
+  {
+    Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
+    Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
+    Node* andit = __ Word32And(bit_field2, mask);
+    Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
+    kind = __ Word32Shr(andit, shift);
+  }
+
+  auto do_store = __ MakeLabel();
+
+  // {value} is a float64.
+  auto transition_smi_array = __ MakeDeferredLabel();
+  {
+    __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_SMI_ELEMENTS),
+                 &transition_smi_array);
+    // We expect that our input array started at HOLEY_SMI_ELEMENTS, and
+    // climbs the lattice up to HOLEY_DOUBLE_ELEMENTS. Force a debug break
+    // if this assumption is broken. It also would be the case that
+    // loop peeling can break this assumption.
+    __ GotoIf(__ Word32Equal(kind, __ Int32Constant(HOLEY_DOUBLE_ELEMENTS)),
+              &do_store);
+    // TODO(turbofan): It would be good to have an "Unreachable()" node type.
+    __ DebugBreak();
+    __ Goto(&do_store);
+  }
+
+  __ Bind(&transition_smi_array);  // deferred code.
+  {
+    // Transition {array} from HOLEY_SMI_ELEMENTS to HOLEY_DOUBLE_ELEMENTS.
+    TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS,
+                         HOLEY_DOUBLE_ELEMENTS);
+    __ Goto(&do_store);
+  }
+
+  __ Bind(&do_store);
+
+  Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
+  __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements, index,
+                  value);
+}
+
+void EffectControlLinearizer::LowerTransitionAndStoreNonNumberElement(
+    Node* node) {
   Node* array = node->InputAt(0);
   Node* index = node->InputAt(1);
   Node* value = node->InputAt(2);
 
+  // Possibly transition array based on input and store.
+  //
+  //   -- TRANSITION PHASE -----------------
+  //   kind = ElementsKind(array)
+  //   if kind == HOLEY_SMI_ELEMENTS {
+  //     Transition array to HOLEY_ELEMENTS
+  //   } else if kind == HOLEY_DOUBLE_ELEMENTS {
+  //     Transition array to HOLEY_ELEMENTS
+  //   }
+  //
+  //   -- STORE PHASE ----------------------
+  //   // kind is HOLEY_ELEMENTS
+  //   Store array[index] = value
+  //
+  Node* map = __ LoadField(AccessBuilder::ForMap(), array);
+  Node* kind;
+  {
+    Node* bit_field2 = __ LoadField(AccessBuilder::ForMapBitField2(), map);
+    Node* mask = __ Int32Constant(Map::ElementsKindBits::kMask);
+    Node* andit = __ Word32And(bit_field2, mask);
+    Node* shift = __ Int32Constant(Map::ElementsKindBits::kShift);
+    kind = __ Word32Shr(andit, shift);
+  }
+
+  auto do_store = __ MakeLabel();
+
+  auto transition_smi_array = __ MakeDeferredLabel();
+  auto transition_double_to_fast = __ MakeDeferredLabel();
+  {
+    __ GotoIfNot(IsElementsKindGreaterThan(kind, HOLEY_SMI_ELEMENTS),
+                 &transition_smi_array);
+    __ GotoIf(IsElementsKindGreaterThan(kind, HOLEY_ELEMENTS),
+              &transition_double_to_fast);
+    __ Goto(&do_store);
+  }
+
+  __ Bind(&transition_smi_array);  // deferred code.
+  {
+    // Transition {array} from HOLEY_SMI_ELEMENTS to HOLEY_ELEMENTS.
+    TransitionElementsTo(node, array, HOLEY_SMI_ELEMENTS, HOLEY_ELEMENTS);
+    __ Goto(&do_store);
+  }
+
+  __ Bind(&transition_double_to_fast);  // deferred code.
+  {
+    TransitionElementsTo(node, array, HOLEY_DOUBLE_ELEMENTS, HOLEY_ELEMENTS);
+    __ Goto(&do_store);
+  }
+
+  __ Bind(&do_store);
+
+  Node* elements = __ LoadField(AccessBuilder::ForJSObjectElements(), array);
+  // Our ElementsKind is HOLEY_ELEMENTS.
+  ElementAccess access = AccessBuilder::ForFixedArrayElement(HOLEY_ELEMENTS);
+  Type* value_type = ValueTypeParameterOf(node->op());
+  if (value_type->Is(Type::BooleanOrNullOrUndefined())) {
+    access.type = value_type;
+    access.write_barrier_kind = kNoWriteBarrier;
+  }
+  __ StoreElement(access, elements, index, value);
+}
+
+void EffectControlLinearizer::LowerStoreSignedSmallElement(Node* node) {
+  Node* array = node->InputAt(0);
+  Node* index = node->InputAt(1);
+  Node* value = node->InputAt(2);  // int32
+
   // Store a signed small in an output array.
   //
   //   kind = ElementsKind(array)
   //
   //   -- STORE PHASE ----------------------
   //   if kind == HOLEY_DOUBLE_ELEMENTS {
-  //     float_value = convert smi to float
+  //     float_value = convert int32 to float
   //     Store array[index] = float_value
   //   } else {
   //     // kind is HOLEY_SMI_ELEMENTS or HOLEY_ELEMENTS
-  //     Store array[index] = value
+  //     smi_value = convert int32 to smi
+  //     Store array[index] = smi_value
   //   }
   //
   Node* map = __ LoadField(AccessBuilder::ForMap(), array);
@@ -3316,14 +3452,14 @@ void EffectControlLinearizer::LowerStoreSignedSmallElement(Node* node) {
     access.type = Type::SignedSmall();
     access.machine_type = MachineType::TaggedSigned();
     access.write_barrier_kind = kNoWriteBarrier;
-    __ StoreElement(access, elements, index, value);
+    Node* smi_value = ChangeInt32ToSmi(value);
+    __ StoreElement(access, elements, index, smi_value);
     __ Goto(&done);
   }
   __ Bind(&if_kind_is_double);
   {
     // Our ElementsKind is HOLEY_DOUBLE_ELEMENTS.
-    Node* int_value = ChangeSmiToInt32(value);
-    Node* float_value = __ ChangeInt32ToFloat64(int_value);
+    Node* float_value = __ ChangeInt32ToFloat64(value);
     __ StoreElement(AccessBuilder::ForFixedDoubleArrayElement(), elements,
                     index, float_value);
     __ Goto(&done);

src/compiler/effect-control-linearizer.h

@@ -130,6 +130,8 @@ class V8_EXPORT_PRIVATE EffectControlLinearizer {
   Node* LowerFindOrderedHashMapEntry(Node* node);
   Node* LowerFindOrderedHashMapEntryForInt32Key(Node* node);
   void LowerTransitionAndStoreElement(Node* node);
+  void LowerTransitionAndStoreNumberElement(Node* node);
+  void LowerTransitionAndStoreNonNumberElement(Node* node);
   void LowerRuntimeAbort(Node* node);
 
   // Lowering of optional operators.

src/compiler/opcodes.h

@@ -353,6 +353,8 @@
   V(StoreTypedElement)                  \
   V(StoreSignedSmallElement)            \
   V(TransitionAndStoreElement)          \
+  V(TransitionAndStoreNumberElement)    \
+  V(TransitionAndStoreNonNumberElement) \
   V(ObjectIsArrayBufferView)            \
   V(ObjectIsCallable)                   \
   V(ObjectIsConstructor)                \

src/compiler/simplified-lowering.cc

@@ -2581,13 +2581,31 @@ class RepresentationSelector {
 
         ProcessInput(node, 0, UseInfo::AnyTagged());         // array
         ProcessInput(node, 1, UseInfo::TruncatingWord32());  // index
-        ProcessInput(node, 2, UseInfo::AnyTagged());         // value
 
         if (value_type->Is(Type::SignedSmall())) {
+          ProcessInput(node, 2, UseInfo::TruncatingWord32());  // value
           if (lower()) {
             NodeProperties::ChangeOp(node,
                                      simplified()->StoreSignedSmallElement());
           }
+        } else if (value_type->Is(Type::Number())) {
+          ProcessInput(node, 2, UseInfo::TruncatingFloat64());  // value
+          if (lower()) {
+            Handle<Map> double_map = DoubleMapParameterOf(node->op());
+            NodeProperties::ChangeOp(
+                node,
+                simplified()->TransitionAndStoreNumberElement(double_map));
+          }
+        } else if (value_type->Is(Type::NonNumber())) {
+          ProcessInput(node, 2, UseInfo::AnyTagged());  // value
+          if (lower()) {
+            Handle<Map> fast_map = FastMapParameterOf(node->op());
+            NodeProperties::ChangeOp(
+                node, simplified()->TransitionAndStoreNonNumberElement(
+                          fast_map, value_type));
+          }
+        } else {
+          ProcessInput(node, 2, UseInfo::AnyTagged());  // value
         }
 
         ProcessRemainingInputs(node, 3);

src/compiler/simplified-operator.cc

@@ -327,14 +327,105 @@ std::ostream& operator<<(std::ostream& os,
 
 }  // namespace
 
+namespace {
+
+// Parameters for the TransitionAndStoreNonNumberElement opcode.
+class TransitionAndStoreNonNumberElementParameters final {
+ public:
+  TransitionAndStoreNonNumberElementParameters(Handle<Map> fast_map,
+                                               Type* value_type);
+
+  Handle<Map> fast_map() const { return fast_map_; }
+  Type* value_type() const { return value_type_; }
+
+ private:
+  Handle<Map> const fast_map_;
+  Type* value_type_;
+};
+
+TransitionAndStoreNonNumberElementParameters::
+    TransitionAndStoreNonNumberElementParameters(Handle<Map> fast_map,
+                                                 Type* value_type)
+    : fast_map_(fast_map), value_type_(value_type) {}
+
+bool operator==(TransitionAndStoreNonNumberElementParameters const& lhs,
+                TransitionAndStoreNonNumberElementParameters const& rhs) {
+  return lhs.fast_map().address() == rhs.fast_map().address() &&
+         lhs.value_type() == rhs.value_type();
+}
+
+size_t hash_value(TransitionAndStoreNonNumberElementParameters parameters) {
+  return base::hash_combine(parameters.fast_map().address(),
+                            parameters.value_type());
+}
+
+std::ostream& operator<<(
+    std::ostream& os, TransitionAndStoreNonNumberElementParameters parameters) {
+  parameters.value_type()->PrintTo(os);
+  return os << ", fast-map" << Brief(*parameters.fast_map());
+}
+
+}  // namespace
+
+namespace {
+
+// Parameters for the TransitionAndStoreNumberElement opcode.
+class TransitionAndStoreNumberElementParameters final {
+ public:
+  explicit TransitionAndStoreNumberElementParameters(Handle<Map> double_map);
+
+  Handle<Map> double_map() const { return double_map_; }
+
+ private:
+  Handle<Map> const double_map_;
+};
+
+TransitionAndStoreNumberElementParameters::
+    TransitionAndStoreNumberElementParameters(Handle<Map> double_map)
+    : double_map_(double_map) {}
+
+bool operator==(TransitionAndStoreNumberElementParameters const& lhs,
+                TransitionAndStoreNumberElementParameters const& rhs) {
+  return lhs.double_map().address() == rhs.double_map().address();
+}
+
+size_t hash_value(TransitionAndStoreNumberElementParameters parameters) {
+  return base::hash_combine(parameters.double_map().address());
+}
+
+std::ostream& operator<<(std::ostream& os,
+                         TransitionAndStoreNumberElementParameters parameters) {
+  return os << "double-map" << Brief(*parameters.double_map());
+}
+
+}  // namespace
+
 Handle<Map> DoubleMapParameterOf(const Operator* op) {
-  DCHECK_EQ(IrOpcode::kTransitionAndStoreElement, op->opcode());
-  return OpParameter<TransitionAndStoreElementParameters>(op).double_map();
+  if (op->opcode() == IrOpcode::kTransitionAndStoreElement) {
+    return OpParameter<TransitionAndStoreElementParameters>(op).double_map();
+  } else if (op->opcode() == IrOpcode::kTransitionAndStoreNumberElement) {
+    return OpParameter<TransitionAndStoreNumberElementParameters>(op)
+        .double_map();
+  }
+  UNREACHABLE();
+  return Handle<Map>::null();
+}
+
+Type* ValueTypeParameterOf(const Operator* op) {
+  DCHECK_EQ(IrOpcode::kTransitionAndStoreNonNumberElement, op->opcode());
+  return OpParameter<TransitionAndStoreNonNumberElementParameters>(op)
+      .value_type();
 }
 
 Handle<Map> FastMapParameterOf(const Operator* op) {
-  DCHECK_EQ(IrOpcode::kTransitionAndStoreElement, op->opcode());
-  return OpParameter<TransitionAndStoreElementParameters>(op).fast_map();
+  if (op->opcode() == IrOpcode::kTransitionAndStoreElement) {
+    return OpParameter<TransitionAndStoreElementParameters>(op).fast_map();
+  } else if (op->opcode() == IrOpcode::kTransitionAndStoreNonNumberElement) {
+    return OpParameter<TransitionAndStoreNonNumberElementParameters>(op)
+        .fast_map();
+  }
+  UNREACHABLE();
+  return Handle<Map>::null();
 }
 
 std::ostream& operator<<(std::ostream& os, NumberOperationHint hint) {
@@ -1111,6 +1202,24 @@ const Operator* SimplifiedOperatorBuilder::StoreSignedSmallElement() {
                                "StoreSignedSmallElement", 3, 1, 1, 0, 1, 0);
 }
 
+const Operator* SimplifiedOperatorBuilder::TransitionAndStoreNumberElement(
+    Handle<Map> double_map) {
+  TransitionAndStoreNumberElementParameters parameters(double_map);
+  return new (zone()) Operator1<TransitionAndStoreNumberElementParameters>(
+      IrOpcode::kTransitionAndStoreNumberElement,
+      Operator::kNoDeopt | Operator::kNoThrow,
+      "TransitionAndStoreNumberElement", 3, 1, 1, 0, 1, 0, parameters);
+}
+
+const Operator* SimplifiedOperatorBuilder::TransitionAndStoreNonNumberElement(
+    Handle<Map> fast_map, Type* value_type) {
+  TransitionAndStoreNonNumberElementParameters parameters(fast_map, value_type);
+  return new (zone()) Operator1<TransitionAndStoreNonNumberElementParameters>(
+      IrOpcode::kTransitionAndStoreNonNumberElement,
+      Operator::kNoDeopt | Operator::kNoThrow,
+      "TransitionAndStoreNonNumberElement", 3, 1, 1, 0, 1, 0, parameters);
+}
+
 #undef PURE_OP_LIST
 #undef SPECULATIVE_NUMBER_BINOP_LIST
 #undef CHECKED_OP_LIST

src/compiler/simplified-operator.h

@@ -207,10 +207,15 @@ std::ostream& operator<<(std::ostream&, ElementsTransition);
 ElementsTransition const& ElementsTransitionOf(const Operator* op)
     WARN_UNUSED_RESULT;
 
-// Parameters for TransitionAndStoreElement.
+// Parameters for TransitionAndStoreElement, or
+// TransitionAndStoreNonNumberElement, or
+// TransitionAndStoreNumberElement.
 Handle<Map> DoubleMapParameterOf(const Operator* op);
 Handle<Map> FastMapParameterOf(const Operator* op);
 
+// Parameters for TransitionAndStoreNonNumberElement.
+Type* ValueTypeParameterOf(const Operator* op);
+
 // A hint for speculative number operations.
 enum class NumberOperationHint : uint8_t {
   kSignedSmall,        // Inputs were Smi, output was in Smi.
@@ -492,6 +497,13 @@ class V8_EXPORT_PRIVATE SimplifiedOperatorBuilder final
   // store-element [base + index], smi value, only with fast arrays.
   const Operator* StoreSignedSmallElement();
 
+  // store-element [base + index], double value, only with fast arrays.
+  const Operator* TransitionAndStoreNumberElement(Handle<Map> double_map);
+
+  // store-element [base + index], object value, only with fast arrays.
+  const Operator* TransitionAndStoreNonNumberElement(Handle<Map> fast_map,
+                                                     Type* value_type);
+
   // load-typed-element buffer, [base + external + index]
   const Operator* LoadTypedElement(ExternalArrayType const&);
 

src/compiler/typer.cc

@@ -1977,6 +1977,14 @@ Type* Typer::Visitor::TypeTransitionAndStoreElement(Node* node) {
   UNREACHABLE();
 }
 
+Type* Typer::Visitor::TypeTransitionAndStoreNumberElement(Node* node) {
+  UNREACHABLE();
+}
+
+Type* Typer::Visitor::TypeTransitionAndStoreNonNumberElement(Node* node) {
+  UNREACHABLE();
+}
+
 Type* Typer::Visitor::TypeStoreSignedSmallElement(Node* node) { UNREACHABLE(); }
 
 Type* Typer::Visitor::TypeStoreTypedElement(Node* node) {

src/compiler/verifier.cc

@@ -1302,8 +1302,13 @@ void Verifier::Visitor::Check(Node* node) {
     case IrOpcode::kTransitionAndStoreElement:
       CheckNotTyped(node);
       break;
+    case IrOpcode::kTransitionAndStoreNumberElement:
+      CheckNotTyped(node);
+      break;
+    case IrOpcode::kTransitionAndStoreNonNumberElement:
+      CheckNotTyped(node);
+      break;
     case IrOpcode::kStoreSignedSmallElement:
-      CheckValueInputIs(node, 1, Type::SignedSmall());
       CheckNotTyped(node);
       break;
     case IrOpcode::kStoreTypedElement:

test/mjsunit/optimized-map.js

@@ -436,6 +436,38 @@ var c = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25];
   assertOptimized(to_fast);
 })();
 
+// TurboFan specializes on number results, ensure the code path is
+// tested.
+(function() {
+  var a = [1, 2, 3];
+  function double_results() {
+    // TurboFan recognizes the result is a double.
+    var callback = v => v + 0.5;
+    return a.map(callback);
+  }
+  double_results();
+  double_results();
+  %OptimizeFunctionOnNextCall(double_results);
+  double_results();
+  assertEquals(1.5, double_results()[0]);
+})();
+
+// TurboFan specializes on non-number results, ensure the code path is
+// tested.
+(function() {
+  var a = [1, 2, 3];
+  function string_results() {
+    // TurboFan recognizes the result is a string.
+    var callback = v => "hello" + v.toString();
+    return a.map(callback);
+  }
+  string_results();
+  string_results();
+  %OptimizeFunctionOnNextCall(string_results);
+  string_results();
+  assertEquals("hello1", string_results()[0]);
+})();
+
 // Messing with the Array species constructor causes deoptimization.
 (function() {
   var result = 0;
@@ -457,3 +489,11 @@ var c = [1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25];
   assertUnoptimized(species_breakage);
   assertEquals(24, result);
 })();
+
+/////////////////////////////////////////////////////////////////////////
+//
+// Any tests added below species_breakage won't test optimized map calls
+// because the array species constructor change disables inlining of
+// Array.prototype.map across the isolate.
+//
+/////////////////////////////////////////////////////////////////////////