[Torque] Continue reducing code-size in Array builtins

Since the performance implications of the patch
"[Torque] Reduce code size by combining FixedArray/FixedDoubleArray paths"
are negligible, I'll extend the pattern to all the array builtins,
providing a savings of about 20% per builtin.

Bug: v8:7672
Change-Id: Ib9aace4da38369842154065f5b4bcfb3ce2355d7
Reviewed-on: https://chromium-review.googlesource.com/c/1488768
Commit-Queue: Michael Stanton <mvstanton@chromium.org>
Reviewed-by: Tobias Tebbi <tebbi@chromium.org>
Cr-Commit-Position: refs/heads/master@{#59944}

src/builtins/array-filter.tq

@@ -94,73 +94,42 @@ namespace array_filter {
     return array;
   }
 
-  transitioning macro
-  FilterVisitAllElements<FixedArrayType: type>(implicit context: Context)(
-      kind: constexpr ElementsKind, o: JSArray, len: Smi, callbackfn: Callable,
-      thisArg: Object, a: JSArray) labels Bailout(Smi, Smi) {
+  transitioning macro FastArrayFilter(implicit context: Context)(
+      fastO: FastJSArray, len: Smi, callbackfn: Callable, thisArg: Object,
+      output: FastJSArray) labels Bailout(Number, Number) {
     let k: Smi = 0;
     let to: Smi = 0;
-    const arrayO = Cast<FastJSArray>(o) otherwise goto Bailout(k, to);
-    let fastO = NewFastJSArrayWitness(arrayO);
-    const arrayA = Cast<FastJSArray>(a) otherwise goto Bailout(k, to);
-    let fastA = NewFastJSArrayWitness(arrayA);
+    let fastOW = NewFastJSArrayWitness(fastO);
+    let fastOutputW = NewFastJSArrayWitness(output);
+
+    fastOutputW.EnsureArrayPushable() otherwise goto Bailout(k, to);
 
-    // Build a fast loop over the smi array.
+    // Build a fast loop over the array.
     for (; k < len; k++) {
-      fastO.Recheck() otherwise goto Bailout(k, to);
+      fastOW.Recheck() otherwise goto Bailout(k, to);
 
       // Ensure that we haven't walked beyond a possibly updated length.
-      if (k >= fastO.Get().length) goto Bailout(k, to);
-      const value: Object = LoadElementNoHole<FixedArrayType>(fastO.Get(), k)
-          otherwise continue;
+      if (k >= fastOW.Get().length) goto Bailout(k, to);
+      const value: Object = fastOW.LoadElementNoHole(k) otherwise continue;
       const result: Object =
-          Call(context, callbackfn, thisArg, value, k, fastO.Get());
+          Call(context, callbackfn, thisArg, value, k, fastOW.Get());
       if (ToBoolean(result)) {
         try {
           // Since the call to {callbackfn} is observable, we can't
           // use the Bailout label until we've successfully stored.
           // Hence the {SlowStore} label.
-          fastA.Recheck() otherwise SlowStore;
-          if (fastA.Get().length != to) goto SlowStore;
-          BuildAppendJSArray(kind, fastA.Get(), value)
-              otherwise SlowStore;
+          fastOutputW.Recheck() otherwise SlowStore;
+          if (fastOutputW.Get().length != to) goto SlowStore;
+          fastOutputW.Push(value) otherwise SlowStore;
         }
         label SlowStore {
-          CreateDataProperty(a, to, value);
+          CreateDataProperty(fastOutputW.stable, to, value);
         }
         to = to + 1;
       }
     }
   }
 
-  transitioning macro FastArrayFilter(implicit context: Context)(
-      o: JSReceiver, len: Number, callbackfn: Callable, thisArg: Object,
-      array: JSReceiver): Object
-      labels Bailout(Smi, Smi) {
-    let k: Smi = 0;
-    let to: Smi = 0;
-    const smiLen = Cast<Smi>(len) otherwise goto Bailout(k, to);
-    const fastArray = Cast<FastJSArray>(array) otherwise goto Bailout(k, to);
-    let fastO = Cast<FastJSArray>(o) otherwise goto Bailout(k, to);
-    EnsureArrayPushable(fastArray.map) otherwise goto Bailout(k, to);
-    const elementsKind: ElementsKind = fastO.map.elements_kind;
-    if (IsElementsKindLessThanOrEqual(elementsKind, HOLEY_SMI_ELEMENTS)) {
-      FilterVisitAllElements<FixedArray>(
-          HOLEY_SMI_ELEMENTS, fastO, smiLen, callbackfn, thisArg, fastArray)
-          otherwise Bailout;
-    } else if (IsElementsKindLessThanOrEqual(elementsKind, HOLEY_ELEMENTS)) {
-      FilterVisitAllElements<FixedArray>(
-          HOLEY_ELEMENTS, fastO, smiLen, callbackfn, thisArg, fastArray)
-          otherwise Bailout;
-    } else {
-      assert(IsDoubleElementsKind(elementsKind));
-      FilterVisitAllElements<FixedDoubleArray>(
-          HOLEY_DOUBLE_ELEMENTS, fastO, smiLen, callbackfn, thisArg, fastArray)
-          otherwise Bailout;
-    }
-    return array;
-  }
-
   // This method creates a 0-length array with the ElementsKind of the
   // receiver if possible, otherwise, bails out. It makes sense for the
   // caller to know that the slow case needs to be invoked.
@@ -197,29 +166,35 @@ namespace array_filter {
 
       // 4. If thisArg is present, let T be thisArg; else let T be undefined.
       const thisArg: Object = arguments.length > 1 ? arguments[1] : Undefined;
-      let array: JSReceiver;
+      let output: JSReceiver;
 
       // Special cases.
       let k: Number = 0;
       let to: Number = 0;
       try {
-        array = FastFilterSpeciesCreate(o) otherwise SlowSpeciesCreate;
+        output = FastFilterSpeciesCreate(o) otherwise SlowSpeciesCreate;
 
         try {
-          return FastArrayFilter(o, len, callbackfn, thisArg, array)
+          const smiLen: Smi = Cast<Smi>(len) otherwise goto Bailout(k, to);
+          const fastOutput =
+              Cast<FastJSArray>(output) otherwise goto Bailout(k, to);
+          const fastO = Cast<FastJSArray>(o) otherwise goto Bailout(k, to);
+
+          FastArrayFilter(fastO, smiLen, callbackfn, thisArg, fastOutput)
               otherwise Bailout;
+          return output;
         }
-        label Bailout(kValue: Smi, toValue: Smi) deferred {
+        label Bailout(kValue: Number, toValue: Number) deferred {
           k = kValue;
           to = toValue;
         }
       }
       label SlowSpeciesCreate {
-        array = ArraySpeciesCreate(context, receiver, 0);
+        output = ArraySpeciesCreate(context, receiver, 0);
       }
 
       return ArrayFilterLoopContinuation(
-          o, callbackfn, thisArg, array, o, k, len, to);
+          o, callbackfn, thisArg, output, o, k, len, to);
     }
     label TypeError deferred {
       ThrowTypeError(kCalledNonCallable, arguments[0]);

src/builtins/array-foreach.tq

@@ -67,39 +67,23 @@ namespace array_foreach {
     return Undefined;
   }
 
-  transitioning macro VisitAllElements<FixedArrayType: type>(implicit context:
-                                                                  Context)(
-      o: JSArray, len: Smi, callbackfn: Callable, thisArg: Object) labels
-  Bailout(Smi) {
-    let k: Smi = 0;
-    let fastO =
-        NewFastJSArrayWitness(Cast<FastJSArray>(o) otherwise goto Bailout(k));
-
-    // Build a fast loop over the smi array.
-    for (; k < len; k++) {
-      fastO.Recheck() otherwise goto Bailout(k);
-
-      // Ensure that we haven't walked beyond a possibly updated length.
-      if (k >= fastO.Get().length) goto Bailout(k);
-      const value: Object = LoadElementNoHole<FixedArrayType>(fastO.Get(), k)
-          otherwise continue;
-      Call(context, callbackfn, thisArg, value, k, fastO.Get());
-    }
-  }
-
   transitioning macro FastArrayForEach(implicit context: Context)(
       o: JSReceiver, len: Number, callbackfn: Callable, thisArg: Object): Object
       labels Bailout(Smi) {
     let k: Smi = 0;
     const smiLen = Cast<Smi>(len) otherwise goto Bailout(k);
     let fastO = Cast<FastJSArray>(o) otherwise goto Bailout(k);
-    const elementsKind: ElementsKind = fastO.map.elements_kind;
-    if (IsElementsKindGreaterThan(elementsKind, HOLEY_ELEMENTS)) {
-      VisitAllElements<FixedDoubleArray>(fastO, smiLen, callbackfn, thisArg)
-          otherwise Bailout;
-    } else {
-      VisitAllElements<FixedArray>(fastO, smiLen, callbackfn, thisArg)
-          otherwise Bailout;
+    let fastOW = NewFastJSArrayWitness(fastO);
+
+    // Build a fast loop over the smi array.
+    for (; k < smiLen; k++) {
+      fastOW.Recheck() otherwise goto Bailout(k);
+
+      // Ensure that we haven't walked beyond a possibly updated length.
+      if (k >= fastOW.Get().length) goto Bailout(k);
+      const value: Object = fastOW.LoadElementNoHole(k)
+          otherwise continue;
+      Call(context, callbackfn, thisArg, value, k, fastOW.Get());
     }
     return Undefined;
   }

src/builtins/array-map.tq

@@ -22,7 +22,7 @@ namespace array_map {
 
     return ArrayMapLoopContinuation(
         jsreceiver, callbackfn, thisArg, outputArray, jsreceiver, numberK,
-        numberLength, Undefined);
+        numberLength);
   }
 
   transitioning javascript builtin
@@ -51,15 +51,13 @@ namespace array_map {
 
     return ArrayMapLoopContinuation(
         jsreceiver, callbackfn, thisArg, outputArray, jsreceiver, numberK,
-        numberLength, Undefined);
+        numberLength);
   }
 
   transitioning builtin ArrayMapLoopContinuation(implicit context: Context)(
       receiver: JSReceiver, callbackfn: Callable, thisArg: Object,
-      array: JSReceiver, o: JSReceiver, initialK: Number, length: Number,
-      initialTo: Object): Object {
-    // {initialTo} is ignored.
-
+      array: JSReceiver, o: JSReceiver, initialK: Number,
+      length: Number): Object {
     // 6. Let k be 0.
     // 7. Repeat, while k < len
     for (let k: Number = initialK; k < length; k++) {
@@ -178,66 +176,39 @@ namespace array_map {
     return Vector{fixedArray, true, true, false};
   }
 
-  transitioning macro
-  MapVisitAllElements<FixedArrayType: type>(implicit context: Context)(
-      o: JSArray, len: Smi, callbackfn: Callable, thisArg: Object,
-      vector: Vector): Vector labels Bailout(Vector, Smi) {
+  transitioning macro FastArrayMap(implicit context: Context)(
+      fastO: FastJSArray, len: Smi, callbackfn: Callable,
+      thisArg: Object): JSArray
+      labels Bailout(JSArray, Smi) {
     let k: Smi = 0;
-    let v: Vector = vector;
-    const array = Cast<FastJSArray>(o) otherwise goto Bailout(v, k);
-    let fastO = NewFastJSArrayWitness(array);
+    let fastOW = NewFastJSArrayWitness(fastO);
+    let vector = NewVector(len);
 
     // Build a fast loop over the smi array.
     // 7. Repeat, while k < len.
+    try {
       for (; k < len; k++) {
-      fastO.Recheck() otherwise goto Bailout(v, k);
+        fastOW.Recheck() otherwise goto PrepareBailout(k);
 
         // Ensure that we haven't walked beyond a possibly updated length.
-      if (k >= fastO.Get().length) goto Bailout(v, k);
+        if (k >= fastOW.Get().length) goto PrepareBailout(k);
 
         try {
-        const value: Object = LoadElementNoHole<FixedArrayType>(fastO.Get(), k)
+          const value: Object = fastOW.LoadElementNoHole(k)
               otherwise FoundHole;
           const result: Object =
-            Call(context, callbackfn, thisArg, value, k, fastO.Get());
-        v.StoreResult(k, result);
+              Call(context, callbackfn, thisArg, value, k, fastOW.Get());
+          vector.StoreResult(k, result);
         }
         label FoundHole {
           // Our output array must necessarily be holey because of holes in
           // the input array.
-        v.ReportSkippedElement();
-      }
-    }
-
-    return v;
+          vector.ReportSkippedElement();
         }
-
-  transitioning macro FastArrayMap(implicit context: Context)(
-      o: JSReceiver, len: Smi, callbackfn: Callable, thisArg: Object): JSArray
-      labels Bailout(JSArray, Smi) {
-    let k: Smi = 0;
-    let fastO = Cast<FastJSArray>(o) otherwise unreachable;
-    let vector = NewVector(len);
-    const elementsKind: ElementsKind = fastO.map.elements_kind;
-    try {
-      if (IsElementsKindLessThanOrEqual(elementsKind, HOLEY_SMI_ELEMENTS)) {
-        vector = MapVisitAllElements<FixedArray>(
-            fastO, len, callbackfn, thisArg, vector)
-            otherwise InnerBailout;
-      } else if (IsElementsKindLessThanOrEqual(elementsKind, HOLEY_ELEMENTS)) {
-        vector = MapVisitAllElements<FixedArray>(
-            fastO, len, callbackfn, thisArg, vector)
-            otherwise InnerBailout;
-      } else {
-        assert(IsDoubleElementsKind(elementsKind));
-        vector = MapVisitAllElements<FixedDoubleArray>(
-            fastO, len, callbackfn, thisArg, vector)
-            otherwise InnerBailout;
       }
     }
-    label InnerBailout(v: Vector, k: Smi) {
-      // Transform the Vector {v} into a JSArray and bail out.
-      let vector: Vector = v;
+    label PrepareBailout(k: Smi) deferred {
+      // Transform {vector} into a JSArray and bail out.
       goto Bailout(vector.CreateJSArray(k), k);
     }
 
@@ -298,8 +269,7 @@ namespace array_map {
         k = kValue;
       }
 
-      return ArrayMapLoopContinuation(
-          o, callbackfn, thisArg, array, o, k, len, Undefined);
+      return ArrayMapLoopContinuation(o, callbackfn, thisArg, array, o, k, len);
     }
     label TypeError deferred {
       ThrowTypeError(kCalledNonCallable, arguments[0]);

src/builtins/array-reduce-right.tq

@@ -105,25 +105,27 @@ namespace array {
     return accumulator;
   }
 
-  transitioning macro
-  ReduceRightVisitAllElements<FixedArrayType: type>(implicit context: Context)(
-      o: FastJSArray, len: Smi, callbackfn: Callable,
+  transitioning macro FastArrayReduceRight(implicit context: Context)(
+      o: JSReceiver, len: Number, callbackfn: Callable,
       initialAccumulator: Object): Object
       labels Bailout(Number, Object) {
-    let fastO = NewFastJSArrayWitness(o);
     let accumulator = initialAccumulator;
+    const smiLen = Cast<Smi>(len) otherwise goto Bailout(len - 1, accumulator);
+    let fastO =
+        Cast<FastJSArray>(o) otherwise goto Bailout(len - 1, accumulator);
+    let fastOW = NewFastJSArrayWitness(fastO);
 
     // Build a fast loop over the array.
-    for (let k: Smi = len - 1; k >= 0; k--) {
-      fastO.Recheck() otherwise goto Bailout(k, accumulator);
+    for (let k: Smi = smiLen - 1; k >= 0; k--) {
+      fastOW.Recheck() otherwise goto Bailout(k, accumulator);
 
-      const value: Object = LoadElementNoHole<FixedArrayType>(fastO.Get(), k)
-          otherwise continue;
+      const value: Object = fastOW.LoadElementNoHole(k) otherwise continue;
       if (accumulator == Hole) {
         accumulator = value;
       } else {
         accumulator = Call(
-            context, callbackfn, Undefined, accumulator, value, k, fastO.Get());
+            context, callbackfn, Undefined, accumulator, value, k,
+            fastOW.Get());
       }
     }
     if (accumulator == Hole) {
@@ -132,23 +134,6 @@ namespace array {
     return accumulator;
   }
 
-  transitioning macro FastArrayReduceRight(implicit context: Context)(
-      o: JSReceiver, len: Number, callbackfn: Callable,
-      accumulator: Object): Object
-      labels Bailout(Number, Object) {
-    const k = len - 1;
-    const smiLen = Cast<Smi>(len) otherwise goto Bailout(k, accumulator);
-    let fastO = Cast<FastJSArray>(o) otherwise goto Bailout(k, accumulator);
-    const elementsKind: ElementsKind = fastO.map.elements_kind;
-    if (IsElementsKindGreaterThan(elementsKind, HOLEY_ELEMENTS)) {
-      return ReduceRightVisitAllElements<FixedDoubleArray>(
-          fastO, smiLen, callbackfn, accumulator) otherwise Bailout;
-    } else {
-      return ReduceRightVisitAllElements<FixedArray>(
-          fastO, smiLen, callbackfn, accumulator) otherwise Bailout;
-    }
-  }
-
   // https://tc39.github.io/ecma262/#sec-array.prototype.reduceRight
   transitioning javascript builtin
   ArrayReduceRight(implicit context: Context)(receiver: Object, ...arguments):

src/builtins/array-reduce.tq

@@ -104,25 +104,27 @@ namespace array {
     return accumulator;
   }
 
-  transitioning macro
-  ReduceVisitAllElements<FixedArrayType: type>(implicit context: Context)(
-      o: FastJSArray, len: Smi, callbackfn: Callable,
+  transitioning macro FastArrayReduce(implicit context: Context)(
+      o: JSReceiver, len: Number, callbackfn: Callable,
       initialAccumulator: Object): Object
       labels Bailout(Number, Object) {
-    let fastO = NewFastJSArrayWitness(o);
+    const k = 0;
     let accumulator = initialAccumulator;
+    const smiLen = Cast<Smi>(len) otherwise goto Bailout(k, accumulator);
+    let fastO = Cast<FastJSArray>(o) otherwise goto Bailout(k, accumulator);
+    let fastOW = NewFastJSArrayWitness(fastO);
 
     // Build a fast loop over the array.
     for (let k: Smi = 0; k < len; k++) {
-      fastO.Recheck() otherwise goto Bailout(k, accumulator);
+      fastOW.Recheck() otherwise goto Bailout(k, accumulator);
 
-      const value: Object = LoadElementNoHole<FixedArrayType>(fastO.Get(), k)
-          otherwise continue;
+      const value: Object = fastOW.LoadElementNoHole(k) otherwise continue;
       if (accumulator == Hole) {
         accumulator = value;
       } else {
         accumulator = Call(
-            context, callbackfn, Undefined, accumulator, value, k, fastO.Get());
+            context, callbackfn, Undefined, accumulator, value, k,
+            fastOW.Get());
       }
     }
     if (accumulator == Hole) {
@@ -131,23 +133,6 @@ namespace array {
     return accumulator;
   }
 
-  transitioning macro FastArrayReduce(implicit context: Context)(
-      o: JSReceiver, len: Number, callbackfn: Callable,
-      accumulator: Object): Object
-      labels Bailout(Number, Object) {
-    const k = 0;
-    const smiLen = Cast<Smi>(len) otherwise goto Bailout(k, accumulator);
-    let fastO = Cast<FastJSArray>(o) otherwise goto Bailout(k, accumulator);
-    const elementsKind: ElementsKind = fastO.map.elements_kind;
-    if (IsElementsKindGreaterThan(elementsKind, HOLEY_ELEMENTS)) {
-      return ReduceVisitAllElements<FixedDoubleArray>(
-          fastO, smiLen, callbackfn, accumulator) otherwise Bailout;
-    } else {
-      return ReduceVisitAllElements<FixedArray>(
-          fastO, smiLen, callbackfn, accumulator) otherwise Bailout;
-    }
-  }
-
   // https://tc39.github.io/ecma262/#sec-array.prototype.reduce
   transitioning javascript builtin
   ArrayReduce(implicit context: Context)(receiver: Object, ...arguments):

src/builtins/base.tq

@@ -1552,10 +1552,34 @@ struct FastJSArrayWitness {
     }
   }
 
+  EnsureArrayPushable() labels Failed {
+    EnsureArrayPushable(this.map) otherwise Failed;
+    this.arrayIsPushable = true;
+  }
+
+  Push(value: Object) labels Failed {
+    assert(this.arrayIsPushable);
+    if (this.hasDoubles) {
+      BuildAppendJSArray(HOLEY_DOUBLE_ELEMENTS, this.unstable, value)
+          otherwise Failed;
+    } else if (this.hasSmis) {
+      BuildAppendJSArray(HOLEY_SMI_ELEMENTS, this.unstable, value)
+          otherwise Failed;
+    } else {
+      assert(
+          this.map.elements_kind == HOLEY_ELEMENTS ||
+          this.map.elements_kind == PACKED_ELEMENTS);
+      BuildAppendJSArray(HOLEY_ELEMENTS, this.unstable, value)
+          otherwise Failed;
+    }
+  }
+
   stable: JSArray;
   unstable: FastJSArray;
   map: Map;
   hasDoubles: bool;
+  hasSmis: bool;
+  arrayIsPushable: bool;
 }
 
 macro NewFastJSArrayWitness(array: FastJSArray): FastJSArrayWitness {
@@ -1564,7 +1588,9 @@ macro NewFastJSArrayWitness(array: FastJSArray): FastJSArrayWitness {
     array,
     array,
     array.map,
-    !IsElementsKindLessThanOrEqual(kind, HOLEY_ELEMENTS)
+    !IsElementsKindLessThanOrEqual(kind, HOLEY_ELEMENTS),
+    IsElementsKindLessThanOrEqual(kind, HOLEY_SMI_ELEMENTS),
+    false
   };
 }