[typedarray] Change TypedArray.p.sort implementation.

This CL uses the new function pointers and generic features of Torque
to improve the performance of TypedArray.p.sort.

Instead of one Load/Store builtin that dispatches at runtime based on
the element kind, there are now many small builtins (one for each
element kind). The sorting algorithm then uses function pointers to
those small builtins, which get set once.

Changes in the relevant benchmarks:

Benchmark   Original (JS)   Current   This CL
IntTypes             83.9     202.3     240.7
BigIntTypes          32.1      47.2      53.3
FloatTypes           99.3     109.3     129.3

Bug: v8:7382
Change-Id: I8684410524d546615b19f6edcbfdc615068196aa
Reviewed-on: https://chromium-review.googlesource.com/1070069
Commit-Queue: Simon Zünd <szuend@google.com>
Reviewed-by: Jakob Gruber <jgruber@chromium.org>
Cr-Commit-Position: refs/heads/master@{#53322}

src/builtins/base.tq

@@ -33,6 +33,10 @@ type FixedArrayBase extends HeapObject generates 'TNode<FixedArrayBase>';
 type FixedArray extends FixedArrayBase generates 'TNode<FixedArray>';
 type FixedDoubleArray extends FixedArrayBase generates
 'TNode<FixedDoubleArray>';
+type FixedTypedArrayBase extends FixedArrayBase generates
+'TNode<FixedTypedArrayBase>';
+type FixedTypedArray extends FixedTypedArrayBase generates
+'TNode<FixedTypedArray>';
 
 type JSArrayBuffer extends Object generates 'TNode<JSArrayBuffer>';
 type JSArrayBufferView extends Object generates 'TNode<JSArrayBufferView>';
@@ -47,6 +51,8 @@ type ExtractFixedArrayFlags generates
 type MessageTemplate;
 type HasPropertyFlag generates 'HasPropertyLookupMode';
 
+const NO_ELEMENTS: constexpr ElementsKind = 'NO_ELEMENTS';
+
 const PACKED_SMI_ELEMENTS: constexpr ElementsKind = 'PACKED_SMI_ELEMENTS';
 const HOLEY_SMI_ELEMENTS: constexpr ElementsKind = 'HOLEY_SMI_ELEMENTS';
 const PACKED_ELEMENTS: constexpr ElementsKind = 'PACKED_ELEMENTS';
@@ -66,6 +72,18 @@ const UINT8_CLAMPED_ELEMENTS: constexpr ElementsKind = 'UINT8_CLAMPED_ELEMENTS';
 const BIGUINT64_ELEMENTS: constexpr ElementsKind = 'BIGUINT64_ELEMENTS';
 const BIGINT64_ELEMENTS: constexpr ElementsKind = 'BIGINT64_ELEMENTS';
 
+type FixedUint8Array extends FixedTypedArray;
+type FixedInt8Array extends FixedTypedArray;
+type FixedUint16Array extends FixedTypedArray;
+type FixedInt16Array extends FixedTypedArray;
+type FixedUint32Array extends FixedTypedArray;
+type FixedInt32Array extends FixedTypedArray;
+type FixedFloat32Array extends FixedTypedArray;
+type FixedFloat64Array extends FixedTypedArray;
+type FixedUint8ClampedArray extends FixedTypedArray;
+type FixedBigUint64Array extends FixedTypedArray;
+type FixedBigInt64Array extends FixedTypedArray;
+
 const kAllFixedArrays: constexpr ExtractFixedArrayFlags =
     'ExtractFixedArrayFlag::kAllFixedArrays';
 
@@ -223,7 +241,8 @@ extern implicit operator
 extern implicit operator 'convert<>' macro BoolConstant(constexpr bool): bool;
 extern implicit operator 'convert<>' macro LanguageModeConstant(
     constexpr LanguageMode): LanguageMode;
-
+extern operator
+'convert<>' macro Int32Constant(constexpr ElementsKind): ElementsKind;
 extern implicit operator 'convert<>' macro SmiFromInt32(ElementsKind): Smi;
 
 extern operator 'convert<>' macro ChangeInt32ToTagged(int32): Number;

src/builtins/builtins-definitions.h

@@ -1147,8 +1147,6 @@ namespace internal {
   TFJ(TypedArrayOf, SharedFunctionInfo::kDontAdaptArgumentsSentinel)           \
   /* ES6 %TypedArray%.from */                                                  \
   TFJ(TypedArrayFrom, SharedFunctionInfo::kDontAdaptArgumentsSentinel)         \
-  TFS(TypedArrayLoadElementAsTagged, kArray, kKind, kIndex)                    \
-  TFS(TypedArrayStoreElementFromTagged, kArray, kKind, kIndex, kValue)         \
                                                                                \
   /* Wasm */                                                                   \
   ASM(WasmCompileLazy)                                                         \

src/builtins/builtins-typed-array-gen.cc

@@ -873,42 +873,6 @@ TNode<IntPtrT> TypedArrayBuiltinsAssembler::GetTypedArrayElementSize(
   return element_size.value();
 }
 
-TF_BUILTIN(TypedArrayLoadElementAsTagged, TypedArrayBuiltinsAssembler) {
-  TVARIABLE(Object, result);
-  TNode<JSTypedArray> array = CAST(Parameter(Descriptor::kArray));
-  TNode<Smi> kind = CAST(Parameter(Descriptor::kKind));
-  TNode<Smi> index_node = CAST(Parameter(Descriptor::kIndex));
-
-  TNode<RawPtrT> data_pointer = UncheckedCast<RawPtrT>(LoadDataPtr(array));
-  TNode<Int32T> elements_kind = SmiToInt32(kind);
-
-  DispatchTypedArrayByElementsKind(
-      elements_kind, [&](ElementsKind el_kind, int, int) {
-        result = CAST(LoadFixedTypedArrayElementAsTagged(
-            data_pointer, index_node, el_kind, SMI_PARAMETERS));
-      });
-
-  Return(result.value());
-}
-
-TF_BUILTIN(TypedArrayStoreElementFromTagged, TypedArrayBuiltinsAssembler) {
-  TNode<Context> context = CAST(Parameter(Descriptor::kContext));
-  TNode<JSTypedArray> array = CAST(Parameter(Descriptor::kArray));
-  TNode<Smi> kind = CAST(Parameter(Descriptor::kKind));
-  TNode<Smi> index_node = CAST(Parameter(Descriptor::kIndex));
-  TNode<Object> value = CAST(Parameter(Descriptor::kValue));
-
-  TNode<FixedTypedArrayBase> elements = CAST(LoadElements(array));
-  TNode<Int32T> elements_kind = SmiToInt32(kind);
-
-  DispatchTypedArrayByElementsKind(
-      elements_kind, [&](ElementsKind el_kind, int, int) {
-        StoreFixedTypedArrayElementFromTagged(context, elements, index_node,
-                                              value, el_kind, SMI_PARAMETERS);
-      });
-  Return(UndefinedConstant());
-}
-
 TNode<Object> TypedArrayBuiltinsAssembler::GetDefaultConstructor(
     TNode<Context> context, TNode<JSTypedArray> exemplar) {
   TVARIABLE(IntPtrT, context_slot);

src/builtins/builtins-typed-array-gen.h

@@ -141,6 +141,12 @@ class TypedArrayBuiltinsAssembler : public CodeStubAssembler {
   // Returns true iff number is NaN.
   // TOOD(szuend): Remove when UncheckedCasts are supported in Torque.
   TNode<BoolT> NumberIsNaN(TNode<Number> number);
+
+  // TODO(szuend): Remove when UncheckedCasts are supported in Torque.
+  TNode<FixedTypedArrayBase> UncheckedCastFixedArrayBaseToFixedTypedArrayBase(
+      TNode<FixedArrayBase> array) {
+    return CAST(array);
+  }
 };
 
 }  // namespace internal

src/builtins/typed-array.tq

@@ -9,13 +9,74 @@ module typed_array {
   const kTypedArrayProtoSort: MethodName = '\"%TypedArray%.prototype.sort\"';
   extern macro ValidateTypedArray(Context, Object, MethodName): JSTypedArray;
 
-  extern builtin TypedArrayLoadElementAsTagged(
-      Context, JSTypedArray, Smi, Smi): Object;
-  extern builtin TypedArrayStoreElementFromTagged(
-      Context, JSTypedArray, Smi, Smi, Object);
-
   extern macro NumberIsNaN(Number): bool;
 
+  type ParameterMode constexpr 'ParameterMode';
+  const SMI_PARAMETERS: constexpr ParameterMode = 'SMI_PARAMETERS';
+  extern macro LoadFixedTypedArrayElementAsTagged(
+      RawPtr, Smi, constexpr ElementsKind, constexpr ParameterMode): Object;
+  extern macro StoreFixedTypedArrayElementFromTagged(
+      Context, FixedArrayBase, Smi, Object, constexpr ElementsKind,
+      constexpr ParameterMode);
+
+  type LoadFn = builtin(Context, JSTypedArray, Smi) => Object;
+  type StoreFn = builtin(Context, JSTypedArray, Smi, Object) => Object;
+
+  macro KindForArrayType<T : type>(): constexpr ElementsKind;
+  KindForArrayType<FixedUint8Array>(): constexpr ElementsKind {
+    return UINT8_ELEMENTS;
+  }
+  KindForArrayType<FixedInt8Array>(): constexpr ElementsKind {
+    return INT8_ELEMENTS;
+  }
+  KindForArrayType<FixedUint16Array>(): constexpr ElementsKind {
+    return UINT16_ELEMENTS;
+  }
+  KindForArrayType<FixedInt16Array>(): constexpr ElementsKind {
+    return INT16_ELEMENTS;
+  }
+  KindForArrayType<FixedUint32Array>(): constexpr ElementsKind {
+    return UINT32_ELEMENTS;
+  }
+  KindForArrayType<FixedInt32Array>(): constexpr ElementsKind {
+    return INT32_ELEMENTS;
+  }
+  KindForArrayType<FixedFloat32Array>(): constexpr ElementsKind {
+    return FLOAT32_ELEMENTS;
+  }
+  KindForArrayType<FixedFloat64Array>(): constexpr ElementsKind {
+    return FLOAT64_ELEMENTS;
+  }
+  KindForArrayType<FixedUint8ClampedArray>(): constexpr ElementsKind {
+    return UINT8_CLAMPED_ELEMENTS;
+  }
+  KindForArrayType<FixedBigUint64Array>(): constexpr ElementsKind {
+    return BIGUINT64_ELEMENTS;
+  }
+  KindForArrayType<FixedBigInt64Array>(): constexpr ElementsKind {
+    return BIGINT64_ELEMENTS;
+  }
+
+  extern operator
+  'convert<>' macro UncheckedCastFixedArrayBaseToFixedTypedArrayBase(
+      FixedArrayBase): FixedTypedArrayBase;
+
+  builtin LoadFixedElement<T : type>(
+      context: Context, array: JSTypedArray, index: Smi): Object {
+    return LoadFixedTypedArrayElementAsTagged(
+        array.data_ptr, index, KindForArrayType<T>(), SMI_PARAMETERS);
+  }
+
+  builtin StoreFixedElement<T : type>(
+      context: Context, array: JSTypedArray, index: Smi,
+      value: Object): Object {
+    let elements: FixedTypedArrayBase =
+        convert<FixedTypedArrayBase>(array.elements);
+    StoreFixedTypedArrayElementFromTagged(
+        context, elements, index, value, KindForArrayType<T>(), SMI_PARAMETERS);
+    return Undefined;
+  }
+
   macro CallCompareWithDetachedCheck(
       context: Context, array: JSTypedArray, comparefn: Callable, a: Object,
       b: Object): Number labels Detached {
@@ -33,32 +94,10 @@ module typed_array {
     return v;
   }
 
-  // Wrapped CSA macro for better readability. Ideally we want to map this
-  // as the array operator "[]".
-  // TODO(szuend): Change Load/Store macros so they use function pointers to
-  //               the correct builtins as soon as they are available in Torque.
-  //
-  //               Currently the dispatch to the correct load/store instruction
-  //               is done during runtime in a builtin. This costs around 20%
-  //               performance in relevant benchmarks, but greatly reduces the
-  //               code size - compared to sort macro "copies" for each
-  //               ElementsKind that inline the correct load/store.
-  macro Load(
-      context: Context, array: JSTypedArray, kind: Smi, index: Smi): Object {
-    return TypedArrayLoadElementAsTagged(context, array, kind, index);
-  }
-
-  // Wrapped array store CSA macro for better readability.
-  macro Store(
-      context: Context, array: JSTypedArray, kind: Smi, index: Smi,
-      value: Object) {
-    TypedArrayStoreElementFromTagged(context, array, kind, index, value);
-  }
-
   // InsertionSort is used for smaller arrays.
   macro TypedArrayInsertionSort(
-      context: Context, array: JSTypedArray, kind: Smi, from_arg: Smi,
-      to_arg: Smi, comparefn: Callable)
+      context: Context, array: JSTypedArray, from_arg: Smi, to_arg: Smi,
+      comparefn: Callable, Load: LoadFn, Store: StoreFn)
   labels Detached {
     let from: Smi = from_arg;
     let to: Smi = to_arg;
@@ -66,25 +105,25 @@ module typed_array {
     if (IsDetachedBuffer(array.buffer)) goto Detached;
 
     for (let i: Smi = from + 1; i < to; ++i) {
-      let element: Object = Load(context, array, kind, i);
+      let element: Object = Load(context, array, i);
       let j: Smi = i - 1;
       for (; j >= from; --j) {
-        let tmp: Object = Load(context, array, kind, j);
+        let tmp: Object = Load(context, array, j);
         let order: Number = CallCompareWithDetachedCheck(
             context, array, comparefn, tmp, element) otherwise Detached;
         if (order > 0) {
-          Store(context, array, kind, j + 1, tmp);
+          Store(context, array, j + 1, tmp);
         } else {
           break;
         }
       }
-      Store(context, array, kind, j + 1, element);
+      Store(context, array, j + 1, element);
     }
   }
 
   macro TypedArrayQuickSortImpl(
-      context: Context, array: JSTypedArray, kind: Smi, from_arg: Smi,
-      to_arg: Smi, comparefn: Callable)
+      context: Context, array: JSTypedArray, from_arg: Smi, to_arg: Smi,
+      comparefn: Callable, Load: LoadFn, Store: StoreFn)
   labels Detached {
     let from: Smi = from_arg;
     let to: Smi = to_arg;
@@ -94,7 +133,8 @@ module typed_array {
         // TODO(szuend): Investigate InsertionSort removal.
         //               Currently it does not make any difference when the
         //               benchmarks are run locally.
-        TypedArrayInsertionSort(context, array, kind, from, to, comparefn)
+        TypedArrayInsertionSort(
+            context, array, from, to, comparefn, Load, Store)
         otherwise Detached;
         break;
       }
@@ -106,9 +146,9 @@ module typed_array {
       if (IsDetachedBuffer(array.buffer)) goto Detached;
 
       // Find a pivot as the median of first, last and middle element.
-      let v0: Object = Load(context, array, kind, from);
-      let v1: Object = Load(context, array, kind, to - 1);
-      let v2: Object = Load(context, array, kind, third_index);
+      let v0: Object = Load(context, array, from);
+      let v1: Object = Load(context, array, to - 1);
+      let v2: Object = Load(context, array, third_index);
 
       let c01: Number = CallCompareWithDetachedCheck(
           context, array, comparefn, v0, v1) otherwise Detached;
@@ -140,28 +180,28 @@ module typed_array {
       }
 
       // v0 <= v1 <= v2.
-      Store(context, array, kind, from, v0);
-      Store(context, array, kind, to - 1, v2);
+      Store(context, array, from, v0);
+      Store(context, array, to - 1, v2);
 
       let pivot: Object = v1;
       let low_end: Smi = from + 1;   // Upper bound of elems lower than pivot.
       let high_start: Smi = to - 1;  // Lower bound of elems greater than pivot.
 
-      let low_end_value: Object = Load(context, array, kind, low_end);
-      Store(context, array, kind, third_index, low_end_value);
-      Store(context, array, kind, low_end, pivot);
+      let low_end_value: Object = Load(context, array, low_end);
+      Store(context, array, third_index, low_end_value);
+      Store(context, array, low_end, pivot);
 
       // From low_end to idx are elements equal to pivot.
       // From idx to high_start are elements that haven"t been compared yet.
       for (let idx: Smi = low_end + 1; idx < high_start; idx++) {
-        let element: Object = Load(context, array, kind, idx);
+        let element: Object = Load(context, array, idx);
         let order: Number = CallCompareWithDetachedCheck(
             context, array, comparefn, element, pivot) otherwise Detached;
 
         if (order < 0) {
-          low_end_value = Load(context, array, kind, low_end);
-          Store(context, array, kind, idx, low_end_value);
-          Store(context, array, kind, low_end, element);
+          low_end_value = Load(context, array, low_end);
+          Store(context, array, idx, low_end_value);
+          Store(context, array, low_end, element);
           low_end++;
         } else if (order > 0) {
           let break_for: bool = false;
@@ -173,7 +213,7 @@ module typed_array {
               break;
             }
 
-            let top_elem: Object = Load(context, array, kind, high_start);
+            let top_elem: Object = Load(context, array, high_start);
             order = CallCompareWithDetachedCheck(
                 context, array, comparefn, top_elem, pivot) otherwise Detached;
           }
@@ -182,36 +222,38 @@ module typed_array {
             break;
           }
 
-          let high_start_value: Object = Load(context, array, kind, high_start);
-          Store(context, array, kind, idx, high_start_value);
-          Store(context, array, kind, high_start, element);
+          let high_start_value: Object = Load(context, array, high_start);
+          Store(context, array, idx, high_start_value);
+          Store(context, array, high_start, element);
 
           if (order < 0) {
-            element = Load(context, array, kind, idx);
+            element = Load(context, array, idx);
 
-            low_end_value = Load(context, array, kind, low_end);
-            Store(context, array, kind, idx, low_end_value);
-            Store(context, array, kind, low_end, element);
+            low_end_value = Load(context, array, low_end);
+            Store(context, array, idx, low_end_value);
+            Store(context, array, low_end, element);
             low_end++;
           }
         }
       }
 
       if ((to - high_start) < (low_end - from)) {
-        TypedArrayQuickSort(context, array, kind, high_start, to, comparefn);
+        TypedArrayQuickSort(
+            context, array, high_start, to, comparefn, Load, Store);
         to = low_end;
       } else {
-        TypedArrayQuickSort(context, array, kind, from, low_end, comparefn);
+        TypedArrayQuickSort(
+            context, array, from, low_end, comparefn, Load, Store);
         from = high_start;
       }
     }
   }
 
   builtin TypedArrayQuickSort(
-      context: Context, array: JSTypedArray, kind: Smi, from: Smi, to: Smi,
-      comparefn: Callable): JSTypedArray {
+      context: Context, array: JSTypedArray, from: Smi, to: Smi,
+      comparefn: Callable, Load: LoadFn, Store: StoreFn): JSTypedArray {
     try {
-      TypedArrayQuickSortImpl(context, array, kind, from, to, comparefn)
+      TypedArrayQuickSortImpl(context, array, from, to, comparefn, Load, Store)
       otherwise Detached;
     }
     label Detached {
@@ -250,8 +292,56 @@ module typed_array {
     try {
       let comparefn: Callable =
           cast<Callable>(comparefn_obj) otherwise CastError;
-      let elements_kind: Smi = convert<Smi>(array.elements_kind);
-      TypedArrayQuickSort(context, array, elements_kind, 0, len, comparefn);
+      let loadfn: LoadFn;
+      let storefn: StoreFn;
+
+      let elements_kind: ElementsKind = array.elements_kind;
+
+      if (IsElementsKindGreaterThan(elements_kind, UINT32_ELEMENTS)) {
+        if (elements_kind == convert<ElementsKind>(INT32_ELEMENTS)) {
+          loadfn = LoadFixedElement<FixedInt32Array>;
+          storefn = StoreFixedElement<FixedInt32Array>;
+        } else if (elements_kind == convert<ElementsKind>(FLOAT32_ELEMENTS)) {
+          loadfn = LoadFixedElement<FixedFloat32Array>;
+          storefn = StoreFixedElement<FixedFloat32Array>;
+        } else if (elements_kind == convert<ElementsKind>(FLOAT64_ELEMENTS)) {
+          loadfn = LoadFixedElement<FixedFloat64Array>;
+          storefn = StoreFixedElement<FixedFloat64Array>;
+        } else if (
+            elements_kind == convert<ElementsKind>(UINT8_CLAMPED_ELEMENTS)) {
+          loadfn = LoadFixedElement<FixedUint8ClampedArray>;
+          storefn = StoreFixedElement<FixedUint8ClampedArray>;
+        } else if (elements_kind == convert<ElementsKind>(BIGUINT64_ELEMENTS)) {
+          loadfn = LoadFixedElement<FixedBigUint64Array>;
+          storefn = StoreFixedElement<FixedBigUint64Array>;
+        } else if (elements_kind == convert<ElementsKind>(BIGINT64_ELEMENTS)) {
+          loadfn = LoadFixedElement<FixedBigInt64Array>;
+          storefn = StoreFixedElement<FixedBigInt64Array>;
+        } else {
+          unreachable;
+        }
+      } else {
+        if (elements_kind == convert<ElementsKind>(UINT8_ELEMENTS)) {
+          loadfn = LoadFixedElement<FixedUint8Array>;
+          storefn = StoreFixedElement<FixedUint8Array>;
+        } else if (elements_kind == convert<ElementsKind>(INT8_ELEMENTS)) {
+          loadfn = LoadFixedElement<FixedInt8Array>;
+          storefn = StoreFixedElement<FixedInt8Array>;
+        } else if (elements_kind == convert<ElementsKind>(UINT16_ELEMENTS)) {
+          loadfn = LoadFixedElement<FixedUint16Array>;
+          storefn = StoreFixedElement<FixedUint16Array>;
+        } else if (elements_kind == convert<ElementsKind>(INT16_ELEMENTS)) {
+          loadfn = LoadFixedElement<FixedInt16Array>;
+          storefn = StoreFixedElement<FixedInt16Array>;
+        } else if (elements_kind == convert<ElementsKind>(UINT32_ELEMENTS)) {
+          loadfn = LoadFixedElement<FixedUint32Array>;
+          storefn = StoreFixedElement<FixedUint32Array>;
+        } else {
+          unreachable;
+        }
+      }
+
+      TypedArrayQuickSort(context, array, 0, len, comparefn, loadfn, storefn);
     }
     label CastError {
       unreachable;

test/mjsunit/es6/typedarray-sort.js

@@ -97,6 +97,7 @@ let constructorsWithArrays = [
     ctor: Float64Array,
     array: [2 ** 53, 2 ** 53 - 1, 1, 0, -1, -(2 ** 53 - 1), -(2 ** 53)]
   },
+  {ctor: Uint8ClampedArray, array: [255, 254, 4, 3, 2, 1, 0]},
   {
     ctor: BigUint64Array,
     array: [2n ** 64n - 1n, 2n ** 64n - 2n, 4n, 3n, 2n, 1n, 0n]