[builtins] Port ReplaceGlobalCallableFastPath to Torque

Bug: v8:8976
Change-Id: Idc896770fd0f448c37d8d83b7970e3f8e16f5f2e
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/1572682
Commit-Queue: Peter Wong <peter.wm.wong@gmail.com>
Reviewed-by: Simon Zünd <szuend@chromium.org>
Reviewed-by: Jakob Gruber <jgruber@chromium.org>
Cr-Commit-Position: refs/heads/master@{#60979}

src/builtins/base.tq

@@ -420,10 +420,16 @@ const OBJECT_FUNCTION_INDEX: constexpr NativeContextSlot
     generates 'Context::OBJECT_FUNCTION_INDEX';
 const ITERATOR_RESULT_MAP_INDEX: constexpr NativeContextSlot
     generates 'Context::ITERATOR_RESULT_MAP_INDEX';
+const JS_ARRAY_PACKED_ELEMENTS_MAP_INDEX: constexpr NativeContextSlot
+    generates 'Context::JS_ARRAY_PACKED_ELEMENTS_MAP_INDEX';
 const JS_ARRAY_PACKED_SMI_ELEMENTS_MAP_INDEX: constexpr NativeContextSlot
     generates 'Context::JS_ARRAY_PACKED_SMI_ELEMENTS_MAP_INDEX';
 const PROXY_REVOCABLE_RESULT_MAP_INDEX: constexpr NativeContextSlot
     generates 'Context::PROXY_REVOCABLE_RESULT_MAP_INDEX';
+const REFLECT_APPLY_INDEX: constexpr NativeContextSlot
+    generates 'Context::REFLECT_APPLY_INDEX';
+const REGEXP_LAST_MATCH_INFO_INDEX: constexpr NativeContextSlot
+    generates 'Context::REGEXP_LAST_MATCH_INFO_INDEX';
 extern operator '[]' macro LoadContextElement(
     NativeContext, NativeContextSlot): Object;
 extern operator '[]=' macro StoreContextElement(
@@ -987,6 +993,9 @@ extern class JSRegExpStringIterator extends JSObject {
 
 const kRegExpMatchInfoFirstCaptureIndex:
     constexpr int31 generates 'RegExpMatchInfo::kFirstCaptureIndex';
+const kRegExpMatchInfoNumberOfCapturesIndex:
+    constexpr int31 generates 'RegExpMatchInfo::kNumberOfCapturesIndex';
+
 macro GetStartOfCaptureIndex(captureIndex: constexpr int31): constexpr int31 {
   return kRegExpMatchInfoFirstCaptureIndex + (captureIndex * 2);
 }
@@ -1002,6 +1011,9 @@ extern class RegExpMatchInfo extends FixedArray {
     const index: constexpr int31 = GetStartOfCaptureIndex(captureIndex) + 1;
     return UnsafeCast<Smi>(this.objects[index]);
   }
+  NumberOfCaptures(implicit context: Context)(): Smi {
+    return UnsafeCast<Smi>(this.objects[kRegExpMatchInfoNumberOfCapturesIndex]);
+  }
 }
 
 extern class AccessorInfo extends Struct {
@@ -1409,6 +1421,8 @@ extern macro HeapObjectToHeapNumber(HeapObject): HeapNumber
 extern macro HeapObjectToSloppyArgumentsElements(HeapObject):
     SloppyArgumentsElements
     labels CastError;
+extern macro HeapObjectToRegExpMatchInfo(HeapObject):
+    RegExpMatchInfo labels CastError;
 extern macro TaggedToNumber(Object): Number
     labels CastError;
 
@@ -1441,6 +1455,12 @@ Cast<HeapObject>(o: HeapObject): HeapObject
   return o;
 }
 
+Cast<Null>(o: HeapObject): Null
+    labels CastError {
+  if (o != Null) goto CastError;
+  return %RawDownCast<Null>(o);
+}
+
 Cast<FixedArray>(o: HeapObject): FixedArray
     labels CastError {
   return HeapObjectToFixedArray(o) otherwise CastError;
@@ -2073,16 +2093,25 @@ macro GetArrayBufferNoInitFunction(implicit context: Context)(): JSFunction {
   return UnsafeCast<JSFunction>(
       LoadNativeContext(context)[ARRAY_BUFFER_NOINIT_FUN_INDEX]);
 }
-
+macro GetFastPackedElementsJSArrayMap(implicit context: Context)(): Map {
+  return UnsafeCast<Map>(
+      LoadNativeContext(context)[JS_ARRAY_PACKED_ELEMENTS_MAP_INDEX]);
+}
 macro GetFastPackedSmiElementsJSArrayMap(implicit context: Context)(): Map {
   return UnsafeCast<Map>(
       LoadNativeContext(context)[JS_ARRAY_PACKED_SMI_ELEMENTS_MAP_INDEX]);
 }
-
 macro GetProxyRevocableResultMap(implicit context: Context)(): Map {
   return UnsafeCast<Map>(
       LoadNativeContext(context)[PROXY_REVOCABLE_RESULT_MAP_INDEX]);
 }
+macro GetReflectApply(implicit context: Context)(): Callable {
+  return UnsafeCast<Callable>(LoadNativeContext(context)[REFLECT_APPLY_INDEX]);
+}
+macro GetRegExpLastMatchInfo(implicit context: Context)(): RegExpMatchInfo {
+  return %RawDownCast<RegExpMatchInfo>(
+             LoadNativeContext(context)[REGEXP_LAST_MATCH_INFO_INDEX]);
+}
 
 extern transitioning macro Call(Context, Callable, Object): Object;
 extern transitioning macro Call(Context, Callable, Object, Object): Object;

src/builtins/builtins-regexp-gen.cc

@@ -2685,213 +2685,6 @@ TF_BUILTIN(RegExpPrototypeSplit, RegExpBuiltinsAssembler) {
                                 string, maybe_limit));
 }
 
-Node* RegExpBuiltinsAssembler::ReplaceGlobalCallableFastPath(
-    Node* context, Node* regexp, Node* string, Node* replace_callable) {
-  // The fast path is reached only if {receiver} is a global unmodified
-  // JSRegExp instance and {replace_callable} is callable.
-
-  CSA_ASSERT(this, IsFastRegExp(context, regexp));
-  CSA_ASSERT(this, IsCallable(replace_callable));
-  CSA_ASSERT(this, IsString(string));
-
-  Isolate* const isolate = this->isolate();
-
-  Node* const undefined = UndefinedConstant();
-  TNode<IntPtrT> int_one = IntPtrConstant(1);
-
-  Node* const native_context = LoadNativeContext(context);
-
-  Label out(this);
-  VARIABLE(var_result, MachineRepresentation::kTagged);
-
-  // Set last index to 0.
-  FastStoreLastIndex(regexp, SmiZero());
-
-  // Allocate {result_array}.
-  Node* result_array;
-  {
-    ElementsKind kind = PACKED_ELEMENTS;
-    TNode<Map> array_map = LoadJSArrayElementsMap(kind, native_context);
-    TNode<IntPtrT> capacity = IntPtrConstant(16);
-    TNode<Smi> length = SmiZero();
-    Node* const allocation_site = nullptr;
-    ParameterMode capacity_mode = CodeStubAssembler::INTPTR_PARAMETERS;
-
-    result_array = AllocateJSArray(kind, array_map, capacity, length,
-                                   allocation_site, capacity_mode);
-  }
-
-  // Call into runtime for RegExpExecMultiple.
-  TNode<FixedArray> last_match_info = CAST(LoadContextElement(
-      native_context, Context::REGEXP_LAST_MATCH_INFO_INDEX));
-  Node* const res = CallRuntime(Runtime::kRegExpExecMultiple, context, regexp,
-                                string, last_match_info, result_array);
-
-  // Reset last index to 0.
-  FastStoreLastIndex(regexp, SmiZero());
-
-  // If no matches, return the subject string.
-  var_result.Bind(string);
-  GotoIf(IsNull(res), &out);
-
-  // Reload last match info since it might have changed.
-  last_match_info = CAST(LoadContextElement(
-      native_context, Context::REGEXP_LAST_MATCH_INFO_INDEX));
-
-  Node* const res_length = LoadJSArrayLength(res);
-  TNode<FixedArray> const res_elems = CAST(LoadElements(res));
-
-  TNode<Smi> const num_capture_registers = CAST(LoadFixedArrayElement(
-      last_match_info, RegExpMatchInfo::kNumberOfCapturesIndex));
-
-  Label if_hasexplicitcaptures(this), if_noexplicitcaptures(this),
-      create_result(this);
-  Branch(SmiEqual(num_capture_registers, SmiConstant(2)),
-         &if_noexplicitcaptures, &if_hasexplicitcaptures);
-
-  BIND(&if_noexplicitcaptures);
-  {
-    // If the number of captures is two then there are no explicit captures in
-    // the regexp, just the implicit capture that captures the whole match. In
-    // this case we can simplify quite a bit and end up with something faster.
-    // The builder will consist of some integers that indicate slices of the
-    // input string and some replacements that were returned from the replace
-    // function.
-
-    TVARIABLE(Smi, var_match_start, SmiZero());
-
-    TNode<IntPtrT> const end = SmiUntag(res_length);
-    TVARIABLE(IntPtrT, var_i, IntPtrZero());
-
-    Variable* vars[] = {&var_i, &var_match_start};
-    Label loop(this, 2, vars);
-    Goto(&loop);
-    BIND(&loop);
-    {
-      GotoIfNot(IntPtrLessThan(var_i.value(), end), &create_result);
-
-      Node* const elem = LoadFixedArrayElement(res_elems, var_i.value());
-
-      Label if_issmi(this), if_isstring(this), loop_epilogue(this);
-      Branch(TaggedIsSmi(elem), &if_issmi, &if_isstring);
-
-      BIND(&if_issmi);
-      {
-        TNode<Smi> smi_elem = CAST(elem);
-        // Integers represent slices of the original string.
-        Label if_isnegativeorzero(this), if_ispositive(this);
-        BranchIfSmiLessThanOrEqual(smi_elem, SmiZero(), &if_isnegativeorzero,
-                                   &if_ispositive);
-
-        BIND(&if_ispositive);
-        {
-          TNode<IntPtrT> int_elem = SmiUntag(smi_elem);
-          TNode<IntPtrT> new_match_start =
-              Signed(IntPtrAdd(WordShr(int_elem, IntPtrConstant(11)),
-                               WordAnd(int_elem, IntPtrConstant(0x7FF))));
-          var_match_start = SmiTag(new_match_start);
-          Goto(&loop_epilogue);
-        }
-
-        BIND(&if_isnegativeorzero);
-        {
-          var_i = IntPtrAdd(var_i.value(), int_one);
-
-          TNode<Smi> const next_elem =
-              CAST(LoadFixedArrayElement(res_elems, var_i.value()));
-
-          var_match_start = SmiSub(next_elem, smi_elem);
-          Goto(&loop_epilogue);
-        }
-      }
-
-      BIND(&if_isstring);
-      {
-        CSA_ASSERT(this, IsString(elem));
-
-        Callable call_callable = CodeFactory::Call(isolate);
-        TNode<Smi> match_start = var_match_start.value();
-        Node* const replacement_obj =
-            CallJS(call_callable, context, replace_callable, undefined, elem,
-                   match_start, string);
-
-        TNode<String> const replacement_str =
-            ToString_Inline(context, replacement_obj);
-        StoreFixedArrayElement(res_elems, var_i.value(), replacement_str);
-
-        TNode<Smi> const elem_length = LoadStringLengthAsSmi(elem);
-        var_match_start = SmiAdd(match_start, elem_length);
-
-        Goto(&loop_epilogue);
-      }
-
-      BIND(&loop_epilogue);
-      {
-        var_i = IntPtrAdd(var_i.value(), int_one);
-        Goto(&loop);
-      }
-    }
-  }
-
-  BIND(&if_hasexplicitcaptures);
-  {
-    Node* const from = IntPtrZero();
-    Node* const to = SmiUntag(res_length);
-    const int increment = 1;
-
-    BuildFastLoop(
-        from, to,
-        [this, res_elems, isolate, native_context, context, undefined,
-         replace_callable](Node* index) {
-          Node* const elem = LoadFixedArrayElement(res_elems, index);
-
-          Label do_continue(this);
-          GotoIf(TaggedIsSmi(elem), &do_continue);
-
-          // elem must be an Array.
-          // Use the apply argument as backing for global RegExp
-          // properties.
-
-          CSA_ASSERT(this, HasInstanceType(elem, JS_ARRAY_TYPE));
-
-          // TODO(jgruber): Remove indirection through
-          // Call->ReflectApply.
-          Callable call_callable = CodeFactory::Call(isolate);
-          Node* const reflect_apply =
-              LoadContextElement(native_context, Context::REFLECT_APPLY_INDEX);
-
-          Node* const replacement_obj =
-              CallJS(call_callable, context, reflect_apply, undefined,
-                     replace_callable, undefined, elem);
-
-          // Overwrite the i'th element in the results with the string
-          // we got back from the callback function.
-
-          TNode<String> const replacement_str =
-              ToString_Inline(context, replacement_obj);
-          StoreFixedArrayElement(res_elems, index, replacement_str);
-
-          Goto(&do_continue);
-          BIND(&do_continue);
-        },
-        increment, CodeStubAssembler::INTPTR_PARAMETERS,
-        CodeStubAssembler::IndexAdvanceMode::kPost);
-
-    Goto(&create_result);
-  }
-
-  BIND(&create_result);
-  {
-    Node* const result = CallRuntime(Runtime::kStringBuilderConcat, context,
-                                     res, res_length, string);
-    var_result.Bind(result);
-    Goto(&out);
-  }
-
-  BIND(&out);
-  return var_result.value();
-}
-
 class RegExpStringIteratorAssembler : public RegExpBuiltinsAssembler {
  public:
   explicit RegExpStringIteratorAssembler(compiler::CodeAssemblerState* state)

src/builtins/builtins-regexp-gen.h

@@ -148,9 +148,6 @@ class RegExpBuiltinsAssembler : public CodeStubAssembler {
   void RegExpPrototypeSplitBody(Node* const context, Node* const regexp,
                                 TNode<String> const string,
                                 TNode<Smi> const limit);
-
-  Node* ReplaceGlobalCallableFastPath(Node* context, Node* regexp, Node* string,
-                                      Node* replace_callable);
 };
 
 class RegExpMatchAllAssembler : public RegExpBuiltinsAssembler {

src/builtins/regexp-replace.tq

@@ -11,23 +11,125 @@ namespace regexp_replace {
   extern builtin
   SubString(implicit context: Context)(String, Smi, Smi): String;
 
-  extern transitioning macro
-  RegExpBuiltinsAssembler::ReplaceGlobalCallableFastPath(
-      implicit context: Context)(JSRegExp, String, Callable): String;
-  extern macro
-  RegExpBuiltinsAssembler::AdvanceStringIndexFast(String, Number, bool): Smi;
-
+  extern runtime RegExpExecMultiple(implicit context: Context)(
+      JSRegExp, String, RegExpMatchInfo, JSArray): Null | JSArray;
   extern transitioning runtime
   RegExpReplaceRT(Context, JSReceiver, String, Object): String;
   extern transitioning runtime
+  StringBuilderConcat(implicit context: Context)(JSArray, Smi, String): String;
+  extern transitioning runtime
   StringReplaceNonGlobalRegExpWithFunction(implicit context: Context)(
       String, JSRegExp, Callable): String;
 
+  extern macro
+  RegExpBuiltinsAssembler::AdvanceStringIndexFast(String, Number, bool): Smi;
   extern macro
   RegExpBuiltinsAssembler::RegExpPrototypeExecBodyWithoutResultFast(
       implicit context: Context)(JSReceiver, String):
       RegExpMatchInfo labels IfDidNotMatch;
 
+  transitioning macro RegExpReplaceCallableNoExplicitCaptures(implicit context:
+                                                                  Context)(
+      matchesElements: FixedArray, matchesLength: intptr, string: String,
+      replaceFn: Callable) {
+    let matchStart: Smi = 0;
+    for (let i: intptr = 0; i < matchesLength; i++) {
+      typeswitch (matchesElements.objects[i]) {
+        // Element represents a slice.
+        case (elSmi: Smi): {
+          // The slice's match start and end is either encoded as one or two
+          // smis. A positive smi indicates a single smi encoding (see
+          // ReplacementStringBuilder::AddSubjectSlice()).
+          if (elSmi > 0) {
+            // For single smi encoding, see
+            // StringBuilderSubstringLength::encode() and
+            // StringBuilderSubstringPosition::encode().
+            const elInt: intptr = Convert<intptr>(elSmi);
+            const newMatchStart: intptr = (elInt >> 11) + (elInt & 0x7FF);
+            matchStart = Convert<Smi>(newMatchStart);
+          } else {
+            // For two smi encoding, the length is negative followed by the
+            // match start.
+            const nextEl: Smi = UnsafeCast<Smi>(matchesElements.objects[++i]);
+            matchStart = nextEl - elSmi;
+          }
+        }
+        // Element represents the matched substring, which is then passed to the
+        // replace function.
+        case (elString: String): {
+          const replacementObj: Object =
+              Call(context, replaceFn, Undefined, elString, matchStart, string);
+          const replacement: String = ToString_Inline(context, replacementObj);
+          matchesElements.objects[i] = replacement;
+          matchStart += elString.length_smi;
+        }
+        case (Object): deferred {
+          unreachable;
+        }
+      }
+    }
+  }
+
+  transitioning macro
+  RegExpReplaceCallableWithExplicitCaptures(implicit context: Context)(
+      matchesElements: FixedArray, matchesLength: intptr, string: String,
+      replaceFn: Callable) {
+    for (let i: intptr = 0; i < matchesLength; i++) {
+      const elArray =
+          Cast<JSArray>(matchesElements.objects[i]) otherwise continue;
+
+      // The JSArray is expanded into the function args by Reflect.apply().
+      // TODO(jgruber): Remove indirection through Call->ReflectApply.
+      const replacementObj: Object = Call(
+          context, GetReflectApply(), Undefined, replaceFn, Undefined, elArray);
+
+      // Overwrite the i'th element in the results with the string
+      // we got back from the callback function.
+      matchesElements.objects[i] = ToString_Inline(context, replacementObj);
+    }
+  }
+
+  transitioning macro RegExpReplaceFastGlobalCallable(implicit context:
+                                                          Context)(
+      regexp: FastJSRegExp, string: String, replaceFn: Callable): String {
+    regexp.lastIndex = 0;
+
+    const kInitialCapacity: Smi = 16;
+    const kInitialLength: Smi = 0;
+    const result: Null | JSArray = RegExpExecMultiple(
+        regexp, string, GetRegExpLastMatchInfo(),
+        AllocateJSArray(
+            PACKED_ELEMENTS, GetFastPackedElementsJSArrayMap(),
+            kInitialCapacity, kInitialLength));
+
+    regexp.lastIndex = 0;
+
+    // If no matches, return the subject string.
+    if (result == Null) return string;
+
+    const matches: JSArray = UnsafeCast<JSArray>(result);
+    const matchesLength: Smi = Cast<Smi>(matches.length) otherwise unreachable;
+    const matchesLengthInt: intptr = Convert<intptr>(matchesLength);
+    const matchesElements: FixedArray =
+        UnsafeCast<FixedArray>(matches.elements);
+
+    // Reload last match info since it might have changed.
+    const nofCaptures: Smi = GetRegExpLastMatchInfo().NumberOfCaptures();
+
+    // If the number of captures is two then there are no explicit captures in
+    // the regexp, just the implicit capture that captures the whole match. In
+    // this case we can simplify quite a bit and end up with something faster.
+    if (nofCaptures == 2) {
+      RegExpReplaceCallableNoExplicitCaptures(
+          matchesElements, matchesLengthInt, string, replaceFn);
+    } else {
+      RegExpReplaceCallableWithExplicitCaptures(
+          matchesElements, matchesLengthInt, string, replaceFn);
+    }
+
+    return StringBuilderConcat(matches, matchesLength, string);
+  }
+
   transitioning macro RegExpReplaceFastString(implicit context: Context)(
       regexp: FastJSRegExp, string: String, replaceString: String): String {
     // The fast path is reached only if {receiver} is an unmodified JSRegExp
@@ -80,7 +182,7 @@ namespace regexp_replace {
     typeswitch (replaceValue) {
       case (replaceFn: Callable): {
         return regexp.global ?
-            ReplaceGlobalCallableFastPath(regexp, string, replaceFn) :
+            RegExpReplaceFastGlobalCallable(regexp, string, replaceFn) :
             StringReplaceNonGlobalRegExpWithFunction(string, regexp, replaceFn);
       }
       case (Object): {
@@ -116,7 +218,7 @@ namespace regexp_replace {
     // if (IsCallable(replace)) {
     //   if (IsGlobal(receiver)) {
     //     // Called 'fast-path' but contains several runtime calls.
-    //     ReplaceGlobalCallableFastPath()
+    //     RegExpReplaceFastGlobalCallable()
     //   } else {
     //     CallRuntime(StringReplaceNonGlobalRegExpWithFunction)
     //   }
@@ -124,7 +226,7 @@ namespace regexp_replace {
     //   if (replace.contains("$")) {
     //     CallRuntime(RegExpReplace)
     //   } else {
-    //     ReplaceSimpleStringFastPath()
+    //     RegExpReplaceFastString()
     //   }
     // }