[wasm] When inlining with specialized signature fails, retry

Opportunistically specializing the inlined function's signature
based on statically available type information in the caller is
currently important for performance, but can make inlining fail
if parts of the inlinee relied on the more generic types.
This patch addresses that problem by retrying with the original
signature in such cases.
Long-term, check elimination should be based on typed IR nodes
instead.

Bug: v8:12166
Change-Id: I4b68d0b056daec25844f6386da11b933cc343d8e
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3579144Reviewed-by: Manos Koukoutos <manoskouk@chromium.org>
Commit-Queue: Jakob Kummerow <jkummerow@chromium.org>
Cr-Commit-Position: refs/heads/main@{#79894}

src/compiler/wasm-inlining.cc

@@ -171,37 +171,37 @@ void WasmInliner::Finalize() {
     // We use the signature based on the real argument types stored in the call
     // node. This is more specific than the callee's formal signature and might
     // enable some optimizations.
-    const wasm::FunctionSig* real_sig =
+    const wasm::FunctionSig* specialized_sig =
         CallDescriptorOf(call->op())->wasm_sig();
 
 #if DEBUG
     // Check that the real signature is a subtype of the formal one.
     const wasm::FunctionSig* formal_sig =
         WasmGraphBuilder::Int64LoweredSig(zone(), inlinee->sig);
-    CHECK_EQ(real_sig->parameter_count(), formal_sig->parameter_count());
-    CHECK_EQ(real_sig->return_count(), formal_sig->return_count());
-    for (size_t i = 0; i < real_sig->parameter_count(); i++) {
-      CHECK(wasm::IsSubtypeOf(real_sig->GetParam(i), formal_sig->GetParam(i),
-                              module()));
+    CHECK_EQ(specialized_sig->parameter_count(), formal_sig->parameter_count());
+    CHECK_EQ(specialized_sig->return_count(), formal_sig->return_count());
+    for (size_t i = 0; i < specialized_sig->parameter_count(); i++) {
+      CHECK(wasm::IsSubtypeOf(specialized_sig->GetParam(i),
+                              formal_sig->GetParam(i), module()));
     }
-    for (size_t i = 0; i < real_sig->return_count(); i++) {
-      CHECK(wasm::IsSubtypeOf(formal_sig->GetReturn(i), real_sig->GetReturn(i),
-                              module()));
+    for (size_t i = 0; i < specialized_sig->return_count(); i++) {
+      CHECK(wasm::IsSubtypeOf(formal_sig->GetReturn(i),
+                              specialized_sig->GetReturn(i), module()));
     }
 #endif
 
-    const wasm::FunctionBody inlinee_body(real_sig, inlinee->code.offset(),
-                                          function_bytes.begin(),
-                                          function_bytes.end());
     wasm::WasmFeatures detected;
-    WasmGraphBuilder builder(env_, zone(), mcgraph_, inlinee_body.sig,
-                             source_positions_);
     std::vector<WasmLoopInfo> inlinee_loop_infos;
 
     size_t subgraph_min_node_id = graph()->NodeCount();
     Node* inlinee_start;
     Node* inlinee_end;
-    {
+    for (const wasm::FunctionSig* sig = specialized_sig;;) {
+      const wasm::FunctionBody inlinee_body(sig, inlinee->code.offset(),
+                                            function_bytes.begin(),
+                                            function_bytes.end());
+      WasmGraphBuilder builder(env_, zone(), mcgraph_, inlinee_body.sig,
+                               source_positions_);
       Graph::SubgraphScope scope(graph());
       wasm::DecodeResult result = wasm::BuildTFGraph(
           zone()->allocator(), env_->enabled_features, module(), &builder,
@@ -210,25 +210,23 @@ void WasmInliner::Finalize() {
           NodeProperties::IsExceptionalCall(call)
               ? wasm::kInlinedHandledCall
               : wasm::kInlinedNonHandledCall);
-      if (result.failed()) {
-        // This can happen if the inlinee has never been compiled before and is
-        // invalid. Return, as there is no point to keep optimizing.
-
-        // TODO(jkummerow): This can also happen as a consequence of the
-        // opportunistic signature specialization we did above! When parameters
-        // are reassigned (as locals), the subtypes can make that invalid.
-        // Fix this for now by detecting when it happens and retrying the
-        // inlining with the original signature.
-        // A better long-term fix would be to port check elimination to the
-        // TF graph, so we won't need the signature "trick" and more.
-
-        Trace(candidate, "failed to compile");
-        return;
+      if (result.ok()) {
+        builder.LowerInt64(WasmGraphBuilder::kCalledFromWasm);
+        inlinee_start = graph()->start();
+        inlinee_end = graph()->end();
+        break;
       }
-
-      builder.LowerInt64(WasmGraphBuilder::kCalledFromWasm);
-      inlinee_start = graph()->start();
-      inlinee_end = graph()->end();
+      if (sig == specialized_sig) {
+        // One possible reason for failure is the opportunistic signature
+        // specialization. Try again without that.
+        sig = inlinee->sig;
+        inlinee_loop_infos.clear();
+        Trace(candidate, "retrying with original signature");
+        continue;
+      }
+      // Otherwise report failure.
+      Trace(candidate, "failed to compile");
+      return;
     }
 
     size_t additional_nodes = graph()->NodeCount() - subgraph_min_node_id;