[wasm] Generate exception handlers in inlined functions

In the WebAssembly Turbofan pipeline, inlining should come before
unrolling. When we inline a function, we link unhandled throwing calls
in it to the handler of the caller node. If a throwing call is in a
loop, we need to generate loop exits between the call and the handler if
we want to unroll later.
This CL adds dangling IfException/LoopExit nodes following each throwing
call in an inlined function. These nodes are connected as required in
inlining.
Drive-by: Remove CheckForException from tail calls, which are kNoThrow.

Bug: v8:12166
Change-Id: Icb8371a0a27234f07d4880e5b3005fc90a91a4b6
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/3322975Reviewed-by: Jakob Kummerow <jkummerow@chromium.org>
Commit-Queue: Manos Koukoutos <manoskouk@chromium.org>
Cr-Commit-Position: refs/heads/main@{#78391}

src/compiler/wasm-inlining.cc

@@ -158,10 +158,13 @@ void WasmInliner::Finalize() {
     Node* inlinee_end;
     {
       Graph::SubgraphScope scope(graph());
-      wasm::DecodeResult result = wasm::BuildTFGraph(
-          zone()->allocator(), env_->enabled_features, module(), &builder,
-          &detected, inlinee_body, &infos, node_origins_,
-          candidate.inlinee_index, wasm::kInlinedFunction);
+      wasm::DecodeResult result =
+          wasm::BuildTFGraph(zone()->allocator(), env_->enabled_features,
+                             module(), &builder, &detected, inlinee_body,
+                             &infos, node_origins_, candidate.inlinee_index,
+                             NodeProperties::IsExceptionalCall(call)
+                                 ? wasm::kInlinedHandledCall
+                                 : wasm::kInlinedNonHandledCall);
       if (result.failed()) {
         // This can happen if the inlinee has never been compiled before and is
         // invalid. Return, as there is no point to keep optimizing.
@@ -230,7 +233,7 @@ void WasmInliner::RewireFunctionEntry(Node* call, Node* callee_start) {
 
 void WasmInliner::InlineTailCall(Node* call, Node* callee_start,
                                  Node* callee_end) {
-  DCHECK(call->opcode() == IrOpcode::kTailCall);
+  DCHECK_EQ(call->opcode(), IrOpcode::kTailCall);
   // 1) Rewire function entry.
   RewireFunctionEntry(call, callee_start);
   // 2) For tail calls, all we have to do is rewire all terminators of the
@@ -248,22 +251,59 @@ void WasmInliner::InlineTailCall(Node* call, Node* callee_start,
   Revisit(graph()->end());
 }
 
+namespace {
+// graph-builder-interface generates a dangling exception handler for each
+// throwing call in the inlinee. This might be followed by a LoopExit node.
+Node* DanglingHandler(Node* call) {
+  Node* if_exception = nullptr;
+  for (Node* use : call->uses()) {
+    if (use->opcode() == IrOpcode::kIfException) {
+      if_exception = use;
+      break;
+    }
+  }
+  DCHECK_NOT_NULL(if_exception);
+
+  // If this handler is dangling, return it.
+  if (if_exception->UseCount() == 0) return if_exception;
+
+  for (Node* use : if_exception->uses()) {
+    // Otherwise, look for a LoopExit use of this handler.
+    if (use->opcode() == IrOpcode::kLoopExit) {
+      for (Node* loop_exit_use : use->uses()) {
+        if (loop_exit_use->opcode() != IrOpcode::kLoopExitEffect &&
+            loop_exit_use->opcode() != IrOpcode::kLoopExitValue) {
+          // This LoopExit has a use other than LoopExitEffect/Value, so it is
+          // not dangling.
+          return nullptr;
+        }
+      }
+      return use;
+    }
+  }
+
+  return nullptr;
+}
+}  // namespace
+
 void WasmInliner::InlineCall(Node* call, Node* callee_start, Node* callee_end,
                              const wasm::FunctionSig* inlinee_sig,
                              size_t subgraph_min_node_id) {
-  DCHECK(call->opcode() == IrOpcode::kCall);
+  DCHECK_EQ(call->opcode(), IrOpcode::kCall);
 
   // 0) Before doing anything, if {call} has an exception handler, collect all
   // unhandled calls in the subgraph.
   Node* handler = nullptr;
-  std::vector<Node*> unhandled_subcalls;
+  std::vector<Node*> dangling_handlers;
   if (NodeProperties::IsExceptionalCall(call, &handler)) {
     AllNodes subgraph_nodes(zone(), callee_end, graph());
     for (Node* node : subgraph_nodes.reachable) {
       if (node->id() >= subgraph_min_node_id &&
-          !node->op()->HasProperty(Operator::kNoThrow) &&
-          !NodeProperties::IsExceptionalCall(node)) {
-        unhandled_subcalls.push_back(node);
+          !node->op()->HasProperty(Operator::kNoThrow)) {
+        Node* dangling_handler = DanglingHandler(node);
+        if (dangling_handler != nullptr) {
+          dangling_handlers.push_back(dangling_handler);
+        }
       }
     }
   }
@@ -326,29 +366,37 @@ void WasmInliner::InlineCall(Node* call, Node* callee_start, Node* callee_end,
   callee_end->Kill();
 
   // 3) Rewire unhandled calls to the handler.
-  std::vector<Node*> on_exception_nodes;
-  for (Node* subcall : unhandled_subcalls) {
-    Node* on_success = graph()->NewNode(common()->IfSuccess(), subcall);
-    NodeProperties::ReplaceUses(subcall, subcall, subcall, on_success);
-    NodeProperties::ReplaceControlInput(on_success, subcall);
-    Node* on_exception =
-        graph()->NewNode(common()->IfException(), subcall, subcall);
-    on_exception_nodes.push_back(on_exception);
-  }
+  int handler_count = static_cast<int>(dangling_handlers.size());
 
-  int subcall_count = static_cast<int>(on_exception_nodes.size());
-
-  if (subcall_count > 0) {
+  if (handler_count > 0) {
     Node* control_output =
-        graph()->NewNode(common()->Merge(subcall_count), subcall_count,
-                         on_exception_nodes.data());
-    on_exception_nodes.push_back(control_output);
+        graph()->NewNode(common()->Merge(handler_count), handler_count,
+                         dangling_handlers.data());
+    std::vector<Node*> effects;
+    std::vector<Node*> values;
+    for (Node* control : dangling_handlers) {
+      if (control->opcode() == IrOpcode::kIfException) {
+        effects.push_back(control);
+        values.push_back(control);
+      } else {
+        DCHECK_EQ(control->opcode(), IrOpcode::kLoopExit);
+        Node* if_exception = control->InputAt(0);
+        DCHECK_EQ(if_exception->opcode(), IrOpcode::kIfException);
+        effects.push_back(graph()->NewNode(common()->LoopExitEffect(),
+                                           if_exception, control));
+        values.push_back(graph()->NewNode(
+            common()->LoopExitValue(MachineRepresentation::kTagged),
+            if_exception, control));
+      }
+    }
+
+    effects.push_back(control_output);
+    values.push_back(control_output);
     Node* value_output = graph()->NewNode(
-        common()->Phi(MachineRepresentation::kTagged, subcall_count),
-        subcall_count + 1, on_exception_nodes.data());
-    Node* effect_output =
-        graph()->NewNode(common()->EffectPhi(subcall_count), subcall_count + 1,
-                         on_exception_nodes.data());
+        common()->Phi(MachineRepresentation::kTagged, handler_count),
+        handler_count + 1, values.data());
+    Node* effect_output = graph()->NewNode(common()->EffectPhi(handler_count),
+                                           handler_count + 1, effects.data());
     ReplaceWithValue(handler, value_output, effect_output, control_output);
   } else if (handler != nullptr) {
     // Nothing in the inlined function can throw. Remove the handler.

src/wasm/graph-builder-interface.cc

@@ -1354,17 +1354,18 @@ class WasmGraphBuildingInterface {
     builder_->set_instance_cache(&env->instance_cache);
   }
 
-  V8_INLINE TFNode* CheckForException(FullDecoder* decoder, TFNode* node) {
-    if (node == nullptr) return nullptr;
+  TFNode* CheckForException(FullDecoder* decoder, TFNode* node) {
+    DCHECK_NOT_NULL(node);
 
+    // We need to emit IfSuccess/IfException nodes if this node throws and has
+    // an exception handler. An exception handler can either be a try-scope
+    // around this node, or if this function is being inlined, the IfException
+    // output of the inlined Call node.
     const bool inside_try_scope = decoder->current_catch() != -1;
-    if (!inside_try_scope) return node;
-
-    return CheckForExceptionImpl(decoder, node);
-  }
+    if (inlined_status_ != kInlinedHandledCall && !inside_try_scope) {
+      return node;
+    }
 
-  V8_NOINLINE TFNode* CheckForExceptionImpl(FullDecoder* decoder,
-                                            TFNode* node) {
     TFNode* if_success = nullptr;
     TFNode* if_exception = nullptr;
     if (!builder_->ThrowsException(node, &if_success, &if_exception)) {
@@ -1378,21 +1379,33 @@ class WasmGraphBuildingInterface {
     exception_env->control = if_exception;
     exception_env->effect = if_exception;
     SetEnv(exception_env);
-    TryInfo* try_info = current_try_info(decoder);
+
     if (emit_loop_exits()) {
       ValueVector values;
-      BuildNestedLoopExits(decoder, decoder->control_depth_of_current_catch(),
+      BuildNestedLoopExits(decoder,
+                           inside_try_scope
+                               ? decoder->control_depth_of_current_catch()
+                               : decoder->control_depth() - 1,
                            true, values, &if_exception);
     }
-    Goto(decoder, try_info->catch_env);
-    if (try_info->exception == nullptr) {
-      DCHECK_EQ(SsaEnv::kReached, try_info->catch_env->state);
-      try_info->exception = if_exception;
+    if (inside_try_scope) {
+      TryInfo* try_info = current_try_info(decoder);
+      Goto(decoder, try_info->catch_env);
+      if (try_info->exception == nullptr) {
+        DCHECK_EQ(SsaEnv::kReached, try_info->catch_env->state);
+        try_info->exception = if_exception;
+      } else {
+        DCHECK_EQ(SsaEnv::kMerged, try_info->catch_env->state);
+        try_info->exception = builder_->CreateOrMergeIntoPhi(
+            MachineRepresentation::kTaggedPointer, try_info->catch_env->control,
+            try_info->exception, if_exception);
+      }
     } else {
-      DCHECK_EQ(SsaEnv::kMerged, try_info->catch_env->state);
-      try_info->exception = builder_->CreateOrMergeIntoPhi(
-          MachineRepresentation::kTaggedPointer, try_info->catch_env->control,
-          try_info->exception, if_exception);
+      DCHECK_EQ(inlined_status_, kInlinedHandledCall);
+      // Leave the IfException/LoopExit node dangling. We will connect it during
+      // inlining to the handler of the inlined call.
+      // Note: We have to generate the handler now since we have no way of
+      // generating a LoopExit if needed in the inlining code.
     }
 
     SetEnv(success_env);
@@ -1725,23 +1738,17 @@ class WasmGraphBuildingInterface {
 
     switch (call_info.call_mode()) {
       case CallInfo::kCallIndirect:
-        CheckForException(
-            decoder,
-            builder_->ReturnCallIndirect(
-                call_info.table_index(), call_info.sig_index(), real_sig,
-                base::VectorOf(arg_nodes), decoder->position()));
+        builder_->ReturnCallIndirect(
+            call_info.table_index(), call_info.sig_index(), real_sig,
+            base::VectorOf(arg_nodes), decoder->position());
         break;
       case CallInfo::kCallDirect:
-        CheckForException(
-            decoder, builder_->ReturnCall(call_info.callee_index(), real_sig,
-                                          base::VectorOf(arg_nodes),
-                                          decoder->position()));
+        builder_->ReturnCall(call_info.callee_index(), real_sig,
+                             base::VectorOf(arg_nodes), decoder->position());
         break;
       case CallInfo::kCallRef:
-        CheckForException(decoder,
-                          builder_->ReturnCallRef(
-                              real_sig, base::VectorOf(arg_nodes),
-                              call_info.null_check(), decoder->position()));
+        builder_->ReturnCallRef(real_sig, base::VectorOf(arg_nodes),
+                                call_info.null_check(), decoder->position());
         break;
     }
   }
@@ -1847,9 +1854,8 @@ DecodeResult BuildTFGraph(AccountingAllocator* allocator,
   if (node_origins) {
     builder->RemoveBytecodePositionDecorator();
   }
-  if (FLAG_wasm_loop_unrolling && inlined_status == kRegularFunction) {
-    *loop_infos = decoder.interface().loop_infos();
-  }
+  *loop_infos = decoder.interface().loop_infos();
+
   return decoder.toResult(nullptr);
 }
 

src/wasm/graph-builder-interface.h

@@ -27,7 +27,15 @@ struct FunctionBody;
 class WasmFeatures;
 struct WasmModule;
 
-enum InlinedStatus { kInlinedFunction, kRegularFunction };
+enum InlinedStatus {
+  // Inlined function whose call node has IfSuccess/IfException outputs.
+  kInlinedHandledCall,
+  // Inlined function whose call node does not have IfSuccess/IfException
+  // outputs.
+  kInlinedNonHandledCall,
+  // Not an inlined call.
+  kRegularFunction
+};
 
 V8_EXPORT_PRIVATE DecodeResult
 BuildTFGraph(AccountingAllocator* allocator, const WasmFeatures& enabled,