[wasm] Merge FallThruTo into PopControl

Since FallThruTo is often followed by PopControl, we can save a few
cache state copy operations in Liftoff by merging the two.
On epic workloads, this saves ~18% of Steal operations and ~22% of
Split operations.

R=herhut@chromium.org

Bug: v8:8423
Change-Id: I99900eaa49ae14ead6a0c08b69da2175730e077e
Reviewed-on: https://chromium-review.googlesource.com/c/1384093Reviewed-by: Stephan Herhut <herhut@chromium.org>
Commit-Queue: Clemens Hammacher <clemensh@chromium.org>
Cr-Commit-Position: refs/heads/master@{#58420}

src/wasm/baseline/liftoff-compiler.cc

@@ -502,38 +502,61 @@ class LiftoffCompiler {
   void FallThruTo(FullDecoder* decoder, Control* c) {
     if (c->end_merge.reached) {
       __ MergeFullStackWith(c->label_state, *__ cache_state());
-    } else if (c->is_onearmed_if()) {
-      // Init the merge point from the else state, then merge the if state into
-      // that.
+    } else {
+      c->label_state.Split(*__ cache_state());
+    }
+    TraceCacheState(decoder);
+  }
+
+  void FinishOneArmedIf(FullDecoder* decoder, Control* c) {
+    DCHECK(c->is_onearmed_if());
+    if (c->end_merge.reached) {
+      // Someone already merged to the end of the if. Merge both arms into that.
+      if (c->reachable()) {
+        // Merge the if state into the end state.
+        __ MergeFullStackWith(c->label_state, *__ cache_state());
+        __ emit_jump(c->label.get());
+      }
+      // Merge the else state into the end state.
+      __ bind(c->else_state->label.get());
+      __ MergeFullStackWith(c->label_state, c->else_state->state);
+      __ cache_state()->Steal(c->label_state);
+    } else if (c->reachable()) {
+      // No merge yet at the end of the if, but we need to create a merge for
+      // the both arms of this if. Thus init the merge point from the else
+      // state, then merge the if state into that.
       DCHECK_EQ(0, c->end_merge.arity);
       c->label_state.InitMerge(c->else_state->state, __ num_locals(), 0,
                                c->stack_depth);
       __ MergeFullStackWith(c->label_state, *__ cache_state());
+      __ emit_jump(c->label.get());
+      // Merge the else state into the end state.
+      __ bind(c->else_state->label.get());
+      __ MergeFullStackWith(c->label_state, c->else_state->state);
+      __ cache_state()->Steal(c->label_state);
     } else {
-      c->label_state.Split(*__ cache_state());
+      // No merge needed, just continue with the else state.
+      __ bind(c->else_state->label.get());
+      __ cache_state()->Steal(c->else_state->state);
     }
-    TraceCacheState(decoder);
   }
 
   void PopControl(FullDecoder* decoder, Control* c) {
-    // A loop just falls through.
-    if (c->is_loop()) return;
+    if (c->is_loop()) return;  // A loop just falls through.
     if (c->is_onearmed_if()) {
-      if (c->end_merge.reached) {
-        // Generate the code to merge the else state into the end state.
-        __ emit_jump(c->label.get());
-        __ bind(c->else_state->label.get());
-        __ MergeFullStackWith(c->label_state, c->else_state->state);
-        __ cache_state()->Steal(c->label_state);
-      } else {
-        // There is no merge at the end of the if, so just continue with the
-        // else state.
-        __ bind(c->else_state->label.get());
-        __ cache_state()->Steal(c->else_state->state);
-      }
+      // Special handling for one-armed ifs.
+      FinishOneArmedIf(decoder, c);
     } else if (c->end_merge.reached) {
+      // There is a merge already. Merge our state into that, then continue with
+      // that state.
+      if (c->reachable()) {
+        __ MergeFullStackWith(c->label_state, *__ cache_state());
+      }
       __ cache_state()->Steal(c->label_state);
+    } else {
+      // No merge, just continue with our current state.
     }
+
     if (!c->label.get()->is_bound()) __ bind(c->label.get());
   }
 

src/wasm/function-body-decoder-impl.h

@@ -1741,11 +1741,9 @@ class WasmFullDecoder : public WasmDecoder<validate> {
             EndControl();
           }
 
-          if (control_.size() > 1) {
-            FallThruTo(c);
-            // A loop just leaves the values on the stack.
-            if (!c->is_loop()) PushMergeValues(c, &c->end_merge);
-          } else {
+          if (!TypeCheckFallThru(c)) break;
+
+          if (control_.size() == 1) {
             // If at the last (implicit) control, check we are at end.
             if (!VALIDATE(this->pc_ + 1 == this->end_)) {
               this->error(this->pc_ + 1, "trailing code after function end");
@@ -1754,8 +1752,9 @@ class WasmFullDecoder : public WasmDecoder<validate> {
             // The result of the block is the return value.
             TRACE_PART("\n" TRACE_INST_FORMAT, startrel(this->pc_),
                        "(implicit) return");
-            if (!TypeCheckFallThru(c)) break;
             DoReturn();
+            control_.clear();
+            break;
           }
 
           PopControl(c);
@@ -2242,11 +2241,16 @@ class WasmFullDecoder : public WasmDecoder<validate> {
   void PopControl(Control* c) {
     DCHECK_EQ(c, &control_.back());
     CALL_INTERFACE_IF_PARENT_REACHABLE(PopControl, c);
-    bool reached = c->end_merge.reached || c->is_onearmed_if();
+
+    // A loop just leaves the values on the stack.
+    if (!c->is_loop()) PushMergeValues(c, &c->end_merge);
+
+    bool parent_reached =
+        c->reachable() || c->end_merge.reached || c->is_onearmed_if();
     control_.pop_back();
     // If the parent block was reachable before, but the popped control does not
-    // return to here, this block becomes indirectly unreachable.
-    if (!control_.empty() && !reached && control_.back().reachable()) {
+    // return to here, this block becomes "spec only reachable".
+    if (!parent_reached && control_.back().reachable()) {
       control_.back().reachability = kSpecOnlyReachable;
     }
   }

src/wasm/graph-builder-interface.cc

@@ -217,12 +217,17 @@ class WasmGraphBuildingInterface {
   }
 
   void PopControl(FullDecoder* decoder, Control* block) {
+    // A loop just continues with the end environment. There is no merge.
+    if (block->is_loop()) return;
+    // Any other block falls through to the parent block.
+    if (block->reachable()) FallThruTo(decoder, block);
     if (block->is_onearmed_if()) {
       // Merge the else branch into the end merge.
       SetEnv(block->false_env);
       MergeValuesInto(decoder, block, &block->end_merge);
     }
-    if (!block->is_loop()) SetEnv(block->end_env);
+    // Now continue with the merged environment.
+    SetEnv(block->end_env);
   }
 
   void EndControl(FullDecoder* decoder, Control* block) { ssa_env_->Kill(); }