[wasm-gc] Implement br_on_non_null

Bug: v8:7748
Change-Id: I9a4dad42f433ce0adf928461cf0db589df3d69e9
Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2897087
Commit-Queue: Manos Koukoutos <manoskouk@chromium.org>
Reviewed-by: Jakob Kummerow <jkummerow@chromium.org>
Cr-Commit-Position: refs/heads/master@{#74571}

src/wasm/baseline/liftoff-compiler.cc

@@ -3146,6 +3146,32 @@ class LiftoffCompiler {
     __ PushRegister(kRef, ref);
   }
 
+  void BrOnNonNull(FullDecoder* decoder, const Value& ref_object,
+                   uint32_t depth) {
+    // Before branching, materialize all constants. This avoids repeatedly
+    // materializing them for each conditional branch.
+    if (depth != decoder->control_depth() - 1) {
+      __ MaterializeMergedConstants(
+          decoder->control_at(depth)->br_merge()->arity);
+    }
+
+    Label cont_false;
+    LiftoffRegList pinned;
+    LiftoffRegister ref = pinned.set(__ PopToRegister(pinned));
+    // Put the reference back onto the stack for the branch.
+    __ PushRegister(kRef, ref);
+
+    Register null = __ GetUnusedRegister(kGpReg, pinned).gp();
+    LoadNullValue(null, pinned);
+    __ emit_cond_jump(kEqual, &cont_false, ref_object.type.kind(), ref.gp(),
+                      null);
+
+    BrOrRet(decoder, depth, 0);
+    // Drop the reference if we are not branching.
+    __ DropValues(1);
+    __ bind(&cont_false);
+  }
+
   template <ValueKind src_kind, ValueKind result_kind, typename EmitFn>
   void EmitTerOp(EmitFn fn) {
     static constexpr RegClass src_rc = reg_class_for(src_kind);

src/wasm/function-body-decoder-impl.h

@@ -1093,6 +1093,7 @@ struct ControlBase : public PcForErrors<validate> {
   F(ReturnCallIndirect, const Value& index,                                    \
     const CallIndirectImmediate<validate>& imm, const Value args[])            \
   F(BrOnNull, const Value& ref_object, uint32_t depth)                         \
+  F(BrOnNonNull, const Value& ref_object, uint32_t depth)                      \
   F(SimdOp, WasmOpcode opcode, Vector<Value> args, Value* result)              \
   F(SimdLaneOp, WasmOpcode opcode, const SimdLaneImmediate<validate>& imm,     \
     const Vector<Value> inputs, Value* result)                                 \
@@ -1697,6 +1698,7 @@ class WasmDecoder : public Decoder {
       case kExprBr:
       case kExprBrIf:
       case kExprBrOnNull:
+      case kExprBrOnNonNull:
       case kExprDelegate: {
         BranchDepthImmediate<validate> imm(decoder, pc + 1);
         return 1 + imm.length;
@@ -2031,6 +2033,7 @@ class WasmDecoder : public Decoder {
       case kExprBrIf:
       case kExprBrTable:
       case kExprIf:
+      case kExprBrOnNonNull:
         return {1, 0};
       case kExprLocalGet:
       case kExprGlobalGet:
@@ -2695,6 +2698,48 @@ class WasmFullDecoder : public WasmDecoder<validate> {
     return 1 + imm.length;
   }
 
+  DECODE(BrOnNonNull) {
+    CHECK_PROTOTYPE_OPCODE(gc);
+    BranchDepthImmediate<validate> imm(this, this->pc_ + 1);
+    if (!this->Validate(this->pc_ + 1, imm, control_.size())) return 0;
+    Value ref_object = Peek(0, 0, kWasmAnyRef);
+    Drop(ref_object);
+    // Typechecking the branch and creating the branch merges requires the
+    // non-null value on the stack, so we push it temporarily.
+    Value result = CreateValue(ref_object.type.AsNonNull());
+    Push(result);
+    Control* c = control_at(imm.depth);
+    if (!VALIDATE(TypeCheckBranch<true>(c, 0))) return 0;
+    switch (ref_object.type.kind()) {
+      case kBottom:
+        // We are in unreachable code. Do nothing.
+        DCHECK(!current_code_reachable_and_ok_);
+        break;
+      case kRef:
+        // For a non-nullable value, we always take the branch.
+        if (V8_LIKELY(current_code_reachable_and_ok_)) {
+          CALL_INTERFACE(Forward, ref_object, stack_value(1));
+          CALL_INTERFACE(BrOrRet, imm.depth, 0);
+          c->br_merge()->reached = true;
+        }
+        break;
+      case kOptRef: {
+        if (V8_LIKELY(current_code_reachable_and_ok_)) {
+          CALL_INTERFACE(Forward, ref_object, stack_value(1));
+          CALL_INTERFACE(BrOnNonNull, ref_object, imm.depth);
+          c->br_merge()->reached = true;
+        }
+        break;
+      }
+      default:
+        PopTypeError(0, ref_object, "object reference");
+        return 0;
+    }
+    // If we stay in the branch, {ref_object} is null. Drop it from the stack.
+    Drop(result);
+    return 1 + imm.length;
+  }
+
   DECODE(Let) {
     CHECK_PROTOTYPE_OPCODE(typed_funcref);
     BlockTypeImmediate<validate> imm(this->enabled_, this, this->pc_ + 1,
@@ -3403,6 +3448,7 @@ class WasmFullDecoder : public WasmDecoder<validate> {
     DECODE_IMPL(CatchAll);
     DECODE_IMPL(Unwind);
     DECODE_IMPL(BrOnNull);
+    DECODE_IMPL(BrOnNonNull);
     DECODE_IMPL(Let);
     DECODE_IMPL(Loop);
     DECODE_IMPL(If);

src/wasm/graph-builder-interface.cc

@@ -670,6 +670,19 @@ class WasmGraphBuildingInterface {
     SetEnv(false_env);
   }
 
+  void BrOnNonNull(FullDecoder* decoder, const Value& ref_object,
+                   uint32_t depth) {
+    SsaEnv* false_env = ssa_env_;
+    SsaEnv* true_env = Split(decoder->zone(), false_env);
+    false_env->SetNotMerged();
+    builder_->BrOnNull(ref_object.node, &false_env->control,
+                       &true_env->control);
+    builder_->SetControl(false_env->control);
+    SetEnv(true_env);
+    BrOrRet(decoder, depth, 0);
+    SetEnv(false_env);
+  }
+
   void SimdOp(FullDecoder* decoder, WasmOpcode opcode, Vector<Value> args,
               Value* result) {
     NodeVector inputs(args.size());

src/wasm/value-type.h

@@ -340,6 +340,13 @@ class ValueType {
     return is_packed() ? Primitive(kI32) : *this;
   }
 
+  // Returns the version of this type that does not allow null values. Handles
+  // bottom.
+  constexpr ValueType AsNonNull() const {
+    DCHECK(is_object_reference() || is_bottom());
+    return is_nullable() ? Ref(heap_type(), kNonNullable) : *this;
+  }
+
   /***************************** Field Accessors ******************************/
   constexpr ValueKind kind() const { return KindField::decode(bit_field_); }
   constexpr HeapType::Representation heap_representation() const {

src/wasm/wasm-opcodes-inl.h

@@ -162,6 +162,7 @@ constexpr const char* WasmOpcodes::OpcodeName(WasmOpcode opcode) {
     CASE_OP(CallRef, "call_ref")
     CASE_OP(ReturnCallRef, "return_call_ref")
     CASE_OP(BrOnNull, "br_on_null")
+    CASE_OP(BrOnNonNull, "br_on_non_null")
     CASE_OP(Drop, "drop")
     CASE_OP(Select, "select")
     CASE_OP(SelectWithType, "select")

src/wasm/wasm-opcodes.h

@@ -52,6 +52,7 @@ bool V8_EXPORT_PRIVATE IsJSCompatibleSignature(const FunctionSig* sig,
   V(Delegate, 0x18, _ /* eh_prototype */)       \
   V(CatchAll, 0x19, _ /* eh_prototype */)       \
   V(BrOnNull, 0xd4, _ /* gc prototype */)       \
+  V(BrOnNonNull, 0xd6, _ /* gc prototype */)    \
   V(NopForTestingUnsupportedInLiftoff, 0x16, _)
 
 // Constants, locals, globals, and calls.

test/cctest/wasm/test-gc.cc

@@ -313,23 +313,23 @@ WASM_COMPILED_EXEC_TEST(WasmBrOnNull) {
   ValueType kRefTypes[] = {ref(type_index)};
   ValueType kOptRefType = optref(type_index);
   FunctionSig sig_q_v(1, 0, kRefTypes);
-  const byte l_local_index = 0;
+  const byte local_index = 0;
   const byte kTaken = tester.DefineFunction(
       tester.sigs.i_v(), {kOptRefType},
       {WASM_BLOCK_I(WASM_I32V(42),
                     // Branch will be taken.
                     // 42 left on stack outside the block (not 52).
-                    WASM_BR_ON_NULL(0, WASM_LOCAL_GET(l_local_index)),
+                    WASM_BR_ON_NULL(0, WASM_LOCAL_GET(local_index)),
                     WASM_I32V(52), WASM_BR(0)),
        kExprEnd});
 
-  const byte m_field_index = 0;
+  const byte field_index = 0;
   const byte kNotTaken = tester.DefineFunction(
       tester.sigs.i_v(), {},
       {WASM_BLOCK_I(
            WASM_I32V(42),
            WASM_STRUCT_GET(
-               type_index, m_field_index,
+               type_index, field_index,
                // Branch will not be taken.
                // 52 left on stack outside the block (not 42).
                WASM_BR_ON_NULL(0, WASM_STRUCT_NEW_WITH_RTT(
@@ -343,6 +343,49 @@ WASM_COMPILED_EXEC_TEST(WasmBrOnNull) {
   tester.CheckResult(kNotTaken, 52);
 }
 
+WASM_COMPILED_EXEC_TEST(WasmBrOnNonNull) {
+  WasmGCTester tester(execution_tier);
+  const byte type_index =
+      tester.DefineStruct({F(kWasmI32, true), F(kWasmI32, true)});
+  ValueType kRefType = ref(type_index);
+  ValueType kOptRefType = optref(type_index);
+  FunctionSig sig_q_v(1, 0, &kRefType);
+  const byte field_index = 0;
+
+  const byte kTaken = tester.DefineFunction(
+      tester.sigs.i_v(), {kOptRefType, kOptRefType},
+      {WASM_LOCAL_SET(
+           0, WASM_STRUCT_NEW_WITH_RTT(type_index, WASM_I32V(52), WASM_I32V(62),
+                                       WASM_RTT_CANON(type_index))),
+       WASM_LOCAL_SET(
+           1, WASM_STRUCT_NEW_WITH_RTT(type_index, WASM_I32V(11), WASM_I32V(22),
+                                       WASM_RTT_CANON(type_index))),
+       WASM_STRUCT_GET(type_index, field_index,
+                       WASM_BLOCK_R(ref(type_index),
+                                    // Branch will be taken, and the block will
+                                    // return struct(52, 62).
+                                    WASM_BR_ON_NON_NULL(0, WASM_LOCAL_GET(0)),
+                                    WASM_REF_AS_NON_NULL(WASM_LOCAL_GET(1)))),
+       kExprEnd});
+
+  const byte kNotTaken = tester.DefineFunction(
+      tester.sigs.i_v(), {kOptRefType, kOptRefType},
+      {WASM_LOCAL_SET(0, WASM_REF_NULL(type_index)),
+       WASM_LOCAL_SET(
+           1, WASM_STRUCT_NEW_WITH_RTT(type_index, WASM_I32V(11), WASM_I32V(22),
+                                       WASM_RTT_CANON(type_index))),
+       WASM_STRUCT_GET(type_index, field_index,
+                       WASM_BLOCK_R(ref(type_index),
+                                    // Branch will not be taken, and the block
+                                    // will return struct(11, 22).
+                                    WASM_BR_ON_NON_NULL(0, WASM_LOCAL_GET(0)),
+                                    WASM_REF_AS_NON_NULL(WASM_LOCAL_GET(1)))),
+       kExprEnd});
+  tester.CompileModule();
+  tester.CheckResult(kTaken, 52);
+  tester.CheckResult(kNotTaken, 11);
+}
+
 WASM_COMPILED_EXEC_TEST(BrOnCast) {
   WasmGCTester tester(execution_tier);
   ValueType kDataRefNull = ValueType::Ref(HeapType::kData, kNullable);

test/common/wasm/wasm-macro-gen.h

@@ -564,6 +564,9 @@ inline WasmOpcode LoadStoreOpcodeOf(MachineType type, bool store) {
 #define WASM_BR_ON_NULL(depth, ref_object) \
   ref_object, kExprBrOnNull, static_cast<byte>(depth)
 
+#define WASM_BR_ON_NON_NULL(depth, ref_object) \
+  ref_object, kExprBrOnNonNull, static_cast<byte>(depth)
+
 // Pass: sig_index, ...args, func_index
 #define WASM_CALL_INDIRECT(sig_index, ...) \
   __VA_ARGS__, kExprCallIndirect, static_cast<byte>(sig_index), TABLE_ZERO

test/mjsunit/wasm/wasm-module-builder.js

@@ -415,6 +415,7 @@ const kWasmOpcodes = {
   'RefAsNonNull': 0xd3,
   'BrOnNull': 0xd4,
   'RefEq': 0xd5,
+  'BrOnNonNull': 0xd6
 };
 
 function defineWasmOpcode(name, value) {

test/unittests/wasm/function-body-decoder-unittest.cc

@@ -3845,6 +3845,31 @@ TEST_F(FunctionBodyDecoderTest, BrOnNull) {
                 "expected 1 elements on the stack for branch, found 0");
 }
 
+TEST_F(FunctionBodyDecoderTest, BrOnNonNull) {
+  WASM_FEATURE_SCOPE(reftypes);
+  WASM_FEATURE_SCOPE(typed_funcref);
+  WASM_FEATURE_SCOPE(gc);
+
+  const ValueType reps[] = {ValueType::Ref(HeapType::kFunc, kNonNullable),
+                            ValueType::Ref(HeapType::kFunc, kNullable)};
+  const FunctionSig sig(1, 1, reps);
+  ExpectValidates(
+      &sig,
+      {WASM_BLOCK_R(reps[0], WASM_BR_ON_NON_NULL(0, WASM_LOCAL_GET(0)),
+                    WASM_RETURN(WASM_REF_AS_NON_NULL(WASM_LOCAL_GET(0))))});
+
+  // Wrong branch type.
+  ExpectFailure(
+      &sig,
+      {WASM_BLOCK_I(WASM_BR_ON_NON_NULL(0, WASM_LOCAL_GET(0)),
+                    WASM_RETURN(WASM_REF_AS_NON_NULL(WASM_LOCAL_GET(0))))},
+      kAppendEnd, "type error in branch[0] (expected i32, got (ref func))");
+
+  // br_on_non_null does not leave a value on the stack.
+  ExpectFailure(&sig, {WASM_BR_ON_NON_NULL(0, WASM_LOCAL_GET(0))}, kAppendEnd,
+                "expected 1 elements on the stack for fallthru, found 0");
+}
+
 TEST_F(FunctionBodyDecoderTest, GCStruct) {
   WASM_FEATURE_SCOPE(reftypes);
   WASM_FEATURE_SCOPE(typed_funcref);