avdevice/decklink: fix checking video mode in SDK version 11

Apparently in the new SDK one cannot query if VANC output is supported, so we
will fall back to non-VANC output if enabling the video output with VANC fails.

Fixes ticket #7867.
Signed-off-by: Marton Balint <cus@passwd.hu>

libavdevice/decklink_common.cpp

@@ -273,21 +273,15 @@ int ff_decklink_set_format(AVFormatContext *avctx,
 #if BLACKMAGIC_DECKLINK_API_VERSION >= 0x0b000000
     if (direction == DIRECTION_IN) {
         if (ctx->dli->DoesSupportVideoMode(ctx->video_input, ctx->bmd_mode, (BMDPixelFormat) cctx->raw_format,
-                                           bmdVideoInputFlagDefault,
+                                           bmdSupportedVideoModeDefault,
                                            &support) != S_OK)
             return -1;
     } else {
         BMDDisplayMode actualMode = ctx->bmd_mode;
-        if (!ctx->supports_vanc || ctx->dlo->DoesSupportVideoMode(bmdVideoConnectionUnspecified, ctx->bmd_mode, ctx->raw_format,
-                                                                  bmdVideoOutputVANC,
-                                                                  &actualMode, &support) != S_OK || !support || ctx->bmd_mode != actualMode) {
-            /* Try without VANC enabled */
-            if (ctx->dlo->DoesSupportVideoMode(bmdVideoConnectionUnspecified, ctx->bmd_mode, ctx->raw_format,
-                                               bmdVideoOutputFlagDefault,
-                                               &actualMode, &support) != S_OK || !support || ctx->bmd_mode != actualMode) {
-                return -1;
-            }
-            ctx->supports_vanc = 0;
+        if (ctx->dlo->DoesSupportVideoMode(bmdVideoConnectionUnspecified, ctx->bmd_mode, ctx->raw_format,
+                                           bmdSupportedVideoModeDefault,
+                                           &actualMode, &support) != S_OK || !support || ctx->bmd_mode != actualMode) {
+            return -1;
         }
 
     }

libavdevice/decklink_enc.cpp

@@ -197,8 +197,11 @@ static int decklink_setup_video(AVFormatContext *avctx, AVStream *st)
                " Check available formats with -list_formats 1.\n");
         return -1;
     }
-    if (ctx->dlo->EnableVideoOutput(ctx->bmd_mode,
-                                    ctx->supports_vanc ? bmdVideoOutputVANC : bmdVideoOutputFlagDefault) != S_OK) {
+    if (ctx->supports_vanc && ctx->dlo->EnableVideoOutput(ctx->bmd_mode, bmdVideoOutputVANC) != S_OK) {
+        av_log(avctx, AV_LOG_WARNING, "Could not enable video output with VANC! Trying without...\n");
+        ctx->supports_vanc = 0;
+    }
+    if (!ctx->supports_vanc && ctx->dlo->EnableVideoOutput(ctx->bmd_mode, bmdVideoOutputFlagDefault) != S_OK) {
         av_log(avctx, AV_LOG_ERROR, "Could not enable video output!\n");
         return -1;
     }