qsvdec: Pass the correct profile to libmfx

This was correct for H.26[45], because libmfx uses the same values
derived from profile_idc and the constraint_set flags, but it is
wrong for other codecs.

Also avoid passing FF_LEVEL_UNKNOWN (-99) as the level, as this is
certainly invalid.

libavcodec/qsv.c

@@ -55,6 +55,22 @@ int ff_qsv_codec_id_to_mfx(enum AVCodecID codec_id)
     return AVERROR(ENOSYS);
 }
 
+int ff_qsv_profile_to_mfx(enum AVCodecID codec_id, int profile)
+{
+    if (profile == FF_PROFILE_UNKNOWN)
+        return MFX_PROFILE_UNKNOWN;
+    switch (codec_id) {
+    case AV_CODEC_ID_H264:
+    case AV_CODEC_ID_HEVC:
+        return profile;
+    case AV_CODEC_ID_VC1:
+        return 4 * profile + 1;
+    case AV_CODEC_ID_MPEG2VIDEO:
+        return 0x10 * profile;
+    }
+    return MFX_PROFILE_UNKNOWN;
+}
+
 static const struct {
     mfxStatus   mfxerr;
     int         averr;

libavcodec/qsv_internal.h

@@ -80,6 +80,7 @@ int ff_qsv_print_warning(void *log_ctx, mfxStatus err,
                          const char *warning_string);
 
 int ff_qsv_codec_id_to_mfx(enum AVCodecID codec_id);
+int ff_qsv_profile_to_mfx(enum AVCodecID codec_id, int profile);
 
 int ff_qsv_map_pixfmt(enum AVPixelFormat format, uint32_t *fourcc);
 

libavcodec/qsvdec.c

@@ -144,8 +144,8 @@ static int qsv_decode_init(AVCodecContext *avctx, QSVContext *q)
         return ret;
 
     param.mfx.CodecId      = ret;
-    param.mfx.CodecProfile = avctx->profile;
-    param.mfx.CodecLevel   = avctx->level;
+    param.mfx.CodecProfile = ff_qsv_profile_to_mfx(avctx->codec_id, avctx->profile);
+    param.mfx.CodecLevel   = avctx->level == FF_LEVEL_UNKNOWN ? MFX_LEVEL_UNKNOWN : avctx->level;
 
     param.mfx.FrameInfo.BitDepthLuma   = desc->comp[0].depth;
     param.mfx.FrameInfo.BitDepthChroma = desc->comp[0].depth;