avcodec/nvenc: Fix profile parameter handling

It was not possible to set a profile before, the builtin profile
parameter does not seem to work propperly.
To be compatible with libx264, this overlays it with a local parameter
that expects a string, instead of an int, that takes the well known values
"high", "main" or "baseline".
Reviewed-by: Philip Langdale <philipl@overt.org>
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/nvenc.c

@@ -151,6 +151,7 @@ typedef struct NvencContext
     void *nvencoder;
 
     char *preset;
+    char *profile;
     int cbr;
     int twopass;
     int gpu;
@@ -729,10 +730,22 @@ static av_cold int nvenc_encode_init(AVCodecContext *avctx)
         ctx->encode_config.frameFieldMode = NV_ENC_PARAMS_FRAME_FIELD_MODE_FRAME;
     }
 
+    switch (avctx->codec->id) {
+    case AV_CODEC_ID_H264:
+        ctx->encode_config.encodeCodecConfig.h264Config.h264VUIParameters.colourDescriptionPresentFlag = 1;
+        ctx->encode_config.encodeCodecConfig.h264Config.h264VUIParameters.videoSignalTypePresentFlag = 1;
+
+        ctx->encode_config.encodeCodecConfig.h264Config.h264VUIParameters.colourMatrix = avctx->colorspace;
+        ctx->encode_config.encodeCodecConfig.h264Config.h264VUIParameters.colourPrimaries = avctx->color_primaries;
+        ctx->encode_config.encodeCodecConfig.h264Config.h264VUIParameters.transferCharacteristics = avctx->color_trc;
+
+        ctx->encode_config.encodeCodecConfig.h264Config.h264VUIParameters.videoFullRangeFlag = avctx->color_range == AVCOL_RANGE_JPEG;
+
+        ctx->encode_config.encodeCodecConfig.h264Config.disableSPSPPS = (avctx->flags & CODEC_FLAG_GLOBAL_HEADER) ? 1 : 0;
+        ctx->encode_config.encodeCodecConfig.h264Config.repeatSPSPPS = (avctx->flags & CODEC_FLAG_GLOBAL_HEADER) ? 0 : 1;
+
+        if (!ctx->profile) {
             switch (avctx->profile) {
-    case FF_PROFILE_HEVC_MAIN:
-        ctx->encode_config.profileGUID = NV_ENC_HEVC_PROFILE_MAIN_GUID;
-        break;
             case FF_PROFILE_H264_BASELINE:
                 ctx->encode_config.profileGUID = NV_ENC_H264_PROFILE_BASELINE_GUID;
                 break;
@@ -748,24 +761,30 @@ static av_cold int nvenc_encode_init(AVCodecContext *avctx)
                 ctx->encode_config.profileGUID = NV_ENC_H264_PROFILE_HIGH_GUID;
                 break;
             }
-
-    switch (avctx->codec->id) {
-    case AV_CODEC_ID_H264:
-        ctx->encode_config.encodeCodecConfig.h264Config.h264VUIParameters.colourDescriptionPresentFlag = 1;
-        ctx->encode_config.encodeCodecConfig.h264Config.h264VUIParameters.videoSignalTypePresentFlag = 1;
-
-        ctx->encode_config.encodeCodecConfig.h264Config.h264VUIParameters.colourMatrix = avctx->colorspace;
-        ctx->encode_config.encodeCodecConfig.h264Config.h264VUIParameters.colourPrimaries = avctx->color_primaries;
-        ctx->encode_config.encodeCodecConfig.h264Config.h264VUIParameters.transferCharacteristics = avctx->color_trc;
-
-        ctx->encode_config.encodeCodecConfig.h264Config.h264VUIParameters.videoFullRangeFlag = avctx->color_range == AVCOL_RANGE_JPEG;
-
-        ctx->encode_config.encodeCodecConfig.h264Config.disableSPSPPS = (avctx->flags & CODEC_FLAG_GLOBAL_HEADER) ? 1 : 0;
-        ctx->encode_config.encodeCodecConfig.h264Config.repeatSPSPPS = (avctx->flags & CODEC_FLAG_GLOBAL_HEADER) ? 0 : 1;
+        } else {
+            if (!strcmp(ctx->profile, "high")) {
+                ctx->encode_config.profileGUID = NV_ENC_H264_PROFILE_HIGH_GUID;
+                avctx->profile = FF_PROFILE_H264_HIGH;
+            } else if (!strcmp(ctx->profile, "main")) {
+                ctx->encode_config.profileGUID = NV_ENC_H264_PROFILE_MAIN_GUID;
+                avctx->profile = FF_PROFILE_H264_MAIN;
+            } else if (!strcmp(ctx->profile, "baseline")) {
+                ctx->encode_config.profileGUID = NV_ENC_H264_PROFILE_BASELINE_GUID;
+                avctx->profile = FF_PROFILE_H264_BASELINE;
+            } else {
+                av_log(avctx, AV_LOG_FATAL, "Profile \"%s\" is unknown! Supported profiles: high, main, baseline\n", ctx->profile);
+                res = AVERROR(EINVAL);
+                goto error;
+            }
+        }
         break;
     case AV_CODEC_ID_H265:
         ctx->encode_config.encodeCodecConfig.hevcConfig.disableSPSPPS = (avctx->flags & CODEC_FLAG_GLOBAL_HEADER) ? 1 : 0;
         ctx->encode_config.encodeCodecConfig.hevcConfig.repeatSPSPPS = (avctx->flags & CODEC_FLAG_GLOBAL_HEADER) ? 0 : 1;
+
+        /* No other profile is supported in the current SDK version 5 */
+        ctx->encode_config.profileGUID = NV_ENC_HEVC_PROFILE_MAIN_GUID;
+        avctx->profile = FF_PROFILE_HEVC_MAIN;
         break;
     /* Earlier switch/case will return if unknown codec is passed. */
     }
@@ -1258,6 +1277,7 @@ static enum AVPixelFormat pix_fmts_nvenc[] = {
 #define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
 static const AVOption options[] = {
     { "preset", "Set the encoding preset (one of hq, hp, bd, ll, llhq, llhp, default)", OFFSET(preset), AV_OPT_TYPE_STRING, { .str = "hq" }, 0, 0, VE },
+    { "profile", "Set the encoding profile (high, main or baseline)", OFFSET(profile), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VE},
     { "cbr", "Use cbr encoding mode", OFFSET(cbr), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, VE },
     { "2pass", "Use 2pass cbr encoding mode (low latency mode only)", OFFSET(twopass), AV_OPT_TYPE_INT, { .i64 = -1 }, -1, 1, VE },
     { "gpu", "Selects which NVENC capable GPU to use. First GPU is 0, second is 1, and so on.", OFFSET(gpu), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, VE },