nvenc: add support for lossless encoding

Based on a patch by Philip Langdale <philipl@overt.org>

libavcodec/nvenc.c

@@ -494,6 +494,14 @@ static void set_vbr(AVCodecContext *avctx, NV_ENC_RC_PARAMS *rc)
     }
 }
 
+static void set_lossless(AVCodecContext *avctx, NV_ENC_RC_PARAMS *rc)
+{
+    rc->rateControlMode  = NV_ENC_PARAMS_RC_CONSTQP;
+    rc->constQP.qpInterB = 0;
+    rc->constQP.qpInterP = 0;
+    rc->constQP.qpIntra  = 0;
+}
+
 static void nvenc_override_rate_control(AVCodecContext *avctx,
                                         NV_ENC_RC_PARAMS *rc)
 {
@@ -554,6 +562,8 @@ static void nvenc_setup_rate_control(AVCodecContext *avctx)
 
     if (ctx->rc > 0) {
         nvenc_override_rate_control(avctx, rc);
+    } else if (ctx->flags & NVENC_LOSSLESS) {
+        set_lossless(avctx, rc);
     } else if (avctx->global_quality > 0) {
         set_constqp(avctx, rc);
     } else if (avctx->qmin >= 0 && avctx->qmax >= 0) {
@@ -591,6 +601,9 @@ static int nvenc_setup_h264_config(AVCodecContext *avctx)
     h264->maxNumRefFrames = avctx->refs;
     h264->idrPeriod       = cc->gopLength;
 
+    if (ctx->flags & NVENC_LOSSLESS)
+        h264->qpPrimeYZeroTransformBypassFlag = 1;
+
     if (ctx->profile)
         avctx->profile = ctx->profile;
 

libavcodec/nvenc_h264.c

@@ -35,6 +35,8 @@ static const AVOption options[] = {
     { "ll",         "low latency",                        0,                   AV_OPT_TYPE_CONST,  { .i64 = PRESET_LOW_LATENCY_DEFAULT }, 0, 0, VE, "preset" },
     { "llhq",       "low latency hq",                     0,                   AV_OPT_TYPE_CONST,  { .i64 = PRESET_LOW_LATENCY_HQ }, 0, 0, VE, "preset" },
     { "llhp",       "low latency hp",                     0,                   AV_OPT_TYPE_CONST,  { .i64 = PRESET_LOW_LATENCY_HP }, 0, 0, VE, "preset" },
+    { "lossless",   NULL,                                 0,                   AV_OPT_TYPE_CONST,  { .i64 = PRESET_LOSSLESS_DEFAULT }, 0, 0, VE, "preset" },
+    { "losslesshp", NULL,                                 0,                   AV_OPT_TYPE_CONST,  { .i64 = PRESET_LOSSLESS_HP }, 0, 0, VE, "preset" },
     { "profile",  "Set the encoding profile",             OFFSET(profile),     AV_OPT_TYPE_INT,    { .i64 = NV_ENC_H264_PROFILE_HIGH }, NV_ENC_H264_PROFILE_BASELINE, NV_ENC_H264_PROFILE_CONSTRAINED_HIGH, VE, "profile" },
     { "baseline", "",                                     0,                   AV_OPT_TYPE_CONST,  { .i64 = NV_ENC_H264_PROFILE_BASELINE },            0, 0, VE, "profile" },
     { "main",     "",                                     0,                   AV_OPT_TYPE_CONST,  { .i64 = NV_ENC_H264_PROFILE_MAIN },                0, 0, VE, "profile" },