vaapi_encode: Support VBR mode

This includes a backward-compatibility hack to choose CBR anyway on
old drivers which have no CBR support, so that existing programs will
continue to work their options now map to VBR.

(cherry picked from commit f033ba47)

libavcodec/vaapi_encode.c

@@ -1034,6 +1034,19 @@ static av_cold int vaapi_encode_config_attributes(AVCodecContext *avctx)
             };
             break;
         case VAConfigAttribRateControl:
+            // Hack for backward compatibility: CBR was the only
+            // usable RC mode for a long time, so old drivers will
+            // only have it.  Normal default options may now choose
+            // VBR and then fail, however, so override it here with
+            // CBR if that is the only supported mode.
+            if (ctx->va_rc_mode == VA_RC_VBR &&
+                !(attr[i].value & VA_RC_VBR) &&
+                (attr[i].value & VA_RC_CBR)) {
+                av_log(avctx, AV_LOG_WARNING, "VBR rate control is "
+                       "not supported with this driver version; "
+                       "using CBR instead.\n");
+                ctx->va_rc_mode = VA_RC_CBR;
+            }
             if (!(ctx->va_rc_mode & attr[i].value)) {
                 av_log(avctx, AV_LOG_ERROR, "Rate control mode %#x "
                        "is not supported (mask: %#x).\n",
@@ -1098,6 +1111,9 @@ fail:
 static av_cold int vaapi_encode_init_rate_control(AVCodecContext *avctx)
 {
     VAAPIEncodeContext *ctx = avctx->priv_data;
+    int rc_bits_per_second;
+    int rc_target_percentage;
+    int rc_window_size;
     int hrd_buffer_size;
     int hrd_initial_buffer_fullness;
 
@@ -1116,13 +1132,29 @@ static av_cold int vaapi_encode_init_rate_control(AVCodecContext *avctx)
     else
         hrd_initial_buffer_fullness = hrd_buffer_size * 3 / 4;
 
+    if (ctx->va_rc_mode == VA_RC_CBR) {
+        rc_bits_per_second   = avctx->bit_rate;
+        rc_target_percentage = 100;
+        rc_window_size       = 1000;
+    } else {
+        if (avctx->rc_max_rate < avctx->bit_rate) {
+            // Max rate is unset or invalid, just use the normal bitrate.
+            rc_bits_per_second   = avctx->bit_rate;
+            rc_target_percentage = 100;
+        } else {
+            rc_bits_per_second   = avctx->rc_max_rate;
+            rc_target_percentage = (avctx->bit_rate * 100) / rc_bits_per_second;
+        }
+        rc_window_size = (hrd_buffer_size * 1000) / avctx->bit_rate;
+    }
+
     ctx->rc_params.misc.type = VAEncMiscParameterTypeRateControl;
     ctx->rc_params.rc = (VAEncMiscParameterRateControl) {
-        .bits_per_second   = avctx->bit_rate,
-        .target_percentage = 66,
-        .window_size       = 1000,
-        .initial_qp        = (avctx->qmax >= 0 ? avctx->qmax : 40),
-        .min_qp            = (avctx->qmin >= 0 ? avctx->qmin : 18),
+        .bits_per_second   = rc_bits_per_second,
+        .target_percentage = rc_target_percentage,
+        .window_size       = rc_window_size,
+        .initial_qp        = 0,
+        .min_qp            = (avctx->qmin > 0 ? avctx->qmin : 0),
         .basic_unit_size   = 0,
     };
     ctx->global_params[ctx->nb_global_params] =