vaapi_encode: Check packed header capabilities

This improves behaviour with drivers which do not support packed
headers, such as AMD VCE on mesa/gallium.

(cherry picked from commit 892bbbcd)

libavcodec/vaapi_encode.c

@@ -237,7 +237,8 @@ static int vaapi_encode_issue(AVCodecContext *avctx,
     }
 
     if (pic->type == PICTURE_TYPE_IDR) {
-        if (ctx->codec->write_sequence_header) {
+        if (ctx->va_packed_headers & VA_ENC_PACKED_HEADER_SEQUENCE &&
+            ctx->codec->write_sequence_header) {
             bit_len = 8 * sizeof(data);
             err = ctx->codec->write_sequence_header(avctx, data, &bit_len);
             if (err < 0) {
@@ -253,7 +254,8 @@ static int vaapi_encode_issue(AVCodecContext *avctx,
         }
     }
 
-    if (ctx->codec->write_picture_header) {
+    if (ctx->va_packed_headers & VA_ENC_PACKED_HEADER_PICTURE &&
+        ctx->codec->write_picture_header) {
         bit_len = 8 * sizeof(data);
         err = ctx->codec->write_picture_header(avctx, pic, data, &bit_len);
         if (err < 0) {
@@ -289,7 +291,8 @@ static int vaapi_encode_issue(AVCodecContext *avctx,
         }
     }
 
-    if (ctx->codec->write_extra_header) {
+    if (ctx->va_packed_headers & VA_ENC_PACKED_HEADER_MISC &&
+        ctx->codec->write_extra_header) {
         for (i = 0;; i++) {
             int type;
             bit_len = 8 * sizeof(data);
@@ -336,7 +339,8 @@ static int vaapi_encode_issue(AVCodecContext *avctx,
             }
         }
 
-        if (ctx->codec->write_slice_header) {
+        if (ctx->va_packed_headers & VA_ENC_PACKED_HEADER_SLICE &&
+            ctx->codec->write_slice_header) {
             bit_len = 8 * sizeof(data);
             err = ctx->codec->write_slice_header(avctx, pic, slice,
                                                  data, &bit_len);
@@ -930,9 +934,10 @@ static av_cold int vaapi_encode_config_attributes(AVCodecContext *avctx)
     VAProfile    *profiles    = NULL;
     VAEntrypoint *entrypoints = NULL;
     VAConfigAttrib attr[] = {
-        { VAConfigAttribRTFormat        },
-        { VAConfigAttribRateControl     },
-        { VAConfigAttribEncMaxRefFrames },
+        { VAConfigAttribRTFormat         },
+        { VAConfigAttribRateControl      },
+        { VAConfigAttribEncMaxRefFrames  },
+        { VAConfigAttribEncPackedHeaders },
     };
 
     n = vaMaxNumProfiles(ctx->hwctx->display);
@@ -1049,6 +1054,23 @@ static av_cold int vaapi_encode_config_attributes(AVCodecContext *avctx)
             }
         }
         break;
+        case VAConfigAttribEncPackedHeaders:
+            if (ctx->va_packed_headers & ~attr[i].value) {
+                // This isn't fatal, but packed headers are always
+                // preferable because they are under our control.
+                // When absent, the driver is generating them and some
+                // features may not work (e.g. VUI or SEI in H.264).
+                av_log(avctx, AV_LOG_WARNING, "Warning: some packed "
+                       "headers are not supported (want %#x, got %#x).\n",
+                       ctx->va_packed_headers, attr[i].value);
+                ctx->va_packed_headers &= attr[i].value;
+            }
+            ctx->config_attributes[ctx->nb_config_attributes++] =
+                (VAConfigAttrib) {
+                .type  = VAConfigAttribEncPackedHeaders,
+                .value = ctx->va_packed_headers,
+            };
+            break;
         default:
             av_assert0(0 && "Unexpected config attribute.");
         }

libavcodec/vaapi_encode.h

@@ -101,6 +101,9 @@ typedef struct VAAPIEncodeContext {
     unsigned int    va_rt_format;
     // Rate control mode.
     unsigned int    va_rc_mode;
+    // Supported packed headers (initially the desired set, modified
+    // later to what is actually supported).
+    unsigned int    va_packed_headers;
 
     // The required size of surfaces.  This is probably the input
     // size (AVCodecContext.width|height) aligned up to whatever

libavcodec/vaapi_encode_h264.c

@@ -1221,6 +1221,10 @@ static av_cold int vaapi_encode_h264_init(AVCodecContext *avctx)
     else
         ctx->va_rc_mode = VA_RC_CQP;
 
+    ctx->va_packed_headers =
+        VA_ENC_PACKED_HEADER_SEQUENCE | // SPS and PPS.
+        VA_ENC_PACKED_HEADER_SLICE    | // Slice headers.
+        VA_ENC_PACKED_HEADER_MISC;      // SEI.
 
     ctx->surface_width  = FFALIGN(avctx->width,  16);
     ctx->surface_height = FFALIGN(avctx->height, 16);

libavcodec/vaapi_encode_h265.c

@@ -1251,6 +1251,9 @@ static av_cold int vaapi_encode_h265_init(AVCodecContext *avctx)
     else
         ctx->va_rc_mode = VA_RC_CQP;
 
+    ctx->va_packed_headers =
+        VA_ENC_PACKED_HEADER_SEQUENCE | // VPS, SPS and PPS.
+        VA_ENC_PACKED_HEADER_SLICE;     // Slice headers.
 
     ctx->surface_width  = FFALIGN(avctx->width,  16);
     ctx->surface_height = FFALIGN(avctx->height, 16);

libavcodec/vaapi_encode_mjpeg.c

@@ -345,6 +345,17 @@ static av_cold int vaapi_encode_mjpeg_configure(AVCodecContext *avctx)
         return AVERROR(EINVAL);
     }
 
+    // Hack: the implementation calls the JPEG image header (which we
+    // will use in the same way as a slice header) generic "raw data".
+    // Therefore, if after the packed header capability check we have
+    // PACKED_HEADER_RAW_DATA available, rewrite it as
+    // PACKED_HEADER_SLICE so that the header-writing code can do the
+    // right thing.
+    if (ctx->va_packed_headers & VA_ENC_PACKED_HEADER_RAW_DATA) {
+        ctx->va_packed_headers &= ~VA_ENC_PACKED_HEADER_RAW_DATA;
+        ctx->va_packed_headers |=  VA_ENC_PACKED_HEADER_SLICE;
+    }
+
     vaapi_encode_mjpeg_init_tables(avctx);
 
     return 0;
@@ -380,6 +391,10 @@ static av_cold int vaapi_encode_mjpeg_init(AVCodecContext *avctx)
 
     ctx->va_rc_mode = VA_RC_CQP;
 
+    // The JPEG image header - see note above.
+    ctx->va_packed_headers =
+        VA_ENC_PACKED_HEADER_RAW_DATA;
+
     ctx->surface_width  = FFALIGN(avctx->width,  8);
     ctx->surface_height = FFALIGN(avctx->height, 8);