vaapi_h265: Add main 10 encode support

(cherry picked from commit 5a5df90d)
(cherry picked from commit d08e02d9)

libavcodec/vaapi_encode_h265.c

@@ -803,8 +803,10 @@ static int vaapi_encode_h265_init_sequence_params(AVCodecContext *avctx)
 
         vseq->seq_fields.bits.chroma_format_idc = 1; // 4:2:0.
         vseq->seq_fields.bits.separate_colour_plane_flag = 0;
-        vseq->seq_fields.bits.bit_depth_luma_minus8 = 0; // 8-bit luma.
-        vseq->seq_fields.bits.bit_depth_chroma_minus8 = 0; // 8-bit chroma.
+        vseq->seq_fields.bits.bit_depth_luma_minus8 =
+            avctx->profile == FF_PROFILE_HEVC_MAIN_10 ? 2 : 0;
+        vseq->seq_fields.bits.bit_depth_chroma_minus8 =
+            avctx->profile == FF_PROFILE_HEVC_MAIN_10 ? 2 : 0;
         // Other misc flags all zero.
 
         // These have to come from the capabilities of the encoder.  We have
@@ -1231,11 +1233,18 @@ static av_cold int vaapi_encode_h265_init(AVCodecContext *avctx)
     case FF_PROFILE_HEVC_MAIN:
     case FF_PROFILE_UNKNOWN:
         ctx->va_profile = VAProfileHEVCMain;
+        ctx->va_rt_format = VA_RT_FORMAT_YUV420;
         break;
     case FF_PROFILE_HEVC_MAIN_10:
-        av_log(avctx, AV_LOG_ERROR, "H.265 main 10-bit profile "
-               "is not supported.\n");
-        return AVERROR_PATCHWELCOME;
+#ifdef VA_RT_FORMAT_YUV420_10BPP
+        ctx->va_profile = VAProfileHEVCMain10;
+        ctx->va_rt_format = VA_RT_FORMAT_YUV420_10BPP;
+        break;
+#else
+        av_log(avctx, AV_LOG_ERROR, "10-bit encoding is not "
+               "supported with this VAAPI version.\n");
+        return AVERROR(ENOSYS);
+#endif
     default:
         av_log(avctx, AV_LOG_ERROR, "Unknown H.265 profile %d.\n",
                avctx->profile);
@@ -1243,9 +1252,6 @@ static av_cold int vaapi_encode_h265_init(AVCodecContext *avctx)
     }
     ctx->va_entrypoint = VAEntrypointEncSlice;
 
-    // This will be dependent on profile when 10-bit is supported.
-    ctx->va_rt_format = VA_RT_FORMAT_YUV420;
-
     if (avctx->bit_rate > 0)
         ctx->va_rc_mode = VA_RC_CBR;
     else