dxva: support DXGI_FORMAT_420_OPAQUE decoding

Some devices (some phones, apparently) will support only this opaque
format. Of course this won't work with CLI, because copying data
directly is not supported.

Automatic frame allocation (setting AVCodecContext.hw_device_ctx) does
not support this mode, even if it's the only supported mode. But since
opaque surfaces are generally less useful, that's probably ok.

Merges Libav commit 5030e385.
Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

libavcodec/dxva2.c

@@ -393,19 +393,28 @@ static int d3d11va_get_decoder_configuration(AVCodecContext *avctx,
     return ret;
 }
 
+static DXGI_FORMAT d3d11va_map_sw_to_hw_format(enum AVPixelFormat pix_fmt)
+{
+    switch (pix_fmt) {
+    case AV_PIX_FMT_NV12:       return DXGI_FORMAT_NV12;
+    case AV_PIX_FMT_P010:       return DXGI_FORMAT_P010;
+    case AV_PIX_FMT_YUV420P:    return DXGI_FORMAT_420_OPAQUE;
+    default:                    return DXGI_FORMAT_UNKNOWN;
+    }
+}
+
 static int d3d11va_create_decoder(AVCodecContext *avctx)
 {
     FFDXVASharedContext *sctx = DXVA_SHARED_CONTEXT(avctx);
     GUID *guid_list;
     unsigned guid_count, i;
     GUID decoder_guid;
-    DXGI_FORMAT surface_format = avctx->sw_pix_fmt == AV_PIX_FMT_YUV420P10 ?
-                                 DXGI_FORMAT_P010 : DXGI_FORMAT_NV12;
     D3D11_VIDEO_DECODER_DESC desc = { 0 };
     D3D11_VIDEO_DECODER_CONFIG config;
     AVHWFramesContext *frames_ctx = (AVHWFramesContext *)avctx->hw_frames_ctx->data;
     AVD3D11VADeviceContext *device_hwctx = frames_ctx->device_ctx->hwctx;
     AVD3D11VAFramesContext *frames_hwctx = frames_ctx->hwctx;
+    DXGI_FORMAT surface_format = d3d11va_map_sw_to_hw_format(frames_ctx->sw_format);
     D3D11_TEXTURE2D_DESC texdesc;
     HRESULT hr;
     int ret;

libavutil/hwcontext_d3d11va.c

@@ -83,8 +83,11 @@ static const struct {
     DXGI_FORMAT d3d_format;
     enum AVPixelFormat pix_fmt;
 } supported_formats[] = {
-    { DXGI_FORMAT_NV12, AV_PIX_FMT_NV12 },
-    { DXGI_FORMAT_P010, AV_PIX_FMT_P010 },
+    { DXGI_FORMAT_NV12,         AV_PIX_FMT_NV12 },
+    { DXGI_FORMAT_P010,         AV_PIX_FMT_P010 },
+    // Special opaque formats. The pix_fmt is merely a place holder, as the
+    // opaque format cannot be accessed directly.
+    { DXGI_FORMAT_420_OPAQUE,   AV_PIX_FMT_YUV420P },
 };
 
 static void d3d11va_default_lock(void *ctx)
@@ -270,6 +273,7 @@ static int d3d11va_transfer_get_formats(AVHWFramesContext *ctx,
                                         enum AVHWFrameTransferDirection dir,
                                         enum AVPixelFormat **formats)
 {
+    D3D11VAFramesContext *s = ctx->internal->priv;
     enum AVPixelFormat *fmts;
 
     fmts = av_malloc_array(2, sizeof(*fmts));
@@ -279,6 +283,10 @@ static int d3d11va_transfer_get_formats(AVHWFramesContext *ctx,
     fmts[0] = ctx->sw_format;
     fmts[1] = AV_PIX_FMT_NONE;
 
+    // Don't signal support for opaque formats. Actual access would fail.
+    if (s->format == DXGI_FORMAT_420_OPAQUE)
+        fmts[0] = AV_PIX_FMT_NONE;
+
     *formats = fmts;
 
     return 0;

libavutil/hwcontext_d3d11va.h

@@ -26,6 +26,10 @@
  * The default pool implementation will be fixed-size if initial_pool_size is
  * set (and allocate elements from an array texture). Otherwise it will allocate
  * individual textures. Be aware that decoding requires a single array texture.
+ *
+ * Using sw_format==AV_PIX_FMT_YUV420P has special semantics, and maps to
+ * DXGI_FORMAT_420_OPAQUE. av_hwframe_transfer_data() is not supported for
+ * this format. Refer to MSDN for details.
  */
 
 #include <d3d11.h>