hwcontext_d3d11va: add option to enable debug mode

Basically copied from VLC (LGPL):

http://git.videolan.org/?p=vlc.git;a=blob;f=modules/video_output/win32/direct3d11.c;h=e9fcb83dcabfe778f26e63d19f218caf06a7c3ae;hb=HEAD#l1482
http://git.videolan.org/?p=vlc.git;a=blob;f=modules/codec/avcodec/d3d11va.c;h=85e7d25caebc059a9770da2ef4bb8fe90816d76d;hb=HEAD#l599

Merges Libav commit cfc9e7c9.
Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

configure

@@ -1859,6 +1859,7 @@ HEADERS_LIST="
     direct_h
     dirent_h
     dlfcn_h
+    dxgidebug_h
     dxva_h
     ES2_gl_h
     gsm_h
@@ -5683,6 +5684,7 @@ check_header d3d11.h
 check_header direct.h
 check_header dirent.h
 check_header dlfcn.h
+check_header dxgidebug.h
 check_header dxva.h
 check_header dxva2api.h -D_WIN32_WINNT=0x0600
 check_header io.h

libavutil/hwcontext_d3d11va.c

@@ -16,6 +16,8 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
+#include "config.h"
+
 #include <windows.h>
 
 // Include thread.h before redefining _WIN32_WINNT, to get
@@ -32,6 +34,10 @@
 #include <d3d11.h>
 #include <dxgi1_2.h>
 
+#if HAVE_DXGIDEBUG_H
+#include <dxgidebug.h>
+#endif
+
 #include "avassert.h"
 #include "common.h"
 #include "hwcontext.h"
@@ -476,8 +482,18 @@ static int d3d11va_device_create(AVHWDeviceContext *ctx, const char *device,
     IDXGIAdapter           *pAdapter = NULL;
     ID3D10Multithread      *pMultithread;
     UINT creationFlags = D3D11_CREATE_DEVICE_VIDEO_SUPPORT;
+    int is_debug       = !!av_dict_get(opts, "debug", NULL, 0);
     int ret;
 
+    // (On UWP we can't check this.)
+#if HAVE_LOADLIBRARY
+    if (!LoadLibrary("d3d11_1sdklayers.dll"))
+        is_debug = 0;
+#endif
+
+    if (is_debug)
+        creationFlags |= D3D11_CREATE_DEVICE_DEBUG;
+
     if ((ret = ff_thread_once(&functions_loaded, load_functions)) != 0)
         return AVERROR_UNKNOWN;
     if (!mD3D11CreateDevice || !mCreateDXGIFactory) {
@@ -511,6 +527,22 @@ static int d3d11va_device_create(AVHWDeviceContext *ctx, const char *device,
         ID3D10Multithread_Release(pMultithread);
     }
 
+#if HAVE_LOADLIBRARY && HAVE_DXGIDEBUG_H
+    if (is_debug) {
+        HANDLE dxgidebug_dll = LoadLibrary("dxgidebug.dll");
+        if (dxgidebug_dll) {
+            HRESULT (WINAPI  * pf_DXGIGetDebugInterface)(const GUID *riid, void **ppDebug)
+                = (void *)GetProcAddress(dxgidebug_dll, "DXGIGetDebugInterface");
+            if (pf_DXGIGetDebugInterface) {
+                IDXGIDebug *dxgi_debug = NULL;
+                hr = pf_DXGIGetDebugInterface(&IID_IDXGIDebug, (void**)&dxgi_debug);
+                if (SUCCEEDED(hr) && dxgi_debug)
+                    IDXGIDebug_ReportLiveObjects(dxgi_debug, DXGI_DEBUG_ALL, DXGI_DEBUG_RLO_ALL);
+            }
+        }
+    }
+#endif
+
     return 0;
 }
 

libavutil/hwcontext_d3d11va.h

@@ -30,6 +30,10 @@
  * Using sw_format==AV_PIX_FMT_YUV420P has special semantics, and maps to
  * DXGI_FORMAT_420_OPAQUE. av_hwframe_transfer_data() is not supported for
  * this format. Refer to MSDN for details.
+ *
+ * av_hwdevice_ctx_create() for this device type supports a key named "debug"
+ * for the AVDictionary entry. If this is set to any value, the device creation
+ * code will try to load various supported D3D debugging layers.
  */
 
 #include <d3d11.h>