lavc: Rewrite VAAPI decode infrastructure

Moves much of the setup logic for VAAPI decoding into lavc; the user
now need only provide the hw_frames_ctx.

(cherry picked from commit 123ccd07)
(cherry picked from commit 5e879b54)
(cherry picked from commit 0aec37e6)
(cherry picked from commit cfa4eb4f)

libavcodec/Makefile

@@ -771,7 +771,7 @@ OBJS-$(CONFIG_ADPCM_YAMAHA_ENCODER)       += adpcmenc.o adpcm_data.o
 # hardware accelerators
 OBJS-$(CONFIG_D3D11VA)                    += dxva2.o
 OBJS-$(CONFIG_DXVA2)                      += dxva2.o
-OBJS-$(CONFIG_VAAPI)                      += vaapi.o
+OBJS-$(CONFIG_VAAPI)                      += vaapi.o vaapi_decode.o
 OBJS-$(CONFIG_VDA)                        += vda.o videotoolbox.o
 OBJS-$(CONFIG_VIDEOTOOLBOX)               += videotoolbox.o
 OBJS-$(CONFIG_VDPAU)                      += vdpau.o
@@ -1003,7 +1003,8 @@ SKIPHEADERS-$(CONFIG_QSV)              += qsv.h qsv_internal.h
 SKIPHEADERS-$(CONFIG_QSVDEC)           += qsvdec.h
 SKIPHEADERS-$(CONFIG_QSVENC)           += qsvenc.h
 SKIPHEADERS-$(CONFIG_XVMC)             += xvmc.h
-SKIPHEADERS-$(CONFIG_VAAPI)            += vaapi_encode.h vaapi_internal.h
+SKIPHEADERS-$(CONFIG_VAAPI)            += vaapi_decode.h vaapi_encode.h \
+                                          vaapi_internal.h
 SKIPHEADERS-$(CONFIG_VDA)              += vda.h vda_vt_internal.h
 SKIPHEADERS-$(CONFIG_VDPAU)            += vdpau.h vdpau_internal.h
 SKIPHEADERS-$(CONFIG_VIDEOTOOLBOX)     += videotoolbox.h vda_vt_internal.h

libavcodec/vaapi_decode.h

+/*
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVCODEC_VAAPI_DECODE_H
+#define AVCODEC_VAAPI_DECODE_H
+
+#include <va/va.h>
+
+#include "libavutil/frame.h"
+#include "libavutil/hwcontext.h"
+#include "libavutil/hwcontext_vaapi.h"
+
+#include "avcodec.h"
+#include "vaapi.h"
+
+static inline VASurfaceID ff_vaapi_get_surface_id(AVFrame *pic)
+{
+    return (uintptr_t)pic->data[3];
+}
+
+enum {
+    MAX_PARAM_BUFFERS = 16,
+};
+
+typedef struct VAAPIDecodePicture {
+    VASurfaceID           output_surface;
+
+    int                nb_param_buffers;
+    VABufferID            param_buffers[MAX_PARAM_BUFFERS];
+
+    int                nb_slices;
+    VABufferID           *slice_buffers;
+    int                   slices_allocated;
+} VAAPIDecodePicture;
+
+typedef struct VAAPIDecodeContext {
+    VAProfile             va_profile;
+    VAEntrypoint          va_entrypoint;
+    VAConfigID            va_config;
+    VAContextID           va_context;
+
+    int                   have_old_context;
+    struct vaapi_context *old_context;
+    AVBufferRef          *device_ref;
+
+    AVHWDeviceContext    *device;
+    AVVAAPIDeviceContext *hwctx;
+
+    AVHWFramesContext    *frames;
+    AVVAAPIFramesContext *hwfc;
+} VAAPIDecodeContext;
+
+
+int ff_vaapi_decode_make_param_buffer(AVCodecContext *avctx,
+                                      VAAPIDecodePicture *pic,
+                                      int type,
+                                      const void *data,
+                                      size_t size);
+
+int ff_vaapi_decode_make_slice_buffer(AVCodecContext *avctx,
+                                      VAAPIDecodePicture *pic,
+                                      const void *params_data,
+                                      size_t params_size,
+                                      const void *slice_data,
+                                      size_t slice_size);
+
+int ff_vaapi_decode_issue(AVCodecContext *avctx,
+                          VAAPIDecodePicture *pic);
+int ff_vaapi_decode_cancel(AVCodecContext *avctx,
+                           VAAPIDecodePicture *pic);
+
+int ff_vaapi_decode_init(AVCodecContext *avctx);
+int ff_vaapi_decode_uninit(AVCodecContext *avctx);
+
+#endif /* AVCODEC_VAAPI_DECODE_H */