lavc: add an Intel libmfx-based H.264 decoder

Based on the code by Luca Barbato <lu_zero@gentoo.org> and Yukinori
Yamazoe <drocon11@gmail.com>.

Changelog

@@ -14,6 +14,7 @@ version <next>:
 - Support DNx100 (960x720@8)
 - DXVA2-accelerated HEVC decoding
 - AAC ELD 480 decoding
+- Intel QSV-accelerated H.264 decoding
 
 
 version 11:

configure

@@ -189,6 +189,7 @@ External library support:
   --enable-libfreetype     enable libfreetype [no]
   --enable-libgsm          enable GSM de/encoding via libgsm [no]
   --enable-libilbc         enable iLBC de/encoding via libilbc [no]
+  --enable-libmfx          enable HW acceleration through libmfx
   --enable-libmp3lame      enable MP3 encoding via libmp3lame [no]
   --enable-libopencore-amrnb enable AMR-NB de/encoding via libopencore-amrnb [no]
   --enable-libopencore-amrwb enable AMR-WB decoding via libopencore-amrwb [no]
@@ -1153,6 +1154,7 @@ EXTERNAL_LIBRARY_LIST="
     libfreetype
     libgsm
     libilbc
+    libmfx
     libmp3lame
     libopencore_amrnb
     libopencore_amrwb
@@ -1235,6 +1237,7 @@ SUBSYSTEM_LIST="
     lzo
     mdct
     network
+    qsv
     rdft
 "
 
@@ -1820,6 +1823,8 @@ h263i_decoder_select="h263_decoder"
 h263p_encoder_select="h263_encoder"
 h264_decoder_select="cabac golomb h264chroma h264dsp h264pred h264qpel startcode videodsp"
 h264_decoder_suggest="error_resilience"
+h264_qsv_decoder_deps="libmfx"
+h264_qsv_decoder_select="h264_mp4toannexb_bsf h264_parser qsv h264_qsv_hwaccel"
 hevc_decoder_select="bswapdsp cabac golomb videodsp"
 huffyuv_decoder_select="bswapdsp huffyuvdsp"
 huffyuv_encoder_select="bswapdsp huffman huffyuvencdsp"
@@ -4195,6 +4200,7 @@ enabled libgsm            && { for gsm_hdr in "gsm.h" "gsm/gsm.h"; do
                                    check_lib "${gsm_hdr}" gsm_create -lgsm && break;
                                done || die "ERROR: libgsm not found"; }
 enabled libilbc           && require libilbc ilbc.h WebRtcIlbcfix_InitDecode -lilbc
+enabled libmfx            && require_pkg_config libmfx "mfx/mfxvideo.h" MFXInit
 enabled libmp3lame        && require "libmp3lame >= 3.98.3" lame/lame.h lame_set_VBR_quality -lmp3lame
 enabled libopencore_amrnb && require libopencore_amrnb opencore-amrnb/interf_dec.h Decoder_Interface_init -lopencore-amrnb
 enabled libopencore_amrwb && require libopencore_amrwb opencore-amrwb/dec_if.h D_IF_init -lopencore-amrwb

libavcodec/Makefile

@@ -4,6 +4,7 @@ HEADERS = avcodec.h                                                     \
           avfft.h                                                       \
           dv_profile.h                                                  \
           dxva2.h                                                       \
+          qsv.h                                                         \
           vaapi.h                                                       \
           vda.h                                                         \
           vdpau.h                                                       \
@@ -24,6 +25,7 @@ OBJS = allcodecs.o                                                      \
        mathtables.o                                                     \
        options.o                                                        \
        parser.o                                                         \
+       qsv_api.o                                                        \
        raw.o                                                            \
        utils.o                                                          \
        vorbis_parser.o                                                  \
@@ -78,6 +80,7 @@ OBJS-$(CONFIG_MPEGVIDEOENC)            += mpegvideo_enc.o mpeg12data.o  \
                                           mpegvideoencdsp.o
 OBJS-$(CONFIG_PIXBLOCKDSP)             += pixblockdsp.o
 OBJS-$(CONFIG_QPELDSP)                 += qpeldsp.o
+OBJS-$(CONFIG_QSV)                     += qsv.o
 OBJS-$(CONFIG_RANGECODER)              += rangecoder.o
 RDFT-OBJS-$(CONFIG_HARDCODED_TABLES)   += sin_tables.o
 OBJS-$(CONFIG_RDFT)                    += rdft.o $(RDFT-OBJS-yes)
@@ -218,6 +221,7 @@ OBJS-$(CONFIG_H264_DECODER)            += h264.o h264_cabac.o h264_cavlc.o \
                                           h264_direct.o h264_loopfilter.o  \
                                           h264_mb.o h264_picture.o h264_ps.o \
                                           h264_refs.o h264_sei.o h264_slice.o
+OBJS-$(CONFIG_H264_QSV_DECODER)        += qsv_h264.o
 OBJS-$(CONFIG_HEVC_DECODER)            += hevc.o hevc_mvs.o hevc_ps.o hevc_sei.o \
                                           hevc_cabac.o hevc_refs.o hevcpred.o    \
                                           hevcdsp.o hevc_filter.o
@@ -712,6 +716,7 @@ SKIPHEADERS                            += %_tablegen.h                  \
 SKIPHEADERS-$(CONFIG_DXVA2)            += dxva2.h dxva2_internal.h
 SKIPHEADERS-$(CONFIG_LIBSCHROEDINGER)  += libschroedinger.h
 SKIPHEADERS-$(CONFIG_MPEG_XVMC_DECODER) += xvmc.h
+SKIPHEADERS-$(CONFIG_QSV)              += qsv_internal.h
 SKIPHEADERS-$(CONFIG_VAAPI)            += vaapi_internal.h
 SKIPHEADERS-$(CONFIG_VDA)              += vda.h vda_internal.h
 SKIPHEADERS-$(CONFIG_VDPAU)            += vdpau.h vdpau_internal.h

libavcodec/allcodecs.c

@@ -77,6 +77,7 @@ void avcodec_register_all(void)
     REGISTER_HWACCEL(H263_VAAPI,        h263_vaapi);
     REGISTER_HWACCEL(H263_VDPAU,        h263_vdpau);
     REGISTER_HWACCEL(H264_DXVA2,        h264_dxva2);
+    REGISTER_HWACCEL(H264_QSV,          h264_qsv);
     REGISTER_HWACCEL(H264_VAAPI,        h264_vaapi);
     REGISTER_HWACCEL(H264_VDA,          h264_vda);
     REGISTER_HWACCEL(H264_VDA_OLD,      h264_vda_old);
@@ -160,6 +161,7 @@ void avcodec_register_all(void)
     REGISTER_DECODER(H263I,             h263i);
     REGISTER_ENCODER(H263P,             h263p);
     REGISTER_DECODER(H264,              h264);
+    REGISTER_DECODER(H264_QSV,          h264_qsv);
     REGISTER_DECODER(HEVC,              hevc);
     REGISTER_DECODER(HNM4_VIDEO,        hnm4_video);
     REGISTER_ENCDEC (HUFFYUV,           huffyuv);

libavcodec/qsv.h

+/*
+ * Intel MediaSDK QSV public API
+ *
+ * This file is part of Libav.
+ *
+ * Libav is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * Libav is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Libav; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVCODEC_QSV_H
+#define AVCODEC_QSV_H
+
+#include <mfx/mfxvideo.h>
+
+typedef struct AVQSVContext {
+    mfxSession session;
+    int iopattern;
+
+    mfxExtBuffer **ext_buffers;
+    int         nb_ext_buffers;
+} AVQSVContext;
+
+/**
+ * Allocate a new context.
+ *
+ * It must be freed by the caller with av_free().
+ */
+AVQSVContext *av_qsv_alloc_context(void);
+
+#endif /* AVCODEC_QSV_H */

libavcodec/qsv_api.c

+/*
+ * Intel MediaSDK QSV public API functions
+ *
+ * This file is part of Libav.
+ *
+ * Libav is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * Libav is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Libav; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include "libavutil/mem.h"
+
+#include "qsv.h"
+
+AVQSVContext *av_qsv_alloc_context(void)
+{
+    return av_mallocz(sizeof(AVQSVContext));
+}

libavcodec/qsv_h264.c

+/*
+ * Intel MediaSDK QSV based H.264 decoder
+ *
+ * copyright (c) 2013 Luca Barbato
+ * copyright (c) 2015 Anton Khirnov
+ *
+ * This file is part of Libav.
+ *
+ * Libav is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * Libav is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Libav; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+
+#include <stdint.h>
+#include <string.h>
+
+#include <mfx/mfxvideo.h>
+
+#include "libavutil/common.h"
+#include "libavutil/fifo.h"
+#include "libavutil/opt.h"
+
+#include "avcodec.h"
+#include "internal.h"
+#include "qsv_internal.h"
+#include "qsv.h"
+
+typedef struct QSVH264Context {
+    AVClass *class;
+    QSVContext qsv;
+
+    // the internal parser and codec context for parsing the data
+    AVCodecParserContext *parser;
+    AVCodecContext *avctx_internal;
+    enum AVPixelFormat orig_pix_fmt;
+
+    // the filter for converting to Annex B
+    AVBitStreamFilterContext *bsf;
+
+    AVFifoBuffer *packet_fifo;
+
+    AVPacket input_ref;
+    AVPacket pkt_filtered;
+    uint8_t *filtered_data;
+} QSVH264Context;
+
+static void qsv_clear_buffers(QSVH264Context *s)
+{
+    AVPacket pkt;
+    while (av_fifo_size(s->packet_fifo) >= sizeof(pkt)) {
+        av_fifo_generic_read(s->packet_fifo, &pkt, sizeof(pkt), NULL);
+        av_packet_unref(&pkt);
+    }
+
+    if (s->filtered_data != s->input_ref.data)
+        av_freep(&s->filtered_data);
+    s->filtered_data = NULL;
+    av_packet_unref(&s->input_ref);
+}
+
+static av_cold int qsv_decode_close(AVCodecContext *avctx)
+{
+    QSVH264Context *s = avctx->priv_data;
+
+    ff_qsv_close(&s->qsv);
+
+    qsv_clear_buffers(s);
+
+    av_fifo_free(s->packet_fifo);
+
+    av_bitstream_filter_close(s->bsf);
+    av_parser_close(s->parser);
+    avcodec_free_context(&s->avctx_internal);
+
+    return 0;
+}
+
+static av_cold int qsv_decode_init(AVCodecContext *avctx)
+{
+    QSVH264Context *s = avctx->priv_data;
+    int ret;
+
+    s->orig_pix_fmt = AV_PIX_FMT_NONE;
+
+    s->packet_fifo = av_fifo_alloc(sizeof(AVPacket));
+    if (!s->packet_fifo) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+
+    s->bsf = av_bitstream_filter_init("h264_mp4toannexb");
+    if (!s->bsf) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+
+    s->avctx_internal = avcodec_alloc_context3(NULL);
+    if (!s->avctx_internal) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+
+    if (avctx->extradata) {
+        s->avctx_internal->extradata = av_mallocz(avctx->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
+        if (!s->avctx_internal->extradata) {
+            ret = AVERROR(ENOMEM);
+            goto fail;
+        }
+        memcpy(s->avctx_internal->extradata, avctx->extradata,
+               avctx->extradata_size);
+        s->avctx_internal->extradata_size = avctx->extradata_size;
+    }
+
+    s->parser = av_parser_init(AV_CODEC_ID_H264);
+    if (!s->parser) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+    s->parser->flags |= PARSER_FLAG_COMPLETE_FRAMES;
+
+    s->qsv.iopattern = MFX_IOPATTERN_OUT_SYSTEM_MEMORY;
+
+    return 0;
+fail:
+    qsv_decode_close(avctx);
+    return ret;
+}
+
+static int qsv_process_data(AVCodecContext *avctx, AVFrame *frame,
+                            int *got_frame, AVPacket *pkt)
+{
+    QSVH264Context *s = avctx->priv_data;
+    uint8_t *dummy_data;
+    int dummy_size;
+    int ret;
+
+    /* we assume the packets are already split properly and want
+     * just the codec parameters here */
+    av_parser_parse2(s->parser, s->avctx_internal,
+                     &dummy_data, &dummy_size,
+                     pkt->data, pkt->size, pkt->pts, pkt->dts,
+                     pkt->pos);
+
+    /* TODO: flush delayed frames on reinit */
+    if (s->parser->format       != s->orig_pix_fmt    ||
+        s->parser->coded_width  != avctx->coded_width ||
+        s->parser->coded_height != avctx->coded_height) {
+        mfxSession session = NULL;
+
+        enum AVPixelFormat pix_fmts[3] = { AV_PIX_FMT_QSV,
+                                           AV_PIX_FMT_NONE,
+                                           AV_PIX_FMT_NONE };
+        enum AVPixelFormat qsv_format;
+
+        qsv_format = ff_qsv_map_pixfmt(s->parser->format);
+        if (qsv_format < 0) {
+            av_log(avctx, AV_LOG_ERROR,
+                   "Only 8-bit YUV420 streams are supported.\n");
+            ret = AVERROR(ENOSYS);
+            goto reinit_fail;
+        }
+
+        s->orig_pix_fmt     = s->parser->format;
+        avctx->pix_fmt      = pix_fmts[1] = qsv_format;
+        avctx->width        = s->parser->width;
+        avctx->height       = s->parser->height;
+        avctx->coded_width  = s->parser->coded_width;
+        avctx->coded_height = s->parser->coded_height;
+        avctx->level        = s->avctx_internal->level;
+        avctx->profile      = s->avctx_internal->profile;
+
+        ret = ff_get_format(avctx, pix_fmts);
+        if (ret < 0)
+            goto reinit_fail;
+
+        avctx->pix_fmt = ret;
+
+        if (avctx->hwaccel_context) {
+            AVQSVContext *user_ctx = avctx->hwaccel_context;
+            session               = user_ctx->session;
+            s->qsv.iopattern      = user_ctx->iopattern;
+            s->qsv.ext_buffers    = user_ctx->ext_buffers;
+            s->qsv.nb_ext_buffers = user_ctx->nb_ext_buffers;
+        }
+
+        ret = ff_qsv_init(avctx, &s->qsv, session);
+        if (ret < 0)
+            goto reinit_fail;
+    }
+
+    return ff_qsv_decode(avctx, &s->qsv, frame, got_frame, &s->pkt_filtered);
+
+reinit_fail:
+    s->orig_pix_fmt = s->parser->format = avctx->pix_fmt = AV_PIX_FMT_NONE;
+    return ret;
+}
+
+static int qsv_decode_frame(AVCodecContext *avctx, void *data,
+                            int *got_frame, AVPacket *avpkt)
+{
+    QSVH264Context *s = avctx->priv_data;
+    AVFrame *frame    = data;
+    int ret;
+
+    /* buffer the input packet */
+    if (avpkt->size) {
+        AVPacket input_ref = { 0 };
+
+        if (av_fifo_space(s->packet_fifo) < sizeof(input_ref)) {
+            ret = av_fifo_realloc2(s->packet_fifo,
+                                   av_fifo_size(s->packet_fifo) + sizeof(input_ref));
+            if (ret < 0)
+                return ret;
+        }
+
+        ret = av_packet_ref(&input_ref, avpkt);
+        if (ret < 0)
+            return ret;
+        av_fifo_generic_write(s->packet_fifo, &input_ref, sizeof(input_ref), NULL);
+    }
+
+    /* process buffered data */
+    while (!*got_frame) {
+        /* prepare the input data -- convert to Annex B if needed */
+        if (s->pkt_filtered.size <= 0) {
+            int size;
+
+            /* no more data */
+            if (av_fifo_size(s->packet_fifo) < sizeof(AVPacket))
+                return avpkt->size ? avpkt->size : ff_qsv_decode(avctx, &s->qsv, frame, got_frame, avpkt);
+
+            if (s->filtered_data != s->input_ref.data)
+                av_freep(&s->filtered_data);
+            s->filtered_data = NULL;
+            av_packet_unref(&s->input_ref);
+
+            av_fifo_generic_read(s->packet_fifo, &s->input_ref, sizeof(s->input_ref), NULL);
+            ret = av_bitstream_filter_filter(s->bsf, avctx, NULL,
+                                             &s->filtered_data, &size,
+                                             s->input_ref.data, s->input_ref.size, 0);
+            if (ret < 0) {
+                s->filtered_data = s->input_ref.data;
+                size             = s->input_ref.size;
+            }
+            s->pkt_filtered      = s->input_ref;
+            s->pkt_filtered.data = s->filtered_data;
+            s->pkt_filtered.size = size;
+        }
+
+        ret = qsv_process_data(avctx, frame, got_frame, &s->pkt_filtered);
+        if (ret < 0)
+            return ret;
+
+        s->pkt_filtered.size -= ret;
+        s->pkt_filtered.data += ret;
+    }
+
+    return avpkt->size;
+}
+
+static void qsv_decode_flush(AVCodecContext *avctx)
+{
+    QSVH264Context *s = avctx->priv_data;
+
+    qsv_clear_buffers(s);
+    s->orig_pix_fmt = AV_PIX_FMT_NONE;
+}
+
+AVHWAccel ff_h264_qsv_hwaccel = {
+    .name           = "h264_qsv",
+    .type           = AVMEDIA_TYPE_VIDEO,
+    .id             = AV_CODEC_ID_H264,
+    .pix_fmt        = AV_PIX_FMT_QSV,
+};
+
+#define OFFSET(x) offsetof(QSVH264Context, x)
+#define VD AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM
+static const AVOption options[] = {
+    { "async_depth", "Internal parallelization depth, the higher the value the higher the latency.", OFFSET(qsv.async_depth), AV_OPT_TYPE_INT, { .i64 = ASYNC_DEPTH_DEFAULT }, 0, INT_MAX, VD },
+    { NULL },
+};
+
+static const AVClass class = {
+    .class_name = "h264_qsv",
+    .item_name  = av_default_item_name,
+    .option     = options,
+    .version    = LIBAVUTIL_VERSION_INT,
+};
+
+AVCodec ff_h264_qsv_decoder = {
+    .name           = "h264_qsv",
+    .long_name      = NULL_IF_CONFIG_SMALL("H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10 (Intel Quick Sync Video acceleration)"),
+    .priv_data_size = sizeof(QSVH264Context),
+    .type           = AVMEDIA_TYPE_VIDEO,
+    .id             = AV_CODEC_ID_H264,
+    .init           = qsv_decode_init,
+    .decode         = qsv_decode_frame,
+    .flush          = qsv_decode_flush,
+    .close          = qsv_decode_close,
+    .capabilities   = CODEC_CAP_DELAY,
+    .priv_class     = &class,
+};

libavcodec/qsv_internal.h

+/*
+ * Intel MediaSDK QSV utility functions
+ *
+ * copyright (c) 2013 Luca Barbato
+ *
+ * This file is part of Libav.
+ *
+ * Libav is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * Libav is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with Libav; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVCODEC_QSV_INTERNAL_H
+#define AVCODEC_QSV_INTERNAL_H
+
+#include <stdint.h>
+#include <sys/types.h>
+
+#include <mfx/mfxvideo.h>
+
+#include "libavutil/frame.h"
+#include "libavutil/pixfmt.h"
+
+#include "avcodec.h"
+
+#define QSV_VERSION_MAJOR 1
+#define QSV_VERSION_MINOR 1
+
+#define ASYNC_DEPTH_DEFAULT 4       // internal parallelism
+
+typedef struct QSVFrame {
+    AVFrame *frame;
+    mfxFrameSurface1 *surface;
+
+    mfxFrameSurface1 surface_internal;
+
+    struct QSVFrame *next;
+} QSVFrame;
+
+typedef struct QSVContext {
+    // the session used for decoding
+    mfxSession session;
+
+    // the session we allocated internally, in case the caller did not provide
+    // one
+    mfxSession internal_session;
+
+    /**
+     * a linked list of frames currently being used by QSV
+     */
+    QSVFrame *work_frames;
+
+    // options set by the caller
+    int async_depth;
+    int iopattern;
+
+    mfxExtBuffer **ext_buffers;
+    int         nb_ext_buffers;
+} QSVContext;
+
+/**
+ * Convert a libmfx error code into a libav error code.
+ */
+int ff_qsv_error(int mfx_err);
+
+int ff_qsv_map_pixfmt(enum AVPixelFormat format);
+
+int ff_qsv_init(AVCodecContext *s, QSVContext *q, mfxSession session);
+
+int ff_qsv_decode(AVCodecContext *s, QSVContext *q,
+                  AVFrame *frame, int *got_frame,
+                  AVPacket *avpkt);
+
+int ff_qsv_close(QSVContext *q);
+
+#endif /* AVCODEC_QSV_INTERNAL_H */

libavcodec/version.h

@@ -29,7 +29,7 @@
 #include "libavutil/version.h"
 
 #define LIBAVCODEC_VERSION_MAJOR 56
-#define LIBAVCODEC_VERSION_MINOR 13
+#define LIBAVCODEC_VERSION_MINOR 14
 #define LIBAVCODEC_VERSION_MICRO  0
 
 #define LIBAVCODEC_VERSION_INT  AV_VERSION_INT(LIBAVCODEC_VERSION_MAJOR, \