ffmpeg: init filtergraphs only after we have a frame on each input

This makes sure the actual stream parameters are used, which is
important mainly for hardware decoding+filtering cases, which would
previously require various weird workarounds to handle the fact that a
fake software graph has to be constructed, but never used.
This should also improve behaviour in rare cases where
avformat_find_stream_info() does not provide accurate information.

This merges Libav commit a3a0230a. It was previously skipped.

The code in flush_encoders() which sets up a "fake" format wasn't in
Libav. I'm not sure if it's a good idea, but it tends to give
behavior closer to the old one in certain corner cases.

The vp8-size-change gives different result, because now the size of
the first frame is used. libavformat reported the size of the largest
frame for some reason.

The exr tests now use the sample aspect ratio of the first frame. For
some reason libavformat determines 0/1 as aspect ratio, while the
decoder returns the correct one.

The ffm and mxf tests change the field_order values. I'm assuming
another libavformat/decoding mismatch.
Signed-off-by: wm4 <nfxjfg@googlemail.com>

ffmpeg.h

@@ -233,6 +233,9 @@ typedef struct InputFilter {
     struct InputStream *ist;
     struct FilterGraph *graph;
     uint8_t            *name;
+    enum AVMediaType    type;   // AVMEDIA_TYPE_SUBTITLE for sub2video
+
+    AVFifoBuffer *frame_queue;
 
     // parameters configured for this input
     int format;
@@ -321,14 +324,6 @@ typedef struct InputStream {
     int guess_layout_max;
 
     int autorotate;
-    int resample_height;
-    int resample_width;
-    int resample_pix_fmt;
-
-    int      resample_sample_fmt;
-    int      resample_sample_rate;
-    int      resample_channels;
-    uint64_t resample_channel_layout;
 
     int fix_sub_duration;
     struct { /* previous decoded subtitle and related variables */
@@ -379,6 +374,8 @@ typedef struct InputStream {
 
     int64_t *dts_buffer;
     int nb_dts_buffer;
+
+    int got_output;
 } InputStream;
 
 typedef struct InputFile {
@@ -507,6 +504,8 @@ typedef struct OutputStream {
     // parameters are set in the AVStream.
     int initialized;
 
+    int inputs_done;
+
     const char *attachment_filename;
     int copy_initial_nonkeyframes;
     int copy_prior_start;
@@ -636,7 +635,6 @@ int init_simple_filtergraph(InputStream *ist, OutputStream *ost);
 int init_complex_filtergraph(FilterGraph *fg);
 
 int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *frame);
-int ifilter_parameters_from_decoder(InputFilter *ifilter, const AVCodecContext *avctx);
 
 int ffmpeg_parse_options(int argc, char **argv);
 
@@ -645,7 +643,6 @@ int dxva2_init(AVCodecContext *s);
 int vda_init(AVCodecContext *s);
 int videotoolbox_init(AVCodecContext *s);
 int qsv_init(AVCodecContext *s);
-int qsv_transcode_init(OutputStream *ost);
 int vaapi_decode_init(AVCodecContext *avctx);
 int vaapi_device_init(const char *device);
 int cuvid_init(AVCodecContext *s);

ffmpeg_cuvid.c

@@ -125,7 +125,6 @@ int cuvid_transcode_init(OutputStream *ost)
         }
 
         ist->hwaccel_ctx = ctx;
-        ist->resample_pix_fmt = AV_PIX_FMT_CUDA;
         ist->hwaccel_uninit = cuvid_uninit;
 
         /* This is a bit hacky, av_hwframe_ctx_init is called by the cuvid decoder

ffmpeg_filter.c

@@ -217,6 +217,10 @@ int init_simple_filtergraph(InputStream *ist, OutputStream *ost)
     fg->inputs[0]->graph = fg;
     fg->inputs[0]->format = -1;
 
+    fg->inputs[0]->frame_queue = av_fifo_alloc(8 * sizeof(AVFrame*));
+    if (!fg->inputs[0]->frame_queue)
+        exit_program(1);
+
     GROW_ARRAY(ist->filters, ist->nb_filters);
     ist->filters[ist->nb_filters - 1] = fg->inputs[0];
 
@@ -295,6 +299,11 @@ static void init_input_filter(FilterGraph *fg, AVFilterInOut *in)
     fg->inputs[fg->nb_inputs - 1]->ist   = ist;
     fg->inputs[fg->nb_inputs - 1]->graph = fg;
     fg->inputs[fg->nb_inputs - 1]->format = -1;
+    fg->inputs[fg->nb_inputs - 1]->type = ist->st->codecpar->codec_type;
+
+    fg->inputs[fg->nb_inputs - 1]->frame_queue = av_fifo_alloc(8 * sizeof(AVFrame*));
+    if (!fg->inputs[fg->nb_inputs - 1]->frame_queue)
+        exit_program(1);
 
     GROW_ARRAY(ist->filters, ist->nb_filters);
     ist->filters[ist->nb_filters - 1] = fg->inputs[fg->nb_inputs - 1];
@@ -691,12 +700,15 @@ static int sub2video_prepare(InputStream *ist, InputFilter *ifilter)
         }
         av_log(avf, AV_LOG_INFO, "sub2video: using %dx%d canvas\n", w, h);
     }
-    ist->sub2video.w = ist->resample_width  = ifilter->width  = w;
-    ist->sub2video.h = ist->resample_height = ifilter->height = h;
+    ist->sub2video.w = ifilter->width  = w;
+    ist->sub2video.h = ifilter->height = h;
+
+    ifilter->width  = ist->dec_ctx->width  ? ist->dec_ctx->width  : ist->sub2video.w;
+    ifilter->height = ist->dec_ctx->height ? ist->dec_ctx->height : ist->sub2video.h;
 
     /* rectangles are AV_PIX_FMT_PAL8, but we have no guarantee that the
        palettes for all rectangles are identical or compatible */
-    ist->resample_pix_fmt = ifilter->format = AV_PIX_FMT_RGB32;
+    ifilter->format = AV_PIX_FMT_RGB32;
 
     ist->sub2video.frame = av_frame_alloc();
     if (!ist->sub2video.frame)
@@ -1133,36 +1145,6 @@ int ifilter_parameters_from_frame(InputFilter *ifilter, const AVFrame *frame)
     return 0;
 }
 
-int ifilter_parameters_from_decoder(InputFilter *ifilter, const AVCodecContext *avctx)
-{
-    av_buffer_unref(&ifilter->hw_frames_ctx);
-
-    if (avctx->codec_type == AVMEDIA_TYPE_VIDEO)
-        ifilter->format = avctx->pix_fmt;
-    else
-        ifilter->format = avctx->sample_fmt;
-
-    ifilter->width               = avctx->width;
-    ifilter->height              = avctx->height;
-    if (ifilter->ist && ifilter->ist->st && ifilter->ist->st->sample_aspect_ratio.num)
-        ifilter->sample_aspect_ratio = ifilter->ist->st->sample_aspect_ratio;
-    else
-        ifilter->sample_aspect_ratio = avctx->sample_aspect_ratio;
-
-    ifilter->sample_rate         = avctx->sample_rate;
-    ifilter->channels            = avctx->channels;
-    ifilter->channel_layout      = avctx->channel_layout;
-
-    if (ifilter->ist && ifilter->ist->hw_frames_ctx) {
-        ifilter->format = ifilter->ist->resample_pix_fmt;
-        ifilter->hw_frames_ctx = av_buffer_ref(ifilter->ist->hw_frames_ctx);
-        if (!ifilter->hw_frames_ctx)
-            return AVERROR(ENOMEM);
-    }
-
-    return 0;
-}
-
 int ist_in_filtergraph(FilterGraph *fg, InputStream *ist)
 {
     int i;

ffmpeg_opt.c

@@ -736,10 +736,6 @@ static void add_input_streams(OptionsContext *o, AVFormatContext *ic)
             // avformat_find_stream_info() doesn't set this for us anymore.
             ist->dec_ctx->framerate = st->avg_frame_rate;
 
-            ist->resample_height  = ist->dec_ctx->height;
-            ist->resample_width   = ist->dec_ctx->width;
-            ist->resample_pix_fmt = ist->dec_ctx->pix_fmt;
-
             MATCH_PER_STREAM_OPT(frame_rates, str, framerate, ic, st);
             if (framerate && av_parse_video_rate(&ist->framerate,
                                                  framerate) < 0) {
@@ -804,12 +800,6 @@ static void add_input_streams(OptionsContext *o, AVFormatContext *ic)
             ist->guess_layout_max = INT_MAX;
             MATCH_PER_STREAM_OPT(guess_layout_max, i, ist->guess_layout_max, ic, st);
             guess_input_channel_layout(ist);
-
-            ist->resample_sample_fmt     = ist->dec_ctx->sample_fmt;
-            ist->resample_sample_rate    = ist->dec_ctx->sample_rate;
-            ist->resample_channels       = ist->dec_ctx->channels;
-            ist->resample_channel_layout = ist->dec_ctx->channel_layout;
-
             break;
         case AVMEDIA_TYPE_DATA:
         case AVMEDIA_TYPE_SUBTITLE: {
@@ -2021,33 +2011,6 @@ static int init_complex_filters(void)
     return 0;
 }
 
-static int configure_complex_filters(void)
-{
-    int i, j, ret = 0;
-
-    for (i = 0; i < nb_filtergraphs; i++) {
-        FilterGraph *fg = filtergraphs[i];
-
-        if (filtergraph_is_simple(fg))
-            continue;
-
-        for (j = 0; j < fg->nb_inputs; j++) {
-            ret = ifilter_parameters_from_decoder(fg->inputs[j],
-                                                  fg->inputs[j]->ist->dec_ctx);
-            if (ret < 0) {
-                av_log(NULL, AV_LOG_ERROR,
-                       "Error initializing filtergraph %d input %d\n", i, j);
-                return ret;
-            }
-        }
-
-        ret = configure_filtergraph(filtergraphs[i]);
-        if (ret < 0)
-            return ret;
-    }
-    return 0;
-}
-
 static int open_output_file(OptionsContext *o, const char *filename)
 {
     AVFormatContext *oc;
@@ -3291,13 +3254,6 @@ int ffmpeg_parse_options(int argc, char **argv)
         goto fail;
     }
 
-    /* configure the complex filtergraphs */
-    ret = configure_complex_filters();
-    if (ret < 0) {
-        av_log(NULL, AV_LOG_FATAL, "Error configuring complex filters.\n");
-        goto fail;
-    }
-
 fail:
     uninit_parse_context(&octx);
     if (ret < 0) {

ffmpeg_qsv.c

@@ -107,85 +107,3 @@ int qsv_init(AVCodecContext *s)
 
     return 0;
 }
-
-int qsv_transcode_init(OutputStream *ost)
-{
-    InputStream *ist;
-    const enum AVPixelFormat *pix_fmt;
-
-    int err, i;
-    AVBufferRef *encode_frames_ref = NULL;
-    AVHWFramesContext *encode_frames;
-    AVQSVFramesContext *qsv_frames;
-
-    /* check if the encoder supports QSV */
-    if (!ost->enc->pix_fmts)
-        return 0;
-    for (pix_fmt = ost->enc->pix_fmts; *pix_fmt != AV_PIX_FMT_NONE; pix_fmt++)
-        if (*pix_fmt == AV_PIX_FMT_QSV)
-            break;
-    if (*pix_fmt == AV_PIX_FMT_NONE)
-        return 0;
-
-    if (strcmp(ost->avfilter, "null") || ost->source_index < 0)
-        return 0;
-
-    /* check if the decoder supports QSV and the output only goes to this stream */
-    ist = input_streams[ost->source_index];
-    if (ist->hwaccel_id != HWACCEL_QSV || !ist->dec || !ist->dec->pix_fmts)
-        return 0;
-    for (pix_fmt = ist->dec->pix_fmts; *pix_fmt != AV_PIX_FMT_NONE; pix_fmt++)
-        if (*pix_fmt == AV_PIX_FMT_QSV)
-            break;
-    if (*pix_fmt == AV_PIX_FMT_NONE)
-        return 0;
-
-    for (i = 0; i < nb_output_streams; i++)
-        if (output_streams[i] != ost &&
-            output_streams[i]->source_index == ost->source_index)
-            return 0;
-
-    av_log(NULL, AV_LOG_VERBOSE, "Setting up QSV transcoding\n");
-
-    if (!hw_device_ctx) {
-        err = qsv_device_init(ist);
-        if (err < 0)
-            goto fail;
-    }
-
-    // This creates a dummy hw_frames_ctx for the encoder to be
-    // suitably initialised.  It only contains one real frame, so
-    // hopefully doesn't waste too much memory.
-
-    encode_frames_ref = av_hwframe_ctx_alloc(hw_device_ctx);
-    if (!encode_frames_ref) {
-        err = AVERROR(ENOMEM);
-        goto fail;
-    }
-    encode_frames = (AVHWFramesContext*)encode_frames_ref->data;
-    qsv_frames = encode_frames->hwctx;
-
-    encode_frames->width     = FFALIGN(ist->resample_width,  32);
-    encode_frames->height    = FFALIGN(ist->resample_height, 32);
-    encode_frames->format    = AV_PIX_FMT_QSV;
-    encode_frames->sw_format = AV_PIX_FMT_NV12;
-    encode_frames->initial_pool_size = 1;
-
-    qsv_frames->frame_type = MFX_MEMTYPE_VIDEO_MEMORY_DECODER_TARGET;
-
-    err = av_hwframe_ctx_init(encode_frames_ref);
-    if (err < 0)
-        goto fail;
-
-    ist->dec_ctx->pix_fmt       = AV_PIX_FMT_QSV;
-    ist->resample_pix_fmt       = AV_PIX_FMT_QSV;
-
-    ost->enc_ctx->pix_fmt       = AV_PIX_FMT_QSV;
-    ost->enc_ctx->hw_frames_ctx = encode_frames_ref;
-
-    return 0;
-
-fail:
-    av_buffer_unref(&encode_frames_ref);
-    return err;
-}

tests/ref/fate/exr-rgb-scanline-pxr24-half-uint32-13x9

@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 13x9
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      936, 0x7b44246e

tests/ref/fate/exr-rgba-scanline-float-half-b44-12x8-l1

@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 12x8
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      768, 0x1de5c7f1

tests/ref/fate/exr-rgba-scanline-float-half-b44-12x8-l2

@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 12x8
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      768, 0xe08ca6d3

tests/ref/fate/exr-rgba-scanline-float-half-b44-13x9-l1

@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 13x9
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      936, 0xdcb42186

tests/ref/fate/exr-rgba-scanline-float-half-b44-13x9-l2

@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 13x9
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      936, 0x7f710bf5

tests/ref/fate/exr-rgba-scanline-float-half-b44a-12x8-l1

@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 12x8
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      768, 0xe200c160

tests/ref/fate/exr-rgba-scanline-float-half-b44a-12x8-l2

@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 12x8
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      768, 0xe08ca6d3

tests/ref/fate/exr-rgba-scanline-float-half-b44a-13x9-l1

@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 13x9
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      936, 0x911718ac

tests/ref/fate/exr-rgba-scanline-float-half-b44a-13x9-l2

@@ -2,5 +2,5 @@
 #media_type 0: video
 #codec_id 0: rawvideo
 #dimensions 0: 13x9
-#sar 0: 0/1
+#sar 0: 9/10
 0,          0,          0,        1,      936, 0x7f710bf5

tests/ref/fate/vp8-size-change

@@ -4,36 +4,36 @@
 #tb 0: 1/30
 #media_type 0: video
 #codec_id 0: rawvideo
-#dimensions 0: 1920x1080
+#dimensions 0: 160x90
 #sar 0: 1/1
 #stream#, dts,        pts, duration,     size, hash
-0,          0,          0,        1,  3110400, 7dde8cd136ab4b04a95d9856b941697e
-0,          1,          1,        1,  3110400, aa885f78cb6374b5bfcc66a4fc57026f
-0,          2,          2,        1,  3110400, b69b7b56f549a3f9b0a603940bac85ed
-0,          3,          3,        1,  3110400, 20e2e0f0c89c58828b6a3b10d9e175e5
-0,          4,          4,        1,  3110400, 483997936e7d6bb849e64d50426ec689
-0,          5,          5,        1,  3110400, c85ef97a853ff594e2bfdf0a0a581dcc
-0,          6,          6,        1,  3110400, c5e7b9ff4c25391793446da788cb83a9
-0,          7,          7,        1,  3110400, 63f93e89d24162e2f6328edbc6716b33
-0,          8,          8,        1,  3110400, 0e346ab4831ce8c69001153c72b7b827
-0,          9,          9,        1,  3110400, c526c21511d8bec2659d3d43d93734f2
-0,         10,         10,        1,  3110400, e95d01d5f9fb81a98bd34305c7ab30f8
-0,         11,         11,        1,  3110400, 177e75e7516e8746d31b43ea9d39e6b1
-0,         12,         12,        1,  3110400, 489d2bc0da93f118dc9a2697275697a7
-0,         13,         13,        1,  3110400, a2dc00d627350ff1ab302bcbad5ca5ac
-0,         14,         14,        1,  3110400, 20ce143831b0189f763ee5bee9c51188
-0,         15,         15,        1,  3110400, 7822fd908bd81b521c23fa4a639caf9e
-0,         16,         16,        1,  3110400, dabc4febbe09734126ac6f5a5180ba8c
-0,         17,         17,        1,  3110400, ef88f0d6667feefac1471b065208e1c8
-0,         18,         18,        1,  3110400, 7c7fc665a6fd9e19af9358bbdc162a51
-0,         19,         19,        1,  3110400, f2bcf32f734f99506bdd0a0376badf82
-0,         20,         20,        1,  3110400, 06809c2d277fd3b3918ebb4b65c27661
-0,         21,         21,        1,  3110400, e403e9e86fa5d519f65c565b3add84b5
-0,         22,         22,        1,  3110400, d2b876730e12245cacb578307794349a
-0,         23,         23,        1,  3110400, dfdfd8cb626a96138f6a2c1953dcf5ec
-0,         24,         24,        1,  3110400, 0ac58c28575b804d9e63395653c3aef2
-0,         25,         25,        1,  3110400, 641f2a78e338c733ef159bd36ec7966f
-0,         26,         26,        1,  3110400, 9402d455fa5bd556b85f479c42c3a4d2
-0,         27,         27,        1,  3110400, 0044d42b4048bc93112aa59789dbdc2d
-0,         28,         28,        1,  3110400, 5d9e5c5ba35f6f452e5f31ccff9e819c
-0,         29,         29,        1,  3110400, 307a55a94739b4cfdf41f7da7e5c0135
+0,          0,          0,        1,    21600, 5abd6c163522c7e882f7e9c369293bf9
+0,          1,          1,        1,    21600, 5c11d8cc9cc9102d0ef5afd1dc64aff1
+0,          2,          2,        1,    21600, cbeffa9ea9d682af77d3fd0fdf12c8c1
+0,          3,          3,        1,    21600, ea7cec515fcf8ccbc932d9e9b472cdc7
+0,          4,          4,        1,    21600, 23038b711dbac95ce710657b1fef5901
+0,          5,          5,        1,    21600, e0d6fb46bb5c0d939ee33af82b887668
+0,          6,          6,        1,    21600, 694518f14d3a2bd3c319bc0b098c78bb
+0,          7,          7,        1,    21600, c1c7394bd4236afbc773af52ef7a10ea
+0,          8,          8,        1,    21600, 4d8d3b2c9a637f963521585ea879357b
+0,          9,          9,        1,    21600, b4444dc3cbf1b6cdd8047d3dcd497ffd
+0,         10,         10,        1,    21600, 65e5d667ec9ceb636e21357f032ce800
+0,         11,         11,        1,    21600, fd9a4c67598051074387b640df7edaa9
+0,         12,         12,        1,    21600, 0e54e22d90f6296ae6989c83846272cd
+0,         13,         13,        1,    21600, db4b1727450243b202bfec5ed6c73ae0
+0,         14,         14,        1,    21600, ab37a84be075ca42cc7351ff9fb1cb47
+0,         15,         15,        1,    21600, ae4d2d297e646bd8e05e76b457d9b576
+0,         16,         16,        1,    21600, e7cfd580e3c3d7c3f2f5136d1e548595
+0,         17,         17,        1,    21600, cbec09314a0b7ad53f4893eb474e1c65
+0,         18,         18,        1,    21600, e1fa89cd63c37496bc86f18694324d88
+0,         19,         19,        1,    21600, e9655b151253950313810228278ca104
+0,         20,         20,        1,    21600, 69ba31c0eff7bc93f4180173d8e64c60
+0,         21,         21,        1,    21600, 368a1f6a1172d7d56f695153b234a330
+0,         22,         22,        1,    21600, 6c298b196e16c64f7c2f407ba1242937
+0,         23,         23,        1,    21600, bf54474112ed5592c4d890e3313881a0
+0,         24,         24,        1,    21600, 945d49abedb0606b6a009c8b5d8face3
+0,         25,         25,        1,    21600, dd6ebef7b6f24619910de811918d3437
+0,         26,         26,        1,    21600, 7952346fc0f1eff3914e0d7646b3cf28
+0,         27,         27,        1,    21600, 26bd0d6b21e8a2df17af8d1446fba745
+0,         28,         28,        1,    21600, b0d91600416716d81c1f73ac141a0b62
+0,         29,         29,        1,    21600, 08f16698beb9cc15f7115961bd69e995

tests/ref/lavf/ffm

-a0e9616f0d9a8c1029f3220b1b9175f4 *./tests/data/lavf/lavf.ffm
+ca2a450cd0d1e299514a345923b4c82a *./tests/data/lavf/lavf.ffm
 376832 ./tests/data/lavf/lavf.ffm
 ./tests/data/lavf/lavf.ffm CRC=0x000e23ae

tests/ref/lavf/mxf

-f9b570c7b4fbbc2b71f2236b32e7cbb6 *./tests/data/lavf/lavf.mxf
+c61e677b4facc407ca82b1d7c6298484 *./tests/data/lavf/lavf.mxf
 525369 ./tests/data/lavf/lavf.mxf
 ./tests/data/lavf/lavf.mxf CRC=0xdbfff6f1
-8f6a9a6b409f0f5a0bf003f8dea26314 *./tests/data/lavf/lavf.mxf
+1ae017b5879992431614b1c7ab3b48c5 *./tests/data/lavf/lavf.mxf
 560697 ./tests/data/lavf/lavf.mxf
 ./tests/data/lavf/lavf.mxf CRC=0x11a6178e
-10ac0f158fc0af356439b818de7601e3 *./tests/data/lavf/lavf.mxf
+27fac91dce279630a04b94b93518a9d8 *./tests/data/lavf/lavf.mxf
 525369 ./tests/data/lavf/lavf.mxf
 ./tests/data/lavf/lavf.mxf CRC=0xdbfff6f1