lavfi: add error handling to start_frame().

libavfilter/avfilter.h

@@ -258,8 +258,11 @@ struct AVFilterPad {
      * picture inside the link structure.
      *
      * Input video pads only.
+     *
+     * @return >= 0 on success, a negative AVERROR on error. picref will be
+     * unreferenced by the caller in case of error.
      */
-    void (*start_frame)(AVFilterLink *link, AVFilterBufferRef *picref);
+    int (*start_frame)(AVFilterLink *link, AVFilterBufferRef *picref);
 
     /**
      * Callback function to get a video buffer. If NULL, the filter system will

libavfilter/buffersink.c

@@ -47,20 +47,16 @@ static av_cold void uninit(AVFilterContext *ctx)
         av_audio_fifo_free(sink->audio_fifo);
 }
 
-static void start_frame(AVFilterLink *link, AVFilterBufferRef *buf)
+static int start_frame(AVFilterLink *link, AVFilterBufferRef *buf)
 {
     BufferSinkContext *s = link->dst->priv;
 
     av_assert0(!s->cur_buf);
     s->cur_buf    = buf;
     link->cur_buf = NULL;
-};
 
-static int filter_samples(AVFilterLink *link, AVFilterBufferRef *buf)
-{
-    start_frame(link, buf);
     return 0;
-}
+};
 
 int av_buffersink_read(AVFilterContext *ctx, AVFilterBufferRef **buf)
 {
@@ -166,7 +162,7 @@ AVFilter avfilter_asink_abuffer = {
 
     .inputs    = (const AVFilterPad[]) {{ .name           = "default",
                                           .type           = AVMEDIA_TYPE_AUDIO,
-                                          .filter_samples = filter_samples,
+                                          .filter_samples = start_frame,
                                           .min_perms      = AV_PERM_READ,
                                           .needs_fifo     = 1 },
                                         { .name = NULL }},

libavfilter/fifo.c

@@ -76,6 +76,7 @@ static int add_to_queue(AVFilterLink *inlink, AVFilterBufferRef *buf)
 {
     FifoContext *fifo = inlink->dst->priv;
 
+    inlink->cur_buf = NULL;
     fifo->last->next = av_mallocz(sizeof(Buf));
     if (!fifo->last->next) {
         avfilter_unref_buffer(buf);
@@ -88,12 +89,6 @@ static int add_to_queue(AVFilterLink *inlink, AVFilterBufferRef *buf)
     return 0;
 }
 
-static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *buf)
-{
-    add_to_queue(inlink, buf);
-    inlink->cur_buf = NULL;
-}
-
 static void queue_pop(FifoContext *s)
 {
     Buf *tmp = s->root.next->next;
@@ -272,7 +267,7 @@ AVFilter avfilter_vf_fifo = {
     .inputs    = (const AVFilterPad[]) {{ .name            = "default",
                                           .type            = AVMEDIA_TYPE_VIDEO,
                                           .get_video_buffer= ff_null_get_video_buffer,
-                                          .start_frame     = start_frame,
+                                          .start_frame     = add_to_queue,
                                           .draw_slice      = draw_slice,
                                           .end_frame       = end_frame,
                                           .rej_perms       = AV_PERM_REUSE2, },

libavfilter/internal.h

@@ -69,6 +69,9 @@ struct AVFilterPad {
      * picture inside the link structure.
      *
      * Input video pads only.
+     *
+     * @return >= 0 on success, a negative AVERROR on error. picref will be
+     * unreferenced by the caller in case of error.
      */
     void (*start_frame)(AVFilterLink *link, AVFilterBufferRef *picref);
 

libavfilter/split.c

@@ -63,14 +63,18 @@ static void split_uninit(AVFilterContext *ctx)
         av_freep(&ctx->output_pads[i].name);
 }
 
-static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
+static int start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
 {
     AVFilterContext *ctx = inlink->dst;
-    int i;
+    int i, ret = 0;
 
-    for (i = 0; i < ctx->nb_outputs; i++)
-        ff_start_frame(ctx->outputs[i],
-                       avfilter_ref_buffer(picref, ~AV_PERM_WRITE));
+    for (i = 0; i < ctx->nb_outputs; i++) {
+        ret = ff_start_frame(ctx->outputs[i],
+                             avfilter_ref_buffer(picref, ~AV_PERM_WRITE));
+        if (ret < 0)
+            break;
+    }
+    return ret;
 }
 
 static void draw_slice(AVFilterLink *inlink, int y, int h, int slice_dir)

libavfilter/vf_aspect.c

@@ -64,13 +64,13 @@ static av_cold int init(AVFilterContext *ctx, const char *args)
     return 0;
 }
 
-static void start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
+static int start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
 {
     AspectContext *aspect = link->dst->priv;
 
     picref->video->pixel_aspect = aspect->aspect;
     link->cur_buf = NULL;
-    ff_start_frame(link->dst->outputs[0], picref);
+    return ff_start_frame(link->dst->outputs[0], picref);
 }
 
 #if CONFIG_SETDAR_FILTER

libavfilter/vf_crop.c

@@ -240,7 +240,7 @@ static int config_output(AVFilterLink *link)
     return 0;
 }
 
-static void start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
+static int start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
 {
     AVFilterContext *ctx = link->dst;
     CropContext *crop = ctx->priv;
@@ -248,6 +248,9 @@ static void start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
     int i;
 
     ref2 = avfilter_ref_buffer(picref, ~0);
+    if (!ref2)
+        return AVERROR(ENOMEM);
+
     ref2->video->w = crop->w;
     ref2->video->h = crop->h;
 
@@ -291,7 +294,7 @@ static void start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
         ref2->data[3] += crop->x * crop->max_step[3];
     }
 
-    ff_start_frame(link->dst->outputs[0], ref2);
+    return ff_start_frame(link->dst->outputs[0], ref2);
 }
 
 static void draw_slice(AVFilterLink *link, int y, int h, int slice_dir)

libavfilter/vf_delogo.c

@@ -214,22 +214,35 @@ static av_cold int init(AVFilterContext *ctx, const char *args)
     return 0;
 }
 
-static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
+static int start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
 {
     AVFilterLink *outlink = inlink->dst->outputs[0];
-    AVFilterBufferRef *outpicref;
+    AVFilterBufferRef *outpicref = NULL;
+    int ret = 0;
 
     if (inpicref->perms & AV_PERM_PRESERVE) {
         outpicref = ff_get_video_buffer(outlink, AV_PERM_WRITE,
                                         outlink->w, outlink->h);
+        if (!outpicref)
+            return AVERROR(ENOMEM);
+
         avfilter_copy_buffer_ref_props(outpicref, inpicref);
         outpicref->video->w = outlink->w;
         outpicref->video->h = outlink->h;
-    } else
+    } else {
         outpicref = avfilter_ref_buffer(inpicref, ~0);
+        if (!outpicref)
+            return AVERROR(ENOMEM);
+    }
+
+    ret = ff_start_frame(outlink, avfilter_ref_buffer(outpicref, ~0));
+    if (ret < 0) {
+        avfilter_unref_bufferp(&outpicref);
+        return ret;
+    }
 
     outlink->out_buf = outpicref;
-    ff_start_frame(outlink, avfilter_ref_buffer(outpicref, ~0));
+    return 0;
 }
 
 static void null_draw_slice(AVFilterLink *link, int y, int h, int slice_dir) { }

libavfilter/vf_drawtext.c

@@ -808,16 +808,16 @@ static inline int normalize_double(int *n, double d)
     return ret;
 }
 
-static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
+static int start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
 {
     AVFilterContext *ctx = inlink->dst;
     DrawTextContext *dtext = ctx->priv;
     AVFilterBufferRef *buf_out;
-    int fail = 0;
+    int ret = 0;
 
-    if (dtext_prepare_text(ctx) < 0) {
+    if ((ret = dtext_prepare_text(ctx)) < 0) {
         av_log(ctx, AV_LOG_ERROR, "Can't draw text\n");
-        fail = 1;
+        return ret;
     }
 
     dtext->var_values[VAR_T] = inpicref->pts == AV_NOPTS_VALUE ?
@@ -829,8 +829,7 @@ static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
     dtext->var_values[VAR_X] =
         av_expr_eval(dtext->x_pexpr, dtext->var_values, &dtext->prng);
 
-    dtext->draw = fail ? 0 :
-        av_expr_eval(dtext->d_pexpr, dtext->var_values, &dtext->prng);
+    dtext->draw = av_expr_eval(dtext->d_pexpr, dtext->var_values, &dtext->prng);
 
     normalize_double(&dtext->x, dtext->var_values[VAR_X]);
     normalize_double(&dtext->y, dtext->var_values[VAR_Y]);
@@ -852,7 +851,10 @@ static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
             dtext->x, dtext->y, dtext->x+dtext->w, dtext->y+dtext->h);
 
     buf_out = avfilter_ref_buffer(inpicref, ~0);
-    ff_start_frame(inlink->dst->outputs[0], buf_out);
+    if (!buf_out)
+        return AVERROR(ENOMEM);
+
+    return ff_start_frame(inlink->dst->outputs[0], buf_out);
 }
 
 static void end_frame(AVFilterLink *inlink)

libavfilter/vf_fieldorder.c

@@ -116,18 +116,32 @@ static AVFilterBufferRef *get_video_buffer(AVFilterLink *inlink, int perms, int
     return ff_get_video_buffer(outlink, perms, w, h);
 }
 
-static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
+static int start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
 {
     AVFilterContext   *ctx        = inlink->dst;
     AVFilterLink      *outlink    = ctx->outputs[0];
 
     AVFilterBufferRef *outpicref, *for_next_filter;
+    int ret = 0;
 
     outpicref = avfilter_ref_buffer(inpicref, ~0);
-    outlink->out_buf = outpicref;
+    if (!outpicref)
+        return AVERROR(ENOMEM);
 
     for_next_filter = avfilter_ref_buffer(outpicref, ~0);
-    ff_start_frame(outlink, for_next_filter);
+    if (!for_next_filter) {
+        avfilter_unref_bufferp(&outpicref);
+        return AVERROR(ENOMEM);
+    }
+
+    ret = ff_start_frame(outlink, for_next_filter);
+    if (ret < 0) {
+        avfilter_unref_bufferp(&outpicref);
+        return ret;
+    }
+
+    outlink->out_buf = outpicref;
+    return 0;
 }
 
 static void draw_slice(AVFilterLink *inlink, int y, int h, int slice_dir)

libavfilter/vf_fps.c

@@ -243,8 +243,9 @@ static void end_frame(AVFilterLink *inlink)
     s->pts = s->first_pts + av_rescale_q(s->frames_out, outlink->time_base, inlink->time_base);
 }
 
-static void null_start_frame(AVFilterLink *link, AVFilterBufferRef *buf)
+static int null_start_frame(AVFilterLink *link, AVFilterBufferRef *buf)
 {
+    return 0;
 }
 
 static void null_draw_slice(AVFilterLink *link, int y, int h, int slice_dir)

libavfilter/vf_gradfun.c

@@ -180,21 +180,34 @@ static int config_input(AVFilterLink *inlink)
     return 0;
 }
 
-static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
+static int start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
 {
     AVFilterLink *outlink = inlink->dst->outputs[0];
-    AVFilterBufferRef *outpicref;
+    AVFilterBufferRef *outpicref = NULL;
+    int ret = 0;
 
     if (inpicref->perms & AV_PERM_PRESERVE) {
         outpicref = ff_get_video_buffer(outlink, AV_PERM_WRITE, outlink->w, outlink->h);
+        if (!outpicref)
+            return AVERROR(ENOMEM);
+
         avfilter_copy_buffer_ref_props(outpicref, inpicref);
         outpicref->video->w = outlink->w;
         outpicref->video->h = outlink->h;
-    } else
+    } else {
         outpicref = avfilter_ref_buffer(inpicref, ~0);
+        if (!outpicref)
+            return AVERROR(ENOMEM);
+    }
+
+    ret = ff_start_frame(outlink, avfilter_ref_buffer(outpicref, ~0));
+    if (ret < 0) {
+        avfilter_unref_bufferp(&outpicref);
+        return ret;
+    }
 
     outlink->out_buf = outpicref;
-    ff_start_frame(outlink, avfilter_ref_buffer(outpicref, ~0));
+    return 0;
 }
 
 static void null_draw_slice(AVFilterLink *link, int y, int h, int slice_dir) { }

libavfilter/vf_overlay.c

@@ -208,7 +208,7 @@ static AVFilterBufferRef *get_video_buffer(AVFilterLink *link, int perms, int w,
     return ff_get_video_buffer(link->dst->outputs[0], perms, w, h);
 }
 
-static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
+static int start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
 {
     AVFilterBufferRef *outpicref = avfilter_ref_buffer(inpicref, ~0);
     AVFilterContext *ctx = inlink->dst;
@@ -228,10 +228,10 @@ static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
             over->overpicref = old;
     }
 
-    ff_start_frame(inlink->dst->outputs[0], outpicref);
+    return ff_start_frame(inlink->dst->outputs[0], outpicref);
 }
 
-static void start_frame_overlay(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
+static int start_frame_overlay(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
 {
     AVFilterContext *ctx = inlink->dst;
     OverlayContext *over = ctx->priv;
@@ -239,6 +239,7 @@ static void start_frame_overlay(AVFilterLink *inlink, AVFilterBufferRef *inpicre
     over->overpicref = inpicref;
     over->overpicref->pts = av_rescale_q(inpicref->pts, ctx->inputs[OVERLAY]->time_base,
                                          ctx->outputs[0]->time_base);
+    return 0;
 }
 
 static void blend_slice(AVFilterContext *ctx,

libavfilter/vf_pad.c

@@ -299,12 +299,15 @@ static int does_clip(PadContext *pad, AVFilterBufferRef *outpicref, int plane, i
     return 0;
 }
 
-static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
+static int start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
 {
     PadContext *pad = inlink->dst->priv;
     AVFilterBufferRef *outpicref = avfilter_ref_buffer(inpicref, ~0);
     AVFilterBufferRef *for_next_filter;
-    int plane;
+    int plane, ret = 0;
+
+    if (!outpicref)
+        return AVERROR(ENOMEM);
 
     for (plane = 0; plane < 4 && outpicref->data[plane]; plane++) {
         int hsub = (plane == 1 || plane == 2) ? pad->hsub : 0;
@@ -332,16 +335,31 @@ static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
         outpicref = ff_get_video_buffer(inlink->dst->outputs[0], AV_PERM_WRITE | AV_PERM_NEG_LINESIZES,
                                         FFMAX(inlink->w, pad->w),
                                         FFMAX(inlink->h, pad->h));
+        if (!outpicref)
+            return AVERROR(ENOMEM);
+
         avfilter_copy_buffer_ref_props(outpicref, inpicref);
     }
 
-    inlink->dst->outputs[0]->out_buf = outpicref;
-
     outpicref->video->w = pad->w;
     outpicref->video->h = pad->h;
 
     for_next_filter = avfilter_ref_buffer(outpicref, ~0);
-    ff_start_frame(inlink->dst->outputs[0], for_next_filter);
+    if (!for_next_filter) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+
+    ret = ff_start_frame(inlink->dst->outputs[0], for_next_filter);
+    if (ret < 0)
+        goto fail;
+
+    inlink->dst->outputs[0]->out_buf = outpicref;
+    return 0;
+
+fail:
+    avfilter_unref_bufferp(&outpicref);
+    return ret;
 }
 
 static void end_frame(AVFilterLink *link)

libavfilter/vf_pixdesctest.c

@@ -51,16 +51,18 @@ static int config_props(AVFilterLink *inlink)
     return 0;
 }
 
-static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
+static int start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
 {
     PixdescTestContext *priv = inlink->dst->priv;
     AVFilterLink *outlink    = inlink->dst->outputs[0];
     AVFilterBufferRef *outpicref;
-    int i;
+    int i, ret = 0;
+
+    outpicref = ff_get_video_buffer(outlink, AV_PERM_WRITE,
+                                    outlink->w, outlink->h);
+    if (!outpicref)
+        return AVERROR(ENOMEM);
 
-    outlink->out_buf = ff_get_video_buffer(outlink, AV_PERM_WRITE,
-                                           outlink->w, outlink->h);
-    outpicref = outlink->out_buf;
     avfilter_copy_buffer_ref_props(outpicref, picref);
 
     for (i = 0; i < 4; i++) {
@@ -78,7 +80,14 @@ static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
         priv->pix_desc->flags & PIX_FMT_PSEUDOPAL)
         memcpy(outpicref->data[1], outpicref->data[1], 256*4);
 
-    ff_start_frame(outlink, avfilter_ref_buffer(outpicref, ~0));
+    ret = ff_start_frame(outlink, avfilter_ref_buffer(outpicref, ~0));
+    if (ret < 0) {
+        avfilter_unref_bufferp(&outpicref);
+        return ret;
+    }
+
+    outlink->out_buf = outpicref;
+    return 0;
 }
 
 static void draw_slice(AVFilterLink *inlink, int y, int h, int slice_dir)

libavfilter/vf_scale.c

@@ -252,26 +252,28 @@ fail:
     return ret;
 }
 
-static void start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
+static int start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
 {
     ScaleContext *scale = link->dst->priv;
     AVFilterLink *outlink = link->dst->outputs[0];
     AVFilterBufferRef *outpicref;
+    int ret = 0;
 
     if (!scale->sws) {
-        ff_start_frame(outlink, avfilter_ref_buffer(picref, ~0));
-        return;
+        return ff_start_frame(outlink, avfilter_ref_buffer(picref, ~0));
     }
 
     scale->hsub = av_pix_fmt_descriptors[link->format].log2_chroma_w;
     scale->vsub = av_pix_fmt_descriptors[link->format].log2_chroma_h;
 
     outpicref = ff_get_video_buffer(outlink, AV_PERM_WRITE, outlink->w, outlink->h);
+    if (!outpicref)
+        return AVERROR(ENOMEM);
+
     avfilter_copy_buffer_ref_props(outpicref, picref);
     outpicref->video->w = outlink->w;
     outpicref->video->h = outlink->h;
 
-    outlink->out_buf = outpicref;
 
     av_reduce(&outpicref->video->pixel_aspect.num, &outpicref->video->pixel_aspect.den,
               (int64_t)picref->video->pixel_aspect.num * outlink->h * link->w,
@@ -279,7 +281,14 @@ static void start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
               INT_MAX);
 
     scale->slice_y = 0;
-    ff_start_frame(outlink, avfilter_ref_buffer(outpicref, ~0));
+    ret = ff_start_frame(outlink, avfilter_ref_buffer(outpicref, ~0));
+    if (ret < 0) {
+        avfilter_unref_bufferp(&outpicref);
+        return ret;
+    }
+
+    outlink->out_buf = outpicref;
+    return 0;
 }
 
 static void draw_slice(AVFilterLink *link, int y, int h, int slice_dir)

libavfilter/vf_select.c

@@ -227,7 +227,7 @@ static int select_frame(AVFilterContext *ctx, AVFilterBufferRef *picref)
     return res;
 }
 
-static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
+static int start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
 {
     SelectContext *select = inlink->dst->priv;
 
@@ -241,10 +241,12 @@ static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
             else
                 av_fifo_generic_write(select->pending_frames, &picref,
                                       sizeof(picref), NULL);
-            return;
+            return 0;
         }
-        ff_start_frame(inlink->dst->outputs[0], avfilter_ref_buffer(picref, ~0));
+        return ff_start_frame(inlink->dst->outputs[0], avfilter_ref_buffer(picref, ~0));
     }
+
+    return 0;
 }
 
 static void draw_slice(AVFilterLink *inlink, int y, int h, int slice_dir)

libavfilter/vf_setpts.c

@@ -101,12 +101,15 @@ static int config_input(AVFilterLink *inlink)
 #define D2TS(d)  (isnan(d) ? AV_NOPTS_VALUE : (int64_t)(d))
 #define TS2D(ts) ((ts) == AV_NOPTS_VALUE ? NAN : (double)(ts))
 
-static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
+static int start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
 {
     SetPTSContext *setpts = inlink->dst->priv;
     double d;
     AVFilterBufferRef *outpicref = avfilter_ref_buffer(inpicref, ~0);
 
+    if (!outpicref)
+        return AVERROR(ENOMEM);
+
     if (isnan(setpts->var_values[VAR_STARTPTS]))
         setpts->var_values[VAR_STARTPTS] = TS2D(inpicref->pts);
 
@@ -130,7 +133,7 @@ static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *inpicref)
     setpts->var_values[VAR_N] += 1.0;
     setpts->var_values[VAR_PREV_INPTS ] = TS2D(inpicref ->pts);
     setpts->var_values[VAR_PREV_OUTPTS] = TS2D(outpicref->pts);
-    ff_start_frame(inlink->dst->outputs[0], outpicref);
+    return ff_start_frame(inlink->dst->outputs[0], outpicref);
 }
 
 static av_cold void uninit(AVFilterContext *ctx)

libavfilter/vf_settb.c

@@ -104,7 +104,7 @@ static int config_output_props(AVFilterLink *outlink)
     return 0;
 }
 
-static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
+static int start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
 {
     AVFilterContext *ctx = inlink->dst;
     AVFilterLink *outlink = ctx->outputs[0];
@@ -118,7 +118,7 @@ static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
     }
     inlink->cur_buf = NULL;
 
-    ff_start_frame(outlink, picref);
+    return ff_start_frame(outlink, picref);
 }
 
 AVFilter avfilter_vf_settb = {

libavfilter/vf_slicify.c

@@ -59,7 +59,7 @@ static int config_props(AVFilterLink *link)
     return 0;
 }
 
-static void start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
+static int start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
 {
     SliceContext *slice = link->dst->priv;
 
@@ -75,7 +75,7 @@ static void start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
     av_log(link->dst, AV_LOG_DEBUG, "h:%d\n", slice->h);
     link->cur_buf = NULL;
 
-    ff_start_frame(link->dst->outputs[0], picref);
+    return ff_start_frame(link->dst->outputs[0], picref);
 }
 
 static void draw_slice(AVFilterLink *link, int y, int h, int slice_dir)

libavfilter/vf_transpose.c

@@ -117,12 +117,15 @@ static int config_props_output(AVFilterLink *outlink)
     return 0;
 }
 
-static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
+static int start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
 {
     AVFilterLink *outlink = inlink->dst->outputs[0];
 
     outlink->out_buf = ff_get_video_buffer(outlink, AV_PERM_WRITE,
                                            outlink->w, outlink->h);
+    if (!outlink->out_buf)
+        return AVERROR(ENOMEM);
+
     outlink->out_buf->pts = picref->pts;
 
     if (picref->video->pixel_aspect.num == 0) {
@@ -132,7 +135,7 @@ static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
         outlink->out_buf->video->pixel_aspect.den = picref->video->pixel_aspect.num;
     }
 
-    ff_start_frame(outlink, avfilter_ref_buffer(outlink->out_buf, ~0));
+    return ff_start_frame(outlink, avfilter_ref_buffer(outlink->out_buf, ~0));
 }
 
 static void end_frame(AVFilterLink *inlink)

libavfilter/vf_vflip.c

@@ -64,7 +64,7 @@ static AVFilterBufferRef *get_video_buffer(AVFilterLink *link, int perms,
     return picref;
 }
 
-static void start_frame(AVFilterLink *link, AVFilterBufferRef *inpicref)
+static int start_frame(AVFilterLink *link, AVFilterBufferRef *inpicref)
 {
     FlipContext *flip = link->dst->priv;
     AVFilterBufferRef *outpicref = avfilter_ref_buffer(inpicref, ~0);
@@ -79,7 +79,7 @@ static void start_frame(AVFilterLink *link, AVFilterBufferRef *inpicref)
         }
     }
 
-    ff_start_frame(link->dst->outputs[0], outpicref);
+    return ff_start_frame(link->dst->outputs[0], outpicref);
 }
 
 static void draw_slice(AVFilterLink *link, int y, int h, int slice_dir)

libavfilter/vf_yadif.c

@@ -201,7 +201,7 @@ static void return_frame(AVFilterContext *ctx, int is_second)
     yadif->frame_pending = (yadif->mode&1) && !is_second;
 }
 
-static void start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
+static int start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
 {
     AVFilterContext *ctx = link->dst;
     YADIFContext *yadif = ctx->priv;
@@ -216,7 +216,7 @@ static void start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
     yadif->next = picref;
 
     if (!yadif->cur)
-        return;
+        return 0;
 
     if (yadif->auto_enable && !yadif->cur->video->interlaced) {
         yadif->out  = avfilter_ref_buffer(yadif->cur, AV_PERM_READ);
@@ -224,8 +224,7 @@ static void start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
         yadif->prev = NULL;
         if (yadif->out->pts != AV_NOPTS_VALUE)
             yadif->out->pts *= 2;
-        ff_start_frame(ctx->outputs[0], yadif->out);
-        return;
+        return ff_start_frame(ctx->outputs[0], yadif->out);
     }
 
     if (!yadif->prev)
@@ -238,7 +237,7 @@ static void start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
     yadif->out->video->interlaced = 0;
     if (yadif->out->pts != AV_NOPTS_VALUE)
         yadif->out->pts *= 2;
-    ff_start_frame(ctx->outputs[0], yadif->out);
+    return ff_start_frame(ctx->outputs[0], yadif->out);
 }
 
 static void end_frame(AVFilterLink *link)

libavfilter/video.c

@@ -160,13 +160,15 @@ AVFilterBufferRef *ff_get_video_buffer(AVFilterLink *link, int perms, int w, int
     return ret;
 }
 
-void ff_null_start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
+int ff_null_start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
 {
     AVFilterBufferRef *buf_out = avfilter_ref_buffer(picref, ~0);
-    ff_start_frame(link->dst->outputs[0], buf_out);
+    if (!buf_out)
+        return AVERROR(ENOMEM);
+    return ff_start_frame(link->dst->outputs[0], buf_out);
 }
 
-static void default_start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
+static int default_start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
 {
     AVFilterLink *outlink = NULL;
 
@@ -175,18 +177,29 @@ static void default_start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
 
     if (outlink) {
         outlink->out_buf = ff_get_video_buffer(outlink, AV_PERM_WRITE, outlink->w, outlink->h);
+        if (!outlink->out_buf)
+            return AVERROR(ENOMEM);
+
         avfilter_copy_buffer_ref_props(outlink->out_buf, picref);
-        ff_start_frame(outlink, avfilter_ref_buffer(outlink->out_buf, ~0));
+        return ff_start_frame(outlink, avfilter_ref_buffer(outlink->out_buf, ~0));
     }
+    return 0;
+}
+
+static void clear_link(AVFilterLink *link)
+{
+    avfilter_unref_bufferp(&link->cur_buf);
+    avfilter_unref_bufferp(&link->src_buf);
+    avfilter_unref_bufferp(&link->out_buf);
 }
 
 /* XXX: should we do the duplicating of the picture ref here, instead of
  * forcing the source filter to do it? */
-void ff_start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
+int ff_start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
 {
-    void (*start_frame)(AVFilterLink *, AVFilterBufferRef *);
+    int (*start_frame)(AVFilterLink *, AVFilterBufferRef *);
     AVFilterPad *dst = link->dstpad;
-    int perms = picref->perms;
+    int ret, perms = picref->perms;
 
     FF_DPRINTF_START(NULL, start_frame); ff_dlog_link(NULL, link, 0); av_dlog(NULL, " "); ff_dlog_ref(NULL, picref, 1);
 
@@ -203,13 +216,22 @@ void ff_start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
                 link->dstpad->min_perms, link->dstpad->rej_perms);
 
         link->cur_buf = ff_get_video_buffer(link, dst->min_perms, link->w, link->h);
+        if (!link->cur_buf) {
+            avfilter_unref_bufferp(&picref);
+            return AVERROR(ENOMEM);
+        }
+
         link->src_buf = picref;
         avfilter_copy_buffer_ref_props(link->cur_buf, link->src_buf);
     }
     else
         link->cur_buf = picref;
 
-    start_frame(link, link->cur_buf);
+    ret = start_frame(link, link->cur_buf);
+    if (ret < 0)
+        clear_link(link);
+
+    return ret;
 }
 
 void ff_null_end_frame(AVFilterLink *link)
@@ -238,14 +260,7 @@ void ff_end_frame(AVFilterLink *link)
 
     end_frame(link);
 
-    /* unreference the source picture if we're feeding the destination filter
-     * a copied version dues to permission issues */
-    if (link->src_buf) {
-        avfilter_unref_buffer(link->src_buf);
-        link->src_buf = NULL;
-    }
-    avfilter_unref_bufferp(&link->cur_buf);
-    avfilter_unref_bufferp(&link->out_buf);
+    clear_link(link);
 }
 
 void ff_null_draw_slice(AVFilterLink *link, int y, int h, int slice_dir)

libavfilter/video.h

@@ -39,7 +39,7 @@ AVFilterBufferRef *ff_null_get_video_buffer(AVFilterLink *link, int perms, int w
 AVFilterBufferRef *ff_get_video_buffer(AVFilterLink *link, int perms,
                                        int w, int h);
 
-void ff_null_start_frame(AVFilterLink *link, AVFilterBufferRef *picref);
+int ff_null_start_frame(AVFilterLink *link, AVFilterBufferRef *picref);
 void ff_null_draw_slice(AVFilterLink *link, int y, int h, int slice_dir);
 void ff_null_end_frame(AVFilterLink *link);
 
@@ -51,8 +51,11 @@ void ff_null_end_frame(AVFilterLink *link);
  *               frame need only be valid once draw_slice() is called for that
  *               portion. The receiving filter will free this reference when
  *               it no longer needs it.
+ *
+ * @return >= 0 on success, a negative AVERROR on error. This function will
+ * unreference picref in case of error.
  */
-void ff_start_frame(AVFilterLink *link, AVFilterBufferRef *picref);
+int ff_start_frame(AVFilterLink *link, AVFilterBufferRef *picref);
 
 /**
  * Notify the next filter that the current frame has finished.

libavfilter/vsink_nullsink.c

@@ -19,8 +19,9 @@
 #include "avfilter.h"
 #include "internal.h"
 
-static void start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
+static int start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
 {
+    return 0;
 }
 
 static void end_frame(AVFilterLink *link)