lavfi: Always propagate hw_frames_ctx through links

Also adds a new flag to mark filters which are aware of hwframes and
will perform this task themselves, and marks all appropriate filters
with this flag.

This is required to allow software-mapped hardware frames to work,
because we need to have the frames context available for any later
mapping operation in the filter graph.

The output from the filter graph should only propagate further to an
encoder if the hardware format actually matches the visible format
(mapped frames are valid here and have an hw_frames_ctx, but this
should not be given to the encoder as its hardware context).

avconv.c

@@ -38,6 +38,7 @@
 #include "libavutil/parseutils.h"
 #include "libavutil/samplefmt.h"
 #include "libavutil/fifo.h"
+#include "libavutil/hwcontext.h"
 #include "libavutil/internal.h"
 #include "libavutil/intreadwrite.h"
 #include "libavutil/dict.h"
@@ -2035,7 +2036,9 @@ static int init_output_stream(OutputStream *ost, char *error, int error_len)
         if (!av_dict_get(ost->encoder_opts, "threads", NULL, 0))
             av_dict_set(&ost->encoder_opts, "threads", "auto", 0);
 
-        if (ost->filter && ost->filter->filter->inputs[0]->hw_frames_ctx) {
+        if (ost->filter && ost->filter->filter->inputs[0]->hw_frames_ctx &&
+            ((AVHWFramesContext*)ost->filter->filter->inputs[0]->hw_frames_ctx->data)->format ==
+            ost->filter->filter->inputs[0]->format) {
             ost->enc_ctx->hw_frames_ctx = av_buffer_ref(ost->filter->filter->inputs[0]->hw_frames_ctx);
             if (!ost->enc_ctx->hw_frames_ctx)
                 return AVERROR(ENOMEM);

libavfilter/avfilter.c

@@ -19,6 +19,7 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
+#include "libavutil/avassert.h"
 #include "libavutil/avstring.h"
 #include "libavutil/buffer.h"
 #include "libavutil/channel_layout.h"
@@ -212,14 +213,12 @@ int avfilter_config_links(AVFilterContext *filter)
             }
 
             if (link->src->nb_inputs && link->src->inputs[0]->hw_frames_ctx &&
-                !link->hw_frames_ctx) {
-                AVHWFramesContext *input_ctx = (AVHWFramesContext*)link->src->inputs[0]->hw_frames_ctx->data;
-
-                if (input_ctx->format == link->format) {
-                    link->hw_frames_ctx = av_buffer_ref(link->src->inputs[0]->hw_frames_ctx);
-                    if (!link->hw_frames_ctx)
-                        return AVERROR(ENOMEM);
-                }
+                !(link->src->filter->flags_internal & FF_FILTER_FLAG_HWFRAME_AWARE)) {
+                av_assert0(!link->hw_frames_ctx &&
+                           "should not be set by non-hwframe-aware filter");
+                link->hw_frames_ctx = av_buffer_ref(link->src->inputs[0]->hw_frames_ctx);
+                if (!link->hw_frames_ctx)
+                    return AVERROR(ENOMEM);
             }
 
             if ((config_link = link->dstpad->config_props))

libavfilter/avfilter.h

@@ -244,6 +244,8 @@ typedef struct AVFilter {
 
     int priv_size;      ///< size of private data to allocate for the filter
 
+    int flags_internal; ///< Additional flags for avfilter internal use only.
+
     /**
      * Used by the filter registration system. Must not be touched by any other
      * code.

libavfilter/internal.h

@@ -220,4 +220,10 @@ AVFilterContext *ff_filter_alloc(const AVFilter *filter, const char *inst_name);
  */
 void ff_filter_graph_remove_filter(AVFilterGraph *graph, AVFilterContext *filter);
 
+/**
+ * The filter is aware of hardware frames, and any hardware frame context
+ * should not be automatically propagated through it.
+ */
+#define FF_FILTER_FLAG_HWFRAME_AWARE (1 << 0)
+
 #endif /* AVFILTER_INTERNAL_H */

libavfilter/vf_deinterlace_qsv.c

@@ -575,4 +575,6 @@ AVFilter ff_vf_deinterlace_qsv = {
 
     .inputs    = qsvdeint_inputs,
     .outputs   = qsvdeint_outputs,
+
+    .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 };

libavfilter/vf_hwdownload.c

@@ -212,4 +212,5 @@ AVFilter ff_vf_hwdownload = {
     .priv_class    = &hwdownload_class,
     .inputs        = hwdownload_inputs,
     .outputs       = hwdownload_outputs,
+    .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 };

libavfilter/vf_hwupload.c

@@ -233,4 +233,5 @@ AVFilter ff_vf_hwupload = {
     .priv_class    = &hwupload_class,
     .inputs        = hwupload_inputs,
     .outputs       = hwupload_outputs,
+    .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 };

libavfilter/vf_hwupload_cuda.c

@@ -230,4 +230,6 @@ AVFilter ff_vf_hwupload_cuda = {
 
     .inputs    = cudaupload_inputs,
     .outputs   = cudaupload_outputs,
+
+    .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 };

libavfilter/vf_scale_npp.c

@@ -657,4 +657,6 @@ AVFilter ff_vf_scale_npp = {
 
     .inputs    = nppscale_inputs,
     .outputs   = nppscale_outputs,
+
+    .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 };

libavfilter/vf_scale_qsv.c

@@ -626,4 +626,6 @@ AVFilter ff_vf_scale_qsv = {
 
     .inputs    = qsvscale_inputs,
     .outputs   = qsvscale_outputs,
+
+    .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 };

libavfilter/vf_scale_vaapi.c

@@ -463,4 +463,5 @@ AVFilter ff_vf_scale_vaapi = {
     .inputs        = scale_vaapi_inputs,
     .outputs       = scale_vaapi_outputs,
     .priv_class    = &scale_vaapi_class,
+    .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
 };