avfilter/edgedetect: add a colormix mode.

doc/filters.texi

@@ -3990,13 +3990,37 @@ by the low threshold.
 
 Default value for @var{low} is @code{20/255}, and default value for @var{high}
 is @code{50/255}.
+
+@item mode
+Define the drawing mode.
+
+@table @samp
+@item wires
+Draw white/gray wires on black background.
+
+@item colormix
+Mix the colors to create a paint/cartoon effect.
+@end table
+
+Default value is @var{wires}.
 @end table
 
-Example:
+@subsection Examples
+
+@itemize
+@item
+Standard edge detection with custom values for the hysteresis thresholding:
 @example
 edgedetect=low=0.1:high=0.4
 @end example
 
+@item
+Painting effect without thresholding:
+@example
+edgedetect=mode=colormix:high=0
+@end example
+@end itemize
+
 @section extractplanes
 
 Extract color channel components from input video stream into

libavfilter/vf_edgedetect.c

@@ -25,19 +25,32 @@
  * @see https://en.wikipedia.org/wiki/Canny_edge_detector
  */
 
+#include "libavutil/avassert.h"
 #include "libavutil/opt.h"
 #include "avfilter.h"
 #include "formats.h"
 #include "internal.h"
 #include "video.h"
 
-typedef struct {
-    const AVClass *class;
+enum FilterMode {
+    MODE_WIRES,
+    MODE_COLORMIX,
+    NB_MODE
+};
+
+struct plane_info {
     uint8_t  *tmpbuf;
     uint16_t *gradients;
     char     *directions;
+};
+
+typedef struct {
+    const AVClass *class;
+    struct plane_info planes[3];
+    int nb_planes;
     double   low, high;
     uint8_t  low_u8, high_u8;
+    enum FilterMode mode;
 } EdgeDetectContext;
 
 #define OFFSET(x) offsetof(EdgeDetectContext, x)
@@ -45,6 +58,9 @@ typedef struct {
 static const AVOption edgedetect_options[] = {
     { "high", "set high threshold", OFFSET(high), AV_OPT_TYPE_DOUBLE, {.dbl=50/255.}, 0, 1, FLAGS },
     { "low",  "set low threshold",  OFFSET(low),  AV_OPT_TYPE_DOUBLE, {.dbl=20/255.}, 0, 1, FLAGS },
+    { "mode", "set mode", OFFSET(mode), AV_OPT_TYPE_INT, {.i64=MODE_WIRES}, 0, NB_MODE-1, FLAGS, "mode" },
+        { "wires",    "white/gray wires on black",  0, AV_OPT_TYPE_CONST, {.i64=MODE_WIRES},    INT_MIN, INT_MAX, FLAGS, "mode" },
+        { "colormix", "mix colors",                 0, AV_OPT_TYPE_CONST, {.i64=MODE_COLORMIX}, INT_MIN, INT_MAX, FLAGS, "mode" },
     { NULL }
 };
 
@@ -61,21 +77,37 @@ static av_cold int init(AVFilterContext *ctx)
 
 static int query_formats(AVFilterContext *ctx)
 {
+    const EdgeDetectContext *edgedetect = ctx->priv;
+
+    if (edgedetect->mode == MODE_WIRES) {
+    /* TODO: reindent */
     static const enum AVPixelFormat pix_fmts[] = {AV_PIX_FMT_GRAY8, AV_PIX_FMT_NONE};
     ff_set_common_formats(ctx, ff_make_format_list(pix_fmts));
+    } else if (edgedetect->mode == MODE_COLORMIX) {
+        static const enum AVPixelFormat pix_fmts[] = {AV_PIX_FMT_GBRP, AV_PIX_FMT_GRAY8, AV_PIX_FMT_NONE};
+        ff_set_common_formats(ctx, ff_make_format_list(pix_fmts));
+    } else {
+        av_assert0(0);
+    }
     return 0;
 }
 
 static int config_props(AVFilterLink *inlink)
 {
+    int p;
     AVFilterContext *ctx = inlink->dst;
     EdgeDetectContext *edgedetect = ctx->priv;
 
-    edgedetect->tmpbuf     = av_malloc(inlink->w * inlink->h);
-    edgedetect->gradients  = av_calloc(inlink->w * inlink->h, sizeof(*edgedetect->gradients));
-    edgedetect->directions = av_malloc(inlink->w * inlink->h);
-    if (!edgedetect->tmpbuf || !edgedetect->gradients || !edgedetect->directions)
-        return AVERROR(ENOMEM);
+    edgedetect->nb_planes = inlink->format == AV_PIX_FMT_GRAY8 ? 1 : 3;
+    for (p = 0; p < edgedetect->nb_planes; p++) {
+        struct plane_info *plane = &edgedetect->planes[p];
+
+        plane->tmpbuf     = av_malloc(inlink->w * inlink->h);
+        plane->gradients  = av_calloc(inlink->w * inlink->h, sizeof(*plane->gradients));
+        plane->directions = av_malloc(inlink->w * inlink->h);
+        if (!plane->tmpbuf || !plane->gradients || !plane->directions)
+            return AVERROR(ENOMEM);
+    }
     return 0;
 }
 
@@ -241,18 +273,29 @@ static void double_threshold(int low, int high, int w, int h,
     }
 }
 
+static void color_mix(int w, int h,
+                            uint8_t *dst, int dst_linesize,
+                      const uint8_t *src, int src_linesize)
+{
+    int i, j;
+
+    for (j = 0; j < h; j++) {
+        for (i = 0; i < w; i++)
+            dst[i] = (dst[i] + src[i]) >> 1;
+        dst += dst_linesize;
+        src += src_linesize;
+    }
+}
+
 static int filter_frame(AVFilterLink *inlink, AVFrame *in)
 {
     AVFilterContext *ctx = inlink->dst;
     EdgeDetectContext *edgedetect = ctx->priv;
     AVFilterLink *outlink = inlink->dst->outputs[0];
-    uint8_t  *tmpbuf    = edgedetect->tmpbuf;
-    uint16_t *gradients = edgedetect->gradients;
-    int8_t   *directions= edgedetect->directions;
-    int direct = 0;
+    int p, direct = 0;
     AVFrame *out;
 
-    if (av_frame_is_writable(in)) {
+    if (edgedetect->mode != MODE_COLORMIX && av_frame_is_writable(in)) {
         direct = 1;
         out = in;
     } else {
@@ -264,10 +307,17 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
         av_frame_copy_props(out, in);
     }
 
+    for (p = 0; p < edgedetect->nb_planes; p++) {
+        struct plane_info *plane = &edgedetect->planes[p];
+        uint8_t  *tmpbuf     = plane->tmpbuf;
+        uint16_t *gradients  = plane->gradients;
+        int8_t   *directions = plane->directions;
+
+        /* TODO: reindent */
     /* gaussian filter to reduce noise  */
     gaussian_blur(ctx, inlink->w, inlink->h,
                   tmpbuf,      inlink->w,
-                  in->data[0], in->linesize[0]);
+                  in->data[p], in->linesize[p]);
 
     /* compute the 16-bits gradients and directions for the next step */
     sobel(inlink->w, inlink->h,
@@ -286,9 +336,16 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
     /* keep high values, or low values surrounded by high values */
     double_threshold(edgedetect->low_u8, edgedetect->high_u8,
                      inlink->w, inlink->h,
-                     out->data[0], out->linesize[0],
+                     out->data[p], out->linesize[p],
                      tmpbuf,       inlink->w);
 
+        if (edgedetect->mode == MODE_COLORMIX) {
+            color_mix(inlink->w, inlink->h,
+                      out->data[p], out->linesize[p],
+                      in->data[p], in->linesize[p]);
+        }
+    }
+
     if (!direct)
         av_frame_free(&in);
     return ff_filter_frame(outlink, out);
@@ -296,10 +353,15 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in)
 
 static av_cold void uninit(AVFilterContext *ctx)
 {
+    int p;
     EdgeDetectContext *edgedetect = ctx->priv;
-    av_freep(&edgedetect->tmpbuf);
-    av_freep(&edgedetect->gradients);
-    av_freep(&edgedetect->directions);
+
+    for (p = 0; p < edgedetect->nb_planes; p++) {
+        struct plane_info *plane = &edgedetect->planes[p];
+        av_freep(&plane->tmpbuf);
+        av_freep(&plane->gradients);
+        av_freep(&plane->directions);
+    }
 }
 
 static const AVFilterPad edgedetect_inputs[] = {

tests/fate/filter-video.mak

@@ -191,6 +191,9 @@ fate-filter-vflip_vflip: CMD = video_filter "vflip,vflip"
 FATE_FILTER_VSYNTH-$(call ALLYES, FORMAT_FILTER PERMS_FILTER EDGEDETECT_FILTER) += fate-filter-edgedetect
 fate-filter-edgedetect: CMD = video_filter "format=gray,perms=random,edgedetect"
 
+FATE_FILTER_VSYNTH-$(call ALLYES, FORMAT_FILTER PERMS_FILTER EDGEDETECT_FILTER) += fate-filter-edgedetect-colormix
+fate-filter-edgedetect-colormix: CMD = video_filter "format=gbrp,perms=random,edgedetect=mode=colormix"
+
 FATE_FILTER_VSYNTH-$(call ALLYES, PERMS_FILTER HUE_FILTER) += fate-filter-hue
 fate-filter-hue: CMD = video_filter "perms=random,hue=s=sin(2*PI*t)+1"
 

tests/ref/fate/filter-edgedetect-colormix

+edgedetect-colormix c84a2be00652610f968bef8c97d71ef4