libavfilter: add photosensitivity filter

Changelog

@@ -13,6 +13,7 @@ version <next>:
 - streamhash muxer
 - sierpinski video source
 - scroll video filter
+- photosensitivity filter
 
 
 version 4.2:

doc/filters.texi

@@ -14040,6 +14040,26 @@ Filter selects among @samp{t}, @samp{b} and @samp{p} using image analysis only.
 @end table
 @end table
 
+@section photosensitivity
+Reduce various flashes in video, so to help users with epilepsy.
+
+It accepts the following options:
+@table @option
+@item frames, f
+Set how many frames to use when filtering. Default is 30.
+
+@item threshold, t
+Set detection threshold factor. Default is 1.
+Lower is stricter.
+
+@item skip
+Set how many pixels to skip when sampling frames. Defalt is 1.
+Allowed range is from 1 to 1024.
+
+@item bypass
+Leave frames unchanged. Default is disabled.
+@end table
+
 @section pixdesctest
 
 Pixel format descriptor test filter, mainly useful for internal

libavfilter/Makefile

@@ -321,6 +321,7 @@ OBJS-$(CONFIG_PALETTEUSE_FILTER)             += vf_paletteuse.o framesync.o
 OBJS-$(CONFIG_PERMS_FILTER)                  += f_perms.o
 OBJS-$(CONFIG_PERSPECTIVE_FILTER)            += vf_perspective.o
 OBJS-$(CONFIG_PHASE_FILTER)                  += vf_phase.o
+OBJS-$(CONFIG_PHOTOSENSITIVITY_FILTER)       += vf_photosensitivity.o
 OBJS-$(CONFIG_PIXDESCTEST_FILTER)            += vf_pixdesctest.o
 OBJS-$(CONFIG_PIXSCOPE_FILTER)               += vf_datascope.o
 OBJS-$(CONFIG_PP_FILTER)                     += vf_pp.o

libavfilter/allfilters.c

@@ -305,6 +305,7 @@ extern AVFilter ff_vf_paletteuse;
 extern AVFilter ff_vf_perms;
 extern AVFilter ff_vf_perspective;
 extern AVFilter ff_vf_phase;
+extern AVFilter ff_vf_photosensitivity;
 extern AVFilter ff_vf_pixdesctest;
 extern AVFilter ff_vf_pixscope;
 extern AVFilter ff_vf_pp;

libavfilter/version.h

@@ -30,7 +30,7 @@
 #include "libavutil/version.h"
 
 #define LIBAVFILTER_VERSION_MAJOR   7
-#define LIBAVFILTER_VERSION_MINOR  60
+#define LIBAVFILTER_VERSION_MINOR  61
 #define LIBAVFILTER_VERSION_MICRO 100
 
 

libavfilter/vf_photosensitivity.c

+/*
+ * Copyright (c) 2019 Vladimir Panteleev
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <float.h>
+
+#include "libavutil/imgutils.h"
+#include "libavutil/opt.h"
+#include "libavutil/pixdesc.h"
+#include "avfilter.h"
+
+#include "formats.h"
+#include "internal.h"
+#include "video.h"
+
+#define MAX_FRAMES 240
+#define GRID_SIZE 8
+#define NUM_CHANNELS 3
+
+typedef struct PhotosensitivityFrame {
+    uint8_t grid[GRID_SIZE][GRID_SIZE][4];
+} PhotosensitivityFrame;
+
+typedef struct PhotosensitivityContext {
+    const AVClass *class;
+
+    int nb_frames;
+    int skip;
+    float threshold_multiplier;
+    int bypass;
+
+    int badness_threshold;
+
+    /* Circular buffer */
+    int history[MAX_FRAMES];
+    int history_pos;
+
+    PhotosensitivityFrame last_frame_e;
+    AVFrame *last_frame_av;
+} PhotosensitivityContext;
+
+#define OFFSET(x) offsetof(PhotosensitivityContext, x)
+#define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
+
+static const AVOption photosensitivity_options[] = {
+    { "frames",    "set how many frames to use",                          OFFSET(nb_frames),            AV_OPT_TYPE_INT,   {.i64=30}, 2, MAX_FRAMES, FLAGS },
+    { "f",         "set how many frames to use",                          OFFSET(nb_frames),            AV_OPT_TYPE_INT,   {.i64=30}, 2, MAX_FRAMES, FLAGS },
+    { "threshold", "set detection threshold factor (lower is stricter)",  OFFSET(threshold_multiplier), AV_OPT_TYPE_FLOAT, {.dbl=1},  0.1, FLT_MAX,  FLAGS },
+    { "t",         "set detection threshold factor (lower is stricter)",  OFFSET(threshold_multiplier), AV_OPT_TYPE_FLOAT, {.dbl=1},  0.1, FLT_MAX,  FLAGS },
+    { "skip",      "set pixels to skip when sampling frames",             OFFSET(skip),                 AV_OPT_TYPE_INT,   {.i64=1},  1, 1024,       FLAGS },
+    { "bypass",    "leave frames unchanged",                              OFFSET(bypass),               AV_OPT_TYPE_BOOL,  {.i64=0},  0, 1,          FLAGS },
+    { NULL }
+};
+
+AVFILTER_DEFINE_CLASS(photosensitivity);
+
+static int query_formats(AVFilterContext *ctx)
+{
+    static const enum AVPixelFormat pixel_fmts[] = {
+        AV_PIX_FMT_RGB24,
+        AV_PIX_FMT_BGR24,
+        AV_PIX_FMT_NONE
+    };
+    AVFilterFormats *formats = ff_make_format_list(pixel_fmts);
+    if (!formats)
+        return AVERROR(ENOMEM);
+    return ff_set_common_formats(ctx, formats);
+}
+
+typedef struct ThreadData_convert_frame
+{
+    AVFrame *in;
+    PhotosensitivityFrame *out;
+    int skip;
+} ThreadData_convert_frame;
+
+#define NUM_CELLS (GRID_SIZE * GRID_SIZE)
+
+static int convert_frame_partial(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
+{
+    int cell, gx, gy, x0, x1, y0, y1, x, y, c, area;
+    int sum[NUM_CHANNELS];
+    const uint8_t *p;
+
+    ThreadData_convert_frame *td = arg;
+
+    const int slice_start = (NUM_CELLS * jobnr) / nb_jobs;
+    const int slice_end = (NUM_CELLS * (jobnr+1)) / nb_jobs;
+
+    int width = td->in->width, height = td->in->height, linesize = td->in->linesize[0], skip = td->skip;
+    const uint8_t *data = td->in->data[0];
+
+    for (cell = slice_start; cell < slice_end; cell++) {
+        gx = cell % GRID_SIZE;
+        gy = cell / GRID_SIZE;
+
+        x0 = width  *  gx    / GRID_SIZE;
+        x1 = width  * (gx+1) / GRID_SIZE;
+        y0 = height *  gy    / GRID_SIZE;
+        y1 = height * (gy+1) / GRID_SIZE;
+
+        for (c = 0; c < NUM_CHANNELS; c++) {
+            sum[c] = 0;
+        }
+        for (y = y0; y < y1; y += skip) {
+            p = data + y * linesize + x0 * NUM_CHANNELS;
+            for (x = x0; x < x1; x += skip) {
+                //av_log(NULL, AV_LOG_VERBOSE, "%d %d %d : (%d,%d) (%d,%d) -> %d,%d | *%d\n", c, gx, gy, x0, y0, x1, y1, x, y, (int)row);
+                sum[0] += p[0];
+                sum[1] += p[1];
+                sum[2] += p[2];
+                p += NUM_CHANNELS * skip;
+                // TODO: variable size
+            }
+        }
+
+        area = ((x1 - x0 + skip - 1) / skip) * ((y1 - y0 + skip - 1) / skip);
+        for (c = 0; c < NUM_CHANNELS; c++) {
+            if (area)
+                sum[c] /= area;
+            td->out->grid[gy][gx][c] = sum[c];
+        }
+    }
+    return 0;
+}
+
+static void convert_frame(AVFilterContext *ctx, AVFrame *in, PhotosensitivityFrame *out, int skip)
+{
+    ThreadData_convert_frame td;
+    td.in = in;
+    td.out = out;
+    td.skip = skip;
+    ctx->internal->execute(ctx, convert_frame_partial, &td, NULL, FFMIN(NUM_CELLS, ff_filter_get_nb_threads(ctx)));
+}
+
+typedef struct ThreadData_blend_frame
+{
+    AVFrame *target;
+    AVFrame *source;
+    uint16_t s_mul;
+} ThreadData_blend_frame;
+
+static int blend_frame_partial(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs)
+{
+    int x, y;
+    uint8_t *t, *s;
+
+    ThreadData_blend_frame *td = arg;
+    const uint16_t s_mul = td->s_mul;
+    const uint16_t t_mul = 0x100 - s_mul;
+    const int slice_start = (td->target->height * jobnr) / nb_jobs;
+    const int slice_end = (td->target->height * (jobnr+1)) / nb_jobs;
+    const int linesize = td->target->linesize[0];
+
+    for (y = slice_start; y < slice_end; y++) {
+        t = td->target->data[0] + y * td->target->linesize[0];
+        s = td->source->data[0] + y * td->source->linesize[0];
+        for (x = 0; x < linesize; x++) {
+            *t = (*t * t_mul + *s * s_mul) >> 8;
+            t++; s++;
+        }
+    }
+    return 0;
+}
+
+static void blend_frame(AVFilterContext *ctx, AVFrame *target, AVFrame *source, float factor)
+{
+    ThreadData_blend_frame td;
+    td.target = target;
+    td.source = source;
+    td.s_mul = (uint16_t)(factor * 0x100);
+    ctx->internal->execute(ctx, blend_frame_partial, &td, NULL, FFMIN(ctx->outputs[0]->h, ff_filter_get_nb_threads(ctx)));
+}
+
+static int get_badness(PhotosensitivityFrame *a, PhotosensitivityFrame *b)
+{
+    int badness, x, y, c;
+    badness = 0;
+    for (c = 0; c < NUM_CHANNELS; c++) {
+        for (y = 0; y < GRID_SIZE; y++) {
+            for (x = 0; x < GRID_SIZE; x++) {
+                badness += abs((int)a->grid[y][x][c] - (int)b->grid[y][x][c]);
+                //av_log(NULL, AV_LOG_VERBOSE, "%d - %d -> %d \n", a->grid[y][x], b->grid[y][x], badness);
+                //av_log(NULL, AV_LOG_VERBOSE, "%d -> %d \n", abs((int)a->grid[y][x] - (int)b->grid[y][x]), badness);
+            }
+        }
+    }
+    return badness;
+}
+
+static int config_input(AVFilterLink *inlink)
+{
+    /* const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(inlink->format); */
+    AVFilterContext *ctx = inlink->dst;
+    PhotosensitivityContext *s = ctx->priv;
+
+    s->badness_threshold = (int)(GRID_SIZE * GRID_SIZE * 4 * 256 * s->nb_frames * s->threshold_multiplier / 128);
+
+    return 0;
+}
+
+static int filter_frame(AVFilterLink *inlink, AVFrame *in)
+{
+    int this_badness, current_badness, fixed_badness, new_badness, i, res;
+    PhotosensitivityFrame ef;
+    AVFrame *src, *out;
+    float factor;
+    AVDictionary **metadata;
+
+    AVFilterContext *ctx = inlink->dst;
+    AVFilterLink *outlink = ctx->outputs[0];
+    PhotosensitivityContext *s = ctx->priv;
+
+    /* weighted moving average */
+    current_badness = 0;
+    for (i = 1; i < s->nb_frames; i++)
+        current_badness += i * s->history[(s->history_pos + i) % s->nb_frames];
+    current_badness /= s->nb_frames;
+
+    convert_frame(ctx, in, &ef, s->skip);
+    this_badness = get_badness(&ef, &s->last_frame_e);
+    new_badness = current_badness + this_badness;
+    av_log(s, AV_LOG_VERBOSE, "badness: %6d -> %6d / %6d (%3d%% - %s)\n",
+        current_badness, new_badness, s->badness_threshold,
+        100 * new_badness / s->badness_threshold, new_badness < s->badness_threshold ? "OK" : "EXCEEDED");
+
+    fixed_badness = new_badness;
+    if (new_badness < s->badness_threshold || !s->last_frame_av || s->bypass) {
+        factor = 1; /* for metadata */
+        av_frame_free(&s->last_frame_av);
+        s->last_frame_av = src = in;
+        s->last_frame_e = ef;
+        s->history[s->history_pos] = this_badness;
+    } else {
+        factor = (float)(s->badness_threshold - current_badness) / (new_badness - current_badness);
+        if (factor <= 0) {
+            /* just duplicate the frame */
+            s->history[s->history_pos] = 0; /* frame was duplicated, thus, delta is zero */
+        } else {
+            res = av_frame_make_writable(s->last_frame_av);
+            if (res) {
+                av_frame_free(&in);
+                return res;
+            }
+            blend_frame(ctx, s->last_frame_av, in, factor);
+
+            convert_frame(ctx, s->last_frame_av, &ef, s->skip);
+            this_badness = get_badness(&ef, &s->last_frame_e);
+            fixed_badness = current_badness + this_badness;
+            av_log(s, AV_LOG_VERBOSE, "  fixed: %6d -> %6d / %6d (%3d%%) factor=%5.3f\n",
+                current_badness, fixed_badness, s->badness_threshold,
+                100 * new_badness / s->badness_threshold, factor);
+            s->last_frame_e = ef;
+            s->history[s->history_pos] = this_badness;
+        }
+        src = s->last_frame_av;
+    }
+    s->history_pos = (s->history_pos + 1) % s->nb_frames;
+
+    out = ff_get_video_buffer(outlink, in->width, in->height);
+    if (!out) {
+        av_frame_free(&in);
+        return AVERROR(ENOMEM);
+    }
+    av_frame_copy_props(out, in);
+    metadata = &out->metadata;
+    if (metadata) {
+        char value[128];
+
+        snprintf(value, sizeof(value), "%f", (float)new_badness / s->badness_threshold);
+        av_dict_set(metadata, "lavfi.photosensitivity.badness", value, 0);
+
+        snprintf(value, sizeof(value), "%f", (float)fixed_badness / s->badness_threshold);
+        av_dict_set(metadata, "lavfi.photosensitivity.fixed-badness", value, 0);
+
+        snprintf(value, sizeof(value), "%f", (float)this_badness / s->badness_threshold);
+        av_dict_set(metadata, "lavfi.photosensitivity.frame-badness", value, 0);
+
+        snprintf(value, sizeof(value), "%f", factor);
+        av_dict_set(metadata, "lavfi.photosensitivity.factor", value, 0);
+    }
+    av_frame_copy(out, src);
+    return ff_filter_frame(outlink, out);
+}
+
+static av_cold void uninit(AVFilterContext *ctx)
+{
+    PhotosensitivityContext *s = ctx->priv;
+
+    av_frame_free(&s->last_frame_av);
+}
+
+static const AVFilterPad inputs[] = {
+    {
+        .name         = "default",
+        .type         = AVMEDIA_TYPE_VIDEO,
+        .filter_frame = filter_frame,
+        .config_props = config_input,
+    },
+    { NULL }
+};
+
+static const AVFilterPad outputs[] = {
+    {
+        .name          = "default",
+        .type          = AVMEDIA_TYPE_VIDEO,
+    },
+    { NULL }
+};
+
+AVFilter ff_vf_photosensitivity = {
+    .name          = "photosensitivity",
+    .description   = NULL_IF_CONFIG_SMALL("Filter out photosensitive epilepsy seizure-inducing flashes."),
+    .priv_size     = sizeof(PhotosensitivityContext),
+    .priv_class    = &photosensitivity_class,
+    .uninit        = uninit,
+    .query_formats = query_formats,
+    .inputs        = inputs,
+    .outputs       = outputs,
+};