avfilter/vf_colorkey: fix formula for calculation of difference

Also fixes colorhold filtering.

libavfilter/vf_colorkey.c

@@ -45,7 +45,7 @@ static uint8_t do_colorkey_pixel(ColorkeyContext *ctx, uint8_t r, uint8_t g, uin
     int dg = (int)g - ctx->colorkey_rgba[1];
     int db = (int)b - ctx->colorkey_rgba[2];
 
-    double diff = sqrt((dr * dr + dg * dg + db * db) / (255.0 * 255.0));
+    double diff = sqrt((dr * dr + dg * dg + db * db) / (255.0 * 255.0 * 3.0));
 
     if (ctx->blend > 0.0001) {
         return av_clipd((diff - ctx->similarity) / ctx->blend, 0.0, 1.0) * 255.0;

tests/ref/fate/ffmpeg-filter_colorkey

@@ -3,13 +3,13 @@
 #codec_id 0: rawvideo
 #dimensions 0: 720x576
 #sar 0: 0/1
-0,          0,          0,        1,   622080, 0x78efb628
-0,          1,          1,        1,   622080, 0x641f2564
-0,          2,          2,        1,   622080, 0x348f25c3
-0,          3,          3,        1,   622080, 0x6afc485a
-0,          4,          4,        1,   622080, 0xe949107f
-0,          5,          5,        1,   622080, 0x171716e5
-0,          6,          6,        1,   622080, 0x2985a01f
-0,          7,          7,        1,   622080, 0xc5ddabd7
-0,          8,          8,        1,   622080, 0xb4dd2b7f
-0,          9,          9,        1,   622080, 0x6e75ba82
+0,          0,          0,        1,   622080, 0x5775bb12
+0,          1,          1,        1,   622080, 0x84d8395a
+0,          2,          2,        1,   622080, 0x599d3a6b
+0,          3,          3,        1,   622080, 0x1df1e9dc
+0,          4,          4,        1,   622080, 0xf3255836
+0,          5,          5,        1,   622080, 0xf46bc26e
+0,          6,          6,        1,   622080, 0x76d491da
+0,          7,          7,        1,   622080, 0x2c9f3ca6
+0,          8,          8,        1,   622080, 0x5713e203
+0,          9,          9,        1,   622080, 0x47776493