avfilter/vf_colormatrix: increase precision

Signed-off-by: Thomas Mundt <loudmax@yahoo.de>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavfilter/vf_colormatrix.c

@@ -40,19 +40,11 @@
 #define NS(n) ((n) < 0 ? (int)((n)*65536.0-0.5+DBL_EPSILON) : (int)((n)*65536.0+0.5))
 #define CB(n) av_clip_uint8(n)
 
-static const double yuv_coeff[4][3][3] = {
-    { { +0.7152, +0.0722, +0.2126 }, // Rec.709 (0)
-      { -0.3850, +0.5000, -0.1150 },
-      { -0.4540, -0.0460, +0.5000 } },
-    { { +0.5900, +0.1100, +0.3000 }, // FCC (1)
-      { -0.3310, +0.5000, -0.1690 },
-      { -0.4210, -0.0790, +0.5000 } },
-    { { +0.5870, +0.1140, +0.2990 }, // Rec.601 (ITU-R BT.470-2/SMPTE 170M) (2)
-      { -0.3313, +0.5000, -0.1687 },
-      { -0.4187, -0.0813, +0.5000 } },
-    { { +0.7010, +0.0870, +0.2120 }, // SMPTE 240M (3)
-      { -0.3840, +0.5000, -0.1160 },
-      { -0.4450, -0.0550, +0.5000 } },
+static const double yuv_coeff_luma[4][3] = {
+    { +0.7152, +0.0722, +0.2126 }, // Rec.709 (0)
+    { +0.5900, +0.1100, +0.3000 }, // FCC (1)
+    { +0.5870, +0.1140, +0.2990 }, // Rec.601 (ITU-R BT.470-2/SMPTE 170M) (2)
+    { +0.7010, +0.0870, +0.2120 }, // SMPTE 240M (3)
 };
 
 enum ColorMode {
@@ -122,7 +114,7 @@ AVFILTER_DEFINE_CLASS(colormatrix);
 #define imh im[2][1]
 #define imi im[2][2]
 
-static void inverse3x3(double im[3][3], const double m[3][3])
+static void inverse3x3(double im[3][3], double m[3][3])
 {
     double det = ma * (me * mi - mf * mh) - mb * (md * mi - mf * mg) + mc * (md * mh - me * mg);
     det = 1.0 / det;
@@ -137,7 +129,7 @@ static void inverse3x3(double im[3][3], const double m[3][3])
     imi = det * (ma * me - mb * md);
 }
 
-static void solve_coefficients(double cm[3][3], double rgb[3][3], const double yuv[3][3])
+static void solve_coefficients(double cm[3][3], double rgb[3][3], double yuv[3][3])
 {
     int i, j;
     for (i = 0; i < 3; i++)
@@ -148,11 +140,25 @@ static void solve_coefficients(double cm[3][3], double rgb[3][3], const double y
 static void calc_coefficients(AVFilterContext *ctx)
 {
     ColorMatrixContext *color = ctx->priv;
+    double yuv_coeff[4][3][3];
     double rgb_coeffd[4][3][3];
     double yuv_convertd[16][3][3];
+    double bscale, rscale;
     int v = 0;
     int i, j, k;
-
+    for (i = 0; i < 4; i++) {
+        yuv_coeff[i][0][0] = yuv_coeff_luma[i][0];
+        yuv_coeff[i][0][1] = yuv_coeff_luma[i][1];
+        yuv_coeff[i][0][2] = yuv_coeff_luma[i][2];
+        bscale = 0.5 / (yuv_coeff[i][0][1] - 1.0);
+        rscale = 0.5 / (yuv_coeff[i][0][2] - 1.0);
+        yuv_coeff[i][1][0] = bscale * yuv_coeff[i][0][0];
+        yuv_coeff[i][1][1] = 0.5;
+        yuv_coeff[i][1][2] = bscale * yuv_coeff[i][0][2];
+        yuv_coeff[i][2][0] = rscale * yuv_coeff[i][0][0];
+        yuv_coeff[i][2][1] = rscale * yuv_coeff[i][0][1];
+        yuv_coeff[i][2][2] = 0.5;
+    }
     for (i = 0; i < 4; i++)
         inverse3x3(rgb_coeffd[i], yuv_coeff[i]);
     for (i = 0; i < 4; i++) {

tests/ref/fate/filter-colormatrix1

-colormatrix1        0a0640b2d4ccd4e793f4919d82a89523
+colormatrix1        1593e24a71db42d07781b92665cf99a9

tests/ref/fate/filter-colormatrix2

-colormatrix2        5eb17671c03496ae43723e49832ab17a
+colormatrix2        cafba294debb89f59143b519f3cd1673