avfilter/vf_stereo3d: improve dubois anaglyphs

libavfilter/vf_stereo3d.c

@@ -100,9 +100,9 @@ static const int ana_coeff[][3][6] = {
      {    0,     0,     0,     0, 65536,     0},
      {    0,     0,     0,     0,     0, 65536}},
   [ANAGLYPH_RC_DUBOIS] =
-    {{29891, 32800, 11559, -2849, -5763,  -102},
-     {-2627, -2479, -1033, 24804, 48080, -1209},
-     { -997, -1350,  -358, -4729, -7403, 80373}},
+    {{29884, 32768, 11534, -2818, -5767,  -131},
+     {-2621, -2490, -1049, 24773, 48103, -1180},
+     { -983, -1376,  -328, -4719, -7406, 80347}},
   [ANAGLYPH_GM_GRAY]   =
     {{    0,     0,     0, 19595, 38470,  7471},
      {19595, 38470,  7471,     0,     0,     0},
@@ -132,9 +132,9 @@ static const int ana_coeff[][3][6] = {
      {    0,     0,     0,     0, 65536,     0},
      {    0,     0, 65536,     0,     0,     0}},
   [ANAGLYPH_YB_DUBOIS] =
-    {{65535,-12650,18451,   -987, -7590, -1049},
-     {-1604, 56032, 4196,    370,  3826, -1049},
-     {-2345,-10676, 1358,   5801, 11416, 56217}},
+    {{69599,-13435,19595,  -1048, -8061, -1114},
+     {-1704, 59507, 4456,    393,  4063, -1114},
+     {-2490,-11338, 1442,   6160, 12124, 59703}},
 };
 
 typedef struct Stereo3DContext {

tests/ref/fate/filter-stereo3d-sbsl-arcd

@@ -3,8 +3,8 @@
 #codec_id 0: rawvideo
 #dimensions 0: 176x288
 #sar 0: 0/1
-0,          0,          0,        1,   152064, 0xa0261570
-0,          1,          1,        1,   152064, 0x678403c8
-0,          2,          2,        1,   152064, 0x1087e7b6
-0,          3,          3,        1,   152064, 0xa3909df3
-0,          4,          4,        1,   152064, 0x87e4c4d4
+0,          0,          0,        1,   152064, 0x34b1fb03
+0,          1,          1,        1,   152064, 0x2045eadd
+0,          2,          2,        1,   152064, 0x1266cdde
+0,          3,          3,        1,   152064, 0xc9f083bd
+0,          4,          4,        1,   152064, 0x0701ab16

tests/ref/fate/filter-stereo3d-sbsl-aybd

@@ -3,8 +3,8 @@
 #codec_id 0: rawvideo
 #dimensions 0: 176x288
 #sar 0: 0/1
-0,          0,          0,        1,   152064, 0x7dc98468
-0,          1,          1,        1,   152064, 0xf72db6c9
-0,          2,          2,        1,   152064, 0x1630f53f
-0,          3,          3,        1,   152064, 0xc1765599
-0,          4,          4,        1,   152064, 0x12e35db1
+0,          0,          0,        1,   152064, 0xf576742c
+0,          1,          1,        1,   152064, 0x2ce36ae4
+0,          2,          2,        1,   152064, 0x2dc99849
+0,          3,          3,        1,   152064, 0x940413b6
+0,          4,          4,        1,   152064, 0x48a600d3