Fix trellis quant + AAN DCT.

Originally committed as revision 12221 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavcodec/mpegvideo_enc.c

@@ -61,6 +61,17 @@ static const uint16_t aanscales[64] = {
     4520 ,  6270,  5906,  5315,  4520,  3552,  2446,  1247
 };
 
+static const uint16_t inv_aanscales[64] = {
+  4096,  2953,  3135,  3483,  4096,  5213,  7568, 14846,
+  2953,  2129,  2260,  2511,  2953,  3759,  5457, 10703,
+  3135,  2260,  2399,  2666,  3135,  3990,  5793, 11363,
+  3483,  2511,  2666,  2962,  3483,  4433,  6436, 12625,
+  4096,  2953,  3135,  3483,  4096,  5213,  7568, 14846,
+  5213,  3759,  3990,  4433,  5213,  6635,  9633, 18895,
+  7568,  5457,  5793,  6436,  7568,  9633, 13985, 27432,
+ 14846, 10703, 11363, 12625, 14846, 18895, 27432, 53809,
+};
+
 static uint8_t default_mv_penalty[MAX_FCODE+1][MAX_MV*2+1];
 static uint8_t default_fcode_tab[MAX_MV*2+1];
 
@@ -3067,10 +3078,18 @@ int dct_quantize_trellis_c(MpegEncContext *s,
     survivor_count= 1;
 
     for(i=start_i; i<=last_non_zero; i++){
-        int level_index, j;
-        const int dct_coeff= FFABS(block[ scantable[i] ]);
-        const int zero_distoration= dct_coeff*dct_coeff;
+        int level_index, j, zero_distoration;
+        int dct_coeff= FFABS(block[ scantable[i] ]);
         int best_score=256*256*256*120;
+
+        if (   s->dsp.fdct == fdct_ifast
+#ifndef FAAN_POSTSCALE
+            || s->dsp.fdct == ff_faandct
+#endif
+           )
+            dct_coeff= (dct_coeff*inv_aanscales[ scantable[i] ]) >> 12;
+        zero_distoration= dct_coeff*dct_coeff;
+
         for(level_index=0; level_index < coeff_count[i]; level_index++){
             int distoration;
             int level= coeff[level_index][i];

tests/ffmpeg.regression.ref

@@ -6,10 +6,10 @@ f3b008355f68394b6cad694f3488ea2b *./tests/data/a-mpeg2.mpg
 736978 ./tests/data/a-mpeg2.mpg
 9f364a477987c3b14412e303b94377ca *./tests/data/mpeg2.vsynth.out.yuv
 stddev:  7.68 PSNR:30.41 bytes:7602176
-3236c7550f66fe1f0a1526b02b72c0df *./tests/data/a-mpeg2ivlc-qprd.mpg
-707911 ./tests/data/a-mpeg2ivlc-qprd.mpg
-8a0112b8a26301ccaca49c28b30d387b *./tests/data/mpeg2.vsynth.out.yuv
-stddev: 12.21 PSNR:26.38 bytes:7602176
+103f4b5f90feaec3c04e200409501982 *./tests/data/a-mpeg2ivlc-qprd.mpg
+787469 ./tests/data/a-mpeg2ivlc-qprd.mpg
+985170ca0ab1ec1edbb37c44d8cfde90 *./tests/data/mpeg2.vsynth.out.yuv
+stddev: 10.09 PSNR:28.04 bytes:7602176
 2566ea5760247a9485c8281cb52291a8 *./tests/data/a-mpeg2.mpg
 735853 ./tests/data/a-mpeg2.mpg
 55c22a09e4924977ee2cc4180078d3ae *./tests/data/mpeg2.vsynth.out.yuv
@@ -70,26 +70,26 @@ b27e911f1563455700c93e6ae39715ca *./tests/data/a-mpeg4-rc.avi
 813988 ./tests/data/a-mpeg4-rc.avi
 f57311745de96b6551c175679b138efc *./tests/data/rc.vsynth.out.yuv
 stddev: 10.40 PSNR:27.78 bytes:7602176
-85e2456a673041d528b242d78318fb65 *./tests/data/a-mpeg4-adv.avi
-600188 ./tests/data/a-mpeg4-adv.avi
-60edc5a67271e425d0a2a52981895b81 *./tests/data/mpeg4adv.vsynth.out.yuv
-stddev: 10.25 PSNR:27.91 bytes:7602176
-6ae74b04d5265132837eb32a5e03a98a *./tests/data/a-mpeg4-qprd.avi
-657854 ./tests/data/a-mpeg4-qprd.avi
-c2c3c675bd1ffdc0b92dd0f27a83f37a *./tests/data/mpeg4adv.vsynth.out.yuv
-stddev: 12.12 PSNR:26.45 bytes:7602176
-69a0f715145536849413e30e950e9cc2 *./tests/data/a-mpeg4-adap.avi
-399984 ./tests/data/a-mpeg4-adap.avi
-f3e2093420cd44101da2f708f19fcbd1 *./tests/data/mpeg4adv.vsynth.out.yuv
-stddev: 14.66 PSNR:24.80 bytes:7602176
+76d3b26ef7e51146e1194956cb0275e6 *./tests/data/a-mpeg4-adv.avi
+604580 ./tests/data/a-mpeg4-adv.avi
+ee7f2a0be286f09e20372e127364d690 *./tests/data/mpeg4adv.vsynth.out.yuv
+stddev:  6.99 PSNR:31.23 bytes:7602176
+13bc53ae93709323b8989cc4f8ea4b1e *./tests/data/a-mpeg4-qprd.avi
+736920 ./tests/data/a-mpeg4-qprd.avi
+745ccd9185f579cde533c1500fc4adc7 *./tests/data/mpeg4adv.vsynth.out.yuv
+stddev: 10.04 PSNR:28.08 bytes:7602176
+5b105ca591f94fa1c2b420cd74b290a1 *./tests/data/a-mpeg4-adap.avi
+409360 ./tests/data/a-mpeg4-adap.avi
+6ae220d766c44af3bd58e3d2f5aefa8d *./tests/data/mpeg4adv.vsynth.out.yuv
+stddev: 14.15 PSNR:25.10 bytes:7602176
 41b27141442f773eca9ef3d48d8d555a *./tests/data/a-mpeg4-Q.avi
 878264 ./tests/data/a-mpeg4-Q.avi
 8995abbcc97ed4767fcbc0bf46accd01 *./tests/data/mpeg4adv.vsynth.out.yuv
 stddev:  5.61 PSNR:33.13 bytes:7602176
-a20b1d73bb0671dbd2012494c31719a6 *./tests/data/a-mpeg4-thread.avi
-761506 ./tests/data/a-mpeg4-thread.avi
-a9835b5b6a34a3fd70c168dc60fb9687 *./tests/data/mpeg4thread.vsynth.out.yuv
-stddev: 12.31 PSNR:26.31 bytes:7602176
+6271c9f3b8a6ec3a78315699952d9a99 *./tests/data/a-mpeg4-thread.avi
+778604 ./tests/data/a-mpeg4-thread.avi
+152d46da2a9d7fa3c22c56767c1760db *./tests/data/mpeg4thread.vsynth.out.yuv
+stddev: 10.26 PSNR:27.89 bytes:7602176
 743cf18c1e8941591f8ab41350247c42 *./tests/data/a-mpeg4-PSP.mp4
 406441 ./tests/data/a-mpeg4-PSP.mp4
 7315281e07830456208dff61337c982b *./tests/data/a-error-mpeg4-adv.avi

tests/rotozoom.regression.ref

@@ -6,10 +6,10 @@ stddev:  4.95 PSNR:34.21 bytes:7602176
 198667 ./tests/data/a-mpeg2.mpg
 b7cae8a1f751b821cddcbe4d5dbc518c *./tests/data/mpeg2.rotozoom.out.yuv
 stddev:  4.96 PSNR:34.19 bytes:7602176
-9e7542fb0dfea5ffa9de6bfee3d26fdb *./tests/data/a-mpeg2ivlc-qprd.mpg
-248055 ./tests/data/a-mpeg2ivlc-qprd.mpg
-610ec1a25a3f823bf083d8bf20348ebb *./tests/data/mpeg2.rotozoom.out.yuv
-stddev:  4.44 PSNR:35.16 bytes:7602176
+788b83a7c8c6c1e648c05076dd40dc30 *./tests/data/a-mpeg2ivlc-qprd.mpg
+247161 ./tests/data/a-mpeg2ivlc-qprd.mpg
+cc6f08c1a24b58c19de325e79cfa40a3 *./tests/data/mpeg2.rotozoom.out.yuv
+stddev:  4.25 PSNR:35.55 bytes:7602176
 f979bcca866e6e4cad5dc6cb06e56cfb *./tests/data/a-mpeg2.mpg
 198041 ./tests/data/a-mpeg2.mpg
 f6d9bf24ff8676a7f6076c05cd2c81a3 *./tests/data/mpeg2.rotozoom.out.yuv
@@ -70,26 +70,26 @@ b517389e425d3065ab26ef4cc0658237 *./tests/data/a-mpeg4-rc.avi
 227762 ./tests/data/a-mpeg4-rc.avi
 cc947bbac9187bf08d3e2d425556aaa5 *./tests/data/rc.rotozoom.out.yuv
 stddev:  4.26 PSNR:35.53 bytes:7602176
-9001cf571eb7f26fa5592bdec6538583 *./tests/data/a-mpeg4-adv.avi
-173590 ./tests/data/a-mpeg4-adv.avi
-699edf05648fdc42196b7bebef9be269 *./tests/data/mpeg4adv.rotozoom.out.yuv
-stddev:  4.84 PSNR:34.41 bytes:7602176
-23a44888361723e233e25147e42aa8d8 *./tests/data/a-mpeg4-qprd.avi
-233288 ./tests/data/a-mpeg4-qprd.avi
-4c8bebe5ceec416d84d99f3afd5fac19 *./tests/data/mpeg4adv.rotozoom.out.yuv
-stddev:  4.26 PSNR:35.52 bytes:7602176
-3921cc04f007f2ba73ca8a8ccbaf42eb *./tests/data/a-mpeg4-adap.avi
-202438 ./tests/data/a-mpeg4-adap.avi
-82379c0cb726088fc4de7546ae696fa8 *./tests/data/mpeg4adv.rotozoom.out.yuv
-stddev:  4.05 PSNR:35.95 bytes:7602176
+dee7be19486a76d96c88d18eefba8f86 *./tests/data/a-mpeg4-adv.avi
+141546 ./tests/data/a-mpeg4-adv.avi
+3f3a21e9db85a9c0f7022f557a5374c1 *./tests/data/mpeg4adv.rotozoom.out.yuv
+stddev:  4.94 PSNR:34.24 bytes:7602176
+1b1a692e11969deb1fe3c7144f8cb103 *./tests/data/a-mpeg4-qprd.avi
+233604 ./tests/data/a-mpeg4-qprd.avi
+95860ad446ca0665d873ae6781736095 *./tests/data/mpeg4adv.rotozoom.out.yuv
+stddev:  4.01 PSNR:36.05 bytes:7602176
+78c93c522b23bcc7f84f8b592b0191b6 *./tests/data/a-mpeg4-adap.avi
+200120 ./tests/data/a-mpeg4-adap.avi
+fd7db0b14fa76d0734bbfa36dbb513f8 *./tests/data/mpeg4adv.rotozoom.out.yuv
+stddev:  3.78 PSNR:36.57 bytes:7602176
 a5150067914ee1dee50f8fc8dcaee841 *./tests/data/a-mpeg4-Q.avi
 165802 ./tests/data/a-mpeg4-Q.avi
 4dcc71ad79bee90777cf5299044be362 *./tests/data/mpeg4adv.rotozoom.out.yuv
 stddev:  4.00 PSNR:36.08 bytes:7602176
-b4c8104d9fd36db9ea5c74f231107d70 *./tests/data/a-mpeg4-thread.avi
-251520 ./tests/data/a-mpeg4-thread.avi
-dbe9a74a4ca34f263900002c333eb342 *./tests/data/mpeg4thread.rotozoom.out.yuv
-stddev:  4.07 PSNR:35.91 bytes:7602176
+8496ffe953dc3398c657d99e962e4d77 *./tests/data/a-mpeg4-thread.avi
+250162 ./tests/data/a-mpeg4-thread.avi
+58165c879707aedeab460bab86dae4ef *./tests/data/mpeg4thread.rotozoom.out.yuv
+stddev:  3.73 PSNR:36.67 bytes:7602176
 6cd200bf844fd1c0e48c23d542a860e8 *./tests/data/a-mpeg4-PSP.mp4
 324998 ./tests/data/a-mpeg4-PSP.mp4
 90e65096aa9ebafa3fe3f44a5a47cdc4 *./tests/data/a-error-mpeg4-adv.avi