check quality too, not only md5 (but dont take the scores too serious, the...

check quality too, not only md5 (but dont take the scores too serious, the different codecs use different quality/bitrates and its just based on a artificially generated testvideo)
small changes to some test to check a broader area of features

Originally committed as revision 1751 to svn://svn.ffmpeg.org/ffmpeg/trunk

tests/Makefile

@@ -19,7 +19,7 @@ test-server: vsynth1/0.pgm asynth1.sw
 	@$(SRC_PATH)/tests/server-regression.sh $(SERVER_REFFILE)
 
 # fast regression tests for all codecs
-test mpeg4 mpeg: vsynth1/0.pgm asynth1.sw
+test mpeg4 mpeg: vsynth1/0.pgm asynth1.sw tiny_psnr
 	@$(SRC_PATH)/tests/regression.sh $@ $(REFFILE) || true
 
 # fast regression for libav formats
@@ -43,6 +43,9 @@ asynth1.sw: audiogen
 audiogen: audiogen.c
 	$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $<
 
+tiny_psnr: tiny_psnr.c
+	$(CC) $(LDFLAGS) $(CFLAGS) -o $@ $<
+        
 DSPDEPS = $(SRC_PATH)/libavcodec/i386/dsputil_mmx.c \
  $(SRC_PATH)/libavcodec/i386/dsputil_mmx_avg.h \
  $(SRC_PATH)/libavcodec/i386/dsputil_mmx_rnd.h \

tests/ffmpeg.regression.ref

 ffmpeg regression test
 dda0ba041aef50a5101884291c06d4d9 *./data/a-mpeg1.mpg
 6713259d72260740bbddaea30631ea18 *./data/out.yuv
+stddev:  7.58 bytes:7299072
 e608f387c4ee7227fb4d0042e528ded7 *./data/a-msmpeg4v2.avi
 712aa6c959d1d90a78fe98657cbff19c *./data/out.yuv
+stddev:  8.11 bytes:7602176
 5957d6460c4b8fef35d68159e9cf2db0 *./data/a-msmpeg4.avi
 8786aa956838234fe3e48d0ef8cbd46c *./data/out.yuv
+stddev:  8.12 bytes:7602176
 78c3826f0a813cf666d9eb36caca5831 *./data/a-wmv1.avi
 7261e23fd8ad1de6efee022051b936be *./data/out.yuv
+stddev:  8.10 bytes:7602176
 58fa570e0867f30d7503482b8690c9dc *./data/a-wmv2.avi
 7261e23fd8ad1de6efee022051b936be *./data/out.yuv
+stddev:  8.10 bytes:7602176
 04a77cf9d7a3b4dcb394440d0bb67ea7 *./data/a-h263.avi
 545df74e0aa443499600faedd10a7065 *./data/out.yuv
-0d949fbabab1d67e76829e95b031a28f *./data/a-h263p.avi
-668ba3cb87859ca4d9a4269bad47b3f5 *./data/out.yuv
-fbda0de97cb28e359ca8b47aacba7d84 *./data/a-odivx.avi
-98bb113f0fa0d61fd3b0b1699ac6c69a *./data/out.yuv
+stddev:  8.18 bytes:7602176
+6d5ac74b9749c71ec1aef92909d002be *./data/a-h263p.avi
+79649b61321beaaaa6413af8c18e4696 *./data/out.yuv
+stddev:  2.05 bytes:7602176
+435965a98cb23b48b1d532bdb7afeb6b *./data/a-odivx.avi
+c753223d02441b59704c9859dbd7ff30 *./data/out.yuv
+stddev:  8.01 bytes:7602176
 16049c232bcd37dd42b0f39ed308fe05 *./data/a-huffyuv.avi
 799d3db687f6cdd7a837ec156efc171f *./data/out.yuv
+stddev:  0.00 bytes:7602176
 78f8b142f5841d0bda2faa0467633154 *./data/a-mpeg4-rc.avi
 580000bfe2d4359d9aa9a9415f953b3b *./data/out.yuv
+stddev: 10.53 bytes:7145472
 12cd2282b00644e35999f19969ec9bd9 *./data/a-mpeg4-adv.avi
 b54262af56f6681186fa2c44e4ef6ec7 *./data/out.yuv
+stddev:  7.32 bytes:7602176
 a38cb11e3035a280f3dad3ccdff5997b *./data/a-mpeg1b.mpg
 da8e21c7b78b7a25558dc319524b91d8 *./data/out.yuv
+stddev:  6.32 bytes:6842368
 2ecdef2ebfcc94e8ce90ce88c90ae8f4 *./data/a-mjpeg.avi
 f23a9e50a559e174766ee808c48fea22 *./data/out.yuv
+stddev:  8.87 bytes:7602176
 4b37703d3dc03873f99603165c0fe11e *./data/a-rv10.rm
 255469fef47bee94cfb2e3385ebb736b *./data/out.yuv
+stddev: 13.42 bytes:7602176
 21f8ff9f1daacd9133683bb4ea0f50a4 *./data/a-mp2.mp2
 116d1290ba1b4eb98fdee52e423417b1 *./data/out.wav
 048b9c3444c788bac6ce5cc3a8f4db00 *./data/a-ac3.rm

tests/regression.sh

@@ -42,11 +42,13 @@ fi
 
 # various files
 ffmpeg="../ffmpeg_g"
+tiny_psnr="./tiny_psnr"
 outfile="$datadir/a-"
 reffile="$2"
 benchfile="$datadir/ffmpeg.bench"
 raw_src="vsynth1/%d.pgm"
 raw_dst="$datadir/out.yuv"
+raw_ref="$datadir/ref.yuv"
 pcm_src="asynth1.sw"
 pcm_dst="$datadir/out.wav"
 
@@ -60,6 +62,9 @@ do_ffmpeg()
     echo $ffmpeg -bitexact -dct_algo 1 -idct_algo 2 $*
     $ffmpeg -bitexact -dct_algo 1 -idct_algo 2 -benchmark $* > $datadir/bench.tmp
     md5sum -b $f >> $logfile
+    if [ $f = $raw_dst ] ; then
+        $tiny_psnr $f $raw_ref >> $logfile
+    fi
     expr "`cat $datadir/bench.tmp`" : '.*utime=\(.*s\)' > $datadir/bench2.tmp
     echo `cat $datadir/bench2.tmp` $f >> $benchfile
 }
@@ -74,9 +79,23 @@ do_ffmpeg_crc()
     cat $datadir/ffmpeg.crc >> $logfile
 }
 
+do_ffmpeg_nocheck()
+{
+    f="$1"
+    shift
+    echo $ffmpeg -bitexact -dct_algo 1 -idct_algo 2 $*
+    $ffmpeg -bitexact -dct_algo 1 -idct_algo 2 -benchmark $* > $datadir/bench.tmp
+    expr "`cat $datadir/bench.tmp`" : '.*utime=\(.*s\)' > $datadir/bench2.tmp
+    echo `cat $datadir/bench2.tmp` $f >> $benchfile
+}
+
 echo "ffmpeg regression test" > $logfile
 echo "ffmpeg benchmarks" > $benchfile
 
+###################################
+# generate reference for quality check
+do_ffmpeg_nocheck $raw_ref -y -f pgmyuv -i $raw_src -an -f rawvideo $raw_ref
+
 ###################################
 if [ -n "$do_mpeg" ] ; then
 # mpeg1 encoding
@@ -145,7 +164,7 @@ fi
 if [ -n "$do_h263p" ] ; then
 # h263p encoding
 file=${outfile}h263p.avi
-do_ffmpeg $file -y -qscale 10 -umv -f pgmyuv -i $raw_src -s 352x288 -an -vcodec h263p -ps 300 $file
+do_ffmpeg $file -y -qscale 2 -umv -f pgmyuv -i $raw_src -s 352x288 -an -vcodec h263p -ps 300 $file
 
 # h263p decoding
 do_ffmpeg $raw_dst -y -i $file -f rawvideo $raw_dst 
@@ -155,7 +174,7 @@ fi
 if [ -n "$do_mpeg4" ] ; then
 # mpeg4
 file=${outfile}odivx.avi
-do_ffmpeg $file -y -qscale 10 -f pgmyuv -i $raw_src -an -vcodec mpeg4 $file
+do_ffmpeg $file -y -4mv -qscale 10 -f pgmyuv -i $raw_src -an -vcodec mpeg4 $file
 
 # mpeg4 decoding
 do_ffmpeg $raw_dst -y -i $file -f rawvideo $raw_dst 

tests/tiny_psnr.c

+/*
+ * Copyright (c) 2003 Michael Niedermayer <michaelni@gmx.at>
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+
+#include <stdio.h>
+#include <inttypes.h>
+
+#define F 100
+#define SIZE 2048
+
+static uint64_t int_sqrt(uint64_t a)
+{
+    uint64_t ret=0;
+    int s;
+    uint64_t ret_sq=0;
+
+    for(s=31; s>=0; s--){
+        uint64_t b= ret_sq + (1ULL<<(s*2)) + (ret<<s)*2;
+        if(b<=a){
+            ret_sq=b;
+            ret+= 1ULL<<s;
+        }
+    }
+    return ret;
+}
+
+int main(int argc,char* argv[]){
+    int i, j;
+    uint64_t sse=0;
+    uint64_t dev;
+    FILE *f[2];
+    uint8_t buf[2][SIZE];
+    
+    if(argc!=3){
+        printf("tiny_psnr <file1> <file2>\n");
+        return -1;
+    }
+    
+    f[0]= fopen(argv[1], "r");
+    f[1]= fopen(argv[2], "r");
+
+    for(i=0;;){
+        if( fread(buf[0], SIZE, 1, f[0]) != 1) break;
+        if( fread(buf[1], SIZE, 1, f[1]) != 1) break;
+        
+        for(j=0; j<SIZE; i++,j++){
+            const int a= buf[0][j];
+            const int b= buf[1][j];
+            sse += (a-b) * (a-b);
+        }
+    }
+    
+    dev= int_sqrt((sse*F*F)/i);
+    
+    //FIXME someone should write a integer fixpoint log() function for bitexact PSNR scores ...
+    printf("stddev:%3d.%02d bytes:%d\n", (int)(dev/F), (int)(dev%F), i);
+    
+    return 0;
+}