fixing variance scaling for b frames (messed adaptive quants up)

cliping too long MVs in b frames instead of setting them to 0

Originally committed as revision 965 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavcodec/motion_est.c

@@ -1546,7 +1546,7 @@ void ff_estimate_b_frame_motion(MpegEncContext * s,
             score=fbmin;
             type= MB_TYPE_BIDIR;
         }
-        score= (score*score)>>8;
+        score= (score*score + 128*256)>>16;
         s->mc_mb_var_sum += score;
         s->mc_mb_var[mb_y*s->mb_width + mb_x] = score; //FIXME use SSD
     }
@@ -1698,18 +1698,15 @@ void ff_fix_long_b_mvs(MpegEncContext * s, int16_t (*mv_table)[2], int f_code, i
         int xy= (y+1)* (s->mb_width+2)+1;
         int i= y*s->mb_width;
         for(x=0; x<s->mb_width; x++){
-            if(s->mb_type[i]&type){
             if(   fcode_tab[mv_table[xy][0] + MAX_MV] > f_code
-                   || fcode_tab[mv_table[xy][0] + MAX_MV] == 0
-                   || fcode_tab[mv_table[xy][1] + MAX_MV] > f_code
-                   || fcode_tab[mv_table[xy][1] + MAX_MV] == 0 ){
-                    if(s->mb_type[i]&(~type)) s->mb_type[i] &= ~type;
-                    else{
-                        mv_table[xy][0] = 0;
-                        mv_table[xy][1] = 0;
-                        //this is certainly bad FIXME            
-                    }
-                }
+               || fcode_tab[mv_table[xy][0] + MAX_MV] == 0){
+                if(mv_table[xy][0]>0) mv_table[xy][0]=  (16<<f_code)-1;
+                else                  mv_table[xy][0]= -(16<<f_code);
+            }
+            if(   fcode_tab[mv_table[xy][1] + MAX_MV] > f_code
+               || fcode_tab[mv_table[xy][1] + MAX_MV] == 0){
+                if(mv_table[xy][1]>0) mv_table[xy][1]=  (16<<f_code)-1;
+                else                  mv_table[xy][1]= -(16<<f_code);
             }
             xy++;
             i++;

tests/ffmpeg.regression.ref

@@ -11,8 +11,8 @@ abe11239875a32f00fa2910828bba4fb *./data/a-h263.avi
 c1f6c8ee7a24d8345deddf1a24ca3756 *./data/out.yuv
 440192aca11c310e01168ec24ea7807e *./data/a-odivx.avi
 145c98a175e760f8ba1997edf15b2964 *./data/out.yuv
-e68a6f6f8a228605cc4b131b100e06df *./data/a-mpeg4-rc.avi
-d63755afb8a7bb851d1da7a54786acbb *./data/out.yuv
+850ba6da4c9dad60b6719b0b06b204de *./data/a-mpeg4-rc.avi
+0ab0a58082e485478f9feab5ccc6fc42 *./data/out.yuv
 f5f44dad09c2d4d16524e539645f693c *./data/a-mpeg4-adv.avi
 d0f077a3b42367d7432b73c0ddad7438 *./data/out.yuv
 2846c8e3d97d7395eb746bfce44e0443 *./data/a-mjpeg.avi