use lagrange multipler instead of qp for ratecontrol, this may break some...

use lagrange multipler instead of qp for ratecontrol, this may break some things, tell me ASAP if u notice anything broken
quality which was 1..31 float is now a 1..FF_LAMBDA_MAX int, and FF_QP2LAMBDA * qp can be used to convert to the new range

Originally committed as revision 2348 to svn://svn.ffmpeg.org/ffmpeg/trunk

ffmpeg.c

@@ -79,7 +79,7 @@ static int frame_rate = 25;
 static int frame_rate_base = 1;
 static int video_bit_rate = 200*1000;
 static int video_bit_rate_tolerance = 4000*1000;
-static int video_qscale = 0;
+static float video_qscale = 0;
 static int video_qmin = 2;
 static int video_qmax = 31;
 static int video_mb_qmin = 2;
@@ -680,7 +680,7 @@ static void do_video_stats(AVFormatContext *os, AVOutputStream *ost,
     total_size += frame_size;
     if (enc->codec_type == CODEC_TYPE_VIDEO) {
         frame_number = ost->frame_number;
-        fprintf(fvstats, "frame= %5d q= %2.1f ", frame_number, enc->coded_frame->quality);
+        fprintf(fvstats, "frame= %5d q= %2.1f ", frame_number, enc->coded_frame->quality/(float)FF_QP2LAMBDA);
         if (enc->flags&CODEC_FLAG_PSNR)
             fprintf(fvstats, "PSNR= %6.2f ", psnr(enc->coded_frame->error[0]/(enc->width*enc->height*255.0*255.0)));
         
@@ -738,12 +738,12 @@ static void print_report(AVFormatContext **output_files,
         enc = &ost->st->codec;
         if (vid && enc->codec_type == CODEC_TYPE_VIDEO) {
             sprintf(buf + strlen(buf), "q=%2.1f ",
-                    enc->coded_frame->quality);
+                    enc->coded_frame->quality/(float)FF_QP2LAMBDA);
         }
         if (!vid && enc->codec_type == CODEC_TYPE_VIDEO) {
             frame_number = ost->frame_number;
             sprintf(buf + strlen(buf), "frame=%5d q=%2.1f ",
-                    frame_number, enc->coded_frame ? enc->coded_frame->quality : 0);
+                    frame_number, enc->coded_frame ? enc->coded_frame->quality/(float)FF_QP2LAMBDA : 0);
             if (enc->flags&CODEC_FLAG_PSNR)
                 sprintf(buf + strlen(buf), "PSNR= %6.2f ", psnr(enc->coded_frame->error[0]/(enc->width*enc->height*255.0*255.0)));
             vid = 1;
@@ -1774,10 +1774,10 @@ static void opt_mb_decision(const char *arg)
 
 static void opt_qscale(const char *arg)
 {
-    video_qscale = atoi(arg);
-    if (video_qscale < 0 ||
-        video_qscale > 31) {
-        fprintf(stderr, "qscale must be >= 1 and <= 31\n");
+    video_qscale = atof(arg);
+    if (video_qscale < 0.01 ||
+        video_qscale > 255) {
+        fprintf(stderr, "qscale must be >= 0.01 and <= 255\n");
         exit(1);
     }
 }
@@ -2245,7 +2245,7 @@ static void opt_output_file(const char *filename)
                     video_enc->gop_size = 0;
                 if (video_qscale || same_quality) {
                     video_enc->flags |= CODEC_FLAG_QSCALE;
-                    st->quality = video_qscale;
+                    st->quality = FF_QP2LAMBDA * video_qscale;
                 }
                 
                 if(bitexact)

libavcodec/avcodec.h

@@ -15,7 +15,7 @@ extern "C" {
 
 #define FFMPEG_VERSION_INT     0x000408
 #define FFMPEG_VERSION         "0.4.8"
-#define LIBAVCODEC_BUILD       4683
+#define LIBAVCODEC_BUILD       4684
 
 #define LIBAVCODEC_VERSION_INT FFMPEG_VERSION_INT
 #define LIBAVCODEC_VERSION     FFMPEG_VERSION
@@ -316,15 +316,15 @@ static const int Motion_Est_QTab[] = { ME_ZERO, ME_PHODS, ME_LOG,
     int display_picture_number;\
 \
     /**\
-     * quality (between 1 (good) and 31 (bad)) \
+     * quality (between 1 (good) and FF_LAMBDA_MAX (bad)) \
      * - encoding: set by lavc for coded_picture (and set by user for input)\
      * - decoding: set by lavc\
      */\
-    float quality; \
+    int quality; \
 \
     /**\
      * buffer age (1->was last buffer and dint change, 2->..., ...).\
-     * set to something large if the buffer has not been used yet \
+     * set to INT_MAX if the buffer has not been used yet \
      * - encoding: unused\
      * - decoding: MUST be set by get_buffer()\
      */\
@@ -1181,8 +1181,13 @@ typedef struct AVCodecContext {
      * Dont touch, used by lavc default_get_buffer()
      */
     void *internal_buffer;
-    
-#define FF_QUALITY_SCALE 256
+
+#define FF_LAMBDA_SHIFT 7
+#define FF_LAMBDA_SCALE (1<<FF_LAMBDA_SHIFT)
+#define FF_QP2LAMBDA 118 ///< factor to convert from H.263 QP to lambda
+#define FF_LAMBDA_MAX (256*128-1)
+
+#define FF_QUALITY_SCALE FF_LAMBDA_SCALE //FIXME maybe remove
     /**
      * global quality for codecs which cannot change it per frame.
      * this should be proportional to MPEG1/2/4 qscale.
@@ -1263,6 +1268,20 @@ typedef struct AVCodecContext {
      * - decoding: unused
      */
     int scenechange_threshold;
+
+    /**
+     * minimum lagrange multipler
+     * - encoding: set by user.
+     * - decoding: unused
+     */
+    int lmin;
+
+    /**
+     * maximum lagrange multipler
+     * - encoding: set by user.
+     * - decoding: unused
+     */
+    int lmax;
 } AVCodecContext;
 
 

libavcodec/h263.c

@@ -626,7 +626,7 @@ static inline int get_p_cbp(MpegEncContext * s,
         int best_cbpc_score= INT_MAX;
         int cbpc, cbpy;
         const int offset= (s->mv_type==MV_TYPE_16X16 ? 0 : 16) + (s->dquant ? 8 : 0);
-        const int lambda= (s->qscale*s->qscale*64*105 + 64)>>7; 
+        const int lambda= s->lambda2 >> (FF_LAMBDA_SHIFT - 6);
 
         for(i=0; i<4; i++){
             int score= inter_MCBPC_bits[i + offset] * lambda;
@@ -679,7 +679,7 @@ static inline int get_b_cbp(MpegEncContext * s, DCTELEM block[6][64],
 
     if(s->flags & CODEC_FLAG_CBP_RD){
         int score=0;
-        const int lambda= (s->qscale*s->qscale*64*105 + 64)>>7; 
+        const int lambda= s->lambda2 >> (FF_LAMBDA_SHIFT - 6);
         
         for(i=0; i<6; i++){
             if(s->coded_score[i] < 0){

libavcodec/mjpeg.c

@@ -1877,7 +1877,8 @@ eoi_parser:
                             picture->qscale_table= s->qscale_table;
                             memset(picture->qscale_table, picture->quality, (s->width+15)/16);
                             if(avctx->debug & FF_DEBUG_QP)
-                                printf("QP: %d\n", (int)picture->quality);
+                                printf("QP: %d\n", picture->quality);
+                            picture->quality*= FF_QP2LAMBDA;
                         }
                         
                         goto the_end;
@@ -2052,7 +2053,8 @@ read_header:
         picture->qscale_table= s->qscale_table;
         memset(picture->qscale_table, picture->quality, (s->width+15)/16);
         if(avctx->debug & FF_DEBUG_QP)
-            printf("QP: %f\n", picture->quality);
+            printf("QP: %d\n", picture->quality);
+        picture->quality*= FF_QP2LAMBDA;
     }
 
     return buf_ptr - buf;

libavcodec/mpegvideo.c

@@ -140,6 +140,13 @@ static void convert_matrix(MpegEncContext *s, int (*qmat)[64], uint16_t (*qmat16
         }
     }
 }
+
+static inline void update_qscale(MpegEncContext *s){
+    s->qscale= (s->lambda*139 + FF_LAMBDA_SCALE*64) >> (FF_LAMBDA_SHIFT + 7);
+    s->qscale= clip(s->qscale, s->avctx->qmin, s->avctx->qmax);
+    
+    s->lambda2= (s->lambda*s->lambda + FF_LAMBDA_SCALE/2) >> FF_LAMBDA_SHIFT;
+}
 #endif //CONFIG_ENCODERS
 
 void ff_init_scantable(uint8_t *permutation, ScanTable *st, const uint8_t *src_scantable){
@@ -424,6 +431,8 @@ int MPV_common_init(MpegEncContext *s)
 
         /* Allocate MB type table */
         CHECKED_ALLOCZ(s->mb_type  , mb_array_size * sizeof(uint8_t)) //needed for encoding
+        
+        CHECKED_ALLOCZ(s->lambda_table, mb_array_size * sizeof(int))
     }
         
     CHECKED_ALLOCZ(s->error_status_table, mb_array_size*sizeof(uint8_t))
@@ -539,6 +548,7 @@ void MPV_common_end(MpegEncContext *s)
     av_freep(&s->ac_stats);
     av_freep(&s->error_status_table);
     av_freep(&s->mb_index2xy);
+    av_freep(&s->lambda_table);
 
     for(i=0; i<MAX_PICTURE_COUNT; i++){
         free_picture(s, &s->picture[i]);
@@ -1028,7 +1038,8 @@ alloc:
     }
 
     s->current_picture_ptr->pict_type= s->pict_type;
-    s->current_picture_ptr->quality= s->qscale;
+//    if(s->flags && CODEC_FLAG_QSCALE) 
+  //      s->current_picture_ptr->quality= s->new_picture_ptr->quality;
     s->current_picture_ptr->key_frame= s->pict_type == I_TYPE;
 
     s->current_picture= *s->current_picture_ptr;
@@ -1602,10 +1613,6 @@ int MPV_encode_picture(AVCodecContext *avctx,
     if(s->new_picture.data[0]){
 
         s->pict_type= s->new_picture.pict_type;
-        if (s->fixed_qscale){ /* the ratecontrol needs the last qscale so we dont touch it for CBR */
-            s->qscale= (int)(s->new_picture.quality+0.5);
-            assert(s->qscale);
-        }
 //emms_c();
 //printf("qs:%f %f %d\n", s->new_picture.quality, s->current_picture.quality, s->qscale);
         MPV_frame_start(s, avctx);
@@ -2815,22 +2822,23 @@ static void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
     for(i=0; i<6; i++) skip_dct[i]=0;
     
     if(s->adaptive_quant){
-        s->dquant= s->current_picture.qscale_table[mb_x + mb_y*s->mb_stride] - s->qscale;
+        const int last_qp= s->qscale;
+        const int mb_xy= mb_x + mb_y*s->mb_stride;
 
-        if(s->out_format==FMT_H263){
-            if     (s->dquant> 2) s->dquant= 2;
-            else if(s->dquant<-2) s->dquant=-2;
-        }
+        s->lambda= s->lambda_table[mb_xy];
+        update_qscale(s);
+        s->dquant= s->qscale - last_qp;
+
+        if(s->out_format==FMT_H263)
+            s->dquant= clip(s->dquant, -2, 2); //FIXME RD
             
         if(s->codec_id==CODEC_ID_MPEG4){        
             if(!s->mb_intra){
-                if(s->mv_dir&MV_DIRECT)
+                if((s->mv_dir&MV_DIRECT) || s->mv_type==MV_TYPE_8X8)
                     s->dquant=0;
-
-                assert(s->dquant==0 || s->mv_type!=MV_TYPE_8X8);
             }
         }
-        s->qscale+= s->dquant;
+        s->qscale= last_qp + s->dquant;
         s->y_dc_scale= s->y_dc_scale_table[ s->qscale ];
         s->c_dc_scale= s->c_dc_scale_table[ s->qscale ];
     }
@@ -2993,19 +3001,6 @@ static void encode_mb(MpegEncContext *s, int motion_x, int motion_y)
 
     }
             
-#if 0
-            {
-                float adap_parm;
-                
-                adap_parm = ((s->avg_mb_var << 1) + s->mb_var[s->mb_stride*mb_y+mb_x] + 1.0) /
-                            ((s->mb_var[s->mb_stride*mb_y+mb_x] << 1) + s->avg_mb_var + 1.0);
-            
-                printf("\ntype=%c qscale=%2d adap=%0.2f dquant=%4.2f var=%4d avgvar=%4d", 
-                        (s->mb_type[s->mb_stride*mb_y+mb_x] > 0) ? 'I' : 'P', 
-                        s->qscale, adap_parm, s->qscale*adap_parm,
-                        s->mb_var[s->mb_stride*mb_y+mb_x], s->avg_mb_var);
-            }
-#endif
     /* DCT & quantize */
     if(s->out_format==FMT_MJPEG){
         for(i=0;i<6;i++) {
@@ -3265,8 +3260,8 @@ static inline void encode_mb_hq(MpegEncContext *s, MpegEncContext *backup, MpegE
     if(s->avctx->mb_decision == FF_MB_DECISION_RD){
         MPV_decode_mb(s, s->block);
 
-        score *= s->qscale * s->qscale * 109;
-        score += sse_mb(s) << 7;
+        score *= s->lambda2;
+        score += sse_mb(s) << FF_LAMBDA_SHIFT;
     }
     
     if(*next_block){
@@ -3351,7 +3346,7 @@ static void encode_picture(MpegEncContext *s, int picture_number)
         
     s->scene_change_score=0;
     
-    s->qscale= (int)(s->frame_qscale + 0.5); //FIXME qscale / ... stuff for ME ratedistoration
+    s->lambda= s->current_picture_ptr->quality; //FIXME qscale / ... stuff for ME ratedistoration
     
     if(s->pict_type==I_TYPE){
         if(s->msmpeg4_version >= 3) s->no_rounding=1;
@@ -3460,10 +3455,8 @@ static void encode_picture(MpegEncContext *s, int picture_number)
         }
     }
     
-    if (s->fixed_qscale) 
-        s->frame_qscale = s->current_picture.quality;
-    else
-        s->frame_qscale = ff_rate_estimate_qscale(s);
+    if (!s->fixed_qscale) 
+        s->current_picture.quality = ff_rate_estimate_qscale(s);
 
     if(s->adaptive_quant){
 #ifdef CONFIG_RISKY
@@ -3479,9 +3472,15 @@ static void encode_picture(MpegEncContext *s, int picture_number)
         }
 #endif
 
-        s->qscale= s->current_picture.qscale_table[0];
+        s->lambda= s->lambda_table[0];
+        //FIXME broken
     }else
-        s->qscale= (int)(s->frame_qscale + 0.5);
+        s->lambda= s->current_picture.quality;
+//printf("%d %d\n", s->avctx->global_quality, s->current_picture.quality);
+    update_qscale(s);
+    
+    if(s->qscale < 3 && s->max_qcoeff<=128 && s->pict_type==I_TYPE && !(s->flags & CODEC_FLAG_QSCALE)) 
+        s->qscale= 3; //reduce cliping problems
         
     if (s->out_format == FMT_MJPEG) {
         /* for mjpeg, we do include qscale in the matrix */
@@ -4020,12 +4019,14 @@ static int dct_quantize_trellis_c(MpegEncContext *s,
     int not_coded_score= 0;
     int coeff[3][64];
     int coeff_count[64];
-    int lambda, qmul, qadd, start_i, last_non_zero, i, dc;
+    int qmul, qadd, start_i, last_non_zero, i, dc;
     const int esc_length= s->ac_esc_length;
     uint8_t * length;
     uint8_t * last_length;
     int score_limit=0;
     int left_limit= 0;
+    const int lambda= s->lambda2 >> (FF_LAMBDA_SHIFT - 6);
+    const int patch_table= s->out_format == FMT_MPEG1 && !s->mb_intra;
         
     s->dsp.fdct (block);
 
@@ -4103,9 +4104,13 @@ static int dct_quantize_trellis_c(MpegEncContext *s,
         return last_non_zero;
     }
 
-    lambda= (qscale*qscale*64*105 + 64)>>7; //FIXME finetune
-        
     score_tab[0]= 0;
+    
+    if(patch_table){
+//        length[UNI_AC_ENC_INDEX(0, 63)]=
+//        length[UNI_AC_ENC_INDEX(0, 65)]= 2;
+    }
+
     for(i=0; i<=last_non_zero - start_i; i++){
         int level_index, run, j;
         const int dct_coeff= block[ scantable[i + start_i] ];
@@ -4212,10 +4217,13 @@ static int dct_quantize_trellis_c(MpegEncContext *s,
         
         //Note: there is a vlc code in mpeg4 which is 1 bit shorter then another one with a shorter run and the same level
         while(score_tab[ left_limit ] > score_limit + lambda) left_limit++;
+    
+        if(patch_table){
+//            length[UNI_AC_ENC_INDEX(0, 63)]=
+//            length[UNI_AC_ENC_INDEX(0, 65)]= 3;
+        }
     }
 
-        //FIXME add some cbp penalty
-
     if(s->out_format != FMT_H263){
         last_score= 256*256*256*120;
         for(i= left_limit; i<=last_non_zero - start_i + 1; i++){

libavcodec/mpegvideo.h

@@ -354,7 +354,9 @@ typedef struct MpegEncContext {
     uint8_t *edge_emu_buffer;     ///< points into the middle of allocated_edge_emu_buffer  
 
     int qscale;                 ///< QP 
-    float frame_qscale;         ///< qscale from the frame level rc FIXME remove
+    int lambda;                 ///< lagrange multipler used in rate distortion
+    int lambda2;                ///< (lambda*lambda) >> FF_LAMBDA_SHIFT 
+    int *lambda_table;
     int adaptive_quant;         ///< use adaptive quantization 
     int dquant;                 ///< qscale difference to prev qscale  
     int pict_type;              ///< I_TYPE, P_TYPE, B_TYPE, ... 

libavcodec/ratecontrol.c

@@ -38,9 +38,9 @@ static int init_pass2(MpegEncContext *s);
 static double get_qscale(MpegEncContext *s, RateControlEntry *rce, double rate_factor, int frame_num);
 
 void ff_write_pass1_stats(MpegEncContext *s){
-    sprintf(s->avctx->stats_out, "in:%d out:%d type:%d q:%f itex:%d ptex:%d mv:%d misc:%d fcode:%d bcode:%d mc-var:%d var:%d icount:%d;\n",
+    sprintf(s->avctx->stats_out, "in:%d out:%d type:%d q:%d itex:%d ptex:%d mv:%d misc:%d fcode:%d bcode:%d mc-var:%d var:%d icount:%d;\n",
             s->picture_number, s->input_picture_number - s->max_b_frames, s->pict_type, 
-            s->frame_qscale, s->i_tex_bits, s->p_tex_bits, s->mv_bits, s->misc_bits, 
+            s->current_picture.quality, s->i_tex_bits, s->p_tex_bits, s->mv_bits, s->misc_bits, 
             s->f_code, s->b_code, s->current_picture.mc_mb_var_sum, s->current_picture.mb_var_sum, s->i_count);
 }
 
@@ -51,7 +51,7 @@ int ff_rate_control_init(MpegEncContext *s)
     emms_c();
 
     for(i=0; i<5; i++){
-        rcc->pred[i].coeff= 7.0;
+        rcc->pred[i].coeff= FF_QP2LAMBDA * 7.0;
         rcc->pred[i].count= 1.0;
     
         rcc->pred[i].decay= 0.4;
@@ -60,7 +60,7 @@ int ff_rate_control_init(MpegEncContext *s)
         rcc->mv_bits_sum[i]=
         rcc->qscale_sum [i]=
         rcc->frame_count[i]= 1; // 1 is better cuz of 1/0 and such
-        rcc->last_qscale_for[i]=5;
+        rcc->last_qscale_for[i]=FF_QP2LAMBDA * 5;
     }
     rcc->buffer_index= s->avctx->rc_buffer_size/2;
 
@@ -81,7 +81,7 @@ int ff_rate_control_init(MpegEncContext *s)
         for(i=0; i<rcc->num_entries; i++){
             RateControlEntry *rce= &rcc->entry[i];
             rce->pict_type= rce->new_pict_type=P_TYPE;
-            rce->qscale= rce->new_qscale=2;
+            rce->qscale= rce->new_qscale=FF_QP2LAMBDA * 2;
             rce->misc_bits= s->mb_num + 10;
             rce->mb_var_sum= s->mb_num*100;
         }        
@@ -140,7 +140,7 @@ int ff_rate_control_init(MpegEncContext *s)
                 rce.new_pict_type= rce.pict_type;
                 rce.mc_mb_var_sum= bits*s->mb_num/100000;
                 rce.mb_var_sum   = s->mb_num;
-                rce.qscale   = 2;
+                rce.qscale   = FF_QP2LAMBDA * 2;
                 rce.f_code   = 2;
                 rce.b_code   = 1;
                 rce.misc_bits= 1;
@@ -339,9 +339,10 @@ static double get_diff_limited_q(MpegEncContext *s, RateControlEntry *rce, doubl
     /* last qscale / qdiff stuff */
     if(rcc->last_non_b_pict_type==pict_type || pict_type!=I_TYPE){
         double last_q= rcc->last_qscale_for[pict_type];
+        const int maxdiff= FF_QP2LAMBDA * a->max_qdiff;
 
-        if     (q > last_q + a->max_qdiff) q= last_q + a->max_qdiff;
-        else if(q < last_q - a->max_qdiff) q= last_q - a->max_qdiff;
+        if     (q > last_q + maxdiff) q= last_q + maxdiff;
+        else if(q < last_q - maxdiff) q= last_q - maxdiff;
     }
 
     rcc->last_qscale_for[pict_type]= q; //Note we cant do that after blurring
@@ -356,8 +357,8 @@ static double get_diff_limited_q(MpegEncContext *s, RateControlEntry *rce, doubl
  * gets the qmin & qmax for pict_type
  */
 static void get_qminmax(int *qmin_ret, int *qmax_ret, MpegEncContext *s, int pict_type){
-    int qmin= s->avctx->qmin;                                                       
-    int qmax= s->avctx->qmax;
+    int qmin= s->avctx->lmin;                                                       
+    int qmax= s->avctx->lmax;
     
     assert(qmin <= qmax);
 
@@ -369,12 +370,8 @@ static void get_qminmax(int *qmin_ret, int *qmax_ret, MpegEncContext *s, int pic
         qmax= (int)(qmax*ABS(s->avctx->i_quant_factor)+s->avctx->i_quant_offset + 0.5);
     }
 
-    qmin= clip(qmin, 1, 31);
-    qmax= clip(qmax, 1, 31);
-
-    if(qmin==1 && s->avctx->qmin>1) qmin=2; //avoid qmin=1 unless the user wants qmin=1
-
-    if(qmin<3 && s->max_qcoeff<=128 && pict_type==I_TYPE) qmin=3; //reduce cliping problems
+    qmin= clip(qmin, 1, FF_LAMBDA_MAX);
+    qmax= clip(qmax, 1, FF_LAMBDA_MAX);
 
     if(qmax<qmin) qmax= qmin;
     
@@ -479,21 +476,23 @@ static void adaptive_quantization(MpegEncContext *s, double q){
     float cplx_sum= 0.0;
     float cplx_tab[s->mb_num];
     float bits_tab[s->mb_num];
-    const int qmin= s->avctx->mb_qmin;
-    const int qmax= s->avctx->mb_qmax;
+    const int qmin= s->avctx->lmin;
+    const int qmax= s->avctx->lmax;
     Picture * const pic= &s->current_picture;
-    int last_qscale=0;
     
     for(i=0; i<s->mb_num; i++){
         const int mb_xy= s->mb_index2xy[i];
-        float temp_cplx= sqrt(pic->mc_mb_var[mb_xy]);
+        float temp_cplx= sqrt(pic->mc_mb_var[mb_xy]); //FIXME merge in pow()
         float spat_cplx= sqrt(pic->mb_var[mb_xy]);
         const int lumi= pic->mb_mean[mb_xy];
         float bits, cplx, factor;
-        
+#if 0        
         if(spat_cplx < q/3) spat_cplx= q/3; //FIXME finetune
         if(temp_cplx < q/3) temp_cplx= q/3; //FIXME finetune
-        
+#endif   
+        if(spat_cplx < 4) spat_cplx= 4; //FIXME finetune
+        if(temp_cplx < 4) temp_cplx= 4; //FIXME finetune
+
         if((s->mb_type[mb_xy]&MB_TYPE_INTRA)){//FIXME hq mode 
             cplx= spat_cplx;
             factor= 1.0 + p_masking;
@@ -543,19 +542,16 @@ static void adaptive_quantization(MpegEncContext *s, double q){
             newq*= bits_sum/cplx_sum;
         }
 
-        if(i && ABS(last_qscale - newq)<0.75)
-            intq= last_qscale;
-        else
-            intq= (int)(newq + 0.5);
+        intq= (int)(newq + 0.5);
 
         if     (intq > qmax) intq= qmax;
         else if(intq < qmin) intq= qmin;
 //if(i%s->mb_width==0) printf("\n");
 //printf("%2d%3d ", intq, ff_sqrt(s->mc_mb_var[i]));
-        last_qscale=
-        pic->qscale_table[mb_xy]= intq;
+        s->lambda_table[mb_xy]= intq;
     }
 }
+//FIXME rd or at least approx for dquant
 
 float ff_rate_estimate_qscale(MpegEncContext *s)
 {
@@ -614,7 +610,7 @@ float ff_rate_estimate_qscale(MpegEncContext *s)
         rce->new_pict_type= pict_type;
         rce->mc_mb_var_sum= pic->mc_mb_var_sum;
         rce->mb_var_sum   = pic->   mb_var_sum;
-        rce->qscale   = 2;
+        rce->qscale   = FF_QP2LAMBDA * 2;
         rce->f_code   = s->f_code;
         rce->b_code   = s->b_code;
         rce->misc_bits= 1;

libavcodec/utils.c

@@ -272,6 +272,8 @@ void avcodec_get_context_defaults(AVCodecContext *s){
     s->release_buffer= avcodec_default_release_buffer;
     s->get_format= avcodec_default_get_format;
     s->me_subpel_quality=8;
+    s->lmin= FF_QP2LAMBDA * s->qmin;
+    s->lmax= FF_QP2LAMBDA * s->qmax;
     
     s->intra_quant_bias= FF_DEFAULT_QUANT_BIAS;
     s->inter_quant_bias= FF_DEFAULT_QUANT_BIAS;

tests/ffmpeg.regression.ref

@@ -29,9 +29,9 @@ stddev:  8.02 bytes:7602176
 5704a082cc5c5970620123ae20566286 *./data/a-huffyuv.avi
 799d3db687f6cdd7a837ec156efc171f *./data/out.yuv
 stddev:  0.00 bytes:7602176
-2fce40764b56ac4eade4bf29ee962ba0 *./data/a-mpeg4-rc.avi
-580000bfe2d4359d9aa9a9415f953b3b *./data/out.yuv
-stddev: 10.53 bytes:7145472
+0cb2324e62eba4bb94d1cd2946355137 *./data/a-mpeg4-rc.avi
+94f5f1d46c5da6d929eb8633835e73cb *./data/out.yuv
+stddev: 10.18 bytes:7145472
 dfe8fcd409a0605ef0cb37fc5e76ecf7 *./data/a-mpeg4-adv.avi
 c8877a6b8ed290a637c2119860b468b0 *./data/out.yuv
 stddev:  7.31 bytes:7602176

tests/rotozoom.regression.ref

@@ -29,9 +29,9 @@ stddev:  5.28 bytes:7602176
 242a7a18c2793e115007bc163861ef4e *./data/a-huffyuv.avi
 dde5895817ad9d219f79a52d0bdfb001 *./data/out.yuv
 stddev:  0.00 bytes:7602176
-24de1707e32f6a651df9d8c72216aa68 *./data/a-mpeg4-rc.avi
-25f3d554edf174034b5e484a7cfb04d3 *./data/out.yuv
-stddev:  4.32 bytes:7145472
+910c4f83e031bfb0024380d674d79091 *./data/a-mpeg4-rc.avi
+ec7bb694f4eaca71ec7b106d4fe20092 *./data/out.yuv
+stddev:  4.20 bytes:7145472
 bb11d9808375911ea63d4f0889306037 *./data/a-mpeg4-adv.avi
 e465df12f2c8a6492a6ec0b1bddc1d0d *./data/out.yuv
 stddev:  4.97 bytes:7602176