clip input motion vectors

be somewhat more tollerant on invalid input
return INT_MAX instead of -1 for unuseable mv/mb types as that ensures nicely that they arent used
initalize limits earlier for b frames
a few more asserts to check for out of picture vectors

Originally committed as revision 3213 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavcodec/motion_est.c

@@ -980,6 +980,16 @@ static int interlaced_search(MpegEncContext *s, int ref_index,
     }
 }
 
+static void clip_input_mv(MpegEncContext * s, int16_t *mv, int interlaced){
+    int ymax= s->me.ymax>>interlaced;
+    int ymin= s->me.ymin>>interlaced;
+    
+    if(mv[0] < s->me.xmin) mv[0] = s->me.xmin;
+    if(mv[0] > s->me.xmax) mv[0] = s->me.xmax;
+    if(mv[1] <       ymin) mv[1] =       ymin;
+    if(mv[1] >       ymax) mv[1] =       ymax;
+}
+
 static inline int check_input_motion(MpegEncContext * s, int mb_x, int mb_y, int p_type){
     MotionEstContext * const c= &s->me;
     Picture *p= s->current_picture_ptr;
@@ -994,9 +1004,18 @@ static inline int check_input_motion(MpegEncContext * s, int mb_x, int mb_y, int
     me_cmp_func cmpf= s->dsp.sse[0];
     me_cmp_func chroma_cmpf= s->dsp.sse[1];
     
-    assert(p_type==0 || !USES_LIST(mb_type, 1));
+    if(p_type && USES_LIST(mb_type, 1)){
+        av_log(c->avctx, AV_LOG_ERROR, "backward motion vector in P frame\n");
+        return INT_MAX;
+    }
     assert(IS_INTRA(mb_type) || USES_LIST(mb_type,0) || USES_LIST(mb_type,1));
     
+    for(i=0; i<4; i++){
+        int xy= s->block_index[i];
+        clip_input_mv(s, p->motion_val[0][xy], !!IS_INTERLACED(mb_type));
+        clip_input_mv(s, p->motion_val[1][xy], !!IS_INTERLACED(mb_type));
+    }
+
     if(IS_INTERLACED(mb_type)){
         int xy2= xy  + s->b8_stride;
         s->mb_type[mb_xy]=CANDIDATE_MB_TYPE_INTRA;
@@ -1005,7 +1024,7 @@ static inline int check_input_motion(MpegEncContext * s, int mb_x, int mb_y, int
         
         if(!(s->flags & CODEC_FLAG_INTERLACED_ME)){
             av_log(c->avctx, AV_LOG_ERROR, "Interlaced macroblock selected but interlaced motion estimation disabled\n");
-            return -1;
+            return INT_MAX;
         }
 
         if(USES_LIST(mb_type, 0)){
@@ -1066,7 +1085,7 @@ static inline int check_input_motion(MpegEncContext * s, int mb_x, int mb_y, int
     }else if(IS_8X8(mb_type)){
         if(!(s->flags & CODEC_FLAG_4MV)){
             av_log(c->avctx, AV_LOG_ERROR, "4MV macroblock selected but 4MV encoding disabled\n");
-            return -1;
+            return INT_MAX;
         }
         cmpf= s->dsp.sse[1];
         chroma_cmpf= s->dsp.sse[1];
@@ -1723,6 +1742,7 @@ void ff_estimate_b_frame_motion(MpegEncContext * s,
     const int xy = mb_y*s->mb_stride + mb_x;
     init_ref(c, s->new_picture.data, s->last_picture.data, s->next_picture.data, 16*mb_x, 16*mb_y, 2);
 
+    get_limits(s, 16*mb_x, 16*mb_y);
     
     c->skip=0;
     if(c->avctx->me_threshold){

libavcodec/motion_est_template.c

@@ -487,6 +487,10 @@ static int qpel_motion_search(MpegEncContext * s,
 {\
     const int key= ((y)<<ME_MAP_MV_BITS) + (x) + map_generation;\
     const int index= (((y)<<ME_MAP_SHIFT) + (x))&(ME_MAP_SIZE-1);\
+    assert((x) >= xmin);\
+    assert((x) <= xmax);\
+    assert((y) >= ymin);\
+    assert((y) <= ymax);\
 /*printf("check_mv %d %d\n", x, y);*/\
     if(map[index]!=key){\
         d= cmp(s, x, y, 0, 0, size, h, ref_index, src_index, cmpf, chroma_cmpf, flags);\