Commit cec93959 authored by Loren Merritt's avatar Loren Merritt

fix some potential arithmetic overflows in pred_direct_motion() and

ff_h264_weight_WxH_mmx2().

Originally committed as revision 4795 to svn://svn.ffmpeg.org/ffmpeg/trunk
parent 003cd80d
......@@ -1286,8 +1286,8 @@ static inline void pred_direct_motion(H264Context * const h, int *mb_type){
}
if(IS_16X16(*mb_type)){
fill_rectangle(&h->ref_cache[0][scan8[0]], 4, 4, 8, ref[0], 1);
fill_rectangle(&h->ref_cache[1][scan8[0]], 4, 4, 8, ref[1], 1);
fill_rectangle(&h->ref_cache[0][scan8[0]], 4, 4, 8, (uint8_t)ref[0], 1);
fill_rectangle(&h->ref_cache[1][scan8[0]], 4, 4, 8, (uint8_t)ref[1], 1);
if(!IS_INTRA(mb_type_col)
&& ( (l1ref0[0] == 0 && ABS(l1mv0[0][0]) <= 1 && ABS(l1mv0[0][1]) <= 1)
|| (l1ref0[0] < 0 && l1ref1[0] == 0 && ABS(l1mv1[0][0]) <= 1 && ABS(l1mv1[0][1]) <= 1
......@@ -1315,8 +1315,8 @@ static inline void pred_direct_motion(H264Context * const h, int *mb_type){
fill_rectangle(&h->mv_cache[0][scan8[i8*4]], 2, 2, 8, pack16to32(mv[0][0],mv[0][1]), 4);
fill_rectangle(&h->mv_cache[1][scan8[i8*4]], 2, 2, 8, pack16to32(mv[1][0],mv[1][1]), 4);
fill_rectangle(&h->ref_cache[0][scan8[i8*4]], 2, 2, 8, ref[0], 1);
fill_rectangle(&h->ref_cache[1][scan8[i8*4]], 2, 2, 8, ref[1], 1);
fill_rectangle(&h->ref_cache[0][scan8[i8*4]], 2, 2, 8, (uint8_t)ref[0], 1);
fill_rectangle(&h->ref_cache[1][scan8[i8*4]], 2, 2, 8, (uint8_t)ref[1], 1);
/* col_zero_flag */
if(!IS_INTRA(mb_type_col) && ( l1ref0[x8 + y8*h->b8_stride] == 0
......
......@@ -518,9 +518,9 @@ static void OPNAME ## h264_qpel4_hv_lowpass_ ## MMX(uint8_t *dst, int16_t *tmp,
"psraw $2, %%mm0 \n\t"/*(a-b)/4 */\
"psubw %%mm1, %%mm0 \n\t"/*(a-b)/4-b */\
"paddsw %%mm2, %%mm0 \n\t"\
"psraw $2, %%mm0 \n\t"/*((a-b)/4-b)/4 */\
"psraw $2, %%mm0 \n\t"/*((a-b)/4-b+c)/4 */\
"paddw %%mm6, %%mm2 \n\t"\
"paddw %%mm2, %%mm0 \n\t"\
"paddw %%mm2, %%mm0 \n\t"/*(a-5*b+20*c)/16 +32 */\
"psraw $6, %%mm0 \n\t"\
"packuswb %%mm0, %%mm0 \n\t"\
OP(%%mm0, (%1),%%mm7, d)\
......@@ -955,8 +955,8 @@ static inline void ff_h264_weight_WxH_mmx2(uint8_t *dst, int stride, int log2_de
"punpcklbw %%mm7, %%mm1 \n\t"
"pmullw %%mm4, %%mm0 \n\t"
"pmullw %%mm4, %%mm1 \n\t"
"paddw %%mm5, %%mm0 \n\t"
"paddw %%mm5, %%mm1 \n\t"
"paddsw %%mm5, %%mm0 \n\t"
"paddsw %%mm5, %%mm1 \n\t"
"psraw %%mm6, %%mm0 \n\t"
"psraw %%mm6, %%mm1 \n\t"
"packuswb %%mm7, %%mm0 \n\t"
......@@ -995,8 +995,8 @@ static inline void ff_h264_biweight_WxH_mmx2(uint8_t *dst, uint8_t *src, int str
"punpcklbw %%mm7, %%mm1 \n\t"
"pmullw %%mm3, %%mm0 \n\t"
"pmullw %%mm4, %%mm1 \n\t"
"paddw %%mm5, %%mm0 \n\t"
"paddw %%mm1, %%mm0 \n\t"
"paddsw %%mm1, %%mm0 \n\t"
"paddsw %%mm5, %%mm0 \n\t"
"psraw %%mm6, %%mm0 \n\t"
"packuswb %%mm0, %%mm0 \n\t"
"movd %%mm0, %0 \n\t"
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment