Commit ac78014f authored by Michael Niedermayer's avatar Michael Niedermayer

avcodec/motion_est: Fix some undefined shifts

Fixes: asan_heap-oob_1dd60fd_1049_cov_4200102444_P4250048.MOV

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: 's avatarMichael Niedermayer <michaelni@gmx.at>
parent f9112664
......@@ -426,13 +426,13 @@ static int sad_hpel_motion_search(MpegEncContext * s,
my > ymin && my < ymax) {
int dx=0, dy=0;
int d, pen_x, pen_y;
const int index= (my<<ME_MAP_SHIFT) + mx;
const int index= my*(1<<ME_MAP_SHIFT) + mx;
const int t= score_map[(index-(1<<ME_MAP_SHIFT))&(ME_MAP_SIZE-1)];
const int l= score_map[(index- 1 )&(ME_MAP_SIZE-1)];
const int r= score_map[(index+ 1 )&(ME_MAP_SIZE-1)];
const int b= score_map[(index+(1<<ME_MAP_SHIFT))&(ME_MAP_SIZE-1)];
mx<<=1;
my<<=1;
mx += mx;
my += my;
pen_x= pred_x + mx;
......@@ -490,8 +490,8 @@ static int sad_hpel_motion_search(MpegEncContext * s,
my+=dy;
}else{
mx<<=1;
my<<=1;
mx += mx;
my += my;
}
*mx_ptr = mx;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment