Commit 58cf31ce authored by Michael Niedermayer's avatar Michael Niedermayer

avcodec/x86/mpegvideodsp: Fix signedness bug in need_emu

Fixes: out of array read
Fixes: 3516/attachment-311488.dat

Found-by: Insu Yun, Georgia Tech.
Tested-by: wuninsu@gmail.com
Signed-off-by: 's avatarMichael Niedermayer <michael@niedermayer.cc>
parent 880b2993
...@@ -52,8 +52,9 @@ static void gmc_mmx(uint8_t *dst, uint8_t *src, ...@@ -52,8 +52,9 @@ static void gmc_mmx(uint8_t *dst, uint8_t *src,
const int dyh = (dyy - (1 << (16 + shift))) * (h - 1); const int dyh = (dyy - (1 << (16 + shift))) * (h - 1);
const int dxh = dxy * (h - 1); const int dxh = dxy * (h - 1);
const int dyw = dyx * (w - 1); const int dyw = dyx * (w - 1);
int need_emu = (unsigned) ix >= width - w || int need_emu = (unsigned) ix >= width - w || width < w ||
(unsigned) iy >= height - h; (unsigned) iy >= height - h || height< h
;
if ( // non-constant fullpel offset (3% of blocks) if ( // non-constant fullpel offset (3% of blocks)
((ox ^ (ox + dxw)) | (ox ^ (ox + dxh)) | (ox ^ (ox + dxw + dxh)) | ((ox ^ (ox + dxw)) | (ox ^ (ox + dxh)) | (ox ^ (ox + dxw + dxh)) |
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment