Commit 8824b737 authored by Michael Niedermayer's avatar Michael Niedermayer

avcodec/vp8dsp: Fixes: runtime error: signed integer overflow: 1330143360 -...

avcodec/vp8dsp: Fixes: runtime error: signed integer overflow: 1330143360 - -1023040530 cannot be represented in type 'int'

Fixes: 1406/clusterfuzz-testcase-minimized-5064865125236736

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpegSigned-off-by: 's avatarMichael Niedermayer <michael@niedermayer.cc>
parent 5d5118f8
......@@ -95,7 +95,8 @@ static void vp7_luma_dc_wht_dc_c(int16_t block[4][4][16], int16_t dc[16])
static void vp7_idct_add_c(uint8_t *dst, int16_t block[16], ptrdiff_t stride)
{
int i, a1, b1, c1, d1;
int i;
unsigned a1, b1, c1, d1;
int16_t tmp[16];
for (i = 0; i < 4; i++) {
......@@ -104,10 +105,10 @@ static void vp7_idct_add_c(uint8_t *dst, int16_t block[16], ptrdiff_t stride)
c1 = block[i * 4 + 1] * 12540 - block[i * 4 + 3] * 30274;
d1 = block[i * 4 + 1] * 30274 + block[i * 4 + 3] * 12540;
AV_ZERO64(block + i * 4);
tmp[i * 4 + 0] = (a1 + d1) >> 14;
tmp[i * 4 + 3] = (a1 - d1) >> 14;
tmp[i * 4 + 1] = (b1 + c1) >> 14;
tmp[i * 4 + 2] = (b1 - c1) >> 14;
tmp[i * 4 + 0] = (int)(a1 + d1) >> 14;
tmp[i * 4 + 3] = (int)(a1 - d1) >> 14;
tmp[i * 4 + 1] = (int)(b1 + c1) >> 14;
tmp[i * 4 + 2] = (int)(b1 - c1) >> 14;
}
for (i = 0; i < 4; i++) {
......@@ -116,13 +117,13 @@ static void vp7_idct_add_c(uint8_t *dst, int16_t block[16], ptrdiff_t stride)
c1 = tmp[i + 4] * 12540 - tmp[i + 12] * 30274;
d1 = tmp[i + 4] * 30274 + tmp[i + 12] * 12540;
dst[0 * stride + i] = av_clip_uint8(dst[0 * stride + i] +
((a1 + d1 + 0x20000) >> 18));
((int)(a1 + d1 + 0x20000) >> 18));
dst[3 * stride + i] = av_clip_uint8(dst[3 * stride + i] +
((a1 - d1 + 0x20000) >> 18));
((int)(a1 - d1 + 0x20000) >> 18));
dst[1 * stride + i] = av_clip_uint8(dst[1 * stride + i] +
((b1 + c1 + 0x20000) >> 18));
((int)(b1 + c1 + 0x20000) >> 18));
dst[2 * stride + i] = av_clip_uint8(dst[2 * stride + i] +
((b1 - c1 + 0x20000) >> 18));
((int)(b1 - c1 + 0x20000) >> 18));
}
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment