Skip to content

F
ffmpeg.wasm-core
Commit 071e2937 authored by Michael Niedermayer's avatar Michael Niedermayer
Browse files
Options

avcodec/vp9dsp_template: Fix integer overflow(s) in iadst16_1d() 

Fixes: signed integer overflow: 1080285923 - -1130879337 cannot be represented in type 'int'
Fixes: 22002/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_VP9_fuzzer-6260237310099456

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpegSigned-off-by: 's avatarMichael Niedermayer <michael@niedermayer.cc>
parent bafaf951
Hide whitespace changes
Inline Side-by-side
Showing with 76 additions and 76 deletions
libavcodec/vp9dsp_template.c
View file @ 071e2937
......@@ -1378,48 +1378,48 @@ static av_always_inline void iadst16_1d(const dctcoef *in, ptrdiff_t stride,
dctint t0a, t1a, t2a, t3a, t4a, t5a, t6a, t7a;
dctint t8a, t9a, t10a, t11a, t12a, t13a, t14a, t15a;
t0 = IN(15) * 16364 + IN(0) * 804;
t1 = IN(15) * 804 - IN(0) * 16364;
t2 = IN(13) * 15893 + IN(2) * 3981;
t3 = IN(13) * 3981 - IN(2) * 15893;
t4 = IN(11) * 14811 + IN(4) * 7005;
t5 = IN(11) * 7005 - IN(4) * 14811;
t6 = IN(9) * 13160 + IN(6) * 9760;
t7 = IN(9) * 9760 - IN(6) * 13160;
t8 = IN(7) * 11003 + IN(8) * 12140;
t9 = IN(7) * 12140 - IN(8) * 11003;
t10 = IN(5) * 8423 + IN(10) * 14053;
t11 = IN(5) * 14053 - IN(10) * 8423;
t12 = IN(3) * 5520 + IN(12) * 15426;
t13 = IN(3) * 15426 - IN(12) * 5520;
t14 = IN(1) * 2404 + IN(14) * 16207;
t15 = IN(1) * 16207 - IN(14) * 2404;
t0a = (t0 + t8 + (1 << 13)) >> 14;
t1a = (t1 + t9 + (1 << 13)) >> 14;
t2a = (t2 + t10 + (1 << 13)) >> 14;
t3a = (t3 + t11 + (1 << 13)) >> 14;
t4a = (t4 + t12 + (1 << 13)) >> 14;
t5a = (t5 + t13 + (1 << 13)) >> 14;
t6a = (t6 + t14 + (1 << 13)) >> 14;
t7a = (t7 + t15 + (1 << 13)) >> 14;
t8a = (t0 - t8 + (1 << 13)) >> 14;
t9a = (t1 - t9 + (1 << 13)) >> 14;
t10a = (t2 - t10 + (1 << 13)) >> 14;
t11a = (t3 - t11 + (1 << 13)) >> 14;
t12a = (t4 - t12 + (1 << 13)) >> 14;
t13a = (t5 - t13 + (1 << 13)) >> 14;
t14a = (t6 - t14 + (1 << 13)) >> 14;
t15a = (t7 - t15 + (1 << 13)) >> 14;
t8 = t8a * 16069 + t9a * 3196;
t9 = t8a * 3196 - t9a * 16069;
t10 = t10a * 9102 + t11a * 13623;
t11 = t10a * 13623 - t11a * 9102;
t12 = t13a * 16069 - t12a * 3196;
t13 = t13a * 3196 + t12a * 16069;
t14 = t15a * 9102 - t14a * 13623;
t15 = t15a * 13623 + t14a * 9102;
t0 = IN(15) * 16364U + IN(0) * 804U;
t1 = IN(15) * 804U - IN(0) * 16364U;
t2 = IN(13) * 15893U + IN(2) * 3981U;
t3 = IN(13) * 3981U - IN(2) * 15893U;
t4 = IN(11) * 14811U + IN(4) * 7005U;
t5 = IN(11) * 7005U - IN(4) * 14811U;
t6 = IN(9) * 13160U + IN(6) * 9760U;
t7 = IN(9) * 9760U - IN(6) * 13160U;
t8 = IN(7) * 11003U + IN(8) * 12140U;
t9 = IN(7) * 12140U - IN(8) * 11003U;
t10 = IN(5) * 8423U + IN(10) * 14053U;
t11 = IN(5) * 14053U - IN(10) * 8423U;
t12 = IN(3) * 5520U + IN(12) * 15426U;
t13 = IN(3) * 15426U - IN(12) * 5520U;
t14 = IN(1) * 2404U + IN(14) * 16207U;
t15 = IN(1) * 16207U - IN(14) * 2404U;
t0a = (dctint)((1U << 13) + t0 + t8 ) >> 14;
t1a = (dctint)((1U << 13) + t1 + t9 ) >> 14;
t2a = (dctint)((1U << 13) + t2 + t10) >> 14;
t3a = (dctint)((1U << 13) + t3 + t11) >> 14;
t4a = (dctint)((1U << 13) + t4 + t12) >> 14;
t5a = (dctint)((1U << 13) + t5 + t13) >> 14;
t6a = (dctint)((1U << 13) + t6 + t14) >> 14;
t7a = (dctint)((1U << 13) + t7 + t15) >> 14;
t8a = (dctint)((1U << 13) + t0 - t8 ) >> 14;
t9a = (dctint)((1U << 13) + t1 - t9 ) >> 14;
t10a = (dctint)((1U << 13) + t2 - t10) >> 14;
t11a = (dctint)((1U << 13) + t3 - t11) >> 14;
t12a = (dctint)((1U << 13) + t4 - t12) >> 14;
t13a = (dctint)((1U << 13) + t5 - t13) >> 14;
t14a = (dctint)((1U << 13) + t6 - t14) >> 14;
t15a = (dctint)((1U << 13) + t7 - t15) >> 14;
t8 = t8a * 16069U + t9a * 3196U;
t9 = t8a * 3196U - t9a * 16069U;
t10 = t10a * 9102U + t11a * 13623U;
t11 = t10a * 13623U - t11a * 9102U;
t12 = t13a * 16069U - t12a * 3196U;
t13 = t13a * 3196U + t12a * 16069U;
t14 = t15a * 9102U - t14a * 13623U;
t15 = t15a * 13623U + t14a * 9102U;
t0 = t0a + t4a;
t1 = t1a + t5a;
......@@ -1429,49 +1429,49 @@ static av_always_inline void iadst16_1d(const dctcoef *in, ptrdiff_t stride,
t5 = t1a - t5a;
t6 = t2a - t6a;
t7 = t3a - t7a;
t8a = (t8 + t12 + (1 << 13)) >> 14;
t9a = (t9 + t13 + (1 << 13)) >> 14;
t10a = (t10 + t14 + (1 << 13)) >> 14;
t11a = (t11 + t15 + (1 << 13)) >> 14;
t12a = (t8 - t12 + (1 << 13)) >> 14;
t13a = (t9 - t13 + (1 << 13)) >> 14;
t14a = (t10 - t14 + (1 << 13)) >> 14;
t15a = (t11 - t15 + (1 << 13)) >> 14;
t4a = t4 * 15137 + t5 * 6270;
t5a = t4 * 6270 - t5 * 15137;
t6a = t7 * 15137 - t6 * 6270;
t7a = t7 * 6270 + t6 * 15137;
t12 = t12a * 15137 + t13a * 6270;
t13 = t12a * 6270 - t13a * 15137;
t14 = t15a * 15137 - t14a * 6270;
t15 = t15a * 6270 + t14a * 15137;
t8a = (dctint)((1U << 13) + t8 + t12) >> 14;
t9a = (dctint)((1U << 13) + t9 + t13) >> 14;
t10a = (dctint)((1U << 13) + t10 + t14) >> 14;
t11a = (dctint)((1U << 13) + t11 + t15) >> 14;
t12a = (dctint)((1U << 13) + t8 - t12) >> 14;
t13a = (dctint)((1U << 13) + t9 - t13) >> 14;
t14a = (dctint)((1U << 13) + t10 - t14) >> 14;
t15a = (dctint)((1U << 13) + t11 - t15) >> 14;
t4a = t4 * 15137U + t5 * 6270U;
t5a = t4 * 6270U - t5 * 15137U;
t6a = t7 * 15137U - t6 * 6270U;
t7a = t7 * 6270U + t6 * 15137U;
t12 = t12a * 15137U + t13a * 6270U;
t13 = t12a * 6270U - t13a * 15137U;
t14 = t15a * 15137U - t14a * 6270U;
t15 = t15a * 6270U + t14a * 15137U;
out[ 0] = t0 + t2;
out[15] = -(t1 + t3);
t2a = t0 - t2;
t3a = t1 - t3;
out[ 3] = -((t4a + t6a + (1 << 13)) >> 14);
out[12] = (t5a + t7a + (1 << 13)) >> 14;
t6 = (t4a - t6a + (1 << 13)) >> 14;
t7 = (t5a - t7a + (1 << 13)) >> 14;
out[ 3] = -((dctint)((1U << 13) + t4a + t6a) >> 14);
out[12] = (dctint)((1U << 13) + t5a + t7a) >> 14;
t6 = (dctint)((1U << 13) + t4a - t6a) >> 14;
t7 = (dctint)((1U << 13) + t5a - t7a) >> 14;
out[ 1] = -(t8a + t10a);
out[14] = t9a + t11a;
t10 = t8a - t10a;
t11 = t9a - t11a;
out[ 2] = (t12 + t14 + (1 << 13)) >> 14;
out[13] = -((t13 + t15 + (1 << 13)) >> 14);
t14a = (t12 - t14 + (1 << 13)) >> 14;
t15a = (t13 - t15 + (1 << 13)) >> 14;
out[ 7] = ((t2a + t3a) * -11585 + (1 << 13)) >> 14;
out[ 8] = ((t2a - t3a) * 11585 + (1 << 13)) >> 14;
out[ 4] = ((t7 + t6) * 11585 + (1 << 13)) >> 14;
out[11] = ((t7 - t6) * 11585 + (1 << 13)) >> 14;
out[ 6] = ((t11 + t10) * 11585 + (1 << 13)) >> 14;
out[ 9] = ((t11 - t10) * 11585 + (1 << 13)) >> 14;
out[ 5] = ((t14a + t15a) * -11585 + (1 << 13)) >> 14;
out[10] = ((t14a - t15a) * 11585 + (1 << 13)) >> 14;
out[ 2] = (dctint)((1U << 13) + t12 + t14) >> 14;
out[13] = -((dctint)((1U << 13) + t13 + t15) >> 14);
t14a = (dctint)((1U << 13) + t12 - t14) >> 14;
t15a = (dctint)((1U << 13) + t13 - t15) >> 14;
out[ 7] = (dctint)(-(t2a + t3a) * 11585U + (1 << 13)) >> 14;
out[ 8] = (dctint)( (t2a - t3a) * 11585U + (1 << 13)) >> 14;
out[ 4] = (dctint)( (t7 + t6) * 11585U + (1 << 13)) >> 14;
out[11] = (dctint)( (t7 - t6) * 11585U + (1 << 13)) >> 14;
out[ 6] = (dctint)( (t11 + t10) * 11585U + (1 << 13)) >> 14;
out[ 9] = (dctint)( (t11 - t10) * 11585U + (1 << 13)) >> 14;
out[ 5] = (dctint)(-(t14a + t15a) * 11585U + (1 << 13)) >> 14;
out[10] = (dctint)( (t14a - t15a) * 11585U + (1 << 13)) >> 14;
}
itxfm_wrap(16, 6)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment