Skip to content

F
ffmpeg.wasm-core
Commit 6d93307f authored by Janne Grunau's avatar Janne Grunau
Browse files
Options

mpeg12: check scantable indices in all decode_block functions 

Add checks to the fast functions used with CODEC_FLAGS2_FAST and move
the check for all other functions to before the invalid memory is
accessed. Fixes https://trac.videolan.org/vlc/ticket/9713 with
CODEC_FLAGS2_FAST.

CC: libav-stable@libav.org
parent fb0c9d41
Show whitespace changes
Inline Side-by-side
Showing with 31 additions and 23 deletions
libavcodec/mpeg12dec.c
View file @ 6d93307f
...@@ -119,6 +119,15 @@ static int mpeg_decode_motion(MpegEncContext *s, int fcode, int pred) ...@@ -119,6 +119,15 @@ static int mpeg_decode_motion(MpegEncContext *s, int fcode, int pred)
return sign_extend(val, 5 + shift); return sign_extend(val, 5 + shift);
} }
#define check_scantable_index(ctx, x) \
do { \
if ((x) > 63) { \
av_log(ctx->avctx, AV_LOG_ERROR, "ac-tex damaged at %d %d\n", \
ctx->mb_x, ctx->mb_y); \
return AVERROR_INVALIDDATA; \
} \
} while (0) \
static inline int mpeg1_decode_block_intra(MpegEncContext *s, int16_t *block, int n) static inline int mpeg1_decode_block_intra(MpegEncContext *s, int16_t *block, int n)
{ {
int level, dc, diff, i, j, run; int level, dc, diff, i, j, run;
...@@ -150,6 +159,7 @@ static inline int mpeg1_decode_block_intra(MpegEncContext *s, int16_t *block, in ...@@ -150,6 +159,7 @@ static inline int mpeg1_decode_block_intra(MpegEncContext *s, int16_t *block, in
break; break;
} else if (level != 0) { } else if (level != 0) {
i += run; i += run;
check_scantable_index(s, i);
j = scantable[i]; j = scantable[i];
level = (level * qscale * quant_matrix[j]) >> 4; level = (level * qscale * quant_matrix[j]) >> 4;
level = (level - 1) | 1; level = (level - 1) | 1;
...@@ -166,6 +176,7 @@ static inline int mpeg1_decode_block_intra(MpegEncContext *s, int16_t *block, in ...@@ -166,6 +176,7 @@ static inline int mpeg1_decode_block_intra(MpegEncContext *s, int16_t *block, in
level = SHOW_UBITS(re, &s->gb, 8) ; LAST_SKIP_BITS(re, &s->gb, 8); level = SHOW_UBITS(re, &s->gb, 8) ; LAST_SKIP_BITS(re, &s->gb, 8);
} }
i += run; i += run;
check_scantable_index(s, i);
j = scantable[i]; j = scantable[i];
if (level < 0) { if (level < 0) {
level = -level; level = -level;
...@@ -177,10 +188,6 @@ static inline int mpeg1_decode_block_intra(MpegEncContext *s, int16_t *block, in ...@@ -177,10 +188,6 @@ static inline int mpeg1_decode_block_intra(MpegEncContext *s, int16_t *block, in
level = (level - 1) | 1; level = (level - 1) | 1;
} }
} }
if (i > 63) {
av_log(s->avctx, AV_LOG_ERROR, "ac-tex damaged at %d %d\n", s->mb_x, s->mb_y);
return -1;
}
block[j] = level; block[j] = level;
} }
...@@ -225,6 +232,7 @@ static inline int mpeg1_decode_block_inter(MpegEncContext *s, int16_t *block, in ...@@ -225,6 +232,7 @@ static inline int mpeg1_decode_block_inter(MpegEncContext *s, int16_t *block, in
if (level != 0) { if (level != 0) {
i += run; i += run;
check_scantable_index(s, i);
j = scantable[i]; j = scantable[i];
level = ((level * 2 + 1) * qscale * quant_matrix[j]) >> 5; level = ((level * 2 + 1) * qscale * quant_matrix[j]) >> 5;
level = (level - 1) | 1; level = (level - 1) | 1;
...@@ -241,6 +249,7 @@ static inline int mpeg1_decode_block_inter(MpegEncContext *s, int16_t *block, in ...@@ -241,6 +249,7 @@ static inline int mpeg1_decode_block_inter(MpegEncContext *s, int16_t *block, in
level = SHOW_UBITS(re, &s->gb, 8) ; SKIP_BITS(re, &s->gb, 8); level = SHOW_UBITS(re, &s->gb, 8) ; SKIP_BITS(re, &s->gb, 8);
} }
i += run; i += run;
check_scantable_index(s, i);
j = scantable[i]; j = scantable[i];
if (level < 0) { if (level < 0) {
level = -level; level = -level;
...@@ -252,10 +261,6 @@ static inline int mpeg1_decode_block_inter(MpegEncContext *s, int16_t *block, in ...@@ -252,10 +261,6 @@ static inline int mpeg1_decode_block_inter(MpegEncContext *s, int16_t *block, in
level = (level - 1) | 1; level = (level - 1) | 1;
} }
} }
if (i > 63) {
av_log(s->avctx, AV_LOG_ERROR, "ac-tex damaged at %d %d\n", s->mb_x, s->mb_y);
return -1;
}
block[j] = level; block[j] = level;
if (((int32_t)GET_CACHE(re, &s->gb)) <= (int32_t)0xBFFFFFFF) if (((int32_t)GET_CACHE(re, &s->gb)) <= (int32_t)0xBFFFFFFF)
...@@ -300,6 +305,7 @@ static inline int mpeg1_fast_decode_block_inter(MpegEncContext *s, int16_t *bloc ...@@ -300,6 +305,7 @@ static inline int mpeg1_fast_decode_block_inter(MpegEncContext *s, int16_t *bloc
if (level != 0) { if (level != 0) {
i += run; i += run;
check_scantable_index(s, i);
j = scantable[i]; j = scantable[i];
level = ((level * 2 + 1) * qscale) >> 1; level = ((level * 2 + 1) * qscale) >> 1;
level = (level - 1) | 1; level = (level - 1) | 1;
...@@ -316,6 +322,7 @@ static inline int mpeg1_fast_decode_block_inter(MpegEncContext *s, int16_t *bloc ...@@ -316,6 +322,7 @@ static inline int mpeg1_fast_decode_block_inter(MpegEncContext *s, int16_t *bloc
level = SHOW_UBITS(re, &s->gb, 8) ; SKIP_BITS(re, &s->gb, 8); level = SHOW_UBITS(re, &s->gb, 8) ; SKIP_BITS(re, &s->gb, 8);
} }
i += run; i += run;
check_scantable_index(s, i);
j = scantable[i]; j = scantable[i];
if (level < 0) { if (level < 0) {
level = -level; level = -level;
...@@ -381,6 +388,7 @@ static inline int mpeg2_decode_block_non_intra(MpegEncContext *s, int16_t *block ...@@ -381,6 +388,7 @@ static inline int mpeg2_decode_block_non_intra(MpegEncContext *s, int16_t *block
if (level != 0) { if (level != 0) {
i += run; i += run;
check_scantable_index(s, i);
j = scantable[i]; j = scantable[i];
level = ((level * 2 + 1) * qscale * quant_matrix[j]) >> 5; level = ((level * 2 + 1) * qscale * quant_matrix[j]) >> 5;
level = (level ^ SHOW_SBITS(re, &s->gb, 1)) - SHOW_SBITS(re, &s->gb, 1); level = (level ^ SHOW_SBITS(re, &s->gb, 1)) - SHOW_SBITS(re, &s->gb, 1);
...@@ -392,6 +400,7 @@ static inline int mpeg2_decode_block_non_intra(MpegEncContext *s, int16_t *block ...@@ -392,6 +400,7 @@ static inline int mpeg2_decode_block_non_intra(MpegEncContext *s, int16_t *block
level = SHOW_SBITS(re, &s->gb, 12); SKIP_BITS(re, &s->gb, 12); level = SHOW_SBITS(re, &s->gb, 12); SKIP_BITS(re, &s->gb, 12);
i += run; i += run;
check_scantable_index(s, i);
j = scantable[i]; j = scantable[i];
if (level < 0) { if (level < 0) {
level = ((-level * 2 + 1) * qscale * quant_matrix[j]) >> 5; level = ((-level * 2 + 1) * qscale * quant_matrix[j]) >> 5;
...@@ -400,10 +409,6 @@ static inline int mpeg2_decode_block_non_intra(MpegEncContext *s, int16_t *block ...@@ -400,10 +409,6 @@ static inline int mpeg2_decode_block_non_intra(MpegEncContext *s, int16_t *block
level = ((level * 2 + 1) * qscale * quant_matrix[j]) >> 5; level = ((level * 2 + 1) * qscale * quant_matrix[j]) >> 5;
} }
} }
if (i > 63) {
av_log(s->avctx, AV_LOG_ERROR, "ac-tex damaged at %d %d\n", s->mb_x, s->mb_y);
return -1;
}
mismatch ^= level; mismatch ^= level;
block[j] = level; block[j] = level;
...@@ -450,6 +455,7 @@ static inline int mpeg2_fast_decode_block_non_intra(MpegEncContext *s, ...@@ -450,6 +455,7 @@ static inline int mpeg2_fast_decode_block_non_intra(MpegEncContext *s,
if (level != 0) { if (level != 0) {
i += run; i += run;
check_scantable_index(s, i);
j = scantable[i]; j = scantable[i];
level = ((level * 2 + 1) * qscale) >> 1; level = ((level * 2 + 1) * qscale) >> 1;
level = (level ^ SHOW_SBITS(re, &s->gb, 1)) - SHOW_SBITS(re, &s->gb, 1); level = (level ^ SHOW_SBITS(re, &s->gb, 1)) - SHOW_SBITS(re, &s->gb, 1);
...@@ -461,6 +467,7 @@ static inline int mpeg2_fast_decode_block_non_intra(MpegEncContext *s, ...@@ -461,6 +467,7 @@ static inline int mpeg2_fast_decode_block_non_intra(MpegEncContext *s,
level = SHOW_SBITS(re, &s->gb, 12); SKIP_BITS(re, &s->gb, 12); level = SHOW_SBITS(re, &s->gb, 12); SKIP_BITS(re, &s->gb, 12);
i += run; i += run;
check_scantable_index(s, i);
j = scantable[i]; j = scantable[i];
if (level < 0) { if (level < 0) {
level = ((-level * 2 + 1) * qscale) >> 1; level = ((-level * 2 + 1) * qscale) >> 1;
...@@ -527,6 +534,7 @@ static inline int mpeg2_decode_block_intra(MpegEncContext *s, int16_t *block, in ...@@ -527,6 +534,7 @@ static inline int mpeg2_decode_block_intra(MpegEncContext *s, int16_t *block, in
break; break;
} else if (level != 0) { } else if (level != 0) {
i += run; i += run;
check_scantable_index(s, i);
j = scantable[i]; j = scantable[i];
level = (level * qscale * quant_matrix[j]) >> 4; level = (level * qscale * quant_matrix[j]) >> 4;
level = (level ^ SHOW_SBITS(re, &s->gb, 1)) - SHOW_SBITS(re, &s->gb, 1); level = (level ^ SHOW_SBITS(re, &s->gb, 1)) - SHOW_SBITS(re, &s->gb, 1);
...@@ -537,6 +545,7 @@ static inline int mpeg2_decode_block_intra(MpegEncContext *s, int16_t *block, in ...@@ -537,6 +545,7 @@ static inline int mpeg2_decode_block_intra(MpegEncContext *s, int16_t *block, in
UPDATE_CACHE(re, &s->gb); UPDATE_CACHE(re, &s->gb);
level = SHOW_SBITS(re, &s->gb, 12); SKIP_BITS(re, &s->gb, 12); level = SHOW_SBITS(re, &s->gb, 12); SKIP_BITS(re, &s->gb, 12);
i += run; i += run;
check_scantable_index(s, i);
j = scantable[i]; j = scantable[i];
if (level < 0) { if (level < 0) {
level = (-level * qscale * quant_matrix[j]) >> 4; level = (-level * qscale * quant_matrix[j]) >> 4;
...@@ -545,10 +554,6 @@ static inline int mpeg2_decode_block_intra(MpegEncContext *s, int16_t *block, in ...@@ -545,10 +554,6 @@ static inline int mpeg2_decode_block_intra(MpegEncContext *s, int16_t *block, in
level = (level * qscale * quant_matrix[j]) >> 4; level = (level * qscale * quant_matrix[j]) >> 4;
} }
} }
if (i > 63) {
av_log(s->avctx, AV_LOG_ERROR, "ac-tex damaged at %d %d\n", s->mb_x, s->mb_y);
return -1;
}
mismatch ^= level; mismatch ^= level;
block[j] = level; block[j] = level;
...@@ -563,10 +568,10 @@ static inline int mpeg2_decode_block_intra(MpegEncContext *s, int16_t *block, in ...@@ -563,10 +568,10 @@ static inline int mpeg2_decode_block_intra(MpegEncContext *s, int16_t *block, in
static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s, int16_t *block, int n) static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s, int16_t *block, int n)
{ {
int level, dc, diff, j, run; int level, dc, diff, i, j, run;
int component; int component;
RLTable *rl; RLTable *rl;
uint8_t * scantable = s->intra_scantable.permutated; uint8_t * const scantable = s->intra_scantable.permutated;
const uint16_t *quant_matrix; const uint16_t *quant_matrix;
const int qscale = s->qscale; const int qscale = s->qscale;
...@@ -585,6 +590,7 @@ static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s, int16_t *bloc ...@@ -585,6 +590,7 @@ static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s, int16_t *bloc
dc += diff; dc += diff;
s->last_dc[component] = dc; s->last_dc[component] = dc;
block[0] = dc << (3 - s->intra_dc_precision); block[0] = dc << (3 - s->intra_dc_precision);
i = 0;
if (s->intra_vlc_format) if (s->intra_vlc_format)
rl = &ff_rl_mpeg2; rl = &ff_rl_mpeg2;
else else
...@@ -600,8 +606,9 @@ static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s, int16_t *bloc ...@@ -600,8 +606,9 @@ static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s, int16_t *bloc
if (level == 127) { if (level == 127) {
break; break;
} else if (level != 0) { } else if (level != 0) {
scantable += run; i += run;
j = *scantable; check_scantable_index(s, i);
j = scantable[i];
level = (level * qscale * quant_matrix[j]) >> 4; level = (level * qscale * quant_matrix[j]) >> 4;
level = (level ^ SHOW_SBITS(re, &s->gb, 1)) - SHOW_SBITS(re, &s->gb, 1); level = (level ^ SHOW_SBITS(re, &s->gb, 1)) - SHOW_SBITS(re, &s->gb, 1);
LAST_SKIP_BITS(re, &s->gb, 1); LAST_SKIP_BITS(re, &s->gb, 1);
...@@ -610,8 +617,9 @@ static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s, int16_t *bloc ...@@ -610,8 +617,9 @@ static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s, int16_t *bloc
run = SHOW_UBITS(re, &s->gb, 6) + 1; LAST_SKIP_BITS(re, &s->gb, 6); run = SHOW_UBITS(re, &s->gb, 6) + 1; LAST_SKIP_BITS(re, &s->gb, 6);
UPDATE_CACHE(re, &s->gb); UPDATE_CACHE(re, &s->gb);
level = SHOW_SBITS(re, &s->gb, 12); SKIP_BITS(re, &s->gb, 12); level = SHOW_SBITS(re, &s->gb, 12); SKIP_BITS(re, &s->gb, 12);
scantable += run; i += run;
j = *scantable; check_scantable_index(s, i);
j = scantable[i];
if (level < 0) { if (level < 0) {
level = (-level * qscale * quant_matrix[j]) >> 4; level = (-level * qscale * quant_matrix[j]) >> 4;
level = -level; level = -level;
...@@ -625,7 +633,7 @@ static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s, int16_t *bloc ...@@ -625,7 +633,7 @@ static inline int mpeg2_fast_decode_block_intra(MpegEncContext *s, int16_t *bloc
CLOSE_READER(re, &s->gb); CLOSE_READER(re, &s->gb);
} }
s->block_last_index[n] = scantable - s->intra_scantable.permutated; s->block_last_index[n] = i;
return 0; return 0;
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment