Commit 23f3f923 authored by Michael Niedermayer's avatar Michael Niedermayer

avcodec/mjpegdec: quant_matrixes can be up to 65535, use uint16_t

Fixes invalid shift
Fixes: 870/clusterfuzz-testcase-5649105424482304

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpegSigned-off-by: 's avatarMichael Niedermayer <michael@niedermayer.cc>
parent 656a17e1
...@@ -686,7 +686,7 @@ static inline int mjpeg_decode_dc(MJpegDecodeContext *s, int dc_index) ...@@ -686,7 +686,7 @@ static inline int mjpeg_decode_dc(MJpegDecodeContext *s, int dc_index)
/* decode block and dequantize */ /* decode block and dequantize */
static int decode_block(MJpegDecodeContext *s, int16_t *block, int component, static int decode_block(MJpegDecodeContext *s, int16_t *block, int component,
int dc_index, int ac_index, int16_t *quant_matrix) int dc_index, int ac_index, uint16_t *quant_matrix)
{ {
int code, i, j, level, val; int code, i, j, level, val;
...@@ -736,7 +736,7 @@ static int decode_block(MJpegDecodeContext *s, int16_t *block, int component, ...@@ -736,7 +736,7 @@ static int decode_block(MJpegDecodeContext *s, int16_t *block, int component,
static int decode_dc_progressive(MJpegDecodeContext *s, int16_t *block, static int decode_dc_progressive(MJpegDecodeContext *s, int16_t *block,
int component, int dc_index, int component, int dc_index,
int16_t *quant_matrix, int Al) uint16_t *quant_matrix, int Al)
{ {
int val; int val;
s->bdsp.clear_block(block); s->bdsp.clear_block(block);
...@@ -754,7 +754,7 @@ static int decode_dc_progressive(MJpegDecodeContext *s, int16_t *block, ...@@ -754,7 +754,7 @@ static int decode_dc_progressive(MJpegDecodeContext *s, int16_t *block,
/* decode block and dequantize - progressive JPEG version */ /* decode block and dequantize - progressive JPEG version */
static int decode_block_progressive(MJpegDecodeContext *s, int16_t *block, static int decode_block_progressive(MJpegDecodeContext *s, int16_t *block,
uint8_t *last_nnz, int ac_index, uint8_t *last_nnz, int ac_index,
int16_t *quant_matrix, uint16_t *quant_matrix,
int ss, int se, int Al, int *EOBRUN) int ss, int se, int Al, int *EOBRUN)
{ {
int code, i, j, level, val, run; int code, i, j, level, val, run;
...@@ -852,7 +852,7 @@ for (; ; i++) { \ ...@@ -852,7 +852,7 @@ for (; ; i++) { \
/* decode block and dequantize - progressive JPEG refinement pass */ /* decode block and dequantize - progressive JPEG refinement pass */
static int decode_block_refinement(MJpegDecodeContext *s, int16_t *block, static int decode_block_refinement(MJpegDecodeContext *s, int16_t *block,
uint8_t *last_nnz, uint8_t *last_nnz,
int ac_index, int16_t *quant_matrix, int ac_index, uint16_t *quant_matrix,
int ss, int se, int Al, int *EOBRUN) int ss, int se, int Al, int *EOBRUN)
{ {
int code, i = ss, j, sign, val, run; int code, i = ss, j, sign, val, run;
...@@ -1383,7 +1383,7 @@ static int mjpeg_decode_scan_progressive_ac(MJpegDecodeContext *s, int ss, ...@@ -1383,7 +1383,7 @@ static int mjpeg_decode_scan_progressive_ac(MJpegDecodeContext *s, int ss,
int mb_x, mb_y; int mb_x, mb_y;
int EOBRUN = 0; int EOBRUN = 0;
int c = s->comp_index[0]; int c = s->comp_index[0];
int16_t *quant_matrix = s->quant_matrixes[s->quant_sindex[0]]; uint16_t *quant_matrix = s->quant_matrixes[s->quant_sindex[0]];
av_assert0(ss>=0 && Ah>=0 && Al>=0); av_assert0(ss>=0 && Ah>=0 && Al>=0);
if (se < ss || se > 63) { if (se < ss || se > 63) {
......
...@@ -50,7 +50,7 @@ typedef struct MJpegDecodeContext { ...@@ -50,7 +50,7 @@ typedef struct MJpegDecodeContext {
int buffer_size; int buffer_size;
uint8_t *buffer; uint8_t *buffer;
int16_t quant_matrixes[4][64]; uint16_t quant_matrixes[4][64];
VLC vlcs[3][4]; VLC vlcs[3][4];
int qscale[4]; ///< quantizer scale calculated from quant_matrixes int qscale[4]; ///< quantizer scale calculated from quant_matrixes
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment