Commit 23f3f923 authored by Michael Niedermayer's avatar Michael Niedermayer

avcodec/mjpegdec: quant_matrixes can be up to 65535, use uint16_t

Fixes invalid shift
Fixes: 870/clusterfuzz-testcase-5649105424482304

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpegSigned-off-by: 's avatarMichael Niedermayer <michael@niedermayer.cc>
parent 656a17e1
......@@ -686,7 +686,7 @@ static inline int mjpeg_decode_dc(MJpegDecodeContext *s, int dc_index)
/* decode block and dequantize */
static int decode_block(MJpegDecodeContext *s, int16_t *block, int component,
int dc_index, int ac_index, int16_t *quant_matrix)
int dc_index, int ac_index, uint16_t *quant_matrix)
{
int code, i, j, level, val;
......@@ -736,7 +736,7 @@ static int decode_block(MJpegDecodeContext *s, int16_t *block, int component,
static int decode_dc_progressive(MJpegDecodeContext *s, int16_t *block,
int component, int dc_index,
int16_t *quant_matrix, int Al)
uint16_t *quant_matrix, int Al)
{
int val;
s->bdsp.clear_block(block);
......@@ -754,7 +754,7 @@ static int decode_dc_progressive(MJpegDecodeContext *s, int16_t *block,
/* decode block and dequantize - progressive JPEG version */
static int decode_block_progressive(MJpegDecodeContext *s, int16_t *block,
uint8_t *last_nnz, int ac_index,
int16_t *quant_matrix,
uint16_t *quant_matrix,
int ss, int se, int Al, int *EOBRUN)
{
int code, i, j, level, val, run;
......@@ -852,7 +852,7 @@ for (; ; i++) { \
/* decode block and dequantize - progressive JPEG refinement pass */
static int decode_block_refinement(MJpegDecodeContext *s, int16_t *block,
uint8_t *last_nnz,
int ac_index, int16_t *quant_matrix,
int ac_index, uint16_t *quant_matrix,
int ss, int se, int Al, int *EOBRUN)
{
int code, i = ss, j, sign, val, run;
......@@ -1383,7 +1383,7 @@ static int mjpeg_decode_scan_progressive_ac(MJpegDecodeContext *s, int ss,
int mb_x, mb_y;
int EOBRUN = 0;
int c = s->comp_index[0];
int16_t *quant_matrix = s->quant_matrixes[s->quant_sindex[0]];
uint16_t *quant_matrix = s->quant_matrixes[s->quant_sindex[0]];
av_assert0(ss>=0 && Ah>=0 && Al>=0);
if (se < ss || se > 63) {
......
......@@ -50,7 +50,7 @@ typedef struct MJpegDecodeContext {
int buffer_size;
uint8_t *buffer;
int16_t quant_matrixes[4][64];
uint16_t quant_matrixes[4][64];
VLC vlcs[3][4];
int qscale[4]; ///< quantizer scale calculated from quant_matrixes
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment