Commit 7651c0e4 authored by Carl Eugen Hoyos's avatar Carl Eugen Hoyos

Avoid overflows when reading pgm files with maxval != 255 and != 65535.

libjpeg v6b apparently does not initialize the most significant bits
to zero when writing 12bit pgm's with maxval 4095.
parent 7e8e8ba9
...@@ -35,6 +35,7 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data, ...@@ -35,6 +35,7 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data,
int i, j, n, linesize, h, upgrade = 0, is_mono = 0; int i, j, n, linesize, h, upgrade = 0, is_mono = 0;
unsigned char *ptr; unsigned char *ptr;
int components, sample_len, ret; int components, sample_len, ret;
unsigned int maskval = 0;
s->bytestream_start = s->bytestream_start =
s->bytestream = (uint8_t *)buf; s->bytestream = (uint8_t *)buf;
...@@ -75,8 +76,10 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data, ...@@ -75,8 +76,10 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data,
n = avctx->width; n = avctx->width;
components=1; components=1;
sample_len=8; sample_len=8;
if (s->maxval < 255) if (s->maxval < 255) {
upgrade = 1; upgrade = 1;
maskval = (2 << av_log2(s->maxval)) - 1;
}
goto do_read; goto do_read;
case AV_PIX_FMT_GRAY8A: case AV_PIX_FMT_GRAY8A:
n = avctx->width * 2; n = avctx->width * 2;
...@@ -88,8 +91,10 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data, ...@@ -88,8 +91,10 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data,
n = avctx->width * 2; n = avctx->width * 2;
components=1; components=1;
sample_len=16; sample_len=16;
if (s->maxval < 65535) if (s->maxval < 65535) {
upgrade = 2; upgrade = 2;
maskval = (2 << av_log2(s->maxval)) - 1;
}
goto do_read; goto do_read;
case AV_PIX_FMT_MONOWHITE: case AV_PIX_FMT_MONOWHITE:
case AV_PIX_FMT_MONOBLACK: case AV_PIX_FMT_MONOBLACK:
...@@ -136,11 +141,11 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data, ...@@ -136,11 +141,11 @@ static int pnm_decode_frame(AVCodecContext *avctx, void *data,
else if (upgrade == 1) { else if (upgrade == 1) {
unsigned int j, f = (255 * 128 + s->maxval / 2) / s->maxval; unsigned int j, f = (255 * 128 + s->maxval / 2) / s->maxval;
for (j = 0; j < n; j++) for (j = 0; j < n; j++)
ptr[j] = (s->bytestream[j] * f + 64) >> 7; ptr[j] = ((s->bytestream[j] & maskval) * f + 64) >> 7;
} else if (upgrade == 2) { } else if (upgrade == 2) {
unsigned int j, v, f = (65535 * 32768 + s->maxval / 2) / s->maxval; unsigned int j, v, f = (65535 * 32768 + s->maxval / 2) / s->maxval;
for (j = 0; j < n / 2; j++) { for (j = 0; j < n / 2; j++) {
v = av_be2ne16(((uint16_t *)s->bytestream)[j]); v = av_be2ne16(((uint16_t *)s->bytestream)[j]) & maskval;
((uint16_t *)ptr)[j] = (v * f + 16384) >> 15; ((uint16_t *)ptr)[j] = (v * f + 16384) >> 15;
} }
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment