pcx: convert to bytestream2 API

Protects against overreads in input buffer.
Signed-off-by: Paul B Mahol <onemda@gmail.com>

libavcodec/pcx.c

@@ -31,7 +31,8 @@ typedef struct PCXContext {
     AVFrame picture;
 } PCXContext;
 
-static av_cold int pcx_init(AVCodecContext *avctx) {
+static av_cold int pcx_init(AVCodecContext *avctx)
+{
     PCXContext *s = avctx->priv_data;
 
     avcodec_get_frame_defaults(&s->picture);
@@ -40,67 +41,67 @@ static av_cold int pcx_init(AVCodecContext *avctx) {
     return 0;
 }
 
-/**
- * @return advanced src pointer
- */
-static const uint8_t *pcx_rle_decode(const uint8_t *src, uint8_t *dst,
-                            unsigned int bytes_per_scanline, int compressed) {
+static void pcx_rle_decode(GetByteContext *gb, uint8_t *dst,
+                           unsigned int bytes_per_scanline, int compressed)
+{
     unsigned int i = 0;
     unsigned char run, value;
 
     if (compressed) {
         while (i<bytes_per_scanline) {
             run = 1;
-            value = *src++;
+            value = bytestream2_get_byte(gb);
             if (value >= 0xc0) {
                 run = value & 0x3f;
-                value = *src++;
+                value = bytestream2_get_byte(gb);
             }
             while (i<bytes_per_scanline && run--)
                 dst[i++] = value;
         }
     } else {
-        memcpy(dst, src, bytes_per_scanline);
-        src += bytes_per_scanline;
+        bytestream2_get_buffer(gb, dst, bytes_per_scanline);
     }
-
-    return src;
 }
 
-static void pcx_palette(const uint8_t **src, uint32_t *dst, unsigned int pallen) {
-    unsigned int i;
+static void pcx_palette(GetByteContext *gb, uint32_t *dst, int pallen)
+{
+    int i;
 
+    pallen = FFMIN(pallen, bytestream2_get_bytes_left(gb) / 3);
     for (i=0; i<pallen; i++)
-        *dst++ = 0xFF000000 | bytestream_get_be24(src);
+        *dst++ = 0xFF000000 | bytestream2_get_be24u(gb);
     if (pallen < 256)
         memset(dst, 0, (256 - pallen) * sizeof(*dst));
 }
 
 static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
-                            AVPacket *avpkt) {
-    const uint8_t *buf = avpkt->data;
-    int buf_size = avpkt->size;
+                            AVPacket *avpkt)
+{
     PCXContext * const s = avctx->priv_data;
     AVFrame *picture = data;
     AVFrame * const p = &s->picture;
-    int compressed, xmin, ymin, xmax, ymax;
+    GetByteContext gb;
+    int compressed, xmin, ymin, xmax, ymax, ret;
     unsigned int w, h, bits_per_pixel, bytes_per_line, nplanes, stride, y, x,
                  bytes_per_scanline;
-    uint8_t *ptr;
-    uint8_t const *bufstart = buf;
-    uint8_t *scanline;
-    int ret = -1;
+    uint8_t *ptr, *scanline;
+
+    if (avpkt->size < 128)
+        return AVERROR_INVALIDDATA;
+
+    bytestream2_init(&gb, avpkt->data, avpkt->size);
 
-    if (buf[0] != 0x0a || buf[1] > 5) {
+    if (bytestream2_get_byteu(&gb) != 0x0a || bytestream2_get_byteu(&gb) > 5) {
         av_log(avctx, AV_LOG_ERROR, "this is not PCX encoded data\n");
         return AVERROR_INVALIDDATA;
     }
 
-    compressed = buf[2];
-    xmin = AV_RL16(buf+ 4);
-    ymin = AV_RL16(buf+ 6);
-    xmax = AV_RL16(buf+ 8);
-    ymax = AV_RL16(buf+10);
+    compressed = bytestream2_get_byteu(&gb);
+    bits_per_pixel = bytestream2_get_byteu(&gb);
+    xmin = bytestream2_get_le16u(&gb);
+    ymin = bytestream2_get_le16u(&gb);
+    xmax = bytestream2_get_le16u(&gb);
+    ymax = bytestream2_get_le16u(&gb);
 
     if (xmax < xmin || ymax < ymin) {
         av_log(avctx, AV_LOG_ERROR, "invalid image dimensions\n");
@@ -110,9 +111,9 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
     w = xmax - xmin + 1;
     h = ymax - ymin + 1;
 
-    bits_per_pixel     = buf[3];
-    bytes_per_line     = AV_RL16(buf+66);
-    nplanes            = buf[65];
+    bytestream2_skipu(&gb, 53);
+    nplanes            = bytestream2_get_byteu(&gb);
+    bytes_per_line     = bytestream2_get_le16u(&gb);
     bytes_per_scanline = nplanes * bytes_per_line;
 
     if (bytes_per_scanline < w * bits_per_pixel * nplanes / 8) {
@@ -138,7 +139,7 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
             return AVERROR_INVALIDDATA;
     }
 
-    buf += 128;
+    bytestream2_skipu(&gb, 60);
 
     if (p->data[0])
         avctx->release_buffer(avctx, p);
@@ -163,7 +164,7 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
 
     if (nplanes == 3 && bits_per_pixel == 8) {
         for (y=0; y<h; y++) {
-            buf = pcx_rle_decode(buf, scanline, bytes_per_scanline, compressed);
+            pcx_rle_decode(&gb, scanline, bytes_per_scanline, compressed);
 
             for (x=0; x<w; x++) {
                 ptr[3*x  ] = scanline[x                    ];
@@ -175,18 +176,18 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
         }
 
     } else if (nplanes == 1 && bits_per_pixel == 8) {
-        const uint8_t *palstart = bufstart + buf_size - 769;
+        int palstart = avpkt->size - 769;
 
         for (y=0; y<h; y++, ptr+=stride) {
-            buf = pcx_rle_decode(buf, scanline, bytes_per_scanline, compressed);
+            pcx_rle_decode(&gb, scanline, bytes_per_scanline, compressed);
             memcpy(ptr, scanline, w);
         }
 
-        if (buf != palstart) {
+        if (bytestream2_tell(&gb) != palstart) {
             av_log(avctx, AV_LOG_WARNING, "image data possibly corrupted\n");
-            buf = palstart;
+            bytestream2_seek(&gb, palstart, SEEK_SET);
         }
-        if (*buf++ != 12) {
+        if (bytestream2_get_byte(&gb) != 12) {
             av_log(avctx, AV_LOG_ERROR, "expected palette after image data\n");
             ret = AVERROR_INVALIDDATA;
             goto end;
@@ -198,7 +199,7 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
         for (y=0; y<h; y++) {
             init_get_bits(&s, scanline, bytes_per_scanline<<3);
 
-            buf = pcx_rle_decode(buf, scanline, bytes_per_scanline, compressed);
+            pcx_rle_decode(&gb, scanline, bytes_per_scanline, compressed);
 
             for (x=0; x<w; x++)
                 ptr[x] = get_bits(&s, bits_per_pixel);
@@ -209,7 +210,7 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
         int i;
 
         for (y=0; y<h; y++) {
-            buf = pcx_rle_decode(buf, scanline, bytes_per_scanline, compressed);
+            pcx_rle_decode(&gb, scanline, bytes_per_scanline, compressed);
 
             for (x=0; x<w; x++) {
                 int m = 0x80 >> (x&7), v = 0;
@@ -223,26 +224,28 @@ static int pcx_decode_frame(AVCodecContext *avctx, void *data, int *data_size,
         }
     }
 
+    ret = bytestream2_tell(&gb);
     if (nplanes == 1 && bits_per_pixel == 8) {
-        pcx_palette(&buf, (uint32_t *) p->data[1], 256);
+        pcx_palette(&gb, (uint32_t *) p->data[1], 256);
+        ret += 256 * 3;
     } else if (bits_per_pixel * nplanes == 1) {
         AV_WN32A(p->data[1]  , 0xFF000000);
         AV_WN32A(p->data[1]+4, 0xFFFFFFFF);
     } else if (bits_per_pixel < 8) {
-        const uint8_t *palette = bufstart+16;
-        pcx_palette(&palette, (uint32_t *) p->data[1], 16);
+        bytestream2_seek(&gb, 16, SEEK_SET);
+        pcx_palette(&gb, (uint32_t *) p->data[1], 16);
     }
 
     *picture = s->picture;
     *data_size = sizeof(AVFrame);
 
-    ret = buf - bufstart;
 end:
     av_free(scanline);
     return ret;
 }
 
-static av_cold int pcx_end(AVCodecContext *avctx) {
+static av_cold int pcx_end(AVCodecContext *avctx)
+{
     PCXContext *s = avctx->priv_data;
 
     if(s->picture.data[0])