qtrle: Use bytestream2 functions to prevent buffer overreads.

Signed-off-by: Ronald S. Bultje <rsbultje@gmail.com>

libavcodec/qtrle.c

@@ -35,27 +35,17 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "libavutil/intreadwrite.h"
 #include "avcodec.h"
+#include "bytestream.h"
 
 typedef struct QtrleContext {
-
     AVCodecContext *avctx;
     AVFrame frame;
 
-    const unsigned char *buf;
-    int size;
-
+    GetByteContext g;
     uint32_t pal[256];
 } QtrleContext;
 
-#define CHECK_STREAM_PTR(n) \
-  if ((stream_ptr + n) > s->size) { \
-    av_log (s->avctx, AV_LOG_INFO, "Problem: stream_ptr out of bounds (%d >= %d)\n", \
-      stream_ptr + n, s->size); \
-    return; \
-  }
-
 #define CHECK_PIXEL_PTR(n) \
   if ((pixel_ptr + n > pixel_limit) || (pixel_ptr + n < 0)) { \
     av_log (s->avctx, AV_LOG_INFO, "Problem: pixel_ptr = %d, pixel_limit = %d\n", \
@@ -63,7 +53,7 @@ typedef struct QtrleContext {
     return; \
   } \
 
-static void qtrle_decode_1bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change)
+static void qtrle_decode_1bpp(QtrleContext *s, int row_ptr, int lines_to_change)
 {
     int rle_code;
     int pixel_ptr = 0;
@@ -74,9 +64,8 @@ static void qtrle_decode_1bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
     int skip;
 
     while (lines_to_change) {
-        CHECK_STREAM_PTR(2);
-        skip = s->buf[stream_ptr++];
-        rle_code = (signed char)s->buf[stream_ptr++];
+        skip     =              bytestream2_get_byte(&s->g);
+        rle_code = (signed char)bytestream2_get_byte(&s->g);
         if (rle_code == 0)
             break;
         if(skip & 0x80) {
@@ -92,9 +81,9 @@ static void qtrle_decode_1bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
             rle_code = -rle_code;
             /* get the next 2 bytes from the stream, treat them as groups
              * of 8 pixels, and output them rle_code times */
-            CHECK_STREAM_PTR(2);
-            pi0 = s->buf[stream_ptr++];
-            pi1 = s->buf[stream_ptr++];
+
+            pi0 = bytestream2_get_byte(&s->g);
+            pi1 = bytestream2_get_byte(&s->g);
             CHECK_PIXEL_PTR(rle_code * 2);
 
             while (rle_code--) {
@@ -104,17 +93,16 @@ static void qtrle_decode_1bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
         } else {
             /* copy the same pixel directly to output 2 times */
             rle_code *= 2;
-            CHECK_STREAM_PTR(rle_code);
             CHECK_PIXEL_PTR(rle_code);
 
             while (rle_code--)
-                rgb[pixel_ptr++] = s->buf[stream_ptr++];
+                rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
         }
     }
 }
 
-static inline void qtrle_decode_2n4bpp(QtrleContext *s, int stream_ptr,
-                             int row_ptr, int lines_to_change, int bpp)
+static inline void qtrle_decode_2n4bpp(QtrleContext *s, int row_ptr,
+                                       int lines_to_change, int bpp)
 {
     int rle_code, i;
     int pixel_ptr;
@@ -125,24 +113,21 @@ static inline void qtrle_decode_2n4bpp(QtrleContext *s, int stream_ptr,
     int num_pixels = (bpp == 4) ? 8 : 16;
 
     while (lines_to_change--) {
-        CHECK_STREAM_PTR(2);
-        pixel_ptr = row_ptr + (num_pixels * (s->buf[stream_ptr++] - 1));
+        pixel_ptr = row_ptr + (num_pixels * (bytestream2_get_byte(&s->g) - 1));
 
-        while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) {
+        while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
             if (rle_code == 0) {
                 /* there's another skip code in the stream */
-                CHECK_STREAM_PTR(1);
-                pixel_ptr += (num_pixels * (s->buf[stream_ptr++] - 1));
+                pixel_ptr += (num_pixels * (bytestream2_get_byte(&s->g) - 1));
                 CHECK_PIXEL_PTR(0);  /* make sure pixel_ptr is positive */
             } else if (rle_code < 0) {
                 /* decode the run length code */
                 rle_code = -rle_code;
                 /* get the next 4 bytes from the stream, treat them as palette
                  * indexes, and output them rle_code times */
-                CHECK_STREAM_PTR(4);
                 for (i = num_pixels-1; i >= 0; i--) {
-                    pi[num_pixels-1-i] = (s->buf[stream_ptr] >> ((i*bpp) & 0x07)) & ((1<<bpp)-1);
-                    stream_ptr+= ((i & ((num_pixels>>2)-1)) == 0);
+                    pi[num_pixels-1-i] = (bytestream2_peek_byte(&s->g) >> ((i*bpp) & 0x07)) & ((1<<bpp)-1);
+                    bytestream2_skip(&s->g, ((i & ((num_pixels>>2)-1)) == 0));
                 }
                 CHECK_PIXEL_PTR(rle_code * num_pixels);
                 while (rle_code--) {
@@ -152,17 +137,18 @@ static inline void qtrle_decode_2n4bpp(QtrleContext *s, int stream_ptr,
             } else {
                 /* copy the same pixel directly to output 4 times */
                 rle_code *= 4;
-                CHECK_STREAM_PTR(rle_code);
                 CHECK_PIXEL_PTR(rle_code*(num_pixels>>2));
                 while (rle_code--) {
                     if(bpp == 4) {
-                        rgb[pixel_ptr++] = ((s->buf[stream_ptr]) >> 4) & 0x0f;
-                        rgb[pixel_ptr++] = (s->buf[stream_ptr++]) & 0x0f;
+                        int x = bytestream2_get_byte(&s->g);
+                        rgb[pixel_ptr++] = (x >> 4) & 0x0f;
+                        rgb[pixel_ptr++] =  x       & 0x0f;
                     } else {
-                        rgb[pixel_ptr++] = ((s->buf[stream_ptr]) >> 6) & 0x03;
-                        rgb[pixel_ptr++] = ((s->buf[stream_ptr]) >> 4) & 0x03;
-                        rgb[pixel_ptr++] = ((s->buf[stream_ptr]) >> 2) & 0x03;
-                        rgb[pixel_ptr++] = (s->buf[stream_ptr++]) & 0x03;
+                        int x = bytestream2_get_byte(&s->g);
+                        rgb[pixel_ptr++] = (x >> 6) & 0x03;
+                        rgb[pixel_ptr++] = (x >> 4) & 0x03;
+                        rgb[pixel_ptr++] = (x >> 2) & 0x03;
+                        rgb[pixel_ptr++] =  x       & 0x03;
                     }
                 }
             }
@@ -171,7 +157,7 @@ static inline void qtrle_decode_2n4bpp(QtrleContext *s, int stream_ptr,
     }
 }
 
-static void qtrle_decode_8bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change)
+static void qtrle_decode_8bpp(QtrleContext *s, int row_ptr, int lines_to_change)
 {
     int rle_code;
     int pixel_ptr;
@@ -181,25 +167,22 @@ static void qtrle_decode_8bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
     int pixel_limit = s->frame.linesize[0] * s->avctx->height;
 
     while (lines_to_change--) {
-        CHECK_STREAM_PTR(2);
-        pixel_ptr = row_ptr + (4 * (s->buf[stream_ptr++] - 1));
+        pixel_ptr = row_ptr + (4 * (bytestream2_get_byte(&s->g) - 1));
 
-        while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) {
+        while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
             if (rle_code == 0) {
                 /* there's another skip code in the stream */
-                CHECK_STREAM_PTR(1);
-                pixel_ptr += (4 * (s->buf[stream_ptr++] - 1));
+                pixel_ptr += (4 * (bytestream2_get_byte(&s->g) - 1));
                 CHECK_PIXEL_PTR(0);  /* make sure pixel_ptr is positive */
             } else if (rle_code < 0) {
                 /* decode the run length code */
                 rle_code = -rle_code;
                 /* get the next 4 bytes from the stream, treat them as palette
                  * indexes, and output them rle_code times */
-                CHECK_STREAM_PTR(4);
-                pi1 = s->buf[stream_ptr++];
-                pi2 = s->buf[stream_ptr++];
-                pi3 = s->buf[stream_ptr++];
-                pi4 = s->buf[stream_ptr++];
+                pi1 = bytestream2_get_byte(&s->g);
+                pi2 = bytestream2_get_byte(&s->g);
+                pi3 = bytestream2_get_byte(&s->g);
+                pi4 = bytestream2_get_byte(&s->g);
 
                 CHECK_PIXEL_PTR(rle_code * 4);
 
@@ -212,11 +195,10 @@ static void qtrle_decode_8bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
             } else {
                 /* copy the same pixel directly to output 4 times */
                 rle_code *= 4;
-                CHECK_STREAM_PTR(rle_code);
                 CHECK_PIXEL_PTR(rle_code);
 
                 while (rle_code--) {
-                    rgb[pixel_ptr++] = s->buf[stream_ptr++];
+                    rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
                 }
             }
         }
@@ -224,7 +206,7 @@ static void qtrle_decode_8bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
     }
 }
 
-static void qtrle_decode_16bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change)
+static void qtrle_decode_16bpp(QtrleContext *s, int row_ptr, int lines_to_change)
 {
     int rle_code;
     int pixel_ptr;
@@ -234,21 +216,17 @@ static void qtrle_decode_16bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
     int pixel_limit = s->frame.linesize[0] * s->avctx->height;
 
     while (lines_to_change--) {
-        CHECK_STREAM_PTR(2);
-        pixel_ptr = row_ptr + (s->buf[stream_ptr++] - 1) * 2;
+        pixel_ptr = row_ptr + (bytestream2_get_byte(&s->g) - 1) * 2;
 
-        while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) {
+        while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
             if (rle_code == 0) {
                 /* there's another skip code in the stream */
-                CHECK_STREAM_PTR(1);
-                pixel_ptr += (s->buf[stream_ptr++] - 1) * 2;
+                pixel_ptr += (bytestream2_get_byte(&s->g) - 1) * 2;
                 CHECK_PIXEL_PTR(0);  /* make sure pixel_ptr is positive */
             } else if (rle_code < 0) {
                 /* decode the run length code */
                 rle_code = -rle_code;
-                CHECK_STREAM_PTR(2);
-                rgb16 = AV_RB16(&s->buf[stream_ptr]);
-                stream_ptr += 2;
+                rgb16 = bytestream2_get_be16(&s->g);
 
                 CHECK_PIXEL_PTR(rle_code * 2);
 
@@ -257,13 +235,11 @@ static void qtrle_decode_16bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
                     pixel_ptr += 2;
                 }
             } else {
-                CHECK_STREAM_PTR(rle_code * 2);
                 CHECK_PIXEL_PTR(rle_code * 2);
 
                 /* copy pixels directly to output */
                 while (rle_code--) {
-                    rgb16 = AV_RB16(&s->buf[stream_ptr]);
-                    stream_ptr += 2;
+                    rgb16 = bytestream2_get_be16(&s->g);
                     *(unsigned short *)(&rgb[pixel_ptr]) = rgb16;
                     pixel_ptr += 2;
                 }
@@ -273,7 +249,7 @@ static void qtrle_decode_16bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
     }
 }
 
-static void qtrle_decode_24bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change)
+static void qtrle_decode_24bpp(QtrleContext *s, int row_ptr, int lines_to_change)
 {
     int rle_code;
     int pixel_ptr;
@@ -283,22 +259,19 @@ static void qtrle_decode_24bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
     int pixel_limit = s->frame.linesize[0] * s->avctx->height;
 
     while (lines_to_change--) {
-        CHECK_STREAM_PTR(2);
-        pixel_ptr = row_ptr + (s->buf[stream_ptr++] - 1) * 3;
+        pixel_ptr = row_ptr + (bytestream2_get_byte(&s->g) - 1) * 3;
 
-        while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) {
+        while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
             if (rle_code == 0) {
                 /* there's another skip code in the stream */
-                CHECK_STREAM_PTR(1);
-                pixel_ptr += (s->buf[stream_ptr++] - 1) * 3;
+                pixel_ptr += (bytestream2_get_byte(&s->g) - 1) * 3;
                 CHECK_PIXEL_PTR(0);  /* make sure pixel_ptr is positive */
             } else if (rle_code < 0) {
                 /* decode the run length code */
                 rle_code = -rle_code;
-                CHECK_STREAM_PTR(3);
-                r = s->buf[stream_ptr++];
-                g = s->buf[stream_ptr++];
-                b = s->buf[stream_ptr++];
+                r = bytestream2_get_byte(&s->g);
+                g = bytestream2_get_byte(&s->g);
+                b = bytestream2_get_byte(&s->g);
 
                 CHECK_PIXEL_PTR(rle_code * 3);
 
@@ -308,14 +281,13 @@ static void qtrle_decode_24bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
                     rgb[pixel_ptr++] = b;
                 }
             } else {
-                CHECK_STREAM_PTR(rle_code * 3);
                 CHECK_PIXEL_PTR(rle_code * 3);
 
                 /* copy pixels directly to output */
                 while (rle_code--) {
-                    rgb[pixel_ptr++] = s->buf[stream_ptr++];
-                    rgb[pixel_ptr++] = s->buf[stream_ptr++];
-                    rgb[pixel_ptr++] = s->buf[stream_ptr++];
+                    rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
+                    rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
+                    rgb[pixel_ptr++] = bytestream2_get_byte(&s->g);
                 }
             }
         }
@@ -323,7 +295,7 @@ static void qtrle_decode_24bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
     }
 }
 
-static void qtrle_decode_32bpp(QtrleContext *s, int stream_ptr, int row_ptr, int lines_to_change)
+static void qtrle_decode_32bpp(QtrleContext *s, int row_ptr, int lines_to_change)
 {
     int rle_code;
     int pixel_ptr;
@@ -333,21 +305,17 @@ static void qtrle_decode_32bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
     int pixel_limit = s->frame.linesize[0] * s->avctx->height;
 
     while (lines_to_change--) {
-        CHECK_STREAM_PTR(2);
-        pixel_ptr = row_ptr + (s->buf[stream_ptr++] - 1) * 4;
+        pixel_ptr = row_ptr + (bytestream2_get_byte(&s->g) - 1) * 4;
 
-        while ((rle_code = (signed char)s->buf[stream_ptr++]) != -1) {
+        while ((rle_code = (signed char)bytestream2_get_byte(&s->g)) != -1) {
             if (rle_code == 0) {
                 /* there's another skip code in the stream */
-                CHECK_STREAM_PTR(1);
-                pixel_ptr += (s->buf[stream_ptr++] - 1) * 4;
+                pixel_ptr += (bytestream2_get_byte(&s->g) - 1) * 4;
                 CHECK_PIXEL_PTR(0);  /* make sure pixel_ptr is positive */
             } else if (rle_code < 0) {
                 /* decode the run length code */
                 rle_code = -rle_code;
-                CHECK_STREAM_PTR(4);
-                argb = AV_RB32(s->buf + stream_ptr);
-                stream_ptr += 4;
+                argb = bytestream2_get_be32(&s->g);
 
                 CHECK_PIXEL_PTR(rle_code * 4);
 
@@ -356,14 +324,12 @@ static void qtrle_decode_32bpp(QtrleContext *s, int stream_ptr, int row_ptr, int
                     pixel_ptr += 4;
                 }
             } else {
-                CHECK_STREAM_PTR(rle_code * 4);
                 CHECK_PIXEL_PTR(rle_code * 4);
 
                 /* copy pixels directly to output */
                 while (rle_code--) {
-                    argb = AV_RB32(s->buf + stream_ptr);
+                    argb = bytestream2_get_be32(&s->g);
                     AV_WN32A(rgb + pixel_ptr, argb);
-                    stream_ptr += 4;
                     pixel_ptr  += 4;
                 }
             }
@@ -419,16 +385,12 @@ static int qtrle_decode_frame(AVCodecContext *avctx,
                               void *data, int *data_size,
                               AVPacket *avpkt)
 {
-    const uint8_t *buf = avpkt->data;
-    int buf_size = avpkt->size;
     QtrleContext *s = avctx->priv_data;
     int header, start_line;
-    int stream_ptr, height, row_ptr;
+    int height, row_ptr;
     int has_palette = 0;
 
-    s->buf = buf;
-    s->size = buf_size;
-
+    bytestream2_init(&s->g, avpkt->data, avpkt->size);
     s->frame.reference = 1;
     s->frame.buffer_hints = FF_BUFFER_HINTS_VALID | FF_BUFFER_HINTS_PRESERVE |
                             FF_BUFFER_HINTS_REUSABLE | FF_BUFFER_HINTS_READABLE;
@@ -438,64 +400,63 @@ static int qtrle_decode_frame(AVCodecContext *avctx,
     }
 
     /* check if this frame is even supposed to change */
-    if (s->size < 8)
+    if (avpkt->size < 8)
         goto done;
 
     /* start after the chunk size */
-    stream_ptr = 4;
+    bytestream2_seek(&s->g, 4, SEEK_SET);
 
     /* fetch the header */
-    header = AV_RB16(&s->buf[stream_ptr]);
-    stream_ptr += 2;
+    header = bytestream2_get_be16(&s->g);
 
     /* if a header is present, fetch additional decoding parameters */
     if (header & 0x0008) {
-        if(s->size < 14)
+        if (avpkt->size < 14)
             goto done;
-        start_line = AV_RB16(&s->buf[stream_ptr]);
-        stream_ptr += 4;
-        height = AV_RB16(&s->buf[stream_ptr]);
-        stream_ptr += 4;
+        start_line = bytestream2_get_be16(&s->g);
+        bytestream2_skip(&s->g, 2);
+        height     = bytestream2_get_be16(&s->g);
+        bytestream2_skip(&s->g, 2);
     } else {
         start_line = 0;
-        height = s->avctx->height;
+        height     = s->avctx->height;
     }
     row_ptr = s->frame.linesize[0] * start_line;
 
     switch (avctx->bits_per_coded_sample) {
     case 1:
     case 33:
-        qtrle_decode_1bpp(s, stream_ptr, row_ptr, height);
+        qtrle_decode_1bpp(s, row_ptr, height);
         break;
 
     case 2:
     case 34:
-        qtrle_decode_2n4bpp(s, stream_ptr, row_ptr, height, 2);
+        qtrle_decode_2n4bpp(s, row_ptr, height, 2);
         has_palette = 1;
         break;
 
     case 4:
     case 36:
-        qtrle_decode_2n4bpp(s, stream_ptr, row_ptr, height, 4);
+        qtrle_decode_2n4bpp(s, row_ptr, height, 4);
         has_palette = 1;
         break;
 
     case 8:
     case 40:
-        qtrle_decode_8bpp(s, stream_ptr, row_ptr, height);
+        qtrle_decode_8bpp(s, row_ptr, height);
         has_palette = 1;
         break;
 
     case 16:
-        qtrle_decode_16bpp(s, stream_ptr, row_ptr, height);
+        qtrle_decode_16bpp(s, row_ptr, height);
         break;
 
     case 24:
-        qtrle_decode_24bpp(s, stream_ptr, row_ptr, height);
+        qtrle_decode_24bpp(s, row_ptr, height);
         break;
 
     case 32:
-        qtrle_decode_32bpp(s, stream_ptr, row_ptr, height);
+        qtrle_decode_32bpp(s, row_ptr, height);
         break;
 
     default:
@@ -521,7 +482,7 @@ done:
     *(AVFrame*)data = s->frame;
 
     /* always report that the buffer was completely consumed */
-    return buf_size;
+    return avpkt->size;
 }
 
 static av_cold int qtrle_decode_end(AVCodecContext *avctx)