tiff: add smarter checks if there is enough data left

Signed-off-by: Paul B Mahol <onemda@gmail.com>

libavcodec/tiff.c

@@ -65,26 +65,26 @@ typedef struct TiffContext {
 
 static unsigned tget_short(GetByteContext *gb, int le)
 {
-    unsigned v = le ? bytestream2_get_le16(gb) : bytestream2_get_be16(gb);
+    unsigned v = le ? bytestream2_get_le16u(gb) : bytestream2_get_be16u(gb);
     return v;
 }
 
 static unsigned tget_long(GetByteContext *gb, int le)
 {
-    unsigned v = le ? bytestream2_get_le32(gb) : bytestream2_get_be32(gb);
+    unsigned v = le ? bytestream2_get_le32u(gb) : bytestream2_get_be32u(gb);
     return v;
 }
 
 static double tget_double(GetByteContext *gb, int le)
 {
-    av_alias64 i = { .u64 = le ? bytestream2_get_le64(gb) : bytestream2_get_be64(gb)};
+    av_alias64 i = { .u64 = le ? bytestream2_get_le64u(gb) : bytestream2_get_be64u(gb)};
     return i.f64;
 }
 
 static unsigned tget(GetByteContext *gb, int type, int le)
 {
     switch (type) {
-    case TIFF_BYTE : return bytestream2_get_byte(gb);
+    case TIFF_BYTE : return bytestream2_get_byteu(gb);
     case TIFF_SHORT: return tget_short(gb, le);
     case TIFF_LONG : return tget_long(gb, le);
     default        : return UINT_MAX;
@@ -246,8 +246,10 @@ static int add_doubles_metadata(int count,
     int i;
     double *dp;
 
+    if (count >= INT_MAX / sizeof(int64_t))
+        return AVERROR_INVALIDDATA;
     if (bytestream2_get_bytes_left(&s->gb) < count * sizeof(int64_t))
-        return -1;
+        return AVERROR_INVALIDDATA;
 
     dp = av_malloc(count * sizeof(double));
     if (!dp)
@@ -270,8 +272,10 @@ static int add_shorts_metadata(int count, const char *name,
     int i;
     int16_t *sp;
 
+    if (count >= INT_MAX / sizeof(int16_t))
+        return AVERROR_INVALIDDATA;
     if (bytestream2_get_bytes_left(&s->gb) < count * sizeof(int16_t))
-        return -1;
+        return AVERROR_INVALIDDATA;
 
     sp = av_malloc(count * sizeof(int16_t));
     if (!sp)
@@ -590,8 +594,6 @@ static int tiff_decode_tag(TiffContext *s)
     uint32_t *pal;
     double *dp;
 
-    if (bytestream2_get_bytes_left(&s->gb) < 12)
-        return -1;
     tag   = tget_short(&s->gb, s->le);
     type  = tget_short(&s->gb, s->le);
     count = tget_long(&s->gb, s->le);
@@ -657,7 +659,9 @@ static int tiff_decode_tag(TiffContext *s)
             case TIFF_SHORT:
             case TIFF_LONG:
                 s->bpp = 0;
-                for (i = 0; i < count && bytestream2_get_bytes_left(&s->gb) > 0; i++)
+                if (bytestream2_get_bytes_left(&s->gb) < type_sizes[type] * count)
+                    return -1;
+                for (i = 0; i < count; i++)
                     s->bpp += tget(&s->gb, type, s->le);
                 break;
             default:
@@ -836,6 +840,8 @@ static int tiff_decode_tag(TiffContext *s)
             s->geotag_count = count / 4 - 1;
             av_log(s->avctx, AV_LOG_WARNING, "GeoTIFF key directory buffer shorter than specified\n");
         }
+        if (bytestream2_get_bytes_left(&s->gb) < s->geotag_count * sizeof(int16_t) * 4)
+            return -1;
         s->geotags = av_mallocz(sizeof(TiffGeoTag) * s->geotag_count);
         if (!s->geotags) {
             av_log(s->avctx, AV_LOG_ERROR, "Error allocating temporary buffer\n");
@@ -853,6 +859,10 @@ static int tiff_decode_tag(TiffContext *s)
         }
         break;
     case TIFF_GEO_DOUBLE_PARAMS:
+        if (count >= INT_MAX / sizeof(int64_t))
+            return AVERROR_INVALIDDATA;
+        if (bytestream2_get_bytes_left(&s->gb) < count * sizeof(int64_t))
+            return AVERROR_INVALIDDATA;
         dp = av_malloc(count * sizeof(double));
         if (!dp) {
             av_log(s->avctx, AV_LOG_ERROR, "Error allocating temporary buffer\n");
@@ -886,13 +896,17 @@ static int tiff_decode_tag(TiffContext *s)
                     || s->geotags[i].offset +  s->geotags[i].count > count) {
                     av_log(s->avctx, AV_LOG_WARNING, "Invalid GeoTIFF key %d\n", s->geotags[i].key);
                 } else {
-                    char *ap = av_malloc(s->geotags[i].count);
+                    char *ap;
+
+                    bytestream2_seek(&s->gb, pos + s->geotags[i].offset, SEEK_SET);
+                    if (bytestream2_get_bytes_left(&s->gb) < s->geotags[i].count)
+                        return -1;
+                    ap = av_malloc(s->geotags[i].count);
                     if (!ap) {
                         av_log(s->avctx, AV_LOG_ERROR, "Error allocating temporary buffer\n");
                         return AVERROR(ENOMEM);
                     }
-                    bytestream2_seek(&s->gb, pos + s->geotags[i].offset, SEEK_SET);
-                    bytestream2_get_buffer(&s->gb, ap, s->geotags[i].count);
+                    bytestream2_get_bufferu(&s->gb, ap, s->geotags[i].count);
                     ap[s->geotags[i].count - 1] = '\0'; //replace the "|" delimiter with a 0 byte
                     s->geotags[i].val = ap;
                 }
@@ -927,7 +941,7 @@ static int decode_frame(AVCodecContext *avctx,
     //parse image header
     if (avpkt->size < 8)
         return AVERROR_INVALIDDATA;
-    id = bytestream2_get_le16(&s->gb);
+    id = bytestream2_get_le16u(&s->gb);
     if (id == 0x4949)
         le = 1;
     else if (id == 0x4D4D)
@@ -961,6 +975,8 @@ static int decode_frame(AVCodecContext *avctx,
     }
     bytestream2_seek(&s->gb, off, SEEK_SET);
     entries = tget_short(&s->gb, le);
+    if (bytestream2_get_bytes_left(&s->gb) < entries * 12)
+        return AVERROR_INVALIDDATA;
     for (i = 0; i < entries; i++) {
         if (tiff_decode_tag(s) < 0)
             return -1;