sanity checks some might have been exploitable

Originally committed as revision 5370 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavformat/rm.c

@@ -555,6 +555,12 @@ static void rm_read_audio_stream_info(AVFormatContext *s, AVStream *st,
             st->codec->extradata_size= 0;
             rm->audio_framesize = st->codec->block_align;
             st->codec->block_align = coded_framesize;
+
+            if(rm->audio_framesize >= UINT_MAX / sub_packet_h){
+                av_log(s, AV_LOG_ERROR, "rm->audio_framesize * sub_packet_h too large\n");
+                return -1;
+            }
+
             rm->audiobuf = av_malloc(rm->audio_framesize * sub_packet_h);
         } else if (!strcmp(buf, "cook")) {
             int codecdata_length, i;
@@ -562,6 +568,11 @@ static void rm_read_audio_stream_info(AVFormatContext *s, AVStream *st,
             if (((version >> 16) & 0xff) == 5)
                 get_byte(pb);
             codecdata_length = get_be32(pb);
+            if(codecdata_length + FF_INPUT_BUFFER_PADDING_SIZE <= (unsigned)codecdata_length){
+                av_log(s, AV_LOG_ERROR, "codecdata_length too large\n");
+                return -1;
+            }
+
             st->codec->codec_id = CODEC_ID_COOK;
             st->codec->extradata_size= codecdata_length;
             st->codec->extradata= av_mallocz(st->codec->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
@@ -569,6 +580,12 @@ static void rm_read_audio_stream_info(AVFormatContext *s, AVStream *st,
                 ((uint8_t*)st->codec->extradata)[i] = get_byte(pb);
             rm->audio_framesize = st->codec->block_align;
             st->codec->block_align = rm->sub_packet_size;
+
+            if(rm->audio_framesize >= UINT_MAX / sub_packet_h){
+                av_log(s, AV_LOG_ERROR, "rm->audio_framesize * sub_packet_h too large\n");
+                return -1;
+            }
+
             rm->audiobuf = av_malloc(rm->audio_framesize * sub_packet_h);
         } else {
             st->codec->codec_id = CODEC_ID_NONE;
@@ -715,6 +732,12 @@ static int rm_read_header(AVFormatContext *s, AVFormatParameters *ap)
                 get_be16(pb);
 
                 st->codec->extradata_size= codec_data_size - (url_ftell(pb) - codec_pos);
+
+                if(st->codec->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE <= (unsigned)st->codec->extradata_size){
+                    //check is redundant as get_buffer() will catch this
+                    av_log(s, AV_LOG_ERROR, "st->codec->extradata_size too large\n");
+                    return -1;
+                }
                 st->codec->extradata= av_mallocz(st->codec->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
                 get_buffer(pb, st->codec->extradata, st->codec->extradata_size);
 

libavformat/sierravmd.c

@@ -196,6 +196,10 @@ static int vmd_read_header(AVFormatContext *s,
     vmd->frame_table = NULL;
     raw_frame_table_size = vmd->frame_count * 6;
     raw_frame_table = av_malloc(raw_frame_table_size);
+    if(vmd->frame_count * vmd->frames_per_block  >= UINT_MAX / sizeof(vmd_frame_t)){
+        av_log(s, AV_LOG_ERROR, "vmd->frame_count * vmd->frames_per_block too large\n");
+        return -1;
+    }
     vmd->frame_table = av_malloc(vmd->frame_count * vmd->frames_per_block * sizeof(vmd_frame_t));
     if (!raw_frame_table || !vmd->frame_table) {
         av_free(raw_frame_table);

libavformat/smacker.c

@@ -114,6 +114,13 @@ static int smacker_read_header(AVFormatContext *s, AVFormatParameters *ap)
     for(i = 0; i < 7; i++)
         smk->audio[i] = get_le32(pb);
     smk->treesize = get_le32(pb);
+
+    if(smk->treesize >= UINT_MAX/4){ // smk->treesize + 16 must not overflow (this check is probably redundant)
+        av_log(s, AV_LOG_ERROR, "treesize too large\n");
+        return -1;
+    }
+
+//FIXME remove extradata "rebuilding"
     smk->mmap_size = get_le32(pb);
     smk->mclr_size = get_le32(pb);
     smk->full_size = get_le32(pb);

libavformat/tta.c

@@ -50,13 +50,27 @@ static int tta_read_header(AVFormatContext *s, AVFormatParameters *ap)
     channels = get_le16(&s->pb);
     bps = get_le16(&s->pb);
     samplerate = get_le32(&s->pb);
+    if(samplerate <= 0 || samplerate > 1000000){
+        av_log(s, AV_LOG_ERROR, "nonsense samplerate\n");
+        return -1;
+    }
+
     datalen = get_le32(&s->pb);
+    if(datalen < 0){
+        av_log(s, AV_LOG_ERROR, "nonsense datalen\n");
+        return -1;
+    }
+
     url_fskip(&s->pb, 4); // header crc
 
     framelen = 1.04489795918367346939 * samplerate;
     c->totalframes = datalen / framelen + ((datalen % framelen) ? 1 : 0);
     c->currentframe = 0;
 
+    if(c->totalframes >= UINT_MAX/sizeof(uint32_t)){
+        av_log(s, AV_LOG_ERROR, "totalframes too large\n");
+        return -1;
+    }
     c->seektable = av_mallocz(sizeof(uint32_t)*c->totalframes);
     if (!c->seektable)
         return AVERROR_NOMEM;
@@ -76,6 +90,11 @@ static int tta_read_header(AVFormatContext *s, AVFormatParameters *ap)
     st->codec->bits_per_sample = bps;
 
     st->codec->extradata_size = url_ftell(&s->pb) - start;
+    if(st->codec->extradata_size+FF_INPUT_BUFFER_PADDING_SIZE <= (unsigned)st->codec->extradata_size){
+        //this check is redundant as get_buffer should fail
+        av_log(s, AV_LOG_ERROR, "extradata_size too large\n");
+        return -1;
+    }
     st->codec->extradata = av_mallocz(st->codec->extradata_size+FF_INPUT_BUFFER_PADDING_SIZE);
     url_fseek(&s->pb, start, SEEK_SET); // or SEEK_CUR and -size ? :)
     get_buffer(&s->pb, st->codec->extradata, st->codec->extradata_size);