handle malloc failures in ff_get_wav_header

ff_get_wav_header is reading data from a WAVE file and then uses it
(without validation) to malloc a buffer. It then proceeded to read
data into the buffer, without verifying that the allocation succeeded.

To address this, change ff_get_wav_header to return an error if
allocation failed, and adapted all calling code to handle that error.
Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

libavformat/asfdec.c

@@ -288,7 +288,9 @@ static int asf_read_stream_properties(AVFormatContext *s, int64_t size)
 
     st->codec->codec_type = type;
     if (type == AVMEDIA_TYPE_AUDIO) {
-        ff_get_wav_header(pb, st->codec, type_specific_size);
+        int ret = ff_get_wav_header(pb, st->codec, type_specific_size);
+        if (ret < 0)
+            return ret;
         if (is_dvr_ms_audio) {
             // codec_id and codec_tag are unreliable in dvr_ms
             // files. Set them later by probing stream.

libavformat/avidec.c

@@ -345,6 +345,7 @@ static int avi_read_header(AVFormatContext *s, AVFormatParameters *ap)
     int avih_width=0, avih_height=0;
     int amv_file_format=0;
     uint64_t list_end = 0;
+    int ret;
 
     avi->stream_index= -1;
 
@@ -623,7 +624,9 @@ static int avi_read_header(AVFormatContext *s, AVFormatParameters *ap)
 //                    avio_skip(pb, size - 5 * 4);
                     break;
                 case AVMEDIA_TYPE_AUDIO:
-                    ff_get_wav_header(pb, st->codec, size);
+                    ret = ff_get_wav_header(pb, st->codec, size);
+                    if (ret < 0)
+                        return ret;
                     ast->dshow_block_align= st->codec->block_align;
                     if(ast->sample_size && st->codec->block_align && ast->sample_size != st->codec->block_align){
                         av_log(s, AV_LOG_WARNING, "sample size (%d) != block align (%d)\n", ast->sample_size, st->codec->block_align);

libavformat/dxa.c

@@ -60,6 +60,7 @@ static int dxa_read_header(AVFormatContext *s, AVFormatParameters *ap)
     int w, h;
     int num, den;
     int flags;
+    int ret;
 
     tag = avio_rl32(pb);
     if (tag != MKTAG('D', 'E', 'X', 'A'))
@@ -102,7 +103,9 @@ static int dxa_read_header(AVFormatContext *s, AVFormatParameters *ap)
         ast = av_new_stream(s, 0);
         if (!ast)
             return -1;
-        ff_get_wav_header(pb, ast->codec, fsize);
+        ret = ff_get_wav_header(pb, ast->codec, fsize);
+        if (ret < 0)
+            return ret;
         // find 'data' chunk
         while(avio_tell(pb) < c->vidpos && !pb->eof_reached){
             tag = avio_rl32(pb);

libavformat/matroskadec.c

@@ -1329,9 +1329,12 @@ static int matroska_read_header(AVFormatContext *s, AVFormatParameters *ap)
         } else if (!strcmp(track->codec_id, "A_MS/ACM")
                    && track->codec_priv.size >= 14
                    && track->codec_priv.data != NULL) {
+            int ret;
             ffio_init_context(&b, track->codec_priv.data, track->codec_priv.size,
                           AVIO_RDONLY, NULL, NULL, NULL, NULL);
-            ff_get_wav_header(&b, st->codec, track->codec_priv.size);
+            ret = ff_get_wav_header(&b, st->codec, track->codec_priv.size);
+            if (ret < 0)
+                return ret;
             codec_id = st->codec->codec_id;
             extradata_offset = FFMIN(track->codec_priv.size, 18);
         } else if (!strcmp(track->codec_id, "V_QUICKTIME")

libavformat/riff.c

@@ -480,7 +480,7 @@ void ff_put_bmp_header(AVIOContext *pb, AVCodecContext *enc, const AVCodecTag *t
  * WAVEFORMATEX adds 'WORD  cbSize' and basically makes itself
  * an openended structure.
  */
-void ff_get_wav_header(AVIOContext *pb, AVCodecContext *codec, int size)
+int ff_get_wav_header(AVIOContext *pb, AVCodecContext *codec, int size)
 {
     int id;
 
@@ -510,6 +510,8 @@ void ff_get_wav_header(AVIOContext *pb, AVCodecContext *codec, int size)
         codec->extradata_size = cbSize;
         if (cbSize > 0) {
             codec->extradata = av_mallocz(codec->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
+            if (!codec->extradata)
+                return AVERROR(ENOMEM);
             avio_read(pb, codec->extradata, codec->extradata_size);
             size -= cbSize;
         }
@@ -524,6 +526,8 @@ void ff_get_wav_header(AVIOContext *pb, AVCodecContext *codec, int size)
         codec->channels    = 0;
         codec->sample_rate = 0;
     }
+
+    return 0;
 }
 
 

libavformat/riff.h

@@ -45,7 +45,7 @@ int ff_get_bmp_header(AVIOContext *pb, AVStream *st);
 void ff_put_bmp_header(AVIOContext *pb, AVCodecContext *enc, const AVCodecTag *tags, int for_asf);
 int ff_put_wav_header(AVIOContext *pb, AVCodecContext *enc);
 enum CodecID ff_wav_codec_get_id(unsigned int tag, int bps);
-void ff_get_wav_header(AVIOContext *pb, AVCodecContext *codec, int size);
+int ff_get_wav_header(AVIOContext *pb, AVCodecContext *codec, int size);
 
 extern const AVCodecTag ff_codec_bmp_tags[];
 extern const AVCodecTag ff_codec_wav_tags[];

libavformat/wav.c

@@ -196,6 +196,7 @@ static int wav_read_header(AVFormatContext *s,
     AVIOContext *pb = s->pb;
     AVStream *st;
     WAVContext *wav = s->priv_data;
+    int ret;
 
     /* check RIFF header */
     tag = avio_rl32(pb);
@@ -228,7 +229,9 @@ static int wav_read_header(AVFormatContext *s,
     if (!st)
         return AVERROR(ENOMEM);
 
-    ff_get_wav_header(pb, st->codec, size);
+    ret = ff_get_wav_header(pb, st->codec, size);
+    if (ret < 0)
+        return ret;
     st->need_parsing = AVSTREAM_PARSE_FULL;
 
     av_set_pts_info(st, 64, 1, st->codec->sample_rate);
@@ -384,6 +387,7 @@ static int w64_read_header(AVFormatContext *s, AVFormatParameters *ap)
     WAVContext    *wav = s->priv_data;
     AVStream *st;
     uint8_t guid[16];
+    int ret;
 
     avio_read(pb, guid, 16);
     if (memcmp(guid, guid_riff, 16))
@@ -409,7 +413,9 @@ static int w64_read_header(AVFormatContext *s, AVFormatParameters *ap)
         return AVERROR(ENOMEM);
 
     /* subtract chunk header size - normal wav file doesn't count it */
-    ff_get_wav_header(pb, st->codec, size - 24);
+    ret = ff_get_wav_header(pb, st->codec, size - 24);
+    if (ret < 0)
+        return ret;
     avio_skip(pb, FFALIGN(size, INT64_C(8)) - size);
 
     st->need_parsing = AVSTREAM_PARSE_FULL;

libavformat/wtv.c

@@ -675,7 +675,9 @@ static AVStream * parse_media_type(AVFormatContext *s, AVStream *st, int sid,
         if (!st)
             return NULL;
         if (!ff_guidcmp(formattype, format_waveformatex)) {
-            ff_get_wav_header(pb, st->codec, size);
+            int ret = ff_get_wav_header(pb, st->codec, size);
+            if (ret < 0)
+                return NULL;
         } else {
             if (ff_guidcmp(formattype, format_none))
                 av_log(s, AV_LOG_WARNING, "unknown formattype:"PRI_GUID"\n", ARG_GUID(formattype));

libavformat/xwma.c

@@ -40,6 +40,7 @@ static int xwma_probe(AVProbeData *p)
 static int xwma_read_header(AVFormatContext *s, AVFormatParameters *ap)
 {
     int64_t size, av_uninit(data_size);
+    int ret;
     uint32_t dpds_table_size = 0;
     uint32_t *dpds_table = 0;
     unsigned int tag;
@@ -70,7 +71,9 @@ static int xwma_read_header(AVFormatContext *s, AVFormatParameters *ap)
     if (!st)
         return AVERROR(ENOMEM);
 
-    ff_get_wav_header(pb, st->codec, size);
+    ret = ff_get_wav_header(pb, st->codec, size);
+    if (ret < 0)
+        return ret;
     st->need_parsing = AVSTREAM_PARSE_NONE;
 
     /* All xWMA files I have seen contained WMAv2 data. If there are files