Fix several security issues in matroskadec.c (MSVR-11-0080).

Whitespace of the patch cleaned up by Aurel
Some of the issues have been reported by Steve Manzuik / Microsoft Vulnerability Research (MSVR)
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavformat/matroskadec.c

@@ -829,11 +829,15 @@ static int ebml_parse_elem(MatroskaDemuxContext *matroska,
     uint32_t id = syntax->id;
     uint64_t length;
     int res;
+    void *newelem;
 
     data = (char *)data + syntax->data_offset;
     if (syntax->list_elem_size) {
         EbmlList *list = data;
-        list->elem = av_realloc(list->elem, (list->nb_elem+1)*syntax->list_elem_size);
+        newelem = av_realloc(list->elem, (list->nb_elem+1)*syntax->list_elem_size);
+        if (!newelem)
+            return AVERROR(ENOMEM);
+        list->elem = newelem;
         data = (char*)list->elem + list->nb_elem*syntax->list_elem_size;
         memset(data, 0, syntax->list_elem_size);
         list->nb_elem++;
@@ -995,7 +999,10 @@ static int matroska_decode_buffer(uint8_t** buf, int* buf_size,
             pkt_data = av_realloc(pkt_data, pkt_size);
             zstream.avail_out = pkt_size - zstream.total_out;
             zstream.next_out = pkt_data + zstream.total_out;
-            result = inflate(&zstream, Z_NO_FLUSH);
+            if (pkt_data) {
+                result = inflate(&zstream, Z_NO_FLUSH);
+            } else
+                result = Z_MEM_ERROR;
         } while (result==Z_OK && pkt_size<10000000);
         pkt_size = zstream.total_out;
         inflateEnd(&zstream);
@@ -1016,7 +1023,10 @@ static int matroska_decode_buffer(uint8_t** buf, int* buf_size,
             pkt_data = av_realloc(pkt_data, pkt_size);
             bzstream.avail_out = pkt_size - bzstream.total_out_lo32;
             bzstream.next_out = pkt_data + bzstream.total_out_lo32;
-            result = BZ2_bzDecompress(&bzstream);
+            if (pkt_data) {
+                result = BZ2_bzDecompress(&bzstream);
+            } else
+                result = BZ_MEM_ERROR;
         } while (result==BZ_OK && pkt_size<10000000);
         pkt_size = bzstream.total_out_lo32;
         BZ2_bzDecompressEnd(&bzstream);
@@ -1069,13 +1079,17 @@ static void matroska_fix_ass_packet(MatroskaDemuxContext *matroska,
     }
 }
 
-static void matroska_merge_packets(AVPacket *out, AVPacket *in)
+static int matroska_merge_packets(AVPacket *out, AVPacket *in)
 {
-    out->data = av_realloc(out->data, out->size+in->size);
+    void *newdata = av_realloc(out->data, out->size+in->size);
+    if (!newdata)
+        return AVERROR(ENOMEM);
+    out->data = newdata;
     memcpy(out->data+out->size, in->data, in->size);
     out->size += in->size;
     av_destruct_packet(in);
     av_free(in);
+    return 0;
 }
 
 static void matroska_convert_tag(AVFormatContext *s, EbmlList *list,
@@ -1661,11 +1675,13 @@ static int matroska_deliver_packet(MatroskaDemuxContext *matroska,
         memcpy(pkt, matroska->packets[0], sizeof(AVPacket));
         av_free(matroska->packets[0]);
         if (matroska->num_packets > 1) {
+            void *newpackets;
             memmove(&matroska->packets[0], &matroska->packets[1],
                     (matroska->num_packets - 1) * sizeof(AVPacket *));
-            matroska->packets =
-                av_realloc(matroska->packets, (matroska->num_packets - 1) *
-                           sizeof(AVPacket *));
+            newpackets = av_realloc(matroska->packets,
+                            (matroska->num_packets - 1) * sizeof(AVPacket *));
+            if (newpackets)
+                matroska->packets = newpackets;
         } else {
             av_freep(&matroska->packets);
         }