avformat/rmenc: Check for av_malloc() failure

Fixes CID1257832
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavformat/rmenc.c

@@ -364,6 +364,8 @@ static int rm_write_audio(AVFormatContext *s, const uint8_t *buf, int size, int
 
     /* XXX: suppress this malloc */
     buf1 = av_malloc(size * sizeof(uint8_t));
+    if (!buf1)
+        return AVERROR(ENOMEM);
 
     write_packet_header(s, stream, size, !!(flags & AV_PKT_FLAG_KEY));