rmdec: Check memory allocations from ff_rm_alloc_rmstream()

Bug-Id: CID 1257835

libavformat/rdt.c

@@ -434,6 +434,8 @@ rdt_parse_sdp_line (AVFormatContext *s, int st_index,
                     rdt->nb_rmst = count;
                 }
                 rdt->rmst[s->streams[n]->index] = ff_rm_alloc_rmstream();
+                if (!rdt->rmst[s->streams[n]->index])
+                    return AVERROR(ENOMEM);
                 rdt_load_mdpr(rdt, s->streams[n], (n - first) * 2);
            }
     }

libavformat/rmdec.c

@@ -109,6 +109,8 @@ static void rm_read_metadata(AVFormatContext *s, AVIOContext *pb, int wide)
 RMStream *ff_rm_alloc_rmstream (void)
 {
     RMStream *rms = av_mallocz(sizeof(RMStream));
+    if (!rms)
+        return NULL;
     rms->curpic_num = -1;
     return rms;
 }
@@ -416,6 +418,8 @@ static int rm_read_header_old(AVFormatContext *s)
     if (!st)
         return -1;
     st->priv_data = ff_rm_alloc_rmstream();
+    if (!st->priv_data)
+        return AVERROR(ENOMEM);
     return rm_read_audio_stream_info(s, s->pb, st, st->priv_data, 1);
 }
 
@@ -493,6 +497,8 @@ static int rm_read_header(AVFormatContext *s)
             get_str8(pb, buf, sizeof(buf)); /* mimetype */
             st->codec->codec_type = AVMEDIA_TYPE_DATA;
             st->priv_data = ff_rm_alloc_rmstream();
+            if (!st->priv_data)
+                return AVERROR(ENOMEM);
             if (ff_rm_read_mdpr_codecdata(s, s->pb, st, st->priv_data,
                                           avio_rb32(pb)) < 0)
                 return -1;