rtpdec: Forward the memory failure

And avoid a memory leak.
Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

libavformat/rtpdec.c

@@ -684,7 +684,7 @@ void ff_rtp_reset_packet_queue(RTPDemuxContext *s)
     s->prev_ret  = 0;
 }
 
-static void enqueue_packet(RTPDemuxContext *s, uint8_t *buf, int len)
+static int enqueue_packet(RTPDemuxContext *s, uint8_t *buf, int len)
 {
     uint16_t seq   = AV_RB16(buf + 2);
     RTPPacket **cur = &s->queue, *packet;
@@ -699,7 +699,7 @@ static void enqueue_packet(RTPDemuxContext *s, uint8_t *buf, int len)
 
     packet = av_mallocz(sizeof(*packet));
     if (!packet)
-        return;
+        return AVERROR(ENOMEM);
     packet->recvtime = av_gettime_relative();
     packet->seq      = seq;
     packet->len      = len;
@@ -707,6 +707,8 @@ static void enqueue_packet(RTPDemuxContext *s, uint8_t *buf, int len)
     packet->next     = *cur;
     *cur = packet;
     s->queue_len++;
+
+    return 0;
 }
 
 static int has_next_packet(RTPDemuxContext *s)
@@ -804,7 +806,9 @@ static int rtp_parse_one_packet(RTPDemuxContext *s, AVPacket *pkt,
             return rv;
         } else {
             /* Still missing some packet, enqueue this one. */
-            enqueue_packet(s, buf, len);
+            rv = enqueue_packet(s, buf, len);
+            if (rv < 0)
+                return rv;
             *bufptr = NULL;
             /* Return the first enqueued packet if the queue is full,
              * even if we're missing something */