rtpenc_chain: Return an error code instead of just a plain pointer

Also check the return value in sapenc.
Signed-off-by: Martin Storsjö <martin@martin.st>

libavformat/movenchint.c

@@ -43,9 +43,9 @@ int ff_mov_init_hinting(AVFormatContext *s, int index, int src_index)
     track->enc->codec_type = AVMEDIA_TYPE_DATA;
     track->enc->codec_tag  = track->tag;
 
-    track->rtp_ctx = ff_rtp_chain_mux_open(s, src_st, NULL,
-                                           RTP_MAX_PACKET_SIZE);
-    if (!track->rtp_ctx)
+    ret = ff_rtp_chain_mux_open(&track->rtp_ctx, s, src_st, NULL,
+                                RTP_MAX_PACKET_SIZE);
+    if (ret < 0)
         goto fail;
 
     /* Copy the RTP AVStream timebase back to the hint AVStream */

libavformat/rtpenc_chain.c

@@ -25,8 +25,8 @@
 #include "avio_internal.h"
 #include "libavutil/opt.h"
 
-AVFormatContext *ff_rtp_chain_mux_open(AVFormatContext *s, AVStream *st,
-                                       URLContext *handle, int packet_size)
+int ff_rtp_chain_mux_open(AVFormatContext **out, AVFormatContext *s,
+                          AVStream *st, URLContext *handle, int packet_size)
 {
     AVFormatContext *rtpctx = NULL;
     int ret;
@@ -34,17 +34,23 @@ AVFormatContext *ff_rtp_chain_mux_open(AVFormatContext *s, AVStream *st,
     uint8_t *rtpflags;
     AVDictionary *opts = NULL;
 
-    if (!rtp_format)
+    if (!rtp_format) {
+        ret = AVERROR(ENOSYS);
         goto fail;
+    }
 
     /* Allocate an AVFormatContext for each output stream */
     rtpctx = avformat_alloc_context();
-    if (!rtpctx)
+    if (!rtpctx) {
+        ret = AVERROR(ENOMEM);
         goto fail;
+    }
 
     rtpctx->oformat = rtp_format;
-    if (!avformat_new_stream(rtpctx, NULL))
+    if (!avformat_new_stream(rtpctx, NULL)) {
+        ret = AVERROR(ENOMEM);
         goto fail;
+    }
     /* Pass the interrupt callback on */
     rtpctx->interrupt_callback = s->interrupt_callback;
     /* Copy the max delay setting; the rtp muxer reads this. */
@@ -76,14 +82,15 @@ AVFormatContext *ff_rtp_chain_mux_open(AVFormatContext *s, AVStream *st,
             av_free(ptr);
         }
         avformat_free_context(rtpctx);
-        return NULL;
+        return ret;
     }
 
-    return rtpctx;
+    *out = rtpctx;
+    return 0;
 
 fail:
     av_free(rtpctx);
     if (handle)
         ffurl_close(handle);
-    return NULL;
+    return ret;
 }

libavformat/rtpenc_chain.h

@@ -25,7 +25,7 @@
 #include "avformat.h"
 #include "url.h"
 
-AVFormatContext *ff_rtp_chain_mux_open(AVFormatContext *s, AVStream *st,
-                                       URLContext *handle, int packet_size);
+int ff_rtp_chain_mux_open(AVFormatContext **out, AVFormatContext *s,
+                          AVStream *st, URLContext *handle, int packet_size);
 
 #endif /* AVFORMAT_RTPENC_CHAIN_H */

libavformat/rtsp.c

@@ -606,11 +606,13 @@ static int rtsp_open_transport_ctx(AVFormatContext *s, RTSPStream *rtsp_st)
         s->ctx_flags |= AVFMTCTX_NOHEADER;
 
     if (s->oformat && CONFIG_RTSP_MUXER) {
-        rtsp_st->transport_priv = ff_rtp_chain_mux_open(s, st,
-                                      rtsp_st->rtp_handle,
-                                      RTSP_TCP_MAX_PACKET_SIZE);
+        int ret = ff_rtp_chain_mux_open(&rtsp_st->transport_priv, s, st,
+                                        rtsp_st->rtp_handle,
+                                        RTSP_TCP_MAX_PACKET_SIZE);
         /* Ownership of rtp_handle is passed to the rtp mux context */
         rtsp_st->rtp_handle = NULL;
+        if (ret < 0)
+            return ret;
     } else if (rt->transport == RTSP_TRANSPORT_RDT && CONFIG_RTPDEC)
         rtsp_st->transport_priv = ff_rdt_parse_open(s, st->index,
                                             rtsp_st->dynamic_protocol_context,

libavformat/sapenc.c

@@ -150,8 +150,10 @@ static int sap_write_header(AVFormatContext *s)
             ret = AVERROR(EIO);
             goto fail;
         }
-        s->streams[i]->priv_data = contexts[i] =
-            ff_rtp_chain_mux_open(s, s->streams[i], fd, 0);
+        ret = ff_rtp_chain_mux_open(&contexts[i], s, s->streams[i], fd, 0);
+        if (ret < 0)
+            goto fail;
+        s->streams[i]->priv_data = contexts[i];
         av_strlcpy(contexts[i]->filename, url, sizeof(contexts[i]->filename));
     }