nuv: check for malloc failure when allocating extradata

Also make sure extradata is freed in the case where multiple
NUV_EXTRADATA frame types are found. This may not happen in practice,
but it could happen in a malformed stream, which would lead to a memleak
if not handled.

libavformat/nuv.c

@@ -63,7 +63,7 @@ static int nuv_probe(AVProbeData *p)
  * @param vst video stream of which to change parameters
  * @param ast video stream of which to change parameters
  * @param myth set if this is a MythTVVideo format file
- * @return 1 if all required codec data was found
+ * @return 0 or AVERROR code
  */
 static int get_codec_data(AVIOContext *pb, AVStream *vst,
                           AVStream *ast, int myth)
@@ -82,12 +82,18 @@ static int get_codec_data(AVIOContext *pb, AVStream *vst,
             avio_skip(pb, 6);
             size = PKTSIZE(avio_rl32(pb));
             if (vst && subtype == 'R') {
+                if (vst->codec->extradata) {
+                    av_freep(&vst->codec->extradata);
+                    vst->codec->extradata_size = 0;
+                }
+                vst->codec->extradata = av_malloc(size);
+                if (!vst->codec->extradata)
+                    return AVERROR(ENOMEM);
                 vst->codec->extradata_size = size;
-                vst->codec->extradata      = av_malloc(size);
                 avio_read(pb, vst->codec->extradata, size);
                 size = 0;
                 if (!myth)
-                    return 1;
+                    return 0;
             }
             break;
         case NUV_MYTHEXT:
@@ -130,7 +136,7 @@ static int get_codec_data(AVIOContext *pb, AVStream *vst,
 
             size -= 6 * 4;
             avio_skip(pb, size);
-            return 1;
+            return 0;
         case NUV_SEEKP:
             size = 11;
             break;
@@ -151,7 +157,7 @@ static int nuv_header(AVFormatContext *s)
     AVIOContext *pb = s->pb;
     char id_string[12];
     double aspect, fps;
-    int is_mythtv, width, height, v_packs, a_packs;
+    int is_mythtv, width, height, v_packs, a_packs, ret;
     AVStream *vst = NULL, *ast = NULL;
 
     avio_read(pb, id_string, 12);
@@ -215,7 +221,9 @@ static int nuv_header(AVFormatContext *s)
     } else
         ctx->a_id = -1;
 
-    get_codec_data(pb, vst, ast, is_mythtv);
+    if ((ret = get_codec_data(pb, vst, ast, is_mythtv)) < 0)
+        return ret;
+
     ctx->rtjpg_video = vst && vst->codec->codec_id == AV_CODEC_ID_NUV;
 
     return 0;