lavf: Reset the entry count and allocation size variables on av_reallocp failures

When av_reallocp fails, the associated variables that keep track of
the number of elements in the array (and in some cases, the
separate number of allocated elements) need to be reset.

Not all of these might technically be needed, but it's better to
reset them if in doubt, to make sure variables don't end up
conflicting.
Signed-off-by: Martin Storsjö <martin@martin.st>

libavformat/avidec.c

@@ -657,9 +657,10 @@ static int avi_read_header(AVFormatContext *s)
                         st->codec->extradata_size += 9;
                         if ((ret = av_reallocp(&st->codec->extradata,
                                                st->codec->extradata_size +
-                                               FF_INPUT_BUFFER_PADDING_SIZE)) < 0)
+                                               FF_INPUT_BUFFER_PADDING_SIZE)) < 0) {
+                            st->codec->extradata_size = 0;
                             return ret;
-                        else
+                        } else
                             memcpy(st->codec->extradata + st->codec->extradata_size - 9,
                                    "BottomUp", 9);
                     }

libavformat/avienc.c

@@ -538,8 +538,11 @@ static int avi_write_packet(AVFormatContext *s, AVPacket *pkt)
         int cl = idx->entry / AVI_INDEX_CLUSTER_SIZE;
         int id = idx->entry % AVI_INDEX_CLUSTER_SIZE;
         if (idx->ents_allocated <= idx->entry) {
-            if ((err = av_reallocp(&idx->cluster, (cl + 1) * sizeof(*idx->cluster))) < 0)
+            if ((err = av_reallocp(&idx->cluster, (cl + 1) * sizeof(*idx->cluster))) < 0) {
+                idx->ents_allocated = 0;
+                idx->entry = 0;
                 return err;
+            }
             idx->cluster[cl] = av_malloc(AVI_INDEX_CLUSTER_SIZE*sizeof(AVIIentry));
             if (!idx->cluster[cl])
                 return -1;

libavformat/aviobuf.c

@@ -880,8 +880,11 @@ static int dyn_buf_write(void *opaque, uint8_t *buf, int buf_size)
 
     if (new_allocated_size > d->allocated_size) {
         int err;
-        if ((err = av_reallocp(&d->buffer, new_allocated_size)) < 0)
+        if ((err = av_reallocp(&d->buffer, new_allocated_size)) < 0) {
+            d->allocated_size = 0;
+            d->size = 0;
             return err;
+        }
         d->allocated_size = new_allocated_size;
     }
     memcpy(d->buffer + d->pos, buf, buf_size);

libavformat/mmst.c

@@ -337,8 +337,10 @@ static MMSSCPacketType get_tcp_server_response(MMSTContext *mmst)
                 if(!mms->header_parsed) {
                     if ((err = av_reallocp(&mms->asf_header,
                                            mms->asf_header_size +
-                                           mms->remaining_in_len)) < 0)
+                                           mms->remaining_in_len)) < 0) {
+                        mms->asf_header_size = 0;
                         return err;
+                    }
                     memcpy(mms->asf_header + mms->asf_header_size,
                            mms->read_in_ptr, mms->remaining_in_len);
                     mms->asf_header_size += mms->remaining_in_len;

libavformat/mov.c

@@ -886,8 +886,10 @@ static int mov_read_extradata(MOVContext *c, AVIOContext *pb, MOVAtom atom)
     size= (uint64_t)st->codec->extradata_size + atom.size + 8 + FF_INPUT_BUFFER_PADDING_SIZE;
     if (size > INT_MAX || (uint64_t)atom.size > INT_MAX)
         return AVERROR_INVALIDDATA;
-    if ((err = av_reallocp(&st->codec->extradata, size)) < 0)
+    if ((err = av_reallocp(&st->codec->extradata, size)) < 0) {
+        st->codec->extradata_size = 0;
         return err;
+    }
     buf = st->codec->extradata + st->codec->extradata_size;
     st->codec->extradata_size= size - FF_INPUT_BUFFER_PADDING_SIZE;
     AV_WB32(       buf    , atom.size + 8);

libavformat/oggparsetheora.c

@@ -124,8 +124,10 @@ theora_header (AVFormatContext * s, int idx)
     }
 
     if ((err = av_reallocp(&st->codec->extradata,
-                           cds + FF_INPUT_BUFFER_PADDING_SIZE)) < 0)
+                           cds + FF_INPUT_BUFFER_PADDING_SIZE)) < 0) {
+        st->codec->extradata_size = 0;
         return err;
+    }
     cdp = st->codec->extradata + st->codec->extradata_size;
     *cdp++ = os->psize >> 8;
     *cdp++ = os->psize & 0xff;

libavformat/oggparsevorbis.c

@@ -283,9 +283,12 @@ vorbis_header (AVFormatContext * s, int idx)
             }
         }
     } else {
-        int ret;
-        st->codec->extradata_size =
-            fixup_vorbis_headers(s, priv, &st->codec->extradata);
+        int ret = fixup_vorbis_headers(s, priv, &st->codec->extradata);
+        if (ret < 0) {
+            st->codec->extradata_size = 0;
+            return ret;
+        }
+        st->codec->extradata_size = ret;
         if ((ret = avpriv_vorbis_parse_extradata(st->codec, &priv->vp))) {
             av_freep(&st->codec->extradata);
             st->codec->extradata_size = 0;

libavformat/rdt.c

@@ -423,8 +423,10 @@ rdt_parse_sdp_line (AVFormatContext *s, int st_index,
                 if (first == -1) first = n;
                 if (rdt->nb_rmst < count) {
                     if ((err = av_reallocp(&rdt->rmst,
-                                           count * sizeof(*rdt->rmst))) < 0)
+                                           count * sizeof(*rdt->rmst))) < 0) {
+                        rdt->nb_rmst = 0;
                         return err;
+                    }
                     memset(rdt->rmst + rdt->nb_rmst, 0,
                            (count - rdt->nb_rmst) * sizeof(*rdt->rmst));
                     rdt->nb_rmst = count;

libavformat/rtmphttp.c

@@ -89,8 +89,11 @@ static int rtmp_http_write(URLContext *h, const uint8_t *buf, int size)
     if (rt->out_size + size > rt->out_capacity) {
         int err;
         rt->out_capacity = (rt->out_size + size) * 2;
-        if ((err = av_reallocp(&rt->out_data, rt->out_capacity)) < 0)
+        if ((err = av_reallocp(&rt->out_data, rt->out_capacity)) < 0) {
+            rt->out_size = 0;
+            rt->out_capacity = 0;
             return err;
+        }
     }
 
     memcpy(rt->out_data + rt->out_size, buf, size);

libavformat/rtmpproto.c

@@ -156,8 +156,11 @@ static int add_tracked_method(RTMPContext *rt, const char *name, int id)
     if (rt->nb_tracked_methods + 1 > rt->tracked_methods_size) {
         rt->tracked_methods_size = (rt->nb_tracked_methods + 1) * 2;
         if ((err = av_reallocp(&rt->tracked_methods, rt->tracked_methods_size *
-                               sizeof(*rt->tracked_methods))) < 0)
+                               sizeof(*rt->tracked_methods))) < 0) {
+            rt->nb_tracked_methods = 0;
+            rt->tracked_methods_size = 0;
             return err;
+        }
     }
 
     rt->tracked_methods[rt->nb_tracked_methods].name = av_strdup(name);

libavformat/rtpdec_qt.c

@@ -174,8 +174,10 @@ static int qt_rtp_parse_packet(AVFormatContext *s, PayloadContext *qt,
         if (qt->pkt.size > 0 && qt->timestamp == *timestamp) {
             int err;
             if ((err = av_reallocp(&qt->pkt.data, qt->pkt.size + alen +
-                                   FF_INPUT_BUFFER_PADDING_SIZE)) < 0)
+                                   FF_INPUT_BUFFER_PADDING_SIZE)) < 0) {
+                qt->pkt.size = 0;
                 return err;
+            }
         } else {
             av_freep(&qt->pkt.data);
             av_init_packet(&qt->pkt);

libavformat/smacker.c

@@ -315,8 +315,10 @@ static int smacker_read_packet(AVFormatContext *s, AVPacket *pkt)
                 frame_size -= size;
                 frame_size -= 4;
                 smk->curstream++;
-                if ((err = av_reallocp(&smk->bufs[smk->curstream], size)) < 0)
+                if ((err = av_reallocp(&smk->bufs[smk->curstream], size)) < 0) {
+                    smk->buf_sizes[smk->curstream] = 0;
                     return err;
+                }
                 smk->buf_sizes[smk->curstream] = size;
                 ret = avio_read(s->pb, smk->bufs[smk->curstream], size);
                 if(ret != size)

libavformat/smoothstreamingenc.c

@@ -450,8 +450,11 @@ static int add_fragment(OutputStream *os, const char *file, const char *infofile
     if (os->nb_fragments >= os->fragments_size) {
         os->fragments_size = (os->fragments_size + 1) * 2;
         if ((err = av_reallocp(&os->fragments, sizeof(*os->fragments) *
-                               os->fragments_size)) < 0)
+                               os->fragments_size)) < 0) {
+            os->fragments_size = 0;
+            os->nb_fragments = 0;
             return err;
+        }
     }
     frag = av_mallocz(sizeof(*frag));
     if (!frag)