avformat/ape: Cleanup after ape_read_header() failure

Fixes: memleaks
Fixes: 23306/clusterfuzz-testcase-minimized-ffmpeg_DEMUXER_fuzzer-5635436931448832

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpegSigned-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 9b5fc789fb52af8769ec66e634ea362a67cb5d06)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavformat/ape.c

@@ -83,6 +83,8 @@ typedef struct APEContext {
     uint8_t  *bittable;
 } APEContext;
 
+static int ape_read_close(AVFormatContext * s);
+
 static int ape_probe(const AVProbeData * p)
 {
     int version = AV_RL16(p->buf+4);
@@ -281,14 +283,18 @@ static int ape_read_header(AVFormatContext * s)
 
     if (ape->seektablelength > 0) {
         ape->seektable = av_mallocz(ape->seektablelength);
-        if (!ape->seektable)
-            return AVERROR(ENOMEM);
+        if (!ape->seektable) {
+            ret = AVERROR(ENOMEM);
+            goto fail;
+        }
         for (i = 0; i < ape->seektablelength / sizeof(uint32_t) && !pb->eof_reached; i++)
             ape->seektable[i] = avio_rl32(pb);
         if (ape->fileversion < 3810) {
             ape->bittable = av_mallocz(ape->totalframes);
-            if (!ape->bittable)
-                return AVERROR(ENOMEM);
+            if (!ape->bittable) {
+                ret = AVERROR(ENOMEM);
+                goto fail;
+            }
             for (i = 0; i < ape->totalframes && !pb->eof_reached; i++)
                 ape->bittable[i] = avio_r8(pb);
         }
@@ -341,8 +347,10 @@ static int ape_read_header(AVFormatContext * s)
 
     /* now we are ready: build format streams */
     st = avformat_new_stream(s, NULL);
-    if (!st)
-        return AVERROR(ENOMEM);
+    if (!st) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
 
     total_blocks = (ape->totalframes == 0) ? 0 : ((ape->totalframes - 1) * ape->blocksperframe) + ape->finalframeblocks;
 
@@ -359,7 +367,7 @@ static int ape_read_header(AVFormatContext * s)
     avpriv_set_pts_info(st, 64, 1, ape->samplerate);
 
     if ((ret = ff_alloc_extradata(st->codecpar, APE_EXTRADATA_SIZE)) < 0)
-        return ret;
+        goto fail;
     AV_WL16(st->codecpar->extradata + 0, ape->fileversion);
     AV_WL16(st->codecpar->extradata + 2, ape->compressiontype);
     AV_WL16(st->codecpar->extradata + 4, ape->formatflags);
@@ -378,6 +386,10 @@ static int ape_read_header(AVFormatContext * s)
     }
 
     return 0;
+fail:
+    ape_read_close(s);
+
+    return ret;
 }
 
 static int ape_read_packet(AVFormatContext * s, AVPacket * pkt)