avformat/matroskaenc: Only write Cues at the front if space has been reserved

If the AVIOContext for output was unseekable when writing the header,
no space for Cues would be reserved even if the reserve_index_space
option was used (because it is reasonable to expect that one can't seek
back to the beginning to write the Cues anyway). But if the AVIOContext
was seekable when writing the trailer, it was presumed that space for
the Cues had been reserved when the reserve_index_space option indicated
so even when it was not. As a result, the beginning of the file would be
overwritten.

This commit fixes this: If the reserve_index_space option had been used
and no space has been reserved in advance because of unseekability when
writing the header, then no attempt to write Cues will be performed
when writing the trailer; after all, writing them at the front is
impossible and writing them at the end is probably undesired.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>

libavformat/matroskaenc.c

@@ -1916,7 +1916,8 @@ static int mkv_write_header(AVFormatContext *s)
         if (mkv->reserve_cues_space == 1)
             mkv->reserve_cues_space++;
         put_ebml_void(pb, mkv->reserve_cues_space);
-        }
+        } else
+            mkv->reserve_cues_space = -1;
     }
 
     av_init_packet(&mkv->cur_audio_pkt);
@@ -2457,7 +2458,7 @@ static int mkv_write_trailer(AVFormatContext *s)
     MatroskaMuxContext *mkv = s->priv_data;
     AVIOContext *pb = s->pb;
     int64_t endpos, ret64;
-    int ret;
+    int ret, ret2 = 0;
 
     // check if we have an audio packet cached
     if (mkv->cur_audio_pkt.size > 0) {
@@ -2485,7 +2486,7 @@ static int mkv_write_trailer(AVFormatContext *s)
 
         endpos = avio_tell(pb);
 
-        if (mkv->cues.num_entries) {
+        if (mkv->cues.num_entries && mkv->reserve_cues_space >= 0) {
             AVIOContext *cues = NULL;
             uint64_t size;
             int length_size = 0;
@@ -2510,7 +2511,7 @@ static int mkv_write_trailer(AVFormatContext *s)
                            "Insufficient space reserved for Cues: "
                            "%d < %"PRIu64". No Cues will be output.\n",
                            mkv->reserve_cues_space, size);
-                    mkv->reserve_cues_space = -1;
+                    ret2 = AVERROR(EINVAL);
                     ffio_free_dyn_buf(&cues);
                     goto after_cues;
                 } else {
@@ -2600,7 +2601,7 @@ static int mkv_write_trailer(AVFormatContext *s)
 
         end_ebml_master(pb, mkv->segment);
 
-    return mkv->reserve_cues_space < 0 ? AVERROR(EINVAL) : 0;
+    return ret2;
 }
 
 static int mkv_query_codec(enum AVCodecID codec_id, int std_compliance)