ffm: redesign header format to make it extensible

Currently FFM files generated with one versions of ffmpeg generally
cannot be read by another.
By spliting data into chunks, more fields can saftely be appended to
chunks as well as new chunks added.
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

Changelog

@@ -11,6 +11,7 @@ version <next>:
 - TAK demuxer, decoder and parser
 - DTS-HD demuxer
 - remove -same_quant, it hasn't worked for years
+- FFM2 support
 - X-Face image encoder and decoder
 - metadata (INFO tag) support in WAV muxer
 - subtitles raw text decoder

libavformat/ffmdec.c

@@ -228,6 +228,141 @@ static int ffm_close(AVFormatContext *s)
     return 0;
 }
 
+static int ffm2_read_header(AVFormatContext *s)
+{
+    FFMContext *ffm = s->priv_data;
+    AVStream *st;
+    AVIOContext *pb = s->pb;
+    AVCodecContext *codec;
+    int i;
+
+    ffm->packet_size = avio_rb32(pb);
+    if (ffm->packet_size != FFM_PACKET_SIZE)
+        goto fail;
+    ffm->write_index = avio_rb64(pb);
+    /* get also filesize */
+    if (pb->seekable) {
+        ffm->file_size = avio_size(pb);
+        if (ffm->write_index && 0)
+            adjust_write_index(s);
+    } else {
+        ffm->file_size = (UINT64_C(1) << 63) - 1;
+    }
+
+    while(!url_feof(pb)) {
+        unsigned id = avio_rb32(pb);
+        unsigned size = avio_rb32(pb);
+        int64_t next = avio_tell(pb) + size;
+        char rc_eq_buf[128];
+
+        if(!id)
+            break;
+
+        switch(id) {
+        case MKBETAG('M', 'A', 'I', 'N'):
+            avio_rb32(pb); /* nb_streams */
+            avio_rb32(pb); /* total bitrate */
+            break;
+        case MKBETAG('C', 'O', 'M', 'M'):
+            st = avformat_new_stream(s, NULL);
+            if (!st)
+                goto fail;
+
+            avpriv_set_pts_info(st, 64, 1, 1000000);
+
+            codec = st->codec;
+            /* generic info */
+            codec->codec_id = avio_rb32(pb);
+            codec->codec_type = avio_r8(pb);
+            codec->bit_rate = avio_rb32(pb);
+            codec->flags = avio_rb32(pb);
+            codec->flags2 = avio_rb32(pb);
+            codec->debug = avio_rb32(pb);
+            if (codec->flags & CODEC_FLAG_GLOBAL_HEADER) {
+                codec->extradata_size = avio_rb32(pb);
+                codec->extradata = av_malloc(codec->extradata_size);
+                if (!codec->extradata)
+                    return AVERROR(ENOMEM);
+                avio_read(pb, codec->extradata, codec->extradata_size);
+            }
+            avio_seek(pb, next, SEEK_SET);
+            id = avio_rb32(pb);
+            size = avio_rb32(pb);
+            next = avio_tell(pb) + size;
+            switch(id) {
+            case MKBETAG('S', 'T', 'V', 'I'):
+                codec->time_base.num = avio_rb32(pb);
+                codec->time_base.den = avio_rb32(pb);
+                codec->width = avio_rb16(pb);
+                codec->height = avio_rb16(pb);
+                codec->gop_size = avio_rb16(pb);
+                codec->pix_fmt = avio_rb32(pb);
+                codec->qmin = avio_r8(pb);
+                codec->qmax = avio_r8(pb);
+                codec->max_qdiff = avio_r8(pb);
+                codec->qcompress = avio_rb16(pb) / 10000.0;
+                codec->qblur = avio_rb16(pb) / 10000.0;
+                codec->bit_rate_tolerance = avio_rb32(pb);
+                avio_get_str(pb, INT_MAX, rc_eq_buf, sizeof(rc_eq_buf));
+                codec->rc_eq = av_strdup(rc_eq_buf);
+                codec->rc_max_rate = avio_rb32(pb);
+                codec->rc_min_rate = avio_rb32(pb);
+                codec->rc_buffer_size = avio_rb32(pb);
+                codec->i_quant_factor = av_int2double(avio_rb64(pb));
+                codec->b_quant_factor = av_int2double(avio_rb64(pb));
+                codec->i_quant_offset = av_int2double(avio_rb64(pb));
+                codec->b_quant_offset = av_int2double(avio_rb64(pb));
+                codec->dct_algo = avio_rb32(pb);
+                codec->strict_std_compliance = avio_rb32(pb);
+                codec->max_b_frames = avio_rb32(pb);
+                codec->mpeg_quant = avio_rb32(pb);
+                codec->intra_dc_precision = avio_rb32(pb);
+                codec->me_method = avio_rb32(pb);
+                codec->mb_decision = avio_rb32(pb);
+                codec->nsse_weight = avio_rb32(pb);
+                codec->frame_skip_cmp = avio_rb32(pb);
+                codec->rc_buffer_aggressivity = av_int2double(avio_rb64(pb));
+                codec->codec_tag = avio_rb32(pb);
+                codec->thread_count = avio_r8(pb);
+                codec->coder_type = avio_rb32(pb);
+                codec->me_cmp = avio_rb32(pb);
+                codec->me_subpel_quality = avio_rb32(pb);
+                codec->me_range = avio_rb32(pb);
+                codec->keyint_min = avio_rb32(pb);
+                codec->scenechange_threshold = avio_rb32(pb);
+                codec->b_frame_strategy = avio_rb32(pb);
+                codec->qcompress = av_int2double(avio_rb64(pb));
+                codec->qblur = av_int2double(avio_rb64(pb));
+                codec->max_qdiff = avio_rb32(pb);
+                codec->refs = avio_rb32(pb);
+                break;
+            case MKBETAG('S', 'T', 'A', 'U'):
+                codec->sample_rate = avio_rb32(pb);
+                codec->channels = avio_rl16(pb);
+                codec->frame_size = avio_rl16(pb);
+                break;
+            }
+            break;
+        }
+        avio_seek(pb, next, SEEK_SET);
+    }
+
+    /* get until end of block reached */
+    while ((avio_tell(pb) % ffm->packet_size) != 0)
+        avio_r8(pb);
+
+    /* init packet demux */
+    ffm->packet_ptr = ffm->packet;
+    ffm->packet_end = ffm->packet;
+    ffm->frame_offset = 0;
+    ffm->dts = 0;
+    ffm->read_state = READ_HEADER;
+    ffm->first_packet = 1;
+    return 0;
+ fail:
+    ffm_close(s);
+    return -1;
+}
 
 static int ffm_read_header(AVFormatContext *s)
 {
@@ -240,6 +375,8 @@ static int ffm_read_header(AVFormatContext *s)
 
     /* header */
     tag = avio_rl32(pb);
+    if (tag == MKTAG('F', 'F', 'M', '2'))
+        return ffm2_read_header(s);
     if (tag != MKTAG('F', 'F', 'M', '1'))
         goto fail;
     ffm->packet_size = avio_rb32(pb);
@@ -486,7 +623,7 @@ static int ffm_probe(AVProbeData *p)
 {
     if (
         p->buf[0] == 'F' && p->buf[1] == 'F' && p->buf[2] == 'M' &&
-        p->buf[3] == '1')
+        (p->buf[3] == '1' || p->buf[3] == '2'))
         return AVPROBE_SCORE_MAX + 1;
     return 0;
 }

libavformat/ffmenc.c

@@ -83,6 +83,16 @@ static void ffm_write_data(AVFormatContext *s,
     }
 }
 
+static void write_header_chunk(AVIOContext *pb, AVIOContext *dpb, unsigned id)
+{
+    uint8_t *dyn_buf;
+    int dyn_size= avio_close_dyn_buf(dpb, &dyn_buf);
+    avio_wb32(pb, id);
+    avio_wb32(pb, dyn_size);
+    avio_write(pb, dyn_buf, dyn_size);
+    av_free(dyn_buf);
+}
+
 static int ffm_write_header(AVFormatContext *s)
 {
     FFMContext *ffm = s->priv_data;
@@ -101,10 +111,13 @@ static int ffm_write_header(AVFormatContext *s)
     ffm->packet_size = FFM_PACKET_SIZE;
 
     /* header */
-    avio_wl32(pb, MKTAG('F', 'F', 'M', '1'));
+    avio_wl32(pb, MKTAG('F', 'F', 'M', '2'));
     avio_wb32(pb, ffm->packet_size);
     avio_wb64(pb, 0); /* current write position */
 
+    if(avio_open_dyn_buf(&pb) < 0)
+        return AVERROR(ENOMEM);
+
     avio_wb32(pb, s->nb_streams);
     bit_rate = 0;
     for(i=0;i<s->nb_streams;i++) {
@@ -113,10 +126,14 @@ static int ffm_write_header(AVFormatContext *s)
     }
     avio_wb32(pb, bit_rate);
 
+    write_header_chunk(s->pb, pb, MKBETAG('M', 'A', 'I', 'N'));
+
     /* list of streams */
     for(i=0;i<s->nb_streams;i++) {
         st = s->streams[i];
         avpriv_set_pts_info(st, 64, 1, 1000000);
+        if(avio_open_dyn_buf(&pb) < 0)
+            return AVERROR(ENOMEM);
 
         codec = st->codec;
         /* generic info */
@@ -126,6 +143,13 @@ static int ffm_write_header(AVFormatContext *s)
         avio_wb32(pb, codec->flags);
         avio_wb32(pb, codec->flags2);
         avio_wb32(pb, codec->debug);
+        if (codec->flags & CODEC_FLAG_GLOBAL_HEADER) {
+            avio_wb32(pb, codec->extradata_size);
+            avio_write(pb, codec->extradata, codec->extradata_size);
+        }
+        write_header_chunk(s->pb, pb, MKBETAG('C', 'O', 'M', 'M'));
+        if(avio_open_dyn_buf(&pb) < 0)
+            return AVERROR(ENOMEM);
         /* specific info */
         switch(codec->codec_type) {
         case AVMEDIA_TYPE_VIDEO:
@@ -172,20 +196,21 @@ static int ffm_write_header(AVFormatContext *s)
             avio_wb64(pb, av_double2int(codec->qblur));
             avio_wb32(pb, codec->max_qdiff);
             avio_wb32(pb, codec->refs);
+            write_header_chunk(s->pb, pb, MKBETAG('S', 'T', 'V', 'I'));
             break;
         case AVMEDIA_TYPE_AUDIO:
             avio_wb32(pb, codec->sample_rate);
             avio_wl16(pb, codec->channels);
             avio_wl16(pb, codec->frame_size);
+            write_header_chunk(s->pb, pb, MKBETAG('S', 'T', 'A', 'U'));
             break;
         default:
             return -1;
         }
-        if (codec->flags & CODEC_FLAG_GLOBAL_HEADER) {
-            avio_wb32(pb, codec->extradata_size);
-            avio_write(pb, codec->extradata, codec->extradata_size);
-        }
     }
+    pb = s->pb;
+
+    avio_wb64(pb, 0); // end of header
 
     /* flush until end of block reached */
     while ((avio_tell(pb) % ffm->packet_size) != 0)

tests/ref/lavf/ffm

-c76e8f9a9bcd04379dfa3239e272d049 *./tests/data/lavf/lavf.ffm
+d33fae310a7f6db1dc7fb74d1a9e0e6a *./tests/data/lavf/lavf.ffm
 376832 ./tests/data/lavf/lavf.ffm
 ./tests/data/lavf/lavf.ffm CRC=0x5b136bb1