avformat/matroskadec: Fix lzo decompression

When a Matroska Block is only stored in compressed form, the size of
the uncompressed block is not explicitly coded and therefore not known
before decompressing it. Therefore the demuxer uses a guess for the
uncompressed size: The first guess is three times the compressed size
and if this is not enough, it is repeatedly incremented by a factor of
three. But when this happens with lzo, the decompression is neither
resumed nor started again. Instead when av_lzo1x_decode indicates that x
bytes of input data could not be decoded, because the output buffer is
already full, the first (not the last) x bytes of the input buffer are
resent for decoding in the next try; they overwrite already decoded
data.

This commit fixes this by instead restarting the decompression anew,
just with a bigger buffer.

This seems to be a regression since 935ec5a1.

A FATE-test for this has been added.
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@gmail.com>
Signed-off-by: James Almer <jamrial@gmail.com>

libavformat/matroskadec.c

@@ -1599,6 +1599,7 @@ static int matroska_decode_buffer(uint8_t **buf, int *buf_size,
 #if CONFIG_LZO
     case MATROSKA_TRACK_ENCODING_COMP_LZO:
         do {
+            int insize = isize;
             olen       = pkt_size *= 3;
             newpktdata = av_realloc(pkt_data, pkt_size + AV_LZO_OUTPUT_PADDING
                                                        + AV_INPUT_BUFFER_PADDING_SIZE);
@@ -1607,7 +1608,7 @@ static int matroska_decode_buffer(uint8_t **buf, int *buf_size,
                 goto failed;
             }
             pkt_data = newpktdata;
-            result   = av_lzo1x_decode(pkt_data, &olen, data, &isize);
+            result   = av_lzo1x_decode(pkt_data, &olen, data, &insize);
         } while (result == AV_LZO_OUTPUT_FULL && pkt_size < 10000000);
         if (result) {
             result = AVERROR_INVALIDDATA;

tests/fate/matroska.mak

@@ -22,6 +22,10 @@ fate-matroska-xiph-lacing: CMD = framecrc -i $(TARGET_SAMPLES)/mkv/xiph_lacing.m
 FATE_MATROSKA-$(call ALLYES, MATROSKA_DEMUXER ZLIB) += fate-matroska-zlib-decompression
 fate-matroska-zlib-decompression: CMD = framecrc -i $(TARGET_SAMPLES)/mkv/subtitle_zlib.mks -c:s copy
 
+# This tests that the matroska demuxer can decompress lzo compressed tracks.
+FATE_MATROSKA-$(call ALLYES, MATROSKA_DEMUXER LZO) += fate-matroska-lzo-decompression
+fate-matroska-lzo-decompression: CMD = framecrc -i $(TARGET_SAMPLES)/mkv/lzo.mka -c copy
+
 # This tests that the matroska demuxer correctly propagates
 # the channel layout contained in vorbis comments in the CodecPrivate
 # of flac tracks. It also tests header removal compression.

tests/ref/fate/matroska-lzo-decompression

+#tb 0: 11337/500000000
+#media_type 0: audio
+#codec_id 0: pcm_s16le
+#sample_rate 0: 44100
+#channel_layout 0: 3
+#channel_layout_name 0: stereo
+0,          0,          0,     4096,    16384, 0x00000000
+0,       4096,       4096,     4096,    16384, 0xad7eebf4
+0,       8192,       8192,     4096,    16384, 0x1d1ff9d9
+0,      12288,      12288,     4097,    16384, 0xf1d9e2e2