indeo: Sanitize ff_ivi_init_planes fail paths

Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org

libavcodec/indeo4.c

@@ -209,6 +209,7 @@ static int decode_pic_hdr(IVI45DecContext *ctx, AVCodecContext *avctx)
     if (ivi_pic_config_cmp(&pic_conf, &ctx->pic_conf)) {
         if (ff_ivi_init_planes(ctx->planes, &pic_conf)) {
             av_log(avctx, AV_LOG_ERROR, "Couldn't reallocate color planes!\n");
+            ctx->pic_conf.luma_bands = 0;
             return AVERROR(ENOMEM);
         }
 

libavcodec/indeo5.c

@@ -113,7 +113,7 @@ static int decode_gop_header(IVI45DecContext *ctx, AVCodecContext *avctx)
     }
 
     /* check if picture layout was changed and reallocate buffers */
-    if (ivi_pic_config_cmp(&pic_conf, &ctx->pic_conf)) {
+    if (ivi_pic_config_cmp(&pic_conf, &ctx->pic_conf) || ctx->gop_invalid) {
         result = ff_ivi_init_planes(ctx->planes, &pic_conf);
         if (result < 0) {
             av_log(avctx, AV_LOG_ERROR, "Couldn't reallocate color planes!\n");
@@ -314,9 +314,9 @@ static int decode_pic_hdr(IVI45DecContext *ctx, AVCodecContext *avctx)
     ctx->frame_num = get_bits(&ctx->gb, 8);
 
     if (ctx->frame_type == FRAMETYPE_INTRA) {
-        ctx->gop_invalid = 1;
         if ((ret = decode_gop_header(ctx, avctx)) < 0) {
             av_log(avctx, AV_LOG_ERROR, "Invalid GOP header, skipping frames.\n");
+            ctx->gop_invalid = 1;
             return ret;
         }
         ctx->gop_invalid = 0;

libavcodec/ivi_common.c

@@ -244,6 +244,7 @@ static av_cold void ivi_free_buffers(IVIPlaneDesc *planes)
             av_freep(&planes[p].bands[b].tiles);
         }
         av_freep(&planes[p].bands);
+        planes[p].num_bands = 0;
     }
 }
 
@@ -256,6 +257,10 @@ av_cold int ff_ivi_init_planes(IVIPlaneDesc *planes, const IVIPicConfig *cfg)
 
     ivi_free_buffers(planes);
 
+    if (cfg->pic_width < 1 || cfg->pic_height < 1 ||
+        cfg->luma_bands < 1 || cfg->chroma_bands < 1)
+        return AVERROR_INVALIDDATA;
+
     /* fill in the descriptor of the luminance plane */
     planes[0].width     = cfg->pic_width;
     planes[0].height    = cfg->pic_height;