avcodec/hapdec: Check tex_size more strictly and before using it

Fixes: OOM
Fixes: 20774/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_HAP_fuzzer-5678608951803904
Fixes: 20956/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_HAP_fuzzer-5713643025203200

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpegSigned-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/hapdec.c

@@ -305,7 +305,6 @@ static int hap_decode(AVCodecContext *avctx, void *data,
     HapContext *ctx = avctx->priv_data;
     ThreadFrame tframe;
     int ret, i, t;
-    int tex_size;
     int section_size;
     enum HapSectionType section_type;
     int start_texture_section = 0;
@@ -342,6 +341,13 @@ static int hap_decode(AVCodecContext *avctx, void *data,
         if (ret < 0)
             return ret;
 
+        if (ctx->tex_size != (avctx->coded_width  / TEXTURE_BLOCK_W)
+            *(avctx->coded_height / TEXTURE_BLOCK_H)
+            *tex_rat[t]) {
+            av_log(avctx, AV_LOG_ERROR, "uncompressed size mismatches\n");
+            return AVERROR_INVALIDDATA;
+        }
+
         start_texture_section += ctx->texture_section_size + 4;
 
         if (avctx->codec->update_thread_context)
@@ -349,9 +355,16 @@ static int hap_decode(AVCodecContext *avctx, void *data,
 
         /* Unpack the DXT texture */
         if (hap_can_use_tex_in_place(ctx)) {
+            int tex_size;
             /* Only DXTC texture compression in a contiguous block */
             ctx->tex_data = ctx->gbc.buffer;
             tex_size = FFMIN(ctx->texture_section_size, bytestream2_get_bytes_left(&ctx->gbc));
+            if (tex_size < (avctx->coded_width  / TEXTURE_BLOCK_W)
+                *(avctx->coded_height / TEXTURE_BLOCK_H)
+                *tex_rat[t]) {
+                av_log(avctx, AV_LOG_ERROR, "Insufficient data\n");
+                return AVERROR_INVALIDDATA;
+            }
         } else {
             /* Perform the second-stage decompression */
             ret = av_reallocp(&ctx->tex_buf, ctx->tex_size);
@@ -367,14 +380,6 @@ static int hap_decode(AVCodecContext *avctx, void *data,
             }
 
             ctx->tex_data = ctx->tex_buf;
-            tex_size = ctx->tex_size;
-        }
-
-        if (tex_size < (avctx->coded_width  / TEXTURE_BLOCK_W)
-            *(avctx->coded_height / TEXTURE_BLOCK_H)
-            *tex_rat[t]) {
-            av_log(avctx, AV_LOG_ERROR, "Insufficient data\n");
-            return AVERROR_INVALIDDATA;
         }
 
         /* Use the decompress function on the texture, one block per thread */