exr: fix overflow check in lineoffset processing

Also read data size for raw compressions too and
make sure its value is sane.

Remove code that fills missing blocks with zeroes.
It is marginally useful and make implementation
of actually useful features harder.
Signed-off-by: Paul B Mahol <onemda@gmail.com>

libavcodec/exr.c

@@ -229,6 +229,7 @@ static int decode_frame(AVCodecContext *avctx,
     const uint8_t *buf      = avpkt->data;
     unsigned int   buf_size = avpkt->size;
     const uint8_t *buf_end  = buf + buf_size;
+    const uint8_t *src;
 
     const AVPixFmtDescriptor *desc;
     EXRContext *const s = avctx->priv_data;
@@ -519,36 +520,37 @@ static int decode_frame(AVCodecContext *avctx,
     for (y = ymin; y <= ymax; y += scan_lines_per_block) {
         uint16_t *ptr_x = (uint16_t *)ptr;
         if (buf_end - buf > 8) {
-            /* Read the lineoffset from the line offset table and add 8 bytes
-               to skip the coordinates and data size fields */
-            const uint64_t line_offset = bytestream_get_le64(&buf) + 8;
+            const uint8_t *red_channel_buffer, *green_channel_buffer, *blue_channel_buffer, *alpha_channel_buffer = 0;
+            const uint64_t line_offset = bytestream_get_le64(&buf);
             int32_t data_size;
 
             // Check if the buffer has the required bytes needed from the offset
-            if ((line_offset > buf_size) ||
-                (s->compr == EXR_RAW && line_offset > avpkt->size - xdelta * current_channel_offset) ||
-                (s->compr != EXR_RAW && line_offset > buf_size - (data_size = AV_RL32(avpkt->data + line_offset - 4)))) {
-                // Line offset is probably wrong and not inside the buffer
-                av_log(avctx, AV_LOG_WARNING, "Line offset for line %d is out of reach setting it to black\n", y);
-                for (i = 0; i < scan_lines_per_block && y + i <= ymax; i++, ptr += stride) {
-                    ptr_x = (uint16_t *)ptr;
-                    memset(ptr_x, 0, out_line_size);
-                }
-            } else {
-                const uint8_t *red_channel_buffer, *green_channel_buffer, *blue_channel_buffer, *alpha_channel_buffer = 0;
+            if (line_offset > (uint64_t)buf_size - 8)
+                return AVERROR_INVALIDDATA;
+
+            src = avpkt->data + line_offset + 8;
+            data_size = AV_RL32(src - 4);
+            if (data_size <= 0 || data_size > buf_size)
+                return AVERROR_INVALIDDATA;
+
+            if ((s->compr == EXR_RAW && (data_size != uncompressed_size ||
+                                         line_offset > buf_size - uncompressed_size)) ||
+                (s->compr != EXR_RAW && line_offset > buf_size - data_size)) {
+                return AVERROR_INVALIDDATA;
+            }
 
                 if (scan_lines_per_block > 1)
                     uncompressed_size = scan_line_size * FFMIN(scan_lines_per_block, ymax - y + 1);
                 if ((s->compr == EXR_ZIP1 || s->compr == EXR_ZIP16) && data_size < uncompressed_size) {
                     unsigned long dest_len = uncompressed_size;
 
-                    if (uncompress(s->tmp, &dest_len, avpkt->data + line_offset, data_size) != Z_OK ||
+                    if (uncompress(s->tmp, &dest_len, src, data_size) != Z_OK ||
                         dest_len != uncompressed_size) {
                         av_log(avctx, AV_LOG_ERROR, "error during zlib decompression\n");
                         return AVERROR(EINVAL);
                     }
                 } else if (s->compr == EXR_RLE && data_size < uncompressed_size) {
-                    if (rle_uncompress(avpkt->data + line_offset, data_size, s->tmp, uncompressed_size)) {
+                    if (rle_uncompress(src, data_size, s->tmp, uncompressed_size)) {
                         av_log(avctx, AV_LOG_ERROR, "error during rle decompression\n");
                         return AVERROR(EINVAL);
                     }
@@ -564,11 +566,11 @@ static int decode_frame(AVCodecContext *avctx,
                     if (s->channel_offsets[3] >= 0)
                         alpha_channel_buffer = s->uncompressed_data + xdelta * s->channel_offsets[3];
                 } else {
-                    red_channel_buffer   = avpkt->data + line_offset + xdelta * s->channel_offsets[0];
-                    green_channel_buffer = avpkt->data + line_offset + xdelta * s->channel_offsets[1];
-                    blue_channel_buffer  = avpkt->data + line_offset + xdelta * s->channel_offsets[2];
+                    red_channel_buffer   = src + xdelta * s->channel_offsets[0];
+                    green_channel_buffer = src + xdelta * s->channel_offsets[1];
+                    blue_channel_buffer  = src + xdelta * s->channel_offsets[2];
                     if (s->channel_offsets[3] >= 0)
-                        alpha_channel_buffer = avpkt->data + line_offset + xdelta * s->channel_offsets[3];
+                        alpha_channel_buffer = src + xdelta * s->channel_offsets[3];
                 }
 
                 for (i = 0; i < scan_lines_per_block && y + i <= ymax; i++, ptr += stride) {
@@ -616,7 +618,6 @@ static int decode_frame(AVCodecContext *avctx,
                 }
             }
         }
-    }
 
     // Zero out the end if ymax+1 is not h
     for (y = ymax + 1; y < avctx->height; y++) {