dnxhddec: check and report bitstream errors

This only occur when an overrun in coefficient decoding is
detected.
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/dnxhddec.c

@@ -39,6 +39,7 @@ typedef struct RowContext {
     GetBitContext gb;
     int last_dc[3];
     int last_qscale;
+    int errors;
 } RowContext;
 
 typedef struct DNXHDContext {
@@ -61,18 +62,18 @@ typedef struct DNXHDContext {
     int is_444;
     int mbaff;
     int act;
-    void (*decode_dct_block)(const struct DNXHDContext *ctx,
+    int (*decode_dct_block)(const struct DNXHDContext *ctx,
                              RowContext *row, int n);
 } DNXHDContext;
 
 #define DNXHD_VLC_BITS 9
 #define DNXHD_DC_VLC_BITS 7
 
-static void dnxhd_decode_dct_block_8(const DNXHDContext *ctx,
+static int dnxhd_decode_dct_block_8(const DNXHDContext *ctx,
                                      RowContext *row, int n);
-static void dnxhd_decode_dct_block_10(const DNXHDContext *ctx,
+static int dnxhd_decode_dct_block_10(const DNXHDContext *ctx,
                                       RowContext *row, int n);
-static void dnxhd_decode_dct_block_10_444(const DNXHDContext *ctx,
+static int dnxhd_decode_dct_block_10_444(const DNXHDContext *ctx,
                                           RowContext *row, int n);
 
 static av_cold int dnxhd_decode_init(AVCodecContext *avctx)
@@ -265,7 +266,7 @@ static int dnxhd_decode_header(DNXHDContext *ctx, AVFrame *frame,
     return 0;
 }
 
-static av_always_inline void dnxhd_decode_dct_block(const DNXHDContext *ctx,
+static av_always_inline int dnxhd_decode_dct_block(const DNXHDContext *ctx,
                                                     RowContext *row,
                                                     int n,
                                                     int index_bits,
@@ -280,6 +281,7 @@ static av_always_inline void dnxhd_decode_dct_block(const DNXHDContext *ctx,
     const uint8_t *ac_flags = ctx->cid_table->ac_flags;
     int16_t *block = row->blocks[n];
     const int eob_index     = ctx->cid_table->eob_index;
+    int ret = 0;
     OPEN_READER(bs, &row->gb);
 
     ctx->bdsp.clear_block(block);
@@ -343,6 +345,7 @@ static av_always_inline void dnxhd_decode_dct_block(const DNXHDContext *ctx,
 
         if (++i > 63) {
             av_log(ctx->avctx, AV_LOG_ERROR, "ac tex damaged %d, %d\n", n, i);
+            ret = -1;
             break;
         }
 
@@ -360,24 +363,25 @@ static av_always_inline void dnxhd_decode_dct_block(const DNXHDContext *ctx,
     }
 
     CLOSE_READER(bs, &row->gb);
+    return ret;
 }
 
-static void dnxhd_decode_dct_block_8(const DNXHDContext *ctx,
+static int dnxhd_decode_dct_block_8(const DNXHDContext *ctx,
                                      RowContext *row, int n)
 {
-    dnxhd_decode_dct_block(ctx, row, n, 4, 32, 6);
+    return dnxhd_decode_dct_block(ctx, row, n, 4, 32, 6);
 }
 
-static void dnxhd_decode_dct_block_10(const DNXHDContext *ctx,
+static int dnxhd_decode_dct_block_10(const DNXHDContext *ctx,
                                       RowContext *row, int n)
 {
-    dnxhd_decode_dct_block(ctx, row, n, 6, 8, 4);
+    return dnxhd_decode_dct_block(ctx, row, n, 6, 8, 4);
 }
 
-static void dnxhd_decode_dct_block_10_444(const DNXHDContext *ctx,
+static int dnxhd_decode_dct_block_10_444(const DNXHDContext *ctx,
                                           RowContext *row, int n)
 {
-    dnxhd_decode_dct_block(ctx, row, n, 6, 32, 6);
+    return dnxhd_decode_dct_block(ctx, row, n, 6, 32, 6);
 }
 
 static int dnxhd_decode_macroblock(const DNXHDContext *ctx, RowContext *row,
@@ -415,7 +419,8 @@ static int dnxhd_decode_macroblock(const DNXHDContext *ctx, RowContext *row,
     }
 
     for (i = 0; i < 8 + 4 * ctx->is_444; i++) {
-        ctx->decode_dct_block(ctx, row, i);
+        if (ctx->decode_dct_block(ctx, row, i) < 0)
+            return AVERROR_INVALIDDATA;
     }
 
     if (frame->interlaced_frame) {
@@ -488,7 +493,11 @@ static int dnxhd_decode_row(AVCodecContext *avctx, void *data,
     init_get_bits(&row->gb, ctx->buf + offset, (ctx->buf_size - offset) << 3);
     for (x = 0; x < ctx->mb_width; x++) {
         //START_TIMER;
-        dnxhd_decode_macroblock(ctx, row, data, x, rownb);
+        int ret = dnxhd_decode_macroblock(ctx, row, data, x, rownb);
+        if (ret < 0) {
+            row->errors++;
+            return ret;
+        }
         //STOP_TIMER("decode macroblock");
     }
 
@@ -504,7 +513,7 @@ static int dnxhd_decode_frame(AVCodecContext *avctx, void *data,
     ThreadFrame frame = { .f = data };
     AVFrame *picture = data;
     int first_field = 1;
-    int ret;
+    int ret, i;
 
     ff_dlog(avctx, "frame size %d\n", buf_size);
 
@@ -547,6 +556,17 @@ decode_coding_unit:
         goto decode_coding_unit;
     }
 
+    ret = 0;
+    for (i = 0; i < avctx->thread_count; i++) {
+        ret += ctx->rows[i].errors;
+        ctx->rows[i].errors = 0;
+    }
+
+    if (ret) {
+        av_log(ctx->avctx, AV_LOG_ERROR, "%d lines with errors\n", ret);
+        return AVERROR_INVALIDDATA;
+    }
+
     *got_frame = 1;
     return avpkt->size;
 }