mlvdec: validate bits_per_coded_sample

A negative bits_per_coded_sample doesn't make sense.
If it is too large, the size calculation for av_get_packet overflows,
resulting in allocation of a too small buffer.

Also make sure width and height are sane.
Reviewed-by: Michael Niedermayer <michael@niedermayer.cc>
Signed-off-by: Andreas Cadhalpun <Andreas.Cadhalpun@googlemail.com>

libavformat/mlvdec.c

@@ -25,6 +25,7 @@
  */
 
 #include "libavutil/eval.h"
+#include "libavutil/imgutils.h"
 #include "libavutil/intreadwrite.h"
 #include "libavutil/rational.h"
 #include "avformat.h"
@@ -131,10 +132,21 @@ static int scan_file(AVFormatContext *avctx, AVStream *vst, AVStream *ast, int f
         if (vst && type == MKTAG('R','A','W','I') && size >= 164) {
             vst->codec->width  = avio_rl16(pb);
             vst->codec->height = avio_rl16(pb);
+            ret = av_image_check_size(vst->codec->width, vst->codec->height, 0, avctx);
+            if (ret < 0)
+                return ret;
             if (avio_rl32(pb) != 1)
                 avpriv_request_sample(avctx, "raw api version");
             avio_skip(pb, 20); // pointer, width, height, pitch, frame_size
             vst->codec->bits_per_coded_sample = avio_rl32(pb);
+            if (vst->codec->bits_per_coded_sample < 0 ||
+                vst->codec->bits_per_coded_sample > (INT_MAX - 7) / (vst->codec->width * vst->codec->height)) {
+                av_log(avctx, AV_LOG_ERROR,
+                       "invalid bits_per_coded_sample %d (size: %dx%d)\n",
+                       vst->codec->bits_per_coded_sample,
+                       vst->codec->width, vst->codec->height);
+                return AVERROR_INVALIDDATA;
+            }
             avio_skip(pb, 8 + 16 + 24); // black_level, white_level, xywh, active_area, exposure_bias
             if (avio_rl32(pb) != 0x2010100) /* RGGB */
                 avpriv_request_sample(avctx, "cfa_pattern");