avidec: check for valid bit_rate range

If bit_rate is negative, it can trigger an av_assert2 in av_rescale_rnd.

Since av_rescale returns int64_t, but st->codec_bit_rate is int, it can
also overflow into a negative value.
Signed-off-by: Andreas Cadhalpun <Andreas.Cadhalpun@googlemail.com>
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavformat/avidec.c

@@ -451,6 +451,7 @@ static int calculate_bitrate(AVFormatContext *s)
         int64_t len = 0;
         AVStream *st = s->streams[i];
         int64_t duration;
+        int64_t bitrate;
 
         for (j = 0; j < st->nb_index_entries; j++)
             len += st->index_entries[j].size;
@@ -458,7 +459,10 @@ static int calculate_bitrate(AVFormatContext *s)
         if (st->nb_index_entries < 2 || st->codec->bit_rate > 0)
             continue;
         duration = st->index_entries[j-1].timestamp - st->index_entries[0].timestamp;
-        st->codec->bit_rate = av_rescale(8*len, st->time_base.den, duration * st->time_base.num);
+        bitrate = av_rescale(8*len, st->time_base.den, duration * st->time_base.num);
+        if (bitrate <= INT_MAX && bitrate > 0) {
+            st->codec->bit_rate = bitrate;
+        }
     }
     return 1;
 }