wamenc: handle failure to encode.

This is necessary since the switch to floating point
input means there is no longer sufficient input data
validation.
There is a good chance that other encoders are affected
by similar issues.
This problem can be triggered by trying to encode
extremely large values and probably also with Inf and
possibly also NaNs.
Signed-off-by: Reimar Döffinger <Reimar.Doeffinger@gmx.de>

libavcodec/wmaenc.c

@@ -376,6 +376,11 @@ static int encode_superframe(AVCodecContext *avctx, AVPacket *avpkt,
 
     while(total_gain <= 128 && error > 0)
         error = encode_frame(s, s->coefs, avpkt->data, avpkt->size, total_gain++);
+    if (error > 0) {
+        av_log(avctx, AV_LOG_ERROR, "Invalid input data or requested bitrate too low, cannot encode\n");
+        avpkt->size = 0;
+        return AVERROR(EINVAL);
+    }
     av_assert0((put_bits_count(&s->pb) & 7) == 0);
     i= avctx->block_align - (put_bits_count(&s->pb)+7)/8;
     av_assert0(i>=0);