aacenc: Fix issues with huge values of bit_rate.

Do not pointlessly call ff_alloc_packet multiple times,
and fix an infinite loop by clamping the maximum
number of bits to target in the algorithm that does
not use lambda.
Signed-off-by: Reimar Döffinger <Reimar.Doeffinger@gmx.de>
Signed-off-by: Derek Buitenhuis <derek.buitenhuis@gmail.com>

libavcodec/aaccoder.c

@@ -721,6 +721,9 @@ static void search_for_quantizers_twoloop(AVCodecContext *avctx,
     int allz = 0;
     float minthr = INFINITY;
 
+    // for values above this the decoder might end up in an endless loop
+    // due to always having more bits than what can be encoded.
+    destbits = FFMIN(destbits, 5800);
     //XXX: some heuristic to determine initial quantizers will reduce search time
     //determine zero bands and upper limits
     for (w = 0; w < sce->ics.num_windows; w += sce->ics.group_len[w]) {

libavcodec/aacenc.c

@@ -571,13 +571,14 @@ static int aac_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
         }
         start_ch += chans;
     }
+    if ((ret = ff_alloc_packet(avpkt, 768 * s->channels))) {
+        av_log(avctx, AV_LOG_ERROR, "Error getting output packet\n");
+        return ret;
+    }
+
     do {
         int frame_bits;
 
-        if ((ret = ff_alloc_packet(avpkt, 768 * s->channels))) {
-            av_log(avctx, AV_LOG_ERROR, "Error getting output packet\n");
-            return ret;
-        }
         init_put_bits(&s->pb, avpkt->data, avpkt->size);
 
         if ((avctx->frame_number & 0xFF)==1 && !(avctx->flags & CODEC_FLAG_BITEXACT))