aacenc_pred: rework the way prediction is done

This commit completely alters the algorithm of prediction.
The original commit which introduced prediction was completely
incorrect to even remotely care about what the actual coefficients
contain or whether any options were enabled. Not my actual fault.

This commit treats prediction the way the decoder does and expects
to do: like lossy encryption. Everything related to prediction now
happens at the very end but just before quantization and encoding
of coefficients. On the decoder side, prediction happens before
anything has had a chance to even access the coefficients.

Also the original implementation had problems because it actually
touched the band_type of special bands which already had their
scalefactor indices marked and it's a wonder the asserion wasn't
triggered when transmitting those.

Overall, this now drastically increases audio quality and you should
think about enabling it if you don't plan on playing anything encoded
on really old low power ultra-embedded devices since they might not
support decoding of prediction or AAC-Main. Though the specifications
were written ages ago and as times change so do the FLOPS.
Signed-off-by: Rostislav Pehlivanov <atomnuker@gmail.com>

libavcodec/aac.h

@@ -247,7 +247,7 @@ typedef struct SingleChannelElement {
     TemporalNoiseShaping tns;
     Pulse pulse;
     enum BandType band_type[128];                   ///< band types
-    enum BandType orig_band_type[128];              ///< band type backups for undoing prediction
+    enum BandType band_alt[128];                    ///< alternative band type (used by encoder)
     int band_type_run_end[120];                     ///< band type run end points
     INTFLOAT sf[120];                               ///< scalefactors
     int sf_idx[128];                                ///< scalefactor indices (used by encoder)

libavcodec/aaccoder.c

@@ -964,7 +964,6 @@ AACCoefficientsEncoder ff_aac_coders[AAC_CODER_NB] = {
         ff_aac_encode_main_pred,
         ff_aac_adjust_common_prediction,
         ff_aac_apply_main_pred,
-        ff_aac_update_main_pred,
         set_special_band_scalefactors,
         search_for_pns,
         ff_aac_search_for_tns,
@@ -980,7 +979,6 @@ AACCoefficientsEncoder ff_aac_coders[AAC_CODER_NB] = {
         ff_aac_encode_main_pred,
         ff_aac_adjust_common_prediction,
         ff_aac_apply_main_pred,
-        ff_aac_update_main_pred,
         set_special_band_scalefactors,
         search_for_pns,
         ff_aac_search_for_tns,
@@ -996,7 +994,6 @@ AACCoefficientsEncoder ff_aac_coders[AAC_CODER_NB] = {
         ff_aac_encode_main_pred,
         ff_aac_adjust_common_prediction,
         ff_aac_apply_main_pred,
-        ff_aac_update_main_pred,
         set_special_band_scalefactors,
         search_for_pns,
         ff_aac_search_for_tns,
@@ -1012,7 +1009,6 @@ AACCoefficientsEncoder ff_aac_coders[AAC_CODER_NB] = {
         ff_aac_encode_main_pred,
         ff_aac_adjust_common_prediction,
         ff_aac_apply_main_pred,
-        ff_aac_update_main_pred,
         set_special_band_scalefactors,
         search_for_pns,
         ff_aac_search_for_tns,

libavcodec/aacenc.c

@@ -354,15 +354,15 @@ static void encode_spectral_coeffs(AACEncContext *s, SingleChannelElement *sce)
                 start += sce->ics.swb_sizes[i];
                 continue;
             }
-            for (w2 = w; w2 < w + sce->ics.group_len[w]; w2++)
+            for (w2 = w; w2 < w + sce->ics.group_len[w]; w2++) {
                 s->coder->quantize_and_encode_band(s, &s->pb,
                                                    &sce->coeffs[start + w2*128],
-                                                   &sce->pqcoeffs[start + w2*128],
-                                                   sce->ics.swb_sizes[i],
+                                                   NULL, sce->ics.swb_sizes[i],
                                                    sce->sf_idx[w*16 + i],
                                                    sce->band_type[w*16 + i],
                                                    s->lambda,
                                                    sce->ics.window_clipping[w]);
+            }
             start += sce->ics.swb_sizes[i];
         }
     }
@@ -609,12 +609,8 @@ static int aac_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
                     s->coder->search_for_pns(s, avctx, sce);
                 if (s->options.tns && s->coder->search_for_tns)
                     s->coder->search_for_tns(s, sce);
-                if (s->options.pred && s->coder->search_for_pred)
-                    s->coder->search_for_pred(s, sce);
                 if (sce->tns.present)
                     tns_mode = 1;
-                if (sce->ics.predictor_present)
-                    pred_mode = 1;
             }
             s->cur_channel = start_ch;
             if (s->options.stereo_mode && cpe->common_window) {
@@ -631,15 +627,26 @@ static int aac_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
                 s->coder->search_for_is(s, avctx, cpe);
                 if (cpe->is_mode) is_mode = 1;
             }
-            if (s->options.pred && s->coder->adjust_common_prediction)
-                s->coder->adjust_common_prediction(s, cpe);
             if (s->coder->set_special_band_scalefactors)
                 for (ch = 0; ch < chans; ch++)
                     s->coder->set_special_band_scalefactors(s, &cpe->ch[ch]);
-            if (s->options.pred && s->coder->apply_main_pred)
-                for (ch = 0; ch < chans; ch++)
-                    s->coder->apply_main_pred(s, &cpe->ch[ch]);
             adjust_frame_information(cpe, chans);
+            for (ch = 0; ch < chans; ch++) {
+                sce = &cpe->ch[ch];
+                s->cur_channel = start_ch + ch;
+                if (s->options.pred && s->coder->search_for_pred)
+                    s->coder->search_for_pred(s, sce);
+                if (cpe->ch[ch].ics.predictor_present) pred_mode = 1;
+            }
+            if (s->options.pred && s->coder->adjust_common_prediction)
+                s->coder->adjust_common_prediction(s, cpe);
+            for (ch = 0; ch < chans; ch++) {
+                sce = &cpe->ch[ch];
+                s->cur_channel = start_ch + ch;
+                if (s->options.pred && s->coder->apply_main_pred)
+                    s->coder->apply_main_pred(s, sce);
+            }
+            s->cur_channel = start_ch;
             if (chans == 2) {
                 put_bits(&s->pb, 1, cpe->common_window);
                 if (cpe->common_window) {
@@ -676,16 +683,6 @@ static int aac_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
 
     } while (1);
 
-    // update predictor state
-    if (s->options.pred && s->coder->update_main_pred) {
-        for (i = 0; i < s->chan_map[0]; i++) {
-            cpe = &s->cpe[i];
-            for (ch = 0; ch < chans; ch++)
-                s->coder->update_main_pred(s, &cpe->ch[ch],
-                                           (cpe->common_window && !ch) ? cpe : NULL);
-        }
-    }
-
     put_bits(&s->pb, 3, TYPE_END);
     flush_put_bits(&s->pb);
     avctx->frame_bits = put_bits_count(&s->pb);

libavcodec/aacenc.h

@@ -63,7 +63,6 @@ typedef struct AACCoefficientsEncoder {
     void (*encode_main_pred)(struct AACEncContext *s, SingleChannelElement *sce);
     void (*adjust_common_prediction)(struct AACEncContext *s, ChannelElement *cpe);
     void (*apply_main_pred)(struct AACEncContext *s, SingleChannelElement *sce);
-    void (*update_main_pred)(struct AACEncContext *s, SingleChannelElement *sce, ChannelElement *cpe);
     void (*set_special_band_scalefactors)(struct AACEncContext *s, SingleChannelElement *sce);
     void (*search_for_pns)(struct AACEncContext *s, AVCodecContext *avctx, SingleChannelElement *sce);
     void (*search_for_tns)(struct AACEncContext *s, SingleChannelElement *sce);

libavcodec/aacenc_pred.h

@@ -34,16 +34,12 @@
 #define PRED_RESET_FRAME_MIN 240
 
 /* Any frame with less than this amount of frames since last reset is ok */
-#define PRED_RESET_MIN 128
+#define PRED_RESET_MIN 64
 
 /* Raise to filter any low frequency artifacts due to prediction */
 #define PRED_SFB_START 10
 
-/* Offset for the number of bits to encode normal coefficients */
-#define PRICE_OFFSET 440
-
 void ff_aac_apply_main_pred(AACEncContext *s, SingleChannelElement *sce);
-void ff_aac_update_main_pred(AACEncContext *s, SingleChannelElement *sce, ChannelElement *cpe);
 void ff_aac_adjust_common_prediction(AACEncContext *s, ChannelElement *cpe);
 void ff_aac_search_for_pred(AACEncContext *s, SingleChannelElement *sce);
 void ff_aac_encode_main_pred(AACEncContext *s, SingleChannelElement *sce);