aacenc_tns: rework the way coefficients are calculated

This commit abandons the way the specifications state to
quantize the coefficients, makes use of the new LPC float
functions and is much better.

The original way of converting non-normalized float samples
to int32_t which out LPC system expects was wrong and it was
wrong to assume the coefficients that are generated are also
valid. It was essentially a full garbage-in, garbage-out
system and it definitely shows when looking at spectrals
and listening. The high frequencies were very overattenuated.
The new LPC function performs the analysis directly.

The specifications state to quantize the coefficients into
four bit index values using an asin() function which of course
had to have ugly ternary operators because the function turns
negative if the coefficients are negative which when encoding
causes invalid bitstream to get generated.

This deviates from this by using the direct TNS tables, which
are fairly small since you only have 4 bits at most for index
values. The LPC values are directly quantized against the tables
and are then used to perform filtering after the requantization,
which simply fetches the array values.

The end result is that TNS works much better now and doesn't
attenuate anything but the actual signal, e.g. TNS removes
quantization errors and does it's job correctly now.

It might be enabled by default soon since it doesn't hurt and
helps reduce nastyness at low bitrates.
Signed-off-by: Rostislav Pehlivanov <atomnuker@gmail.com>

libavcodec/aaccoder.c

@@ -964,6 +964,7 @@ AACCoefficientsEncoder ff_aac_coders[AAC_CODER_NB] = {
         ff_aac_encode_main_pred,
         ff_aac_adjust_common_prediction,
         ff_aac_apply_main_pred,
+        ff_aac_apply_tns,
         set_special_band_scalefactors,
         search_for_pns,
         ff_aac_search_for_tns,
@@ -979,6 +980,7 @@ AACCoefficientsEncoder ff_aac_coders[AAC_CODER_NB] = {
         ff_aac_encode_main_pred,
         ff_aac_adjust_common_prediction,
         ff_aac_apply_main_pred,
+        ff_aac_apply_tns,
         set_special_band_scalefactors,
         search_for_pns,
         ff_aac_search_for_tns,
@@ -994,6 +996,7 @@ AACCoefficientsEncoder ff_aac_coders[AAC_CODER_NB] = {
         ff_aac_encode_main_pred,
         ff_aac_adjust_common_prediction,
         ff_aac_apply_main_pred,
+        ff_aac_apply_tns,
         set_special_band_scalefactors,
         search_for_pns,
         ff_aac_search_for_tns,
@@ -1009,6 +1012,7 @@ AACCoefficientsEncoder ff_aac_coders[AAC_CODER_NB] = {
         ff_aac_encode_main_pred,
         ff_aac_adjust_common_prediction,
         ff_aac_apply_main_pred,
+        ff_aac_apply_tns,
         set_special_band_scalefactors,
         search_for_pns,
         ff_aac_search_for_tns,

libavcodec/aacenc.c

@@ -404,10 +404,9 @@ static int encode_individual_channel(AVCodecContext *avctx, AACEncContext *s,
     encode_band_info(s, sce);
     encode_scale_factors(avctx, s, sce);
     encode_pulses(s, &sce->pulse);
+    put_bits(&s->pb, 1, !!sce->tns.present);
     if (s->coder->encode_tns_info)
         s->coder->encode_tns_info(s, sce);
-    else
-        put_bits(&s->pb, 1, 0);
     put_bits(&s->pb, 1, 0); //ssr
     encode_spectral_coeffs(s, sce);
     return 0;
@@ -609,6 +608,8 @@ static int aac_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
                     s->coder->search_for_pns(s, avctx, sce);
                 if (s->options.tns && s->coder->search_for_tns)
                     s->coder->search_for_tns(s, sce);
+                if (s->options.tns && s->coder->apply_tns_filt)
+                    s->coder->apply_tns_filt(sce);
                 if (sce->tns.present)
                     tns_mode = 1;
             }

libavcodec/aacenc.h

@@ -63,6 +63,7 @@ typedef struct AACCoefficientsEncoder {
     void (*encode_main_pred)(struct AACEncContext *s, SingleChannelElement *sce);
     void (*adjust_common_prediction)(struct AACEncContext *s, ChannelElement *cpe);
     void (*apply_main_pred)(struct AACEncContext *s, SingleChannelElement *sce);
+    void (*apply_tns_filt)(SingleChannelElement *sce);
     void (*set_special_band_scalefactors)(struct AACEncContext *s, SingleChannelElement *sce);
     void (*search_for_pns)(struct AACEncContext *s, AVCodecContext *avctx, SingleChannelElement *sce);
     void (*search_for_tns)(struct AACEncContext *s, SingleChannelElement *sce);

libavcodec/aacenc_tns.h

@@ -30,16 +30,11 @@
 
 #include "aacenc.h"
 
-/** Frequency in Hz for lower limit of TNS **/
-#define TNS_LOW_LIMIT 2150
-
-/** LPC settings */
-#define TNS_MIN_PRED_ORDER 0
-#define MAX_LPC_PRECISION  4   /* 4 bits ltp coeff precision */
-#define TNS_LPC_PASSES     2
-#define MAX_LPC_SHIFT      4
+/** Lower limit of TNS in SFBs **/
+#define TNS_LOW_LIMIT 24
 
 void ff_aac_encode_tns_info(AACEncContext *s, SingleChannelElement *sce);
+void ff_aac_apply_tns(SingleChannelElement *sce);
 void ff_aac_search_for_tns(AACEncContext *s, SingleChannelElement *sce);
 
 #endif /* AVCODEC_AACENC_TNS_H */