aacenc_tns: rework coefficient quantization and filter application

This commit reworks the TNS implementation to a hybrid between what
the specifications say, what the decoder does and what's the best
thing to do.

The filter application function was copied from the decoder and
modified such that it applies the inverse AR filter to the
coefficients. The LPC coefficients themselves are fed into the
same quantization expression that the specifications say should
be used however further processing is not done, instead they're
converted to the form that the decoder expects them to be in
and are sent off to the compute_lpc_coeffs function exactly the
way the decoder does. This function does all conversions and will
return the exact coefficients that the decoder will generate, which
are then applied to the coefficients.
Having the exact same coefficients on both the encoder and decoder
is a must since otherwise the entire sfb's over which the filter
is applied will be attenuated.

Despite this major rework, TNS might not work fine on some audio
types at very low bitrates (e.g. sub 90kbps) as it can attenuate
some coefficients too much. Users are advised to experiment with
TNS at higher bitrates if they wish to use this tool or simply
wait for the implementation to be improved.
Signed-off-by: Rostislav Pehlivanov <atomnuker@gmail.com>

libavcodec/aacenc.c

@@ -611,7 +611,7 @@ static int aac_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
                 if (s->options.tns && s->coder->search_for_tns)
                     s->coder->search_for_tns(s, sce);
                 if (s->options.tns && s->coder->apply_tns_filt)
-                    s->coder->apply_tns_filt(sce);
+                    s->coder->apply_tns_filt(s, sce);
                 if (sce->tns.present)
                     tns_mode = 1;
             }

libavcodec/aacenc.h

@@ -63,7 +63,7 @@ typedef struct AACCoefficientsEncoder {
     void (*encode_main_pred)(struct AACEncContext *s, SingleChannelElement *sce);
     void (*adjust_common_prediction)(struct AACEncContext *s, ChannelElement *cpe);
     void (*apply_main_pred)(struct AACEncContext *s, SingleChannelElement *sce);
-    void (*apply_tns_filt)(SingleChannelElement *sce);
+    void (*apply_tns_filt)(struct AACEncContext *s, SingleChannelElement *sce);
     void (*set_special_band_scalefactors)(struct AACEncContext *s, SingleChannelElement *sce);
     void (*search_for_pns)(struct AACEncContext *s, AVCodecContext *avctx, SingleChannelElement *sce);
     void (*search_for_tns)(struct AACEncContext *s, SingleChannelElement *sce);

libavcodec/aacenc_tns.h

@@ -30,11 +30,21 @@
 
 #include "aacenc.h"
 
-/** Lower limit of TNS in SFBs **/
-#define TNS_LOW_LIMIT 24
+/* Could be set to 3 to save an additional bit at the cost of little quality */
+#define TNS_Q_BITS 4
+
+/* TNS will only be used if the LPC gain is within these margins */
+#define TNS_GAIN_THRESHOLD_LOW  1.395f
+#define TNS_GAIN_THRESHOLD_HIGH 11.19f
+
+/* Do not use TNS if the psy band spread is below this value */
+#define TNS_SPREAD_THRESHOLD 20.081512f
+
+/* Allows to reverse the filter direction if the band energy is uneven */
+#define TNS_DIRECTION_VARY 1
 
 void ff_aac_encode_tns_info(AACEncContext *s, SingleChannelElement *sce);
-void ff_aac_apply_tns(SingleChannelElement *sce);
+void ff_aac_apply_tns(AACEncContext *s, SingleChannelElement *sce);
 void ff_aac_search_for_tns(AACEncContext *s, SingleChannelElement *sce);
 
 #endif /* AVCODEC_AACENC_TNS_H */