ac3dec: fix non-optimal dithering of zero bit mantissas

Use a noise range of -0.707 to 0.707 instead of -0.5 to 0.5
Signed-off-by: Justin Ruggles <justin.ruggles@gmail.com>

libavcodec/ac3dec.c

@@ -442,8 +442,9 @@ static void ac3_decode_transform_coeffs_ch(AC3DecodeContext *s, int ch_index, ma
         int mantissa;
         switch (bap) {
         case 0:
+            /* random noise with approximate range of -0.707 to 0.707 */
             if (dither)
-                mantissa = (av_lfg_get(&s->dith_state) & 0x7FFFFF) - 0x400000;
+                mantissa = (av_lfg_get(&s->dith_state) / 362) - 5932275;
             else
                 mantissa = 0;
             break;

tests/fate/ac3.mak

 FATE_AC3 += fate-ac3-2.0
 fate-ac3-2.0: CMD = pcm -i $(SAMPLES)/ac3/monsters_inc_2.0_192_small.ac3
-fate-ac3-2.0: REF = $(SAMPLES)/ac3/monsters_inc_2.0_192_small.pcm
+fate-ac3-2.0: REF = $(SAMPLES)/ac3/monsters_inc_2.0_192_small_v2.pcm
 
 FATE_AC3 += fate-ac3-4.0
 fate-ac3-4.0: CMD = pcm -i $(SAMPLES)/ac3/millers_crossing_4.0.ac3
-fate-ac3-4.0: REF = $(SAMPLES)/ac3/millers_crossing_4.0.pcm
+fate-ac3-4.0: REF = $(SAMPLES)/ac3/millers_crossing_4.0_v2.pcm
 
 FATE_AC3 += fate-ac3-4.0-downmix-mono
 fate-ac3-4.0-downmix-mono: CMD = pcm -request_channels 1 -i $(SAMPLES)/ac3/millers_crossing_4.0.ac3
-fate-ac3-4.0-downmix-mono: REF = $(SAMPLES)/ac3/millers_crossing_4.0_mono.pcm
+fate-ac3-4.0-downmix-mono: REF = $(SAMPLES)/ac3/millers_crossing_4.0_mono_v2.pcm
 
 FATE_AC3 += fate-ac3-4.0-downmix-stereo
 fate-ac3-4.0-downmix-stereo: CMD = pcm -request_channels 2 -i $(SAMPLES)/ac3/millers_crossing_4.0.ac3
-fate-ac3-4.0-downmix-stereo: REF = $(SAMPLES)/ac3/millers_crossing_4.0_stereo.pcm
+fate-ac3-4.0-downmix-stereo: REF = $(SAMPLES)/ac3/millers_crossing_4.0_stereo_v2.pcm
 
 FATE_AC3 += fate-ac3-5.1
 fate-ac3-5.1: CMD = pcm -i $(SAMPLES)/ac3/monsters_inc_5.1_448_small.ac3
-fate-ac3-5.1: REF = $(SAMPLES)/ac3/monsters_inc_5.1_448_small.pcm
+fate-ac3-5.1: REF = $(SAMPLES)/ac3/monsters_inc_5.1_448_small_v2.pcm
 
 FATE_AC3 += fate-ac3-5.1-downmix-mono
 fate-ac3-5.1-downmix-mono: CMD = pcm -request_channels 1 -i $(SAMPLES)/ac3/monsters_inc_5.1_448_small.ac3
-fate-ac3-5.1-downmix-mono: REF = $(SAMPLES)/ac3/monsters_inc_5.1_448_small_mono.pcm
+fate-ac3-5.1-downmix-mono: REF = $(SAMPLES)/ac3/monsters_inc_5.1_448_small_mono_v2.pcm
 
 FATE_AC3 += fate-ac3-5.1-downmix-stereo
 fate-ac3-5.1-downmix-stereo: CMD = pcm -request_channels 2 -i $(SAMPLES)/ac3/monsters_inc_5.1_448_small.ac3
-fate-ac3-5.1-downmix-stereo: REF = $(SAMPLES)/ac3/monsters_inc_5.1_448_small_stereo.pcm
+fate-ac3-5.1-downmix-stereo: REF = $(SAMPLES)/ac3/monsters_inc_5.1_448_small_stereo_v2.pcm
 
 FATE_EAC3 += fate-eac3-1
 fate-eac3-1: CMD = pcm -i $(SAMPLES)/eac3/csi_miami_5.1_256_spx_small.eac3
-fate-eac3-1: REF = $(SAMPLES)/eac3/csi_miami_5.1_256_spx_small.pcm
+fate-eac3-1: REF = $(SAMPLES)/eac3/csi_miami_5.1_256_spx_small_v2.pcm
 
 FATE_EAC3 += fate-eac3-2
 fate-eac3-2: CMD = pcm -i $(SAMPLES)/eac3/csi_miami_stereo_128_spx_small.eac3
-fate-eac3-2: REF = $(SAMPLES)/eac3/csi_miami_stereo_128_spx_small.pcm
+fate-eac3-2: REF = $(SAMPLES)/eac3/csi_miami_stereo_128_spx_small_v2.pcm
 
 FATE_EAC3 += fate-eac3-3
 fate-eac3-3: CMD = pcm -i $(SAMPLES)/eac3/matrix2_commentary1_stereo_192_small.eac3
-fate-eac3-3: REF = $(SAMPLES)/eac3/matrix2_commentary1_stereo_192_small.pcm
+fate-eac3-3: REF = $(SAMPLES)/eac3/matrix2_commentary1_stereo_192_small_v2.pcm
 
 FATE_EAC3 += fate-eac3-4
 fate-eac3-4: CMD = pcm -i $(SAMPLES)/eac3/serenity_english_5.1_1536_small.eac3
-fate-eac3-4: REF = $(SAMPLES)/eac3/serenity_english_5.1_1536_small.pcm
+fate-eac3-4: REF = $(SAMPLES)/eac3/serenity_english_5.1_1536_small_v2.pcm
 
 $(FATE_AC3) $(FATE_EAC3): CMP = oneoff
 
@@ -50,14 +50,14 @@ FATE_EAC3-$(call DEMDEC, EAC3, EAC3) += $(FATE_EAC3)
 FATE_AC3-$(call ENCDEC, AC3, AC3) += fate-ac3-encode
 fate-ac3-encode: CMD = enc_dec_pcm ac3 wav s16le $(REF) -c:a ac3 -b:a 128k
 fate-ac3-encode: CMP_SHIFT = -1024
-fate-ac3-encode: CMP_TARGET = 399.62
+fate-ac3-encode: CMP_TARGET = 404.53
 fate-ac3-encode: SIZE_TOLERANCE = 488
 fate-ac3-encode: FUZZ = 3
 
 FATE_EAC3-$(call ENCDEC, EAC3, EAC3) += fate-eac3-encode
 fate-eac3-encode: CMD = enc_dec_pcm eac3 wav s16le $(REF) -c:a eac3 -b:a 128k
 fate-eac3-encode: CMP_SHIFT = -1024
-fate-eac3-encode: CMP_TARGET = 514.02
+fate-eac3-encode: CMP_TARGET = 516.94
 fate-eac3-encode: SIZE_TOLERANCE = 488
 fate-eac3-encode: FUZZ = 3