avcodec/vp6: use ff_vp3dsp_[hv]_loop_filter_12

Partially fixes: https://trac.ffmpeg.org/ticket/1282Signed-off-by: Peter Ross <pross@xvid.org>

libavcodec/vp56.c

@@ -33,6 +33,8 @@
 
 void ff_vp56_init_dequant(VP56Context *s, int quantizer)
 {
+    if (s->quantizer != quantizer)
+        ff_vp3dsp_set_bounding_values(s->bounding_values_array, ff_vp56_filter_threshold[quantizer]);
     s->quantizer = quantizer;
     s->dequant_dc = ff_vp56_dc_dequant[quantizer] << 2;
     s->dequant_ac = ff_vp56_ac_dequant[quantizer] << 2;
@@ -320,9 +322,17 @@ static void vp56_add_predictors_dc(VP56Context *s, VP56Frame ref_frame)
 static void vp56_deblock_filter(VP56Context *s, uint8_t *yuv,
                                 ptrdiff_t stride, int dx, int dy)
 {
+    if (s->avctx->codec->id == AV_CODEC_ID_VP5) {
     int t = ff_vp56_filter_threshold[s->quantizer];
     if (dx)  s->vp56dsp.edge_filter_hor(yuv +         10-dx , stride, t);
     if (dy)  s->vp56dsp.edge_filter_ver(yuv + stride*(10-dy), stride, t);
+    } else {
+        int * bounding_values = s->bounding_values_array + 127;
+        if (dx)
+            ff_vp3dsp_h_loop_filter_12(yuv +         10-dx, stride, bounding_values);
+        if (dy)
+            ff_vp3dsp_v_loop_filter_12(yuv + stride*(10-dy), stride, bounding_values);
+    }
 }
 
 static void vp56_mc(VP56Context *s, int b, int plane, uint8_t *src,

libavcodec/vp56.h

@@ -170,6 +170,7 @@ struct vp56_context {
     int filter_mode;
     int max_vector_length;
     int sample_variance_threshold;
+    DECLARE_ALIGNED(8, int, bounding_values_array)[256];
 
     uint8_t coeff_ctx[4][64];              /* used in vp5 only */
     uint8_t coeff_ctx_last[4];             /* used in vp5 only */

libavcodec/vp56dsp.c

@@ -72,27 +72,8 @@ av_cold void ff_vp5dsp_init(VP56DSPContext *s)
 #endif /* CONFIG_VP5_DECODER */
 
 #if CONFIG_VP6_DECODER
-static int vp6_adjust(int v, int t)
-{
-    int V = v, s = v >> 31;
-    V ^= s;
-    V -= s;
-    if (V-t-1 >= (unsigned)(t-1))
-        return v;
-    V = 2*t - V;
-    V += s;
-    V ^= s;
-    return V;
-}
-
-VP56_EDGE_FILTER(vp6, hor, 1, stride)
-VP56_EDGE_FILTER(vp6, ver, stride, 1)
-
 av_cold void ff_vp6dsp_init(VP56DSPContext *s)
 {
-    s->edge_filter_hor = vp6_edge_filter_hor;
-    s->edge_filter_ver = vp6_edge_filter_ver;
-
     s->vp6_filter_diag4 = ff_vp6_filter_diag4_c;
 
     if (ARCH_ARM)