avcodec/vp6: select idct based (loosely) on number of coefficients decoded

The VP3/4/5/6 reference decoders all use three IDCT versions: one for the
DC-only case, another for blocks with more than 10 coefficients, and an
optimised one for blocks with up to 10 AC coefficents. VP6 relies on the
sparse 10 coefficient version, and without it, IDCT drift occurs.

Fixes: https://trac.ffmpeg.org/ticket/1282Signed-off-by: Peter Ross <pross@xvid.org>

libavcodec/vp5.c

@@ -252,6 +252,7 @@ static int vp5_parse_coeff(VP56Context *s)
             for (i=coeff_idx; i<=ctx_last; i++)
                 s->coeff_ctx[ff_vp56_b6to4[b]][i] = 5;
         s->above_blocks[s->above_block_idx[b]].not_null_dc = s->coeff_ctx[ff_vp56_b6to4[b]][0];
+        s->idct_selector[b] = 63;
     }
     return 0;
 }

libavcodec/vp56.c

@@ -406,6 +406,24 @@ static void vp56_mc(VP56Context *s, int b, int plane, uint8_t *src,
     }
 }
 
+static void vp56_idct_put(VP56Context *s, uint8_t * dest, ptrdiff_t stride, int16_t *block, int selector)
+{
+    if (selector > 10 || selector == 1)
+        s->vp3dsp.idct_put(dest, stride, block);
+    else
+        ff_vp3dsp_idct10_put(dest, stride, block);
+}
+
+static void vp56_idct_add(VP56Context *s, uint8_t * dest, ptrdiff_t stride, int16_t *block, int selector)
+{
+    if (selector > 10)
+        s->vp3dsp.idct_add(dest, stride, block);
+    else if (selector > 1)
+        ff_vp3dsp_idct10_add(dest, stride, block);
+    else
+        s->vp3dsp.idct_dc_add(dest, stride, block);
+}
+
 static av_always_inline void vp56_render_mb(VP56Context *s, int row, int col, int is_alpha, VP56mb mb_type)
 {
     int b, ab, b_max, plane, off;
@@ -426,8 +444,8 @@ static av_always_inline void vp56_render_mb(VP56Context *s, int row, int col, in
         case VP56_MB_INTRA:
             for (b=0; b<b_max; b++) {
                 plane = ff_vp56_b2p[b+ab];
-                s->vp3dsp.idct_put(frame_current->data[plane] + s->block_offset[b],
-                                s->stride[plane], s->block_coeff[b]);
+                vp56_idct_put(s, frame_current->data[plane] + s->block_offset[b],
+                                s->stride[plane], s->block_coeff[b], s->idct_selector[b]);
             }
             break;
 
@@ -439,8 +457,8 @@ static av_always_inline void vp56_render_mb(VP56Context *s, int row, int col, in
                 s->hdsp.put_pixels_tab[1][0](frame_current->data[plane] + off,
                                              frame_ref->data[plane] + off,
                                              s->stride[plane], 8);
-                s->vp3dsp.idct_add(frame_current->data[plane] + off,
-                                s->stride[plane], s->block_coeff[b]);
+                vp56_idct_add(s, frame_current->data[plane] + off,
+                              s->stride[plane], s->block_coeff[b], s->idct_selector[b]);
             }
             break;
 
@@ -457,8 +475,8 @@ static av_always_inline void vp56_render_mb(VP56Context *s, int row, int col, in
                 plane = ff_vp56_b2p[b+ab];
                 vp56_mc(s, b, plane, frame_ref->data[plane], s->stride[plane],
                         16*col+x_off, 16*row+y_off);
-                s->vp3dsp.idct_add(frame_current->data[plane] + s->block_offset[b],
-                                s->stride[plane], s->block_coeff[b]);
+                vp56_idct_add(s, frame_current->data[plane] + s->block_offset[b],
+                              s->stride[plane], s->block_coeff[b], s->idct_selector[b]);
             }
             break;
     }

libavcodec/vp56.h

@@ -105,6 +105,7 @@ typedef struct VP56Macroblock {
 typedef struct VP56Model {
     uint8_t coeff_reorder[64];       /* used in vp6 only */
     uint8_t coeff_index_to_pos[64];  /* used in vp6 only */
+    uint8_t coeff_index_to_idct_selector[64]; /* used in vp6 only */
     uint8_t vector_sig[2];           /* delta sign */
     uint8_t vector_dct[2];           /* delta coding types */
     uint8_t vector_pdi[2][2];        /* predefined delta init */
@@ -157,6 +158,7 @@ struct vp56_context {
     VP56mb mb_type;
     VP56Macroblock *macroblocks;
     DECLARE_ALIGNED(16, int16_t, block_coeff)[6][64];
+    int idct_selector[6];
 
     /* motion vectors */
     VP56mv mv[6];  /* vectors for each block in MB */

libavcodec/vp6.c

@@ -194,6 +194,18 @@ static void vp6_coeff_order_table_init(VP56Context *s)
         for (pos=1; pos<64; pos++)
             if (s->modelp->coeff_reorder[pos] == i)
                 s->modelp->coeff_index_to_pos[idx++] = pos;
+
+    for (idx = 0; idx < 64; idx++) {
+        int max = 0;
+        for (i = 0; i <= idx; i++) {
+            int v = s->modelp->coeff_index_to_pos[i];
+            if (v > max)
+                max = v;
+        }
+        if (s->sub_version > 6)
+            max++;
+        s->modelp->coeff_index_to_idct_selector[idx] = max;
+    }
 }
 
 static void vp6_default_models_init(VP56Context *s)
@@ -446,6 +458,7 @@ static int vp6_parse_coeff_huffman(VP56Context *s)
             cg = FFMIN(vp6_coeff_groups[coeff_idx], 3);
             vlc_coeff = &s->ract_vlc[pt][ct][cg];
         }
+        s->idct_selector[b] = model->coeff_index_to_idct_selector[FFMIN(coeff_idx, 63)];
     }
     return 0;
 }
@@ -527,6 +540,7 @@ static int vp6_parse_coeff(VP56Context *s)
 
         s->left_block[ff_vp56_b6to4[b]].not_null_dc =
         s->above_blocks[s->above_block_idx[b]].not_null_dc = !!s->block_coeff[b][0];
+        s->idct_selector[b] = model->coeff_index_to_idct_selector[FFMIN(coeff_idx, 63)];
     }
     return 0;
 }