avcodec/vc1: fix overlap smoothing filter for P frames

The v_overlap_filter needs to run on the colocated block of the previous
macroblock. For the luma plane, the colocated block is located two blocks
on the left instead of one. In addition, the overlap filter needs to run
on the non-edge blocks of the first macroblock row and column.

Fixes ticket #7171.
Signed-off-by: Jerome Borsboom <jerome.borsboom@carpalis.nl>

libavcodec/vc1_loopfilter.c

@@ -64,27 +64,23 @@ void ff_vc1_loop_filter_iblk(VC1Context *v, int pq)
 static av_always_inline void vc1_h_overlap_filter(VC1Context *v, int16_t (*left_block)[64],
                                                   int16_t (*right_block)[64], int block_num)
 {
-    if (left_block != right_block || (block_num & 5) == 1) {
-        if (block_num > 3)
-            v->vc1dsp.vc1_h_s_overlap(left_block[block_num], right_block[block_num]);
-        else if (block_num & 1)
-            v->vc1dsp.vc1_h_s_overlap(right_block[block_num - 1], right_block[block_num]);
-        else
-            v->vc1dsp.vc1_h_s_overlap(left_block[block_num + 1], right_block[block_num]);
-    }
+    if (block_num > 3)
+        v->vc1dsp.vc1_h_s_overlap(left_block[block_num], right_block[block_num]);
+    else if (block_num & 1)
+        v->vc1dsp.vc1_h_s_overlap(right_block[block_num - 1], right_block[block_num]);
+    else
+        v->vc1dsp.vc1_h_s_overlap(left_block[block_num + 1], right_block[block_num]);
 }
 
 static av_always_inline void vc1_v_overlap_filter(VC1Context *v, int16_t (*top_block)[64],
                                                   int16_t (*bottom_block)[64], int block_num)
 {
-    if (top_block != bottom_block || block_num & 2) {
-        if (block_num > 3)
-            v->vc1dsp.vc1_v_s_overlap(top_block[block_num], bottom_block[block_num]);
-        else if (block_num & 2)
-            v->vc1dsp.vc1_v_s_overlap(bottom_block[block_num - 2], bottom_block[block_num]);
-        else
-            v->vc1dsp.vc1_v_s_overlap(top_block[block_num + 2], bottom_block[block_num]);
-    }
+    if (block_num > 3)
+        v->vc1dsp.vc1_v_s_overlap(top_block[block_num], bottom_block[block_num]);
+    else if (block_num & 2)
+        v->vc1dsp.vc1_v_s_overlap(bottom_block[block_num - 2], bottom_block[block_num]);
+    else
+        v->vc1dsp.vc1_v_s_overlap(top_block[block_num + 2], bottom_block[block_num]);
 }
 
 void ff_vc1_i_overlap_filter(VC1Context *v)
@@ -108,21 +104,28 @@ void ff_vc1_i_overlap_filter(VC1Context *v)
      * borders. Therefore, the H overlap trails by one MB col and the
      * V overlap trails by one MB row. This is reflected in the time at which
      * we run the put_pixels loop, i.e. delayed by one row and one column. */
-    for (i = 0; i < block_count; i++)
+    for (i = 0; i < block_count; i++) {
+        if (s->mb_x == 0 && (i & 5) != 1)
+            continue;
+
         if (v->pq >= 9 || v->condover == CONDOVER_ALL ||
-            (v->over_flags_plane[mb_pos] && ((i & 5) == 1 || (s->mb_x && v->over_flags_plane[mb_pos - 1]))))
+            (v->over_flags_plane[mb_pos] && ((i & 5) == 1 || v->over_flags_plane[mb_pos - 1])))
             vc1_h_overlap_filter(v, s->mb_x ? left_blk : cur_blk, cur_blk, i);
+    }
 
     if (v->fcm != ILACE_FRAME)
         for (i = 0; i < block_count; i++) {
+            if (s->first_slice_line && !(i & 2))
+                continue;
+
             if (s->mb_x && (v->pq >= 9 || v->condover == CONDOVER_ALL ||
                 (v->over_flags_plane[mb_pos - 1] &&
-                 ((i & 2) || (!s->first_slice_line && v->over_flags_plane[mb_pos - 1 - s->mb_stride])))))
+                 ((i & 2) || v->over_flags_plane[mb_pos - 1 - s->mb_stride]))))
                 vc1_v_overlap_filter(v, s->first_slice_line ? left_blk : topleft_blk, left_blk, i);
             if (s->mb_x == s->mb_width - 1)
                 if (v->pq >= 9 || v->condover == CONDOVER_ALL ||
                     (v->over_flags_plane[mb_pos] &&
-                     ((i & 2) || (!s->first_slice_line && v->over_flags_plane[mb_pos - s->mb_stride]))))
+                     ((i & 2) || v->over_flags_plane[mb_pos - s->mb_stride])))
                     vc1_v_overlap_filter(v, s->first_slice_line ? cur_blk : top_blk, cur_blk, i);
         }
 }
@@ -139,18 +142,25 @@ void ff_vc1_p_overlap_filter(VC1Context *v)
     left_blk = v->block[v->left_blk_idx];
     cur_blk = v->block[v->cur_blk_idx];
 
-    for (i = 0; i < block_count; i++)
-        if (v->mb_type[0][s->block_index[i]] && (s->mb_x == 0 || v->mb_type[0][s->block_index[i] - 1]))
+    for (i = 0; i < block_count; i++) {
+        if (s->mb_x == 0 && (i & 5) != 1)
+            continue;
+
+        if (v->mb_type[0][s->block_index[i]] && v->mb_type[0][s->block_index[i] - 1])
             vc1_h_overlap_filter(v, s->mb_x ? left_blk : cur_blk, cur_blk, i);
+    }
 
     if (v->fcm != ILACE_FRAME)
         for (i = 0; i < block_count; i++) {
-            if (s->mb_x && v->mb_type[0][s->block_index[i] - 1] &&
-                (s->first_slice_line || v->mb_type[0][s->block_index[i] - s->block_wrap[i] - 1]))
+            if (s->first_slice_line && !(i & 2))
+                continue;
+
+            if (s->mb_x && v->mb_type[0][s->block_index[i] - 2 + (i > 3)] &&
+                v->mb_type[0][s->block_index[i] - s->block_wrap[i] - 2 + (i > 3)])
                 vc1_v_overlap_filter(v, s->first_slice_line ? left_blk : topleft_blk, left_blk, i);
             if (s->mb_x == s->mb_width - 1)
                 if (v->mb_type[0][s->block_index[i]] &&
-                    (s->first_slice_line || v->mb_type[0][s->block_index[i] - s->block_wrap[i]]))
+                    v->mb_type[0][s->block_index[i] - s->block_wrap[i]])
                     vc1_v_overlap_filter(v, s->first_slice_line ? cur_blk : top_blk, cur_blk, i);
         }
 }