libavfilter/transform: fix mirroring.

mirror() borrowed from dwt.c, this could be moved to libavutil and made
public API, Ill submit a patch for this

Fixes Ticket1278
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavfilter/transform.c

@@ -25,6 +25,7 @@
  */
 
 #include "libavutil/common.h"
+#include "libavutil/avassert.h"
 
 #include "transform.h"
 
@@ -135,6 +136,16 @@ void avfilter_mul_matrix(const float *m1, float scalar, float *result)
         result[i] = m1[i] * scalar;
 }
 
+static inline int mirror(int v, int m)
+{
+    while ((unsigned)v > (unsigned)m) {
+        v = -v;
+        if (v < 0)
+            v += 2 * m;
+    }
+    return v;
+}
+
 void avfilter_transform(const uint8_t *src, uint8_t *dst,
                         int src_stride, int dst_stride,
                         int width, int height, const float *matrix,
@@ -173,8 +184,11 @@ void avfilter_transform(const uint8_t *src, uint8_t *dst,
                     def = src[(int)y_s * src_stride + (int)x_s];
                     break;
                 case FILL_MIRROR:
-                    y_s = (y_s < 0) ? -y_s : (y_s >= height) ? (height + height - y_s) : y_s;
-                    x_s = (x_s < 0) ? -x_s : (x_s >= width) ? (width + width - x_s) : x_s;
+                    x_s = mirror(x_s,  width-1);
+                    y_s = mirror(y_s, height-1);
+
+                    av_assert2(x_s >= 0 && y_s >= 0);
+                    av_assert2(x_s < width && y_s < height);
                     def = src[(int)y_s * src_stride + (int)x_s];
             }