Commit 4b0cad65 authored by Michael Niedermayer's avatar Michael Niedermayer

avcodec/simple_idct_template: fix rounding of the special DC case for 10bit

MSE doesnt change for the test as the code only triggers
for very sparse matrixes
Signed-off-by: 's avatarMichael Niedermayer <michaelni@gmx.at>
parent a7ea733b
......@@ -107,10 +107,10 @@ static inline void FUNC(idctRowCondDC)(int16_t *row, int extra_shift)
#define ROW0_MASK (0xffffLL << 48 * HAVE_BIGENDIAN)
if (((((uint64_t *)row)[0] & ~ROW0_MASK) | ((uint64_t *)row)[1]) == 0) {
uint64_t temp;
if (DC_SHIFT - extra_shift > 0) {
if (DC_SHIFT - extra_shift >= 0) {
temp = (row[0] << (DC_SHIFT - extra_shift)) & 0xffff;
} else {
temp = (row[0] >> (extra_shift - DC_SHIFT)) & 0xffff;
temp = ((row[0] + (1<<(extra_shift - DC_SHIFT-1))) >> (extra_shift - DC_SHIFT)) & 0xffff;
}
temp += temp << 16;
temp += temp << 32;
......@@ -124,10 +124,10 @@ static inline void FUNC(idctRowCondDC)(int16_t *row, int extra_shift)
((uint32_t*)row)[3] |
row[1])) {
uint32_t temp;
if (DC_SHIFT - extra_shift > 0) {
if (DC_SHIFT - extra_shift >= 0) {
temp = (row[0] << (DC_SHIFT - extra_shift)) & 0xffff;
} else {
temp = (row[0] >> (extra_shift - DC_SHIFT)) & 0xffff;
temp = ((row[0] + (1<<(extra_shift - DC_SHIFT-1))) >> (extra_shift - DC_SHIFT)) & 0xffff;
}
temp += temp << 16;
((uint32_t*)row)[0]=((uint32_t*)row)[1] =
......
#tb 0: 100/2997
0, 0, 0, 1, 8294400, 0xdaff4019
0, 1, 1, 1, 8294400, 0xdaff4019
0, 0, 0, 1, 8294400, 0x4ca110c7
0, 1, 1, 1, 8294400, 0x4ca110c7
af885572d77d28e5ede27932bd68cdde *tests/data/fate/vsynth2-prores.mov
2844076 tests/data/fate/vsynth2-prores.mov
387a3c23586c679b411c3eff2c9d9d06 *tests/data/fate/vsynth2-prores.out.rawvideo
03fd29e3963716a09d232b6f817ecb57 *tests/data/fate/vsynth2-prores.out.rawvideo
stddev: 1.31 PSNR: 45.77 MAXDIFF: 11 bytes: 7603200/ 7603200
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment