avcodec/jpeg2000dec: Fix some 5/3 bitexactness issues

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/jpeg2000dec.c

@@ -1145,7 +1145,21 @@ static void dequantization_int(int x, int y, Jpeg2000Cblk *cblk,
         int32_t *datap = &comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * (y + j) + x];
         int *src = t1->data[j];
         for (i = 0; i < w; ++i)
-            datap[i] = (src[i] * band->i_stepsize + (1 << 14)) >> 15;
+            datap[i] = (src[i] * band->i_stepsize) / 32768;
+    }
+}
+
+static void dequantization_int_97(int x, int y, Jpeg2000Cblk *cblk,
+                               Jpeg2000Component *comp,
+                               Jpeg2000T1Context *t1, Jpeg2000Band *band)
+{
+    int i, j;
+    int w = cblk->coord[0][1] - cblk->coord[0][0];
+    for (j = 0; j < (cblk->coord[1][1] - cblk->coord[1][0]); ++j) {
+        int32_t *datap = &comp->i_data[(comp->coord[0][1] - comp->coord[0][0]) * (y + j) + x];
+        int *src = t1->data[j];
+        for (i = 0; i < w; ++i)
+            datap[i] = (src[i] * band->i_stepsize + (1<<14)) >> 15;
     }
 }
 
@@ -1228,6 +1242,8 @@ static int jpeg2000_decode_tile(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile,
 
                         if (codsty->transform == FF_DWT97)
                             dequantization_float(x, y, cblk, comp, &t1, band);
+                        else if (codsty->transform == FF_DWT97_INT)
+                            dequantization_int_97(x, y, cblk, comp, &t1, band);
                         else
                             dequantization_int(x, y, cblk, comp, &t1, band);
                    } /* end cblk */

tests/ref/vsynth/vsynth1-jpeg2000

 ed9c45dc090a03c2eb9c35cf76e4d914 *tests/data/fate/vsynth1-jpeg2000.avi
 2306906 tests/data/fate/vsynth1-jpeg2000.avi
-1774b621bd92a53a24712cb77e9f0b28 *tests/data/fate/vsynth1-jpeg2000.out.rawvideo
-stddev:    5.37 PSNR: 33.52 MAXDIFF:   63 bytes:  7603200/  7603200
+64166a8fbb730a7a132c50ee89592672 *tests/data/fate/vsynth1-jpeg2000.out.rawvideo
+stddev:    5.41 PSNR: 33.46 MAXDIFF:   63 bytes:  7603200/  7603200

tests/ref/vsynth/vsynth2-jpeg2000

 b918de21dec4310f87af3788ed458462 *tests/data/fate/vsynth2-jpeg2000.avi
 1513144 tests/data/fate/vsynth2-jpeg2000.avi
-1d33de510f21eaad6c3cecfcf29798ba *tests/data/fate/vsynth2-jpeg2000.out.rawvideo
-stddev:    4.99 PSNR: 34.17 MAXDIFF:   70 bytes:  7603200/  7603200
+d30f637670a60de6cc8d177047f32a59 *tests/data/fate/vsynth2-jpeg2000.out.rawvideo
+stddev:    5.04 PSNR: 34.07 MAXDIFF:   71 bytes:  7603200/  7603200

tests/ref/vsynth/vsynth3-jpeg2000

 7ea4e3901817f06a3de59ee70836f5d9 *tests/data/fate/vsynth3-jpeg2000.avi
 63772 tests/data/fate/vsynth3-jpeg2000.avi
-11b954cc1b0091399fa4342a6bb432b2 *tests/data/fate/vsynth3-jpeg2000.out.rawvideo
-stddev:    5.50 PSNR: 33.31 MAXDIFF:   48 bytes:    86700/    86700
+ddfc0e8e033aeecf334c1450148dab44 *tests/data/fate/vsynth3-jpeg2000.out.rawvideo
+stddev:    5.52 PSNR: 33.28 MAXDIFF:   48 bytes:    86700/    86700

tests/ref/vsynth/vsynth_lena-jpeg2000

 9283c83bc8dc830bd48ad66f71ae42e8 *tests/data/fate/vsynth_lena-jpeg2000.avi
 1151148 tests/data/fate/vsynth_lena-jpeg2000.avi
-e7d79c9e11d0fe97f03e38be66c34e4f *tests/data/fate/vsynth_lena-jpeg2000.out.rawvideo
-stddev:    4.41 PSNR: 35.23 MAXDIFF:   63 bytes:  7603200/  7603200
+71a9ee7ad7c3dad60aa6641712c1f9e4 *tests/data/fate/vsynth_lena-jpeg2000.out.rawvideo
+stddev:    4.44 PSNR: 35.17 MAXDIFF:   61 bytes:  7603200/  7603200