sws: add planar RGB formats to isAnyRGB

We have to make some symetric changes elsewhere as this increases
the precission with which samples are stored.
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libswscale/input.c

@@ -729,7 +729,7 @@ static av_always_inline void planar_rgb16_to_y(uint8_t *_dst, const uint8_t *_sr
         int b = rdpx(src[1] + i);
         int r = rdpx(src[2] + i);
 
-        dst[i] = ((RY * r + GY * g + BY * b + (33 << (RGB2YUV_SHIFT + bpc - 9))) >> RGB2YUV_SHIFT);
+        dst[i] = ((RY * r + GY * g + BY * b + (33 << (RGB2YUV_SHIFT + bpc - 9))) >> (RGB2YUV_SHIFT + bpc - 14));
     }
 }
 
@@ -796,8 +796,8 @@ static av_always_inline void planar_rgb16_to_uv(uint8_t *_dstU, uint8_t *_dstV,
         int b = rdpx(src[1] + i);
         int r = rdpx(src[2] + i);
 
-        dstU[i] = (RU * r + GU * g + BU * b + (257 << (RGB2YUV_SHIFT + bpc - 9))) >> RGB2YUV_SHIFT;
-        dstV[i] = (RV * r + GV * g + BV * b + (257 << (RGB2YUV_SHIFT + bpc - 9))) >> RGB2YUV_SHIFT;
+        dstU[i] = (RU * r + GU * g + BU * b + (257 << (RGB2YUV_SHIFT + bpc - 9))) >> (RGB2YUV_SHIFT + bpc - 14);
+        dstV[i] = (RV * r + GV * g + BV * b + (257 << (RGB2YUV_SHIFT + bpc - 9))) >> (RGB2YUV_SHIFT + bpc - 14);
     }
 }
 #undef rdpx

libswscale/swscale_internal.h

@@ -716,6 +716,14 @@ static av_always_inline int isRGB(enum AVPixelFormat pix_fmt)
     (           \
           isRGBinInt(x)       ||    \
           isBGRinInt(x)       ||    \
+          (x)==AV_PIX_FMT_GBRP9LE  || \
+          (x)==AV_PIX_FMT_GBRP9BE  || \
+          (x)==AV_PIX_FMT_GBRP10LE || \
+          (x)==AV_PIX_FMT_GBRP10BE || \
+          (x)==AV_PIX_FMT_GBRP12LE || \
+          (x)==AV_PIX_FMT_GBRP12BE || \
+          (x)==AV_PIX_FMT_GBRP14LE || \
+          (x)==AV_PIX_FMT_GBRP14BE || \
           (x)==AV_PIX_FMT_GBR24P     \
     )
 

tests/ref/lavf/dpx

@@ -5,5 +5,5 @@
 ./tests/data/images/dpx/%02d.dpx CRC=0xe5b9c023
 609920 ./tests/data/images/dpx/02.dpx
 13dc41b1e1e36399a5e1f8b7e3344a81 *./tests/data/images/dpx/02.dpx
-./tests/data/images/dpx/%02d.dpx CRC=0xf0a1c097
+./tests/data/images/dpx/%02d.dpx CRC=0xb6310a70
 407168 ./tests/data/images/dpx/02.dpx

tests/ref/lavfi/pixfmts_scale

@@ -16,14 +16,14 @@ bgr565le            f524e9f16bdd68b247dbcb621e543fc0
 bgr8                68a3a395043dc57335ad1f8e891229c5
 bgra                3eaf5489b8aa13a3388aad3751b597bf
 gbrp                7b83ae32c1f76bd634e50f4797a74e92
-gbrp10be            2bc8c21383a6adb44f15d06615154b2c
-gbrp10le            e5f1fddd18dd44d6e6e1275674b5d7b6
-gbrp12be            56d0b664e9905a20c1d05c15a0b34c2c
-gbrp12le            26650d0e35bdb55b6c699576608df0b4
+gbrp10be            77dae432c42fad019c286753b0e4b0a8
+gbrp10le            7d3e7341423b1206e8a6a3a02b48b5d7
+gbrp12be            2c237c413e765024eb3f73d8c5e03df8
+gbrp12le            9a0c9eda1f16d640663c729d4bb88024
 gbrp14be            2b4f1928a5ef53d0a216b3b10fa11ca0
 gbrp14le            47100ec39fad45579de1dff56fe17117
-gbrp9be             b6240144e63e16fab97b5c38bd8673f1
-gbrp9le             dc0477bf95cf457255b98855a170f6c8
+gbrp9be             c345137de0aa4b0536a00009e8e11267
+gbrp9le             b798662b2ba4cdf21ad6a6d4c1fe1b72
 gray                045c35e6cc7d41460f2b96b3e254deab
 gray16be            70064f9acdc5e3935ccda67e765bf2fb
 gray16le            578241fb43029e5ae841a3c94d940dce