Commit bcf64348 authored by Michael Niedermayer's avatar Michael Niedermayer

sws/fill_rgb2yuv_table: increase precission to 32bit

Signed-off-by: 's avatarMichael Niedermayer <michaelni@gmx.at>
parent 2fa11a94
...@@ -801,7 +801,7 @@ static void getSubSampleFactors(int *h, int *v, enum AVPixelFormat format) ...@@ -801,7 +801,7 @@ static void getSubSampleFactors(int *h, int *v, enum AVPixelFormat format)
static void fill_rgb2yuv_table(SwsContext *c, const int table[4], int dstRange) static void fill_rgb2yuv_table(SwsContext *c, const int table[4], int dstRange)
{ {
int64_t W,V,Z; int64_t W, V, Z, Cy, Cu, Cv;
int64_t vr = table[0]; int64_t vr = table[0];
int64_t ub = table[1]; int64_t ub = table[1];
int64_t ug = -table[2]; int64_t ug = -table[2];
...@@ -858,21 +858,25 @@ static void fill_rgb2yuv_table(SwsContext *c, const int table[4], int dstRange) ...@@ -858,21 +858,25 @@ static void fill_rgb2yuv_table(SwsContext *c, const int table[4], int dstRange)
ug = ug * 224 / 255; ug = ug * 224 / 255;
vg = vg * 224 / 255; vg = vg * 224 / 255;
} }
W = ONE*ug/ub; W = ONE*ONE*ug/ub;
V = ONE*vg/vr; V = ONE*ONE*vg/vr;
Z = ONE-W-V; Z = ONE*ONE-W-V;
c->input_rgb2yuv_table[RY_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*V/(Z*cy); Cy = cy*Z / ONE;
c->input_rgb2yuv_table[GY_IDX] = (1 << RGB2YUV_SHIFT)*ONE*ONE/(Z*cy); Cu = ub*Z / ONE;
c->input_rgb2yuv_table[BY_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*W/(Z*cy); Cv = vr*Z / ONE;
c->input_rgb2yuv_table[RU_IDX] = (1 << RGB2YUV_SHIFT)*ONE*V/(Z*ub); c->input_rgb2yuv_table[RY_IDX] = -(1 << RGB2YUV_SHIFT)*V/Cy;
c->input_rgb2yuv_table[GU_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*ONE/(Z*ub); c->input_rgb2yuv_table[GY_IDX] = (1 << RGB2YUV_SHIFT)*ONE*ONE/Cy;
c->input_rgb2yuv_table[BU_IDX] = (1 << RGB2YUV_SHIFT)*ONE*(Z+W)/(Z*ub); c->input_rgb2yuv_table[BY_IDX] = -(1 << RGB2YUV_SHIFT)*W/Cy;
c->input_rgb2yuv_table[RV_IDX] = (1 << RGB2YUV_SHIFT)*ONE*(V+Z)/(Z*vr); c->input_rgb2yuv_table[RU_IDX] = (1 << RGB2YUV_SHIFT)*V/Cu;
c->input_rgb2yuv_table[GV_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*ONE/(Z*vr); c->input_rgb2yuv_table[GU_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*ONE/Cu;
c->input_rgb2yuv_table[BV_IDX] = (1 << RGB2YUV_SHIFT)*ONE*W/(Z*vr); c->input_rgb2yuv_table[BU_IDX] = (1 << RGB2YUV_SHIFT)*(Z+W)/Cu;
c->input_rgb2yuv_table[RV_IDX] = (1 << RGB2YUV_SHIFT)*(V+Z)/Cv;
c->input_rgb2yuv_table[GV_IDX] = -(1 << RGB2YUV_SHIFT)*ONE*ONE/Cv;
c->input_rgb2yuv_table[BV_IDX] = (1 << RGB2YUV_SHIFT)*W/Cv;
if(/*!dstRange && */table == ff_yuv2rgb_coeffs[SWS_CS_DEFAULT]) { if(/*!dstRange && */table == ff_yuv2rgb_coeffs[SWS_CS_DEFAULT]) {
c->input_rgb2yuv_table[BY_IDX] = ((int)(0.114 * 219 / 255 * (1 << RGB2YUV_SHIFT) + 0.5)); c->input_rgb2yuv_table[BY_IDX] = ((int)(0.114 * 219 / 255 * (1 << RGB2YUV_SHIFT) + 0.5));
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment