swscale/alphablend: support packed pixel formats

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libswscale/alphablend.c

@@ -68,6 +68,48 @@ int ff_sws_alphablendaway(SwsContext *c, const uint8_t *src[],
                 }
             }
         }
+    } else {
+        int alpha_pos = desc->comp[plane_count].offset_plus1 - 1;
+        int w = c->srcW;
+        for (y = srcSliceY; y < srcSliceH; y++) {
+            if (sixteen_bits) {
+                const uint16_t *s = src[0] + srcStride[0] * y + 2*!alpha_pos;
+                const uint16_t *a = src[0] + srcStride[0] * y + alpha_pos;
+                        uint16_t *d = dst[0] + dstStride[0] * y;
+                if ((!isBE(c->srcFormat)) == !HAVE_BIGENDIAN) {
+                    for (x = 0; x < w; x++) {
+                        for (plane = 0; plane < plane_count; plane++) {
+                            unsigned target = plane && !(desc->flags & AV_PIX_FMT_FLAG_RGB) ? 1<<desc->comp[0].depth_minus1 : 0;
+                            int x_index = (plane_count + 1) * x;
+                            unsigned u = s[x_index + plane]*a[x_index] + target*(max-a[x_index]) + off;
+                            d[plane_count*x + plane] = av_clip((u + (u >> shift)) >> shift, 0, max);
+                        }
+                    }
+                } else {
+                    for (x = 0; x < w; x++) {
+                        for (plane = 0; plane < plane_count; plane++) {
+                            unsigned target = plane && !(desc->flags & AV_PIX_FMT_FLAG_RGB) ? 1<<desc->comp[0].depth_minus1 : 0;
+                            int x_index = (plane_count + 1) * x;
+                            unsigned aswap =av_bswap16(a[x_index]);
+                            unsigned u = av_bswap16(s[x_index + plane])*aswap + target*(max-aswap) + off;
+                            d[plane_count*x + plane] = av_clip((u + (u >> shift)) >> shift, 0, max);
+                        }
+                    }
+                }
+            } else {
+                const uint8_t *s = src[0] + srcStride[0] * y + !alpha_pos;
+                const uint8_t *a = src[0] + srcStride[0] * y + alpha_pos;
+                      uint8_t *d = dst[0] + dstStride[0] * y;
+                for (x = 0; x < w; x++) {
+                    for (plane = 0; plane < plane_count; plane++) {
+                        unsigned target = plane && !(desc->flags & AV_PIX_FMT_FLAG_RGB) ? 128 : 0;
+                        int x_index = (plane_count + 1) * x;
+                        unsigned u = s[x_index + plane]*a[x_index] + target*(255-a[x_index]) + 128;
+                        d[plane_count*x + plane] = (257*u) >> 16;
+                    }
+                }
+            }
+        }
     }
 
     return 0;

libswscale/utils.c

@@ -982,11 +982,11 @@ static uint16_t * alloc_gamma_tbl(double e)
 static enum AVPixelFormat alphaless_fmt(enum AVPixelFormat fmt)
 {
     switch(fmt) {
-//     case AV_PIX_FMT_ARGB:       return AV_PIX_FMT_RGB24;
-//     case AV_PIX_FMT_RGBA:       return AV_PIX_FMT_RGB24;
-//     case AV_PIX_FMT_ABGR:       return AV_PIX_FMT_BGR24;
-//     case AV_PIX_FMT_BGRA:       return AV_PIX_FMT_BGR24;
-//     case AV_PIX_FMT_YA8:        return AV_PIX_FMT_GRAY8;
+    case AV_PIX_FMT_ARGB:       return AV_PIX_FMT_RGB24;
+    case AV_PIX_FMT_RGBA:       return AV_PIX_FMT_RGB24;
+    case AV_PIX_FMT_ABGR:       return AV_PIX_FMT_BGR24;
+    case AV_PIX_FMT_BGRA:       return AV_PIX_FMT_BGR24;
+    case AV_PIX_FMT_YA8:        return AV_PIX_FMT_GRAY8;
 //
 //     case AV_PIX_FMT_YUVA420P:   return AV_PIX_FMT_YUV420P;
 //     case AV_PIX_FMT_YUVA422P:   return AV_PIX_FMT_YUV422P;
@@ -997,13 +997,13 @@ static enum AVPixelFormat alphaless_fmt(enum AVPixelFormat fmt)
     case AV_PIX_FMT_GBRAP16LE:          return AV_PIX_FMT_GBRP16;
     case AV_PIX_FMT_GBRAP16BE:          return AV_PIX_FMT_GBRP16;
 
-//     case AV_PIX_FMT_RGBA64LE:   return AV_PIX_FMT_RGB48;
-//     case AV_PIX_FMT_RGBA64BE:   return AV_PIX_FMT_RGB48;
-//     case AV_PIX_FMT_BGRA64LE:   return AV_PIX_FMT_BGR48;
-//     case AV_PIX_FMT_BGRA64BE:   return AV_PIX_FMT_BGR48;
+    case AV_PIX_FMT_RGBA64LE:   return AV_PIX_FMT_RGB48;
+    case AV_PIX_FMT_RGBA64BE:   return AV_PIX_FMT_RGB48;
+    case AV_PIX_FMT_BGRA64LE:   return AV_PIX_FMT_BGR48;
+    case AV_PIX_FMT_BGRA64BE:   return AV_PIX_FMT_BGR48;
 
-//     case AV_PIX_FMT_YA16BE:             return AV_PIX_FMT_GRAY16;
-//     case AV_PIX_FMT_YA16LE:             return AV_PIX_FMT_GRAY16;
+    case AV_PIX_FMT_YA16BE:             return AV_PIX_FMT_GRAY16;
+    case AV_PIX_FMT_YA16LE:             return AV_PIX_FMT_GRAY16;
 
 //     case AV_PIX_FMT_YUVA420P9BE:        return AV_PIX_FMT_YUV420P9;
 //     case AV_PIX_FMT_YUVA422P9BE:        return AV_PIX_FMT_YUV422P9;