avcodec/pngdec: fully support the tRNS chunk

Signed-off-by: Donny Yang <work@kota.moe>

libavcodec/pngdec.c

@@ -21,6 +21,7 @@
 
 //#define DEBUG
 
+#include "libavutil/avassert.h"
 #include "libavutil/bprint.h"
 #include "libavutil/imgutils.h"
 #include "avcodec.h"
@@ -59,6 +60,7 @@ typedef struct PNGDecContext {
     int bits_per_pixel;
     int bpp;
     int has_trns;
+    uint8_t transparent_color_be[6];
 
     uint8_t *image_buf;
     int image_linesize;
@@ -590,6 +592,7 @@ static int decode_idat_chunk(AVCodecContext *avctx, PNGDecContext *s,
                              uint32_t length, AVFrame *p)
 {
     int ret;
+    size_t byte_depth = s->bit_depth > 8 ? 2 : 1;
 
     if (!(s->state & PNG_IHDR)) {
         av_log(avctx, AV_LOG_ERROR, "IDAT without IHDR\n");
@@ -641,6 +644,31 @@ static int decode_idat_chunk(AVCodecContext *avctx, PNGDecContext *s,
             return AVERROR_INVALIDDATA;
         }
 
+        if (s->has_trns && s->color_type != PNG_COLOR_TYPE_PALETTE) {
+            switch (avctx->pix_fmt) {
+            case AV_PIX_FMT_RGB24:
+                avctx->pix_fmt = AV_PIX_FMT_RGBA;
+                break;
+
+            case AV_PIX_FMT_RGB48BE:
+                avctx->pix_fmt = AV_PIX_FMT_RGBA64BE;
+                break;
+
+            case AV_PIX_FMT_GRAY8:
+                avctx->pix_fmt = AV_PIX_FMT_YA8;
+                break;
+
+            case AV_PIX_FMT_GRAY16BE:
+                avctx->pix_fmt = AV_PIX_FMT_YA16BE;
+                break;
+
+            default:
+                av_assert0(0);
+            }
+
+            s->bpp += byte_depth;
+        }
+
         if ((ret = ff_thread_get_buffer(avctx, &s->picture, AV_GET_BUFFER_FLAG_REF)) < 0)
             return ret;
         if (avctx->codec_id == AV_CODEC_ID_APNG && s->last_dispose_op != APNG_DISPOSE_OP_PREVIOUS) {
@@ -691,9 +719,21 @@ static int decode_idat_chunk(AVCodecContext *avctx, PNGDecContext *s,
         s->zstream.avail_out = s->crow_size;
         s->zstream.next_out  = s->crow_buf;
     }
+
     s->state |= PNG_IDAT;
-    if ((ret = png_decode_idat(s, length)) < 0)
+
+    /* set image to non-transparent bpp while decompressing */
+    if (s->has_trns && s->color_type != PNG_COLOR_TYPE_PALETTE)
+        s->bpp -= byte_depth;
+
+    ret = png_decode_idat(s, length);
+
+    if (s->has_trns && s->color_type != PNG_COLOR_TYPE_PALETTE)
+        s->bpp += byte_depth;
+
+    if (ret < 0)
         return ret;
+
     bytestream2_skip(&s->gb, 4); /* crc */
 
     return 0;
@@ -727,17 +767,33 @@ static int decode_trns_chunk(AVCodecContext *avctx, PNGDecContext *s,
 {
     int v, i;
 
-    /* read the transparency. XXX: Only palette mode supported */
-    if (s->color_type != PNG_COLOR_TYPE_PALETTE ||
-            length > 256 ||
-            !(s->state & PNG_PLTE))
+    if (s->color_type == PNG_COLOR_TYPE_PALETTE) {
+        if (length > 256 || !(s->state & PNG_PLTE))
             return AVERROR_INVALIDDATA;
+
         for (i = 0; i < length; i++) {
             v = bytestream2_get_byte(&s->gb);
             s->palette[i] = (s->palette[i] & 0x00ffffff) | (v << 24);
         }
-    bytestream2_skip(&s->gb, 4);     /* crc */
+    } else if (s->color_type == PNG_COLOR_TYPE_GRAY || s->color_type == PNG_COLOR_TYPE_RGB) {
+        if ((s->color_type == PNG_COLOR_TYPE_GRAY && length != 2) ||
+            (s->color_type == PNG_COLOR_TYPE_RGB && length != 6))
+            return AVERROR_INVALIDDATA;
+
+        for (i = 0; i < length / 2; i++) {
+            /* only use the least significant bits */
+            v = bytestream2_get_be16(&s->gb) & ((1 << s->bit_depth) - 1);
+
+            if (s->bit_depth > 8)
+                AV_WB16(&s->transparent_color_be[2 * i], v);
+            else
+                s->transparent_color_be[i] = v;
+        }
+    } else {
+        return AVERROR_INVALIDDATA;
+    }
 
+    bytestream2_skip(&s->gb, 4); /* crc */
     s->has_trns = 1;
 
     return 0;
@@ -1122,6 +1178,29 @@ exit_loop:
     if (s->bits_per_pixel <= 4)
         handle_small_bpp(s, p);
 
+    /* apply transparency if needed */
+    if (s->has_trns && s->color_type != PNG_COLOR_TYPE_PALETTE) {
+        size_t byte_depth = s->bit_depth > 8 ? 2 : 1;
+        size_t raw_bpp = s->bpp - byte_depth;
+        unsigned x, y;
+
+        for (y = 0; y < s->height; ++y) {
+            uint8_t *row = &s->image_buf[s->image_linesize * y];
+
+            /* since we're updating in-place, we have to go from right to left */
+            for (x = s->width; x > 0; --x) {
+                uint8_t *pixel = &row[s->bpp * (x - 1)];
+                memmove(pixel, &row[raw_bpp * (x - 1)], raw_bpp);
+
+                if (!memcmp(pixel, s->transparent_color_be, raw_bpp)) {
+                    memset(&pixel[raw_bpp], 0, byte_depth);
+                } else {
+                    memset(&pixel[raw_bpp], 0xff, byte_depth);
+                }
+            }
+        }
+    }
+
     /* handle p-frames only if a predecessor frame is available */
     if (s->last_picture.f->data[0]) {
         if (   !(avpkt->flags & AV_PKT_FLAG_KEY) && avctx->codec_tag != AV_RL32("MPNG")
@@ -1285,6 +1364,7 @@ static int update_thread_context(AVCodecContext *dst, const AVCodecContext *src)
         pdst->x_offset = psrc->x_offset;
         pdst->y_offset = psrc->y_offset;
         pdst->has_trns = psrc->has_trns;
+        memcpy(pdst->transparent_color_be, psrc->transparent_color_be, sizeof(pdst->transparent_color_be));
 
         pdst->dispose_op = psrc->dispose_op;