avcodec/gif: support crop and transparency with PAL8

libavcodec/gif.c

@@ -46,6 +46,9 @@ typedef struct {
     AVFrame *last_frame;
     int flags;
     uint32_t palette[AVPALETTE_COUNT];  ///< local reference palette for !pal8
+    int palette_loaded;
+    int transparent_index;
+    uint8_t *pal_exdata;
     uint8_t *tmpl;                      ///< temporary line buffer
 } GIFContext;
 
@@ -78,11 +81,11 @@ static int gif_image_write_image(AVCodecContext *avctx,
 {
     GIFContext *s = avctx->priv_data;
     int len = 0, height = avctx->height, width = avctx->width, x, y;
-    int x_start = 0, y_start = 0, trans = -1;
+    int x_start = 0, y_start = 0, trans = s->transparent_index;
+    int honor_transparency = (s->flags & GF_TRANSDIFF) && s->last_frame;
     const uint8_t *ptr;
 
     /* Crop image */
-    // TODO support with palette change
     if ((s->flags & GF_OFFSETTING) && s->last_frame && !palette) {
         const uint8_t *ref = s->last_frame->data[0];
         const int ref_linesize = s->last_frame->linesize[0];
@@ -151,20 +154,23 @@ static int gif_image_write_image(AVCodecContext *avctx,
         }
     }
 
-    /* TODO: support with palette change (pal8) */
-    if ((s->flags & GF_TRANSDIFF) && s->last_frame && !palette) {
+    if (honor_transparency && trans < 0) {
         trans = pick_palette_entry(buf + y_start*linesize + x_start,
                                    linesize, width, height);
         if (trans < 0) { // TODO, patch welcome
             av_log(avctx, AV_LOG_DEBUG, "No available color, can not use transparency\n");
         } else {
-            uint8_t *pal_exdata = av_packet_new_side_data(pkt, AV_PKT_DATA_PALETTE, AVPALETTE_SIZE);
+            uint8_t *pal_exdata = s->pal_exdata;
+            if (!pal_exdata)
+                pal_exdata = av_packet_new_side_data(pkt, AV_PKT_DATA_PALETTE, AVPALETTE_SIZE);
             if (!pal_exdata)
                 return AVERROR(ENOMEM);
             memcpy(pal_exdata, s->palette, AVPALETTE_SIZE);
             pal_exdata[trans*4 + 3*!HAVE_BIGENDIAN] = 0x00;
         }
     }
+    if (trans < 0)
+        honor_transparency = 0;
 
     bytestream_put_byte(bytestream, 0x08);
 
@@ -172,7 +178,7 @@ static int gif_image_write_image(AVCodecContext *avctx,
                        12, FF_LZW_GIF, put_bits);
 
     ptr = buf + y_start*linesize + x_start;
-    if (trans >= 0) {
+    if (honor_transparency) {
         const int ref_linesize = s->last_frame->linesize[0];
         const uint8_t *ref = s->last_frame->data[0] + y_start*ref_linesize + x_start;
 
@@ -223,6 +229,8 @@ static av_cold int gif_encode_init(AVCodecContext *avctx)
     avctx->coded_frame->pict_type = AV_PICTURE_TYPE_I;
     avctx->coded_frame->key_frame = 1;
 
+    s->transparent_index = -1;
+
     s->lzw = av_mallocz(ff_lzw_encode_state_size);
     s->buf = av_malloc(avctx->width*avctx->height*2);
     s->tmpl = av_malloc(avctx->width);
@@ -235,6 +243,25 @@ static av_cold int gif_encode_init(AVCodecContext *avctx)
     return 0;
 }
 
+/* FIXME: duplicated with lavc */
+static int get_palette_transparency_index(const uint32_t *palette)
+{
+    int transparent_color_index = -1;
+    unsigned i, smallest_alpha = 0xff;
+
+    if (!palette)
+        return -1;
+
+    for (i = 0; i < AVPALETTE_COUNT; i++) {
+        const uint32_t v = palette[i];
+        if (v >> 24 < smallest_alpha) {
+            smallest_alpha = v >> 24;
+            transparent_color_index = i;
+        }
+    }
+    return smallest_alpha < 128 ? transparent_color_index : -1;
+}
+
 static int gif_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
                             const AVFrame *pict, int *got_packet)
 {
@@ -254,6 +281,20 @@ static int gif_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
             return AVERROR(ENOMEM);
         memcpy(pal_exdata, pict->data[1], AVPALETTE_SIZE);
         palette = (uint32_t*)pict->data[1];
+
+        s->pal_exdata = pal_exdata;
+
+        /* The first palette with PAL8 will be used as generic palette by the
+         * muxer so we don't need to write it locally in the packet. We store
+         * it as a reference here in case it changes later. */
+        if (!s->palette_loaded) {
+            memcpy(s->palette, palette, AVPALETTE_SIZE);
+            s->transparent_index = get_palette_transparency_index(palette);
+            s->palette_loaded = 1;
+            palette = NULL;
+        } else if (!memcmp(s->palette, palette, AVPALETTE_SIZE)) {
+            palette = NULL;
+        }
     }
 
     gif_image_write_image(avctx, &outbuf_ptr, end, palette,