avcodec/libvpxenc: only allocate alpha U/V planes on size changes

Signed-off-by: Marton Balint <cus@passwd.hu>
Signed-off-by: James Zern <jzern@google.com>

libavcodec/libvpxenc.c

@@ -347,8 +347,11 @@ static av_cold int vpx_free(AVCodecContext *avctx)
 #endif
 
     vpx_codec_destroy(&ctx->encoder);
-    if (ctx->is_alpha)
+    if (ctx->is_alpha) {
         vpx_codec_destroy(&ctx->encoder_alpha);
+        av_freep(&ctx->rawimg_alpha.planes[VPX_PLANE_U]);
+        av_freep(&ctx->rawimg_alpha.planes[VPX_PLANE_V]);
+    }
     av_freep(&ctx->twopass_stats.buf);
     av_freep(&avctx->stats_out);
     free_frame_list(ctx->coded_frame_list);
@@ -872,10 +875,6 @@ FF_ENABLE_DEPRECATION_WARNINGS
         ctx->rawimg.bit_depth = enccfg.g_bit_depth;
 #endif
 
-    if (ctx->is_alpha)
-        vpx_img_wrap(&ctx->rawimg_alpha, VPX_IMG_FMT_I420, avctx->width, avctx->height, 1,
-                     (unsigned char*)1);
-
     cpb_props = ff_add_cpb_side_data(avctx);
     if (!cpb_props)
         return AVERROR(ENOMEM);
@@ -1292,6 +1291,34 @@ static int vp8_encode_set_roi(AVCodecContext *avctx, int frame_width, int frame_
     return ret;
 }
 
+static int realloc_alpha_uv(AVCodecContext *avctx, int width, int height)
+{
+    VPxContext *ctx = avctx->priv_data;
+    struct vpx_image *rawimg_alpha = &ctx->rawimg_alpha;
+    unsigned char **planes = rawimg_alpha->planes;
+    int *stride = rawimg_alpha->stride;
+
+    if (!planes[VPX_PLANE_U] ||
+        !planes[VPX_PLANE_V] ||
+        width  != (int)rawimg_alpha->d_w ||
+        height != (int)rawimg_alpha->d_h) {
+        av_freep(&planes[VPX_PLANE_U]);
+        av_freep(&planes[VPX_PLANE_V]);
+
+        vpx_img_wrap(rawimg_alpha, VPX_IMG_FMT_I420, width, height, 1,
+                     (unsigned char*)1);
+        planes[VPX_PLANE_U] = av_malloc_array(stride[VPX_PLANE_U], height);
+        planes[VPX_PLANE_V] = av_malloc_array(stride[VPX_PLANE_V], height);
+        if (!planes[VPX_PLANE_U] || !planes[VPX_PLANE_V])
+            return AVERROR(ENOMEM);
+
+        memset(planes[VPX_PLANE_U], 0x80, stride[VPX_PLANE_U] * height);
+        memset(planes[VPX_PLANE_V], 0x80, stride[VPX_PLANE_V] * height);
+    }
+
+    return 0;
+}
+
 static int vpx_encode(AVCodecContext *avctx, AVPacket *pkt,
                       const AVFrame *frame, int *got_packet)
 {
@@ -1312,23 +1339,12 @@ static int vpx_encode(AVCodecContext *avctx, AVPacket *pkt,
         rawimg->stride[VPX_PLANE_U] = frame->linesize[1];
         rawimg->stride[VPX_PLANE_V] = frame->linesize[2];
         if (ctx->is_alpha) {
-            uint8_t *u_plane, *v_plane;
             rawimg_alpha = &ctx->rawimg_alpha;
+            res = realloc_alpha_uv(avctx, frame->width, frame->height);
+            if (res < 0)
+                return res;
             rawimg_alpha->planes[VPX_PLANE_Y] = frame->data[3];
-            u_plane = av_malloc(frame->linesize[1] * frame->height);
-            v_plane = av_malloc(frame->linesize[2] * frame->height);
-            if (!u_plane || !v_plane) {
-                av_free(u_plane);
-                av_free(v_plane);
-                return AVERROR(ENOMEM);
-            }
-            memset(u_plane, 0x80, frame->linesize[1] * frame->height);
-            rawimg_alpha->planes[VPX_PLANE_U] = u_plane;
-            memset(v_plane, 0x80, frame->linesize[2] * frame->height);
-            rawimg_alpha->planes[VPX_PLANE_V] = v_plane;
             rawimg_alpha->stride[VPX_PLANE_Y] = frame->linesize[3];
-            rawimg_alpha->stride[VPX_PLANE_U] = frame->linesize[1];
-            rawimg_alpha->stride[VPX_PLANE_V] = frame->linesize[2];
         }
         timestamp                   = frame->pts;
 #if VPX_IMAGE_ABI_VERSION >= 4
@@ -1390,11 +1406,6 @@ static int vpx_encode(AVCodecContext *avctx, AVPacket *pkt,
                          ctx->twopass_stats.sz);
     }
 
-    if (rawimg_alpha) {
-        av_freep(&rawimg_alpha->planes[VPX_PLANE_U]);
-        av_freep(&rawimg_alpha->planes[VPX_PLANE_V]);
-    }
-
     *got_packet = !!coded_size;
     return 0;
 }