avcodec/libutvideoenc: Check avpicture_get_size() return code

Fixes CID1257656
Reviewed-by: Derek Buitenhuis <derek.buitenhuis@gmail.com>
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/libutvideoenc.cpp

@@ -39,6 +39,7 @@ static av_cold int utvideo_encode_init(AVCodecContext *avctx)
     UtVideoContext *utv = (UtVideoContext *)avctx->priv_data;
     UtVideoExtra *info;
     uint32_t flags, in_format;
+    int ret;
 
     switch (avctx->pix_fmt) {
     case AV_PIX_FMT_YUV420P:
@@ -94,8 +95,11 @@ static av_cold int utvideo_encode_init(AVCodecContext *avctx)
      * We use this buffer to hold the data that Ut Video returns,
      * since we cannot decode planes separately with it.
      */
-    utv->buf_size = avpicture_get_size(avctx->pix_fmt,
-                                       avctx->width, avctx->height);
+    ret = avpicture_get_size(avctx->pix_fmt, avctx->width, avctx->height);
+    if (ret < 0)
+        return ret;
+    utv->buf_size = ret;
+
     utv->buffer = (uint8_t *)av_malloc(utv->buf_size);
 
     if (utv->buffer == NULL) {