avdevice/xcbgrab: check if frame size fits in INT_MAX

Also fixes a possible overflow and sets stream bitrate.
Signed-off-by: Marton Balint <cus@passwd.hu>

libavdevice/xcbgrab.c

@@ -542,6 +542,7 @@ static int create_stream(AVFormatContext *s)
     AVStream *st      = avformat_new_stream(s, NULL);
     xcb_get_geometry_cookie_t gc;
     xcb_get_geometry_reply_t *geo;
+    int64_t frame_size_bits;
     int ret;
 
     if (!st)
@@ -580,17 +581,18 @@ static int create_stream(AVFormatContext *s)
     c->frame_duration = av_rescale_q(1, c->time_base, AV_TIME_BASE_Q);
     c->time_frame = av_gettime();
 
-    st->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
-    st->codecpar->codec_id   = AV_CODEC_ID_RAWVIDEO;
-    st->codecpar->width      = c->width;
-    st->codecpar->height     = c->height;
-
     ret = pixfmt_from_pixmap_format(s, geo->depth, &st->codecpar->format, &c->bpp);
     free(geo);
     if (ret < 0)
         return ret;
 
-    c->frame_size = c->width * c->height * c->bpp / 8;
+    frame_size_bits = (int64_t)c->width * c->height * c->bpp;
+    if (frame_size_bits / 8 + AV_INPUT_BUFFER_PADDING_SIZE > INT_MAX) {
+        av_log(s, AV_LOG_ERROR, "Captured area is too large\n");
+        return AVERROR_PATCHWELCOME;
+    }
+    c->frame_size = frame_size_bits / 8;
+
 #if CONFIG_LIBXCB_SHM
     c->shm_pool = av_buffer_pool_init2(c->frame_size + AV_INPUT_BUFFER_PADDING_SIZE,
                                            c->conn, allocate_shm_buffer, NULL);
@@ -598,6 +600,12 @@ static int create_stream(AVFormatContext *s)
         return AVERROR(ENOMEM);
 #endif
 
+    st->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
+    st->codecpar->codec_id   = AV_CODEC_ID_RAWVIDEO;
+    st->codecpar->width      = c->width;
+    st->codecpar->height     = c->height;
+    st->codecpar->bit_rate   = av_rescale(frame_size_bits, st->avg_frame_rate.num, st->avg_frame_rate.den);
+
     return ret;
 }