4xm: validate the buffer size before parsing it

Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org

libavcodec/4xm.c

@@ -418,6 +418,8 @@ static int decode_p_frame(FourXContext *f, AVFrame *frame,
     src = (uint16_t *)f->last_picture->data[0];
 
     if (f->version > 1) {
+        if (length < 20)
+            return AVERROR_INVALIDDATA;
         extra           = 20;
         bitstream_size  = AV_RL32(buf + 8);
         wordstream_size = AV_RL32(buf + 12);
@@ -788,18 +790,29 @@ static int decode_frame(AVCodecContext *avctx, void *data,
     AVFrame *picture      = data;
     int i, frame_4cc, frame_size, ret;
 
-    frame_4cc = AV_RL32(buf);
-    if (buf_size != AV_RL32(buf + 4) + 8 || buf_size < 20)
+    if (buf_size < 20)
+        return AVERROR_INVALIDDATA;
+
+    if (buf_size < AV_RL32(buf + 4) + 8) {
         av_log(f->avctx, AV_LOG_ERROR, "size mismatch %d %d\n",
                buf_size, AV_RL32(buf + 4));
+        return AVERROR_INVALIDDATA;
+    }
+
+    frame_4cc = AV_RL32(buf);
 
     if (frame_4cc == AV_RL32("cfrm")) {
         int free_index       = -1;
+        int id, whole_size;
         const int data_size  = buf_size - 20;
-        const int id         = AV_RL32(buf + 12);
-        const int whole_size = AV_RL32(buf + 16);
         CFrameBuffer *cfrm;
 
+        if (data_size < 0)
+            return AVERROR_INVALIDDATA;
+
+        id         = AV_RL32(buf + 12);
+        whole_size = AV_RL32(buf + 16);
+
         for (i = 0; i < CFRAME_BUFFER_COUNT; i++)
             if (f->cfrm[i].id && f->cfrm[i].id < avctx->frame_number)
                 av_log(f->avctx, AV_LOG_ERROR, "lost c frame %d\n",