4xm: more thorought check for negative index and negative shift

CC: libav-stable@libav.org
Bug-Id: CID 1087094

libavcodec/4xm.c

@@ -340,22 +340,29 @@ static inline void mcdc(uint16_t *dst, uint16_t *src, int log2w,
 static int decode_p_block(FourXContext *f, uint16_t *dst, uint16_t *src,
                           int log2w, int log2h, int stride)
 {
-    const int index = size2index[log2h][log2w];
-    const int h     = 1 << log2h;
-    int code        = get_vlc2(&f->gb,
-                               block_type_vlc[1 - (f->version > 1)][index].table,
-                               BLOCK_TYPE_VLC_BITS, 1);
-    uint16_t *start = f->last_frame_buffer;
-    uint16_t *end   = start + stride * (f->avctx->height - h + 1) - (1 << log2w);
-    int ret;
-    int scale   = 1;
+    int index, h, code, ret, scale = 1;
+    uint16_t *start, *end;
     unsigned dc = 0;
 
-    if (code < 0 || code > 6 || log2w < 0)
+    if (log2h < 0 || log2w < 0)
+        return AVERROR_INVALIDDATA;
+
+    index = size2index[log2h][log2w];
+    if (index < 0)
         return AVERROR_INVALIDDATA;
 
+    h     = 1 << log2h;
+    code  = get_vlc2(&f->gb, block_type_vlc[1 - (f->version > 1)][index].table,
+                     BLOCK_TYPE_VLC_BITS, 1);
+    if (code < 0 || code > 6)
+        return AVERROR_INVALIDDATA;
+
+    start = f->last_frame_buffer;
+    end   = start + stride * (f->avctx->height - h + 1) - (1 << log2w);
+
     if (code == 1) {
-        log2h--;
+        if (--log2h < 0)
+            return AVERROR_INVALIDDATA;
         if ((ret = decode_p_block(f, dst, src, log2w, log2h, stride)) < 0)
             return ret;
         return decode_p_block(f, dst + (stride << log2h),