xa_adpcm: limit filter to prevent xa_adpcm_table[] array bounds overruns.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org

libavcodec/adpcm.c

@@ -265,7 +265,8 @@ static inline short adpcm_yamaha_expand_nibble(ADPCMChannelStatus *c, unsigned c
     return c->predictor;
 }
 
-static void xa_decode(short *out, const unsigned char *in,
+static int xa_decode(AVCodecContext *avctx,
+                     short *out, const unsigned char *in,
                      ADPCMChannelStatus *left, ADPCMChannelStatus *right, int inc)
 {
     int i, j;
@@ -277,6 +278,12 @@ static void xa_decode(short *out, const unsigned char *in,
 
         shift  = 12 - (in[4+i*2] & 15);
         filter = in[4+i*2] >> 4;
+        if (filter > 4) {
+            av_log(avctx, AV_LOG_ERROR,
+                   "Invalid XA-ADPCM filter %d (max. allowed is 4)\n",
+                   filter);
+            return AVERROR_INVALIDDATA;
+        }
         f0 = xa_adpcm_table[filter][0];
         f1 = xa_adpcm_table[filter][1];
 
@@ -304,7 +311,12 @@ static void xa_decode(short *out, const unsigned char *in,
 
         shift  = 12 - (in[5+i*2] & 15);
         filter = in[5+i*2] >> 4;
-
+        if (filter > 4) {
+            av_log(avctx, AV_LOG_ERROR,
+                   "Invalid XA-ADPCM filter %d (max. allowed is 4)\n",
+                   filter);
+            return AVERROR_INVALIDDATA;
+        }
         f0 = xa_adpcm_table[filter][0];
         f1 = xa_adpcm_table[filter][1];
 
@@ -328,6 +340,8 @@ static void xa_decode(short *out, const unsigned char *in,
             left->sample2 = s_2;
         }
     }
+
+    return 0;
 }
 
 /**
@@ -812,8 +826,9 @@ static int adpcm_decode_frame(AVCodecContext *avctx, void *data,
         break;
     case CODEC_ID_ADPCM_XA:
         while (buf_size >= 128) {
-            xa_decode(samples, src, &c->status[0], &c->status[1],
-                avctx->channels);
+            if ((ret = xa_decode(avctx, samples, src, &c->status[0],
+                                 &c->status[1], avctx->channels)) < 0)
+                return ret;
             src += 128;
             samples += 28 * 8;
             buf_size -= 128;