wmalossless: error out on invalid values for order.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org

libavcodec/wmalosslessdec.c

@@ -34,6 +34,7 @@
 #define MAX_SUBFRAMES          32                       ///< max number of subframes per channel
 #define MAX_BANDS              29                       ///< max number of scale factor bands
 #define MAX_FRAMESIZE       32768                       ///< maximum compressed frame size
+#define MAX_ORDER             256
 
 #define WMALL_BLOCK_MIN_BITS    6                       ///< log2 of min block size
 #define WMALL_BLOCK_MAX_BITS   12                       ///< log2 of max block size
@@ -139,9 +140,9 @@ typedef struct WmallDecodeCtx {
         int scaling;
         int coefsend;
         int bitsend;
-        int16_t coefs[256];
-        int16_t lms_prevvalues[512];
-        int16_t lms_updates[512];
+        int16_t coefs[MAX_ORDER];
+        int16_t lms_prevvalues[MAX_ORDER * 2];
+        int16_t lms_updates[MAX_ORDER * 2];
         int recent;
     } cdlms[2][9];
 
@@ -423,15 +424,23 @@ static void decode_mclms(WmallDecodeCtx *s)
     }
 }
 
-static void decode_cdlms(WmallDecodeCtx *s)
+static int decode_cdlms(WmallDecodeCtx *s)
 {
     int c, i;
     int cdlms_send_coef = get_bits1(&s->gb);
 
     for (c = 0; c < s->num_channels; c++) {
         s->cdlms_ttl[c] = get_bits(&s->gb, 3) + 1;
-        for (i = 0; i < s->cdlms_ttl[c]; i++)
+        for (i = 0; i < s->cdlms_ttl[c]; i++) {
             s->cdlms[c][i].order = (get_bits(&s->gb, 7) + 1) * 8;
+            if (s->cdlms[c][i].order > MAX_ORDER) {
+                av_log(s->avctx, AV_LOG_ERROR,
+                       "Order[%d][%d] %d > max (%d), not supported\n",
+                       c, i, s->cdlms[c][i].order, MAX_ORDER);
+                s->cdlms[0][0].order = 0;
+                return AVERROR_INVALIDDATA;
+            }
+        }
 
         for (i = 0; i < s->cdlms_ttl[c]; i++)
             s->cdlms[c][i].scaling = get_bits(&s->gb, 4);
@@ -457,6 +466,8 @@ static void decode_cdlms(WmallDecodeCtx *s)
             }
         }
     }
+
+    return 0;
 }
 
 static int decode_channel_residues(WmallDecodeCtx *s, int ch, int tile_size)
@@ -820,7 +831,7 @@ static int decode_subframe(WmallDecodeCtx *s)
     int offset        = s->samples_per_frame;
     int subframe_len  = s->samples_per_frame;
     int total_samples = s->samples_per_frame * s->num_channels;
-    int i, j, rawpcm_tile, padding_zeroes;
+    int i, j, rawpcm_tile, padding_zeroes, res;
 
     s->subframe_offset = get_bits_count(&s->gb);
 
@@ -865,8 +876,8 @@ static int decode_subframe(WmallDecodeCtx *s)
 
         s->do_arith_coding    = get_bits1(&s->gb);
         if (s->do_arith_coding) {
-            av_dlog(s->avctx, "do_arith_coding == 1");
-            abort();
+            av_log_missing_feature(s->avctx, "arithmetic coding", 1);
+            return AVERROR_PATCHWELCOME;
         }
         s->do_ac_filter       = get_bits1(&s->gb);
         s->do_inter_ch_decorr = get_bits1(&s->gb);
@@ -878,11 +889,16 @@ static int decode_subframe(WmallDecodeCtx *s)
         if (s->do_mclms)
             decode_mclms(s);
 
-        decode_cdlms(s);
+        if ((res = decode_cdlms(s)) < 0)
+            return res;
         s->movave_scaling = get_bits(&s->gb, 3);
         s->quant_stepsize = get_bits(&s->gb, 8) + 1;
 
         reset_codec(s);
+    } else if (!s->cdlms[0][0].order) {
+        av_log(s->avctx, AV_LOG_DEBUG,
+               "Waiting for seekable tile\n");
+        return -1;
     }
 
     rawpcm_tile = get_bits1(&s->gb);