wma: check memory allocations and propagate errors

libavcodec/wma.c

@@ -33,9 +33,9 @@
 
 /* XXX: use same run/length optimization as mpeg decoders */
 // FIXME maybe split decode / encode or pass flag
-static av_cold void init_coef_vlc(VLC *vlc, uint16_t **prun_table,
-                                  float **plevel_table, uint16_t **pint_table,
-                                  const CoefVLCTable *vlc_table)
+static av_cold int init_coef_vlc(VLC *vlc, uint16_t **prun_table,
+                                 float **plevel_table, uint16_t **pint_table,
+                                 const CoefVLCTable *vlc_table)
 {
     int n                        = vlc_table->n;
     const uint8_t  *table_bits   = vlc_table->huffbits;
@@ -51,6 +51,13 @@ static av_cold void init_coef_vlc(VLC *vlc, uint16_t **prun_table,
     level_table  = av_malloc(n * sizeof(uint16_t));
     flevel_table = av_malloc(n * sizeof(*flevel_table));
     int_table    = av_malloc(n * sizeof(uint16_t));
+    if (!run_table || !level_table || !flevel_table || !int_table) {
+        av_freep(&run_table);
+        av_freep(&level_table);
+        av_freep(&flevel_table);
+        av_freep(&int_table);
+        return AVERROR(ENOMEM);
+    }
     i            = 2;
     level        = 1;
     k            = 0;
@@ -69,12 +76,14 @@ static av_cold void init_coef_vlc(VLC *vlc, uint16_t **prun_table,
     *plevel_table = flevel_table;
     *pint_table   = int_table;
     av_free(level_table);
+
+    return 0;
 }
 
 av_cold int ff_wma_init(AVCodecContext *avctx, int flags2)
 {
     WMACodecContext *s = avctx->priv_data;
-    int i;
+    int i, ret;
     float bps1, high_freq;
     volatile float bps;
     int sample_rate1;
@@ -339,12 +348,13 @@ av_cold int ff_wma_init(AVCodecContext *avctx, int flags2)
     }
     s->coef_vlcs[0] = &coef_vlcs[coef_vlc_table * 2];
     s->coef_vlcs[1] = &coef_vlcs[coef_vlc_table * 2 + 1];
-    init_coef_vlc(&s->coef_vlc[0], &s->run_table[0], &s->level_table[0],
-                  &s->int_table[0], s->coef_vlcs[0]);
-    init_coef_vlc(&s->coef_vlc[1], &s->run_table[1], &s->level_table[1],
-                  &s->int_table[1], s->coef_vlcs[1]);
+    ret = init_coef_vlc(&s->coef_vlc[0], &s->run_table[0], &s->level_table[0],
+                        &s->int_table[0], s->coef_vlcs[0]);
+    if (ret < 0)
+        return ret;
 
-    return 0;
+    return init_coef_vlc(&s->coef_vlc[1], &s->run_table[1], &s->level_table[1],
+                         &s->int_table[1], s->coef_vlcs[1]);
 }
 
 int ff_wma_total_gain_to_bits(int total_gain)