Add memory allocation failure checks to ff_iir_filter_init_coeffs().

Signed-off-by: Mans Rullgard <mans@mansr.com>
(cherry picked from commit d42dc217)

libavcodec/iirfilter.c

@@ -47,7 +47,8 @@ typedef struct FFIIRFilterState{
 /// maximum supported filter order
 #define MAXORDER 30
 
-av_cold struct FFIIRFilterCoeffs* ff_iir_filter_init_coeffs(enum IIRFilterType filt_type,
+av_cold struct FFIIRFilterCoeffs* ff_iir_filter_init_coeffs(void *avc,
+                                                enum IIRFilterType filt_type,
                                                     enum IIRFilterMode filt_mode,
                                                     int order, float cutoff_ratio,
                                                     float stopband, float ripple)
@@ -62,9 +63,12 @@ av_cold struct FFIIRFilterCoeffs* ff_iir_filter_init_coeffs(enum IIRFilterType f
     if(order <= 1 || (order & 1) || order > MAXORDER || cutoff_ratio >= 1.0)
         return NULL;
 
-    c = av_malloc(sizeof(FFIIRFilterCoeffs));
-    c->cx = av_malloc(sizeof(c->cx[0]) * ((order >> 1) + 1));
-    c->cy = av_malloc(sizeof(c->cy[0]) * order);
+    FF_ALLOCZ_OR_GOTO(avc, c,     sizeof(FFIIRFilterCoeffs),
+                      init_fail);
+    FF_ALLOC_OR_GOTO (avc, c->cx, sizeof(c->cx[0]) * ((order >> 1) + 1),
+                      init_fail);
+    FF_ALLOC_OR_GOTO (avc, c->cy, sizeof(c->cy[0]) * order,
+                      init_fail);
     c->order = order;
 
     wa = 2 * tan(M_PI * 0.5 * cutoff_ratio);
@@ -110,6 +114,10 @@ av_cold struct FFIIRFilterCoeffs* ff_iir_filter_init_coeffs(enum IIRFilterType f
     c->gain /= 1 << order;
 
     return c;
+
+init_fail:
+    ff_iir_filter_free_coeffs(c);
+    return NULL;
 }
 
 av_cold struct FFIIRFilterState* ff_iir_filter_init_state(int order)

libavcodec/iirfilter.h

@@ -49,6 +49,8 @@ enum IIRFilterMode{
 /**
  * Initialize filter coefficients.
  *
+ * @param avc          a pointer to an arbitrary struct of which the first
+ *                     field is a pointer to an AVClass struct
  * @param filt_type    filter type (e.g. Butterworth)
  * @param filt_mode    filter mode (e.g. lowpass)
  * @param order        filter order
@@ -58,7 +60,8 @@ enum IIRFilterMode{
  *
  * @return pointer to filter coefficients structure or NULL if filter cannot be created
  */
-struct FFIIRFilterCoeffs* ff_iir_filter_init_coeffs(enum IIRFilterType filt_type,
+struct FFIIRFilterCoeffs* ff_iir_filter_init_coeffs(void *avc,
+                                                enum IIRFilterType filt_type,
                                                     enum IIRFilterMode filt_mode,
                                                     int order, float cutoff_ratio,
                                                     float stopband, float ripple);

libavcodec/psymodel.c

@@ -88,7 +88,7 @@ av_cold struct FFPsyPreprocessContext* ff_psy_preprocess_init(AVCodecContext *av
         cutoff_coeff = 2.0 * avctx->cutoff / avctx->sample_rate;
 
     if (cutoff_coeff)
-    ctx->fcoeffs = ff_iir_filter_init_coeffs(FF_FILTER_TYPE_BUTTERWORTH, FF_FILTER_MODE_LOWPASS,
+    ctx->fcoeffs = ff_iir_filter_init_coeffs(avctx, FF_FILTER_TYPE_BUTTERWORTH, FF_FILTER_MODE_LOWPASS,
                                              FILT_ORDER, cutoff_coeff, 0.0, 0.0);
     if (ctx->fcoeffs) {
         ctx->fstate = av_mallocz(sizeof(ctx->fstate[0]) * avctx->channels);