Mark vectors as NAN instead of dereferencing NULL pointers on malloc failure

Found-by: Daemon404
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libswscale/utils.c

@@ -1654,6 +1654,22 @@ SwsContext *sws_getContext(int srcW, int srcH, enum AVPixelFormat srcFormat,
     return c;
 }
 
+static int isnan_vec(SwsVector *a)
+{
+    int i;
+    for (i=0; i<a->length; i++)
+        if (isnan(a->coeff[i]))
+            return 1;
+    return 0;
+}
+
+static void makenan_vec(SwsVector *a)
+{
+    int i;
+    for (i=0; i<a->length; i++)
+        a->coeff[i] = NAN;
+}
+
 SwsFilter *sws_getDefaultFilter(float lumaGBlur, float chromaGBlur,
                                 float lumaSharpen, float chromaSharpen,
                                 float chromaHShift, float chromaVShift,
@@ -1715,6 +1731,12 @@ SwsFilter *sws_getDefaultFilter(float lumaGBlur, float chromaGBlur,
     sws_normalizeVec(filter->lumH, 1.0);
     sws_normalizeVec(filter->lumV, 1.0);
 
+    if (isnan_vec(filter->chrH) ||
+        isnan_vec(filter->chrV) ||
+        isnan_vec(filter->lumH) ||
+        isnan_vec(filter->lumV))
+        goto fail;
+
     if (verbose)
         sws_printVec2(filter->chrH, NULL, AV_LOG_DEBUG);
     if (verbose)
@@ -1890,6 +1912,10 @@ static SwsVector *sws_getShiftedVec(SwsVector *a, int shift)
 void sws_shiftVec(SwsVector *a, int shift)
 {
     SwsVector *shifted = sws_getShiftedVec(a, shift);
+    if (!shifted) {
+        makenan_vec(a);
+        return;
+    }
     av_free(a->coeff);
     a->coeff  = shifted->coeff;
     a->length = shifted->length;
@@ -1899,6 +1925,10 @@ void sws_shiftVec(SwsVector *a, int shift)
 void sws_addVec(SwsVector *a, SwsVector *b)
 {
     SwsVector *sum = sws_sumVec(a, b);
+    if (!sum) {
+        makenan_vec(a);
+        return;
+    }
     av_free(a->coeff);
     a->coeff  = sum->coeff;
     a->length = sum->length;
@@ -1908,6 +1938,10 @@ void sws_addVec(SwsVector *a, SwsVector *b)
 void sws_subVec(SwsVector *a, SwsVector *b)
 {
     SwsVector *diff = sws_diffVec(a, b);
+    if (!diff) {
+        makenan_vec(a);
+        return;
+    }
     av_free(a->coeff);
     a->coeff  = diff->coeff;
     a->length = diff->length;
@@ -1917,6 +1951,10 @@ void sws_subVec(SwsVector *a, SwsVector *b)
 void sws_convVec(SwsVector *a, SwsVector *b)
 {
     SwsVector *conv = sws_getConvVec(a, b);
+    if (!conv) {
+        makenan_vec(a);
+        return;
+    }
     av_free(a->coeff);
     a->coeff  = conv->coeff;
     a->length = conv->length;