Check for failed extradata malloc, fixes a crash in out-of-memory conditions

or with extremely large extradata.

Originally committed as revision 19333 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavformat/avidec.c

@@ -483,6 +483,10 @@ static int avi_read_header(AVFormatContext *s, AVFormatParameters *ap)
                     if(size > 10*4 && size<(1<<30)){
                         st->codec->extradata_size= size - 10*4;
                         st->codec->extradata= av_malloc(st->codec->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
+                        if (!st->codec->extradata) {
+                            st->codec->extradata_size= 0;
+                            return AVERROR(ENOMEM);
+                        }
                         get_buffer(pb, st->codec->extradata, st->codec->extradata_size);
                     }