avformat/utils: End probing if the expected codec surpasses AVPROBE_SCORE_STREAM_RETRY

Fixes Ticket5800
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavformat/utils.c

@@ -307,7 +307,7 @@ static int set_codec_from_probe_data(AVFormatContext *s, AVStream *st,
     int score;
     AVInputFormat *fmt = av_probe_input_format3(pd, 1, &score);
 
-    if (fmt && st->request_probe <= score) {
+    if (fmt) {
         int i;
         av_log(s, AV_LOG_DEBUG,
                "Probe with size=%d, packets=%d detected %s with score=%d\n",
@@ -318,6 +318,9 @@ static int set_codec_from_probe_data(AVFormatContext *s, AVStream *st,
                 if (fmt_id_type[i].type != AVMEDIA_TYPE_AUDIO &&
                     st->codecpar->sample_rate)
                     continue;
+                if (st->request_probe > score &&
+                    st->codecpar->codec_id != fmt_id_type[i].id)
+                    continue;
                 st->codecpar->codec_id   = fmt_id_type[i].id;
                 st->codecpar->codec_type = fmt_id_type[i].type;
                 st->internal->need_context_update = 1;