pthread/mpegvideo: detect and block attempts to init frames after setup.

This fixes race conditions that ultimately lead to memory corruption.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/internal.h

@@ -157,4 +157,6 @@ static av_always_inline int64_t ff_samples_to_time_base(AVCodecContext *avctx,
                         avctx->time_base);
 }
 
+int ff_thread_can_start_frame(AVCodecContext *avctx);
+
 #endif /* AVCODEC_INTERNAL_H */

libavcodec/mpegvideo.c

@@ -1134,6 +1134,11 @@ int ff_MPV_frame_start(MpegEncContext *s, AVCodecContext *avctx)
     assert(s->last_picture_ptr == NULL || s->out_format != FMT_H264 ||
            s->codec_id == CODEC_ID_SVQ3);
 
+    if (!ff_thread_can_start_frame(avctx)) {
+        av_log(avctx, AV_LOG_ERROR, "Attempt to start a frame outside SETUP state\n");
+        return -1;
+    }
+
     /* mark & release old frames */
     if (s->out_format != FMT_H264 || s->codec_id == CODEC_ID_SVQ3) {
         if (s->pict_type != AV_PICTURE_TYPE_B && s->last_picture_ptr &&

libavcodec/pthread.c

@@ -932,6 +932,17 @@ static int *allocate_progress(PerThreadContext *p)
     return p->progress[i];
 }
 
+int ff_thread_can_start_frame(AVCodecContext *avctx)
+{
+    PerThreadContext *p = avctx->thread_opaque;
+    if ((avctx->active_thread_type&FF_THREAD_FRAME) && p->state != STATE_SETTING_UP &&
+        (avctx->codec->update_thread_context || (!avctx->thread_safe_callbacks &&
+                avctx->get_buffer != avcodec_default_get_buffer))) {
+        return 0;
+    }
+    return 1;
+}
+
 int ff_thread_get_buffer(AVCodecContext *avctx, AVFrame *f)
 {
     PerThreadContext *p = avctx->thread_opaque;

libavcodec/utils.c

@@ -2285,6 +2285,11 @@ void ff_thread_await_progress(AVFrame *f, int progress, int field)
 {
 }
 
+int ff_thread_can_start_frame(AVCodecContext *avctx)
+{
+    return 1;
+}
+
 #endif
 
 enum AVMediaType avcodec_get_type(enum CodecID codec_id)