lavc/videotoolbox: fix threaded decoding

AVHWAccel.end_frame can run on a worker thread. The assumption of the
frame threading code is that the worker thread will change the AVFrame
image data, not the AVFrame fields. So the AVFrame fields are not synced
back to the main thread. But this breaks videotoolbox due to its special
requirements (everything else is fine). It actually wants to update
AVFrame fields.

The actual videotoolbox frame is now stored in the dummy AVBufferRef, so
it mimics what happens in non-videotoolbox cases. (Changing the
AVBufferRef contents is a bit like changing the image data.) The
post_process callback copies that reference to the proper AVFrame field.

Based on a patch by wm4.
Signed-off-by: Aman Gupta <aman@tmm1.net>

libavcodec/h264dec.c

@@ -838,9 +838,6 @@ static int output_frame(H264Context *h, AVFrame *dst, H264Picture *srcp)
     AVFrame *src = srcp->f;
     int ret;
 
-    if (src->format == AV_PIX_FMT_VIDEOTOOLBOX && src->buf[0]->size == 1)
-        return AVERROR_INVALIDDATA;
-
     ret = av_frame_ref(dst, src);
     if (ret < 0)
         return ret;

libavcodec/videotoolbox.c

@@ -45,8 +45,10 @@ enum { kCMVideoCodecType_HEVC = 'hvc1' };
 
 static void videotoolbox_buffer_release(void *opaque, uint8_t *data)
 {
-    CVPixelBufferRef cv_buffer = (CVImageBufferRef)data;
+    CVPixelBufferRef cv_buffer = *(CVPixelBufferRef *)data;
     CVPixelBufferRelease(cv_buffer);
+
+    av_free(data);
 }
 
 static int videotoolbox_buffer_copy(VTContext *vtctx,
@@ -69,19 +71,47 @@ static int videotoolbox_buffer_copy(VTContext *vtctx,
     return 0;
 }
 
+static int videotoolbox_postproc_frame(void *avctx, AVFrame *frame)
+{
+    CVPixelBufferRef ref = *(CVPixelBufferRef *)frame->buf[0]->data;
+
+    if (!ref) {
+        av_log(avctx, AV_LOG_ERROR, "No frame decoded?\n");
+        av_frame_unref(frame);
+        return AVERROR_EXTERNAL;
+    }
+
+    frame->data[3] = (uint8_t*)ref;
+
+    return 0;
+}
+
 int ff_videotoolbox_alloc_frame(AVCodecContext *avctx, AVFrame *frame)
 {
+    size_t      size = sizeof(CVPixelBufferRef);
+    uint8_t    *data = NULL;
+    AVBufferRef *buf = NULL;
     int ret = ff_attach_decode_data(frame);
+    FrameDecodeData *fdd;
     if (ret < 0)
         return ret;
 
+    data = av_mallocz(size);
+    if (!data)
+        return AVERROR(ENOMEM);
+    buf = av_buffer_create(data, size, videotoolbox_buffer_release, NULL, 0);
+    if (!buf) {
+        av_freep(&data);
+        return AVERROR(ENOMEM);
+    }
+    frame->buf[0] = buf;
+
+    fdd = (FrameDecodeData*)frame->private_ref->data;
+    fdd->post_process = videotoolbox_postproc_frame;
+
     frame->width  = avctx->width;
     frame->height = avctx->height;
     frame->format = avctx->pix_fmt;
-    frame->buf[0] = av_buffer_alloc(1);
-
-    if (!frame->buf[0])
-        return AVERROR(ENOMEM);
 
     return 0;
 }
@@ -285,20 +315,24 @@ CFDataRef ff_videotoolbox_hvcc_extradata_create(AVCodecContext *avctx)
     return data;
 }
 
-int ff_videotoolbox_buffer_create(VTContext *vtctx, AVFrame *frame)
+static int videotoolbox_set_frame(AVCodecContext *avctx, AVFrame *frame)
 {
-    av_buffer_unref(&frame->buf[0]);
-
-    frame->buf[0] = av_buffer_create((uint8_t*)vtctx->frame,
-                                     sizeof(vtctx->frame),
-                                     videotoolbox_buffer_release,
-                                     NULL,
-                                     AV_BUFFER_FLAG_READONLY);
-    if (!frame->buf[0]) {
-        return AVERROR(ENOMEM);
+    VTContext *vtctx = avctx->internal->hwaccel_priv_data;
+    if (!frame->buf[0] || frame->data[3]) {
+        av_log(avctx, AV_LOG_ERROR, "videotoolbox: invalid state\n");
+        av_frame_unref(frame);
+        return AVERROR_EXTERNAL;
+    }
+
+    CVPixelBufferRef *ref = (CVPixelBufferRef *)frame->buf[0]->data;
+
+    if (*ref) {
+        av_log(avctx, AV_LOG_ERROR, "videotoolbox: frame already set?\n");
+        av_frame_unref(frame);
+        return AVERROR_EXTERNAL;
     }
 
-    frame->data[3] = (uint8_t*)vtctx->frame;
+    *ref = vtctx->frame;
     vtctx->frame = NULL;
 
     return 0;
@@ -406,7 +440,7 @@ static int videotoolbox_buffer_create(AVCodecContext *avctx, AVFrame *frame)
     AVHWFramesContext *cached_frames;
     int ret;
 
-    ret = ff_videotoolbox_buffer_create(vtctx, frame);
+    ret = videotoolbox_set_frame(avctx, frame);
     if (ret < 0)
         return ret;
 

libavcodec/vt_internal.h

@@ -46,7 +46,6 @@ typedef struct VTContext {
 
 int ff_videotoolbox_alloc_frame(AVCodecContext *avctx, AVFrame *frame);
 int ff_videotoolbox_uninit(AVCodecContext *avctx);
-int ff_videotoolbox_buffer_create(VTContext *vtctx, AVFrame *frame);
 int ff_videotoolbox_h264_start_frame(AVCodecContext *avctx,
                                      const uint8_t *buffer,
                                      uint32_t size);