avfoundation.m 43.7 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32
/*
 * AVFoundation input device
 * Copyright (c) 2014 Thilo Borgmann <thilo.borgmann@mail.de>
 *
 * This file is part of FFmpeg.
 *
 * FFmpeg is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * FFmpeg is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with FFmpeg; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

/**
 * @file
 * AVFoundation input device
 * @author Thilo Borgmann <thilo.borgmann@mail.de>
 */

#import <AVFoundation/AVFoundation.h>
#include <pthread.h>

#include "libavutil/pixdesc.h"
#include "libavutil/opt.h"
33
#include "libavutil/avstring.h"
34 35
#include "libavformat/internal.h"
#include "libavutil/internal.h"
36
#include "libavutil/parseutils.h"
37
#include "libavutil/time.h"
38
#include "libavutil/imgutils.h"
39 40
#include "avdevice.h"

41
static const int avf_time_base = 1000000;
42 43 44 45 46 47

static const AVRational avf_time_base_q = {
    .num = 1,
    .den = avf_time_base
};

48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75
struct AVFPixelFormatSpec {
    enum AVPixelFormat ff_id;
    OSType avf_id;
};

static const struct AVFPixelFormatSpec avf_pixel_formats[] = {
    { AV_PIX_FMT_MONOBLACK,    kCVPixelFormatType_1Monochrome },
    { AV_PIX_FMT_RGB555BE,     kCVPixelFormatType_16BE555 },
    { AV_PIX_FMT_RGB555LE,     kCVPixelFormatType_16LE555 },
    { AV_PIX_FMT_RGB565BE,     kCVPixelFormatType_16BE565 },
    { AV_PIX_FMT_RGB565LE,     kCVPixelFormatType_16LE565 },
    { AV_PIX_FMT_RGB24,        kCVPixelFormatType_24RGB },
    { AV_PIX_FMT_BGR24,        kCVPixelFormatType_24BGR },
    { AV_PIX_FMT_0RGB,         kCVPixelFormatType_32ARGB },
    { AV_PIX_FMT_BGR0,         kCVPixelFormatType_32BGRA },
    { AV_PIX_FMT_0BGR,         kCVPixelFormatType_32ABGR },
    { AV_PIX_FMT_RGB0,         kCVPixelFormatType_32RGBA },
    { AV_PIX_FMT_BGR48BE,      kCVPixelFormatType_48RGB },
    { AV_PIX_FMT_UYVY422,      kCVPixelFormatType_422YpCbCr8 },
    { AV_PIX_FMT_YUVA444P,     kCVPixelFormatType_4444YpCbCrA8R },
    { AV_PIX_FMT_YUVA444P16LE, kCVPixelFormatType_4444AYpCbCr16 },
    { AV_PIX_FMT_YUV444P,      kCVPixelFormatType_444YpCbCr8 },
    { AV_PIX_FMT_YUV422P16,    kCVPixelFormatType_422YpCbCr16 },
    { AV_PIX_FMT_YUV422P10,    kCVPixelFormatType_422YpCbCr10 },
    { AV_PIX_FMT_YUV444P10,    kCVPixelFormatType_444YpCbCr10 },
    { AV_PIX_FMT_YUV420P,      kCVPixelFormatType_420YpCbCr8Planar },
    { AV_PIX_FMT_NV12,         kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange },
    { AV_PIX_FMT_YUYV422,      kCVPixelFormatType_422YpCbCr8_yuvs },
76
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
77
    { AV_PIX_FMT_GRAY8,        kCVPixelFormatType_OneComponent8 },
78
#endif
79 80 81
    { AV_PIX_FMT_NONE, 0 }
};

82 83 84 85 86
typedef struct
{
    AVClass*        class;

    int             frames_captured;
87
    int             audio_frames_captured;
88
    int64_t         first_pts;
89
    int64_t         first_audio_pts;
90 91
    pthread_mutex_t frame_lock;
    id              avf_delegate;
92
    id              avf_audio_delegate;
93

94 95 96
    AVRational      framerate;
    int             width, height;

97
    int             capture_cursor;
98
    int             capture_mouse_clicks;
99
    int             capture_raw_data;
100
    int             drop_late_frames;
101
    int             video_is_muxed;
102

103 104
    int             list_devices;
    int             video_device_index;
105
    int             video_stream_index;
106 107 108 109 110 111
    int             audio_device_index;
    int             audio_stream_index;

    char            *video_filename;
    char            *audio_filename;

112 113
    int             num_video_devices;

114 115 116 117 118 119 120 121 122 123 124
    int             audio_channels;
    int             audio_bits_per_sample;
    int             audio_float;
    int             audio_be;
    int             audio_signed_integer;
    int             audio_packed;
    int             audio_non_interleaved;

    int32_t         *audio_buffer;
    int             audio_buffer_size;

125
    enum AVPixelFormat pixel_format;
126 127 128

    AVCaptureSession         *capture_session;
    AVCaptureVideoDataOutput *video_output;
129
    AVCaptureAudioDataOutput *audio_output;
130
    CMSampleBufferRef         current_frame;
131
    CMSampleBufferRef         current_audio_frame;
132 133 134 135

    AVCaptureDevice          *observed_device;
    AVCaptureDeviceTransportControlsPlaybackMode observed_mode;
    int                      observed_quit;
136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168
} AVFContext;

static void lock_frames(AVFContext* ctx)
{
    pthread_mutex_lock(&ctx->frame_lock);
}

static void unlock_frames(AVFContext* ctx)
{
    pthread_mutex_unlock(&ctx->frame_lock);
}

/** FrameReciever class - delegate for AVCaptureSession
 */
@interface AVFFrameReceiver : NSObject
{
    AVFContext* _context;
}

- (id)initWithContext:(AVFContext*)context;

- (void)  captureOutput:(AVCaptureOutput *)captureOutput
  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
         fromConnection:(AVCaptureConnection *)connection;

@end

@implementation AVFFrameReceiver

- (id)initWithContext:(AVFContext*)context
{
    if (self = [super init]) {
        _context = context;
169 170 171 172 173 174 175 176 177 178 179

        // start observing if a device is set for it
        if (_context->observed_device) {
            NSString *keyPath = NSStringFromSelector(@selector(transportControlsPlaybackMode));
            NSKeyValueObservingOptions options = NSKeyValueObservingOptionNew;

            [_context->observed_device addObserver: self
                                        forKeyPath: keyPath
                                           options: options
                                           context: _context];
        }
180 181 182 183
    }
    return self;
}

184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
- (void)dealloc {
    // stop observing if a device is set for it
    NSString *keyPath = NSStringFromSelector(@selector(transportControlsPlaybackMode));
    [_context->observed_device removeObserver: self forKeyPath: keyPath];
    [super dealloc];
}

- (void)observeValueForKeyPath:(NSString *)keyPath
                      ofObject:(id)object
                        change:(NSDictionary *)change
                       context:(void *)context {
    if (context == _context) {
        AVCaptureDeviceTransportControlsPlaybackMode mode =
            [change[NSKeyValueChangeNewKey] integerValue];

        if (mode != _context->observed_mode) {
            if (mode == AVCaptureDeviceTransportControlsNotPlayingMode) {
                _context->observed_quit = 1;
            }
            _context->observed_mode = mode;
        }
    } else {
        [super observeValueForKeyPath: keyPath
                             ofObject: object
                               change: change
                              context: context];
    }
}

213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231
- (void)  captureOutput:(AVCaptureOutput *)captureOutput
  didOutputSampleBuffer:(CMSampleBufferRef)videoFrame
         fromConnection:(AVCaptureConnection *)connection
{
    lock_frames(_context);

    if (_context->current_frame != nil) {
        CFRelease(_context->current_frame);
    }

    _context->current_frame = (CMSampleBufferRef)CFRetain(videoFrame);

    unlock_frames(_context);

    ++_context->frames_captured;
}

@end

232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275
/** AudioReciever class - delegate for AVCaptureSession
 */
@interface AVFAudioReceiver : NSObject
{
    AVFContext* _context;
}

- (id)initWithContext:(AVFContext*)context;

- (void)  captureOutput:(AVCaptureOutput *)captureOutput
  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
         fromConnection:(AVCaptureConnection *)connection;

@end

@implementation AVFAudioReceiver

- (id)initWithContext:(AVFContext*)context
{
    if (self = [super init]) {
        _context = context;
    }
    return self;
}

- (void)  captureOutput:(AVCaptureOutput *)captureOutput
  didOutputSampleBuffer:(CMSampleBufferRef)audioFrame
         fromConnection:(AVCaptureConnection *)connection
{
    lock_frames(_context);

    if (_context->current_audio_frame != nil) {
        CFRelease(_context->current_audio_frame);
    }

    _context->current_audio_frame = (CMSampleBufferRef)CFRetain(audioFrame);

    unlock_frames(_context);

    ++_context->audio_frames_captured;
}

@end

276 277 278 279 280 281
static void destroy_context(AVFContext* ctx)
{
    [ctx->capture_session stopRunning];

    [ctx->capture_session release];
    [ctx->video_output    release];
282
    [ctx->audio_output    release];
283
    [ctx->avf_delegate    release];
284
    [ctx->avf_audio_delegate release];
285 286 287

    ctx->capture_session = NULL;
    ctx->video_output    = NULL;
288
    ctx->audio_output    = NULL;
289
    ctx->avf_delegate    = NULL;
290 291 292
    ctx->avf_audio_delegate = NULL;

    av_freep(&ctx->audio_buffer);
293 294 295 296 297 298 299 300

    pthread_mutex_destroy(&ctx->frame_lock);

    if (ctx->current_frame) {
        CFRelease(ctx->current_frame);
    }
}

301 302 303
static void parse_device_name(AVFormatContext *s)
{
    AVFContext *ctx = (AVFContext*)s->priv_data;
304
    char *tmp = av_strdup(s->url);
305
    char *save;
306 307

    if (tmp[0] != ':') {
308 309
        ctx->video_filename = av_strtok(tmp,  ":", &save);
        ctx->audio_filename = av_strtok(NULL, ":", &save);
310
    } else {
311
        ctx->audio_filename = av_strtok(tmp,  ":", &save);
312 313 314
    }
}

315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334
/**
 * Configure the video device.
 *
 * Configure the video device using a run-time approach to access properties
 * since formats, activeFormat are available since  iOS >= 7.0 or OSX >= 10.7
 * and activeVideoMaxFrameDuration is available since i0S >= 7.0 and OSX >= 10.9.
 *
 * The NSUndefinedKeyException must be handled by the caller of this function.
 *
 */
static int configure_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
{
    AVFContext *ctx = (AVFContext*)s->priv_data;

    double framerate = av_q2d(ctx->framerate);
    NSObject *range = nil;
    NSObject *format = nil;
    NSObject *selected_range = nil;
    NSObject *selected_format = nil;

335 336 337 338
    // try to configure format by formats list
    // might raise an exception if no format list is given
    // (then fallback to default, no configuration)
    @try {
339 340 341
        for (format in [video_device valueForKey:@"formats"]) {
            CMFormatDescriptionRef formatDescription;
            CMVideoDimensions dimensions;
342

343 344
            formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];
            dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
345

346 347
            if ((ctx->width == 0 && ctx->height == 0) ||
                (dimensions.width == ctx->width && dimensions.height == ctx->height)) {
348

349
                selected_format = format;
350

351 352
                for (range in [format valueForKey:@"videoSupportedFrameRateRanges"]) {
                    double max_framerate;
353

354 355 356 357 358
                    [[range valueForKey:@"maxFrameRate"] getValue:&max_framerate];
                    if (fabs (framerate - max_framerate) < 0.01) {
                        selected_range = range;
                        break;
                    }
359 360 361 362
                }
            }
        }

363 364 365 366
        if (!selected_format) {
            av_log(s, AV_LOG_ERROR, "Selected video size (%dx%d) is not supported by the device.\n",
                ctx->width, ctx->height);
            goto unsupported_format;
367
        }
368

369 370 371 372 373 374 375 376
        if (!selected_range) {
            av_log(s, AV_LOG_ERROR, "Selected framerate (%f) is not supported by the device.\n",
                framerate);
            if (ctx->video_is_muxed) {
                av_log(s, AV_LOG_ERROR, "Falling back to default.\n");
            } else {
                goto unsupported_format;
            }
377
        }
378 379 380 381 382 383 384 385 386 387 388 389 390

        if ([video_device lockForConfiguration:NULL] == YES) {
            if (selected_format) {
                [video_device setValue:selected_format forKey:@"activeFormat"];
            }
            if (selected_range) {
                NSValue *min_frame_duration = [selected_range valueForKey:@"minFrameDuration"];
                [video_device setValue:min_frame_duration forKey:@"activeVideoMinFrameDuration"];
                [video_device setValue:min_frame_duration forKey:@"activeVideoMaxFrameDuration"];
            }
        } else {
            av_log(s, AV_LOG_ERROR, "Could not lock device for configuration.\n");
            return AVERROR(EINVAL);
391 392 393 394
        }
    } @catch(NSException *e) {
        av_log(ctx, AV_LOG_WARNING, "Configuration of video device failed, falling back to default.\n");
    }
395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421

    return 0;

unsupported_format:

    av_log(s, AV_LOG_ERROR, "Supported modes:\n");
    for (format in [video_device valueForKey:@"formats"]) {
        CMFormatDescriptionRef formatDescription;
        CMVideoDimensions dimensions;

        formatDescription = (CMFormatDescriptionRef) [format performSelector:@selector(formatDescription)];
        dimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);

        for (range in [format valueForKey:@"videoSupportedFrameRateRanges"]) {
            double min_framerate;
            double max_framerate;

            [[range valueForKey:@"minFrameRate"] getValue:&min_framerate];
            [[range valueForKey:@"maxFrameRate"] getValue:&max_framerate];
            av_log(s, AV_LOG_ERROR, "  %dx%d@[%f %f]fps\n",
                dimensions.width, dimensions.height,
                min_framerate, max_framerate);
        }
    }
    return AVERROR(EINVAL);
}

422
static int add_video_device(AVFormatContext *s, AVCaptureDevice *video_device)
423
{
424
    AVFContext *ctx = (AVFContext*)s->priv_data;
425
    int ret;
426
    NSError *error  = nil;
427
    AVCaptureInput* capture_input = nil;
428 429 430 431
    struct AVFPixelFormatSpec pxl_fmt_spec;
    NSNumber *pixel_format;
    NSDictionary *capture_dict;
    dispatch_queue_t queue;
432 433 434 435 436 437

    if (ctx->video_device_index < ctx->num_video_devices) {
        capture_input = (AVCaptureInput*) [[[AVCaptureDeviceInput alloc] initWithDevice:video_device error:&error] autorelease];
    } else {
        capture_input = (AVCaptureInput*) video_device;
    }
438

439
    if (!capture_input) {
440 441
        av_log(s, AV_LOG_ERROR, "Failed to create AV capture input device: %s\n",
               [[error localizedDescription] UTF8String]);
442
        return 1;
443 444
    }

445 446
    if ([ctx->capture_session canAddInput:capture_input]) {
        [ctx->capture_session addInput:capture_input];
447 448
    } else {
        av_log(s, AV_LOG_ERROR, "can't add video input to capture session\n");
449
        return 1;
450 451 452 453 454 455 456
    }

    // Attaching output
    ctx->video_output = [[AVCaptureVideoDataOutput alloc] init];

    if (!ctx->video_output) {
        av_log(s, AV_LOG_ERROR, "Failed to init AV video output\n");
457
        return 1;
458 459
    }

460 461 462 463 464 465 466
    // Configure device framerate and video size
    @try {
        if ((ret = configure_video_device(s, video_device)) < 0) {
            return ret;
        }
    } @catch (NSException *exception) {
        if (![[exception name] isEqualToString:NSUndefinedKeyException]) {
467
          av_log (s, AV_LOG_ERROR, "An error occurred: %s", [exception.reason UTF8String]);
468 469 470 471
          return AVERROR_EXTERNAL;
        }
    }

472 473 474 475 476 477 478 479 480 481 482 483 484 485
    // select pixel format
    pxl_fmt_spec.ff_id = AV_PIX_FMT_NONE;

    for (int i = 0; avf_pixel_formats[i].ff_id != AV_PIX_FMT_NONE; i++) {
        if (ctx->pixel_format == avf_pixel_formats[i].ff_id) {
            pxl_fmt_spec = avf_pixel_formats[i];
            break;
        }
    }

    // check if selected pixel format is supported by AVFoundation
    if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
        av_log(s, AV_LOG_ERROR, "Selected pixel format (%s) is not supported by AVFoundation.\n",
               av_get_pix_fmt_name(pxl_fmt_spec.ff_id));
486
        return 1;
487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518
    }

    // check if the pixel format is available for this device
    if ([[ctx->video_output availableVideoCVPixelFormatTypes] indexOfObject:[NSNumber numberWithInt:pxl_fmt_spec.avf_id]] == NSNotFound) {
        av_log(s, AV_LOG_ERROR, "Selected pixel format (%s) is not supported by the input device.\n",
               av_get_pix_fmt_name(pxl_fmt_spec.ff_id));

        pxl_fmt_spec.ff_id = AV_PIX_FMT_NONE;

        av_log(s, AV_LOG_ERROR, "Supported pixel formats:\n");
        for (NSNumber *pxl_fmt in [ctx->video_output availableVideoCVPixelFormatTypes]) {
            struct AVFPixelFormatSpec pxl_fmt_dummy;
            pxl_fmt_dummy.ff_id = AV_PIX_FMT_NONE;
            for (int i = 0; avf_pixel_formats[i].ff_id != AV_PIX_FMT_NONE; i++) {
                if ([pxl_fmt intValue] == avf_pixel_formats[i].avf_id) {
                    pxl_fmt_dummy = avf_pixel_formats[i];
                    break;
                }
            }

            if (pxl_fmt_dummy.ff_id != AV_PIX_FMT_NONE) {
                av_log(s, AV_LOG_ERROR, "  %s\n", av_get_pix_fmt_name(pxl_fmt_dummy.ff_id));

                // select first supported pixel format instead of user selected (or default) pixel format
                if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
                    pxl_fmt_spec = pxl_fmt_dummy;
                }
            }
        }

        // fail if there is no appropriate pixel format or print a warning about overriding the pixel format
        if (pxl_fmt_spec.ff_id == AV_PIX_FMT_NONE) {
519
            return 1;
520 521 522 523 524 525
        } else {
            av_log(s, AV_LOG_WARNING, "Overriding selected pixel format to use %s instead.\n",
                   av_get_pix_fmt_name(pxl_fmt_spec.ff_id));
        }
    }

526 527
    // set videoSettings to an empty dict for receiving raw data of muxed devices
    if (ctx->capture_raw_data) {
528 529
        ctx->pixel_format = pxl_fmt_spec.ff_id;
        ctx->video_output.videoSettings = @{ };
530
    } else {
531 532 533 534
        ctx->pixel_format = pxl_fmt_spec.ff_id;
        pixel_format = [NSNumber numberWithUnsignedInt:pxl_fmt_spec.avf_id];
        capture_dict = [NSDictionary dictionaryWithObject:pixel_format
                                                   forKey:(id)kCVPixelBufferPixelFormatTypeKey];
535

536
        [ctx->video_output setVideoSettings:capture_dict];
537
    }
538
    [ctx->video_output setAlwaysDiscardsLateVideoFrames:ctx->drop_late_frames];
539

540 541 542 543 544 545 546 547 548
    // check for transport control support and set observer device if supported
    int trans_ctrl = [video_device transportControlsSupported];
    AVCaptureDeviceTransportControlsPlaybackMode trans_mode = [video_device transportControlsPlaybackMode];

    if (trans_ctrl) {
        ctx->observed_mode   = trans_mode;
        ctx->observed_device = video_device;
    }

549 550
    ctx->avf_delegate = [[AVFFrameReceiver alloc] initWithContext:ctx];

551
    queue = dispatch_queue_create("avf_queue", NULL);
552 553 554 555 556 557 558
    [ctx->video_output setSampleBufferDelegate:ctx->avf_delegate queue:queue];
    dispatch_release(queue);

    if ([ctx->capture_session canAddOutput:ctx->video_output]) {
        [ctx->capture_session addOutput:ctx->video_output];
    } else {
        av_log(s, AV_LOG_ERROR, "can't add video output to capture session\n");
559
        return 1;
560 561
    }

562 563 564
    return 0;
}

565 566 567 568 569
static int add_audio_device(AVFormatContext *s, AVCaptureDevice *audio_device)
{
    AVFContext *ctx = (AVFContext*)s->priv_data;
    NSError *error  = nil;
    AVCaptureDeviceInput* audio_dev_input = [[[AVCaptureDeviceInput alloc] initWithDevice:audio_device error:&error] autorelease];
570
    dispatch_queue_t queue;
571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594

    if (!audio_dev_input) {
        av_log(s, AV_LOG_ERROR, "Failed to create AV capture input device: %s\n",
               [[error localizedDescription] UTF8String]);
        return 1;
    }

    if ([ctx->capture_session canAddInput:audio_dev_input]) {
        [ctx->capture_session addInput:audio_dev_input];
    } else {
        av_log(s, AV_LOG_ERROR, "can't add audio input to capture session\n");
        return 1;
    }

    // Attaching output
    ctx->audio_output = [[AVCaptureAudioDataOutput alloc] init];

    if (!ctx->audio_output) {
        av_log(s, AV_LOG_ERROR, "Failed to init AV audio output\n");
        return 1;
    }

    ctx->avf_audio_delegate = [[AVFAudioReceiver alloc] initWithContext:ctx];

595
    queue = dispatch_queue_create("avf_audio_queue", NULL);
596 597 598 599 600 601 602 603 604 605 606 607 608
    [ctx->audio_output setSampleBufferDelegate:ctx->avf_audio_delegate queue:queue];
    dispatch_release(queue);

    if ([ctx->capture_session canAddOutput:ctx->audio_output]) {
        [ctx->capture_session addOutput:ctx->audio_output];
    } else {
        av_log(s, AV_LOG_ERROR, "adding audio output to capture session failed\n");
        return 1;
    }

    return 0;
}

609 610 611
static int get_video_config(AVFormatContext *s)
{
    AVFContext *ctx = (AVFContext*)s->priv_data;
612
    CVImageBufferRef image_buffer;
613
    CMBlockBufferRef block_buffer;
614 615 616 617 618 619
    CGSize image_buffer_size;
    AVStream* stream = avformat_new_stream(s, NULL);

    if (!stream) {
        return 1;
    }
620 621 622 623 624 625 626 627

    // Take stream info from the first frame.
    while (ctx->frames_captured < 1) {
        CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
    }

    lock_frames(ctx);

628 629
    ctx->video_stream_index = stream->index;

630 631
    avpriv_set_pts_info(stream, 64, 1, avf_time_base);

632 633 634 635 636
    image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
    block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame);

    if (image_buffer) {
        image_buffer_size = CVImageBufferGetEncodedSize(image_buffer);
637

638 639 640 641 642
        stream->codecpar->codec_id   = AV_CODEC_ID_RAWVIDEO;
        stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
        stream->codecpar->width      = (int)image_buffer_size.width;
        stream->codecpar->height     = (int)image_buffer_size.height;
        stream->codecpar->format     = ctx->pixel_format;
643 644 645 646 647
    } else {
        stream->codecpar->codec_id   = AV_CODEC_ID_DVVIDEO;
        stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
        stream->codecpar->format     = ctx->pixel_format;
    }
648 649 650 651 652

    CFRelease(ctx->current_frame);
    ctx->current_frame = nil;

    unlock_frames(ctx);
653 654 655 656

    return 0;
}

657 658 659
static int get_audio_config(AVFormatContext *s)
{
    AVFContext *ctx = (AVFContext*)s->priv_data;
660 661 662 663 664 665
    CMFormatDescriptionRef format_desc;
    AVStream* stream = avformat_new_stream(s, NULL);

    if (!stream) {
        return 1;
    }
666 667 668 669 670 671 672 673 674 675 676 677

    // Take stream info from the first frame.
    while (ctx->audio_frames_captured < 1) {
        CFRunLoopRunInMode(kCFRunLoopDefaultMode, 0.1, YES);
    }

    lock_frames(ctx);

    ctx->audio_stream_index = stream->index;

    avpriv_set_pts_info(stream, 64, 1, avf_time_base);

678
    format_desc = CMSampleBufferGetFormatDescription(ctx->current_audio_frame);
679 680 681 682 683 684 685
    const AudioStreamBasicDescription *basic_desc = CMAudioFormatDescriptionGetStreamBasicDescription(format_desc);

    if (!basic_desc) {
        av_log(s, AV_LOG_ERROR, "audio format not available\n");
        return 1;
    }

686 687 688 689
    stream->codecpar->codec_type     = AVMEDIA_TYPE_AUDIO;
    stream->codecpar->sample_rate    = basic_desc->mSampleRate;
    stream->codecpar->channels       = basic_desc->mChannelsPerFrame;
    stream->codecpar->channel_layout = av_get_default_channel_layout(stream->codecpar->channels);
690 691 692 693 694 695 696 697 698 699 700

    ctx->audio_channels        = basic_desc->mChannelsPerFrame;
    ctx->audio_bits_per_sample = basic_desc->mBitsPerChannel;
    ctx->audio_float           = basic_desc->mFormatFlags & kAudioFormatFlagIsFloat;
    ctx->audio_be              = basic_desc->mFormatFlags & kAudioFormatFlagIsBigEndian;
    ctx->audio_signed_integer  = basic_desc->mFormatFlags & kAudioFormatFlagIsSignedInteger;
    ctx->audio_packed          = basic_desc->mFormatFlags & kAudioFormatFlagIsPacked;
    ctx->audio_non_interleaved = basic_desc->mFormatFlags & kAudioFormatFlagIsNonInterleaved;

    if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
        ctx->audio_float &&
701
        ctx->audio_bits_per_sample == 32 &&
702
        ctx->audio_packed) {
703
        stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_F32BE : AV_CODEC_ID_PCM_F32LE;
704 705 706 707
    } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
        ctx->audio_signed_integer &&
        ctx->audio_bits_per_sample == 16 &&
        ctx->audio_packed) {
708
        stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S16BE : AV_CODEC_ID_PCM_S16LE;
709 710 711 712
    } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
        ctx->audio_signed_integer &&
        ctx->audio_bits_per_sample == 24 &&
        ctx->audio_packed) {
713
        stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S24BE : AV_CODEC_ID_PCM_S24LE;
714 715 716 717
    } else if (basic_desc->mFormatID == kAudioFormatLinearPCM &&
        ctx->audio_signed_integer &&
        ctx->audio_bits_per_sample == 32 &&
        ctx->audio_packed) {
718
        stream->codecpar->codec_id = ctx->audio_be ? AV_CODEC_ID_PCM_S32BE : AV_CODEC_ID_PCM_S32LE;
719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741
    } else {
        av_log(s, AV_LOG_ERROR, "audio format is not supported\n");
        return 1;
    }

    if (ctx->audio_non_interleaved) {
        CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
        ctx->audio_buffer_size        = CMBlockBufferGetDataLength(block_buffer);
        ctx->audio_buffer             = av_malloc(ctx->audio_buffer_size);
        if (!ctx->audio_buffer) {
            av_log(s, AV_LOG_ERROR, "error allocating audio buffer\n");
            return 1;
        }
    }

    CFRelease(ctx->current_audio_frame);
    ctx->current_audio_frame = nil;

    unlock_frames(ctx);

    return 0;
}

742 743 744
static int avf_read_header(AVFormatContext *s)
{
    NSAutoreleasePool *pool = [[NSAutoreleasePool alloc] init];
745
    int capture_screen      = 0;
746
    uint32_t num_screens    = 0;
747
    AVFContext *ctx         = (AVFContext*)s->priv_data;
748 749 750 751
    AVCaptureDevice *video_device = nil;
    AVCaptureDevice *audio_device = nil;
    // Find capture device
    NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo];
752
    NSArray *devices_muxed = [AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed];
753

754
    ctx->num_video_devices = [devices count] + [devices_muxed count];
755
    ctx->first_pts          = av_gettime();
756
    ctx->first_audio_pts    = av_gettime();
757 758 759

    pthread_mutex_init(&ctx->frame_lock, NULL);

760
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
761
    CGGetActiveDisplayList(0, NULL, &num_screens);
762
#endif
763

764 765
    // List devices if requested
    if (ctx->list_devices) {
766
        int index = 0;
767
        av_log(ctx, AV_LOG_INFO, "AVFoundation video devices:\n");
768 769
        for (AVCaptureDevice *device in devices) {
            const char *name = [[device localizedName] UTF8String];
770
            index            = [devices indexOfObject:device];
771 772
            av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
        }
773 774 775 776 777
        for (AVCaptureDevice *device in devices_muxed) {
            const char *name = [[device localizedName] UTF8String];
            index            = [devices count] + [devices_muxed indexOfObject:device];
            av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
        }
778
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
779 780 781 782
        if (num_screens > 0) {
            CGDirectDisplayID screens[num_screens];
            CGGetActiveDisplayList(num_screens, screens, &num_screens);
            for (int i = 0; i < num_screens; i++) {
783
                av_log(ctx, AV_LOG_INFO, "[%d] Capture screen %d\n", ctx->num_video_devices + i, i);
784 785
            }
        }
786
#endif
787

788 789 790 791 792 793 794 795
        av_log(ctx, AV_LOG_INFO, "AVFoundation audio devices:\n");
        devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
        for (AVCaptureDevice *device in devices) {
            const char *name = [[device localizedName] UTF8String];
            int index  = [devices indexOfObject:device];
            av_log(ctx, AV_LOG_INFO, "[%d] %s\n", index, name);
        }
         goto fail;
796 797
    }

798 799
    // parse input filename for video and audio device
    parse_device_name(s);
800 801

    // check for device index given in filename
802 803 804 805 806
    if (ctx->video_device_index == -1 && ctx->video_filename) {
        sscanf(ctx->video_filename, "%d", &ctx->video_device_index);
    }
    if (ctx->audio_device_index == -1 && ctx->audio_filename) {
        sscanf(ctx->audio_filename, "%d", &ctx->audio_device_index);
807 808 809
    }

    if (ctx->video_device_index >= 0) {
810
        if (ctx->video_device_index < ctx->num_video_devices) {
811
            if (ctx->video_device_index < [devices count]) {
812
                video_device = [devices objectAtIndex:ctx->video_device_index];
813 814 815 816
            } else {
                video_device = [devices_muxed objectAtIndex:(ctx->video_device_index - [devices count])];
                ctx->video_is_muxed = 1;
            }
817
        } else if (ctx->video_device_index < ctx->num_video_devices + num_screens) {
818
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
819 820 821
            CGDirectDisplayID screens[num_screens];
            CGGetActiveDisplayList(num_screens, screens, &num_screens);
            AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[ctx->video_device_index - ctx->num_video_devices]] autorelease];
822 823 824 825 826

            if (ctx->framerate.num > 0) {
                capture_screen_input.minFrameDuration = CMTimeMake(ctx->framerate.den, ctx->framerate.num);
            }

827 828 829 830 831 832 833 834
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
            if (ctx->capture_cursor) {
                capture_screen_input.capturesCursor = YES;
            } else {
                capture_screen_input.capturesCursor = NO;
            }
#endif

835 836 837 838 839 840
            if (ctx->capture_mouse_clicks) {
                capture_screen_input.capturesMouseClicks = YES;
            } else {
                capture_screen_input.capturesMouseClicks = NO;
            }

841
            video_device = (AVCaptureDevice*) capture_screen_input;
842
            capture_screen = 1;
843
#endif
844
         } else {
845 846 847
            av_log(ctx, AV_LOG_ERROR, "Invalid device index\n");
            goto fail;
        }
848
    } else if (ctx->video_filename &&
849 850 851 852
               strncmp(ctx->video_filename, "none", 4)) {
        if (!strncmp(ctx->video_filename, "default", 7)) {
            video_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo];
        } else {
853
        // looking for video inputs
854
        for (AVCaptureDevice *device in devices) {
855
            if (!strncmp(ctx->video_filename, [[device localizedName] UTF8String], strlen(ctx->video_filename))) {
856 857 858 859
                video_device = device;
                break;
            }
        }
860 861 862 863 864 865 866 867
        // looking for muxed inputs
        for (AVCaptureDevice *device in devices_muxed) {
            if (!strncmp(ctx->video_filename, [[device localizedName] UTF8String], strlen(ctx->video_filename))) {
                video_device = device;
                ctx->video_is_muxed = 1;
                break;
            }
        }
868

869
#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1070
870 871 872 873 874 875 876 877 878
        // looking for screen inputs
        if (!video_device) {
            int idx;
            if(sscanf(ctx->video_filename, "Capture screen %d", &idx) && idx < num_screens) {
                CGDirectDisplayID screens[num_screens];
                CGGetActiveDisplayList(num_screens, screens, &num_screens);
                AVCaptureScreenInput* capture_screen_input = [[[AVCaptureScreenInput alloc] initWithDisplayID:screens[idx]] autorelease];
                video_device = (AVCaptureDevice*) capture_screen_input;
                ctx->video_device_index = ctx->num_video_devices + idx;
879 880 881 882 883
                capture_screen = 1;

                if (ctx->framerate.num > 0) {
                    capture_screen_input.minFrameDuration = CMTimeMake(ctx->framerate.den, ctx->framerate.num);
                }
884 885 886 887 888 889 890 891

#if !TARGET_OS_IPHONE && __MAC_OS_X_VERSION_MIN_REQUIRED >= 1080
                if (ctx->capture_cursor) {
                    capture_screen_input.capturesCursor = YES;
                } else {
                    capture_screen_input.capturesCursor = NO;
                }
#endif
892 893 894 895 896 897

                if (ctx->capture_mouse_clicks) {
                    capture_screen_input.capturesMouseClicks = YES;
                } else {
                    capture_screen_input.capturesMouseClicks = NO;
                }
898 899
            }
        }
900
#endif
901
        }
902

903 904 905 906 907 908
        if (!video_device) {
            av_log(ctx, AV_LOG_ERROR, "Video device not found\n");
            goto fail;
        }
    }

909 910 911
    // get audio device
    if (ctx->audio_device_index >= 0) {
        NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
912

913 914
        if (ctx->audio_device_index >= [devices count]) {
            av_log(ctx, AV_LOG_ERROR, "Invalid audio device index\n");
915 916
            goto fail;
        }
917 918 919

        audio_device = [devices objectAtIndex:ctx->audio_device_index];
    } else if (ctx->audio_filename &&
920 921 922 923
               strncmp(ctx->audio_filename, "none", 4)) {
        if (!strncmp(ctx->audio_filename, "default", 7)) {
            audio_device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio];
        } else {
924 925 926 927 928 929 930 931
        NSArray *devices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];

        for (AVCaptureDevice *device in devices) {
            if (!strncmp(ctx->audio_filename, [[device localizedName] UTF8String], strlen(ctx->audio_filename))) {
                audio_device = device;
                break;
            }
        }
932
        }
933 934 935 936 937

        if (!audio_device) {
            av_log(ctx, AV_LOG_ERROR, "Audio device not found\n");
             goto fail;
        }
938 939
    }

940 941 942 943 944 945 946
    // Video nor Audio capture device not found, looking for AVMediaTypeVideo/Audio
    if (!video_device && !audio_device) {
        av_log(s, AV_LOG_ERROR, "No AV capture device found\n");
        goto fail;
    }

    if (video_device) {
947 948 949 950 951
        if (ctx->video_device_index < ctx->num_video_devices) {
            av_log(s, AV_LOG_DEBUG, "'%s' opened\n", [[video_device localizedName] UTF8String]);
        } else {
            av_log(s, AV_LOG_DEBUG, "'%s' opened\n", [[video_device description] UTF8String]);
        }
952 953 954 955
    }
    if (audio_device) {
        av_log(s, AV_LOG_DEBUG, "audio device '%s' opened\n", [[audio_device localizedName] UTF8String]);
    }
956 957 958 959

    // Initialize capture session
    ctx->capture_session = [[AVCaptureSession alloc] init];

960
    if (video_device && add_video_device(s, video_device)) {
961 962
        goto fail;
    }
963 964
    if (audio_device && add_audio_device(s, audio_device)) {
    }
965 966 967

    [ctx->capture_session startRunning];

968 969 970 971 972 973
    /* Unlock device configuration only after the session is started so it
     * does not reset the capture formats */
    if (!capture_screen) {
        [video_device unlockForConfiguration];
    }

974 975 976 977 978 979
    if (video_device && get_video_config(s)) {
        goto fail;
    }

    // set audio stream
    if (audio_device && get_audio_config(s)) {
980 981 982
        goto fail;
    }

983 984 985 986 987 988 989 990 991
    [pool release];
    return 0;

fail:
    [pool release];
    destroy_context(ctx);
    return AVERROR(EIO);
}

992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007
static int copy_cvpixelbuffer(AVFormatContext *s,
                               CVPixelBufferRef image_buffer,
                               AVPacket *pkt)
{
    AVFContext *ctx = s->priv_data;
    int src_linesize[4];
    const uint8_t *src_data[4];
    int width  = CVPixelBufferGetWidth(image_buffer);
    int height = CVPixelBufferGetHeight(image_buffer);
    int status;

    memset(src_linesize, 0, sizeof(src_linesize));
    memset(src_data, 0, sizeof(src_data));

    status = CVPixelBufferLockBaseAddress(image_buffer, 0);
    if (status != kCVReturnSuccess) {
1008
        av_log(s, AV_LOG_ERROR, "Could not lock base address: %d (%dx%d)\n", status, width, height);
1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
        return AVERROR_EXTERNAL;
    }

    if (CVPixelBufferIsPlanar(image_buffer)) {
        size_t plane_count = CVPixelBufferGetPlaneCount(image_buffer);
        int i;
        for(i = 0; i < plane_count; i++){
            src_linesize[i] = CVPixelBufferGetBytesPerRowOfPlane(image_buffer, i);
            src_data[i] = CVPixelBufferGetBaseAddressOfPlane(image_buffer, i);
        }
    } else {
        src_linesize[0] = CVPixelBufferGetBytesPerRow(image_buffer);
        src_data[0] = CVPixelBufferGetBaseAddress(image_buffer);
    }

    status = av_image_copy_to_buffer(pkt->data, pkt->size,
                                     src_data, src_linesize,
                                     ctx->pixel_format, width, height, 1);



    CVPixelBufferUnlockBaseAddress(image_buffer, 0);

    return status;
}

1035 1036 1037 1038 1039
static int avf_read_packet(AVFormatContext *s, AVPacket *pkt)
{
    AVFContext* ctx = (AVFContext*)s->priv_data;

    do {
1040
        CVImageBufferRef image_buffer;
1041
        CMBlockBufferRef block_buffer;
1042 1043 1044
        lock_frames(ctx);

        if (ctx->current_frame != nil) {
1045
            int status;
1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059
            int length = 0;

            image_buffer = CMSampleBufferGetImageBuffer(ctx->current_frame);
            block_buffer = CMSampleBufferGetDataBuffer(ctx->current_frame);

            if (image_buffer != nil) {
                length = (int)CVPixelBufferGetDataSize(image_buffer);
            } else if (block_buffer != nil) {
                length = (int)CMBlockBufferGetDataLength(block_buffer);
            } else  {
                return AVERROR(EINVAL);
            }

            if (av_new_packet(pkt, length) < 0) {
1060 1061 1062
                return AVERROR(EIO);
            }

1063 1064 1065 1066 1067 1068 1069 1070
            CMItemCount count;
            CMSampleTimingInfo timing_info;

            if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_frame, 1, &timing_info, &count) == noErr) {
                AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
                pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
            }

1071
            pkt->stream_index  = ctx->video_stream_index;
1072 1073
            pkt->flags        |= AV_PKT_FLAG_KEY;

1074
            if (image_buffer) {
1075
                status = copy_cvpixelbuffer(s, image_buffer, pkt);
1076 1077 1078 1079 1080 1081 1082
            } else {
                status = 0;
                OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
                if (ret != kCMBlockBufferNoErr) {
                    status = AVERROR(EIO);
                }
             }
1083 1084
            CFRelease(ctx->current_frame);
            ctx->current_frame = nil;
1085 1086 1087

            if (status < 0)
                return status;
1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103
        } else if (ctx->current_audio_frame != nil) {
            CMBlockBufferRef block_buffer = CMSampleBufferGetDataBuffer(ctx->current_audio_frame);
            int block_buffer_size         = CMBlockBufferGetDataLength(block_buffer);

            if (!block_buffer || !block_buffer_size) {
                return AVERROR(EIO);
            }

            if (ctx->audio_non_interleaved && block_buffer_size > ctx->audio_buffer_size) {
                return AVERROR_BUFFER_TOO_SMALL;
            }

            if (av_new_packet(pkt, block_buffer_size) < 0) {
                return AVERROR(EIO);
            }

1104 1105 1106 1107 1108 1109 1110
            CMItemCount count;
            CMSampleTimingInfo timing_info;

            if (CMSampleBufferGetOutputSampleTimingInfoArray(ctx->current_audio_frame, 1, &timing_info, &count) == noErr) {
                AVRational timebase_q = av_make_q(1, timing_info.presentationTimeStamp.timescale);
                pkt->pts = pkt->dts = av_rescale_q(timing_info.presentationTimeStamp.value, timebase_q, avf_time_base_q);
            }
1111 1112 1113 1114 1115

            pkt->stream_index  = ctx->audio_stream_index;
            pkt->flags        |= AV_PKT_FLAG_KEY;

            if (ctx->audio_non_interleaved) {
1116
                int sample, c, shift, num_samples;
1117 1118 1119 1120 1121 1122

                OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, ctx->audio_buffer);
                if (ret != kCMBlockBufferNoErr) {
                    return AVERROR(EIO);
                }

1123
                num_samples = pkt->size / (ctx->audio_channels * (ctx->audio_bits_per_sample >> 3));
1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156

                // transform decoded frame into output format
                #define INTERLEAVE_OUTPUT(bps)                                         \
                {                                                                      \
                    int##bps##_t **src;                                                \
                    int##bps##_t *dest;                                                \
                    src = av_malloc(ctx->audio_channels * sizeof(int##bps##_t*));      \
                    if (!src) return AVERROR(EIO);                                     \
                    for (c = 0; c < ctx->audio_channels; c++) {                        \
                        src[c] = ((int##bps##_t*)ctx->audio_buffer) + c * num_samples; \
                    }                                                                  \
                    dest  = (int##bps##_t*)pkt->data;                                  \
                    shift = bps - ctx->audio_bits_per_sample;                          \
                    for (sample = 0; sample < num_samples; sample++)                   \
                        for (c = 0; c < ctx->audio_channels; c++)                      \
                            *dest++ = src[c][sample] << shift;                         \
                    av_freep(&src);                                                    \
                }

                if (ctx->audio_bits_per_sample <= 16) {
                    INTERLEAVE_OUTPUT(16)
                } else {
                    INTERLEAVE_OUTPUT(32)
                }
            } else {
                OSStatus ret = CMBlockBufferCopyDataBytes(block_buffer, 0, pkt->size, pkt->data);
                if (ret != kCMBlockBufferNoErr) {
                    return AVERROR(EIO);
                }
            }

            CFRelease(ctx->current_audio_frame);
            ctx->current_audio_frame = nil;
1157 1158
        } else {
            pkt->data = NULL;
1159 1160 1161 1162 1163 1164
            unlock_frames(ctx);
            if (ctx->observed_quit) {
                return AVERROR_EOF;
            } else {
                return AVERROR(EAGAIN);
            }
1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180
        }

        unlock_frames(ctx);
    } while (!pkt->data);

    return 0;
}

static int avf_close(AVFormatContext *s)
{
    AVFContext* ctx = (AVFContext*)s->priv_data;
    destroy_context(ctx);
    return 0;
}

static const AVOption options[] = {
1181
    { "list_devices", "list available devices", offsetof(AVFContext, list_devices), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1182
    { "video_device_index", "select video device by index for devices with same name (starts at 0)", offsetof(AVFContext, video_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1183
    { "audio_device_index", "select audio device by index for devices with same name (starts at 0)", offsetof(AVFContext, audio_device_index), AV_OPT_TYPE_INT, {.i64 = -1}, -1, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1184
    { "pixel_format", "set pixel format", offsetof(AVFContext, pixel_format), AV_OPT_TYPE_PIXEL_FMT, {.i64 = AV_PIX_FMT_YUV420P}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM},
1185
    { "framerate", "set frame rate", offsetof(AVFContext, framerate), AV_OPT_TYPE_VIDEO_RATE, {.str = "ntsc"}, 0, INT_MAX, AV_OPT_FLAG_DECODING_PARAM },
1186
    { "video_size", "set video size", offsetof(AVFContext, width), AV_OPT_TYPE_IMAGE_SIZE, {.str = NULL}, 0, 0, AV_OPT_FLAG_DECODING_PARAM },
1187 1188
    { "capture_cursor", "capture the screen cursor", offsetof(AVFContext, capture_cursor), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
    { "capture_mouse_clicks", "capture the screen mouse clicks", offsetof(AVFContext, capture_mouse_clicks), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1189
    { "capture_raw_data", "capture the raw data from device connection", offsetof(AVFContext, capture_raw_data), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1190
    { "drop_late_frames", "drop frames that are available later than expected", offsetof(AVFContext, drop_late_frames), AV_OPT_TYPE_BOOL, {.i64=1}, 0, 1, AV_OPT_FLAG_DECODING_PARAM },
1191

1192 1193 1194 1195
    { NULL },
};

static const AVClass avf_class = {
1196
    .class_name = "AVFoundation indev",
1197 1198 1199
    .item_name  = av_default_item_name,
    .option     = options,
    .version    = LIBAVUTIL_VERSION_INT,
1200
    .category   = AV_CLASS_CATEGORY_DEVICE_VIDEO_INPUT,
1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212
};

AVInputFormat ff_avfoundation_demuxer = {
    .name           = "avfoundation",
    .long_name      = NULL_IF_CONFIG_SMALL("AVFoundation input device"),
    .priv_data_size = sizeof(AVFContext),
    .read_header    = avf_read_header,
    .read_packet    = avf_read_packet,
    .read_close     = avf_close,
    .flags          = AVFMT_NOFILE,
    .priv_class     = &avf_class,
};