mediacodecdec.c 17.1 KB
Newer Older
1
/*
2
 * Android MediaCodec H.264 / H.265 / MPEG-4 / VP8 / VP9 decoders
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
 *
 * Copyright (c) 2015-2016 Matthieu Bouron <matthieu.bouron stupeflix.com>
 *
 * This file is part of FFmpeg.
 *
 * FFmpeg is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * FFmpeg is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with FFmpeg; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

23
#include <stdint.h>
24 25
#include <string.h>

26
#include "libavutil/avassert.h"
27
#include "libavutil/common.h"
28 29 30
#include "libavutil/fifo.h"
#include "libavutil/opt.h"
#include "libavutil/intreadwrite.h"
31
#include "libavutil/pixfmt.h"
32
#include "libavutil/atomic.h"
33 34

#include "avcodec.h"
35 36
#include "h264_parse.h"
#include "hevc_parse.h"
37 38
#include "internal.h"
#include "mediacodec_wrapper.h"
39
#include "mediacodecdec_common.h"
40

41
typedef struct MediaCodecH264DecContext {
42

43
    MediaCodecDecContext *ctx;
44

45
    AVBSFContext *bsf;
46

47
    AVFifoBuffer *fifo;
48

49
    AVPacket filtered_pkt;
50

51
} MediaCodecH264DecContext;
52

53
static av_cold int mediacodec_decode_close(AVCodecContext *avctx)
54
{
55
    MediaCodecH264DecContext *s = avctx->priv_data;
56

57 58
    ff_mediacodec_dec_close(avctx, s->ctx);
    s->ctx = NULL;
59

60
    av_fifo_free(s->fifo);
61

62 63
    av_bsf_free(&s->bsf);
    av_packet_unref(&s->filtered_pkt);
64

65
    return 0;
66 67
}

68 69
#if CONFIG_H264_MEDIACODEC_DECODER || CONFIG_HEVC_MEDIACODEC_DECODER
static int h2645_ps_to_nalu(const uint8_t *src, int src_size, uint8_t **out, int *out_size)
70
{
71 72 73 74
    int i;
    int ret = 0;
    uint8_t *p = NULL;
    static const uint8_t nalu_header[] = { 0x00, 0x00, 0x00, 0x01 };
75

76 77 78
    if (!out || !out_size) {
        return AVERROR(EINVAL);
    }
79

80 81 82 83
    p = av_malloc(sizeof(nalu_header) + src_size);
    if (!p) {
        return AVERROR(ENOMEM);
    }
84

85 86
    *out = p;
    *out_size = sizeof(nalu_header) + src_size;
87

88 89
    memcpy(p, nalu_header, sizeof(nalu_header));
    memcpy(p + sizeof(nalu_header), src, src_size);
90

91 92 93 94 95 96 97
    /* Escape 0x00, 0x00, 0x0{0-3} pattern */
    for (i = 4; i < *out_size; i++) {
        if (i < *out_size - 3 &&
            p[i + 0] == 0 &&
            p[i + 1] == 0 &&
            p[i + 2] <= 3) {
            uint8_t *new;
98

99 100 101 102 103 104 105
            *out_size += 1;
            new = av_realloc(*out, *out_size);
            if (!new) {
                ret = AVERROR(ENOMEM);
                goto done;
            }
            *out = p = new;
106

107 108 109 110 111 112 113 114 115
            i = i + 2;
            memmove(p + i + 1, p + i, *out_size - (i + 1));
            p[i] = 0x03;
        }
    }
done:
    if (ret < 0) {
        av_freep(out);
        *out_size = 0;
116 117
    }

118
    return ret;
119
}
120
#endif
121

122 123 124 125 126
#if CONFIG_H264_MEDIACODEC_DECODER
static int h264_set_extradata(AVCodecContext *avctx, FFAMediaFormat *format)
{
    int i;
    int ret;
127

128 129 130 131 132
    H264ParamSets ps;
    const PPS *pps = NULL;
    const SPS *sps = NULL;
    int is_avc = 0;
    int nal_length_size = 0;
133

134
    memset(&ps, 0, sizeof(ps));
135

136 137 138 139
    ret = ff_h264_decode_extradata(avctx->extradata, avctx->extradata_size,
                                   &ps, &is_avc, &nal_length_size, 0, avctx);
    if (ret < 0) {
        goto done;
140 141
    }

142 143 144 145 146
    for (i = 0; i < MAX_PPS_COUNT; i++) {
        if (ps.pps_list[i]) {
            pps = (const PPS*)ps.pps_list[i]->data;
            break;
        }
147 148
    }

149 150 151 152 153
    if (pps) {
        if (ps.sps_list[pps->sps_id]) {
            sps = (const SPS*)ps.sps_list[pps->sps_id]->data;
        }
    }
154

155 156 157
    if (pps && sps) {
        uint8_t *data = NULL;
        int data_size = 0;
158

159 160 161 162 163
        if ((ret = h2645_ps_to_nalu(sps->data, sps->data_size, &data, &data_size)) < 0) {
            goto done;
        }
        ff_AMediaFormat_setBuffer(format, "csd-0", (void*)data, data_size);
        av_freep(&data);
164

165 166 167 168 169 170 171 172
        if ((ret = h2645_ps_to_nalu(pps->data, pps->data_size, &data, &data_size)) < 0) {
            goto done;
        }
        ff_AMediaFormat_setBuffer(format, "csd-1", (void*)data, data_size);
        av_freep(&data);
    } else {
        av_log(avctx, AV_LOG_ERROR, "Could not extract PPS/SPS from extradata");
        ret = AVERROR_INVALIDDATA;
173 174 175
    }

done:
176
    ff_h264_ps_uninit(&ps);
177 178 179

    return ret;
}
180
#endif
181

182 183
#if CONFIG_HEVC_MEDIACODEC_DECODER
static int hevc_set_extradata(AVCodecContext *avctx, FFAMediaFormat *format)
184
{
185 186
    int i;
    int ret;
187

188
    HEVCParamSets ps;
189

190 191 192 193 194
    const HEVCVPS *vps = NULL;
    const HEVCPPS *pps = NULL;
    const HEVCSPS *sps = NULL;
    int is_nalff = 0;
    int nal_length_size = 0;
195

196 197 198 199 200 201
    uint8_t *vps_data = NULL;
    uint8_t *sps_data = NULL;
    uint8_t *pps_data = NULL;
    int vps_data_size = 0;
    int sps_data_size = 0;
    int pps_data_size = 0;
202

203
    memset(&ps, 0, sizeof(ps));
204

205 206 207 208
    ret = ff_hevc_decode_extradata(avctx->extradata, avctx->extradata_size,
                                &ps, &is_nalff, &nal_length_size, 0, avctx);
    if (ret < 0) {
        goto done;
209 210
    }

211 212 213 214 215
    for (i = 0; i < MAX_VPS_COUNT; i++) {
        if (ps.vps_list[i]) {
            vps = (const HEVCVPS*)ps.vps_list[i]->data;
            break;
        }
216 217
    }

218 219 220 221 222
    for (i = 0; i < MAX_PPS_COUNT; i++) {
        if (ps.pps_list[i]) {
            pps = (const HEVCPPS*)ps.pps_list[i]->data;
            break;
        }
223 224
    }

225 226 227 228
    if (pps) {
        if (ps.sps_list[pps->sps_id]) {
            sps = (const HEVCSPS*)ps.sps_list[pps->sps_id]->data;
        }
229 230
    }

231 232 233
    if (vps && pps && sps) {
        uint8_t *data;
        int data_size;
234

235 236 237 238 239
        if ((ret = h2645_ps_to_nalu(vps->data, vps->data_size, &vps_data, &vps_data_size)) < 0 ||
            (ret = h2645_ps_to_nalu(sps->data, sps->data_size, &sps_data, &sps_data_size)) < 0 ||
            (ret = h2645_ps_to_nalu(pps->data, pps->data_size, &pps_data, &pps_data_size)) < 0) {
            goto done;
        }
240

241 242 243 244 245 246
        data_size = vps_data_size + sps_data_size + pps_data_size;
        data = av_mallocz(data_size);
        if (!data) {
            ret = AVERROR(ENOMEM);
            goto done;
        }
247

248 249 250
        memcpy(data                                , vps_data, vps_data_size);
        memcpy(data + vps_data_size                , sps_data, sps_data_size);
        memcpy(data + vps_data_size + sps_data_size, pps_data, pps_data_size);
251

252
        ff_AMediaFormat_setBuffer(format, "csd-0", data, data_size);
253

254 255 256 257 258
        av_freep(&data);
    } else {
        av_log(avctx, AV_LOG_ERROR, "Could not extract VPS/PPS/SPS from extradata");
        ret = AVERROR_INVALIDDATA;
    }
259

260 261 262 263
done:
    av_freep(&vps_data);
    av_freep(&sps_data);
    av_freep(&pps_data);
264

265 266 267
    return ret;
}
#endif
268

269 270
#if CONFIG_MPEG4_MEDIACODEC_DECODER
static int mpeg4_set_extradata(AVCodecContext *avctx, FFAMediaFormat *format)
271
{
272
    int ret = 0;
273

274 275
    if (avctx->extradata) {
        ff_AMediaFormat_setBuffer(format, "csd-0", avctx->extradata, avctx->extradata_size);
276 277
    }

278
    return ret;
279
}
280
#endif
281

282 283
#if CONFIG_VP8_MEDIACODEC_DECODER || CONFIG_VP9_MEDIACODEC_DECODER
static int vpx_set_extradata(AVCodecContext *avctx, FFAMediaFormat *format)
284 285 286
{
    int ret = 0;

287 288 289
    if (avctx->extradata) {
        ff_AMediaFormat_setBuffer(format, "csd-0", avctx->extradata, avctx->extradata_size);
    }
290

291 292 293
    return ret;
}
#endif
294

295 296 297
static av_cold int mediacodec_decode_init(AVCodecContext *avctx)
{
    int ret;
298

299
    const char *codec_mime = NULL;
300

301 302
    const char *bsf_name = NULL;
    const AVBitStreamFilter *bsf = NULL;
303

304 305
    FFAMediaFormat *format = NULL;
    MediaCodecH264DecContext *s = avctx->priv_data;
306

307 308 309
    format = ff_AMediaFormat_new();
    if (!format) {
        av_log(avctx, AV_LOG_ERROR, "Failed to create media format\n");
310
        ret = AVERROR_EXTERNAL;
311
        goto done;
312 313
    }

314 315 316 317 318
    switch (avctx->codec_id) {
#if CONFIG_H264_MEDIACODEC_DECODER
    case AV_CODEC_ID_H264:
        codec_mime = "video/avc";
        bsf_name = "h264_mp4toannexb";
319

320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337
        ret = h264_set_extradata(avctx, format);
        if (ret < 0)
            goto done;
        break;
#endif
#if CONFIG_HEVC_MEDIACODEC_DECODER
    case AV_CODEC_ID_HEVC:
        codec_mime = "video/hevc";
        bsf_name = "hevc_mp4toannexb";

        ret = hevc_set_extradata(avctx, format);
        if (ret < 0)
            goto done;
        break;
#endif
#if CONFIG_MPEG4_MEDIACODEC_DECODER
    case AV_CODEC_ID_MPEG4:
        codec_mime = "video/mp4v-es",
338

339 340 341 342 343 344 345 346
        ret = mpeg4_set_extradata(avctx, format);
        if (ret < 0)
            goto done;
        break;
#endif
#if CONFIG_VP8_MEDIACODEC_DECODER
    case AV_CODEC_ID_VP8:
        codec_mime = "video/x-vnd.on2.vp8";
347

348 349 350 351 352 353 354 355
        ret = vpx_set_extradata(avctx, format);
        if (ret < 0)
            goto done;
        break;
#endif
#if CONFIG_VP9_MEDIACODEC_DECODER
    case AV_CODEC_ID_VP9:
        codec_mime = "video/x-vnd.on2.vp9";
356

357 358 359 360 361 362 363 364
        ret = vpx_set_extradata(avctx, format);
        if (ret < 0)
            goto done;
        break;
#endif
    default:
        av_assert0(0);
    }
365

366 367 368
    ff_AMediaFormat_setString(format, "mime", codec_mime);
    ff_AMediaFormat_setInt32(format, "width", avctx->width);
    ff_AMediaFormat_setInt32(format, "height", avctx->height);
369

370 371 372 373 374 375
    s->ctx = av_mallocz(sizeof(*s->ctx));
    if (!s->ctx) {
        av_log(avctx, AV_LOG_ERROR, "Failed to allocate MediaCodecDecContext\n");
        ret = AVERROR(ENOMEM);
        goto done;
    }
376

377 378 379
    if ((ret = ff_mediacodec_dec_init(avctx, s->ctx, codec_mime, format)) < 0) {
        s->ctx = NULL;
        goto done;
380 381
    }

382 383 384 385 386 387
    av_log(avctx, AV_LOG_INFO, "MediaCodec started successfully, ret = %d\n", ret);

    s->fifo = av_fifo_alloc(sizeof(AVPacket));
    if (!s->fifo) {
        ret = AVERROR(ENOMEM);
        goto done;
388 389
    }

390 391 392 393 394
    if (bsf_name) {
    bsf = av_bsf_get_by_name(bsf_name);
    if(!bsf) {
        ret = AVERROR_BSF_NOT_FOUND;
        goto done;
395 396
    }

397 398 399
    if ((ret = av_bsf_alloc(bsf, &s->bsf))) {
        goto done;
    }
400

401 402 403 404 405
    if (((ret = avcodec_parameters_from_context(s->bsf->par_in, avctx)) < 0) ||
        ((ret = av_bsf_init(s->bsf)) < 0)) {
          goto done;
    }
    }
406

407
    av_init_packet(&s->filtered_pkt);
408

409 410 411 412
done:
    if (format) {
        ff_AMediaFormat_delete(format);
    }
413

414 415 416
    if (ret < 0) {
        mediacodec_decode_close(avctx);
    }
417

418 419
    return ret;
}
420

421

422 423 424 425
static int mediacodec_process_data(AVCodecContext *avctx, AVFrame *frame,
                                   int *got_frame, AVPacket *pkt)
{
    MediaCodecH264DecContext *s = avctx->priv_data;
426

427 428
    return ff_mediacodec_dec_decode(avctx, s->ctx, frame, got_frame, pkt);
}
429

430 431 432 433 434 435
static int mediacodec_decode_frame(AVCodecContext *avctx, void *data,
                                   int *got_frame, AVPacket *avpkt)
{
    MediaCodecH264DecContext *s = avctx->priv_data;
    AVFrame *frame    = data;
    int ret;
436

437 438 439
    /* buffer the input packet */
    if (avpkt->size) {
        AVPacket input_pkt = { 0 };
440

441 442 443 444 445 446
        if (av_fifo_space(s->fifo) < sizeof(input_pkt)) {
            ret = av_fifo_realloc2(s->fifo,
                                   av_fifo_size(s->fifo) + sizeof(input_pkt));
            if (ret < 0)
                return ret;
        }
447

448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476
        ret = av_packet_ref(&input_pkt, avpkt);
        if (ret < 0)
            return ret;
        av_fifo_generic_write(s->fifo, &input_pkt, sizeof(input_pkt), NULL);
    }

    /*
     * MediaCodec.flush() discards both input and output buffers, thus we
     * need to delay the call to this function until the user has released or
     * renderered the frames he retains.
     *
     * After we have buffered an input packet, check if the codec is in the
     * flushing state. If it is, we need to call ff_mediacodec_dec_flush.
     *
     * ff_mediacodec_dec_flush returns 0 if the flush cannot be performed on
     * the codec (because the user retains frames). The codec stays in the
     * flushing state.
     *
     * ff_mediacodec_dec_flush returns 1 if the flush can actually be
     * performed on the codec. The codec leaves the flushing state and can
     * process again packets.
     *
     * ff_mediacodec_dec_flush returns a negative value if an error has
     * occurred.
     *
     */
    if (ff_mediacodec_dec_is_flushing(avctx, s->ctx)) {
        if (!ff_mediacodec_dec_flush(avctx, s->ctx)) {
            return avpkt->size;
477 478 479
        }
    }

480 481 482 483 484
    /* process buffered data */
    while (!*got_frame) {
        /* prepare the input data -- convert to Annex B if needed */
        if (s->filtered_pkt.size <= 0) {
            AVPacket input_pkt = { 0 };
485

486
            av_packet_unref(&s->filtered_pkt);
487

488 489 490 491 492
            /* no more data */
            if (av_fifo_size(s->fifo) < sizeof(AVPacket)) {
                return avpkt->size ? avpkt->size :
                    ff_mediacodec_dec_decode(avctx, s->ctx, frame, got_frame, avpkt);
            }
493

494
            av_fifo_generic_read(s->fifo, &input_pkt, sizeof(input_pkt), NULL);
495

496 497 498 499
            if (s->bsf) {
            ret = av_bsf_send_packet(s->bsf, &input_pkt);
            if (ret < 0) {
                return ret;
500
            }
501

502 503 504
            ret = av_bsf_receive_packet(s->bsf, &s->filtered_pkt);
            if (ret == AVERROR(EAGAIN)) {
                goto done;
505
            }
506 507
            } else {
                av_packet_move_ref(&s->filtered_pkt, &input_pkt);
508 509
            }

510 511
            /* {h264,hevc}_mp4toannexb are used here and do not require flushing */
            av_assert0(ret != AVERROR_EOF);
512

513 514 515
            if (ret < 0) {
                return ret;
            }
516 517
        }

518 519
        ret = mediacodec_process_data(avctx, frame, got_frame, &s->filtered_pkt);
        if (ret < 0)
520 521
            return ret;

522 523
        s->filtered_pkt.size -= ret;
        s->filtered_pkt.data += ret;
524
    }
525 526
done:
    return avpkt->size;
527 528
}

529
static void mediacodec_decode_flush(AVCodecContext *avctx)
530
{
531
    MediaCodecH264DecContext *s = avctx->priv_data;
532

533 534 535 536
    while (av_fifo_size(s->fifo)) {
        AVPacket pkt;
        av_fifo_generic_read(s->fifo, &pkt, sizeof(pkt), NULL);
        av_packet_unref(&pkt);
537
    }
538
    av_fifo_reset(s->fifo);
539

540
    av_packet_unref(&s->filtered_pkt);
541

542
    ff_mediacodec_dec_flush(avctx, s->ctx);
543 544
}

545 546 547 548 549 550 551 552 553 554 555 556 557
#if CONFIG_H264_MEDIACODEC_DECODER
AVCodec ff_h264_mediacodec_decoder = {
    .name           = "h264_mediacodec",
    .long_name      = NULL_IF_CONFIG_SMALL("H.264 Android MediaCodec decoder"),
    .type           = AVMEDIA_TYPE_VIDEO,
    .id             = AV_CODEC_ID_H264,
    .priv_data_size = sizeof(MediaCodecH264DecContext),
    .init           = mediacodec_decode_init,
    .decode         = mediacodec_decode_frame,
    .flush          = mediacodec_decode_flush,
    .close          = mediacodec_decode_close,
    .capabilities   = CODEC_CAP_DELAY,
    .caps_internal  = FF_CODEC_CAP_SETS_PKT_DTS,
558
};
559
#endif
560

561 562 563 564 565 566 567 568 569 570 571 572 573
#if CONFIG_HEVC_MEDIACODEC_DECODER
AVCodec ff_hevc_mediacodec_decoder = {
    .name           = "hevc_mediacodec",
    .long_name      = NULL_IF_CONFIG_SMALL("H.265 Android MediaCodec decoder"),
    .type           = AVMEDIA_TYPE_VIDEO,
    .id             = AV_CODEC_ID_HEVC,
    .priv_data_size = sizeof(MediaCodecH264DecContext),
    .init           = mediacodec_decode_init,
    .decode         = mediacodec_decode_frame,
    .flush          = mediacodec_decode_flush,
    .close          = mediacodec_decode_close,
    .capabilities   = CODEC_CAP_DELAY,
    .caps_internal  = FF_CODEC_CAP_SETS_PKT_DTS,
574
};
575
#endif
576

577 578 579 580 581 582 583 584 585 586 587 588 589
#if CONFIG_MPEG4_MEDIACODEC_DECODER
AVCodec ff_mpeg4_mediacodec_decoder = {
    .name           = "mpeg4_mediacodec",
    .long_name      = NULL_IF_CONFIG_SMALL("MPEG-4 Android MediaCodec decoder"),
    .type           = AVMEDIA_TYPE_VIDEO,
    .id             = AV_CODEC_ID_MPEG4,
    .priv_data_size = sizeof(MediaCodecH264DecContext),
    .init           = mediacodec_decode_init,
    .decode         = mediacodec_decode_frame,
    .flush          = mediacodec_decode_flush,
    .close          = mediacodec_decode_close,
    .capabilities   = CODEC_CAP_DELAY,
    .caps_internal  = FF_CODEC_CAP_SETS_PKT_DTS,
590
};
591
#endif
592

593 594 595 596 597 598 599 600 601 602 603 604 605
#if CONFIG_VP8_MEDIACODEC_DECODER
AVCodec ff_vp8_mediacodec_decoder = {
    .name           = "vp8_mediacodec",
    .long_name      = NULL_IF_CONFIG_SMALL("VP8 Android MediaCodec decoder"),
    .type           = AVMEDIA_TYPE_VIDEO,
    .id             = AV_CODEC_ID_VP8,
    .priv_data_size = sizeof(MediaCodecH264DecContext),
    .init           = mediacodec_decode_init,
    .decode         = mediacodec_decode_frame,
    .flush          = mediacodec_decode_flush,
    .close          = mediacodec_decode_close,
    .capabilities   = CODEC_CAP_DELAY,
    .caps_internal  = FF_CODEC_CAP_SETS_PKT_DTS,
606
};
607
#endif
608

609 610 611 612 613 614 615 616 617 618 619 620 621
#if CONFIG_VP9_MEDIACODEC_DECODER
AVCodec ff_vp9_mediacodec_decoder = {
    .name           = "vp9_mediacodec",
    .long_name      = NULL_IF_CONFIG_SMALL("VP9 Android MediaCodec decoder"),
    .type           = AVMEDIA_TYPE_VIDEO,
    .id             = AV_CODEC_ID_VP9,
    .priv_data_size = sizeof(MediaCodecH264DecContext),
    .init           = mediacodec_decode_init,
    .decode         = mediacodec_decode_frame,
    .flush          = mediacodec_decode_flush,
    .close          = mediacodec_decode_close,
    .capabilities   = CODEC_CAP_DELAY,
    .caps_internal  = FF_CODEC_CAP_SETS_PKT_DTS,
622
};
623
#endif