avcodec/libvpxdec: Adds decode support for formats other than 420

Handles decoding of new VP9 profiles 1-3 with different color sampling
and bit-depths.

For high bitdepth (profiles 2 and 3) support, we currently need to link
with the highbitdepth branch of libvpx with --enable-experimental
and --enable-vp9-high config options on. But eventually this branch will
be merged into master, whereafter to enable high bitdepth
support you will need to link with libvpx with configure option
--enable-vp9-highbitdepth on.
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/libvpxdec.c

@@ -60,6 +60,57 @@ static av_cold int vpx_init(AVCodecContext *avctx,
     return 0;
 }
 
+// returns 0 on success, AVERROR_INVALIDDATA otherwise
+static int set_pix_fmt(AVCodecContext *avctx, struct vpx_image *img) {
+    if (avctx->codec_id == AV_CODEC_ID_VP8 && img->fmt != VPX_IMG_FMT_I420)
+        return AVERROR_INVALIDDATA;
+    switch (img->fmt) {
+        case VPX_IMG_FMT_I420:
+            avctx->pix_fmt = AV_PIX_FMT_YUV420P;
+            return 0;
+        case VPX_IMG_FMT_I422:
+            avctx->pix_fmt = AV_PIX_FMT_YUV422P;
+            return 0;
+        case VPX_IMG_FMT_I444:
+            avctx->pix_fmt = AV_PIX_FMT_YUV444P;
+            return 0;
+#ifdef VPX_IMG_FMT_HIGHBITDEPTH
+        case VPX_IMG_FMT_I42016:
+            if (img->bit_depth == 10) {
+                avctx->pix_fmt = AV_PIX_FMT_YUV420P10LE;
+                return 0;
+            } else if (img->bit_depth == 12) {
+                avctx->pix_fmt = AV_PIX_FMT_YUV420P12LE;
+                return 0;
+            } else {
+                return AVERROR_INVALIDDATA;
+            }
+        case VPX_IMG_FMT_I42216:
+            if (img->bit_depth == 10) {
+                avctx->pix_fmt = AV_PIX_FMT_YUV422P10LE;
+                return 0;
+            } else if (img->bit_depth == 12) {
+                avctx->pix_fmt = AV_PIX_FMT_YUV422P12LE;
+                return 0;
+            } else {
+                return AVERROR_INVALIDDATA;
+            }
+        case VPX_IMG_FMT_I44416:
+            if (img->bit_depth == 10) {
+                avctx->pix_fmt = AV_PIX_FMT_YUV444P10LE;
+                return 0;
+            } else if (img->bit_depth == 12) {
+                avctx->pix_fmt = AV_PIX_FMT_YUV444P12LE;
+                return 0;
+            } else {
+                return AVERROR_INVALIDDATA;
+            }
+#endif
+        default:
+            return AVERROR_INVALIDDATA;
+    }
+}
+
 static int vp8_decode(AVCodecContext *avctx,
                       void *data, int *got_frame, AVPacket *avpkt)
 {
@@ -82,10 +133,15 @@ static int vp8_decode(AVCodecContext *avctx,
     }
 
     if ((img = vpx_codec_get_frame(&ctx->decoder, &iter))) {
-        if (img->fmt != VPX_IMG_FMT_I420) {
-            av_log(avctx, AV_LOG_ERROR, "Unsupported output colorspace (%d)\n",
-                   img->fmt);
-            return AVERROR_INVALIDDATA;
+        if ((ret = set_pix_fmt(avctx, img)) < 0) {
+#ifdef VPX_IMG_FMT_HIGHBITDEPTH
+            av_log(avctx, AV_LOG_ERROR, "Unsupported output colorspace (%d) / bit_depth (%d)\n",
+                   img->fmt, img->bit_depth);
+#else
+            av_log(avctx, AV_LOG_ERROR, "Unsupported output colorspace (%d) / bit_depth (%d)\n",
+                   img->fmt, 8);
+#endif
+            return ret;
         }
 
         if ((int) img->d_w != avctx->width || (int) img->d_h != avctx->height) {