hwaccel: Call ->get_format again if hwaccel init fails

This allows the application to fall back on another hwaccel or,
more likely, software decoding.
Signed-off-by: Luca Barbato <lu_zero@gentoo.org>

libavcodec/avcodec.h

@@ -1294,6 +1294,10 @@ typedef struct AVCodecContext {
      * @param fmt is the list of formats which are supported by the codec,
      * it is terminated by -1 as 0 is a valid format, the formats are ordered by quality.
      * The first is always the native one.
+     * @note The callback may be called again immediately if initialization for
+     * the selected (hardware-accelerated) pixel format failed.
+     * @warning Behavior is undefined if the callback returns a value not
+     * in the fmt list of formats.
      * @return the chosen format
      * - encoding: unused
      * - decoding: Set by user, if not set the native format will be chosen.

libavcodec/utils.c

@@ -898,49 +898,85 @@ static AVHWAccel *find_hwaccel(enum AVCodecID codec_id,
     return NULL;
 }
 
+static int setup_hwaccel(AVCodecContext *avctx,
+                         const enum AVPixelFormat fmt,
+                         const char *name)
+{
+    AVHWAccel *hwa = find_hwaccel(avctx->codec_id, fmt);
+    int ret        = 0;
+
+    if (!hwa) {
+        av_log(avctx, AV_LOG_ERROR,
+               "Could not find an AVHWAccel for the pixel format: %s",
+               name);
+        return AVERROR(ENOENT);
+    }
+
+    if (hwa->priv_data_size) {
+        avctx->internal->hwaccel_priv_data = av_mallocz(hwa->priv_data_size);
+        if (!avctx->internal->hwaccel_priv_data)
+            return AVERROR(ENOMEM);
+    }
+
+    if (hwa->init) {
+        ret = hwa->init(avctx);
+        if (ret < 0) {
+            av_freep(&avctx->internal->hwaccel_priv_data);
+            return ret;
+        }
+    }
+
+    avctx->hwaccel = hwa;
+
+    return 0;
+}
 
 int ff_get_format(AVCodecContext *avctx, const enum AVPixelFormat *fmt)
 {
     const AVPixFmtDescriptor *desc;
-    enum AVPixelFormat ret = avctx->get_format(avctx, fmt);
+    enum AVPixelFormat *choices;
+    enum AVPixelFormat ret;
+    unsigned n = 0;
 
-    desc = av_pix_fmt_desc_get(ret);
-    if (!desc)
+    while (fmt[n] != AV_PIX_FMT_NONE)
+        ++n;
+
+    choices = av_malloc_array(n + 1, sizeof(*choices));
+    if (!choices)
         return AV_PIX_FMT_NONE;
 
-    if (avctx->hwaccel && avctx->hwaccel->uninit)
-        avctx->hwaccel->uninit(avctx);
-    av_freep(&avctx->internal->hwaccel_priv_data);
-    avctx->hwaccel = NULL;
-
-    if (desc->flags & AV_PIX_FMT_FLAG_HWACCEL) {
-        AVHWAccel *hwaccel;
-        int err;
-
-        hwaccel = find_hwaccel(avctx->codec_id, ret);
-        if (!hwaccel) {
-            av_log(avctx, AV_LOG_ERROR,
-                   "Could not find an AVHWAccel for the pixel format: %s",
-                   desc->name);
-            return AV_PIX_FMT_NONE;
-        }
+    memcpy(choices, fmt, (n + 1) * sizeof(*choices));
 
-        if (hwaccel->priv_data_size) {
-            avctx->internal->hwaccel_priv_data = av_mallocz(hwaccel->priv_data_size);
-            if (!avctx->internal->hwaccel_priv_data)
-                return AV_PIX_FMT_NONE;
-        }
+    for (;;) {
+        ret = avctx->get_format(avctx, choices);
 
-        if (hwaccel->init) {
-            err = hwaccel->init(avctx);
-            if (err < 0) {
-                av_freep(&avctx->internal->hwaccel_priv_data);
-                return AV_PIX_FMT_NONE;
-            }
+        desc = av_pix_fmt_desc_get(ret);
+        if (!desc) {
+            ret = AV_PIX_FMT_NONE;
+            break;
         }
-        avctx->hwaccel = hwaccel;
+
+        if (avctx->hwaccel && avctx->hwaccel->uninit)
+            avctx->hwaccel->uninit(avctx);
+        av_freep(&avctx->internal->hwaccel_priv_data);
+        avctx->hwaccel = NULL;
+
+        if (!(desc->flags & AV_PIX_FMT_FLAG_HWACCEL))
+            break;
+
+        if (!setup_hwaccel(avctx, ret, desc->name))
+            break;
+
+        /* Remove failed hwaccel from choices */
+        for (n = 0; choices[n] != ret; n++)
+            av_assert0(choices[n] != AV_PIX_FMT_NONE);
+
+        do
+            choices[n] = choices[n + 1];
+        while (choices[n] != AV_PIX_FMT_NONE);
     }
 
+    av_freep(&choices);
     return ret;
 }