lavd/v4l2: sanitize logic of device_try_init(), so that it properly signal errors

Make device_try_init() return an error value, and allow to properly
report errors which are not due to a wrong pixel/codec configuration.

In particular, report intelligible feedback in case of busy device.

libavdevice/v4l2.c

@@ -747,19 +747,27 @@ static int v4l2_set_parameters(AVFormatContext *s1)
     return 0;
 }
 
-static uint32_t device_try_init(AVFormatContext *s1,
+static int device_try_init(AVFormatContext *s1,
                            enum AVPixelFormat pix_fmt,
                            int *width,
                            int *height,
+                           uint32_t *desired_format,
                            enum AVCodecID *codec_id)
 {
-    uint32_t desired_format = fmt_ff2v4l(pix_fmt, s1->video_codec_id);
+    int ret, i;
 
-    if (desired_format == 0 ||
-        device_init(s1, width, height, desired_format) < 0) {
-        int i;
+    *desired_format = fmt_ff2v4l(pix_fmt, s1->video_codec_id);
 
-        desired_format = 0;
+    if (*desired_format) {
+        ret = device_init(s1, width, height, *desired_format);
+        if (ret < 0) {
+            *desired_format = 0;
+            if (ret != AVERROR(EINVAL))
+                return ret;
+        }
+    }
+
+    if (!*desired_format) {
         for (i = 0; i<FF_ARRAY_ELEMS(fmt_conversion_table); i++) {
             if (s1->video_codec_id == AV_CODEC_ID_NONE ||
                 fmt_conversion_table[i].codec_id == s1->video_codec_id) {
@@ -767,21 +775,28 @@ static uint32_t device_try_init(AVFormatContext *s1,
                        avcodec_get_name(fmt_conversion_table[i].codec_id),
                        (char *)av_x_if_null(av_get_pix_fmt_name(fmt_conversion_table[i].ff_fmt), "none"));
 
-                desired_format = fmt_conversion_table[i].v4l2_fmt;
-                if (device_init(s1, width, height, desired_format) >= 0) {
+                *desired_format = fmt_conversion_table[i].v4l2_fmt;
+                ret = device_init(s1, width, height, *desired_format);
+                if (ret >= 0)
                     break;
+                else if (ret != AVERROR(EINVAL))
+                    return ret;
+                *desired_format = 0;
             }
-                desired_format = 0;
         }
+
+        if (*desired_format == 0) {
+            av_log(s1, AV_LOG_ERROR, "Cannot find a proper format for "
+                   "codec '%s' (id %d), pixel format '%s' (id %d)\n",
+                   avcodec_get_name(s1->video_codec_id), s1->video_codec_id,
+                   (char *)av_x_if_null(av_get_pix_fmt_name(pix_fmt), "none"), pix_fmt);
+            ret = AVERROR(EINVAL);
         }
     }
 
-    if (desired_format != 0) {
-        *codec_id = fmt_v4l2codec(desired_format);
+    *codec_id = fmt_v4l2codec(*desired_format);
     av_assert0(*codec_id != AV_CODEC_ID_NONE);
-    }
-
-    return desired_format;
+    return ret;
 }
 
 static int v4l2_read_header(AVFormatContext *s1)
@@ -842,8 +857,11 @@ static int v4l2_read_header(AVFormatContext *s1)
                "Setting frame size to %dx%d\n", s->width, s->height);
     }
 
-    desired_format = device_try_init(s1, pix_fmt, &s->width, &s->height,
-                                     &codec_id);
+    res = device_try_init(s1, pix_fmt, &s->width, &s->height, &desired_format, &codec_id);
+    if (res < 0) {
+        v4l2_close(s->fd);
+        return res;
+    }
 
     /* If no pixel_format was specified, the codec_id was not known up
      * until now. Set video_codec_id in the context, as codec_id will
@@ -852,15 +870,6 @@ static int v4l2_read_header(AVFormatContext *s1)
     if (codec_id != AV_CODEC_ID_NONE && s1->video_codec_id == AV_CODEC_ID_NONE)
         s1->video_codec_id = codec_id;
 
-    if (desired_format == 0) {
-        av_log(s1, AV_LOG_ERROR, "Cannot find a proper format for "
-               "codec '%s' (id %d), pixel format '%s' (id %d)\n",
-               avcodec_get_name(s1->video_codec_id), s1->video_codec_id,
-               (char *)av_x_if_null(av_get_pix_fmt_name(pix_fmt), "none"), pix_fmt);
-        v4l2_close(s->fd);
-
-        return AVERROR(EIO);
-    }
     if ((res = av_image_check_size(s->width, s->height, 0, s1)) < 0)
         return res;