Updates avcodec_get_pix_fmt_loss to return maximum loss for an invalid destination pixel format.

Signed-off-by: Matthew Einhorn <moiein2000@gmail.com>
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

libavcodec/avcodec.h

@@ -3484,7 +3484,8 @@ size_t av_get_codec_tag_string(char *buf, size_t buf_size, unsigned int codec_ta
  * @param[in] dst_pix_fmt destination pixel format
  * @param[in] src_pix_fmt source pixel format
  * @param[in] has_alpha Whether the source pixel format alpha channel is used.
- * @return Combination of flags informing you what kind of losses will occur.
+ * @return Combination of flags informing you what kind of losses will occur
+ * (maximum loss for an invalid dst_pix_fmt).
  */
 int avcodec_get_pix_fmt_loss(enum PixelFormat dst_pix_fmt, enum PixelFormat src_pix_fmt,
                              int has_alpha);

libavcodec/imgconvert.c

@@ -359,6 +359,9 @@ static int get_pix_fmt_depth(int *min, int *max, enum PixelFormat pix_fmt)
 int avcodec_get_pix_fmt_loss(enum PixelFormat dst_pix_fmt, enum PixelFormat src_pix_fmt,
                              int has_alpha)
 {
+    if (dst_pix_fmt>=PIX_FMT_NB || dst_pix_fmt<=PIX_FMT_NONE)
+        return ~0;
+
     const PixFmtInfo *pf, *ps;
     const AVPixFmtDescriptor *src_desc = &av_pix_fmt_descriptors[src_pix_fmt];
     const AVPixFmtDescriptor *dst_desc = &av_pix_fmt_descriptors[dst_pix_fmt];