Warn if rawvideo and an unreadable pix_fmt are written.

Print an error if a combination of rawvideo and an unusual pix_fmt
that will be impossible to decode are written to avi or mov.
Fixes ticket #3545.

libavcodec/raw.c

@@ -234,3 +234,28 @@ unsigned int avcodec_pix_fmt_to_codec_tag(enum AVPixelFormat fmt)
     }
     return 0;
 }
+
+const PixelFormatTag avpriv_pix_fmt_bps_avi[] = {
+    { AV_PIX_FMT_MONOWHITE, 1 },
+    { AV_PIX_FMT_PAL8,    2 },
+    { AV_PIX_FMT_PAL8,    4 },
+    { AV_PIX_FMT_PAL8,    8 },
+    { AV_PIX_FMT_RGB444LE, 12 },
+    { AV_PIX_FMT_RGB555LE, 15 },
+    { AV_PIX_FMT_RGB555LE, 16 },
+    { AV_PIX_FMT_BGR24,  24 },
+    { AV_PIX_FMT_BGRA,   32 },
+    { AV_PIX_FMT_NONE,    0 },
+};
+
+const PixelFormatTag avpriv_pix_fmt_bps_mov[] = {
+    { AV_PIX_FMT_MONOWHITE, 1 },
+    { AV_PIX_FMT_PAL8,      2 },
+    { AV_PIX_FMT_PAL8,      4 },
+    { AV_PIX_FMT_PAL8,      8 },
+    { AV_PIX_FMT_RGB555BE, 16 },
+    { AV_PIX_FMT_RGB24,    24 },
+    { AV_PIX_FMT_ARGB,     32 },
+    { AV_PIX_FMT_MONOWHITE,33 },
+    { AV_PIX_FMT_NONE,      0 },
+};

libavcodec/raw.h

@@ -41,4 +41,7 @@ enum AVPixelFormat ff_find_pix_fmt(const PixelFormatTag *tags, unsigned int four
 #endif
 enum AVPixelFormat avpriv_find_pix_fmt(const PixelFormatTag *tags, unsigned int fourcc);
 
+extern av_export const PixelFormatTag avpriv_pix_fmt_bps_avi[];
+extern av_export const PixelFormatTag avpriv_pix_fmt_bps_mov[];
+
 #endif /* AVCODEC_RAW_H */

libavcodec/rawdec.c

@@ -62,31 +62,6 @@ static const AVClass rawdec_class = {
     .version    = LIBAVUTIL_VERSION_INT,
 };
 
-static const PixelFormatTag pix_fmt_bps_avi[] = {
-    { AV_PIX_FMT_MONOWHITE, 1 },
-    { AV_PIX_FMT_PAL8,    2 },
-    { AV_PIX_FMT_PAL8,    4 },
-    { AV_PIX_FMT_PAL8,    8 },
-    { AV_PIX_FMT_RGB444LE, 12 },
-    { AV_PIX_FMT_RGB555LE, 15 },
-    { AV_PIX_FMT_RGB555LE, 16 },
-    { AV_PIX_FMT_BGR24,  24 },
-    { AV_PIX_FMT_BGRA,   32 },
-    { AV_PIX_FMT_NONE,    0 },
-};
-
-static const PixelFormatTag pix_fmt_bps_mov[] = {
-    { AV_PIX_FMT_MONOWHITE, 1 },
-    { AV_PIX_FMT_PAL8,      2 },
-    { AV_PIX_FMT_PAL8,      4 },
-    { AV_PIX_FMT_PAL8,      8 },
-    { AV_PIX_FMT_RGB555BE, 16 },
-    { AV_PIX_FMT_RGB24,    24 },
-    { AV_PIX_FMT_ARGB,     32 },
-    { AV_PIX_FMT_MONOWHITE,33 },
-    { AV_PIX_FMT_NONE,      0 },
-};
-
 #if LIBAVCODEC_VERSION_MAJOR < 55
 enum AVPixelFormat ff_find_pix_fmt(const PixelFormatTag *tags, unsigned int fourcc)
 {
@@ -103,15 +78,15 @@ static av_cold int raw_init_decoder(AVCodecContext *avctx)
 
     if (   avctx->codec_tag == MKTAG('r','a','w',' ')
         || avctx->codec_tag == MKTAG('N','O','1','6'))
-        avctx->pix_fmt = avpriv_find_pix_fmt(pix_fmt_bps_mov,
+        avctx->pix_fmt = avpriv_find_pix_fmt(avpriv_pix_fmt_bps_mov,
                                       avctx->bits_per_coded_sample);
     else if (avctx->codec_tag == MKTAG('W', 'R', 'A', 'W'))
-        avctx->pix_fmt = avpriv_find_pix_fmt(pix_fmt_bps_avi,
+        avctx->pix_fmt = avpriv_find_pix_fmt(avpriv_pix_fmt_bps_avi,
                                       avctx->bits_per_coded_sample);
     else if (avctx->codec_tag && (avctx->codec_tag & 0xFFFFFF) != MKTAG('B','I','T', 0))
         avctx->pix_fmt = avpriv_find_pix_fmt(ff_raw_pix_fmt_tags, avctx->codec_tag);
     else if (avctx->pix_fmt == AV_PIX_FMT_NONE && avctx->bits_per_coded_sample)
-        avctx->pix_fmt = avpriv_find_pix_fmt(pix_fmt_bps_avi,
+        avctx->pix_fmt = avpriv_find_pix_fmt(avpriv_pix_fmt_bps_avi,
                                       avctx->bits_per_coded_sample);
 
     desc = av_pix_fmt_desc_get(avctx->pix_fmt);

libavformat/avienc.c

@@ -32,6 +32,8 @@
 #include "libavutil/dict.h"
 #include "libavutil/avassert.h"
 #include "libavutil/timestamp.h"
+#include "libavutil/pixdesc.h"
+#include "libavcodec/raw.h"
 
 /*
  * TODO:
@@ -296,6 +298,7 @@ static int avi_write_header(AVFormatContext *s)
 
         if (stream->codec_type != AVMEDIA_TYPE_DATA) {
             int ret;
+            enum AVPixelFormat pix_fmt;
 
             strf = ff_start_tag(pb, "strf");
             switch (stream->codec_type) {
@@ -306,6 +309,14 @@ static int avi_write_header(AVFormatContext *s)
                     break;
             case AVMEDIA_TYPE_VIDEO:
                 ff_put_bmp_header(pb, stream, ff_codec_bmp_tags, 0, 0);
+                pix_fmt = avpriv_find_pix_fmt(avpriv_pix_fmt_bps_avi,
+                                              stream->bits_per_coded_sample);
+                if (   !stream->codec_tag
+                    && stream->codec_id == AV_CODEC_ID_RAWVIDEO
+                    && stream->pix_fmt != pix_fmt
+                    && stream->pix_fmt != AV_PIX_FMT_NONE)
+                    av_log(s, AV_LOG_ERROR, "%s rawvideo cannot be written to avi, output file will be unreadable\n",
+                          av_get_pix_fmt_name(stream->pix_fmt));
                 break;
             case AVMEDIA_TYPE_AUDIO:
                 if ((ret = ff_put_wav_header(pb, stream)) < 0)

libavformat/movenc.c

@@ -34,12 +34,14 @@
 #include "libavcodec/get_bits.h"
 #include "libavcodec/put_bits.h"
 #include "libavcodec/vc1.h"
+#include "libavcodec/raw.h"
 #include "internal.h"
 #include "libavutil/avstring.h"
 #include "libavutil/intfloat.h"
 #include "libavutil/mathematics.h"
 #include "libavutil/opt.h"
 #include "libavutil/dict.h"
+#include "libavutil/pixdesc.h"
 #include "hevc.h"
 #include "rtpenc.h"
 #include "mov_chan.h"
@@ -1009,6 +1011,7 @@ static int mov_get_rawvideo_codec_tag(AVFormatContext *s, MOVTrack *track)
 {
     int tag = track->enc->codec_tag;
     int i;
+    enum AVPixelFormat pix_fmt;
 
     for (i = 0; i < FF_ARRAY_ELEMS(mov_pix_fmt_tags); i++) {
         if (track->enc->pix_fmt == mov_pix_fmt_tags[i].pix_fmt) {
@@ -1019,6 +1022,13 @@ static int mov_get_rawvideo_codec_tag(AVFormatContext *s, MOVTrack *track)
         }
     }
 
+    pix_fmt = avpriv_find_pix_fmt(avpriv_pix_fmt_bps_mov,
+                                  track->enc->bits_per_coded_sample);
+    if (tag == MKTAG('r','a','w',' ') &&
+        track->enc->pix_fmt != pix_fmt &&
+        track->enc->pix_fmt != AV_PIX_FMT_NONE)
+        av_log(s, AV_LOG_ERROR, "%s rawvideo cannot be written to mov, output file will be unreadable\n",
+               av_get_pix_fmt_name(track->enc->pix_fmt));
     return tag;
 }