pixdesc: mark pseudopaletted formats with a special flag.

This makes it possible to dintinguish them from PAL8.

Fixes an invalid write in avpicture_layout().

doc/APIchanges

@@ -13,6 +13,9 @@ libavutil:   2011-04-18
 
 API changes, most recent first:
 
+2012-02-xx - xxxxxxx - lavu 51.22.1 - pixdesc.h
+  Add PIX_FMT_PSEUDOPAL flag.
+
 2012-02-01 - xxxxxxx - lavc 54.01.0
   Add avcodec_encode_video2() and deprecate avcodec_encode_video().
 

libavcodec/imgconvert.c

@@ -471,15 +471,9 @@ int avpicture_get_size(enum PixelFormat pix_fmt, int width, int height)
     AVPicture dummy_pict;
     if(av_image_check_size(width, height, 0, NULL))
         return -1;
-    switch (pix_fmt) {
-    case PIX_FMT_RGB8:
-    case PIX_FMT_BGR8:
-    case PIX_FMT_RGB4_BYTE:
-    case PIX_FMT_BGR4_BYTE:
-    case PIX_FMT_GRAY8:
+    if (av_pix_fmt_descriptors[pix_fmt].flags & PIX_FMT_PSEUDOPAL)
         // do not include palette for these pseudo-paletted formats
         return width * height;
-    }
     return avpicture_fill(&dummy_pict, NULL, pix_fmt, width, height);
 }
 

libavcodec/rawdec.c

@@ -158,8 +158,7 @@ static int raw_decode(AVCodecContext *avctx,
 
     avpicture_fill(picture, buf, avctx->pix_fmt, avctx->width, avctx->height);
     if((avctx->pix_fmt==PIX_FMT_PAL8 && buf_size < context->length) ||
-       (avctx->pix_fmt!=PIX_FMT_PAL8 &&
-        (av_pix_fmt_descriptors[avctx->pix_fmt].flags & PIX_FMT_PAL))){
+       (av_pix_fmt_descriptors[avctx->pix_fmt].flags & PIX_FMT_PSEUDOPAL)) {
         frame->data[1]= context->palette;
     }
     if (avctx->pix_fmt == PIX_FMT_PAL8) {

libavfilter/vf_crop.c

@@ -272,7 +272,8 @@ static void start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
     ref2->data[0] += crop->y * ref2->linesize[0];
     ref2->data[0] += crop->x * crop->max_step[0];
 
-    if (!(av_pix_fmt_descriptors[link->format].flags & PIX_FMT_PAL)) {
+    if (!(av_pix_fmt_descriptors[link->format].flags & PIX_FMT_PAL ||
+          av_pix_fmt_descriptors[link->format].flags & PIX_FMT_PSEUDOPAL)) {
         for (i = 1; i < 3; i ++) {
             if (ref2->data[i]) {
                 ref2->data[i] += (crop->y >> crop->vsub) * ref2->linesize[i];

libavfilter/vf_pixdesctest.c

@@ -72,7 +72,8 @@ static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
     }
 
     /* copy palette */
-    if (priv->pix_desc->flags & PIX_FMT_PAL)
+    if (priv->pix_desc->flags & PIX_FMT_PAL ||
+        priv->pix_desc->flags & PIX_FMT_PSEUDOPAL)
         memcpy(outpicref->data[1], outpicref->data[1], 256*4);
 
     avfilter_start_frame(outlink, avfilter_ref_buffer(outpicref, ~0));

libavfilter/vf_scale.c

@@ -208,7 +208,8 @@ static int config_props(AVFilterLink *outlink)
            outlink->w, outlink->h, av_pix_fmt_descriptors[outlink->format].name,
            scale->flags);
 
-    scale->input_is_pal = av_pix_fmt_descriptors[inlink->format].flags & PIX_FMT_PAL;
+    scale->input_is_pal = av_pix_fmt_descriptors[inlink->format].flags & PIX_FMT_PAL ||
+                          av_pix_fmt_descriptors[inlink->format].flags & PIX_FMT_PSEUDOPAL;
 
     if (scale->sws)
         sws_freeContext(scale->sws);

libavutil/avutil.h

@@ -155,7 +155,7 @@
 
 #define LIBAVUTIL_VERSION_MAJOR 51
 #define LIBAVUTIL_VERSION_MINOR 22
-#define LIBAVUTIL_VERSION_MICRO  0
+#define LIBAVUTIL_VERSION_MICRO  1
 
 #define LIBAVUTIL_VERSION_INT   AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \
                                                LIBAVUTIL_VERSION_MINOR, \

libavutil/imgutils.c

@@ -108,7 +108,8 @@ int av_image_fill_pointers(uint8_t *data[4], enum PixelFormat pix_fmt, int heigh
         return AVERROR(EINVAL);
     size[0] = linesizes[0] * height;
 
-    if (desc->flags & PIX_FMT_PAL) {
+    if (desc->flags & PIX_FMT_PAL ||
+        desc->flags & PIX_FMT_PSEUDOPAL) {
         size[0] = (size[0] + 3) & ~3;
         data[1] = ptr + size[0]; /* palette is stored here as 256 32 bits words */
         return size[0] + 256 * 4;
@@ -196,7 +197,8 @@ int av_image_alloc(uint8_t *pointers[4], int linesizes[4],
         av_free(buf);
         return ret;
     }
-    if (av_pix_fmt_descriptors[pix_fmt].flags & PIX_FMT_PAL)
+    if (av_pix_fmt_descriptors[pix_fmt].flags & PIX_FMT_PAL ||
+        av_pix_fmt_descriptors[pix_fmt].flags & PIX_FMT_PSEUDOPAL)
         ff_set_systematic_pal2((uint32_t*)pointers[1], pix_fmt);
 
     return ret;
@@ -243,7 +245,8 @@ void av_image_copy(uint8_t *dst_data[4], int dst_linesizes[4],
     if (desc->flags & PIX_FMT_HWACCEL)
         return;
 
-    if (desc->flags & PIX_FMT_PAL) {
+    if (desc->flags & PIX_FMT_PAL ||
+        desc->flags & PIX_FMT_PSEUDOPAL) {
         av_image_copy_plane(dst_data[0], dst_linesizes[0],
                             src_data[0], src_linesizes[0],
                             width, height);

libavutil/pixdesc.c

@@ -327,7 +327,7 @@ const AVPixFmtDescriptor av_pix_fmt_descriptors[PIX_FMT_NB] = {
             { 0, 0, 1, 3, 2 },        /* G */
             { 0, 0, 1, 0, 2 },        /* R */
         },
-        .flags = PIX_FMT_PAL | PIX_FMT_RGB,
+        .flags = PIX_FMT_RGB | PIX_FMT_PSEUDOPAL,
     },
     [PIX_FMT_BGR4] = {
         .name = "bgr4",
@@ -351,7 +351,7 @@ const AVPixFmtDescriptor av_pix_fmt_descriptors[PIX_FMT_NB] = {
             { 0, 0, 1, 1, 1 },        /* G */
             { 0, 0, 1, 0, 0 },        /* R */
         },
-        .flags = PIX_FMT_PAL | PIX_FMT_RGB,
+        .flags = PIX_FMT_RGB | PIX_FMT_PSEUDOPAL,
     },
     [PIX_FMT_RGB8] = {
         .name = "rgb8",
@@ -363,7 +363,7 @@ const AVPixFmtDescriptor av_pix_fmt_descriptors[PIX_FMT_NB] = {
             { 0, 0, 1, 3, 2 },        /* G */
             { 0, 0, 1, 0, 2 },        /* B */
         },
-        .flags = PIX_FMT_PAL | PIX_FMT_RGB,
+        .flags = PIX_FMT_RGB | PIX_FMT_PSEUDOPAL,
     },
     [PIX_FMT_RGB4] = {
         .name = "rgb4",
@@ -387,7 +387,7 @@ const AVPixFmtDescriptor av_pix_fmt_descriptors[PIX_FMT_NB] = {
             { 0, 0, 1, 1, 1 },        /* G */
             { 0, 0, 1, 0, 0 },        /* B */
         },
-        .flags = PIX_FMT_PAL | PIX_FMT_RGB,
+        .flags = PIX_FMT_RGB | PIX_FMT_PSEUDOPAL,
     },
     [PIX_FMT_NV12] = {
         .name = "nv12",

libavutil/pixdesc.h

@@ -89,6 +89,12 @@ typedef struct AVPixFmtDescriptor{
 #define PIX_FMT_HWACCEL   8 ///< Pixel format is an HW accelerated format.
 #define PIX_FMT_PLANAR   16 ///< At least one pixel component is not in the first data plane
 #define PIX_FMT_RGB      32 ///< The pixel format contains RGB-like data (as opposed to YUV/grayscale)
+/**
+ * The pixel format is "pseudo-paletted". This means that Libav treats it as
+ * paletted internally, but the palette is generated by the decoder and is not
+ * stored in the file.
+ */
+#define PIX_FMT_PSEUDOPAL 64
 
 /**
  * The array of all the pixel format descriptors.

libswscale/swscale_internal.h

@@ -627,7 +627,9 @@ const char *sws_format_name(enum PixelFormat format);
     (av_pix_fmt_descriptors[x].nb_components >= 2          &&  \
      (av_pix_fmt_descriptors[x].flags & PIX_FMT_PLANAR))
 
-#define usePal(x) ((av_pix_fmt_descriptors[x].flags & PIX_FMT_PAL) || (x) == PIX_FMT_Y400A)
+#define usePal(x) ((av_pix_fmt_descriptors[x].flags & PIX_FMT_PAL) ||       \
+                   (av_pix_fmt_descriptors[x].flags & PIX_FMT_PSEUDOPAL) || \
+                   (x) == PIX_FMT_Y400A)
 
 extern const uint64_t ff_dither4[2];
 extern const uint64_t ff_dither8[2];