Commit 38d55332 authored by Anton Khirnov's avatar Anton Khirnov

pixdesc: mark pseudopaletted formats with a special flag.

This makes it possible to dintinguish them from PAL8.

Fixes an invalid write in avpicture_layout().
parent 8e37038a
......@@ -13,6 +13,9 @@ libavutil: 2011-04-18
API changes, most recent first:
2012-02-xx - xxxxxxx - lavu 51.22.1 - pixdesc.h
Add PIX_FMT_PSEUDOPAL flag.
2012-02-01 - xxxxxxx - lavc 54.01.0
Add avcodec_encode_video2() and deprecate avcodec_encode_video().
......
......@@ -471,15 +471,9 @@ int avpicture_get_size(enum PixelFormat pix_fmt, int width, int height)
AVPicture dummy_pict;
if(av_image_check_size(width, height, 0, NULL))
return -1;
switch (pix_fmt) {
case PIX_FMT_RGB8:
case PIX_FMT_BGR8:
case PIX_FMT_RGB4_BYTE:
case PIX_FMT_BGR4_BYTE:
case PIX_FMT_GRAY8:
if (av_pix_fmt_descriptors[pix_fmt].flags & PIX_FMT_PSEUDOPAL)
// do not include palette for these pseudo-paletted formats
return width * height;
}
return avpicture_fill(&dummy_pict, NULL, pix_fmt, width, height);
}
......
......@@ -158,8 +158,7 @@ static int raw_decode(AVCodecContext *avctx,
avpicture_fill(picture, buf, avctx->pix_fmt, avctx->width, avctx->height);
if((avctx->pix_fmt==PIX_FMT_PAL8 && buf_size < context->length) ||
(avctx->pix_fmt!=PIX_FMT_PAL8 &&
(av_pix_fmt_descriptors[avctx->pix_fmt].flags & PIX_FMT_PAL))){
(av_pix_fmt_descriptors[avctx->pix_fmt].flags & PIX_FMT_PSEUDOPAL)) {
frame->data[1]= context->palette;
}
if (avctx->pix_fmt == PIX_FMT_PAL8) {
......
......@@ -272,7 +272,8 @@ static void start_frame(AVFilterLink *link, AVFilterBufferRef *picref)
ref2->data[0] += crop->y * ref2->linesize[0];
ref2->data[0] += crop->x * crop->max_step[0];
if (!(av_pix_fmt_descriptors[link->format].flags & PIX_FMT_PAL)) {
if (!(av_pix_fmt_descriptors[link->format].flags & PIX_FMT_PAL ||
av_pix_fmt_descriptors[link->format].flags & PIX_FMT_PSEUDOPAL)) {
for (i = 1; i < 3; i ++) {
if (ref2->data[i]) {
ref2->data[i] += (crop->y >> crop->vsub) * ref2->linesize[i];
......
......@@ -72,7 +72,8 @@ static void start_frame(AVFilterLink *inlink, AVFilterBufferRef *picref)
}
/* copy palette */
if (priv->pix_desc->flags & PIX_FMT_PAL)
if (priv->pix_desc->flags & PIX_FMT_PAL ||
priv->pix_desc->flags & PIX_FMT_PSEUDOPAL)
memcpy(outpicref->data[1], outpicref->data[1], 256*4);
avfilter_start_frame(outlink, avfilter_ref_buffer(outpicref, ~0));
......
......@@ -208,7 +208,8 @@ static int config_props(AVFilterLink *outlink)
outlink->w, outlink->h, av_pix_fmt_descriptors[outlink->format].name,
scale->flags);
scale->input_is_pal = av_pix_fmt_descriptors[inlink->format].flags & PIX_FMT_PAL;
scale->input_is_pal = av_pix_fmt_descriptors[inlink->format].flags & PIX_FMT_PAL ||
av_pix_fmt_descriptors[inlink->format].flags & PIX_FMT_PSEUDOPAL;
if (scale->sws)
sws_freeContext(scale->sws);
......
......@@ -155,7 +155,7 @@
#define LIBAVUTIL_VERSION_MAJOR 51
#define LIBAVUTIL_VERSION_MINOR 22
#define LIBAVUTIL_VERSION_MICRO 0
#define LIBAVUTIL_VERSION_MICRO 1
#define LIBAVUTIL_VERSION_INT AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \
LIBAVUTIL_VERSION_MINOR, \
......
......@@ -108,7 +108,8 @@ int av_image_fill_pointers(uint8_t *data[4], enum PixelFormat pix_fmt, int heigh
return AVERROR(EINVAL);
size[0] = linesizes[0] * height;
if (desc->flags & PIX_FMT_PAL) {
if (desc->flags & PIX_FMT_PAL ||
desc->flags & PIX_FMT_PSEUDOPAL) {
size[0] = (size[0] + 3) & ~3;
data[1] = ptr + size[0]; /* palette is stored here as 256 32 bits words */
return size[0] + 256 * 4;
......@@ -196,7 +197,8 @@ int av_image_alloc(uint8_t *pointers[4], int linesizes[4],
av_free(buf);
return ret;
}
if (av_pix_fmt_descriptors[pix_fmt].flags & PIX_FMT_PAL)
if (av_pix_fmt_descriptors[pix_fmt].flags & PIX_FMT_PAL ||
av_pix_fmt_descriptors[pix_fmt].flags & PIX_FMT_PSEUDOPAL)
ff_set_systematic_pal2((uint32_t*)pointers[1], pix_fmt);
return ret;
......@@ -243,7 +245,8 @@ void av_image_copy(uint8_t *dst_data[4], int dst_linesizes[4],
if (desc->flags & PIX_FMT_HWACCEL)
return;
if (desc->flags & PIX_FMT_PAL) {
if (desc->flags & PIX_FMT_PAL ||
desc->flags & PIX_FMT_PSEUDOPAL) {
av_image_copy_plane(dst_data[0], dst_linesizes[0],
src_data[0], src_linesizes[0],
width, height);
......
......@@ -327,7 +327,7 @@ const AVPixFmtDescriptor av_pix_fmt_descriptors[PIX_FMT_NB] = {
{ 0, 0, 1, 3, 2 }, /* G */
{ 0, 0, 1, 0, 2 }, /* R */
},
.flags = PIX_FMT_PAL | PIX_FMT_RGB,
.flags = PIX_FMT_RGB | PIX_FMT_PSEUDOPAL,
},
[PIX_FMT_BGR4] = {
.name = "bgr4",
......@@ -351,7 +351,7 @@ const AVPixFmtDescriptor av_pix_fmt_descriptors[PIX_FMT_NB] = {
{ 0, 0, 1, 1, 1 }, /* G */
{ 0, 0, 1, 0, 0 }, /* R */
},
.flags = PIX_FMT_PAL | PIX_FMT_RGB,
.flags = PIX_FMT_RGB | PIX_FMT_PSEUDOPAL,
},
[PIX_FMT_RGB8] = {
.name = "rgb8",
......@@ -363,7 +363,7 @@ const AVPixFmtDescriptor av_pix_fmt_descriptors[PIX_FMT_NB] = {
{ 0, 0, 1, 3, 2 }, /* G */
{ 0, 0, 1, 0, 2 }, /* B */
},
.flags = PIX_FMT_PAL | PIX_FMT_RGB,
.flags = PIX_FMT_RGB | PIX_FMT_PSEUDOPAL,
},
[PIX_FMT_RGB4] = {
.name = "rgb4",
......@@ -387,7 +387,7 @@ const AVPixFmtDescriptor av_pix_fmt_descriptors[PIX_FMT_NB] = {
{ 0, 0, 1, 1, 1 }, /* G */
{ 0, 0, 1, 0, 0 }, /* B */
},
.flags = PIX_FMT_PAL | PIX_FMT_RGB,
.flags = PIX_FMT_RGB | PIX_FMT_PSEUDOPAL,
},
[PIX_FMT_NV12] = {
.name = "nv12",
......
......@@ -89,6 +89,12 @@ typedef struct AVPixFmtDescriptor{
#define PIX_FMT_HWACCEL 8 ///< Pixel format is an HW accelerated format.
#define PIX_FMT_PLANAR 16 ///< At least one pixel component is not in the first data plane
#define PIX_FMT_RGB 32 ///< The pixel format contains RGB-like data (as opposed to YUV/grayscale)
/**
* The pixel format is "pseudo-paletted". This means that Libav treats it as
* paletted internally, but the palette is generated by the decoder and is not
* stored in the file.
*/
#define PIX_FMT_PSEUDOPAL 64
/**
* The array of all the pixel format descriptors.
......
......@@ -627,7 +627,9 @@ const char *sws_format_name(enum PixelFormat format);
(av_pix_fmt_descriptors[x].nb_components >= 2 && \
(av_pix_fmt_descriptors[x].flags & PIX_FMT_PLANAR))
#define usePal(x) ((av_pix_fmt_descriptors[x].flags & PIX_FMT_PAL) || (x) == PIX_FMT_Y400A)
#define usePal(x) ((av_pix_fmt_descriptors[x].flags & PIX_FMT_PAL) || \
(av_pix_fmt_descriptors[x].flags & PIX_FMT_PSEUDOPAL) || \
(x) == PIX_FMT_Y400A)
extern const uint64_t ff_dither4[2];
extern const uint64_t ff_dither8[2];
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment