Commit e7073004 authored by Uoti Urpala's avatar Uoti Urpala Committed by Michael Niedermayer

h264: vdpau: fix crash with unsupported colorspace

The h264_vdpau decoder crashed if output colorspace was not 8-bit 420.
Add a check to error out instead (current hardware does not support
other colorspaces, so successful decoding is not possible).

Check implemented at a different place by michael, thus blame for bugs goes to michael
Signed-off-by: 's avatarMichael Niedermayer <michaelni@gmx.at>
parent 6c180b35
...@@ -2494,6 +2494,15 @@ static int decode_slice_header(H264Context *h, H264Context *h0) ...@@ -2494,6 +2494,15 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
s->avctx->sample_aspect_ratio = h->sps.sar; s->avctx->sample_aspect_ratio = h->sps.sar;
av_assert0(s->avctx->sample_aspect_ratio.den); av_assert0(s->avctx->sample_aspect_ratio.den);
if (s->avctx->codec->capabilities & CODEC_CAP_HWACCEL_VDPAU
&& (h->sps.bit_depth_luma != 8 ||
h->sps.chroma_format_idc > 1)) {
av_log(s->avctx, AV_LOG_ERROR,
"VDPAU decoding does not support video "
"colorspace\n");
return -1;
}
if (s->avctx->bits_per_raw_sample != h->sps.bit_depth_luma || if (s->avctx->bits_per_raw_sample != h->sps.bit_depth_luma ||
h->cur_chroma_format_idc != h->sps.chroma_format_idc) { h->cur_chroma_format_idc != h->sps.chroma_format_idc) {
if (h->sps.bit_depth_luma >= 8 && h->sps.bit_depth_luma <= 14 && h->sps.bit_depth_luma != 11 && h->sps.bit_depth_luma != 13 && if (h->sps.bit_depth_luma >= 8 && h->sps.bit_depth_luma <= 14 && h->sps.bit_depth_luma != 11 && h->sps.bit_depth_luma != 13 &&
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment