Commit 676da248 authored by Luca Barbato's avatar Luca Barbato

vmd: refactor the inner decode loop

Simplify a little, assume empty frames are acceptable and
do not pointlessly reinit the bytestream2 contexts using
possibly wrong size values.

Reported-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org
parent c8f3cb91
...@@ -268,9 +268,11 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame) ...@@ -268,9 +268,11 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame)
} }
s->size -= PALETTE_COUNT * 3 + 2; s->size -= PALETTE_COUNT * 3 + 2;
} }
if (s->size > 0) {
if (!s->size)
return 0;
/* originally UnpackFrame in VAG's code */ /* originally UnpackFrame in VAG's code */
bytestream2_init(&gb, gb.buffer, s->buf + s->size - gb.buffer);
if (bytestream2_get_bytes_left(&gb) < 1) if (bytestream2_get_bytes_left(&gb) < 1)
return AVERROR_INVALIDDATA; return AVERROR_INVALIDDATA;
meth = bytestream2_get_byteu(&gb); meth = bytestream2_get_byteu(&gb);
...@@ -291,7 +293,8 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame) ...@@ -291,7 +293,8 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame)
len = bytestream2_get_byte(&gb); len = bytestream2_get_byte(&gb);
if (len & 0x80) { if (len & 0x80) {
len = (len & 0x7F) + 1; len = (len & 0x7F) + 1;
if (ofs + len > frame_width || bytestream2_get_bytes_left(&gb) < len) if (ofs + len > frame_width ||
bytestream2_get_bytes_left(&gb) < len)
return AVERROR_INVALIDDATA; return AVERROR_INVALIDDATA;
bytestream2_get_buffer(&gb, &dp[ofs], len); bytestream2_get_buffer(&gb, &dp[ofs], len);
ofs += len; ofs += len;
...@@ -304,7 +307,8 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame) ...@@ -304,7 +307,8 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame)
} }
} while (ofs < frame_width); } while (ofs < frame_width);
if (ofs > frame_width) { if (ofs > frame_width) {
av_log(s->avctx, AV_LOG_ERROR, "VMD video: offset > width (%d > %d)\n", av_log(s->avctx, AV_LOG_ERROR,
"VMD video: offset > width (%d > %d)\n",
ofs, frame_width); ofs, frame_width);
return AVERROR_INVALIDDATA; return AVERROR_INVALIDDATA;
} }
...@@ -344,7 +348,8 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame) ...@@ -344,7 +348,8 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame)
} }
} while (ofs < frame_width); } while (ofs < frame_width);
if (ofs > frame_width) { if (ofs > frame_width) {
av_log(s->avctx, AV_LOG_ERROR, "VMD video: offset > width (%d > %d)\n", av_log(s->avctx, AV_LOG_ERROR,
"VMD video: offset > width (%d > %d)\n",
ofs, frame_width); ofs, frame_width);
return AVERROR_INVALIDDATA; return AVERROR_INVALIDDATA;
} }
...@@ -353,7 +358,6 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame) ...@@ -353,7 +358,6 @@ static int vmd_decode(VmdVideoContext *s, AVFrame *frame)
} }
break; break;
} }
}
return 0; return 0;
} }
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment