Commit 330deb75 authored by Ronald S. Bultje's avatar Ronald S. Bultje

mpegvideo: set correct offset for edge emulation buffer.

Using the old code, half of it was unused and the other half was too
small for e.g. >8bpp interlaced data, causing random buffer overruns.
parent 0884dd5a
...@@ -377,8 +377,7 @@ static int init_duplicate_context(MpegEncContext *s, MpegEncContext *base){ ...@@ -377,8 +377,7 @@ static int init_duplicate_context(MpegEncContext *s, MpegEncContext *base){
int i; int i;
// edge emu needs blocksize + filter length - 1 (=17x17 for halfpel / 21x21 for h264) // edge emu needs blocksize + filter length - 1 (=17x17 for halfpel / 21x21 for h264)
FF_ALLOCZ_OR_GOTO(s->avctx, s->allocated_edge_emu_buffer, (s->width+64)*2*21*2, fail); //(width + edge + align)*interlaced*MBsize*tolerance FF_ALLOCZ_OR_GOTO(s->avctx, s->edge_emu_buffer, (s->width+64)*2*21*2, fail); //(width + edge + align)*interlaced*MBsize*tolerance
s->edge_emu_buffer= s->allocated_edge_emu_buffer + (s->width+64)*2*21;
//FIXME should be linesize instead of s->width*2 but that is not known before get_buffer() //FIXME should be linesize instead of s->width*2 but that is not known before get_buffer()
FF_ALLOCZ_OR_GOTO(s->avctx, s->me.scratchpad, (s->width+64)*4*16*2*sizeof(uint8_t), fail) FF_ALLOCZ_OR_GOTO(s->avctx, s->me.scratchpad, (s->width+64)*4*16*2*sizeof(uint8_t), fail)
...@@ -416,7 +415,7 @@ fail: ...@@ -416,7 +415,7 @@ fail:
static void free_duplicate_context(MpegEncContext *s){ static void free_duplicate_context(MpegEncContext *s){
if(s==NULL) return; if(s==NULL) return;
av_freep(&s->allocated_edge_emu_buffer); s->edge_emu_buffer= NULL; av_freep(&s->edge_emu_buffer);
av_freep(&s->me.scratchpad); av_freep(&s->me.scratchpad);
s->me.temp= s->me.temp=
s->rd_scratchpad= s->rd_scratchpad=
...@@ -433,7 +432,6 @@ static void free_duplicate_context(MpegEncContext *s){ ...@@ -433,7 +432,6 @@ static void free_duplicate_context(MpegEncContext *s){
static void backup_duplicate_context(MpegEncContext *bak, MpegEncContext *src){ static void backup_duplicate_context(MpegEncContext *bak, MpegEncContext *src){
#define COPY(a) bak->a= src->a #define COPY(a) bak->a= src->a
COPY(allocated_edge_emu_buffer);
COPY(edge_emu_buffer); COPY(edge_emu_buffer);
COPY(me.scratchpad); COPY(me.scratchpad);
COPY(me.temp); COPY(me.temp);
......
...@@ -319,8 +319,7 @@ typedef struct MpegEncContext { ...@@ -319,8 +319,7 @@ typedef struct MpegEncContext {
uint8_t *mbintra_table; ///< used to avoid setting {ac, dc, cbp}-pred stuff to zero on inter MB decoding uint8_t *mbintra_table; ///< used to avoid setting {ac, dc, cbp}-pred stuff to zero on inter MB decoding
uint8_t *cbp_table; ///< used to store cbp, ac_pred for partitioned decoding uint8_t *cbp_table; ///< used to store cbp, ac_pred for partitioned decoding
uint8_t *pred_dir_table; ///< used to store pred_dir for partitioned decoding uint8_t *pred_dir_table; ///< used to store pred_dir for partitioned decoding
uint8_t *allocated_edge_emu_buffer; uint8_t *edge_emu_buffer; ///< temporary buffer for if MVs point to out-of-frame data
uint8_t *edge_emu_buffer; ///< points into the middle of allocated_edge_emu_buffer
uint8_t *rd_scratchpad; ///< scratchpad for rate distortion mb decision uint8_t *rd_scratchpad; ///< scratchpad for rate distortion mb decision
uint8_t *obmc_scratchpad; uint8_t *obmc_scratchpad;
uint8_t *b_scratchpad; ///< scratchpad used for writing into write only buffers uint8_t *b_scratchpad; ///< scratchpad used for writing into write only buffers
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment