motion_est.c 67.5 KB
Newer Older
Fabrice Bellard's avatar
Fabrice Bellard committed
1
/*
2
 * Motion estimation
3
 * Copyright (c) 2000,2001 Fabrice Bellard
4
 * Copyright (c) 2002-2004 Michael Niedermayer
5
 *
6
 * new motion estimation (X1/EPZS) by Michael Niedermayer <michaelni@gmx.at>
Fabrice Bellard's avatar
Fabrice Bellard committed
7
 *
8 9 10
 * This file is part of FFmpeg.
 *
 * FFmpeg is free software; you can redistribute it and/or
11 12
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
13
 * version 2.1 of the License, or (at your option) any later version.
Fabrice Bellard's avatar
Fabrice Bellard committed
14
 *
15
 * FFmpeg is distributed in the hope that it will be useful,
Fabrice Bellard's avatar
Fabrice Bellard committed
16
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 18
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
Fabrice Bellard's avatar
Fabrice Bellard committed
19
 *
20
 * You should have received a copy of the GNU Lesser General Public
21
 * License along with FFmpeg; if not, write to the Free Software
22
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
Fabrice Bellard's avatar
Fabrice Bellard committed
23
 */
24

Michael Niedermayer's avatar
Michael Niedermayer committed
25
/**
26
 * @file
Michael Niedermayer's avatar
Michael Niedermayer committed
27 28
 * Motion estimation.
 */
29

Fabrice Bellard's avatar
Fabrice Bellard committed
30 31
#include <stdlib.h>
#include <stdio.h>
32
#include <limits.h>
33

Fabrice Bellard's avatar
Fabrice Bellard committed
34
#include "avcodec.h"
35
#include "mathops.h"
Fabrice Bellard's avatar
Fabrice Bellard committed
36 37
#include "mpegvideo.h"

38 39
#undef NDEBUG
#include <assert.h>
Michael Niedermayer's avatar
Michael Niedermayer committed
40

41 42 43 44 45 46
#define P_LEFT P[1]
#define P_TOP P[2]
#define P_TOPRIGHT P[3]
#define P_MEDIAN P[4]
#define P_MV1 P[9]

47
static int sad_hpel_motion_search(MpegEncContext * s,
48
                                  int *mx_ptr, int *my_ptr, int dmin,
49 50
                                  int src_index, int ref_index,
                                  int size, int h);
Michael Niedermayer's avatar
Michael Niedermayer committed
51

52
static inline unsigned update_map_generation(MotionEstContext *c)
Michael Niedermayer's avatar
Michael Niedermayer committed
53
{
54 55 56 57
    c->map_generation+= 1<<(ME_MAP_MV_BITS*2);
    if(c->map_generation==0){
        c->map_generation= 1<<(ME_MAP_MV_BITS*2);
        memset(c->map, 0, sizeof(uint32_t)*ME_MAP_SIZE);
Michael Niedermayer's avatar
Michael Niedermayer committed
58
    }
59
    return c->map_generation;
Michael Niedermayer's avatar
Michael Niedermayer committed
60 61
}

62 63 64 65 66 67
/* shape adaptive search stuff */
typedef struct Minima{
    int height;
    int x, y;
    int checked;
}Minima;
Michael Niedermayer's avatar
Michael Niedermayer committed
68

69
static int minima_cmp(const void *a, const void *b){
70 71
    const Minima *da = (const Minima *) a;
    const Minima *db = (const Minima *) b;
72

73 74
    return da->height - db->height;
}
Michael Niedermayer's avatar
Michael Niedermayer committed
75

76 77 78
#define FLAG_QPEL   1 //must be 1
#define FLAG_CHROMA 2
#define FLAG_DIRECT 4
Michael Niedermayer's avatar
Michael Niedermayer committed
79

80
static inline void init_ref(MotionEstContext *c, uint8_t *src[3], uint8_t *ref[3], uint8_t *ref2[3], int x, int y, int ref_index){
81 82 83 84 85 86 87 88 89 90 91 92 93 94 95
    const int offset[3]= {
          y*c->  stride + x,
        ((y*c->uvstride + x)>>1),
        ((y*c->uvstride + x)>>1),
    };
    int i;
    for(i=0; i<3; i++){
        c->src[0][i]= src [i] + offset[i];
        c->ref[0][i]= ref [i] + offset[i];
    }
    if(ref_index){
        for(i=0; i<3; i++){
            c->ref[ref_index][i]= ref2[i] + offset[i];
        }
    }
Michael Niedermayer's avatar
Michael Niedermayer committed
96 97
}

98 99
static int get_flags(MotionEstContext *c, int direct, int chroma){
    return   ((c->avctx->flags&CODEC_FLAG_QPEL) ? FLAG_QPEL : 0)
100
           + (direct ? FLAG_DIRECT : 0)
101
           + (chroma ? FLAG_CHROMA : 0);
Michael Niedermayer's avatar
Michael Niedermayer committed
102 103
}

104
static av_always_inline int cmp_direct_inline(MpegEncContext *s, const int x, const int y, const int subx, const int suby,
105
                      const int size, const int h, int ref_index, int src_index,
106
                      me_cmp_func cmp_func, me_cmp_func chroma_cmp_func, int qpel){
107 108 109 110 111 112 113 114
    MotionEstContext * const c= &s->me;
    const int stride= c->stride;
    const int hx= subx + (x<<(1+qpel));
    const int hy= suby + (y<<(1+qpel));
    uint8_t * const * const ref= c->ref[ref_index];
    uint8_t * const * const src= c->src[src_index];
    int d;
    //FIXME check chroma 4mv, (no crashes ...)
115
        av_assert2(x >= c->xmin && hx <= c->xmax<<(qpel+1) && y >= c->ymin && hy <= c->ymax<<(qpel+1));
116 117 118 119 120 121 122 123 124 125 126 127 128
        if(x >= c->xmin && hx <= c->xmax<<(qpel+1) && y >= c->ymin && hy <= c->ymax<<(qpel+1)){
            const int time_pp= s->pp_time;
            const int time_pb= s->pb_time;
            const int mask= 2*qpel+1;
            if(s->mv_type==MV_TYPE_8X8){
                int i;
                for(i=0; i<4; i++){
                    int fx = c->direct_basis_mv[i][0] + hx;
                    int fy = c->direct_basis_mv[i][1] + hy;
                    int bx = hx ? fx - c->co_located_mv[i][0] : c->co_located_mv[i][0]*(time_pb - time_pp)/time_pp + ((i &1)<<(qpel+4));
                    int by = hy ? fy - c->co_located_mv[i][1] : c->co_located_mv[i][1]*(time_pb - time_pp)/time_pp + ((i>>1)<<(qpel+4));
                    int fxy= (fx&mask) + ((fy&mask)<<(qpel+1));
                    int bxy= (bx&mask) + ((by&mask)<<(qpel+1));
129

130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145
                    uint8_t *dst= c->temp + 8*(i&1) + 8*stride*(i>>1);
                    if(qpel){
                        c->qpel_put[1][fxy](dst, ref[0] + (fx>>2) + (fy>>2)*stride, stride);
                        c->qpel_avg[1][bxy](dst, ref[8] + (bx>>2) + (by>>2)*stride, stride);
                    }else{
                        c->hpel_put[1][fxy](dst, ref[0] + (fx>>1) + (fy>>1)*stride, stride, 8);
                        c->hpel_avg[1][bxy](dst, ref[8] + (bx>>1) + (by>>1)*stride, stride, 8);
                    }
                }
            }else{
                int fx = c->direct_basis_mv[0][0] + hx;
                int fy = c->direct_basis_mv[0][1] + hy;
                int bx = hx ? fx - c->co_located_mv[0][0] : (c->co_located_mv[0][0]*(time_pb - time_pp)/time_pp);
                int by = hy ? fy - c->co_located_mv[0][1] : (c->co_located_mv[0][1]*(time_pb - time_pp)/time_pp);
                int fxy= (fx&mask) + ((fy&mask)<<(qpel+1));
                int bxy= (bx&mask) + ((by&mask)<<(qpel+1));
146

147 148 149 150 151 152 153 154 155
                if(qpel){
                    c->qpel_put[1][fxy](c->temp               , ref[0] + (fx>>2) + (fy>>2)*stride               , stride);
                    c->qpel_put[1][fxy](c->temp + 8           , ref[0] + (fx>>2) + (fy>>2)*stride + 8           , stride);
                    c->qpel_put[1][fxy](c->temp     + 8*stride, ref[0] + (fx>>2) + (fy>>2)*stride     + 8*stride, stride);
                    c->qpel_put[1][fxy](c->temp + 8 + 8*stride, ref[0] + (fx>>2) + (fy>>2)*stride + 8 + 8*stride, stride);
                    c->qpel_avg[1][bxy](c->temp               , ref[8] + (bx>>2) + (by>>2)*stride               , stride);
                    c->qpel_avg[1][bxy](c->temp + 8           , ref[8] + (bx>>2) + (by>>2)*stride + 8           , stride);
                    c->qpel_avg[1][bxy](c->temp     + 8*stride, ref[8] + (bx>>2) + (by>>2)*stride     + 8*stride, stride);
                    c->qpel_avg[1][bxy](c->temp + 8 + 8*stride, ref[8] + (bx>>2) + (by>>2)*stride + 8 + 8*stride, stride);
156
                }else{
157 158 159 160 161 162 163 164
                    av_assert2((fx>>1) + 16*s->mb_x >= -16);
                    av_assert2((fy>>1) + 16*s->mb_y >= -16);
                    av_assert2((fx>>1) + 16*s->mb_x <= s->width);
                    av_assert2((fy>>1) + 16*s->mb_y <= s->height);
                    av_assert2((bx>>1) + 16*s->mb_x >= -16);
                    av_assert2((by>>1) + 16*s->mb_y >= -16);
                    av_assert2((bx>>1) + 16*s->mb_x <= s->width);
                    av_assert2((by>>1) + 16*s->mb_y <= s->height);
165 166 167 168 169 170 171 172

                    c->hpel_put[0][fxy](c->temp, ref[0] + (fx>>1) + (fy>>1)*stride, stride, 16);
                    c->hpel_avg[0][bxy](c->temp, ref[8] + (bx>>1) + (by>>1)*stride, stride, 16);
                }
            }
            d = cmp_func(s, c->temp, src[0], stride, 16);
        }else
            d= 256*256*256*32;
173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188
    return d;
}

static av_always_inline int cmp_inline(MpegEncContext *s, const int x, const int y, const int subx, const int suby,
                      const int size, const int h, int ref_index, int src_index,
                      me_cmp_func cmp_func, me_cmp_func chroma_cmp_func, int qpel, int chroma){
    MotionEstContext * const c= &s->me;
    const int stride= c->stride;
    const int uvstride= c->uvstride;
    const int dxy= subx + (suby<<(1+qpel)); //FIXME log2_subpel?
    const int hx= subx + (x<<(1+qpel));
    const int hy= suby + (y<<(1+qpel));
    uint8_t * const * const ref= c->ref[ref_index];
    uint8_t * const * const src= c->src[src_index];
    int d;
    //FIXME check chroma 4mv, (no crashes ...)
189
        int uvdxy;              /* no, it might not be used uninitialized */
190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205
        if(dxy){
            if(qpel){
                c->qpel_put[size][dxy](c->temp, ref[0] + x + y*stride, stride); //FIXME prototype (add h)
                if(chroma){
                    int cx= hx/2;
                    int cy= hy/2;
                    cx= (cx>>1)|(cx&1);
                    cy= (cy>>1)|(cy&1);
                    uvdxy= (cx&1) + 2*(cy&1);
                    //FIXME x/y wrong, but mpeg4 qpel is sick anyway, we should drop as much of it as possible in favor for h264
                }
            }else{
                c->hpel_put[size][dxy](c->temp, ref[0] + x + y*stride, stride, h);
                if(chroma)
                    uvdxy= dxy | (x&1) | (2*(y&1));
            }
206
            d = cmp_func(s, c->temp, src[0], stride, h);
207
        }else{
208
            d = cmp_func(s, src[0], ref[0] + x + y*stride, stride, h);
209 210 211 212 213 214 215
            if(chroma)
                uvdxy= (x&1) + 2*(y&1);
        }
        if(chroma){
            uint8_t * const uvtemp= c->temp + 16*stride;
            c->hpel_put[size+1][uvdxy](uvtemp  , ref[1] + (x>>1) + (y>>1)*uvstride, uvstride, h>>1);
            c->hpel_put[size+1][uvdxy](uvtemp+8, ref[2] + (x>>1) + (y>>1)*uvstride, uvstride, h>>1);
216 217
            d += chroma_cmp_func(s, uvtemp  , src[1], uvstride, h>>1);
            d += chroma_cmp_func(s, uvtemp+8, src[2], uvstride, h>>1);
218
        }
219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234
    return d;
}

static int cmp_simple(MpegEncContext *s, const int x, const int y,
                      int ref_index, int src_index,
                      me_cmp_func cmp_func, me_cmp_func chroma_cmp_func){
    return cmp_inline(s,x,y,0,0,0,16,ref_index,src_index, cmp_func, chroma_cmp_func, 0, 0);
}

static int cmp_fpel_internal(MpegEncContext *s, const int x, const int y,
                      const int size, const int h, int ref_index, int src_index,
                      me_cmp_func cmp_func, me_cmp_func chroma_cmp_func, const int flags){
    if(flags&FLAG_DIRECT){
        return cmp_direct_inline(s,x,y,0,0,size,h,ref_index,src_index, cmp_func, chroma_cmp_func, flags&FLAG_QPEL);
    }else{
        return cmp_inline(s,x,y,0,0,size,h,ref_index,src_index, cmp_func, chroma_cmp_func, 0, flags&FLAG_CHROMA);
235
    }
236 237 238 239 240 241 242 243 244
}

static int cmp_internal(MpegEncContext *s, const int x, const int y, const int subx, const int suby,
                      const int size, const int h, int ref_index, int src_index,
                      me_cmp_func cmp_func, me_cmp_func chroma_cmp_func, const int flags){
    if(flags&FLAG_DIRECT){
        return cmp_direct_inline(s,x,y,subx,suby,size,h,ref_index,src_index, cmp_func, chroma_cmp_func, flags&FLAG_QPEL);
    }else{
        return cmp_inline(s,x,y,subx,suby,size,h,ref_index,src_index, cmp_func, chroma_cmp_func, flags&FLAG_QPEL, flags&FLAG_CHROMA);
245
    }
246
}
247

248
/** @brief compares a block (either a full macroblock or a partition thereof)
249 250
    against a proposed motion-compensated prediction of that block
 */
251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283
static av_always_inline int cmp(MpegEncContext *s, const int x, const int y, const int subx, const int suby,
                      const int size, const int h, int ref_index, int src_index,
                      me_cmp_func cmp_func, me_cmp_func chroma_cmp_func, const int flags){
    if(av_builtin_constant_p(flags) && av_builtin_constant_p(h) && av_builtin_constant_p(size)
       && av_builtin_constant_p(subx) && av_builtin_constant_p(suby)
       && flags==0 && h==16 && size==0 && subx==0 && suby==0){
        return cmp_simple(s,x,y,ref_index,src_index, cmp_func, chroma_cmp_func);
    }else if(av_builtin_constant_p(subx) && av_builtin_constant_p(suby)
       && subx==0 && suby==0){
        return cmp_fpel_internal(s,x,y,size,h,ref_index,src_index, cmp_func, chroma_cmp_func,flags);
    }else{
        return cmp_internal(s,x,y,subx,suby,size,h,ref_index,src_index, cmp_func, chroma_cmp_func, flags);
    }
}

static int cmp_hpel(MpegEncContext *s, const int x, const int y, const int subx, const int suby,
                      const int size, const int h, int ref_index, int src_index,
                      me_cmp_func cmp_func, me_cmp_func chroma_cmp_func, const int flags){
    if(flags&FLAG_DIRECT){
        return cmp_direct_inline(s,x,y,subx,suby,size,h,ref_index,src_index, cmp_func, chroma_cmp_func, 0);
    }else{
        return cmp_inline(s,x,y,subx,suby,size,h,ref_index,src_index, cmp_func, chroma_cmp_func, 0, flags&FLAG_CHROMA);
    }
}

static int cmp_qpel(MpegEncContext *s, const int x, const int y, const int subx, const int suby,
                      const int size, const int h, int ref_index, int src_index,
                      me_cmp_func cmp_func, me_cmp_func chroma_cmp_func, const int flags){
    if(flags&FLAG_DIRECT){
        return cmp_direct_inline(s,x,y,subx,suby,size,h,ref_index,src_index, cmp_func, chroma_cmp_func, 1);
    }else{
        return cmp_inline(s,x,y,subx,suby,size,h,ref_index,src_index, cmp_func, chroma_cmp_func, 1, flags&FLAG_CHROMA);
    }
284
}
Michael Niedermayer's avatar
Michael Niedermayer committed
285 286 287

#include "motion_est_template.c"

Michael Niedermayer's avatar
Michael Niedermayer committed
288 289 290 291
static int zero_cmp(void *s, uint8_t *a, uint8_t *b, int stride, int h){
    return 0;
}

292
static void zero_hpel(uint8_t *a, const uint8_t *b, ptrdiff_t stride, int h){
Michael Niedermayer's avatar
Michael Niedermayer committed
293 294
}

295
int ff_init_me(MpegEncContext *s){
296
    MotionEstContext * const c= &s->me;
297 298
    int cache_size= FFMIN(ME_MAP_SIZE>>ME_MAP_SHIFT, 1<<ME_MAP_SHIFT);
    int dia_size= FFMAX(FFABS(s->avctx->dia_size)&255, FFABS(s->avctx->pre_dia_size)&255);
299

300
    if(FFMIN(s->avctx->dia_size, s->avctx->pre_dia_size) < -FFMIN(ME_MAP_SIZE, MAX_SAB_SIZE)){
301 302 303
        av_log(s->avctx, AV_LOG_ERROR, "ME_MAP size is too small for SAB diamond\n");
        return -1;
    }
304
    //special case of snow is needed because snow uses its own iterative ME code
305
    if(s->me_method!=ME_ZERO && s->me_method!=ME_EPZS && s->me_method!=ME_X1 && s->avctx->codec_id != AV_CODEC_ID_SNOW){
306 307 308
        av_log(s->avctx, AV_LOG_ERROR, "me_method is only allowed to be set to zero and epzs; for hex,umh,full and others see dia_size\n");
        return -1;
    }
309

Michael Niedermayer's avatar
Michael Niedermayer committed
310
    c->avctx= s->avctx;
311 312 313 314

    if(cache_size < 2*dia_size && !c->stride){
        av_log(s->avctx, AV_LOG_INFO, "ME_MAP size may be a little small for the selected diamond size\n");
    }
315

316 317 318 319
    ff_set_cmp(&s->dsp, s->dsp.me_pre_cmp, c->avctx->me_pre_cmp);
    ff_set_cmp(&s->dsp, s->dsp.me_cmp, c->avctx->me_cmp);
    ff_set_cmp(&s->dsp, s->dsp.me_sub_cmp, c->avctx->me_sub_cmp);
    ff_set_cmp(&s->dsp, s->dsp.mb_cmp, c->avctx->mb_cmp);
320

321 322 323
    c->flags    = get_flags(c, 0, c->avctx->me_cmp    &FF_CMP_CHROMA);
    c->sub_flags= get_flags(c, 0, c->avctx->me_sub_cmp&FF_CMP_CHROMA);
    c->mb_flags = get_flags(c, 0, c->avctx->mb_cmp    &FF_CMP_CHROMA);
Fabrice Bellard's avatar
Fabrice Bellard committed
324

325
/*FIXME s->no_rounding b_type*/
Michael Niedermayer's avatar
Michael Niedermayer committed
326
    if(s->flags&CODEC_FLAG_QPEL){
327
        c->sub_motion_search= qpel_motion_search;
328 329 330
        c->qpel_avg= s->dsp.avg_qpel_pixels_tab;
        if(s->no_rounding) c->qpel_put= s->dsp.put_no_rnd_qpel_pixels_tab;
        else               c->qpel_put= s->dsp.put_qpel_pixels_tab;
Michael Niedermayer's avatar
Michael Niedermayer committed
331
    }else{
332
        if(c->avctx->me_sub_cmp&FF_CMP_CHROMA)
333
            c->sub_motion_search= hpel_motion_search;
334 335
        else if(   c->avctx->me_sub_cmp == FF_CMP_SAD
                && c->avctx->    me_cmp == FF_CMP_SAD
336
                && c->avctx->    mb_cmp == FF_CMP_SAD)
337
            c->sub_motion_search= sad_hpel_motion_search; // 2050 vs. 2450 cycles
Michael Niedermayer's avatar
Michael Niedermayer committed
338
        else
339
            c->sub_motion_search= hpel_motion_search;
Michael Niedermayer's avatar
Michael Niedermayer committed
340
    }
341 342 343
    c->hpel_avg= s->hdsp.avg_pixels_tab;
    if(s->no_rounding) c->hpel_put= s->hdsp.put_no_rnd_pixels_tab;
    else               c->hpel_put= s->hdsp.put_pixels_tab;
Michael Niedermayer's avatar
Michael Niedermayer committed
344

345
    if(s->linesize){
346
        c->stride  = s->linesize;
347
        c->uvstride= s->uvlinesize;
348
    }else{
349 350
        c->stride  = 16*s->mb_width + 32;
        c->uvstride=  8*s->mb_width + 16;
351
    }
352

Diego Biurrun's avatar
Diego Biurrun committed
353 354 355
    /* 8x8 fullpel search would need a 4x4 chroma compare, which we do
     * not have yet, and even if we had, the motion estimation code
     * does not expect it. */
356
    if(s->codec_id != AV_CODEC_ID_SNOW){
357
        if((c->avctx->me_cmp&FF_CMP_CHROMA)/* && !s->dsp.me_cmp[2]*/){
358 359 360 361 362 363 364
            s->dsp.me_cmp[2]= zero_cmp;
        }
        if((c->avctx->me_sub_cmp&FF_CMP_CHROMA) && !s->dsp.me_sub_cmp[2]){
            s->dsp.me_sub_cmp[2]= zero_cmp;
        }
        c->hpel_put[2][0]= c->hpel_put[2][1]=
        c->hpel_put[2][2]= c->hpel_put[2][3]= zero_hpel;
Michael Niedermayer's avatar
Michael Niedermayer committed
365 366
    }

367
    if(s->codec_id == AV_CODEC_ID_H261){
368 369 370
        c->sub_motion_search= no_sub_motion_search;
    }

371
    return 0;
Michael Niedermayer's avatar
Michael Niedermayer committed
372
}
373

Michael Niedermayer's avatar
Michael Niedermayer committed
374
#define CHECK_SAD_HALF_MV(suffix, x, y) \
375
{\
376
    d= s->dsp.pix_abs[size][(x?1:0)+(y?2:0)](NULL, pix, ptr+((x)>>1), stride, h);\
Michael Niedermayer's avatar
Michael Niedermayer committed
377
    d += (mv_penalty[pen_x + x] + mv_penalty[pen_y + y])*penalty_factor;\
378 379
    COPY3_IF_LT(dminh, d, dx, x, dy, y)\
}
380

381
static int sad_hpel_motion_search(MpegEncContext * s,
382
                                  int *mx_ptr, int *my_ptr, int dmin,
383 384
                                  int src_index, int ref_index,
                                  int size, int h)
Fabrice Bellard's avatar
Fabrice Bellard committed
385
{
386 387
    MotionEstContext * const c= &s->me;
    const int penalty_factor= c->sub_penalty_factor;
388
    int mx, my, dminh;
389
    uint8_t *pix, *ptr;
390 391
    int stride= c->stride;
    const int flags= c->sub_flags;
392
    LOAD_COMMON
393

394
    av_assert2(flags == 0);
Fabrice Bellard's avatar
Fabrice Bellard committed
395

396
    if(c->skip){
397 398 399 400
        *mx_ptr = 0;
        *my_ptr = 0;
        return dmin;
    }
401

402
    pix = c->src[src_index][0];
403

404 405
    mx = *mx_ptr;
    my = *my_ptr;
406
    ptr = c->ref[ref_index][0] + (my * stride) + mx;
407

408 409
    dminh = dmin;

410
    if (mx > xmin && mx < xmax &&
411
        my > ymin && my < ymax) {
412
        int dx=0, dy=0;
413
        int d, pen_x, pen_y;
414 415 416 417 418 419 420
        const int index= (my<<ME_MAP_SHIFT) + mx;
        const int t= score_map[(index-(1<<ME_MAP_SHIFT))&(ME_MAP_SIZE-1)];
        const int l= score_map[(index- 1               )&(ME_MAP_SIZE-1)];
        const int r= score_map[(index+ 1               )&(ME_MAP_SIZE-1)];
        const int b= score_map[(index+(1<<ME_MAP_SHIFT))&(ME_MAP_SIZE-1)];
        mx<<=1;
        my<<=1;
421

422

423 424 425
        pen_x= pred_x + mx;
        pen_y= pred_y + my;

426
        ptr-= stride;
427
        if(t<=b){
Michael Niedermayer's avatar
Michael Niedermayer committed
428
            CHECK_SAD_HALF_MV(y2 , 0, -1)
429
            if(l<=r){
Michael Niedermayer's avatar
Michael Niedermayer committed
430
                CHECK_SAD_HALF_MV(xy2, -1, -1)
431
                if(t+r<=b+l){
Michael Niedermayer's avatar
Michael Niedermayer committed
432
                    CHECK_SAD_HALF_MV(xy2, +1, -1)
433
                    ptr+= stride;
434
                }else{
435
                    ptr+= stride;
Michael Niedermayer's avatar
Michael Niedermayer committed
436
                    CHECK_SAD_HALF_MV(xy2, -1, +1)
437
                }
Michael Niedermayer's avatar
Michael Niedermayer committed
438
                CHECK_SAD_HALF_MV(x2 , -1,  0)
439
            }else{
Michael Niedermayer's avatar
Michael Niedermayer committed
440
                CHECK_SAD_HALF_MV(xy2, +1, -1)
441
                if(t+l<=b+r){
Michael Niedermayer's avatar
Michael Niedermayer committed
442
                    CHECK_SAD_HALF_MV(xy2, -1, -1)
443
                    ptr+= stride;
444
                }else{
445
                    ptr+= stride;
Michael Niedermayer's avatar
Michael Niedermayer committed
446
                    CHECK_SAD_HALF_MV(xy2, +1, +1)
447
                }
Michael Niedermayer's avatar
Michael Niedermayer committed
448
                CHECK_SAD_HALF_MV(x2 , +1,  0)
449 450 451 452
            }
        }else{
            if(l<=r){
                if(t+l<=b+r){
Michael Niedermayer's avatar
Michael Niedermayer committed
453
                    CHECK_SAD_HALF_MV(xy2, -1, -1)
454
                    ptr+= stride;
455
                }else{
456
                    ptr+= stride;
Michael Niedermayer's avatar
Michael Niedermayer committed
457
                    CHECK_SAD_HALF_MV(xy2, +1, +1)
458
                }
Michael Niedermayer's avatar
Michael Niedermayer committed
459 460
                CHECK_SAD_HALF_MV(x2 , -1,  0)
                CHECK_SAD_HALF_MV(xy2, -1, +1)
461 462
            }else{
                if(t+r<=b+l){
Michael Niedermayer's avatar
Michael Niedermayer committed
463
                    CHECK_SAD_HALF_MV(xy2, +1, -1)
464
                    ptr+= stride;
465
                }else{
466
                    ptr+= stride;
Michael Niedermayer's avatar
Michael Niedermayer committed
467
                    CHECK_SAD_HALF_MV(xy2, -1, +1)
468
                }
Michael Niedermayer's avatar
Michael Niedermayer committed
469 470
                CHECK_SAD_HALF_MV(x2 , +1,  0)
                CHECK_SAD_HALF_MV(xy2, +1, +1)
471
            }
Michael Niedermayer's avatar
Michael Niedermayer committed
472
            CHECK_SAD_HALF_MV(y2 ,  0, +1)
473 474 475
        }
        mx+=dx;
        my+=dy;
476 477

    }else{
478 479
        mx<<=1;
        my<<=1;
480 481 482 483
    }

    *mx_ptr = mx;
    *my_ptr = my;
484
    return dminh;
485 486
}

487
static inline void set_p_mv_tables(MpegEncContext * s, int mx, int my, int mv4)
488
{
489
    const int xy= s->mb_x + s->mb_y*s->mb_stride;
490

491 492
    s->p_mv_table[xy][0] = mx;
    s->p_mv_table[xy][1] = my;
493

Diego Biurrun's avatar
Diego Biurrun committed
494
    /* has already been set to the 4 MV if 4MV is done */
495
    if(mv4){
496 497
        int mot_xy= s->block_index[0];

498 499 500 501
        s->current_picture.motion_val[0][mot_xy    ][0] = mx;
        s->current_picture.motion_val[0][mot_xy    ][1] = my;
        s->current_picture.motion_val[0][mot_xy + 1][0] = mx;
        s->current_picture.motion_val[0][mot_xy + 1][1] = my;
502

503
        mot_xy += s->b8_stride;
504 505 506 507
        s->current_picture.motion_val[0][mot_xy    ][0] = mx;
        s->current_picture.motion_val[0][mot_xy    ][1] = my;
        s->current_picture.motion_val[0][mot_xy + 1][0] = mx;
        s->current_picture.motion_val[0][mot_xy + 1][1] = my;
508 509 510
    }
}

511 512 513
/**
 * get fullpel ME search limits.
 */
514
static inline void get_limits(MpegEncContext *s, int x, int y)
Fabrice Bellard's avatar
Fabrice Bellard committed
515
{
516
    MotionEstContext * const c= &s->me;
517
    int range= c->avctx->me_range >> (1 + !!(c->flags&FLAG_QPEL));
518
    int max_range = MAX_MV >> (1 + !!(c->flags&FLAG_QPEL));
519
/*
520
    if(c->avctx->me_range) c->range= c->avctx->me_range >> 1;
521
    else                   c->range= 16;
522
*/
Fabrice Bellard's avatar
Fabrice Bellard committed
523
    if (s->unrestricted_mv) {
524 525
        c->xmin = - x - 16;
        c->ymin = - y - 16;
526 527
        c->xmax = - x + s->width;
        c->ymax = - y + s->height;
528 529 530 531
    } else if (s->out_format == FMT_H261){
        // Search range of H261 is different from other codec standards
        c->xmin = (x > 15) ? - 15 : 0;
        c->ymin = (y > 15) ? - 15 : 0;
532
        c->xmax = (x < s->mb_width * 16 - 16) ? 15 : 0;
533
        c->ymax = (y < s->mb_height * 16 - 16) ? 15 : 0;
Fabrice Bellard's avatar
Fabrice Bellard committed
534
    } else {
535 536 537 538
        c->xmin = - x;
        c->ymin = - y;
        c->xmax = - x + s->mb_width *16 - 16;
        c->ymax = - y + s->mb_height*16 - 16;
Fabrice Bellard's avatar
Fabrice Bellard committed
539
    }
540 541
    if(!range || range > max_range)
        range = max_range;
542 543 544 545 546 547
    if(range){
        c->xmin = FFMAX(c->xmin,-range);
        c->xmax = FFMIN(c->xmax, range);
        c->ymin = FFMAX(c->ymin,-range);
        c->ymax = FFMIN(c->ymax, range);
    }
548 549
}

550 551
static inline void init_mv4_ref(MotionEstContext *c){
    const int stride= c->stride;
552 553 554 555 556 557 558 559 560

    c->ref[1][0] = c->ref[0][0] + 8;
    c->ref[2][0] = c->ref[0][0] + 8*stride;
    c->ref[3][0] = c->ref[2][0] + 8;
    c->src[1][0] = c->src[0][0] + 8;
    c->src[2][0] = c->src[0][0] + 8*stride;
    c->src[3][0] = c->src[2][0] + 8;
}

561
static inline int h263_mv4_search(MpegEncContext *s, int mx, int my, int shift)
562
{
563
    MotionEstContext * const c= &s->me;
564 565
    const int size= 1;
    const int h=8;
566 567
    int block;
    int P[10][2];
568
    int dmin_sum=0, mx4_sum=0, my4_sum=0, i;
569
    int same=1;
570
    const int stride= c->stride;
571
    uint8_t *mv_penalty= c->current_mv_penalty;
572
    int saftey_cliping= s->unrestricted_mv && (s->width&15) && (s->height&15);
573

574
    init_mv4_ref(c);
575

576 577 578 579 580
    for(block=0; block<4; block++){
        int mx4, my4;
        int pred_x4, pred_y4;
        int dmin4;
        static const int off[4]= {2, 1, 1, -1};
581
        const int mot_stride = s->b8_stride;
582
        const int mot_xy = s->block_index[block];
583

584 585 586 587 588
        if(saftey_cliping){
            c->xmax = - 16*s->mb_x + s->width  - 8*(block &1);
            c->ymax = - 16*s->mb_y + s->height - 8*(block>>1);
        }

589 590
        P_LEFT[0] = s->current_picture.motion_val[0][mot_xy - 1][0];
        P_LEFT[1] = s->current_picture.motion_val[0][mot_xy - 1][1];
591

592
        if(P_LEFT[0]       > (c->xmax<<shift)) P_LEFT[0]       = (c->xmax<<shift);
593 594

        /* special case for first line */
595
        if (s->first_slice_line && block<2) {
596 597
            c->pred_x= pred_x4= P_LEFT[0];
            c->pred_y= pred_y4= P_LEFT[1];
598
        } else {
599 600 601 602
            P_TOP[0]      = s->current_picture.motion_val[0][mot_xy - mot_stride             ][0];
            P_TOP[1]      = s->current_picture.motion_val[0][mot_xy - mot_stride             ][1];
            P_TOPRIGHT[0] = s->current_picture.motion_val[0][mot_xy - mot_stride + off[block]][0];
            P_TOPRIGHT[1] = s->current_picture.motion_val[0][mot_xy - mot_stride + off[block]][1];
603 604 605 606
            if(P_TOP[1]      > (c->ymax<<shift)) P_TOP[1]     = (c->ymax<<shift);
            if(P_TOPRIGHT[0] < (c->xmin<<shift)) P_TOPRIGHT[0]= (c->xmin<<shift);
            if(P_TOPRIGHT[0] > (c->xmax<<shift)) P_TOPRIGHT[0]= (c->xmax<<shift);
            if(P_TOPRIGHT[1] > (c->ymax<<shift)) P_TOPRIGHT[1]= (c->ymax<<shift);
607

608 609 610
            P_MEDIAN[0]= mid_pred(P_LEFT[0], P_TOP[0], P_TOPRIGHT[0]);
            P_MEDIAN[1]= mid_pred(P_LEFT[1], P_TOP[1], P_TOPRIGHT[1]);

611 612
            c->pred_x= pred_x4 = P_MEDIAN[0];
            c->pred_y= pred_y4 = P_MEDIAN[1];
613 614 615
        }
        P_MV1[0]= mx;
        P_MV1[1]= my;
616 617 618 619 620
        if(saftey_cliping)
            for(i=0; i<10; i++){
                if(P[i][0] > (c->xmax<<shift)) P[i][0]= (c->xmax<<shift);
                if(P[i][1] > (c->ymax<<shift)) P[i][1]= (c->ymax<<shift);
            }
621

622
        dmin4 = epzs_motion_search4(s, &mx4, &my4, P, block, block, s->p_mv_table, (1<<16)>>shift);
623

624
        dmin4= c->sub_motion_search(s, &mx4, &my4, dmin4, block, block, size, h);
625

626
        if(s->dsp.me_sub_cmp[0] != s->dsp.mb_cmp[0]){
627
            int dxy;
628
            const int offset= ((block&1) + (block>>1)*stride)*8;
629
            uint8_t *dest_y = c->scratchpad + offset;
630
            if(s->quarter_sample){
631
                uint8_t *ref= c->ref[block][0] + (mx4>>2) + (my4>>2)*stride;
632 633 634
                dxy = ((my4 & 3) << 2) | (mx4 & 3);

                if(s->no_rounding)
635
                    s->dsp.put_no_rnd_qpel_pixels_tab[1][dxy](dest_y   , ref    , stride);
636
                else
637
                    s->dsp.put_qpel_pixels_tab       [1][dxy](dest_y   , ref    , stride);
638
            }else{
639
                uint8_t *ref= c->ref[block][0] + (mx4>>1) + (my4>>1)*stride;
640 641 642
                dxy = ((my4 & 1) << 1) | (mx4 & 1);

                if(s->no_rounding)
643
                    s->hdsp.put_no_rnd_pixels_tab[1][dxy](dest_y    , ref    , stride, h);
644
                else
645
                    s->hdsp.put_pixels_tab       [1][dxy](dest_y    , ref    , stride, h);
646
            }
647
            dmin_sum+= (mv_penalty[mx4-pred_x4] + mv_penalty[my4-pred_y4])*c->mb_penalty_factor;
648 649 650 651 652 653 654 655 656 657
        }else
            dmin_sum+= dmin4;

        if(s->quarter_sample){
            mx4_sum+= mx4/2;
            my4_sum+= my4/2;
        }else{
            mx4_sum+= mx4;
            my4_sum+= my4;
        }
658

659 660
        s->current_picture.motion_val[0][s->block_index[block]][0] = mx4;
        s->current_picture.motion_val[0][s->block_index[block]][1] = my4;
661 662

        if(mx4 != mx || my4 != my) same=0;
663
    }
664

665 666
    if(same)
        return INT_MAX;
667

668
    if(s->dsp.me_sub_cmp[0] != s->dsp.mb_cmp[0]){
669
        dmin_sum += s->dsp.mb_cmp[0](s, s->new_picture.f.data[0] + s->mb_x*16 + s->mb_y*16*stride, c->scratchpad, stride, 16);
670
    }
671

672
    if(c->avctx->mb_cmp&FF_CMP_CHROMA){
673 674 675 676 677 678 679
        int dxy;
        int mx, my;
        int offset;

        mx= ff_h263_round_chroma(mx4_sum);
        my= ff_h263_round_chroma(my4_sum);
        dxy = ((my & 1) << 1) | (mx & 1);
680

681
        offset= (s->mb_x*8 + (mx>>1)) + (s->mb_y*8 + (my>>1))*s->uvlinesize;
682

683
        if(s->no_rounding){
684 685
            s->hdsp.put_no_rnd_pixels_tab[1][dxy](c->scratchpad    , s->last_picture.f.data[1] + offset, s->uvlinesize, 8);
            s->hdsp.put_no_rnd_pixels_tab[1][dxy](c->scratchpad + 8, s->last_picture.f.data[2] + offset, s->uvlinesize, 8);
686
        }else{
687 688
            s->hdsp.put_pixels_tab       [1][dxy](c->scratchpad    , s->last_picture.f.data[1] + offset, s->uvlinesize, 8);
            s->hdsp.put_pixels_tab       [1][dxy](c->scratchpad + 8, s->last_picture.f.data[2] + offset, s->uvlinesize, 8);
689 690
        }

691 692
        dmin_sum += s->dsp.mb_cmp[1](s, s->new_picture.f.data[1] + s->mb_x*8 + s->mb_y*8*s->uvlinesize, c->scratchpad  , s->uvlinesize, 8);
        dmin_sum += s->dsp.mb_cmp[1](s, s->new_picture.f.data[2] + s->mb_x*8 + s->mb_y*8*s->uvlinesize, c->scratchpad+8, s->uvlinesize, 8);
693
    }
694

695 696
    c->pred_x= mx;
    c->pred_y= my;
697

698
    switch(c->avctx->mb_cmp&0xFF){
699 700 701 702 703
    /*case FF_CMP_SSE:
        return dmin_sum+ 32*s->qscale*s->qscale;*/
    case FF_CMP_RD:
        return dmin_sum;
    default:
704
        return dmin_sum+ 11*c->mb_penalty_factor;
705
    }
706 707
}

708 709 710 711 712 713 714 715 716 717 718 719 720
static inline void init_interlaced_ref(MpegEncContext *s, int ref_index){
    MotionEstContext * const c= &s->me;

    c->ref[1+ref_index][0] = c->ref[0+ref_index][0] + s->linesize;
    c->src[1][0] = c->src[0][0] + s->linesize;
    if(c->flags & FLAG_CHROMA){
        c->ref[1+ref_index][1] = c->ref[0+ref_index][1] + s->uvlinesize;
        c->ref[1+ref_index][2] = c->ref[0+ref_index][2] + s->uvlinesize;
        c->src[1][1] = c->src[0][1] + s->uvlinesize;
        c->src[1][2] = c->src[0][2] + s->uvlinesize;
    }
}

721
static int interlaced_search(MpegEncContext *s, int ref_index,
722
                             int16_t (*mv_tables[2][2])[2], uint8_t *field_select_tables[2], int mx, int my, int user_field_select)
723
{
724
    MotionEstContext * const c= &s->me;
725 726 727 728
    const int size=0;
    const int h=8;
    int block;
    int P[10][2];
729
    uint8_t * const mv_penalty= c->current_mv_penalty;
730 731 732 733 734
    int same=1;
    const int stride= 2*s->linesize;
    int dmin_sum= 0;
    const int mot_stride= s->mb_stride;
    const int xy= s->mb_x + s->mb_y*mot_stride;
735

736 737 738 739
    c->ymin>>=1;
    c->ymax>>=1;
    c->stride<<=1;
    c->uvstride<<=1;
740
    init_interlaced_ref(s, ref_index);
741

742 743 744 745 746 747
    for(block=0; block<2; block++){
        int field_select;
        int best_dmin= INT_MAX;
        int best_field= -1;

        for(field_select=0; field_select<2; field_select++){
748
            int dmin, mx_i, my_i;
749
            int16_t (*mv_table)[2]= mv_tables[block][field_select];
750

751
            if(user_field_select){
752 753
                av_assert1(field_select==0 || field_select==1);
                av_assert1(field_select_tables[block][xy]==0 || field_select_tables[block][xy]==1);
754 755 756
                if(field_select_tables[block][xy] != field_select)
                    continue;
            }
757

758 759
            P_LEFT[0] = mv_table[xy - 1][0];
            P_LEFT[1] = mv_table[xy - 1][1];
760
            if(P_LEFT[0]       > (c->xmax<<1)) P_LEFT[0]       = (c->xmax<<1);
761

762 763
            c->pred_x= P_LEFT[0];
            c->pred_y= P_LEFT[1];
764

765
            if(!s->first_slice_line){
766 767 768 769
                P_TOP[0]      = mv_table[xy - mot_stride][0];
                P_TOP[1]      = mv_table[xy - mot_stride][1];
                P_TOPRIGHT[0] = mv_table[xy - mot_stride + 1][0];
                P_TOPRIGHT[1] = mv_table[xy - mot_stride + 1][1];
770 771 772 773
                if(P_TOP[1]      > (c->ymax<<1)) P_TOP[1]     = (c->ymax<<1);
                if(P_TOPRIGHT[0] < (c->xmin<<1)) P_TOPRIGHT[0]= (c->xmin<<1);
                if(P_TOPRIGHT[0] > (c->xmax<<1)) P_TOPRIGHT[0]= (c->xmax<<1);
                if(P_TOPRIGHT[1] > (c->ymax<<1)) P_TOPRIGHT[1]= (c->ymax<<1);
774

775 776 777 778 779
                P_MEDIAN[0]= mid_pred(P_LEFT[0], P_TOP[0], P_TOPRIGHT[0]);
                P_MEDIAN[1]= mid_pred(P_LEFT[1], P_TOP[1], P_TOPRIGHT[1]);
            }
            P_MV1[0]= mx; //FIXME not correct if block != field_select
            P_MV1[1]= my / 2;
780

781
            dmin = epzs_motion_search2(s, &mx_i, &my_i, P, block, field_select+ref_index, mv_table, (1<<16)>>1);
782

783
            dmin= c->sub_motion_search(s, &mx_i, &my_i, dmin, block, field_select+ref_index, size, h);
784

785 786
            mv_table[xy][0]= mx_i;
            mv_table[xy][1]= my_i;
787

788
            if(s->dsp.me_sub_cmp[0] != s->dsp.mb_cmp[0]){
789 790 791
                int dxy;

                //FIXME chroma ME
792
                uint8_t *ref= c->ref[field_select+ref_index][0] + (mx_i>>1) + (my_i>>1)*stride;
793 794 795
                dxy = ((my_i & 1) << 1) | (mx_i & 1);

                if(s->no_rounding){
796
                    s->hdsp.put_no_rnd_pixels_tab[size][dxy](c->scratchpad, ref    , stride, h);
797
                }else{
798
                    s->hdsp.put_pixels_tab       [size][dxy](c->scratchpad, ref    , stride, h);
799
                }
800
                dmin= s->dsp.mb_cmp[size](s, c->src[block][0], c->scratchpad, stride, h);
801
                dmin+= (mv_penalty[mx_i-c->pred_x] + mv_penalty[my_i-c->pred_y] + 1)*c->mb_penalty_factor;
802
            }else
803
                dmin+= c->mb_penalty_factor; //field_select bits
804

805
            dmin += field_select != block; //slightly prefer same field
806

807 808 809 810 811 812 813 814 815 816
            if(dmin < best_dmin){
                best_dmin= dmin;
                best_field= field_select;
            }
        }
        {
            int16_t (*mv_table)[2]= mv_tables[block][best_field];

            if(mv_table[xy][0] != mx) same=0; //FIXME check if these checks work and are any good at all
            if(mv_table[xy][1]&1) same=0;
817
            if(mv_table[xy][1]*2 != my) same=0;
818 819 820 821 822 823
            if(best_field != block) same=0;
        }

        field_select_tables[block][xy]= best_field;
        dmin_sum += best_dmin;
    }
824

825 826 827 828
    c->ymin<<=1;
    c->ymax<<=1;
    c->stride>>=1;
    c->uvstride>>=1;
829 830 831

    if(same)
        return INT_MAX;
832

833
    switch(c->avctx->mb_cmp&0xFF){
834 835 836 837 838
    /*case FF_CMP_SSE:
        return dmin_sum+ 32*s->qscale*s->qscale;*/
    case FF_CMP_RD:
        return dmin_sum;
    default:
839
        return dmin_sum+ 11*c->mb_penalty_factor;
840 841 842
    }
}

843 844 845 846 847 848 849
static inline int get_penalty_factor(int lambda, int lambda2, int type){
    switch(type&0xFF){
    default:
    case FF_CMP_SAD:
        return lambda>>FF_LAMBDA_SHIFT;
    case FF_CMP_DCT:
        return (3*lambda)>>(FF_LAMBDA_SHIFT+1);
850 851 852 853
    case FF_CMP_W53:
        return (4*lambda)>>(FF_LAMBDA_SHIFT);
    case FF_CMP_W97:
        return (2*lambda)>>(FF_LAMBDA_SHIFT);
854 855 856 857 858 859 860 861 862 863 864 865 866
    case FF_CMP_SATD:
    case FF_CMP_DCT264:
        return (2*lambda)>>FF_LAMBDA_SHIFT;
    case FF_CMP_RD:
    case FF_CMP_PSNR:
    case FF_CMP_SSE:
    case FF_CMP_NSSE:
        return lambda2>>FF_LAMBDA_SHIFT;
    case FF_CMP_BIT:
        return 1;
    }
}

867 868 869
void ff_estimate_p_frame_motion(MpegEncContext * s,
                                int mb_x, int mb_y)
{
870
    MotionEstContext * const c= &s->me;
871
    uint8_t *pix, *ppix;
872 873 874
    int sum, mx, my, dmin;
    int varc;            ///< the variance of the block (sum of squared (p[y][x]-average))
    int vard;            ///< sum of squared differences with the estimated motion vector
875
    int P[10][2];
876 877
    const int shift= 1+s->quarter_sample;
    int mb_type=0;
Michael Niedermayer's avatar
Michael Niedermayer committed
878
    Picture * const pic= &s->current_picture;
879

880
    init_ref(c, s->new_picture.f.data, s->last_picture.f.data, NULL, 16*mb_x, 16*mb_y, 0);
881

882 883 884
    av_assert0(s->quarter_sample==0 || s->quarter_sample==1);
    av_assert0(s->linesize == c->stride);
    av_assert0(s->uvlinesize == c->uvstride);
Michael Niedermayer's avatar
Michael Niedermayer committed
885

886 887 888
    c->penalty_factor    = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp);
    c->sub_penalty_factor= get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_sub_cmp);
    c->mb_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->mb_cmp);
889
    c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV;
890

891
    get_limits(s, 16*mb_x, 16*mb_y);
892
    c->skip=0;
893

894 895 896
    /* intra / predictive decision */
    pix = c->src[0][0];
    sum = s->dsp.pix_sum(pix, s->linesize);
897
    varc = s->dsp.pix_norm1(pix, s->linesize) - (((unsigned)sum*sum)>>8) + 500;
898 899

    pic->mb_mean[s->mb_stride * mb_y + mb_x] = (sum+128)>>8;
900 901
    pic->mb_var [s->mb_stride * mb_y + mb_x] = (varc+128)>>8;
    c->mb_var_sum_temp += (varc+128)>>8;
902

903
    switch(s->me_method) {
Fabrice Bellard's avatar
Fabrice Bellard committed
904 905
    case ME_ZERO:
    default:
906 907
        mx   = 0;
        my   = 0;
Fabrice Bellard's avatar
Fabrice Bellard committed
908 909
        dmin = 0;
        break;
910
    case ME_X1:
911
    case ME_EPZS:
912
       {
913
            const int mot_stride = s->b8_stride;
914
            const int mot_xy = s->block_index[0];
915

916 917
            P_LEFT[0] = s->current_picture.motion_val[0][mot_xy - 1][0];
            P_LEFT[1] = s->current_picture.motion_val[0][mot_xy - 1][1];
918

919
            if(P_LEFT[0]       > (c->xmax<<shift)) P_LEFT[0]       = (c->xmax<<shift);
920

921
            if(!s->first_slice_line) {
922 923 924 925
                P_TOP[0]      = s->current_picture.motion_val[0][mot_xy - mot_stride    ][0];
                P_TOP[1]      = s->current_picture.motion_val[0][mot_xy - mot_stride    ][1];
                P_TOPRIGHT[0] = s->current_picture.motion_val[0][mot_xy - mot_stride + 2][0];
                P_TOPRIGHT[1] = s->current_picture.motion_val[0][mot_xy - mot_stride + 2][1];
926 927 928
                if(P_TOP[1]      > (c->ymax<<shift)) P_TOP[1]     = (c->ymax<<shift);
                if(P_TOPRIGHT[0] < (c->xmin<<shift)) P_TOPRIGHT[0]= (c->xmin<<shift);
                if(P_TOPRIGHT[1] > (c->ymax<<shift)) P_TOPRIGHT[1]= (c->ymax<<shift);
929

930 931 932 933
                P_MEDIAN[0]= mid_pred(P_LEFT[0], P_TOP[0], P_TOPRIGHT[0]);
                P_MEDIAN[1]= mid_pred(P_LEFT[1], P_TOP[1], P_TOPRIGHT[1]);

                if(s->out_format == FMT_H263){
934 935
                    c->pred_x = P_MEDIAN[0];
                    c->pred_y = P_MEDIAN[1];
936
                }else { /* mpeg1 at least */
937 938
                    c->pred_x= P_LEFT[0];
                    c->pred_y= P_LEFT[1];
939
                }
940
            }else{
941 942
                c->pred_x= P_LEFT[0];
                c->pred_y= P_LEFT[1];
943
            }
944

945
        }
946
        dmin = ff_epzs_motion_search(s, &mx, &my, P, 0, 0, s->p_mv_table, (1<<16)>>shift, 0, 16);
947

948
        break;
Fabrice Bellard's avatar
Fabrice Bellard committed
949 950
    }

951
    /* At this point (mx,my) are full-pell and the relative displacement */
952
    ppix = c->ref[0][0] + (my * s->linesize) + mx;
953

954
    vard = s->dsp.sse[0](NULL, pix, ppix, s->linesize, 16);
955

956 957
    pic->mc_mb_var[s->mb_stride * mb_y + mb_x] = (vard+128)>>8;
    c->mc_mb_var_sum_temp += (vard+128)>>8;
958

959
    if(mb_type){
960 961 962
        int p_score= FFMIN(vard, varc-500+(s->lambda2>>FF_LAMBDA_SHIFT)*100);
        int i_score= varc-500+(s->lambda2>>FF_LAMBDA_SHIFT)*20;
        c->scene_change_score+= ff_sqrt(p_score) - ff_sqrt(i_score);
963 964

        if(mb_type == CANDIDATE_MB_TYPE_INTER){
965
            c->sub_motion_search(s, &mx, &my, dmin, 0, 0, 0, 16);
966 967 968 969 970 971 972 973 974 975 976 977 978
            set_p_mv_tables(s, mx, my, 1);
        }else{
            mx <<=shift;
            my <<=shift;
        }
        if(mb_type == CANDIDATE_MB_TYPE_INTER4V){
            h263_mv4_search(s, mx, my, shift);

            set_p_mv_tables(s, mx, my, 0);
        }
        if(mb_type == CANDIDATE_MB_TYPE_INTER_I){
            interlaced_search(s, 0, s->p_field_mv_table, s->p_field_select_table, mx, my, 1);
        }
979
    }else if(c->avctx->mb_decision > FF_MB_DECISION_SIMPLE){
980 981 982
        int p_score= FFMIN(vard, varc-500+(s->lambda2>>FF_LAMBDA_SHIFT)*100);
        int i_score= varc-500+(s->lambda2>>FF_LAMBDA_SHIFT)*20;
        c->scene_change_score+= ff_sqrt(p_score) - ff_sqrt(i_score);
983

984
        if (vard*2 + 200*256 > varc)
985
            mb_type|= CANDIDATE_MB_TYPE_INTRA;
986 987
        if (varc*2 + 200*256 > vard || s->qscale > 24){
//        if (varc*2 + 200*256 + 50*(s->lambda2>>FF_LAMBDA_SHIFT) > vard){
988
            mb_type|= CANDIDATE_MB_TYPE_INTER;
989
            c->sub_motion_search(s, &mx, &my, dmin, 0, 0, 0, 16);
990 991
            if(s->flags&CODEC_FLAG_MV0)
                if(mx || my)
992
                    mb_type |= CANDIDATE_MB_TYPE_SKIPPED; //FIXME check difference
993
        }else{
Michael Niedermayer's avatar
Michael Niedermayer committed
994 995
            mx <<=shift;
            my <<=shift;
Fabrice Bellard's avatar
Fabrice Bellard committed
996
        }
997
        if((s->flags&CODEC_FLAG_4MV)
998
           && !c->skip && varc>50<<8 && vard>10<<8){
999 1000
            if(h263_mv4_search(s, mx, my, shift) < INT_MAX)
                mb_type|=CANDIDATE_MB_TYPE_INTER4V;
1001 1002 1003 1004

            set_p_mv_tables(s, mx, my, 0);
        }else
            set_p_mv_tables(s, mx, my, 1);
1005
        if((s->flags&CODEC_FLAG_INTERLACED_ME)
1006
           && !c->skip){ //FIXME varc/d checks
1007
            if(interlaced_search(s, 0, s->p_field_mv_table, s->p_field_select_table, mx, my, 0) < INT_MAX)
1008 1009
                mb_type |= CANDIDATE_MB_TYPE_INTER_I;
        }
1010
    }else{
1011
        int intra_score, i;
1012
        mb_type= CANDIDATE_MB_TYPE_INTER;
1013

1014
        dmin= c->sub_motion_search(s, &mx, &my, dmin, 0, 0, 0, 16);
1015
        if(c->avctx->me_sub_cmp != c->avctx->mb_cmp && !c->skip)
1016
            dmin= get_mb_score(s, mx, my, 0, 0, 0, 16, 1);
1017 1018

        if((s->flags&CODEC_FLAG_4MV)
1019
           && !c->skip && varc>50<<8 && vard>10<<8){
1020
            int dmin4= h263_mv4_search(s, mx, my, shift);
1021
            if(dmin4 < dmin){
1022
                mb_type= CANDIDATE_MB_TYPE_INTER4V;
1023
                dmin=dmin4;
1024
            }
1025
        }
1026
        if((s->flags&CODEC_FLAG_INTERLACED_ME)
1027
           && !c->skip){ //FIXME varc/d checks
1028
            int dmin_i= interlaced_search(s, 0, s->p_field_mv_table, s->p_field_select_table, mx, my, 0);
1029 1030 1031 1032 1033
            if(dmin_i < dmin){
                mb_type = CANDIDATE_MB_TYPE_INTER_I;
                dmin= dmin_i;
            }
        }
1034

1035
        set_p_mv_tables(s, mx, my, mb_type!=CANDIDATE_MB_TYPE_INTER4V);
1036 1037

        /* get intra luma score */
1038
        if((c->avctx->mb_cmp&0xFF)==FF_CMP_SSE){
1039
            intra_score= varc - 500;
1040
        }else{
1041
            unsigned mean = (sum+128)>>8;
1042
            mean*= 0x01010101;
1043

1044
            for(i=0; i<16; i++){
1045 1046 1047 1048
                *(uint32_t*)(&c->scratchpad[i*s->linesize+ 0]) = mean;
                *(uint32_t*)(&c->scratchpad[i*s->linesize+ 4]) = mean;
                *(uint32_t*)(&c->scratchpad[i*s->linesize+ 8]) = mean;
                *(uint32_t*)(&c->scratchpad[i*s->linesize+12]) = mean;
1049 1050
            }

1051
            intra_score= s->dsp.mb_cmp[0](s, c->scratchpad, pix, s->linesize, 16);
1052
        }
1053
        intra_score += c->mb_penalty_factor*16;
1054

1055
        if(intra_score < dmin){
1056
            mb_type= CANDIDATE_MB_TYPE_INTRA;
1057
            s->current_picture.mb_type[mb_y*s->mb_stride + mb_x] = CANDIDATE_MB_TYPE_INTRA; //FIXME cleanup
1058
        }else
1059
            s->current_picture.mb_type[mb_y*s->mb_stride + mb_x] = 0;
1060

1061 1062 1063 1064
        {
            int p_score= FFMIN(vard, varc-500+(s->lambda2>>FF_LAMBDA_SHIFT)*100);
            int i_score= varc-500+(s->lambda2>>FF_LAMBDA_SHIFT)*20;
            c->scene_change_score+= ff_sqrt(p_score) - ff_sqrt(i_score);
1065
        }
Fabrice Bellard's avatar
Fabrice Bellard committed
1066
    }
1067

1068
    s->mb_type[mb_y*s->mb_stride + mb_x]= mb_type;
1069 1070
}

1071 1072 1073
int ff_pre_estimate_p_frame_motion(MpegEncContext * s,
                                    int mb_x, int mb_y)
{
1074
    MotionEstContext * const c= &s->me;
1075
    int mx, my, dmin;
1076 1077
    int P[10][2];
    const int shift= 1+s->quarter_sample;
1078
    const int xy= mb_x + mb_y*s->mb_stride;
1079
    init_ref(c, s->new_picture.f.data, s->last_picture.f.data, NULL, 16*mb_x, 16*mb_y, 0);
1080

1081
    av_assert0(s->quarter_sample==0 || s->quarter_sample==1);
1082

1083
    c->pre_penalty_factor    = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_pre_cmp);
1084
    c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV;
1085

1086
    get_limits(s, 16*mb_x, 16*mb_y);
1087
    c->skip=0;
1088 1089 1090 1091

    P_LEFT[0]       = s->p_mv_table[xy + 1][0];
    P_LEFT[1]       = s->p_mv_table[xy + 1][1];

1092
    if(P_LEFT[0]       < (c->xmin<<shift)) P_LEFT[0]       = (c->xmin<<shift);
1093 1094

    /* special case for first line */
1095
    if (s->first_slice_line) {
1096 1097
        c->pred_x= P_LEFT[0];
        c->pred_y= P_LEFT[1];
1098
        P_TOP[0]= P_TOPRIGHT[0]= P_MEDIAN[0]=
1099
        P_TOP[1]= P_TOPRIGHT[1]= P_MEDIAN[1]= 0; //FIXME
1100
    } else {
1101 1102 1103 1104
        P_TOP[0]      = s->p_mv_table[xy + s->mb_stride    ][0];
        P_TOP[1]      = s->p_mv_table[xy + s->mb_stride    ][1];
        P_TOPRIGHT[0] = s->p_mv_table[xy + s->mb_stride - 1][0];
        P_TOPRIGHT[1] = s->p_mv_table[xy + s->mb_stride - 1][1];
1105 1106 1107
        if(P_TOP[1]      < (c->ymin<<shift)) P_TOP[1]     = (c->ymin<<shift);
        if(P_TOPRIGHT[0] > (c->xmax<<shift)) P_TOPRIGHT[0]= (c->xmax<<shift);
        if(P_TOPRIGHT[1] < (c->ymin<<shift)) P_TOPRIGHT[1]= (c->ymin<<shift);
1108

1109 1110 1111
        P_MEDIAN[0]= mid_pred(P_LEFT[0], P_TOP[0], P_TOPRIGHT[0]);
        P_MEDIAN[1]= mid_pred(P_LEFT[1], P_TOP[1], P_TOPRIGHT[1]);

1112 1113
        c->pred_x = P_MEDIAN[0];
        c->pred_y = P_MEDIAN[1];
1114
    }
1115

1116
    dmin = ff_epzs_motion_search(s, &mx, &my, P, 0, 0, s->p_mv_table, (1<<16)>>shift, 0, 16);
1117

1118 1119
    s->p_mv_table[xy][0] = mx<<shift;
    s->p_mv_table[xy][1] = my<<shift;
1120

1121 1122 1123
    return dmin;
}

1124
static int ff_estimate_motion_b(MpegEncContext * s,
1125
                       int mb_x, int mb_y, int16_t (*mv_table)[2], int ref_index, int f_code)
1126
{
1127
    MotionEstContext * const c= &s->me;
1128
    int mx, my, dmin;
1129
    int P[10][2];
1130
    const int shift= 1+s->quarter_sample;
1131 1132
    const int mot_stride = s->mb_stride;
    const int mot_xy = mb_y*mot_stride + mb_x;
1133
    uint8_t * const mv_penalty= c->mv_penalty[f_code] + MAX_MV;
1134
    int mv_scale;
1135

1136 1137 1138
    c->penalty_factor    = get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_cmp);
    c->sub_penalty_factor= get_penalty_factor(s->lambda, s->lambda2, c->avctx->me_sub_cmp);
    c->mb_penalty_factor = get_penalty_factor(s->lambda, s->lambda2, c->avctx->mb_cmp);
1139
    c->current_mv_penalty= mv_penalty;
Michael Niedermayer's avatar
Michael Niedermayer committed
1140

1141
    get_limits(s, 16*mb_x, 16*mb_y);
1142 1143 1144 1145

    switch(s->me_method) {
    case ME_ZERO:
    default:
1146 1147
        mx   = 0;
        my   = 0;
1148 1149 1150 1151
        dmin = 0;
        break;
    case ME_X1:
    case ME_EPZS:
1152 1153
        P_LEFT[0] = mv_table[mot_xy - 1][0];
        P_LEFT[1] = mv_table[mot_xy - 1][1];
1154

1155
        if (P_LEFT[0] > (c->xmax << shift)) P_LEFT[0] = (c->xmax << shift);
1156

1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168
        /* special case for first line */
        if (!s->first_slice_line) {
            P_TOP[0]      = mv_table[mot_xy - mot_stride    ][0];
            P_TOP[1]      = mv_table[mot_xy - mot_stride    ][1];
            P_TOPRIGHT[0] = mv_table[mot_xy - mot_stride + 1][0];
            P_TOPRIGHT[1] = mv_table[mot_xy - mot_stride + 1][1];
            if (P_TOP[1] > (c->ymax << shift)) P_TOP[1] = (c->ymax << shift);
            if (P_TOPRIGHT[0] < (c->xmin << shift)) P_TOPRIGHT[0] = (c->xmin << shift);
            if (P_TOPRIGHT[1] > (c->ymax << shift)) P_TOPRIGHT[1] = (c->ymax << shift);

            P_MEDIAN[0] = mid_pred(P_LEFT[0], P_TOP[0], P_TOPRIGHT[0]);
            P_MEDIAN[1] = mid_pred(P_LEFT[1], P_TOP[1], P_TOPRIGHT[1]);
1169
        }
1170 1171
        c->pred_x = P_LEFT[0];
        c->pred_y = P_LEFT[1];
1172

1173 1174 1175 1176 1177
        if(mv_table == s->b_forw_mv_table){
            mv_scale= (s->pb_time<<16) / (s->pp_time<<shift);
        }else{
            mv_scale= ((s->pb_time - s->pp_time)<<16) / (s->pp_time<<shift);
        }
1178

1179
        dmin = ff_epzs_motion_search(s, &mx, &my, P, 0, ref_index, s->p_mv_table, mv_scale, 0, 16);
1180

1181 1182
        break;
    }
1183

1184
    dmin= c->sub_motion_search(s, &mx, &my, dmin, 0, ref_index, 0, 16);
1185

1186
    if(c->avctx->me_sub_cmp != c->avctx->mb_cmp && !c->skip)
1187
        dmin= get_mb_score(s, mx, my, 0, ref_index, 0, 16, 1);
1188

1189 1190 1191
//    s->mb_type[mb_y*s->mb_width + mb_x]= mb_type;
    mv_table[mot_xy][0]= mx;
    mv_table[mot_xy][1]= my;
Michael Niedermayer's avatar
Michael Niedermayer committed
1192

1193
    return dmin;
Fabrice Bellard's avatar
Fabrice Bellard committed
1194 1195
}

1196
static inline int check_bidir_mv(MpegEncContext * s,
1197 1198 1199
                   int motion_fx, int motion_fy,
                   int motion_bx, int motion_by,
                   int pred_fx, int pred_fy,
1200 1201
                   int pred_bx, int pred_by,
                   int size, int h)
Fabrice Bellard's avatar
Fabrice Bellard committed
1202
{
1203
    //FIXME optimize?
Michael Niedermayer's avatar
Michael Niedermayer committed
1204
    //FIXME better f_code prediction (max mv & distance)
1205
    //FIXME pointers
1206
    MotionEstContext * const c= &s->me;
Michael Niedermayer's avatar
Michael Niedermayer committed
1207 1208
    uint8_t * const mv_penalty_f= c->mv_penalty[s->f_code] + MAX_MV; // f_code of the prev frame
    uint8_t * const mv_penalty_b= c->mv_penalty[s->b_code] + MAX_MV; // f_code of the prev frame
1209 1210
    int stride= c->stride;
    uint8_t *dest_y = c->scratchpad;
1211 1212 1213 1214
    uint8_t *ptr;
    int dxy;
    int src_x, src_y;
    int fbmin;
1215 1216 1217
    uint8_t **src_data= c->src[0];
    uint8_t **ref_data= c->ref[0];
    uint8_t **ref2_data= c->ref[2];
1218

Michael Niedermayer's avatar
Michael Niedermayer committed
1219 1220
    if(s->quarter_sample){
        dxy = ((motion_fy & 3) << 2) | (motion_fx & 3);
1221 1222
        src_x = motion_fx >> 2;
        src_y = motion_fy >> 2;
Michael Niedermayer's avatar
Michael Niedermayer committed
1223

1224 1225
        ptr = ref_data[0] + (src_y * stride) + src_x;
        s->dsp.put_qpel_pixels_tab[0][dxy](dest_y    , ptr    , stride);
Michael Niedermayer's avatar
Michael Niedermayer committed
1226 1227

        dxy = ((motion_by & 3) << 2) | (motion_bx & 3);
1228 1229
        src_x = motion_bx >> 2;
        src_y = motion_by >> 2;
1230

1231
        ptr = ref2_data[0] + (src_y * stride) + src_x;
1232
        s->dsp.avg_qpel_pixels_tab[size][dxy](dest_y    , ptr    , stride);
Michael Niedermayer's avatar
Michael Niedermayer committed
1233 1234
    }else{
        dxy = ((motion_fy & 1) << 1) | (motion_fx & 1);
1235 1236
        src_x = motion_fx >> 1;
        src_y = motion_fy >> 1;
Michael Niedermayer's avatar
Michael Niedermayer committed
1237

1238
        ptr = ref_data[0] + (src_y * stride) + src_x;
1239
        s->hdsp.put_pixels_tab[size][dxy](dest_y    , ptr    , stride, h);
Michael Niedermayer's avatar
Michael Niedermayer committed
1240 1241

        dxy = ((motion_by & 1) << 1) | (motion_bx & 1);
1242 1243
        src_x = motion_bx >> 1;
        src_y = motion_by >> 1;
1244

1245
        ptr = ref2_data[0] + (src_y * stride) + src_x;
1246
        s->hdsp.avg_pixels_tab[size][dxy](dest_y    , ptr    , stride, h);
Michael Niedermayer's avatar
Michael Niedermayer committed
1247 1248
    }

Michael Niedermayer's avatar
Michael Niedermayer committed
1249 1250
    fbmin = (mv_penalty_f[motion_fx-pred_fx] + mv_penalty_f[motion_fy-pred_fy])*c->mb_penalty_factor
           +(mv_penalty_b[motion_bx-pred_bx] + mv_penalty_b[motion_by-pred_by])*c->mb_penalty_factor
1251
           + s->dsp.mb_cmp[size](s, src_data[0], dest_y, stride, h); //FIXME new_pic
1252

1253
    if(c->avctx->mb_cmp&FF_CMP_CHROMA){
1254 1255
    }
    //FIXME CHROMA !!!
1256

1257 1258
    return fbmin;
}
1259

1260
/* refine the bidir vectors in hq mode and return the score in both lq & hq mode*/
1261
static inline int bidir_refine(MpegEncContext * s, int mb_x, int mb_y)
1262
{
1263
    MotionEstContext * const c= &s->me;
1264 1265
    const int mot_stride = s->mb_stride;
    const int xy = mb_y *mot_stride + mb_x;
1266 1267 1268 1269 1270 1271 1272 1273 1274
    int fbmin;
    int pred_fx= s->b_bidir_forw_mv_table[xy-1][0];
    int pred_fy= s->b_bidir_forw_mv_table[xy-1][1];
    int pred_bx= s->b_bidir_back_mv_table[xy-1][0];
    int pred_by= s->b_bidir_back_mv_table[xy-1][1];
    int motion_fx= s->b_bidir_forw_mv_table[xy][0]= s->b_forw_mv_table[xy][0];
    int motion_fy= s->b_bidir_forw_mv_table[xy][1]= s->b_forw_mv_table[xy][1];
    int motion_bx= s->b_bidir_back_mv_table[xy][0]= s->b_back_mv_table[xy][0];
    int motion_by= s->b_bidir_back_mv_table[xy][1]= s->b_back_mv_table[xy][1];
1275 1276 1277 1278 1279 1280 1281
    const int flags= c->sub_flags;
    const int qpel= flags&FLAG_QPEL;
    const int shift= 1+qpel;
    const int xmin= c->xmin<<shift;
    const int ymin= c->ymin<<shift;
    const int xmax= c->xmax<<shift;
    const int ymax= c->ymax<<shift;
1282
#define HASH(fx,fy,bx,by) ((fx)+17*(fy)+63*(bx)+117*(by))
1283
#define HASH8(fx,fy,bx,by) ((uint8_t)HASH(fx,fy,bx,by))
1284
    int hashidx= HASH(motion_fx,motion_fy, motion_bx, motion_by);
1285
    uint8_t map[256] = { 0 };
1286

1287
    map[hashidx&255] = 1;
1288

1289
    fbmin= check_bidir_mv(s, motion_fx, motion_fy,
1290 1291
                          motion_bx, motion_by,
                          pred_fx, pred_fy,
1292 1293
                          pred_bx, pred_by,
                          0, 16);
1294

1295
    if(s->avctx->bidir_refine){
1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316
        int end;
        static const uint8_t limittab[5]={0,8,32,64,80};
        const int limit= limittab[s->avctx->bidir_refine];
        static const int8_t vect[][4]={
{ 0, 0, 0, 1}, { 0, 0, 0,-1}, { 0, 0, 1, 0}, { 0, 0,-1, 0}, { 0, 1, 0, 0}, { 0,-1, 0, 0}, { 1, 0, 0, 0}, {-1, 0, 0, 0},

{ 0, 0, 1, 1}, { 0, 0,-1,-1}, { 0, 1, 1, 0}, { 0,-1,-1, 0}, { 1, 1, 0, 0}, {-1,-1, 0, 0}, { 1, 0, 0, 1}, {-1, 0, 0,-1},
{ 0, 1, 0, 1}, { 0,-1, 0,-1}, { 1, 0, 1, 0}, {-1, 0,-1, 0},
{ 0, 0,-1, 1}, { 0, 0, 1,-1}, { 0,-1, 1, 0}, { 0, 1,-1, 0}, {-1, 1, 0, 0}, { 1,-1, 0, 0}, { 1, 0, 0,-1}, {-1, 0, 0, 1},
{ 0,-1, 0, 1}, { 0, 1, 0,-1}, {-1, 0, 1, 0}, { 1, 0,-1, 0},

{ 0, 1, 1, 1}, { 0,-1,-1,-1}, { 1, 1, 1, 0}, {-1,-1,-1, 0}, { 1, 1, 0, 1}, {-1,-1, 0,-1}, { 1, 0, 1, 1}, {-1, 0,-1,-1},
{ 0,-1, 1, 1}, { 0, 1,-1,-1}, {-1, 1, 1, 0}, { 1,-1,-1, 0}, { 1, 1, 0,-1}, {-1,-1, 0, 1}, { 1, 0,-1, 1}, {-1, 0, 1,-1},
{ 0, 1,-1, 1}, { 0,-1, 1,-1}, { 1,-1, 1, 0}, {-1, 1,-1, 0}, {-1, 1, 0, 1}, { 1,-1, 0,-1}, { 1, 0, 1,-1}, {-1, 0,-1, 1},
{ 0, 1, 1,-1}, { 0,-1,-1, 1}, { 1, 1,-1, 0}, {-1,-1, 1, 0}, { 1,-1, 0, 1}, {-1, 1, 0,-1}, {-1, 0, 1, 1}, { 1, 0,-1,-1},

{ 1, 1, 1, 1}, {-1,-1,-1,-1},
{ 1, 1, 1,-1}, {-1,-1,-1, 1}, { 1, 1,-1, 1}, {-1,-1, 1,-1}, { 1,-1, 1, 1}, {-1, 1,-1,-1}, {-1, 1, 1, 1}, { 1,-1,-1,-1},
{ 1, 1,-1,-1}, {-1,-1, 1, 1}, { 1,-1,-1, 1}, {-1, 1, 1,-1}, { 1,-1, 1,-1}, {-1, 1,-1, 1},
        };
        static const uint8_t hash[]={
1317
HASH8( 0, 0, 0, 1), HASH8( 0, 0, 0,-1), HASH8( 0, 0, 1, 0), HASH8( 0, 0,-1, 0), HASH8( 0, 1, 0, 0), HASH8( 0,-1, 0, 0), HASH8( 1, 0, 0, 0), HASH8(-1, 0, 0, 0),
1318

1319 1320 1321 1322
HASH8( 0, 0, 1, 1), HASH8( 0, 0,-1,-1), HASH8( 0, 1, 1, 0), HASH8( 0,-1,-1, 0), HASH8( 1, 1, 0, 0), HASH8(-1,-1, 0, 0), HASH8( 1, 0, 0, 1), HASH8(-1, 0, 0,-1),
HASH8( 0, 1, 0, 1), HASH8( 0,-1, 0,-1), HASH8( 1, 0, 1, 0), HASH8(-1, 0,-1, 0),
HASH8( 0, 0,-1, 1), HASH8( 0, 0, 1,-1), HASH8( 0,-1, 1, 0), HASH8( 0, 1,-1, 0), HASH8(-1, 1, 0, 0), HASH8( 1,-1, 0, 0), HASH8( 1, 0, 0,-1), HASH8(-1, 0, 0, 1),
HASH8( 0,-1, 0, 1), HASH8( 0, 1, 0,-1), HASH8(-1, 0, 1, 0), HASH8( 1, 0,-1, 0),
1323

1324 1325 1326 1327
HASH8( 0, 1, 1, 1), HASH8( 0,-1,-1,-1), HASH8( 1, 1, 1, 0), HASH8(-1,-1,-1, 0), HASH8( 1, 1, 0, 1), HASH8(-1,-1, 0,-1), HASH8( 1, 0, 1, 1), HASH8(-1, 0,-1,-1),
HASH8( 0,-1, 1, 1), HASH8( 0, 1,-1,-1), HASH8(-1, 1, 1, 0), HASH8( 1,-1,-1, 0), HASH8( 1, 1, 0,-1), HASH8(-1,-1, 0, 1), HASH8( 1, 0,-1, 1), HASH8(-1, 0, 1,-1),
HASH8( 0, 1,-1, 1), HASH8( 0,-1, 1,-1), HASH8( 1,-1, 1, 0), HASH8(-1, 1,-1, 0), HASH8(-1, 1, 0, 1), HASH8( 1,-1, 0,-1), HASH8( 1, 0, 1,-1), HASH8(-1, 0,-1, 1),
HASH8( 0, 1, 1,-1), HASH8( 0,-1,-1, 1), HASH8( 1, 1,-1, 0), HASH8(-1,-1, 1, 0), HASH8( 1,-1, 0, 1), HASH8(-1, 1, 0,-1), HASH8(-1, 0, 1, 1), HASH8( 1, 0,-1,-1),
1328

1329 1330 1331
HASH8( 1, 1, 1, 1), HASH8(-1,-1,-1,-1),
HASH8( 1, 1, 1,-1), HASH8(-1,-1,-1, 1), HASH8( 1, 1,-1, 1), HASH8(-1,-1, 1,-1), HASH8( 1,-1, 1, 1), HASH8(-1, 1,-1,-1), HASH8(-1, 1, 1, 1), HASH8( 1,-1,-1,-1),
HASH8( 1, 1,-1,-1), HASH8(-1,-1, 1, 1), HASH8( 1,-1,-1, 1), HASH8(-1, 1, 1,-1), HASH8( 1,-1, 1,-1), HASH8(-1, 1,-1, 1),
1332 1333
};

1334
#define CHECK_BIDIR(fx,fy,bx,by)\
1335
    if( !map[(hashidx+HASH(fx,fy,bx,by))&255]\
1336
       &&(fx<=0 || motion_fx+fx<=xmax) && (fy<=0 || motion_fy+fy<=ymax) && (bx<=0 || motion_bx+bx<=xmax) && (by<=0 || motion_by+by<=ymax)\
Michael Niedermayer's avatar
Michael Niedermayer committed
1337
       &&(fx>=0 || motion_fx+fx>=xmin) && (fy>=0 || motion_fy+fy>=ymin) && (bx>=0 || motion_bx+bx>=xmin) && (by>=0 || motion_by+by>=ymin)){\
1338
        int score;\
1339
        map[(hashidx+HASH(fx,fy,bx,by))&255] = 1;\
Michael Niedermayer's avatar
Michael Niedermayer committed
1340 1341
        score= check_bidir_mv(s, motion_fx+fx, motion_fy+fy, motion_bx+bx, motion_by+by, pred_fx, pred_fy, pred_bx, pred_by, 0, 16);\
        if(score < fbmin){\
1342
            hashidx += HASH(fx,fy,bx,by);\
Michael Niedermayer's avatar
Michael Niedermayer committed
1343 1344 1345 1346 1347 1348 1349
            fbmin= score;\
            motion_fx+=fx;\
            motion_fy+=fy;\
            motion_bx+=bx;\
            motion_by+=by;\
            end=0;\
        }\
1350 1351 1352
    }
#define CHECK_BIDIR2(a,b,c,d)\
CHECK_BIDIR(a,b,c,d)\
1353
CHECK_BIDIR(-(a),-(b),-(c),-(d))
1354 1355

        do{
1356 1357
            int i;
            int borderdist=0;
1358 1359
            end=1;

1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393
            CHECK_BIDIR2(0,0,0,1)
            CHECK_BIDIR2(0,0,1,0)
            CHECK_BIDIR2(0,1,0,0)
            CHECK_BIDIR2(1,0,0,0)

            for(i=8; i<limit; i++){
                int fx= motion_fx+vect[i][0];
                int fy= motion_fy+vect[i][1];
                int bx= motion_bx+vect[i][2];
                int by= motion_by+vect[i][3];
                if(borderdist<=0){
                    int a= (xmax - FFMAX(fx,bx))|(FFMIN(fx,bx) - xmin);
                    int b= (ymax - FFMAX(fy,by))|(FFMIN(fy,by) - ymin);
                    if((a|b) < 0)
                        map[(hashidx+hash[i])&255] = 1;
                }
                if(!map[(hashidx+hash[i])&255]){
                    int score;
                    map[(hashidx+hash[i])&255] = 1;
                    score= check_bidir_mv(s, fx, fy, bx, by, pred_fx, pred_fy, pred_bx, pred_by, 0, 16);
                    if(score < fbmin){
                        hashidx += hash[i];
                        fbmin= score;
                        motion_fx=fx;
                        motion_fy=fy;
                        motion_bx=bx;
                        motion_by=by;
                        end=0;
                        borderdist--;
                        if(borderdist<=0){
                            int a= FFMIN(xmax - FFMAX(fx,bx), FFMIN(fx,bx) - xmin);
                            int b= FFMIN(ymax - FFMAX(fy,by), FFMIN(fy,by) - ymin);
                            borderdist= FFMIN(a,b);
                        }
1394 1395 1396 1397 1398 1399
                    }
                }
            }
        }while(!end);
    }

1400 1401 1402 1403 1404
    s->b_bidir_forw_mv_table[xy][0]= motion_fx;
    s->b_bidir_forw_mv_table[xy][1]= motion_fy;
    s->b_bidir_back_mv_table[xy][0]= motion_bx;
    s->b_bidir_back_mv_table[xy][1]= motion_by;

1405
    return fbmin;
1406 1407
}

1408
static inline int direct_search(MpegEncContext * s, int mb_x, int mb_y)
1409
{
1410
    MotionEstContext * const c= &s->me;
1411
    int P[10][2];
1412 1413
    const int mot_stride = s->mb_stride;
    const int mot_xy = mb_y*mot_stride + mb_x;
Michael Niedermayer's avatar
Michael Niedermayer committed
1414 1415
    const int shift= 1+s->quarter_sample;
    int dmin, i;
1416
    const int time_pp= s->pp_time;
1417
    const int time_pb= s->pb_time;
Michael Niedermayer's avatar
Michael Niedermayer committed
1418
    int mx, my, xmin, xmax, ymin, ymax;
1419
    int16_t (*mv_table)[2]= s->b_direct_mv_table;
1420

1421
    c->current_mv_penalty= c->mv_penalty[1] + MAX_MV;
Michael Niedermayer's avatar
Michael Niedermayer committed
1422 1423 1424
    ymin= xmin=(-32)>>shift;
    ymax= xmax=   31>>shift;

1425
    if (IS_8X8(s->next_picture.mb_type[mot_xy])) {
Michael Niedermayer's avatar
Michael Niedermayer committed
1426 1427 1428
        s->mv_type= MV_TYPE_8X8;
    }else{
        s->mv_type= MV_TYPE_16X16;
1429
    }
Michael Niedermayer's avatar
Michael Niedermayer committed
1430 1431 1432 1433

    for(i=0; i<4; i++){
        int index= s->block_index[i];
        int min, max;
1434

1435 1436
        c->co_located_mv[i][0] = s->next_picture.motion_val[0][index][0];
        c->co_located_mv[i][1] = s->next_picture.motion_val[0][index][1];
1437 1438 1439 1440 1441 1442 1443
        c->direct_basis_mv[i][0]= c->co_located_mv[i][0]*time_pb/time_pp + ((i& 1)<<(shift+3));
        c->direct_basis_mv[i][1]= c->co_located_mv[i][1]*time_pb/time_pp + ((i>>1)<<(shift+3));
//        c->direct_basis_mv[1][i][0]= c->co_located_mv[i][0]*(time_pb - time_pp)/time_pp + ((i &1)<<(shift+3);
//        c->direct_basis_mv[1][i][1]= c->co_located_mv[i][1]*(time_pb - time_pp)/time_pp + ((i>>1)<<(shift+3);

        max= FFMAX(c->direct_basis_mv[i][0], c->direct_basis_mv[i][0] - c->co_located_mv[i][0])>>shift;
        min= FFMIN(c->direct_basis_mv[i][0], c->direct_basis_mv[i][0] - c->co_located_mv[i][0])>>shift;
1444 1445
        max+= 16*mb_x + 1; // +-1 is for the simpler rounding
        min+= 16*mb_x - 1;
1446 1447
        xmax= FFMIN(xmax, s->width - max);
        xmin= FFMAX(xmin, - 16     - min);
Michael Niedermayer's avatar
Michael Niedermayer committed
1448

1449 1450
        max= FFMAX(c->direct_basis_mv[i][1], c->direct_basis_mv[i][1] - c->co_located_mv[i][1])>>shift;
        min= FFMIN(c->direct_basis_mv[i][1], c->direct_basis_mv[i][1] - c->co_located_mv[i][1])>>shift;
1451 1452
        max+= 16*mb_y + 1; // +-1 is for the simpler rounding
        min+= 16*mb_y - 1;
1453 1454
        ymax= FFMIN(ymax, s->height - max);
        ymin= FFMAX(ymin, - 16      - min);
1455

Michael Niedermayer's avatar
Michael Niedermayer committed
1456
        if(s->mv_type == MV_TYPE_16X16) break;
1457
    }
1458

1459
    av_assert2(xmax <= 15 && ymax <= 15 && xmin >= -16 && ymin >= -16);
1460

Michael Niedermayer's avatar
Michael Niedermayer committed
1461 1462 1463 1464 1465 1466
    if(xmax < 0 || xmin >0 || ymax < 0 || ymin > 0){
        s->b_direct_mv_table[mot_xy][0]= 0;
        s->b_direct_mv_table[mot_xy][1]= 0;

        return 256*256*256*64;
    }
1467

1468 1469 1470 1471 1472 1473 1474 1475
    c->xmin= xmin;
    c->ymin= ymin;
    c->xmax= xmax;
    c->ymax= ymax;
    c->flags     |= FLAG_DIRECT;
    c->sub_flags |= FLAG_DIRECT;
    c->pred_x=0;
    c->pred_y=0;
Michael Niedermayer's avatar
Michael Niedermayer committed
1476

1477 1478
    P_LEFT[0]        = av_clip(mv_table[mot_xy - 1][0], xmin<<shift, xmax<<shift);
    P_LEFT[1]        = av_clip(mv_table[mot_xy - 1][1], ymin<<shift, ymax<<shift);
Michael Niedermayer's avatar
Michael Niedermayer committed
1479 1480

    /* special case for first line */
Diego Biurrun's avatar
Diego Biurrun committed
1481
    if (!s->first_slice_line) { //FIXME maybe allow this over thread boundary as it is clipped
1482 1483 1484 1485
        P_TOP[0]      = av_clip(mv_table[mot_xy - mot_stride             ][0], xmin<<shift, xmax<<shift);
        P_TOP[1]      = av_clip(mv_table[mot_xy - mot_stride             ][1], ymin<<shift, ymax<<shift);
        P_TOPRIGHT[0] = av_clip(mv_table[mot_xy - mot_stride + 1         ][0], xmin<<shift, xmax<<shift);
        P_TOPRIGHT[1] = av_clip(mv_table[mot_xy - mot_stride + 1         ][1], ymin<<shift, ymax<<shift);
1486

Michael Niedermayer's avatar
Michael Niedermayer committed
1487 1488 1489
        P_MEDIAN[0]= mid_pred(P_LEFT[0], P_TOP[0], P_TOPRIGHT[0]);
        P_MEDIAN[1]= mid_pred(P_LEFT[1], P_TOP[1], P_TOPRIGHT[1]);
    }
1490

1491
    dmin = ff_epzs_motion_search(s, &mx, &my, P, 0, 0, mv_table, 1<<(16-shift), 0, 16);
1492
    if(c->sub_flags&FLAG_QPEL)
1493 1494 1495
        dmin = qpel_motion_search(s, &mx, &my, dmin, 0, 0, 0, 16);
    else
        dmin = hpel_motion_search(s, &mx, &my, dmin, 0, 0, 0, 16);
1496

1497
    if(c->avctx->me_sub_cmp != c->avctx->mb_cmp && !c->skip)
1498
        dmin= get_mb_score(s, mx, my, 0, 0, 0, 16, 1);
1499

1500
    get_limits(s, 16*mb_x, 16*mb_y); //restore c->?min/max, maybe not needed
1501

Michael Niedermayer's avatar
Michael Niedermayer committed
1502 1503
    mv_table[mot_xy][0]= mx;
    mv_table[mot_xy][1]= my;
1504 1505
    c->flags     &= ~FLAG_DIRECT;
    c->sub_flags &= ~FLAG_DIRECT;
1506

1507
    return dmin;
1508 1509 1510 1511 1512
}

void ff_estimate_b_frame_motion(MpegEncContext * s,
                             int mb_x, int mb_y)
{
1513 1514
    MotionEstContext * const c= &s->me;
    const int penalty_factor= c->mb_penalty_factor;
1515
    int fmin, bmin, dmin, fbmin, bimin, fimin;
1516
    int type=0;
1517
    const int xy = mb_y*s->mb_stride + mb_x;
1518 1519
    init_ref(c, s->new_picture.f.data, s->last_picture.f.data,
             s->next_picture.f.data, 16 * mb_x, 16 * mb_y, 2);
1520

1521
    get_limits(s, 16*mb_x, 16*mb_y);
1522

1523
    c->skip=0;
1524

1525
    if (s->codec_id == AV_CODEC_ID_MPEG4 && s->next_picture.mbskip_table[xy]) {
1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
        int score= direct_search(s, mb_x, mb_y); //FIXME just check 0,0

        score= ((unsigned)(score*score + 128*256))>>16;
        c->mc_mb_var_sum_temp += score;
        s->current_picture.mc_mb_var[mb_y*s->mb_stride + mb_x] = score; //FIXME use SSE
        s->mb_type[mb_y*s->mb_stride + mb_x]= CANDIDATE_MB_TYPE_DIRECT0;

        return;
    }

1536
    if (s->codec_id == AV_CODEC_ID_MPEG4)
1537
        dmin= direct_search(s, mb_x, mb_y);
1538 1539
    else
        dmin= INT_MAX;
1540
//FIXME penalty stuff for non mpeg4
1541
    c->skip=0;
1542
    fmin= ff_estimate_motion_b(s, mb_x, mb_y, s->b_forw_mv_table, 0, s->f_code) + 3*penalty_factor;
1543

1544
    c->skip=0;
1545
    bmin= ff_estimate_motion_b(s, mb_x, mb_y, s->b_back_mv_table, 2, s->b_code) + 2*penalty_factor;
1546
    av_dlog(s, " %d %d ", s->b_forw_mv_table[xy][0], s->b_forw_mv_table[xy][1]);
1547

1548
    c->skip=0;
1549
    fbmin= bidir_refine(s, mb_x, mb_y) + penalty_factor;
1550
    av_dlog(s, "%d %d %d %d\n", dmin, fmin, bmin, fbmin);
1551

1552 1553
    if(s->flags & CODEC_FLAG_INTERLACED_ME){
//FIXME mb type penalty
1554 1555
        c->skip=0;
        c->current_mv_penalty= c->mv_penalty[s->f_code] + MAX_MV;
1556 1557
        fimin= interlaced_search(s, 0,
                                 s->b_field_mv_table[0], s->b_field_select_table[0],
1558
                                 s->b_forw_mv_table[xy][0], s->b_forw_mv_table[xy][1], 0);
1559
        c->current_mv_penalty= c->mv_penalty[s->b_code] + MAX_MV;
1560 1561
        bimin= interlaced_search(s, 2,
                                 s->b_field_mv_table[1], s->b_field_select_table[1],
1562
                                 s->b_back_mv_table[xy][0], s->b_back_mv_table[xy][1], 0);
1563 1564 1565
    }else
        fimin= bimin= INT_MAX;

1566
    {
1567
        int score= fmin;
1568
        type = CANDIDATE_MB_TYPE_FORWARD;
1569

1570
        if (dmin <= score){
1571
            score = dmin;
1572
            type = CANDIDATE_MB_TYPE_DIRECT;
1573 1574 1575
        }
        if(bmin<score){
            score=bmin;
1576
            type= CANDIDATE_MB_TYPE_BACKWARD;
1577 1578 1579
        }
        if(fbmin<score){
            score=fbmin;
1580 1581 1582 1583 1584 1585 1586 1587 1588
            type= CANDIDATE_MB_TYPE_BIDIR;
        }
        if(fimin<score){
            score=fimin;
            type= CANDIDATE_MB_TYPE_FORWARD_I;
        }
        if(bimin<score){
            score=bimin;
            type= CANDIDATE_MB_TYPE_BACKWARD_I;
1589
        }
1590

1591
        score= ((unsigned)(score*score + 128*256))>>16;
1592
        c->mc_mb_var_sum_temp += score;
1593
        s->current_picture.mc_mb_var[mb_y*s->mb_stride + mb_x] = score; //FIXME use SSE
1594
    }
1595

1596
    if(c->avctx->mb_decision > FF_MB_DECISION_SIMPLE){
1597 1598 1599 1600 1601 1602 1603 1604 1605
        type= CANDIDATE_MB_TYPE_FORWARD | CANDIDATE_MB_TYPE_BACKWARD | CANDIDATE_MB_TYPE_BIDIR | CANDIDATE_MB_TYPE_DIRECT;
        if(fimin < INT_MAX)
            type |= CANDIDATE_MB_TYPE_FORWARD_I;
        if(bimin < INT_MAX)
            type |= CANDIDATE_MB_TYPE_BACKWARD_I;
        if(fimin < INT_MAX && bimin < INT_MAX){
            type |= CANDIDATE_MB_TYPE_BIDIR_I;
        }
         //FIXME something smarter
Diego Biurrun's avatar
Diego Biurrun committed
1606
        if(dmin>256*256*16) type&= ~CANDIDATE_MB_TYPE_DIRECT; //do not try direct mode if it is invalid for this MB
1607
        if(s->codec_id == AV_CODEC_ID_MPEG4 && type&CANDIDATE_MB_TYPE_DIRECT && s->flags&CODEC_FLAG_MV0 && *(uint32_t*)s->b_direct_mv_table[xy])
1608
            type |= CANDIDATE_MB_TYPE_DIRECT0;
1609 1610
    }

1611
    s->mb_type[mb_y*s->mb_stride + mb_x]= type;
1612 1613 1614 1615 1616 1617
}

/* find best f_code for ME which do unlimited searches */
int ff_get_best_fcode(MpegEncContext * s, int16_t (*mv_table)[2], int type)
{
    if(s->me_method>=ME_EPZS){
1618
        int score[8];
1619
        int i, y, range= s->avctx->me_range ? s->avctx->me_range : (INT_MAX/2);
1620
        uint8_t * fcode_tab= s->fcode_tab;
1621 1622
        int best_fcode=-1;
        int best_score=-10000000;
1623

1624
        if(s->msmpeg4_version)
1625
            range= FFMIN(range, 16);
1626
        else if(s->codec_id == AV_CODEC_ID_MPEG2VIDEO && s->avctx->strict_std_compliance >= FF_COMPLIANCE_NORMAL)
1627 1628
            range= FFMIN(range, 256);

Michael Niedermayer's avatar
Michael Niedermayer committed
1629
        for(i=0; i<8; i++) score[i]= s->mb_num*(8-i);
1630 1631 1632

        for(y=0; y<s->mb_height; y++){
            int x;
1633
            int xy= y*s->mb_stride;
1634
            for(x=0; x<s->mb_width; x++){
Michael Niedermayer's avatar
Michael Niedermayer committed
1635
                if(s->mb_type[xy] & type){
1636 1637 1638 1639
                    int mx= mv_table[xy][0];
                    int my= mv_table[xy][1];
                    int fcode= FFMAX(fcode_tab[mx + MAX_MV],
                                     fcode_tab[my + MAX_MV]);
1640
                    int j;
1641 1642

                        if(mx >= range || mx < -range ||
1643 1644
                           my >= range || my < -range)
                            continue;
1645

1646
                    for(j=0; j<fcode && j<8; j++){
1647
                        if(s->pict_type==AV_PICTURE_TYPE_B || s->current_picture.mc_mb_var[xy] < s->current_picture.mb_var[xy])
1648 1649
                            score[j]-= 170;
                    }
1650 1651 1652 1653
                }
                xy++;
            }
        }
1654

1655 1656 1657 1658 1659
        for(i=1; i<8; i++){
            if(score[i] > best_score){
                best_score= score[i];
                best_fcode= i;
            }
1660
        }
1661 1662

        return best_fcode;
1663 1664
    }else{
        return 1;
Fabrice Bellard's avatar
Fabrice Bellard committed
1665 1666 1667
    }
}

1668 1669
void ff_fix_long_p_mvs(MpegEncContext * s)
{
1670
    MotionEstContext * const c= &s->me;
1671
    const int f_code= s->f_code;
1672
    int y, range;
1673
    av_assert0(s->pict_type==AV_PICTURE_TYPE_P);
1674

1675 1676
    range = (((s->out_format == FMT_MPEG1 || s->msmpeg4_version) ? 8 : 16) << f_code);

1677 1678
    av_assert0(range <= 16 || !s->msmpeg4_version);
    av_assert0(range <=256 || !(s->codec_id == AV_CODEC_ID_MPEG2VIDEO && s->avctx->strict_std_compliance >= FF_COMPLIANCE_NORMAL));
1679

1680
    if(c->avctx->me_range && range > c->avctx->me_range) range= c->avctx->me_range;
1681

1682
    if(s->flags&CODEC_FLAG_4MV){
1683
        const int wrap= s->b8_stride;
1684 1685 1686

        /* clip / convert to intra 8x8 type MVs */
        for(y=0; y<s->mb_height; y++){
1687
            int xy= y*2*wrap;
1688
            int i= y*s->mb_stride;
1689 1690 1691
            int x;

            for(x=0; x<s->mb_width; x++){
1692
                if(s->mb_type[i]&CANDIDATE_MB_TYPE_INTER4V){
1693 1694 1695
                    int block;
                    for(block=0; block<4; block++){
                        int off= (block& 1) + (block>>1)*wrap;
1696 1697
                        int mx = s->current_picture.motion_val[0][ xy + off ][0];
                        int my = s->current_picture.motion_val[0][ xy + off ][1];
1698

1699 1700
                        if(   mx >=range || mx <-range
                           || my >=range || my <-range){
1701 1702
                            s->mb_type[i] &= ~CANDIDATE_MB_TYPE_INTER4V;
                            s->mb_type[i] |= CANDIDATE_MB_TYPE_INTRA;
1703
                            s->current_picture.mb_type[i] = CANDIDATE_MB_TYPE_INTRA;
1704 1705 1706
                        }
                    }
                }
Michael Niedermayer's avatar
Michael Niedermayer committed
1707 1708
                xy+=2;
                i++;
1709 1710 1711 1712 1713
            }
        }
    }
}

1714 1715 1716 1717
/**
 *
 * @param truncate 1 for truncation, 0 for using intra
 */
1718
void ff_fix_long_mvs(MpegEncContext * s, uint8_t *field_select_table, int field_select,
1719
                     int16_t (*mv_table)[2], int f_code, int type, int truncate)
1720
{
1721
    MotionEstContext * const c= &s->me;
1722
    int y, h_range, v_range;
1723

1724
    // RAL: 8 in MPEG-1, 16 in MPEG-4
1725
    int range = (((s->out_format == FMT_MPEG1 || s->msmpeg4_version) ? 8 : 16) << f_code);
1726

1727
    if(c->avctx->me_range && range > c->avctx->me_range) range= c->avctx->me_range;
1728

1729 1730 1731
    h_range= range;
    v_range= field_select_table ? range>>1 : range;

1732 1733 1734
    /* clip / convert to intra 16x16 type MVs */
    for(y=0; y<s->mb_height; y++){
        int x;
1735
        int xy= y*s->mb_stride;
1736
        for(x=0; x<s->mb_width; x++){
1737
            if (s->mb_type[xy] & type){    // RAL: "type" test added...
1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752
                if(field_select_table==NULL || field_select_table[xy] == field_select){
                    if(   mv_table[xy][0] >=h_range || mv_table[xy][0] <-h_range
                       || mv_table[xy][1] >=v_range || mv_table[xy][1] <-v_range){

                        if(truncate){
                            if     (mv_table[xy][0] > h_range-1) mv_table[xy][0]=  h_range-1;
                            else if(mv_table[xy][0] < -h_range ) mv_table[xy][0]= -h_range;
                            if     (mv_table[xy][1] > v_range-1) mv_table[xy][1]=  v_range-1;
                            else if(mv_table[xy][1] < -v_range ) mv_table[xy][1]= -v_range;
                        }else{
                            s->mb_type[xy] &= ~type;
                            s->mb_type[xy] |= CANDIDATE_MB_TYPE_INTRA;
                            mv_table[xy][0]=
                            mv_table[xy][1]= 0;
                        }
1753
                    }
1754
                }
1755 1756 1757 1758 1759
            }
            xy++;
        }
    }
}