h264pred_template.c 41 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
/*
 * H.26L/H.264/AVC/JVT/14496-10/... encoder/decoder
 * Copyright (c) 2003-2011 Michael Niedermayer <michaelni@gmx.at>
 *
 * This file is part of Libav.
 *
 * Libav is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * Libav is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with Libav; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

/**
 * @file
 * H.264 / AVC / MPEG4 part10 prediction functions.
 * @author Michael Niedermayer <michaelni@gmx.at>
 */

#include "mathops.h"
29 30

#include "bit_depth_template.c"
31

32 33 34
static void FUNCC(pred4x4_vertical)(uint8_t *_src, const uint8_t *topright,
                                    ptrdiff_t _stride)
{
35 36
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
37 38 39 40 41 42
    const pixel4 a= AV_RN4PA(src-stride);

    AV_WN4PA(src+0*stride, a);
    AV_WN4PA(src+1*stride, a);
    AV_WN4PA(src+2*stride, a);
    AV_WN4PA(src+3*stride, a);
43 44
}

45 46 47
static void FUNCC(pred4x4_horizontal)(uint8_t *_src, const uint8_t *topright,
                                      ptrdiff_t _stride)
{
48 49
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
50 51 52 53
    AV_WN4PA(src+0*stride, PIXEL_SPLAT_X4(src[-1+0*stride]));
    AV_WN4PA(src+1*stride, PIXEL_SPLAT_X4(src[-1+1*stride]));
    AV_WN4PA(src+2*stride, PIXEL_SPLAT_X4(src[-1+2*stride]));
    AV_WN4PA(src+3*stride, PIXEL_SPLAT_X4(src[-1+3*stride]));
54 55
}

56 57 58
static void FUNCC(pred4x4_dc)(uint8_t *_src, const uint8_t *topright,
                              ptrdiff_t _stride)
{
59 60
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
61 62
    const int dc= (  src[-stride] + src[1-stride] + src[2-stride] + src[3-stride]
                   + src[-1+0*stride] + src[-1+1*stride] + src[-1+2*stride] + src[-1+3*stride] + 4) >>3;
63
    const pixel4 a = PIXEL_SPLAT_X4(dc);
64

65 66 67 68
    AV_WN4PA(src+0*stride, a);
    AV_WN4PA(src+1*stride, a);
    AV_WN4PA(src+2*stride, a);
    AV_WN4PA(src+3*stride, a);
69 70
}

71 72 73
static void FUNCC(pred4x4_left_dc)(uint8_t *_src, const uint8_t *topright,
                                   ptrdiff_t _stride)
{
74 75
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
76
    const int dc= (  src[-1+0*stride] + src[-1+1*stride] + src[-1+2*stride] + src[-1+3*stride] + 2) >>2;
77
    const pixel4 a = PIXEL_SPLAT_X4(dc);
78

79 80 81 82
    AV_WN4PA(src+0*stride, a);
    AV_WN4PA(src+1*stride, a);
    AV_WN4PA(src+2*stride, a);
    AV_WN4PA(src+3*stride, a);
83 84
}

85 86 87
static void FUNCC(pred4x4_top_dc)(uint8_t *_src, const uint8_t *topright,
                                  ptrdiff_t _stride)
{
88 89
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
90
    const int dc= (  src[-stride] + src[1-stride] + src[2-stride] + src[3-stride] + 2) >>2;
91
    const pixel4 a = PIXEL_SPLAT_X4(dc);
92

93 94 95 96
    AV_WN4PA(src+0*stride, a);
    AV_WN4PA(src+1*stride, a);
    AV_WN4PA(src+2*stride, a);
    AV_WN4PA(src+3*stride, a);
97 98
}

99 100 101
static void FUNCC(pred4x4_128_dc)(uint8_t *_src, const uint8_t *topright,
                                  ptrdiff_t _stride)
{
102 103
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
104 105 106 107 108 109
    const pixel4 a = PIXEL_SPLAT_X4(1<<(BIT_DEPTH-1));

    AV_WN4PA(src+0*stride, a);
    AV_WN4PA(src+1*stride, a);
    AV_WN4PA(src+2*stride, a);
    AV_WN4PA(src+3*stride, a);
110 111
}

112 113 114
static void FUNCC(pred4x4_127_dc)(uint8_t *_src, const uint8_t *topright,
                                  ptrdiff_t _stride)
{
115 116
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
117 118 119 120 121 122
    const pixel4 a = PIXEL_SPLAT_X4((1<<(BIT_DEPTH-1))-1);

    AV_WN4PA(src+0*stride, a);
    AV_WN4PA(src+1*stride, a);
    AV_WN4PA(src+2*stride, a);
    AV_WN4PA(src+3*stride, a);
123 124
}

125 126 127
static void FUNCC(pred4x4_129_dc)(uint8_t *_src, const uint8_t *topright,
                                  ptrdiff_t _stride)
{
128 129
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
130 131 132 133 134 135
    const pixel4 a = PIXEL_SPLAT_X4((1<<(BIT_DEPTH-1))+1);

    AV_WN4PA(src+0*stride, a);
    AV_WN4PA(src+1*stride, a);
    AV_WN4PA(src+2*stride, a);
    AV_WN4PA(src+3*stride, a);
136 137 138 139
}


#define LOAD_TOP_RIGHT_EDGE\
140 141 142 143
    const unsigned av_unused t4 = topright[0];\
    const unsigned av_unused t5 = topright[1];\
    const unsigned av_unused t6 = topright[2];\
    const unsigned av_unused t7 = topright[3];\
144 145

#define LOAD_DOWN_LEFT_EDGE\
146 147 148 149
    const unsigned av_unused l4 = src[-1+4*stride];\
    const unsigned av_unused l5 = src[-1+5*stride];\
    const unsigned av_unused l6 = src[-1+6*stride];\
    const unsigned av_unused l7 = src[-1+7*stride];\
150 151

#define LOAD_LEFT_EDGE\
152 153 154 155
    const unsigned av_unused l0 = src[-1+0*stride];\
    const unsigned av_unused l1 = src[-1+1*stride];\
    const unsigned av_unused l2 = src[-1+2*stride];\
    const unsigned av_unused l3 = src[-1+3*stride];\
156 157

#define LOAD_TOP_EDGE\
158 159 160 161
    const unsigned av_unused t0 = src[ 0-1*stride];\
    const unsigned av_unused t1 = src[ 1-1*stride];\
    const unsigned av_unused t2 = src[ 2-1*stride];\
    const unsigned av_unused t3 = src[ 3-1*stride];\
162

163 164 165
static void FUNCC(pred4x4_down_right)(uint8_t *_src, const uint8_t *topright,
                                      ptrdiff_t _stride)
{
166 167
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189
    const int lt= src[-1-1*stride];
    LOAD_TOP_EDGE
    LOAD_LEFT_EDGE

    src[0+3*stride]=(l3 + 2*l2 + l1 + 2)>>2;
    src[0+2*stride]=
    src[1+3*stride]=(l2 + 2*l1 + l0 + 2)>>2;
    src[0+1*stride]=
    src[1+2*stride]=
    src[2+3*stride]=(l1 + 2*l0 + lt + 2)>>2;
    src[0+0*stride]=
    src[1+1*stride]=
    src[2+2*stride]=
    src[3+3*stride]=(l0 + 2*lt + t0 + 2)>>2;
    src[1+0*stride]=
    src[2+1*stride]=
    src[3+2*stride]=(lt + 2*t0 + t1 + 2)>>2;
    src[2+0*stride]=
    src[3+1*stride]=(t0 + 2*t1 + t2 + 2)>>2;
    src[3+0*stride]=(t1 + 2*t2 + t3 + 2)>>2;
}

190 191 192
static void FUNCC(pred4x4_down_left)(uint8_t *_src, const uint8_t *_topright,
                                     ptrdiff_t _stride)
{
193 194 195
    pixel *src = (pixel*)_src;
    const pixel *topright = (const pixel*)_topright;
    int stride = _stride/sizeof(pixel);
196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217
    LOAD_TOP_EDGE
    LOAD_TOP_RIGHT_EDGE
//    LOAD_LEFT_EDGE

    src[0+0*stride]=(t0 + t2 + 2*t1 + 2)>>2;
    src[1+0*stride]=
    src[0+1*stride]=(t1 + t3 + 2*t2 + 2)>>2;
    src[2+0*stride]=
    src[1+1*stride]=
    src[0+2*stride]=(t2 + t4 + 2*t3 + 2)>>2;
    src[3+0*stride]=
    src[2+1*stride]=
    src[1+2*stride]=
    src[0+3*stride]=(t3 + t5 + 2*t4 + 2)>>2;
    src[3+1*stride]=
    src[2+2*stride]=
    src[1+3*stride]=(t4 + t6 + 2*t5 + 2)>>2;
    src[3+2*stride]=
    src[2+3*stride]=(t5 + t7 + 2*t6 + 2)>>2;
    src[3+3*stride]=(t6 + 3*t7 + 2)>>2;
}

218 219 220 221
static void FUNCC(pred4x4_vertical_right)(uint8_t *_src,
                                          const uint8_t *topright,
                                          ptrdiff_t _stride)
{
222 223
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245
    const int lt= src[-1-1*stride];
    LOAD_TOP_EDGE
    LOAD_LEFT_EDGE

    src[0+0*stride]=
    src[1+2*stride]=(lt + t0 + 1)>>1;
    src[1+0*stride]=
    src[2+2*stride]=(t0 + t1 + 1)>>1;
    src[2+0*stride]=
    src[3+2*stride]=(t1 + t2 + 1)>>1;
    src[3+0*stride]=(t2 + t3 + 1)>>1;
    src[0+1*stride]=
    src[1+3*stride]=(l0 + 2*lt + t0 + 2)>>2;
    src[1+1*stride]=
    src[2+3*stride]=(lt + 2*t0 + t1 + 2)>>2;
    src[2+1*stride]=
    src[3+3*stride]=(t0 + 2*t1 + t2 + 2)>>2;
    src[3+1*stride]=(t1 + 2*t2 + t3 + 2)>>2;
    src[0+2*stride]=(lt + 2*l0 + l1 + 2)>>2;
    src[0+3*stride]=(l0 + 2*l1 + l2 + 2)>>2;
}

246 247 248 249
static void FUNCC(pred4x4_vertical_left)(uint8_t *_src,
                                         const uint8_t *_topright,
                                         ptrdiff_t _stride)
{
250 251 252
    pixel *src = (pixel*)_src;
    const pixel *topright = (const pixel*)_topright;
    int stride = _stride/sizeof(pixel);
253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
    LOAD_TOP_EDGE
    LOAD_TOP_RIGHT_EDGE

    src[0+0*stride]=(t0 + t1 + 1)>>1;
    src[1+0*stride]=
    src[0+2*stride]=(t1 + t2 + 1)>>1;
    src[2+0*stride]=
    src[1+2*stride]=(t2 + t3 + 1)>>1;
    src[3+0*stride]=
    src[2+2*stride]=(t3 + t4+ 1)>>1;
    src[3+2*stride]=(t4 + t5+ 1)>>1;
    src[0+1*stride]=(t0 + 2*t1 + t2 + 2)>>2;
    src[1+1*stride]=
    src[0+3*stride]=(t1 + 2*t2 + t3 + 2)>>2;
    src[2+1*stride]=
    src[1+3*stride]=(t2 + 2*t3 + t4 + 2)>>2;
    src[3+1*stride]=
    src[2+3*stride]=(t3 + 2*t4 + t5 + 2)>>2;
    src[3+3*stride]=(t4 + 2*t5 + t6 + 2)>>2;
}

274 275 276
static void FUNCC(pred4x4_horizontal_up)(uint8_t *_src, const uint8_t *topright,
                                         ptrdiff_t _stride)
{
277 278
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298
    LOAD_LEFT_EDGE

    src[0+0*stride]=(l0 + l1 + 1)>>1;
    src[1+0*stride]=(l0 + 2*l1 + l2 + 2)>>2;
    src[2+0*stride]=
    src[0+1*stride]=(l1 + l2 + 1)>>1;
    src[3+0*stride]=
    src[1+1*stride]=(l1 + 2*l2 + l3 + 2)>>2;
    src[2+1*stride]=
    src[0+2*stride]=(l2 + l3 + 1)>>1;
    src[3+1*stride]=
    src[1+2*stride]=(l2 + 2*l3 + l3 + 2)>>2;
    src[3+2*stride]=
    src[1+3*stride]=
    src[0+3*stride]=
    src[2+2*stride]=
    src[2+3*stride]=
    src[3+3*stride]=l3;
}

299 300 301 302
static void FUNCC(pred4x4_horizontal_down)(uint8_t *_src,
                                           const uint8_t *topright,
                                           ptrdiff_t _stride)
{
303 304
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326
    const int lt= src[-1-1*stride];
    LOAD_TOP_EDGE
    LOAD_LEFT_EDGE

    src[0+0*stride]=
    src[2+1*stride]=(lt + l0 + 1)>>1;
    src[1+0*stride]=
    src[3+1*stride]=(l0 + 2*lt + t0 + 2)>>2;
    src[2+0*stride]=(lt + 2*t0 + t1 + 2)>>2;
    src[3+0*stride]=(t0 + 2*t1 + t2 + 2)>>2;
    src[0+1*stride]=
    src[2+2*stride]=(l0 + l1 + 1)>>1;
    src[1+1*stride]=
    src[3+2*stride]=(lt + 2*l0 + l1 + 2)>>2;
    src[0+2*stride]=
    src[2+3*stride]=(l1 + l2+ 1)>>1;
    src[1+2*stride]=
    src[3+3*stride]=(l0 + 2*l1 + l2 + 2)>>2;
    src[0+3*stride]=(l2 + l3 + 1)>>1;
    src[1+3*stride]=(l1 + 2*l2 + l3 + 2)>>2;
}

327 328
static void FUNCC(pred16x16_vertical)(uint8_t *_src, ptrdiff_t _stride)
{
329
    int i;
330 331
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
332 333 334 335
    const pixel4 a = AV_RN4PA(((pixel4*)(src-stride))+0);
    const pixel4 b = AV_RN4PA(((pixel4*)(src-stride))+1);
    const pixel4 c = AV_RN4PA(((pixel4*)(src-stride))+2);
    const pixel4 d = AV_RN4PA(((pixel4*)(src-stride))+3);
336 337

    for(i=0; i<16; i++){
338 339 340 341
        AV_WN4PA(((pixel4*)(src+i*stride))+0, a);
        AV_WN4PA(((pixel4*)(src+i*stride))+1, b);
        AV_WN4PA(((pixel4*)(src+i*stride))+2, c);
        AV_WN4PA(((pixel4*)(src+i*stride))+3, d);
342 343 344
    }
}

345 346
static void FUNCC(pred16x16_horizontal)(uint8_t *_src, ptrdiff_t stride)
{
347
    int i;
348 349
    pixel *src = (pixel*)_src;
    stride /= sizeof(pixel);
350 351

    for(i=0; i<16; i++){
352 353 354 355 356 357
        const pixel4 a = PIXEL_SPLAT_X4(src[-1+i*stride]);

        AV_WN4PA(((pixel4*)(src+i*stride))+0, a);
        AV_WN4PA(((pixel4*)(src+i*stride))+1, a);
        AV_WN4PA(((pixel4*)(src+i*stride))+2, a);
        AV_WN4PA(((pixel4*)(src+i*stride))+3, a);
358 359 360
    }
}

361 362
#define PREDICT_16x16_DC(v)\
    for(i=0; i<16; i++){\
363 364 365 366
        AV_WN4PA(src+ 0, v);\
        AV_WN4PA(src+ 4, v);\
        AV_WN4PA(src+ 8, v);\
        AV_WN4PA(src+12, v);\
367 368 369
        src += stride;\
    }

370 371
static void FUNCC(pred16x16_dc)(uint8_t *_src, ptrdiff_t stride)
{
372
    int i, dc=0;
373 374 375
    pixel *src = (pixel*)_src;
    pixel4 dcsplat;
    stride /= sizeof(pixel);
376 377 378 379 380 381 382 383 384

    for(i=0;i<16; i++){
        dc+= src[-1+i*stride];
    }

    for(i=0;i<16; i++){
        dc+= src[i-stride];
    }

385 386
    dcsplat = PIXEL_SPLAT_X4((dc+16)>>5);
    PREDICT_16x16_DC(dcsplat);
387 388
}

389 390
static void FUNCC(pred16x16_left_dc)(uint8_t *_src, ptrdiff_t stride)
{
391
    int i, dc=0;
392 393 394
    pixel *src = (pixel*)_src;
    pixel4 dcsplat;
    stride /= sizeof(pixel);
395 396 397 398 399

    for(i=0;i<16; i++){
        dc+= src[-1+i*stride];
    }

400 401
    dcsplat = PIXEL_SPLAT_X4((dc+8)>>4);
    PREDICT_16x16_DC(dcsplat);
402 403
}

404 405
static void FUNCC(pred16x16_top_dc)(uint8_t *_src, ptrdiff_t stride)
{
406
    int i, dc=0;
407 408 409
    pixel *src = (pixel*)_src;
    pixel4 dcsplat;
    stride /= sizeof(pixel);
410 411 412 413 414

    for(i=0;i<16; i++){
        dc+= src[i-stride];
    }

415 416
    dcsplat = PIXEL_SPLAT_X4((dc+8)>>4);
    PREDICT_16x16_DC(dcsplat);
417 418
}

419
#define PRED16x16_X(n, v) \
420 421
static void FUNCC(pred16x16_##n##_dc)(uint8_t *_src, ptrdiff_t stride)\
{\
422 423 424 425
    int i;\
    pixel *src = (pixel*)_src;\
    stride /= sizeof(pixel);\
    PREDICT_16x16_DC(PIXEL_SPLAT_X4(v));\
426 427
}

428 429 430
PRED16x16_X(127, (1<<(BIT_DEPTH-1))-1)
PRED16x16_X(128, (1<<(BIT_DEPTH-1))+0)
PRED16x16_X(129, (1<<(BIT_DEPTH-1))+1)
431

432 433 434 435 436
static inline void FUNCC(pred16x16_plane_compat)(uint8_t *_src,
                                                 ptrdiff_t _stride,
                                                 const int svq3,
                                                 const int rv40)
{
437 438
  int i, j, k;
  int a;
439 440 441 442 443 444
  INIT_CLIP
  pixel *src = (pixel*)_src;
  int stride = _stride/sizeof(pixel);
  const pixel * const src0 = src +7-stride;
  const pixel *       src1 = src +8*stride-1;
  const pixel *       src2 = src1-2*stride;    // == src+6*stride-1;
445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470
  int H = src0[1] - src0[-1];
  int V = src1[0] - src2[ 0];
  for(k=2; k<=8; ++k) {
    src1 += stride; src2 -= stride;
    H += k*(src0[k] - src0[-k]);
    V += k*(src1[0] - src2[ 0]);
  }
  if(svq3){
    H = ( 5*(H/4) ) / 16;
    V = ( 5*(V/4) ) / 16;

    /* required for 100% accuracy */
    i = H; H = V; V = i;
  }else if(rv40){
    H = ( H + (H>>2) ) >> 4;
    V = ( V + (V>>2) ) >> 4;
  }else{
    H = ( 5*H+32 ) >> 6;
    V = ( 5*V+32 ) >> 6;
  }

  a = 16*(src1[0] + src2[16] + 1) - 7*(V+H);
  for(j=16; j>0; --j) {
    int b = a;
    a += V;
    for(i=-16; i<0; i+=4) {
471 472 473 474
      src[16+i] = CLIP((b    ) >> 5);
      src[17+i] = CLIP((b+  H) >> 5);
      src[18+i] = CLIP((b+2*H) >> 5);
      src[19+i] = CLIP((b+3*H) >> 5);
475 476 477 478 479 480
      b += 4*H;
    }
    src += stride;
  }
}

481 482
static void FUNCC(pred16x16_plane)(uint8_t *src, ptrdiff_t stride)
{
483
    FUNCC(pred16x16_plane_compat)(src, stride, 0, 0);
484 485
}

486 487
static void FUNCC(pred8x8_vertical)(uint8_t *_src, ptrdiff_t _stride)
{
488
    int i;
489 490
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
491 492
    const pixel4 a= AV_RN4PA(((pixel4*)(src-stride))+0);
    const pixel4 b= AV_RN4PA(((pixel4*)(src-stride))+1);
493 494

    for(i=0; i<8; i++){
495 496
        AV_WN4PA(((pixel4*)(src+i*stride))+0, a);
        AV_WN4PA(((pixel4*)(src+i*stride))+1, b);
497 498 499
    }
}

500 501
static void FUNCC(pred8x16_vertical)(uint8_t *_src, ptrdiff_t _stride)
{
502 503 504 505 506 507 508 509 510 511 512 513
    int i;
    pixel *src = (pixel*)_src;
    int stride = _stride>>(sizeof(pixel)-1);
    const pixel4 a= AV_RN4PA(((pixel4*)(src-stride))+0);
    const pixel4 b= AV_RN4PA(((pixel4*)(src-stride))+1);

    for(i=0; i<16; i++){
        AV_WN4PA(((pixel4*)(src+i*stride))+0, a);
        AV_WN4PA(((pixel4*)(src+i*stride))+1, b);
    }
}

514 515
static void FUNCC(pred8x8_horizontal)(uint8_t *_src, ptrdiff_t stride)
{
516
    int i;
517 518
    pixel *src = (pixel*)_src;
    stride /= sizeof(pixel);
519 520

    for(i=0; i<8; i++){
521 522 523
        const pixel4 a = PIXEL_SPLAT_X4(src[-1+i*stride]);
        AV_WN4PA(((pixel4*)(src+i*stride))+0, a);
        AV_WN4PA(((pixel4*)(src+i*stride))+1, a);
524 525 526
    }
}

527 528
static void FUNCC(pred8x16_horizontal)(uint8_t *_src, ptrdiff_t stride)
{
529 530 531 532 533 534 535 536 537 538
    int i;
    pixel *src = (pixel*)_src;
    stride >>= sizeof(pixel)-1;
    for(i=0; i<16; i++){
        const pixel4 a = PIXEL_SPLAT_X4(src[-1+i*stride]);
        AV_WN4PA(((pixel4*)(src+i*stride))+0, a);
        AV_WN4PA(((pixel4*)(src+i*stride))+1, a);
    }
}

539
#define PRED8x8_X(n, v)\
540 541
static void FUNCC(pred8x8_##n##_dc)(uint8_t *_src, ptrdiff_t stride)\
{\
542
    int i;\
543
    const pixel4 a = PIXEL_SPLAT_X4(v);\
544 545 546
    pixel *src = (pixel*)_src;\
    stride /= sizeof(pixel);\
    for(i=0; i<8; i++){\
547 548
        AV_WN4PA(((pixel4*)(src+i*stride))+0, a);\
        AV_WN4PA(((pixel4*)(src+i*stride))+1, a);\
549
    }\
550 551
}

552 553 554
PRED8x8_X(127, (1<<(BIT_DEPTH-1))-1)
PRED8x8_X(128, (1<<(BIT_DEPTH-1))+0)
PRED8x8_X(129, (1<<(BIT_DEPTH-1))+1)
555

556 557
static void FUNCC(pred8x16_128_dc)(uint8_t *_src, ptrdiff_t stride)
{
558 559 560 561
    FUNCC(pred8x8_128_dc)(_src, stride);
    FUNCC(pred8x8_128_dc)(_src+8*stride, stride);
}

562 563
static void FUNCC(pred8x8_left_dc)(uint8_t *_src, ptrdiff_t stride)
{
564 565
    int i;
    int dc0, dc2;
566 567 568
    pixel4 dc0splat, dc2splat;
    pixel *src = (pixel*)_src;
    stride /= sizeof(pixel);
569 570 571 572 573 574

    dc0=dc2=0;
    for(i=0;i<4; i++){
        dc0+= src[-1+i*stride];
        dc2+= src[-1+(i+4)*stride];
    }
575 576
    dc0splat = PIXEL_SPLAT_X4((dc0 + 2)>>2);
    dc2splat = PIXEL_SPLAT_X4((dc2 + 2)>>2);
577 578

    for(i=0; i<4; i++){
579 580
        AV_WN4PA(((pixel4*)(src+i*stride))+0, dc0splat);
        AV_WN4PA(((pixel4*)(src+i*stride))+1, dc0splat);
581 582
    }
    for(i=4; i<8; i++){
583 584
        AV_WN4PA(((pixel4*)(src+i*stride))+0, dc2splat);
        AV_WN4PA(((pixel4*)(src+i*stride))+1, dc2splat);
585 586 587
    }
}

588 589
static void FUNCC(pred8x16_left_dc)(uint8_t *_src, ptrdiff_t stride)
{
590 591 592 593
    FUNCC(pred8x8_left_dc)(_src, stride);
    FUNCC(pred8x8_left_dc)(_src+8*stride, stride);
}

594 595
static void FUNCC(pred8x8_top_dc)(uint8_t *_src, ptrdiff_t stride)
{
596 597
    int i;
    int dc0, dc1;
598 599 600
    pixel4 dc0splat, dc1splat;
    pixel *src = (pixel*)_src;
    stride /= sizeof(pixel);
601 602 603 604 605 606

    dc0=dc1=0;
    for(i=0;i<4; i++){
        dc0+= src[i-stride];
        dc1+= src[4+i-stride];
    }
607 608
    dc0splat = PIXEL_SPLAT_X4((dc0 + 2)>>2);
    dc1splat = PIXEL_SPLAT_X4((dc1 + 2)>>2);
609 610

    for(i=0; i<4; i++){
611 612
        AV_WN4PA(((pixel4*)(src+i*stride))+0, dc0splat);
        AV_WN4PA(((pixel4*)(src+i*stride))+1, dc1splat);
613 614
    }
    for(i=4; i<8; i++){
615 616
        AV_WN4PA(((pixel4*)(src+i*stride))+0, dc0splat);
        AV_WN4PA(((pixel4*)(src+i*stride))+1, dc1splat);
617 618 619
    }
}

620 621
static void FUNCC(pred8x16_top_dc)(uint8_t *_src, ptrdiff_t stride)
{
622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641
    int i;
    int dc0, dc1;
    pixel4 dc0splat, dc1splat;
    pixel *src = (pixel*)_src;
    stride >>= sizeof(pixel)-1;

    dc0=dc1=0;
    for(i=0;i<4; i++){
        dc0+= src[i-stride];
        dc1+= src[4+i-stride];
    }
    dc0splat = PIXEL_SPLAT_X4((dc0 + 2)>>2);
    dc1splat = PIXEL_SPLAT_X4((dc1 + 2)>>2);

    for(i=0; i<16; i++){
        AV_WN4PA(((pixel4*)(src+i*stride))+0, dc0splat);
        AV_WN4PA(((pixel4*)(src+i*stride))+1, dc1splat);
    }
}

642 643
static void FUNCC(pred8x8_dc)(uint8_t *_src, ptrdiff_t stride)
{
644
    int i;
645 646 647 648
    int dc0, dc1, dc2;
    pixel4 dc0splat, dc1splat, dc2splat, dc3splat;
    pixel *src = (pixel*)_src;
    stride /= sizeof(pixel);
649 650 651 652 653 654 655

    dc0=dc1=dc2=0;
    for(i=0;i<4; i++){
        dc0+= src[-1+i*stride] + src[i-stride];
        dc1+= src[4+i-stride];
        dc2+= src[-1+(i+4)*stride];
    }
656 657 658 659
    dc0splat = PIXEL_SPLAT_X4((dc0 + 4)>>3);
    dc1splat = PIXEL_SPLAT_X4((dc1 + 2)>>2);
    dc2splat = PIXEL_SPLAT_X4((dc2 + 2)>>2);
    dc3splat = PIXEL_SPLAT_X4((dc1 + dc2 + 4)>>3);
660 661

    for(i=0; i<4; i++){
662 663
        AV_WN4PA(((pixel4*)(src+i*stride))+0, dc0splat);
        AV_WN4PA(((pixel4*)(src+i*stride))+1, dc1splat);
664 665
    }
    for(i=4; i<8; i++){
666 667
        AV_WN4PA(((pixel4*)(src+i*stride))+0, dc2splat);
        AV_WN4PA(((pixel4*)(src+i*stride))+1, dc3splat);
668 669 670
    }
}

671 672
static void FUNCC(pred8x16_dc)(uint8_t *_src, ptrdiff_t stride)
{
673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713
    int i;
    int dc0, dc1, dc2, dc3, dc4;
    pixel4 dc0splat, dc1splat, dc2splat, dc3splat, dc4splat, dc5splat, dc6splat, dc7splat;
    pixel *src = (pixel*)_src;
    stride >>= sizeof(pixel)-1;

    dc0=dc1=dc2=dc3=dc4=0;
    for(i=0;i<4; i++){
        dc0+= src[-1+i*stride] + src[i-stride];
        dc1+= src[4+i-stride];
        dc2+= src[-1+(i+4)*stride];
        dc3+= src[-1+(i+8)*stride];
        dc4+= src[-1+(i+12)*stride];
    }
    dc0splat = PIXEL_SPLAT_X4((dc0 + 4)>>3);
    dc1splat = PIXEL_SPLAT_X4((dc1 + 2)>>2);
    dc2splat = PIXEL_SPLAT_X4((dc2 + 2)>>2);
    dc3splat = PIXEL_SPLAT_X4((dc1 + dc2 + 4)>>3);
    dc4splat = PIXEL_SPLAT_X4((dc3 + 2)>>2);
    dc5splat = PIXEL_SPLAT_X4((dc1 + dc3 + 4)>>3);
    dc6splat = PIXEL_SPLAT_X4((dc4 + 2)>>2);
    dc7splat = PIXEL_SPLAT_X4((dc1 + dc4 + 4)>>3);

    for(i=0; i<4; i++){
        AV_WN4PA(((pixel4*)(src+i*stride))+0, dc0splat);
        AV_WN4PA(((pixel4*)(src+i*stride))+1, dc1splat);
    }
    for(i=4; i<8; i++){
        AV_WN4PA(((pixel4*)(src+i*stride))+0, dc2splat);
        AV_WN4PA(((pixel4*)(src+i*stride))+1, dc3splat);
    }
    for(i=8; i<12; i++){
        AV_WN4PA(((pixel4*)(src+i*stride))+0, dc4splat);
        AV_WN4PA(((pixel4*)(src+i*stride))+1, dc5splat);
    }
    for(i=12; i<16; i++){
        AV_WN4PA(((pixel4*)(src+i*stride))+0, dc6splat);
        AV_WN4PA(((pixel4*)(src+i*stride))+1, dc7splat);
    }
}

714 715
static void FUNC(pred8x8_mad_cow_dc_l0t)(uint8_t *src, ptrdiff_t stride)
{
716 717
    FUNCC(pred8x8_top_dc)(src, stride);
    FUNCC(pred4x4_dc)(src, NULL, stride);
718 719
}

720 721
static void FUNC(pred8x16_mad_cow_dc_l0t)(uint8_t *src, ptrdiff_t stride)
{
722 723 724 725
    FUNCC(pred8x16_top_dc)(src, stride);
    FUNCC(pred4x4_dc)(src, NULL, stride);
}

726 727
static void FUNC(pred8x8_mad_cow_dc_0lt)(uint8_t *src, ptrdiff_t stride)
{
728 729
    FUNCC(pred8x8_dc)(src, stride);
    FUNCC(pred4x4_top_dc)(src, NULL, stride);
730 731
}

732 733
static void FUNC(pred8x16_mad_cow_dc_0lt)(uint8_t *src, ptrdiff_t stride)
{
734 735 736 737
    FUNCC(pred8x16_dc)(src, stride);
    FUNCC(pred4x4_top_dc)(src, NULL, stride);
}

738 739
static void FUNC(pred8x8_mad_cow_dc_l00)(uint8_t *src, ptrdiff_t stride)
{
740 741 742
    FUNCC(pred8x8_left_dc)(src, stride);
    FUNCC(pred4x4_128_dc)(src + 4*stride                  , NULL, stride);
    FUNCC(pred4x4_128_dc)(src + 4*stride + 4*sizeof(pixel), NULL, stride);
743 744
}

745 746
static void FUNC(pred8x16_mad_cow_dc_l00)(uint8_t *src, ptrdiff_t stride)
{
747 748 749 750 751
    FUNCC(pred8x16_left_dc)(src, stride);
    FUNCC(pred4x4_128_dc)(src + 4*stride                  , NULL, stride);
    FUNCC(pred4x4_128_dc)(src + 4*stride + 4*sizeof(pixel), NULL, stride);
}

752 753
static void FUNC(pred8x8_mad_cow_dc_0l0)(uint8_t *src, ptrdiff_t stride)
{
754 755 756
    FUNCC(pred8x8_left_dc)(src, stride);
    FUNCC(pred4x4_128_dc)(src                  , NULL, stride);
    FUNCC(pred4x4_128_dc)(src + 4*sizeof(pixel), NULL, stride);
757 758
}

759 760
static void FUNC(pred8x16_mad_cow_dc_0l0)(uint8_t *src, ptrdiff_t stride)
{
761 762 763 764 765
    FUNCC(pred8x16_left_dc)(src, stride);
    FUNCC(pred4x4_128_dc)(src                  , NULL, stride);
    FUNCC(pred4x4_128_dc)(src + 4*sizeof(pixel), NULL, stride);
}

766 767
static void FUNCC(pred8x8_plane)(uint8_t *_src, ptrdiff_t _stride)
{
768 769
  int j, k;
  int a;
770 771 772 773 774 775
  INIT_CLIP
  pixel *src = (pixel*)_src;
  int stride = _stride/sizeof(pixel);
  const pixel * const src0 = src +3-stride;
  const pixel *       src1 = src +4*stride-1;
  const pixel *       src2 = src1-2*stride;    // == src+2*stride-1;
776 777 778 779 780 781 782 783 784 785 786 787 788 789
  int H = src0[1] - src0[-1];
  int V = src1[0] - src2[ 0];
  for(k=2; k<=4; ++k) {
    src1 += stride; src2 -= stride;
    H += k*(src0[k] - src0[-k]);
    V += k*(src1[0] - src2[ 0]);
  }
  H = ( 17*H+16 ) >> 5;
  V = ( 17*V+16 ) >> 5;

  a = 16*(src1[0] + src2[8]+1) - 3*(V+H);
  for(j=8; j>0; --j) {
    int b = a;
    a += V;
790 791 792 793 794
    src[0] = CLIP((b    ) >> 5);
    src[1] = CLIP((b+  H) >> 5);
    src[2] = CLIP((b+2*H) >> 5);
    src[3] = CLIP((b+3*H) >> 5);
    src[4] = CLIP((b+4*H) >> 5);
795 796 797 798 799 800 801
    src[5] = CLIP((b+5*H) >> 5);
    src[6] = CLIP((b+6*H) >> 5);
    src[7] = CLIP((b+7*H) >> 5);
    src += stride;
  }
}

802 803
static void FUNCC(pred8x16_plane)(uint8_t *_src, ptrdiff_t _stride)
{
804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836
  int j, k;
  int a;
  INIT_CLIP
  pixel *src = (pixel*)_src;
  int stride = _stride>>(sizeof(pixel)-1);
  const pixel * const src0 = src +3-stride;
  const pixel *       src1 = src +8*stride-1;
  const pixel *       src2 = src1-2*stride;    // == src+6*stride-1;
  int H = src0[1] - src0[-1];
  int V = src1[0] - src2[ 0];

  for (k = 2; k <= 4; ++k) {
      src1 += stride; src2 -= stride;
      H += k*(src0[k] - src0[-k]);
      V += k*(src1[0] - src2[ 0]);
  }
  for (; k <= 8; ++k) {
      src1 += stride; src2 -= stride;
      V += k*(src1[0] - src2[0]);
  }

  H = (17*H+16) >> 5;
  V = (5*V+32) >> 6;

  a = 16*(src1[0] + src2[8] + 1) - 7*V - 3*H;
  for(j=16; j>0; --j) {
    int b = a;
    a += V;
    src[0] = CLIP((b    ) >> 5);
    src[1] = CLIP((b+  H) >> 5);
    src[2] = CLIP((b+2*H) >> 5);
    src[3] = CLIP((b+3*H) >> 5);
    src[4] = CLIP((b+4*H) >> 5);
837 838 839
    src[5] = CLIP((b+5*H) >> 5);
    src[6] = CLIP((b+6*H) >> 5);
    src[7] = CLIP((b+7*H) >> 5);
840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876
    src += stride;
  }
}

#define SRC(x,y) src[(x)+(y)*stride]
#define PL(y) \
    const int l##y = (SRC(-1,y-1) + 2*SRC(-1,y) + SRC(-1,y+1) + 2) >> 2;
#define PREDICT_8x8_LOAD_LEFT \
    const int l0 = ((has_topleft ? SRC(-1,-1) : SRC(-1,0)) \
                     + 2*SRC(-1,0) + SRC(-1,1) + 2) >> 2; \
    PL(1) PL(2) PL(3) PL(4) PL(5) PL(6) \
    const int l7 av_unused = (SRC(-1,6) + 3*SRC(-1,7) + 2) >> 2

#define PT(x) \
    const int t##x = (SRC(x-1,-1) + 2*SRC(x,-1) + SRC(x+1,-1) + 2) >> 2;
#define PREDICT_8x8_LOAD_TOP \
    const int t0 = ((has_topleft ? SRC(-1,-1) : SRC(0,-1)) \
                     + 2*SRC(0,-1) + SRC(1,-1) + 2) >> 2; \
    PT(1) PT(2) PT(3) PT(4) PT(5) PT(6) \
    const int t7 av_unused = ((has_topright ? SRC(8,-1) : SRC(7,-1)) \
                     + 2*SRC(7,-1) + SRC(6,-1) + 2) >> 2

#define PTR(x) \
    t##x = (SRC(x-1,-1) + 2*SRC(x,-1) + SRC(x+1,-1) + 2) >> 2;
#define PREDICT_8x8_LOAD_TOPRIGHT \
    int t8, t9, t10, t11, t12, t13, t14, t15; \
    if(has_topright) { \
        PTR(8) PTR(9) PTR(10) PTR(11) PTR(12) PTR(13) PTR(14) \
        t15 = (SRC(14,-1) + 3*SRC(15,-1) + 2) >> 2; \
    } else t8=t9=t10=t11=t12=t13=t14=t15= SRC(7,-1);

#define PREDICT_8x8_LOAD_TOPLEFT \
    const int lt = (SRC(-1,0) + 2*SRC(-1,-1) + SRC(0,-1) + 2) >> 2

#define PREDICT_8x8_DC(v) \
    int y; \
    for( y = 0; y < 8; y++ ) { \
877 878
        AV_WN4PA(((pixel4*)src)+0, v); \
        AV_WN4PA(((pixel4*)src)+1, v); \
879 880 881
        src += stride; \
    }

882 883
static void FUNCC(pred8x8l_128_dc)(uint8_t *_src, int has_topleft,
                                   int has_topright, ptrdiff_t _stride)
884
{
885 886 887 888
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);

    PREDICT_8x8_DC(PIXEL_SPLAT_X4(1<<(BIT_DEPTH-1)));
889
}
890 891
static void FUNCC(pred8x8l_left_dc)(uint8_t *_src, int has_topleft,
                                    int has_topright, ptrdiff_t _stride)
892
{
893 894 895
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);

896
    PREDICT_8x8_LOAD_LEFT;
897
    const pixel4 dc = PIXEL_SPLAT_X4((l0+l1+l2+l3+l4+l5+l6+l7+4) >> 3);
898 899
    PREDICT_8x8_DC(dc);
}
900 901
static void FUNCC(pred8x8l_top_dc)(uint8_t *_src, int has_topleft,
                                   int has_topright, ptrdiff_t _stride)
902
{
903 904 905
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);

906
    PREDICT_8x8_LOAD_TOP;
907
    const pixel4 dc = PIXEL_SPLAT_X4((t0+t1+t2+t3+t4+t5+t6+t7+4) >> 3);
908 909
    PREDICT_8x8_DC(dc);
}
910 911
static void FUNCC(pred8x8l_dc)(uint8_t *_src, int has_topleft,
                               int has_topright, ptrdiff_t _stride)
912
{
913 914 915
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);

916 917
    PREDICT_8x8_LOAD_LEFT;
    PREDICT_8x8_LOAD_TOP;
918 919
    const pixel4 dc = PIXEL_SPLAT_X4((l0+l1+l2+l3+l4+l5+l6+l7
                                     +t0+t1+t2+t3+t4+t5+t6+t7+8) >> 4);
920 921
    PREDICT_8x8_DC(dc);
}
922 923
static void FUNCC(pred8x8l_horizontal)(uint8_t *_src, int has_topleft,
                                       int has_topright, ptrdiff_t _stride)
924
{
925 926
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
927
    pixel4 a;
928

929
    PREDICT_8x8_LOAD_LEFT;
930 931 932
#define ROW(y) a = PIXEL_SPLAT_X4(l##y); \
               AV_WN4PA(src+y*stride, a); \
               AV_WN4PA(src+y*stride+4, a);
933 934 935
    ROW(0); ROW(1); ROW(2); ROW(3); ROW(4); ROW(5); ROW(6); ROW(7);
#undef ROW
}
936 937
static void FUNCC(pred8x8l_vertical)(uint8_t *_src, int has_topleft,
                                     int has_topright, ptrdiff_t _stride)
938 939
{
    int y;
940 941
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
942
    pixel4 a, b;
943

944 945 946 947 948 949 950 951 952
    PREDICT_8x8_LOAD_TOP;
    src[0] = t0;
    src[1] = t1;
    src[2] = t2;
    src[3] = t3;
    src[4] = t4;
    src[5] = t5;
    src[6] = t6;
    src[7] = t7;
953 954
    a = AV_RN4PA(((pixel4*)src)+0);
    b = AV_RN4PA(((pixel4*)src)+1);
955
    for( y = 1; y < 8; y++ ) {
956 957
        AV_WN4PA(((pixel4*)(src+y*stride))+0, a);
        AV_WN4PA(((pixel4*)(src+y*stride))+1, b);
958
    }
959
}
960 961
static void FUNCC(pred8x8l_down_left)(uint8_t *_src, int has_topleft,
                                      int has_topright, ptrdiff_t _stride)
962
{
963 964
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982
    PREDICT_8x8_LOAD_TOP;
    PREDICT_8x8_LOAD_TOPRIGHT;
    SRC(0,0)= (t0 + 2*t1 + t2 + 2) >> 2;
    SRC(0,1)=SRC(1,0)= (t1 + 2*t2 + t3 + 2) >> 2;
    SRC(0,2)=SRC(1,1)=SRC(2,0)= (t2 + 2*t3 + t4 + 2) >> 2;
    SRC(0,3)=SRC(1,2)=SRC(2,1)=SRC(3,0)= (t3 + 2*t4 + t5 + 2) >> 2;
    SRC(0,4)=SRC(1,3)=SRC(2,2)=SRC(3,1)=SRC(4,0)= (t4 + 2*t5 + t6 + 2) >> 2;
    SRC(0,5)=SRC(1,4)=SRC(2,3)=SRC(3,2)=SRC(4,1)=SRC(5,0)= (t5 + 2*t6 + t7 + 2) >> 2;
    SRC(0,6)=SRC(1,5)=SRC(2,4)=SRC(3,3)=SRC(4,2)=SRC(5,1)=SRC(6,0)= (t6 + 2*t7 + t8 + 2) >> 2;
    SRC(0,7)=SRC(1,6)=SRC(2,5)=SRC(3,4)=SRC(4,3)=SRC(5,2)=SRC(6,1)=SRC(7,0)= (t7 + 2*t8 + t9 + 2) >> 2;
    SRC(1,7)=SRC(2,6)=SRC(3,5)=SRC(4,4)=SRC(5,3)=SRC(6,2)=SRC(7,1)= (t8 + 2*t9 + t10 + 2) >> 2;
    SRC(2,7)=SRC(3,6)=SRC(4,5)=SRC(5,4)=SRC(6,3)=SRC(7,2)= (t9 + 2*t10 + t11 + 2) >> 2;
    SRC(3,7)=SRC(4,6)=SRC(5,5)=SRC(6,4)=SRC(7,3)= (t10 + 2*t11 + t12 + 2) >> 2;
    SRC(4,7)=SRC(5,6)=SRC(6,5)=SRC(7,4)= (t11 + 2*t12 + t13 + 2) >> 2;
    SRC(5,7)=SRC(6,6)=SRC(7,5)= (t12 + 2*t13 + t14 + 2) >> 2;
    SRC(6,7)=SRC(7,6)= (t13 + 2*t14 + t15 + 2) >> 2;
    SRC(7,7)= (t14 + 3*t15 + 2) >> 2;
}
983 984
static void FUNCC(pred8x8l_down_right)(uint8_t *_src, int has_topleft,
                                       int has_topright, ptrdiff_t _stride)
985
{
986 987
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006
    PREDICT_8x8_LOAD_TOP;
    PREDICT_8x8_LOAD_LEFT;
    PREDICT_8x8_LOAD_TOPLEFT;
    SRC(0,7)= (l7 + 2*l6 + l5 + 2) >> 2;
    SRC(0,6)=SRC(1,7)= (l6 + 2*l5 + l4 + 2) >> 2;
    SRC(0,5)=SRC(1,6)=SRC(2,7)= (l5 + 2*l4 + l3 + 2) >> 2;
    SRC(0,4)=SRC(1,5)=SRC(2,6)=SRC(3,7)= (l4 + 2*l3 + l2 + 2) >> 2;
    SRC(0,3)=SRC(1,4)=SRC(2,5)=SRC(3,6)=SRC(4,7)= (l3 + 2*l2 + l1 + 2) >> 2;
    SRC(0,2)=SRC(1,3)=SRC(2,4)=SRC(3,5)=SRC(4,6)=SRC(5,7)= (l2 + 2*l1 + l0 + 2) >> 2;
    SRC(0,1)=SRC(1,2)=SRC(2,3)=SRC(3,4)=SRC(4,5)=SRC(5,6)=SRC(6,7)= (l1 + 2*l0 + lt + 2) >> 2;
    SRC(0,0)=SRC(1,1)=SRC(2,2)=SRC(3,3)=SRC(4,4)=SRC(5,5)=SRC(6,6)=SRC(7,7)= (l0 + 2*lt + t0 + 2) >> 2;
    SRC(1,0)=SRC(2,1)=SRC(3,2)=SRC(4,3)=SRC(5,4)=SRC(6,5)=SRC(7,6)= (lt + 2*t0 + t1 + 2) >> 2;
    SRC(2,0)=SRC(3,1)=SRC(4,2)=SRC(5,3)=SRC(6,4)=SRC(7,5)= (t0 + 2*t1 + t2 + 2) >> 2;
    SRC(3,0)=SRC(4,1)=SRC(5,2)=SRC(6,3)=SRC(7,4)= (t1 + 2*t2 + t3 + 2) >> 2;
    SRC(4,0)=SRC(5,1)=SRC(6,2)=SRC(7,3)= (t2 + 2*t3 + t4 + 2) >> 2;
    SRC(5,0)=SRC(6,1)=SRC(7,2)= (t3 + 2*t4 + t5 + 2) >> 2;
    SRC(6,0)=SRC(7,1)= (t4 + 2*t5 + t6 + 2) >> 2;
    SRC(7,0)= (t5 + 2*t6 + t7 + 2) >> 2;
}
1007 1008
static void FUNCC(pred8x8l_vertical_right)(uint8_t *_src, int has_topleft,
                                           int has_topright, ptrdiff_t _stride)
1009
{
1010 1011
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037
    PREDICT_8x8_LOAD_TOP;
    PREDICT_8x8_LOAD_LEFT;
    PREDICT_8x8_LOAD_TOPLEFT;
    SRC(0,6)= (l5 + 2*l4 + l3 + 2) >> 2;
    SRC(0,7)= (l6 + 2*l5 + l4 + 2) >> 2;
    SRC(0,4)=SRC(1,6)= (l3 + 2*l2 + l1 + 2) >> 2;
    SRC(0,5)=SRC(1,7)= (l4 + 2*l3 + l2 + 2) >> 2;
    SRC(0,2)=SRC(1,4)=SRC(2,6)= (l1 + 2*l0 + lt + 2) >> 2;
    SRC(0,3)=SRC(1,5)=SRC(2,7)= (l2 + 2*l1 + l0 + 2) >> 2;
    SRC(0,1)=SRC(1,3)=SRC(2,5)=SRC(3,7)= (l0 + 2*lt + t0 + 2) >> 2;
    SRC(0,0)=SRC(1,2)=SRC(2,4)=SRC(3,6)= (lt + t0 + 1) >> 1;
    SRC(1,1)=SRC(2,3)=SRC(3,5)=SRC(4,7)= (lt + 2*t0 + t1 + 2) >> 2;
    SRC(1,0)=SRC(2,2)=SRC(3,4)=SRC(4,6)= (t0 + t1 + 1) >> 1;
    SRC(2,1)=SRC(3,3)=SRC(4,5)=SRC(5,7)= (t0 + 2*t1 + t2 + 2) >> 2;
    SRC(2,0)=SRC(3,2)=SRC(4,4)=SRC(5,6)= (t1 + t2 + 1) >> 1;
    SRC(3,1)=SRC(4,3)=SRC(5,5)=SRC(6,7)= (t1 + 2*t2 + t3 + 2) >> 2;
    SRC(3,0)=SRC(4,2)=SRC(5,4)=SRC(6,6)= (t2 + t3 + 1) >> 1;
    SRC(4,1)=SRC(5,3)=SRC(6,5)=SRC(7,7)= (t2 + 2*t3 + t4 + 2) >> 2;
    SRC(4,0)=SRC(5,2)=SRC(6,4)=SRC(7,6)= (t3 + t4 + 1) >> 1;
    SRC(5,1)=SRC(6,3)=SRC(7,5)= (t3 + 2*t4 + t5 + 2) >> 2;
    SRC(5,0)=SRC(6,2)=SRC(7,4)= (t4 + t5 + 1) >> 1;
    SRC(6,1)=SRC(7,3)= (t4 + 2*t5 + t6 + 2) >> 2;
    SRC(6,0)=SRC(7,2)= (t5 + t6 + 1) >> 1;
    SRC(7,1)= (t5 + 2*t6 + t7 + 2) >> 2;
    SRC(7,0)= (t6 + t7 + 1) >> 1;
}
1038 1039
static void FUNCC(pred8x8l_horizontal_down)(uint8_t *_src, int has_topleft,
                                            int has_topright, ptrdiff_t _stride)
1040
{
1041 1042
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068
    PREDICT_8x8_LOAD_TOP;
    PREDICT_8x8_LOAD_LEFT;
    PREDICT_8x8_LOAD_TOPLEFT;
    SRC(0,7)= (l6 + l7 + 1) >> 1;
    SRC(1,7)= (l5 + 2*l6 + l7 + 2) >> 2;
    SRC(0,6)=SRC(2,7)= (l5 + l6 + 1) >> 1;
    SRC(1,6)=SRC(3,7)= (l4 + 2*l5 + l6 + 2) >> 2;
    SRC(0,5)=SRC(2,6)=SRC(4,7)= (l4 + l5 + 1) >> 1;
    SRC(1,5)=SRC(3,6)=SRC(5,7)= (l3 + 2*l4 + l5 + 2) >> 2;
    SRC(0,4)=SRC(2,5)=SRC(4,6)=SRC(6,7)= (l3 + l4 + 1) >> 1;
    SRC(1,4)=SRC(3,5)=SRC(5,6)=SRC(7,7)= (l2 + 2*l3 + l4 + 2) >> 2;
    SRC(0,3)=SRC(2,4)=SRC(4,5)=SRC(6,6)= (l2 + l3 + 1) >> 1;
    SRC(1,3)=SRC(3,4)=SRC(5,5)=SRC(7,6)= (l1 + 2*l2 + l3 + 2) >> 2;
    SRC(0,2)=SRC(2,3)=SRC(4,4)=SRC(6,5)= (l1 + l2 + 1) >> 1;
    SRC(1,2)=SRC(3,3)=SRC(5,4)=SRC(7,5)= (l0 + 2*l1 + l2 + 2) >> 2;
    SRC(0,1)=SRC(2,2)=SRC(4,3)=SRC(6,4)= (l0 + l1 + 1) >> 1;
    SRC(1,1)=SRC(3,2)=SRC(5,3)=SRC(7,4)= (lt + 2*l0 + l1 + 2) >> 2;
    SRC(0,0)=SRC(2,1)=SRC(4,2)=SRC(6,3)= (lt + l0 + 1) >> 1;
    SRC(1,0)=SRC(3,1)=SRC(5,2)=SRC(7,3)= (l0 + 2*lt + t0 + 2) >> 2;
    SRC(2,0)=SRC(4,1)=SRC(6,2)= (t1 + 2*t0 + lt + 2) >> 2;
    SRC(3,0)=SRC(5,1)=SRC(7,2)= (t2 + 2*t1 + t0 + 2) >> 2;
    SRC(4,0)=SRC(6,1)= (t3 + 2*t2 + t1 + 2) >> 2;
    SRC(5,0)=SRC(7,1)= (t4 + 2*t3 + t2 + 2) >> 2;
    SRC(6,0)= (t5 + 2*t4 + t3 + 2) >> 2;
    SRC(7,0)= (t6 + 2*t5 + t4 + 2) >> 2;
}
1069 1070
static void FUNCC(pred8x8l_vertical_left)(uint8_t *_src, int has_topleft,
                                          int has_topright, ptrdiff_t _stride)
1071
{
1072 1073
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098
    PREDICT_8x8_LOAD_TOP;
    PREDICT_8x8_LOAD_TOPRIGHT;
    SRC(0,0)= (t0 + t1 + 1) >> 1;
    SRC(0,1)= (t0 + 2*t1 + t2 + 2) >> 2;
    SRC(0,2)=SRC(1,0)= (t1 + t2 + 1) >> 1;
    SRC(0,3)=SRC(1,1)= (t1 + 2*t2 + t3 + 2) >> 2;
    SRC(0,4)=SRC(1,2)=SRC(2,0)= (t2 + t3 + 1) >> 1;
    SRC(0,5)=SRC(1,3)=SRC(2,1)= (t2 + 2*t3 + t4 + 2) >> 2;
    SRC(0,6)=SRC(1,4)=SRC(2,2)=SRC(3,0)= (t3 + t4 + 1) >> 1;
    SRC(0,7)=SRC(1,5)=SRC(2,3)=SRC(3,1)= (t3 + 2*t4 + t5 + 2) >> 2;
    SRC(1,6)=SRC(2,4)=SRC(3,2)=SRC(4,0)= (t4 + t5 + 1) >> 1;
    SRC(1,7)=SRC(2,5)=SRC(3,3)=SRC(4,1)= (t4 + 2*t5 + t6 + 2) >> 2;
    SRC(2,6)=SRC(3,4)=SRC(4,2)=SRC(5,0)= (t5 + t6 + 1) >> 1;
    SRC(2,7)=SRC(3,5)=SRC(4,3)=SRC(5,1)= (t5 + 2*t6 + t7 + 2) >> 2;
    SRC(3,6)=SRC(4,4)=SRC(5,2)=SRC(6,0)= (t6 + t7 + 1) >> 1;
    SRC(3,7)=SRC(4,5)=SRC(5,3)=SRC(6,1)= (t6 + 2*t7 + t8 + 2) >> 2;
    SRC(4,6)=SRC(5,4)=SRC(6,2)=SRC(7,0)= (t7 + t8 + 1) >> 1;
    SRC(4,7)=SRC(5,5)=SRC(6,3)=SRC(7,1)= (t7 + 2*t8 + t9 + 2) >> 2;
    SRC(5,6)=SRC(6,4)=SRC(7,2)= (t8 + t9 + 1) >> 1;
    SRC(5,7)=SRC(6,5)=SRC(7,3)= (t8 + 2*t9 + t10 + 2) >> 2;
    SRC(6,6)=SRC(7,4)= (t9 + t10 + 1) >> 1;
    SRC(6,7)=SRC(7,5)= (t9 + 2*t10 + t11 + 2) >> 2;
    SRC(7,6)= (t10 + t11 + 1) >> 1;
    SRC(7,7)= (t10 + 2*t11 + t12 + 2) >> 2;
}
1099 1100
static void FUNCC(pred8x8l_horizontal_up)(uint8_t *_src, int has_topleft,
                                          int has_topright, ptrdiff_t _stride)
1101
{
1102 1103
    pixel *src = (pixel*)_src;
    int stride = _stride/sizeof(pixel);
1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133
    PREDICT_8x8_LOAD_LEFT;
    SRC(0,0)= (l0 + l1 + 1) >> 1;
    SRC(1,0)= (l0 + 2*l1 + l2 + 2) >> 2;
    SRC(0,1)=SRC(2,0)= (l1 + l2 + 1) >> 1;
    SRC(1,1)=SRC(3,0)= (l1 + 2*l2 + l3 + 2) >> 2;
    SRC(0,2)=SRC(2,1)=SRC(4,0)= (l2 + l3 + 1) >> 1;
    SRC(1,2)=SRC(3,1)=SRC(5,0)= (l2 + 2*l3 + l4 + 2) >> 2;
    SRC(0,3)=SRC(2,2)=SRC(4,1)=SRC(6,0)= (l3 + l4 + 1) >> 1;
    SRC(1,3)=SRC(3,2)=SRC(5,1)=SRC(7,0)= (l3 + 2*l4 + l5 + 2) >> 2;
    SRC(0,4)=SRC(2,3)=SRC(4,2)=SRC(6,1)= (l4 + l5 + 1) >> 1;
    SRC(1,4)=SRC(3,3)=SRC(5,2)=SRC(7,1)= (l4 + 2*l5 + l6 + 2) >> 2;
    SRC(0,5)=SRC(2,4)=SRC(4,3)=SRC(6,2)= (l5 + l6 + 1) >> 1;
    SRC(1,5)=SRC(3,4)=SRC(5,3)=SRC(7,2)= (l5 + 2*l6 + l7 + 2) >> 2;
    SRC(0,6)=SRC(2,5)=SRC(4,4)=SRC(6,3)= (l6 + l7 + 1) >> 1;
    SRC(1,6)=SRC(3,5)=SRC(5,4)=SRC(7,3)= (l6 + 3*l7 + 2) >> 2;
    SRC(0,7)=SRC(1,7)=SRC(2,6)=SRC(2,7)=SRC(3,6)=
    SRC(3,7)=SRC(4,5)=SRC(4,6)=SRC(4,7)=SRC(5,5)=
    SRC(5,6)=SRC(5,7)=SRC(6,4)=SRC(6,5)=SRC(6,6)=
    SRC(6,7)=SRC(7,4)=SRC(7,5)=SRC(7,6)=SRC(7,7)= l7;
}
#undef PREDICT_8x8_LOAD_LEFT
#undef PREDICT_8x8_LOAD_TOP
#undef PREDICT_8x8_LOAD_TOPLEFT
#undef PREDICT_8x8_LOAD_TOPRIGHT
#undef PREDICT_8x8_DC
#undef PTR
#undef PT
#undef PL
#undef SRC

1134
static void FUNCC(pred4x4_vertical_add)(uint8_t *_pix, int16_t *_block,
1135 1136
                                        ptrdiff_t stride)
{
1137
    int i;
1138 1139 1140
    pixel *pix = (pixel*)_pix;
    const dctcoef *block = (const dctcoef*)_block;
    stride /= sizeof(pixel);
1141 1142
    pix -= stride;
    for(i=0; i<4; i++){
1143
        pixel v = pix[0];
1144 1145 1146 1147 1148 1149 1150
        pix[1*stride]= v += block[0];
        pix[2*stride]= v += block[4];
        pix[3*stride]= v += block[8];
        pix[4*stride]= v +  block[12];
        pix++;
        block++;
    }
1151 1152

    memset(_block, 0, sizeof(dctcoef) * 16);
1153 1154
}

1155
static void FUNCC(pred4x4_horizontal_add)(uint8_t *_pix, int16_t *_block,
1156 1157
                                          ptrdiff_t stride)
{
1158
    int i;
1159 1160 1161
    pixel *pix = (pixel*)_pix;
    const dctcoef *block = (const dctcoef*)_block;
    stride /= sizeof(pixel);
1162
    for(i=0; i<4; i++){
1163
        pixel v = pix[-1];
1164 1165 1166 1167 1168 1169 1170
        pix[0]= v += block[0];
        pix[1]= v += block[1];
        pix[2]= v += block[2];
        pix[3]= v +  block[3];
        pix+= stride;
        block+= 4;
    }
1171 1172

    memset(_block, 0, sizeof(dctcoef) * 16);
1173 1174
}

1175
static void FUNCC(pred8x8l_vertical_add)(uint8_t *_pix, int16_t *_block,
1176 1177
                                         ptrdiff_t stride)
{
1178
    int i;
1179 1180 1181
    pixel *pix = (pixel*)_pix;
    const dctcoef *block = (const dctcoef*)_block;
    stride /= sizeof(pixel);
1182 1183
    pix -= stride;
    for(i=0; i<8; i++){
1184
        pixel v = pix[0];
1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195
        pix[1*stride]= v += block[0];
        pix[2*stride]= v += block[8];
        pix[3*stride]= v += block[16];
        pix[4*stride]= v += block[24];
        pix[5*stride]= v += block[32];
        pix[6*stride]= v += block[40];
        pix[7*stride]= v += block[48];
        pix[8*stride]= v +  block[56];
        pix++;
        block++;
    }
1196 1197

    memset(_block, 0, sizeof(dctcoef) * 64);
1198 1199
}

1200
static void FUNCC(pred8x8l_horizontal_add)(uint8_t *_pix, int16_t *_block,
1201 1202
                                           ptrdiff_t stride)
{
1203
    int i;
1204 1205 1206
    pixel *pix = (pixel*)_pix;
    const dctcoef *block = (const dctcoef*)_block;
    stride /= sizeof(pixel);
1207
    for(i=0; i<8; i++){
1208
        pixel v = pix[-1];
1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219
        pix[0]= v += block[0];
        pix[1]= v += block[1];
        pix[2]= v += block[2];
        pix[3]= v += block[3];
        pix[4]= v += block[4];
        pix[5]= v += block[5];
        pix[6]= v += block[6];
        pix[7]= v +  block[7];
        pix+= stride;
        block+= 8;
    }
1220 1221

    memset(_block, 0, sizeof(dctcoef) * 64);
1222 1223
}

1224
static void FUNCC(pred16x16_vertical_add)(uint8_t *pix, const int *block_offset,
1225
                                          int16_t *block,
1226 1227
                                          ptrdiff_t stride)
{
1228 1229
    int i;
    for(i=0; i<16; i++)
1230
        FUNCC(pred4x4_vertical_add)(pix + block_offset[i], block + i*16*sizeof(pixel), stride);
1231 1232
}

1233 1234
static void FUNCC(pred16x16_horizontal_add)(uint8_t *pix,
                                            const int *block_offset,
1235
                                            int16_t *block,
1236 1237
                                            ptrdiff_t stride)
{
1238 1239
    int i;
    for(i=0; i<16; i++)
1240
        FUNCC(pred4x4_horizontal_add)(pix + block_offset[i], block + i*16*sizeof(pixel), stride);
1241 1242
}

1243
static void FUNCC(pred8x8_vertical_add)(uint8_t *pix, const int *block_offset,
1244
                                        int16_t *block, ptrdiff_t stride)
1245
{
1246 1247
    int i;
    for(i=0; i<4; i++)
1248
        FUNCC(pred4x4_vertical_add)(pix + block_offset[i], block + i*16*sizeof(pixel), stride);
1249 1250
}

1251
static void FUNCC(pred8x16_vertical_add)(uint8_t *pix, const int *block_offset,
1252
                                         int16_t *block, ptrdiff_t stride)
1253
{
1254 1255 1256 1257 1258 1259 1260
    int i;
    for(i=0; i<4; i++)
        FUNCC(pred4x4_vertical_add)(pix + block_offset[i], block + i*16*sizeof(pixel), stride);
    for(i=4; i<8; i++)
        FUNCC(pred4x4_vertical_add)(pix + block_offset[i+4], block + i*16*sizeof(pixel), stride);
}

1261
static void FUNCC(pred8x8_horizontal_add)(uint8_t *pix, const int *block_offset,
1262
                                          int16_t *block,
1263 1264
                                          ptrdiff_t stride)
{
1265 1266
    int i;
    for(i=0; i<4; i++)
1267
        FUNCC(pred4x4_horizontal_add)(pix + block_offset[i], block + i*16*sizeof(pixel), stride);
1268
}
1269

1270 1271
static void FUNCC(pred8x16_horizontal_add)(uint8_t *pix,
                                           const int *block_offset,
1272
                                           int16_t *block, ptrdiff_t stride)
1273
{
1274 1275 1276 1277 1278 1279
    int i;
    for(i=0; i<4; i++)
        FUNCC(pred4x4_horizontal_add)(pix + block_offset[i], block + i*16*sizeof(pixel), stride);
    for(i=4; i<8; i++)
        FUNCC(pred4x4_horizontal_add)(pix + block_offset[i+4], block + i*16*sizeof(pixel), stride);
}