hevcdsp_template.c 48.6 KB
Newer Older
1
/*
2
 * HEVC video decoder
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
 *
 * Copyright (C) 2012 - 2013 Guillaume Martres
 *
 * This file is part of FFmpeg.
 *
 * FFmpeg is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * FFmpeg is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with FFmpeg; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */

#include "get_bits.h"
24 25
#include "hevc.h"

26 27
#include "bit_depth_template.c"
#include "hevcdsp.h"
28

29

30
static void FUNC(put_pcm)(uint8_t *_dst, ptrdiff_t stride, int size,
31 32 33
                          GetBitContext *gb, int pcm_bit_depth)
{
    int x, y;
34 35
    pixel *dst = (pixel *)_dst;

36
    stride /= sizeof(pixel);
37 38 39 40 41 42 43 44

    for (y = 0; y < size; y++) {
        for (x = 0; x < size; x++)
            dst[x] = get_bits(gb, pcm_bit_depth) << (BIT_DEPTH - pcm_bit_depth);
        dst += stride;
    }
}

45
static void FUNC(transquant_bypass4x4)(uint8_t *_dst, int16_t *coeffs,
46
                                       ptrdiff_t stride)
47 48
{
    int x, y;
49 50
    pixel *dst = (pixel *)_dst;

51
    stride /= sizeof(pixel);
52 53 54 55 56 57 58 59 60 61

    for (y = 0; y < 4; y++) {
        for (x = 0; x < 4; x++) {
            dst[x] += *coeffs;
            coeffs++;
        }
        dst += stride;
    }
}

62
static void FUNC(transquant_bypass8x8)(uint8_t *_dst, int16_t *coeffs,
63
                                       ptrdiff_t stride)
64 65
{
    int x, y;
66 67
    pixel *dst = (pixel *)_dst;

68
    stride /= sizeof(pixel);
69 70 71 72 73 74 75 76 77 78

    for (y = 0; y < 8; y++) {
        for (x = 0; x < 8; x++) {
            dst[x] += *coeffs;
            coeffs++;
        }
        dst += stride;
    }
}

79
static void FUNC(transquant_bypass16x16)(uint8_t *_dst, int16_t *coeffs,
80
                                         ptrdiff_t stride)
81 82
{
    int x, y;
83 84
    pixel *dst = (pixel *)_dst;

85
    stride /= sizeof(pixel);
86 87 88 89 90 91 92 93 94 95

    for (y = 0; y < 16; y++) {
        for (x = 0; x < 16; x++) {
            dst[x] += *coeffs;
            coeffs++;
        }
        dst += stride;
    }
}

96
static void FUNC(transquant_bypass32x32)(uint8_t *_dst, int16_t *coeffs,
97
                                         ptrdiff_t stride)
98 99
{
    int x, y;
100 101
    pixel *dst = (pixel *)_dst;

102
    stride /= sizeof(pixel);
103 104 105 106 107 108 109 110 111 112

    for (y = 0; y < 32; y++) {
        for (x = 0; x < 32; x++) {
            dst[x] += *coeffs;
            coeffs++;
        }
        dst += stride;
    }
}

113
static void FUNC(transform_skip)(uint8_t *_dst, int16_t *coeffs,
114
                                 ptrdiff_t stride)
115
{
116 117
    pixel *dst = (pixel *)_dst;
    int shift  = 13 - BIT_DEPTH;
118 119 120 121 122 123
#if BIT_DEPTH <= 13
    int offset = 1 << (shift - 1);
#else
    int offset = 0;
#endif
    int x, y;
124

125 126 127 128
    stride /= sizeof(pixel);

    for (y = 0; y < 4 * 4; y += 4) {
        for (x = 0; x < 4; x++)
129 130
            dst[x] = av_clip_pixel(dst[x] + ((coeffs[y + x] + offset) >> shift));
        dst += stride;
131 132 133
    }
}

134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151
#define SET(dst, x)   (dst) = (x)
#define SCALE(dst, x) (dst) = av_clip_int16(((x) + add) >> shift)
#define ADD_AND_SCALE(dst, x)                                           \
    (dst) = av_clip_pixel((dst) + av_clip_int16(((x) + add) >> shift))

#define TR_4x4_LUMA(dst, src, step, assign)                             \
    do {                                                                \
        int c0 = src[0 * step] + src[2 * step];                         \
        int c1 = src[2 * step] + src[3 * step];                         \
        int c2 = src[0 * step] - src[3 * step];                         \
        int c3 = 74 * src[1 * step];                                    \
                                                                        \
        assign(dst[2 * step], 74 * (src[0 * step] -                     \
                                    src[2 * step] +                     \
                                    src[3 * step]));                    \
        assign(dst[0 * step], 29 * c0 + 55 * c1 + c3);                  \
        assign(dst[1 * step], 55 * c2 - 29 * c1 + c3);                  \
        assign(dst[3 * step], 55 * c0 + 29 * c2 - c3);                  \
152 153
    } while (0)

154 155 156
static void FUNC(transform_4x4_luma_add)(uint8_t *_dst, int16_t *coeffs,
                                         ptrdiff_t stride)
{
157
    int i;
158
    pixel *dst   = (pixel *)_dst;
159
    int shift    = 7;
160
    int add      = 1 << (shift - 1);
161 162
    int16_t *src = coeffs;

163 164
    stride /= sizeof(pixel);

165 166 167 168 169 170
    for (i = 0; i < 4; i++) {
        TR_4x4_LUMA(src, src, 4, SCALE);
        src++;
    }

    shift = 20 - BIT_DEPTH;
171
    add   = 1 << (shift - 1);
172 173 174
    for (i = 0; i < 4; i++) {
        TR_4x4_LUMA(dst, coeffs, 1, ADD_AND_SCALE);
        coeffs += 4;
175
        dst    += stride;
176
    }
177
}
178 179 180

#undef TR_4x4_LUMA

181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
#define TR_4(dst, src, dstep, sstep, assign)                            \
    do {                                                                \
        const int e0 = transform[8 * 0][0] * src[0 * sstep] +           \
                       transform[8 * 2][0] * src[2 * sstep];            \
        const int e1 = transform[8 * 0][1] * src[0 * sstep] +           \
                       transform[8 * 2][1] * src[2 * sstep];            \
        const int o0 = transform[8 * 1][0] * src[1 * sstep] +           \
                       transform[8 * 3][0] * src[3 * sstep];            \
        const int o1 = transform[8 * 1][1] * src[1 * sstep] +           \
                       transform[8 * 3][1] * src[3 * sstep];            \
                                                                        \
        assign(dst[0 * dstep], e0 + o0);                                \
        assign(dst[1 * dstep], e1 + o1);                                \
        assign(dst[2 * dstep], e1 - o1);                                \
        assign(dst[3 * dstep], e0 - o0);                                \
196 197
    } while (0)

198 199
static void FUNC(transform_4x4_add)(uint8_t *_dst, int16_t *coeffs,
                                    ptrdiff_t stride)
200 201
{
    int i;
202
    pixel *dst   = (pixel *)_dst;
203
    int shift    = 7;
204
    int add      = 1 << (shift - 1);
205 206
    int16_t *src = coeffs;

207 208
    stride /= sizeof(pixel);

209
    for (i = 0; i < 4; i++) {
210
        TR_4(src, src, 4, 4, SCALE);
211 212 213 214
        src++;
    }

    shift = 20 - BIT_DEPTH;
215
    add   = 1 << (shift - 1);
216
    for (i = 0; i < 4; i++) {
217
        TR_4(dst, coeffs, 1, 1, ADD_AND_SCALE);
218
        coeffs += 4;
219
        dst    += stride;
220 221 222
    }
}

223 224
#define TR_8(dst, src, dstep, sstep, assign)                      \
    do {                                                          \
225 226 227 228 229 230 231 232 233 234 235 236
        int i, j;                                                 \
        int e_8[4];                                               \
        int o_8[4] = { 0 };                                       \
        for (i = 0; i < 4; i++)                                   \
            for (j = 1; j < 8; j += 2)                            \
                o_8[i] += transform[4 * j][i] * src[j * sstep];   \
        TR_4(e_8, src, 1, 2 * sstep, SET);                        \
                                                                  \
        for (i = 0; i < 4; i++) {                                 \
            assign(dst[i * dstep], e_8[i] + o_8[i]);              \
            assign(dst[(7 - i) * dstep], e_8[i] - o_8[i]);        \
        }                                                         \
237
    } while (0)
238 239 240 241 242 243 244 245 246 247 248 249 250 251 252

#define TR_16(dst, src, dstep, sstep, assign)                     \
    do {                                                          \
        int i, j;                                                 \
        int e_16[8];                                              \
        int o_16[8] = { 0 };                                      \
        for (i = 0; i < 8; i++)                                   \
            for (j = 1; j < 16; j += 2)                           \
                o_16[i] += transform[2 * j][i] * src[j * sstep];  \
        TR_8(e_16, src, 1, 2 * sstep, SET);                       \
                                                                  \
        for (i = 0; i < 8; i++) {                                 \
            assign(dst[i * dstep], e_16[i] + o_16[i]);            \
            assign(dst[(15 - i) * dstep], e_16[i] - o_16[i]);     \
        }                                                         \
253
    } while (0)
254 255 256 257 258 259 260 261 262 263 264 265 266 267 268

#define TR_32(dst, src, dstep, sstep, assign)                     \
    do {                                                          \
        int i, j;                                                 \
        int e_32[16];                                             \
        int o_32[16] = { 0 };                                     \
        for (i = 0; i < 16; i++)                                  \
            for (j = 1; j < 32; j += 2)                           \
                o_32[i] += transform[j][i] * src[j * sstep];      \
        TR_16(e_32, src, 1, 2 * sstep, SET);                      \
                                                                  \
        for (i = 0; i < 16; i++) {                                \
            assign(dst[i * dstep], e_32[i] + o_32[i]);            \
            assign(dst[(31 - i) * dstep], e_32[i] - o_32[i]);     \
        }                                                         \
269 270 271 272
    } while (0)



273 274
static void FUNC(transform_8x8_add)(uint8_t *_dst, int16_t *coeffs,
                                    ptrdiff_t stride)
275 276
{
    int i;
277
    pixel *dst   = (pixel *)_dst;
278
    int shift    = 7;
279
    int add      = 1 << (shift - 1);
280 281
    int16_t *src = coeffs;

282 283
    stride /= sizeof(pixel);

284
    for (i = 0; i < 8; i++) {
285
        TR_8(src, src, 8, 8, SCALE);
286 287 288 289
        src++;
    }

    shift = 20 - BIT_DEPTH;
290
    add   = 1 << (shift - 1);
291
    for (i = 0; i < 8; i++) {
292
        TR_8(dst, coeffs, 1, 1, ADD_AND_SCALE);
293
        coeffs += 8;
294
        dst    += stride;
295 296 297
    }
}

298
static void FUNC(transform_16x16_add)(uint8_t *_dst, int16_t *coeffs,
299
                                      ptrdiff_t stride)
300 301
{
    int i;
302
    pixel *dst   = (pixel *)_dst;
303
    int shift    = 7;
304
    int add      = 1 << (shift - 1);
305 306
    int16_t *src = coeffs;

307 308
    stride /= sizeof(pixel);

309
    for (i = 0; i < 16; i++) {
310
        TR_16(src, src, 16, 16, SCALE);
311 312 313 314
        src++;
    }

    shift = 20 - BIT_DEPTH;
315
    add   = 1 << (shift - 1);
316
    for (i = 0; i < 16; i++) {
317
        TR_16(dst, coeffs, 1, 1, ADD_AND_SCALE);
318
        coeffs += 16;
319
        dst    += stride;
320 321 322
    }
}

323
static void FUNC(transform_32x32_add)(uint8_t *_dst, int16_t *coeffs,
324
                                      ptrdiff_t stride)
325 326
{
    int i;
327
    pixel *dst   = (pixel *)_dst;
328
    int shift    = 7;
329
    int add      = 1 << (shift - 1);
330 331
    int16_t *src = coeffs;

332 333
    stride /= sizeof(pixel);

334
    for (i = 0; i < 32; i++) {
335
        TR_32(src, src, 32, 32, SCALE);
336 337 338 339 340 341
        src++;
    }
    src   = coeffs;
    shift = 20 - BIT_DEPTH;
    add   = 1 << (shift - 1);
    for (i = 0; i < 32; i++) {
342
        TR_32(dst, coeffs, 1, 1, ADD_AND_SCALE);
343
        coeffs += 32;
344
        dst    += stride;
345 346 347 348
    }
}

static void FUNC(sao_band_filter)(uint8_t *_dst, uint8_t *_src,
349
                                  ptrdiff_t stride, SAOParams *sao,
350
                                  int *borders, int width, int height,
351 352
                                  int c_idx, int class)
{
353 354
    pixel *dst = (pixel *)_dst;
    pixel *src = (pixel *)_src;
355 356 357
    int offset_table[32] = { 0 };
    int k, y, x;
    int chroma = !!c_idx;
358
    int shift  = BIT_DEPTH - 5;
359 360 361 362
    int *sao_offset_val = sao->offset_val[c_idx];
    int sao_left_class  = sao->band_position[c_idx];
    int init_y = 0, init_x = 0;

363 364
    stride /= sizeof(pixel);

365 366 367
    switch (class) {
    case 0:
        if (!borders[2])
368
            width -= (8 >> chroma) + 2;
369
        if (!borders[3])
370
            height -= (4 >> chroma) + 2;
371 372 373 374
        break;
    case 1:
        init_y = -(4 >> chroma) - 2;
        if (!borders[2])
375
            width -= (8 >> chroma) + 2;
376 377 378 379 380 381
        height = (4 >> chroma) + 2;
        break;
    case 2:
        init_x = -(8 >> chroma) - 2;
        width  =  (8 >> chroma) + 2;
        if (!borders[3])
382
            height -= (4 >> chroma) + 2;
383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
        break;
    case 3:
        init_y = -(4 >> chroma) - 2;
        init_x = -(8 >> chroma) - 2;
        width  =  (8 >> chroma) + 2;
        height =  (4 >> chroma) + 2;
        break;
    }

    dst = dst + (init_y * stride + init_x);
    src = src + (init_y * stride + init_x);
    for (k = 0; k < 4; k++)
        offset_table[(k + sao_left_class) & 31] = sao_offset_val[k + 1];
    for (y = 0; y < height; y++) {
        for (x = 0; x < width; x++)
            dst[x] = av_clip_pixel(src[x] + offset_table[av_clip_pixel(src[x] >> shift)]);
        dst += stride;
        src += stride;
    }
}

static void FUNC(sao_band_filter_0)(uint8_t *dst, uint8_t *src,
                                    ptrdiff_t stride, SAOParams *sao,
                                    int *borders, int width, int height,
                                    int c_idx)
{
409 410
    FUNC(sao_band_filter)(dst, src, stride, sao, borders,
                          width, height, c_idx, 0);
411 412 413 414 415 416 417
}

static void FUNC(sao_band_filter_1)(uint8_t *dst, uint8_t *src,
                                    ptrdiff_t stride, SAOParams *sao,
                                    int *borders, int width, int height,
                                    int c_idx)
{
418 419
    FUNC(sao_band_filter)(dst, src, stride, sao, borders,
                          width, height, c_idx, 1);
420 421 422 423 424 425 426
}

static void FUNC(sao_band_filter_2)(uint8_t *dst, uint8_t *src,
                                    ptrdiff_t stride, SAOParams *sao,
                                    int *borders, int width, int height,
                                    int c_idx)
{
427 428
    FUNC(sao_band_filter)(dst, src, stride, sao, borders,
                          width, height, c_idx, 2);
429 430 431
}

static void FUNC(sao_band_filter_3)(uint8_t *_dst, uint8_t *_src,
432
                                    ptrdiff_t stride, SAOParams *sao,
433 434 435
                                    int *borders, int width, int height,
                                    int c_idx)
{
436
    FUNC(sao_band_filter)(_dst, _src, stride, sao, borders,
437
                          width, height, c_idx, 3);
438 439 440
}

static void FUNC(sao_edge_filter_0)(uint8_t *_dst, uint8_t *_src,
441
                                    ptrdiff_t stride, SAOParams *sao,
442
                                    int *borders, int _width, int _height,
443 444
                                    int c_idx, uint8_t vert_edge,
                                    uint8_t horiz_edge, uint8_t diag_edge)
445 446
{
    int x, y;
447 448
    pixel *dst = (pixel *)_dst;
    pixel *src = (pixel *)_src;
449 450
    int chroma = !!c_idx;
    int *sao_offset_val = sao->offset_val[c_idx];
451
    int sao_eo_class    = sao->eo_class[c_idx];
452
    int init_x = 0, init_y = 0, width = _width, height = _height;
453 454

    static const int8_t pos[4][2][2] = {
455 456 457 458
        { { -1,  0 }, {  1, 0 } }, // horizontal
        { {  0, -1 }, {  0, 1 } }, // vertical
        { { -1, -1 }, {  1, 1 } }, // 45 degree
        { {  1, -1 }, { -1, 1 } }, // 135 degree
459 460 461 462
    };
    static const uint8_t edge_idx[] = { 1, 2, 0, 3, 4 };

#define CMP(a, b) ((a) > (b) ? 1 : ((a) == (b) ? 0 : -1))
463

464
    stride /= sizeof(pixel);
465 466 467 468 469 470 471 472 473 474 475 476 477 478 479

    if (!borders[2])
        width -= (8 >> chroma) + 2;
    if (!borders[3])
        height -= (4 >> chroma) + 2;

    dst = dst + (init_y * stride + init_x);
    src = src + (init_y * stride + init_x);
    init_y = init_x = 0;
    if (sao_eo_class != SAO_EO_VERT) {
        if (borders[0]) {
            int offset_val = sao_offset_val[0];
            int y_stride   = 0;
            for (y = 0; y < height; y++) {
                dst[y_stride] = av_clip_pixel(src[y_stride] + offset_val);
480
                y_stride     += stride;
481 482 483 484 485 486 487 488
            }
            init_x = 1;
        }
        if (borders[2]) {
            int offset_val = sao_offset_val[0];
            int x_stride   = width - 1;
            for (x = 0; x < height; x++) {
                dst[x_stride] = av_clip_pixel(src[x_stride] + offset_val);
489
                x_stride     += stride;
490
            }
491
            width--;
492 493
        }
    }
494 495
    if (sao_eo_class != SAO_EO_HORIZ) {
        if (borders[1]) {
496
            int offset_val = sao_offset_val[0];
497
            for (x = init_x; x < width; x++)
498 499 500
                dst[x] = av_clip_pixel(src[x] + offset_val);
            init_y = 1;
        }
501
        if (borders[3]) {
502 503
            int offset_val = sao_offset_val[0];
            int y_stride   = stride * (height - 1);
504
            for (x = init_x; x < width; x++)
505 506 507 508 509
                dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + offset_val);
            height--;
        }
    }
    {
510 511 512 513 514
        int y_stride = init_y * stride;
        int pos_0_0  = pos[sao_eo_class][0][0];
        int pos_0_1  = pos[sao_eo_class][0][1];
        int pos_1_0  = pos[sao_eo_class][1][0];
        int pos_1_1  = pos[sao_eo_class][1][1];
515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547

        int y_stride_0_1 = (init_y + pos_0_1) * stride;
        int y_stride_1_1 = (init_y + pos_1_1) * stride;
        for (y = init_y; y < height; y++) {
            for (x = init_x; x < width; x++) {
                int diff0         = CMP(src[x + y_stride], src[x + pos_0_0 + y_stride_0_1]);
                int diff1         = CMP(src[x + y_stride], src[x + pos_1_0 + y_stride_1_1]);
                int offset_val    = edge_idx[2 + diff0 + diff1];
                dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + sao_offset_val[offset_val]);
            }
            y_stride     += stride;
            y_stride_0_1 += stride;
            y_stride_1_1 += stride;
        }
    }

    {
        // Restore pixels that can't be modified
        int save_upper_left = !diag_edge && sao_eo_class == SAO_EO_135D && !borders[0] && !borders[1];
        if (vert_edge && sao_eo_class != SAO_EO_VERT)
            for (y = init_y+save_upper_left; y< height; y++)
                dst[y*stride] = src[y*stride];
        if(horiz_edge && sao_eo_class != SAO_EO_HORIZ)
            for(x = init_x+save_upper_left; x<width; x++)
                dst[x] = src[x];
        if(diag_edge && sao_eo_class == SAO_EO_135D)
            dst[0] = src[0];
    }

#undef CMP
}

static void FUNC(sao_edge_filter_1)(uint8_t *_dst, uint8_t *_src,
548
                                    ptrdiff_t stride, SAOParams *sao,
549
                                    int *borders, int _width, int _height,
550 551
                                    int c_idx, uint8_t vert_edge,
                                    uint8_t horiz_edge, uint8_t diag_edge)
552 553
{
    int x, y;
554 555
    pixel *dst = (pixel *)_dst;
    pixel *src = (pixel *)_src;
556 557
    int chroma = !!c_idx;
    int *sao_offset_val = sao->offset_val[c_idx];
558
    int sao_eo_class    = sao->eo_class[c_idx];
559
    int init_x = 0, init_y = 0, width = _width, height = _height;
560 561

    static const int8_t pos[4][2][2] = {
562 563 564 565
        { { -1, 0  }, { 1,  0 } }, // horizontal
        { { 0,  -1 }, { 0,  1 } }, // vertical
        { { -1, -1 }, { 1,  1 } }, // 45 degree
        { { 1,  -1 }, { -1, 1 } }, // 135 degree
566 567 568 569
    };
    static const uint8_t edge_idx[] = { 1, 2, 0, 3, 4 };

#define CMP(a, b) ((a) > (b) ? 1 : ((a) == (b) ? 0 : -1))
570

571
    stride /= sizeof(pixel);
572 573 574 575 576 577 578 579 580 581 582 583 584 585 586

    init_y = -(4 >> chroma) - 2;
    if (!borders[2])
        width -= (8 >> chroma) + 2;
    height = (4 >> chroma) + 2;

    dst = dst + (init_y * stride + init_x);
    src = src + (init_y * stride + init_x);
    init_y = init_x = 0;
    if (sao_eo_class != SAO_EO_VERT) {
        if (borders[0]) {
            int offset_val = sao_offset_val[0];
            int y_stride   = 0;
            for (y = 0; y < height; y++) {
                dst[y_stride] = av_clip_pixel(src[y_stride] + offset_val);
587
                y_stride     += stride;
588 589 590 591 592 593 594 595
            }
            init_x = 1;
        }
        if (borders[2]) {
            int offset_val = sao_offset_val[0];
            int x_stride   = width - 1;
            for (x = 0; x < height; x++) {
                dst[x_stride] = av_clip_pixel(src[x_stride] + offset_val);
596
                x_stride     += stride;
597 598 599 600 601
            }
            width--;
        }
    }
    {
602 603 604 605 606
        int y_stride = init_y * stride;
        int pos_0_0  = pos[sao_eo_class][0][0];
        int pos_0_1  = pos[sao_eo_class][0][1];
        int pos_1_0  = pos[sao_eo_class][1][0];
        int pos_1_1  = pos[sao_eo_class][1][1];
607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639

        int y_stride_0_1 = (init_y + pos_0_1) * stride;
        int y_stride_1_1 = (init_y + pos_1_1) * stride;
        for (y = init_y; y < height; y++) {
            for (x = init_x; x < width; x++) {
                int diff0         = CMP(src[x + y_stride], src[x + pos_0_0 + y_stride_0_1]);
                int diff1         = CMP(src[x + y_stride], src[x + pos_1_0 + y_stride_1_1]);
                int offset_val    = edge_idx[2 + diff0 + diff1];
                dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + sao_offset_val[offset_val]);
            }
            y_stride     += stride;
            y_stride_0_1 += stride;
            y_stride_1_1 += stride;
        }
    }

    {
        // Restore pixels that can't be modified
        int save_lower_left = !diag_edge && sao_eo_class == SAO_EO_45D && !borders[0];
        if(vert_edge && sao_eo_class != SAO_EO_VERT)
            for(y = init_y; y< height-save_lower_left; y++)
                dst[y*stride] = src[y*stride];
        if(horiz_edge && sao_eo_class != SAO_EO_HORIZ)
            for(x = init_x+save_lower_left; x<width; x++)
                dst[(height-1)*stride+x] = src[(height-1)*stride+x];
        if(diag_edge && sao_eo_class == SAO_EO_45D)
            dst[stride*(height-1)] = src[stride*(height-1)];
    }

#undef CMP
}

static void FUNC(sao_edge_filter_2)(uint8_t *_dst, uint8_t *_src,
640
                                    ptrdiff_t stride, SAOParams *sao,
641
                                    int *borders, int _width, int _height,
642 643
                                    int c_idx, uint8_t vert_edge,
                                    uint8_t horiz_edge, uint8_t diag_edge)
644 645
{
    int x, y;
646 647
    pixel *dst = (pixel *)_dst;
    pixel *src = (pixel *)_src;
648 649
    int chroma = !!c_idx;
    int *sao_offset_val = sao->offset_val[c_idx];
650
    int sao_eo_class    = sao->eo_class[c_idx];
651
    int init_x = 0, init_y = 0, width = _width, height = _height;
652 653

    static const int8_t pos[4][2][2] = {
654 655 656 657
        { { -1,  0 }, {  1, 0 } }, // horizontal
        { {  0, -1 }, {  0, 1 } }, // vertical
        { { -1, -1 }, {  1, 1 } }, // 45 degree
        { {  1, -1 }, { -1, 1 } }, // 135 degree
658 659 660 661
    };
    static const uint8_t edge_idx[] = { 1, 2, 0, 3, 4 };

#define CMP(a, b) ((a) > (b) ? 1 : ((a) == (b) ? 0 : -1))
662

663
    stride /= sizeof(pixel);
664 665 666 667 668 669 670 671 672 673

    init_x = -(8 >> chroma) - 2;
    width  =  (8 >> chroma) + 2;
    if (!borders[3])
        height -= (4 >> chroma) + 2;

    dst = dst + (init_y * stride + init_x);
    src = src + (init_y * stride + init_x);
    init_y = init_x = 0;
    if (sao_eo_class != SAO_EO_HORIZ) {
674
        if (borders[1]) {
675
            int offset_val = sao_offset_val[0];
676
            for (x = init_x; x < width; x++)
677 678 679
                dst[x] = av_clip_pixel(src[x] + offset_val);
            init_y = 1;
        }
680
        if (borders[3]) {
681 682
            int offset_val = sao_offset_val[0];
            int y_stride   = stride * (height - 1);
683
            for (x = init_x; x < width; x++)
684 685 686 687 688
                dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + offset_val);
            height--;
        }
    }
    {
689 690 691 692 693
        int y_stride = init_y * stride;
        int pos_0_0  = pos[sao_eo_class][0][0];
        int pos_0_1  = pos[sao_eo_class][0][1];
        int pos_1_0  = pos[sao_eo_class][1][0];
        int pos_1_1  = pos[sao_eo_class][1][1];
694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725

        int y_stride_0_1 = (init_y + pos_0_1) * stride;
        int y_stride_1_1 = (init_y + pos_1_1) * stride;
        for (y = init_y; y < height; y++) {
            for (x = init_x; x < width; x++) {
                int diff0         = CMP(src[x + y_stride], src[x + pos_0_0 + y_stride_0_1]);
                int diff1         = CMP(src[x + y_stride], src[x + pos_1_0 + y_stride_1_1]);
                int offset_val    = edge_idx[2 + diff0 + diff1];
                dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + sao_offset_val[offset_val]);
            }
            y_stride     += stride;
            y_stride_0_1 += stride;
            y_stride_1_1 += stride;
        }
    }

    {
        // Restore pixels that can't be modified
        int save_upper_right = !diag_edge && sao_eo_class == SAO_EO_45D && !borders[1];
        if(vert_edge && sao_eo_class != SAO_EO_VERT)
            for(y = init_y+save_upper_right; y< height; y++)
                dst[y*stride+width-1] = src[y*stride+width-1];
        if(horiz_edge && sao_eo_class != SAO_EO_HORIZ)
            for(x = init_x; x<width-save_upper_right; x++)
                dst[x] = src[x];
        if(diag_edge && sao_eo_class == SAO_EO_45D)
            dst[width-1] = src[width-1];
    }
#undef CMP
}

static void FUNC(sao_edge_filter_3)(uint8_t *_dst, uint8_t *_src,
726
                                    ptrdiff_t stride, SAOParams *sao,
727
                                    int *borders, int _width, int _height,
728 729
                                    int c_idx, uint8_t vert_edge,
                                    uint8_t horiz_edge, uint8_t diag_edge)
730 731
{
    int x, y;
732 733
    pixel *dst = (pixel *)_dst;
    pixel *src = (pixel *)_src;
734 735 736
    int chroma = !!c_idx;
    int *sao_offset_val = sao->offset_val[c_idx];
    int sao_eo_class    = sao->eo_class[c_idx];
737
    int init_x = 0, init_y = 0, width = _width, height = _height;
738 739

    static const int8_t pos[4][2][2] = {
740 741 742 743
        { { -1,  0 }, {  1, 0 } }, // horizontal
        { {  0, -1 }, {  0, 1 } }, // vertical
        { { -1, -1 }, {  1, 1 } }, // 45 degree
        { {  1, -1 }, { -1, 1 } }, // 135 degree
744 745 746 747
    };
    static const uint8_t edge_idx[] = { 1, 2, 0, 3, 4 };

#define CMP(a, b) ((a) > (b) ? 1 : ((a) == (b) ? 0 : -1))
748

749
    stride /= sizeof(pixel);
750 751 752 753 754 755 756

    init_y = -(4 >> chroma) - 2;
    init_x = -(8 >> chroma) - 2;
    width  =  (8 >> chroma) + 2;
    height =  (4 >> chroma) + 2;


757 758
    dst    = dst + (init_y * stride + init_x);
    src    = src + (init_y * stride + init_x);
759 760 761
    init_y = init_x = 0;

    {
762 763 764 765 766
        int y_stride = init_y * stride;
        int pos_0_0  = pos[sao_eo_class][0][0];
        int pos_0_1  = pos[sao_eo_class][0][1];
        int pos_1_0  = pos[sao_eo_class][1][0];
        int pos_1_1  = pos[sao_eo_class][1][1];
767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811

        int y_stride_0_1 = (init_y + pos_0_1) * stride;
        int y_stride_1_1 = (init_y + pos_1_1) * stride;

        for (y = init_y; y < height; y++) {
            for (x = init_x; x < width; x++) {
                int diff0         = CMP(src[x + y_stride], src[x + pos_0_0 + y_stride_0_1]);
                int diff1         = CMP(src[x + y_stride], src[x + pos_1_0 + y_stride_1_1]);
                int offset_val    = edge_idx[2 + diff0 + diff1];
                dst[x + y_stride] = av_clip_pixel(src[x + y_stride] + sao_offset_val[offset_val]);
            }
            y_stride     += stride;
            y_stride_0_1 += stride;
            y_stride_1_1 += stride;
        }
    }

    {
        // Restore pixels that can't be modified
        int save_lower_right = !diag_edge && sao_eo_class == SAO_EO_135D;
        if(vert_edge && sao_eo_class != SAO_EO_VERT)
            for(y = init_y; y< height-save_lower_right; y++)
                dst[y*stride+width-1] = src[y*stride+width-1];
        if(horiz_edge && sao_eo_class != SAO_EO_HORIZ)
            for(x = init_x; x<width-save_lower_right; x++)
                dst[(height-1)*stride+x] = src[(height-1)*stride+x];
        if(diag_edge && sao_eo_class == SAO_EO_135D)
            dst[stride*(height-1)+width-1] = src[stride*(height-1)+width-1];
    }
#undef CMP
}

#undef SET
#undef SCALE
#undef ADD_AND_SCALE
#undef TR_4
#undef TR_8
#undef TR_16
#undef TR_32

static void FUNC(put_hevc_qpel_pixels)(int16_t *dst, ptrdiff_t dststride,
                                       uint8_t *_src, ptrdiff_t _srcstride,
                                       int width, int height, int16_t* mcbuffer)
{
    int x, y;
812
    pixel *src          = (pixel *)_src;
813 814 815 816 817 818 819 820 821 822
    ptrdiff_t srcstride = _srcstride / sizeof(pixel);

    for (y = 0; y < height; y++) {
        for (x = 0; x < width; x++)
            dst[x] = src[x] << (14 - BIT_DEPTH);
        src += srcstride;
        dst += dststride;
    }
}

823
#define QPEL_FILTER_1(src, stride)      \
824 825 826 827 828 829 830
    (1 * -src[x - 3 * stride] +         \
     4 *  src[x - 2 * stride] -         \
    10 *  src[x -     stride] +         \
    58 *  src[x]              +         \
    17 *  src[x +     stride] -         \
     5 *  src[x + 2 * stride] +         \
     1 *  src[x + 3 * stride])
831 832

#define QPEL_FILTER_2(src, stride)      \
833 834 835 836 837 838 839 840
    (1  * -src[x - 3 * stride] +        \
     4  *  src[x - 2 * stride] -        \
    11  *  src[x -     stride] +        \
    40  *  src[x]              +        \
    40  *  src[x +     stride] -        \
    11  *  src[x + 2 * stride] +        \
     4  *  src[x + 3 * stride] -        \
     1  *  src[x + 4 * stride])
841 842

#define QPEL_FILTER_3(src, stride)      \
843 844 845 846 847 848 849
    (1  * src[x - 2 * stride] -         \
     5  * src[x -     stride] +         \
    17  * src[x]              +         \
    58  * src[x + stride]     -         \
    10  * src[x + 2 * stride] +         \
     4  * src[x + 3 * stride] -         \
     1  * src[x + 4 * stride])
850 851 852 853 854 855 856 857 858 859 860 861 862 863


#define PUT_HEVC_QPEL_H(H)                                                     \
static void FUNC(put_hevc_qpel_h ## H)(int16_t *dst,  ptrdiff_t dststride,     \
                                       uint8_t *_src, ptrdiff_t _srcstride,    \
                                       int width, int height,                  \
                                       int16_t* mcbuffer)                      \
{                                                                              \
    int x, y;                                                                  \
    pixel *src = (pixel*)_src;                                                 \
    ptrdiff_t srcstride = _srcstride / sizeof(pixel);                          \
                                                                               \
    for (y = 0; y < height; y++) {                                             \
        for (x = 0; x < width; x++)                                            \
864
            dst[x] = QPEL_FILTER_ ## H(src, 1) >> (BIT_DEPTH - 8);             \
865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881
        src += srcstride;                                                      \
        dst += dststride;                                                      \
    }                                                                          \
}

#define PUT_HEVC_QPEL_V(V)                                                     \
static void FUNC(put_hevc_qpel_v ## V)(int16_t *dst,  ptrdiff_t dststride,     \
                                       uint8_t *_src, ptrdiff_t _srcstride,    \
                                       int width, int height,                  \
                                       int16_t* mcbuffer)                      \
{                                                                              \
    int x, y;                                                                  \
    pixel *src = (pixel*)_src;                                                 \
    ptrdiff_t srcstride = _srcstride / sizeof(pixel);                          \
                                                                               \
    for (y = 0; y < height; y++)  {                                            \
        for (x = 0; x < width; x++)                                            \
882
            dst[x] = QPEL_FILTER_ ## V(src, srcstride) >> (BIT_DEPTH - 8);     \
883 884 885 886 887
        src += srcstride;                                                      \
        dst += dststride;                                                      \
    }                                                                          \
}

888
#define PUT_HEVC_QPEL_HV(H, V)                                                 \
889 890 891 892
static void FUNC(put_hevc_qpel_h ## H ## v ## V)(int16_t *dst,                 \
                                                 ptrdiff_t dststride,          \
                                                 uint8_t *_src,                \
                                                 ptrdiff_t _srcstride,         \
893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919
                                                 int width, int height,        \
                                                 int16_t* mcbuffer)            \
{                                                                              \
    int x, y;                                                                  \
    pixel *src = (pixel*)_src;                                                 \
    ptrdiff_t srcstride = _srcstride / sizeof(pixel);                          \
                                                                               \
    int16_t tmp_array[(MAX_PB_SIZE + 7) * MAX_PB_SIZE];                        \
    int16_t *tmp = tmp_array;                                                  \
                                                                               \
    src -= ff_hevc_qpel_extra_before[V] * srcstride;                           \
                                                                               \
    for (y = 0; y < height + ff_hevc_qpel_extra[V]; y++) {                     \
        for (x = 0; x < width; x++)                                            \
            tmp[x] = QPEL_FILTER_ ## H(src, 1) >> (BIT_DEPTH - 8);             \
        src += srcstride;                                                      \
        tmp += MAX_PB_SIZE;                                                    \
    }                                                                          \
                                                                               \
    tmp = tmp_array + ff_hevc_qpel_extra_before[V] * MAX_PB_SIZE;              \
                                                                               \
    for (y = 0; y < height; y++) {                                             \
        for (x = 0; x < width; x++)                                            \
            dst[x] = QPEL_FILTER_ ## V(tmp, MAX_PB_SIZE) >> 6;                 \
        tmp += MAX_PB_SIZE;                                                    \
        dst += dststride;                                                      \
    }                                                                          \
920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943
}

PUT_HEVC_QPEL_H(1)
PUT_HEVC_QPEL_H(2)
PUT_HEVC_QPEL_H(3)
PUT_HEVC_QPEL_V(1)
PUT_HEVC_QPEL_V(2)
PUT_HEVC_QPEL_V(3)
PUT_HEVC_QPEL_HV(1, 1)
PUT_HEVC_QPEL_HV(1, 2)
PUT_HEVC_QPEL_HV(1, 3)
PUT_HEVC_QPEL_HV(2, 1)
PUT_HEVC_QPEL_HV(2, 2)
PUT_HEVC_QPEL_HV(2, 3)
PUT_HEVC_QPEL_HV(3, 1)
PUT_HEVC_QPEL_HV(3, 2)
PUT_HEVC_QPEL_HV(3, 3)

static void FUNC(put_hevc_epel_pixels)(int16_t *dst, ptrdiff_t dststride,
                                       uint8_t *_src, ptrdiff_t _srcstride,
                                       int width, int height, int mx, int my,
                                       int16_t* mcbuffer)
{
    int x, y;
944
    pixel *src          = (pixel *)_src;
945 946 947
    ptrdiff_t srcstride = _srcstride / sizeof(pixel);

    for (y = 0; y < height; y++) {
948
        for (x = 0; x < width; x++)
949 950 951 952 953 954
            dst[x] = src[x] << (14 - BIT_DEPTH);
        src += srcstride;
        dst += dststride;
    }
}

955
#define EPEL_FILTER(src, stride)                \
956 957 958 959
    (filter_0 * src[x - stride] +               \
     filter_1 * src[x]          +               \
     filter_2 * src[x + stride] +               \
     filter_3 * src[x + 2 * stride])
960 961 962 963 964 965 966

static void FUNC(put_hevc_epel_h)(int16_t *dst, ptrdiff_t dststride,
                                  uint8_t *_src, ptrdiff_t _srcstride,
                                  int width, int height, int mx, int my,
                                  int16_t* mcbuffer)
{
    int x, y;
967 968
    pixel *src = (pixel *)_src;
    ptrdiff_t srcstride  = _srcstride / sizeof(pixel);
969 970 971 972 973 974
    const int8_t *filter = ff_hevc_epel_filters[mx - 1];
    int8_t filter_0 = filter[0];
    int8_t filter_1 = filter[1];
    int8_t filter_2 = filter[2];
    int8_t filter_3 = filter[3];
    for (y = 0; y < height; y++) {
975
        for (x = 0; x < width; x++)
976 977 978 979 980 981 982 983 984 985 986 987
            dst[x] = EPEL_FILTER(src, 1) >> (BIT_DEPTH - 8);
        src += srcstride;
        dst += dststride;
    }
}

static void FUNC(put_hevc_epel_v)(int16_t *dst, ptrdiff_t dststride,
                                  uint8_t *_src, ptrdiff_t _srcstride,
                                  int width, int height, int mx, int my,
                                  int16_t* mcbuffer)
{
    int x, y;
988
    pixel *src = (pixel *)_src;
989
    ptrdiff_t srcstride = _srcstride / sizeof(pixel);
990
    const int8_t *filter = ff_hevc_epel_filters[my - 1];
991 992 993 994 995 996
    int8_t filter_0 = filter[0];
    int8_t filter_1 = filter[1];
    int8_t filter_2 = filter[2];
    int8_t filter_3 = filter[3];

    for (y = 0; y < height; y++) {
997
        for (x = 0; x < width; x++)
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009
            dst[x] = EPEL_FILTER(src, srcstride) >> (BIT_DEPTH - 8);
        src += srcstride;
        dst += dststride;
    }
}

static void FUNC(put_hevc_epel_hv)(int16_t *dst, ptrdiff_t dststride,
                                   uint8_t *_src, ptrdiff_t _srcstride,
                                   int width, int height, int mx, int my,
                                   int16_t* mcbuffer)
{
    int x, y;
1010
    pixel *src = (pixel *)_src;
1011
    ptrdiff_t srcstride = _srcstride / sizeof(pixel);
1012 1013
    const int8_t *filter_h = ff_hevc_epel_filters[mx - 1];
    const int8_t *filter_v = ff_hevc_epel_filters[my - 1];
1014 1015 1016 1017
    int8_t filter_0 = filter_h[0];
    int8_t filter_1 = filter_h[1];
    int8_t filter_2 = filter_h[2];
    int8_t filter_3 = filter_h[3];
1018
    int16_t tmp_array[(MAX_PB_SIZE + 3) * MAX_PB_SIZE];
1019 1020 1021 1022 1023
    int16_t *tmp = tmp_array;

    src -= EPEL_EXTRA_BEFORE * srcstride;

    for (y = 0; y < height + EPEL_EXTRA; y++) {
1024
        for (x = 0; x < width; x++)
1025 1026 1027 1028 1029
            tmp[x] = EPEL_FILTER(src, 1) >> (BIT_DEPTH - 8);
        src += srcstride;
        tmp += MAX_PB_SIZE;
    }

1030
    tmp      = tmp_array + EPEL_EXTRA_BEFORE * MAX_PB_SIZE;
1031 1032 1033 1034 1035
    filter_0 = filter_v[0];
    filter_1 = filter_v[1];
    filter_2 = filter_v[2];
    filter_3 = filter_v[3];
    for (y = 0; y < height; y++) {
1036
        for (x = 0; x < width; x++)
1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047
            dst[x] = EPEL_FILTER(tmp, MAX_PB_SIZE) >> 6;
        tmp += MAX_PB_SIZE;
        dst += dststride;
    }
}

static void FUNC(put_unweighted_pred)(uint8_t *_dst, ptrdiff_t _dststride,
                                      int16_t *src, ptrdiff_t srcstride,
                                      int width, int height)
{
    int x, y;
1048
    pixel *dst          = (pixel *)_dst;
1049 1050 1051 1052 1053 1054 1055 1056 1057
    ptrdiff_t dststride = _dststride / sizeof(pixel);

    int shift = 14 - BIT_DEPTH;
#if BIT_DEPTH < 14
    int offset = 1 << (shift - 1);
#else
    int offset = 0;
#endif
    for (y = 0; y < height; y++) {
1058
        for (x = 0; x < width; x++)
1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
            dst[x] = av_clip_pixel((src[x] + offset) >> shift);
        dst += dststride;
        src += srcstride;
    }
}

static void FUNC(put_weighted_pred_avg)(uint8_t *_dst, ptrdiff_t _dststride,
                                        int16_t *src1, int16_t *src2,
                                        ptrdiff_t srcstride,
                                        int width, int height)
{
    int x, y;
1071
    pixel *dst          = (pixel *)_dst;
1072 1073 1074 1075 1076 1077 1078 1079 1080 1081
    ptrdiff_t dststride = _dststride / sizeof(pixel);

    int shift = 14 + 1 - BIT_DEPTH;
#if BIT_DEPTH < 14
    int offset = 1 << (shift - 1);
#else
    int offset = 0;
#endif

    for (y = 0; y < height; y++) {
1082
        for (x = 0; x < width; x++)
1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094
            dst[x] = av_clip_pixel((src1[x] + src2[x] + offset) >> shift);
        dst  += dststride;
        src1 += srcstride;
        src2 += srcstride;
    }
}

static void FUNC(weighted_pred)(uint8_t denom, int16_t wlxFlag, int16_t olxFlag,
                                uint8_t *_dst, ptrdiff_t _dststride,
                                int16_t *src, ptrdiff_t srcstride,
                                int width, int height)
{
1095
    int shift, log2Wd, wx, ox, x, y, offset;
1096
    pixel *dst          = (pixel *)_dst;
1097 1098 1099 1100 1101
    ptrdiff_t dststride = _dststride / sizeof(pixel);

    shift  = 14 - BIT_DEPTH;
    log2Wd = denom + shift;
    offset = 1 << (log2Wd - 1);
1102 1103
    wx     = wlxFlag;
    ox     = olxFlag * (1 << (BIT_DEPTH - 8));
1104 1105 1106 1107 1108 1109 1110 1111 1112

    for (y = 0; y < height; y++) {
        for (x = 0; x < width; x++) {
            if (log2Wd >= 1) {
                dst[x] = av_clip_pixel(((src[x] * wx + offset) >> log2Wd) + ox);
            } else {
                dst[x] = av_clip_pixel(src[x] * wx + ox);
            }
        }
1113 1114
        dst += dststride;
        src += srcstride;
1115 1116 1117
    }
}

1118 1119
static void FUNC(weighted_pred_avg)(uint8_t denom,
                                    int16_t wl0Flag, int16_t wl1Flag,
1120 1121
                                    int16_t ol0Flag, int16_t ol1Flag,
                                    uint8_t *_dst, ptrdiff_t _dststride,
1122 1123
                                    int16_t *src1, int16_t *src2,
                                    ptrdiff_t srcstride,
1124 1125
                                    int width, int height)
{
1126
    int shift, log2Wd, w0, w1, o0, o1, x, y;
1127
    pixel *dst = (pixel *)_dst;
1128 1129 1130 1131
    ptrdiff_t dststride = _dststride / sizeof(pixel);

    shift  = 14 - BIT_DEPTH;
    log2Wd = denom + shift;
1132 1133
    w0     = wl0Flag;
    w1     = wl1Flag;
1134 1135
    o0     = ol0Flag * (1 << (BIT_DEPTH - 8));
    o1     = ol1Flag * (1 << (BIT_DEPTH - 8));
1136 1137

    for (y = 0; y < height; y++) {
1138
        for (x = 0; x < width; x++)
1139 1140 1141 1142 1143 1144 1145 1146 1147
            dst[x] = av_clip_pixel((src1[x] * w0 + src2[x] * w1 +
                                    ((o0 + o1 + 1) << log2Wd)) >> (log2Wd + 1));
        dst  += dststride;
        src1 += srcstride;
        src2 += srcstride;
    }
}

// line zero
1148 1149 1150
#define P3 pix[-4 * xstride]
#define P2 pix[-3 * xstride]
#define P1 pix[-2 * xstride]
1151
#define P0 pix[-1 * xstride]
1152
#define Q0 pix[0 * xstride]
1153
#define Q1 pix[1 * xstride]
1154 1155
#define Q2 pix[2 * xstride]
#define Q3 pix[3 * xstride]
1156 1157

// line three. used only for deblocking decision
1158 1159 1160
#define TP3 pix[-4 * xstride + 3 * ystride]
#define TP2 pix[-3 * xstride + 3 * ystride]
#define TP1 pix[-2 * xstride + 3 * ystride]
1161
#define TP0 pix[-1 * xstride + 3 * ystride]
1162
#define TQ0 pix[0  * xstride + 3 * ystride]
1163
#define TQ1 pix[1  * xstride + 3 * ystride]
1164 1165 1166 1167 1168 1169
#define TQ2 pix[2  * xstride + 3 * ystride]
#define TQ3 pix[3  * xstride + 3 * ystride]

static void FUNC(hevc_loop_filter_luma)(uint8_t *_pix,
                                        ptrdiff_t _xstride, ptrdiff_t _ystride,
                                        int *_beta, int *_tc,
1170 1171 1172
                                        uint8_t *_no_p, uint8_t *_no_q)
{
    int d, j;
1173
    pixel *pix        = (pixel *)_pix;
1174 1175 1176 1177
    ptrdiff_t xstride = _xstride / sizeof(pixel);
    ptrdiff_t ystride = _ystride / sizeof(pixel);

    for (j = 0; j < 2; j++) {
1178 1179
        const int dp0  = abs(P2  - 2 * P1  + P0);
        const int dq0  = abs(Q2  - 2 * Q1  + Q0);
1180 1181 1182 1183
        const int dp3  = abs(TP2 - 2 * TP1 + TP0);
        const int dq3  = abs(TQ2 - 2 * TQ1 + TQ0);
        const int d0   = dp0 + dq0;
        const int d3   = dp3 + dq3;
1184
        const int beta = _beta[j] << (BIT_DEPTH - 8);
1185
        const int tc   = _tc[j]   << (BIT_DEPTH - 8);
1186 1187 1188
        const int no_p = _no_p[j];
        const int no_q = _no_q[j];

1189
        if (d0 + d3 >= beta) {
1190 1191 1192 1193 1194
            pix += 4 * ystride;
            continue;
        } else {
            const int beta_3 = beta >> 3;
            const int beta_2 = beta >> 2;
1195
            const int tc25   = ((tc * 5 + 1) >> 1);
1196

1197
            if (abs(P3  -  P0) + abs(Q3  -  Q0) < beta_3 && abs(P0  -  Q0) < tc25 &&
1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211
                abs(TP3 - TP0) + abs(TQ3 - TQ0) < beta_3 && abs(TP0 - TQ0) < tc25 &&
                                      (d0 << 1) < beta_2 &&      (d3 << 1) < beta_2) {
                // strong filtering
                const int tc2 = tc << 1;
                for (d = 0; d < 4; d++) {
                    const int p3 = P3;
                    const int p2 = P2;
                    const int p1 = P1;
                    const int p0 = P0;
                    const int q0 = Q0;
                    const int q1 = Q1;
                    const int q2 = Q2;
                    const int q3 = Q3;
                    if (!no_p) {
1212 1213 1214
                        P0 = p0 + av_clip(((p2 + 2 * p1 + 2 * p0 + 2 * q0 + q1 + 4) >> 3) - p0, -tc2, tc2);
                        P1 = p1 + av_clip(((p2 + p1 + p0 + q0 + 2) >> 2) - p1, -tc2, tc2);
                        P2 = p2 + av_clip(((2 * p3 + 3 * p2 + p1 + p0 + q0 + 4) >> 3) - p2, -tc2, tc2);
1215 1216
                    }
                    if (!no_q) {
1217 1218 1219
                        Q0 = q0 + av_clip(((p1 + 2 * p0 + 2 * q0 + 2 * q1 + q2 + 4) >> 3) - q0, -tc2, tc2);
                        Q1 = q1 + av_clip(((p0 + q0 + q1 + q2 + 2) >> 2) - q1, -tc2, tc2);
                        Q2 = q2 + av_clip(((2 * q3 + 3 * q2 + q1 + q0 + p0 + 4) >> 3) - q2, -tc2, tc2);
1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
                    }
                    pix += ystride;
                }
            } else { // normal filtering
                int nd_p = 1;
                int nd_q = 1;
                const int tc_2 = tc >> 1;
                if (dp0 + dp3 < ((beta + (beta >> 1)) >> 3))
                    nd_p = 2;
                if (dq0 + dq3 < ((beta + (beta >> 1)) >> 3))
                    nd_q = 2;

                for (d = 0; d < 4; d++) {
                    const int p2 = P2;
                    const int p1 = P1;
                    const int p0 = P0;
                    const int q0 = Q0;
                    const int q1 = Q1;
                    const int q2 = Q2;
1239
                    int delta0   = (9 * (q0 - p0) - 3 * (q1 - p1) + 8) >> 4;
1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265
                    if (abs(delta0) < 10 * tc) {
                        delta0 = av_clip(delta0, -tc, tc);
                        if (!no_p)
                            P0 = av_clip_pixel(p0 + delta0);
                        if (!no_q)
                            Q0 = av_clip_pixel(q0 - delta0);
                        if (!no_p && nd_p > 1) {
                            const int deltap1 = av_clip((((p2 + p0 + 1) >> 1) - p1 + delta0) >> 1, -tc_2, tc_2);
                            P1 = av_clip_pixel(p1 + deltap1);
                        }
                        if (!no_q && nd_q > 1) {
                            const int deltaq1 = av_clip((((q2 + q0 + 1) >> 1) - q1 - delta0) >> 1, -tc_2, tc_2);
                            Q1 = av_clip_pixel(q1 + deltaq1);
                        }
                    }
                    pix += ystride;
                }
            }
        }
    }
}

static void FUNC(hevc_loop_filter_chroma)(uint8_t *_pix, ptrdiff_t _xstride,
                                          ptrdiff_t _ystride, int *_tc,
                                          uint8_t *_no_p, uint8_t *_no_q)
{
1266
    int d, j, no_p, no_q;
1267
    pixel *pix        = (pixel *)_pix;
1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296
    ptrdiff_t xstride = _xstride / sizeof(pixel);
    ptrdiff_t ystride = _ystride / sizeof(pixel);

    for (j = 0; j < 2; j++) {
        const int tc = _tc[j] << (BIT_DEPTH - 8);
        if (tc <= 0) {
            pix += 4 * ystride;
            continue;
        }
        no_p = _no_p[j];
        no_q = _no_q[j];

        for (d = 0; d < 4; d++) {
            int delta0;
            const int p1 = P1;
            const int p0 = P0;
            const int q0 = Q0;
            const int q1 = Q1;
            delta0 = av_clip((((q0 - p0) << 2) + p1 - q1 + 4) >> 3, -tc, tc);
            if (!no_p)
                P0 = av_clip_pixel(p0 + delta0);
            if (!no_q)
                Q0 = av_clip_pixel(q0 - delta0);
            pix += ystride;
        }
    }
}

static void FUNC(hevc_h_loop_filter_chroma)(uint8_t *pix, ptrdiff_t stride,
1297 1298
                                            int *tc, uint8_t *no_p,
                                            uint8_t *no_q)
1299 1300 1301 1302 1303
{
    FUNC(hevc_loop_filter_chroma)(pix, stride, sizeof(pixel), tc, no_p, no_q);
}

static void FUNC(hevc_v_loop_filter_chroma)(uint8_t *pix, ptrdiff_t stride,
1304 1305
                                            int *tc, uint8_t *no_p,
                                            uint8_t *no_q)
1306 1307 1308 1309 1310 1311 1312 1313
{
    FUNC(hevc_loop_filter_chroma)(pix, sizeof(pixel), stride, tc, no_p, no_q);
}

static void FUNC(hevc_h_loop_filter_luma)(uint8_t *pix, ptrdiff_t stride,
                                          int *beta, int *tc, uint8_t *no_p,
                                          uint8_t *no_q)
{
1314 1315
    FUNC(hevc_loop_filter_luma)(pix, stride, sizeof(pixel),
                                beta, tc, no_p, no_q);
1316 1317 1318 1319 1320 1321
}

static void FUNC(hevc_v_loop_filter_luma)(uint8_t *pix, ptrdiff_t stride,
                                          int *beta, int *tc, uint8_t *no_p,
                                          uint8_t *no_q)
{
1322 1323
    FUNC(hevc_loop_filter_luma)(pix, sizeof(pixel), stride,
                                beta, tc, no_p, no_q);
1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342
}

#undef P3
#undef P2
#undef P1
#undef P0
#undef Q0
#undef Q1
#undef Q2
#undef Q3

#undef TP3
#undef TP2
#undef TP1
#undef TP0
#undef TQ0
#undef TQ1
#undef TQ2
#undef TQ3