/*
 * Copyright © 2011 SCore Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 * DEALINGS IN THE SOFTWARE.
 *
 * Author:  Siarhei Siamashka (siarhei.siamashka@nokia.com)
 * Author:  Taekyun Kim (tkq.kim@samsung.com)
 */

/*
 * This file contains scaled bilinear scanline functions implemented
 * using older siarhei's bilinear macro template.
 *
 * << General scanline function procedures >>
 *  1. bilinear interpolate source pixels
 *  2. load mask pixels
 *  3. load destination pixels
 *  4. duplicate mask to fill whole register
 *  5. interleave source & destination pixels
 *  6. apply mask to source pixels
 *  7. combine source & destination pixels
 *  8, Deinterleave final result
 *  9. store destination pixels
 *
 * All registers with single number (i.e. src0, tmp0) are 64-bits registers.
 * Registers with double numbers(src01, dst01) are 128-bits registers.
 * All temp registers can be used freely outside the code block.
 * Assume that symbol(register .req) OUT and MASK are defined at caller of these macro blocks.
 *
 * Remarks
 *  There can be lots of pipeline stalls inside code block and between code blocks.
 *  Further optimizations will be done by new macro templates using head/tail_head/tail scheme.
 */

/* Prevent the stack from becoming executable for no reason... */
#if defined(__linux__) && defined (__ELF__)
.section .note.GNU-stack,"",%progbits
#endif

.text
.arch armv8-a
.altmacro
.p2align 2

#include "pixman-private.h"
#include "pixman-arm-asm.h"
#include "pixman-arma64-neon-asm.h"

/*
 * Bilinear macros from pixman-arm-neon-asm.S
 */

/*
 * Bilinear scaling support code which tries to provide pixel fetching, color
 * format conversion, and interpolation as separate macros which can be used
 * as the basic building blocks for constructing bilinear scanline functions.
 */

.macro bilinear_load_8888 reg1, reg2, tmp
    asr       WTMP1, X, #16
    add       X, X, UX
    add       TMP1, TOP, TMP1, lsl #2
    ld1       {&reg1&.2s}, [TMP1], STRIDE
    ld1       {&reg2&.2s}, [TMP1]
.endm

.macro bilinear_load_0565 reg1, reg2, tmp
    asr       WTMP1, X, #16
    add       X, X, UX
    add       TMP1, TOP, TMP1, lsl #1
    ld1       {&reg2&.s}[0], [TMP1], STRIDE
    ld1       {&reg2&.s}[1], [TMP1]
    convert_four_0565_to_x888_packed reg2, reg1, reg2, tmp
.endm

.macro bilinear_load_and_vertical_interpolate_two_8888 \
                    acc1, acc2, reg1, reg2, reg3, reg4, tmp1, tmp2

    bilinear_load_8888 reg1, reg2, tmp1
    umull     &acc1&.8h, &reg1&.8b, v28.8b
    umlal     &acc1&.8h, &reg2&.8b, v29.8b
    bilinear_load_8888 reg3, reg4, tmp2
    umull     &acc2&.8h, &reg3&.8b, v28.8b
    umlal     &acc2&.8h, &reg4&.8b, v29.8b
.endm

.macro bilinear_load_and_vertical_interpolate_four_8888 \
                xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi \
                yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi

    bilinear_load_and_vertical_interpolate_two_8888 \
                xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi
    bilinear_load_and_vertical_interpolate_two_8888 \
                yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi
.endm

.macro vzip reg1, reg2
    zip1      v24.8b, reg1, reg2
    zip2      reg2,   reg1, reg2
    mov       reg1,   v24.8b
.endm

.macro vuzp reg1, reg2
    uzp1     v24.8b, reg1, reg2
    uzp2     reg2,   reg1, reg2
    mov      reg1,   v24.8b
.endm

.macro bilinear_load_and_vertical_interpolate_two_0565 \
                acc1, acc2, reg1, reg2, reg3, reg4, acc2lo, acc2hi
    asr       WTMP1, X, #16
    add       X, X, UX
    add       TMP1, TOP, TMP1, lsl #1
    asr       WTMP2, X, #16
    add       X, X, UX
    add       TMP2, TOP, TMP2, lsl #1
    ld1       {&acc2&.s}[0], [TMP1], STRIDE
    ld1       {&acc2&.s}[2], [TMP2], STRIDE
    ld1       {&acc2&.s}[1], [TMP1]
    ld1       {&acc2&.s}[3], [TMP2]
    convert_0565_to_x888 acc2, reg3, reg2, reg1
    vzip      &reg1&.8b, &reg3&.8b
    vzip      &reg2&.8b, &reg4&.8b
    vzip      &reg3&.8b, &reg4&.8b
    vzip      &reg1&.8b, &reg2&.8b
    umull     &acc1&.8h, &reg1&.8b, v28.8b
    umlal     &acc1&.8h, &reg2&.8b, v29.8b
    umull     &acc2&.8h, &reg3&.8b, v28.8b
    umlal     &acc2&.8h, &reg4&.8b, v29.8b
.endm

.macro bilinear_load_and_vertical_interpolate_four_0565 \
                xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi \
                yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi

    asr       WTMP1, X, #16
    add       X, X, UX
    add       TMP1, TOP, TMP1, lsl #1
    asr       WTMP2, X, #16
    add       X, X, UX
    add       TMP2, TOP, TMP2, lsl #1
    ld1       {&xacc2&.s}[0], [TMP1], STRIDE
    ld1       {&xacc2&.s}[2], [TMP2], STRIDE
    ld1       {&xacc2&.s}[1], [TMP1]
    ld1       {&xacc2&.s}[3], [TMP2]
    convert_0565_to_x888 xacc2, xreg3, xreg2, xreg1
    asr       WTMP1, X, #16
    add       X, X, UX
    add       TMP1, TOP, TMP1, lsl #1
    asr       WTMP2, X, #16
    add       X, X, UX
    add       TMP2, TOP, TMP2, lsl #1
    ld1       {&yacc2&.s}[0], [TMP1], STRIDE
    vzip      &xreg1&.8b, &xreg3&.8b
    ld1       {&yacc2&.s}[2], [TMP2], STRIDE
    vzip      &xreg2&.8b, &xreg4&.8b
    ld1       {&yacc2&.s}[1], [TMP1]
    vzip      &xreg3&.8b, &xreg4&.8b
    ld1       {&yacc2&.s}[3], [TMP2]
    vzip      &xreg1&.8b, &xreg2&.8b
    convert_0565_to_x888 yacc2, yreg3, yreg2, yreg1
    umull     &xacc1&.8h, &xreg1&.8b, v28.8b
    vzip      &yreg1&.8b, &yreg3&.8b
    umlal     &xacc1&.8h, &xreg2&.8b, v29.8b
    vzip      &yreg2&.8b, &yreg4&.8b
    umull     &xacc2&.8h, &xreg3&.8b, v28.8b
    vzip      &yreg3&.8b, &yreg4&.8b
    umlal     &xacc2&.8h, &xreg4&.8b, v29.8b
    vzip      &yreg1&.8b, &yreg2&.8b
    umull     &yacc1&.8h, &yreg1&.8b, v28.8b
    umlal     &yacc1&.8h, &yreg2&.8b, v29.8b
    umull     &yacc2&.8h, &yreg3&.8b, v28.8b
    umlal     &yacc2&.8h, &yreg4&.8b, v29.8b
.endm

.macro bilinear_store_8888 numpix, tmp1, tmp2
.if numpix == 4
    st1       {v0.2s, v1.2s}, [OUT], #16
.elseif numpix == 2
    st1       {v0.2s}, [OUT], #8
.elseif numpix == 1
    st1       {v0.s}[0], [OUT], #4
.else
    .error bilinear_store_8888 numpix is unsupported
.endif
.endm

.macro bilinear_store_0565 numpix, tmp1, tmp2
    vuzp    v0.8b, v1.8b
    vuzp    v2.8b, v3.8b
    vuzp    v1.8b, v3.8b
    vuzp    v0.8b, v2.8b
    convert_8888_to_0565 v2, v1, v0, v1, tmp1, tmp2
.if numpix == 4
    st1       {v1.4h}, [OUT], #8
.elseif numpix == 2
    st1       {v1.s}[0], [OUT], #4
.elseif numpix == 1
    st1       {v1.h}[0], [OUT], #2
.else
    .error bilinear_store_0565 numpix is unsupported
.endif
.endm


/*
 * Macros for loading mask pixels into register 'mask'.
 * dup must be done in somewhere else.
 */
.macro bilinear_load_mask_x numpix, mask
.endm

.macro bilinear_load_mask_8 numpix, mask
.if numpix == 4
    ld1         {&mask&.s}[0], [MASK], #4
.elseif numpix == 2
    ld1         {&mask&.h}[0], [MASK], #2
.elseif numpix == 1
    ld1         {&mask&.b}[0], [MASK], #1
.else
    .error bilinear_load_mask_8 numpix is unsupported
.endif
    prfm        PREFETCH_MODE, [MASK, #prefetch_offset]
.endm

.macro bilinear_load_mask mask_fmt, numpix, mask
    bilinear_load_mask_&mask_fmt numpix, mask
.endm


/*
 * Macros for loading destination pixels into register 'dst0' and 'dst1'.
 * Interleave should be done somewhere else.
 */
.macro bilinear_load_dst_0565_src numpix, dst0, dst1, dst01
.endm

.macro bilinear_load_dst_8888_src numpix, dst0, dst1, dst01
.endm

.macro bilinear_load_dst_8888 numpix, dst0, dst1, dst01
.if numpix == 4
    ld1         {&dst0&.2s, &dst1&.2s}, [OUT]
.elseif numpix == 2
    ld1         {&dst0&.2s}, [OUT]
.elseif numpix == 1
    ld1         {&dst0&.s}[0], [OUT]
.else
    .error bilinear_load_dst_8888 numpix is unsupported
.endif
    mov         &dst01&.d[0], &dst0&.d[0]
    mov         &dst01&.d[1], &dst1&.d[0]
    prfm        PREFETCH_MODE, [OUT, #(prefetch_offset * 4)]
.endm

.macro bilinear_load_dst_8888_over numpix, dst0, dst1, dst01
    bilinear_load_dst_8888 numpix, dst0, dst1, dst01
.endm

.macro bilinear_load_dst_8888_add numpix, dst0, dst1, dst01
    bilinear_load_dst_8888 numpix, dst0, dst1, dst01
.endm

.macro bilinear_load_dst dst_fmt, op, numpix, dst0, dst1, dst01
    bilinear_load_dst_&dst_fmt&_&op numpix, dst0, dst1, dst01
.endm

/*
 * Macros for duplicating partially loaded mask to fill entire register.
 * We will apply mask to interleaved source pixels, that is
 *  (r0, r1, r2, r3, g0, g1, g2, g3) x (m0, m1, m2, m3, m0, m1, m2, m3)
 *  (b0, b1, b2, b3, a0, a1, a2, a3) x (m0, m1, m2, m3, m0, m1, m2, m3)
 * So, we need to duplicate loaded mask into whole register.
 *
 * For two pixel case
 *  (r0, r1, x, x, g0, g1, x, x) x (m0, m1, m0, m1, m0, m1, m0, m1)
 *  (b0, b1, x, x, a0, a1, x, x) x (m0, m1, m0, m1, m0, m1, m0, m1)
 * We can do some optimizations for this including last pixel cases.
 */
.macro bilinear_duplicate_mask_x numpix, mask
.endm

.macro bilinear_duplicate_mask_8 numpix, mask
.if numpix == 4
    dup         &mask&.2s, &mask&.s[0]
.elseif numpix == 2
    dup         &mask&.4h, &mask&.h[0]
.elseif numpix == 1
    dup         &mask&.8b, &mask&.b[0]
.else
    .error bilinear_duplicate_mask_8 is unsupported
.endif
.endm

.macro bilinear_duplicate_mask mask_fmt, numpix, mask
    bilinear_duplicate_mask_&mask_fmt numpix, mask
.endm

/*
 * Macros for interleaving src and dst pixels to rrrr gggg bbbb aaaa form.
 * Interleave should be done when maks is enabled or operator is 'over'.
 */
.macro bilinear_interleave src0, src1, src01, dst0, dst1, dst01
    vuzp       &src0&.8b, &src1&.8b
    vuzp       &dst0&.8b, &dst1&.8b
    vuzp       &src0&.8b, &src1&.8b
    vuzp       &dst0&.8b, &dst1&.8b
    mov        &src01&.d[1], &src1&.d[0]
    mov        &src01&.d[0], &src0&.d[0]
    mov        &dst01&.d[1], &dst1&.d[0]
    mov        &dst01&.d[0], &dst0&.d[0]
.endm

.macro bilinear_interleave_src_dst_x_src \
                numpix, src0, src1, src01, dst0, dst1, dst01
.endm

.macro bilinear_interleave_src_dst_x_over \
                numpix, src0, src1, src01, dst0, dst1, dst01

    bilinear_interleave src0, src1, src01, dst0, dst1, dst01
.endm

.macro bilinear_interleave_src_dst_x_add \
                numpix, src0, src1, src01, dst0, dst1, dst01
    bilinear_interleave src0, src1, src01, dst0, dst1, dst01
.endm

.macro bilinear_interleave_src_dst_8_src \
                numpix, src0, src1, src01, dst0, dst1, dst01

    bilinear_interleave src0, src1, src01, dst0, dst1, dst01
.endm

.macro bilinear_interleave_src_dst_8_over \
                numpix, src0, src1, src01, dst0, dst1, dst01

    bilinear_interleave src0, src1, src01, dst0, dst1, dst01
.endm

.macro bilinear_interleave_src_dst_8_add \
                numpix, src0, src1, src01, dst0, dst1, dst01

    bilinear_interleave src0, src1, src01, dst0, dst1, dst01
.endm

.macro bilinear_interleave_src_dst \
                mask_fmt, op, numpix, src0, src1, src01, dst0, dst1, dst01

    bilinear_interleave_src_dst_&mask_fmt&_&op \
                numpix, src0, src1, src01, dst0, dst1, dst01
.endm


/*
 * Macros for applying masks to src pixels. (see combine_mask_u() function)
 * src, dst should be in interleaved form.
 * mask register should be in form (m0, m1, m2, m3).
 */
.macro bilinear_apply_mask_to_src_x \
                numpix, src0, src1, src01, mask, \
                tmp01, tmp23, tmp45, tmp67
.endm

.macro bilinear_apply_mask_to_src_8 \
                numpix, src0, src1, src01, mask, \
                tmp01, tmp23, tmp45, tmp67

    umull           &tmp01&.8h, &src0&.8b, &mask&.8b
    umull           &tmp23&.8h, &src1&.8b, &mask&.8b
    /* bubbles */
    urshr           &tmp45&.8h, &tmp01&.8h, #8
    urshr           &tmp67&.8h, &tmp23&.8h, #8
    /* bubbles */
    raddhn          &src0&.8b, &tmp45&.8h, &tmp01&.8h
    raddhn          &src1&.8b, &tmp67&.8h, &tmp23&.8h
    mov             &src01&.d[0], &src0&.d[0]
    mov             &src01&.d[1], &src1&.d[0]
.endm

.macro bilinear_apply_mask_to_src \
                mask_fmt, numpix, src0, src1, src01, mask, \
                tmp01, tmp23, tmp45, tmp67

    bilinear_apply_mask_to_src_&mask_fmt \
                numpix, src0, src1, src01, mask, \
                tmp01, tmp23, tmp45, tmp67
.endm


/*
 * Macros for combining src and destination pixels.
 * Interleave or not is depending on operator 'op'.
 */
.macro bilinear_combine_src \
                numpix, src0, src1, src01, dst0, dst1, dst01, \
                tmp01, tmp23, tmp45, tmp67, tmp8
.endm

.macro bilinear_combine_over \
                numpix, src0, src1, src01, dst0, dst1, dst01, \
                tmp01, tmp23, tmp45, tmp67, tmp8

    dup         &tmp8&.2s, &src1&.s[1]
    /* bubbles */
    mvn         &tmp8&.8b, &tmp8&.8b
    /* bubbles */
    umull       &tmp01&.8h, &dst0&.8b, &tmp8&.8b
    /* bubbles */
    umull       &tmp23&.8h, &dst1&.8b, &tmp8&.8b
    /* bubbles */
    urshr       &tmp45&.8h, &tmp01&.8h, #8
    urshr       &tmp67&.8h, &tmp23&.8h, #8
    /* bubbles */
    raddhn      &dst0&.8b, &tmp45&.8h, &tmp01&.8h
    raddhn      &dst1&.8b, &tmp67&.8h, &tmp23&.8h
    mov         &dst01&.d[0], &dst0&.d[0]
    mov         &dst01&.d[1], &dst1&.d[0]
    /* bubbles */
    uqadd       &src0&.8b, &dst0&.8b, &src0&.8b
    uqadd       &src1&.8b, &dst1&.8b, &src1&.8b
    mov         &src01&.d[0], &src0&.d[0]
    mov         &src01&.d[1], &src1&.d[0]
.endm

.macro bilinear_combine_add \
                numpix, src0, src1, src01, dst0, dst1, dst01, \
                tmp01, tmp23, tmp45, tmp67, tmp8

    uqadd       &src0&.8b, &dst0&.8b, &src0&.8b
    uqadd       &src1&.8b, &dst1&.8b, &src1&.8b
    mov         &src01&.d[0], &src0&.d[0]
    mov         &src01&.d[1], &src1&.d[0]
.endm

.macro bilinear_combine \
                op, numpix, src0, src1, src01, dst0, dst1, dst01, \
                tmp01, tmp23, tmp45, tmp67, tmp8

    bilinear_combine_&op \
                numpix, src0, src1, src01, dst0, dst1, dst01, \
                tmp01, tmp23, tmp45, tmp67, tmp8
.endm

/*
 * Macros for final deinterleaving of destination pixels if needed.
 */
.macro bilinear_deinterleave numpix, dst0, dst1, dst01
    vuzp       &dst0&.8b, &dst1&.8b
    /* bubbles */
    vuzp       &dst0&.8b, &dst1&.8b
    mov        &dst01&.d[0], &dst0&.d[0]
    mov        &dst01&.d[1], &dst1&.d[0]
.endm

.macro bilinear_deinterleave_dst_x_src numpix, dst0, dst1, dst01
.endm

.macro bilinear_deinterleave_dst_x_over numpix, dst0, dst1, dst01
    bilinear_deinterleave numpix, dst0, dst1, dst01
.endm

.macro bilinear_deinterleave_dst_x_add numpix, dst0, dst1, dst01
    bilinear_deinterleave numpix, dst0, dst1, dst01
.endm

.macro bilinear_deinterleave_dst_8_src numpix, dst0, dst1, dst01
    bilinear_deinterleave numpix, dst0, dst1, dst01
.endm

.macro bilinear_deinterleave_dst_8_over numpix, dst0, dst1, dst01
    bilinear_deinterleave numpix, dst0, dst1, dst01
.endm

.macro bilinear_deinterleave_dst_8_add numpix, dst0, dst1, dst01
    bilinear_deinterleave numpix, dst0, dst1, dst01
.endm

.macro bilinear_deinterleave_dst mask_fmt, op, numpix, dst0, dst1, dst01
    bilinear_deinterleave_dst_&mask_fmt&_&op numpix, dst0, dst1, dst01
.endm


.macro bilinear_interpolate_last_pixel src_fmt, mask_fmt, dst_fmt, op
    bilinear_load_&src_fmt v0, v1, v2
    bilinear_load_mask mask_fmt, 1, v4
    bilinear_load_dst dst_fmt, op, 1, v18, v19, v9
    umull     v2.8h, v0.8b, v28.8b
    umlal     v2.8h, v1.8b, v29.8b
    /* 5 cycles bubble */
    ushll     v0.4s, v2.4h, #BILINEAR_INTERPOLATION_BITS
    umlsl     v0.4s, v2.4h, v15.h[0]
    umlal2    v0.4s, v2.8h, v15.h[0]
    /* 5 cycles bubble */
    bilinear_duplicate_mask mask_fmt, 1, v4
    shrn      v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
    /* 3 cycles bubble */
    xtn       v0.8b, v0.8h
    /* 1 cycle bubble */
    bilinear_interleave_src_dst \
                mask_fmt, op, 1, v0, v1, v0, v18, v19, v9
    bilinear_apply_mask_to_src \
                mask_fmt, 1, v0, v1, v0, v4, \
                v3, v8, v10, v11
    bilinear_combine \
                op, 1, v0, v1, v0, v18, v19, v9, \
                v3, v8, v10, v11, v5
    bilinear_deinterleave_dst mask_fmt, op, 1, v0, v1, v0
    bilinear_store_&dst_fmt 1, v17, v18
.endm

.macro bilinear_interpolate_two_pixels src_fmt, mask_fmt, dst_fmt, op
    bilinear_load_and_vertical_interpolate_two_&src_fmt \
                v1, v11, v18, v19, v20, v21, v22, v23
    bilinear_load_mask mask_fmt, 2, v4
    bilinear_load_dst dst_fmt, op, 2, v18, v19, v9
    ushll     v0.4s, v1.4h, #BILINEAR_INTERPOLATION_BITS
    umlsl     v0.4s, v1.4h, v15.h[0]
    umlal2    v0.4s, v1.8h, v15.h[0]
    ushll     v10.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS
    umlsl     v10.4s, v11.4h, v15.h[4]
    umlal2    v10.4s, v11.8h, v15.h[4]
    shrn      v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
    shrn2     v0.8h, v10.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
    bilinear_duplicate_mask mask_fmt, 2, v4
    ushr      v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
    add       v12.8h, v12.8h, v13.8h
    xtn       v0.8b, v0.8h
    bilinear_interleave_src_dst \
                mask_fmt, op, 2, v0, v1, v0, v18, v19, v9
    bilinear_apply_mask_to_src \
                mask_fmt, 2, v0, v1, v0, v4, \
                v3, v8, v10, v11
    bilinear_combine \
                op, 2, v0, v1, v0, v18, v19, v9, \
                v3, v8, v10, v11, v5
    bilinear_deinterleave_dst mask_fmt, op, 2, v0, v1, v0
    bilinear_store_&dst_fmt 2, v16, v17
.endm

.macro bilinear_interpolate_four_pixels src_fmt, mask_fmt, dst_fmt, op
    bilinear_load_and_vertical_interpolate_four_&src_fmt \
                v1, v11, v4,  v5,  v6,  v7,  v22, v23 \
                v3, v9,  v16, v17, v20, v21, v18, v19
    prfm      PREFETCH_MODE, [TMP1, PF_OFFS]
    sub       TMP1, TMP1, STRIDE
    prfm      PREFETCH_MODE, [TMP1, PF_OFFS]
    ushll     v0.4s, v1.4h, #BILINEAR_INTERPOLATION_BITS
    umlsl     v0.4s, v1.4h, v15.h[0]
    umlal2    v0.4s, v1.8h, v15.h[0]
    ushll     v10.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS
    umlsl     v10.4s, v11.4h, v15.h[4]
    umlal2    v10.4s, v11.8h, v15.h[4]
    ushr      v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
    ushll     v2.4s, v3.4h, #BILINEAR_INTERPOLATION_BITS
    umlsl     v2.4s, v3.4h, v15.h[0]
    umlal2    v2.4s, v3.8h, v15.h[0]
    ushll     v8.4s, v9.4h, #BILINEAR_INTERPOLATION_BITS
    umlsl     v8.4s, v9.4h, v15.h[4]
    umlal2    v8.4s, v9.8h, v15.h[4]
    add       v12.8h, v12.8h, v13.8h
    shrn      v0.4h,  v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
    shrn2     v0.8h, v10.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
    shrn      v2.4h,  v2.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
    shrn2     v2.8h,  v8.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
    bilinear_load_mask mask_fmt, 4, v4
    bilinear_duplicate_mask mask_fmt, 4, v4
    ushr      v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
    xtn       v0.8b, v0.8h
    xtn       v1.8b, v2.8h
    add       v12.8h, v12.8h, v13.8h
    bilinear_load_dst dst_fmt, op, 4, v2, v3, v21
    bilinear_interleave_src_dst \
                mask_fmt, op, 4, v0, v1, v0, v2, v3, v11
    bilinear_apply_mask_to_src \
                mask_fmt, 4, v0, v1, v0, v4, \
                v6, v8, v9, v10
    bilinear_combine \
                op, 4, v0, v1, v0, v2, v3, v1, \
                v6, v8, v9, v10, v23
    bilinear_deinterleave_dst mask_fmt, op, 4, v0, v1, v0
    bilinear_store_&dst_fmt 4, v6, v7
.endm

.set BILINEAR_FLAG_USE_MASK,        1
.set BILINEAR_FLAG_USE_ALL_NEON_REGS,    2

/*
 * Main template macro for generating NEON optimized bilinear scanline functions.
 *
 * Bilinear scanline generator macro take folling arguments:
 *  fname            - name of the function to generate
 *  src_fmt            - source color format (8888 or 0565)
 *  dst_fmt            - destination color format (8888 or 0565)
 *  src/dst_bpp_shift        - (1 << bpp_shift) is the size of src/dst pixel in bytes
 *  process_last_pixel        - code block that interpolate one pixel and does not
 *                  update horizontal weight
 *  process_two_pixels        - code block that interpolate two pixels and update
 *                  horizontal weight
 *  process_four_pixels        - code block that interpolate four pixels and update
 *                  horizontal weight
 *  process_pixblock_head    - head part of middle loop
 *  process_pixblock_tail    - tail part of middle loop
 *  process_pixblock_tail_head    - tail_head of middle loop
 *  pixblock_size        - number of pixels processed in a single middle loop
 *  prefetch_distance        - prefetch in the source image by that many pixels ahead
 */

.macro generate_bilinear_scanline_func \
    fname, \
    src_fmt, dst_fmt, src_bpp_shift, dst_bpp_shift, \
    bilinear_process_last_pixel, \
    bilinear_process_two_pixels, \
    bilinear_process_four_pixels, \
    bilinear_process_pixblock_head, \
    bilinear_process_pixblock_tail, \
    bilinear_process_pixblock_tail_head, \
    pixblock_size, \
    prefetch_distance, \
    flags

pixman_asm_function fname
.if pixblock_size == 8
.elseif pixblock_size == 4
.else
    .error unsupported pixblock size
.endif

.if ((flags) & BILINEAR_FLAG_USE_MASK) == 0
    OUT       .req    x0
    TOP       .req    x1
    BOTTOM    .req    x2
    WT        .req    x3
    WWT       .req    w3
    WB        .req    x4
    WWB       .req    w4
    X         .req    w5
    UX        .req    w6
    WIDTH     .req    x7
    TMP1      .req    x10
    WTMP1     .req    w10
    TMP2      .req    x11
    WTMP2     .req    w11
    PF_OFFS   .req    x12
    TMP3      .req    x13
    WTMP3     .req    w13
    TMP4      .req    x14
    WTMP4     .req    w14
    STRIDE    .req    x15
    DUMMY     .req    x30

    stp       x29, x30, [sp, -16]!
    mov       x29, sp
    sub       sp, sp, 112
    sub       x29, x29, 64
    st1       {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32
    st1       {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32
    stp       x10, x11, [x29, -80]
    stp       x12, x13, [x29, -96]
    stp       x14, x15, [x29, -112]
.else
    OUT       .req      x0
    MASK      .req      x1
    TOP       .req      x2
    BOTTOM    .req      x3
    WT        .req      x4
    WWT       .req      w4
    WB        .req      x5
    WWB       .req      w5
    X         .req      w6
    UX        .req      w7
    WIDTH     .req      x8
    TMP1      .req      x10
    WTMP1     .req      w10
    TMP2      .req      x11
    WTMP2     .req      w11
    PF_OFFS   .req      x12
    TMP3      .req      x13
    WTMP3     .req      w13
    TMP4      .req      x14
    WTMP4     .req      w14
    STRIDE    .req      x15
    DUMMY     .req      x30

    .set prefetch_offset, prefetch_distance

    stp      x29, x30, [sp, -16]!
    mov      x29, sp
    sub      x29, x29, 64
    st1      {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32
    st1      {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32
    stp      x10, x11, [x29, -80]
    stp      x12, x13, [x29, -96]
    stp      x14, x15, [x29, -112]
    str      x8, [x29, -120]
    ldr      w8, [x29, 16]
    sub      sp, sp, 120
.endif

    mov      WTMP1, #prefetch_distance
    umull    PF_OFFS, WTMP1, UX

    sub      STRIDE, BOTTOM, TOP
    .unreq   BOTTOM

    cmp      WIDTH, #0
    ble      300f

    dup      v12.8h, X
    dup      v13.8h, UX
    dup      v28.8b, WWT
    dup      v29.8b, WWB
    mov      v25.d[0], v12.d[1]
    mov      v26.d[0], v13.d[0]
    add      v25.4h, v25.4h, v26.4h
    mov      v12.d[1], v25.d[0]

    /* ensure good destination alignment  */
    cmp       WIDTH, #1
    blt       100f
    tst       OUT, #(1 << dst_bpp_shift)
    beq       100f
    ushr      v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
    add       v12.8h, v12.8h, v13.8h
    bilinear_process_last_pixel
    sub       WIDTH, WIDTH, #1
100:
    add       v13.8h, v13.8h, v13.8h
    ushr      v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
    add       v12.8h, v12.8h, v13.8h

    cmp       WIDTH, #2
    blt       100f
    tst       OUT, #(1 << (dst_bpp_shift + 1))
    beq       100f
    bilinear_process_two_pixels
    sub       WIDTH, WIDTH, #2
100:
.if pixblock_size == 8
    cmp       WIDTH, #4
    blt       100f
    tst       OUT, #(1 << (dst_bpp_shift + 2))
    beq       100f
    bilinear_process_four_pixels
    sub       WIDTH, WIDTH, #4
100:
.endif
    subs      WIDTH, WIDTH, #pixblock_size
    blt       100f
    asr       PF_OFFS, PF_OFFS, #(16 - src_bpp_shift)
    bilinear_process_pixblock_head
    subs      WIDTH, WIDTH, #pixblock_size
    blt       500f
0:
    bilinear_process_pixblock_tail_head
    subs      WIDTH, WIDTH, #pixblock_size
    bge       0b
500:
    bilinear_process_pixblock_tail
100:
.if pixblock_size == 8
    tst       WIDTH, #4
    beq       200f
    bilinear_process_four_pixels
200:
.endif
    /* handle the remaining trailing pixels */
    tst       WIDTH, #2
    beq       200f
    bilinear_process_two_pixels
200:
    tst       WIDTH, #1
    beq       300f
    bilinear_process_last_pixel
300:

.if ((flags) & BILINEAR_FLAG_USE_MASK) == 0
    sub       x29, x29, 64
    ld1       {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32
    ld1       {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32
    ldp       x10, x11, [x29, -80]
    ldp       x12, x13, [x29, -96]
    ldp       x14, x15, [x29, -112]
    mov       sp, x29
    ldp       x29, x30, [sp], 16
.else
    sub       x29, x29, 64
    ld1       {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], 32
    ld1       {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], 32
    ldp       x10, x11, [x29, -80]
    ldp       x12, x13, [x29, -96]
    ldp       x14, x15, [x29, -112]
    ldr       x8, [x29, -120]
    mov       sp, x29
    ldp       x29, x30, [sp], 16
.endif
    ret

    .unreq    OUT
    .unreq    TOP
    .unreq    WT
    .unreq    WWT
    .unreq    WB
    .unreq    WWB
    .unreq    X
    .unreq    UX
    .unreq    WIDTH
    .unreq    TMP1
    .unreq    WTMP1
    .unreq    TMP2
    .unreq    PF_OFFS
    .unreq    TMP3
    .unreq    TMP4
    .unreq    STRIDE
.if ((flags) & BILINEAR_FLAG_USE_MASK) != 0
    .unreq    MASK
.endif

.endfunc

.endm

/* src_8888_8_8888 */
.macro bilinear_src_8888_8_8888_process_last_pixel
    bilinear_interpolate_last_pixel 8888, 8, 8888, src
.endm

.macro bilinear_src_8888_8_8888_process_two_pixels
    bilinear_interpolate_two_pixels 8888, 8, 8888, src
.endm

.macro bilinear_src_8888_8_8888_process_four_pixels
    bilinear_interpolate_four_pixels 8888, 8, 8888, src
.endm

.macro bilinear_src_8888_8_8888_process_pixblock_head
    bilinear_src_8888_8_8888_process_four_pixels
.endm

.macro bilinear_src_8888_8_8888_process_pixblock_tail
.endm

.macro bilinear_src_8888_8_8888_process_pixblock_tail_head
    bilinear_src_8888_8_8888_process_pixblock_tail
    bilinear_src_8888_8_8888_process_pixblock_head
.endm

/* src_8888_8_0565 */
.macro bilinear_src_8888_8_0565_process_last_pixel
    bilinear_interpolate_last_pixel 8888, 8, 0565, src
.endm

.macro bilinear_src_8888_8_0565_process_two_pixels
    bilinear_interpolate_two_pixels 8888, 8, 0565, src
.endm

.macro bilinear_src_8888_8_0565_process_four_pixels
    bilinear_interpolate_four_pixels 8888, 8, 0565, src
.endm

.macro bilinear_src_8888_8_0565_process_pixblock_head
    bilinear_src_8888_8_0565_process_four_pixels
.endm

.macro bilinear_src_8888_8_0565_process_pixblock_tail
.endm

.macro bilinear_src_8888_8_0565_process_pixblock_tail_head
    bilinear_src_8888_8_0565_process_pixblock_tail
    bilinear_src_8888_8_0565_process_pixblock_head
.endm

/* src_0565_8_x888 */
.macro bilinear_src_0565_8_x888_process_last_pixel
    bilinear_interpolate_last_pixel 0565, 8, 8888, src
.endm

.macro bilinear_src_0565_8_x888_process_two_pixels
    bilinear_interpolate_two_pixels 0565, 8, 8888, src
.endm

.macro bilinear_src_0565_8_x888_process_four_pixels
    bilinear_interpolate_four_pixels 0565, 8, 8888, src
.endm

.macro bilinear_src_0565_8_x888_process_pixblock_head
    bilinear_src_0565_8_x888_process_four_pixels
.endm

.macro bilinear_src_0565_8_x888_process_pixblock_tail
.endm

.macro bilinear_src_0565_8_x888_process_pixblock_tail_head
    bilinear_src_0565_8_x888_process_pixblock_tail
    bilinear_src_0565_8_x888_process_pixblock_head
.endm

/* src_0565_8_0565 */
.macro bilinear_src_0565_8_0565_process_last_pixel
    bilinear_interpolate_last_pixel 0565, 8, 0565, src
.endm

.macro bilinear_src_0565_8_0565_process_two_pixels
    bilinear_interpolate_two_pixels 0565, 8, 0565, src
.endm

.macro bilinear_src_0565_8_0565_process_four_pixels
    bilinear_interpolate_four_pixels 0565, 8, 0565, src
.endm

.macro bilinear_src_0565_8_0565_process_pixblock_head
    bilinear_src_0565_8_0565_process_four_pixels
.endm

.macro bilinear_src_0565_8_0565_process_pixblock_tail
.endm

.macro bilinear_src_0565_8_0565_process_pixblock_tail_head
    bilinear_src_0565_8_0565_process_pixblock_tail
    bilinear_src_0565_8_0565_process_pixblock_head
.endm

/* over_8888_8888 */
.macro bilinear_over_8888_8888_process_last_pixel
    bilinear_interpolate_last_pixel 8888, x, 8888, over
.endm

.macro bilinear_over_8888_8888_process_two_pixels
    bilinear_interpolate_two_pixels 8888, x, 8888, over
.endm

.macro bilinear_over_8888_8888_process_four_pixels
    bilinear_interpolate_four_pixels 8888, x, 8888, over
.endm

.macro bilinear_over_8888_8888_process_pixblock_head
    asr         WTMP1, X, #16
    add         X, X, UX
    add         TMP1, TOP, TMP1, lsl #2
    asr         WTMP2, X, #16
    add         X, X, UX
    add         TMP2, TOP, TMP2, lsl #2

    ld1         {v22.2s}, [TMP1], STRIDE
    ld1         {v23.2s}, [TMP1]
    asr         WTMP3, X, #16
    add         X, X, UX
    add         TMP3, TOP, TMP3, lsl #2
    umull       v8.8h, v22.8b, v28.8b
    umlal       v8.8h, v23.8b, v29.8b

    ld1         {v22.2s}, [TMP2], STRIDE
    ld1         {v23.2s}, [TMP2]
    asr         WTMP4, X, #16
    add         X, X, UX
    add         TMP4, TOP, TMP4, lsl #2
    umull       v9.8h, v22.8b, v28.8b
    umlal       v9.8h, v23.8b, v29.8b

    ld1         {v22.2s}, [TMP3], STRIDE
    ld1         {v23.2s}, [TMP3]
    umull       v10.8h, v22.8b, v28.8b
    umlal       v10.8h, v23.8b, v29.8b

    ushll       v0.4s, v8.4h, #BILINEAR_INTERPOLATION_BITS
    umlsl       v0.4s, v8.4h, v15.h[0]
    umlal2      v0.4s, v8.8h, v15.h[0]

    prfm        PREFETCH_MODE, [TMP4, PF_OFFS]
    ld1         {v16.2s}, [TMP4], STRIDE
    ld1         {v17.2s}, [TMP4]
    prfm        PREFETCH_MODE, [TMP4, PF_OFFS]
    umull       v11.8h, v16.8b, v28.8b
    umlal       v11.8h, v17.8b, v29.8b

    ushll       v1.4s, v9.4h, #BILINEAR_INTERPOLATION_BITS
    umlsl       v1.4s, v9.4h, v15.h[4]
    umlal2      v1.4s, v9.8h, v15.h[4]
    ushr        v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
    add         v12.8h, v12.8h, v13.8h
.endm

.macro bilinear_over_8888_8888_process_pixblock_tail
    ushll       v2.4s, v10.4h, #BILINEAR_INTERPOLATION_BITS
    umlsl       v2.4s, v10.4h, v15.h[0]
    umlal2      v2.4s, v10.8h, v15.h[0]
    ushll       v3.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS
    umlsl       v3.4s, v11.4h, v15.h[4]
    umlal2      v3.4s, v11.8h, v15.h[4]
    shrn        v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
    shrn2       v0.8h, v1.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
    shrn        v2.4h, v2.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
    ushr        v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
    shrn2       v2.8h, v3.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
    xtn         v6.8b, v0.8h
    xtn         v7.8b, v2.8h
    ld1         {v2.2s, v3.2s}, [OUT]
    prfm        PREFETCH_MODE, [OUT, #(prefetch_offset * 4)]
    vuzp        v6.8b, v7.8b
    vuzp        v2.8b, v3.8b
    vuzp        v6.8b, v7.8b
    vuzp        v2.8b, v3.8b
    dup         v4.2s, v7.s[1]
    mvn         v4.8b, v4.8b
    umull       v11.8h, v2.8b, v4.8b
    umull       v2.8h,  v3.8b, v4.8b
    urshr       v1.8h, v11.8h, #8
    urshr       v10.8h, v2.8h, #8
    raddhn      v3.8b, v10.8h, v2.8h
    raddhn      v2.8b, v1.8h, v11.8h
    uqadd       v6.8b, v2.8b,  v6.8b
    uqadd       v7.8b, v3.8b,  v7.8b
    vuzp        v6.8b, v7.8b
    vuzp        v6.8b, v7.8b
    add         v12.8h, v12.8h, v13.8h
    st1         {v6.2s, v7.2s}, [OUT], #16
.endm

.macro bilinear_over_8888_8888_process_pixblock_tail_head
                                            ushll       v2.4s, v10.4h, #BILINEAR_INTERPOLATION_BITS
    asr         WTMP1, X, #16
    add         X, X, UX
    add         TMP1, TOP, TMP1, lsl #2
                                            umlsl       v2.4s, v10.4h, v15.h[0]
    asr         WTMP2, X, #16
    add         X, X, UX
    add         TMP2, TOP, TMP2, lsl #2
                                            umlal2      v2.4s, v10.8h, v15.h[0]
                                            ushll       v3.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS
    ld1         {v20.2s}, [TMP1], STRIDE
                                            umlsl       v3.4s, v11.4h, v15.h[4]
                                            umlal2      v3.4s, v11.8h, v15.h[4]
    ld1         {v21.2s}, [TMP1]
    umull       v8.8h, v20.8b, v28.8b
    umlal       v8.8h, v21.8b, v29.8b
                                            shrn        v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
                                            shrn2       v0.8h, v1.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
                                            shrn        v2.4h, v2.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
                                            ushr        v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
    ld1         {v22.2s}, [TMP2], STRIDE
                                            shrn2       v2.8h, v3.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
                                            xtn         v6.8b, v0.8h
    ld1         {v23.2s}, [TMP2]
    umull       v9.8h, v22.8b, v28.8b
    asr         WTMP3, X, #16
    add         X, X, UX
    add         TMP3, TOP, TMP3, lsl #2
    asr         WTMP4, X, #16
    add         X, X, UX
    add         TMP4, TOP, TMP4, lsl #2
    umlal       v9.8h, v23.8b, v29.8b
                                            xtn         v7.8b, v2.8h
                                            ld1         {v2.2s, v3.2s}, [OUT]
                                            prfm        PREFETCH_MODE, [OUT, PF_OFFS]
    ld1         {v22.2s}, [TMP3], STRIDE
                                            vuzp        v6.8b, v7.8b
                                            vuzp        v2.8b, v3.8b
                                            vuzp        v6.8b, v7.8b
                                            vuzp        v2.8b, v3.8b
                                            dup         v4.2s, v7.s[1]
    ld1         {v23.2s}, [TMP3]
                                            mvn         v4.8b, v4.8b
    umull       v10.8h, v22.8b, v28.8b
    umlal       v10.8h, v23.8b, v29.8b
                                            umull       v11.8h, v2.8b, v4.8b
                                            umull        v2.8h, v3.8b, v4.8b
    ushll       v0.4s, v8.4h, #BILINEAR_INTERPOLATION_BITS
    umlsl       v0.4s, v8.4h, v15.h[0]
                                            urshr       v1.8h, v11.8h, #8
    umlal2      v0.4s, v8.8h, v15.h[0]
                                            urshr       v8.8h, v2.8h, #8
                                            raddhn      v3.8b, v8.8h, v2.8h
                                            raddhn      v2.8b, v1.8h, v11.8h
    prfm        PREFETCH_MODE, [TMP4, PF_OFFS]
    ld1         {v16.2s}, [TMP4], STRIDE
                                            uqadd       v6.8b, v2.8b, v6.8b
                                            uqadd       v7.8b, v3.8b, v7.8b
    ld1         {v17.2s}, [TMP4]
    prfm        PREFETCH_MODE, [TMP4, PF_OFFS]
    umull       v11.8h, v16.8b, v28.8b
    umlal       v11.8h, v17.8b, v29.8b
                                            vuzp        v6.8b, v7.8b
    ushll       v1.4s, v9.4h, #BILINEAR_INTERPOLATION_BITS
                                            vuzp        v6.8b, v7.8b
    umlsl       v1.4s, v9.4h, v15.h[4]
                                            add         v12.8h, v12.8h, v13.8h
    umlal2      v1.4s, v9.8h, v15.h[4]
    ushr        v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
    add         v12.8h, v12.8h, v13.8h
                                            st1         {v6.2s, v7.2s}, [OUT], #16
.endm

/* over_8888_8_8888 */
.macro bilinear_over_8888_8_8888_process_last_pixel
    bilinear_interpolate_last_pixel 8888, 8, 8888, over
.endm

.macro bilinear_over_8888_8_8888_process_two_pixels
    bilinear_interpolate_two_pixels 8888, 8, 8888, over
.endm

.macro bilinear_over_8888_8_8888_process_four_pixels
    bilinear_interpolate_two_pixels 8888, 8, 8888, over
    bilinear_interpolate_two_pixels 8888, 8, 8888, over
.endm

.macro bilinear_over_8888_8_8888_process_pixblock_head
    bilinear_over_8888_8_8888_process_four_pixels
.endm

.macro bilinear_over_8888_8_8888_process_pixblock_tail
.endm

.macro bilinear_over_8888_8_8888_process_pixblock_tail_head
     bilinear_over_8888_8_8888_process_pixblock_tail
     bilinear_over_8888_8_8888_process_pixblock_head
.endm

/* add_8888_8888 */
.macro bilinear_add_8888_8888_process_last_pixel
    bilinear_interpolate_last_pixel 8888, x, 8888, add
.endm

.macro bilinear_add_8888_8888_process_two_pixels
    bilinear_interpolate_two_pixels 8888, x, 8888, add
.endm

.macro bilinear_add_8888_8888_process_four_pixels
    bilinear_interpolate_two_pixels 8888, x, 8888, add
    bilinear_interpolate_two_pixels 8888, x, 8888, add
.endm

.macro bilinear_add_8888_8888_process_pixblock_head
    bilinear_add_8888_8888_process_four_pixels
.endm

.macro bilinear_add_8888_8888_process_pixblock_tail
.endm

.macro bilinear_add_8888_8888_process_pixblock_tail_head
    bilinear_add_8888_8888_process_pixblock_tail
    bilinear_add_8888_8888_process_pixblock_head
.endm

/* add_8888_8_8888 */
.macro bilinear_add_8888_8_8888_process_last_pixel
    bilinear_interpolate_last_pixel 8888, 8, 8888, add
.endm

.macro bilinear_add_8888_8_8888_process_two_pixels
    bilinear_interpolate_two_pixels 8888, 8, 8888, add
.endm

.macro bilinear_add_8888_8_8888_process_four_pixels
    bilinear_interpolate_four_pixels 8888, 8, 8888, add
.endm

.macro bilinear_add_8888_8_8888_process_pixblock_head
    bilinear_add_8888_8_8888_process_four_pixels
.endm

.macro bilinear_add_8888_8_8888_process_pixblock_tail
.endm

.macro bilinear_add_8888_8_8888_process_pixblock_tail_head
    bilinear_add_8888_8_8888_process_pixblock_tail
    bilinear_add_8888_8_8888_process_pixblock_head
.endm


/* Bilinear scanline functions */
generate_bilinear_scanline_func \
    pixman_scaled_bilinear_scanline_8888_8_8888_SRC_asm_neon, \
    8888, 8888, 2, 2, \
    bilinear_src_8888_8_8888_process_last_pixel, \
    bilinear_src_8888_8_8888_process_two_pixels, \
    bilinear_src_8888_8_8888_process_four_pixels, \
    bilinear_src_8888_8_8888_process_pixblock_head, \
    bilinear_src_8888_8_8888_process_pixblock_tail, \
    bilinear_src_8888_8_8888_process_pixblock_tail_head, \
    4, 28, BILINEAR_FLAG_USE_MASK

generate_bilinear_scanline_func \
    pixman_scaled_bilinear_scanline_8888_8_0565_SRC_asm_neon, \
    8888, 0565, 2, 1, \
    bilinear_src_8888_8_0565_process_last_pixel, \
    bilinear_src_8888_8_0565_process_two_pixels, \
    bilinear_src_8888_8_0565_process_four_pixels, \
    bilinear_src_8888_8_0565_process_pixblock_head, \
    bilinear_src_8888_8_0565_process_pixblock_tail, \
    bilinear_src_8888_8_0565_process_pixblock_tail_head, \
    4, 28, BILINEAR_FLAG_USE_MASK

generate_bilinear_scanline_func \
    pixman_scaled_bilinear_scanline_0565_8_x888_SRC_asm_neon, \
    0565, 8888, 1, 2, \
    bilinear_src_0565_8_x888_process_last_pixel, \
    bilinear_src_0565_8_x888_process_two_pixels, \
    bilinear_src_0565_8_x888_process_four_pixels, \
    bilinear_src_0565_8_x888_process_pixblock_head, \
    bilinear_src_0565_8_x888_process_pixblock_tail, \
    bilinear_src_0565_8_x888_process_pixblock_tail_head, \
    4, 28, BILINEAR_FLAG_USE_MASK

generate_bilinear_scanline_func \
    pixman_scaled_bilinear_scanline_0565_8_0565_SRC_asm_neon, \
    0565, 0565, 1, 1, \
    bilinear_src_0565_8_0565_process_last_pixel, \
    bilinear_src_0565_8_0565_process_two_pixels, \
    bilinear_src_0565_8_0565_process_four_pixels, \
    bilinear_src_0565_8_0565_process_pixblock_head, \
    bilinear_src_0565_8_0565_process_pixblock_tail, \
    bilinear_src_0565_8_0565_process_pixblock_tail_head, \
    4, 28, BILINEAR_FLAG_USE_MASK

generate_bilinear_scanline_func \
    pixman_scaled_bilinear_scanline_8888_8888_OVER_asm_neon, \
    8888, 8888, 2, 2, \
    bilinear_over_8888_8888_process_last_pixel, \
    bilinear_over_8888_8888_process_two_pixels, \
    bilinear_over_8888_8888_process_four_pixels, \
    bilinear_over_8888_8888_process_pixblock_head, \
    bilinear_over_8888_8888_process_pixblock_tail, \
    bilinear_over_8888_8888_process_pixblock_tail_head, \
    4, 28, 0

generate_bilinear_scanline_func \
    pixman_scaled_bilinear_scanline_8888_8_8888_OVER_asm_neon, \
    8888, 8888, 2, 2, \
    bilinear_over_8888_8_8888_process_last_pixel, \
    bilinear_over_8888_8_8888_process_two_pixels, \
    bilinear_over_8888_8_8888_process_four_pixels, \
    bilinear_over_8888_8_8888_process_pixblock_head, \
    bilinear_over_8888_8_8888_process_pixblock_tail, \
    bilinear_over_8888_8_8888_process_pixblock_tail_head, \
    4, 28, BILINEAR_FLAG_USE_MASK

generate_bilinear_scanline_func \
    pixman_scaled_bilinear_scanline_8888_8888_ADD_asm_neon, \
    8888, 8888, 2, 2, \
    bilinear_add_8888_8888_process_last_pixel, \
    bilinear_add_8888_8888_process_two_pixels, \
    bilinear_add_8888_8888_process_four_pixels, \
    bilinear_add_8888_8888_process_pixblock_head, \
    bilinear_add_8888_8888_process_pixblock_tail, \
    bilinear_add_8888_8888_process_pixblock_tail_head, \
    4, 28, 0

generate_bilinear_scanline_func \
    pixman_scaled_bilinear_scanline_8888_8_8888_ADD_asm_neon, \
    8888, 8888, 2, 2, \
    bilinear_add_8888_8_8888_process_last_pixel, \
    bilinear_add_8888_8_8888_process_two_pixels, \
    bilinear_add_8888_8_8888_process_four_pixels, \
    bilinear_add_8888_8_8888_process_pixblock_head, \
    bilinear_add_8888_8_8888_process_pixblock_tail, \
    bilinear_add_8888_8_8888_process_pixblock_tail_head, \
    4, 28, BILINEAR_FLAG_USE_MASK
