mirror of
https://github.com/ghostty-org/ghostty.git
synced 2025-07-30 13:27:51 +03:00
3705 lines
136 KiB
ArmAsm
Vendored
3705 lines
136 KiB
ArmAsm
Vendored
/*
|
|
* Copyright © 2009 Nokia Corporation
|
|
*
|
|
* Permission is hereby granted, free of charge, to any person obtaining a
|
|
* copy of this software and associated documentation files (the "Software"),
|
|
* to deal in the Software without restriction, including without limitation
|
|
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
|
* and/or sell copies of the Software, and to permit persons to whom the
|
|
* Software is furnished to do so, subject to the following conditions:
|
|
*
|
|
* The above copyright notice and this permission notice (including the next
|
|
* paragraph) shall be included in all copies or substantial portions of the
|
|
* Software.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
|
|
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
|
|
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
|
|
* DEALINGS IN THE SOFTWARE.
|
|
*
|
|
* Author: Siarhei Siamashka (siarhei.siamashka@nokia.com)
|
|
*/
|
|
|
|
/*
|
|
* This file contains implementations of NEON optimized pixel processing
|
|
* functions. There is no full and detailed tutorial, but some functions
|
|
* (those which are exposing some new or interesting features) are
|
|
* extensively commented and can be used as examples.
|
|
*
|
|
* You may want to have a look at the comments for following functions:
|
|
* - pixman_composite_over_8888_0565_asm_neon
|
|
* - pixman_composite_over_n_8_0565_asm_neon
|
|
*/
|
|
|
|
/* Prevent the stack from becoming executable for no reason... */
|
|
#if defined(__linux__) && defined(__ELF__)
|
|
.section .note.GNU-stack,"",%progbits
|
|
#endif
|
|
|
|
.text
|
|
.arch armv8-a
|
|
|
|
.altmacro
|
|
.p2align 2
|
|
|
|
#include "pixman-private.h"
|
|
#include "pixman-arm-asm.h"
|
|
#include "pixman-arma64-neon-asm.h"
|
|
|
|
/* Global configuration options and preferences */
|
|
|
|
/*
|
|
* The code can optionally make use of unaligned memory accesses to improve
|
|
* performance of handling leading/trailing pixels for each scanline.
|
|
* Configuration variable RESPECT_STRICT_ALIGNMENT can be set to 0 for
|
|
* example in linux if unaligned memory accesses are not configured to
|
|
* generate.exceptions.
|
|
*/
|
|
.set RESPECT_STRICT_ALIGNMENT, 1
|
|
|
|
/*
|
|
* Set default prefetch type. There is a choice between the following options:
|
|
*
|
|
* PREFETCH_TYPE_NONE (may be useful for the ARM cores where PLD is set to work
|
|
* as NOP to workaround some HW bugs or for whatever other reason)
|
|
*
|
|
* PREFETCH_TYPE_SIMPLE (may be useful for simple single-issue ARM cores where
|
|
* advanced prefetch intruduces heavy overhead)
|
|
*
|
|
* PREFETCH_TYPE_ADVANCED (useful for superscalar cores such as ARM Cortex-A8
|
|
* which can run ARM and NEON instructions simultaneously so that extra ARM
|
|
* instructions do not add (many) extra cycles, but improve prefetch efficiency)
|
|
*
|
|
* Note: some types of function can't support advanced prefetch and fallback
|
|
* to simple one (those which handle 24bpp pixels)
|
|
*/
|
|
.set PREFETCH_TYPE_DEFAULT, PREFETCH_TYPE_ADVANCED
|
|
|
|
/* Prefetch distance in pixels for simple prefetch */
|
|
.set PREFETCH_DISTANCE_SIMPLE, 64
|
|
|
|
/*
|
|
* Implementation of pixman_composite_over_8888_0565_asm_neon
|
|
*
|
|
* This function takes a8r8g8b8 source buffer, r5g6b5 destination buffer and
|
|
* performs OVER compositing operation. Function fast_composite_over_8888_0565
|
|
* from pixman-fast-path.c does the same in C and can be used as a reference.
|
|
*
|
|
* First we need to have some NEON assembly code which can do the actual
|
|
* operation on the pixels and provide it to the template macro.
|
|
*
|
|
* Template macro quite conveniently takes care of emitting all the necessary
|
|
* code for memory reading and writing (including quite tricky cases of
|
|
* handling unaligned leading/trailing pixels), so we only need to deal with
|
|
* the data in NEON registers.
|
|
*
|
|
* NEON registers allocation in general is recommented to be the following:
|
|
* v0, v1, v2, v3 - contain loaded source pixel data
|
|
* v4, v5, v6, v7 - contain loaded destination pixels (if they are needed)
|
|
* v24, v25, v26, v27 - contain loading mask pixel data (if mask is used)
|
|
* v28, v29, v30, v31 - place for storing the result (destination pixels)
|
|
*
|
|
* As can be seen above, four 64-bit NEON registers are used for keeping
|
|
* intermediate pixel data and up to 8 pixels can be processed in one step
|
|
* for 32bpp formats (16 pixels for 16bpp, 32 pixels for 8bpp).
|
|
*
|
|
* This particular function uses the following registers allocation:
|
|
* v0, v1, v2, v3 - contain loaded source pixel data
|
|
* v4, v5 - contain loaded destination pixels (they are needed)
|
|
* v28, v29 - place for storing the result (destination pixels)
|
|
*/
|
|
|
|
/*
|
|
* Step one. We need to have some code to do some arithmetics on pixel data.
|
|
* This is implemented as a pair of macros: '*_head' and '*_tail'. When used
|
|
* back-to-back, they take pixel data from {v0, v1, v2, v3} and {v4, v5},
|
|
* perform all the needed calculations and write the result to {v28, v29}.
|
|
* The rationale for having two macros and not just one will be explained
|
|
* later. In practice, any single monolitic function which does the work can
|
|
* be split into two parts in any arbitrary way without affecting correctness.
|
|
*
|
|
* There is one special trick here too. Common template macro can optionally
|
|
* make our life a bit easier by doing R, G, B, A color components
|
|
* deinterleaving for 32bpp pixel formats (and this feature is used in
|
|
* 'pixman_composite_over_8888_0565_asm_neon' function). So it means that
|
|
* instead of having 8 packed pixels in {v0, v1, v2, v3} registers, we
|
|
* actually use v0 register for blue channel (a vector of eight 8-bit
|
|
* values), v1 register for green, v2 for red and v3 for alpha. This
|
|
* simple conversion can be also done with a few NEON instructions:
|
|
*
|
|
* Packed to planar conversion: // vuzp8 is a wrapper macro
|
|
* vuzp8 v0, v1
|
|
* vuzp8 v2, v3
|
|
* vuzp8 v1, v3
|
|
* vuzp8 v0, v2
|
|
*
|
|
* Planar to packed conversion: // vzip8 is a wrapper macro
|
|
* vzip8 v0, v2
|
|
* vzip8 v1, v3
|
|
* vzip8 v2, v3
|
|
* vzip8 v0, v1
|
|
*
|
|
* But pixel can be loaded directly in planar format using LD4 / b NEON
|
|
* instruction. It is 1 cycle slower than LD1 / s, so this is not always
|
|
* desirable, that's why deinterleaving is optional.
|
|
*
|
|
* But anyway, here is the code:
|
|
*/
|
|
|
|
.macro pixman_composite_over_8888_0565_process_pixblock_head
|
|
/* convert 8 r5g6b5 pixel data from {v4} to planar 8-bit format
|
|
and put data into v6 - red, v7 - green, v30 - blue */
|
|
mov v4.d[1], v5.d[0]
|
|
shrn v6.8b, v4.8h, #8
|
|
shrn v7.8b, v4.8h, #3
|
|
sli v4.8h, v4.8h, #5
|
|
sri v6.8b, v6.8b, #5
|
|
mvn v3.8b, v3.8b /* invert source alpha */
|
|
sri v7.8b, v7.8b, #6
|
|
shrn v30.8b, v4.8h, #2
|
|
/* now do alpha blending, storing results in 8-bit planar format
|
|
into v20 - red, v23 - green, v22 - blue */
|
|
umull v10.8h, v3.8b, v6.8b
|
|
umull v11.8h, v3.8b, v7.8b
|
|
umull v12.8h, v3.8b, v30.8b
|
|
urshr v17.8h, v10.8h, #8
|
|
urshr v18.8h, v11.8h, #8
|
|
urshr v19.8h, v12.8h, #8
|
|
raddhn v20.8b, v10.8h, v17.8h
|
|
raddhn v23.8b, v11.8h, v18.8h
|
|
raddhn v22.8b, v12.8h, v19.8h
|
|
.endm
|
|
|
|
.macro pixman_composite_over_8888_0565_process_pixblock_tail
|
|
/* ... continue alpha blending */
|
|
uqadd v17.8b, v2.8b, v20.8b
|
|
uqadd v18.8b, v0.8b, v22.8b
|
|
uqadd v19.8b, v1.8b, v23.8b
|
|
/* convert the result to r5g6b5 and store it into {v14} */
|
|
ushll v14.8h, v17.8b, #7
|
|
sli v14.8h, v14.8h, #1
|
|
ushll v8.8h, v19.8b, #7
|
|
sli v8.8h, v8.8h, #1
|
|
ushll v9.8h, v18.8b, #7
|
|
sli v9.8h, v9.8h, #1
|
|
sri v14.8h, v8.8h, #5
|
|
sri v14.8h, v9.8h, #11
|
|
mov v28.d[0], v14.d[0]
|
|
mov v29.d[0], v14.d[1]
|
|
.endm
|
|
|
|
/*
|
|
* OK, now we got almost everything that we need. Using the above two
|
|
* macros, the work can be done right. But now we want to optimize
|
|
* it a bit. ARM Cortex-A8 is an in-order core, and benefits really
|
|
* a lot from good code scheduling and software pipelining.
|
|
*
|
|
* Let's construct some code, which will run in the core main loop.
|
|
* Some pseudo-code of the main loop will look like this:
|
|
* head
|
|
* while (...) {
|
|
* tail
|
|
* head
|
|
* }
|
|
* tail
|
|
*
|
|
* It may look a bit weird, but this setup allows to hide instruction
|
|
* latencies better and also utilize dual-issue capability more
|
|
* efficiently (make pairs of load-store and ALU instructions).
|
|
*
|
|
* So what we need now is a '*_tail_head' macro, which will be used
|
|
* in the core main loop. A trivial straightforward implementation
|
|
* of this macro would look like this:
|
|
*
|
|
* pixman_composite_over_8888_0565_process_pixblock_tail
|
|
* st1 {v28.4h, v29.4h}, [DST_W], #32
|
|
* ld1 {v4.4h, v5.4h}, [DST_R], #16
|
|
* ld4 {v0.2s, v1.2s, v2.2s, v3.2s}, [SRC], #32
|
|
* pixman_composite_over_8888_0565_process_pixblock_head
|
|
* cache_preload 8, 8
|
|
*
|
|
* Now it also got some VLD/VST instructions. We simply can't move from
|
|
* processing one block of pixels to the other one with just arithmetics.
|
|
* The previously processed data needs to be written to memory and new
|
|
* data needs to be fetched. Fortunately, this main loop does not deal
|
|
* with partial leading/trailing pixels and can load/store a full block
|
|
* of pixels in a bulk. Additionally, destination buffer is already
|
|
* 16 bytes aligned here (which is good for performance).
|
|
*
|
|
* New things here are DST_R, DST_W, SRC and MASK identifiers. These
|
|
* are the aliases for ARM registers which are used as pointers for
|
|
* accessing data. We maintain separate pointers for reading and writing
|
|
* destination buffer (DST_R and DST_W).
|
|
*
|
|
* Another new thing is 'cache_preload' macro. It is used for prefetching
|
|
* data into CPU L2 cache and improve performance when dealing with large
|
|
* images which are far larger than cache size. It uses one argument
|
|
* (actually two, but they need to be the same here) - number of pixels
|
|
* in a block. Looking into 'pixman-arm-neon-asm.h' can provide some
|
|
* details about this macro. Moreover, if good performance is needed
|
|
* the code from this macro needs to be copied into '*_tail_head' macro
|
|
* and mixed with the rest of code for optimal instructions scheduling.
|
|
* We are actually doing it below.
|
|
*
|
|
* Now after all the explanations, here is the optimized code.
|
|
* Different instruction streams (originaling from '*_head', '*_tail'
|
|
* and 'cache_preload' macro) use different indentation levels for
|
|
* better readability. Actually taking the code from one of these
|
|
* indentation levels and ignoring a few LD/ST instructions would
|
|
* result in exactly the code from '*_head', '*_tail' or 'cache_preload'
|
|
* macro!
|
|
*/
|
|
|
|
#if 1
|
|
|
|
.macro pixman_composite_over_8888_0565_process_pixblock_tail_head
|
|
uqadd v17.8b, v2.8b, v20.8b
|
|
ld1 {v4.4h, v5.4h}, [DST_R], #16
|
|
mov v4.d[1], v5.d[0]
|
|
uqadd v18.8b, v0.8b, v22.8b
|
|
uqadd v19.8b, v1.8b, v23.8b
|
|
shrn v6.8b, v4.8h, #8
|
|
fetch_src_pixblock
|
|
shrn v7.8b, v4.8h, #3
|
|
sli v4.8h, v4.8h, #5
|
|
ushll v14.8h, v17.8b, #7
|
|
sli v14.8h, v14.8h, #1
|
|
PF add PF_X, PF_X, #8
|
|
ushll v8.8h, v19.8b, #7
|
|
sli v8.8h, v8.8h, #1
|
|
PF tst PF_CTL, #0xF
|
|
sri v6.8b, v6.8b, #5
|
|
PF beq 10f
|
|
PF add PF_X, PF_X, #8
|
|
10:
|
|
mvn v3.8b, v3.8b
|
|
PF beq 10f
|
|
PF sub PF_CTL, PF_CTL, #1
|
|
10:
|
|
sri v7.8b, v7.8b, #6
|
|
shrn v30.8b, v4.8h, #2
|
|
umull v10.8h, v3.8b, v6.8b
|
|
PF lsl DUMMY, PF_X, #src_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_SRC, DUMMY]
|
|
umull v11.8h, v3.8b, v7.8b
|
|
umull v12.8h, v3.8b, v30.8b
|
|
PF lsl DUMMY, PF_X, #dst_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_DST, DUMMY]
|
|
sri v14.8h, v8.8h, #5
|
|
PF cmp PF_X, ORIG_W
|
|
ushll v9.8h, v18.8b, #7
|
|
sli v9.8h, v9.8h, #1
|
|
urshr v17.8h, v10.8h, #8
|
|
PF ble 10f
|
|
PF sub PF_X, PF_X, ORIG_W
|
|
10:
|
|
urshr v19.8h, v11.8h, #8
|
|
urshr v18.8h, v12.8h, #8
|
|
PF ble 10f
|
|
PF subs PF_CTL, PF_CTL, #0x10
|
|
10:
|
|
sri v14.8h, v9.8h, #11
|
|
mov v28.d[0], v14.d[0]
|
|
mov v29.d[0], v14.d[1]
|
|
PF ble 10f
|
|
PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift
|
|
PF ldrsb DUMMY, [PF_SRC, DUMMY]
|
|
PF add PF_SRC, PF_SRC, #1
|
|
10:
|
|
raddhn v20.8b, v10.8h, v17.8h
|
|
raddhn v23.8b, v11.8h, v19.8h
|
|
PF ble 10f
|
|
PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift
|
|
PF ldrsb DUMMY, [PF_DST, DUMMY]
|
|
PF add PF_DST, PF_SRC, #1
|
|
10:
|
|
raddhn v22.8b, v12.8h, v18.8h
|
|
st1 {v14.8h}, [DST_W], #16
|
|
.endm
|
|
|
|
#else
|
|
|
|
/* If we did not care much about the performance, we would just use this... */
|
|
.macro pixman_composite_over_8888_0565_process_pixblock_tail_head
|
|
pixman_composite_over_8888_0565_process_pixblock_tail
|
|
st1 {v14.8h}, [DST_W], #16
|
|
ld1 {v4.4h, v4.5h}, [DST_R], #16
|
|
fetch_src_pixblock
|
|
pixman_composite_over_8888_0565_process_pixblock_head
|
|
cache_preload 8, 8
|
|
.endm
|
|
|
|
#endif
|
|
|
|
/*
|
|
* And now the final part. We are using 'generate_composite_function' macro
|
|
* to put all the stuff together. We are specifying the name of the function
|
|
* which we want to get, number of bits per pixel for the source, mask and
|
|
* destination (0 if unused, like mask in this case). Next come some bit
|
|
* flags:
|
|
* FLAG_DST_READWRITE - tells that the destination buffer is both read
|
|
* and written, for write-only buffer we would use
|
|
* FLAG_DST_WRITEONLY flag instead
|
|
* FLAG_DEINTERLEAVE_32BPP - tells that we prefer to work with planar data
|
|
* and separate color channels for 32bpp format.
|
|
* The next things are:
|
|
* - the number of pixels processed per iteration (8 in this case, because
|
|
* that's the maximum what can fit into four 64-bit NEON registers).
|
|
* - prefetch distance, measured in pixel blocks. In this case it is 5 times
|
|
* by 8 pixels. That would be 40 pixels, or up to 160 bytes. Optimal
|
|
* prefetch distance can be selected by running some benchmarks.
|
|
*
|
|
* After that we specify some macros, these are 'default_init',
|
|
* 'default_cleanup' here which are empty (but it is possible to have custom
|
|
* init/cleanup macros to be able to save/restore some extra NEON registers
|
|
* like d8-d15 or do anything else) followed by
|
|
* 'pixman_composite_over_8888_0565_process_pixblock_head',
|
|
* 'pixman_composite_over_8888_0565_process_pixblock_tail' and
|
|
* 'pixman_composite_over_8888_0565_process_pixblock_tail_head'
|
|
* which we got implemented above.
|
|
*
|
|
* The last part is the NEON registers allocation scheme.
|
|
*/
|
|
generate_composite_function \
|
|
pixman_composite_over_8888_0565_asm_neon, 32, 0, 16, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_over_8888_0565_process_pixblock_head, \
|
|
pixman_composite_over_8888_0565_process_pixblock_tail, \
|
|
pixman_composite_over_8888_0565_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
24 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_over_n_0565_process_pixblock_head
|
|
/* convert 8 r5g6b5 pixel data from {v4} to planar 8-bit format
|
|
and put data into v6 - red, v7 - green, v30 - blue */
|
|
mov v4.d[1], v5.d[0]
|
|
shrn v6.8b, v4.8h, #8
|
|
shrn v7.8b, v4.8h, #3
|
|
sli v4.8h, v4.8h, #5
|
|
sri v6.8b, v6.8b, #5
|
|
sri v7.8b, v7.8b, #6
|
|
shrn v30.8b, v4.8h, #2
|
|
/* now do alpha blending, storing results in 8-bit planar format
|
|
into v20 - red, v23 - green, v22 - blue */
|
|
umull v10.8h, v3.8b, v6.8b
|
|
umull v11.8h, v3.8b, v7.8b
|
|
umull v12.8h, v3.8b, v30.8b
|
|
urshr v13.8h, v10.8h, #8
|
|
urshr v14.8h, v11.8h, #8
|
|
urshr v15.8h, v12.8h, #8
|
|
raddhn v20.8b, v10.8h, v13.8h
|
|
raddhn v23.8b, v11.8h, v14.8h
|
|
raddhn v22.8b, v12.8h, v15.8h
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_0565_process_pixblock_tail
|
|
/* ... continue alpha blending */
|
|
uqadd v17.8b, v2.8b, v20.8b
|
|
uqadd v18.8b, v0.8b, v22.8b
|
|
uqadd v19.8b, v1.8b, v23.8b
|
|
/* convert the result to r5g6b5 and store it into {v14} */
|
|
ushll v14.8h, v17.8b, #7
|
|
sli v14.8h, v14.8h, #1
|
|
ushll v8.8h, v19.8b, #7
|
|
sli v8.8h, v8.8h, #1
|
|
ushll v9.8h, v18.8b, #7
|
|
sli v9.8h, v9.8h, #1
|
|
sri v14.8h, v8.8h, #5
|
|
sri v14.8h, v9.8h, #11
|
|
mov v28.d[0], v14.d[0]
|
|
mov v29.d[0], v14.d[1]
|
|
.endm
|
|
|
|
/* TODO: expand macros and do better instructions scheduling */
|
|
.macro pixman_composite_over_n_0565_process_pixblock_tail_head
|
|
pixman_composite_over_n_0565_process_pixblock_tail
|
|
ld1 {v4.4h, v5.4h}, [DST_R], #16
|
|
st1 {v14.8h}, [DST_W], #16
|
|
pixman_composite_over_n_0565_process_pixblock_head
|
|
cache_preload 8, 8
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_0565_init
|
|
mov v3.s[0], w4
|
|
dup v0.8b, v3.b[0]
|
|
dup v1.8b, v3.b[1]
|
|
dup v2.8b, v3.b[2]
|
|
dup v3.8b, v3.b[3]
|
|
mvn v3.8b, v3.8b /* invert source alpha */
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_over_n_0565_asm_neon, 0, 0, 16, \
|
|
FLAG_DST_READWRITE, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_over_n_0565_init, \
|
|
default_cleanup, \
|
|
pixman_composite_over_n_0565_process_pixblock_head, \
|
|
pixman_composite_over_n_0565_process_pixblock_tail, \
|
|
pixman_composite_over_n_0565_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
24 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_src_8888_0565_process_pixblock_head
|
|
ushll v8.8h, v1.8b, #7
|
|
sli v8.8h, v8.8h, #1
|
|
ushll v14.8h, v2.8b, #7
|
|
sli v14.8h, v14.8h, #1
|
|
ushll v9.8h, v0.8b, #7
|
|
sli v9.8h, v9.8h, #1
|
|
.endm
|
|
|
|
.macro pixman_composite_src_8888_0565_process_pixblock_tail
|
|
sri v14.8h, v8.8h, #5
|
|
sri v14.8h, v9.8h, #11
|
|
mov v28.d[0], v14.d[0]
|
|
mov v29.d[0], v14.d[1]
|
|
.endm
|
|
|
|
.macro pixman_composite_src_8888_0565_process_pixblock_tail_head
|
|
sri v14.8h, v8.8h, #5
|
|
PF add PF_X, PF_X, #8
|
|
PF tst PF_CTL, #0xF
|
|
fetch_src_pixblock
|
|
PF beq 10f
|
|
PF add PF_X, PF_X, #8
|
|
PF sub PF_CTL, PF_CTL, #1
|
|
10:
|
|
sri v14.8h, v9.8h, #11
|
|
mov v28.d[0], v14.d[0]
|
|
mov v29.d[0], v14.d[1]
|
|
PF cmp PF_X, ORIG_W
|
|
PF lsl DUMMY, PF_X, #src_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_SRC, DUMMY]
|
|
ushll v8.8h, v1.8b, #7
|
|
sli v8.8h, v8.8h, #1
|
|
st1 {v14.8h}, [DST_W], #16
|
|
PF ble 10f
|
|
PF sub PF_X, PF_X, ORIG_W
|
|
PF subs PF_CTL, PF_CTL, #0x10
|
|
10:
|
|
ushll v14.8h, v2.8b, #7
|
|
sli v14.8h, v14.8h, #1
|
|
PF ble 10f
|
|
PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift
|
|
PF ldrsb DUMMY, [PF_SRC, DUMMY]
|
|
PF add PF_SRC, PF_SRC, #1
|
|
10:
|
|
ushll v9.8h, v0.8b, #7
|
|
sli v9.8h, v9.8h, #1
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_src_8888_0565_asm_neon, 32, 0, 16, \
|
|
FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
10, /* prefetch distance */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_src_8888_0565_process_pixblock_head, \
|
|
pixman_composite_src_8888_0565_process_pixblock_tail, \
|
|
pixman_composite_src_8888_0565_process_pixblock_tail_head
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_src_0565_8888_process_pixblock_head
|
|
mov v0.d[1], v1.d[0]
|
|
shrn v30.8b, v0.8h, #8
|
|
shrn v29.8b, v0.8h, #3
|
|
sli v0.8h, v0.8h, #5
|
|
movi v31.8b, #255
|
|
sri v30.8b, v30.8b, #5
|
|
sri v29.8b, v29.8b, #6
|
|
shrn v28.8b, v0.8h, #2
|
|
.endm
|
|
|
|
.macro pixman_composite_src_0565_8888_process_pixblock_tail
|
|
.endm
|
|
|
|
/* TODO: expand macros and do better instructions scheduling */
|
|
.macro pixman_composite_src_0565_8888_process_pixblock_tail_head
|
|
pixman_composite_src_0565_8888_process_pixblock_tail
|
|
st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
fetch_src_pixblock
|
|
pixman_composite_src_0565_8888_process_pixblock_head
|
|
cache_preload 8, 8
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_src_0565_8888_asm_neon, 16, 0, 32, \
|
|
FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
10, /* prefetch distance */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_src_0565_8888_process_pixblock_head, \
|
|
pixman_composite_src_0565_8888_process_pixblock_tail, \
|
|
pixman_composite_src_0565_8888_process_pixblock_tail_head
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_add_8_8_process_pixblock_head
|
|
uqadd v28.8b, v0.8b, v4.8b
|
|
uqadd v29.8b, v1.8b, v5.8b
|
|
uqadd v30.8b, v2.8b, v6.8b
|
|
uqadd v31.8b, v3.8b, v7.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_add_8_8_process_pixblock_tail
|
|
.endm
|
|
|
|
.macro pixman_composite_add_8_8_process_pixblock_tail_head
|
|
fetch_src_pixblock
|
|
PF add PF_X, PF_X, #32
|
|
PF tst PF_CTL, #0xF
|
|
ld1 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
|
|
PF beq 10f
|
|
PF add PF_X, PF_X, #32
|
|
PF sub PF_CTL, PF_CTL, #1
|
|
10:
|
|
st1 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
PF cmp PF_X, ORIG_W
|
|
PF lsl DUMMY, PF_X, #src_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_SRC, DUMMY]
|
|
PF lsl DUMMY, PF_X, #dst_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_DST, DUMMY]
|
|
PF ble 10f
|
|
PF sub PF_X, PF_X, ORIG_W
|
|
PF subs PF_CTL, PF_CTL, #0x10
|
|
10:
|
|
uqadd v28.8b, v0.8b, v4.8b
|
|
PF ble 10f
|
|
PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift
|
|
PF ldrsb DUMMY, [PF_SRC, DUMMY]
|
|
PF add PF_SRC, PF_SRC, #1
|
|
PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift
|
|
PF ldrsb DUMMY, [PF_DST, DUMMY]
|
|
PF add PF_DST, PF_DST, #1
|
|
10:
|
|
uqadd v29.8b, v1.8b, v5.8b
|
|
uqadd v30.8b, v2.8b, v6.8b
|
|
uqadd v31.8b, v3.8b, v7.8b
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_add_8_8_asm_neon, 8, 0, 8, \
|
|
FLAG_DST_READWRITE, \
|
|
32, /* number of pixels, processed in a single block */ \
|
|
10, /* prefetch distance */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_add_8_8_process_pixblock_head, \
|
|
pixman_composite_add_8_8_process_pixblock_tail, \
|
|
pixman_composite_add_8_8_process_pixblock_tail_head
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_add_8888_8888_process_pixblock_tail_head
|
|
fetch_src_pixblock
|
|
PF add PF_X, PF_X, #8
|
|
PF tst PF_CTL, #0xF
|
|
ld1 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
|
|
PF beq 10f
|
|
PF add PF_X, PF_X, #8
|
|
PF sub PF_CTL, PF_CTL, #1
|
|
10:
|
|
st1 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
PF cmp PF_X, ORIG_W
|
|
PF lsl DUMMY, PF_X, #src_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_SRC, DUMMY]
|
|
PF lsl DUMMY, PF_X, #dst_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_DST, DUMMY]
|
|
PF ble 10f
|
|
PF sub PF_X, PF_X, ORIG_W
|
|
PF subs PF_CTL, PF_CTL, #0x10
|
|
10:
|
|
uqadd v28.8b, v0.8b, v4.8b
|
|
PF ble 10f
|
|
PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift
|
|
PF ldrsb DUMMY, [PF_SRC, DUMMY]
|
|
PF add PF_SRC, PF_SRC, #1
|
|
PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift
|
|
PF ldrsb DUMMY, [PF_DST, DUMMY]
|
|
PF add PF_DST, PF_DST, #1
|
|
10:
|
|
uqadd v29.8b, v1.8b, v5.8b
|
|
uqadd v30.8b, v2.8b, v6.8b
|
|
uqadd v31.8b, v3.8b, v7.8b
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_add_8888_8888_asm_neon, 32, 0, 32, \
|
|
FLAG_DST_READWRITE, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
10, /* prefetch distance */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_add_8_8_process_pixblock_head, \
|
|
pixman_composite_add_8_8_process_pixblock_tail, \
|
|
pixman_composite_add_8888_8888_process_pixblock_tail_head
|
|
|
|
generate_composite_function_single_scanline \
|
|
pixman_composite_scanline_add_asm_neon, 32, 0, 32, \
|
|
FLAG_DST_READWRITE, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_add_8_8_process_pixblock_head, \
|
|
pixman_composite_add_8_8_process_pixblock_tail, \
|
|
pixman_composite_add_8888_8888_process_pixblock_tail_head
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_out_reverse_8888_8888_process_pixblock_head
|
|
mvn v24.8b, v3.8b /* get inverted alpha */
|
|
/* do alpha blending */
|
|
umull v8.8h, v24.8b, v4.8b
|
|
umull v9.8h, v24.8b, v5.8b
|
|
umull v10.8h, v24.8b, v6.8b
|
|
umull v11.8h, v24.8b, v7.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_out_reverse_8888_8888_process_pixblock_tail
|
|
urshr v14.8h, v8.8h, #8
|
|
urshr v15.8h, v9.8h, #8
|
|
urshr v16.8h, v10.8h, #8
|
|
urshr v17.8h, v11.8h, #8
|
|
raddhn v28.8b, v14.8h, v8.8h
|
|
raddhn v29.8b, v15.8h, v9.8h
|
|
raddhn v30.8b, v16.8h, v10.8h
|
|
raddhn v31.8b, v17.8h, v11.8h
|
|
.endm
|
|
|
|
.macro pixman_composite_out_reverse_8888_8888_process_pixblock_tail_head
|
|
ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
|
|
urshr v14.8h, v8.8h, #8
|
|
PF add PF_X, PF_X, #8
|
|
PF tst PF_CTL, #0xF
|
|
urshr v15.8h, v9.8h, #8
|
|
urshr v16.8h, v10.8h, #8
|
|
urshr v17.8h, v11.8h, #8
|
|
PF beq 10f
|
|
PF add PF_X, PF_X, #8
|
|
PF sub PF_CTL, PF_CTL, #1
|
|
10:
|
|
raddhn v28.8b, v14.8h, v8.8h
|
|
raddhn v29.8b, v15.8h, v9.8h
|
|
PF cmp PF_X, ORIG_W
|
|
raddhn v30.8b, v16.8h, v10.8h
|
|
raddhn v31.8b, v17.8h, v11.8h
|
|
fetch_src_pixblock
|
|
PF lsl DUMMY, PF_X, #src_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_SRC, DUMMY]
|
|
mvn v22.8b, v3.8b
|
|
PF lsl DUMMY, PF_X, #dst_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_DST, DUMMY]
|
|
st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
PF ble 10f
|
|
PF sub PF_X, PF_X, ORIG_W
|
|
10:
|
|
umull v8.8h, v22.8b, v4.8b
|
|
PF ble 10f
|
|
PF subs PF_CTL, PF_CTL, #0x10
|
|
10:
|
|
umull v9.8h, v22.8b, v5.8b
|
|
PF ble 10f
|
|
PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift
|
|
PF ldrsb DUMMY, [PF_SRC, DUMMY]
|
|
PF add PF_SRC, PF_SRC, #1
|
|
10:
|
|
umull v10.8h, v22.8b, v6.8b
|
|
PF ble 10f
|
|
PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift
|
|
PF ldrsb DUMMY, [PF_DST, DUMMY]
|
|
PF add PF_DST, PF_DST, #1
|
|
10:
|
|
umull v11.8h, v22.8b, v7.8b
|
|
.endm
|
|
|
|
generate_composite_function_single_scanline \
|
|
pixman_composite_scanline_out_reverse_asm_neon, 32, 0, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_out_reverse_8888_8888_process_pixblock_head, \
|
|
pixman_composite_out_reverse_8888_8888_process_pixblock_tail, \
|
|
pixman_composite_out_reverse_8888_8888_process_pixblock_tail_head
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_over_8888_8888_process_pixblock_head
|
|
pixman_composite_out_reverse_8888_8888_process_pixblock_head
|
|
.endm
|
|
|
|
.macro pixman_composite_over_8888_8888_process_pixblock_tail
|
|
pixman_composite_out_reverse_8888_8888_process_pixblock_tail
|
|
uqadd v28.8b, v0.8b, v28.8b
|
|
uqadd v29.8b, v1.8b, v29.8b
|
|
uqadd v30.8b, v2.8b, v30.8b
|
|
uqadd v31.8b, v3.8b, v31.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_over_8888_8888_process_pixblock_tail_head
|
|
ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
|
|
urshr v14.8h, v8.8h, #8
|
|
PF add PF_X, PF_X, #8
|
|
PF tst PF_CTL, #0xF
|
|
urshr v15.8h, v9.8h, #8
|
|
urshr v16.8h, v10.8h, #8
|
|
urshr v17.8h, v11.8h, #8
|
|
PF beq 10f
|
|
PF add PF_X, PF_X, #8
|
|
PF sub PF_CTL, PF_CTL, #1
|
|
10:
|
|
raddhn v28.8b, v14.8h, v8.8h
|
|
raddhn v29.8b, v15.8h, v9.8h
|
|
PF cmp PF_X, ORIG_W
|
|
raddhn v30.8b, v16.8h, v10.8h
|
|
raddhn v31.8b, v17.8h, v11.8h
|
|
uqadd v28.8b, v0.8b, v28.8b
|
|
uqadd v29.8b, v1.8b, v29.8b
|
|
uqadd v30.8b, v2.8b, v30.8b
|
|
uqadd v31.8b, v3.8b, v31.8b
|
|
fetch_src_pixblock
|
|
PF lsl DUMMY, PF_X, #src_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_SRC, DUMMY]
|
|
mvn v22.8b, v3.8b
|
|
PF lsl DUMMY, PF_X, #dst_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_DST, DUMMY]
|
|
st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
PF ble 10f
|
|
PF sub PF_X, PF_X, ORIG_W
|
|
10:
|
|
umull v8.8h, v22.8b, v4.8b
|
|
PF ble 10f
|
|
PF subs PF_CTL, PF_CTL, #0x10
|
|
10:
|
|
umull v9.8h, v22.8b, v5.8b
|
|
PF ble 10f
|
|
PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift
|
|
PF ldrsb DUMMY, [PF_SRC, DUMMY]
|
|
PF add PF_SRC, PF_SRC, #1
|
|
10:
|
|
umull v10.8h, v22.8b, v6.8b
|
|
PF ble 10f
|
|
PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift
|
|
PF ldrsb DUMMY, [PF_DST, DUMMY]
|
|
PF add PF_DST, PF_DST, #1
|
|
10:
|
|
umull v11.8h, v22.8b, v7.8b
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_over_8888_8888_asm_neon, 32, 0, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_over_8888_8888_process_pixblock_head, \
|
|
pixman_composite_over_8888_8888_process_pixblock_tail, \
|
|
pixman_composite_over_8888_8888_process_pixblock_tail_head
|
|
|
|
generate_composite_function_single_scanline \
|
|
pixman_composite_scanline_over_asm_neon, 32, 0, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_over_8888_8888_process_pixblock_head, \
|
|
pixman_composite_over_8888_8888_process_pixblock_tail, \
|
|
pixman_composite_over_8888_8888_process_pixblock_tail_head
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_over_n_8888_process_pixblock_head
|
|
/* deinterleaved source pixels in {v0, v1, v2, v3} */
|
|
/* inverted alpha in {v24} */
|
|
/* destination pixels in {v4, v5, v6, v7} */
|
|
umull v8.8h, v24.8b, v4.8b
|
|
umull v9.8h, v24.8b, v5.8b
|
|
umull v10.8h, v24.8b, v6.8b
|
|
umull v11.8h, v24.8b, v7.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8888_process_pixblock_tail
|
|
urshr v14.8h, v8.8h, #8
|
|
urshr v15.8h, v9.8h, #8
|
|
urshr v16.8h, v10.8h, #8
|
|
urshr v17.8h, v11.8h, #8
|
|
raddhn v28.8b, v14.8h, v8.8h
|
|
raddhn v29.8b, v15.8h, v9.8h
|
|
raddhn v30.8b, v16.8h, v10.8h
|
|
raddhn v31.8b, v17.8h, v11.8h
|
|
uqadd v28.8b, v0.8b, v28.8b
|
|
uqadd v29.8b, v1.8b, v29.8b
|
|
uqadd v30.8b, v2.8b, v30.8b
|
|
uqadd v31.8b, v3.8b, v31.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8888_process_pixblock_tail_head
|
|
urshr v14.8h, v8.8h, #8
|
|
urshr v15.8h, v9.8h, #8
|
|
urshr v16.8h, v10.8h, #8
|
|
urshr v17.8h, v11.8h, #8
|
|
raddhn v28.8b, v14.8h, v8.8h
|
|
raddhn v29.8b, v15.8h, v9.8h
|
|
raddhn v30.8b, v16.8h, v10.8h
|
|
raddhn v31.8b, v17.8h, v11.8h
|
|
ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
|
|
uqadd v28.8b, v0.8b, v28.8b
|
|
PF add PF_X, PF_X, #8
|
|
PF tst PF_CTL, #0x0F
|
|
PF beq 10f
|
|
PF add PF_X, PF_X, #8
|
|
PF sub PF_CTL, PF_CTL, #1
|
|
10:
|
|
uqadd v29.8b, v1.8b, v29.8b
|
|
uqadd v30.8b, v2.8b, v30.8b
|
|
uqadd v31.8b, v3.8b, v31.8b
|
|
PF cmp PF_X, ORIG_W
|
|
umull v8.8h, v24.8b, v4.8b
|
|
PF lsl DUMMY, PF_X, #dst_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_DST, DUMMY]
|
|
umull v9.8h, v24.8b, v5.8b
|
|
PF ble 10f
|
|
PF sub PF_X, PF_X, ORIG_W
|
|
10:
|
|
umull v10.8h, v24.8b, v6.8b
|
|
PF subs PF_CTL, PF_CTL, #0x10
|
|
umull v11.8h, v24.8b, v7.8b
|
|
PF ble 10f
|
|
PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift
|
|
PF ldrsb DUMMY, [PF_DST, DUMMY]
|
|
PF add PF_DST, PF_DST, #1
|
|
10:
|
|
st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8888_init
|
|
mov v3.s[0], w4
|
|
dup v0.8b, v3.b[0]
|
|
dup v1.8b, v3.b[1]
|
|
dup v2.8b, v3.b[2]
|
|
dup v3.8b, v3.b[3]
|
|
mvn v24.8b, v3.8b /* get inverted alpha */
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_over_n_8888_asm_neon, 0, 0, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_over_n_8888_init, \
|
|
default_cleanup, \
|
|
pixman_composite_over_8888_8888_process_pixblock_head, \
|
|
pixman_composite_over_8888_8888_process_pixblock_tail, \
|
|
pixman_composite_over_n_8888_process_pixblock_tail_head
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_over_reverse_n_8888_process_pixblock_tail_head
|
|
urshr v14.8h, v8.8h, #8
|
|
PF add PF_X, PF_X, #8
|
|
PF tst PF_CTL, #0xF
|
|
urshr v15.8h, v9.8h, #8
|
|
urshr v12.8h, v10.8h, #8
|
|
urshr v13.8h, v11.8h, #8
|
|
PF beq 10f
|
|
PF add PF_X, PF_X, #8
|
|
PF sub PF_CTL, PF_CTL, #1
|
|
10:
|
|
raddhn v28.8b, v14.8h, v8.8h
|
|
raddhn v29.8b, v15.8h, v9.8h
|
|
PF cmp PF_X, ORIG_W
|
|
raddhn v30.8b, v12.8h, v10.8h
|
|
raddhn v31.8b, v13.8h, v11.8h
|
|
uqadd v28.8b, v0.8b, v28.8b
|
|
uqadd v29.8b, v1.8b, v29.8b
|
|
uqadd v30.8b, v2.8b, v30.8b
|
|
uqadd v31.8b, v3.8b, v31.8b
|
|
ld4 {v0.8b, v1.8b, v2.8b, v3.8b}, [DST_R], #32
|
|
mvn v22.8b, v3.8b
|
|
PF lsl DUMMY, PF_X, #dst_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_DST, DUMMY]
|
|
st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
PF blt 10f
|
|
PF sub PF_X, PF_X, ORIG_W
|
|
10:
|
|
umull v8.8h, v22.8b, v4.8b
|
|
PF blt 10f
|
|
PF subs PF_CTL, PF_CTL, #0x10
|
|
10:
|
|
umull v9.8h, v22.8b, v5.8b
|
|
umull v10.8h, v22.8b, v6.8b
|
|
PF blt 10f
|
|
PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift
|
|
PF ldrsb DUMMY, [PF_DST, DUMMY]
|
|
PF add PF_DST, PF_DST, #1
|
|
10:
|
|
umull v11.8h, v22.8b, v7.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_over_reverse_n_8888_init
|
|
mov v7.s[0], w4
|
|
dup v4.8b, v7.b[0]
|
|
dup v5.8b, v7.b[1]
|
|
dup v6.8b, v7.b[2]
|
|
dup v7.8b, v7.b[3]
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_over_reverse_n_8888_asm_neon, 0, 0, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_over_reverse_n_8888_init, \
|
|
default_cleanup, \
|
|
pixman_composite_over_8888_8888_process_pixblock_head, \
|
|
pixman_composite_over_8888_8888_process_pixblock_tail, \
|
|
pixman_composite_over_reverse_n_8888_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
0, /* dst_r_basereg */ \
|
|
4, /* src_basereg */ \
|
|
24 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_over_8888_8_0565_process_pixblock_head
|
|
umull v0.8h, v24.8b, v8.8b /* IN for SRC pixels (part1) */
|
|
umull v1.8h, v24.8b, v9.8b
|
|
umull v2.8h, v24.8b, v10.8b
|
|
umull v3.8h, v24.8b, v11.8b
|
|
mov v4.d[1], v5.d[0]
|
|
shrn v25.8b, v4.8h, #8 /* convert DST_R data to 32-bpp (part1) */
|
|
shrn v26.8b, v4.8h, #3
|
|
sli v4.8h, v4.8h, #5
|
|
urshr v17.8h, v0.8h, #8 /* IN for SRC pixels (part2) */
|
|
urshr v18.8h, v1.8h, #8
|
|
urshr v19.8h, v2.8h, #8
|
|
urshr v20.8h, v3.8h, #8
|
|
raddhn v0.8b, v0.8h, v17.8h
|
|
raddhn v1.8b, v1.8h, v18.8h
|
|
raddhn v2.8b, v2.8h, v19.8h
|
|
raddhn v3.8b, v3.8h, v20.8h
|
|
sri v25.8b, v25.8b, #5 /* convert DST_R data to 32-bpp (part2) */
|
|
sri v26.8b, v26.8b, #6
|
|
mvn v3.8b, v3.8b
|
|
shrn v30.8b, v4.8h, #2
|
|
umull v18.8h, v3.8b, v25.8b /* now do alpha blending */
|
|
umull v19.8h, v3.8b, v26.8b
|
|
umull v20.8h, v3.8b, v30.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_over_8888_8_0565_process_pixblock_tail
|
|
/* 3 cycle bubble (after vmull.u8) */
|
|
urshr v5.8h, v18.8h, #8
|
|
urshr v6.8h, v19.8h, #8
|
|
urshr v7.8h, v20.8h, #8
|
|
raddhn v17.8b, v18.8h, v5.8h
|
|
raddhn v19.8b, v19.8h, v6.8h
|
|
raddhn v18.8b, v20.8h, v7.8h
|
|
uqadd v5.8b, v2.8b, v17.8b
|
|
/* 1 cycle bubble */
|
|
uqadd v6.8b, v0.8b, v18.8b
|
|
uqadd v7.8b, v1.8b, v19.8b
|
|
ushll v14.8h, v5.8b, #7 /* convert to 16bpp */
|
|
sli v14.8h, v14.8h, #1
|
|
ushll v18.8h, v7.8b, #7
|
|
sli v18.8h, v18.8h, #1
|
|
ushll v19.8h, v6.8b, #7
|
|
sli v19.8h, v19.8h, #1
|
|
sri v14.8h, v18.8h, #5
|
|
/* 1 cycle bubble */
|
|
sri v14.8h, v19.8h, #11
|
|
mov v28.d[0], v14.d[0]
|
|
mov v29.d[0], v14.d[1]
|
|
.endm
|
|
|
|
.macro pixman_composite_over_8888_8_0565_process_pixblock_tail_head
|
|
#if 0
|
|
ld1 {v4.8h}, [DST_R], #16
|
|
shrn v25.8b, v4.8h, #8
|
|
fetch_mask_pixblock
|
|
shrn v26.8b, v4.8h, #3
|
|
fetch_src_pixblock
|
|
umull v22.8h, v24.8b, v10.8b
|
|
urshr v13.8h, v18.8h, #8
|
|
urshr v11.8h, v19.8h, #8
|
|
urshr v15.8h, v20.8h, #8
|
|
raddhn v17.8b, v18.8h, v13.8h
|
|
raddhn v19.8b, v19.8h, v11.8h
|
|
raddhn v18.8b, v20.8h, v15.8h
|
|
uqadd v17.8b, v2.8b, v17.8b
|
|
umull v21.8h, v24.8b, v9.8b
|
|
uqadd v18.8b, v0.8b, v18.8b
|
|
uqadd v19.8b, v1.8b, v19.8b
|
|
ushll v14.8h, v17.8b, #7
|
|
sli v14.8h, v14.8h, #1
|
|
umull v20.8h, v24.8b, v8.8b
|
|
ushll v18.8h, v18.8b, #7
|
|
sli v18.8h, v18.8h, #1
|
|
ushll v19.8h, v19.8b, #7
|
|
sli v19.8h, v19.8h, #1
|
|
sri v14.8h, v18.8h, #5
|
|
umull v23.8h, v24.8b, v11.8b
|
|
sri v14.8h, v19.8h, #11
|
|
mov v28.d[0], v14.d[0]
|
|
mov v29.d[0], v14.d[1]
|
|
|
|
cache_preload 8, 8
|
|
|
|
sli v4.8h, v4.8h, #5
|
|
urshr v16.8h, v20.8h, #8
|
|
urshr v17.8h, v21.8h, #8
|
|
urshr v18.8h, v22.8h, #8
|
|
urshr v19.8h, v23.8h, #8
|
|
raddhn v0.8b, v20.8h, v16.8h
|
|
raddhn v1.8b, v21.8h, v17.8h
|
|
raddhn v2.8b, v22.8h, v18.8h
|
|
raddhn v3.8b, v23.8h, v19.8h
|
|
sri v25.8b, v25.8b, #5
|
|
sri v26.8b, v26.8b, #6
|
|
mvn v3.8b, v3.8b
|
|
shrn v30.8b, v4.8h, #2
|
|
st1 {v14.8h}, [DST_W], #16
|
|
umull v18.8h, v3.8b, v25.8b
|
|
umull v19.8h, v3.8b, v26.8b
|
|
umull v20.8h, v3.8b, v30.8b
|
|
#else
|
|
pixman_composite_over_8888_8_0565_process_pixblock_tail
|
|
st1 {v28.4h, v29.4h}, [DST_W], #16
|
|
ld1 {v4.4h, v5.4h}, [DST_R], #16
|
|
fetch_mask_pixblock
|
|
fetch_src_pixblock
|
|
pixman_composite_over_8888_8_0565_process_pixblock_head
|
|
#endif
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_over_8888_8_0565_asm_neon, 32, 8, 16, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
default_init_need_all_regs, \
|
|
default_cleanup_need_all_regs, \
|
|
pixman_composite_over_8888_8_0565_process_pixblock_head, \
|
|
pixman_composite_over_8888_8_0565_process_pixblock_tail, \
|
|
pixman_composite_over_8888_8_0565_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
8, /* src_basereg */ \
|
|
24 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
/*
|
|
* This function needs a special initialization of solid mask.
|
|
* Solid source pixel data is fetched from stack at ARGS_STACK_OFFSET
|
|
* offset, split into color components and replicated in d8-d11
|
|
* registers. Additionally, this function needs all the NEON registers,
|
|
* so it has to save d8-d15 registers which are callee saved according
|
|
* to ABI. These registers are restored from 'cleanup' macro. All the
|
|
* other NEON registers are caller saved, so can be clobbered freely
|
|
* without introducing any problems.
|
|
*/
|
|
.macro pixman_composite_over_n_8_0565_init
|
|
mov v11.s[0], w4
|
|
dup v8.8b, v11.b[0]
|
|
dup v9.8b, v11.b[1]
|
|
dup v10.8b, v11.b[2]
|
|
dup v11.8b, v11.b[3]
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8_0565_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_over_n_8_0565_asm_neon, 0, 8, 16, \
|
|
FLAG_DST_READWRITE, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_over_n_8_0565_init, \
|
|
pixman_composite_over_n_8_0565_cleanup, \
|
|
pixman_composite_over_8888_8_0565_process_pixblock_head, \
|
|
pixman_composite_over_8888_8_0565_process_pixblock_tail, \
|
|
pixman_composite_over_8888_8_0565_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
8, /* src_basereg */ \
|
|
24 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_over_8888_n_0565_init
|
|
mov v24.s[0], w6
|
|
dup v24.8b, v24.b[3]
|
|
.endm
|
|
|
|
.macro pixman_composite_over_8888_n_0565_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_over_8888_n_0565_asm_neon, 32, 0, 16, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_over_8888_n_0565_init, \
|
|
pixman_composite_over_8888_n_0565_cleanup, \
|
|
pixman_composite_over_8888_8_0565_process_pixblock_head, \
|
|
pixman_composite_over_8888_8_0565_process_pixblock_tail, \
|
|
pixman_composite_over_8888_8_0565_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
8, /* src_basereg */ \
|
|
24 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_src_0565_0565_process_pixblock_head
|
|
.endm
|
|
|
|
.macro pixman_composite_src_0565_0565_process_pixblock_tail
|
|
.endm
|
|
|
|
.macro pixman_composite_src_0565_0565_process_pixblock_tail_head
|
|
st1 {v0.4h, v1.4h, v2.4h, v3.4h}, [DST_W], #32
|
|
fetch_src_pixblock
|
|
cache_preload 16, 16
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_src_0565_0565_asm_neon, 16, 0, 16, \
|
|
FLAG_DST_WRITEONLY, \
|
|
16, /* number of pixels, processed in a single block */ \
|
|
10, /* prefetch distance */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_src_0565_0565_process_pixblock_head, \
|
|
pixman_composite_src_0565_0565_process_pixblock_tail, \
|
|
pixman_composite_src_0565_0565_process_pixblock_tail_head, \
|
|
0, /* dst_w_basereg */ \
|
|
0, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
0 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_src_n_8_process_pixblock_head
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_8_process_pixblock_tail
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_8_process_pixblock_tail_head
|
|
st1 {v0.8b, v1.8b, v2.8b, v3.8b}, [DST_W], 32
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_8_init
|
|
mov v0.s[0], w4
|
|
dup v3.8b, v0.b[0]
|
|
dup v2.8b, v0.b[0]
|
|
dup v1.8b, v0.b[0]
|
|
dup v0.8b, v0.b[0]
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_8_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_src_n_8_asm_neon, 0, 0, 8, \
|
|
FLAG_DST_WRITEONLY, \
|
|
32, /* number of pixels, processed in a single block */ \
|
|
0, /* prefetch distance */ \
|
|
pixman_composite_src_n_8_init, \
|
|
pixman_composite_src_n_8_cleanup, \
|
|
pixman_composite_src_n_8_process_pixblock_head, \
|
|
pixman_composite_src_n_8_process_pixblock_tail, \
|
|
pixman_composite_src_n_8_process_pixblock_tail_head, \
|
|
0, /* dst_w_basereg */ \
|
|
0, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
0 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_src_n_0565_process_pixblock_head
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_0565_process_pixblock_tail
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_0565_process_pixblock_tail_head
|
|
st1 {v0.4h, v1.4h, v2.4h, v3.4h}, [DST_W], #32
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_0565_init
|
|
mov v0.s[0], w4
|
|
dup v3.4h, v0.h[0]
|
|
dup v2.4h, v0.h[0]
|
|
dup v1.4h, v0.h[0]
|
|
dup v0.4h, v0.h[0]
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_0565_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_src_n_0565_asm_neon, 0, 0, 16, \
|
|
FLAG_DST_WRITEONLY, \
|
|
16, /* number of pixels, processed in a single block */ \
|
|
0, /* prefetch distance */ \
|
|
pixman_composite_src_n_0565_init, \
|
|
pixman_composite_src_n_0565_cleanup, \
|
|
pixman_composite_src_n_0565_process_pixblock_head, \
|
|
pixman_composite_src_n_0565_process_pixblock_tail, \
|
|
pixman_composite_src_n_0565_process_pixblock_tail_head, \
|
|
0, /* dst_w_basereg */ \
|
|
0, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
0 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_src_n_8888_process_pixblock_head
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_8888_process_pixblock_tail
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_8888_process_pixblock_tail_head
|
|
st1 {v0.2s, v1.2s, v2.2s, v3.2s}, [DST_W], #32
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_8888_init
|
|
mov v0.s[0], w4
|
|
dup v3.2s, v0.s[0]
|
|
dup v2.2s, v0.s[0]
|
|
dup v1.2s, v0.s[0]
|
|
dup v0.2s, v0.s[0]
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_8888_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_src_n_8888_asm_neon, 0, 0, 32, \
|
|
FLAG_DST_WRITEONLY, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
0, /* prefetch distance */ \
|
|
pixman_composite_src_n_8888_init, \
|
|
pixman_composite_src_n_8888_cleanup, \
|
|
pixman_composite_src_n_8888_process_pixblock_head, \
|
|
pixman_composite_src_n_8888_process_pixblock_tail, \
|
|
pixman_composite_src_n_8888_process_pixblock_tail_head, \
|
|
0, /* dst_w_basereg */ \
|
|
0, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
0 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_src_8888_8888_process_pixblock_head
|
|
.endm
|
|
|
|
.macro pixman_composite_src_8888_8888_process_pixblock_tail
|
|
.endm
|
|
|
|
.macro pixman_composite_src_8888_8888_process_pixblock_tail_head
|
|
st1 {v0.2s, v1.2s, v2.2s, v3.2s}, [DST_W], #32
|
|
fetch_src_pixblock
|
|
cache_preload 8, 8
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_src_8888_8888_asm_neon, 32, 0, 32, \
|
|
FLAG_DST_WRITEONLY, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
10, /* prefetch distance */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_src_8888_8888_process_pixblock_head, \
|
|
pixman_composite_src_8888_8888_process_pixblock_tail, \
|
|
pixman_composite_src_8888_8888_process_pixblock_tail_head, \
|
|
0, /* dst_w_basereg */ \
|
|
0, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
0 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_src_x888_8888_process_pixblock_head
|
|
orr v0.8b, v0.8b, v4.8b
|
|
orr v1.8b, v1.8b, v4.8b
|
|
orr v2.8b, v2.8b, v4.8b
|
|
orr v3.8b, v3.8b, v4.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_src_x888_8888_process_pixblock_tail
|
|
.endm
|
|
|
|
.macro pixman_composite_src_x888_8888_process_pixblock_tail_head
|
|
st1 {v0.2s, v1.2s, v2.2s, v3.2s}, [DST_W], #32
|
|
fetch_src_pixblock
|
|
orr v0.8b, v0.8b, v4.8b
|
|
orr v1.8b, v1.8b, v4.8b
|
|
orr v2.8b, v2.8b, v4.8b
|
|
orr v3.8b, v3.8b, v4.8b
|
|
cache_preload 8, 8
|
|
.endm
|
|
|
|
.macro pixman_composite_src_x888_8888_init
|
|
movi v4.2s, #0xff, lsl 24
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_src_x888_8888_asm_neon, 32, 0, 32, \
|
|
FLAG_DST_WRITEONLY, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
10, /* prefetch distance */ \
|
|
pixman_composite_src_x888_8888_init, \
|
|
default_cleanup, \
|
|
pixman_composite_src_x888_8888_process_pixblock_head, \
|
|
pixman_composite_src_x888_8888_process_pixblock_tail, \
|
|
pixman_composite_src_x888_8888_process_pixblock_tail_head, \
|
|
0, /* dst_w_basereg */ \
|
|
0, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
0 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_src_n_8_8888_process_pixblock_head
|
|
/* expecting solid source in {v0, v1, v2, v3} */
|
|
/* mask is in v24 (v25, v26, v27 are unused) */
|
|
|
|
/* in */
|
|
umull v8.8h, v24.8b, v0.8b
|
|
umull v9.8h, v24.8b, v1.8b
|
|
umull v10.8h, v24.8b, v2.8b
|
|
umull v11.8h, v24.8b, v3.8b
|
|
ursra v8.8h, v8.8h, #8
|
|
ursra v9.8h, v9.8h, #8
|
|
ursra v10.8h, v10.8h, #8
|
|
ursra v11.8h, v11.8h, #8
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_8_8888_process_pixblock_tail
|
|
rshrn v28.8b, v8.8h, #8
|
|
rshrn v29.8b, v9.8h, #8
|
|
rshrn v30.8b, v10.8h, #8
|
|
rshrn v31.8b, v11.8h, #8
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_8_8888_process_pixblock_tail_head
|
|
fetch_mask_pixblock
|
|
PF add PF_X, PF_X, #8
|
|
rshrn v28.8b, v8.8h, #8
|
|
PF tst PF_CTL, #0x0F
|
|
rshrn v29.8b, v9.8h, #8
|
|
PF beq 10f
|
|
PF add PF_X, PF_X, #8
|
|
10:
|
|
rshrn v30.8b, v10.8h, #8
|
|
PF beq 10f
|
|
PF sub PF_CTL, PF_CTL, #1
|
|
10:
|
|
rshrn v31.8b, v11.8h, #8
|
|
PF cmp PF_X, ORIG_W
|
|
umull v8.8h, v24.8b, v0.8b
|
|
PF lsl DUMMY, PF_X, #mask_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_MASK, DUMMY]
|
|
umull v9.8h, v24.8b, v1.8b
|
|
PF ble 10f
|
|
PF sub PF_X, PF_X, ORIG_W
|
|
10:
|
|
umull v10.8h, v24.8b, v2.8b
|
|
PF ble 10f
|
|
PF subs PF_CTL, PF_CTL, #0x10
|
|
10:
|
|
umull v11.8h, v24.8b, v3.8b
|
|
PF ble 10f
|
|
PF lsl DUMMY, MASK_STRIDE, #mask_bpp_shift
|
|
PF ldrsb DUMMY, [PF_MASK, DUMMY]
|
|
PF add PF_MASK, PF_MASK, #1
|
|
10:
|
|
st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
ursra v8.8h, v8.8h, #8
|
|
ursra v9.8h, v9.8h, #8
|
|
ursra v10.8h, v10.8h, #8
|
|
ursra v11.8h, v11.8h, #8
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_8_8888_init
|
|
mov v3.s[0], w4
|
|
dup v0.8b, v3.b[0]
|
|
dup v1.8b, v3.b[1]
|
|
dup v2.8b, v3.b[2]
|
|
dup v3.8b, v3.b[3]
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_8_8888_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_src_n_8_8888_asm_neon, 0, 8, 32, \
|
|
FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_src_n_8_8888_init, \
|
|
pixman_composite_src_n_8_8888_cleanup, \
|
|
pixman_composite_src_n_8_8888_process_pixblock_head, \
|
|
pixman_composite_src_n_8_8888_process_pixblock_tail, \
|
|
pixman_composite_src_n_8_8888_process_pixblock_tail_head, \
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_src_n_8_8_process_pixblock_head
|
|
umull v0.8h, v24.8b, v16.8b
|
|
umull v1.8h, v25.8b, v16.8b
|
|
umull v2.8h, v26.8b, v16.8b
|
|
umull v3.8h, v27.8b, v16.8b
|
|
ursra v0.8h, v0.8h, #8
|
|
ursra v1.8h, v1.8h, #8
|
|
ursra v2.8h, v2.8h, #8
|
|
ursra v3.8h, v3.8h, #8
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_8_8_process_pixblock_tail
|
|
rshrn v28.8b, v0.8h, #8
|
|
rshrn v29.8b, v1.8h, #8
|
|
rshrn v30.8b, v2.8h, #8
|
|
rshrn v31.8b, v3.8h, #8
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_8_8_process_pixblock_tail_head
|
|
fetch_mask_pixblock
|
|
PF add PF_X, PF_X, #8
|
|
rshrn v28.8b, v0.8h, #8
|
|
PF tst PF_CTL, #0x0F
|
|
rshrn v29.8b, v1.8h, #8
|
|
PF beq 10f
|
|
PF add PF_X, PF_X, #8
|
|
10:
|
|
rshrn v30.8b, v2.8h, #8
|
|
PF beq 10f
|
|
PF sub PF_CTL, PF_CTL, #1
|
|
10:
|
|
rshrn v31.8b, v3.8h, #8
|
|
PF cmp PF_X, ORIG_W
|
|
umull v0.8h, v24.8b, v16.8b
|
|
PF lsl DUMMY, PF_X, mask_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_MASK, DUMMY]
|
|
umull v1.8h, v25.8b, v16.8b
|
|
PF ble 10f
|
|
PF sub PF_X, PF_X, ORIG_W
|
|
10:
|
|
umull v2.8h, v26.8b, v16.8b
|
|
PF ble 10f
|
|
PF subs PF_CTL, PF_CTL, #0x10
|
|
10:
|
|
umull v3.8h, v27.8b, v16.8b
|
|
PF ble 10f
|
|
PF lsl DUMMY, MASK_STRIDE, #mask_bpp_shift
|
|
PF ldrsb DUMMY, [PF_MASK, DUMMY]
|
|
PF add PF_MASK, PF_MASK, #1
|
|
10:
|
|
st1 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
ursra v0.8h, v0.8h, #8
|
|
ursra v1.8h, v1.8h, #8
|
|
ursra v2.8h, v2.8h, #8
|
|
ursra v3.8h, v3.8h, #8
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_8_8_init
|
|
mov v16.s[0], w4
|
|
dup v16.8b, v16.b[3]
|
|
.endm
|
|
|
|
.macro pixman_composite_src_n_8_8_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_src_n_8_8_asm_neon, 0, 8, 8, \
|
|
FLAG_DST_WRITEONLY, \
|
|
32, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_src_n_8_8_init, \
|
|
pixman_composite_src_n_8_8_cleanup, \
|
|
pixman_composite_src_n_8_8_process_pixblock_head, \
|
|
pixman_composite_src_n_8_8_process_pixblock_tail, \
|
|
pixman_composite_src_n_8_8_process_pixblock_tail_head
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_over_n_8_8888_process_pixblock_head
|
|
/* expecting deinterleaved source data in {v8, v9, v10, v11} */
|
|
/* v8 - blue, v9 - green, v10 - red, v11 - alpha */
|
|
/* and destination data in {v4, v5, v6, v7} */
|
|
/* mask is in v24 (v25, v26, v27 are unused) */
|
|
|
|
/* in */
|
|
umull v12.8h, v24.8b, v8.8b
|
|
umull v13.8h, v24.8b, v9.8b
|
|
umull v14.8h, v24.8b, v10.8b
|
|
umull v15.8h, v24.8b, v11.8b
|
|
urshr v16.8h, v12.8h, #8
|
|
urshr v17.8h, v13.8h, #8
|
|
urshr v18.8h, v14.8h, #8
|
|
urshr v19.8h, v15.8h, #8
|
|
raddhn v0.8b, v12.8h, v16.8h
|
|
raddhn v1.8b, v13.8h, v17.8h
|
|
raddhn v2.8b, v14.8h, v18.8h
|
|
raddhn v3.8b, v15.8h, v19.8h
|
|
mvn v25.8b, v3.8b /* get inverted alpha */
|
|
/* source: v0 - blue, v1 - green, v2 - red, v3 - alpha */
|
|
/* destination: v4 - blue, v5 - green, v6 - red, v7 - alpha */
|
|
/* now do alpha blending */
|
|
umull v12.8h, v25.8b, v4.8b
|
|
umull v13.8h, v25.8b, v5.8b
|
|
umull v14.8h, v25.8b, v6.8b
|
|
umull v15.8h, v25.8b, v7.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8_8888_process_pixblock_tail
|
|
urshr v16.8h, v12.8h, #8
|
|
urshr v17.8h, v13.8h, #8
|
|
urshr v18.8h, v14.8h, #8
|
|
urshr v19.8h, v15.8h, #8
|
|
raddhn v28.8b, v16.8h, v12.8h
|
|
raddhn v29.8b, v17.8h, v13.8h
|
|
raddhn v30.8b, v18.8h, v14.8h
|
|
raddhn v31.8b, v19.8h, v15.8h
|
|
uqadd v28.8b, v0.8b, v28.8b
|
|
uqadd v29.8b, v1.8b, v29.8b
|
|
uqadd v30.8b, v2.8b, v30.8b
|
|
uqadd v31.8b, v3.8b, v31.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8_8888_process_pixblock_tail_head
|
|
urshr v16.8h, v12.8h, #8
|
|
ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
|
|
urshr v17.8h, v13.8h, #8
|
|
fetch_mask_pixblock
|
|
urshr v18.8h, v14.8h, #8
|
|
PF add PF_X, PF_X, #8
|
|
urshr v19.8h, v15.8h, #8
|
|
PF tst PF_CTL, #0x0F
|
|
raddhn v28.8b, v16.8h, v12.8h
|
|
PF beq 10f
|
|
PF add PF_X, PF_X, #8
|
|
10:
|
|
raddhn v29.8b, v17.8h, v13.8h
|
|
PF beq 10f
|
|
PF sub PF_CTL, PF_CTL, #1
|
|
10:
|
|
raddhn v30.8b, v18.8h, v14.8h
|
|
PF cmp PF_X, ORIG_W
|
|
raddhn v31.8b, v19.8h, v15.8h
|
|
PF lsl DUMMY, PF_X, #dst_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_DST, DUMMY]
|
|
umull v16.8h, v24.8b, v8.8b
|
|
PF lsl DUMMY, PF_X, #mask_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_MASK, DUMMY]
|
|
umull v17.8h, v24.8b, v9.8b
|
|
PF ble 10f
|
|
PF sub PF_X, PF_X, ORIG_W
|
|
10:
|
|
umull v18.8h, v24.8b, v10.8b
|
|
PF ble 10f
|
|
PF subs PF_CTL, PF_CTL, #0x10
|
|
10:
|
|
umull v19.8h, v24.8b, v11.8b
|
|
PF ble 10f
|
|
PF lsl DUMMY, DST_STRIDE, #dst_bpp_shift
|
|
PF ldrsb DUMMY, [PF_DST, DUMMY]
|
|
PF add PF_DST, PF_DST, #1
|
|
10:
|
|
uqadd v28.8b, v0.8b, v28.8b
|
|
PF ble 10f
|
|
PF lsl DUMMY, MASK_STRIDE, #mask_bpp_shift
|
|
PF ldrsb DUMMY, [PF_MASK, DUMMY]
|
|
PF add PF_MASK, PF_MASK, #1
|
|
10:
|
|
uqadd v29.8b, v1.8b, v29.8b
|
|
uqadd v30.8b, v2.8b, v30.8b
|
|
uqadd v31.8b, v3.8b, v31.8b
|
|
urshr v12.8h, v16.8h, #8
|
|
urshr v13.8h, v17.8h, #8
|
|
urshr v14.8h, v18.8h, #8
|
|
urshr v15.8h, v19.8h, #8
|
|
raddhn v0.8b, v16.8h, v12.8h
|
|
raddhn v1.8b, v17.8h, v13.8h
|
|
raddhn v2.8b, v18.8h, v14.8h
|
|
raddhn v3.8b, v19.8h, v15.8h
|
|
st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
mvn v25.8b, v3.8b
|
|
umull v12.8h, v25.8b, v4.8b
|
|
umull v13.8h, v25.8b, v5.8b
|
|
umull v14.8h, v25.8b, v6.8b
|
|
umull v15.8h, v25.8b, v7.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8_8888_init
|
|
mov v11.s[0], w4
|
|
dup v8.8b, v11.b[0]
|
|
dup v9.8b, v11.b[1]
|
|
dup v10.8b, v11.b[2]
|
|
dup v11.8b, v11.b[3]
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8_8888_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_over_n_8_8888_asm_neon, 0, 8, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_over_n_8_8888_init, \
|
|
pixman_composite_over_n_8_8888_cleanup, \
|
|
pixman_composite_over_n_8_8888_process_pixblock_head, \
|
|
pixman_composite_over_n_8_8888_process_pixblock_tail, \
|
|
pixman_composite_over_n_8_8888_process_pixblock_tail_head
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_over_n_8_8_process_pixblock_head
|
|
umull v0.8h, v24.8b, v8.8b
|
|
umull v1.8h, v25.8b, v8.8b
|
|
umull v2.8h, v26.8b, v8.8b
|
|
umull v3.8h, v27.8b, v8.8b
|
|
urshr v10.8h, v0.8h, #8
|
|
urshr v11.8h, v1.8h, #8
|
|
urshr v12.8h, v2.8h, #8
|
|
urshr v13.8h, v3.8h, #8
|
|
raddhn v0.8b, v0.8h, v10.8h
|
|
raddhn v1.8b, v1.8h, v11.8h
|
|
raddhn v2.8b, v2.8h, v12.8h
|
|
raddhn v3.8b, v3.8h, v13.8h
|
|
mvn v24.8b, v0.8b
|
|
mvn v25.8b, v1.8b
|
|
mvn v26.8b, v2.8b
|
|
mvn v27.8b, v3.8b
|
|
umull v10.8h, v24.8b, v4.8b
|
|
umull v11.8h, v25.8b, v5.8b
|
|
umull v12.8h, v26.8b, v6.8b
|
|
umull v13.8h, v27.8b, v7.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8_8_process_pixblock_tail
|
|
urshr v14.8h, v10.8h, #8
|
|
urshr v15.8h, v11.8h, #8
|
|
urshr v16.8h, v12.8h, #8
|
|
urshr v17.8h, v13.8h, #8
|
|
raddhn v28.8b, v14.8h, v10.8h
|
|
raddhn v29.8b, v15.8h, v11.8h
|
|
raddhn v30.8b, v16.8h, v12.8h
|
|
raddhn v31.8b, v17.8h, v13.8h
|
|
uqadd v28.8b, v0.8b, v28.8b
|
|
uqadd v29.8b, v1.8b, v29.8b
|
|
uqadd v30.8b, v2.8b, v30.8b
|
|
uqadd v31.8b, v3.8b, v31.8b
|
|
.endm
|
|
|
|
/* TODO: expand macros and do better instructions scheduling */
|
|
.macro pixman_composite_over_n_8_8_process_pixblock_tail_head
|
|
ld1 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
|
|
pixman_composite_over_n_8_8_process_pixblock_tail
|
|
fetch_mask_pixblock
|
|
cache_preload 32, 32
|
|
st1 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
pixman_composite_over_n_8_8_process_pixblock_head
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8_8_init
|
|
mov v8.s[0], w4
|
|
dup v8.8b, v8.b[3]
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8_8_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_over_n_8_8_asm_neon, 0, 8, 8, \
|
|
FLAG_DST_READWRITE, \
|
|
32, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_over_n_8_8_init, \
|
|
pixman_composite_over_n_8_8_cleanup, \
|
|
pixman_composite_over_n_8_8_process_pixblock_head, \
|
|
pixman_composite_over_n_8_8_process_pixblock_tail, \
|
|
pixman_composite_over_n_8_8_process_pixblock_tail_head
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_head
|
|
/*
|
|
* 'combine_mask_ca' replacement
|
|
*
|
|
* input: solid src (n) in {v8, v9, v10, v11}
|
|
* dest in {v4, v5, v6, v7 }
|
|
* mask in {v24, v25, v26, v27}
|
|
* output: updated src in {v0, v1, v2, v3 }
|
|
* updated mask in {v24, v25, v26, v3 }
|
|
*/
|
|
umull v0.8h, v24.8b, v8.8b
|
|
umull v1.8h, v25.8b, v9.8b
|
|
umull v2.8h, v26.8b, v10.8b
|
|
umull v3.8h, v27.8b, v11.8b
|
|
umull v12.8h, v11.8b, v25.8b
|
|
umull v13.8h, v11.8b, v24.8b
|
|
umull v14.8h, v11.8b, v26.8b
|
|
urshr v15.8h, v0.8h, #8
|
|
urshr v16.8h, v1.8h, #8
|
|
urshr v17.8h, v2.8h, #8
|
|
raddhn v0.8b, v0.8h, v15.8h
|
|
raddhn v1.8b, v1.8h, v16.8h
|
|
raddhn v2.8b, v2.8h, v17.8h
|
|
urshr v15.8h, v13.8h, #8
|
|
urshr v16.8h, v12.8h, #8
|
|
urshr v17.8h, v14.8h, #8
|
|
urshr v18.8h, v3.8h, #8
|
|
raddhn v24.8b, v13.8h, v15.8h
|
|
raddhn v25.8b, v12.8h, v16.8h
|
|
raddhn v26.8b, v14.8h, v17.8h
|
|
raddhn v3.8b, v3.8h, v18.8h
|
|
/*
|
|
* 'combine_over_ca' replacement
|
|
*
|
|
* output: updated dest in {v28, v29, v30, v31}
|
|
*/
|
|
mvn v24.8b, v24.8b
|
|
mvn v25.8b, v25.8b
|
|
mvn v26.8b, v26.8b
|
|
mvn v27.8b, v3.8b
|
|
umull v12.8h, v24.8b, v4.8b
|
|
umull v13.8h, v25.8b, v5.8b
|
|
umull v14.8h, v26.8b, v6.8b
|
|
umull v15.8h, v27.8b, v7.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_tail
|
|
/* ... continue 'combine_over_ca' replacement */
|
|
urshr v16.8h, v12.8h, #8
|
|
urshr v17.8h, v13.8h, #8
|
|
urshr v18.8h, v14.8h, #8
|
|
urshr v19.8h, v15.8h, #8
|
|
raddhn v28.8b, v16.8h, v12.8h
|
|
raddhn v29.8b, v17.8h, v13.8h
|
|
raddhn v30.8b, v18.8h, v14.8h
|
|
raddhn v31.8b, v19.8h, v15.8h
|
|
uqadd v28.8b, v0.8b, v28.8b
|
|
uqadd v29.8b, v1.8b, v29.8b
|
|
uqadd v30.8b, v2.8b, v30.8b
|
|
uqadd v31.8b, v3.8b, v31.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8888_8888_ca_process_pixblock_tail_head
|
|
urshr v16.8h, v12.8h, #8
|
|
urshr v17.8h, v13.8h, #8
|
|
ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
|
|
urshr v18.8h, v14.8h, #8
|
|
urshr v19.8h, v15.8h, #8
|
|
raddhn v28.8b, v16.8h, v12.8h
|
|
raddhn v29.8b, v17.8h, v13.8h
|
|
raddhn v30.8b, v18.8h, v14.8h
|
|
raddhn v31.8b, v19.8h, v15.8h
|
|
fetch_mask_pixblock
|
|
uqadd v28.8b, v0.8b, v28.8b
|
|
uqadd v29.8b, v1.8b, v29.8b
|
|
uqadd v30.8b, v2.8b, v30.8b
|
|
uqadd v31.8b, v3.8b, v31.8b
|
|
cache_preload 8, 8
|
|
pixman_composite_over_n_8888_8888_ca_process_pixblock_head
|
|
st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8888_8888_ca_init
|
|
mov v13.s[0], w4
|
|
dup v8.8b, v13.b[0]
|
|
dup v9.8b, v13.b[1]
|
|
dup v10.8b, v13.b[2]
|
|
dup v11.8b, v13.b[3]
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8888_8888_ca_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_over_n_8888_8888_ca_asm_neon, 0, 32, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_over_n_8888_8888_ca_init, \
|
|
pixman_composite_over_n_8888_8888_ca_cleanup, \
|
|
pixman_composite_over_n_8888_8888_ca_process_pixblock_head, \
|
|
pixman_composite_over_n_8888_8888_ca_process_pixblock_tail, \
|
|
pixman_composite_over_n_8888_8888_ca_process_pixblock_tail_head
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_over_n_8888_0565_ca_process_pixblock_head
|
|
/*
|
|
* 'combine_mask_ca' replacement
|
|
*
|
|
* input: solid src (n) in {v8, v9, v10, v11} [B, G, R, A]
|
|
* mask in {v24, v25, v26} [B, G, R]
|
|
* output: updated src in {v0, v1, v2 } [B, G, R]
|
|
* updated mask in {v24, v25, v26} [B, G, R]
|
|
*/
|
|
umull v0.8h, v24.8b, v8.8b
|
|
umull v1.8h, v25.8b, v9.8b
|
|
umull v2.8h, v26.8b, v10.8b
|
|
umull v12.8h, v11.8b, v24.8b
|
|
umull v13.8h, v11.8b, v25.8b
|
|
umull v14.8h, v11.8b, v26.8b
|
|
urshr v15.8h, v0.8h, #8
|
|
urshr v16.8h, v1.8h, #8
|
|
urshr v17.8h, v2.8h, #8
|
|
raddhn v0.8b, v0.8h, v15.8h
|
|
raddhn v1.8b, v1.8h, v16.8h
|
|
raddhn v2.8b, v2.8h, v17.8h
|
|
urshr v19.8h, v12.8h, #8
|
|
urshr v20.8h, v13.8h, #8
|
|
urshr v21.8h, v14.8h, #8
|
|
raddhn v24.8b, v12.8h, v19.8h
|
|
raddhn v25.8b, v13.8h, v20.8h
|
|
/*
|
|
* convert 8 r5g6b5 pixel data from {v4} to planar 8-bit format
|
|
* and put data into v16 - blue, v17 - green, v18 - red
|
|
*/
|
|
mov v4.d[1], v5.d[0]
|
|
shrn v17.8b, v4.8h, #3
|
|
shrn v18.8b, v4.8h, #8
|
|
raddhn v26.8b, v14.8h, v21.8h
|
|
sli v4.8h, v4.8h, #5
|
|
sri v18.8b, v18.8b, #5
|
|
sri v17.8b, v17.8b, #6
|
|
/*
|
|
* 'combine_over_ca' replacement
|
|
*
|
|
* output: updated dest in v16 - blue, v17 - green, v18 - red
|
|
*/
|
|
mvn v24.8b, v24.8b
|
|
mvn v25.8b, v25.8b
|
|
shrn v16.8b, v4.8h, #2
|
|
mvn v26.8b, v26.8b
|
|
umull v5.8h, v16.8b, v24.8b
|
|
umull v6.8h, v17.8b, v25.8b
|
|
umull v7.8h, v18.8b, v26.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8888_0565_ca_process_pixblock_tail
|
|
/* ... continue 'combine_over_ca' replacement */
|
|
urshr v13.8h, v5.8h, #8
|
|
urshr v14.8h, v6.8h, #8
|
|
urshr v15.8h, v7.8h, #8
|
|
raddhn v16.8b, v13.8h, v5.8h
|
|
raddhn v17.8b, v14.8h, v6.8h
|
|
raddhn v18.8b, v15.8h, v7.8h
|
|
uqadd v16.8b, v0.8b, v16.8b
|
|
uqadd v17.8b, v1.8b, v17.8b
|
|
uqadd v18.8b, v2.8b, v18.8b
|
|
/*
|
|
* convert the results in v16, v17, v18 to r5g6b5 and store
|
|
* them into {v14}
|
|
*/
|
|
ushll v14.8h, v18.8b, #7
|
|
sli v14.8h, v14.8h, #1
|
|
ushll v12.8h, v17.8b, #7
|
|
sli v12.8h, v12.8h, #1
|
|
ushll v13.8h, v16.8b, #7
|
|
sli v13.8h, v13.8h, #1
|
|
sri v14.8h, v12.8h, #5
|
|
sri v14.8h, v13.8h, #11
|
|
mov v28.d[0], v14.d[0]
|
|
mov v29.d[0], v14.d[1]
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8888_0565_ca_process_pixblock_tail_head
|
|
fetch_mask_pixblock
|
|
urshr v13.8h, v5.8h, #8
|
|
urshr v14.8h, v6.8h, #8
|
|
ld1 {v4.8h}, [DST_R], #16
|
|
urshr v15.8h, v7.8h, #8
|
|
raddhn v16.8b, v13.8h, v5.8h
|
|
raddhn v17.8b, v14.8h, v6.8h
|
|
raddhn v18.8b, v15.8h, v7.8h
|
|
mov v5.d[0], v4.d[1]
|
|
/* process_pixblock_head */
|
|
/*
|
|
* 'combine_mask_ca' replacement
|
|
*
|
|
* input: solid src (n) in {v8, v9, v10, v11} [B, G, R, A]
|
|
* mask in {v24, v25, v26} [B, G, R]
|
|
* output: updated src in {v0, v1, v2 } [B, G, R]
|
|
* updated mask in {v24, v25, v26} [B, G, R]
|
|
*/
|
|
uqadd v16.8b, v0.8b, v16.8b
|
|
uqadd v17.8b, v1.8b, v17.8b
|
|
uqadd v18.8b, v2.8b, v18.8b
|
|
umull v0.8h, v24.8b, v8.8b
|
|
umull v1.8h, v25.8b, v9.8b
|
|
umull v2.8h, v26.8b, v10.8b
|
|
/*
|
|
* convert the result in v16, v17, v18 to r5g6b5 and store
|
|
* it into {v14}
|
|
*/
|
|
ushll v14.8h, v18.8b, #7
|
|
sli v14.8h, v14.8h, #1
|
|
ushll v18.8h, v16.8b, #7
|
|
sli v18.8h, v18.8h, #1
|
|
ushll v19.8h, v17.8b, #7
|
|
sli v19.8h, v19.8h, #1
|
|
umull v12.8h, v11.8b, v24.8b
|
|
sri v14.8h, v19.8h, #5
|
|
umull v13.8h, v11.8b, v25.8b
|
|
umull v15.8h, v11.8b, v26.8b
|
|
sri v14.8h, v18.8h, #11
|
|
mov v28.d[0], v14.d[0]
|
|
mov v29.d[0], v14.d[1]
|
|
cache_preload 8, 8
|
|
urshr v16.8h, v0.8h, #8
|
|
urshr v17.8h, v1.8h, #8
|
|
urshr v18.8h, v2.8h, #8
|
|
raddhn v0.8b, v0.8h, v16.8h
|
|
raddhn v1.8b, v1.8h, v17.8h
|
|
raddhn v2.8b, v2.8h, v18.8h
|
|
urshr v19.8h, v12.8h, #8
|
|
urshr v20.8h, v13.8h, #8
|
|
urshr v21.8h, v15.8h, #8
|
|
raddhn v24.8b, v12.8h, v19.8h
|
|
raddhn v25.8b, v13.8h, v20.8h
|
|
/*
|
|
* convert 8 r5g6b5 pixel data from {v4, v5} to planar
|
|
* 8-bit format and put data into v16 - blue, v17 - green,
|
|
* v18 - red
|
|
*/
|
|
mov v4.d[1], v5.d[0]
|
|
shrn v17.8b, v4.8h, #3
|
|
shrn v18.8b, v4.8h, #8
|
|
raddhn v26.8b, v15.8h, v21.8h
|
|
sli v4.8h, v4.8h, #5
|
|
sri v17.8b, v17.8b, #6
|
|
sri v18.8b, v18.8b, #5
|
|
/*
|
|
* 'combine_over_ca' replacement
|
|
*
|
|
* output: updated dest in v16 - blue, v17 - green, v18 - red
|
|
*/
|
|
mvn v24.8b, v24.8b
|
|
mvn v25.8b, v25.8b
|
|
shrn v16.8b, v4.8h, #2
|
|
mvn v26.8b, v26.8b
|
|
umull v5.8h, v16.8b, v24.8b
|
|
umull v6.8h, v17.8b, v25.8b
|
|
umull v7.8h, v18.8b, v26.8b
|
|
st1 {v14.8h}, [DST_W], #16
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8888_0565_ca_init
|
|
mov v13.s[0], w4
|
|
dup v8.8b, v13.b[0]
|
|
dup v9.8b, v13.b[1]
|
|
dup v10.8b, v13.b[2]
|
|
dup v11.8b, v13.b[3]
|
|
.endm
|
|
|
|
.macro pixman_composite_over_n_8888_0565_ca_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_over_n_8888_0565_ca_asm_neon, 0, 32, 16, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_over_n_8888_0565_ca_init, \
|
|
pixman_composite_over_n_8888_0565_ca_cleanup, \
|
|
pixman_composite_over_n_8888_0565_ca_process_pixblock_head, \
|
|
pixman_composite_over_n_8888_0565_ca_process_pixblock_tail, \
|
|
pixman_composite_over_n_8888_0565_ca_process_pixblock_tail_head
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_in_n_8_process_pixblock_head
|
|
/* expecting source data in {v0, v1, v2, v3} */
|
|
/* and destination data in {v4, v5, v6, v7} */
|
|
umull v8.8h, v4.8b, v3.8b
|
|
umull v9.8h, v5.8b, v3.8b
|
|
umull v10.8h, v6.8b, v3.8b
|
|
umull v11.8h, v7.8b, v3.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_in_n_8_process_pixblock_tail
|
|
urshr v14.8h, v8.8h, #8
|
|
urshr v15.8h, v9.8h, #8
|
|
urshr v12.8h, v10.8h, #8
|
|
urshr v13.8h, v11.8h, #8
|
|
raddhn v28.8b, v8.8h, v14.8h
|
|
raddhn v29.8b, v9.8h, v15.8h
|
|
raddhn v30.8b, v10.8h, v12.8h
|
|
raddhn v31.8b, v11.8h, v13.8h
|
|
.endm
|
|
|
|
.macro pixman_composite_in_n_8_process_pixblock_tail_head
|
|
pixman_composite_in_n_8_process_pixblock_tail
|
|
ld1 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
|
|
cache_preload 32, 32
|
|
pixman_composite_in_n_8_process_pixblock_head
|
|
st1 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
.endm
|
|
|
|
.macro pixman_composite_in_n_8_init
|
|
mov v3.s[0], w4
|
|
dup v3.8b, v3.b[3]
|
|
.endm
|
|
|
|
.macro pixman_composite_in_n_8_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_in_n_8_asm_neon, 0, 0, 8, \
|
|
FLAG_DST_READWRITE, \
|
|
32, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_in_n_8_init, \
|
|
pixman_composite_in_n_8_cleanup, \
|
|
pixman_composite_in_n_8_process_pixblock_head, \
|
|
pixman_composite_in_n_8_process_pixblock_tail, \
|
|
pixman_composite_in_n_8_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
24 /* mask_basereg */
|
|
|
|
.macro pixman_composite_add_n_8_8_process_pixblock_head
|
|
/* expecting source data in {v8, v9, v10, v11} */
|
|
/* v8 - blue, v9 - green, v10 - red, v11 - alpha */
|
|
/* and destination data in {v4, v5, v6, v7} */
|
|
/* mask is in v24, v25, v26, v27 */
|
|
umull v0.8h, v24.8b, v11.8b
|
|
umull v1.8h, v25.8b, v11.8b
|
|
umull v2.8h, v26.8b, v11.8b
|
|
umull v3.8h, v27.8b, v11.8b
|
|
urshr v12.8h, v0.8h, #8
|
|
urshr v13.8h, v1.8h, #8
|
|
urshr v14.8h, v2.8h, #8
|
|
urshr v15.8h, v3.8h, #8
|
|
raddhn v0.8b, v0.8h, v12.8h
|
|
raddhn v1.8b, v1.8h, v13.8h
|
|
raddhn v2.8b, v2.8h, v14.8h
|
|
raddhn v3.8b, v3.8h, v15.8h
|
|
uqadd v28.8b, v0.8b, v4.8b
|
|
uqadd v29.8b, v1.8b, v5.8b
|
|
uqadd v30.8b, v2.8b, v6.8b
|
|
uqadd v31.8b, v3.8b, v7.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_add_n_8_8_process_pixblock_tail
|
|
.endm
|
|
|
|
/* TODO: expand macros and do better instructions scheduling */
|
|
.macro pixman_composite_add_n_8_8_process_pixblock_tail_head
|
|
pixman_composite_add_n_8_8_process_pixblock_tail
|
|
st1 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
ld1 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
|
|
fetch_mask_pixblock
|
|
cache_preload 32, 32
|
|
pixman_composite_add_n_8_8_process_pixblock_head
|
|
.endm
|
|
|
|
.macro pixman_composite_add_n_8_8_init
|
|
mov v11.s[0], w4
|
|
dup v11.8b, v11.b[3]
|
|
.endm
|
|
|
|
.macro pixman_composite_add_n_8_8_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_add_n_8_8_asm_neon, 0, 8, 8, \
|
|
FLAG_DST_READWRITE, \
|
|
32, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_add_n_8_8_init, \
|
|
pixman_composite_add_n_8_8_cleanup, \
|
|
pixman_composite_add_n_8_8_process_pixblock_head, \
|
|
pixman_composite_add_n_8_8_process_pixblock_tail, \
|
|
pixman_composite_add_n_8_8_process_pixblock_tail_head
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_add_8_8_8_process_pixblock_head
|
|
/* expecting source data in {v0, v1, v2, v3} */
|
|
/* destination data in {v4, v5, v6, v7} */
|
|
/* mask in {v24, v25, v26, v27} */
|
|
umull v8.8h, v24.8b, v0.8b
|
|
umull v9.8h, v25.8b, v1.8b
|
|
umull v10.8h, v26.8b, v2.8b
|
|
umull v11.8h, v27.8b, v3.8b
|
|
urshr v0.8h, v8.8h, #8
|
|
urshr v1.8h, v9.8h, #8
|
|
urshr v12.8h, v10.8h, #8
|
|
urshr v13.8h, v11.8h, #8
|
|
raddhn v0.8b, v0.8h, v8.8h
|
|
raddhn v1.8b, v1.8h, v9.8h
|
|
raddhn v2.8b, v12.8h, v10.8h
|
|
raddhn v3.8b, v13.8h, v11.8h
|
|
uqadd v28.8b, v0.8b, v4.8b
|
|
uqadd v29.8b, v1.8b, v5.8b
|
|
uqadd v30.8b, v2.8b, v6.8b
|
|
uqadd v31.8b, v3.8b, v7.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_add_8_8_8_process_pixblock_tail
|
|
.endm
|
|
|
|
/* TODO: expand macros and do better instructions scheduling */
|
|
.macro pixman_composite_add_8_8_8_process_pixblock_tail_head
|
|
pixman_composite_add_8_8_8_process_pixblock_tail
|
|
st1 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
ld1 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
|
|
fetch_mask_pixblock
|
|
fetch_src_pixblock
|
|
cache_preload 32, 32
|
|
pixman_composite_add_8_8_8_process_pixblock_head
|
|
.endm
|
|
|
|
.macro pixman_composite_add_8_8_8_init
|
|
.endm
|
|
|
|
.macro pixman_composite_add_8_8_8_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_add_8_8_8_asm_neon, 8, 8, 8, \
|
|
FLAG_DST_READWRITE, \
|
|
32, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_add_8_8_8_init, \
|
|
pixman_composite_add_8_8_8_cleanup, \
|
|
pixman_composite_add_8_8_8_process_pixblock_head, \
|
|
pixman_composite_add_8_8_8_process_pixblock_tail, \
|
|
pixman_composite_add_8_8_8_process_pixblock_tail_head
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_add_8888_8888_8888_process_pixblock_head
|
|
/* expecting source data in {v0, v1, v2, v3} */
|
|
/* destination data in {v4, v5, v6, v7} */
|
|
/* mask in {v24, v25, v26, v27} */
|
|
umull v8.8h, v27.8b, v0.8b
|
|
umull v9.8h, v27.8b, v1.8b
|
|
umull v10.8h, v27.8b, v2.8b
|
|
umull v11.8h, v27.8b, v3.8b
|
|
/* 1 cycle bubble */
|
|
ursra v8.8h, v8.8h, #8
|
|
ursra v9.8h, v9.8h, #8
|
|
ursra v10.8h, v10.8h, #8
|
|
ursra v11.8h, v11.8h, #8
|
|
.endm
|
|
|
|
.macro pixman_composite_add_8888_8888_8888_process_pixblock_tail
|
|
/* 2 cycle bubble */
|
|
rshrn v28.8b, v8.8h, #8
|
|
rshrn v29.8b, v9.8h, #8
|
|
rshrn v30.8b, v10.8h, #8
|
|
rshrn v31.8b, v11.8h, #8
|
|
uqadd v28.8b, v4.8b, v28.8b
|
|
uqadd v29.8b, v5.8b, v29.8b
|
|
uqadd v30.8b, v6.8b, v30.8b
|
|
uqadd v31.8b, v7.8b, v31.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_add_8888_8888_8888_process_pixblock_tail_head
|
|
fetch_src_pixblock
|
|
rshrn v28.8b, v8.8h, #8
|
|
fetch_mask_pixblock
|
|
rshrn v29.8b, v9.8h, #8
|
|
umull v8.8h, v27.8b, v0.8b
|
|
rshrn v30.8b, v10.8h, #8
|
|
umull v9.8h, v27.8b, v1.8b
|
|
rshrn v31.8b, v11.8h, #8
|
|
umull v10.8h, v27.8b, v2.8b
|
|
umull v11.8h, v27.8b, v3.8b
|
|
uqadd v28.8b, v4.8b, v28.8b
|
|
uqadd v29.8b, v5.8b, v29.8b
|
|
uqadd v30.8b, v6.8b, v30.8b
|
|
uqadd v31.8b, v7.8b, v31.8b
|
|
ursra v8.8h, v8.8h, #8
|
|
ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
|
|
ursra v9.8h, v9.8h, #8
|
|
st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
ursra v10.8h, v10.8h, #8
|
|
|
|
cache_preload 8, 8
|
|
|
|
ursra v11.8h, v11.8h, #8
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_add_8888_8888_8888_asm_neon, 32, 32, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
10, /* prefetch distance */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_add_8888_8888_8888_process_pixblock_head, \
|
|
pixman_composite_add_8888_8888_8888_process_pixblock_tail, \
|
|
pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
24 /* mask_basereg */
|
|
|
|
generate_composite_function_single_scanline \
|
|
pixman_composite_scanline_add_mask_asm_neon, 32, 32, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_add_8888_8888_8888_process_pixblock_head, \
|
|
pixman_composite_add_8888_8888_8888_process_pixblock_tail, \
|
|
pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
24 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
generate_composite_function \
|
|
pixman_composite_add_8888_8_8888_asm_neon, 32, 8, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_add_8888_8888_8888_process_pixblock_head, \
|
|
pixman_composite_add_8888_8888_8888_process_pixblock_tail, \
|
|
pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
27 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_add_n_8_8888_init
|
|
mov v3.s[0], w4
|
|
dup v0.8b, v3.b[0]
|
|
dup v1.8b, v3.b[1]
|
|
dup v2.8b, v3.b[2]
|
|
dup v3.8b, v3.b[3]
|
|
.endm
|
|
|
|
.macro pixman_composite_add_n_8_8888_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_add_n_8_8888_asm_neon, 0, 8, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_add_n_8_8888_init, \
|
|
pixman_composite_add_n_8_8888_cleanup, \
|
|
pixman_composite_add_8888_8888_8888_process_pixblock_head, \
|
|
pixman_composite_add_8888_8888_8888_process_pixblock_tail, \
|
|
pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
27 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_add_8888_n_8888_init
|
|
mov v27.s[0], w6
|
|
dup v27.8b, v27.b[3]
|
|
.endm
|
|
|
|
.macro pixman_composite_add_8888_n_8888_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_add_8888_n_8888_asm_neon, 32, 0, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_add_8888_n_8888_init, \
|
|
pixman_composite_add_8888_n_8888_cleanup, \
|
|
pixman_composite_add_8888_8888_8888_process_pixblock_head, \
|
|
pixman_composite_add_8888_8888_8888_process_pixblock_tail, \
|
|
pixman_composite_add_8888_8888_8888_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
27 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_out_reverse_8888_n_8888_process_pixblock_head
|
|
/* expecting source data in {v0, v1, v2, v3} */
|
|
/* destination data in {v4, v5, v6, v7} */
|
|
/* solid mask is in v15 */
|
|
|
|
/* 'in' */
|
|
umull v11.8h, v15.8b, v3.8b
|
|
umull v10.8h, v15.8b, v2.8b
|
|
umull v9.8h, v15.8b, v1.8b
|
|
umull v8.8h, v15.8b, v0.8b
|
|
urshr v16.8h, v11.8h, #8
|
|
urshr v14.8h, v10.8h, #8
|
|
urshr v13.8h, v9.8h, #8
|
|
urshr v12.8h, v8.8h, #8
|
|
raddhn v3.8b, v11.8h, v16.8h
|
|
raddhn v2.8b, v10.8h, v14.8h
|
|
raddhn v1.8b, v9.8h, v13.8h
|
|
raddhn v0.8b, v8.8h, v12.8h
|
|
mvn v24.8b, v3.8b /* get inverted alpha */
|
|
/* now do alpha blending */
|
|
umull v8.8h, v24.8b, v4.8b
|
|
umull v9.8h, v24.8b, v5.8b
|
|
umull v10.8h, v24.8b, v6.8b
|
|
umull v11.8h, v24.8b, v7.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail
|
|
urshr v16.8h, v8.8h, #8
|
|
urshr v17.8h, v9.8h, #8
|
|
urshr v18.8h, v10.8h, #8
|
|
urshr v19.8h, v11.8h, #8
|
|
raddhn v28.8b, v16.8h, v8.8h
|
|
raddhn v29.8b, v17.8h, v9.8h
|
|
raddhn v30.8b, v18.8h, v10.8h
|
|
raddhn v31.8b, v19.8h, v11.8h
|
|
.endm
|
|
|
|
/* TODO: expand macros and do better instructions scheduling */
|
|
.macro pixman_composite_out_reverse_8888_8888_8888_process_pixblock_tail_head
|
|
ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
|
|
pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail
|
|
fetch_src_pixblock
|
|
cache_preload 8, 8
|
|
fetch_mask_pixblock
|
|
pixman_composite_out_reverse_8888_n_8888_process_pixblock_head
|
|
st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
.endm
|
|
|
|
generate_composite_function_single_scanline \
|
|
pixman_composite_scanline_out_reverse_mask_asm_neon, 32, 32, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
default_init_need_all_regs, \
|
|
default_cleanup_need_all_regs, \
|
|
pixman_composite_out_reverse_8888_n_8888_process_pixblock_head, \
|
|
pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail, \
|
|
pixman_composite_out_reverse_8888_8888_8888_process_pixblock_tail_head \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
12 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_over_8888_n_8888_process_pixblock_head
|
|
pixman_composite_out_reverse_8888_n_8888_process_pixblock_head
|
|
.endm
|
|
|
|
.macro pixman_composite_over_8888_n_8888_process_pixblock_tail
|
|
pixman_composite_out_reverse_8888_n_8888_process_pixblock_tail
|
|
uqadd v28.8b, v0.8b, v28.8b
|
|
uqadd v29.8b, v1.8b, v29.8b
|
|
uqadd v30.8b, v2.8b, v30.8b
|
|
uqadd v31.8b, v3.8b, v31.8b
|
|
.endm
|
|
|
|
/* TODO: expand macros and do better instructions scheduling */
|
|
.macro pixman_composite_over_8888_n_8888_process_pixblock_tail_head
|
|
ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
|
|
pixman_composite_over_8888_n_8888_process_pixblock_tail
|
|
fetch_src_pixblock
|
|
cache_preload 8, 8
|
|
pixman_composite_over_8888_n_8888_process_pixblock_head
|
|
st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
.endm
|
|
|
|
.macro pixman_composite_over_8888_n_8888_init
|
|
mov v15.s[0], w6
|
|
dup v15.8b, v15.b[3]
|
|
.endm
|
|
|
|
.macro pixman_composite_over_8888_n_8888_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_over_8888_n_8888_asm_neon, 32, 0, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_over_8888_n_8888_init, \
|
|
pixman_composite_over_8888_n_8888_cleanup, \
|
|
pixman_composite_over_8888_n_8888_process_pixblock_head, \
|
|
pixman_composite_over_8888_n_8888_process_pixblock_tail, \
|
|
pixman_composite_over_8888_n_8888_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
12 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
/* TODO: expand macros and do better instructions scheduling */
|
|
.macro pixman_composite_over_8888_8888_8888_process_pixblock_tail_head
|
|
ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
|
|
pixman_composite_over_8888_n_8888_process_pixblock_tail
|
|
fetch_src_pixblock
|
|
cache_preload 8, 8
|
|
fetch_mask_pixblock
|
|
pixman_composite_over_8888_n_8888_process_pixblock_head
|
|
st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_over_8888_8888_8888_asm_neon, 32, 32, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
default_init_need_all_regs, \
|
|
default_cleanup_need_all_regs, \
|
|
pixman_composite_over_8888_n_8888_process_pixblock_head, \
|
|
pixman_composite_over_8888_n_8888_process_pixblock_tail, \
|
|
pixman_composite_over_8888_8888_8888_process_pixblock_tail_head \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
12 /* mask_basereg */
|
|
|
|
generate_composite_function_single_scanline \
|
|
pixman_composite_scanline_over_mask_asm_neon, 32, 32, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
default_init_need_all_regs, \
|
|
default_cleanup_need_all_regs, \
|
|
pixman_composite_over_8888_n_8888_process_pixblock_head, \
|
|
pixman_composite_over_8888_n_8888_process_pixblock_tail, \
|
|
pixman_composite_over_8888_8888_8888_process_pixblock_tail_head \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
12 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
/* TODO: expand macros and do better instructions scheduling */
|
|
.macro pixman_composite_over_8888_8_8888_process_pixblock_tail_head
|
|
ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
|
|
pixman_composite_over_8888_n_8888_process_pixblock_tail
|
|
fetch_src_pixblock
|
|
cache_preload 8, 8
|
|
fetch_mask_pixblock
|
|
pixman_composite_over_8888_n_8888_process_pixblock_head
|
|
st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_over_8888_8_8888_asm_neon, 32, 8, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
default_init_need_all_regs, \
|
|
default_cleanup_need_all_regs, \
|
|
pixman_composite_over_8888_n_8888_process_pixblock_head, \
|
|
pixman_composite_over_8888_n_8888_process_pixblock_tail, \
|
|
pixman_composite_over_8888_8_8888_process_pixblock_tail_head \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
15 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_src_0888_0888_process_pixblock_head
|
|
.endm
|
|
|
|
.macro pixman_composite_src_0888_0888_process_pixblock_tail
|
|
.endm
|
|
|
|
.macro pixman_composite_src_0888_0888_process_pixblock_tail_head
|
|
st3 {v0.8b, v1.8b, v2.8b}, [DST_W], #24
|
|
fetch_src_pixblock
|
|
cache_preload 8, 8
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_src_0888_0888_asm_neon, 24, 0, 24, \
|
|
FLAG_DST_WRITEONLY, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
10, /* prefetch distance */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_src_0888_0888_process_pixblock_head, \
|
|
pixman_composite_src_0888_0888_process_pixblock_tail, \
|
|
pixman_composite_src_0888_0888_process_pixblock_tail_head, \
|
|
0, /* dst_w_basereg */ \
|
|
0, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
0 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_src_0888_8888_rev_process_pixblock_head
|
|
mov v31.8b, v2.8b
|
|
mov v2.8b, v0.8b
|
|
mov v0.8b, v31.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_src_0888_8888_rev_process_pixblock_tail
|
|
.endm
|
|
|
|
.macro pixman_composite_src_0888_8888_rev_process_pixblock_tail_head
|
|
st4 {v0.8b, v1.8b, v2.8b, v3.8b}, [DST_W], #32
|
|
fetch_src_pixblock
|
|
mov v31.8b, v2.8b
|
|
mov v2.8b, v0.8b
|
|
mov v0.8b, v31.8b
|
|
cache_preload 8, 8
|
|
.endm
|
|
|
|
.macro pixman_composite_src_0888_8888_rev_init
|
|
eor v3.8b, v3.8b, v3.8b
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_src_0888_8888_rev_asm_neon, 24, 0, 32, \
|
|
FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
10, /* prefetch distance */ \
|
|
pixman_composite_src_0888_8888_rev_init, \
|
|
default_cleanup, \
|
|
pixman_composite_src_0888_8888_rev_process_pixblock_head, \
|
|
pixman_composite_src_0888_8888_rev_process_pixblock_tail, \
|
|
pixman_composite_src_0888_8888_rev_process_pixblock_tail_head, \
|
|
0, /* dst_w_basereg */ \
|
|
0, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
0 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_src_0888_0565_rev_process_pixblock_head
|
|
ushll v8.8h, v1.8b, #7
|
|
sli v8.8h, v8.8h, #1
|
|
ushll v9.8h, v2.8b, #7
|
|
sli v9.8h, v9.8h, #1
|
|
.endm
|
|
|
|
.macro pixman_composite_src_0888_0565_rev_process_pixblock_tail
|
|
ushll v14.8h, v0.8b, #7
|
|
sli v14.8h, v14.8h, #1
|
|
sri v14.8h, v8.8h, #5
|
|
sri v14.8h, v9.8h, #11
|
|
mov v28.d[0], v14.d[0]
|
|
mov v29.d[0], v14.d[1]
|
|
.endm
|
|
|
|
.macro pixman_composite_src_0888_0565_rev_process_pixblock_tail_head
|
|
ushll v14.8h, v0.8b, #7
|
|
sli v14.8h, v14.8h, #1
|
|
fetch_src_pixblock
|
|
sri v14.8h, v8.8h, #5
|
|
sri v14.8h, v9.8h, #11
|
|
mov v28.d[0], v14.d[0]
|
|
mov v29.d[0], v14.d[1]
|
|
ushll v8.8h, v1.8b, #7
|
|
sli v8.8h, v8.8h, #1
|
|
st1 {v14.8h}, [DST_W], #16
|
|
ushll v9.8h, v2.8b, #7
|
|
sli v9.8h, v9.8h, #1
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_src_0888_0565_rev_asm_neon, 24, 0, 16, \
|
|
FLAG_DST_WRITEONLY, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
10, /* prefetch distance */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_src_0888_0565_rev_process_pixblock_head, \
|
|
pixman_composite_src_0888_0565_rev_process_pixblock_tail, \
|
|
pixman_composite_src_0888_0565_rev_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
0, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
0 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_src_pixbuf_8888_process_pixblock_head
|
|
umull v8.8h, v3.8b, v0.8b
|
|
umull v9.8h, v3.8b, v1.8b
|
|
umull v10.8h, v3.8b, v2.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_src_pixbuf_8888_process_pixblock_tail
|
|
urshr v11.8h, v8.8h, #8
|
|
mov v30.8b, v31.8b
|
|
mov v31.8b, v3.8b
|
|
mov v3.8b, v30.8b
|
|
urshr v12.8h, v9.8h, #8
|
|
urshr v13.8h, v10.8h, #8
|
|
raddhn v30.8b, v11.8h, v8.8h
|
|
raddhn v29.8b, v12.8h, v9.8h
|
|
raddhn v28.8b, v13.8h, v10.8h
|
|
.endm
|
|
|
|
.macro pixman_composite_src_pixbuf_8888_process_pixblock_tail_head
|
|
urshr v11.8h, v8.8h, #8
|
|
mov v30.8b, v31.8b
|
|
mov v31.8b, v3.8b
|
|
mov v3.8b, v31.8b
|
|
urshr v12.8h, v9.8h, #8
|
|
urshr v13.8h, v10.8h, #8
|
|
fetch_src_pixblock
|
|
raddhn v30.8b, v11.8h, v8.8h
|
|
PF add PF_X, PF_X, #8
|
|
PF tst PF_CTL, #0xF
|
|
PF beq 10f
|
|
PF add PF_X, PF_X, #8
|
|
PF sub PF_CTL, PF_CTL, #1
|
|
10:
|
|
raddhn v29.8b, v12.8h, v9.8h
|
|
raddhn v28.8b, v13.8h, v10.8h
|
|
umull v8.8h, v3.8b, v0.8b
|
|
umull v9.8h, v3.8b, v1.8b
|
|
umull v10.8h, v3.8b, v2.8b
|
|
st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
PF cmp PF_X, ORIG_W
|
|
PF lsl DUMMY, PF_X, src_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_SRC, DUMMY]
|
|
PF ble 10f
|
|
PF sub PF_X, PF_X, ORIG_W
|
|
PF subs PF_CTL, PF_CTL, #0x10
|
|
PF ble 10f
|
|
PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift
|
|
PF ldrsb DUMMY, [PF_SRC, DUMMY]
|
|
PF add PF_SRC, PF_SRC, #1
|
|
10:
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_src_pixbuf_8888_asm_neon, 32, 0, 32, \
|
|
FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
10, /* prefetch distance */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_src_pixbuf_8888_process_pixblock_head, \
|
|
pixman_composite_src_pixbuf_8888_process_pixblock_tail, \
|
|
pixman_composite_src_pixbuf_8888_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
0, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
0 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_src_rpixbuf_8888_process_pixblock_head
|
|
umull v8.8h, v3.8b, v0.8b
|
|
umull v9.8h, v3.8b, v1.8b
|
|
umull v10.8h, v3.8b, v2.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_src_rpixbuf_8888_process_pixblock_tail
|
|
urshr v11.8h, v8.8h, #8
|
|
mov v30.8b, v31.8b
|
|
mov v31.8b, v3.8b
|
|
mov v3.8b, v30.8b
|
|
urshr v12.8h, v9.8h, #8
|
|
urshr v13.8h, v10.8h, #8
|
|
raddhn v28.8b, v11.8h, v8.8h
|
|
raddhn v29.8b, v12.8h, v9.8h
|
|
raddhn v30.8b, v13.8h, v10.8h
|
|
.endm
|
|
|
|
.macro pixman_composite_src_rpixbuf_8888_process_pixblock_tail_head
|
|
urshr v11.8h, v8.8h, #8
|
|
mov v30.8b, v31.8b
|
|
mov v31.8b, v3.8b
|
|
mov v3.8b, v30.8b
|
|
urshr v12.8h, v9.8h, #8
|
|
urshr v13.8h, v10.8h, #8
|
|
fetch_src_pixblock
|
|
raddhn v28.8b, v11.8h, v8.8h
|
|
PF add PF_X, PF_X, #8
|
|
PF tst PF_CTL, #0xF
|
|
PF beq 10f
|
|
PF add PF_X, PF_X, #8
|
|
PF sub PF_CTL, PF_CTL, #1
|
|
10:
|
|
raddhn v29.8b, v12.8h, v9.8h
|
|
raddhn v30.8b, v13.8h, v10.8h
|
|
umull v8.8h, v3.8b, v0.8b
|
|
umull v9.8h, v3.8b, v1.8b
|
|
umull v10.8h, v3.8b, v2.8b
|
|
st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
PF cmp PF_X, ORIG_W
|
|
PF lsl DUMMY, PF_X, src_bpp_shift
|
|
PF prfm PREFETCH_MODE, [PF_SRC, DUMMY]
|
|
PF ble 10f
|
|
PF sub PF_X, PF_X, ORIG_W
|
|
PF subs PF_CTL, PF_CTL, #0x10
|
|
PF ble 10f
|
|
PF lsl DUMMY, SRC_STRIDE, #src_bpp_shift
|
|
PF ldrsb DUMMY, [PF_SRC, DUMMY]
|
|
PF add PF_SRC, PF_SRC, #1
|
|
10:
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_src_rpixbuf_8888_asm_neon, 32, 0, 32, \
|
|
FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
10, /* prefetch distance */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_src_rpixbuf_8888_process_pixblock_head, \
|
|
pixman_composite_src_rpixbuf_8888_process_pixblock_tail, \
|
|
pixman_composite_src_rpixbuf_8888_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
0, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
0 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_over_0565_8_0565_process_pixblock_head
|
|
/* mask is in v15 */
|
|
mov v4.d[0], v8.d[0]
|
|
mov v4.d[1], v9.d[0]
|
|
mov v13.d[0], v10.d[0]
|
|
mov v13.d[1], v11.d[0]
|
|
convert_0565_to_x888 v4, v2, v1, v0
|
|
convert_0565_to_x888 v13, v6, v5, v4
|
|
/* source pixel data is in {v0, v1, v2, XX} */
|
|
/* destination pixel data is in {v4, v5, v6, XX} */
|
|
mvn v7.8b, v15.8b
|
|
umull v10.8h, v15.8b, v2.8b
|
|
umull v9.8h, v15.8b, v1.8b
|
|
umull v8.8h, v15.8b, v0.8b
|
|
umull v11.8h, v7.8b, v4.8b
|
|
umull v12.8h, v7.8b, v5.8b
|
|
umull v13.8h, v7.8b, v6.8b
|
|
urshr v19.8h, v10.8h, #8
|
|
urshr v18.8h, v9.8h, #8
|
|
urshr v17.8h, v8.8h, #8
|
|
raddhn v2.8b, v10.8h, v19.8h
|
|
raddhn v1.8b, v9.8h, v18.8h
|
|
raddhn v0.8b, v8.8h, v17.8h
|
|
.endm
|
|
|
|
.macro pixman_composite_over_0565_8_0565_process_pixblock_tail
|
|
urshr v17.8h, v11.8h, #8
|
|
urshr v18.8h, v12.8h, #8
|
|
urshr v19.8h, v13.8h, #8
|
|
raddhn v28.8b, v17.8h, v11.8h
|
|
raddhn v29.8b, v18.8h, v12.8h
|
|
raddhn v30.8b, v19.8h, v13.8h
|
|
uqadd v0.8b, v0.8b, v28.8b
|
|
uqadd v1.8b, v1.8b, v29.8b
|
|
uqadd v2.8b, v2.8b, v30.8b
|
|
/* 32bpp result is in {v0, v1, v2, XX} */
|
|
convert_8888_to_0565 v2, v1, v0, v14, v30, v13
|
|
mov v28.d[0], v14.d[0]
|
|
mov v29.d[0], v14.d[1]
|
|
.endm
|
|
|
|
/* TODO: expand macros and do better instructions scheduling */
|
|
.macro pixman_composite_over_0565_8_0565_process_pixblock_tail_head
|
|
fetch_mask_pixblock
|
|
pixman_composite_over_0565_8_0565_process_pixblock_tail
|
|
fetch_src_pixblock
|
|
ld1 {v10.4h, v11.4h}, [DST_R], #16
|
|
cache_preload 8, 8
|
|
pixman_composite_over_0565_8_0565_process_pixblock_head
|
|
st1 {v14.8h}, [DST_W], #16
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_over_0565_8_0565_asm_neon, 16, 8, 16, \
|
|
FLAG_DST_READWRITE, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
default_init_need_all_regs, \
|
|
default_cleanup_need_all_regs, \
|
|
pixman_composite_over_0565_8_0565_process_pixblock_head, \
|
|
pixman_composite_over_0565_8_0565_process_pixblock_tail, \
|
|
pixman_composite_over_0565_8_0565_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
10, /* dst_r_basereg */ \
|
|
8, /* src_basereg */ \
|
|
15 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_over_0565_n_0565_init
|
|
mov v15.s[0], w6
|
|
dup v15.8b, v15.b[3]
|
|
.endm
|
|
|
|
.macro pixman_composite_over_0565_n_0565_cleanup
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_over_0565_n_0565_asm_neon, 16, 0, 16, \
|
|
FLAG_DST_READWRITE, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
pixman_composite_over_0565_n_0565_init, \
|
|
pixman_composite_over_0565_n_0565_cleanup, \
|
|
pixman_composite_over_0565_8_0565_process_pixblock_head, \
|
|
pixman_composite_over_0565_8_0565_process_pixblock_tail, \
|
|
pixman_composite_over_0565_8_0565_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
10, /* dst_r_basereg */ \
|
|
8, /* src_basereg */ \
|
|
15 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_add_0565_8_0565_process_pixblock_head
|
|
/* mask is in v15 */
|
|
mov v4.d[0], v8.d[0]
|
|
mov v4.d[1], v9.d[0]
|
|
mov v13.d[0], v10.d[0]
|
|
mov v13.d[1], v11.d[0]
|
|
convert_0565_to_x888 v4, v2, v1, v0
|
|
convert_0565_to_x888 v13, v6, v5, v4
|
|
/* source pixel data is in {v0, v1, v2, XX} */
|
|
/* destination pixel data is in {v4, v5, v6, XX} */
|
|
umull v9.8h, v15.8b, v2.8b
|
|
umull v8.8h, v15.8b, v1.8b
|
|
umull v7.8h, v15.8b, v0.8b
|
|
urshr v12.8h, v9.8h, #8
|
|
urshr v11.8h, v8.8h, #8
|
|
urshr v10.8h, v7.8h, #8
|
|
raddhn v2.8b, v9.8h, v12.8h
|
|
raddhn v1.8b, v8.8h, v11.8h
|
|
raddhn v0.8b, v7.8h, v10.8h
|
|
.endm
|
|
|
|
.macro pixman_composite_add_0565_8_0565_process_pixblock_tail
|
|
uqadd v0.8b, v0.8b, v4.8b
|
|
uqadd v1.8b, v1.8b, v5.8b
|
|
uqadd v2.8b, v2.8b, v6.8b
|
|
/* 32bpp result is in {v0, v1, v2, XX} */
|
|
convert_8888_to_0565 v2, v1, v0, v14, v30, v13
|
|
mov v28.d[0], v14.d[0]
|
|
mov v29.d[0], v14.d[1]
|
|
.endm
|
|
|
|
/* TODO: expand macros and do better instructions scheduling */
|
|
.macro pixman_composite_add_0565_8_0565_process_pixblock_tail_head
|
|
fetch_mask_pixblock
|
|
pixman_composite_add_0565_8_0565_process_pixblock_tail
|
|
fetch_src_pixblock
|
|
ld1 {v10.4h, v11.4h}, [DST_R], #16
|
|
cache_preload 8, 8
|
|
pixman_composite_add_0565_8_0565_process_pixblock_head
|
|
st1 {v14.8h}, [DST_W], #16
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_add_0565_8_0565_asm_neon, 16, 8, 16, \
|
|
FLAG_DST_READWRITE, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
default_init_need_all_regs, \
|
|
default_cleanup_need_all_regs, \
|
|
pixman_composite_add_0565_8_0565_process_pixblock_head, \
|
|
pixman_composite_add_0565_8_0565_process_pixblock_tail, \
|
|
pixman_composite_add_0565_8_0565_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
10, /* dst_r_basereg */ \
|
|
8, /* src_basereg */ \
|
|
15 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_out_reverse_8_0565_process_pixblock_head
|
|
/* mask is in v15 */
|
|
mov v12.d[0], v10.d[0]
|
|
mov v12.d[1], v11.d[0]
|
|
convert_0565_to_x888 v12, v6, v5, v4
|
|
/* destination pixel data is in {v4, v5, v6, xx} */
|
|
mvn v24.8b, v15.8b /* get inverted alpha */
|
|
/* now do alpha blending */
|
|
umull v8.8h, v24.8b, v4.8b
|
|
umull v9.8h, v24.8b, v5.8b
|
|
umull v10.8h, v24.8b, v6.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_out_reverse_8_0565_process_pixblock_tail
|
|
urshr v11.8h, v8.8h, #8
|
|
urshr v12.8h, v9.8h, #8
|
|
urshr v13.8h, v10.8h, #8
|
|
raddhn v0.8b, v11.8h, v8.8h
|
|
raddhn v1.8b, v12.8h, v9.8h
|
|
raddhn v2.8b, v13.8h, v10.8h
|
|
/* 32bpp result is in {v0, v1, v2, XX} */
|
|
convert_8888_to_0565 v2, v1, v0, v14, v12, v3
|
|
mov v28.d[0], v14.d[0]
|
|
mov v29.d[0], v14.d[1]
|
|
.endm
|
|
|
|
/* TODO: expand macros and do better instructions scheduling */
|
|
.macro pixman_composite_out_reverse_8_0565_process_pixblock_tail_head
|
|
fetch_src_pixblock
|
|
pixman_composite_out_reverse_8_0565_process_pixblock_tail
|
|
ld1 {v10.4h, v11.4h}, [DST_R], #16
|
|
cache_preload 8, 8
|
|
pixman_composite_out_reverse_8_0565_process_pixblock_head
|
|
st1 {v14.8h}, [DST_W], #16
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_out_reverse_8_0565_asm_neon, 8, 0, 16, \
|
|
FLAG_DST_READWRITE, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
default_init_need_all_regs, \
|
|
default_cleanup_need_all_regs, \
|
|
pixman_composite_out_reverse_8_0565_process_pixblock_head, \
|
|
pixman_composite_out_reverse_8_0565_process_pixblock_tail, \
|
|
pixman_composite_out_reverse_8_0565_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
10, /* dst_r_basereg */ \
|
|
15, /* src_basereg */ \
|
|
0 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
.macro pixman_composite_out_reverse_8_8888_process_pixblock_head
|
|
/* src is in v0 */
|
|
/* destination pixel data is in {v4, v5, v6, v7} */
|
|
mvn v1.8b, v0.8b /* get inverted alpha */
|
|
/* now do alpha blending */
|
|
umull v8.8h, v1.8b, v4.8b
|
|
umull v9.8h, v1.8b, v5.8b
|
|
umull v10.8h, v1.8b, v6.8b
|
|
umull v11.8h, v1.8b, v7.8b
|
|
.endm
|
|
|
|
.macro pixman_composite_out_reverse_8_8888_process_pixblock_tail
|
|
urshr v14.8h, v8.8h, #8
|
|
urshr v15.8h, v9.8h, #8
|
|
urshr v12.8h, v10.8h, #8
|
|
urshr v13.8h, v11.8h, #8
|
|
raddhn v28.8b, v14.8h, v8.8h
|
|
raddhn v29.8b, v15.8h, v9.8h
|
|
raddhn v30.8b, v12.8h, v10.8h
|
|
raddhn v31.8b, v13.8h, v11.8h
|
|
/* 32bpp result is in {v28, v29, v30, v31} */
|
|
.endm
|
|
|
|
/* TODO: expand macros and do better instructions scheduling */
|
|
.macro pixman_composite_out_reverse_8_8888_process_pixblock_tail_head
|
|
fetch_src_pixblock
|
|
pixman_composite_out_reverse_8_8888_process_pixblock_tail
|
|
ld4 {v4.8b, v5.8b, v6.8b, v7.8b}, [DST_R], #32
|
|
cache_preload 8, 8
|
|
pixman_composite_out_reverse_8_8888_process_pixblock_head
|
|
st4 {v28.8b, v29.8b, v30.8b, v31.8b}, [DST_W], #32
|
|
.endm
|
|
|
|
generate_composite_function \
|
|
pixman_composite_out_reverse_8_8888_asm_neon, 8, 0, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
5, /* prefetch distance */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_out_reverse_8_8888_process_pixblock_head, \
|
|
pixman_composite_out_reverse_8_8888_process_pixblock_tail, \
|
|
pixman_composite_out_reverse_8_8888_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
0 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
generate_composite_function_nearest_scanline \
|
|
pixman_scaled_nearest_scanline_8888_8888_OVER_asm_neon, 32, 0, 32, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_over_8888_8888_process_pixblock_head, \
|
|
pixman_composite_over_8888_8888_process_pixblock_tail, \
|
|
pixman_composite_over_8888_8888_process_pixblock_tail_head
|
|
|
|
generate_composite_function_nearest_scanline \
|
|
pixman_scaled_nearest_scanline_8888_0565_OVER_asm_neon, 32, 0, 16, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_over_8888_0565_process_pixblock_head, \
|
|
pixman_composite_over_8888_0565_process_pixblock_tail, \
|
|
pixman_composite_over_8888_0565_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
0, /* src_basereg */ \
|
|
24 /* mask_basereg */
|
|
|
|
generate_composite_function_nearest_scanline \
|
|
pixman_scaled_nearest_scanline_8888_0565_SRC_asm_neon, 32, 0, 16, \
|
|
FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_src_8888_0565_process_pixblock_head, \
|
|
pixman_composite_src_8888_0565_process_pixblock_tail, \
|
|
pixman_composite_src_8888_0565_process_pixblock_tail_head
|
|
|
|
generate_composite_function_nearest_scanline \
|
|
pixman_scaled_nearest_scanline_0565_8888_SRC_asm_neon, 16, 0, 32, \
|
|
FLAG_DST_WRITEONLY | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
default_init, \
|
|
default_cleanup, \
|
|
pixman_composite_src_0565_8888_process_pixblock_head, \
|
|
pixman_composite_src_0565_8888_process_pixblock_tail, \
|
|
pixman_composite_src_0565_8888_process_pixblock_tail_head
|
|
|
|
generate_composite_function_nearest_scanline \
|
|
pixman_scaled_nearest_scanline_8888_8_0565_OVER_asm_neon, 32, 8, 16, \
|
|
FLAG_DST_READWRITE | FLAG_DEINTERLEAVE_32BPP, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
default_init_need_all_regs, \
|
|
default_cleanup_need_all_regs, \
|
|
pixman_composite_over_8888_8_0565_process_pixblock_head, \
|
|
pixman_composite_over_8888_8_0565_process_pixblock_tail, \
|
|
pixman_composite_over_8888_8_0565_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
4, /* dst_r_basereg */ \
|
|
8, /* src_basereg */ \
|
|
24 /* mask_basereg */
|
|
|
|
generate_composite_function_nearest_scanline \
|
|
pixman_scaled_nearest_scanline_0565_8_0565_OVER_asm_neon, 16, 8, 16, \
|
|
FLAG_DST_READWRITE, \
|
|
8, /* number of pixels, processed in a single block */ \
|
|
default_init_need_all_regs, \
|
|
default_cleanup_need_all_regs, \
|
|
pixman_composite_over_0565_8_0565_process_pixblock_head, \
|
|
pixman_composite_over_0565_8_0565_process_pixblock_tail, \
|
|
pixman_composite_over_0565_8_0565_process_pixblock_tail_head, \
|
|
28, /* dst_w_basereg */ \
|
|
10, /* dst_r_basereg */ \
|
|
8, /* src_basereg */ \
|
|
15 /* mask_basereg */
|
|
|
|
/******************************************************************************/
|
|
|
|
/*
|
|
* Bilinear scaling support code which tries to provide pixel fetching, color
|
|
* format conversion, and interpolation as separate macros which can be used
|
|
* as the basic building blocks for constructing bilinear scanline functions.
|
|
*/
|
|
|
|
.macro bilinear_load_8888 reg1, reg2, tmp
|
|
asr TMP1, X, #16
|
|
add X, X, UX
|
|
add TMP1, TOP, TMP1, lsl #2
|
|
ld1 {®1&.2s}, [TMP1], STRIDE
|
|
ld1 {®2&.2s}, [TMP1]
|
|
.endm
|
|
|
|
.macro bilinear_load_0565 reg1, reg2, tmp
|
|
asr TMP1, X, #16
|
|
add X, X, UX
|
|
add TMP1, TOP, TMP1, lsl #1
|
|
ld1 {®2&.s}[0], [TMP1], STRIDE
|
|
ld1 {®2&.s}[1], [TMP1]
|
|
convert_four_0565_to_x888_packed reg2, reg1, reg2, tmp
|
|
.endm
|
|
|
|
.macro bilinear_load_and_vertical_interpolate_two_8888 \
|
|
acc1, acc2, reg1, reg2, reg3, reg4, tmp1, tmp2
|
|
|
|
bilinear_load_8888 reg1, reg2, tmp1
|
|
umull &acc1&.8h, ®1&.8b, v28.8b
|
|
umlal &acc1&.8h, ®2&.8b, v29.8b
|
|
bilinear_load_8888 reg3, reg4, tmp2
|
|
umull &acc2&.8h, ®3&.8b, v28.8b
|
|
umlal &acc2&.8h, ®4&.8b, v29.8b
|
|
.endm
|
|
|
|
.macro bilinear_load_and_vertical_interpolate_four_8888 \
|
|
xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi \
|
|
yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi
|
|
|
|
bilinear_load_and_vertical_interpolate_two_8888 \
|
|
xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi
|
|
bilinear_load_and_vertical_interpolate_two_8888 \
|
|
yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi
|
|
.endm
|
|
|
|
.macro vzip reg1, reg2
|
|
umov TMP4, v31.d[0]
|
|
zip1 v31.8b, reg1, reg2
|
|
zip2 reg2, reg1, reg2
|
|
mov reg1, v31.8b
|
|
mov v31.d[0], TMP4
|
|
.endm
|
|
|
|
.macro vuzp reg1, reg2
|
|
umov TMP4, v31.d[0]
|
|
uzp1 v31.8b, reg1, reg2
|
|
uzp2 reg2, reg1, reg2
|
|
mov reg1, v31.8b
|
|
mov v31.d[0], TMP4
|
|
.endm
|
|
|
|
.macro bilinear_load_and_vertical_interpolate_two_0565 \
|
|
acc1, acc2, reg1, reg2, reg3, reg4, acc2lo, acc2hi
|
|
asr TMP1, X, #16
|
|
add X, X, UX
|
|
add TMP1, TOP, TMP1, lsl #1
|
|
asr TMP2, X, #16
|
|
add X, X, UX
|
|
add TMP2, TOP, TMP2, lsl #1
|
|
ld1 {&acc2&.s}[0], [TMP1], STRIDE
|
|
ld1 {&acc2&.s}[2], [TMP2], STRIDE
|
|
ld1 {&acc2&.s}[1], [TMP1]
|
|
ld1 {&acc2&.s}[3], [TMP2]
|
|
convert_0565_to_x888 acc2, reg3, reg2, reg1
|
|
vzip ®1&.8b, ®3&.8b
|
|
vzip ®2&.8b, ®4&.8b
|
|
vzip ®3&.8b, ®4&.8b
|
|
vzip ®1&.8b, ®2&.8b
|
|
umull &acc1&.8h, ®1&.8b, v28.8b
|
|
umlal &acc1&.8h, ®2&.8b, v29.8b
|
|
umull &acc2&.8h, ®3&.8b, v28.8b
|
|
umlal &acc2&.8h, ®4&.8b, v29.8b
|
|
.endm
|
|
|
|
.macro bilinear_load_and_vertical_interpolate_four_0565 \
|
|
xacc1, xacc2, xreg1, xreg2, xreg3, xreg4, xacc2lo, xacc2hi \
|
|
yacc1, yacc2, yreg1, yreg2, yreg3, yreg4, yacc2lo, yacc2hi
|
|
asr TMP1, X, #16
|
|
add X, X, UX
|
|
add TMP1, TOP, TMP1, lsl #1
|
|
asr TMP2, X, #16
|
|
add X, X, UX
|
|
add TMP2, TOP, TMP2, lsl #1
|
|
ld1 {&xacc2&.s}[0], [TMP1], STRIDE
|
|
ld1 {&xacc2&.s}[2], [TMP2], STRIDE
|
|
ld1 {&xacc2&.s}[1], [TMP1]
|
|
ld1 {&xacc2&.s}[3], [TMP2]
|
|
convert_0565_to_x888 xacc2, xreg3, xreg2, xreg1
|
|
asr TMP1, X, #16
|
|
add X, X, UX
|
|
add TMP1, TOP, TMP1, lsl #1
|
|
asr TMP2, X, #16
|
|
add X, X, UX
|
|
add TMP2, TOP, TMP2, lsl #1
|
|
ld1 {&yacc2&.s}[0], [TMP1], STRIDE
|
|
vzip &xreg1&.8b, &xreg3&.8b
|
|
ld1 {&yacc2&.s}[2], [TMP2], STRIDE
|
|
vzip &xreg2&.8b, &xreg4&.8b
|
|
ld1 {&yacc2&.s}[1], [TMP1]
|
|
vzip &xreg3&.8b, &xreg4&.8b
|
|
ld1 {&yacc2&.s}[3], [TMP2]
|
|
vzip &xreg1&.8b, &xreg2&.8b
|
|
convert_0565_to_x888 yacc2, yreg3, yreg2, yreg1
|
|
umull &xacc1&.8h, &xreg1&.8b, v28.8b
|
|
vzip &yreg1&.8b, &yreg3&.8b
|
|
umlal &xacc1&.8h, &xreg2&.8b, v29.8b
|
|
vzip &yreg2&.8b, &yreg4&.8b
|
|
umull &xacc2&.8h, &xreg3&.8b, v28.8b
|
|
vzip &yreg3&.8b, &yreg4&.8b
|
|
umlal &xacc2&.8h, &xreg4&.8b, v29.8b
|
|
vzip &yreg1&.8b, &yreg2&.8b
|
|
umull &yacc1&.8h, &yreg1&.8b, v28.8b
|
|
umlal &yacc1&.8h, &yreg2&.8b, v29.8b
|
|
umull &yacc2&.8h, &yreg3&.8b, v28.8b
|
|
umlal &yacc2&.8h, &yreg4&.8b, v29.8b
|
|
.endm
|
|
|
|
.macro bilinear_store_8888 numpix, tmp1, tmp2
|
|
.if numpix == 4
|
|
st1 {v0.2s, v1.2s}, [OUT], #16
|
|
.elseif numpix == 2
|
|
st1 {v0.2s}, [OUT], #8
|
|
.elseif numpix == 1
|
|
st1 {v0.s}[0], [OUT], #4
|
|
.else
|
|
.error bilinear_store_8888 numpix is unsupported
|
|
.endif
|
|
.endm
|
|
|
|
.macro bilinear_store_0565 numpix, tmp1, tmp2
|
|
vuzp v0.8b, v1.8b
|
|
vuzp v2.8b, v3.8b
|
|
vuzp v1.8b, v3.8b
|
|
vuzp v0.8b, v2.8b
|
|
convert_8888_to_0565 v2, v1, v0, v1, tmp1, tmp2
|
|
.if numpix == 4
|
|
st1 {v1.4h}, [OUT], #8
|
|
.elseif numpix == 2
|
|
st1 {v1.s}[0], [OUT], #4
|
|
.elseif numpix == 1
|
|
st1 {v1.h}[0], [OUT], #2
|
|
.else
|
|
.error bilinear_store_0565 numpix is unsupported
|
|
.endif
|
|
.endm
|
|
|
|
.macro bilinear_interpolate_last_pixel src_fmt, dst_fmt
|
|
bilinear_load_&src_fmt v0, v1, v2
|
|
umull v2.8h, v0.8b, v28.8b
|
|
umlal v2.8h, v1.8b, v29.8b
|
|
/* 5 cycles bubble */
|
|
ushll v0.4s, v2.4h, #BILINEAR_INTERPOLATION_BITS
|
|
umlsl v0.4s, v2.4h, v15.h[0]
|
|
umlal2 v0.4s, v2.8h, v15.h[0]
|
|
/* 5 cycles bubble */
|
|
shrn v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
|
|
/* 3 cycles bubble */
|
|
xtn v0.8b, v0.8h
|
|
/* 1 cycle bubble */
|
|
bilinear_store_&dst_fmt 1, v3, v4
|
|
.endm
|
|
|
|
.macro bilinear_interpolate_two_pixels src_fmt, dst_fmt
|
|
bilinear_load_and_vertical_interpolate_two_&src_fmt \
|
|
v1, v11, v2, v3, v20, v21, v22, v23
|
|
ushll v0.4s, v1.4h, #BILINEAR_INTERPOLATION_BITS
|
|
umlsl v0.4s, v1.4h, v15.h[0]
|
|
umlal2 v0.4s, v1.8h, v15.h[0]
|
|
ushll v10.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS
|
|
umlsl v10.4s, v11.4h, v15.h[4]
|
|
umlal2 v10.4s, v11.8h, v15.h[4]
|
|
shrn v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
|
|
shrn2 v0.8h, v10.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
|
|
ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
|
|
add v12.8h, v12.8h, v13.8h
|
|
xtn v0.8b, v0.8h
|
|
bilinear_store_&dst_fmt 2, v3, v4
|
|
.endm
|
|
|
|
.macro bilinear_interpolate_four_pixels src_fmt, dst_fmt
|
|
bilinear_load_and_vertical_interpolate_four_&src_fmt \
|
|
v1, v11, v14, v20, v16, v17, v22, v23 \
|
|
v3, v9, v24, v25, v26, v27, v18, v19
|
|
prfm PREFETCH_MODE, [TMP1, PF_OFFS]
|
|
sub TMP1, TMP1, STRIDE
|
|
ushll v0.4s, v1.4h, #BILINEAR_INTERPOLATION_BITS
|
|
umlsl v0.4s, v1.4h, v15.h[0]
|
|
umlal2 v0.4s, v1.8h, v15.h[0]
|
|
ushll v10.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS
|
|
umlsl v10.4s, v11.4h, v15.h[4]
|
|
umlal2 v10.4s, v11.8h, v15.h[4]
|
|
ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
|
|
ushll v2.4s, v3.4h, #BILINEAR_INTERPOLATION_BITS
|
|
umlsl v2.4s, v3.4h, v15.h[0]
|
|
umlal2 v2.4s, v3.8h, v15.h[0]
|
|
ushll v8.4s, v9.4h, #BILINEAR_INTERPOLATION_BITS
|
|
prfm PREFETCH_MODE, [TMP2, PF_OFFS]
|
|
umlsl v8.4s, v9.4h, v15.h[4]
|
|
umlal2 v8.4s, v9.8h, v15.h[4]
|
|
add v12.8h, v12.8h, v13.8h
|
|
shrn v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
|
|
shrn2 v0.8h, v10.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
|
|
shrn v2.4h, v2.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
|
|
shrn2 v2.8h, v8.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
|
|
ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
|
|
xtn v0.8b, v0.8h
|
|
xtn v1.8b, v2.8h
|
|
add v12.8h, v12.8h, v13.8h
|
|
bilinear_store_&dst_fmt 4, v3, v4
|
|
.endm
|
|
|
|
.macro bilinear_interpolate_four_pixels_head src_fmt, dst_fmt
|
|
.ifdef have_bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt
|
|
bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt&_head
|
|
.else
|
|
bilinear_interpolate_four_pixels src_fmt, dst_fmt
|
|
.endif
|
|
.endm
|
|
|
|
.macro bilinear_interpolate_four_pixels_tail src_fmt, dst_fmt
|
|
.ifdef have_bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt
|
|
bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt&_tail
|
|
.endif
|
|
.endm
|
|
|
|
.macro bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt
|
|
.ifdef have_bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt
|
|
bilinear_interpolate_four_pixels_&src_fmt&_&dst_fmt&_tail_head
|
|
.else
|
|
bilinear_interpolate_four_pixels src_fmt, dst_fmt
|
|
.endif
|
|
.endm
|
|
|
|
.macro bilinear_interpolate_eight_pixels_head src_fmt, dst_fmt
|
|
.ifdef have_bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt
|
|
bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt&_head
|
|
.else
|
|
bilinear_interpolate_four_pixels_head src_fmt, dst_fmt
|
|
bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt
|
|
.endif
|
|
.endm
|
|
|
|
.macro bilinear_interpolate_eight_pixels_tail src_fmt, dst_fmt
|
|
.ifdef have_bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt
|
|
bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt&_tail
|
|
.else
|
|
bilinear_interpolate_four_pixels_tail src_fmt, dst_fmt
|
|
.endif
|
|
.endm
|
|
|
|
.macro bilinear_interpolate_eight_pixels_tail_head src_fmt, dst_fmt
|
|
.ifdef have_bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt
|
|
bilinear_interpolate_eight_pixels_&src_fmt&_&dst_fmt&_tail_head
|
|
.else
|
|
bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt
|
|
bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt
|
|
.endif
|
|
.endm
|
|
|
|
.set BILINEAR_FLAG_UNROLL_4, 0
|
|
.set BILINEAR_FLAG_UNROLL_8, 1
|
|
.set BILINEAR_FLAG_USE_ALL_NEON_REGS, 2
|
|
|
|
/*
|
|
* Main template macro for generating NEON optimized bilinear scanline
|
|
* functions.
|
|
*
|
|
* Bilinear scanline scaler macro template uses the following arguments:
|
|
* fname - name of the function to generate
|
|
* src_fmt - source color format (8888 or 0565)
|
|
* dst_fmt - destination color format (8888 or 0565)
|
|
* bpp_shift - (1 << bpp_shift) is the size of source pixel in bytes
|
|
* prefetch_distance - prefetch in the source image by that many
|
|
* pixels ahead
|
|
*/
|
|
|
|
.macro generate_bilinear_scanline_func fname, src_fmt, dst_fmt, \
|
|
src_bpp_shift, dst_bpp_shift, \
|
|
prefetch_distance, flags
|
|
|
|
pixman_asm_function fname
|
|
OUT .req x0
|
|
TOP .req x1
|
|
BOTTOM .req x2
|
|
WT .req x3
|
|
WB .req x4
|
|
X .req x5
|
|
UX .req x6
|
|
WIDTH .req x7
|
|
TMP1 .req x8
|
|
TMP2 .req x9
|
|
PF_OFFS .req x10
|
|
TMP3 .req x11
|
|
TMP4 .req x12
|
|
STRIDE .req x13
|
|
|
|
sxtw x3, w3
|
|
sxtw x4, w4
|
|
sxtw x5, w5
|
|
sxtw x6, w6
|
|
sxtw x7, w7
|
|
|
|
stp x29, x30, [sp, -16]!
|
|
mov x29, sp
|
|
sub sp, sp, 112 /* push all registers */
|
|
sub x29, x29, 64
|
|
st1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], #32
|
|
st1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], #32
|
|
stp x8, x9, [x29, -80]
|
|
stp x10, x11, [x29, -96]
|
|
stp x12, x13, [x29, -112]
|
|
|
|
mov PF_OFFS, #prefetch_distance
|
|
mul PF_OFFS, PF_OFFS, UX
|
|
|
|
subs STRIDE, BOTTOM, TOP
|
|
.unreq BOTTOM
|
|
|
|
cmp WIDTH, #0
|
|
ble 300f
|
|
|
|
dup v12.8h, w5
|
|
dup v13.8h, w6
|
|
dup v28.8b, w3
|
|
dup v29.8b, w4
|
|
mov v25.d[0], v12.d[1]
|
|
mov v26.d[0], v13.d[0]
|
|
add v25.4h, v25.4h, v26.4h
|
|
mov v12.d[1], v25.d[0]
|
|
|
|
/* ensure good destination alignment */
|
|
cmp WIDTH, #1
|
|
blt 100f
|
|
tst OUT, #(1 << dst_bpp_shift)
|
|
beq 100f
|
|
ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
|
|
add v12.8h, v12.8h, v13.8h
|
|
bilinear_interpolate_last_pixel src_fmt, dst_fmt
|
|
sub WIDTH, WIDTH, #1
|
|
100:
|
|
add v13.8h, v13.8h, v13.8h
|
|
ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
|
|
add v12.8h, v12.8h, v13.8h
|
|
|
|
cmp WIDTH, #2
|
|
blt 100f
|
|
tst OUT, #(1 << (dst_bpp_shift + 1))
|
|
beq 100f
|
|
bilinear_interpolate_two_pixels src_fmt, dst_fmt
|
|
sub WIDTH, WIDTH, #2
|
|
100:
|
|
.if ((flags) & BILINEAR_FLAG_UNROLL_8) != 0
|
|
/*********** 8 pixels per iteration *****************/
|
|
cmp WIDTH, #4
|
|
blt 100f
|
|
tst OUT, #(1 << (dst_bpp_shift + 2))
|
|
beq 100f
|
|
bilinear_interpolate_four_pixels src_fmt, dst_fmt
|
|
sub WIDTH, WIDTH, #4
|
|
100:
|
|
subs WIDTH, WIDTH, #8
|
|
blt 100f
|
|
asr PF_OFFS, PF_OFFS, #(16 - src_bpp_shift)
|
|
bilinear_interpolate_eight_pixels_head src_fmt, dst_fmt
|
|
subs WIDTH, WIDTH, #8
|
|
blt 500f
|
|
1000:
|
|
bilinear_interpolate_eight_pixels_tail_head src_fmt, dst_fmt
|
|
subs WIDTH, WIDTH, #8
|
|
bge 1000b
|
|
500:
|
|
bilinear_interpolate_eight_pixels_tail src_fmt, dst_fmt
|
|
100:
|
|
tst WIDTH, #4
|
|
beq 200f
|
|
bilinear_interpolate_four_pixels src_fmt, dst_fmt
|
|
200:
|
|
.else
|
|
/*********** 4 pixels per iteration *****************/
|
|
subs WIDTH, WIDTH, #4
|
|
blt 100f
|
|
asr PF_OFFS, PF_OFFS, #(16 - src_bpp_shift)
|
|
bilinear_interpolate_four_pixels_head src_fmt, dst_fmt
|
|
subs WIDTH, WIDTH, #4
|
|
blt 500f
|
|
1000:
|
|
bilinear_interpolate_four_pixels_tail_head src_fmt, dst_fmt
|
|
subs WIDTH, WIDTH, #4
|
|
bge 1000b
|
|
500:
|
|
bilinear_interpolate_four_pixels_tail src_fmt, dst_fmt
|
|
100:
|
|
/****************************************************/
|
|
.endif
|
|
/* handle the remaining trailing pixels */
|
|
tst WIDTH, #2
|
|
beq 200f
|
|
bilinear_interpolate_two_pixels src_fmt, dst_fmt
|
|
200:
|
|
tst WIDTH, #1
|
|
beq 300f
|
|
bilinear_interpolate_last_pixel src_fmt, dst_fmt
|
|
300:
|
|
sub x29, x29, 64
|
|
ld1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x29], #32
|
|
ld1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x29], #32
|
|
ldp x8, x9, [x29, -80]
|
|
ldp x10, x11, [x29, -96]
|
|
ldp x12, x13, [x29, -104]
|
|
mov sp, x29
|
|
ldp x29, x30, [sp], 16
|
|
ret
|
|
|
|
.unreq OUT
|
|
.unreq TOP
|
|
.unreq WT
|
|
.unreq WB
|
|
.unreq X
|
|
.unreq UX
|
|
.unreq WIDTH
|
|
.unreq TMP1
|
|
.unreq TMP2
|
|
.unreq PF_OFFS
|
|
.unreq TMP3
|
|
.unreq TMP4
|
|
.unreq STRIDE
|
|
.endfunc
|
|
|
|
.endm
|
|
|
|
/*****************************************************************************/
|
|
|
|
.set have_bilinear_interpolate_four_pixels_8888_8888, 1
|
|
|
|
.macro bilinear_interpolate_four_pixels_8888_8888_head
|
|
asr TMP1, X, #16
|
|
add X, X, UX
|
|
add TMP1, TOP, TMP1, lsl #2
|
|
asr TMP2, X, #16
|
|
add X, X, UX
|
|
add TMP2, TOP, TMP2, lsl #2
|
|
|
|
ld1 {v22.2s}, [TMP1], STRIDE
|
|
ld1 {v23.2s}, [TMP1]
|
|
asr TMP3, X, #16
|
|
add X, X, UX
|
|
add TMP3, TOP, TMP3, lsl #2
|
|
umull v8.8h, v22.8b, v28.8b
|
|
umlal v8.8h, v23.8b, v29.8b
|
|
|
|
ld1 {v22.2s}, [TMP2], STRIDE
|
|
ld1 {v23.2s}, [TMP2]
|
|
asr TMP4, X, #16
|
|
add X, X, UX
|
|
add TMP4, TOP, TMP4, lsl #2
|
|
umull v9.8h, v22.8b, v28.8b
|
|
umlal v9.8h, v23.8b, v29.8b
|
|
|
|
ld1 {v22.2s}, [TMP3], STRIDE
|
|
ld1 {v23.2s}, [TMP3]
|
|
umull v10.8h, v22.8b, v28.8b
|
|
umlal v10.8h, v23.8b, v29.8b
|
|
|
|
ushll v0.4s, v8.4h, #BILINEAR_INTERPOLATION_BITS
|
|
umlsl v0.4s, v8.4h, v15.h[0]
|
|
umlal2 v0.4s, v8.8h, v15.h[0]
|
|
|
|
prfm PREFETCH_MODE, [TMP4, PF_OFFS]
|
|
ld1 {v16.2s}, [TMP4], STRIDE
|
|
ld1 {v17.2s}, [TMP4]
|
|
prfm PREFETCH_MODE, [TMP4, PF_OFFS]
|
|
umull v11.8h, v16.8b, v28.8b
|
|
umlal v11.8h, v17.8b, v29.8b
|
|
|
|
ushll v1.4s, v9.4h, #BILINEAR_INTERPOLATION_BITS
|
|
umlsl v1.4s, v9.4h, v15.h[4]
|
|
.endm
|
|
|
|
.macro bilinear_interpolate_four_pixels_8888_8888_tail
|
|
umlal2 v1.4s, v9.8h, v15.h[4]
|
|
ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
|
|
ushll v2.4s, v10.4h, #BILINEAR_INTERPOLATION_BITS
|
|
umlsl v2.4s, v10.4h, v15.h[0]
|
|
umlal2 v2.4s, v10.8h, v15.h[0]
|
|
ushll v3.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS
|
|
umlsl v3.4s, v11.4h, v15.h[4]
|
|
umlal2 v3.4s, v11.8h, v15.h[4]
|
|
add v12.8h, v12.8h, v13.8h
|
|
shrn v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
|
|
shrn2 v0.8h, v1.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
|
|
shrn v2.4h, v2.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
|
|
ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
|
|
shrn2 v2.8h, v3.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
|
|
xtn v6.8b, v0.8h
|
|
xtn v7.8b, v2.8h
|
|
add v12.8h, v12.8h, v13.8h
|
|
st1 {v6.2s, v7.2s}, [OUT], #16
|
|
.endm
|
|
|
|
.macro bilinear_interpolate_four_pixels_8888_8888_tail_head
|
|
asr TMP1, X, #16
|
|
add X, X, UX
|
|
add TMP1, TOP, TMP1, lsl #2
|
|
asr TMP2, X, #16
|
|
add X, X, UX
|
|
add TMP2, TOP, TMP2, lsl #2
|
|
umlal2 v1.4s, v9.8h, v15.h[4]
|
|
ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
|
|
ushll v2.4s, v10.4h, #BILINEAR_INTERPOLATION_BITS
|
|
umlsl v2.4s, v10.4h, v15.h[0]
|
|
umlal2 v2.4s, v10.8h, v15.h[0]
|
|
ushll v3.4s, v11.4h, #BILINEAR_INTERPOLATION_BITS
|
|
ld1 {v20.2s}, [TMP1], STRIDE
|
|
umlsl v3.4s, v11.4h, v15.h[4]
|
|
umlal2 v3.4s, v11.8h, v15.h[4]
|
|
ld1 {v21.2s}, [TMP1]
|
|
umull v8.8h, v20.8b, v28.8b
|
|
umlal v8.8h, v21.8b, v29.8b
|
|
shrn v0.4h, v0.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
|
|
shrn2 v0.8h, v1.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
|
|
shrn v4.4h, v2.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
|
|
ld1 {v22.2s}, [TMP2], STRIDE
|
|
shrn2 v4.8h, v3.4s, #(2 * BILINEAR_INTERPOLATION_BITS)
|
|
add v12.8h, v12.8h, v13.8h
|
|
ld1 {v23.2s}, [TMP2]
|
|
umull v9.8h, v22.8b, v28.8b
|
|
asr TMP3, X, #16
|
|
add X, X, UX
|
|
add TMP3, TOP, TMP3, lsl #2
|
|
asr TMP4, X, #16
|
|
add X, X, UX
|
|
add TMP4, TOP, TMP4, lsl #2
|
|
umlal v9.8h, v23.8b, v29.8b
|
|
ld1 {v22.2s}, [TMP3], STRIDE
|
|
ushr v15.8h, v12.8h, #(16 - BILINEAR_INTERPOLATION_BITS)
|
|
ld1 {v23.2s}, [TMP3]
|
|
umull v10.8h, v22.8b, v28.8b
|
|
umlal v10.8h, v23.8b, v29.8b
|
|
xtn v6.8b, v0.8h
|
|
ushll v0.4s, v8.4h, #BILINEAR_INTERPOLATION_BITS
|
|
xtn v7.8b, v4.8h
|
|
umlsl v0.4s, v8.4h, v15.h[0]
|
|
umlal2 v0.4s, v8.8h, v15.h[0]
|
|
prfm PREFETCH_MODE, [TMP4, PF_OFFS]
|
|
ld1 {v16.2s}, [TMP4], STRIDE
|
|
add v12.8h, v12.8h, v13.8h
|
|
ld1 {v17.2s}, [TMP4]
|
|
prfm PREFETCH_MODE, [TMP4, PF_OFFS]
|
|
umull v11.8h, v16.8b, v28.8b
|
|
umlal v11.8h, v17.8b, v29.8b
|
|
st1 {v6.2s, v7.2s}, [OUT], #16
|
|
ushll v1.4s, v9.4h, #BILINEAR_INTERPOLATION_BITS
|
|
umlsl v1.4s, v9.4h, v15.h[4]
|
|
.endm
|
|
|
|
/*****************************************************************************/
|
|
|
|
generate_bilinear_scanline_func \
|
|
pixman_scaled_bilinear_scanline_8888_8888_SRC_asm_neon, 8888, 8888, \
|
|
2, 2, 28, BILINEAR_FLAG_UNROLL_4
|
|
|
|
generate_bilinear_scanline_func \
|
|
pixman_scaled_bilinear_scanline_8888_0565_SRC_asm_neon, 8888, 0565, \
|
|
2, 1, 28, BILINEAR_FLAG_UNROLL_8 | BILINEAR_FLAG_USE_ALL_NEON_REGS
|
|
|
|
generate_bilinear_scanline_func \
|
|
pixman_scaled_bilinear_scanline_0565_x888_SRC_asm_neon, 0565, 8888, \
|
|
1, 2, 28, BILINEAR_FLAG_UNROLL_4
|
|
|
|
generate_bilinear_scanline_func \
|
|
pixman_scaled_bilinear_scanline_0565_0565_SRC_asm_neon, 0565, 0565, \
|
|
1, 1, 28, BILINEAR_FLAG_UNROLL_4
|