mirror of
https://bitbucket.org/CPMADevs/cnq3
synced 2024-11-29 23:32:03 +00:00
fc9465caab
aside from the speed improvements, this also makes for nicer code in the renderer interaction with libjpeg, thanks to mem_dest support etc
3425 lines
138 KiB
ArmAsm
3425 lines
138 KiB
ArmAsm
/*
|
|
* ARMv8 NEON optimizations for libjpeg-turbo
|
|
*
|
|
* Copyright (C) 2009-2011, Nokia Corporation and/or its subsidiary(-ies).
|
|
* All Rights Reserved.
|
|
* Author: Siarhei Siamashka <siarhei.siamashka@nokia.com>
|
|
* Copyright (C) 2013-2014, Linaro Limited. All Rights Reserved.
|
|
* Author: Ragesh Radhakrishnan <ragesh.r@linaro.org>
|
|
* Copyright (C) 2014-2016, D. R. Commander. All Rights Reserved.
|
|
* Copyright (C) 2015-2016, Matthieu Darbois. All Rights Reserved.
|
|
* Copyright (C) 2016, Siarhei Siamashka. All Rights Reserved.
|
|
*
|
|
* This software is provided 'as-is', without any express or implied
|
|
* warranty. In no event will the authors be held liable for any damages
|
|
* arising from the use of this software.
|
|
*
|
|
* Permission is granted to anyone to use this software for any purpose,
|
|
* including commercial applications, and to alter it and redistribute it
|
|
* freely, subject to the following restrictions:
|
|
*
|
|
* 1. The origin of this software must not be misrepresented; you must not
|
|
* claim that you wrote the original software. If you use this software
|
|
* in a product, an acknowledgment in the product documentation would be
|
|
* appreciated but is not required.
|
|
* 2. Altered source versions must be plainly marked as such, and must not be
|
|
* misrepresented as being the original software.
|
|
* 3. This notice may not be removed or altered from any source distribution.
|
|
*/
|
|
|
|
#if defined(__linux__) && defined(__ELF__)
|
|
.section .note.GNU-stack, "", %progbits /* mark stack as non-executable */
|
|
#endif
|
|
|
|
.text
|
|
|
|
|
|
#define RESPECT_STRICT_ALIGNMENT 1
|
|
|
|
|
|
/*****************************************************************************/
|
|
|
|
/* Supplementary macro for setting function attributes */
|
|
.macro asm_function fname
|
|
#ifdef __APPLE__
|
|
.globl _\fname
|
|
_\fname:
|
|
#else
|
|
.global \fname
|
|
#ifdef __ELF__
|
|
.hidden \fname
|
|
.type \fname, %function
|
|
#endif
|
|
\fname:
|
|
#endif
|
|
.endm
|
|
|
|
/* Transpose elements of single 128 bit registers */
|
|
.macro transpose_single x0, x1, xi, xilen, literal
|
|
ins \xi\xilen[0], \x0\xilen[0]
|
|
ins \x1\xilen[0], \x0\xilen[1]
|
|
trn1 \x0\literal, \x0\literal, \x1\literal
|
|
trn2 \x1\literal, \xi\literal, \x1\literal
|
|
.endm
|
|
|
|
/* Transpose elements of 2 differnet registers */
|
|
.macro transpose x0, x1, xi, xilen, literal
|
|
mov \xi\xilen, \x0\xilen
|
|
trn1 \x0\literal, \x0\literal, \x1\literal
|
|
trn2 \x1\literal, \xi\literal, \x1\literal
|
|
.endm
|
|
|
|
/* Transpose a block of 4x4 coefficients in four 64-bit registers */
|
|
.macro transpose_4x4_32 x0, x0len, x1, x1len, x2, x2len, x3, x3len, xi, xilen
|
|
mov \xi\xilen, \x0\xilen
|
|
trn1 \x0\x0len, \x0\x0len, \x2\x2len
|
|
trn2 \x2\x2len, \xi\x0len, \x2\x2len
|
|
mov \xi\xilen, \x1\xilen
|
|
trn1 \x1\x1len, \x1\x1len, \x3\x3len
|
|
trn2 \x3\x3len, \xi\x1len, \x3\x3len
|
|
.endm
|
|
|
|
.macro transpose_4x4_16 x0, x0len, x1, x1len, x2, x2len, x3, x3len, xi, xilen
|
|
mov \xi\xilen, \x0\xilen
|
|
trn1 \x0\x0len, \x0\x0len, \x1\x1len
|
|
trn2 \x1\x2len, \xi\x0len, \x1\x2len
|
|
mov \xi\xilen, \x2\xilen
|
|
trn1 \x2\x2len, \x2\x2len, \x3\x3len
|
|
trn2 \x3\x2len, \xi\x1len, \x3\x3len
|
|
.endm
|
|
|
|
.macro transpose_4x4 x0, x1, x2, x3, x5
|
|
transpose_4x4_16 \x0, .4h, \x1, .4h, \x2, .4h, \x3, .4h, \x5, .16b
|
|
transpose_4x4_32 \x0, .2s, \x1, .2s, \x2, .2s, \x3, .2s, \x5, .16b
|
|
.endm
|
|
|
|
.macro transpose_8x8 l0, l1, l2, l3, l4, l5, l6, l7, t0, t1, t2, t3
|
|
trn1 \t0\().8h, \l0\().8h, \l1\().8h
|
|
trn1 \t1\().8h, \l2\().8h, \l3\().8h
|
|
trn1 \t2\().8h, \l4\().8h, \l5\().8h
|
|
trn1 \t3\().8h, \l6\().8h, \l7\().8h
|
|
trn2 \l1\().8h, \l0\().8h, \l1\().8h
|
|
trn2 \l3\().8h, \l2\().8h, \l3\().8h
|
|
trn2 \l5\().8h, \l4\().8h, \l5\().8h
|
|
trn2 \l7\().8h, \l6\().8h, \l7\().8h
|
|
|
|
trn1 \l4\().4s, \t2\().4s, \t3\().4s
|
|
trn2 \t3\().4s, \t2\().4s, \t3\().4s
|
|
trn1 \t2\().4s, \t0\().4s, \t1\().4s
|
|
trn2 \l2\().4s, \t0\().4s, \t1\().4s
|
|
trn1 \t0\().4s, \l1\().4s, \l3\().4s
|
|
trn2 \l3\().4s, \l1\().4s, \l3\().4s
|
|
trn2 \t1\().4s, \l5\().4s, \l7\().4s
|
|
trn1 \l5\().4s, \l5\().4s, \l7\().4s
|
|
|
|
trn2 \l6\().2d, \l2\().2d, \t3\().2d
|
|
trn1 \l0\().2d, \t2\().2d, \l4\().2d
|
|
trn1 \l1\().2d, \t0\().2d, \l5\().2d
|
|
trn2 \l7\().2d, \l3\().2d, \t1\().2d
|
|
trn1 \l2\().2d, \l2\().2d, \t3\().2d
|
|
trn2 \l4\().2d, \t2\().2d, \l4\().2d
|
|
trn1 \l3\().2d, \l3\().2d, \t1\().2d
|
|
trn2 \l5\().2d, \t0\().2d, \l5\().2d
|
|
.endm
|
|
|
|
|
|
#define CENTERJSAMPLE 128
|
|
|
|
/*****************************************************************************/
|
|
|
|
/*
|
|
* Perform dequantization and inverse DCT on one block of coefficients.
|
|
*
|
|
* GLOBAL(void)
|
|
* jsimd_idct_islow_neon (void *dct_table, JCOEFPTR coef_block,
|
|
* JSAMPARRAY output_buf, JDIMENSION output_col)
|
|
*/
|
|
|
|
#define CONST_BITS 13
|
|
#define PASS1_BITS 2
|
|
|
|
#define F_0_298 2446 /* FIX(0.298631336) */
|
|
#define F_0_390 3196 /* FIX(0.390180644) */
|
|
#define F_0_541 4433 /* FIX(0.541196100) */
|
|
#define F_0_765 6270 /* FIX(0.765366865) */
|
|
#define F_0_899 7373 /* FIX(0.899976223) */
|
|
#define F_1_175 9633 /* FIX(1.175875602) */
|
|
#define F_1_501 12299 /* FIX(1.501321110) */
|
|
#define F_1_847 15137 /* FIX(1.847759065) */
|
|
#define F_1_961 16069 /* FIX(1.961570560) */
|
|
#define F_2_053 16819 /* FIX(2.053119869) */
|
|
#define F_2_562 20995 /* FIX(2.562915447) */
|
|
#define F_3_072 25172 /* FIX(3.072711026) */
|
|
|
|
.balign 16
|
|
Ljsimd_idct_islow_neon_consts:
|
|
.short F_0_298
|
|
.short -F_0_390
|
|
.short F_0_541
|
|
.short F_0_765
|
|
.short - F_0_899
|
|
.short F_1_175
|
|
.short F_1_501
|
|
.short - F_1_847
|
|
.short - F_1_961
|
|
.short F_2_053
|
|
.short - F_2_562
|
|
.short F_3_072
|
|
.short 0 /* padding */
|
|
.short 0
|
|
.short 0
|
|
.short 0
|
|
|
|
#undef F_0_298
|
|
#undef F_0_390
|
|
#undef F_0_541
|
|
#undef F_0_765
|
|
#undef F_0_899
|
|
#undef F_1_175
|
|
#undef F_1_501
|
|
#undef F_1_847
|
|
#undef F_1_961
|
|
#undef F_2_053
|
|
#undef F_2_562
|
|
#undef F_3_072
|
|
|
|
#define XFIX_P_0_298 v0.h[0]
|
|
#define XFIX_N_0_390 v0.h[1]
|
|
#define XFIX_P_0_541 v0.h[2]
|
|
#define XFIX_P_0_765 v0.h[3]
|
|
#define XFIX_N_0_899 v0.h[4]
|
|
#define XFIX_P_1_175 v0.h[5]
|
|
#define XFIX_P_1_501 v0.h[6]
|
|
#define XFIX_N_1_847 v0.h[7]
|
|
#define XFIX_N_1_961 v1.h[0]
|
|
#define XFIX_P_2_053 v1.h[1]
|
|
#define XFIX_N_2_562 v1.h[2]
|
|
#define XFIX_P_3_072 v1.h[3]
|
|
|
|
asm_function jsimd_idct_islow_neon
|
|
DCT_TABLE .req x0
|
|
COEF_BLOCK .req x1
|
|
OUTPUT_BUF .req x2
|
|
OUTPUT_COL .req x3
|
|
TMP1 .req x0
|
|
TMP2 .req x1
|
|
TMP3 .req x9
|
|
TMP4 .req x10
|
|
TMP5 .req x11
|
|
TMP6 .req x12
|
|
TMP7 .req x13
|
|
TMP8 .req x14
|
|
|
|
/* OUTPUT_COL is a JDIMENSION (unsigned int) argument, so the ABI doesn't
|
|
guarantee that the upper (unused) 32 bits of x3 are valid. This
|
|
instruction ensures that those bits are set to zero. */
|
|
uxtw x3, w3
|
|
|
|
sub sp, sp, #64
|
|
adr x15, Ljsimd_idct_islow_neon_consts
|
|
mov x10, sp
|
|
st1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x10], #32
|
|
st1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x10], #32
|
|
ld1 {v0.8h, v1.8h}, [x15]
|
|
ld1 {v2.8h, v3.8h, v4.8h, v5.8h}, [COEF_BLOCK], #64
|
|
ld1 {v18.8h, v19.8h, v20.8h, v21.8h}, [DCT_TABLE], #64
|
|
ld1 {v6.8h, v7.8h, v8.8h, v9.8h}, [COEF_BLOCK], #64
|
|
ld1 {v22.8h, v23.8h, v24.8h, v25.8h}, [DCT_TABLE], #64
|
|
|
|
cmeq v16.8h, v3.8h, #0
|
|
cmeq v26.8h, v4.8h, #0
|
|
cmeq v27.8h, v5.8h, #0
|
|
cmeq v28.8h, v6.8h, #0
|
|
cmeq v29.8h, v7.8h, #0
|
|
cmeq v30.8h, v8.8h, #0
|
|
cmeq v31.8h, v9.8h, #0
|
|
|
|
and v10.16b, v16.16b, v26.16b
|
|
and v11.16b, v27.16b, v28.16b
|
|
and v12.16b, v29.16b, v30.16b
|
|
and v13.16b, v31.16b, v10.16b
|
|
and v14.16b, v11.16b, v12.16b
|
|
mul v2.8h, v2.8h, v18.8h
|
|
and v15.16b, v13.16b, v14.16b
|
|
shl v10.8h, v2.8h, #(PASS1_BITS)
|
|
sqxtn v16.8b, v15.8h
|
|
mov TMP1, v16.d[0]
|
|
mvn TMP2, TMP1
|
|
|
|
cbnz TMP2, 2f
|
|
/* case all AC coeffs are zeros */
|
|
dup v2.2d, v10.d[0]
|
|
dup v6.2d, v10.d[1]
|
|
mov v3.16b, v2.16b
|
|
mov v7.16b, v6.16b
|
|
mov v4.16b, v2.16b
|
|
mov v8.16b, v6.16b
|
|
mov v5.16b, v2.16b
|
|
mov v9.16b, v6.16b
|
|
1:
|
|
/* for this transpose, we should organise data like this:
|
|
* 00, 01, 02, 03, 40, 41, 42, 43
|
|
* 10, 11, 12, 13, 50, 51, 52, 53
|
|
* 20, 21, 22, 23, 60, 61, 62, 63
|
|
* 30, 31, 32, 33, 70, 71, 72, 73
|
|
* 04, 05, 06, 07, 44, 45, 46, 47
|
|
* 14, 15, 16, 17, 54, 55, 56, 57
|
|
* 24, 25, 26, 27, 64, 65, 66, 67
|
|
* 34, 35, 36, 37, 74, 75, 76, 77
|
|
*/
|
|
trn1 v28.8h, v2.8h, v3.8h
|
|
trn1 v29.8h, v4.8h, v5.8h
|
|
trn1 v30.8h, v6.8h, v7.8h
|
|
trn1 v31.8h, v8.8h, v9.8h
|
|
trn2 v16.8h, v2.8h, v3.8h
|
|
trn2 v17.8h, v4.8h, v5.8h
|
|
trn2 v18.8h, v6.8h, v7.8h
|
|
trn2 v19.8h, v8.8h, v9.8h
|
|
trn1 v2.4s, v28.4s, v29.4s
|
|
trn1 v6.4s, v30.4s, v31.4s
|
|
trn1 v3.4s, v16.4s, v17.4s
|
|
trn1 v7.4s, v18.4s, v19.4s
|
|
trn2 v4.4s, v28.4s, v29.4s
|
|
trn2 v8.4s, v30.4s, v31.4s
|
|
trn2 v5.4s, v16.4s, v17.4s
|
|
trn2 v9.4s, v18.4s, v19.4s
|
|
/* Even part: reverse the even part of the forward DCT. */
|
|
add v18.8h, v4.8h, v8.8h /* z2 + z3 = DEQUANTIZE(inptr[DCTSIZE*2], quantptr[DCTSIZE*2]) + DEQUANTIZE(inptr[DCTSIZE*6], quantptr[DCTSIZE*6]) */
|
|
add v22.8h, v2.8h, v6.8h /* z2 + z3 = DEQUANTIZE(inptr[DCTSIZE*0], quantptr[DCTSIZE*0]) + DEQUANTIZE(inptr[DCTSIZE*4], quantptr[DCTSIZE*4]) */
|
|
smull2 v19.4s, v18.8h, XFIX_P_0_541 /* z1h z1 = MULTIPLY(z2 + z3, FIX_0_541196100); */
|
|
sub v26.8h, v2.8h, v6.8h /* z2 - z3 = DEQUANTIZE(inptr[DCTSIZE*0], quantptr[DCTSIZE*0]) - DEQUANTIZE(inptr[DCTSIZE*4], quantptr[DCTSIZE*4]) */
|
|
smull v18.4s, v18.4h, XFIX_P_0_541 /* z1l z1 = MULTIPLY(z2 + z3, FIX_0_541196100); */
|
|
sshll2 v23.4s, v22.8h, #(CONST_BITS) /* tmp0h tmp0 = LEFT_SHIFT(z2 + z3, CONST_BITS); */
|
|
mov v21.16b, v19.16b /* tmp3 = z1 */
|
|
mov v20.16b, v18.16b /* tmp3 = z1 */
|
|
smlal2 v19.4s, v8.8h, XFIX_N_1_847 /* tmp2h tmp2 = z1 + MULTIPLY(z3, - FIX_1_847759065); */
|
|
smlal v18.4s, v8.4h, XFIX_N_1_847 /* tmp2l tmp2 = z1 + MULTIPLY(z3, - FIX_1_847759065); */
|
|
sshll2 v27.4s, v26.8h, #(CONST_BITS) /* tmp1h tmp1 = LEFT_SHIFT(z2 - z3, CONST_BITS); */
|
|
smlal2 v21.4s, v4.8h, XFIX_P_0_765 /* tmp3h tmp3 = z1 + MULTIPLY(z2, FIX_0_765366865); */
|
|
smlal v20.4s, v4.4h, XFIX_P_0_765 /* tmp3l tmp3 = z1 + MULTIPLY(z2, FIX_0_765366865); */
|
|
sshll v22.4s, v22.4h, #(CONST_BITS) /* tmp0l tmp0 = LEFT_SHIFT(z2 + z3, CONST_BITS); */
|
|
sshll v26.4s, v26.4h, #(CONST_BITS) /* tmp1l tmp1 = LEFT_SHIFT(z2 - z3, CONST_BITS); */
|
|
add v2.4s, v22.4s, v20.4s /* tmp10l tmp10 = tmp0 + tmp3; */
|
|
sub v6.4s, v22.4s, v20.4s /* tmp13l tmp13 = tmp0 - tmp3; */
|
|
add v8.4s, v26.4s, v18.4s /* tmp11l tmp11 = tmp1 + tmp2; */
|
|
sub v4.4s, v26.4s, v18.4s /* tmp12l tmp12 = tmp1 - tmp2; */
|
|
add v28.4s, v23.4s, v21.4s /* tmp10h tmp10 = tmp0 + tmp3; */
|
|
sub v31.4s, v23.4s, v21.4s /* tmp13h tmp13 = tmp0 - tmp3; */
|
|
add v29.4s, v27.4s, v19.4s /* tmp11h tmp11 = tmp1 + tmp2; */
|
|
sub v30.4s, v27.4s, v19.4s /* tmp12h tmp12 = tmp1 - tmp2; */
|
|
|
|
/* Odd part per figure 8; the matrix is unitary and hence its
|
|
* transpose is its inverse. i0..i3 are y7,y5,y3,y1 respectively.
|
|
*/
|
|
|
|
add v22.8h, v9.8h, v5.8h /* z3 = tmp0 + tmp2 = DEQUANTIZE(inptr[DCTSIZE*7], quantptr[DCTSIZE*7]) + DEQUANTIZE(inptr[DCTSIZE*3], quantptr[DCTSIZE*3]) */
|
|
add v24.8h, v7.8h, v3.8h /* z4 = tmp1 + tmp3 = DEQUANTIZE(inptr[DCTSIZE*5], quantptr[DCTSIZE*5]) + DEQUANTIZE(inptr[DCTSIZE*1], quantptr[DCTSIZE*1]) */
|
|
add v18.8h, v9.8h, v3.8h /* z1 = tmp0 + tmp3 = DEQUANTIZE(inptr[DCTSIZE*7], quantptr[DCTSIZE*7]) + DEQUANTIZE(inptr[DCTSIZE*1], quantptr[DCTSIZE*1]) */
|
|
add v20.8h, v7.8h, v5.8h /* z2 = tmp1 + tmp2 = DEQUANTIZE(inptr[DCTSIZE*5], quantptr[DCTSIZE*5]) + DEQUANTIZE(inptr[DCTSIZE*3], quantptr[DCTSIZE*3]) */
|
|
add v26.8h, v22.8h, v24.8h /* z5 = z3 + z4 */
|
|
|
|
smull2 v11.4s, v9.8h, XFIX_P_0_298 /* tmp0 = MULTIPLY(tmp0, FIX_0_298631336) */
|
|
smull2 v13.4s, v7.8h, XFIX_P_2_053 /* tmp1 = MULTIPLY(tmp1, FIX_2_053119869) */
|
|
smull2 v15.4s, v5.8h, XFIX_P_3_072 /* tmp2 = MULTIPLY(tmp2, FIX_3_072711026) */
|
|
smull2 v17.4s, v3.8h, XFIX_P_1_501 /* tmp3 = MULTIPLY(tmp3, FIX_1_501321110) */
|
|
smull2 v27.4s, v26.8h, XFIX_P_1_175 /* z5h z5 = MULTIPLY(z3 + z4, FIX_1_175875602) */
|
|
smull2 v23.4s, v22.8h, XFIX_N_1_961 /* z3 = MULTIPLY(z3, - FIX_1_961570560) */
|
|
smull2 v25.4s, v24.8h, XFIX_N_0_390 /* z4 = MULTIPLY(z4, - FIX_0_390180644) */
|
|
smull2 v19.4s, v18.8h, XFIX_N_0_899 /* z1 = MULTIPLY(z1, - FIX_0_899976223) */
|
|
smull2 v21.4s, v20.8h, XFIX_N_2_562 /* z2 = MULTIPLY(z2, - FIX_2_562915447) */
|
|
|
|
smull v10.4s, v9.4h, XFIX_P_0_298 /* tmp0 = MULTIPLY(tmp0, FIX_0_298631336) */
|
|
smull v12.4s, v7.4h, XFIX_P_2_053 /* tmp1 = MULTIPLY(tmp1, FIX_2_053119869) */
|
|
smull v14.4s, v5.4h, XFIX_P_3_072 /* tmp2 = MULTIPLY(tmp2, FIX_3_072711026) */
|
|
smull v16.4s, v3.4h, XFIX_P_1_501 /* tmp3 = MULTIPLY(tmp3, FIX_1_501321110) */
|
|
smull v26.4s, v26.4h, XFIX_P_1_175 /* z5l z5 = MULTIPLY(z3 + z4, FIX_1_175875602) */
|
|
smull v22.4s, v22.4h, XFIX_N_1_961 /* z3 = MULTIPLY(z3, - FIX_1_961570560) */
|
|
smull v24.4s, v24.4h, XFIX_N_0_390 /* z4 = MULTIPLY(z4, - FIX_0_390180644) */
|
|
smull v18.4s, v18.4h, XFIX_N_0_899 /* z1 = MULTIPLY(z1, - FIX_0_899976223) */
|
|
smull v20.4s, v20.4h, XFIX_N_2_562 /* z2 = MULTIPLY(z2, - FIX_2_562915447) */
|
|
|
|
add v23.4s, v23.4s, v27.4s /* z3 += z5 */
|
|
add v22.4s, v22.4s, v26.4s /* z3 += z5 */
|
|
add v25.4s, v25.4s, v27.4s /* z4 += z5 */
|
|
add v24.4s, v24.4s, v26.4s /* z4 += z5 */
|
|
|
|
add v11.4s, v11.4s, v19.4s /* tmp0 += z1 */
|
|
add v10.4s, v10.4s, v18.4s /* tmp0 += z1 */
|
|
add v13.4s, v13.4s, v21.4s /* tmp1 += z2 */
|
|
add v12.4s, v12.4s, v20.4s /* tmp1 += z2 */
|
|
add v15.4s, v15.4s, v21.4s /* tmp2 += z2 */
|
|
add v14.4s, v14.4s, v20.4s /* tmp2 += z2 */
|
|
add v17.4s, v17.4s, v19.4s /* tmp3 += z1 */
|
|
add v16.4s, v16.4s, v18.4s /* tmp3 += z1 */
|
|
|
|
add v11.4s, v11.4s, v23.4s /* tmp0 += z3 */
|
|
add v10.4s, v10.4s, v22.4s /* tmp0 += z3 */
|
|
add v13.4s, v13.4s, v25.4s /* tmp1 += z4 */
|
|
add v12.4s, v12.4s, v24.4s /* tmp1 += z4 */
|
|
add v17.4s, v17.4s, v25.4s /* tmp3 += z4 */
|
|
add v16.4s, v16.4s, v24.4s /* tmp3 += z4 */
|
|
add v15.4s, v15.4s, v23.4s /* tmp2 += z3 */
|
|
add v14.4s, v14.4s, v22.4s /* tmp2 += z3 */
|
|
|
|
/* Final output stage: inputs are tmp10..tmp13, tmp0..tmp3 */
|
|
|
|
add v18.4s, v2.4s, v16.4s /* tmp10 + tmp3 */
|
|
add v19.4s, v28.4s, v17.4s /* tmp10 + tmp3 */
|
|
sub v20.4s, v2.4s, v16.4s /* tmp10 - tmp3 */
|
|
sub v21.4s, v28.4s, v17.4s /* tmp10 - tmp3 */
|
|
add v22.4s, v8.4s, v14.4s /* tmp11 + tmp2 */
|
|
add v23.4s, v29.4s, v15.4s /* tmp11 + tmp2 */
|
|
sub v24.4s, v8.4s, v14.4s /* tmp11 - tmp2 */
|
|
sub v25.4s, v29.4s, v15.4s /* tmp11 - tmp2 */
|
|
add v26.4s, v4.4s, v12.4s /* tmp12 + tmp1 */
|
|
add v27.4s, v30.4s, v13.4s /* tmp12 + tmp1 */
|
|
sub v28.4s, v4.4s, v12.4s /* tmp12 - tmp1 */
|
|
sub v29.4s, v30.4s, v13.4s /* tmp12 - tmp1 */
|
|
add v14.4s, v6.4s, v10.4s /* tmp13 + tmp0 */
|
|
add v15.4s, v31.4s, v11.4s /* tmp13 + tmp0 */
|
|
sub v16.4s, v6.4s, v10.4s /* tmp13 - tmp0 */
|
|
sub v17.4s, v31.4s, v11.4s /* tmp13 - tmp0 */
|
|
|
|
shrn v2.4h, v18.4s, #16 /* wsptr[DCTSIZE*0] = (int) DESCALE(tmp10 + tmp3, CONST_BITS+PASS1_BITS+3) */
|
|
shrn v9.4h, v20.4s, #16 /* wsptr[DCTSIZE*7] = (int) DESCALE(tmp10 - tmp3, CONST_BITS+PASS1_BITS+3) */
|
|
shrn v3.4h, v22.4s, #16 /* wsptr[DCTSIZE*1] = (int) DESCALE(tmp11 + tmp2, CONST_BITS+PASS1_BITS+3) */
|
|
shrn v8.4h, v24.4s, #16 /* wsptr[DCTSIZE*6] = (int) DESCALE(tmp11 - tmp2, CONST_BITS+PASS1_BITS+3) */
|
|
shrn v4.4h, v26.4s, #16 /* wsptr[DCTSIZE*2] = (int) DESCALE(tmp12 + tmp1, CONST_BITS+PASS1_BITS+3) */
|
|
shrn v7.4h, v28.4s, #16 /* wsptr[DCTSIZE*5] = (int) DESCALE(tmp12 - tmp1, CONST_BITS+PASS1_BITS+3) */
|
|
shrn v5.4h, v14.4s, #16 /* wsptr[DCTSIZE*3] = (int) DESCALE(tmp13 + tmp0, CONST_BITS+PASS1_BITS+3) */
|
|
shrn v6.4h, v16.4s, #16 /* wsptr[DCTSIZE*4] = (int) DESCALE(tmp13 - tmp0, CONST_BITS+PASS1_BITS+3) */
|
|
shrn2 v2.8h, v19.4s, #16 /* wsptr[DCTSIZE*0] = (int) DESCALE(tmp10 + tmp3, CONST_BITS+PASS1_BITS+3) */
|
|
shrn2 v9.8h, v21.4s, #16 /* wsptr[DCTSIZE*7] = (int) DESCALE(tmp10 - tmp3, CONST_BITS+PASS1_BITS+3) */
|
|
shrn2 v3.8h, v23.4s, #16 /* wsptr[DCTSIZE*1] = (int) DESCALE(tmp11 + tmp2, CONST_BITS+PASS1_BITS+3) */
|
|
shrn2 v8.8h, v25.4s, #16 /* wsptr[DCTSIZE*6] = (int) DESCALE(tmp11 - tmp2, CONST_BITS+PASS1_BITS+3) */
|
|
shrn2 v4.8h, v27.4s, #16 /* wsptr[DCTSIZE*2] = (int) DESCALE(tmp12 + tmp1, CONST_BITS+PASS1_BITS+3) */
|
|
shrn2 v7.8h, v29.4s, #16 /* wsptr[DCTSIZE*5] = (int) DESCALE(tmp12 - tmp1, CONST_BITS+PASS1_BITS+3) */
|
|
shrn2 v5.8h, v15.4s, #16 /* wsptr[DCTSIZE*3] = (int) DESCALE(tmp13 + tmp0, CONST_BITS+PASS1_BITS+3) */
|
|
shrn2 v6.8h, v17.4s, #16 /* wsptr[DCTSIZE*4] = (int) DESCALE(tmp13 - tmp0, CONST_BITS+PASS1_BITS+3) */
|
|
movi v0.16b, #(CENTERJSAMPLE)
|
|
/* Prepare pointers (dual-issue with NEON instructions) */
|
|
ldp TMP1, TMP2, [OUTPUT_BUF], 16
|
|
sqrshrn v28.8b, v2.8h, #(CONST_BITS+PASS1_BITS+3-16)
|
|
ldp TMP3, TMP4, [OUTPUT_BUF], 16
|
|
sqrshrn v29.8b, v3.8h, #(CONST_BITS+PASS1_BITS+3-16)
|
|
add TMP1, TMP1, OUTPUT_COL
|
|
sqrshrn v30.8b, v4.8h, #(CONST_BITS+PASS1_BITS+3-16)
|
|
add TMP2, TMP2, OUTPUT_COL
|
|
sqrshrn v31.8b, v5.8h, #(CONST_BITS+PASS1_BITS+3-16)
|
|
add TMP3, TMP3, OUTPUT_COL
|
|
sqrshrn2 v28.16b, v6.8h, #(CONST_BITS+PASS1_BITS+3-16)
|
|
add TMP4, TMP4, OUTPUT_COL
|
|
sqrshrn2 v29.16b, v7.8h, #(CONST_BITS+PASS1_BITS+3-16)
|
|
ldp TMP5, TMP6, [OUTPUT_BUF], 16
|
|
sqrshrn2 v30.16b, v8.8h, #(CONST_BITS+PASS1_BITS+3-16)
|
|
ldp TMP7, TMP8, [OUTPUT_BUF], 16
|
|
sqrshrn2 v31.16b, v9.8h, #(CONST_BITS+PASS1_BITS+3-16)
|
|
add TMP5, TMP5, OUTPUT_COL
|
|
add v16.16b, v28.16b, v0.16b
|
|
add TMP6, TMP6, OUTPUT_COL
|
|
add v18.16b, v29.16b, v0.16b
|
|
add TMP7, TMP7, OUTPUT_COL
|
|
add v20.16b, v30.16b, v0.16b
|
|
add TMP8, TMP8, OUTPUT_COL
|
|
add v22.16b, v31.16b, v0.16b
|
|
|
|
/* Transpose the final 8-bit samples */
|
|
trn1 v28.16b, v16.16b, v18.16b
|
|
trn1 v30.16b, v20.16b, v22.16b
|
|
trn2 v29.16b, v16.16b, v18.16b
|
|
trn2 v31.16b, v20.16b, v22.16b
|
|
|
|
trn1 v16.8h, v28.8h, v30.8h
|
|
trn2 v18.8h, v28.8h, v30.8h
|
|
trn1 v20.8h, v29.8h, v31.8h
|
|
trn2 v22.8h, v29.8h, v31.8h
|
|
|
|
uzp1 v28.4s, v16.4s, v18.4s
|
|
uzp2 v30.4s, v16.4s, v18.4s
|
|
uzp1 v29.4s, v20.4s, v22.4s
|
|
uzp2 v31.4s, v20.4s, v22.4s
|
|
|
|
/* Store results to the output buffer */
|
|
st1 {v28.d}[0], [TMP1]
|
|
st1 {v29.d}[0], [TMP2]
|
|
st1 {v28.d}[1], [TMP3]
|
|
st1 {v29.d}[1], [TMP4]
|
|
st1 {v30.d}[0], [TMP5]
|
|
st1 {v31.d}[0], [TMP6]
|
|
st1 {v30.d}[1], [TMP7]
|
|
st1 {v31.d}[1], [TMP8]
|
|
ld1 {v8.8b, v9.8b, v10.8b, v11.8b}, [sp], #32
|
|
ld1 {v12.8b, v13.8b, v14.8b, v15.8b}, [sp], #32
|
|
blr x30
|
|
|
|
.balign 16
|
|
2:
|
|
mul v3.8h, v3.8h, v19.8h
|
|
mul v4.8h, v4.8h, v20.8h
|
|
mul v5.8h, v5.8h, v21.8h
|
|
add TMP4, xzr, TMP2, LSL #32
|
|
mul v6.8h, v6.8h, v22.8h
|
|
mul v7.8h, v7.8h, v23.8h
|
|
adds TMP3, xzr, TMP2, LSR #32
|
|
mul v8.8h, v8.8h, v24.8h
|
|
mul v9.8h, v9.8h, v25.8h
|
|
b.ne 3f
|
|
/* Right AC coef is zero */
|
|
dup v15.2d, v10.d[1]
|
|
/* Even part: reverse the even part of the forward DCT. */
|
|
add v18.4h, v4.4h, v8.4h /* z2 + z3 = DEQUANTIZE(inptr[DCTSIZE*2], quantptr[DCTSIZE*2]) + DEQUANTIZE(inptr[DCTSIZE*6], quantptr[DCTSIZE*6]) */
|
|
add v22.4h, v2.4h, v6.4h /* z2 + z3 = DEQUANTIZE(inptr[DCTSIZE*0], quantptr[DCTSIZE*0]) + DEQUANTIZE(inptr[DCTSIZE*4], quantptr[DCTSIZE*4]) */
|
|
sub v26.4h, v2.4h, v6.4h /* z2 - z3 = DEQUANTIZE(inptr[DCTSIZE*0], quantptr[DCTSIZE*0]) - DEQUANTIZE(inptr[DCTSIZE*4], quantptr[DCTSIZE*4]) */
|
|
smull v18.4s, v18.4h, XFIX_P_0_541 /* z1l z1 = MULTIPLY(z2 + z3, FIX_0_541196100); */
|
|
sshll v22.4s, v22.4h, #(CONST_BITS) /* tmp0l tmp0 = LEFT_SHIFT(z2 + z3, CONST_BITS); */
|
|
mov v20.16b, v18.16b /* tmp3 = z1 */
|
|
sshll v26.4s, v26.4h, #(CONST_BITS) /* tmp1l tmp1 = LEFT_SHIFT(z2 - z3, CONST_BITS); */
|
|
smlal v18.4s, v8.4h, XFIX_N_1_847 /* tmp2l tmp2 = z1 + MULTIPLY(z3, - FIX_1_847759065); */
|
|
smlal v20.4s, v4.4h, XFIX_P_0_765 /* tmp3l tmp3 = z1 + MULTIPLY(z2, FIX_0_765366865); */
|
|
add v2.4s, v22.4s, v20.4s /* tmp10l tmp10 = tmp0 + tmp3; */
|
|
sub v6.4s, v22.4s, v20.4s /* tmp13l tmp13 = tmp0 - tmp3; */
|
|
add v8.4s, v26.4s, v18.4s /* tmp11l tmp11 = tmp1 + tmp2; */
|
|
sub v4.4s, v26.4s, v18.4s /* tmp12l tmp12 = tmp1 - tmp2; */
|
|
|
|
/* Odd part per figure 8; the matrix is unitary and hence its
|
|
* transpose is its inverse. i0..i3 are y7,y5,y3,y1 respectively.
|
|
*/
|
|
|
|
add v22.4h, v9.4h, v5.4h /* z3 = tmp0 + tmp2 = DEQUANTIZE(inptr[DCTSIZE*7], quantptr[DCTSIZE*7]) + DEQUANTIZE(inptr[DCTSIZE*3], quantptr[DCTSIZE*3]) */
|
|
add v24.4h, v7.4h, v3.4h /* z4 = tmp1 + tmp3 = DEQUANTIZE(inptr[DCTSIZE*5], quantptr[DCTSIZE*5]) + DEQUANTIZE(inptr[DCTSIZE*1], quantptr[DCTSIZE*1]) */
|
|
add v18.4h, v9.4h, v3.4h /* z1 = tmp0 + tmp3 = DEQUANTIZE(inptr[DCTSIZE*7], quantptr[DCTSIZE*7]) + DEQUANTIZE(inptr[DCTSIZE*1], quantptr[DCTSIZE*1]) */
|
|
add v20.4h, v7.4h, v5.4h /* z2 = tmp1 + tmp2 = DEQUANTIZE(inptr[DCTSIZE*5], quantptr[DCTSIZE*5]) + DEQUANTIZE(inptr[DCTSIZE*3], quantptr[DCTSIZE*3]) */
|
|
add v26.4h, v22.4h, v24.4h /* z5 = z3 + z4 */
|
|
|
|
smull v10.4s, v9.4h, XFIX_P_0_298 /* tmp0 = MULTIPLY(tmp0, FIX_0_298631336) */
|
|
smull v12.4s, v7.4h, XFIX_P_2_053 /* tmp1 = MULTIPLY(tmp1, FIX_2_053119869) */
|
|
smull v14.4s, v5.4h, XFIX_P_3_072 /* tmp2 = MULTIPLY(tmp2, FIX_3_072711026) */
|
|
smull v16.4s, v3.4h, XFIX_P_1_501 /* tmp3 = MULTIPLY(tmp3, FIX_1_501321110) */
|
|
smull v26.4s, v26.4h, XFIX_P_1_175 /* z5l z5 = MULTIPLY(z3 + z4, FIX_1_175875602) */
|
|
smull v22.4s, v22.4h, XFIX_N_1_961 /* z3 = MULTIPLY(z3, - FIX_1_961570560) */
|
|
smull v24.4s, v24.4h, XFIX_N_0_390 /* z4 = MULTIPLY(z4, - FIX_0_390180644) */
|
|
smull v18.4s, v18.4h, XFIX_N_0_899 /* z1 = MULTIPLY(z1, - FIX_0_899976223) */
|
|
smull v20.4s, v20.4h, XFIX_N_2_562 /* z2 = MULTIPLY(z2, - FIX_2_562915447) */
|
|
|
|
add v22.4s, v22.4s, v26.4s /* z3 += z5 */
|
|
add v24.4s, v24.4s, v26.4s /* z4 += z5 */
|
|
|
|
add v10.4s, v10.4s, v18.4s /* tmp0 += z1 */
|
|
add v12.4s, v12.4s, v20.4s /* tmp1 += z2 */
|
|
add v14.4s, v14.4s, v20.4s /* tmp2 += z2 */
|
|
add v16.4s, v16.4s, v18.4s /* tmp3 += z1 */
|
|
|
|
add v10.4s, v10.4s, v22.4s /* tmp0 += z3 */
|
|
add v12.4s, v12.4s, v24.4s /* tmp1 += z4 */
|
|
add v16.4s, v16.4s, v24.4s /* tmp3 += z4 */
|
|
add v14.4s, v14.4s, v22.4s /* tmp2 += z3 */
|
|
|
|
/* Final output stage: inputs are tmp10..tmp13, tmp0..tmp3 */
|
|
|
|
add v18.4s, v2.4s, v16.4s /* tmp10 + tmp3 */
|
|
sub v20.4s, v2.4s, v16.4s /* tmp10 - tmp3 */
|
|
add v22.4s, v8.4s, v14.4s /* tmp11 + tmp2 */
|
|
sub v24.4s, v8.4s, v14.4s /* tmp11 - tmp2 */
|
|
add v26.4s, v4.4s, v12.4s /* tmp12 + tmp1 */
|
|
sub v28.4s, v4.4s, v12.4s /* tmp12 - tmp1 */
|
|
add v14.4s, v6.4s, v10.4s /* tmp13 + tmp0 */
|
|
sub v16.4s, v6.4s, v10.4s /* tmp13 - tmp0 */
|
|
|
|
rshrn v2.4h, v18.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*0] = (int) DESCALE(tmp10 + tmp3, CONST_BITS-PASS1_BITS) */
|
|
rshrn v3.4h, v22.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*1] = (int) DESCALE(tmp11 + tmp2, CONST_BITS-PASS1_BITS) */
|
|
rshrn v4.4h, v26.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*2] = (int) DESCALE(tmp12 + tmp1, CONST_BITS-PASS1_BITS) */
|
|
rshrn v5.4h, v14.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*3] = (int) DESCALE(tmp13 + tmp0, CONST_BITS-PASS1_BITS) */
|
|
rshrn2 v2.8h, v16.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*4] = (int) DESCALE(tmp13 - tmp0, CONST_BITS-PASS1_BITS) */
|
|
rshrn2 v3.8h, v28.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*5] = (int) DESCALE(tmp12 - tmp1, CONST_BITS-PASS1_BITS) */
|
|
rshrn2 v4.8h, v24.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*6] = (int) DESCALE(tmp11 - tmp2, CONST_BITS-PASS1_BITS) */
|
|
rshrn2 v5.8h, v20.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*7] = (int) DESCALE(tmp10 - tmp3, CONST_BITS-PASS1_BITS) */
|
|
mov v6.16b, v15.16b
|
|
mov v7.16b, v15.16b
|
|
mov v8.16b, v15.16b
|
|
mov v9.16b, v15.16b
|
|
b 1b
|
|
|
|
.balign 16
|
|
3:
|
|
cbnz TMP4, 4f
|
|
/* Left AC coef is zero */
|
|
dup v14.2d, v10.d[0]
|
|
/* Even part: reverse the even part of the forward DCT. */
|
|
add v18.8h, v4.8h, v8.8h /* z2 + z3 = DEQUANTIZE(inptr[DCTSIZE*2], quantptr[DCTSIZE*2]) + DEQUANTIZE(inptr[DCTSIZE*6], quantptr[DCTSIZE*6]) */
|
|
add v22.8h, v2.8h, v6.8h /* z2 + z3 = DEQUANTIZE(inptr[DCTSIZE*0], quantptr[DCTSIZE*0]) + DEQUANTIZE(inptr[DCTSIZE*4], quantptr[DCTSIZE*4]) */
|
|
smull2 v19.4s, v18.8h, XFIX_P_0_541 /* z1h z1 = MULTIPLY(z2 + z3, FIX_0_541196100); */
|
|
sub v26.8h, v2.8h, v6.8h /* z2 - z3 = DEQUANTIZE(inptr[DCTSIZE*0], quantptr[DCTSIZE*0]) - DEQUANTIZE(inptr[DCTSIZE*4], quantptr[DCTSIZE*4]) */
|
|
sshll2 v23.4s, v22.8h, #(CONST_BITS) /* tmp0h tmp0 = LEFT_SHIFT(z2 + z3, CONST_BITS); */
|
|
mov v21.16b, v19.16b /* tmp3 = z1 */
|
|
smlal2 v19.4s, v8.8h, XFIX_N_1_847 /* tmp2h tmp2 = z1 + MULTIPLY(z3, - FIX_1_847759065); */
|
|
sshll2 v27.4s, v26.8h, #(CONST_BITS) /* tmp1h tmp1 = LEFT_SHIFT(z2 - z3, CONST_BITS); */
|
|
smlal2 v21.4s, v4.8h, XFIX_P_0_765 /* tmp3h tmp3 = z1 + MULTIPLY(z2, FIX_0_765366865); */
|
|
add v28.4s, v23.4s, v21.4s /* tmp10h tmp10 = tmp0 + tmp3; */
|
|
sub v31.4s, v23.4s, v21.4s /* tmp13h tmp13 = tmp0 - tmp3; */
|
|
add v29.4s, v27.4s, v19.4s /* tmp11h tmp11 = tmp1 + tmp2; */
|
|
sub v30.4s, v27.4s, v19.4s /* tmp12h tmp12 = tmp1 - tmp2; */
|
|
|
|
/* Odd part per figure 8; the matrix is unitary and hence its
|
|
* transpose is its inverse. i0..i3 are y7,y5,y3,y1 respectively.
|
|
*/
|
|
|
|
add v22.8h, v9.8h, v5.8h /* z3 = tmp0 + tmp2 = DEQUANTIZE(inptr[DCTSIZE*7], quantptr[DCTSIZE*7]) + DEQUANTIZE(inptr[DCTSIZE*3], quantptr[DCTSIZE*3]) */
|
|
add v24.8h, v7.8h, v3.8h /* z4 = tmp1 + tmp3 = DEQUANTIZE(inptr[DCTSIZE*5], quantptr[DCTSIZE*5]) + DEQUANTIZE(inptr[DCTSIZE*1], quantptr[DCTSIZE*1]) */
|
|
add v18.8h, v9.8h, v3.8h /* z1 = tmp0 + tmp3 = DEQUANTIZE(inptr[DCTSIZE*7], quantptr[DCTSIZE*7]) + DEQUANTIZE(inptr[DCTSIZE*1], quantptr[DCTSIZE*1]) */
|
|
add v20.8h, v7.8h, v5.8h /* z2 = tmp1 + tmp2 = DEQUANTIZE(inptr[DCTSIZE*5], quantptr[DCTSIZE*5]) + DEQUANTIZE(inptr[DCTSIZE*3], quantptr[DCTSIZE*3]) */
|
|
add v26.8h, v22.8h, v24.8h /* z5 = z3 + z4 */
|
|
|
|
smull2 v11.4s, v9.8h, XFIX_P_0_298 /* tmp0 = MULTIPLY(tmp0, FIX_0_298631336) */
|
|
smull2 v13.4s, v7.8h, XFIX_P_2_053 /* tmp1 = MULTIPLY(tmp1, FIX_2_053119869) */
|
|
smull2 v15.4s, v5.8h, XFIX_P_3_072 /* tmp2 = MULTIPLY(tmp2, FIX_3_072711026) */
|
|
smull2 v17.4s, v3.8h, XFIX_P_1_501 /* tmp3 = MULTIPLY(tmp3, FIX_1_501321110) */
|
|
smull2 v27.4s, v26.8h, XFIX_P_1_175 /* z5h z5 = MULTIPLY(z3 + z4, FIX_1_175875602) */
|
|
smull2 v23.4s, v22.8h, XFIX_N_1_961 /* z3 = MULTIPLY(z3, - FIX_1_961570560) */
|
|
smull2 v25.4s, v24.8h, XFIX_N_0_390 /* z4 = MULTIPLY(z4, - FIX_0_390180644) */
|
|
smull2 v19.4s, v18.8h, XFIX_N_0_899 /* z1 = MULTIPLY(z1, - FIX_0_899976223) */
|
|
smull2 v21.4s, v20.8h, XFIX_N_2_562 /* z2 = MULTIPLY(z2, - FIX_2_562915447) */
|
|
|
|
add v23.4s, v23.4s, v27.4s /* z3 += z5 */
|
|
add v22.4s, v22.4s, v26.4s /* z3 += z5 */
|
|
add v25.4s, v25.4s, v27.4s /* z4 += z5 */
|
|
add v24.4s, v24.4s, v26.4s /* z4 += z5 */
|
|
|
|
add v11.4s, v11.4s, v19.4s /* tmp0 += z1 */
|
|
add v13.4s, v13.4s, v21.4s /* tmp1 += z2 */
|
|
add v15.4s, v15.4s, v21.4s /* tmp2 += z2 */
|
|
add v17.4s, v17.4s, v19.4s /* tmp3 += z1 */
|
|
|
|
add v11.4s, v11.4s, v23.4s /* tmp0 += z3 */
|
|
add v13.4s, v13.4s, v25.4s /* tmp1 += z4 */
|
|
add v17.4s, v17.4s, v25.4s /* tmp3 += z4 */
|
|
add v15.4s, v15.4s, v23.4s /* tmp2 += z3 */
|
|
|
|
/* Final output stage: inputs are tmp10..tmp13, tmp0..tmp3 */
|
|
|
|
add v19.4s, v28.4s, v17.4s /* tmp10 + tmp3 */
|
|
sub v21.4s, v28.4s, v17.4s /* tmp10 - tmp3 */
|
|
add v23.4s, v29.4s, v15.4s /* tmp11 + tmp2 */
|
|
sub v25.4s, v29.4s, v15.4s /* tmp11 - tmp2 */
|
|
add v27.4s, v30.4s, v13.4s /* tmp12 + tmp1 */
|
|
sub v29.4s, v30.4s, v13.4s /* tmp12 - tmp1 */
|
|
add v15.4s, v31.4s, v11.4s /* tmp13 + tmp0 */
|
|
sub v17.4s, v31.4s, v11.4s /* tmp13 - tmp0 */
|
|
|
|
mov v2.16b, v14.16b
|
|
mov v3.16b, v14.16b
|
|
mov v4.16b, v14.16b
|
|
mov v5.16b, v14.16b
|
|
rshrn v6.4h, v19.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*0] = (int) DESCALE(tmp10 + tmp3, CONST_BITS-PASS1_BITS) */
|
|
rshrn v7.4h, v23.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*1] = (int) DESCALE(tmp11 + tmp2, CONST_BITS-PASS1_BITS) */
|
|
rshrn v8.4h, v27.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*2] = (int) DESCALE(tmp12 + tmp1, CONST_BITS-PASS1_BITS) */
|
|
rshrn v9.4h, v15.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*3] = (int) DESCALE(tmp13 + tmp0, CONST_BITS-PASS1_BITS) */
|
|
rshrn2 v6.8h, v17.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*4] = (int) DESCALE(tmp13 - tmp0, CONST_BITS-PASS1_BITS) */
|
|
rshrn2 v7.8h, v29.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*5] = (int) DESCALE(tmp12 - tmp1, CONST_BITS-PASS1_BITS) */
|
|
rshrn2 v8.8h, v25.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*6] = (int) DESCALE(tmp11 - tmp2, CONST_BITS-PASS1_BITS) */
|
|
rshrn2 v9.8h, v21.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*7] = (int) DESCALE(tmp10 - tmp3, CONST_BITS-PASS1_BITS) */
|
|
b 1b
|
|
|
|
.balign 16
|
|
4:
|
|
/* "No" AC coef is zero */
|
|
/* Even part: reverse the even part of the forward DCT. */
|
|
add v18.8h, v4.8h, v8.8h /* z2 + z3 = DEQUANTIZE(inptr[DCTSIZE*2], quantptr[DCTSIZE*2]) + DEQUANTIZE(inptr[DCTSIZE*6], quantptr[DCTSIZE*6]) */
|
|
add v22.8h, v2.8h, v6.8h /* z2 + z3 = DEQUANTIZE(inptr[DCTSIZE*0], quantptr[DCTSIZE*0]) + DEQUANTIZE(inptr[DCTSIZE*4], quantptr[DCTSIZE*4]) */
|
|
smull2 v19.4s, v18.8h, XFIX_P_0_541 /* z1h z1 = MULTIPLY(z2 + z3, FIX_0_541196100); */
|
|
sub v26.8h, v2.8h, v6.8h /* z2 - z3 = DEQUANTIZE(inptr[DCTSIZE*0], quantptr[DCTSIZE*0]) - DEQUANTIZE(inptr[DCTSIZE*4], quantptr[DCTSIZE*4]) */
|
|
smull v18.4s, v18.4h, XFIX_P_0_541 /* z1l z1 = MULTIPLY(z2 + z3, FIX_0_541196100); */
|
|
sshll2 v23.4s, v22.8h, #(CONST_BITS) /* tmp0h tmp0 = LEFT_SHIFT(z2 + z3, CONST_BITS); */
|
|
mov v21.16b, v19.16b /* tmp3 = z1 */
|
|
mov v20.16b, v18.16b /* tmp3 = z1 */
|
|
smlal2 v19.4s, v8.8h, XFIX_N_1_847 /* tmp2h tmp2 = z1 + MULTIPLY(z3, - FIX_1_847759065); */
|
|
smlal v18.4s, v8.4h, XFIX_N_1_847 /* tmp2l tmp2 = z1 + MULTIPLY(z3, - FIX_1_847759065); */
|
|
sshll2 v27.4s, v26.8h, #(CONST_BITS) /* tmp1h tmp1 = LEFT_SHIFT(z2 - z3, CONST_BITS); */
|
|
smlal2 v21.4s, v4.8h, XFIX_P_0_765 /* tmp3h tmp3 = z1 + MULTIPLY(z2, FIX_0_765366865); */
|
|
smlal v20.4s, v4.4h, XFIX_P_0_765 /* tmp3l tmp3 = z1 + MULTIPLY(z2, FIX_0_765366865); */
|
|
sshll v22.4s, v22.4h, #(CONST_BITS) /* tmp0l tmp0 = LEFT_SHIFT(z2 + z3, CONST_BITS); */
|
|
sshll v26.4s, v26.4h, #(CONST_BITS) /* tmp1l tmp1 = LEFT_SHIFT(z2 - z3, CONST_BITS); */
|
|
add v2.4s, v22.4s, v20.4s /* tmp10l tmp10 = tmp0 + tmp3; */
|
|
sub v6.4s, v22.4s, v20.4s /* tmp13l tmp13 = tmp0 - tmp3; */
|
|
add v8.4s, v26.4s, v18.4s /* tmp11l tmp11 = tmp1 + tmp2; */
|
|
sub v4.4s, v26.4s, v18.4s /* tmp12l tmp12 = tmp1 - tmp2; */
|
|
add v28.4s, v23.4s, v21.4s /* tmp10h tmp10 = tmp0 + tmp3; */
|
|
sub v31.4s, v23.4s, v21.4s /* tmp13h tmp13 = tmp0 - tmp3; */
|
|
add v29.4s, v27.4s, v19.4s /* tmp11h tmp11 = tmp1 + tmp2; */
|
|
sub v30.4s, v27.4s, v19.4s /* tmp12h tmp12 = tmp1 - tmp2; */
|
|
|
|
/* Odd part per figure 8; the matrix is unitary and hence its
|
|
* transpose is its inverse. i0..i3 are y7,y5,y3,y1 respectively.
|
|
*/
|
|
|
|
add v22.8h, v9.8h, v5.8h /* z3 = tmp0 + tmp2 = DEQUANTIZE(inptr[DCTSIZE*7], quantptr[DCTSIZE*7]) + DEQUANTIZE(inptr[DCTSIZE*3], quantptr[DCTSIZE*3]) */
|
|
add v24.8h, v7.8h, v3.8h /* z4 = tmp1 + tmp3 = DEQUANTIZE(inptr[DCTSIZE*5], quantptr[DCTSIZE*5]) + DEQUANTIZE(inptr[DCTSIZE*1], quantptr[DCTSIZE*1]) */
|
|
add v18.8h, v9.8h, v3.8h /* z1 = tmp0 + tmp3 = DEQUANTIZE(inptr[DCTSIZE*7], quantptr[DCTSIZE*7]) + DEQUANTIZE(inptr[DCTSIZE*1], quantptr[DCTSIZE*1]) */
|
|
add v20.8h, v7.8h, v5.8h /* z2 = tmp1 + tmp2 = DEQUANTIZE(inptr[DCTSIZE*5], quantptr[DCTSIZE*5]) + DEQUANTIZE(inptr[DCTSIZE*3], quantptr[DCTSIZE*3]) */
|
|
add v26.8h, v22.8h, v24.8h /* z5 = z3 + z4 */
|
|
|
|
smull2 v11.4s, v9.8h, XFIX_P_0_298 /* tmp0 = MULTIPLY(tmp0, FIX_0_298631336) */
|
|
smull2 v13.4s, v7.8h, XFIX_P_2_053 /* tmp1 = MULTIPLY(tmp1, FIX_2_053119869) */
|
|
smull2 v15.4s, v5.8h, XFIX_P_3_072 /* tmp2 = MULTIPLY(tmp2, FIX_3_072711026) */
|
|
smull2 v17.4s, v3.8h, XFIX_P_1_501 /* tmp3 = MULTIPLY(tmp3, FIX_1_501321110) */
|
|
smull2 v27.4s, v26.8h, XFIX_P_1_175 /* z5h z5 = MULTIPLY(z3 + z4, FIX_1_175875602) */
|
|
smull2 v23.4s, v22.8h, XFIX_N_1_961 /* z3 = MULTIPLY(z3, - FIX_1_961570560) */
|
|
smull2 v25.4s, v24.8h, XFIX_N_0_390 /* z4 = MULTIPLY(z4, - FIX_0_390180644) */
|
|
smull2 v19.4s, v18.8h, XFIX_N_0_899 /* z1 = MULTIPLY(z1, - FIX_0_899976223) */
|
|
smull2 v21.4s, v20.8h, XFIX_N_2_562 /* z2 = MULTIPLY(z2, - FIX_2_562915447) */
|
|
|
|
smull v10.4s, v9.4h, XFIX_P_0_298 /* tmp0 = MULTIPLY(tmp0, FIX_0_298631336) */
|
|
smull v12.4s, v7.4h, XFIX_P_2_053 /* tmp1 = MULTIPLY(tmp1, FIX_2_053119869) */
|
|
smull v14.4s, v5.4h, XFIX_P_3_072 /* tmp2 = MULTIPLY(tmp2, FIX_3_072711026) */
|
|
smull v16.4s, v3.4h, XFIX_P_1_501 /* tmp3 = MULTIPLY(tmp3, FIX_1_501321110) */
|
|
smull v26.4s, v26.4h, XFIX_P_1_175 /* z5l z5 = MULTIPLY(z3 + z4, FIX_1_175875602) */
|
|
smull v22.4s, v22.4h, XFIX_N_1_961 /* z3 = MULTIPLY(z3, - FIX_1_961570560) */
|
|
smull v24.4s, v24.4h, XFIX_N_0_390 /* z4 = MULTIPLY(z4, - FIX_0_390180644) */
|
|
smull v18.4s, v18.4h, XFIX_N_0_899 /* z1 = MULTIPLY(z1, - FIX_0_899976223) */
|
|
smull v20.4s, v20.4h, XFIX_N_2_562 /* z2 = MULTIPLY(z2, - FIX_2_562915447) */
|
|
|
|
add v23.4s, v23.4s, v27.4s /* z3 += z5 */
|
|
add v22.4s, v22.4s, v26.4s /* z3 += z5 */
|
|
add v25.4s, v25.4s, v27.4s /* z4 += z5 */
|
|
add v24.4s, v24.4s, v26.4s /* z4 += z5 */
|
|
|
|
add v11.4s, v11.4s, v19.4s /* tmp0 += z1 */
|
|
add v10.4s, v10.4s, v18.4s /* tmp0 += z1 */
|
|
add v13.4s, v13.4s, v21.4s /* tmp1 += z2 */
|
|
add v12.4s, v12.4s, v20.4s /* tmp1 += z2 */
|
|
add v15.4s, v15.4s, v21.4s /* tmp2 += z2 */
|
|
add v14.4s, v14.4s, v20.4s /* tmp2 += z2 */
|
|
add v17.4s, v17.4s, v19.4s /* tmp3 += z1 */
|
|
add v16.4s, v16.4s, v18.4s /* tmp3 += z1 */
|
|
|
|
add v11.4s, v11.4s, v23.4s /* tmp0 += z3 */
|
|
add v10.4s, v10.4s, v22.4s /* tmp0 += z3 */
|
|
add v13.4s, v13.4s, v25.4s /* tmp1 += z4 */
|
|
add v12.4s, v12.4s, v24.4s /* tmp1 += z4 */
|
|
add v17.4s, v17.4s, v25.4s /* tmp3 += z4 */
|
|
add v16.4s, v16.4s, v24.4s /* tmp3 += z4 */
|
|
add v15.4s, v15.4s, v23.4s /* tmp2 += z3 */
|
|
add v14.4s, v14.4s, v22.4s /* tmp2 += z3 */
|
|
|
|
/* Final output stage: inputs are tmp10..tmp13, tmp0..tmp3 */
|
|
|
|
add v18.4s, v2.4s, v16.4s /* tmp10 + tmp3 */
|
|
add v19.4s, v28.4s, v17.4s /* tmp10 + tmp3 */
|
|
sub v20.4s, v2.4s, v16.4s /* tmp10 - tmp3 */
|
|
sub v21.4s, v28.4s, v17.4s /* tmp10 - tmp3 */
|
|
add v22.4s, v8.4s, v14.4s /* tmp11 + tmp2 */
|
|
add v23.4s, v29.4s, v15.4s /* tmp11 + tmp2 */
|
|
sub v24.4s, v8.4s, v14.4s /* tmp11 - tmp2 */
|
|
sub v25.4s, v29.4s, v15.4s /* tmp11 - tmp2 */
|
|
add v26.4s, v4.4s, v12.4s /* tmp12 + tmp1 */
|
|
add v27.4s, v30.4s, v13.4s /* tmp12 + tmp1 */
|
|
sub v28.4s, v4.4s, v12.4s /* tmp12 - tmp1 */
|
|
sub v29.4s, v30.4s, v13.4s /* tmp12 - tmp1 */
|
|
add v14.4s, v6.4s, v10.4s /* tmp13 + tmp0 */
|
|
add v15.4s, v31.4s, v11.4s /* tmp13 + tmp0 */
|
|
sub v16.4s, v6.4s, v10.4s /* tmp13 - tmp0 */
|
|
sub v17.4s, v31.4s, v11.4s /* tmp13 - tmp0 */
|
|
|
|
rshrn v2.4h, v18.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*0] = (int) DESCALE(tmp10 + tmp3, CONST_BITS-PASS1_BITS) */
|
|
rshrn v3.4h, v22.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*1] = (int) DESCALE(tmp11 + tmp2, CONST_BITS-PASS1_BITS) */
|
|
rshrn v4.4h, v26.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*2] = (int) DESCALE(tmp12 + tmp1, CONST_BITS-PASS1_BITS) */
|
|
rshrn v5.4h, v14.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*3] = (int) DESCALE(tmp13 + tmp0, CONST_BITS-PASS1_BITS) */
|
|
rshrn v6.4h, v19.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*0] = (int) DESCALE(tmp10 + tmp3, CONST_BITS-PASS1_BITS) */
|
|
rshrn v7.4h, v23.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*1] = (int) DESCALE(tmp11 + tmp2, CONST_BITS-PASS1_BITS) */
|
|
rshrn v8.4h, v27.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*2] = (int) DESCALE(tmp12 + tmp1, CONST_BITS-PASS1_BITS) */
|
|
rshrn v9.4h, v15.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*3] = (int) DESCALE(tmp13 + tmp0, CONST_BITS-PASS1_BITS) */
|
|
rshrn2 v2.8h, v16.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*4] = (int) DESCALE(tmp13 - tmp0, CONST_BITS-PASS1_BITS) */
|
|
rshrn2 v3.8h, v28.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*5] = (int) DESCALE(tmp12 - tmp1, CONST_BITS-PASS1_BITS) */
|
|
rshrn2 v4.8h, v24.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*6] = (int) DESCALE(tmp11 - tmp2, CONST_BITS-PASS1_BITS) */
|
|
rshrn2 v5.8h, v20.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*7] = (int) DESCALE(tmp10 - tmp3, CONST_BITS-PASS1_BITS) */
|
|
rshrn2 v6.8h, v17.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*4] = (int) DESCALE(tmp13 - tmp0, CONST_BITS-PASS1_BITS) */
|
|
rshrn2 v7.8h, v29.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*5] = (int) DESCALE(tmp12 - tmp1, CONST_BITS-PASS1_BITS) */
|
|
rshrn2 v8.8h, v25.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*6] = (int) DESCALE(tmp11 - tmp2, CONST_BITS-PASS1_BITS) */
|
|
rshrn2 v9.8h, v21.4s, #(CONST_BITS-PASS1_BITS) /* wsptr[DCTSIZE*7] = (int) DESCALE(tmp10 - tmp3, CONST_BITS-PASS1_BITS) */
|
|
b 1b
|
|
|
|
.unreq DCT_TABLE
|
|
.unreq COEF_BLOCK
|
|
.unreq OUTPUT_BUF
|
|
.unreq OUTPUT_COL
|
|
.unreq TMP1
|
|
.unreq TMP2
|
|
.unreq TMP3
|
|
.unreq TMP4
|
|
.unreq TMP5
|
|
.unreq TMP6
|
|
.unreq TMP7
|
|
.unreq TMP8
|
|
|
|
#undef CENTERJSAMPLE
|
|
#undef CONST_BITS
|
|
#undef PASS1_BITS
|
|
#undef XFIX_P_0_298
|
|
#undef XFIX_N_0_390
|
|
#undef XFIX_P_0_541
|
|
#undef XFIX_P_0_765
|
|
#undef XFIX_N_0_899
|
|
#undef XFIX_P_1_175
|
|
#undef XFIX_P_1_501
|
|
#undef XFIX_N_1_847
|
|
#undef XFIX_N_1_961
|
|
#undef XFIX_P_2_053
|
|
#undef XFIX_N_2_562
|
|
#undef XFIX_P_3_072
|
|
|
|
|
|
/*****************************************************************************/
|
|
|
|
/*
|
|
* jsimd_idct_ifast_neon
|
|
*
|
|
* This function contains a fast, not so accurate integer implementation of
|
|
* the inverse DCT (Discrete Cosine Transform). It uses the same calculations
|
|
* and produces exactly the same output as IJG's original 'jpeg_idct_ifast'
|
|
* function from jidctfst.c
|
|
*
|
|
* Normally 1-D AAN DCT needs 5 multiplications and 29 additions.
|
|
* But in ARM NEON case some extra additions are required because VQDMULH
|
|
* instruction can't handle the constants larger than 1. So the expressions
|
|
* like "x * 1.082392200" have to be converted to "x * 0.082392200 + x",
|
|
* which introduces an extra addition. Overall, there are 6 extra additions
|
|
* per 1-D IDCT pass, totalling to 5 VQDMULH and 35 VADD/VSUB instructions.
|
|
*/
|
|
|
|
#define XFIX_1_082392200 v0.h[0]
|
|
#define XFIX_1_414213562 v0.h[1]
|
|
#define XFIX_1_847759065 v0.h[2]
|
|
#define XFIX_2_613125930 v0.h[3]
|
|
|
|
.balign 16
|
|
Ljsimd_idct_ifast_neon_consts:
|
|
.short (277 * 128 - 256 * 128) /* XFIX_1_082392200 */
|
|
.short (362 * 128 - 256 * 128) /* XFIX_1_414213562 */
|
|
.short (473 * 128 - 256 * 128) /* XFIX_1_847759065 */
|
|
.short (669 * 128 - 512 * 128) /* XFIX_2_613125930 */
|
|
|
|
asm_function jsimd_idct_ifast_neon
|
|
|
|
DCT_TABLE .req x0
|
|
COEF_BLOCK .req x1
|
|
OUTPUT_BUF .req x2
|
|
OUTPUT_COL .req x3
|
|
TMP1 .req x0
|
|
TMP2 .req x1
|
|
TMP3 .req x9
|
|
TMP4 .req x10
|
|
TMP5 .req x11
|
|
TMP6 .req x12
|
|
TMP7 .req x13
|
|
TMP8 .req x14
|
|
|
|
/* OUTPUT_COL is a JDIMENSION (unsigned int) argument, so the ABI doesn't
|
|
guarantee that the upper (unused) 32 bits of x3 are valid. This
|
|
instruction ensures that those bits are set to zero. */
|
|
uxtw x3, w3
|
|
|
|
/* Load and dequantize coefficients into NEON registers
|
|
* with the following allocation:
|
|
* 0 1 2 3 | 4 5 6 7
|
|
* ---------+--------
|
|
* 0 | d16 | d17 ( v16.8h )
|
|
* 1 | d18 | d19 ( v17.8h )
|
|
* 2 | d20 | d21 ( v18.8h )
|
|
* 3 | d22 | d23 ( v19.8h )
|
|
* 4 | d24 | d25 ( v20.8h )
|
|
* 5 | d26 | d27 ( v21.8h )
|
|
* 6 | d28 | d29 ( v22.8h )
|
|
* 7 | d30 | d31 ( v23.8h )
|
|
*/
|
|
/* Save NEON registers used in fast IDCT */
|
|
adr TMP5, Ljsimd_idct_ifast_neon_consts
|
|
ld1 {v16.8h, v17.8h}, [COEF_BLOCK], 32
|
|
ld1 {v0.8h, v1.8h}, [DCT_TABLE], 32
|
|
ld1 {v18.8h, v19.8h}, [COEF_BLOCK], 32
|
|
mul v16.8h, v16.8h, v0.8h
|
|
ld1 {v2.8h, v3.8h}, [DCT_TABLE], 32
|
|
mul v17.8h, v17.8h, v1.8h
|
|
ld1 {v20.8h, v21.8h}, [COEF_BLOCK], 32
|
|
mul v18.8h, v18.8h, v2.8h
|
|
ld1 {v0.8h, v1.8h}, [DCT_TABLE], 32
|
|
mul v19.8h, v19.8h, v3.8h
|
|
ld1 {v22.8h, v23.8h}, [COEF_BLOCK], 32
|
|
mul v20.8h, v20.8h, v0.8h
|
|
ld1 {v2.8h, v3.8h}, [DCT_TABLE], 32
|
|
mul v22.8h, v22.8h, v2.8h
|
|
mul v21.8h, v21.8h, v1.8h
|
|
ld1 {v0.4h}, [TMP5] /* load constants */
|
|
mul v23.8h, v23.8h, v3.8h
|
|
|
|
/* 1-D IDCT, pass 1 */
|
|
sub v2.8h, v18.8h, v22.8h
|
|
add v22.8h, v18.8h, v22.8h
|
|
sub v1.8h, v19.8h, v21.8h
|
|
add v21.8h, v19.8h, v21.8h
|
|
sub v5.8h, v17.8h, v23.8h
|
|
add v23.8h, v17.8h, v23.8h
|
|
sqdmulh v4.8h, v2.8h, XFIX_1_414213562
|
|
sqdmulh v6.8h, v1.8h, XFIX_2_613125930
|
|
add v3.8h, v1.8h, v1.8h
|
|
sub v1.8h, v5.8h, v1.8h
|
|
add v18.8h, v2.8h, v4.8h
|
|
sqdmulh v4.8h, v1.8h, XFIX_1_847759065
|
|
sub v2.8h, v23.8h, v21.8h
|
|
add v3.8h, v3.8h, v6.8h
|
|
sqdmulh v6.8h, v2.8h, XFIX_1_414213562
|
|
add v1.8h, v1.8h, v4.8h
|
|
sqdmulh v4.8h, v5.8h, XFIX_1_082392200
|
|
sub v18.8h, v18.8h, v22.8h
|
|
add v2.8h, v2.8h, v6.8h
|
|
sub v6.8h, v16.8h, v20.8h
|
|
add v20.8h, v16.8h, v20.8h
|
|
add v17.8h, v5.8h, v4.8h
|
|
add v5.8h, v6.8h, v18.8h
|
|
sub v18.8h, v6.8h, v18.8h
|
|
add v6.8h, v23.8h, v21.8h
|
|
add v16.8h, v20.8h, v22.8h
|
|
sub v3.8h, v6.8h, v3.8h
|
|
sub v20.8h, v20.8h, v22.8h
|
|
sub v3.8h, v3.8h, v1.8h
|
|
sub v1.8h, v17.8h, v1.8h
|
|
add v2.8h, v3.8h, v2.8h
|
|
sub v23.8h, v16.8h, v6.8h
|
|
add v1.8h, v1.8h, v2.8h
|
|
add v16.8h, v16.8h, v6.8h
|
|
add v22.8h, v5.8h, v3.8h
|
|
sub v17.8h, v5.8h, v3.8h
|
|
sub v21.8h, v18.8h, v2.8h
|
|
add v18.8h, v18.8h, v2.8h
|
|
sub v19.8h, v20.8h, v1.8h
|
|
add v20.8h, v20.8h, v1.8h
|
|
transpose_8x8 v16, v17, v18, v19, v20, v21, v22, v23, v28, v29, v30, v31
|
|
/* 1-D IDCT, pass 2 */
|
|
sub v2.8h, v18.8h, v22.8h
|
|
add v22.8h, v18.8h, v22.8h
|
|
sub v1.8h, v19.8h, v21.8h
|
|
add v21.8h, v19.8h, v21.8h
|
|
sub v5.8h, v17.8h, v23.8h
|
|
add v23.8h, v17.8h, v23.8h
|
|
sqdmulh v4.8h, v2.8h, XFIX_1_414213562
|
|
sqdmulh v6.8h, v1.8h, XFIX_2_613125930
|
|
add v3.8h, v1.8h, v1.8h
|
|
sub v1.8h, v5.8h, v1.8h
|
|
add v18.8h, v2.8h, v4.8h
|
|
sqdmulh v4.8h, v1.8h, XFIX_1_847759065
|
|
sub v2.8h, v23.8h, v21.8h
|
|
add v3.8h, v3.8h, v6.8h
|
|
sqdmulh v6.8h, v2.8h, XFIX_1_414213562
|
|
add v1.8h, v1.8h, v4.8h
|
|
sqdmulh v4.8h, v5.8h, XFIX_1_082392200
|
|
sub v18.8h, v18.8h, v22.8h
|
|
add v2.8h, v2.8h, v6.8h
|
|
sub v6.8h, v16.8h, v20.8h
|
|
add v20.8h, v16.8h, v20.8h
|
|
add v17.8h, v5.8h, v4.8h
|
|
add v5.8h, v6.8h, v18.8h
|
|
sub v18.8h, v6.8h, v18.8h
|
|
add v6.8h, v23.8h, v21.8h
|
|
add v16.8h, v20.8h, v22.8h
|
|
sub v3.8h, v6.8h, v3.8h
|
|
sub v20.8h, v20.8h, v22.8h
|
|
sub v3.8h, v3.8h, v1.8h
|
|
sub v1.8h, v17.8h, v1.8h
|
|
add v2.8h, v3.8h, v2.8h
|
|
sub v23.8h, v16.8h, v6.8h
|
|
add v1.8h, v1.8h, v2.8h
|
|
add v16.8h, v16.8h, v6.8h
|
|
add v22.8h, v5.8h, v3.8h
|
|
sub v17.8h, v5.8h, v3.8h
|
|
sub v21.8h, v18.8h, v2.8h
|
|
add v18.8h, v18.8h, v2.8h
|
|
sub v19.8h, v20.8h, v1.8h
|
|
add v20.8h, v20.8h, v1.8h
|
|
/* Descale to 8-bit and range limit */
|
|
movi v0.16b, #0x80
|
|
/* Prepare pointers (dual-issue with NEON instructions) */
|
|
ldp TMP1, TMP2, [OUTPUT_BUF], 16
|
|
sqshrn v28.8b, v16.8h, #5
|
|
ldp TMP3, TMP4, [OUTPUT_BUF], 16
|
|
sqshrn v29.8b, v17.8h, #5
|
|
add TMP1, TMP1, OUTPUT_COL
|
|
sqshrn v30.8b, v18.8h, #5
|
|
add TMP2, TMP2, OUTPUT_COL
|
|
sqshrn v31.8b, v19.8h, #5
|
|
add TMP3, TMP3, OUTPUT_COL
|
|
sqshrn2 v28.16b, v20.8h, #5
|
|
add TMP4, TMP4, OUTPUT_COL
|
|
sqshrn2 v29.16b, v21.8h, #5
|
|
ldp TMP5, TMP6, [OUTPUT_BUF], 16
|
|
sqshrn2 v30.16b, v22.8h, #5
|
|
ldp TMP7, TMP8, [OUTPUT_BUF], 16
|
|
sqshrn2 v31.16b, v23.8h, #5
|
|
add TMP5, TMP5, OUTPUT_COL
|
|
add v16.16b, v28.16b, v0.16b
|
|
add TMP6, TMP6, OUTPUT_COL
|
|
add v18.16b, v29.16b, v0.16b
|
|
add TMP7, TMP7, OUTPUT_COL
|
|
add v20.16b, v30.16b, v0.16b
|
|
add TMP8, TMP8, OUTPUT_COL
|
|
add v22.16b, v31.16b, v0.16b
|
|
|
|
/* Transpose the final 8-bit samples */
|
|
trn1 v28.16b, v16.16b, v18.16b
|
|
trn1 v30.16b, v20.16b, v22.16b
|
|
trn2 v29.16b, v16.16b, v18.16b
|
|
trn2 v31.16b, v20.16b, v22.16b
|
|
|
|
trn1 v16.8h, v28.8h, v30.8h
|
|
trn2 v18.8h, v28.8h, v30.8h
|
|
trn1 v20.8h, v29.8h, v31.8h
|
|
trn2 v22.8h, v29.8h, v31.8h
|
|
|
|
uzp1 v28.4s, v16.4s, v18.4s
|
|
uzp2 v30.4s, v16.4s, v18.4s
|
|
uzp1 v29.4s, v20.4s, v22.4s
|
|
uzp2 v31.4s, v20.4s, v22.4s
|
|
|
|
/* Store results to the output buffer */
|
|
st1 {v28.d}[0], [TMP1]
|
|
st1 {v29.d}[0], [TMP2]
|
|
st1 {v28.d}[1], [TMP3]
|
|
st1 {v29.d}[1], [TMP4]
|
|
st1 {v30.d}[0], [TMP5]
|
|
st1 {v31.d}[0], [TMP6]
|
|
st1 {v30.d}[1], [TMP7]
|
|
st1 {v31.d}[1], [TMP8]
|
|
blr x30
|
|
|
|
.unreq DCT_TABLE
|
|
.unreq COEF_BLOCK
|
|
.unreq OUTPUT_BUF
|
|
.unreq OUTPUT_COL
|
|
.unreq TMP1
|
|
.unreq TMP2
|
|
.unreq TMP3
|
|
.unreq TMP4
|
|
.unreq TMP5
|
|
.unreq TMP6
|
|
.unreq TMP7
|
|
.unreq TMP8
|
|
|
|
|
|
/*****************************************************************************/
|
|
|
|
/*
|
|
* jsimd_idct_4x4_neon
|
|
*
|
|
* This function contains inverse-DCT code for getting reduced-size
|
|
* 4x4 pixels output from an 8x8 DCT block. It uses the same calculations
|
|
* and produces exactly the same output as IJG's original 'jpeg_idct_4x4'
|
|
* function from jpeg-6b (jidctred.c).
|
|
*
|
|
* NOTE: jpeg-8 has an improved implementation of 4x4 inverse-DCT, which
|
|
* requires much less arithmetic operations and hence should be faster.
|
|
* The primary purpose of this particular NEON optimized function is
|
|
* bit exact compatibility with jpeg-6b.
|
|
*
|
|
* TODO: a bit better instructions scheduling can be achieved by expanding
|
|
* idct_helper/transpose_4x4 macros and reordering instructions,
|
|
* but readability will suffer somewhat.
|
|
*/
|
|
|
|
#define CONST_BITS 13
|
|
|
|
#define FIX_0_211164243 (1730) /* FIX(0.211164243) */
|
|
#define FIX_0_509795579 (4176) /* FIX(0.509795579) */
|
|
#define FIX_0_601344887 (4926) /* FIX(0.601344887) */
|
|
#define FIX_0_720959822 (5906) /* FIX(0.720959822) */
|
|
#define FIX_0_765366865 (6270) /* FIX(0.765366865) */
|
|
#define FIX_0_850430095 (6967) /* FIX(0.850430095) */
|
|
#define FIX_0_899976223 (7373) /* FIX(0.899976223) */
|
|
#define FIX_1_061594337 (8697) /* FIX(1.061594337) */
|
|
#define FIX_1_272758580 (10426) /* FIX(1.272758580) */
|
|
#define FIX_1_451774981 (11893) /* FIX(1.451774981) */
|
|
#define FIX_1_847759065 (15137) /* FIX(1.847759065) */
|
|
#define FIX_2_172734803 (17799) /* FIX(2.172734803) */
|
|
#define FIX_2_562915447 (20995) /* FIX(2.562915447) */
|
|
#define FIX_3_624509785 (29692) /* FIX(3.624509785) */
|
|
|
|
.balign 16
|
|
Ljsimd_idct_4x4_neon_consts:
|
|
.short FIX_1_847759065 /* v0.h[0] */
|
|
.short -FIX_0_765366865 /* v0.h[1] */
|
|
.short -FIX_0_211164243 /* v0.h[2] */
|
|
.short FIX_1_451774981 /* v0.h[3] */
|
|
.short -FIX_2_172734803 /* d1[0] */
|
|
.short FIX_1_061594337 /* d1[1] */
|
|
.short -FIX_0_509795579 /* d1[2] */
|
|
.short -FIX_0_601344887 /* d1[3] */
|
|
.short FIX_0_899976223 /* v2.h[0] */
|
|
.short FIX_2_562915447 /* v2.h[1] */
|
|
.short 1 << (CONST_BITS+1) /* v2.h[2] */
|
|
.short 0 /* v2.h[3] */
|
|
|
|
.macro idct_helper x4, x6, x8, x10, x12, x14, x16, shift, y26, y27, y28, y29
|
|
smull v28.4s, \x4, v2.h[2]
|
|
smlal v28.4s, \x8, v0.h[0]
|
|
smlal v28.4s, \x14, v0.h[1]
|
|
|
|
smull v26.4s, \x16, v1.h[2]
|
|
smlal v26.4s, \x12, v1.h[3]
|
|
smlal v26.4s, \x10, v2.h[0]
|
|
smlal v26.4s, \x6, v2.h[1]
|
|
|
|
smull v30.4s, \x4, v2.h[2]
|
|
smlsl v30.4s, \x8, v0.h[0]
|
|
smlsl v30.4s, \x14, v0.h[1]
|
|
|
|
smull v24.4s, \x16, v0.h[2]
|
|
smlal v24.4s, \x12, v0.h[3]
|
|
smlal v24.4s, \x10, v1.h[0]
|
|
smlal v24.4s, \x6, v1.h[1]
|
|
|
|
add v20.4s, v28.4s, v26.4s
|
|
sub v28.4s, v28.4s, v26.4s
|
|
|
|
.if \shift > 16
|
|
srshr v20.4s, v20.4s, #\shift
|
|
srshr v28.4s, v28.4s, #\shift
|
|
xtn \y26, v20.4s
|
|
xtn \y29, v28.4s
|
|
.else
|
|
rshrn \y26, v20.4s, #\shift
|
|
rshrn \y29, v28.4s, #\shift
|
|
.endif
|
|
|
|
add v20.4s, v30.4s, v24.4s
|
|
sub v30.4s, v30.4s, v24.4s
|
|
|
|
.if \shift > 16
|
|
srshr v20.4s, v20.4s, #\shift
|
|
srshr v30.4s, v30.4s, #\shift
|
|
xtn \y27, v20.4s
|
|
xtn \y28, v30.4s
|
|
.else
|
|
rshrn \y27, v20.4s, #\shift
|
|
rshrn \y28, v30.4s, #\shift
|
|
.endif
|
|
.endm
|
|
|
|
asm_function jsimd_idct_4x4_neon
|
|
|
|
DCT_TABLE .req x0
|
|
COEF_BLOCK .req x1
|
|
OUTPUT_BUF .req x2
|
|
OUTPUT_COL .req x3
|
|
TMP1 .req x0
|
|
TMP2 .req x1
|
|
TMP3 .req x2
|
|
TMP4 .req x15
|
|
|
|
/* OUTPUT_COL is a JDIMENSION (unsigned int) argument, so the ABI doesn't
|
|
guarantee that the upper (unused) 32 bits of x3 are valid. This
|
|
instruction ensures that those bits are set to zero. */
|
|
uxtw x3, w3
|
|
|
|
/* Save all used NEON registers */
|
|
sub sp, sp, 64
|
|
mov x9, sp
|
|
/* Load constants (v3.4h is just used for padding) */
|
|
adr TMP4, Ljsimd_idct_4x4_neon_consts
|
|
st1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x9], 32
|
|
st1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x9], 32
|
|
ld1 {v0.4h, v1.4h, v2.4h, v3.4h}, [TMP4]
|
|
|
|
/* Load all COEF_BLOCK into NEON registers with the following allocation:
|
|
* 0 1 2 3 | 4 5 6 7
|
|
* ---------+--------
|
|
* 0 | v4.4h | v5.4h
|
|
* 1 | v6.4h | v7.4h
|
|
* 2 | v8.4h | v9.4h
|
|
* 3 | v10.4h | v11.4h
|
|
* 4 | - | -
|
|
* 5 | v12.4h | v13.4h
|
|
* 6 | v14.4h | v15.4h
|
|
* 7 | v16.4h | v17.4h
|
|
*/
|
|
ld1 {v4.4h, v5.4h, v6.4h, v7.4h}, [COEF_BLOCK], 32
|
|
ld1 {v8.4h, v9.4h, v10.4h, v11.4h}, [COEF_BLOCK], 32
|
|
add COEF_BLOCK, COEF_BLOCK, #16
|
|
ld1 {v12.4h, v13.4h, v14.4h, v15.4h}, [COEF_BLOCK], 32
|
|
ld1 {v16.4h, v17.4h}, [COEF_BLOCK], 16
|
|
/* dequantize */
|
|
ld1 {v18.4h, v19.4h, v20.4h, v21.4h}, [DCT_TABLE], 32
|
|
mul v4.4h, v4.4h, v18.4h
|
|
mul v5.4h, v5.4h, v19.4h
|
|
ins v4.d[1], v5.d[0] /* 128 bit q4 */
|
|
ld1 {v22.4h, v23.4h, v24.4h, v25.4h}, [DCT_TABLE], 32
|
|
mul v6.4h, v6.4h, v20.4h
|
|
mul v7.4h, v7.4h, v21.4h
|
|
ins v6.d[1], v7.d[0] /* 128 bit q6 */
|
|
mul v8.4h, v8.4h, v22.4h
|
|
mul v9.4h, v9.4h, v23.4h
|
|
ins v8.d[1], v9.d[0] /* 128 bit q8 */
|
|
add DCT_TABLE, DCT_TABLE, #16
|
|
ld1 {v26.4h, v27.4h, v28.4h, v29.4h}, [DCT_TABLE], 32
|
|
mul v10.4h, v10.4h, v24.4h
|
|
mul v11.4h, v11.4h, v25.4h
|
|
ins v10.d[1], v11.d[0] /* 128 bit q10 */
|
|
mul v12.4h, v12.4h, v26.4h
|
|
mul v13.4h, v13.4h, v27.4h
|
|
ins v12.d[1], v13.d[0] /* 128 bit q12 */
|
|
ld1 {v30.4h, v31.4h}, [DCT_TABLE], 16
|
|
mul v14.4h, v14.4h, v28.4h
|
|
mul v15.4h, v15.4h, v29.4h
|
|
ins v14.d[1], v15.d[0] /* 128 bit q14 */
|
|
mul v16.4h, v16.4h, v30.4h
|
|
mul v17.4h, v17.4h, v31.4h
|
|
ins v16.d[1], v17.d[0] /* 128 bit q16 */
|
|
|
|
/* Pass 1 */
|
|
idct_helper v4.4h, v6.4h, v8.4h, v10.4h, v12.4h, v14.4h, v16.4h, 12, \
|
|
v4.4h, v6.4h, v8.4h, v10.4h
|
|
transpose_4x4 v4, v6, v8, v10, v3
|
|
ins v10.d[1], v11.d[0]
|
|
idct_helper v5.4h, v7.4h, v9.4h, v11.4h, v13.4h, v15.4h, v17.4h, 12, \
|
|
v5.4h, v7.4h, v9.4h, v11.4h
|
|
transpose_4x4 v5, v7, v9, v11, v3
|
|
ins v10.d[1], v11.d[0]
|
|
|
|
/* Pass 2 */
|
|
idct_helper v4.4h, v6.4h, v8.4h, v10.4h, v7.4h, v9.4h, v11.4h, 19, \
|
|
v26.4h, v27.4h, v28.4h, v29.4h
|
|
transpose_4x4 v26, v27, v28, v29, v3
|
|
|
|
/* Range limit */
|
|
movi v30.8h, #0x80
|
|
ins v26.d[1], v27.d[0]
|
|
ins v28.d[1], v29.d[0]
|
|
add v26.8h, v26.8h, v30.8h
|
|
add v28.8h, v28.8h, v30.8h
|
|
sqxtun v26.8b, v26.8h
|
|
sqxtun v27.8b, v28.8h
|
|
|
|
/* Store results to the output buffer */
|
|
ldp TMP1, TMP2, [OUTPUT_BUF], 16
|
|
ldp TMP3, TMP4, [OUTPUT_BUF]
|
|
add TMP1, TMP1, OUTPUT_COL
|
|
add TMP2, TMP2, OUTPUT_COL
|
|
add TMP3, TMP3, OUTPUT_COL
|
|
add TMP4, TMP4, OUTPUT_COL
|
|
|
|
#if defined(__ARMEL__) && !RESPECT_STRICT_ALIGNMENT
|
|
/* We can use much less instructions on little endian systems if the
|
|
* OS kernel is not configured to trap unaligned memory accesses
|
|
*/
|
|
st1 {v26.s}[0], [TMP1], 4
|
|
st1 {v27.s}[0], [TMP3], 4
|
|
st1 {v26.s}[1], [TMP2], 4
|
|
st1 {v27.s}[1], [TMP4], 4
|
|
#else
|
|
st1 {v26.b}[0], [TMP1], 1
|
|
st1 {v27.b}[0], [TMP3], 1
|
|
st1 {v26.b}[1], [TMP1], 1
|
|
st1 {v27.b}[1], [TMP3], 1
|
|
st1 {v26.b}[2], [TMP1], 1
|
|
st1 {v27.b}[2], [TMP3], 1
|
|
st1 {v26.b}[3], [TMP1], 1
|
|
st1 {v27.b}[3], [TMP3], 1
|
|
|
|
st1 {v26.b}[4], [TMP2], 1
|
|
st1 {v27.b}[4], [TMP4], 1
|
|
st1 {v26.b}[5], [TMP2], 1
|
|
st1 {v27.b}[5], [TMP4], 1
|
|
st1 {v26.b}[6], [TMP2], 1
|
|
st1 {v27.b}[6], [TMP4], 1
|
|
st1 {v26.b}[7], [TMP2], 1
|
|
st1 {v27.b}[7], [TMP4], 1
|
|
#endif
|
|
|
|
/* vpop {v8.4h - v15.4h} ;not available */
|
|
ld1 {v8.8b, v9.8b, v10.8b, v11.8b}, [sp], 32
|
|
ld1 {v12.8b, v13.8b, v14.8b, v15.8b}, [sp], 32
|
|
blr x30
|
|
|
|
.unreq DCT_TABLE
|
|
.unreq COEF_BLOCK
|
|
.unreq OUTPUT_BUF
|
|
.unreq OUTPUT_COL
|
|
.unreq TMP1
|
|
.unreq TMP2
|
|
.unreq TMP3
|
|
.unreq TMP4
|
|
|
|
.purgem idct_helper
|
|
|
|
|
|
/*****************************************************************************/
|
|
|
|
/*
|
|
* jsimd_idct_2x2_neon
|
|
*
|
|
* This function contains inverse-DCT code for getting reduced-size
|
|
* 2x2 pixels output from an 8x8 DCT block. It uses the same calculations
|
|
* and produces exactly the same output as IJG's original 'jpeg_idct_2x2'
|
|
* function from jpeg-6b (jidctred.c).
|
|
*
|
|
* NOTE: jpeg-8 has an improved implementation of 2x2 inverse-DCT, which
|
|
* requires much less arithmetic operations and hence should be faster.
|
|
* The primary purpose of this particular NEON optimized function is
|
|
* bit exact compatibility with jpeg-6b.
|
|
*/
|
|
|
|
.balign 8
|
|
Ljsimd_idct_2x2_neon_consts:
|
|
.short -FIX_0_720959822 /* v14[0] */
|
|
.short FIX_0_850430095 /* v14[1] */
|
|
.short -FIX_1_272758580 /* v14[2] */
|
|
.short FIX_3_624509785 /* v14[3] */
|
|
|
|
.macro idct_helper x4, x6, x10, x12, x16, shift, y26, y27
|
|
sshll v15.4s, \x4, #15
|
|
smull v26.4s, \x6, v14.h[3]
|
|
smlal v26.4s, \x10, v14.h[2]
|
|
smlal v26.4s, \x12, v14.h[1]
|
|
smlal v26.4s, \x16, v14.h[0]
|
|
|
|
add v20.4s, v15.4s, v26.4s
|
|
sub v15.4s, v15.4s, v26.4s
|
|
|
|
.if \shift > 16
|
|
srshr v20.4s, v20.4s, #\shift
|
|
srshr v15.4s, v15.4s, #\shift
|
|
xtn \y26, v20.4s
|
|
xtn \y27, v15.4s
|
|
.else
|
|
rshrn \y26, v20.4s, #\shift
|
|
rshrn \y27, v15.4s, #\shift
|
|
.endif
|
|
.endm
|
|
|
|
asm_function jsimd_idct_2x2_neon
|
|
|
|
DCT_TABLE .req x0
|
|
COEF_BLOCK .req x1
|
|
OUTPUT_BUF .req x2
|
|
OUTPUT_COL .req x3
|
|
TMP1 .req x0
|
|
TMP2 .req x15
|
|
|
|
/* OUTPUT_COL is a JDIMENSION (unsigned int) argument, so the ABI doesn't
|
|
guarantee that the upper (unused) 32 bits of x3 are valid. This
|
|
instruction ensures that those bits are set to zero. */
|
|
uxtw x3, w3
|
|
|
|
/* vpush {v8.4h - v15.4h} ; not available */
|
|
sub sp, sp, 64
|
|
mov x9, sp
|
|
|
|
/* Load constants */
|
|
adr TMP2, Ljsimd_idct_2x2_neon_consts
|
|
st1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x9], 32
|
|
st1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x9], 32
|
|
ld1 {v14.4h}, [TMP2]
|
|
|
|
/* Load all COEF_BLOCK into NEON registers with the following allocation:
|
|
* 0 1 2 3 | 4 5 6 7
|
|
* ---------+--------
|
|
* 0 | v4.4h | v5.4h
|
|
* 1 | v6.4h | v7.4h
|
|
* 2 | - | -
|
|
* 3 | v10.4h | v11.4h
|
|
* 4 | - | -
|
|
* 5 | v12.4h | v13.4h
|
|
* 6 | - | -
|
|
* 7 | v16.4h | v17.4h
|
|
*/
|
|
ld1 {v4.4h, v5.4h, v6.4h, v7.4h}, [COEF_BLOCK], 32
|
|
add COEF_BLOCK, COEF_BLOCK, #16
|
|
ld1 {v10.4h, v11.4h}, [COEF_BLOCK], 16
|
|
add COEF_BLOCK, COEF_BLOCK, #16
|
|
ld1 {v12.4h, v13.4h}, [COEF_BLOCK], 16
|
|
add COEF_BLOCK, COEF_BLOCK, #16
|
|
ld1 {v16.4h, v17.4h}, [COEF_BLOCK], 16
|
|
/* Dequantize */
|
|
ld1 {v18.4h, v19.4h, v20.4h, v21.4h}, [DCT_TABLE], 32
|
|
mul v4.4h, v4.4h, v18.4h
|
|
mul v5.4h, v5.4h, v19.4h
|
|
ins v4.d[1], v5.d[0]
|
|
mul v6.4h, v6.4h, v20.4h
|
|
mul v7.4h, v7.4h, v21.4h
|
|
ins v6.d[1], v7.d[0]
|
|
add DCT_TABLE, DCT_TABLE, #16
|
|
ld1 {v24.4h, v25.4h}, [DCT_TABLE], 16
|
|
mul v10.4h, v10.4h, v24.4h
|
|
mul v11.4h, v11.4h, v25.4h
|
|
ins v10.d[1], v11.d[0]
|
|
add DCT_TABLE, DCT_TABLE, #16
|
|
ld1 {v26.4h, v27.4h}, [DCT_TABLE], 16
|
|
mul v12.4h, v12.4h, v26.4h
|
|
mul v13.4h, v13.4h, v27.4h
|
|
ins v12.d[1], v13.d[0]
|
|
add DCT_TABLE, DCT_TABLE, #16
|
|
ld1 {v30.4h, v31.4h}, [DCT_TABLE], 16
|
|
mul v16.4h, v16.4h, v30.4h
|
|
mul v17.4h, v17.4h, v31.4h
|
|
ins v16.d[1], v17.d[0]
|
|
|
|
/* Pass 1 */
|
|
#if 0
|
|
idct_helper v4.4h, v6.4h, v10.4h, v12.4h, v16.4h, 13, v4.4h, v6.4h
|
|
transpose_4x4 v4.4h, v6.4h, v8.4h, v10.4h
|
|
idct_helper v5.4h, v7.4h, v11.4h, v13.4h, v17.4h, 13, v5.4h, v7.4h
|
|
transpose_4x4 v5.4h, v7.4h, v9.4h, v11.4h
|
|
#else
|
|
smull v26.4s, v6.4h, v14.h[3]
|
|
smlal v26.4s, v10.4h, v14.h[2]
|
|
smlal v26.4s, v12.4h, v14.h[1]
|
|
smlal v26.4s, v16.4h, v14.h[0]
|
|
smull v24.4s, v7.4h, v14.h[3]
|
|
smlal v24.4s, v11.4h, v14.h[2]
|
|
smlal v24.4s, v13.4h, v14.h[1]
|
|
smlal v24.4s, v17.4h, v14.h[0]
|
|
sshll v15.4s, v4.4h, #15
|
|
sshll v30.4s, v5.4h, #15
|
|
add v20.4s, v15.4s, v26.4s
|
|
sub v15.4s, v15.4s, v26.4s
|
|
rshrn v4.4h, v20.4s, #13
|
|
rshrn v6.4h, v15.4s, #13
|
|
add v20.4s, v30.4s, v24.4s
|
|
sub v15.4s, v30.4s, v24.4s
|
|
rshrn v5.4h, v20.4s, #13
|
|
rshrn v7.4h, v15.4s, #13
|
|
ins v4.d[1], v5.d[0]
|
|
ins v6.d[1], v7.d[0]
|
|
transpose v4, v6, v3, .16b, .8h
|
|
transpose v6, v10, v3, .16b, .4s
|
|
ins v11.d[0], v10.d[1]
|
|
ins v7.d[0], v6.d[1]
|
|
#endif
|
|
|
|
/* Pass 2 */
|
|
idct_helper v4.4h, v6.4h, v10.4h, v7.4h, v11.4h, 20, v26.4h, v27.4h
|
|
|
|
/* Range limit */
|
|
movi v30.8h, #0x80
|
|
ins v26.d[1], v27.d[0]
|
|
add v26.8h, v26.8h, v30.8h
|
|
sqxtun v30.8b, v26.8h
|
|
ins v26.d[0], v30.d[0]
|
|
sqxtun v27.8b, v26.8h
|
|
|
|
/* Store results to the output buffer */
|
|
ldp TMP1, TMP2, [OUTPUT_BUF]
|
|
add TMP1, TMP1, OUTPUT_COL
|
|
add TMP2, TMP2, OUTPUT_COL
|
|
|
|
st1 {v26.b}[0], [TMP1], 1
|
|
st1 {v27.b}[4], [TMP1], 1
|
|
st1 {v26.b}[1], [TMP2], 1
|
|
st1 {v27.b}[5], [TMP2], 1
|
|
|
|
ld1 {v8.8b, v9.8b, v10.8b, v11.8b}, [sp], 32
|
|
ld1 {v12.8b, v13.8b, v14.8b, v15.8b}, [sp], 32
|
|
blr x30
|
|
|
|
.unreq DCT_TABLE
|
|
.unreq COEF_BLOCK
|
|
.unreq OUTPUT_BUF
|
|
.unreq OUTPUT_COL
|
|
.unreq TMP1
|
|
.unreq TMP2
|
|
|
|
.purgem idct_helper
|
|
|
|
|
|
/*****************************************************************************/
|
|
|
|
/*
|
|
* jsimd_ycc_extrgb_convert_neon
|
|
* jsimd_ycc_extbgr_convert_neon
|
|
* jsimd_ycc_extrgbx_convert_neon
|
|
* jsimd_ycc_extbgrx_convert_neon
|
|
* jsimd_ycc_extxbgr_convert_neon
|
|
* jsimd_ycc_extxrgb_convert_neon
|
|
*
|
|
* Colorspace conversion YCbCr -> RGB
|
|
*/
|
|
|
|
.macro do_load size
|
|
.if \size == 8
|
|
ld1 {v4.8b}, [U], 8
|
|
ld1 {v5.8b}, [V], 8
|
|
ld1 {v0.8b}, [Y], 8
|
|
prfm pldl1keep, [U, #64]
|
|
prfm pldl1keep, [V, #64]
|
|
prfm pldl1keep, [Y, #64]
|
|
.elseif \size == 4
|
|
ld1 {v4.b}[0], [U], 1
|
|
ld1 {v4.b}[1], [U], 1
|
|
ld1 {v4.b}[2], [U], 1
|
|
ld1 {v4.b}[3], [U], 1
|
|
ld1 {v5.b}[0], [V], 1
|
|
ld1 {v5.b}[1], [V], 1
|
|
ld1 {v5.b}[2], [V], 1
|
|
ld1 {v5.b}[3], [V], 1
|
|
ld1 {v0.b}[0], [Y], 1
|
|
ld1 {v0.b}[1], [Y], 1
|
|
ld1 {v0.b}[2], [Y], 1
|
|
ld1 {v0.b}[3], [Y], 1
|
|
.elseif \size == 2
|
|
ld1 {v4.b}[4], [U], 1
|
|
ld1 {v4.b}[5], [U], 1
|
|
ld1 {v5.b}[4], [V], 1
|
|
ld1 {v5.b}[5], [V], 1
|
|
ld1 {v0.b}[4], [Y], 1
|
|
ld1 {v0.b}[5], [Y], 1
|
|
.elseif \size == 1
|
|
ld1 {v4.b}[6], [U], 1
|
|
ld1 {v5.b}[6], [V], 1
|
|
ld1 {v0.b}[6], [Y], 1
|
|
.else
|
|
.error unsupported macroblock size
|
|
.endif
|
|
.endm
|
|
|
|
.macro do_store bpp, size, fast_st3
|
|
.if \bpp == 24
|
|
.if \size == 8
|
|
.if \fast_st3 == 1
|
|
st3 {v10.8b, v11.8b, v12.8b}, [RGB], 24
|
|
.else
|
|
st1 {v10.b}[0], [RGB], #1
|
|
st1 {v11.b}[0], [RGB], #1
|
|
st1 {v12.b}[0], [RGB], #1
|
|
|
|
st1 {v10.b}[1], [RGB], #1
|
|
st1 {v11.b}[1], [RGB], #1
|
|
st1 {v12.b}[1], [RGB], #1
|
|
|
|
st1 {v10.b}[2], [RGB], #1
|
|
st1 {v11.b}[2], [RGB], #1
|
|
st1 {v12.b}[2], [RGB], #1
|
|
|
|
st1 {v10.b}[3], [RGB], #1
|
|
st1 {v11.b}[3], [RGB], #1
|
|
st1 {v12.b}[3], [RGB], #1
|
|
|
|
st1 {v10.b}[4], [RGB], #1
|
|
st1 {v11.b}[4], [RGB], #1
|
|
st1 {v12.b}[4], [RGB], #1
|
|
|
|
st1 {v10.b}[5], [RGB], #1
|
|
st1 {v11.b}[5], [RGB], #1
|
|
st1 {v12.b}[5], [RGB], #1
|
|
|
|
st1 {v10.b}[6], [RGB], #1
|
|
st1 {v11.b}[6], [RGB], #1
|
|
st1 {v12.b}[6], [RGB], #1
|
|
|
|
st1 {v10.b}[7], [RGB], #1
|
|
st1 {v11.b}[7], [RGB], #1
|
|
st1 {v12.b}[7], [RGB], #1
|
|
.endif
|
|
.elseif \size == 4
|
|
st3 {v10.b, v11.b, v12.b}[0], [RGB], 3
|
|
st3 {v10.b, v11.b, v12.b}[1], [RGB], 3
|
|
st3 {v10.b, v11.b, v12.b}[2], [RGB], 3
|
|
st3 {v10.b, v11.b, v12.b}[3], [RGB], 3
|
|
.elseif \size == 2
|
|
st3 {v10.b, v11.b, v12.b}[4], [RGB], 3
|
|
st3 {v10.b, v11.b, v12.b}[5], [RGB], 3
|
|
.elseif \size == 1
|
|
st3 {v10.b, v11.b, v12.b}[6], [RGB], 3
|
|
.else
|
|
.error unsupported macroblock size
|
|
.endif
|
|
.elseif \bpp == 32
|
|
.if \size == 8
|
|
st4 {v10.8b, v11.8b, v12.8b, v13.8b}, [RGB], 32
|
|
.elseif \size == 4
|
|
st4 {v10.b, v11.b, v12.b, v13.b}[0], [RGB], 4
|
|
st4 {v10.b, v11.b, v12.b, v13.b}[1], [RGB], 4
|
|
st4 {v10.b, v11.b, v12.b, v13.b}[2], [RGB], 4
|
|
st4 {v10.b, v11.b, v12.b, v13.b}[3], [RGB], 4
|
|
.elseif \size == 2
|
|
st4 {v10.b, v11.b, v12.b, v13.b}[4], [RGB], 4
|
|
st4 {v10.b, v11.b, v12.b, v13.b}[5], [RGB], 4
|
|
.elseif \size == 1
|
|
st4 {v10.b, v11.b, v12.b, v13.b}[6], [RGB], 4
|
|
.else
|
|
.error unsupported macroblock size
|
|
.endif
|
|
.elseif \bpp==16
|
|
.if \size == 8
|
|
st1 {v25.8h}, [RGB], 16
|
|
.elseif \size == 4
|
|
st1 {v25.4h}, [RGB], 8
|
|
.elseif \size == 2
|
|
st1 {v25.h}[4], [RGB], 2
|
|
st1 {v25.h}[5], [RGB], 2
|
|
.elseif \size == 1
|
|
st1 {v25.h}[6], [RGB], 2
|
|
.else
|
|
.error unsupported macroblock size
|
|
.endif
|
|
.else
|
|
.error unsupported bpp
|
|
.endif
|
|
.endm
|
|
|
|
.macro generate_jsimd_ycc_rgb_convert_neon colorid, bpp, r_offs, rsize, \
|
|
g_offs, gsize, b_offs, bsize, \
|
|
defsize, fast_st3
|
|
|
|
/*
|
|
* 2-stage pipelined YCbCr->RGB conversion
|
|
*/
|
|
|
|
.macro do_yuv_to_rgb_stage1
|
|
uaddw v6.8h, v2.8h, v4.8b /* q3 = u - 128 */
|
|
uaddw v8.8h, v2.8h, v5.8b /* q2 = v - 128 */
|
|
smull v20.4s, v6.4h, v1.h[1] /* multiply by -11277 */
|
|
smlal v20.4s, v8.4h, v1.h[2] /* multiply by -23401 */
|
|
smull2 v22.4s, v6.8h, v1.h[1] /* multiply by -11277 */
|
|
smlal2 v22.4s, v8.8h, v1.h[2] /* multiply by -23401 */
|
|
smull v24.4s, v8.4h, v1.h[0] /* multiply by 22971 */
|
|
smull2 v26.4s, v8.8h, v1.h[0] /* multiply by 22971 */
|
|
smull v28.4s, v6.4h, v1.h[3] /* multiply by 29033 */
|
|
smull2 v30.4s, v6.8h, v1.h[3] /* multiply by 29033 */
|
|
.endm
|
|
|
|
.macro do_yuv_to_rgb_stage2
|
|
rshrn v20.4h, v20.4s, #15
|
|
rshrn2 v20.8h, v22.4s, #15
|
|
rshrn v24.4h, v24.4s, #14
|
|
rshrn2 v24.8h, v26.4s, #14
|
|
rshrn v28.4h, v28.4s, #14
|
|
rshrn2 v28.8h, v30.4s, #14
|
|
uaddw v20.8h, v20.8h, v0.8b
|
|
uaddw v24.8h, v24.8h, v0.8b
|
|
uaddw v28.8h, v28.8h, v0.8b
|
|
.if \bpp != 16
|
|
sqxtun v1\g_offs\defsize, v20.8h
|
|
sqxtun v1\r_offs\defsize, v24.8h
|
|
sqxtun v1\b_offs\defsize, v28.8h
|
|
.else
|
|
sqshlu v21.8h, v20.8h, #8
|
|
sqshlu v25.8h, v24.8h, #8
|
|
sqshlu v29.8h, v28.8h, #8
|
|
sri v25.8h, v21.8h, #5
|
|
sri v25.8h, v29.8h, #11
|
|
.endif
|
|
.endm
|
|
|
|
.macro do_yuv_to_rgb_stage2_store_load_stage1 fast_st3
|
|
rshrn v20.4h, v20.4s, #15
|
|
rshrn v24.4h, v24.4s, #14
|
|
rshrn v28.4h, v28.4s, #14
|
|
ld1 {v4.8b}, [U], 8
|
|
rshrn2 v20.8h, v22.4s, #15
|
|
rshrn2 v24.8h, v26.4s, #14
|
|
rshrn2 v28.8h, v30.4s, #14
|
|
ld1 {v5.8b}, [V], 8
|
|
uaddw v20.8h, v20.8h, v0.8b
|
|
uaddw v24.8h, v24.8h, v0.8b
|
|
uaddw v28.8h, v28.8h, v0.8b
|
|
.if \bpp != 16 /**************** rgb24/rgb32 ******************************/
|
|
sqxtun v1\g_offs\defsize, v20.8h
|
|
ld1 {v0.8b}, [Y], 8
|
|
sqxtun v1\r_offs\defsize, v24.8h
|
|
prfm pldl1keep, [U, #64]
|
|
prfm pldl1keep, [V, #64]
|
|
prfm pldl1keep, [Y, #64]
|
|
sqxtun v1\b_offs\defsize, v28.8h
|
|
uaddw v6.8h, v2.8h, v4.8b /* v6.16b = u - 128 */
|
|
uaddw v8.8h, v2.8h, v5.8b /* q2 = v - 128 */
|
|
smull v20.4s, v6.4h, v1.h[1] /* multiply by -11277 */
|
|
smlal v20.4s, v8.4h, v1.h[2] /* multiply by -23401 */
|
|
smull2 v22.4s, v6.8h, v1.h[1] /* multiply by -11277 */
|
|
smlal2 v22.4s, v8.8h, v1.h[2] /* multiply by -23401 */
|
|
smull v24.4s, v8.4h, v1.h[0] /* multiply by 22971 */
|
|
smull2 v26.4s, v8.8h, v1.h[0] /* multiply by 22971 */
|
|
.else /**************************** rgb565 ********************************/
|
|
sqshlu v21.8h, v20.8h, #8
|
|
sqshlu v25.8h, v24.8h, #8
|
|
sqshlu v29.8h, v28.8h, #8
|
|
uaddw v6.8h, v2.8h, v4.8b /* v6.16b = u - 128 */
|
|
uaddw v8.8h, v2.8h, v5.8b /* q2 = v - 128 */
|
|
ld1 {v0.8b}, [Y], 8
|
|
smull v20.4s, v6.4h, v1.h[1] /* multiply by -11277 */
|
|
smlal v20.4s, v8.4h, v1.h[2] /* multiply by -23401 */
|
|
smull2 v22.4s, v6.8h, v1.h[1] /* multiply by -11277 */
|
|
smlal2 v22.4s, v8.8h, v1.h[2] /* multiply by -23401 */
|
|
sri v25.8h, v21.8h, #5
|
|
smull v24.4s, v8.4h, v1.h[0] /* multiply by 22971 */
|
|
smull2 v26.4s, v8.8h, v1.h[0] /* multiply by 22971 */
|
|
prfm pldl1keep, [U, #64]
|
|
prfm pldl1keep, [V, #64]
|
|
prfm pldl1keep, [Y, #64]
|
|
sri v25.8h, v29.8h, #11
|
|
.endif
|
|
do_store \bpp, 8, \fast_st3
|
|
smull v28.4s, v6.4h, v1.h[3] /* multiply by 29033 */
|
|
smull2 v30.4s, v6.8h, v1.h[3] /* multiply by 29033 */
|
|
.endm
|
|
|
|
.macro do_yuv_to_rgb
|
|
do_yuv_to_rgb_stage1
|
|
do_yuv_to_rgb_stage2
|
|
.endm
|
|
|
|
/* Apple gas crashes on adrl, work around that by using adr.
|
|
* But this requires a copy of these constants for each function.
|
|
*/
|
|
|
|
.balign 16
|
|
.if \fast_st3 == 1
|
|
Ljsimd_ycc_\colorid\()_neon_consts:
|
|
.else
|
|
Ljsimd_ycc_\colorid\()_neon_slowst3_consts:
|
|
.endif
|
|
.short 0, 0, 0, 0
|
|
.short 22971, -11277, -23401, 29033
|
|
.short -128, -128, -128, -128
|
|
.short -128, -128, -128, -128
|
|
|
|
.if \fast_st3 == 1
|
|
asm_function jsimd_ycc_\colorid\()_convert_neon
|
|
.else
|
|
asm_function jsimd_ycc_\colorid\()_convert_neon_slowst3
|
|
.endif
|
|
OUTPUT_WIDTH .req w0
|
|
INPUT_BUF .req x1
|
|
INPUT_ROW .req w2
|
|
OUTPUT_BUF .req x3
|
|
NUM_ROWS .req w4
|
|
|
|
INPUT_BUF0 .req x5
|
|
INPUT_BUF1 .req x6
|
|
INPUT_BUF2 .req x1
|
|
|
|
RGB .req x7
|
|
Y .req x9
|
|
U .req x10
|
|
V .req x11
|
|
N .req w15
|
|
|
|
sub sp, sp, 64
|
|
mov x9, sp
|
|
|
|
/* Load constants to d1, d2, d3 (v0.4h is just used for padding) */
|
|
.if \fast_st3 == 1
|
|
adr x15, Ljsimd_ycc_\colorid\()_neon_consts
|
|
.else
|
|
adr x15, Ljsimd_ycc_\colorid\()_neon_slowst3_consts
|
|
.endif
|
|
|
|
/* Save NEON registers */
|
|
st1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x9], 32
|
|
st1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x9], 32
|
|
ld1 {v0.4h, v1.4h}, [x15], 16
|
|
ld1 {v2.8h}, [x15]
|
|
|
|
ldr INPUT_BUF0, [INPUT_BUF]
|
|
ldr INPUT_BUF1, [INPUT_BUF, #8]
|
|
ldr INPUT_BUF2, [INPUT_BUF, #16]
|
|
.unreq INPUT_BUF
|
|
|
|
/* Initially set v10, v11.4h, v12.8b, d13 to 0xFF */
|
|
movi v10.16b, #255
|
|
movi v13.16b, #255
|
|
|
|
/* Outer loop over scanlines */
|
|
cmp NUM_ROWS, #1
|
|
b.lt 9f
|
|
0:
|
|
ldr Y, [INPUT_BUF0, INPUT_ROW, uxtw #3]
|
|
ldr U, [INPUT_BUF1, INPUT_ROW, uxtw #3]
|
|
mov N, OUTPUT_WIDTH
|
|
ldr V, [INPUT_BUF2, INPUT_ROW, uxtw #3]
|
|
add INPUT_ROW, INPUT_ROW, #1
|
|
ldr RGB, [OUTPUT_BUF], #8
|
|
|
|
/* Inner loop over pixels */
|
|
subs N, N, #8
|
|
b.lt 3f
|
|
do_load 8
|
|
do_yuv_to_rgb_stage1
|
|
subs N, N, #8
|
|
b.lt 2f
|
|
1:
|
|
do_yuv_to_rgb_stage2_store_load_stage1 \fast_st3
|
|
subs N, N, #8
|
|
b.ge 1b
|
|
2:
|
|
do_yuv_to_rgb_stage2
|
|
do_store \bpp, 8, \fast_st3
|
|
tst N, #7
|
|
b.eq 8f
|
|
3:
|
|
tst N, #4
|
|
b.eq 3f
|
|
do_load 4
|
|
3:
|
|
tst N, #2
|
|
b.eq 4f
|
|
do_load 2
|
|
4:
|
|
tst N, #1
|
|
b.eq 5f
|
|
do_load 1
|
|
5:
|
|
do_yuv_to_rgb
|
|
tst N, #4
|
|
b.eq 6f
|
|
do_store \bpp, 4, \fast_st3
|
|
6:
|
|
tst N, #2
|
|
b.eq 7f
|
|
do_store \bpp, 2, \fast_st3
|
|
7:
|
|
tst N, #1
|
|
b.eq 8f
|
|
do_store \bpp, 1, \fast_st3
|
|
8:
|
|
subs NUM_ROWS, NUM_ROWS, #1
|
|
b.gt 0b
|
|
9:
|
|
/* Restore all registers and return */
|
|
ld1 {v8.8b, v9.8b, v10.8b, v11.8b}, [sp], 32
|
|
ld1 {v12.8b, v13.8b, v14.8b, v15.8b}, [sp], 32
|
|
br x30
|
|
.unreq OUTPUT_WIDTH
|
|
.unreq INPUT_ROW
|
|
.unreq OUTPUT_BUF
|
|
.unreq NUM_ROWS
|
|
.unreq INPUT_BUF0
|
|
.unreq INPUT_BUF1
|
|
.unreq INPUT_BUF2
|
|
.unreq RGB
|
|
.unreq Y
|
|
.unreq U
|
|
.unreq V
|
|
.unreq N
|
|
|
|
.purgem do_yuv_to_rgb
|
|
.purgem do_yuv_to_rgb_stage1
|
|
.purgem do_yuv_to_rgb_stage2
|
|
.purgem do_yuv_to_rgb_stage2_store_load_stage1
|
|
|
|
.endm
|
|
|
|
/*--------------------------------- id ----- bpp R rsize G gsize B bsize defsize fast_st3*/
|
|
generate_jsimd_ycc_rgb_convert_neon extrgb, 24, 0, .4h, 1, .4h, 2, .4h, .8b, 1
|
|
generate_jsimd_ycc_rgb_convert_neon extbgr, 24, 2, .4h, 1, .4h, 0, .4h, .8b, 1
|
|
generate_jsimd_ycc_rgb_convert_neon extrgbx, 32, 0, .4h, 1, .4h, 2, .4h, .8b, 1
|
|
generate_jsimd_ycc_rgb_convert_neon extbgrx, 32, 2, .4h, 1, .4h, 0, .4h, .8b, 1
|
|
generate_jsimd_ycc_rgb_convert_neon extxbgr, 32, 3, .4h, 2, .4h, 1, .4h, .8b, 1
|
|
generate_jsimd_ycc_rgb_convert_neon extxrgb, 32, 1, .4h, 2, .4h, 3, .4h, .8b, 1
|
|
generate_jsimd_ycc_rgb_convert_neon rgb565, 16, 0, .4h, 0, .4h, 0, .4h, .8b, 1
|
|
|
|
generate_jsimd_ycc_rgb_convert_neon extrgb, 24, 0, .4h, 1, .4h, 2, .4h, .8b, 0
|
|
generate_jsimd_ycc_rgb_convert_neon extbgr, 24, 2, .4h, 1, .4h, 0, .4h, .8b, 0
|
|
|
|
.purgem do_load
|
|
.purgem do_store
|
|
|
|
|
|
/*****************************************************************************/
|
|
|
|
/*
|
|
* jsimd_extrgb_ycc_convert_neon
|
|
* jsimd_extbgr_ycc_convert_neon
|
|
* jsimd_extrgbx_ycc_convert_neon
|
|
* jsimd_extbgrx_ycc_convert_neon
|
|
* jsimd_extxbgr_ycc_convert_neon
|
|
* jsimd_extxrgb_ycc_convert_neon
|
|
*
|
|
* Colorspace conversion RGB -> YCbCr
|
|
*/
|
|
|
|
.macro do_store size
|
|
.if \size == 8
|
|
st1 {v20.8b}, [Y], #8
|
|
st1 {v21.8b}, [U], #8
|
|
st1 {v22.8b}, [V], #8
|
|
.elseif \size == 4
|
|
st1 {v20.b}[0], [Y], #1
|
|
st1 {v20.b}[1], [Y], #1
|
|
st1 {v20.b}[2], [Y], #1
|
|
st1 {v20.b}[3], [Y], #1
|
|
st1 {v21.b}[0], [U], #1
|
|
st1 {v21.b}[1], [U], #1
|
|
st1 {v21.b}[2], [U], #1
|
|
st1 {v21.b}[3], [U], #1
|
|
st1 {v22.b}[0], [V], #1
|
|
st1 {v22.b}[1], [V], #1
|
|
st1 {v22.b}[2], [V], #1
|
|
st1 {v22.b}[3], [V], #1
|
|
.elseif \size == 2
|
|
st1 {v20.b}[4], [Y], #1
|
|
st1 {v20.b}[5], [Y], #1
|
|
st1 {v21.b}[4], [U], #1
|
|
st1 {v21.b}[5], [U], #1
|
|
st1 {v22.b}[4], [V], #1
|
|
st1 {v22.b}[5], [V], #1
|
|
.elseif \size == 1
|
|
st1 {v20.b}[6], [Y], #1
|
|
st1 {v21.b}[6], [U], #1
|
|
st1 {v22.b}[6], [V], #1
|
|
.else
|
|
.error unsupported macroblock size
|
|
.endif
|
|
.endm
|
|
|
|
.macro do_load bpp, size, fast_ld3
|
|
.if \bpp == 24
|
|
.if \size == 8
|
|
.if \fast_ld3 == 1
|
|
ld3 {v10.8b, v11.8b, v12.8b}, [RGB], #24
|
|
.else
|
|
ld1 {v10.b}[0], [RGB], #1
|
|
ld1 {v11.b}[0], [RGB], #1
|
|
ld1 {v12.b}[0], [RGB], #1
|
|
|
|
ld1 {v10.b}[1], [RGB], #1
|
|
ld1 {v11.b}[1], [RGB], #1
|
|
ld1 {v12.b}[1], [RGB], #1
|
|
|
|
ld1 {v10.b}[2], [RGB], #1
|
|
ld1 {v11.b}[2], [RGB], #1
|
|
ld1 {v12.b}[2], [RGB], #1
|
|
|
|
ld1 {v10.b}[3], [RGB], #1
|
|
ld1 {v11.b}[3], [RGB], #1
|
|
ld1 {v12.b}[3], [RGB], #1
|
|
|
|
ld1 {v10.b}[4], [RGB], #1
|
|
ld1 {v11.b}[4], [RGB], #1
|
|
ld1 {v12.b}[4], [RGB], #1
|
|
|
|
ld1 {v10.b}[5], [RGB], #1
|
|
ld1 {v11.b}[5], [RGB], #1
|
|
ld1 {v12.b}[5], [RGB], #1
|
|
|
|
ld1 {v10.b}[6], [RGB], #1
|
|
ld1 {v11.b}[6], [RGB], #1
|
|
ld1 {v12.b}[6], [RGB], #1
|
|
|
|
ld1 {v10.b}[7], [RGB], #1
|
|
ld1 {v11.b}[7], [RGB], #1
|
|
ld1 {v12.b}[7], [RGB], #1
|
|
.endif
|
|
prfm pldl1keep, [RGB, #128]
|
|
.elseif \size == 4
|
|
ld3 {v10.b, v11.b, v12.b}[0], [RGB], #3
|
|
ld3 {v10.b, v11.b, v12.b}[1], [RGB], #3
|
|
ld3 {v10.b, v11.b, v12.b}[2], [RGB], #3
|
|
ld3 {v10.b, v11.b, v12.b}[3], [RGB], #3
|
|
.elseif \size == 2
|
|
ld3 {v10.b, v11.b, v12.b}[4], [RGB], #3
|
|
ld3 {v10.b, v11.b, v12.b}[5], [RGB], #3
|
|
.elseif \size == 1
|
|
ld3 {v10.b, v11.b, v12.b}[6], [RGB], #3
|
|
.else
|
|
.error unsupported macroblock size
|
|
.endif
|
|
.elseif \bpp == 32
|
|
.if \size == 8
|
|
ld4 {v10.8b, v11.8b, v12.8b, v13.8b}, [RGB], #32
|
|
prfm pldl1keep, [RGB, #128]
|
|
.elseif \size == 4
|
|
ld4 {v10.b, v11.b, v12.b, v13.b}[0], [RGB], #4
|
|
ld4 {v10.b, v11.b, v12.b, v13.b}[1], [RGB], #4
|
|
ld4 {v10.b, v11.b, v12.b, v13.b}[2], [RGB], #4
|
|
ld4 {v10.b, v11.b, v12.b, v13.b}[3], [RGB], #4
|
|
.elseif \size == 2
|
|
ld4 {v10.b, v11.b, v12.b, v13.b}[4], [RGB], #4
|
|
ld4 {v10.b, v11.b, v12.b, v13.b}[5], [RGB], #4
|
|
.elseif \size == 1
|
|
ld4 {v10.b, v11.b, v12.b, v13.b}[6], [RGB], #4
|
|
.else
|
|
.error unsupported macroblock size
|
|
.endif
|
|
.else
|
|
.error unsupported bpp
|
|
.endif
|
|
.endm
|
|
|
|
.macro generate_jsimd_rgb_ycc_convert_neon colorid, bpp, r_offs, g_offs, \
|
|
b_offs, fast_ld3
|
|
|
|
/*
|
|
* 2-stage pipelined RGB->YCbCr conversion
|
|
*/
|
|
|
|
.macro do_rgb_to_yuv_stage1
|
|
ushll v4.8h, v1\r_offs\().8b, #0 /* r = v4 */
|
|
ushll v6.8h, v1\g_offs\().8b, #0 /* g = v6 */
|
|
ushll v8.8h, v1\b_offs\().8b, #0 /* b = v8 */
|
|
rev64 v18.4s, v1.4s
|
|
rev64 v26.4s, v1.4s
|
|
rev64 v28.4s, v1.4s
|
|
rev64 v30.4s, v1.4s
|
|
umull v14.4s, v4.4h, v0.h[0]
|
|
umull2 v16.4s, v4.8h, v0.h[0]
|
|
umlsl v18.4s, v4.4h, v0.h[3]
|
|
umlsl2 v26.4s, v4.8h, v0.h[3]
|
|
umlal v28.4s, v4.4h, v0.h[5]
|
|
umlal2 v30.4s, v4.8h, v0.h[5]
|
|
umlal v14.4s, v6.4h, v0.h[1]
|
|
umlal2 v16.4s, v6.8h, v0.h[1]
|
|
umlsl v18.4s, v6.4h, v0.h[4]
|
|
umlsl2 v26.4s, v6.8h, v0.h[4]
|
|
umlsl v28.4s, v6.4h, v0.h[6]
|
|
umlsl2 v30.4s, v6.8h, v0.h[6]
|
|
umlal v14.4s, v8.4h, v0.h[2]
|
|
umlal2 v16.4s, v8.8h, v0.h[2]
|
|
umlal v18.4s, v8.4h, v0.h[5]
|
|
umlal2 v26.4s, v8.8h, v0.h[5]
|
|
umlsl v28.4s, v8.4h, v0.h[7]
|
|
umlsl2 v30.4s, v8.8h, v0.h[7]
|
|
.endm
|
|
|
|
.macro do_rgb_to_yuv_stage2
|
|
rshrn v20.4h, v14.4s, #16
|
|
shrn v22.4h, v18.4s, #16
|
|
shrn v24.4h, v28.4s, #16
|
|
rshrn2 v20.8h, v16.4s, #16
|
|
shrn2 v22.8h, v26.4s, #16
|
|
shrn2 v24.8h, v30.4s, #16
|
|
xtn v20.8b, v20.8h /* v20 = y */
|
|
xtn v21.8b, v22.8h /* v21 = u */
|
|
xtn v22.8b, v24.8h /* v22 = v */
|
|
.endm
|
|
|
|
.macro do_rgb_to_yuv
|
|
do_rgb_to_yuv_stage1
|
|
do_rgb_to_yuv_stage2
|
|
.endm
|
|
|
|
/* TODO: expand macros and interleave instructions if some in-order
|
|
* ARM64 processor actually can dual-issue LOAD/STORE with ALU */
|
|
.macro do_rgb_to_yuv_stage2_store_load_stage1 fast_ld3
|
|
do_rgb_to_yuv_stage2
|
|
do_load \bpp, 8, \fast_ld3
|
|
st1 {v20.8b}, [Y], #8
|
|
st1 {v21.8b}, [U], #8
|
|
st1 {v22.8b}, [V], #8
|
|
do_rgb_to_yuv_stage1
|
|
.endm
|
|
|
|
.balign 16
|
|
.if \fast_ld3 == 1
|
|
Ljsimd_\colorid\()_ycc_neon_consts:
|
|
.else
|
|
Ljsimd_\colorid\()_ycc_neon_slowld3_consts:
|
|
.endif
|
|
.short 19595, 38470, 7471, 11059
|
|
.short 21709, 32768, 27439, 5329
|
|
.short 32767, 128, 32767, 128
|
|
.short 32767, 128, 32767, 128
|
|
|
|
.if \fast_ld3 == 1
|
|
asm_function jsimd_\colorid\()_ycc_convert_neon
|
|
.else
|
|
asm_function jsimd_\colorid\()_ycc_convert_neon_slowld3
|
|
.endif
|
|
OUTPUT_WIDTH .req w0
|
|
INPUT_BUF .req x1
|
|
OUTPUT_BUF .req x2
|
|
OUTPUT_ROW .req w3
|
|
NUM_ROWS .req w4
|
|
|
|
OUTPUT_BUF0 .req x5
|
|
OUTPUT_BUF1 .req x6
|
|
OUTPUT_BUF2 .req x2 /* OUTPUT_BUF */
|
|
|
|
RGB .req x7
|
|
Y .req x9
|
|
U .req x10
|
|
V .req x11
|
|
N .req w12
|
|
|
|
/* Load constants to d0, d1, d2, d3 */
|
|
.if \fast_ld3 == 1
|
|
adr x13, Ljsimd_\colorid\()_ycc_neon_consts
|
|
.else
|
|
adr x13, Ljsimd_\colorid\()_ycc_neon_slowld3_consts
|
|
.endif
|
|
ld1 {v0.8h, v1.8h}, [x13]
|
|
|
|
ldr OUTPUT_BUF0, [OUTPUT_BUF]
|
|
ldr OUTPUT_BUF1, [OUTPUT_BUF, #8]
|
|
ldr OUTPUT_BUF2, [OUTPUT_BUF, #16]
|
|
.unreq OUTPUT_BUF
|
|
|
|
/* Save NEON registers */
|
|
sub sp, sp, #64
|
|
mov x9, sp
|
|
st1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x9], 32
|
|
st1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x9], 32
|
|
|
|
/* Outer loop over scanlines */
|
|
cmp NUM_ROWS, #1
|
|
b.lt 9f
|
|
0:
|
|
ldr Y, [OUTPUT_BUF0, OUTPUT_ROW, uxtw #3]
|
|
ldr U, [OUTPUT_BUF1, OUTPUT_ROW, uxtw #3]
|
|
mov N, OUTPUT_WIDTH
|
|
ldr V, [OUTPUT_BUF2, OUTPUT_ROW, uxtw #3]
|
|
add OUTPUT_ROW, OUTPUT_ROW, #1
|
|
ldr RGB, [INPUT_BUF], #8
|
|
|
|
/* Inner loop over pixels */
|
|
subs N, N, #8
|
|
b.lt 3f
|
|
do_load \bpp, 8, \fast_ld3
|
|
do_rgb_to_yuv_stage1
|
|
subs N, N, #8
|
|
b.lt 2f
|
|
1:
|
|
do_rgb_to_yuv_stage2_store_load_stage1 \fast_ld3
|
|
subs N, N, #8
|
|
b.ge 1b
|
|
2:
|
|
do_rgb_to_yuv_stage2
|
|
do_store 8
|
|
tst N, #7
|
|
b.eq 8f
|
|
3:
|
|
tbz N, #2, 3f
|
|
do_load \bpp, 4, \fast_ld3
|
|
3:
|
|
tbz N, #1, 4f
|
|
do_load \bpp, 2, \fast_ld3
|
|
4:
|
|
tbz N, #0, 5f
|
|
do_load \bpp, 1, \fast_ld3
|
|
5:
|
|
do_rgb_to_yuv
|
|
tbz N, #2, 6f
|
|
do_store 4
|
|
6:
|
|
tbz N, #1, 7f
|
|
do_store 2
|
|
7:
|
|
tbz N, #0, 8f
|
|
do_store 1
|
|
8:
|
|
subs NUM_ROWS, NUM_ROWS, #1
|
|
b.gt 0b
|
|
9:
|
|
/* Restore all registers and return */
|
|
ld1 {v8.8b, v9.8b, v10.8b, v11.8b}, [sp], 32
|
|
ld1 {v12.8b, v13.8b, v14.8b, v15.8b}, [sp], 32
|
|
br x30
|
|
|
|
.unreq OUTPUT_WIDTH
|
|
.unreq OUTPUT_ROW
|
|
.unreq INPUT_BUF
|
|
.unreq NUM_ROWS
|
|
.unreq OUTPUT_BUF0
|
|
.unreq OUTPUT_BUF1
|
|
.unreq OUTPUT_BUF2
|
|
.unreq RGB
|
|
.unreq Y
|
|
.unreq U
|
|
.unreq V
|
|
.unreq N
|
|
|
|
.purgem do_rgb_to_yuv
|
|
.purgem do_rgb_to_yuv_stage1
|
|
.purgem do_rgb_to_yuv_stage2
|
|
.purgem do_rgb_to_yuv_stage2_store_load_stage1
|
|
|
|
.endm
|
|
|
|
/*--------------------------------- id ----- bpp R G B Fast LD3 */
|
|
generate_jsimd_rgb_ycc_convert_neon extrgb, 24, 0, 1, 2, 1
|
|
generate_jsimd_rgb_ycc_convert_neon extbgr, 24, 2, 1, 0, 1
|
|
generate_jsimd_rgb_ycc_convert_neon extrgbx, 32, 0, 1, 2, 1
|
|
generate_jsimd_rgb_ycc_convert_neon extbgrx, 32, 2, 1, 0, 1
|
|
generate_jsimd_rgb_ycc_convert_neon extxbgr, 32, 3, 2, 1, 1
|
|
generate_jsimd_rgb_ycc_convert_neon extxrgb, 32, 1, 2, 3, 1
|
|
|
|
generate_jsimd_rgb_ycc_convert_neon extrgb, 24, 0, 1, 2, 0
|
|
generate_jsimd_rgb_ycc_convert_neon extbgr, 24, 2, 1, 0, 0
|
|
|
|
.purgem do_load
|
|
.purgem do_store
|
|
|
|
|
|
/*****************************************************************************/
|
|
|
|
/*
|
|
* Load data into workspace, applying unsigned->signed conversion
|
|
*
|
|
* TODO: can be combined with 'jsimd_fdct_ifast_neon' to get
|
|
* rid of VST1.16 instructions
|
|
*/
|
|
|
|
asm_function jsimd_convsamp_neon
|
|
SAMPLE_DATA .req x0
|
|
START_COL .req x1
|
|
WORKSPACE .req x2
|
|
TMP1 .req x9
|
|
TMP2 .req x10
|
|
TMP3 .req x11
|
|
TMP4 .req x12
|
|
TMP5 .req x13
|
|
TMP6 .req x14
|
|
TMP7 .req x15
|
|
TMP8 .req x4
|
|
TMPDUP .req w3
|
|
|
|
/* START_COL is a JDIMENSION (unsigned int) argument, so the ABI doesn't
|
|
guarantee that the upper (unused) 32 bits of x1 are valid. This
|
|
instruction ensures that those bits are set to zero. */
|
|
uxtw x1, w1
|
|
|
|
mov TMPDUP, #128
|
|
ldp TMP1, TMP2, [SAMPLE_DATA], 16
|
|
ldp TMP3, TMP4, [SAMPLE_DATA], 16
|
|
dup v0.8b, TMPDUP
|
|
add TMP1, TMP1, START_COL
|
|
add TMP2, TMP2, START_COL
|
|
ldp TMP5, TMP6, [SAMPLE_DATA], 16
|
|
add TMP3, TMP3, START_COL
|
|
add TMP4, TMP4, START_COL
|
|
ldp TMP7, TMP8, [SAMPLE_DATA], 16
|
|
add TMP5, TMP5, START_COL
|
|
add TMP6, TMP6, START_COL
|
|
ld1 {v16.8b}, [TMP1]
|
|
add TMP7, TMP7, START_COL
|
|
add TMP8, TMP8, START_COL
|
|
ld1 {v17.8b}, [TMP2]
|
|
usubl v16.8h, v16.8b, v0.8b
|
|
ld1 {v18.8b}, [TMP3]
|
|
usubl v17.8h, v17.8b, v0.8b
|
|
ld1 {v19.8b}, [TMP4]
|
|
usubl v18.8h, v18.8b, v0.8b
|
|
ld1 {v20.8b}, [TMP5]
|
|
usubl v19.8h, v19.8b, v0.8b
|
|
ld1 {v21.8b}, [TMP6]
|
|
st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [WORKSPACE], 64
|
|
usubl v20.8h, v20.8b, v0.8b
|
|
ld1 {v22.8b}, [TMP7]
|
|
usubl v21.8h, v21.8b, v0.8b
|
|
ld1 {v23.8b}, [TMP8]
|
|
usubl v22.8h, v22.8b, v0.8b
|
|
usubl v23.8h, v23.8b, v0.8b
|
|
st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [WORKSPACE], 64
|
|
|
|
br x30
|
|
|
|
.unreq SAMPLE_DATA
|
|
.unreq START_COL
|
|
.unreq WORKSPACE
|
|
.unreq TMP1
|
|
.unreq TMP2
|
|
.unreq TMP3
|
|
.unreq TMP4
|
|
.unreq TMP5
|
|
.unreq TMP6
|
|
.unreq TMP7
|
|
.unreq TMP8
|
|
.unreq TMPDUP
|
|
|
|
/*****************************************************************************/
|
|
|
|
/*
|
|
* jsimd_fdct_islow_neon
|
|
*
|
|
* This file contains a slow-but-accurate integer implementation of the
|
|
* forward DCT (Discrete Cosine Transform). The following code is based
|
|
* directly on the IJG''s original jfdctint.c; see the jfdctint.c for
|
|
* more details.
|
|
*
|
|
* TODO: can be combined with 'jsimd_convsamp_neon' to get
|
|
* rid of a bunch of VLD1.16 instructions
|
|
*/
|
|
|
|
#define CONST_BITS 13
|
|
#define PASS1_BITS 2
|
|
|
|
#define DESCALE_P1 (CONST_BITS-PASS1_BITS)
|
|
#define DESCALE_P2 (CONST_BITS+PASS1_BITS)
|
|
|
|
#define F_0_298 2446 /* FIX(0.298631336) */
|
|
#define F_0_390 3196 /* FIX(0.390180644) */
|
|
#define F_0_541 4433 /* FIX(0.541196100) */
|
|
#define F_0_765 6270 /* FIX(0.765366865) */
|
|
#define F_0_899 7373 /* FIX(0.899976223) */
|
|
#define F_1_175 9633 /* FIX(1.175875602) */
|
|
#define F_1_501 12299 /* FIX(1.501321110) */
|
|
#define F_1_847 15137 /* FIX(1.847759065) */
|
|
#define F_1_961 16069 /* FIX(1.961570560) */
|
|
#define F_2_053 16819 /* FIX(2.053119869) */
|
|
#define F_2_562 20995 /* FIX(2.562915447) */
|
|
#define F_3_072 25172 /* FIX(3.072711026) */
|
|
|
|
.balign 16
|
|
Ljsimd_fdct_islow_neon_consts:
|
|
.short F_0_298
|
|
.short -F_0_390
|
|
.short F_0_541
|
|
.short F_0_765
|
|
.short - F_0_899
|
|
.short F_1_175
|
|
.short F_1_501
|
|
.short - F_1_847
|
|
.short - F_1_961
|
|
.short F_2_053
|
|
.short - F_2_562
|
|
.short F_3_072
|
|
.short 0 /* padding */
|
|
.short 0
|
|
.short 0
|
|
.short 0
|
|
|
|
#undef F_0_298
|
|
#undef F_0_390
|
|
#undef F_0_541
|
|
#undef F_0_765
|
|
#undef F_0_899
|
|
#undef F_1_175
|
|
#undef F_1_501
|
|
#undef F_1_847
|
|
#undef F_1_961
|
|
#undef F_2_053
|
|
#undef F_2_562
|
|
#undef F_3_072
|
|
#define XFIX_P_0_298 v0.h[0]
|
|
#define XFIX_N_0_390 v0.h[1]
|
|
#define XFIX_P_0_541 v0.h[2]
|
|
#define XFIX_P_0_765 v0.h[3]
|
|
#define XFIX_N_0_899 v0.h[4]
|
|
#define XFIX_P_1_175 v0.h[5]
|
|
#define XFIX_P_1_501 v0.h[6]
|
|
#define XFIX_N_1_847 v0.h[7]
|
|
#define XFIX_N_1_961 v1.h[0]
|
|
#define XFIX_P_2_053 v1.h[1]
|
|
#define XFIX_N_2_562 v1.h[2]
|
|
#define XFIX_P_3_072 v1.h[3]
|
|
|
|
asm_function jsimd_fdct_islow_neon
|
|
|
|
DATA .req x0
|
|
TMP .req x9
|
|
|
|
/* Load constants */
|
|
adr TMP, Ljsimd_fdct_islow_neon_consts
|
|
ld1 {v0.8h, v1.8h}, [TMP]
|
|
|
|
/* Save NEON registers */
|
|
sub sp, sp, #64
|
|
mov x10, sp
|
|
st1 {v8.8b, v9.8b, v10.8b, v11.8b}, [x10], 32
|
|
st1 {v12.8b, v13.8b, v14.8b, v15.8b}, [x10], 32
|
|
|
|
/* Load all DATA into NEON registers with the following allocation:
|
|
* 0 1 2 3 | 4 5 6 7
|
|
* ---------+--------
|
|
* 0 | d16 | d17 | v16.8h
|
|
* 1 | d18 | d19 | v17.8h
|
|
* 2 | d20 | d21 | v18.8h
|
|
* 3 | d22 | d23 | v19.8h
|
|
* 4 | d24 | d25 | v20.8h
|
|
* 5 | d26 | d27 | v21.8h
|
|
* 6 | d28 | d29 | v22.8h
|
|
* 7 | d30 | d31 | v23.8h
|
|
*/
|
|
|
|
ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [DATA], 64
|
|
ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [DATA]
|
|
sub DATA, DATA, #64
|
|
|
|
/* Transpose */
|
|
transpose_8x8 v16, v17, v18, v19, v20, v21, v22, v23, v31, v2, v3, v4
|
|
/* 1-D FDCT */
|
|
add v24.8h, v16.8h, v23.8h /* tmp0 = dataptr[0] + dataptr[7]; */
|
|
sub v31.8h, v16.8h, v23.8h /* tmp7 = dataptr[0] - dataptr[7]; */
|
|
add v25.8h, v17.8h, v22.8h /* tmp1 = dataptr[1] + dataptr[6]; */
|
|
sub v30.8h, v17.8h, v22.8h /* tmp6 = dataptr[1] - dataptr[6]; */
|
|
add v26.8h, v18.8h, v21.8h /* tmp2 = dataptr[2] + dataptr[5]; */
|
|
sub v29.8h, v18.8h, v21.8h /* tmp5 = dataptr[2] - dataptr[5]; */
|
|
add v27.8h, v19.8h, v20.8h /* tmp3 = dataptr[3] + dataptr[4]; */
|
|
sub v28.8h, v19.8h, v20.8h /* tmp4 = dataptr[3] - dataptr[4]; */
|
|
|
|
/* even part */
|
|
|
|
add v8.8h, v24.8h, v27.8h /* tmp10 = tmp0 + tmp3; */
|
|
sub v9.8h, v24.8h, v27.8h /* tmp13 = tmp0 - tmp3; */
|
|
add v10.8h, v25.8h, v26.8h /* tmp11 = tmp1 + tmp2; */
|
|
sub v11.8h, v25.8h, v26.8h /* tmp12 = tmp1 - tmp2; */
|
|
|
|
add v16.8h, v8.8h, v10.8h /* tmp10 + tmp11 */
|
|
sub v20.8h, v8.8h, v10.8h /* tmp10 - tmp11 */
|
|
|
|
add v18.8h, v11.8h, v9.8h /* tmp12 + tmp13 */
|
|
|
|
shl v16.8h, v16.8h, #PASS1_BITS /* dataptr[0] = (DCTELEM) LEFT_SHIFT(tmp10 + tmp11, PASS1_BITS); */
|
|
shl v20.8h, v20.8h, #PASS1_BITS /* dataptr[4] = (DCTELEM) LEFT_SHIFT(tmp10 - tmp11, PASS1_BITS); */
|
|
|
|
smull2 v24.4s, v18.8h, XFIX_P_0_541 /* z1 hi = MULTIPLY(tmp12 + tmp13, XFIX_P_0_541); */
|
|
smull v18.4s, v18.4h, XFIX_P_0_541 /* z1 lo = MULTIPLY(tmp12 + tmp13, XFIX_P_0_541); */
|
|
mov v22.16b, v18.16b
|
|
mov v25.16b, v24.16b
|
|
|
|
smlal v18.4s, v9.4h, XFIX_P_0_765 /* lo z1 + MULTIPLY(tmp13, XFIX_P_0_765) */
|
|
smlal2 v24.4s, v9.8h, XFIX_P_0_765 /* hi z1 + MULTIPLY(tmp13, XFIX_P_0_765) */
|
|
smlal v22.4s, v11.4h, XFIX_N_1_847 /* lo z1 + MULTIPLY(tmp12, XFIX_N_1_847) */
|
|
smlal2 v25.4s, v11.8h, XFIX_N_1_847 /* hi z1 + MULTIPLY(tmp12, XFIX_N_1_847) */
|
|
|
|
rshrn v18.4h, v18.4s, #DESCALE_P1
|
|
rshrn v22.4h, v22.4s, #DESCALE_P1
|
|
rshrn2 v18.8h, v24.4s, #DESCALE_P1 /* dataptr[2] = (DCTELEM) DESCALE(z1 + MULTIPLY(tmp13, XFIX_P_0_765), CONST_BITS-PASS1_BITS); */
|
|
rshrn2 v22.8h, v25.4s, #DESCALE_P1 /* dataptr[6] = (DCTELEM) DESCALE(z1 + MULTIPLY(tmp12, XFIX_N_1_847), CONST_BITS-PASS1_BITS); */
|
|
|
|
/* Odd part */
|
|
|
|
add v8.8h, v28.8h, v31.8h /* z1 = tmp4 + tmp7; */
|
|
add v9.8h, v29.8h, v30.8h /* z2 = tmp5 + tmp6; */
|
|
add v10.8h, v28.8h, v30.8h /* z3 = tmp4 + tmp6; */
|
|
add v11.8h, v29.8h, v31.8h /* z4 = tmp5 + tmp7; */
|
|
smull v4.4s, v10.4h, XFIX_P_1_175 /* z5 lo = z3 lo * XFIX_P_1_175 */
|
|
smull2 v5.4s, v10.8h, XFIX_P_1_175
|
|
smlal v4.4s, v11.4h, XFIX_P_1_175 /* z5 = MULTIPLY(z3 + z4, FIX_1_175875602); */
|
|
smlal2 v5.4s, v11.8h, XFIX_P_1_175
|
|
|
|
smull2 v24.4s, v28.8h, XFIX_P_0_298
|
|
smull2 v25.4s, v29.8h, XFIX_P_2_053
|
|
smull2 v26.4s, v30.8h, XFIX_P_3_072
|
|
smull2 v27.4s, v31.8h, XFIX_P_1_501
|
|
smull v28.4s, v28.4h, XFIX_P_0_298 /* tmp4 = MULTIPLY(tmp4, FIX_0_298631336); */
|
|
smull v29.4s, v29.4h, XFIX_P_2_053 /* tmp5 = MULTIPLY(tmp5, FIX_2_053119869); */
|
|
smull v30.4s, v30.4h, XFIX_P_3_072 /* tmp6 = MULTIPLY(tmp6, FIX_3_072711026); */
|
|
smull v31.4s, v31.4h, XFIX_P_1_501 /* tmp7 = MULTIPLY(tmp7, FIX_1_501321110); */
|
|
|
|
smull2 v12.4s, v8.8h, XFIX_N_0_899
|
|
smull2 v13.4s, v9.8h, XFIX_N_2_562
|
|
smull2 v14.4s, v10.8h, XFIX_N_1_961
|
|
smull2 v15.4s, v11.8h, XFIX_N_0_390
|
|
smull v8.4s, v8.4h, XFIX_N_0_899 /* z1 = MULTIPLY(z1, - FIX_0_899976223); */
|
|
smull v9.4s, v9.4h, XFIX_N_2_562 /* z2 = MULTIPLY(z2, - FIX_2_562915447); */
|
|
smull v10.4s, v10.4h, XFIX_N_1_961 /* z3 = MULTIPLY(z3, - FIX_1_961570560); */
|
|
smull v11.4s, v11.4h, XFIX_N_0_390 /* z4 = MULTIPLY(z4, - FIX_0_390180644); */
|
|
|
|
add v10.4s, v10.4s, v4.4s /* z3 += z5 */
|
|
add v14.4s, v14.4s, v5.4s
|
|
add v11.4s, v11.4s, v4.4s /* z4 += z5 */
|
|
add v15.4s, v15.4s, v5.4s
|
|
|
|
add v28.4s, v28.4s, v8.4s /* tmp4 += z1 */
|
|
add v24.4s, v24.4s, v12.4s
|
|
add v29.4s, v29.4s, v9.4s /* tmp5 += z2 */
|
|
add v25.4s, v25.4s, v13.4s
|
|
add v30.4s, v30.4s, v10.4s /* tmp6 += z3 */
|
|
add v26.4s, v26.4s, v14.4s
|
|
add v31.4s, v31.4s, v11.4s /* tmp7 += z4 */
|
|
add v27.4s, v27.4s, v15.4s
|
|
|
|
add v28.4s, v28.4s, v10.4s /* tmp4 += z3 */
|
|
add v24.4s, v24.4s, v14.4s
|
|
add v29.4s, v29.4s, v11.4s /* tmp5 += z4 */
|
|
add v25.4s, v25.4s, v15.4s
|
|
add v30.4s, v30.4s, v9.4s /* tmp6 += z2 */
|
|
add v26.4s, v26.4s, v13.4s
|
|
add v31.4s, v31.4s, v8.4s /* tmp7 += z1 */
|
|
add v27.4s, v27.4s, v12.4s
|
|
|
|
rshrn v23.4h, v28.4s, #DESCALE_P1
|
|
rshrn v21.4h, v29.4s, #DESCALE_P1
|
|
rshrn v19.4h, v30.4s, #DESCALE_P1
|
|
rshrn v17.4h, v31.4s, #DESCALE_P1
|
|
rshrn2 v23.8h, v24.4s, #DESCALE_P1 /* dataptr[7] = (DCTELEM) DESCALE(tmp4 + z1 + z3, CONST_BITS-PASS1_BITS); */
|
|
rshrn2 v21.8h, v25.4s, #DESCALE_P1 /* dataptr[5] = (DCTELEM) DESCALE(tmp5 + z2 + z4, CONST_BITS-PASS1_BITS); */
|
|
rshrn2 v19.8h, v26.4s, #DESCALE_P1 /* dataptr[3] = (DCTELEM) DESCALE(tmp6 + z2 + z3, CONST_BITS-PASS1_BITS); */
|
|
rshrn2 v17.8h, v27.4s, #DESCALE_P1 /* dataptr[1] = (DCTELEM) DESCALE(tmp7 + z1 + z4, CONST_BITS-PASS1_BITS); */
|
|
|
|
/* Transpose */
|
|
transpose_8x8 v16, v17, v18, v19, v20, v21, v22, v23, v31, v2, v3, v4
|
|
|
|
/* 1-D FDCT */
|
|
add v24.8h, v16.8h, v23.8h /* tmp0 = dataptr[0] + dataptr[7]; */
|
|
sub v31.8h, v16.8h, v23.8h /* tmp7 = dataptr[0] - dataptr[7]; */
|
|
add v25.8h, v17.8h, v22.8h /* tmp1 = dataptr[1] + dataptr[6]; */
|
|
sub v30.8h, v17.8h, v22.8h /* tmp6 = dataptr[1] - dataptr[6]; */
|
|
add v26.8h, v18.8h, v21.8h /* tmp2 = dataptr[2] + dataptr[5]; */
|
|
sub v29.8h, v18.8h, v21.8h /* tmp5 = dataptr[2] - dataptr[5]; */
|
|
add v27.8h, v19.8h, v20.8h /* tmp3 = dataptr[3] + dataptr[4]; */
|
|
sub v28.8h, v19.8h, v20.8h /* tmp4 = dataptr[3] - dataptr[4]; */
|
|
|
|
/* even part */
|
|
add v8.8h, v24.8h, v27.8h /* tmp10 = tmp0 + tmp3; */
|
|
sub v9.8h, v24.8h, v27.8h /* tmp13 = tmp0 - tmp3; */
|
|
add v10.8h, v25.8h, v26.8h /* tmp11 = tmp1 + tmp2; */
|
|
sub v11.8h, v25.8h, v26.8h /* tmp12 = tmp1 - tmp2; */
|
|
|
|
add v16.8h, v8.8h, v10.8h /* tmp10 + tmp11 */
|
|
sub v20.8h, v8.8h, v10.8h /* tmp10 - tmp11 */
|
|
|
|
add v18.8h, v11.8h, v9.8h /* tmp12 + tmp13 */
|
|
|
|
srshr v16.8h, v16.8h, #PASS1_BITS /* dataptr[0] = (DCTELEM) DESCALE(tmp10 + tmp11, PASS1_BITS); */
|
|
srshr v20.8h, v20.8h, #PASS1_BITS /* dataptr[4] = (DCTELEM) DESCALE(tmp10 - tmp11, PASS1_BITS); */
|
|
|
|
smull2 v24.4s, v18.8h, XFIX_P_0_541 /* z1 hi = MULTIPLY(tmp12 + tmp13, XFIX_P_0_541); */
|
|
smull v18.4s, v18.4h, XFIX_P_0_541 /* z1 lo = MULTIPLY(tmp12 + tmp13, XFIX_P_0_541); */
|
|
mov v22.16b, v18.16b
|
|
mov v25.16b, v24.16b
|
|
|
|
smlal v18.4s, v9.4h, XFIX_P_0_765 /* lo z1 + MULTIPLY(tmp13, XFIX_P_0_765) */
|
|
smlal2 v24.4s, v9.8h, XFIX_P_0_765 /* hi z1 + MULTIPLY(tmp13, XFIX_P_0_765) */
|
|
smlal v22.4s, v11.4h, XFIX_N_1_847 /* lo z1 + MULTIPLY(tmp12, XFIX_N_1_847) */
|
|
smlal2 v25.4s, v11.8h, XFIX_N_1_847 /* hi z1 + MULTIPLY(tmp12, XFIX_N_1_847) */
|
|
|
|
rshrn v18.4h, v18.4s, #DESCALE_P2
|
|
rshrn v22.4h, v22.4s, #DESCALE_P2
|
|
rshrn2 v18.8h, v24.4s, #DESCALE_P2 /* dataptr[2] = (DCTELEM) DESCALE(z1 + MULTIPLY(tmp13, XFIX_P_0_765), CONST_BITS-PASS1_BITS); */
|
|
rshrn2 v22.8h, v25.4s, #DESCALE_P2 /* dataptr[6] = (DCTELEM) DESCALE(z1 + MULTIPLY(tmp12, XFIX_N_1_847), CONST_BITS-PASS1_BITS); */
|
|
|
|
/* Odd part */
|
|
add v8.8h, v28.8h, v31.8h /* z1 = tmp4 + tmp7; */
|
|
add v9.8h, v29.8h, v30.8h /* z2 = tmp5 + tmp6; */
|
|
add v10.8h, v28.8h, v30.8h /* z3 = tmp4 + tmp6; */
|
|
add v11.8h, v29.8h, v31.8h /* z4 = tmp5 + tmp7; */
|
|
|
|
smull v4.4s, v10.4h, XFIX_P_1_175 /* z5 lo = z3 lo * XFIX_P_1_175 */
|
|
smull2 v5.4s, v10.8h, XFIX_P_1_175
|
|
smlal v4.4s, v11.4h, XFIX_P_1_175 /* z5 = MULTIPLY(z3 + z4, FIX_1_175875602); */
|
|
smlal2 v5.4s, v11.8h, XFIX_P_1_175
|
|
|
|
smull2 v24.4s, v28.8h, XFIX_P_0_298
|
|
smull2 v25.4s, v29.8h, XFIX_P_2_053
|
|
smull2 v26.4s, v30.8h, XFIX_P_3_072
|
|
smull2 v27.4s, v31.8h, XFIX_P_1_501
|
|
smull v28.4s, v28.4h, XFIX_P_0_298 /* tmp4 = MULTIPLY(tmp4, FIX_0_298631336); */
|
|
smull v29.4s, v29.4h, XFIX_P_2_053 /* tmp5 = MULTIPLY(tmp5, FIX_2_053119869); */
|
|
smull v30.4s, v30.4h, XFIX_P_3_072 /* tmp6 = MULTIPLY(tmp6, FIX_3_072711026); */
|
|
smull v31.4s, v31.4h, XFIX_P_1_501 /* tmp7 = MULTIPLY(tmp7, FIX_1_501321110); */
|
|
|
|
smull2 v12.4s, v8.8h, XFIX_N_0_899
|
|
smull2 v13.4s, v9.8h, XFIX_N_2_562
|
|
smull2 v14.4s, v10.8h, XFIX_N_1_961
|
|
smull2 v15.4s, v11.8h, XFIX_N_0_390
|
|
smull v8.4s, v8.4h, XFIX_N_0_899 /* z1 = MULTIPLY(z1, - FIX_0_899976223); */
|
|
smull v9.4s, v9.4h, XFIX_N_2_562 /* z2 = MULTIPLY(z2, - FIX_2_562915447); */
|
|
smull v10.4s, v10.4h, XFIX_N_1_961 /* z3 = MULTIPLY(z3, - FIX_1_961570560); */
|
|
smull v11.4s, v11.4h, XFIX_N_0_390 /* z4 = MULTIPLY(z4, - FIX_0_390180644); */
|
|
|
|
add v10.4s, v10.4s, v4.4s
|
|
add v14.4s, v14.4s, v5.4s
|
|
add v11.4s, v11.4s, v4.4s
|
|
add v15.4s, v15.4s, v5.4s
|
|
|
|
add v28.4s, v28.4s, v8.4s /* tmp4 += z1 */
|
|
add v24.4s, v24.4s, v12.4s
|
|
add v29.4s, v29.4s, v9.4s /* tmp5 += z2 */
|
|
add v25.4s, v25.4s, v13.4s
|
|
add v30.4s, v30.4s, v10.4s /* tmp6 += z3 */
|
|
add v26.4s, v26.4s, v14.4s
|
|
add v31.4s, v31.4s, v11.4s /* tmp7 += z4 */
|
|
add v27.4s, v27.4s, v15.4s
|
|
|
|
add v28.4s, v28.4s, v10.4s /* tmp4 += z3 */
|
|
add v24.4s, v24.4s, v14.4s
|
|
add v29.4s, v29.4s, v11.4s /* tmp5 += z4 */
|
|
add v25.4s, v25.4s, v15.4s
|
|
add v30.4s, v30.4s, v9.4s /* tmp6 += z2 */
|
|
add v26.4s, v26.4s, v13.4s
|
|
add v31.4s, v31.4s, v8.4s /* tmp7 += z1 */
|
|
add v27.4s, v27.4s, v12.4s
|
|
|
|
rshrn v23.4h, v28.4s, #DESCALE_P2
|
|
rshrn v21.4h, v29.4s, #DESCALE_P2
|
|
rshrn v19.4h, v30.4s, #DESCALE_P2
|
|
rshrn v17.4h, v31.4s, #DESCALE_P2
|
|
rshrn2 v23.8h, v24.4s, #DESCALE_P2 /* dataptr[7] = (DCTELEM) DESCALE(tmp4 + z1 + z3, CONST_BITS-PASS1_BITS); */
|
|
rshrn2 v21.8h, v25.4s, #DESCALE_P2 /* dataptr[5] = (DCTELEM) DESCALE(tmp5 + z2 + z4, CONST_BITS-PASS1_BITS); */
|
|
rshrn2 v19.8h, v26.4s, #DESCALE_P2 /* dataptr[3] = (DCTELEM) DESCALE(tmp6 + z2 + z3, CONST_BITS-PASS1_BITS); */
|
|
rshrn2 v17.8h, v27.4s, #DESCALE_P2 /* dataptr[1] = (DCTELEM) DESCALE(tmp7 + z1 + z4, CONST_BITS-PASS1_BITS); */
|
|
|
|
/* store results */
|
|
st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [DATA], 64
|
|
st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [DATA]
|
|
|
|
/* Restore NEON registers */
|
|
ld1 {v8.8b, v9.8b, v10.8b, v11.8b}, [sp], 32
|
|
ld1 {v12.8b, v13.8b, v14.8b, v15.8b}, [sp], 32
|
|
|
|
br x30
|
|
|
|
.unreq DATA
|
|
.unreq TMP
|
|
|
|
#undef XFIX_P_0_298
|
|
#undef XFIX_N_0_390
|
|
#undef XFIX_P_0_541
|
|
#undef XFIX_P_0_765
|
|
#undef XFIX_N_0_899
|
|
#undef XFIX_P_1_175
|
|
#undef XFIX_P_1_501
|
|
#undef XFIX_N_1_847
|
|
#undef XFIX_N_1_961
|
|
#undef XFIX_P_2_053
|
|
#undef XFIX_N_2_562
|
|
#undef XFIX_P_3_072
|
|
|
|
|
|
/*****************************************************************************/
|
|
|
|
/*
|
|
* jsimd_fdct_ifast_neon
|
|
*
|
|
* This function contains a fast, not so accurate integer implementation of
|
|
* the forward DCT (Discrete Cosine Transform). It uses the same calculations
|
|
* and produces exactly the same output as IJG's original 'jpeg_fdct_ifast'
|
|
* function from jfdctfst.c
|
|
*
|
|
* TODO: can be combined with 'jsimd_convsamp_neon' to get
|
|
* rid of a bunch of VLD1.16 instructions
|
|
*/
|
|
|
|
#undef XFIX_0_541196100
|
|
#define XFIX_0_382683433 v0.h[0]
|
|
#define XFIX_0_541196100 v0.h[1]
|
|
#define XFIX_0_707106781 v0.h[2]
|
|
#define XFIX_1_306562965 v0.h[3]
|
|
|
|
.balign 16
|
|
Ljsimd_fdct_ifast_neon_consts:
|
|
.short (98 * 128) /* XFIX_0_382683433 */
|
|
.short (139 * 128) /* XFIX_0_541196100 */
|
|
.short (181 * 128) /* XFIX_0_707106781 */
|
|
.short (334 * 128 - 256 * 128) /* XFIX_1_306562965 */
|
|
|
|
asm_function jsimd_fdct_ifast_neon
|
|
|
|
DATA .req x0
|
|
TMP .req x9
|
|
|
|
/* Load constants */
|
|
adr TMP, Ljsimd_fdct_ifast_neon_consts
|
|
ld1 {v0.4h}, [TMP]
|
|
|
|
/* Load all DATA into NEON registers with the following allocation:
|
|
* 0 1 2 3 | 4 5 6 7
|
|
* ---------+--------
|
|
* 0 | d16 | d17 | v0.8h
|
|
* 1 | d18 | d19 | q9
|
|
* 2 | d20 | d21 | q10
|
|
* 3 | d22 | d23 | q11
|
|
* 4 | d24 | d25 | q12
|
|
* 5 | d26 | d27 | q13
|
|
* 6 | d28 | d29 | q14
|
|
* 7 | d30 | d31 | q15
|
|
*/
|
|
|
|
ld1 {v16.8h, v17.8h, v18.8h, v19.8h}, [DATA], 64
|
|
ld1 {v20.8h, v21.8h, v22.8h, v23.8h}, [DATA]
|
|
mov TMP, #2
|
|
sub DATA, DATA, #64
|
|
1:
|
|
/* Transpose */
|
|
transpose_8x8 v16, v17, v18, v19, v20, v21, v22, v23, v1, v2, v3, v4
|
|
subs TMP, TMP, #1
|
|
/* 1-D FDCT */
|
|
add v4.8h, v19.8h, v20.8h
|
|
sub v20.8h, v19.8h, v20.8h
|
|
sub v28.8h, v18.8h, v21.8h
|
|
add v18.8h, v18.8h, v21.8h
|
|
sub v29.8h, v17.8h, v22.8h
|
|
add v17.8h, v17.8h, v22.8h
|
|
sub v21.8h, v16.8h, v23.8h
|
|
add v16.8h, v16.8h, v23.8h
|
|
sub v6.8h, v17.8h, v18.8h
|
|
sub v7.8h, v16.8h, v4.8h
|
|
add v5.8h, v17.8h, v18.8h
|
|
add v6.8h, v6.8h, v7.8h
|
|
add v4.8h, v16.8h, v4.8h
|
|
sqdmulh v6.8h, v6.8h, XFIX_0_707106781
|
|
add v19.8h, v20.8h, v28.8h
|
|
add v16.8h, v4.8h, v5.8h
|
|
sub v20.8h, v4.8h, v5.8h
|
|
add v5.8h, v28.8h, v29.8h
|
|
add v29.8h, v29.8h, v21.8h
|
|
sqdmulh v5.8h, v5.8h, XFIX_0_707106781
|
|
sub v28.8h, v19.8h, v29.8h
|
|
add v18.8h, v7.8h, v6.8h
|
|
sqdmulh v28.8h, v28.8h, XFIX_0_382683433
|
|
sub v22.8h, v7.8h, v6.8h
|
|
sqdmulh v19.8h, v19.8h, XFIX_0_541196100
|
|
sqdmulh v7.8h, v29.8h, XFIX_1_306562965
|
|
add v6.8h, v21.8h, v5.8h
|
|
sub v5.8h, v21.8h, v5.8h
|
|
add v29.8h, v29.8h, v28.8h
|
|
add v19.8h, v19.8h, v28.8h
|
|
add v29.8h, v29.8h, v7.8h
|
|
add v21.8h, v5.8h, v19.8h
|
|
sub v19.8h, v5.8h, v19.8h
|
|
add v17.8h, v6.8h, v29.8h
|
|
sub v23.8h, v6.8h, v29.8h
|
|
|
|
b.ne 1b
|
|
|
|
/* store results */
|
|
st1 {v16.8h, v17.8h, v18.8h, v19.8h}, [DATA], 64
|
|
st1 {v20.8h, v21.8h, v22.8h, v23.8h}, [DATA]
|
|
|
|
br x30
|
|
|
|
.unreq DATA
|
|
.unreq TMP
|
|
#undef XFIX_0_382683433
|
|
#undef XFIX_0_541196100
|
|
#undef XFIX_0_707106781
|
|
#undef XFIX_1_306562965
|
|
|
|
|
|
/*****************************************************************************/
|
|
|
|
/*
|
|
* GLOBAL(void)
|
|
* jsimd_quantize_neon (JCOEFPTR coef_block, DCTELEM *divisors,
|
|
* DCTELEM *workspace);
|
|
*
|
|
*/
|
|
asm_function jsimd_quantize_neon
|
|
|
|
COEF_BLOCK .req x0
|
|
DIVISORS .req x1
|
|
WORKSPACE .req x2
|
|
|
|
RECIPROCAL .req DIVISORS
|
|
CORRECTION .req x9
|
|
SHIFT .req x10
|
|
LOOP_COUNT .req x11
|
|
|
|
mov LOOP_COUNT, #2
|
|
add CORRECTION, DIVISORS, #(64 * 2)
|
|
add SHIFT, DIVISORS, #(64 * 6)
|
|
1:
|
|
subs LOOP_COUNT, LOOP_COUNT, #1
|
|
ld1 {v0.8h, v1.8h, v2.8h, v3.8h}, [WORKSPACE], 64
|
|
ld1 {v4.8h, v5.8h, v6.8h, v7.8h}, [CORRECTION], 64
|
|
abs v20.8h, v0.8h
|
|
abs v21.8h, v1.8h
|
|
abs v22.8h, v2.8h
|
|
abs v23.8h, v3.8h
|
|
ld1 {v28.8h, v29.8h, v30.8h, v31.8h}, [RECIPROCAL], 64
|
|
add v20.8h, v20.8h, v4.8h /* add correction */
|
|
add v21.8h, v21.8h, v5.8h
|
|
add v22.8h, v22.8h, v6.8h
|
|
add v23.8h, v23.8h, v7.8h
|
|
umull v4.4s, v20.4h, v28.4h /* multiply by reciprocal */
|
|
umull2 v16.4s, v20.8h, v28.8h
|
|
umull v5.4s, v21.4h, v29.4h
|
|
umull2 v17.4s, v21.8h, v29.8h
|
|
umull v6.4s, v22.4h, v30.4h /* multiply by reciprocal */
|
|
umull2 v18.4s, v22.8h, v30.8h
|
|
umull v7.4s, v23.4h, v31.4h
|
|
umull2 v19.4s, v23.8h, v31.8h
|
|
ld1 {v24.8h, v25.8h, v26.8h, v27.8h}, [SHIFT], 64
|
|
shrn v4.4h, v4.4s, #16
|
|
shrn v5.4h, v5.4s, #16
|
|
shrn v6.4h, v6.4s, #16
|
|
shrn v7.4h, v7.4s, #16
|
|
shrn2 v4.8h, v16.4s, #16
|
|
shrn2 v5.8h, v17.4s, #16
|
|
shrn2 v6.8h, v18.4s, #16
|
|
shrn2 v7.8h, v19.4s, #16
|
|
neg v24.8h, v24.8h
|
|
neg v25.8h, v25.8h
|
|
neg v26.8h, v26.8h
|
|
neg v27.8h, v27.8h
|
|
sshr v0.8h, v0.8h, #15 /* extract sign */
|
|
sshr v1.8h, v1.8h, #15
|
|
sshr v2.8h, v2.8h, #15
|
|
sshr v3.8h, v3.8h, #15
|
|
ushl v4.8h, v4.8h, v24.8h /* shift */
|
|
ushl v5.8h, v5.8h, v25.8h
|
|
ushl v6.8h, v6.8h, v26.8h
|
|
ushl v7.8h, v7.8h, v27.8h
|
|
|
|
eor v4.16b, v4.16b, v0.16b /* restore sign */
|
|
eor v5.16b, v5.16b, v1.16b
|
|
eor v6.16b, v6.16b, v2.16b
|
|
eor v7.16b, v7.16b, v3.16b
|
|
sub v4.8h, v4.8h, v0.8h
|
|
sub v5.8h, v5.8h, v1.8h
|
|
sub v6.8h, v6.8h, v2.8h
|
|
sub v7.8h, v7.8h, v3.8h
|
|
st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [COEF_BLOCK], 64
|
|
|
|
b.ne 1b
|
|
|
|
br x30 /* return */
|
|
|
|
.unreq COEF_BLOCK
|
|
.unreq DIVISORS
|
|
.unreq WORKSPACE
|
|
.unreq RECIPROCAL
|
|
.unreq CORRECTION
|
|
.unreq SHIFT
|
|
.unreq LOOP_COUNT
|
|
|
|
|
|
/*****************************************************************************/
|
|
|
|
/*
|
|
* Downsample pixel values of a single component.
|
|
* This version handles the common case of 2:1 horizontal and 1:1 vertical,
|
|
* without smoothing.
|
|
*
|
|
* GLOBAL(void)
|
|
* jsimd_h2v1_downsample_neon (JDIMENSION image_width, int max_v_samp_factor,
|
|
* JDIMENSION v_samp_factor,
|
|
* JDIMENSION width_blocks, JSAMPARRAY input_data,
|
|
* JSAMPARRAY output_data);
|
|
*/
|
|
|
|
.balign 16
|
|
Ljsimd_h2_downsample_neon_consts:
|
|
.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, \
|
|
0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F /* diff 0 */
|
|
.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, \
|
|
0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0E /* diff 1 */
|
|
.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, \
|
|
0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0D, 0x0D /* diff 2 */
|
|
.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, \
|
|
0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0C, 0x0C, 0x0C /* diff 3 */
|
|
.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, \
|
|
0x08, 0x09, 0x0A, 0x0B, 0x0B, 0x0B, 0x0B, 0x0B /* diff 4 */
|
|
.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, \
|
|
0x08, 0x09, 0x0A, 0x0A, 0x0A, 0x0A, 0x0A, 0x0A /* diff 5 */
|
|
.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, \
|
|
0x08, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09 /* diff 6 */
|
|
.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, \
|
|
0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08, 0x08 /* diff 7 */
|
|
.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, \
|
|
0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07, 0x07 /* diff 8 */
|
|
.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x06, \
|
|
0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06 /* diff 9 */
|
|
.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x05, 0x05, \
|
|
0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05 /* diff 10 */
|
|
.byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x04, 0x04, 0x04, \
|
|
0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04, 0x04 /* diff 11 */
|
|
.byte 0x00, 0x01, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, \
|
|
0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03, 0x03 /* diff 12 */
|
|
.byte 0x00, 0x01, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, \
|
|
0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02 /* diff 13 */
|
|
.byte 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, \
|
|
0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01 /* diff 14 */
|
|
.byte 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \
|
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 /* diff 15 */
|
|
|
|
asm_function jsimd_h2v1_downsample_neon
|
|
IMAGE_WIDTH .req x0
|
|
MAX_V_SAMP .req x1
|
|
V_SAMP .req x2
|
|
BLOCK_WIDTH .req x3
|
|
INPUT_DATA .req x4
|
|
OUTPUT_DATA .req x5
|
|
OUTPTR .req x9
|
|
INPTR .req x10
|
|
TMP1 .req x11
|
|
TMP2 .req x12
|
|
TMP3 .req x13
|
|
TMPDUP .req w15
|
|
|
|
mov TMPDUP, #0x10000
|
|
lsl TMP2, BLOCK_WIDTH, #4
|
|
sub TMP2, TMP2, IMAGE_WIDTH
|
|
adr TMP3, Ljsimd_h2_downsample_neon_consts
|
|
add TMP3, TMP3, TMP2, lsl #4
|
|
dup v16.4s, TMPDUP
|
|
ld1 {v18.16b}, [TMP3]
|
|
|
|
1: /* row loop */
|
|
ldr INPTR, [INPUT_DATA], #8
|
|
ldr OUTPTR, [OUTPUT_DATA], #8
|
|
subs TMP1, BLOCK_WIDTH, #1
|
|
b.eq 3f
|
|
2: /* columns */
|
|
ld1 {v0.16b}, [INPTR], #16
|
|
mov v4.16b, v16.16b
|
|
subs TMP1, TMP1, #1
|
|
uadalp v4.8h, v0.16b
|
|
shrn v6.8b, v4.8h, #1
|
|
st1 {v6.8b}, [OUTPTR], #8
|
|
b.ne 2b
|
|
3: /* last columns */
|
|
ld1 {v0.16b}, [INPTR]
|
|
mov v4.16b, v16.16b
|
|
subs V_SAMP, V_SAMP, #1
|
|
/* expand right */
|
|
tbl v2.16b, {v0.16b}, v18.16b
|
|
uadalp v4.8h, v2.16b
|
|
shrn v6.8b, v4.8h, #1
|
|
st1 {v6.8b}, [OUTPTR], #8
|
|
b.ne 1b
|
|
|
|
br x30
|
|
|
|
.unreq IMAGE_WIDTH
|
|
.unreq MAX_V_SAMP
|
|
.unreq V_SAMP
|
|
.unreq BLOCK_WIDTH
|
|
.unreq INPUT_DATA
|
|
.unreq OUTPUT_DATA
|
|
.unreq OUTPTR
|
|
.unreq INPTR
|
|
.unreq TMP1
|
|
.unreq TMP2
|
|
.unreq TMP3
|
|
.unreq TMPDUP
|
|
|
|
|
|
/*****************************************************************************/
|
|
|
|
/*
|
|
* Downsample pixel values of a single component.
|
|
* This version handles the common case of 2:1 horizontal and 2:1 vertical,
|
|
* without smoothing.
|
|
*
|
|
* GLOBAL(void)
|
|
* jsimd_h2v2_downsample_neon (JDIMENSION image_width, int max_v_samp_factor,
|
|
* JDIMENSION v_samp_factor, JDIMENSION width_blocks,
|
|
* JSAMPARRAY input_data, JSAMPARRAY output_data);
|
|
*/
|
|
|
|
.balign 16
|
|
asm_function jsimd_h2v2_downsample_neon
|
|
IMAGE_WIDTH .req x0
|
|
MAX_V_SAMP .req x1
|
|
V_SAMP .req x2
|
|
BLOCK_WIDTH .req x3
|
|
INPUT_DATA .req x4
|
|
OUTPUT_DATA .req x5
|
|
OUTPTR .req x9
|
|
INPTR0 .req x10
|
|
INPTR1 .req x14
|
|
TMP1 .req x11
|
|
TMP2 .req x12
|
|
TMP3 .req x13
|
|
TMPDUP .req w15
|
|
|
|
mov TMPDUP, #1
|
|
lsl TMP2, BLOCK_WIDTH, #4
|
|
lsl TMPDUP, TMPDUP, #17
|
|
sub TMP2, TMP2, IMAGE_WIDTH
|
|
adr TMP3, Ljsimd_h2_downsample_neon_consts
|
|
orr TMPDUP, TMPDUP, #1
|
|
add TMP3, TMP3, TMP2, lsl #4
|
|
dup v16.4s, TMPDUP
|
|
ld1 {v18.16b}, [TMP3]
|
|
|
|
1: /* row loop */
|
|
ldr INPTR0, [INPUT_DATA], #8
|
|
ldr OUTPTR, [OUTPUT_DATA], #8
|
|
ldr INPTR1, [INPUT_DATA], #8
|
|
subs TMP1, BLOCK_WIDTH, #1
|
|
b.eq 3f
|
|
2: /* columns */
|
|
ld1 {v0.16b}, [INPTR0], #16
|
|
ld1 {v1.16b}, [INPTR1], #16
|
|
mov v4.16b, v16.16b
|
|
subs TMP1, TMP1, #1
|
|
uadalp v4.8h, v0.16b
|
|
uadalp v4.8h, v1.16b
|
|
shrn v6.8b, v4.8h, #2
|
|
st1 {v6.8b}, [OUTPTR], #8
|
|
b.ne 2b
|
|
3: /* last columns */
|
|
ld1 {v0.16b}, [INPTR0], #16
|
|
ld1 {v1.16b}, [INPTR1], #16
|
|
mov v4.16b, v16.16b
|
|
subs V_SAMP, V_SAMP, #1
|
|
/* expand right */
|
|
tbl v2.16b, {v0.16b}, v18.16b
|
|
tbl v3.16b, {v1.16b}, v18.16b
|
|
uadalp v4.8h, v2.16b
|
|
uadalp v4.8h, v3.16b
|
|
shrn v6.8b, v4.8h, #2
|
|
st1 {v6.8b}, [OUTPTR], #8
|
|
b.ne 1b
|
|
|
|
br x30
|
|
|
|
.unreq IMAGE_WIDTH
|
|
.unreq MAX_V_SAMP
|
|
.unreq V_SAMP
|
|
.unreq BLOCK_WIDTH
|
|
.unreq INPUT_DATA
|
|
.unreq OUTPUT_DATA
|
|
.unreq OUTPTR
|
|
.unreq INPTR0
|
|
.unreq INPTR1
|
|
.unreq TMP1
|
|
.unreq TMP2
|
|
.unreq TMP3
|
|
.unreq TMPDUP
|
|
|
|
|
|
/*****************************************************************************/
|
|
|
|
/*
|
|
* GLOBAL(JOCTET*)
|
|
* jsimd_huff_encode_one_block (working_state *state, JOCTET *buffer,
|
|
* JCOEFPTR block, int last_dc_val,
|
|
* c_derived_tbl *dctbl, c_derived_tbl *actbl)
|
|
*
|
|
*/
|
|
|
|
BUFFER .req x1
|
|
PUT_BUFFER .req x6
|
|
PUT_BITS .req x7
|
|
PUT_BITSw .req w7
|
|
|
|
.macro emit_byte
|
|
sub PUT_BITS, PUT_BITS, #0x8
|
|
lsr x19, PUT_BUFFER, PUT_BITS
|
|
uxtb w19, w19
|
|
strb w19, [BUFFER, #1]!
|
|
cmp w19, #0xff
|
|
b.ne 14f
|
|
strb wzr, [BUFFER, #1]!
|
|
14:
|
|
.endm
|
|
.macro put_bits CODE, SIZE
|
|
lsl PUT_BUFFER, PUT_BUFFER, \SIZE
|
|
add PUT_BITS, PUT_BITS, \SIZE
|
|
orr PUT_BUFFER, PUT_BUFFER, \CODE
|
|
.endm
|
|
.macro checkbuf31
|
|
cmp PUT_BITS, #0x20
|
|
b.lt 31f
|
|
emit_byte
|
|
emit_byte
|
|
emit_byte
|
|
emit_byte
|
|
31:
|
|
.endm
|
|
.macro checkbuf47
|
|
cmp PUT_BITS, #0x30
|
|
b.lt 47f
|
|
emit_byte
|
|
emit_byte
|
|
emit_byte
|
|
emit_byte
|
|
emit_byte
|
|
emit_byte
|
|
47:
|
|
.endm
|
|
|
|
.macro generate_jsimd_huff_encode_one_block fast_tbl
|
|
|
|
.balign 16
|
|
.if \fast_tbl == 1
|
|
Ljsimd_huff_encode_one_block_neon_consts:
|
|
.else
|
|
Ljsimd_huff_encode_one_block_neon_slowtbl_consts:
|
|
.endif
|
|
.byte 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, \
|
|
0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80
|
|
.if \fast_tbl == 1
|
|
.byte 0, 1, 2, 3, 16, 17, 32, 33, \
|
|
18, 19, 4, 5, 6, 7, 20, 21 /* L0 => L3 : 4 lines OK */
|
|
.byte 34, 35, 48, 49, 255, 255, 50, 51, \
|
|
36, 37, 22, 23, 8, 9, 10, 11 /* L0 => L3 : 4 lines OK */
|
|
.byte 8, 9, 22, 23, 36, 37, 50, 51, \
|
|
255, 255, 255, 255, 255, 255, 52, 53 /* L1 => L4 : 4 lines OK */
|
|
.byte 54, 55, 40, 41, 26, 27, 12, 13, \
|
|
14, 15, 28, 29, 42, 43, 56, 57 /* L0 => L3 : 4 lines OK */
|
|
.byte 6, 7, 20, 21, 34, 35, 48, 49, \
|
|
50, 51, 36, 37, 22, 23, 8, 9 /* L4 => L7 : 4 lines OK */
|
|
.byte 42, 43, 28, 29, 14, 15, 30, 31, \
|
|
44, 45, 58, 59, 255, 255, 255, 255 /* L1 => L4 : 4 lines OK */
|
|
.byte 255, 255, 255, 255, 56, 57, 42, 43, \
|
|
28, 29, 14, 15, 30, 31, 44, 45 /* L3 => L6 : 4 lines OK */
|
|
.byte 26, 27, 40, 41, 42, 43, 28, 29, \
|
|
14, 15, 30, 31, 44, 45, 46, 47 /* L5 => L7 : 3 lines OK */
|
|
.byte 255, 255, 255, 255, 0, 1, 255, 255, \
|
|
255, 255, 255, 255, 255, 255, 255, 255 /* L4 : 1 lines OK */
|
|
.byte 255, 255, 255, 255, 255, 255, 255, 255, \
|
|
0, 1, 16, 17, 2, 3, 255, 255 /* L5 => L6 : 2 lines OK */
|
|
.byte 255, 255, 255, 255, 255, 255, 255, 255, \
|
|
255, 255, 255, 255, 8, 9, 22, 23 /* L5 => L6 : 2 lines OK */
|
|
.byte 4, 5, 6, 7, 255, 255, 255, 255, \
|
|
255, 255, 255, 255, 255, 255, 255, 255 /* L7 : 1 line OK */
|
|
.endif
|
|
|
|
.if \fast_tbl == 1
|
|
asm_function jsimd_huff_encode_one_block_neon
|
|
.else
|
|
asm_function jsimd_huff_encode_one_block_neon_slowtbl
|
|
.endif
|
|
sub sp, sp, 272
|
|
sub BUFFER, BUFFER, #0x1 /* BUFFER=buffer-- */
|
|
/* Save ARM registers */
|
|
stp x19, x20, [sp]
|
|
.if \fast_tbl == 1
|
|
adr x15, Ljsimd_huff_encode_one_block_neon_consts
|
|
.else
|
|
adr x15, Ljsimd_huff_encode_one_block_neon_slowtbl_consts
|
|
.endif
|
|
ldr PUT_BUFFER, [x0, #0x10]
|
|
ldr PUT_BITSw, [x0, #0x18]
|
|
ldrsh w12, [x2] /* load DC coeff in w12 */
|
|
/* prepare data */
|
|
.if \fast_tbl == 1
|
|
ld1 {v23.16b}, [x15], #16
|
|
ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [x15], #64
|
|
ld1 {v4.16b, v5.16b, v6.16b, v7.16b}, [x15], #64
|
|
ld1 {v16.16b, v17.16b, v18.16b, v19.16b}, [x15], #64
|
|
ld1 {v24.16b, v25.16b, v26.16b, v27.16b}, [x2], #64
|
|
ld1 {v28.16b, v29.16b, v30.16b, v31.16b}, [x2], #64
|
|
sub w12, w12, w3 /* last_dc_val, not used afterwards */
|
|
/* ZigZag 8x8 */
|
|
tbl v0.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v0.16b
|
|
tbl v1.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v1.16b
|
|
tbl v2.16b, {v25.16b, v26.16b, v27.16b, v28.16b}, v2.16b
|
|
tbl v3.16b, {v24.16b, v25.16b, v26.16b, v27.16b}, v3.16b
|
|
tbl v4.16b, {v28.16b, v29.16b, v30.16b, v31.16b}, v4.16b
|
|
tbl v5.16b, {v25.16b, v26.16b, v27.16b, v28.16b}, v5.16b
|
|
tbl v6.16b, {v27.16b, v28.16b, v29.16b, v30.16b}, v6.16b
|
|
tbl v7.16b, {v29.16b, v30.16b, v31.16b}, v7.16b
|
|
ins v0.h[0], w12
|
|
tbx v1.16b, {v28.16b}, v16.16b
|
|
tbx v2.16b, {v29.16b, v30.16b}, v17.16b
|
|
tbx v5.16b, {v29.16b, v30.16b}, v18.16b
|
|
tbx v6.16b, {v31.16b}, v19.16b
|
|
.else
|
|
add x13, x2, #0x22
|
|
sub w12, w12, w3 /* last_dc_val, not used afterwards */
|
|
ld1 {v23.16b}, [x15]
|
|
add x14, x2, #0x18
|
|
add x3, x2, #0x36
|
|
ins v0.h[0], w12
|
|
add x9, x2, #0x2
|
|
ld1 {v1.h}[0], [x13]
|
|
add x15, x2, #0x30
|
|
ld1 {v2.h}[0], [x14]
|
|
add x19, x2, #0x26
|
|
ld1 {v3.h}[0], [x3]
|
|
add x20, x2, #0x28
|
|
ld1 {v0.h}[1], [x9]
|
|
add x12, x2, #0x10
|
|
ld1 {v1.h}[1], [x15]
|
|
add x13, x2, #0x40
|
|
ld1 {v2.h}[1], [x19]
|
|
add x14, x2, #0x34
|
|
ld1 {v3.h}[1], [x20]
|
|
add x3, x2, #0x1a
|
|
ld1 {v0.h}[2], [x12]
|
|
add x9, x2, #0x20
|
|
ld1 {v1.h}[2], [x13]
|
|
add x15, x2, #0x32
|
|
ld1 {v2.h}[2], [x14]
|
|
add x19, x2, #0x42
|
|
ld1 {v3.h}[2], [x3]
|
|
add x20, x2, #0xc
|
|
ld1 {v0.h}[3], [x9]
|
|
add x12, x2, #0x12
|
|
ld1 {v1.h}[3], [x15]
|
|
add x13, x2, #0x24
|
|
ld1 {v2.h}[3], [x19]
|
|
add x14, x2, #0x50
|
|
ld1 {v3.h}[3], [x20]
|
|
add x3, x2, #0xe
|
|
ld1 {v0.h}[4], [x12]
|
|
add x9, x2, #0x4
|
|
ld1 {v1.h}[4], [x13]
|
|
add x15, x2, #0x16
|
|
ld1 {v2.h}[4], [x14]
|
|
add x19, x2, #0x60
|
|
ld1 {v3.h}[4], [x3]
|
|
add x20, x2, #0x1c
|
|
ld1 {v0.h}[5], [x9]
|
|
add x12, x2, #0x6
|
|
ld1 {v1.h}[5], [x15]
|
|
add x13, x2, #0x8
|
|
ld1 {v2.h}[5], [x19]
|
|
add x14, x2, #0x52
|
|
ld1 {v3.h}[5], [x20]
|
|
add x3, x2, #0x2a
|
|
ld1 {v0.h}[6], [x12]
|
|
add x9, x2, #0x14
|
|
ld1 {v1.h}[6], [x13]
|
|
add x15, x2, #0xa
|
|
ld1 {v2.h}[6], [x14]
|
|
add x19, x2, #0x44
|
|
ld1 {v3.h}[6], [x3]
|
|
add x20, x2, #0x38
|
|
ld1 {v0.h}[7], [x9]
|
|
add x12, x2, #0x46
|
|
ld1 {v1.h}[7], [x15]
|
|
add x13, x2, #0x3a
|
|
ld1 {v2.h}[7], [x19]
|
|
add x14, x2, #0x74
|
|
ld1 {v3.h}[7], [x20]
|
|
add x3, x2, #0x6a
|
|
ld1 {v4.h}[0], [x12]
|
|
add x9, x2, #0x54
|
|
ld1 {v5.h}[0], [x13]
|
|
add x15, x2, #0x2c
|
|
ld1 {v6.h}[0], [x14]
|
|
add x19, x2, #0x76
|
|
ld1 {v7.h}[0], [x3]
|
|
add x20, x2, #0x78
|
|
ld1 {v4.h}[1], [x9]
|
|
add x12, x2, #0x62
|
|
ld1 {v5.h}[1], [x15]
|
|
add x13, x2, #0x1e
|
|
ld1 {v6.h}[1], [x19]
|
|
add x14, x2, #0x68
|
|
ld1 {v7.h}[1], [x20]
|
|
add x3, x2, #0x7a
|
|
ld1 {v4.h}[2], [x12]
|
|
add x9, x2, #0x70
|
|
ld1 {v5.h}[2], [x13]
|
|
add x15, x2, #0x2e
|
|
ld1 {v6.h}[2], [x14]
|
|
add x19, x2, #0x5a
|
|
ld1 {v7.h}[2], [x3]
|
|
add x20, x2, #0x6c
|
|
ld1 {v4.h}[3], [x9]
|
|
add x12, x2, #0x72
|
|
ld1 {v5.h}[3], [x15]
|
|
add x13, x2, #0x3c
|
|
ld1 {v6.h}[3], [x19]
|
|
add x14, x2, #0x4c
|
|
ld1 {v7.h}[3], [x20]
|
|
add x3, x2, #0x5e
|
|
ld1 {v4.h}[4], [x12]
|
|
add x9, x2, #0x64
|
|
ld1 {v5.h}[4], [x13]
|
|
add x15, x2, #0x4a
|
|
ld1 {v6.h}[4], [x14]
|
|
add x19, x2, #0x3e
|
|
ld1 {v7.h}[4], [x3]
|
|
add x20, x2, #0x6e
|
|
ld1 {v4.h}[5], [x9]
|
|
add x12, x2, #0x56
|
|
ld1 {v5.h}[5], [x15]
|
|
add x13, x2, #0x58
|
|
ld1 {v6.h}[5], [x19]
|
|
add x14, x2, #0x4e
|
|
ld1 {v7.h}[5], [x20]
|
|
add x3, x2, #0x7c
|
|
ld1 {v4.h}[6], [x12]
|
|
add x9, x2, #0x48
|
|
ld1 {v5.h}[6], [x13]
|
|
add x15, x2, #0x66
|
|
ld1 {v6.h}[6], [x14]
|
|
add x19, x2, #0x5c
|
|
ld1 {v7.h}[6], [x3]
|
|
add x20, x2, #0x7e
|
|
ld1 {v4.h}[7], [x9]
|
|
ld1 {v5.h}[7], [x15]
|
|
ld1 {v6.h}[7], [x19]
|
|
ld1 {v7.h}[7], [x20]
|
|
.endif
|
|
cmlt v24.8h, v0.8h, #0
|
|
cmlt v25.8h, v1.8h, #0
|
|
cmlt v26.8h, v2.8h, #0
|
|
cmlt v27.8h, v3.8h, #0
|
|
cmlt v28.8h, v4.8h, #0
|
|
cmlt v29.8h, v5.8h, #0
|
|
cmlt v30.8h, v6.8h, #0
|
|
cmlt v31.8h, v7.8h, #0
|
|
abs v0.8h, v0.8h
|
|
abs v1.8h, v1.8h
|
|
abs v2.8h, v2.8h
|
|
abs v3.8h, v3.8h
|
|
abs v4.8h, v4.8h
|
|
abs v5.8h, v5.8h
|
|
abs v6.8h, v6.8h
|
|
abs v7.8h, v7.8h
|
|
eor v24.16b, v24.16b, v0.16b
|
|
eor v25.16b, v25.16b, v1.16b
|
|
eor v26.16b, v26.16b, v2.16b
|
|
eor v27.16b, v27.16b, v3.16b
|
|
eor v28.16b, v28.16b, v4.16b
|
|
eor v29.16b, v29.16b, v5.16b
|
|
eor v30.16b, v30.16b, v6.16b
|
|
eor v31.16b, v31.16b, v7.16b
|
|
cmeq v16.8h, v0.8h, #0
|
|
cmeq v17.8h, v1.8h, #0
|
|
cmeq v18.8h, v2.8h, #0
|
|
cmeq v19.8h, v3.8h, #0
|
|
cmeq v20.8h, v4.8h, #0
|
|
cmeq v21.8h, v5.8h, #0
|
|
cmeq v22.8h, v6.8h, #0
|
|
xtn v16.8b, v16.8h
|
|
xtn v18.8b, v18.8h
|
|
xtn v20.8b, v20.8h
|
|
xtn v22.8b, v22.8h
|
|
umov w14, v0.h[0]
|
|
xtn2 v16.16b, v17.8h
|
|
umov w13, v24.h[0]
|
|
xtn2 v18.16b, v19.8h
|
|
clz w14, w14
|
|
xtn2 v20.16b, v21.8h
|
|
lsl w13, w13, w14
|
|
cmeq v17.8h, v7.8h, #0
|
|
sub w12, w14, #32
|
|
xtn2 v22.16b, v17.8h
|
|
lsr w13, w13, w14
|
|
and v16.16b, v16.16b, v23.16b
|
|
neg w12, w12
|
|
and v18.16b, v18.16b, v23.16b
|
|
add x3, x4, #0x400 /* r1 = dctbl->ehufsi */
|
|
and v20.16b, v20.16b, v23.16b
|
|
add x15, sp, #0x90 /* x15 = t2 */
|
|
and v22.16b, v22.16b, v23.16b
|
|
ldr w10, [x4, x12, lsl #2]
|
|
addp v16.16b, v16.16b, v18.16b
|
|
ldrb w11, [x3, x12]
|
|
addp v20.16b, v20.16b, v22.16b
|
|
checkbuf47
|
|
addp v16.16b, v16.16b, v20.16b
|
|
put_bits x10, x11
|
|
addp v16.16b, v16.16b, v18.16b
|
|
checkbuf47
|
|
umov x9,v16.D[0]
|
|
put_bits x13, x12
|
|
cnt v17.8b, v16.8b
|
|
mvn x9, x9
|
|
addv B18, v17.8b
|
|
add x4, x5, #0x400 /* x4 = actbl->ehufsi */
|
|
umov w12, v18.b[0]
|
|
lsr x9, x9, #0x1 /* clear AC coeff */
|
|
ldr w13, [x5, #0x3c0] /* x13 = actbl->ehufco[0xf0] */
|
|
rbit x9, x9 /* x9 = index0 */
|
|
ldrb w14, [x4, #0xf0] /* x14 = actbl->ehufsi[0xf0] */
|
|
cmp w12, #(64-8)
|
|
add x11, sp, #16
|
|
b.lt 4f
|
|
cbz x9, 6f
|
|
st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x11], #64
|
|
st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x11], #64
|
|
st1 {v24.8h, v25.8h, v26.8h, v27.8h}, [x11], #64
|
|
st1 {v28.8h, v29.8h, v30.8h, v31.8h}, [x11], #64
|
|
1:
|
|
clz x2, x9
|
|
add x15, x15, x2, lsl #1
|
|
lsl x9, x9, x2
|
|
ldrh w20, [x15, #-126]
|
|
2:
|
|
cmp x2, #0x10
|
|
b.lt 3f
|
|
sub x2, x2, #0x10
|
|
checkbuf47
|
|
put_bits x13, x14
|
|
b 2b
|
|
3:
|
|
clz w20, w20
|
|
ldrh w3, [x15, #2]!
|
|
sub w11, w20, #32
|
|
lsl w3, w3, w20
|
|
neg w11, w11
|
|
lsr w3, w3, w20
|
|
add x2, x11, x2, lsl #4
|
|
lsl x9, x9, #0x1
|
|
ldr w12, [x5, x2, lsl #2]
|
|
ldrb w10, [x4, x2]
|
|
checkbuf31
|
|
put_bits x12, x10
|
|
put_bits x3, x11
|
|
cbnz x9, 1b
|
|
b 6f
|
|
4:
|
|
movi v21.8h, #0x0010
|
|
clz v0.8h, v0.8h
|
|
clz v1.8h, v1.8h
|
|
clz v2.8h, v2.8h
|
|
clz v3.8h, v3.8h
|
|
clz v4.8h, v4.8h
|
|
clz v5.8h, v5.8h
|
|
clz v6.8h, v6.8h
|
|
clz v7.8h, v7.8h
|
|
ushl v24.8h, v24.8h, v0.8h
|
|
ushl v25.8h, v25.8h, v1.8h
|
|
ushl v26.8h, v26.8h, v2.8h
|
|
ushl v27.8h, v27.8h, v3.8h
|
|
ushl v28.8h, v28.8h, v4.8h
|
|
ushl v29.8h, v29.8h, v5.8h
|
|
ushl v30.8h, v30.8h, v6.8h
|
|
ushl v31.8h, v31.8h, v7.8h
|
|
neg v0.8h, v0.8h
|
|
neg v1.8h, v1.8h
|
|
neg v2.8h, v2.8h
|
|
neg v3.8h, v3.8h
|
|
neg v4.8h, v4.8h
|
|
neg v5.8h, v5.8h
|
|
neg v6.8h, v6.8h
|
|
neg v7.8h, v7.8h
|
|
ushl v24.8h, v24.8h, v0.8h
|
|
ushl v25.8h, v25.8h, v1.8h
|
|
ushl v26.8h, v26.8h, v2.8h
|
|
ushl v27.8h, v27.8h, v3.8h
|
|
ushl v28.8h, v28.8h, v4.8h
|
|
ushl v29.8h, v29.8h, v5.8h
|
|
ushl v30.8h, v30.8h, v6.8h
|
|
ushl v31.8h, v31.8h, v7.8h
|
|
add v0.8h, v21.8h, v0.8h
|
|
add v1.8h, v21.8h, v1.8h
|
|
add v2.8h, v21.8h, v2.8h
|
|
add v3.8h, v21.8h, v3.8h
|
|
add v4.8h, v21.8h, v4.8h
|
|
add v5.8h, v21.8h, v5.8h
|
|
add v6.8h, v21.8h, v6.8h
|
|
add v7.8h, v21.8h, v7.8h
|
|
st1 {v0.8h, v1.8h, v2.8h, v3.8h}, [x11], #64
|
|
st1 {v4.8h, v5.8h, v6.8h, v7.8h}, [x11], #64
|
|
st1 {v24.8h, v25.8h, v26.8h, v27.8h}, [x11], #64
|
|
st1 {v28.8h, v29.8h, v30.8h, v31.8h}, [x11], #64
|
|
1:
|
|
clz x2, x9
|
|
add x15, x15, x2, lsl #1
|
|
lsl x9, x9, x2
|
|
ldrh w11, [x15, #-126]
|
|
2:
|
|
cmp x2, #0x10
|
|
b.lt 3f
|
|
sub x2, x2, #0x10
|
|
checkbuf47
|
|
put_bits x13, x14
|
|
b 2b
|
|
3:
|
|
ldrh w3, [x15, #2]!
|
|
add x2, x11, x2, lsl #4
|
|
lsl x9, x9, #0x1
|
|
ldr w12, [x5, x2, lsl #2]
|
|
ldrb w10, [x4, x2]
|
|
checkbuf31
|
|
put_bits x12, x10
|
|
put_bits x3, x11
|
|
cbnz x9, 1b
|
|
6:
|
|
add x13, sp, #0x10e
|
|
cmp x15, x13
|
|
b.hs 1f
|
|
ldr w12, [x5]
|
|
ldrb w14, [x4]
|
|
checkbuf47
|
|
put_bits x12, x14
|
|
1:
|
|
str PUT_BUFFER, [x0, #0x10]
|
|
str PUT_BITSw, [x0, #0x18]
|
|
ldp x19, x20, [sp], 16
|
|
add x0, BUFFER, #0x1
|
|
add sp, sp, 256
|
|
br x30
|
|
|
|
.endm
|
|
|
|
generate_jsimd_huff_encode_one_block 1
|
|
generate_jsimd_huff_encode_one_block 0
|
|
|
|
.unreq BUFFER
|
|
.unreq PUT_BUFFER
|
|
.unreq PUT_BITS
|
|
.unreq PUT_BITSw
|
|
|
|
.purgem emit_byte
|
|
.purgem put_bits
|
|
.purgem checkbuf31
|
|
.purgem checkbuf47
|