/* SPDX-License-Identifier: GPL-2.0-only */
/* * linux/arch/arm64/crypto/aes-neon.S - AES cipher for ARMv8 NEON * * Copyright (C) 2013 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
#include <linux/linkage.h>
#include <asm/assembler.h>
#define AES_FUNC_START(func) SYM_FUNC_START(neon_ ## func)
#define AES_FUNC_END(func) SYM_FUNC_END(neon_ ## func)
xtsmask .req v7
cbciv .req v7
vctr .req v4
.macro xts_reload_mask, tmp
xts_load_mask \tmp
.endm
/* special case for the neon-bs driver calling into this one for CTS */
.macro xts_cts_skip_tw, reg, lbl
tbnz \reg, #
1 , \lbl
.endm
/* multiply by polynomial 'x' in GF(2^8) */
.macro mul_by_x, out, in, temp, const
sshr \temp, \in, #
7
shl \out, \in, #
1
and \temp, \temp, \const
eor \out, \out, \temp
.endm
/* multiply by polynomial 'x^2' in GF(2^8) */
.macro mul_by_x2, out, in, temp, const
ushr \temp, \in, #
6
shl \out, \in, #
2
pmul \temp, \temp, \const
eor \out, \out, \temp
.endm
/* preload the entire Sbox */
.macro prepare, sbox, shiftrows, temp
movi v12.
16 b, #
0 x1b
ldr_l q13, \shiftrows, \temp
ldr_l q14, .Lror32by8, \temp
adr_l \temp, \sbox
ld1 {v16.
16 b-v19.
16 b}, [\temp], #
64
ld1 {v20.
16 b-v23.
16 b}, [\temp], #
64
ld1 {v24.
16 b-v27.
16 b}, [\temp], #
64
ld1 {v28.
16 b-v31.
16 b}, [\temp]
.endm
/* do preload for encryption */
.macro enc_prepare, ignore0, ignore1, temp
prepare crypto_aes_sbox, .LForward_ShiftRows, \temp
.endm
.macro enc_switch_key, ignore0, ignore1, temp
/* do nothing */
.endm
/* do preload for decryption */
.macro dec_prepare, ignore0, ignore1, temp
prepare crypto_aes_inv_sbox, .LReverse_ShiftRows, \temp
.endm
/* apply SubBytes transformation using the preloaded Sbox */
.macro sub_bytes, in
sub v9.
16 b, \in\().
16 b, v15.
16 b
tbl \in\().
16 b, {v16.
16 b-v19.
16 b}, \in\().
16 b
sub v10.
16 b, v9.
16 b, v15.
16 b
tbx \in\().
16 b, {v20.
16 b-v23.
16 b}, v9.
16 b
sub v11.
16 b, v10.
16 b, v15.
16 b
tbx \in\().
16 b, {v24.
16 b-v27.
16 b}, v10.
16 b
tbx \in\().
16 b, {v28.
16 b-v31.
16 b}, v11.
16 b
.endm
/* apply MixColumns transformation */
.macro mix_columns, in, enc
.if \enc ==
0
/* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
mul_by_x2 v8.
16 b, \in\().
16 b, v9.
16 b, v12.
16 b
eor \in\().
16 b, \in\().
16 b, v8.
16 b
rev32 v8.
8 h, v8.
8 h
eor \in\().
16 b, \in\().
16 b, v8.
16 b
.endif
mul_by_x v9.
16 b, \in\().
16 b, v8.
16 b, v12.
16 b
rev32 v8.
8 h, \in\().
8 h
eor v8.
16 b, v8.
16 b, v9.
16 b
eor \in\().
16 b, \in\().
16 b, v8.
16 b
tbl \in\().
16 b, {\in\().
16 b}, v14.
16 b
eor \in\().
16 b, \in\().
16 b, v8.
16 b
.endm
.macro do_block, enc, in, rounds, rk, rkp, i
ld1 {v15.
4 s}, [\rk]
add \rkp, \rk, #
16
mov \i, \rounds
.La\@: eor \in\().
16 b, \in\().
16 b, v15.
16 b
/* ^round key */
movi v15.
16 b, #
0 x40
tbl \in\().
16 b, {\in\().
16 b}, v13.
16 b
/* ShiftRows */
sub_bytes \in
sub \i, \i, #
1
ld1 {v15.
4 s}, [\rkp], #
16
cbz \i, .Lb\@
mix_columns \in, \enc
b .La\@
.Lb\@: eor \in\().
16 b, \in\().
16 b, v15.
16 b
/* ^round key */
.endm
.macro encrypt_block, in, rounds, rk, rkp, i
do_block
1 , \in, \rounds, \rk, \rkp, \i
.endm
.macro decrypt_block, in, rounds, rk, rkp, i
do_block
0 , \in, \rounds, \rk, \rkp, \i
.endm
/* * Interleaved versions: functionally equivalent to the * ones above, but applied to AES states in parallel.
.macro sub_bytes_4x, in0, in1, in2, in3
sub v8.
16 b, \in0\().
16 b, v15.
16 b
tbl \in0\().
16 b, {v16.
16 b-v19.
16 b}, \in0\().
16 b
sub v9.
16 b, \in1\().
16 b, v15.
16 b
tbl \in1\().
16 b, {v16.
16 b-v19.
16 b}, \in1\().
16 b
sub v10.
16 b, \in2\().
16 b, v15.
16 b
tbl \in2\().
16 b, {v16.
16 b-v19.
16 b}, \in2\().
16 b
sub v11.
16 b, \in3\().
16 b, v15.
16 b
tbl \in3\().
16 b, {v16.
16 b-v19.
16 b}, \in3\().
16 b
tbx \in0\().
16 b, {v20.
16 b-v23.
16 b}, v8.
16 b
tbx \in1\().
16 b, {v20.
16 b-v23.
16 b}, v9.
16 b
sub v8.
16 b, v8.
16 b, v15.
16 b
tbx \in2\().
16 b, {v20.
16 b-v23.
16 b}, v10.
16 b
sub v9.
16 b, v9.
16 b, v15.
16 b
tbx \in3\().
16 b, {v20.
16 b-v23.
16 b}, v11.
16 b
sub v10.
16 b, v10.
16 b, v15.
16 b
tbx \in0\().
16 b, {v24.
16 b-v27.
16 b}, v8.
16 b
sub v11.
16 b, v11.
16 b, v15.
16 b
tbx \in1\().
16 b, {v24.
16 b-v27.
16 b}, v9.
16 b
sub v8.
16 b, v8.
16 b, v15.
16 b
tbx \in2\().
16 b, {v24.
16 b-v27.
16 b}, v10.
16 b
sub v9.
16 b, v9.
16 b, v15.
16 b
tbx \in3\().
16 b, {v24.
16 b-v27.
16 b}, v11.
16 b
sub v10.
16 b, v10.
16 b, v15.
16 b
tbx \in0\().
16 b, {v28.
16 b-v31.
16 b}, v8.
16 b
sub v11.
16 b, v11.
16 b, v15.
16 b
tbx \in1\().
16 b, {v28.
16 b-v31.
16 b}, v9.
16 b
tbx \in2\().
16 b, {v28.
16 b-v31.
16 b}, v10.
16 b
tbx \in3\().
16 b, {v28.
16 b-v31.
16 b}, v11.
16 b
.endm
.macro mul_by_x_2x, out0, out1, in0, in1, tmp0, tmp1, const
sshr \tmp0\().
16 b, \in0\().
16 b, #
7
shl \out0\().
16 b, \in0\().
16 b, #
1
sshr \tmp1\().
16 b, \in1\().
16 b, #
7
and \tmp0\().
16 b, \tmp0\().
16 b, \const\().
16 b
shl \out1\().
16 b, \in1\().
16 b, #
1
and \tmp1\().
16 b, \tmp1\().
16 b, \const\().
16 b
eor \out0\().
16 b, \out0\().
16 b, \tmp0\().
16 b
eor \out1\().
16 b, \out1\().
16 b, \tmp1\().
16 b
.endm
.macro mul_by_x2_2x, out0, out1, in0, in1, tmp0, tmp1, const
ushr \tmp0\().
16 b, \in0\().
16 b, #
6
shl \out0\().
16 b, \in0\().
16 b, #
2
ushr \tmp1\().
16 b, \in1\().
16 b, #
6
pmul \tmp0\().
16 b, \tmp0\().
16 b, \const\().
16 b
shl \out1\().
16 b, \in1\().
16 b, #
2
pmul \tmp1\().
16 b, \tmp1\().
16 b, \const\().
16 b
eor \out0\().
16 b, \out0\().
16 b, \tmp0\().
16 b
eor \out1\().
16 b, \out1\().
16 b, \tmp1\().
16 b
.endm
.macro mix_columns_2x, in0, in1, enc
.if \enc ==
0
/* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
mul_by_x2_2x v8, v9, \in0, \in1, v10, v11, v12
eor \in0\().
16 b, \in0\().
16 b, v8.
16 b
rev32 v8.
8 h, v8.
8 h
eor \in1\().
16 b, \in1\().
16 b, v9.
16 b
rev32 v9.
8 h, v9.
8 h
eor \in0\().
16 b, \in0\().
16 b, v8.
16 b
eor \in1\().
16 b, \in1\().
16 b, v9.
16 b
.endif
mul_by_x_2x v8, v9, \in0, \in1, v10, v11, v12
rev32 v10.
8 h, \in0\().
8 h
rev32 v11.
8 h, \in1\().
8 h
eor v10.
16 b, v10.
16 b, v8.
16 b
eor v11.
16 b, v11.
16 b, v9.
16 b
eor \in0\().
16 b, \in0\().
16 b, v10.
16 b
eor \in1\().
16 b, \in1\().
16 b, v11.
16 b
tbl \in0\().
16 b, {\in0\().
16 b}, v14.
16 b
tbl \in1\().
16 b, {\in1\().
16 b}, v14.
16 b
eor \in0\().
16 b, \in0\().
16 b, v10.
16 b
eor \in1\().
16 b, \in1\().
16 b, v11.
16 b
.endm
.macro do_block_4x, enc, in0, in1, in2, in3, rounds, rk, rkp, i
ld1 {v15.
4 s}, [\rk]
add \rkp, \rk, #
16
mov \i, \rounds
.La\@: eor \in0\().
16 b, \in0\().
16 b, v15.
16 b
/* ^round key */
eor \in1\().
16 b, \in1\().
16 b, v15.
16 b
/* ^round key */
eor \in2\().
16 b, \in2\().
16 b, v15.
16 b
/* ^round key */
eor \in3\().
16 b, \in3\().
16 b, v15.
16 b
/* ^round key */
movi v15.
16 b, #
0 x40
tbl \in0\().
16 b, {\in0\().
16 b}, v13.
16 b
/* ShiftRows */
tbl \in1\().
16 b, {\in1\().
16 b}, v13.
16 b
/* ShiftRows */
tbl \in2\().
16 b, {\in2\().
16 b}, v13.
16 b
/* ShiftRows */
tbl \in3\().
16 b, {\in3\().
16 b}, v13.
16 b
/* ShiftRows */
sub_bytes_4x \in0, \in1, \in2, \in3
sub \i, \i, #
1
ld1 {v15.
4 s}, [\rkp], #
16
cbz \i, .Lb\@
mix_columns_2x \in0, \in1, \enc
mix_columns_2x \in2, \in3, \enc
b .La\@
.Lb\@: eor \in0\().
16 b, \in0\().
16 b, v15.
16 b
/* ^round key */
eor \in1\().
16 b, \in1\().
16 b, v15.
16 b
/* ^round key */
eor \in2\().
16 b, \in2\().
16 b, v15.
16 b
/* ^round key */
eor \in3\().
16 b, \in3\().
16 b, v15.
16 b
/* ^round key */
.endm
.macro encrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
do_block_4x
1 , \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
.endm
.macro decrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
do_block_4x
0 , \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
.endm
#include
"aes-modes.S"
.
section ".rodata" ,
"a"
.
align 4
.LForward_ShiftRows:
.octa
0 x0b06010c07020d08030e09040f0a0500
.LReverse_ShiftRows:
.octa
0 x0306090c0f0205080b0e0104070a0d00
.Lror32by8:
.octa
0 x0c0f0e0d080b0a090407060500030201
Messung V0.5 in Prozent C=95 H=93 G=93
¤ Dauer der Verarbeitung: 0.8 Sekunden
(vorverarbeitet am 2026-06-05)
¤ *© Formatika GbR, Deutschland
