mirror of
https://github.com/izzy2lost/xemu.git
synced 2026-07-06 00:20:22 -07:00
Merge tag 'pull-tcg-20230709' of https://gitlab.com/rth7680/qemu into staging
crypto: Provide aes-round.h and host accel # -----BEGIN PGP SIGNATURE----- # # iQFRBAABCgA7FiEEekgeeIaLTbaoWgXAZN846K9+IV8FAmSqvGodHHJpY2hhcmQu # aGVuZGVyc29uQGxpbmFyby5vcmcACgkQZN846K9+IV+1bgf9EG57jfnCQLCfMQ6C # 0bQ0MaeAkGg+7+mUwyi3OPB1VO0yjEKv5pWEnolzrGud35P0KsyoO+msqGqxnbMv # IbhPkQZbmfMsGFPG1DbswjiwmQU5cV+ciONDM+C+qepnuUN+JrzIDHoEFzQRFoQo # eQL/LnuyUkYBvR7YCKNJxFHtwILKcYQPH4jiC6a92C11AzYjDfilSxnuQ2RwL3Tn # Zwf8TKJP5QGExvUdtm8f6xJ1LT7WAvsk9ZTwudE/+XRTnw8RWk6RmZSEQPx+cBdI # p3opaoxkkMrdmcaXbr+9eSfBGq2gsVkKYPiyTDuwVW26575Nob9ZmodT3oSBNlkC # +njd4w== # =Nf5i # -----END PGP SIGNATURE----- # gpg: Signature made Sun 09 Jul 2023 02:55:54 PM BST # gpg: using RSA key 7A481E78868B4DB6A85A05C064DF38E8AF7E215F # gpg: issuer "richard.henderson@linaro.org" # gpg: Good signature from "Richard Henderson <richard.henderson@linaro.org>" [ultimate] * tag 'pull-tcg-20230709' of https://gitlab.com/rth7680/qemu: (37 commits) crypto: Unexport AES_*_rot, AES_TeN, AES_TdN crypto: Remove AES_imc crypto: Implement aesdec_IMC with AES_imc_rot crypto: Remove AES_shifts, AES_ishifts target/riscv: Use aesdec_ISB_ISR_IMC_AK target/riscv: Use aesenc_SB_SR_MC_AK target/riscv: Use aesdec_IMC target/riscv: Use aesdec_ISB_ISR_AK target/riscv: Use aesenc_SB_SR_AK target/arm: Use aesdec_IMC target/arm: Use aesenc_MC target/arm: Use aesdec_ISB_ISR_AK target/arm: Use aesenc_SB_SR_AK target/arm: Demultiplex AESE and AESMC target/i386: Use aesdec_ISB_ISR_IMC_AK target/i386: Use aesenc_SB_SR_MC_AK target/i386: Use aesdec_IMC target/i386: Use aesdec_ISB_ISR_AK target/i386: Use aesenc_SB_SR_AK target/ppc: Use aesdec_ISB_ISR_AK_IMC ... Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
This commit is contained in:
Richard Henderson
@@ -3224,6 +3224,7 @@ M: Daniel P. Berrange <berrange@redhat.com>
|
||||
S: Maintained
|
||||
F: crypto/
|
||||
F: include/crypto/
|
||||
F: host/include/*/host/crypto/
|
||||
F: qapi/crypto.json
|
||||
F: tests/unit/test-crypto-*
|
||||
F: tests/bench/benchmark-crypto-*
|
||||
|
||||
+502
-278
File diff suppressed because it is too large
Load Diff
@@ -9,6 +9,7 @@
|
||||
#define CPUINFO_ALWAYS (1u << 0) /* so cpuinfo is nonzero */
|
||||
#define CPUINFO_LSE (1u << 1)
|
||||
#define CPUINFO_LSE2 (1u << 2)
|
||||
#define CPUINFO_AES (1u << 3)
|
||||
|
||||
/* Initialized with a constructor. */
|
||||
extern unsigned cpuinfo;
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
/*
|
||||
* AArch64 specific aes acceleration.
|
||||
* SPDX-License-Identifier: GPL-2.0-or-later
|
||||
*/
|
||||
|
||||
#ifndef AARCH64_HOST_CRYPTO_AES_ROUND_H
|
||||
#define AARCH64_HOST_CRYPTO_AES_ROUND_H
|
||||
|
||||
#include "host/cpuinfo.h"
|
||||
#include <arm_neon.h>
|
||||
|
||||
#ifdef __ARM_FEATURE_AES
|
||||
# define HAVE_AES_ACCEL true
|
||||
#else
|
||||
# define HAVE_AES_ACCEL likely(cpuinfo & CPUINFO_AES)
|
||||
#endif
|
||||
#if !defined(__ARM_FEATURE_AES) && defined(CONFIG_ARM_AES_BUILTIN)
|
||||
# define ATTR_AES_ACCEL __attribute__((target("+crypto")))
|
||||
#else
|
||||
# define ATTR_AES_ACCEL
|
||||
#endif
|
||||
|
||||
static inline uint8x16_t aes_accel_bswap(uint8x16_t x)
|
||||
{
|
||||
return vqtbl1q_u8(x, (uint8x16_t){ 15, 14, 13, 12, 11, 10, 9, 8,
|
||||
7, 6, 5, 4, 3, 2, 1, 0, });
|
||||
}
|
||||
|
||||
#ifdef CONFIG_ARM_AES_BUILTIN
|
||||
# define aes_accel_aesd vaesdq_u8
|
||||
# define aes_accel_aese vaeseq_u8
|
||||
# define aes_accel_aesmc vaesmcq_u8
|
||||
# define aes_accel_aesimc vaesimcq_u8
|
||||
# define aes_accel_aesd_imc(S, K) vaesimcq_u8(vaesdq_u8(S, K))
|
||||
# define aes_accel_aese_mc(S, K) vaesmcq_u8(vaeseq_u8(S, K))
|
||||
#else
|
||||
static inline uint8x16_t aes_accel_aesd(uint8x16_t d, uint8x16_t k)
|
||||
{
|
||||
asm(".arch_extension aes\n\t"
|
||||
"aesd %0.16b, %1.16b" : "+w"(d) : "w"(k));
|
||||
return d;
|
||||
}
|
||||
|
||||
static inline uint8x16_t aes_accel_aese(uint8x16_t d, uint8x16_t k)
|
||||
{
|
||||
asm(".arch_extension aes\n\t"
|
||||
"aese %0.16b, %1.16b" : "+w"(d) : "w"(k));
|
||||
return d;
|
||||
}
|
||||
|
||||
static inline uint8x16_t aes_accel_aesmc(uint8x16_t d)
|
||||
{
|
||||
asm(".arch_extension aes\n\t"
|
||||
"aesmc %0.16b, %1.16b" : "=w"(d) : "w"(d));
|
||||
return d;
|
||||
}
|
||||
|
||||
static inline uint8x16_t aes_accel_aesimc(uint8x16_t d)
|
||||
{
|
||||
asm(".arch_extension aes\n\t"
|
||||
"aesimc %0.16b, %1.16b" : "=w"(d) : "w"(d));
|
||||
return d;
|
||||
}
|
||||
|
||||
/* Most CPUs fuse AESD+AESIMC in the execution pipeline. */
|
||||
static inline uint8x16_t aes_accel_aesd_imc(uint8x16_t d, uint8x16_t k)
|
||||
{
|
||||
asm(".arch_extension aes\n\t"
|
||||
"aesd %0.16b, %1.16b\n\t"
|
||||
"aesimc %0.16b, %0.16b" : "+w"(d) : "w"(k));
|
||||
return d;
|
||||
}
|
||||
|
||||
/* Most CPUs fuse AESE+AESMC in the execution pipeline. */
|
||||
static inline uint8x16_t aes_accel_aese_mc(uint8x16_t d, uint8x16_t k)
|
||||
{
|
||||
asm(".arch_extension aes\n\t"
|
||||
"aese %0.16b, %1.16b\n\t"
|
||||
"aesmc %0.16b, %0.16b" : "+w"(d) : "w"(k));
|
||||
return d;
|
||||
}
|
||||
#endif /* CONFIG_ARM_AES_BUILTIN */
|
||||
|
||||
static inline void ATTR_AES_ACCEL
|
||||
aesenc_MC_accel(AESState *ret, const AESState *st, bool be)
|
||||
{
|
||||
uint8x16_t t = (uint8x16_t)st->v;
|
||||
|
||||
if (be) {
|
||||
t = aes_accel_bswap(t);
|
||||
t = aes_accel_aesmc(t);
|
||||
t = aes_accel_bswap(t);
|
||||
} else {
|
||||
t = aes_accel_aesmc(t);
|
||||
}
|
||||
ret->v = (AESStateVec)t;
|
||||
}
|
||||
|
||||
static inline void ATTR_AES_ACCEL
|
||||
aesenc_SB_SR_AK_accel(AESState *ret, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
uint8x16_t t = (uint8x16_t)st->v;
|
||||
uint8x16_t z = { };
|
||||
|
||||
if (be) {
|
||||
t = aes_accel_bswap(t);
|
||||
t = aes_accel_aese(t, z);
|
||||
t = aes_accel_bswap(t);
|
||||
} else {
|
||||
t = aes_accel_aese(t, z);
|
||||
}
|
||||
ret->v = (AESStateVec)t ^ rk->v;
|
||||
}
|
||||
|
||||
static inline void ATTR_AES_ACCEL
|
||||
aesenc_SB_SR_MC_AK_accel(AESState *ret, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
uint8x16_t t = (uint8x16_t)st->v;
|
||||
uint8x16_t z = { };
|
||||
|
||||
if (be) {
|
||||
t = aes_accel_bswap(t);
|
||||
t = aes_accel_aese_mc(t, z);
|
||||
t = aes_accel_bswap(t);
|
||||
} else {
|
||||
t = aes_accel_aese_mc(t, z);
|
||||
}
|
||||
ret->v = (AESStateVec)t ^ rk->v;
|
||||
}
|
||||
|
||||
static inline void ATTR_AES_ACCEL
|
||||
aesdec_IMC_accel(AESState *ret, const AESState *st, bool be)
|
||||
{
|
||||
uint8x16_t t = (uint8x16_t)st->v;
|
||||
|
||||
if (be) {
|
||||
t = aes_accel_bswap(t);
|
||||
t = aes_accel_aesimc(t);
|
||||
t = aes_accel_bswap(t);
|
||||
} else {
|
||||
t = aes_accel_aesimc(t);
|
||||
}
|
||||
ret->v = (AESStateVec)t;
|
||||
}
|
||||
|
||||
static inline void ATTR_AES_ACCEL
|
||||
aesdec_ISB_ISR_AK_accel(AESState *ret, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
uint8x16_t t = (uint8x16_t)st->v;
|
||||
uint8x16_t z = { };
|
||||
|
||||
if (be) {
|
||||
t = aes_accel_bswap(t);
|
||||
t = aes_accel_aesd(t, z);
|
||||
t = aes_accel_bswap(t);
|
||||
} else {
|
||||
t = aes_accel_aesd(t, z);
|
||||
}
|
||||
ret->v = (AESStateVec)t ^ rk->v;
|
||||
}
|
||||
|
||||
static inline void ATTR_AES_ACCEL
|
||||
aesdec_ISB_ISR_AK_IMC_accel(AESState *ret, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
uint8x16_t t = (uint8x16_t)st->v;
|
||||
uint8x16_t k = (uint8x16_t)rk->v;
|
||||
uint8x16_t z = { };
|
||||
|
||||
if (be) {
|
||||
t = aes_accel_bswap(t);
|
||||
k = aes_accel_bswap(k);
|
||||
t = aes_accel_aesd(t, z);
|
||||
t ^= k;
|
||||
t = aes_accel_aesimc(t);
|
||||
t = aes_accel_bswap(t);
|
||||
} else {
|
||||
t = aes_accel_aesd(t, z);
|
||||
t ^= k;
|
||||
t = aes_accel_aesimc(t);
|
||||
}
|
||||
ret->v = (AESStateVec)t;
|
||||
}
|
||||
|
||||
static inline void ATTR_AES_ACCEL
|
||||
aesdec_ISB_ISR_IMC_AK_accel(AESState *ret, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
uint8x16_t t = (uint8x16_t)st->v;
|
||||
uint8x16_t z = { };
|
||||
|
||||
if (be) {
|
||||
t = aes_accel_bswap(t);
|
||||
t = aes_accel_aesd_imc(t, z);
|
||||
t = aes_accel_bswap(t);
|
||||
} else {
|
||||
t = aes_accel_aesd_imc(t, z);
|
||||
}
|
||||
ret->v = (AESStateVec)t ^ rk->v;
|
||||
}
|
||||
|
||||
#endif /* AARCH64_HOST_CRYPTO_AES_ROUND_H */
|
||||
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* No host specific aes acceleration.
|
||||
* SPDX-License-Identifier: GPL-2.0-or-later
|
||||
*/
|
||||
|
||||
#ifndef GENERIC_HOST_CRYPTO_AES_ROUND_H
|
||||
#define GENERIC_HOST_CRYPTO_AES_ROUND_H
|
||||
|
||||
#define HAVE_AES_ACCEL false
|
||||
#define ATTR_AES_ACCEL
|
||||
|
||||
void aesenc_MC_accel(AESState *, const AESState *, bool)
|
||||
QEMU_ERROR("unsupported accel");
|
||||
void aesenc_SB_SR_AK_accel(AESState *, const AESState *,
|
||||
const AESState *, bool)
|
||||
QEMU_ERROR("unsupported accel");
|
||||
void aesenc_SB_SR_MC_AK_accel(AESState *, const AESState *,
|
||||
const AESState *, bool)
|
||||
QEMU_ERROR("unsupported accel");
|
||||
|
||||
void aesdec_IMC_accel(AESState *, const AESState *, bool)
|
||||
QEMU_ERROR("unsupported accel");
|
||||
void aesdec_ISB_ISR_AK_accel(AESState *, const AESState *,
|
||||
const AESState *, bool)
|
||||
QEMU_ERROR("unsupported accel");
|
||||
void aesdec_ISB_ISR_AK_IMC_accel(AESState *, const AESState *,
|
||||
const AESState *, bool)
|
||||
QEMU_ERROR("unsupported accel");
|
||||
void aesdec_ISB_ISR_IMC_AK_accel(AESState *, const AESState *,
|
||||
const AESState *, bool)
|
||||
QEMU_ERROR("unsupported accel");
|
||||
|
||||
#endif /* GENERIC_HOST_CRYPTO_AES_ROUND_H */
|
||||
@@ -26,6 +26,7 @@
|
||||
#define CPUINFO_AVX512VBMI2 (1u << 15)
|
||||
#define CPUINFO_ATOMIC_VMOVDQA (1u << 16)
|
||||
#define CPUINFO_ATOMIC_VMOVDQU (1u << 17)
|
||||
#define CPUINFO_AES (1u << 18)
|
||||
|
||||
/* Initialized with a constructor. */
|
||||
extern unsigned cpuinfo;
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
/*
|
||||
* x86 specific aes acceleration.
|
||||
* SPDX-License-Identifier: GPL-2.0-or-later
|
||||
*/
|
||||
|
||||
#ifndef X86_HOST_CRYPTO_AES_ROUND_H
|
||||
#define X86_HOST_CRYPTO_AES_ROUND_H
|
||||
|
||||
#include "host/cpuinfo.h"
|
||||
#include <immintrin.h>
|
||||
|
||||
#if defined(__AES__) && defined(__SSSE3__)
|
||||
# define HAVE_AES_ACCEL true
|
||||
# define ATTR_AES_ACCEL
|
||||
#else
|
||||
# define HAVE_AES_ACCEL likely(cpuinfo & CPUINFO_AES)
|
||||
# define ATTR_AES_ACCEL __attribute__((target("aes,ssse3")))
|
||||
#endif
|
||||
|
||||
static inline __m128i ATTR_AES_ACCEL
|
||||
aes_accel_bswap(__m128i x)
|
||||
{
|
||||
return _mm_shuffle_epi8(x, _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8,
|
||||
9, 10, 11, 12, 13, 14, 15));
|
||||
}
|
||||
|
||||
static inline void ATTR_AES_ACCEL
|
||||
aesenc_MC_accel(AESState *ret, const AESState *st, bool be)
|
||||
{
|
||||
__m128i t = (__m128i)st->v;
|
||||
__m128i z = _mm_setzero_si128();
|
||||
|
||||
if (be) {
|
||||
t = aes_accel_bswap(t);
|
||||
t = _mm_aesdeclast_si128(t, z);
|
||||
t = _mm_aesenc_si128(t, z);
|
||||
t = aes_accel_bswap(t);
|
||||
} else {
|
||||
t = _mm_aesdeclast_si128(t, z);
|
||||
t = _mm_aesenc_si128(t, z);
|
||||
}
|
||||
ret->v = (AESStateVec)t;
|
||||
}
|
||||
|
||||
static inline void ATTR_AES_ACCEL
|
||||
aesenc_SB_SR_AK_accel(AESState *ret, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
__m128i t = (__m128i)st->v;
|
||||
__m128i k = (__m128i)rk->v;
|
||||
|
||||
if (be) {
|
||||
t = aes_accel_bswap(t);
|
||||
k = aes_accel_bswap(k);
|
||||
t = _mm_aesenclast_si128(t, k);
|
||||
t = aes_accel_bswap(t);
|
||||
} else {
|
||||
t = _mm_aesenclast_si128(t, k);
|
||||
}
|
||||
ret->v = (AESStateVec)t;
|
||||
}
|
||||
|
||||
static inline void ATTR_AES_ACCEL
|
||||
aesenc_SB_SR_MC_AK_accel(AESState *ret, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
__m128i t = (__m128i)st->v;
|
||||
__m128i k = (__m128i)rk->v;
|
||||
|
||||
if (be) {
|
||||
t = aes_accel_bswap(t);
|
||||
k = aes_accel_bswap(k);
|
||||
t = _mm_aesenc_si128(t, k);
|
||||
t = aes_accel_bswap(t);
|
||||
} else {
|
||||
t = _mm_aesenc_si128(t, k);
|
||||
}
|
||||
ret->v = (AESStateVec)t;
|
||||
}
|
||||
|
||||
static inline void ATTR_AES_ACCEL
|
||||
aesdec_IMC_accel(AESState *ret, const AESState *st, bool be)
|
||||
{
|
||||
__m128i t = (__m128i)st->v;
|
||||
|
||||
if (be) {
|
||||
t = aes_accel_bswap(t);
|
||||
t = _mm_aesimc_si128(t);
|
||||
t = aes_accel_bswap(t);
|
||||
} else {
|
||||
t = _mm_aesimc_si128(t);
|
||||
}
|
||||
ret->v = (AESStateVec)t;
|
||||
}
|
||||
|
||||
static inline void ATTR_AES_ACCEL
|
||||
aesdec_ISB_ISR_AK_accel(AESState *ret, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
__m128i t = (__m128i)st->v;
|
||||
__m128i k = (__m128i)rk->v;
|
||||
|
||||
if (be) {
|
||||
t = aes_accel_bswap(t);
|
||||
k = aes_accel_bswap(k);
|
||||
t = _mm_aesdeclast_si128(t, k);
|
||||
t = aes_accel_bswap(t);
|
||||
} else {
|
||||
t = _mm_aesdeclast_si128(t, k);
|
||||
}
|
||||
ret->v = (AESStateVec)t;
|
||||
}
|
||||
|
||||
static inline void ATTR_AES_ACCEL
|
||||
aesdec_ISB_ISR_AK_IMC_accel(AESState *ret, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
__m128i t = (__m128i)st->v;
|
||||
__m128i k = (__m128i)rk->v;
|
||||
|
||||
if (be) {
|
||||
t = aes_accel_bswap(t);
|
||||
k = aes_accel_bswap(k);
|
||||
t = _mm_aesdeclast_si128(t, k);
|
||||
t = _mm_aesimc_si128(t);
|
||||
t = aes_accel_bswap(t);
|
||||
} else {
|
||||
t = _mm_aesdeclast_si128(t, k);
|
||||
t = _mm_aesimc_si128(t);
|
||||
}
|
||||
ret->v = (AESStateVec)t;
|
||||
}
|
||||
|
||||
static inline void ATTR_AES_ACCEL
|
||||
aesdec_ISB_ISR_IMC_AK_accel(AESState *ret, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
__m128i t = (__m128i)st->v;
|
||||
__m128i k = (__m128i)rk->v;
|
||||
|
||||
if (be) {
|
||||
t = aes_accel_bswap(t);
|
||||
k = aes_accel_bswap(k);
|
||||
t = _mm_aesdec_si128(t, k);
|
||||
t = aes_accel_bswap(t);
|
||||
} else {
|
||||
t = _mm_aesdec_si128(t, k);
|
||||
}
|
||||
ret->v = (AESStateVec)t;
|
||||
}
|
||||
|
||||
#endif /* X86_HOST_CRYPTO_AES_ROUND_H */
|
||||
@@ -0,0 +1,30 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: GPL-2.0-or-later
|
||||
* Host specific cpu indentification for ppc.
|
||||
*/
|
||||
|
||||
#ifndef HOST_CPUINFO_H
|
||||
#define HOST_CPUINFO_H
|
||||
|
||||
/* Digested version of <cpuid.h> */
|
||||
|
||||
#define CPUINFO_ALWAYS (1u << 0) /* so cpuinfo is nonzero */
|
||||
#define CPUINFO_V2_06 (1u << 1)
|
||||
#define CPUINFO_V2_07 (1u << 2)
|
||||
#define CPUINFO_V3_0 (1u << 3)
|
||||
#define CPUINFO_V3_1 (1u << 4)
|
||||
#define CPUINFO_ISEL (1u << 5)
|
||||
#define CPUINFO_ALTIVEC (1u << 6)
|
||||
#define CPUINFO_VSX (1u << 7)
|
||||
#define CPUINFO_CRYPTO (1u << 8)
|
||||
|
||||
/* Initialized with a constructor. */
|
||||
extern unsigned cpuinfo;
|
||||
|
||||
/*
|
||||
* We cannot rely on constructor ordering, so other constructors must
|
||||
* use the function interface rather than the variable above.
|
||||
*/
|
||||
unsigned cpuinfo_init(void);
|
||||
|
||||
#endif /* HOST_CPUINFO_H */
|
||||
@@ -0,0 +1,182 @@
|
||||
/*
|
||||
* Power v2.07 specific aes acceleration.
|
||||
* SPDX-License-Identifier: GPL-2.0-or-later
|
||||
*/
|
||||
|
||||
#ifndef PPC_HOST_CRYPTO_AES_ROUND_H
|
||||
#define PPC_HOST_CRYPTO_AES_ROUND_H
|
||||
|
||||
#ifdef __ALTIVEC__
|
||||
#include "host/cpuinfo.h"
|
||||
|
||||
#ifdef __CRYPTO__
|
||||
# define HAVE_AES_ACCEL true
|
||||
#else
|
||||
# define HAVE_AES_ACCEL likely(cpuinfo & CPUINFO_CRYPTO)
|
||||
#endif
|
||||
#define ATTR_AES_ACCEL
|
||||
|
||||
/*
|
||||
* While there is <altivec.h>, both gcc and clang "aid" with the
|
||||
* endianness issues in different ways. Just use inline asm instead.
|
||||
*/
|
||||
|
||||
/* Bytes in memory are host-endian; bytes in register are @be. */
|
||||
static inline AESStateVec aes_accel_ld(const AESState *p, bool be)
|
||||
{
|
||||
AESStateVec r;
|
||||
|
||||
if (be) {
|
||||
asm("lvx %0, 0, %1" : "=v"(r) : "r"(p), "m"(*p));
|
||||
} else if (HOST_BIG_ENDIAN) {
|
||||
AESStateVec rev = {
|
||||
15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0,
|
||||
};
|
||||
asm("lvx %0, 0, %1\n\t"
|
||||
"vperm %0, %0, %0, %2"
|
||||
: "=v"(r) : "r"(p), "v"(rev), "m"(*p));
|
||||
} else {
|
||||
#ifdef __POWER9_VECTOR__
|
||||
asm("lxvb16x %x0, 0, %1" : "=v"(r) : "r"(p), "m"(*p));
|
||||
#else
|
||||
asm("lxvd2x %x0, 0, %1\n\t"
|
||||
"xxpermdi %x0, %x0, %x0, 2"
|
||||
: "=v"(r) : "r"(p), "m"(*p));
|
||||
#endif
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
static void aes_accel_st(AESState *p, AESStateVec r, bool be)
|
||||
{
|
||||
if (be) {
|
||||
asm("stvx %1, 0, %2" : "=m"(*p) : "v"(r), "r"(p));
|
||||
} else if (HOST_BIG_ENDIAN) {
|
||||
AESStateVec rev = {
|
||||
15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0,
|
||||
};
|
||||
asm("vperm %1, %1, %1, %2\n\t"
|
||||
"stvx %1, 0, %3"
|
||||
: "=m"(*p), "+v"(r) : "v"(rev), "r"(p));
|
||||
} else {
|
||||
#ifdef __POWER9_VECTOR__
|
||||
asm("stxvb16x %x1, 0, %2" : "=m"(*p) : "v"(r), "r"(p));
|
||||
#else
|
||||
asm("xxpermdi %x1, %x1, %x1, 2\n\t"
|
||||
"stxvd2x %x1, 0, %2"
|
||||
: "=m"(*p), "+v"(r) : "r"(p));
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
static inline AESStateVec aes_accel_vcipher(AESStateVec d, AESStateVec k)
|
||||
{
|
||||
asm("vcipher %0, %0, %1" : "+v"(d) : "v"(k));
|
||||
return d;
|
||||
}
|
||||
|
||||
static inline AESStateVec aes_accel_vncipher(AESStateVec d, AESStateVec k)
|
||||
{
|
||||
asm("vncipher %0, %0, %1" : "+v"(d) : "v"(k));
|
||||
return d;
|
||||
}
|
||||
|
||||
static inline AESStateVec aes_accel_vcipherlast(AESStateVec d, AESStateVec k)
|
||||
{
|
||||
asm("vcipherlast %0, %0, %1" : "+v"(d) : "v"(k));
|
||||
return d;
|
||||
}
|
||||
|
||||
static inline AESStateVec aes_accel_vncipherlast(AESStateVec d, AESStateVec k)
|
||||
{
|
||||
asm("vncipherlast %0, %0, %1" : "+v"(d) : "v"(k));
|
||||
return d;
|
||||
}
|
||||
|
||||
static inline void
|
||||
aesenc_MC_accel(AESState *ret, const AESState *st, bool be)
|
||||
{
|
||||
AESStateVec t, z = { };
|
||||
|
||||
t = aes_accel_ld(st, be);
|
||||
t = aes_accel_vncipherlast(t, z);
|
||||
t = aes_accel_vcipher(t, z);
|
||||
aes_accel_st(ret, t, be);
|
||||
}
|
||||
|
||||
static inline void
|
||||
aesenc_SB_SR_AK_accel(AESState *ret, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
AESStateVec t, k;
|
||||
|
||||
t = aes_accel_ld(st, be);
|
||||
k = aes_accel_ld(rk, be);
|
||||
t = aes_accel_vcipherlast(t, k);
|
||||
aes_accel_st(ret, t, be);
|
||||
}
|
||||
|
||||
static inline void
|
||||
aesenc_SB_SR_MC_AK_accel(AESState *ret, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
AESStateVec t, k;
|
||||
|
||||
t = aes_accel_ld(st, be);
|
||||
k = aes_accel_ld(rk, be);
|
||||
t = aes_accel_vcipher(t, k);
|
||||
aes_accel_st(ret, t, be);
|
||||
}
|
||||
|
||||
static inline void
|
||||
aesdec_IMC_accel(AESState *ret, const AESState *st, bool be)
|
||||
{
|
||||
AESStateVec t, z = { };
|
||||
|
||||
t = aes_accel_ld(st, be);
|
||||
t = aes_accel_vcipherlast(t, z);
|
||||
t = aes_accel_vncipher(t, z);
|
||||
aes_accel_st(ret, t, be);
|
||||
}
|
||||
|
||||
static inline void
|
||||
aesdec_ISB_ISR_AK_accel(AESState *ret, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
AESStateVec t, k;
|
||||
|
||||
t = aes_accel_ld(st, be);
|
||||
k = aes_accel_ld(rk, be);
|
||||
t = aes_accel_vncipherlast(t, k);
|
||||
aes_accel_st(ret, t, be);
|
||||
}
|
||||
|
||||
static inline void
|
||||
aesdec_ISB_ISR_AK_IMC_accel(AESState *ret, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
AESStateVec t, k;
|
||||
|
||||
t = aes_accel_ld(st, be);
|
||||
k = aes_accel_ld(rk, be);
|
||||
t = aes_accel_vncipher(t, k);
|
||||
aes_accel_st(ret, t, be);
|
||||
}
|
||||
|
||||
static inline void
|
||||
aesdec_ISB_ISR_IMC_AK_accel(AESState *ret, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
AESStateVec t, k, z = { };
|
||||
|
||||
t = aes_accel_ld(st, be);
|
||||
k = aes_accel_ld(rk, be);
|
||||
t = aes_accel_vncipher(t, z);
|
||||
aes_accel_st(ret, t ^ k, be);
|
||||
}
|
||||
#else
|
||||
/* Without ALTIVEC, we can't even write inline assembly. */
|
||||
#include "host/include/generic/host/crypto/aes-round.h"
|
||||
#endif
|
||||
|
||||
#endif /* PPC_HOST_CRYPTO_AES_ROUND_H */
|
||||
@@ -0,0 +1 @@
|
||||
#include "host/include/ppc/host/cpuinfo.h"
|
||||
@@ -0,0 +1 @@
|
||||
#include "host/include/ppc/host/crypto/aes-round.h"
|
||||
@@ -0,0 +1 @@
|
||||
#include "host/include/i386/host/crypto/aes-round.h"
|
||||
@@ -0,0 +1,164 @@
|
||||
/*
|
||||
* AES round fragments, generic version
|
||||
* SPDX-License-Identifier: GPL-2.0-or-later
|
||||
*
|
||||
* Copyright (C) 2023 Linaro, Ltd.
|
||||
*/
|
||||
|
||||
#ifndef CRYPTO_AES_ROUND_H
|
||||
#define CRYPTO_AES_ROUND_H
|
||||
|
||||
/* Hosts with acceleration will usually need a 16-byte vector type. */
|
||||
typedef uint8_t AESStateVec __attribute__((vector_size(16)));
|
||||
|
||||
typedef union {
|
||||
uint8_t b[16];
|
||||
uint32_t w[4];
|
||||
uint64_t d[2];
|
||||
AESStateVec v;
|
||||
} AESState;
|
||||
|
||||
#include "host/crypto/aes-round.h"
|
||||
|
||||
/*
|
||||
* Perform MixColumns.
|
||||
*/
|
||||
|
||||
void aesenc_MC_gen(AESState *ret, const AESState *st);
|
||||
void aesenc_MC_genrev(AESState *ret, const AESState *st);
|
||||
|
||||
static inline void aesenc_MC(AESState *r, const AESState *st, bool be)
|
||||
{
|
||||
if (HAVE_AES_ACCEL) {
|
||||
aesenc_MC_accel(r, st, be);
|
||||
} else if (HOST_BIG_ENDIAN == be) {
|
||||
aesenc_MC_gen(r, st);
|
||||
} else {
|
||||
aesenc_MC_genrev(r, st);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Perform SubBytes + ShiftRows + AddRoundKey.
|
||||
*/
|
||||
|
||||
void aesenc_SB_SR_AK_gen(AESState *ret, const AESState *st,
|
||||
const AESState *rk);
|
||||
void aesenc_SB_SR_AK_genrev(AESState *ret, const AESState *st,
|
||||
const AESState *rk);
|
||||
|
||||
static inline void aesenc_SB_SR_AK(AESState *r, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
if (HAVE_AES_ACCEL) {
|
||||
aesenc_SB_SR_AK_accel(r, st, rk, be);
|
||||
} else if (HOST_BIG_ENDIAN == be) {
|
||||
aesenc_SB_SR_AK_gen(r, st, rk);
|
||||
} else {
|
||||
aesenc_SB_SR_AK_genrev(r, st, rk);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Perform SubBytes + ShiftRows + MixColumns + AddRoundKey.
|
||||
*/
|
||||
|
||||
void aesenc_SB_SR_MC_AK_gen(AESState *ret, const AESState *st,
|
||||
const AESState *rk);
|
||||
void aesenc_SB_SR_MC_AK_genrev(AESState *ret, const AESState *st,
|
||||
const AESState *rk);
|
||||
|
||||
static inline void aesenc_SB_SR_MC_AK(AESState *r, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
if (HAVE_AES_ACCEL) {
|
||||
aesenc_SB_SR_MC_AK_accel(r, st, rk, be);
|
||||
} else if (HOST_BIG_ENDIAN == be) {
|
||||
aesenc_SB_SR_MC_AK_gen(r, st, rk);
|
||||
} else {
|
||||
aesenc_SB_SR_MC_AK_genrev(r, st, rk);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Perform InvMixColumns.
|
||||
*/
|
||||
|
||||
void aesdec_IMC_gen(AESState *ret, const AESState *st);
|
||||
void aesdec_IMC_genrev(AESState *ret, const AESState *st);
|
||||
|
||||
static inline void aesdec_IMC(AESState *r, const AESState *st, bool be)
|
||||
{
|
||||
if (HAVE_AES_ACCEL) {
|
||||
aesdec_IMC_accel(r, st, be);
|
||||
} else if (HOST_BIG_ENDIAN == be) {
|
||||
aesdec_IMC_gen(r, st);
|
||||
} else {
|
||||
aesdec_IMC_genrev(r, st);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Perform InvSubBytes + InvShiftRows + AddRoundKey.
|
||||
*/
|
||||
|
||||
void aesdec_ISB_ISR_AK_gen(AESState *ret, const AESState *st,
|
||||
const AESState *rk);
|
||||
void aesdec_ISB_ISR_AK_genrev(AESState *ret, const AESState *st,
|
||||
const AESState *rk);
|
||||
|
||||
static inline void aesdec_ISB_ISR_AK(AESState *r, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
if (HAVE_AES_ACCEL) {
|
||||
aesdec_ISB_ISR_AK_accel(r, st, rk, be);
|
||||
} else if (HOST_BIG_ENDIAN == be) {
|
||||
aesdec_ISB_ISR_AK_gen(r, st, rk);
|
||||
} else {
|
||||
aesdec_ISB_ISR_AK_genrev(r, st, rk);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Perform InvSubBytes + InvShiftRows + AddRoundKey + InvMixColumns.
|
||||
*/
|
||||
|
||||
void aesdec_ISB_ISR_AK_IMC_gen(AESState *ret, const AESState *st,
|
||||
const AESState *rk);
|
||||
void aesdec_ISB_ISR_AK_IMC_genrev(AESState *ret, const AESState *st,
|
||||
const AESState *rk);
|
||||
|
||||
static inline void aesdec_ISB_ISR_AK_IMC(AESState *r, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
if (HAVE_AES_ACCEL) {
|
||||
aesdec_ISB_ISR_AK_IMC_accel(r, st, rk, be);
|
||||
} else if (HOST_BIG_ENDIAN == be) {
|
||||
aesdec_ISB_ISR_AK_IMC_gen(r, st, rk);
|
||||
} else {
|
||||
aesdec_ISB_ISR_AK_IMC_genrev(r, st, rk);
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Perform InvSubBytes + InvShiftRows + InvMixColumns + AddRoundKey.
|
||||
*/
|
||||
|
||||
void aesdec_ISB_ISR_IMC_AK_gen(AESState *ret, const AESState *st,
|
||||
const AESState *rk);
|
||||
void aesdec_ISB_ISR_IMC_AK_genrev(AESState *ret, const AESState *st,
|
||||
const AESState *rk);
|
||||
|
||||
static inline void aesdec_ISB_ISR_IMC_AK(AESState *r, const AESState *st,
|
||||
const AESState *rk, bool be)
|
||||
{
|
||||
if (HAVE_AES_ACCEL) {
|
||||
aesdec_ISB_ISR_IMC_AK_accel(r, st, rk, be);
|
||||
} else if (HOST_BIG_ENDIAN == be) {
|
||||
aesdec_ISB_ISR_IMC_AK_gen(r, st, rk);
|
||||
} else {
|
||||
aesdec_ISB_ISR_IMC_AK_genrev(r, st, rk);
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* CRYPTO_AES_ROUND_H */
|
||||
@@ -30,34 +30,4 @@ void AES_decrypt(const unsigned char *in, unsigned char *out,
|
||||
extern const uint8_t AES_sbox[256];
|
||||
extern const uint8_t AES_isbox[256];
|
||||
|
||||
/* AES ShiftRows and InvShiftRows */
|
||||
extern const uint8_t AES_shifts[16];
|
||||
extern const uint8_t AES_ishifts[16];
|
||||
|
||||
/* AES InvMixColumns */
|
||||
/* AES_imc[x][0] = [x].[0e, 09, 0d, 0b]; */
|
||||
/* AES_imc[x][1] = [x].[0b, 0e, 09, 0d]; */
|
||||
/* AES_imc[x][2] = [x].[0d, 0b, 0e, 09]; */
|
||||
/* AES_imc[x][3] = [x].[09, 0d, 0b, 0e]; */
|
||||
extern const uint32_t AES_imc[256][4];
|
||||
|
||||
/*
|
||||
AES_Te0[x] = S [x].[02, 01, 01, 03];
|
||||
AES_Te1[x] = S [x].[03, 02, 01, 01];
|
||||
AES_Te2[x] = S [x].[01, 03, 02, 01];
|
||||
AES_Te3[x] = S [x].[01, 01, 03, 02];
|
||||
AES_Te4[x] = S [x].[01, 01, 01, 01];
|
||||
|
||||
AES_Td0[x] = Si[x].[0e, 09, 0d, 0b];
|
||||
AES_Td1[x] = Si[x].[0b, 0e, 09, 0d];
|
||||
AES_Td2[x] = Si[x].[0d, 0b, 0e, 09];
|
||||
AES_Td3[x] = Si[x].[09, 0d, 0b, 0e];
|
||||
AES_Td4[x] = Si[x].[01, 01, 01, 01];
|
||||
*/
|
||||
|
||||
extern const uint32_t AES_Te0[256], AES_Te1[256], AES_Te2[256],
|
||||
AES_Te3[256], AES_Te4[256];
|
||||
extern const uint32_t AES_Td0[256], AES_Td1[256], AES_Td2[256],
|
||||
AES_Td3[256], AES_Td4[256];
|
||||
|
||||
#endif
|
||||
|
||||
@@ -2665,6 +2665,15 @@ config_host_data.set('CONFIG_AVX512BW_OPT', get_option('avx512bw') \
|
||||
int main(int argc, char *argv[]) { return bar(argv[0]); }
|
||||
'''), error_message: 'AVX512BW not available').allowed())
|
||||
|
||||
# For both AArch64 and AArch32, detect if builtins are available.
|
||||
config_host_data.set('CONFIG_ARM_AES_BUILTIN', cc.compiles('''
|
||||
#include <arm_neon.h>
|
||||
#ifndef __ARM_FEATURE_AES
|
||||
__attribute__((target("+crypto")))
|
||||
#endif
|
||||
void foo(uint8x16_t *p) { *p = vaesmcq_u8(*p); }
|
||||
'''))
|
||||
|
||||
have_pvrdma = get_option('pvrdma') \
|
||||
.require(rdma.found(), error_message: 'PVRDMA requires OpenFabrics libraries') \
|
||||
.require(cc.compiles(gnu_source_prefix + '''
|
||||
|
||||
@@ -552,7 +552,9 @@ DEF_HELPER_FLAGS_2(neon_qzip16, TCG_CALL_NO_RWG, void, ptr, ptr)
|
||||
DEF_HELPER_FLAGS_2(neon_qzip32, TCG_CALL_NO_RWG, void, ptr, ptr)
|
||||
|
||||
DEF_HELPER_FLAGS_4(crypto_aese, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
|
||||
DEF_HELPER_FLAGS_4(crypto_aesd, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
|
||||
DEF_HELPER_FLAGS_3(crypto_aesmc, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
|
||||
DEF_HELPER_FLAGS_3(crypto_aesimc, TCG_CALL_NO_RWG, void, ptr, ptr, i32)
|
||||
|
||||
DEF_HELPER_FLAGS_4(crypto_sha1su0, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
|
||||
DEF_HELPER_FLAGS_4(crypto_sha1c, TCG_CALL_NO_RWG, void, ptr, ptr, ptr, i32)
|
||||
|
||||
+78
-171
@@ -14,7 +14,7 @@
|
||||
#include "cpu.h"
|
||||
#include "exec/helper-proto.h"
|
||||
#include "tcg/tcg-gvec-desc.h"
|
||||
#include "crypto/aes.h"
|
||||
#include "crypto/aes-round.h"
|
||||
#include "crypto/sm4.h"
|
||||
#include "vec_internal.h"
|
||||
|
||||
@@ -45,197 +45,104 @@ static void clear_tail_16(void *vd, uint32_t desc)
|
||||
clear_tail(vd, opr_sz, max_sz);
|
||||
}
|
||||
|
||||
static void do_crypto_aese(uint64_t *rd, uint64_t *rn,
|
||||
uint64_t *rm, bool decrypt)
|
||||
{
|
||||
static uint8_t const * const sbox[2] = { AES_sbox, AES_isbox };
|
||||
static uint8_t const * const shift[2] = { AES_shifts, AES_ishifts };
|
||||
union CRYPTO_STATE rk = { .l = { rm[0], rm[1] } };
|
||||
union CRYPTO_STATE st = { .l = { rn[0], rn[1] } };
|
||||
int i;
|
||||
|
||||
/* xor state vector with round key */
|
||||
rk.l[0] ^= st.l[0];
|
||||
rk.l[1] ^= st.l[1];
|
||||
|
||||
/* combine ShiftRows operation and sbox substitution */
|
||||
for (i = 0; i < 16; i++) {
|
||||
CR_ST_BYTE(st, i) = sbox[decrypt][CR_ST_BYTE(rk, shift[decrypt][i])];
|
||||
}
|
||||
|
||||
rd[0] = st.l[0];
|
||||
rd[1] = st.l[1];
|
||||
}
|
||||
static const AESState aes_zero = { };
|
||||
|
||||
void HELPER(crypto_aese)(void *vd, void *vn, void *vm, uint32_t desc)
|
||||
{
|
||||
intptr_t i, opr_sz = simd_oprsz(desc);
|
||||
bool decrypt = simd_data(desc);
|
||||
|
||||
for (i = 0; i < opr_sz; i += 16) {
|
||||
do_crypto_aese(vd + i, vn + i, vm + i, decrypt);
|
||||
AESState *ad = (AESState *)(vd + i);
|
||||
AESState *st = (AESState *)(vn + i);
|
||||
AESState *rk = (AESState *)(vm + i);
|
||||
AESState t;
|
||||
|
||||
/*
|
||||
* Our uint64_t are in the wrong order for big-endian.
|
||||
* The Arm AddRoundKey comes first, while the API AddRoundKey
|
||||
* comes last: perform the xor here, and provide zero to API.
|
||||
*/
|
||||
if (HOST_BIG_ENDIAN) {
|
||||
t.d[0] = st->d[1] ^ rk->d[1];
|
||||
t.d[1] = st->d[0] ^ rk->d[0];
|
||||
aesenc_SB_SR_AK(&t, &t, &aes_zero, false);
|
||||
ad->d[0] = t.d[1];
|
||||
ad->d[1] = t.d[0];
|
||||
} else {
|
||||
t.v = st->v ^ rk->v;
|
||||
aesenc_SB_SR_AK(ad, &t, &aes_zero, false);
|
||||
}
|
||||
}
|
||||
clear_tail(vd, opr_sz, simd_maxsz(desc));
|
||||
}
|
||||
|
||||
static void do_crypto_aesmc(uint64_t *rd, uint64_t *rm, bool decrypt)
|
||||
void HELPER(crypto_aesd)(void *vd, void *vn, void *vm, uint32_t desc)
|
||||
{
|
||||
static uint32_t const mc[][256] = { {
|
||||
/* MixColumns lookup table */
|
||||
0x00000000, 0x03010102, 0x06020204, 0x05030306,
|
||||
0x0c040408, 0x0f05050a, 0x0a06060c, 0x0907070e,
|
||||
0x18080810, 0x1b090912, 0x1e0a0a14, 0x1d0b0b16,
|
||||
0x140c0c18, 0x170d0d1a, 0x120e0e1c, 0x110f0f1e,
|
||||
0x30101020, 0x33111122, 0x36121224, 0x35131326,
|
||||
0x3c141428, 0x3f15152a, 0x3a16162c, 0x3917172e,
|
||||
0x28181830, 0x2b191932, 0x2e1a1a34, 0x2d1b1b36,
|
||||
0x241c1c38, 0x271d1d3a, 0x221e1e3c, 0x211f1f3e,
|
||||
0x60202040, 0x63212142, 0x66222244, 0x65232346,
|
||||
0x6c242448, 0x6f25254a, 0x6a26264c, 0x6927274e,
|
||||
0x78282850, 0x7b292952, 0x7e2a2a54, 0x7d2b2b56,
|
||||
0x742c2c58, 0x772d2d5a, 0x722e2e5c, 0x712f2f5e,
|
||||
0x50303060, 0x53313162, 0x56323264, 0x55333366,
|
||||
0x5c343468, 0x5f35356a, 0x5a36366c, 0x5937376e,
|
||||
0x48383870, 0x4b393972, 0x4e3a3a74, 0x4d3b3b76,
|
||||
0x443c3c78, 0x473d3d7a, 0x423e3e7c, 0x413f3f7e,
|
||||
0xc0404080, 0xc3414182, 0xc6424284, 0xc5434386,
|
||||
0xcc444488, 0xcf45458a, 0xca46468c, 0xc947478e,
|
||||
0xd8484890, 0xdb494992, 0xde4a4a94, 0xdd4b4b96,
|
||||
0xd44c4c98, 0xd74d4d9a, 0xd24e4e9c, 0xd14f4f9e,
|
||||
0xf05050a0, 0xf35151a2, 0xf65252a4, 0xf55353a6,
|
||||
0xfc5454a8, 0xff5555aa, 0xfa5656ac, 0xf95757ae,
|
||||
0xe85858b0, 0xeb5959b2, 0xee5a5ab4, 0xed5b5bb6,
|
||||
0xe45c5cb8, 0xe75d5dba, 0xe25e5ebc, 0xe15f5fbe,
|
||||
0xa06060c0, 0xa36161c2, 0xa66262c4, 0xa56363c6,
|
||||
0xac6464c8, 0xaf6565ca, 0xaa6666cc, 0xa96767ce,
|
||||
0xb86868d0, 0xbb6969d2, 0xbe6a6ad4, 0xbd6b6bd6,
|
||||
0xb46c6cd8, 0xb76d6dda, 0xb26e6edc, 0xb16f6fde,
|
||||
0x907070e0, 0x937171e2, 0x967272e4, 0x957373e6,
|
||||
0x9c7474e8, 0x9f7575ea, 0x9a7676ec, 0x997777ee,
|
||||
0x887878f0, 0x8b7979f2, 0x8e7a7af4, 0x8d7b7bf6,
|
||||
0x847c7cf8, 0x877d7dfa, 0x827e7efc, 0x817f7ffe,
|
||||
0x9b80801b, 0x98818119, 0x9d82821f, 0x9e83831d,
|
||||
0x97848413, 0x94858511, 0x91868617, 0x92878715,
|
||||
0x8388880b, 0x80898909, 0x858a8a0f, 0x868b8b0d,
|
||||
0x8f8c8c03, 0x8c8d8d01, 0x898e8e07, 0x8a8f8f05,
|
||||
0xab90903b, 0xa8919139, 0xad92923f, 0xae93933d,
|
||||
0xa7949433, 0xa4959531, 0xa1969637, 0xa2979735,
|
||||
0xb398982b, 0xb0999929, 0xb59a9a2f, 0xb69b9b2d,
|
||||
0xbf9c9c23, 0xbc9d9d21, 0xb99e9e27, 0xba9f9f25,
|
||||
0xfba0a05b, 0xf8a1a159, 0xfda2a25f, 0xfea3a35d,
|
||||
0xf7a4a453, 0xf4a5a551, 0xf1a6a657, 0xf2a7a755,
|
||||
0xe3a8a84b, 0xe0a9a949, 0xe5aaaa4f, 0xe6abab4d,
|
||||
0xefacac43, 0xecadad41, 0xe9aeae47, 0xeaafaf45,
|
||||
0xcbb0b07b, 0xc8b1b179, 0xcdb2b27f, 0xceb3b37d,
|
||||
0xc7b4b473, 0xc4b5b571, 0xc1b6b677, 0xc2b7b775,
|
||||
0xd3b8b86b, 0xd0b9b969, 0xd5baba6f, 0xd6bbbb6d,
|
||||
0xdfbcbc63, 0xdcbdbd61, 0xd9bebe67, 0xdabfbf65,
|
||||
0x5bc0c09b, 0x58c1c199, 0x5dc2c29f, 0x5ec3c39d,
|
||||
0x57c4c493, 0x54c5c591, 0x51c6c697, 0x52c7c795,
|
||||
0x43c8c88b, 0x40c9c989, 0x45caca8f, 0x46cbcb8d,
|
||||
0x4fcccc83, 0x4ccdcd81, 0x49cece87, 0x4acfcf85,
|
||||
0x6bd0d0bb, 0x68d1d1b9, 0x6dd2d2bf, 0x6ed3d3bd,
|
||||
0x67d4d4b3, 0x64d5d5b1, 0x61d6d6b7, 0x62d7d7b5,
|
||||
0x73d8d8ab, 0x70d9d9a9, 0x75dadaaf, 0x76dbdbad,
|
||||
0x7fdcdca3, 0x7cdddda1, 0x79dedea7, 0x7adfdfa5,
|
||||
0x3be0e0db, 0x38e1e1d9, 0x3de2e2df, 0x3ee3e3dd,
|
||||
0x37e4e4d3, 0x34e5e5d1, 0x31e6e6d7, 0x32e7e7d5,
|
||||
0x23e8e8cb, 0x20e9e9c9, 0x25eaeacf, 0x26ebebcd,
|
||||
0x2fececc3, 0x2cededc1, 0x29eeeec7, 0x2aefefc5,
|
||||
0x0bf0f0fb, 0x08f1f1f9, 0x0df2f2ff, 0x0ef3f3fd,
|
||||
0x07f4f4f3, 0x04f5f5f1, 0x01f6f6f7, 0x02f7f7f5,
|
||||
0x13f8f8eb, 0x10f9f9e9, 0x15fafaef, 0x16fbfbed,
|
||||
0x1ffcfce3, 0x1cfdfde1, 0x19fefee7, 0x1affffe5,
|
||||
}, {
|
||||
/* Inverse MixColumns lookup table */
|
||||
0x00000000, 0x0b0d090e, 0x161a121c, 0x1d171b12,
|
||||
0x2c342438, 0x27392d36, 0x3a2e3624, 0x31233f2a,
|
||||
0x58684870, 0x5365417e, 0x4e725a6c, 0x457f5362,
|
||||
0x745c6c48, 0x7f516546, 0x62467e54, 0x694b775a,
|
||||
0xb0d090e0, 0xbbdd99ee, 0xa6ca82fc, 0xadc78bf2,
|
||||
0x9ce4b4d8, 0x97e9bdd6, 0x8afea6c4, 0x81f3afca,
|
||||
0xe8b8d890, 0xe3b5d19e, 0xfea2ca8c, 0xf5afc382,
|
||||
0xc48cfca8, 0xcf81f5a6, 0xd296eeb4, 0xd99be7ba,
|
||||
0x7bbb3bdb, 0x70b632d5, 0x6da129c7, 0x66ac20c9,
|
||||
0x578f1fe3, 0x5c8216ed, 0x41950dff, 0x4a9804f1,
|
||||
0x23d373ab, 0x28de7aa5, 0x35c961b7, 0x3ec468b9,
|
||||
0x0fe75793, 0x04ea5e9d, 0x19fd458f, 0x12f04c81,
|
||||
0xcb6bab3b, 0xc066a235, 0xdd71b927, 0xd67cb029,
|
||||
0xe75f8f03, 0xec52860d, 0xf1459d1f, 0xfa489411,
|
||||
0x9303e34b, 0x980eea45, 0x8519f157, 0x8e14f859,
|
||||
0xbf37c773, 0xb43ace7d, 0xa92dd56f, 0xa220dc61,
|
||||
0xf66d76ad, 0xfd607fa3, 0xe07764b1, 0xeb7a6dbf,
|
||||
0xda595295, 0xd1545b9b, 0xcc434089, 0xc74e4987,
|
||||
0xae053edd, 0xa50837d3, 0xb81f2cc1, 0xb31225cf,
|
||||
0x82311ae5, 0x893c13eb, 0x942b08f9, 0x9f2601f7,
|
||||
0x46bde64d, 0x4db0ef43, 0x50a7f451, 0x5baafd5f,
|
||||
0x6a89c275, 0x6184cb7b, 0x7c93d069, 0x779ed967,
|
||||
0x1ed5ae3d, 0x15d8a733, 0x08cfbc21, 0x03c2b52f,
|
||||
0x32e18a05, 0x39ec830b, 0x24fb9819, 0x2ff69117,
|
||||
0x8dd64d76, 0x86db4478, 0x9bcc5f6a, 0x90c15664,
|
||||
0xa1e2694e, 0xaaef6040, 0xb7f87b52, 0xbcf5725c,
|
||||
0xd5be0506, 0xdeb30c08, 0xc3a4171a, 0xc8a91e14,
|
||||
0xf98a213e, 0xf2872830, 0xef903322, 0xe49d3a2c,
|
||||
0x3d06dd96, 0x360bd498, 0x2b1ccf8a, 0x2011c684,
|
||||
0x1132f9ae, 0x1a3ff0a0, 0x0728ebb2, 0x0c25e2bc,
|
||||
0x656e95e6, 0x6e639ce8, 0x737487fa, 0x78798ef4,
|
||||
0x495ab1de, 0x4257b8d0, 0x5f40a3c2, 0x544daacc,
|
||||
0xf7daec41, 0xfcd7e54f, 0xe1c0fe5d, 0xeacdf753,
|
||||
0xdbeec879, 0xd0e3c177, 0xcdf4da65, 0xc6f9d36b,
|
||||
0xafb2a431, 0xa4bfad3f, 0xb9a8b62d, 0xb2a5bf23,
|
||||
0x83868009, 0x888b8907, 0x959c9215, 0x9e919b1b,
|
||||
0x470a7ca1, 0x4c0775af, 0x51106ebd, 0x5a1d67b3,
|
||||
0x6b3e5899, 0x60335197, 0x7d244a85, 0x7629438b,
|
||||
0x1f6234d1, 0x146f3ddf, 0x097826cd, 0x02752fc3,
|
||||
0x335610e9, 0x385b19e7, 0x254c02f5, 0x2e410bfb,
|
||||
0x8c61d79a, 0x876cde94, 0x9a7bc586, 0x9176cc88,
|
||||
0xa055f3a2, 0xab58faac, 0xb64fe1be, 0xbd42e8b0,
|
||||
0xd4099fea, 0xdf0496e4, 0xc2138df6, 0xc91e84f8,
|
||||
0xf83dbbd2, 0xf330b2dc, 0xee27a9ce, 0xe52aa0c0,
|
||||
0x3cb1477a, 0x37bc4e74, 0x2aab5566, 0x21a65c68,
|
||||
0x10856342, 0x1b886a4c, 0x069f715e, 0x0d927850,
|
||||
0x64d90f0a, 0x6fd40604, 0x72c31d16, 0x79ce1418,
|
||||
0x48ed2b32, 0x43e0223c, 0x5ef7392e, 0x55fa3020,
|
||||
0x01b79aec, 0x0aba93e2, 0x17ad88f0, 0x1ca081fe,
|
||||
0x2d83bed4, 0x268eb7da, 0x3b99acc8, 0x3094a5c6,
|
||||
0x59dfd29c, 0x52d2db92, 0x4fc5c080, 0x44c8c98e,
|
||||
0x75ebf6a4, 0x7ee6ffaa, 0x63f1e4b8, 0x68fcedb6,
|
||||
0xb1670a0c, 0xba6a0302, 0xa77d1810, 0xac70111e,
|
||||
0x9d532e34, 0x965e273a, 0x8b493c28, 0x80443526,
|
||||
0xe90f427c, 0xe2024b72, 0xff155060, 0xf418596e,
|
||||
0xc53b6644, 0xce366f4a, 0xd3217458, 0xd82c7d56,
|
||||
0x7a0ca137, 0x7101a839, 0x6c16b32b, 0x671bba25,
|
||||
0x5638850f, 0x5d358c01, 0x40229713, 0x4b2f9e1d,
|
||||
0x2264e947, 0x2969e049, 0x347efb5b, 0x3f73f255,
|
||||
0x0e50cd7f, 0x055dc471, 0x184adf63, 0x1347d66d,
|
||||
0xcadc31d7, 0xc1d138d9, 0xdcc623cb, 0xd7cb2ac5,
|
||||
0xe6e815ef, 0xede51ce1, 0xf0f207f3, 0xfbff0efd,
|
||||
0x92b479a7, 0x99b970a9, 0x84ae6bbb, 0x8fa362b5,
|
||||
0xbe805d9f, 0xb58d5491, 0xa89a4f83, 0xa397468d,
|
||||
} };
|
||||
intptr_t i, opr_sz = simd_oprsz(desc);
|
||||
|
||||
union CRYPTO_STATE st = { .l = { rm[0], rm[1] } };
|
||||
int i;
|
||||
for (i = 0; i < opr_sz; i += 16) {
|
||||
AESState *ad = (AESState *)(vd + i);
|
||||
AESState *st = (AESState *)(vn + i);
|
||||
AESState *rk = (AESState *)(vm + i);
|
||||
AESState t;
|
||||
|
||||
for (i = 0; i < 16; i += 4) {
|
||||
CR_ST_WORD(st, i >> 2) =
|
||||
mc[decrypt][CR_ST_BYTE(st, i)] ^
|
||||
rol32(mc[decrypt][CR_ST_BYTE(st, i + 1)], 8) ^
|
||||
rol32(mc[decrypt][CR_ST_BYTE(st, i + 2)], 16) ^
|
||||
rol32(mc[decrypt][CR_ST_BYTE(st, i + 3)], 24);
|
||||
/* Our uint64_t are in the wrong order for big-endian. */
|
||||
if (HOST_BIG_ENDIAN) {
|
||||
t.d[0] = st->d[1] ^ rk->d[1];
|
||||
t.d[1] = st->d[0] ^ rk->d[0];
|
||||
aesdec_ISB_ISR_AK(&t, &t, &aes_zero, false);
|
||||
ad->d[0] = t.d[1];
|
||||
ad->d[1] = t.d[0];
|
||||
} else {
|
||||
t.v = st->v ^ rk->v;
|
||||
aesdec_ISB_ISR_AK(ad, &t, &aes_zero, false);
|
||||
}
|
||||
}
|
||||
|
||||
rd[0] = st.l[0];
|
||||
rd[1] = st.l[1];
|
||||
clear_tail(vd, opr_sz, simd_maxsz(desc));
|
||||
}
|
||||
|
||||
void HELPER(crypto_aesmc)(void *vd, void *vm, uint32_t desc)
|
||||
{
|
||||
intptr_t i, opr_sz = simd_oprsz(desc);
|
||||
bool decrypt = simd_data(desc);
|
||||
|
||||
for (i = 0; i < opr_sz; i += 16) {
|
||||
do_crypto_aesmc(vd + i, vm + i, decrypt);
|
||||
AESState *ad = (AESState *)(vd + i);
|
||||
AESState *st = (AESState *)(vm + i);
|
||||
AESState t;
|
||||
|
||||
/* Our uint64_t are in the wrong order for big-endian. */
|
||||
if (HOST_BIG_ENDIAN) {
|
||||
t.d[0] = st->d[1];
|
||||
t.d[1] = st->d[0];
|
||||
aesenc_MC(&t, &t, false);
|
||||
ad->d[0] = t.d[1];
|
||||
ad->d[1] = t.d[0];
|
||||
} else {
|
||||
aesenc_MC(ad, st, false);
|
||||
}
|
||||
}
|
||||
clear_tail(vd, opr_sz, simd_maxsz(desc));
|
||||
}
|
||||
|
||||
void HELPER(crypto_aesimc)(void *vd, void *vm, uint32_t desc)
|
||||
{
|
||||
intptr_t i, opr_sz = simd_oprsz(desc);
|
||||
|
||||
for (i = 0; i < opr_sz; i += 16) {
|
||||
AESState *ad = (AESState *)(vd + i);
|
||||
AESState *st = (AESState *)(vm + i);
|
||||
AESState t;
|
||||
|
||||
/* Our uint64_t are in the wrong order for big-endian. */
|
||||
if (HOST_BIG_ENDIAN) {
|
||||
t.d[0] = st->d[1];
|
||||
t.d[1] = st->d[0];
|
||||
aesdec_IMC(&t, &t, false);
|
||||
ad->d[0] = t.d[1];
|
||||
ad->d[1] = t.d[0];
|
||||
} else {
|
||||
aesdec_IMC(ad, st, false);
|
||||
}
|
||||
}
|
||||
clear_tail(vd, opr_sz, simd_maxsz(desc));
|
||||
}
|
||||
|
||||
@@ -1629,8 +1629,8 @@ STNT1_zprz 1110010 .. 10 ..... 001 ... ..... ..... \
|
||||
### SVE2 Crypto Extensions
|
||||
|
||||
# SVE2 crypto unary operations
|
||||
# AESMC and AESIMC
|
||||
AESMC 01000101 00 10000011100 decrypt:1 00000 rd:5
|
||||
AESMC 01000101 00 10000011100 0 00000 rd:5
|
||||
AESIMC 01000101 00 10000011100 1 00000 rd:5
|
||||
|
||||
# SVE2 crypto destructive binary operations
|
||||
AESE 01000101 00 10001 0 11100 0 ..... ..... @rdn_rm_e0
|
||||
|
||||
@@ -13210,7 +13210,6 @@ static void disas_crypto_aes(DisasContext *s, uint32_t insn)
|
||||
int opcode = extract32(insn, 12, 5);
|
||||
int rn = extract32(insn, 5, 5);
|
||||
int rd = extract32(insn, 0, 5);
|
||||
int decrypt;
|
||||
gen_helper_gvec_2 *genfn2 = NULL;
|
||||
gen_helper_gvec_3 *genfn3 = NULL;
|
||||
|
||||
@@ -13221,20 +13220,16 @@ static void disas_crypto_aes(DisasContext *s, uint32_t insn)
|
||||
|
||||
switch (opcode) {
|
||||
case 0x4: /* AESE */
|
||||
decrypt = 0;
|
||||
genfn3 = gen_helper_crypto_aese;
|
||||
break;
|
||||
case 0x6: /* AESMC */
|
||||
decrypt = 0;
|
||||
genfn2 = gen_helper_crypto_aesmc;
|
||||
break;
|
||||
case 0x5: /* AESD */
|
||||
decrypt = 1;
|
||||
genfn3 = gen_helper_crypto_aese;
|
||||
genfn3 = gen_helper_crypto_aesd;
|
||||
break;
|
||||
case 0x7: /* AESIMC */
|
||||
decrypt = 1;
|
||||
genfn2 = gen_helper_crypto_aesmc;
|
||||
genfn2 = gen_helper_crypto_aesimc;
|
||||
break;
|
||||
default:
|
||||
unallocated_encoding(s);
|
||||
@@ -13245,9 +13240,9 @@ static void disas_crypto_aes(DisasContext *s, uint32_t insn)
|
||||
return;
|
||||
}
|
||||
if (genfn2) {
|
||||
gen_gvec_op2_ool(s, true, rd, rn, decrypt, genfn2);
|
||||
gen_gvec_op2_ool(s, true, rd, rn, 0, genfn2);
|
||||
} else {
|
||||
gen_gvec_op3_ool(s, true, rd, rd, rn, decrypt, genfn3);
|
||||
gen_gvec_op3_ool(s, true, rd, rd, rn, 0, genfn3);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3451,9 +3451,9 @@ static bool trans_VMVN(DisasContext *s, arg_2misc *a)
|
||||
}
|
||||
|
||||
WRAP_2M_3_OOL_FN(gen_AESE, gen_helper_crypto_aese, 0)
|
||||
WRAP_2M_3_OOL_FN(gen_AESD, gen_helper_crypto_aese, 1)
|
||||
WRAP_2M_3_OOL_FN(gen_AESD, gen_helper_crypto_aesd, 0)
|
||||
WRAP_2M_2_OOL_FN(gen_AESMC, gen_helper_crypto_aesmc, 0)
|
||||
WRAP_2M_2_OOL_FN(gen_AESIMC, gen_helper_crypto_aesmc, 1)
|
||||
WRAP_2M_2_OOL_FN(gen_AESIMC, gen_helper_crypto_aesimc, 0)
|
||||
WRAP_2M_2_OOL_FN(gen_SHA1H, gen_helper_crypto_sha1h, 0)
|
||||
WRAP_2M_2_OOL_FN(gen_SHA1SU1, gen_helper_crypto_sha1su1, 0)
|
||||
WRAP_2M_2_OOL_FN(gen_SHA256SU0, gen_helper_crypto_sha256su0, 0)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user