dasharo/linux
0
0
Fork 0
mirror of https://github.com/Dasharo/linux.git synced 2026-03-06 15:25:10 -08:00
Code Issues Packages Projects Releases Wiki Activity

crypto: serpent - add AVX2/x86_64 assembler implementation of serpent cipher

Browse Source 
Patch adds AVX2/x86-64 implementation of Serpent cipher, requiring 16 parallel
blocks for input (256 bytes). Implementation is based on the AVX implementation
and extends to use the 256-bit wide YMM registers. Since serpent does not use
table look-ups, this implementation should be close to two times faster than
the AVX implementation.

Signed-off-by: Jussi Kivilinna <jussi.kivilinna@iki.fi>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
This commit is contained in:
Jussi Kivilinna
2013-04-13 13:46:55 +03:00
committed by Herbert Xu
parent cf1521a1a5
commit 56d76c96a9
7 changed files with 1468 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
arch/x86/crypto/Makefile
View File

@@ -43,6 +43,7 @@ endif
# These modules require assembler to support AVX2.
ifeq ($(avx2_supported),yes)
obj-$(CONFIG_CRYPTO_BLOWFISH_AVX2_X86_64) += blowfish-avx2.o
obj-$(CONFIG_CRYPTO_SERPENT_AVX2_X86_64) += serpent-avx2.o
obj-$(CONFIG_CRYPTO_TWOFISH_AVX2_X86_64) += twofish-avx2.o
endif
@@ -72,6 +73,7 @@ endif
ifeq ($(avx2_supported),yes)
blowfish-avx2-y := blowfish-avx2-asm_64.o blowfish_avx2_glue.o
serpent-avx2-y := serpent-avx2-asm_64.o serpent_avx2_glue.o
twofish-avx2-y := twofish-avx2-asm_64.o twofish_avx2_glue.o
endif

800
arch/x86/crypto/serpent-avx2-asm_64.S Normal file
View File

File diff suppressed because it is too large Load Diff

562
arch/x86/crypto/serpent_avx2_glue.c Normal file
View File

File diff suppressed because it is too large Load Diff

62
arch/x86/crypto/serpent_avx_glue.c
View File

@@ -41,7 +41,32 @@
#include <asm/crypto/ablk_helper.h>
#include <asm/crypto/glue_helper.h>
static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
/* 8-way parallel cipher functions */
asmlinkage void serpent_ecb_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst,
const u8 *src);
EXPORT_SYMBOL_GPL(serpent_ecb_enc_8way_avx);
asmlinkage void serpent_ecb_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
const u8 *src);
EXPORT_SYMBOL_GPL(serpent_ecb_dec_8way_avx);
asmlinkage void serpent_cbc_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
const u8 *src);
EXPORT_SYMBOL_GPL(serpent_cbc_dec_8way_avx);
asmlinkage void serpent_ctr_8way_avx(struct serpent_ctx *ctx, u8 *dst,
const u8 *src, le128 *iv);
EXPORT_SYMBOL_GPL(serpent_ctr_8way_avx);
asmlinkage void serpent_xts_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst,
const u8 *src, le128 *iv);
EXPORT_SYMBOL_GPL(serpent_xts_enc_8way_avx);
asmlinkage void serpent_xts_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
const u8 *src, le128 *iv);
EXPORT_SYMBOL_GPL(serpent_xts_dec_8way_avx);
void __serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
be128 ctrblk;
@@ -51,18 +76,22 @@ static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, le128 *iv)
__serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
u128_xor(dst, src, (u128 *)&ctrblk);
}
EXPORT_SYMBOL_GPL(__serpent_crypt_ctr);
static void serpent_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
void serpent_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
glue_xts_crypt_128bit_one(ctx, dst, src, iv,
GLUE_FUNC_CAST(__serpent_encrypt));
}
EXPORT_SYMBOL_GPL(serpent_xts_enc);
static void serpent_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
void serpent_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv)
{
glue_xts_crypt_128bit_one(ctx, dst, src, iv,
GLUE_FUNC_CAST(__serpent_decrypt));
}
EXPORT_SYMBOL_GPL(serpent_xts_dec);
static const struct common_glue_ctx serpent_enc = {
.num_funcs = 2,
@@ -86,7 +115,7 @@ static const struct common_glue_ctx serpent_ctr = {
.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_ctr_8way_avx) }
}, {
.num_blocks = 1,
.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) }
.fn_u = { .ctr = GLUE_CTR_FUNC_CAST(__serpent_crypt_ctr) }
} }
};
@@ -224,13 +253,8 @@ static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
__serpent_decrypt(ctx->ctx, srcdst, srcdst);
}
struct serpent_lrw_ctx {
struct lrw_table_ctx lrw_table;
struct serpent_ctx serpent_ctx;
};
static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
unsigned int keylen)
int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
unsigned int keylen)
{
struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
int err;
@@ -243,6 +267,7 @@ static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
return lrw_init_table(&ctx->lrw_table, key + keylen -
SERPENT_BLOCK_SIZE);
}
EXPORT_SYMBOL_GPL(lrw_serpent_setkey);
static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
@@ -296,20 +321,16 @@ static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
return ret;
}
static void lrw_exit_tfm(struct crypto_tfm *tfm)
void lrw_serpent_exit_tfm(struct crypto_tfm *tfm)
{
struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
lrw_free_table(&ctx->lrw_table);
}
EXPORT_SYMBOL_GPL(lrw_serpent_exit_tfm);
struct serpent_xts_ctx {
struct serpent_ctx tweak_ctx;
struct serpent_ctx crypt_ctx;
};
static int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
unsigned int keylen)
int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
unsigned int keylen)
{
struct serpent_xts_ctx *ctx = crypto_tfm_ctx(tfm);
u32 *flags = &tfm->crt_flags;
@@ -331,6 +352,7 @@ static int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
/* second half of xts-key is for tweak */
return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
}
EXPORT_SYMBOL_GPL(xts_serpent_setkey);
static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
@@ -420,7 +442,7 @@ static struct crypto_alg serpent_algs[10] = { {
.cra_alignmask = 0,
.cra_type = &crypto_blkcipher_type,
.cra_module = THIS_MODULE,
.cra_exit = lrw_exit_tfm,
.cra_exit = lrw_serpent_exit_tfm,
.cra_u = {
.blkcipher = {
.min_keysize = SERPENT_MIN_KEY_SIZE +

24
arch/x86/include/asm/crypto/serpent-avx.h
View File

@@ -6,6 +6,16 @@
#define SERPENT_PARALLEL_BLOCKS 8
struct serpent_lrw_ctx {
struct lrw_table_ctx lrw_table;
struct serpent_ctx serpent_ctx;
};
struct serpent_xts_ctx {
struct serpent_ctx tweak_ctx;
struct serpent_ctx crypt_ctx;
};
asmlinkage void serpent_ecb_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst,
const u8 *src);
asmlinkage void serpent_ecb_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
@@ -21,4 +31,18 @@ asmlinkage void serpent_xts_enc_8way_avx(struct serpent_ctx *ctx, u8 *dst,
asmlinkage void serpent_xts_dec_8way_avx(struct serpent_ctx *ctx, u8 *dst,
const u8 *src, le128 *iv);
extern void __serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src,
le128 *iv);
extern void serpent_xts_enc(void *ctx, u128 *dst, const u128 *src, le128 *iv);
extern void serpent_xts_dec(void *ctx, u128 *dst, const u128 *src, le128 *iv);
extern int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
unsigned int keylen);
extern void lrw_serpent_exit_tfm(struct crypto_tfm *tfm);
extern int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
unsigned int keylen);
#endif

23
crypto/Kconfig
View File

@@ -1131,6 +1131,29 @@ config CRYPTO_SERPENT_AVX_X86_64
See also:
<http://www.cl.cam.ac.uk/~rja14/serpent.html>
config CRYPTO_SERPENT_AVX2_X86_64
tristate "Serpent cipher algorithm (x86_64/AVX2)"
depends on X86 && 64BIT
select CRYPTO_ALGAPI
select CRYPTO_CRYPTD
select CRYPTO_ABLK_HELPER_X86
select CRYPTO_GLUE_HELPER_X86
select CRYPTO_SERPENT
select CRYPTO_SERPENT_AVX_X86_64
select CRYPTO_LRW
select CRYPTO_XTS
help
Serpent cipher algorithm, by Anderson, Biham & Knudsen.
Keys are allowed to be from 0 to 256 bits in length, in steps
of 8 bits.
This module provides Serpent cipher algorithm that processes 16
blocks parallel using AVX2 instruction set.
See also:
<http://www.cl.cam.ac.uk/~rja14/serpent.html>
config CRYPTO_TEA
tristate "TEA, XTEA and XETA cipher algorithms"
select CRYPTO_ALGAPI

15
crypto/testmgr.c
View File

@@ -1644,6 +1644,9 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "__cbc-serpent-avx",
.test = alg_test_null,
}, {
.alg = "__cbc-serpent-avx2",
.test = alg_test_null,
}, {
.alg = "__cbc-serpent-sse2",
.test = alg_test_null,
@@ -1672,6 +1675,9 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "__driver-cbc-serpent-avx",
.test = alg_test_null,
}, {
.alg = "__driver-cbc-serpent-avx2",
.test = alg_test_null,
}, {
.alg = "__driver-cbc-serpent-sse2",
.test = alg_test_null,
@@ -1700,6 +1706,9 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "__driver-ecb-serpent-avx",
.test = alg_test_null,
}, {
.alg = "__driver-ecb-serpent-avx2",
.test = alg_test_null,
}, {
.alg = "__driver-ecb-serpent-sse2",
.test = alg_test_null,
@@ -1968,6 +1977,9 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "cryptd(__driver-cbc-camellia-aesni)",
.test = alg_test_null,
}, {
.alg = "cryptd(__driver-cbc-serpent-avx2)",
.test = alg_test_null,
}, {
.alg = "cryptd(__driver-ecb-aes-aesni)",
.test = alg_test_null,
@@ -1987,6 +1999,9 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "cryptd(__driver-ecb-serpent-avx)",
.test = alg_test_null,
}, {
.alg = "cryptd(__driver-ecb-serpent-avx2)",
.test = alg_test_null,
}, {
.alg = "cryptd(__driver-ecb-serpent-sse2)",
.test = alg_test_null,