mirror of
https://github.com/Dasharo/linux.git
synced 2026-03-06 15:25:10 -08:00
Merge tag 'v6.3-p1' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
Pull crypto update from Herbert Xu: "API: - Use kmap_local instead of kmap_atomic - Change request callback to take void pointer - Print FIPS status in /proc/crypto (when enabled) Algorithms: - Add rfc4106/gcm support on arm64 - Add ARIA AVX2/512 support on x86 Drivers: - Add TRNG driver for StarFive SoC - Delete ux500/hash driver (subsumed by stm32/hash) - Add zlib support in qat - Add RSA support in aspeed" * tag 'v6.3-p1' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (156 commits) crypto: x86/aria-avx - Do not use avx2 instructions crypto: aspeed - Fix modular aspeed-acry crypto: hisilicon/qm - fix coding style issues crypto: hisilicon/qm - update comments to match function crypto: hisilicon/qm - change function names crypto: hisilicon/qm - use min() instead of min_t() crypto: hisilicon/qm - remove some unused defines crypto: proc - Print fips status crypto: crypto4xx - Call dma_unmap_page when done crypto: octeontx2 - Fix objects shared between several modules crypto: nx - Fix sparse warnings crypto: ecc - Silence sparse warning tls: Pass rec instead of aead_req into tls_encrypt_done crypto: api - Remove completion function scaffolding tls: Remove completion function scaffolding tipc: Remove completion function scaffolding net: ipv6: Remove completion function scaffolding net: ipv4: Remove completion function scaffolding net: macsec: Remove completion function scaffolding dm: Remove completion function scaffolding ...
This commit is contained in:
Linus Torvalds
@@ -1,6 +1,6 @@
|
||||
What: /sys/bus/pci/devices/<BDF>/qat/state
|
||||
Date: June 2022
|
||||
KernelVersion: 5.20
|
||||
KernelVersion: 6.0
|
||||
Contact: qat-linux@intel.com
|
||||
Description: (RW) Reports the current state of the QAT device. Write to
|
||||
the file to start or stop the device.
|
||||
@@ -18,7 +18,7 @@ Description: (RW) Reports the current state of the QAT device. Write to
|
||||
|
||||
What: /sys/bus/pci/devices/<BDF>/qat/cfg_services
|
||||
Date: June 2022
|
||||
KernelVersion: 5.20
|
||||
KernelVersion: 6.0
|
||||
Contact: qat-linux@intel.com
|
||||
Description: (RW) Reports the current configuration of the QAT device.
|
||||
Write to the file to change the configured services.
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
|
||||
%YAML 1.2
|
||||
---
|
||||
$id: http://devicetree.org/schemas/bus/aspeed,ast2600-ahbc.yaml#
|
||||
$schema: http://devicetree.org/meta-schemas/core.yaml#
|
||||
|
||||
title: ASPEED Advanced High-Performance Bus Controller (AHBC)
|
||||
|
||||
maintainers:
|
||||
- Neal Liu <neal_liu@aspeedtech.com>
|
||||
- Chia-Wei Wang <chiawei_wang@aspeedtech.com>
|
||||
|
||||
description: |
|
||||
Advanced High-performance Bus Controller (AHBC) supports plenty of mechanisms
|
||||
including a priority arbiter, an address decoder and a data multiplexer
|
||||
to control the overall operations of Advanced High-performance Bus (AHB).
|
||||
|
||||
properties:
|
||||
compatible:
|
||||
enum:
|
||||
- aspeed,ast2600-ahbc
|
||||
|
||||
reg:
|
||||
maxItems: 1
|
||||
|
||||
required:
|
||||
- compatible
|
||||
- reg
|
||||
|
||||
additionalProperties: false
|
||||
|
||||
examples:
|
||||
- |
|
||||
ahbc@1e600000 {
|
||||
compatible = "aspeed,ast2600-ahbc";
|
||||
reg = <0x1e600000 0x100>;
|
||||
};
|
||||
@@ -14,6 +14,7 @@ properties:
|
||||
enum:
|
||||
- allwinner,sun8i-h3-crypto
|
||||
- allwinner,sun8i-r40-crypto
|
||||
- allwinner,sun20i-d1-crypto
|
||||
- allwinner,sun50i-a64-crypto
|
||||
- allwinner,sun50i-h5-crypto
|
||||
- allwinner,sun50i-h6-crypto
|
||||
@@ -29,6 +30,7 @@ properties:
|
||||
- description: Bus clock
|
||||
- description: Module clock
|
||||
- description: MBus clock
|
||||
- description: TRNG clock (RC oscillator)
|
||||
minItems: 2
|
||||
|
||||
clock-names:
|
||||
@@ -36,6 +38,7 @@ properties:
|
||||
- const: bus
|
||||
- const: mod
|
||||
- const: ram
|
||||
- const: trng
|
||||
minItems: 2
|
||||
|
||||
resets:
|
||||
@@ -44,19 +47,33 @@ properties:
|
||||
if:
|
||||
properties:
|
||||
compatible:
|
||||
const: allwinner,sun50i-h6-crypto
|
||||
enum:
|
||||
- allwinner,sun20i-d1-crypto
|
||||
then:
|
||||
properties:
|
||||
clocks:
|
||||
minItems: 3
|
||||
minItems: 4
|
||||
clock-names:
|
||||
minItems: 3
|
||||
minItems: 4
|
||||
else:
|
||||
properties:
|
||||
clocks:
|
||||
maxItems: 2
|
||||
clock-names:
|
||||
maxItems: 2
|
||||
if:
|
||||
properties:
|
||||
compatible:
|
||||
const: allwinner,sun50i-h6-crypto
|
||||
then:
|
||||
properties:
|
||||
clocks:
|
||||
minItems: 3
|
||||
maxItems: 3
|
||||
clock-names:
|
||||
minItems: 3
|
||||
maxItems: 3
|
||||
else:
|
||||
properties:
|
||||
clocks:
|
||||
maxItems: 2
|
||||
clock-names:
|
||||
maxItems: 2
|
||||
|
||||
required:
|
||||
- compatible
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
|
||||
%YAML 1.2
|
||||
---
|
||||
$id: http://devicetree.org/schemas/crypto/aspeed,ast2600-acry.yaml#
|
||||
$schema: http://devicetree.org/meta-schemas/core.yaml#
|
||||
|
||||
title: ASPEED ACRY ECDSA/RSA Hardware Accelerator Engines
|
||||
|
||||
maintainers:
|
||||
- Neal Liu <neal_liu@aspeedtech.com>
|
||||
|
||||
description:
|
||||
The ACRY ECDSA/RSA engines is designed to accelerate the throughput
|
||||
of ECDSA/RSA signature and verification. Basically, ACRY can be
|
||||
divided into two independent engines - ECC Engine and RSA Engine.
|
||||
|
||||
properties:
|
||||
compatible:
|
||||
enum:
|
||||
- aspeed,ast2600-acry
|
||||
|
||||
reg:
|
||||
items:
|
||||
- description: acry base address & size
|
||||
- description: acry sram base address & size
|
||||
|
||||
clocks:
|
||||
maxItems: 1
|
||||
|
||||
interrupts:
|
||||
maxItems: 1
|
||||
|
||||
required:
|
||||
- compatible
|
||||
- reg
|
||||
- clocks
|
||||
- interrupts
|
||||
|
||||
additionalProperties: false
|
||||
|
||||
examples:
|
||||
- |
|
||||
#include <dt-bindings/clock/ast2600-clock.h>
|
||||
acry: crypto@1e6fa000 {
|
||||
compatible = "aspeed,ast2600-acry";
|
||||
reg = <0x1e6fa000 0x400>, <0x1e710000 0x1800>;
|
||||
interrupts = <160>;
|
||||
clocks = <&syscon ASPEED_CLK_GATE_RSACLK>;
|
||||
};
|
||||
@@ -6,12 +6,18 @@ $schema: http://devicetree.org/meta-schemas/core.yaml#
|
||||
|
||||
title: STMicroelectronics STM32 HASH
|
||||
|
||||
description: The STM32 HASH block is built on the HASH block found in
|
||||
the STn8820 SoC introduced in 2007, and subsequently used in the U8500
|
||||
SoC in 2010.
|
||||
|
||||
maintainers:
|
||||
- Lionel Debieve <lionel.debieve@foss.st.com>
|
||||
|
||||
properties:
|
||||
compatible:
|
||||
enum:
|
||||
- st,stn8820-hash
|
||||
- stericsson,ux500-hash
|
||||
- st,stm32f456-hash
|
||||
- st,stm32f756-hash
|
||||
|
||||
@@ -41,11 +47,26 @@ properties:
|
||||
maximum: 2
|
||||
default: 0
|
||||
|
||||
power-domains:
|
||||
maxItems: 1
|
||||
|
||||
required:
|
||||
- compatible
|
||||
- reg
|
||||
- clocks
|
||||
- interrupts
|
||||
|
||||
allOf:
|
||||
- if:
|
||||
properties:
|
||||
compatible:
|
||||
items:
|
||||
const: stericsson,ux500-hash
|
||||
then:
|
||||
properties:
|
||||
interrupts: false
|
||||
else:
|
||||
required:
|
||||
- interrupts
|
||||
|
||||
additionalProperties: false
|
||||
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
|
||||
%YAML 1.2
|
||||
---
|
||||
$id: http://devicetree.org/schemas/rng/starfive,jh7110-trng.yaml#
|
||||
$schema: http://devicetree.org/meta-schemas/core.yaml#
|
||||
|
||||
title: StarFive SoC TRNG Module
|
||||
|
||||
maintainers:
|
||||
- Jia Jie Ho <jiajie.ho@starfivetech.com>
|
||||
|
||||
properties:
|
||||
compatible:
|
||||
const: starfive,jh7110-trng
|
||||
|
||||
reg:
|
||||
maxItems: 1
|
||||
|
||||
clocks:
|
||||
items:
|
||||
- description: Hardware reference clock
|
||||
- description: AHB reference clock
|
||||
|
||||
clock-names:
|
||||
items:
|
||||
- const: hclk
|
||||
- const: ahb
|
||||
|
||||
resets:
|
||||
maxItems: 1
|
||||
|
||||
interrupts:
|
||||
maxItems: 1
|
||||
|
||||
required:
|
||||
- compatible
|
||||
- reg
|
||||
- clocks
|
||||
- clock-names
|
||||
- resets
|
||||
- interrupts
|
||||
|
||||
additionalProperties: false
|
||||
|
||||
examples:
|
||||
- |
|
||||
rng: rng@1600C000 {
|
||||
compatible = "starfive,jh7110-trng";
|
||||
reg = <0x1600C000 0x4000>;
|
||||
clocks = <&clk 15>, <&clk 16>;
|
||||
clock-names = "hclk", "ahb";
|
||||
resets = <&reset 3>;
|
||||
interrupts = <30>;
|
||||
};
|
||||
...
|
||||
@@ -3149,7 +3149,7 @@ ASPEED CRYPTO DRIVER
|
||||
M: Neal Liu <neal_liu@aspeedtech.com>
|
||||
L: linux-aspeed@lists.ozlabs.org (moderated for non-subscribers)
|
||||
S: Maintained
|
||||
F: Documentation/devicetree/bindings/crypto/aspeed,ast2500-hace.yaml
|
||||
F: Documentation/devicetree/bindings/crypto/aspeed,*
|
||||
F: drivers/crypto/aspeed/
|
||||
|
||||
ASUS NOTEBOOKS AND EEEPC ACPI/WMI EXTRAS DRIVERS
|
||||
@@ -19769,6 +19769,12 @@ F: Documentation/devicetree/bindings/reset/starfive,jh7100-reset.yaml
|
||||
F: drivers/reset/reset-starfive-jh7100.c
|
||||
F: include/dt-bindings/reset/starfive-jh7100.h
|
||||
|
||||
STARFIVE TRNG DRIVER
|
||||
M: Jia Jie Ho <jiajie.ho@starfivetech.com>
|
||||
S: Supported
|
||||
F: Documentation/devicetree/bindings/rng/starfive*
|
||||
F: drivers/char/hw_random/jh7110-trng.c
|
||||
|
||||
STATIC BRANCH/CALL
|
||||
M: Peter Zijlstra <peterz@infradead.org>
|
||||
M: Josh Poimboeuf <jpoimboe@kernel.org>
|
||||
|
||||
@@ -98,6 +98,11 @@
|
||||
<0x40466000 0x2000>;
|
||||
};
|
||||
|
||||
ahbc: bus@1e600000 {
|
||||
compatible = "aspeed,ast2600-ahbc", "syscon";
|
||||
reg = <0x1e600000 0x100>;
|
||||
};
|
||||
|
||||
fmc: spi@1e620000 {
|
||||
reg = <0x1e620000 0xc4>, <0x20000000 0x10000000>;
|
||||
#address-cells = <1>;
|
||||
@@ -431,6 +436,14 @@
|
||||
reg = <0x1e6f2000 0x1000>;
|
||||
};
|
||||
|
||||
acry: crypto@1e6fa000 {
|
||||
compatible = "aspeed,ast2600-acry";
|
||||
reg = <0x1e6fa000 0x400>, <0x1e710000 0x1800>;
|
||||
interrupts = <GIC_SPI 160 IRQ_TYPE_LEVEL_HIGH>;
|
||||
clocks = <&syscon ASPEED_CLK_GATE_RSACLK>;
|
||||
aspeed,ahbc = <&ahbc>;
|
||||
};
|
||||
|
||||
video: video@1e700000 {
|
||||
compatible = "aspeed,ast2600-video-engine";
|
||||
reg = <0x1e700000 0x1000>;
|
||||
|
||||
@@ -21,31 +21,29 @@
|
||||
|
||||
#include "sha1.h"
|
||||
|
||||
asmlinkage void sha1_block_data_order(u32 *digest,
|
||||
const unsigned char *data, unsigned int rounds);
|
||||
asmlinkage void sha1_block_data_order(struct sha1_state *digest,
|
||||
const u8 *data, int rounds);
|
||||
|
||||
int sha1_update_arm(struct shash_desc *desc, const u8 *data,
|
||||
unsigned int len)
|
||||
{
|
||||
/* make sure casting to sha1_block_fn() is safe */
|
||||
/* make sure signature matches sha1_block_fn() */
|
||||
BUILD_BUG_ON(offsetof(struct sha1_state, state) != 0);
|
||||
|
||||
return sha1_base_do_update(desc, data, len,
|
||||
(sha1_block_fn *)sha1_block_data_order);
|
||||
return sha1_base_do_update(desc, data, len, sha1_block_data_order);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(sha1_update_arm);
|
||||
|
||||
static int sha1_final(struct shash_desc *desc, u8 *out)
|
||||
{
|
||||
sha1_base_do_finalize(desc, (sha1_block_fn *)sha1_block_data_order);
|
||||
sha1_base_do_finalize(desc, sha1_block_data_order);
|
||||
return sha1_base_finish(desc, out);
|
||||
}
|
||||
|
||||
int sha1_finup_arm(struct shash_desc *desc, const u8 *data,
|
||||
unsigned int len, u8 *out)
|
||||
{
|
||||
sha1_base_do_update(desc, data, len,
|
||||
(sha1_block_fn *)sha1_block_data_order);
|
||||
sha1_base_do_update(desc, data, len, sha1_block_data_order);
|
||||
return sha1_final(desc, out);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(sha1_finup_arm);
|
||||
|
||||
@@ -161,43 +161,39 @@ static int ccm_encrypt(struct aead_request *req)
|
||||
memcpy(buf, req->iv, AES_BLOCK_SIZE);
|
||||
|
||||
err = skcipher_walk_aead_encrypt(&walk, req, false);
|
||||
if (unlikely(err))
|
||||
return err;
|
||||
|
||||
kernel_neon_begin();
|
||||
|
||||
if (req->assoclen)
|
||||
ccm_calculate_auth_mac(req, mac);
|
||||
|
||||
do {
|
||||
while (walk.nbytes) {
|
||||
u32 tail = walk.nbytes % AES_BLOCK_SIZE;
|
||||
bool final = walk.nbytes == walk.total;
|
||||
|
||||
if (walk.nbytes == walk.total)
|
||||
if (final)
|
||||
tail = 0;
|
||||
|
||||
ce_aes_ccm_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
|
||||
walk.nbytes - tail, ctx->key_enc,
|
||||
num_rounds(ctx), mac, walk.iv);
|
||||
|
||||
if (walk.nbytes == walk.total)
|
||||
ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx));
|
||||
if (!final)
|
||||
kernel_neon_end();
|
||||
err = skcipher_walk_done(&walk, tail);
|
||||
if (!final)
|
||||
kernel_neon_begin();
|
||||
}
|
||||
|
||||
kernel_neon_end();
|
||||
ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx));
|
||||
|
||||
if (walk.nbytes) {
|
||||
err = skcipher_walk_done(&walk, tail);
|
||||
if (unlikely(err))
|
||||
return err;
|
||||
if (unlikely(walk.nbytes))
|
||||
kernel_neon_begin();
|
||||
}
|
||||
} while (walk.nbytes);
|
||||
kernel_neon_end();
|
||||
|
||||
/* copy authtag to end of dst */
|
||||
scatterwalk_map_and_copy(mac, req->dst, req->assoclen + req->cryptlen,
|
||||
crypto_aead_authsize(aead), 1);
|
||||
|
||||
return 0;
|
||||
return err;
|
||||
}
|
||||
|
||||
static int ccm_decrypt(struct aead_request *req)
|
||||
@@ -219,37 +215,36 @@ static int ccm_decrypt(struct aead_request *req)
|
||||
memcpy(buf, req->iv, AES_BLOCK_SIZE);
|
||||
|
||||
err = skcipher_walk_aead_decrypt(&walk, req, false);
|
||||
if (unlikely(err))
|
||||
return err;
|
||||
|
||||
kernel_neon_begin();
|
||||
|
||||
if (req->assoclen)
|
||||
ccm_calculate_auth_mac(req, mac);
|
||||
|
||||
do {
|
||||
while (walk.nbytes) {
|
||||
u32 tail = walk.nbytes % AES_BLOCK_SIZE;
|
||||
bool final = walk.nbytes == walk.total;
|
||||
|
||||
if (walk.nbytes == walk.total)
|
||||
if (final)
|
||||
tail = 0;
|
||||
|
||||
ce_aes_ccm_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
|
||||
walk.nbytes - tail, ctx->key_enc,
|
||||
num_rounds(ctx), mac, walk.iv);
|
||||
|
||||
if (walk.nbytes == walk.total)
|
||||
ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx));
|
||||
if (!final)
|
||||
kernel_neon_end();
|
||||
err = skcipher_walk_done(&walk, tail);
|
||||
if (!final)
|
||||
kernel_neon_begin();
|
||||
}
|
||||
|
||||
kernel_neon_end();
|
||||
ce_aes_ccm_final(mac, buf, ctx->key_enc, num_rounds(ctx));
|
||||
|
||||
if (walk.nbytes) {
|
||||
err = skcipher_walk_done(&walk, tail);
|
||||
if (unlikely(err))
|
||||
return err;
|
||||
if (unlikely(walk.nbytes))
|
||||
kernel_neon_begin();
|
||||
}
|
||||
} while (walk.nbytes);
|
||||
kernel_neon_end();
|
||||
|
||||
if (unlikely(err))
|
||||
return err;
|
||||
|
||||
/* compare calculated auth tag with the stored one */
|
||||
scatterwalk_map_and_copy(buf, req->src,
|
||||
|
||||
@@ -9,6 +9,7 @@
|
||||
#include <asm/simd.h>
|
||||
#include <asm/unaligned.h>
|
||||
#include <crypto/aes.h>
|
||||
#include <crypto/gcm.h>
|
||||
#include <crypto/algapi.h>
|
||||
#include <crypto/b128ops.h>
|
||||
#include <crypto/gf128mul.h>
|
||||
@@ -28,7 +29,8 @@ MODULE_ALIAS_CRYPTO("ghash");
|
||||
|
||||
#define GHASH_BLOCK_SIZE 16
|
||||
#define GHASH_DIGEST_SIZE 16
|
||||
#define GCM_IV_SIZE 12
|
||||
|
||||
#define RFC4106_NONCE_SIZE 4
|
||||
|
||||
struct ghash_key {
|
||||
be128 k;
|
||||
@@ -43,6 +45,7 @@ struct ghash_desc_ctx {
|
||||
|
||||
struct gcm_aes_ctx {
|
||||
struct crypto_aes_ctx aes_key;
|
||||
u8 nonce[RFC4106_NONCE_SIZE];
|
||||
struct ghash_key ghash_key;
|
||||
};
|
||||
|
||||
@@ -226,8 +229,8 @@ static int num_rounds(struct crypto_aes_ctx *ctx)
|
||||
return 6 + ctx->key_length / 4;
|
||||
}
|
||||
|
||||
static int gcm_setkey(struct crypto_aead *tfm, const u8 *inkey,
|
||||
unsigned int keylen)
|
||||
static int gcm_aes_setkey(struct crypto_aead *tfm, const u8 *inkey,
|
||||
unsigned int keylen)
|
||||
{
|
||||
struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm);
|
||||
u8 key[GHASH_BLOCK_SIZE];
|
||||
@@ -258,17 +261,9 @@ static int gcm_setkey(struct crypto_aead *tfm, const u8 *inkey,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int gcm_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
|
||||
static int gcm_aes_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
|
||||
{
|
||||
switch (authsize) {
|
||||
case 4:
|
||||
case 8:
|
||||
case 12 ... 16:
|
||||
break;
|
||||
default:
|
||||
return -EINVAL;
|
||||
}
|
||||
return 0;
|
||||
return crypto_gcm_check_authsize(authsize);
|
||||
}
|
||||
|
||||
static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[],
|
||||
@@ -302,13 +297,12 @@ static void gcm_update_mac(u64 dg[], const u8 *src, int count, u8 buf[],
|
||||
}
|
||||
}
|
||||
|
||||
static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[])
|
||||
static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[], u32 len)
|
||||
{
|
||||
struct crypto_aead *aead = crypto_aead_reqtfm(req);
|
||||
struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
|
||||
u8 buf[GHASH_BLOCK_SIZE];
|
||||
struct scatter_walk walk;
|
||||
u32 len = req->assoclen;
|
||||
int buf_count = 0;
|
||||
|
||||
scatterwalk_start(&walk, req->src);
|
||||
@@ -338,27 +332,25 @@ static void gcm_calculate_auth_mac(struct aead_request *req, u64 dg[])
|
||||
}
|
||||
}
|
||||
|
||||
static int gcm_encrypt(struct aead_request *req)
|
||||
static int gcm_encrypt(struct aead_request *req, char *iv, int assoclen)
|
||||
{
|
||||
struct crypto_aead *aead = crypto_aead_reqtfm(req);
|
||||
struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
|
||||
int nrounds = num_rounds(&ctx->aes_key);
|
||||
struct skcipher_walk walk;
|
||||
u8 buf[AES_BLOCK_SIZE];
|
||||
u8 iv[AES_BLOCK_SIZE];
|
||||
u64 dg[2] = {};
|
||||
be128 lengths;
|
||||
u8 *tag;
|
||||
int err;
|
||||
|
||||
lengths.a = cpu_to_be64(req->assoclen * 8);
|
||||
lengths.a = cpu_to_be64(assoclen * 8);
|
||||
lengths.b = cpu_to_be64(req->cryptlen * 8);
|
||||
|
||||
if (req->assoclen)
|
||||
gcm_calculate_auth_mac(req, dg);
|
||||
if (assoclen)
|
||||
gcm_calculate_auth_mac(req, dg, assoclen);
|
||||
|
||||
memcpy(iv, req->iv, GCM_IV_SIZE);
|
||||
put_unaligned_be32(2, iv + GCM_IV_SIZE);
|
||||
put_unaligned_be32(2, iv + GCM_AES_IV_SIZE);
|
||||
|
||||
err = skcipher_walk_aead_encrypt(&walk, req, false);
|
||||
|
||||
@@ -403,7 +395,7 @@ static int gcm_encrypt(struct aead_request *req)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int gcm_decrypt(struct aead_request *req)
|
||||
static int gcm_decrypt(struct aead_request *req, char *iv, int assoclen)
|
||||
{
|
||||
struct crypto_aead *aead = crypto_aead_reqtfm(req);
|
||||
struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
|
||||
@@ -412,21 +404,19 @@ static int gcm_decrypt(struct aead_request *req)
|
||||
struct skcipher_walk walk;
|
||||
u8 otag[AES_BLOCK_SIZE];
|
||||
u8 buf[AES_BLOCK_SIZE];
|
||||
u8 iv[AES_BLOCK_SIZE];
|
||||
u64 dg[2] = {};
|
||||
be128 lengths;
|
||||
u8 *tag;
|
||||
int ret;
|
||||
int err;
|
||||
|
||||
lengths.a = cpu_to_be64(req->assoclen * 8);
|
||||
lengths.a = cpu_to_be64(assoclen * 8);
|
||||
lengths.b = cpu_to_be64((req->cryptlen - authsize) * 8);
|
||||
|
||||
if (req->assoclen)
|
||||
gcm_calculate_auth_mac(req, dg);
|
||||
if (assoclen)
|
||||
gcm_calculate_auth_mac(req, dg, assoclen);
|
||||
|
||||
memcpy(iv, req->iv, GCM_IV_SIZE);
|
||||
put_unaligned_be32(2, iv + GCM_IV_SIZE);
|
||||
put_unaligned_be32(2, iv + GCM_AES_IV_SIZE);
|
||||
|
||||
scatterwalk_map_and_copy(otag, req->src,
|
||||
req->assoclen + req->cryptlen - authsize,
|
||||
@@ -471,14 +461,76 @@ static int gcm_decrypt(struct aead_request *req)
|
||||
return ret ? -EBADMSG : 0;
|
||||
}
|
||||
|
||||
static struct aead_alg gcm_aes_alg = {
|
||||
.ivsize = GCM_IV_SIZE,
|
||||
static int gcm_aes_encrypt(struct aead_request *req)
|
||||
{
|
||||
u8 iv[AES_BLOCK_SIZE];
|
||||
|
||||
memcpy(iv, req->iv, GCM_AES_IV_SIZE);
|
||||
return gcm_encrypt(req, iv, req->assoclen);
|
||||
}
|
||||
|
||||
static int gcm_aes_decrypt(struct aead_request *req)
|
||||
{
|
||||
u8 iv[AES_BLOCK_SIZE];
|
||||
|
||||
memcpy(iv, req->iv, GCM_AES_IV_SIZE);
|
||||
return gcm_decrypt(req, iv, req->assoclen);
|
||||
}
|
||||
|
||||
static int rfc4106_setkey(struct crypto_aead *tfm, const u8 *inkey,
|
||||
unsigned int keylen)
|
||||
{
|
||||
struct gcm_aes_ctx *ctx = crypto_aead_ctx(tfm);
|
||||
int err;
|
||||
|
||||
keylen -= RFC4106_NONCE_SIZE;
|
||||
err = gcm_aes_setkey(tfm, inkey, keylen);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
memcpy(ctx->nonce, inkey + keylen, RFC4106_NONCE_SIZE);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
|
||||
{
|
||||
return crypto_rfc4106_check_authsize(authsize);
|
||||
}
|
||||
|
||||
static int rfc4106_encrypt(struct aead_request *req)
|
||||
{
|
||||
struct crypto_aead *aead = crypto_aead_reqtfm(req);
|
||||
struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
|
||||
u8 iv[AES_BLOCK_SIZE];
|
||||
|
||||
memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE);
|
||||
memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE);
|
||||
|
||||
return crypto_ipsec_check_assoclen(req->assoclen) ?:
|
||||
gcm_encrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE);
|
||||
}
|
||||
|
||||
static int rfc4106_decrypt(struct aead_request *req)
|
||||
{
|
||||
struct crypto_aead *aead = crypto_aead_reqtfm(req);
|
||||
struct gcm_aes_ctx *ctx = crypto_aead_ctx(aead);
|
||||
u8 iv[AES_BLOCK_SIZE];
|
||||
|
||||
memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE);
|
||||
memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE);
|
||||
|
||||
return crypto_ipsec_check_assoclen(req->assoclen) ?:
|
||||
gcm_decrypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE);
|
||||
}
|
||||
|
||||
static struct aead_alg gcm_aes_algs[] = {{
|
||||
.ivsize = GCM_AES_IV_SIZE,
|
||||
.chunksize = AES_BLOCK_SIZE,
|
||||
.maxauthsize = AES_BLOCK_SIZE,
|
||||
.setkey = gcm_setkey,
|
||||
.setauthsize = gcm_setauthsize,
|
||||
.encrypt = gcm_encrypt,
|
||||
.decrypt = gcm_decrypt,
|
||||
.setkey = gcm_aes_setkey,
|
||||
.setauthsize = gcm_aes_setauthsize,
|
||||
.encrypt = gcm_aes_encrypt,
|
||||
.decrypt = gcm_aes_decrypt,
|
||||
|
||||
.base.cra_name = "gcm(aes)",
|
||||
.base.cra_driver_name = "gcm-aes-ce",
|
||||
@@ -487,7 +539,23 @@ static struct aead_alg gcm_aes_alg = {
|
||||
.base.cra_ctxsize = sizeof(struct gcm_aes_ctx) +
|
||||
4 * sizeof(u64[2]),
|
||||
.base.cra_module = THIS_MODULE,
|
||||
};
|
||||
}, {
|
||||
.ivsize = GCM_RFC4106_IV_SIZE,
|
||||
.chunksize = AES_BLOCK_SIZE,
|
||||
.maxauthsize = AES_BLOCK_SIZE,
|
||||
.setkey = rfc4106_setkey,
|
||||
.setauthsize = rfc4106_setauthsize,
|
||||
.encrypt = rfc4106_encrypt,
|
||||
.decrypt = rfc4106_decrypt,
|
||||
|
||||
.base.cra_name = "rfc4106(gcm(aes))",
|
||||
.base.cra_driver_name = "rfc4106-gcm-aes-ce",
|
||||
.base.cra_priority = 300,
|
||||
.base.cra_blocksize = 1,
|
||||
.base.cra_ctxsize = sizeof(struct gcm_aes_ctx) +
|
||||
4 * sizeof(u64[2]),
|
||||
.base.cra_module = THIS_MODULE,
|
||||
}};
|
||||
|
||||
static int __init ghash_ce_mod_init(void)
|
||||
{
|
||||
@@ -495,7 +563,8 @@ static int __init ghash_ce_mod_init(void)
|
||||
return -ENODEV;
|
||||
|
||||
if (cpu_have_named_feature(PMULL))
|
||||
return crypto_register_aead(&gcm_aes_alg);
|
||||
return crypto_register_aeads(gcm_aes_algs,
|
||||
ARRAY_SIZE(gcm_aes_algs));
|
||||
|
||||
return crypto_register_shash(&ghash_alg);
|
||||
}
|
||||
@@ -503,7 +572,7 @@ static int __init ghash_ce_mod_init(void)
|
||||
static void __exit ghash_ce_mod_exit(void)
|
||||
{
|
||||
if (cpu_have_named_feature(PMULL))
|
||||
crypto_unregister_aead(&gcm_aes_alg);
|
||||
crypto_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs));
|
||||
else
|
||||
crypto_unregister_shash(&ghash_alg);
|
||||
}
|
||||
|
||||
@@ -166,7 +166,7 @@ static int ccm_crypt(struct aead_request *req, struct skcipher_walk *walk,
|
||||
unsigned int nbytes, u8 *mac))
|
||||
{
|
||||
u8 __aligned(8) ctr0[SM4_BLOCK_SIZE];
|
||||
int err;
|
||||
int err = 0;
|
||||
|
||||
/* preserve the initial ctr0 for the TAG */
|
||||
memcpy(ctr0, walk->iv, SM4_BLOCK_SIZE);
|
||||
@@ -177,33 +177,37 @@ static int ccm_crypt(struct aead_request *req, struct skcipher_walk *walk,
|
||||
if (req->assoclen)
|
||||
ccm_calculate_auth_mac(req, mac);
|
||||
|
||||
do {
|
||||
while (walk->nbytes && walk->nbytes != walk->total) {
|
||||
unsigned int tail = walk->nbytes % SM4_BLOCK_SIZE;
|
||||
const u8 *src = walk->src.virt.addr;
|
||||
u8 *dst = walk->dst.virt.addr;
|
||||
|
||||
if (walk->nbytes == walk->total)
|
||||
tail = 0;
|
||||
|
||||
if (walk->nbytes - tail)
|
||||
sm4_ce_ccm_crypt(rkey_enc, dst, src, walk->iv,
|
||||
walk->nbytes - tail, mac);
|
||||
|
||||
if (walk->nbytes == walk->total)
|
||||
sm4_ce_ccm_final(rkey_enc, ctr0, mac);
|
||||
sm4_ce_ccm_crypt(rkey_enc, walk->dst.virt.addr,
|
||||
walk->src.virt.addr, walk->iv,
|
||||
walk->nbytes - tail, mac);
|
||||
|
||||
kernel_neon_end();
|
||||
|
||||
if (walk->nbytes) {
|
||||
err = skcipher_walk_done(walk, tail);
|
||||
if (err)
|
||||
return err;
|
||||
if (walk->nbytes)
|
||||
kernel_neon_begin();
|
||||
}
|
||||
} while (walk->nbytes > 0);
|
||||
err = skcipher_walk_done(walk, tail);
|
||||
|
||||
return 0;
|
||||
kernel_neon_begin();
|
||||
}
|
||||
|
||||
if (walk->nbytes) {
|
||||
sm4_ce_ccm_crypt(rkey_enc, walk->dst.virt.addr,
|
||||
walk->src.virt.addr, walk->iv,
|
||||
walk->nbytes, mac);
|
||||
|
||||
sm4_ce_ccm_final(rkey_enc, ctr0, mac);
|
||||
|
||||
kernel_neon_end();
|
||||
|
||||
err = skcipher_walk_done(walk, 0);
|
||||
} else {
|
||||
sm4_ce_ccm_final(rkey_enc, ctr0, mac);
|
||||
|
||||
kernel_neon_end();
|
||||
}
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
static int ccm_encrypt(struct aead_request *req)
|
||||
|
||||
@@ -135,22 +135,23 @@ static void gcm_calculate_auth_mac(struct aead_request *req, u8 ghash[])
|
||||
}
|
||||
|
||||
static int gcm_crypt(struct aead_request *req, struct skcipher_walk *walk,
|
||||
struct sm4_gcm_ctx *ctx, u8 ghash[],
|
||||
u8 ghash[], int err,
|
||||
void (*sm4_ce_pmull_gcm_crypt)(const u32 *rkey_enc,
|
||||
u8 *dst, const u8 *src, u8 *iv,
|
||||
unsigned int nbytes, u8 *ghash,
|
||||
const u8 *ghash_table, const u8 *lengths))
|
||||
{
|
||||
struct crypto_aead *aead = crypto_aead_reqtfm(req);
|
||||
struct sm4_gcm_ctx *ctx = crypto_aead_ctx(aead);
|
||||
u8 __aligned(8) iv[SM4_BLOCK_SIZE];
|
||||
be128 __aligned(8) lengths;
|
||||
int err;
|
||||
|
||||
memset(ghash, 0, SM4_BLOCK_SIZE);
|
||||
|
||||
lengths.a = cpu_to_be64(req->assoclen * 8);
|
||||
lengths.b = cpu_to_be64(walk->total * 8);
|
||||
|
||||
memcpy(iv, walk->iv, GCM_IV_SIZE);
|
||||
memcpy(iv, req->iv, GCM_IV_SIZE);
|
||||
put_unaligned_be32(2, iv + GCM_IV_SIZE);
|
||||
|
||||
kernel_neon_begin();
|
||||
@@ -158,49 +159,51 @@ static int gcm_crypt(struct aead_request *req, struct skcipher_walk *walk,
|
||||
if (req->assoclen)
|
||||
gcm_calculate_auth_mac(req, ghash);
|
||||
|
||||
do {
|
||||
while (walk->nbytes) {
|
||||
unsigned int tail = walk->nbytes % SM4_BLOCK_SIZE;
|
||||
const u8 *src = walk->src.virt.addr;
|
||||
u8 *dst = walk->dst.virt.addr;
|
||||
|
||||
if (walk->nbytes == walk->total) {
|
||||
tail = 0;
|
||||
|
||||
sm4_ce_pmull_gcm_crypt(ctx->key.rkey_enc, dst, src, iv,
|
||||
walk->nbytes, ghash,
|
||||
ctx->ghash_table,
|
||||
(const u8 *)&lengths);
|
||||
} else if (walk->nbytes - tail) {
|
||||
sm4_ce_pmull_gcm_crypt(ctx->key.rkey_enc, dst, src, iv,
|
||||
walk->nbytes - tail, ghash,
|
||||
ctx->ghash_table, NULL);
|
||||
|
||||
kernel_neon_end();
|
||||
|
||||
return skcipher_walk_done(walk, 0);
|
||||
}
|
||||
|
||||
sm4_ce_pmull_gcm_crypt(ctx->key.rkey_enc, dst, src, iv,
|
||||
walk->nbytes - tail, ghash,
|
||||
ctx->ghash_table, NULL);
|
||||
|
||||
kernel_neon_end();
|
||||
|
||||
err = skcipher_walk_done(walk, tail);
|
||||
if (err)
|
||||
return err;
|
||||
if (walk->nbytes)
|
||||
kernel_neon_begin();
|
||||
} while (walk->nbytes > 0);
|
||||
|
||||
return 0;
|
||||
kernel_neon_begin();
|
||||
}
|
||||
|
||||
sm4_ce_pmull_gcm_crypt(ctx->key.rkey_enc, NULL, NULL, iv,
|
||||
walk->nbytes, ghash, ctx->ghash_table,
|
||||
(const u8 *)&lengths);
|
||||
|
||||
kernel_neon_end();
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
static int gcm_encrypt(struct aead_request *req)
|
||||
{
|
||||
struct crypto_aead *aead = crypto_aead_reqtfm(req);
|
||||
struct sm4_gcm_ctx *ctx = crypto_aead_ctx(aead);
|
||||
u8 __aligned(8) ghash[SM4_BLOCK_SIZE];
|
||||
struct skcipher_walk walk;
|
||||
int err;
|
||||
|
||||
err = skcipher_walk_aead_encrypt(&walk, req, false);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
err = gcm_crypt(req, &walk, ctx, ghash, sm4_ce_pmull_gcm_enc);
|
||||
err = gcm_crypt(req, &walk, ghash, err, sm4_ce_pmull_gcm_enc);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
@@ -215,17 +218,13 @@ static int gcm_decrypt(struct aead_request *req)
|
||||
{
|
||||
struct crypto_aead *aead = crypto_aead_reqtfm(req);
|
||||
unsigned int authsize = crypto_aead_authsize(aead);
|
||||
struct sm4_gcm_ctx *ctx = crypto_aead_ctx(aead);
|
||||
u8 __aligned(8) ghash[SM4_BLOCK_SIZE];
|
||||
u8 authtag[SM4_BLOCK_SIZE];
|
||||
struct skcipher_walk walk;
|
||||
int err;
|
||||
|
||||
err = skcipher_walk_aead_decrypt(&walk, req, false);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
err = gcm_crypt(req, &walk, ctx, ghash, sm4_ce_pmull_gcm_dec);
|
||||
err = gcm_crypt(req, &walk, ghash, err, sm4_ce_pmull_gcm_dec);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
|
||||
@@ -398,10 +398,6 @@ static int xts_aes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
/* In fips mode only 128 bit or 256 bit keys are valid */
|
||||
if (fips_enabled && key_len != 32 && key_len != 64)
|
||||
return -EINVAL;
|
||||
|
||||
/* Pick the correct function code based on the key length */
|
||||
fc = (key_len == 32) ? CPACF_KM_XTS_128 :
|
||||
(key_len == 64) ? CPACF_KM_XTS_256 : 0;
|
||||
|
||||
@@ -474,7 +474,7 @@ static int xts_paes_set_key(struct crypto_skcipher *tfm, const u8 *in_key,
|
||||
return rc;
|
||||
|
||||
/*
|
||||
* xts_check_key verifies the key length is not odd and makes
|
||||
* xts_verify_key verifies the key length is not odd and makes
|
||||
* sure that the two keys are not the same. This can be done
|
||||
* on the two protected keys as well
|
||||
*/
|
||||
|
||||
@@ -19,3 +19,8 @@ config AS_TPAUSE
|
||||
def_bool $(as-instr,tpause %ecx)
|
||||
help
|
||||
Supported by binutils >= 2.31.1 and LLVM integrated assembler >= V7
|
||||
|
||||
config AS_GFNI
|
||||
def_bool $(as-instr,vgf2p8mulb %xmm0$(comma)%xmm1$(comma)%xmm2)
|
||||
help
|
||||
Supported by binutils >= 2.30 and LLVM integrated assembler
|
||||
|
||||
@@ -304,6 +304,44 @@ config CRYPTO_ARIA_AESNI_AVX_X86_64
|
||||
|
||||
Processes 16 blocks in parallel.
|
||||
|
||||
config CRYPTO_ARIA_AESNI_AVX2_X86_64
|
||||
tristate "Ciphers: ARIA with modes: ECB, CTR (AES-NI/AVX2/GFNI)"
|
||||
depends on X86 && 64BIT
|
||||
select CRYPTO_SKCIPHER
|
||||
select CRYPTO_SIMD
|
||||
select CRYPTO_ALGAPI
|
||||
select CRYPTO_ARIA
|
||||
select CRYPTO_ARIA_AESNI_AVX_X86_64
|
||||
help
|
||||
Length-preserving cipher: ARIA cipher algorithms
|
||||
(RFC 5794) with ECB and CTR modes
|
||||
|
||||
Architecture: x86_64 using:
|
||||
- AES-NI (AES New Instructions)
|
||||
- AVX2 (Advanced Vector Extensions)
|
||||
- GFNI (Galois Field New Instructions)
|
||||
|
||||
Processes 32 blocks in parallel.
|
||||
|
||||
config CRYPTO_ARIA_GFNI_AVX512_X86_64
|
||||
tristate "Ciphers: ARIA with modes: ECB, CTR (AVX512/GFNI)"
|
||||
depends on X86 && 64BIT && AS_AVX512 && AS_GFNI
|
||||
select CRYPTO_SKCIPHER
|
||||
select CRYPTO_SIMD
|
||||
select CRYPTO_ALGAPI
|
||||
select CRYPTO_ARIA
|
||||
select CRYPTO_ARIA_AESNI_AVX_X86_64
|
||||
select CRYPTO_ARIA_AESNI_AVX2_X86_64
|
||||
help
|
||||
Length-preserving cipher: ARIA cipher algorithms
|
||||
(RFC 5794) with ECB and CTR modes
|
||||
|
||||
Architecture: x86_64 using:
|
||||
- AVX512 (Advanced Vector Extensions)
|
||||
- GFNI (Galois Field New Instructions)
|
||||
|
||||
Processes 64 blocks in parallel.
|
||||
|
||||
config CRYPTO_CHACHA20_X86_64
|
||||
tristate "Ciphers: ChaCha20, XChaCha20, XChaCha12 (SSSE3/AVX2/AVX-512VL)"
|
||||
depends on X86 && 64BIT
|
||||
|
||||
@@ -103,6 +103,12 @@ sm4-aesni-avx2-x86_64-y := sm4-aesni-avx2-asm_64.o sm4_aesni_avx2_glue.o
|
||||
obj-$(CONFIG_CRYPTO_ARIA_AESNI_AVX_X86_64) += aria-aesni-avx-x86_64.o
|
||||
aria-aesni-avx-x86_64-y := aria-aesni-avx-asm_64.o aria_aesni_avx_glue.o
|
||||
|
||||
obj-$(CONFIG_CRYPTO_ARIA_AESNI_AVX2_X86_64) += aria-aesni-avx2-x86_64.o
|
||||
aria-aesni-avx2-x86_64-y := aria-aesni-avx2-asm_64.o aria_aesni_avx2_glue.o
|
||||
|
||||
obj-$(CONFIG_CRYPTO_ARIA_GFNI_AVX512_X86_64) += aria-gfni-avx512-x86_64.o
|
||||
aria-gfni-avx512-x86_64-y := aria-gfni-avx512-asm_64.o aria_gfni_avx512_glue.o
|
||||
|
||||
quiet_cmd_perlasm = PERLASM $@
|
||||
cmd_perlasm = $(PERL) $< > $@
|
||||
$(obj)/%.S: $(src)/%.pl FORCE
|
||||
|
||||
@@ -8,13 +8,9 @@
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <linux/cfi_types.h>
|
||||
#include <asm/asm-offsets.h>
|
||||
#include <asm/frame.h>
|
||||
|
||||
/* struct aria_ctx: */
|
||||
#define enc_key 0
|
||||
#define dec_key 272
|
||||
#define rounds 544
|
||||
|
||||
/* register macros */
|
||||
#define CTX %rdi
|
||||
|
||||
@@ -271,34 +267,44 @@
|
||||
|
||||
#define aria_ark_8way(x0, x1, x2, x3, \
|
||||
x4, x5, x6, x7, \
|
||||
t0, rk, idx, round) \
|
||||
t0, t1, t2, rk, \
|
||||
idx, round) \
|
||||
/* AddRoundKey */ \
|
||||
vpbroadcastb ((round * 16) + idx + 3)(rk), t0; \
|
||||
vpxor t0, x0, x0; \
|
||||
vpbroadcastb ((round * 16) + idx + 2)(rk), t0; \
|
||||
vpxor t0, x1, x1; \
|
||||
vpbroadcastb ((round * 16) + idx + 1)(rk), t0; \
|
||||
vpxor t0, x2, x2; \
|
||||
vpbroadcastb ((round * 16) + idx + 0)(rk), t0; \
|
||||
vpxor t0, x3, x3; \
|
||||
vpbroadcastb ((round * 16) + idx + 7)(rk), t0; \
|
||||
vpxor t0, x4, x4; \
|
||||
vpbroadcastb ((round * 16) + idx + 6)(rk), t0; \
|
||||
vpxor t0, x5, x5; \
|
||||
vpbroadcastb ((round * 16) + idx + 5)(rk), t0; \
|
||||
vpxor t0, x6, x6; \
|
||||
vpbroadcastb ((round * 16) + idx + 4)(rk), t0; \
|
||||
vpxor t0, x7, x7;
|
||||
vbroadcastss ((round * 16) + idx + 0)(rk), t0; \
|
||||
vpsrld $24, t0, t2; \
|
||||
vpshufb t1, t2, t2; \
|
||||
vpxor t2, x0, x0; \
|
||||
vpsrld $16, t0, t2; \
|
||||
vpshufb t1, t2, t2; \
|
||||
vpxor t2, x1, x1; \
|
||||
vpsrld $8, t0, t2; \
|
||||
vpshufb t1, t2, t2; \
|
||||
vpxor t2, x2, x2; \
|
||||
vpshufb t1, t0, t2; \
|
||||
vpxor t2, x3, x3; \
|
||||
vbroadcastss ((round * 16) + idx + 4)(rk), t0; \
|
||||
vpsrld $24, t0, t2; \
|
||||
vpshufb t1, t2, t2; \
|
||||
vpxor t2, x4, x4; \
|
||||
vpsrld $16, t0, t2; \
|
||||
vpshufb t1, t2, t2; \
|
||||
vpxor t2, x5, x5; \
|
||||
vpsrld $8, t0, t2; \
|
||||
vpshufb t1, t2, t2; \
|
||||
vpxor t2, x6, x6; \
|
||||
vpshufb t1, t0, t2; \
|
||||
vpxor t2, x7, x7;
|
||||
|
||||
#ifdef CONFIG_AS_GFNI
|
||||
#define aria_sbox_8way_gfni(x0, x1, x2, x3, \
|
||||
x4, x5, x6, x7, \
|
||||
t0, t1, t2, t3, \
|
||||
t4, t5, t6, t7) \
|
||||
vpbroadcastq .Ltf_s2_bitmatrix, t0; \
|
||||
vpbroadcastq .Ltf_inv_bitmatrix, t1; \
|
||||
vpbroadcastq .Ltf_id_bitmatrix, t2; \
|
||||
vpbroadcastq .Ltf_aff_bitmatrix, t3; \
|
||||
vpbroadcastq .Ltf_x2_bitmatrix, t4; \
|
||||
vmovdqa .Ltf_s2_bitmatrix, t0; \
|
||||
vmovdqa .Ltf_inv_bitmatrix, t1; \
|
||||
vmovdqa .Ltf_id_bitmatrix, t2; \
|
||||
vmovdqa .Ltf_aff_bitmatrix, t3; \
|
||||
vmovdqa .Ltf_x2_bitmatrix, t4; \
|
||||
vgf2p8affineinvqb $(tf_s2_const), t0, x1, x1; \
|
||||
vgf2p8affineinvqb $(tf_s2_const), t0, x5, x5; \
|
||||
vgf2p8affineqb $(tf_inv_const), t1, x2, x2; \
|
||||
@@ -312,14 +318,15 @@
|
||||
vgf2p8affineinvqb $0, t2, x3, x3; \
|
||||
vgf2p8affineinvqb $0, t2, x7, x7
|
||||
|
||||
#endif /* CONFIG_AS_GFNI */
|
||||
|
||||
#define aria_sbox_8way(x0, x1, x2, x3, \
|
||||
x4, x5, x6, x7, \
|
||||
t0, t1, t2, t3, \
|
||||
t4, t5, t6, t7) \
|
||||
vpxor t7, t7, t7; \
|
||||
vmovdqa .Linv_shift_row, t0; \
|
||||
vmovdqa .Lshift_row, t1; \
|
||||
vpbroadcastd .L0f0f0f0f, t6; \
|
||||
vbroadcastss .L0f0f0f0f, t6; \
|
||||
vmovdqa .Ltf_lo__inv_aff__and__s2, t2; \
|
||||
vmovdqa .Ltf_hi__inv_aff__and__s2, t3; \
|
||||
vmovdqa .Ltf_lo__x2__and__fwd_aff, t4; \
|
||||
@@ -414,8 +421,9 @@
|
||||
y0, y1, y2, y3, \
|
||||
y4, y5, y6, y7, \
|
||||
mem_tmp, rk, round) \
|
||||
vpxor y7, y7, y7; \
|
||||
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, rk, 8, round); \
|
||||
y0, y7, y2, rk, 8, round); \
|
||||
\
|
||||
aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \
|
||||
y0, y1, y2, y3, y4, y5, y6, y7); \
|
||||
@@ -430,7 +438,7 @@
|
||||
x4, x5, x6, x7, \
|
||||
mem_tmp, 0); \
|
||||
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, rk, 0, round); \
|
||||
y0, y7, y2, rk, 0, round); \
|
||||
\
|
||||
aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \
|
||||
y0, y1, y2, y3, y4, y5, y6, y7); \
|
||||
@@ -468,8 +476,9 @@
|
||||
y0, y1, y2, y3, \
|
||||
y4, y5, y6, y7, \
|
||||
mem_tmp, rk, round) \
|
||||
vpxor y7, y7, y7; \
|
||||
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, rk, 8, round); \
|
||||
y0, y7, y2, rk, 8, round); \
|
||||
\
|
||||
aria_sbox_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, y1, y2, y3, y4, y5, y6, y7); \
|
||||
@@ -484,7 +493,7 @@
|
||||
x4, x5, x6, x7, \
|
||||
mem_tmp, 0); \
|
||||
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, rk, 0, round); \
|
||||
y0, y7, y2, rk, 0, round); \
|
||||
\
|
||||
aria_sbox_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, y1, y2, y3, y4, y5, y6, y7); \
|
||||
@@ -522,14 +531,15 @@
|
||||
y0, y1, y2, y3, \
|
||||
y4, y5, y6, y7, \
|
||||
mem_tmp, rk, round, last_round) \
|
||||
vpxor y7, y7, y7; \
|
||||
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, rk, 8, round); \
|
||||
y0, y7, y2, rk, 8, round); \
|
||||
\
|
||||
aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \
|
||||
y0, y1, y2, y3, y4, y5, y6, y7); \
|
||||
\
|
||||
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, rk, 8, last_round); \
|
||||
y0, y7, y2, rk, 8, last_round); \
|
||||
\
|
||||
aria_store_state_8way(x0, x1, x2, x3, \
|
||||
x4, x5, x6, x7, \
|
||||
@@ -539,25 +549,27 @@
|
||||
x4, x5, x6, x7, \
|
||||
mem_tmp, 0); \
|
||||
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, rk, 0, round); \
|
||||
y0, y7, y2, rk, 0, round); \
|
||||
\
|
||||
aria_sbox_8way(x2, x3, x0, x1, x6, x7, x4, x5, \
|
||||
y0, y1, y2, y3, y4, y5, y6, y7); \
|
||||
\
|
||||
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, rk, 0, last_round); \
|
||||
y0, y7, y2, rk, 0, last_round); \
|
||||
\
|
||||
aria_load_state_8way(y0, y1, y2, y3, \
|
||||
y4, y5, y6, y7, \
|
||||
mem_tmp, 8);
|
||||
|
||||
#ifdef CONFIG_AS_GFNI
|
||||
#define aria_fe_gfni(x0, x1, x2, x3, \
|
||||
x4, x5, x6, x7, \
|
||||
y0, y1, y2, y3, \
|
||||
y4, y5, y6, y7, \
|
||||
mem_tmp, rk, round) \
|
||||
vpxor y7, y7, y7; \
|
||||
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, rk, 8, round); \
|
||||
y0, y7, y2, rk, 8, round); \
|
||||
\
|
||||
aria_sbox_8way_gfni(x2, x3, x0, x1, \
|
||||
x6, x7, x4, x5, \
|
||||
@@ -574,7 +586,7 @@
|
||||
x4, x5, x6, x7, \
|
||||
mem_tmp, 0); \
|
||||
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, rk, 0, round); \
|
||||
y0, y7, y2, rk, 0, round); \
|
||||
\
|
||||
aria_sbox_8way_gfni(x2, x3, x0, x1, \
|
||||
x6, x7, x4, x5, \
|
||||
@@ -614,8 +626,9 @@
|
||||
y0, y1, y2, y3, \
|
||||
y4, y5, y6, y7, \
|
||||
mem_tmp, rk, round) \
|
||||
vpxor y7, y7, y7; \
|
||||
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, rk, 8, round); \
|
||||
y0, y7, y2, rk, 8, round); \
|
||||
\
|
||||
aria_sbox_8way_gfni(x0, x1, x2, x3, \
|
||||
x4, x5, x6, x7, \
|
||||
@@ -632,7 +645,7 @@
|
||||
x4, x5, x6, x7, \
|
||||
mem_tmp, 0); \
|
||||
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, rk, 0, round); \
|
||||
y0, y7, y2, rk, 0, round); \
|
||||
\
|
||||
aria_sbox_8way_gfni(x0, x1, x2, x3, \
|
||||
x4, x5, x6, x7, \
|
||||
@@ -672,8 +685,9 @@
|
||||
y0, y1, y2, y3, \
|
||||
y4, y5, y6, y7, \
|
||||
mem_tmp, rk, round, last_round) \
|
||||
vpxor y7, y7, y7; \
|
||||
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, rk, 8, round); \
|
||||
y0, y7, y2, rk, 8, round); \
|
||||
\
|
||||
aria_sbox_8way_gfni(x2, x3, x0, x1, \
|
||||
x6, x7, x4, x5, \
|
||||
@@ -681,7 +695,7 @@
|
||||
y4, y5, y6, y7); \
|
||||
\
|
||||
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, rk, 8, last_round); \
|
||||
y0, y7, y2, rk, 8, last_round); \
|
||||
\
|
||||
aria_store_state_8way(x0, x1, x2, x3, \
|
||||
x4, x5, x6, x7, \
|
||||
@@ -691,7 +705,7 @@
|
||||
x4, x5, x6, x7, \
|
||||
mem_tmp, 0); \
|
||||
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, rk, 0, round); \
|
||||
y0, y7, y2, rk, 0, round); \
|
||||
\
|
||||
aria_sbox_8way_gfni(x2, x3, x0, x1, \
|
||||
x6, x7, x4, x5, \
|
||||
@@ -699,12 +713,14 @@
|
||||
y4, y5, y6, y7); \
|
||||
\
|
||||
aria_ark_8way(x0, x1, x2, x3, x4, x5, x6, x7, \
|
||||
y0, rk, 0, last_round); \
|
||||
y0, y7, y2, rk, 0, last_round); \
|
||||
\
|
||||
aria_load_state_8way(y0, y1, y2, y3, \
|
||||
y4, y5, y6, y7, \
|
||||
mem_tmp, 8);
|
||||
|
||||
#endif /* CONFIG_AS_GFNI */
|
||||
|
||||
/* NB: section is mergeable, all elements must be aligned 16-byte blocks */
|
||||
.section .rodata.cst16, "aM", @progbits, 16
|
||||
.align 16
|
||||
@@ -756,6 +772,7 @@
|
||||
.Ltf_hi__x2__and__fwd_aff:
|
||||
.octa 0x3F893781E95FE1576CDA64D2BA0CB204
|
||||
|
||||
#ifdef CONFIG_AS_GFNI
|
||||
.section .rodata.cst8, "aM", @progbits, 8
|
||||
.align 8
|
||||
/* AES affine: */
|
||||
@@ -769,6 +786,14 @@
|
||||
BV8(0, 1, 1, 1, 1, 1, 0, 0),
|
||||
BV8(0, 0, 1, 1, 1, 1, 1, 0),
|
||||
BV8(0, 0, 0, 1, 1, 1, 1, 1))
|
||||
.quad BM8X8(BV8(1, 0, 0, 0, 1, 1, 1, 1),
|
||||
BV8(1, 1, 0, 0, 0, 1, 1, 1),
|
||||
BV8(1, 1, 1, 0, 0, 0, 1, 1),
|
||||
BV8(1, 1, 1, 1, 0, 0, 0, 1),
|
||||
BV8(1, 1, 1, 1, 1, 0, 0, 0),
|
||||
BV8(0, 1, 1, 1, 1, 1, 0, 0),
|
||||
BV8(0, 0, 1, 1, 1, 1, 1, 0),
|
||||
BV8(0, 0, 0, 1, 1, 1, 1, 1))
|
||||
|
||||
/* AES inverse affine: */
|
||||
#define tf_inv_const BV8(1, 0, 1, 0, 0, 0, 0, 0)
|
||||
@@ -781,6 +806,14 @@
|
||||
BV8(0, 0, 1, 0, 1, 0, 0, 1),
|
||||
BV8(1, 0, 0, 1, 0, 1, 0, 0),
|
||||
BV8(0, 1, 0, 0, 1, 0, 1, 0))
|
||||
.quad BM8X8(BV8(0, 0, 1, 0, 0, 1, 0, 1),
|
||||
BV8(1, 0, 0, 1, 0, 0, 1, 0),
|
||||
BV8(0, 1, 0, 0, 1, 0, 0, 1),
|
||||
BV8(1, 0, 1, 0, 0, 1, 0, 0),
|
||||
BV8(0, 1, 0, 1, 0, 0, 1, 0),
|
||||
BV8(0, 0, 1, 0, 1, 0, 0, 1),
|
||||
BV8(1, 0, 0, 1, 0, 1, 0, 0),
|
||||
BV8(0, 1, 0, 0, 1, 0, 1, 0))
|
||||
|
||||
/* S2: */
|
||||
#define tf_s2_const BV8(0, 1, 0, 0, 0, 1, 1, 1)
|
||||
@@ -793,6 +826,14 @@
|
||||
BV8(1, 1, 0, 0, 1, 1, 1, 0),
|
||||
BV8(0, 1, 1, 0, 0, 0, 1, 1),
|
||||
BV8(1, 1, 1, 1, 0, 1, 1, 0))
|
||||
.quad BM8X8(BV8(0, 1, 0, 1, 0, 1, 1, 1),
|
||||
BV8(0, 0, 1, 1, 1, 1, 1, 1),
|
||||
BV8(1, 1, 1, 0, 1, 1, 0, 1),
|
||||
BV8(1, 1, 0, 0, 0, 0, 1, 1),
|
||||
BV8(0, 1, 0, 0, 0, 0, 1, 1),
|
||||
BV8(1, 1, 0, 0, 1, 1, 1, 0),
|
||||
BV8(0, 1, 1, 0, 0, 0, 1, 1),
|
||||
BV8(1, 1, 1, 1, 0, 1, 1, 0))
|
||||
|
||||
/* X2: */
|
||||
#define tf_x2_const BV8(0, 0, 1, 1, 0, 1, 0, 0)
|
||||
@@ -805,6 +846,14 @@
|
||||
BV8(0, 1, 1, 0, 1, 0, 1, 1),
|
||||
BV8(1, 0, 1, 1, 1, 1, 0, 1),
|
||||
BV8(1, 0, 0, 1, 0, 0, 1, 1))
|
||||
.quad BM8X8(BV8(0, 0, 0, 1, 1, 0, 0, 0),
|
||||
BV8(0, 0, 1, 0, 0, 1, 1, 0),
|
||||
BV8(0, 0, 0, 0, 1, 0, 1, 0),
|
||||
BV8(1, 1, 1, 0, 0, 0, 1, 1),
|
||||
BV8(1, 1, 1, 0, 1, 1, 0, 0),
|
||||
BV8(0, 1, 1, 0, 1, 0, 1, 1),
|
||||
BV8(1, 0, 1, 1, 1, 1, 0, 1),
|
||||
BV8(1, 0, 0, 1, 0, 0, 1, 1))
|
||||
|
||||
/* Identity matrix: */
|
||||
.Ltf_id_bitmatrix:
|
||||
@@ -816,6 +865,15 @@
|
||||
BV8(0, 0, 0, 0, 0, 1, 0, 0),
|
||||
BV8(0, 0, 0, 0, 0, 0, 1, 0),
|
||||
BV8(0, 0, 0, 0, 0, 0, 0, 1))
|
||||
.quad BM8X8(BV8(1, 0, 0, 0, 0, 0, 0, 0),
|
||||
BV8(0, 1, 0, 0, 0, 0, 0, 0),
|
||||
BV8(0, 0, 1, 0, 0, 0, 0, 0),
|
||||
BV8(0, 0, 0, 1, 0, 0, 0, 0),
|
||||
BV8(0, 0, 0, 0, 1, 0, 0, 0),
|
||||
BV8(0, 0, 0, 0, 0, 1, 0, 0),
|
||||
BV8(0, 0, 0, 0, 0, 0, 1, 0),
|
||||
BV8(0, 0, 0, 0, 0, 0, 0, 1))
|
||||
#endif /* CONFIG_AS_GFNI */
|
||||
|
||||
/* 4-bit mask */
|
||||
.section .rodata.cst4.L0f0f0f0f, "aM", @progbits, 4
|
||||
@@ -874,7 +932,7 @@ SYM_FUNC_START_LOCAL(__aria_aesni_avx_crypt_16way)
|
||||
aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15,
|
||||
%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
||||
%rax, %r9, 10);
|
||||
cmpl $12, rounds(CTX);
|
||||
cmpl $12, ARIA_CTX_rounds(CTX);
|
||||
jne .Laria_192;
|
||||
aria_ff(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
||||
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
||||
@@ -887,7 +945,7 @@ SYM_FUNC_START_LOCAL(__aria_aesni_avx_crypt_16way)
|
||||
aria_fo(%xmm9, %xmm8, %xmm11, %xmm10, %xmm12, %xmm13, %xmm14, %xmm15,
|
||||
%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
||||
%rax, %r9, 12);
|
||||
cmpl $14, rounds(CTX);
|
||||
cmpl $14, ARIA_CTX_rounds(CTX);
|
||||
jne .Laria_256;
|
||||
aria_ff(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
||||
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
||||
@@ -923,7 +981,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx_encrypt_16way)
|
||||
|
||||
FRAME_BEGIN
|
||||
|
||||
leaq enc_key(CTX), %r9;
|
||||
leaq ARIA_CTX_enc_key(CTX), %r9;
|
||||
|
||||
inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
||||
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
||||
@@ -948,7 +1006,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx_decrypt_16way)
|
||||
|
||||
FRAME_BEGIN
|
||||
|
||||
leaq dec_key(CTX), %r9;
|
||||
leaq ARIA_CTX_dec_key(CTX), %r9;
|
||||
|
||||
inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
||||
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
||||
@@ -1056,7 +1114,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx_ctr_crypt_16way)
|
||||
leaq (%rdx), %r11;
|
||||
leaq (%rcx), %rsi;
|
||||
leaq (%rcx), %rdx;
|
||||
leaq enc_key(CTX), %r9;
|
||||
leaq ARIA_CTX_enc_key(CTX), %r9;
|
||||
|
||||
call __aria_aesni_avx_crypt_16way;
|
||||
|
||||
@@ -1084,6 +1142,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx_ctr_crypt_16way)
|
||||
RET;
|
||||
SYM_FUNC_END(aria_aesni_avx_ctr_crypt_16way)
|
||||
|
||||
#ifdef CONFIG_AS_GFNI
|
||||
SYM_FUNC_START_LOCAL(__aria_aesni_avx_gfni_crypt_16way)
|
||||
/* input:
|
||||
* %r9: rk
|
||||
@@ -1157,7 +1216,7 @@ SYM_FUNC_START_LOCAL(__aria_aesni_avx_gfni_crypt_16way)
|
||||
%xmm0, %xmm1, %xmm2, %xmm3,
|
||||
%xmm4, %xmm5, %xmm6, %xmm7,
|
||||
%rax, %r9, 10);
|
||||
cmpl $12, rounds(CTX);
|
||||
cmpl $12, ARIA_CTX_rounds(CTX);
|
||||
jne .Laria_gfni_192;
|
||||
aria_ff_gfni(%xmm1, %xmm0, %xmm3, %xmm2, %xmm4, %xmm5, %xmm6, %xmm7,
|
||||
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
||||
@@ -1174,7 +1233,7 @@ SYM_FUNC_START_LOCAL(__aria_aesni_avx_gfni_crypt_16way)
|
||||
%xmm0, %xmm1, %xmm2, %xmm3,
|
||||
%xmm4, %xmm5, %xmm6, %xmm7,
|
||||
%rax, %r9, 12);
|
||||
cmpl $14, rounds(CTX);
|
||||
cmpl $14, ARIA_CTX_rounds(CTX);
|
||||
jne .Laria_gfni_256;
|
||||
aria_ff_gfni(%xmm1, %xmm0, %xmm3, %xmm2,
|
||||
%xmm4, %xmm5, %xmm6, %xmm7,
|
||||
@@ -1218,7 +1277,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx_gfni_encrypt_16way)
|
||||
|
||||
FRAME_BEGIN
|
||||
|
||||
leaq enc_key(CTX), %r9;
|
||||
leaq ARIA_CTX_enc_key(CTX), %r9;
|
||||
|
||||
inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
||||
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
||||
@@ -1243,7 +1302,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx_gfni_decrypt_16way)
|
||||
|
||||
FRAME_BEGIN
|
||||
|
||||
leaq dec_key(CTX), %r9;
|
||||
leaq ARIA_CTX_dec_key(CTX), %r9;
|
||||
|
||||
inpack16_pre(%xmm0, %xmm1, %xmm2, %xmm3, %xmm4, %xmm5, %xmm6, %xmm7,
|
||||
%xmm8, %xmm9, %xmm10, %xmm11, %xmm12, %xmm13, %xmm14,
|
||||
@@ -1275,7 +1334,7 @@ SYM_TYPED_FUNC_START(aria_aesni_avx_gfni_ctr_crypt_16way)
|
||||
leaq (%rdx), %r11;
|
||||
leaq (%rcx), %rsi;
|
||||
leaq (%rcx), %rdx;
|
||||
leaq enc_key(CTX), %r9;
|
||||
leaq ARIA_CTX_enc_key(CTX), %r9;
|
||||
|
||||
call __aria_aesni_avx_gfni_crypt_16way;
|
||||
|
||||
@@ -1302,3 +1361,4 @@ SYM_TYPED_FUNC_START(aria_aesni_avx_gfni_ctr_crypt_16way)
|
||||
FRAME_END
|
||||
RET;
|
||||
SYM_FUNC_END(aria_aesni_avx_gfni_ctr_crypt_16way)
|
||||
#endif /* CONFIG_AS_GFNI */
|
||||
|
||||
1441
arch/x86/crypto/aria-aesni-avx2-asm_64.S
Normal file
1441
arch/x86/crypto/aria-aesni-avx2-asm_64.S
Normal file
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user