lib/crypto: sha256: Consolidate into single module

Consolidate the CPU-based SHA-256 code into a single module, following
what I did with SHA-512:

- Each arch now provides a header file lib/crypto/$(SRCARCH)/sha256.h,
  replacing lib/crypto/$(SRCARCH)/sha256.c.  The header defines
  sha256_blocks() and optionally sha256_mod_init_arch().  It is included
  by lib/crypto/sha256.c, and thus the code gets built into the single
  libsha256 module, with proper inlining and dead code elimination.

- sha256_blocks_generic() is moved from lib/crypto/sha256-generic.c into
  lib/crypto/sha256.c.  It's now a static function marked with
  __maybe_unused, so the compiler automatically eliminates it in any
  cases where it's not used.

- Whether arch-optimized SHA-256 is buildable is now controlled
  centrally by lib/crypto/Kconfig instead of by
  lib/crypto/$(SRCARCH)/Kconfig.  The conditions for enabling it remain
  the same as before, and it remains enabled by default.

- Any additional arch-specific translation units for the optimized
  SHA-256 code (such as assembly files) are now compiled by
  lib/crypto/Makefile instead of lib/crypto/$(SRCARCH)/Makefile.

Acked-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20250630160645.3198-13-ebiggers@kernel.org
Signed-off-by: Eric Biggers <ebiggers@kernel.org>
This commit is contained in:
Eric Biggers
2025-07-04 10:23:11 -07:00
parent 9f9846a72e
commit e96cb9507f
29 changed files with 231 additions and 465 deletions
-6
View File
@@ -23,12 +23,6 @@ config CAVIUM_OCTEON_CVMSEG_SIZE
legally range is from zero to 54 cache blocks (i.e. CVMSEG LM is
between zero and 6192 bytes).
config CRYPTO_SHA256_OCTEON
tristate
default CRYPTO_LIB_SHA256
select CRYPTO_ARCH_HAVE_LIB_SHA256
select CRYPTO_LIB_SHA256_GENERIC
endif # CPU_CAVIUM_OCTEON
if CAVIUM_OCTEON_SOC
-1
View File
@@ -7,4 +7,3 @@ obj-y += octeon-crypto.o
obj-$(CONFIG_CRYPTO_MD5_OCTEON) += octeon-md5.o
obj-$(CONFIG_CRYPTO_SHA1_OCTEON) += octeon-sha1.o
obj-$(CONFIG_CRYPTO_SHA256_OCTEON) += octeon-sha256.o
-52
View File
@@ -1,52 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0-only */
#ifndef _CRYPTO_INTERNAL_SHA2_H
#define _CRYPTO_INTERNAL_SHA2_H
#include <crypto/sha2.h>
#include <linux/compiler_attributes.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/unaligned.h>
void sha256_blocks_generic(struct sha256_block_state *state,
const u8 *data, size_t nblocks);
void sha256_blocks_arch(struct sha256_block_state *state,
const u8 *data, size_t nblocks);
static __always_inline void sha256_choose_blocks(
u32 state[SHA256_STATE_WORDS], const u8 *data, size_t nblocks,
bool force_generic, bool force_simd)
{
if (!IS_ENABLED(CONFIG_CRYPTO_ARCH_HAVE_LIB_SHA256) || force_generic)
sha256_blocks_generic((struct sha256_block_state *)state, data, nblocks);
else
sha256_blocks_arch((struct sha256_block_state *)state, data, nblocks);
}
static __always_inline void sha256_finup(
struct crypto_sha256_state *sctx, u8 buf[SHA256_BLOCK_SIZE],
size_t len, u8 out[SHA256_DIGEST_SIZE], size_t digest_size,
bool force_generic, bool force_simd)
{
const size_t bit_offset = SHA256_BLOCK_SIZE - 8;
__be64 *bits = (__be64 *)&buf[bit_offset];
int i;
buf[len++] = 0x80;
if (len > bit_offset) {
memset(&buf[len], 0, SHA256_BLOCK_SIZE - len);
sha256_choose_blocks(sctx->state, buf, 1, force_generic,
force_simd);
len = 0;
}
memset(&buf[len], 0, bit_offset - len);
*bits = cpu_to_be64(sctx->count << 3);
sha256_choose_blocks(sctx->state, buf, 1, force_generic, force_simd);
for (i = 0; i < digest_size; i += 4)
put_unaligned_be32(sctx->state[i / 4], out + i);
}
#endif /* _CRYPTO_INTERNAL_SHA2_H */
+10 -16
View File
@@ -144,20 +144,17 @@ config CRYPTO_LIB_SHA256
by either the generic implementation or an arch-specific one, if one
is available and enabled.
config CRYPTO_ARCH_HAVE_LIB_SHA256
config CRYPTO_LIB_SHA256_ARCH
bool
help
Declares whether the architecture provides an arch-specific
accelerated implementation of the SHA-256 library interface.
config CRYPTO_LIB_SHA256_GENERIC
tristate
default CRYPTO_LIB_SHA256 if !CRYPTO_ARCH_HAVE_LIB_SHA256
help
This symbol can be selected by arch implementations of the SHA-256
library interface that require the generic code as a fallback, e.g.,
for SIMD implementations. If no arch specific implementation is
enabled, this implementation serves the users of CRYPTO_LIB_SHA256.
depends on CRYPTO_LIB_SHA256 && !UML
default y if ARM && !CPU_V7M
default y if ARM64
default y if MIPS && CPU_CAVIUM_OCTEON
default y if PPC && SPE
default y if RISCV && 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
default y if S390
default y if SPARC64
default y if X86_64
config CRYPTO_LIB_SHA512
tristate
@@ -199,9 +196,6 @@ endif
if S390
source "lib/crypto/s390/Kconfig"
endif
if SPARC
source "lib/crypto/sparc/Kconfig"
endif
if X86
source "lib/crypto/x86/Kconfig"
endif
+34 -5
View File
@@ -66,11 +66,39 @@ libpoly1305-generic-y += poly1305-generic.o
obj-$(CONFIG_CRYPTO_LIB_SHA1) += libsha1.o
libsha1-y := sha1.o
obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o
libsha256-y := sha256.o
################################################################################
obj-$(CONFIG_CRYPTO_LIB_SHA256_GENERIC) += libsha256-generic.o
libsha256-generic-y := sha256-generic.o
obj-$(CONFIG_CRYPTO_LIB_SHA256) += libsha256.o
libsha256-y := sha256.o
ifeq ($(CONFIG_CRYPTO_LIB_SHA256_ARCH),y)
CFLAGS_sha256.o += -I$(src)/$(SRCARCH)
ifeq ($(CONFIG_ARM),y)
libsha256-y += arm/sha256-ce.o arm/sha256-core.o
$(obj)/arm/sha256-core.S: $(src)/arm/sha256-armv4.pl
$(call cmd,perlasm)
clean-files += arm/sha256-core.S
AFLAGS_arm/sha256-core.o += $(aflags-thumb2-y)
endif
ifeq ($(CONFIG_ARM64),y)
libsha256-y += arm64/sha256-core.o
$(obj)/arm64/sha256-core.S: $(src)/arm64/sha2-armv8.pl
$(call cmd,perlasm_with_args)
clean-files += arm64/sha256-core.S
libsha256-$(CONFIG_KERNEL_MODE_NEON) += arm64/sha256-ce.o
endif
libsha256-$(CONFIG_PPC) += powerpc/sha256-spe-asm.o
libsha256-$(CONFIG_RISCV) += riscv/sha256-riscv64-zvknha_or_zvknhb-zvkb.o
libsha256-$(CONFIG_SPARC) += sparc/sha256_asm.o
libsha256-$(CONFIG_X86) += x86/sha256-ssse3-asm.o \
x86/sha256-avx-asm.o \
x86/sha256-avx2-asm.o \
x86/sha256-ni-asm.o
endif # CONFIG_CRYPTO_LIB_SHA256_ARCH
################################################################################
obj-$(CONFIG_CRYPTO_LIB_SHA512) += libsha512.o
libsha512-y := sha512.o
@@ -100,6 +128,8 @@ libsha512-$(CONFIG_X86) += x86/sha512-ssse3-asm.o \
x86/sha512-avx2-asm.o
endif # CONFIG_CRYPTO_LIB_SHA512_ARCH
################################################################################
obj-$(CONFIG_MPILIB) += mpi/
obj-$(CONFIG_CRYPTO_SELFTESTS_FULL) += simd.o
@@ -113,5 +143,4 @@ obj-$(CONFIG_MIPS) += mips/
obj-$(CONFIG_PPC) += powerpc/
obj-$(CONFIG_RISCV) += riscv/
obj-$(CONFIG_S390) += s390/
obj-$(CONFIG_SPARC) += sparc/
obj-$(CONFIG_X86) += x86/
-6
View File
@@ -22,9 +22,3 @@ config CRYPTO_POLY1305_ARM
tristate
default CRYPTO_LIB_POLY1305
select CRYPTO_ARCH_HAVE_LIB_POLY1305
config CRYPTO_SHA256_ARM
tristate
depends on !CPU_V7M
default CRYPTO_LIB_SHA256
select CRYPTO_ARCH_HAVE_LIB_SHA256
+1 -7
View File
@@ -10,17 +10,13 @@ chacha-neon-$(CONFIG_KERNEL_MODE_NEON) += chacha-neon-core.o
obj-$(CONFIG_CRYPTO_POLY1305_ARM) += poly1305-arm.o
poly1305-arm-y := poly1305-core.o poly1305-glue.o
obj-$(CONFIG_CRYPTO_SHA256_ARM) += sha256-arm.o
sha256-arm-y := sha256.o sha256-core.o
sha256-arm-$(CONFIG_KERNEL_MODE_NEON) += sha256-ce.o
quiet_cmd_perl = PERL $@
cmd_perl = $(PERL) $(<) > $(@)
$(obj)/%-core.S: $(src)/%-armv4.pl
$(call cmd,perl)
clean-files += poly1305-core.S sha256-core.S
clean-files += poly1305-core.S
aflags-thumb2-$(CONFIG_THUMB2_KERNEL) := -U__thumb2__ -D__thumb2__=1
@@ -28,5 +24,3 @@ aflags-thumb2-$(CONFIG_THUMB2_KERNEL) := -U__thumb2__ -D__thumb2__=1
poly1305-aflags-$(CONFIG_CPU_V7) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=5
poly1305-aflags-$(CONFIG_KERNEL_MODE_NEON) := -U__LINUX_ARM_ARCH__ -D__LINUX_ARM_ARCH__=7
AFLAGS_poly1305-core.o += $(poly1305-aflags-y) $(aflags-thumb2-y)
AFLAGS_sha256-core.o += $(aflags-thumb2-y)
@@ -1,14 +1,11 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SHA-256 optimized for ARM
*
* Copyright 2025 Google LLC
*/
#include <asm/neon.h>
#include <crypto/internal/sha2.h>
#include <crypto/internal/simd.h>
#include <linux/kernel.h>
#include <linux/module.h>
asmlinkage void sha256_block_data_order(struct sha256_block_state *state,
const u8 *data, size_t nblocks);
@@ -20,8 +17,8 @@ asmlinkage void sha256_ce_transform(struct sha256_block_state *state,
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
void sha256_blocks_arch(struct sha256_block_state *state,
const u8 *data, size_t nblocks)
static void sha256_blocks(struct sha256_block_state *state,
const u8 *data, size_t nblocks)
{
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
static_branch_likely(&have_neon) && crypto_simd_usable()) {
@@ -35,23 +32,15 @@ void sha256_blocks_arch(struct sha256_block_state *state,
sha256_block_data_order(state, data, nblocks);
}
}
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
static int __init sha256_arm_mod_init(void)
#ifdef CONFIG_KERNEL_MODE_NEON
#define sha256_mod_init_arch sha256_mod_init_arch
static inline void sha256_mod_init_arch(void)
{
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_NEON)) {
if (elf_hwcap & HWCAP_NEON) {
static_branch_enable(&have_neon);
if (elf_hwcap2 & HWCAP2_SHA2)
static_branch_enable(&have_ce);
}
return 0;
}
subsys_initcall(sha256_arm_mod_init);
static void __exit sha256_arm_mod_exit(void)
{
}
module_exit(sha256_arm_mod_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA-256 optimized for ARM");
#endif /* CONFIG_KERNEL_MODE_NEON */
-5
View File
@@ -12,8 +12,3 @@ config CRYPTO_POLY1305_NEON
depends on KERNEL_MODE_NEON
default CRYPTO_LIB_POLY1305
select CRYPTO_ARCH_HAVE_LIB_POLY1305
config CRYPTO_SHA256_ARM64
tristate
default CRYPTO_LIB_SHA256
select CRYPTO_ARCH_HAVE_LIB_SHA256
+1 -8
View File
@@ -8,17 +8,10 @@ poly1305-neon-y := poly1305-core.o poly1305-glue.o
AFLAGS_poly1305-core.o += -Dpoly1305_init=poly1305_block_init_arch
AFLAGS_poly1305-core.o += -Dpoly1305_emit=poly1305_emit_arch
obj-$(CONFIG_CRYPTO_SHA256_ARM64) += sha256-arm64.o
sha256-arm64-y := sha256.o sha256-core.o
sha256-arm64-$(CONFIG_KERNEL_MODE_NEON) += sha256-ce.o
quiet_cmd_perlasm = PERLASM $@
cmd_perlasm = $(PERL) $(<) void $(@)
$(obj)/%-core.S: $(src)/%-armv8.pl
$(call cmd,perlasm)
$(obj)/sha256-core.S: $(src)/sha2-armv8.pl
$(call cmd,perlasm)
clean-files += poly1305-core.S sha256-core.S
clean-files += poly1305-core.S
@@ -1,14 +1,12 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SHA-256 optimized for ARM64
*
* Copyright 2025 Google LLC
*/
#include <asm/neon.h>
#include <crypto/internal/sha2.h>
#include <crypto/internal/simd.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/cpufeature.h>
asmlinkage void sha256_block_data_order(struct sha256_block_state *state,
const u8 *data, size_t nblocks);
@@ -20,8 +18,8 @@ asmlinkage size_t __sha256_ce_transform(struct sha256_block_state *state,
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_neon);
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_ce);
void sha256_blocks_arch(struct sha256_block_state *state,
const u8 *data, size_t nblocks)
static void sha256_blocks(struct sha256_block_state *state,
const u8 *data, size_t nblocks)
{
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
static_branch_likely(&have_neon) && crypto_simd_usable()) {
@@ -45,24 +43,15 @@ void sha256_blocks_arch(struct sha256_block_state *state,
sha256_block_data_order(state, data, nblocks);
}
}
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
static int __init sha256_arm64_mod_init(void)
#ifdef CONFIG_KERNEL_MODE_NEON
#define sha256_mod_init_arch sha256_mod_init_arch
static inline void sha256_mod_init_arch(void)
{
if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) &&
cpu_have_named_feature(ASIMD)) {
if (cpu_have_named_feature(ASIMD)) {
static_branch_enable(&have_neon);
if (cpu_have_named_feature(SHA2))
static_branch_enable(&have_ce);
}
return 0;
}
subsys_initcall(sha256_arm64_mod_init);
static void __exit sha256_arm64_mod_exit(void)
{
}
module_exit(sha256_arm64_mod_exit);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA-256 optimized for ARM64");
#endif /* CONFIG_KERNEL_MODE_NEON */
@@ -1,4 +1,4 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SHA-256 Secure Hash Algorithm.
*
@@ -14,16 +14,13 @@
#include <asm/octeon/crypto.h>
#include <asm/octeon/octeon.h>
#include <crypto/internal/sha2.h>
#include <linux/kernel.h>
#include <linux/module.h>
/*
* We pass everything as 64-bit. OCTEON can handle misaligned data.
*/
void sha256_blocks_arch(struct sha256_block_state *state,
const u8 *data, size_t nblocks)
static void sha256_blocks(struct sha256_block_state *state,
const u8 *data, size_t nblocks)
{
struct octeon_cop2_state cop2_state;
u64 *state64 = (u64 *)state;
@@ -59,8 +56,3 @@ void sha256_blocks_arch(struct sha256_block_state *state,
state64[3] = read_octeon_64bit_hash_dword(3);
octeon_crypto_disable(&cop2_state, flags);
}
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA-256 Secure Hash Algorithm (OCTEON)");
MODULE_AUTHOR("Aaro Koskinen <aaro.koskinen@iki.fi>");
-6
View File
@@ -14,9 +14,3 @@ config CRYPTO_POLY1305_P10
default CRYPTO_LIB_POLY1305
select CRYPTO_ARCH_HAVE_LIB_POLY1305
select CRYPTO_LIB_POLY1305_GENERIC
config CRYPTO_SHA256_PPC_SPE
tristate
depends on SPE
default CRYPTO_LIB_SHA256
select CRYPTO_ARCH_HAVE_LIB_SHA256
-3
View File
@@ -5,6 +5,3 @@ chacha-p10-crypto-y := chacha-p10-glue.o chacha-p10le-8x.o
obj-$(CONFIG_CRYPTO_POLY1305_P10) += poly1305-p10-crypto.o
poly1305-p10-crypto-y := poly1305-p10-glue.o poly1305-p10le_64.o
obj-$(CONFIG_CRYPTO_SHA256_PPC_SPE) += sha256-ppc-spe.o
sha256-ppc-spe-y := sha256.o sha256-spe-asm.o
@@ -1,4 +1,4 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SHA-256 Secure Hash Algorithm, SPE optimized
*
@@ -9,9 +9,6 @@
*/
#include <asm/switch_to.h>
#include <crypto/internal/sha2.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/preempt.h>
/*
@@ -43,8 +40,8 @@ static void spe_end(void)
preempt_enable();
}
void sha256_blocks_arch(struct sha256_block_state *state,
const u8 *data, size_t nblocks)
static void sha256_blocks(struct sha256_block_state *state,
const u8 *data, size_t nblocks)
{
do {
/* cut input data into smaller blocks */
@@ -59,7 +56,3 @@ void sha256_blocks_arch(struct sha256_block_state *state,
nblocks -= unit;
} while (nblocks);
}
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
MODULE_LICENSE("GPL");
MODULE_DESCRIPTION("SHA-256 Secure Hash Algorithm, SPE optimized");
-7
View File
@@ -6,10 +6,3 @@ config CRYPTO_CHACHA_RISCV64
default CRYPTO_LIB_CHACHA
select CRYPTO_ARCH_HAVE_LIB_CHACHA
select CRYPTO_LIB_CHACHA_GENERIC
config CRYPTO_SHA256_RISCV64
tristate
depends on 64BIT && RISCV_ISA_V && TOOLCHAIN_HAS_VECTOR_CRYPTO
default CRYPTO_LIB_SHA256
select CRYPTO_ARCH_HAVE_LIB_SHA256
select CRYPTO_LIB_SHA256_GENERIC
-3
View File
@@ -2,6 +2,3 @@
obj-$(CONFIG_CRYPTO_CHACHA_RISCV64) += chacha-riscv64.o
chacha-riscv64-y := chacha-riscv64-glue.o chacha-riscv64-zvkb.o
obj-$(CONFIG_CRYPTO_SHA256_RISCV64) += sha256-riscv64.o
sha256-riscv64-y := sha256.o sha256-riscv64-zvknha_or_zvknhb-zvkb.o
@@ -1,4 +1,4 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
* SHA-256 (RISC-V accelerated)
*
@@ -10,10 +10,7 @@
*/
#include <asm/vector.h>
#include <crypto/internal/sha2.h>
#include <crypto/internal/simd.h>
#include <linux/kernel.h>
#include <linux/module.h>
asmlinkage void
sha256_transform_zvknha_or_zvknhb_zvkb(struct sha256_block_state *state,
@@ -21,8 +18,8 @@ sha256_transform_zvknha_or_zvknhb_zvkb(struct sha256_block_state *state,
static __ro_after_init DEFINE_STATIC_KEY_FALSE(have_extensions);
void sha256_blocks_arch(struct sha256_block_state *state,
const u8 *data, size_t nblocks)
static void sha256_blocks(struct sha256_block_state *state,
const u8 *data, size_t nblocks)
{
if (static_branch_likely(&have_extensions) && crypto_simd_usable()) {
kernel_vector_begin();
@@ -32,9 +29,9 @@ void sha256_blocks_arch(struct sha256_block_state *state,
sha256_blocks_generic(state, data, nblocks);
}
}
EXPORT_SYMBOL_GPL(sha256_blocks_arch);
static int __init riscv64_sha256_mod_init(void)
#define sha256_mod_init_arch sha256_mod_init_arch
static inline void sha256_mod_init_arch(void)
{
/* Both zvknha and zvknhb provide the SHA-256 instructions. */
if ((riscv_isa_extension_available(NULL, ZVKNHA) ||
@@ -42,15 +39,4 @@ static int __init riscv64_sha256_mod_init(void)
riscv_isa_extension_available(NULL, ZVKB) &&
riscv_vector_vlen() >= 128)
static_branch_enable(&have_extensions);
return 0;
}
subsys_initcall(riscv64_sha256_mod_init);
static void __exit riscv64_sha256_mod_exit(void)
{
}
module_exit(riscv64_sha256_mod_exit);
MODULE_DESCRIPTION("SHA-256 (RISC-V accelerated)");
MODULE_AUTHOR("Heiko Stuebner <heiko.stuebner@vrull.eu>");
MODULE_LICENSE("GPL");
-6
View File
@@ -5,9 +5,3 @@ config CRYPTO_CHACHA_S390
default CRYPTO_LIB_CHACHA
select CRYPTO_LIB_CHACHA_GENERIC
select CRYPTO_ARCH_HAVE_LIB_CHACHA
config CRYPTO_SHA256_S390
tristate
default CRYPTO_LIB_SHA256
select CRYPTO_ARCH_HAVE_LIB_SHA256
select CRYPTO_LIB_SHA256_GENERIC
-3
View File
@@ -2,6 +2,3 @@
obj-$(CONFIG_CRYPTO_CHACHA_S390) += chacha_s390.o
chacha_s390-y := chacha-glue.o chacha-s390.o
obj-$(CONFIG_CRYPTO_SHA256_S390) += sha256-s390.o
sha256-s390-y := sha256.o

Some files were not shown because too many files have changed in this diff Show More