From 5fbc09eb0b4f4b1a4b33abebacbeee0d29f195e9 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Wed, 24 Dec 2025 12:20:49 +0100 Subject: [PATCH 01/27] powerpc/uaccess: Move barrier_nospec() out of allow_read_{from/write}_user() Commit 74e19ef0ff80 ("uaccess: Add speculation barrier to copy_from_user()") added a redundant barrier_nospec() in copy_from_user(), because powerpc is already calling barrier_nospec() in allow_read_from_user() and allow_read_write_user(). But on other architectures that call to barrier_nospec() was missing. So change powerpc instead of reverting the above commit and having to fix other architectures one by one. This is now possible because barrier_nospec() has also been added in copy_from_user_iter(). Move barrier_nospec() out of allow_read_from_user() and allow_read_write_user(). This will also allow reuse of those functions when implementing masked user access which doesn't require barrier_nospec(). Don't add it back in raw_copy_from_user() as it is already called by copy_from_user() and copy_from_user_iter(). Fixes: 74e19ef0ff80 ("uaccess: Add speculation barrier to copy_from_user()") Signed-off-by: Christophe Leroy Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/f29612105c5fcbc8ceb7303808ddc1a781f0f6b5.1766574657.git.chleroy@kernel.org --- arch/powerpc/include/asm/kup.h | 2 -- arch/powerpc/include/asm/uaccess.h | 4 ++++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h index dab63b82a8d4..f2009d7c8cfa 100644 --- a/arch/powerpc/include/asm/kup.h +++ b/arch/powerpc/include/asm/kup.h @@ -134,7 +134,6 @@ static __always_inline void kuap_assert_locked(void) static __always_inline void allow_read_from_user(const void __user *from, unsigned long size) { - barrier_nospec(); allow_user_access(NULL, from, size, KUAP_READ); } @@ -146,7 +145,6 @@ static __always_inline void allow_write_to_user(void __user *to, unsigned long s static __always_inline void allow_read_write_user(void __user *to, const void __user *from, unsigned long size) { - barrier_nospec(); allow_user_access(to, from, size, KUAP_READ_WRITE); } diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 784a00e681fa..3e622e647d62 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -301,6 +301,7 @@ do { \ __typeof__(sizeof(*(ptr))) __gu_size = sizeof(*(ptr)); \ \ might_fault(); \ + barrier_nospec(); \ allow_read_from_user(__gu_addr, __gu_size); \ __get_user_size_allowed(__gu_val, __gu_addr, __gu_size, __gu_err); \ prevent_read_from_user(__gu_addr, __gu_size); \ @@ -329,6 +330,7 @@ raw_copy_in_user(void __user *to, const void __user *from, unsigned long n) { unsigned long ret; + barrier_nospec(); allow_read_write_user(to, from, n); ret = __copy_tofrom_user(to, from, n); prevent_read_write_user(to, from, n); @@ -415,6 +417,7 @@ static __must_check __always_inline bool user_access_begin(const void __user *pt might_fault(); + barrier_nospec(); allow_read_write_user((void __user *)ptr, ptr, len); return true; } @@ -431,6 +434,7 @@ user_read_access_begin(const void __user *ptr, size_t len) might_fault(); + barrier_nospec(); allow_read_from_user(ptr, len); return true; } From e3efbfbfaaaa739072b2756cfb76cd3de6d91c0f Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Wed, 24 Dec 2025 12:20:50 +0100 Subject: [PATCH 02/27] powerpc/uaccess: Remove unused size and from parameters from allow_access_user() Since commit 16132529cee5 ("powerpc/32s: Rework Kernel Userspace Access Protection") the size parameter is unused on all platforms. And the 'from' parameter has never been used. Remove them. Signed-off-by: Christophe Leroy Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/4552b00707923b71150ee47b925d6eaae1b03261.1766574657.git.chleroy@kernel.org --- arch/powerpc/include/asm/book3s/32/kup.h | 3 +-- arch/powerpc/include/asm/book3s/64/kup.h | 6 ++---- arch/powerpc/include/asm/kup.h | 9 ++++----- arch/powerpc/include/asm/nohash/32/kup-8xx.h | 3 +-- arch/powerpc/include/asm/nohash/kup-booke.h | 3 +-- 5 files changed, 9 insertions(+), 15 deletions(-) diff --git a/arch/powerpc/include/asm/book3s/32/kup.h b/arch/powerpc/include/asm/book3s/32/kup.h index 873c5146e326..a3558419c41b 100644 --- a/arch/powerpc/include/asm/book3s/32/kup.h +++ b/arch/powerpc/include/asm/book3s/32/kup.h @@ -97,8 +97,7 @@ static __always_inline unsigned long __kuap_get_and_assert_locked(void) } #define __kuap_get_and_assert_locked __kuap_get_and_assert_locked -static __always_inline void allow_user_access(void __user *to, const void __user *from, - u32 size, unsigned long dir) +static __always_inline void allow_user_access(void __user *to, unsigned long dir) { BUILD_BUG_ON(!__builtin_constant_p(dir)); diff --git a/arch/powerpc/include/asm/book3s/64/kup.h b/arch/powerpc/include/asm/book3s/64/kup.h index 03aec3c6c851..9ccf8a5e0926 100644 --- a/arch/powerpc/include/asm/book3s/64/kup.h +++ b/arch/powerpc/include/asm/book3s/64/kup.h @@ -353,8 +353,7 @@ __bad_kuap_fault(struct pt_regs *regs, unsigned long address, bool is_write) return (regs->amr & AMR_KUAP_BLOCK_READ) == AMR_KUAP_BLOCK_READ; } -static __always_inline void allow_user_access(void __user *to, const void __user *from, - unsigned long size, unsigned long dir) +static __always_inline void allow_user_access(void __user *to, unsigned long dir) { unsigned long thread_amr = 0; @@ -383,8 +382,7 @@ static __always_inline unsigned long get_kuap(void) static __always_inline void set_kuap(unsigned long value) { } -static __always_inline void allow_user_access(void __user *to, const void __user *from, - unsigned long size, unsigned long dir) +static __always_inline void allow_user_access(void __user *to, unsigned long dir) { } #endif /* !CONFIG_PPC_KUAP */ diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h index f2009d7c8cfa..3963584ac1cf 100644 --- a/arch/powerpc/include/asm/kup.h +++ b/arch/powerpc/include/asm/kup.h @@ -72,8 +72,7 @@ static __always_inline void __kuap_kernel_restore(struct pt_regs *regs, unsigned * platforms. */ #ifndef CONFIG_PPC_BOOK3S_64 -static __always_inline void allow_user_access(void __user *to, const void __user *from, - unsigned long size, unsigned long dir) { } +static __always_inline void allow_user_access(void __user *to, unsigned long dir) { } static __always_inline void prevent_user_access(unsigned long dir) { } static __always_inline unsigned long prevent_user_access_return(void) { return 0UL; } static __always_inline void restore_user_access(unsigned long flags) { } @@ -134,18 +133,18 @@ static __always_inline void kuap_assert_locked(void) static __always_inline void allow_read_from_user(const void __user *from, unsigned long size) { - allow_user_access(NULL, from, size, KUAP_READ); + allow_user_access(NULL, KUAP_READ); } static __always_inline void allow_write_to_user(void __user *to, unsigned long size) { - allow_user_access(to, NULL, size, KUAP_WRITE); + allow_user_access(to, KUAP_WRITE); } static __always_inline void allow_read_write_user(void __user *to, const void __user *from, unsigned long size) { - allow_user_access(to, from, size, KUAP_READ_WRITE); + allow_user_access(to, KUAP_READ_WRITE); } static __always_inline void prevent_read_from_user(const void __user *from, unsigned long size) diff --git a/arch/powerpc/include/asm/nohash/32/kup-8xx.h b/arch/powerpc/include/asm/nohash/32/kup-8xx.h index 08486b15b207..efffb5006d19 100644 --- a/arch/powerpc/include/asm/nohash/32/kup-8xx.h +++ b/arch/powerpc/include/asm/nohash/32/kup-8xx.h @@ -49,8 +49,7 @@ static __always_inline void uaccess_end_8xx(void) "i"(SPRN_MD_AP), "r"(MD_APG_KUAP), "i"(MMU_FTR_KUAP) : "memory"); } -static __always_inline void allow_user_access(void __user *to, const void __user *from, - unsigned long size, unsigned long dir) +static __always_inline void allow_user_access(void __user *to, unsigned long dir) { uaccess_begin_8xx(MD_APG_INIT); } diff --git a/arch/powerpc/include/asm/nohash/kup-booke.h b/arch/powerpc/include/asm/nohash/kup-booke.h index d6bbb6d78bbe..cb2d5a96c3df 100644 --- a/arch/powerpc/include/asm/nohash/kup-booke.h +++ b/arch/powerpc/include/asm/nohash/kup-booke.h @@ -73,8 +73,7 @@ static __always_inline void uaccess_end_booke(void) "i"(SPRN_PID), "r"(0), "i"(MMU_FTR_KUAP) : "memory"); } -static __always_inline void allow_user_access(void __user *to, const void __user *from, - unsigned long size, unsigned long dir) +static __always_inline void allow_user_access(void __user *to, unsigned long dir) { uaccess_begin_booke(current->thread.pid); } From 5458989a0e48c2bd51a7d37a04ff6ae489fe5458 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Wed, 24 Dec 2025 12:20:51 +0100 Subject: [PATCH 03/27] powerpc/uaccess: Remove {allow/prevent}_{read/write/read_write}_{from/to/}_user() The six following functions have become simple single-line fonctions that do not have much added value anymore: - allow_read_from_user() - allow_write_to_user() - allow_read_write_user() - prevent_read_from_user() - prevent_write_to_user() - prevent_read_write_user() Directly call allow_user_access() and prevent_user_access(), it doesn't reduce the readability and it removes unnecessary middle functions. Signed-off-by: Christophe Leroy Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/70971f0ba81eab742a120e5bfdeff6b42d08fd98.1766574657.git.chleroy@kernel.org --- arch/powerpc/include/asm/kup.h | 47 ------------------------------ arch/powerpc/include/asm/uaccess.h | 30 +++++++++---------- 2 files changed, 15 insertions(+), 62 deletions(-) diff --git a/arch/powerpc/include/asm/kup.h b/arch/powerpc/include/asm/kup.h index 3963584ac1cf..4a4145a244f2 100644 --- a/arch/powerpc/include/asm/kup.h +++ b/arch/powerpc/include/asm/kup.h @@ -131,53 +131,6 @@ static __always_inline void kuap_assert_locked(void) kuap_get_and_assert_locked(); } -static __always_inline void allow_read_from_user(const void __user *from, unsigned long size) -{ - allow_user_access(NULL, KUAP_READ); -} - -static __always_inline void allow_write_to_user(void __user *to, unsigned long size) -{ - allow_user_access(to, KUAP_WRITE); -} - -static __always_inline void allow_read_write_user(void __user *to, const void __user *from, - unsigned long size) -{ - allow_user_access(to, KUAP_READ_WRITE); -} - -static __always_inline void prevent_read_from_user(const void __user *from, unsigned long size) -{ - prevent_user_access(KUAP_READ); -} - -static __always_inline void prevent_write_to_user(void __user *to, unsigned long size) -{ - prevent_user_access(KUAP_WRITE); -} - -static __always_inline void prevent_read_write_user(void __user *to, const void __user *from, - unsigned long size) -{ - prevent_user_access(KUAP_READ_WRITE); -} - -static __always_inline void prevent_current_access_user(void) -{ - prevent_user_access(KUAP_READ_WRITE); -} - -static __always_inline void prevent_current_read_from_user(void) -{ - prevent_user_access(KUAP_READ); -} - -static __always_inline void prevent_current_write_to_user(void) -{ - prevent_user_access(KUAP_WRITE); -} - #endif /* !__ASSEMBLER__ */ #endif /* _ASM_POWERPC_KUAP_H_ */ diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 3e622e647d62..7846ee59e374 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -45,14 +45,14 @@ do { \ __label__ __pu_failed; \ \ - allow_write_to_user(__pu_addr, __pu_size); \ + allow_user_access(__pu_addr, KUAP_WRITE); \ __put_user_size_goto(__pu_val, __pu_addr, __pu_size, __pu_failed); \ - prevent_write_to_user(__pu_addr, __pu_size); \ + prevent_user_access(KUAP_WRITE); \ __pu_err = 0; \ break; \ \ __pu_failed: \ - prevent_write_to_user(__pu_addr, __pu_size); \ + prevent_user_access(KUAP_WRITE); \ __pu_err = -EFAULT; \ } while (0); \ \ @@ -302,9 +302,9 @@ do { \ \ might_fault(); \ barrier_nospec(); \ - allow_read_from_user(__gu_addr, __gu_size); \ + allow_user_access(NULL, KUAP_READ); \ __get_user_size_allowed(__gu_val, __gu_addr, __gu_size, __gu_err); \ - prevent_read_from_user(__gu_addr, __gu_size); \ + prevent_user_access(KUAP_READ); \ (x) = (__typeof__(*(ptr)))__gu_val; \ \ __gu_err; \ @@ -331,9 +331,9 @@ raw_copy_in_user(void __user *to, const void __user *from, unsigned long n) unsigned long ret; barrier_nospec(); - allow_read_write_user(to, from, n); + allow_user_access(to, KUAP_READ_WRITE); ret = __copy_tofrom_user(to, from, n); - prevent_read_write_user(to, from, n); + prevent_user_access(KUAP_READ_WRITE); return ret; } #endif /* __powerpc64__ */ @@ -343,9 +343,9 @@ static inline unsigned long raw_copy_from_user(void *to, { unsigned long ret; - allow_read_from_user(from, n); + allow_user_access(NULL, KUAP_READ); ret = __copy_tofrom_user((__force void __user *)to, from, n); - prevent_read_from_user(from, n); + prevent_user_access(KUAP_READ); return ret; } @@ -354,9 +354,9 @@ raw_copy_to_user(void __user *to, const void *from, unsigned long n) { unsigned long ret; - allow_write_to_user(to, n); + allow_user_access(to, KUAP_WRITE); ret = __copy_tofrom_user(to, (__force const void __user *)from, n); - prevent_write_to_user(to, n); + prevent_user_access(KUAP_WRITE); return ret; } @@ -367,9 +367,9 @@ static inline unsigned long __clear_user(void __user *addr, unsigned long size) unsigned long ret; might_fault(); - allow_write_to_user(addr, size); + allow_user_access(addr, KUAP_WRITE); ret = __arch_clear_user(addr, size); - prevent_write_to_user(addr, size); + prevent_user_access(KUAP_WRITE); return ret; } @@ -397,9 +397,9 @@ copy_mc_to_user(void __user *to, const void *from, unsigned long n) { if (check_copy_size(from, n, true)) { if (access_ok(to, n)) { - allow_write_to_user(to, n); + allow_user_access(to, KUAP_WRITE); n = copy_mc_generic((void __force *)to, from, n); - prevent_write_to_user(to, n); + prevent_user_access(KUAP_WRITE); } } From 704f4300315396af94e26ec0127ff0cd3fd309bf Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Wed, 24 Dec 2025 12:20:52 +0100 Subject: [PATCH 04/27] powerpc/uaccess: Refactor user_{read/write/}_access_begin() user_read_access_begin() and user_write_access_begin() and user_access_begin() are now very similar. Create a common __user_access_begin() that takes direction as parameter. In order to avoid a warning with the conditional call of barrier_nospec() which is sometimes an empty macro, change it to a do {} while (0). Signed-off-by: Christophe Leroy Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/2b4f9d4e521e0b56bf5cb239916b4a178c4d2007.1766574657.git.chleroy@kernel.org --- arch/powerpc/include/asm/barrier.h | 2 +- arch/powerpc/include/asm/uaccess.h | 48 +++++++++--------------------- 2 files changed, 15 insertions(+), 35 deletions(-) diff --git a/arch/powerpc/include/asm/barrier.h b/arch/powerpc/include/asm/barrier.h index 9e9833faa4af..9d2f612cfb1d 100644 --- a/arch/powerpc/include/asm/barrier.h +++ b/arch/powerpc/include/asm/barrier.h @@ -102,7 +102,7 @@ do { \ #else /* !CONFIG_PPC_BARRIER_NOSPEC */ #define barrier_nospec_asm -#define barrier_nospec() +#define barrier_nospec() do {} while (0) #endif /* CONFIG_PPC_BARRIER_NOSPEC */ /* diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 7846ee59e374..721d65dbbb2e 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -410,51 +410,31 @@ copy_mc_to_user(void __user *to, const void *from, unsigned long n) extern long __copy_from_user_flushcache(void *dst, const void __user *src, unsigned size); -static __must_check __always_inline bool user_access_begin(const void __user *ptr, size_t len) +static __must_check __always_inline bool __user_access_begin(const void __user *ptr, size_t len, + unsigned long dir) { if (unlikely(!access_ok(ptr, len))) return false; might_fault(); - barrier_nospec(); - allow_read_write_user((void __user *)ptr, ptr, len); + if (dir & KUAP_READ) + barrier_nospec(); + allow_user_access((void __user *)ptr, dir); return true; } -#define user_access_begin user_access_begin -#define user_access_end prevent_current_access_user + +#define user_access_begin(p, l) __user_access_begin(p, l, KUAP_READ_WRITE) +#define user_read_access_begin(p, l) __user_access_begin(p, l, KUAP_READ) +#define user_write_access_begin(p, l) __user_access_begin(p, l, KUAP_WRITE) + +#define user_access_end() prevent_user_access(KUAP_READ_WRITE) +#define user_read_access_end() prevent_user_access(KUAP_READ) +#define user_write_access_end() prevent_user_access(KUAP_WRITE) + #define user_access_save prevent_user_access_return #define user_access_restore restore_user_access -static __must_check __always_inline bool -user_read_access_begin(const void __user *ptr, size_t len) -{ - if (unlikely(!access_ok(ptr, len))) - return false; - - might_fault(); - - barrier_nospec(); - allow_read_from_user(ptr, len); - return true; -} -#define user_read_access_begin user_read_access_begin -#define user_read_access_end prevent_current_read_from_user - -static __must_check __always_inline bool -user_write_access_begin(const void __user *ptr, size_t len) -{ - if (unlikely(!access_ok(ptr, len))) - return false; - - might_fault(); - - allow_write_to_user((void __user *)ptr, len); - return true; -} -#define user_write_access_begin user_write_access_begin -#define user_write_access_end prevent_current_write_to_user - #define arch_unsafe_get_user(x, p, e) do { \ __long_type(*(p)) __gu_val; \ __typeof__(*(p)) __user *__gu_addr = (p); \ From fb7903771c107b0358584a359b8545060e23c530 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Wed, 24 Dec 2025 12:20:53 +0100 Subject: [PATCH 05/27] powerpc/32s: Fix segments setup when TASK_SIZE is not a multiple of 256M For book3s/32 it is assumed that TASK_SIZE is a multiple of 256 Mbytes, but Kconfig allows any value for TASK_SIZE. In all relevant calculations, align TASK_SIZE to the upper 256 Mbytes boundary. Also use ASM_CONST() in the definition of TASK_SIZE to ensure it is seen as an unsigned constant. Signed-off-by: Christophe Leroy Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/8928d906079e156c59794c41e826a684eaaaebb4.1766574657.git.chleroy@kernel.org --- arch/powerpc/include/asm/book3s/32/mmu-hash.h | 5 ++++- arch/powerpc/include/asm/task_size_32.h | 2 +- arch/powerpc/kernel/asm-offsets.c | 2 +- arch/powerpc/kernel/head_book3s_32.S | 6 +++--- arch/powerpc/mm/book3s32/mmu.c | 2 +- arch/powerpc/mm/ptdump/segment_regs.c | 2 +- 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/arch/powerpc/include/asm/book3s/32/mmu-hash.h b/arch/powerpc/include/asm/book3s/32/mmu-hash.h index 8435bf3cdabf..387d370c8a35 100644 --- a/arch/powerpc/include/asm/book3s/32/mmu-hash.h +++ b/arch/powerpc/include/asm/book3s/32/mmu-hash.h @@ -192,12 +192,15 @@ extern s32 patch__hash_page_B, patch__hash_page_C; extern s32 patch__flush_hash_A0, patch__flush_hash_A1, patch__flush_hash_A2; extern s32 patch__flush_hash_B; +#include +#include + #include #include static __always_inline void update_user_segment(u32 n, u32 val) { - if (n << 28 < TASK_SIZE) + if (n << 28 < ALIGN(TASK_SIZE, SZ_256M)) mtsr(val + n * 0x111, n << 28); } diff --git a/arch/powerpc/include/asm/task_size_32.h b/arch/powerpc/include/asm/task_size_32.h index de7290ee770f..30edc21f71fb 100644 --- a/arch/powerpc/include/asm/task_size_32.h +++ b/arch/powerpc/include/asm/task_size_32.h @@ -6,7 +6,7 @@ #error User TASK_SIZE overlaps with KERNEL_START address #endif -#define TASK_SIZE (CONFIG_TASK_SIZE) +#define TASK_SIZE ASM_CONST(CONFIG_TASK_SIZE) /* * This decides where the kernel will search for a free chunk of vm space during diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index a4bc80b30410..46149f326fd4 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -331,7 +331,7 @@ int main(void) #ifndef CONFIG_PPC64 DEFINE(TASK_SIZE, TASK_SIZE); - DEFINE(NUM_USER_SEGMENTS, TASK_SIZE>>28); + DEFINE(NUM_USER_SEGMENTS, ALIGN(TASK_SIZE, SZ_256M) >> 28); #endif /* ! CONFIG_PPC64 */ /* datapage offsets for use by vdso */ diff --git a/arch/powerpc/kernel/head_book3s_32.S b/arch/powerpc/kernel/head_book3s_32.S index cb2bca76be53..c1779455ea32 100644 --- a/arch/powerpc/kernel/head_book3s_32.S +++ b/arch/powerpc/kernel/head_book3s_32.S @@ -420,7 +420,7 @@ InstructionTLBMiss: lwz r2,0(r2) /* get pmd entry */ #ifdef CONFIG_EXECMEM rlwinm r3, r0, 4, 0xf - subi r3, r3, (TASK_SIZE >> 28) & 0xf + subi r3, r3, NUM_USER_SEGMENTS #endif rlwinm. r2,r2,0,0,19 /* extract address of pte page */ beq- InstructionAddressInvalid /* return if no mapping */ @@ -475,7 +475,7 @@ DataLoadTLBMiss: lwz r2,0(r1) /* get pmd entry */ rlwinm r3, r0, 4, 0xf rlwinm. r2,r2,0,0,19 /* extract address of pte page */ - subi r3, r3, (TASK_SIZE >> 28) & 0xf + subi r3, r3, NUM_USER_SEGMENTS beq- 2f /* bail if no mapping */ 1: rlwimi r2,r0,22,20,29 /* insert next 10 bits of address */ lwz r2,0(r2) /* get linux-style pte */ @@ -554,7 +554,7 @@ DataStoreTLBMiss: lwz r2,0(r1) /* get pmd entry */ rlwinm r3, r0, 4, 0xf rlwinm. r2,r2,0,0,19 /* extract address of pte page */ - subi r3, r3, (TASK_SIZE >> 28) & 0xf + subi r3, r3, NUM_USER_SEGMENTS beq- 2f /* bail if no mapping */ 1: rlwimi r2,r0,22,20,29 /* insert next 10 bits of address */ diff --git a/arch/powerpc/mm/book3s32/mmu.c b/arch/powerpc/mm/book3s32/mmu.c index c42ecdf94e48..37eefc6786a7 100644 --- a/arch/powerpc/mm/book3s32/mmu.c +++ b/arch/powerpc/mm/book3s32/mmu.c @@ -225,7 +225,7 @@ int mmu_mark_initmem_nx(void) BUILD_BUG_ON(ALIGN_DOWN(MODULES_VADDR, SZ_256M) < TASK_SIZE); - for (i = TASK_SIZE >> 28; i < 16; i++) { + for (i = ALIGN(TASK_SIZE, SZ_256M) >> 28; i < 16; i++) { /* Do not set NX on VM space for modules */ if (is_module_segment(i << 28)) continue; diff --git a/arch/powerpc/mm/ptdump/segment_regs.c b/arch/powerpc/mm/ptdump/segment_regs.c index 9df3af8d481f..c06704b18a2c 100644 --- a/arch/powerpc/mm/ptdump/segment_regs.c +++ b/arch/powerpc/mm/ptdump/segment_regs.c @@ -31,7 +31,7 @@ static int sr_show(struct seq_file *m, void *v) int i; seq_puts(m, "---[ User Segments ]---\n"); - for (i = 0; i < TASK_SIZE >> 28; i++) + for (i = 0; i < ALIGN(TASK_SIZE, SZ_256M) >> 28; i++) seg_show(m, i); seq_puts(m, "\n---[ Kernel Segments ]---\n"); From 370d841929c3b863b7409047e8c84eabc4d0960f Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Wed, 24 Dec 2025 12:20:54 +0100 Subject: [PATCH 06/27] powerpc/32: Automatically adapt TASK_SIZE based on constraints At the time being, TASK_SIZE can be customized by the user via Kconfig but it is not possible to check all constraints in Kconfig. Impossible setups are detected at compile time with BUILD_BUG() but that leads to build failure when setting crazy values. It is not a problem on its own because the user will usually either use the default value or set a well thought value. However build robots generate crazy random configs that lead to build failures, and build robots see it as a regression every time a patch adds such a constraint. So instead of failing the build when the custom TASK_SIZE is too big, just adjust it to the maximum possible value matching the setup. Several architectures already calculate TASK_SIZE based on other parameters and options. In order to do so, move MODULES_VADDR calculation into task_size_32.h and ensure that: - On book3s/32, userspace and module area have their own segments (256M) - On 8xx, userspace has its own full PGDIR entries (4M) Then TASK_SIZE is guaranteed to be correct so remove related BUILD_BUG()s. Signed-off-by: Christophe Leroy Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/6a2575420770d075cd090b5a316730a2ffafdee4.1766574657.git.chleroy@kernel.org --- arch/powerpc/Kconfig | 3 +-- arch/powerpc/include/asm/book3s/32/pgtable.h | 4 --- arch/powerpc/include/asm/nohash/32/mmu-8xx.h | 4 --- arch/powerpc/include/asm/task_size_32.h | 26 ++++++++++++++++++++ arch/powerpc/mm/book3s32/mmu.c | 2 -- arch/powerpc/mm/mem.c | 2 -- arch/powerpc/mm/nohash/8xx.c | 2 -- 7 files changed, 27 insertions(+), 16 deletions(-) diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 9537a61ebae0..b8d36a261009 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -1293,9 +1293,8 @@ config TASK_SIZE_BOOL Say N here unless you know what you are doing. config TASK_SIZE - hex "Size of user task space" if TASK_SIZE_BOOL + hex "Size of maximum user task space" if TASK_SIZE_BOOL default "0x80000000" if PPC_8xx - default "0xb0000000" if PPC_BOOK3S_32 && EXECMEM default "0xc0000000" config MODULES_SIZE_BOOL diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h index 87dcca962be7..41ae404d0b7a 100644 --- a/arch/powerpc/include/asm/book3s/32/pgtable.h +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h @@ -195,10 +195,6 @@ void unmap_kernel_page(unsigned long va); #define VMALLOC_END ioremap_bot #endif -#define MODULES_END ALIGN_DOWN(PAGE_OFFSET, SZ_256M) -#define MODULES_SIZE (CONFIG_MODULES_SIZE * SZ_1M) -#define MODULES_VADDR (MODULES_END - MODULES_SIZE) - #ifndef __ASSEMBLER__ #include #include diff --git a/arch/powerpc/include/asm/nohash/32/mmu-8xx.h b/arch/powerpc/include/asm/nohash/32/mmu-8xx.h index f19115db8072..74ad32e1588c 100644 --- a/arch/powerpc/include/asm/nohash/32/mmu-8xx.h +++ b/arch/powerpc/include/asm/nohash/32/mmu-8xx.h @@ -170,10 +170,6 @@ #define mmu_linear_psize MMU_PAGE_8M -#define MODULES_END PAGE_OFFSET -#define MODULES_SIZE (CONFIG_MODULES_SIZE * SZ_1M) -#define MODULES_VADDR (MODULES_END - MODULES_SIZE) - #ifndef __ASSEMBLER__ #include diff --git a/arch/powerpc/include/asm/task_size_32.h b/arch/powerpc/include/asm/task_size_32.h index 30edc21f71fb..42a64bbd1964 100644 --- a/arch/powerpc/include/asm/task_size_32.h +++ b/arch/powerpc/include/asm/task_size_32.h @@ -2,11 +2,37 @@ #ifndef _ASM_POWERPC_TASK_SIZE_32_H #define _ASM_POWERPC_TASK_SIZE_32_H +#include + #if CONFIG_TASK_SIZE > CONFIG_KERNEL_START #error User TASK_SIZE overlaps with KERNEL_START address #endif +#ifdef CONFIG_PPC_8xx +#define MODULES_END ASM_CONST(CONFIG_PAGE_OFFSET) +#define MODULES_SIZE (CONFIG_MODULES_SIZE * SZ_1M) +#define MODULES_VADDR (MODULES_END - MODULES_SIZE) +#define MODULES_BASE (MODULES_VADDR & ~(UL(SZ_4M) - 1)) +#define USER_TOP MODULES_BASE +#endif + +#ifdef CONFIG_PPC_BOOK3S_32 +#define MODULES_END (ASM_CONST(CONFIG_PAGE_OFFSET) & ~(UL(SZ_256M) - 1)) +#define MODULES_SIZE (CONFIG_MODULES_SIZE * SZ_1M) +#define MODULES_VADDR (MODULES_END - MODULES_SIZE) +#define MODULES_BASE (MODULES_VADDR & ~(UL(SZ_256M) - 1)) +#define USER_TOP MODULES_BASE +#endif + +#ifndef USER_TOP +#define USER_TOP ASM_CONST(CONFIG_PAGE_OFFSET) +#endif + +#if CONFIG_TASK_SIZE < USER_TOP #define TASK_SIZE ASM_CONST(CONFIG_TASK_SIZE) +#else +#define TASK_SIZE USER_TOP +#endif /* * This decides where the kernel will search for a free chunk of vm space during diff --git a/arch/powerpc/mm/book3s32/mmu.c b/arch/powerpc/mm/book3s32/mmu.c index 37eefc6786a7..07660e8badbd 100644 --- a/arch/powerpc/mm/book3s32/mmu.c +++ b/arch/powerpc/mm/book3s32/mmu.c @@ -223,8 +223,6 @@ int mmu_mark_initmem_nx(void) update_bats(); - BUILD_BUG_ON(ALIGN_DOWN(MODULES_VADDR, SZ_256M) < TASK_SIZE); - for (i = ALIGN(TASK_SIZE, SZ_256M) >> 28; i < 16; i++) { /* Do not set NX on VM space for modules */ if (is_module_segment(i << 28)) diff --git a/arch/powerpc/mm/mem.c b/arch/powerpc/mm/mem.c index 3ddbfdbfa941..bc0f1a9eb0bc 100644 --- a/arch/powerpc/mm/mem.c +++ b/arch/powerpc/mm/mem.c @@ -401,8 +401,6 @@ struct execmem_info __init *execmem_arch_setup(void) #ifdef MODULES_VADDR unsigned long limit = (unsigned long)_etext - SZ_32M; - BUILD_BUG_ON(TASK_SIZE > MODULES_VADDR); - /* First try within 32M limit from _etext to avoid branch trampolines */ if (MODULES_VADDR < PAGE_OFFSET && MODULES_END > limit) { start = limit; diff --git a/arch/powerpc/mm/nohash/8xx.c b/arch/powerpc/mm/nohash/8xx.c index ab1505cf42bf..a9d3f4729ead 100644 --- a/arch/powerpc/mm/nohash/8xx.c +++ b/arch/powerpc/mm/nohash/8xx.c @@ -209,8 +209,6 @@ void __init setup_initial_memory_limit(phys_addr_t first_memblock_base, /* 8xx can only access 32MB at the moment */ memblock_set_current_limit(min_t(u64, first_memblock_size, SZ_32M)); - - BUILD_BUG_ON(ALIGN_DOWN(MODULES_VADDR, PGDIR_SIZE) < TASK_SIZE); } int pud_clear_huge(pud_t *pud) From 861574d51bbd7a06dbf07f658a9a8def012c2f74 Mon Sep 17 00:00:00 2001 From: Christophe Leroy Date: Wed, 24 Dec 2025 12:20:55 +0100 Subject: [PATCH 07/27] powerpc/uaccess: Implement masked user access Masked user access avoids the address/size verification by access_ok(). Allthough its main purpose is to skip the speculation in the verification of user address and size hence avoid the need of spec mitigation, it also has the advantage of reducing the amount of instructions required so it even benefits to platforms that don't need speculation mitigation, especially when the size of the copy is not know at build time. So implement masked user access on powerpc. The only requirement is to have memory gap that faults between the top user space and the real start of kernel area. On 64 bits platforms the address space is divided that way: 0xffffffffffffffff +------------------+ | | | kernel space | | | 0xc000000000000000 +------------------+ <== PAGE_OFFSET |//////////////////| |//////////////////| 0x8000000000000000 |//////////////////| |//////////////////| |//////////////////| 0x0010000000000000 +------------------+ <== TASK_SIZE_MAX | | | user space | | | 0x0000000000000000 +------------------+ Kernel is always above 0x8000000000000000 and user always below, with a gap in-between. It leads to a 3 instructions sequence: 150: 7c 69 fe 76 sradi r9,r3,63 154: 79 29 00 40 clrldi r9,r9,1 158: 7c 63 48 78 andc r3,r3,r9 This sequence leaves r3 unmodified when it is below 0x8000000000000000 and clamps it to 0x8000000000000000 if it is above. On 32 bits it is more tricky. In theory user space can go up to 0xbfffffff while kernel will usually start at 0xc0000000. So a gap needs to be added in-between. Allthough in theory a single 4k page would suffice, it is easier and more efficient to enforce a 128k gap below kernel, as it simplifies the masking. e500 has the isel instruction which allows selecting one value or the other without branch and that instruction is not speculative, so use it. Allthough GCC usually generates code using that instruction, it is safer to use inline assembly to be sure. The result is: 14: 3d 20 bf fe lis r9,-16386 18: 7c 03 48 40 cmplw r3,r9 1c: 7c 69 18 5e iselgt r3,r9,r3 On other ones, when kernel space is over 0x80000000 and user space is below, the logic in mask_user_address_simple() leads to a 3 instruction sequence: 64: 7c 69 fe 70 srawi r9,r3,31 68: 55 29 00 7e clrlwi r9,r9,1 6c: 7c 63 48 78 andc r3,r3,r9 This is the default on powerpc 8xx. When the limit between user space and kernel space is not 0x80000000, mask_user_address_32() is used and a 6 instructions sequence is generated: 24: 54 69 7c 7e srwi r9,r3,17 28: 21 29 57 ff subfic r9,r9,22527 2c: 7d 29 fe 70 srawi r9,r9,31 30: 75 2a b0 00 andis. r10,r9,45056 34: 7c 63 48 78 andc r3,r3,r9 38: 7c 63 53 78 or r3,r3,r10 The constraint is that TASK_SIZE be aligned to 128K in order to get the most optimal number of instructions. When CONFIG_PPC_BARRIER_NOSPEC is not defined, fallback on the test-based masking as it is quicker than the 6 instructions sequence but not quicker than the 3 instructions sequences above. As an exemple, allthough barrier_nospec() voids on the 8xx, this change has the following impact on strncpy_from_user(): the length of the function is reduced from 488 to 340 bytes: Start of the function with the patch: 00000000 : 0: 7c ab 2b 79 mr. r11,r5 4: 40 81 01 40 ble 144 8: 7c 89 fe 70 srawi r9,r4,31 c: 55 29 00 7e clrlwi r9,r9,1 10: 7c 84 48 78 andc r4,r4,r9 14: 3d 20 dc 00 lis r9,-9216 18: 7d 3a c3 a6 mtspr 794,r9 1c: 2f 8b 00 03 cmpwi cr7,r11,3 20: 40 9d 00 b4 ble cr7,d4 ... Start of the function without the patch: 00000000 : 0: 7c a0 2b 79 mr. r0,r5 4: 40 81 01 10 ble 114 8: 2f 84 00 00 cmpwi cr7,r4,0 c: 41 9c 01 30 blt cr7,13c 10: 3d 20 80 00 lis r9,-32768 14: 7d 24 48 50 subf r9,r4,r9 18: 7f 80 48 40 cmplw cr7,r0,r9 1c: 7c 05 03 78 mr r5,r0 20: 41 9d 01 00 bgt cr7,120 24: 3d 20 80 00 lis r9,-32768 28: 7d 25 48 50 subf r9,r5,r9 2c: 7f 84 48 40 cmplw cr7,r4,r9 30: 38 e0 ff f2 li r7,-14 34: 41 9d 00 e4 bgt cr7,118 38: 94 21 ff e0 stwu r1,-32(r1) 3c: 3d 20 dc 00 lis r9,-9216 40: 7d 3a c3 a6 mtspr 794,r9 44: 2b 85 00 03 cmplwi cr7,r5,3 48: 40 9d 01 6c ble cr7,1b4 ... 118: 7c e3 3b 78 mr r3,r7 11c: 4e 80 00 20 blr 120: 7d 25 4b 78 mr r5,r9 124: 3d 20 80 00 lis r9,-32768 128: 7d 25 48 50 subf r9,r5,r9 12c: 7f 84 48 40 cmplw cr7,r4,r9 130: 38 e0 ff f2 li r7,-14 134: 41 bd ff e4 bgt cr7,118 138: 4b ff ff 00 b 38 13c: 38 e0 ff f2 li r7,-14 140: 4b ff ff d8 b 118 ... Signed-off-by: Christophe Leroy Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/8f418183d9125cc0bf23922bc2ef2a1130d8b63a.1766574657.git.chleroy@kernel.org --- arch/powerpc/include/asm/task_size_32.h | 6 +- arch/powerpc/include/asm/uaccess.h | 76 +++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 3 deletions(-) diff --git a/arch/powerpc/include/asm/task_size_32.h b/arch/powerpc/include/asm/task_size_32.h index 42a64bbd1964..725ddbf06217 100644 --- a/arch/powerpc/include/asm/task_size_32.h +++ b/arch/powerpc/include/asm/task_size_32.h @@ -13,7 +13,7 @@ #define MODULES_SIZE (CONFIG_MODULES_SIZE * SZ_1M) #define MODULES_VADDR (MODULES_END - MODULES_SIZE) #define MODULES_BASE (MODULES_VADDR & ~(UL(SZ_4M) - 1)) -#define USER_TOP MODULES_BASE +#define USER_TOP (MODULES_BASE - SZ_4M) #endif #ifdef CONFIG_PPC_BOOK3S_32 @@ -21,11 +21,11 @@ #define MODULES_SIZE (CONFIG_MODULES_SIZE * SZ_1M) #define MODULES_VADDR (MODULES_END - MODULES_SIZE) #define MODULES_BASE (MODULES_VADDR & ~(UL(SZ_256M) - 1)) -#define USER_TOP MODULES_BASE +#define USER_TOP (MODULES_BASE - SZ_4M) #endif #ifndef USER_TOP -#define USER_TOP ASM_CONST(CONFIG_PAGE_OFFSET) +#define USER_TOP ((ASM_CONST(CONFIG_PAGE_OFFSET) - SZ_128K) & ~(UL(SZ_128K) - 1)) #endif #if CONFIG_TASK_SIZE < USER_TOP diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 721d65dbbb2e..ba1d878c3f40 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -2,6 +2,8 @@ #ifndef _ARCH_POWERPC_UACCESS_H #define _ARCH_POWERPC_UACCESS_H +#include + #include #include #include @@ -435,6 +437,80 @@ static __must_check __always_inline bool __user_access_begin(const void __user * #define user_access_save prevent_user_access_return #define user_access_restore restore_user_access +/* + * Masking the user address is an alternative to a conditional + * user_access_begin that can avoid the fencing. This only works + * for dense accesses starting at the address. + */ +static inline void __user *mask_user_address_simple(const void __user *ptr) +{ + unsigned long addr = (unsigned long)ptr; + unsigned long mask = (unsigned long)(((long)addr >> (BITS_PER_LONG - 1)) & LONG_MAX); + + return (void __user *)(addr & ~mask); +} + +static inline void __user *mask_user_address_isel(const void __user *ptr) +{ + unsigned long addr; + + asm("cmplw %1, %2; iselgt %0, %2, %1" : "=r"(addr) : "r"(ptr), "r"(TASK_SIZE) : "cr0"); + + return (void __user *)addr; +} + +/* TASK_SIZE is a multiple of 128K for shifting by 17 to the right */ +static inline void __user *mask_user_address_32(const void __user *ptr) +{ + unsigned long addr = (unsigned long)ptr; + unsigned long mask = (unsigned long)((long)((TASK_SIZE >> 17) - 1 - (addr >> 17)) >> 31); + + addr = (addr & ~mask) | (TASK_SIZE & mask); + + return (void __user *)addr; +} + +static inline void __user *mask_user_address_fallback(const void __user *ptr) +{ + unsigned long addr = (unsigned long)ptr; + + return (void __user *)(likely(addr < TASK_SIZE) ? addr : TASK_SIZE); +} + +static inline void __user *mask_user_address(const void __user *ptr) +{ +#ifdef MODULES_VADDR + const unsigned long border = MODULES_VADDR; +#else + const unsigned long border = PAGE_OFFSET; +#endif + + if (IS_ENABLED(CONFIG_PPC64)) + return mask_user_address_simple(ptr); + if (IS_ENABLED(CONFIG_E500)) + return mask_user_address_isel(ptr); + if (TASK_SIZE <= UL(SZ_2G) && border >= UL(SZ_2G)) + return mask_user_address_simple(ptr); + if (IS_ENABLED(CONFIG_PPC_BARRIER_NOSPEC)) + return mask_user_address_32(ptr); + return mask_user_address_fallback(ptr); +} + +static __always_inline void __user *__masked_user_access_begin(const void __user *p, + unsigned long dir) +{ + void __user *ptr = mask_user_address(p); + + might_fault(); + allow_user_access(ptr, dir); + + return ptr; +} + +#define masked_user_access_begin(p) __masked_user_access_begin(p, KUAP_READ_WRITE) +#define masked_user_read_access_begin(p) __masked_user_access_begin(p, KUAP_READ) +#define masked_user_write_access_begin(p) __masked_user_access_begin(p, KUAP_WRITE) + #define arch_unsafe_get_user(x, p, e) do { \ __long_type(*(p)) __gu_val; \ __typeof__(*(p)) __user *__gu_addr = (p); \ From 58a075adf3fb1701435d29e0113284dc31d6e43e Mon Sep 17 00:00:00 2001 From: Saket Kumar Bhaskar Date: Wed, 10 Dec 2025 12:20:32 +0530 Subject: [PATCH 08/27] powerpc64/bpf: Support internal-only MOV instruction to resolve per-CPU addrs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the introduction of commit 7bdbf7446305 ("bpf: add special internal-only MOV instruction to resolve per-CPU addrs"), a new BPF instruction BPF_MOV64_PERCPU_REG has been added to resolve absolute addresses of per-CPU data from their per-CPU offsets. This update requires enabling support for this instruction in the powerpc JIT compiler. As of commit 7a0268fa1a36 ("[PATCH] powerpc/64: per cpu data optimisations"), the per-CPU data offset for the CPU is stored in the paca. To support this BPF instruction in the powerpc JIT, the following powerpc instructions are emitted: if (IS_ENABLED(CONFIG_SMP)) ld tmp1_reg, 48(13) //Load per-CPU data offset from paca(r13) in tmp1_reg. add dst_reg, src_reg, tmp1_reg //Add the per cpu offset to the dst. else if (src_reg != dst_reg) mr dst_reg, src_reg //Move src_reg to dst_reg, if src_reg != dst_reg To evaluate the performance improvements introduced by this change, the benchmark described in [1] was employed. Before Change: glob-arr-inc : 41.580 ± 0.034M/s arr-inc : 39.592 ± 0.055M/s hash-inc : 25.873 ± 0.012M/s After Change: glob-arr-inc : 42.024 ± 0.049M/s arr-inc : 55.447 ± 0.031M/s hash-inc : 26.565 ± 0.014M/s [1] https://github.com/anakryiko/linux/commit/8dec900975ef Reviewed-by: Puranjay Mohan Signed-off-by: Saket Kumar Bhaskar Acked-by: Hari Bathini Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/667fdaa19c1564141f6cd82e75b2be86a42c0f96.1765343385.git.skb99@linux.ibm.com --- arch/powerpc/net/bpf_jit_comp.c | 5 +++++ arch/powerpc/net/bpf_jit_comp64.c | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 5e976730b2f5..d53e9cd7563f 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -466,6 +466,11 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena) return true; } +bool bpf_jit_supports_percpu_insn(void) +{ + return IS_ENABLED(CONFIG_PPC64); +} + void *arch_alloc_bpf_trampoline(unsigned int size) { return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns); diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 1fe37128c876..37723ee9344e 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -918,6 +918,16 @@ int bpf_jit_build_body(struct bpf_prog *fp, u32 *image, u32 *fimage, struct code case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */ case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */ + if (insn_is_mov_percpu_addr(&insn[i])) { + if (IS_ENABLED(CONFIG_SMP)) { + EMIT(PPC_RAW_LD(tmp1_reg, _R13, offsetof(struct paca_struct, data_offset))); + EMIT(PPC_RAW_ADD(dst_reg, src_reg, tmp1_reg)); + } else if (src_reg != dst_reg) { + EMIT(PPC_RAW_MR(dst_reg, src_reg)); + } + break; + } + if (insn_is_cast_user(&insn[i])) { EMIT(PPC_RAW_RLDICL_DOT(tmp1_reg, src_reg, 0, 32)); PPC_LI64(dst_reg, (ctx->user_vm_start & 0xffffffff00000000UL)); From 20ab1d11265350a9ea3d315d6c70170b8198f329 Mon Sep 17 00:00:00 2001 From: Saket Kumar Bhaskar Date: Wed, 10 Dec 2025 12:20:33 +0530 Subject: [PATCH 09/27] powerpc64/bpf: Inline bpf_get_smp_processor_id() and bpf_get_current_task/_btf() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Inline the calls to bpf_get_smp_processor_id() and bpf_get_current_task/_btf() in the powerpc bpf jit. powerpc saves the Logical processor number (paca_index) and pointer to current task (__current) in paca. Here is how the powerpc JITed assembly changes after this commit: Before: cpu = bpf_get_smp_processor_id(); addis 12, 2, -517 addi 12, 12, -29456 mtctr 12 bctrl mr 8, 3 After: cpu = bpf_get_smp_processor_id(); lhz 8, 8(13) To evaluate the performance improvements introduced by this change, the benchmark described in [1] was employed. +---------------+-------------------+-------------------+--------------+ | Name | Before | After | % change | |---------------+-------------------+-------------------+--------------| | glob-arr-inc | 40.701 ± 0.008M/s | 55.207 ± 0.021M/s | + 35.64% | | arr-inc | 39.401 ± 0.007M/s | 56.275 ± 0.023M/s | + 42.42% | | hash-inc | 24.944 ± 0.004M/s | 26.212 ± 0.003M/s | + 5.08% | +---------------+-------------------+-------------------+--------------+ [1] https://github.com/anakryiko/linux/commit/8dec900975ef Reviewed-by: Puranjay Mohan Signed-off-by: Saket Kumar Bhaskar Acked-by: Hari Bathini Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/89abfdd6f6721fbe7897865e74f2f691e5f7824a.1765343385.git.skb99@linux.ibm.com --- arch/powerpc/net/bpf_jit_comp.c | 12 ++++++++++++ arch/powerpc/net/bpf_jit_comp64.c | 11 +++++++++++ 2 files changed, 23 insertions(+) diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index d53e9cd7563f..b243ee205885 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -471,6 +471,18 @@ bool bpf_jit_supports_percpu_insn(void) return IS_ENABLED(CONFIG_PPC64); } +bool bpf_jit_inlines_helper_call(s32 imm) +{ + switch (imm) { + case BPF_FUNC_get_smp_processor_id: + case BPF_FUNC_get_current_task: + case BPF_FUNC_get_current_task_btf: + return true; + default: + return false; + } +} + void *arch_alloc_bpf_trampoline(unsigned int size) { return bpf_prog_pack_alloc(size, bpf_jit_fill_ill_insns); diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 37723ee9344e..6c827e7aa691 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -1400,6 +1400,17 @@ emit_clear: case BPF_JMP | BPF_CALL: ctx->seen |= SEEN_FUNC; + if (src_reg == bpf_to_ppc(BPF_REG_0)) { + if (imm == BPF_FUNC_get_smp_processor_id) { + EMIT(PPC_RAW_LHZ(src_reg, _R13, offsetof(struct paca_struct, paca_index))); + break; + } else if (imm == BPF_FUNC_get_current_task || + imm == BPF_FUNC_get_current_task_btf) { + EMIT(PPC_RAW_LD(src_reg, _R13, offsetof(struct paca_struct, __current))); + break; + } + } + ret = bpf_jit_get_func_addr(fp, &insn[i], extra_pass, &func_addr, &func_addr_fixed); if (ret < 0) From 1471c517cf7dae1a6342fb821d8ed501af956dd0 Mon Sep 17 00:00:00 2001 From: Gaurav Batra Date: Fri, 7 Nov 2025 10:11:05 -0600 Subject: [PATCH 10/27] powerpc/iommu: bypass DMA APIs for coherent allocations for pre-mapped memory Leverage ARCH_HAS_DMA_MAP_DIRECT config option for coherent allocations as well. This will bypass DMA ops for memory allocations that have been pre-mapped. Always set device bus_dma_limit when memory is pre-mapped. In some architectures, like PowerPC, pmemory can be converted to regular memory via daxctl command. This will gate the coherent allocations to pre-mapped RAM only, by dma_coherent_ok(). Signed-off-by: Gaurav Batra Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20251107161105.85999-1-gbatra@linux.ibm.com --- arch/powerpc/kernel/dma-iommu.c | 30 +++++++++++++++++--------- arch/powerpc/platforms/pseries/iommu.c | 6 ++---- include/linux/dma-map-ops.h | 4 ++++ kernel/dma/mapping.c | 4 ++-- 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/arch/powerpc/kernel/dma-iommu.c b/arch/powerpc/kernel/dma-iommu.c index aa3689d61917..73e10bd4d56d 100644 --- a/arch/powerpc/kernel/dma-iommu.c +++ b/arch/powerpc/kernel/dma-iommu.c @@ -65,6 +65,21 @@ bool arch_dma_unmap_sg_direct(struct device *dev, struct scatterlist *sg, return true; } +bool arch_dma_alloc_direct(struct device *dev) +{ + if (dev->dma_ops_bypass) + return true; + + return false; +} + +bool arch_dma_free_direct(struct device *dev, dma_addr_t dma_handle) +{ + if (!dev->dma_ops_bypass) + return false; + + return is_direct_handle(dev, dma_handle); +} #endif /* CONFIG_ARCH_HAS_DMA_MAP_DIRECT */ /* @@ -146,17 +161,12 @@ int dma_iommu_dma_supported(struct device *dev, u64 mask) if (dev_is_pci(dev) && dma_iommu_bypass_supported(dev, mask)) { /* - * dma_iommu_bypass_supported() sets dma_max when there is - * 1:1 mapping but it is somehow limited. - * ibm,pmemory is one example. + * fixed ops will be used for RAM. This is limited by + * bus_dma_limit which is set when RAM is pre-mapped. */ - dev->dma_ops_bypass = dev->bus_dma_limit == 0; - if (!dev->dma_ops_bypass) - dev_warn(dev, - "iommu: 64-bit OK but direct DMA is limited by %llx\n", - dev->bus_dma_limit); - else - dev_dbg(dev, "iommu: 64-bit OK, using fixed ops\n"); + dev->dma_ops_bypass = true; + dev_info(dev, "iommu: 64-bit OK but direct DMA is limited by %llx\n", + dev->bus_dma_limit); return 1; } diff --git a/arch/powerpc/platforms/pseries/iommu.c b/arch/powerpc/platforms/pseries/iommu.c index eec333dd2e59..5497b130e026 100644 --- a/arch/powerpc/platforms/pseries/iommu.c +++ b/arch/powerpc/platforms/pseries/iommu.c @@ -1769,10 +1769,8 @@ out_failed: out_unlock: mutex_unlock(&dma_win_init_mutex); - /* If we have persistent memory and the window size is not big enough - * to directly map both RAM and vPMEM, then we need to set DMA limit. - */ - if (pmem_present && direct_mapping && len != MAX_PHYSMEM_BITS) + /* For pre-mapped memory, set bus_dma_limit to the max RAM */ + if (direct_mapping) dev->dev.bus_dma_limit = dev->dev.archdata.dma_offset + (1ULL << max_ram_len); diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 4809204c674c..8eff2f53fd86 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -395,11 +395,15 @@ bool arch_dma_map_sg_direct(struct device *dev, struct scatterlist *sg, int nents); bool arch_dma_unmap_sg_direct(struct device *dev, struct scatterlist *sg, int nents); +bool arch_dma_alloc_direct(struct device *dev); +bool arch_dma_free_direct(struct device *dev, dma_addr_t dma_handle); #else #define arch_dma_map_phys_direct(d, a) (false) #define arch_dma_unmap_phys_direct(d, a) (false) #define arch_dma_map_sg_direct(d, s, n) (false) #define arch_dma_unmap_sg_direct(d, s, n) (false) +#define arch_dma_alloc_direct(d) (false) +#define arch_dma_free_direct(d, a) (false) #endif #ifdef CONFIG_ARCH_HAS_SETUP_DMA_OPS diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index 37163eb49f9f..ee29c47781e3 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -638,7 +638,7 @@ void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, /* let the implementation decide on the zone to allocate from: */ flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM); - if (dma_alloc_direct(dev, ops)) { + if (dma_alloc_direct(dev, ops) || arch_dma_alloc_direct(dev)) { cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs); } else if (use_dma_iommu(dev)) { cpu_addr = iommu_dma_alloc(dev, size, dma_handle, flag, attrs); @@ -679,7 +679,7 @@ void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, return; debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); - if (dma_alloc_direct(dev, ops)) + if (dma_alloc_direct(dev, ops) || arch_dma_free_direct(dev, dma_handle)) dma_direct_free(dev, size, cpu_addr, dma_handle, attrs); else if (use_dma_iommu(dev)) iommu_dma_free(dev, size, cpu_addr, dma_handle, attrs); From c0215e2d72debcd9cbc1c002fb012d50a3140387 Mon Sep 17 00:00:00 2001 From: Nam Cao Date: Wed, 7 Jan 2026 10:02:30 +0000 Subject: [PATCH 11/27] powerpc/pseries: Fix MSI-X allocation failure when quota is exceeded Nilay reported that since commit daaa574aba6f ("powerpc/pseries/msi: Switch to msi_create_parent_irq_domain()"), the NVMe driver cannot enable MSI-X when the device's MSI-X table size is larger than the firmware's MSI quota for the device. This is because the commit changes how rtas_prepare_msi_irqs() is called: - Before, it is called when interrupts are allocated at the global interrupt domain with nvec_in being the number of allocated interrupts. rtas_prepare_msi_irqs() can return a positive number and the allocation will be retried. - Now, it is called at the creation of per-device interrupt domain with nvec_in being the number of interrupts that the device supports. If rtas_prepare_msi_irqs() returns positive, domain creation just fails. For Nilay's NVMe driver case, rtas_prepare_msi_irqs() returns a positive number (the quota). This causes per-device interrupt domain creation to fail and thus the NVMe driver cannot enable MSI-X. Rework to make this scenario works again: - pseries_msi_ops_prepare() only prepares as many interrupts as the quota permit. - pseries_irq_domain_alloc() fails if the device's quota is exceeded. Now, if the quota is exceeded, pseries_msi_ops_prepare() will only prepare as allowed by the quota. If device drivers attempt to allocate more interrupts than the quota permits, pseries_irq_domain_alloc() will return an error code and msi_handle_pci_fail() will allow device drivers a retry. Reported-by: Nilay Shroff Closes: https://lore.kernel.org/linuxppc-dev/6af2c4c2-97f6-4758-be33-256638ef39e5@linux.ibm.com/ Fixes: daaa574aba6f ("powerpc/pseries/msi: Switch to msi_create_parent_irq_domain()") Signed-off-by: Nam Cao Cc: stable@vger.kernel.org Tested-by: Nilay Shroff Acked-by: Nilay Shroff Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20260107100230.1466093-1-namcao@linutronix.de --- arch/powerpc/platforms/pseries/msi.c | 44 ++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/arch/powerpc/platforms/pseries/msi.c b/arch/powerpc/platforms/pseries/msi.c index a82aaa786e9e..edc30cda5dbc 100644 --- a/arch/powerpc/platforms/pseries/msi.c +++ b/arch/powerpc/platforms/pseries/msi.c @@ -19,6 +19,11 @@ #include "pseries.h" +struct pseries_msi_device { + unsigned int msi_quota; + unsigned int msi_used; +}; + static int query_token, change_token; #define RTAS_QUERY_FN 0 @@ -433,8 +438,28 @@ static int pseries_msi_ops_prepare(struct irq_domain *domain, struct device *dev struct msi_domain_info *info = domain->host_data; struct pci_dev *pdev = to_pci_dev(dev); int type = (info->flags & MSI_FLAG_PCI_MSIX) ? PCI_CAP_ID_MSIX : PCI_CAP_ID_MSI; + int ret; - return rtas_prepare_msi_irqs(pdev, nvec, type, arg); + struct pseries_msi_device *pseries_dev __free(kfree) + = kmalloc(sizeof(*pseries_dev), GFP_KERNEL); + if (!pseries_dev) + return -ENOMEM; + + while (1) { + ret = rtas_prepare_msi_irqs(pdev, nvec, type, arg); + if (!ret) + break; + else if (ret > 0) + nvec = ret; + else + return ret; + } + + pseries_dev->msi_quota = nvec; + pseries_dev->msi_used = 0; + + arg->scratchpad[0].ptr = no_free_ptr(pseries_dev); + return 0; } /* @@ -443,9 +468,13 @@ static int pseries_msi_ops_prepare(struct irq_domain *domain, struct device *dev */ static void pseries_msi_ops_teardown(struct irq_domain *domain, msi_alloc_info_t *arg) { + struct pseries_msi_device *pseries_dev = arg->scratchpad[0].ptr; struct pci_dev *pdev = to_pci_dev(domain->dev); rtas_disable_msi(pdev); + + WARN_ON(pseries_dev->msi_used); + kfree(pseries_dev); } static void pseries_msi_shutdown(struct irq_data *d) @@ -546,12 +575,18 @@ static int pseries_irq_domain_alloc(struct irq_domain *domain, unsigned int virq unsigned int nr_irqs, void *arg) { struct pci_controller *phb = domain->host_data; + struct pseries_msi_device *pseries_dev; msi_alloc_info_t *info = arg; struct msi_desc *desc = info->desc; struct pci_dev *pdev = msi_desc_to_pci_dev(desc); int hwirq; int i, ret; + pseries_dev = info->scratchpad[0].ptr; + + if (pseries_dev->msi_used + nr_irqs > pseries_dev->msi_quota) + return -ENOSPC; + hwirq = rtas_query_irq_number(pci_get_pdn(pdev), desc->msi_index); if (hwirq < 0) { dev_err(&pdev->dev, "Failed to query HW IRQ: %d\n", hwirq); @@ -567,9 +602,10 @@ static int pseries_irq_domain_alloc(struct irq_domain *domain, unsigned int virq goto out; irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, - &pseries_msi_irq_chip, domain->host_data); + &pseries_msi_irq_chip, pseries_dev); } + pseries_dev->msi_used++; return 0; out: @@ -582,9 +618,11 @@ static void pseries_irq_domain_free(struct irq_domain *domain, unsigned int virq unsigned int nr_irqs) { struct irq_data *d = irq_domain_get_irq_data(domain, virq); - struct pci_controller *phb = irq_data_get_irq_chip_data(d); + struct pseries_msi_device *pseries_dev = irq_data_get_irq_chip_data(d); + struct pci_controller *phb = domain->host_data; pr_debug("%s bridge %pOF %d #%d\n", __func__, phb->dn, virq, nr_irqs); + pseries_dev->msi_used -= nr_irqs; irq_domain_free_irqs_parent(domain, virq, nr_irqs); } From 815a8d2feb5615ae7f0b5befd206af0b0160614c Mon Sep 17 00:00:00 2001 From: Narayana Murty N Date: Wed, 10 Dec 2025 08:25:59 -0600 Subject: [PATCH 12/27] powerpc/eeh: fix recursive pci_lock_rescan_remove locking in EEH event handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recent commit 1010b4c012b0 ("powerpc/eeh: Make EEH driver device hotplug safe") restructured the EEH driver to improve synchronization with the PCI hotplug layer. However, it inadvertently moved pci_lock_rescan_remove() outside its intended scope in eeh_handle_normal_event(), leading to broken PCI error reporting and improper EEH event triggering. Specifically, eeh_handle_normal_event() acquired pci_lock_rescan_remove() before calling eeh_pe_bus_get(), but eeh_pe_bus_get() itself attempts to acquire the same lock internally, causing nested locking and disrupting normal EEH event handling paths. This patch adds a boolean parameter do_lock to _eeh_pe_bus_get(), with two public wrappers: eeh_pe_bus_get() with locking enabled. eeh_pe_bus_get_nolock() that skips locking. Callers that already hold pci_lock_rescan_remove() now use eeh_pe_bus_get_nolock() to avoid recursive lock acquisition. Additionally, pci_lock_rescan_remove() calls are restored to the correct position—after eeh_pe_bus_get() and immediately before iterating affected PEs and devices. This ensures EEH-triggered PCI removes occur under proper bus rescan locking without recursive lock contention. The eeh_pe_loc_get() function has been split into two functions: eeh_pe_loc_get(struct eeh_pe *pe) which retrieves the loc for given PE. eeh_pe_loc_get_bus(struct pci_bus *bus) which retrieves the location code for given bus. This resolves lockdep warnings such as: [ 84.964298] [ T928] ============================================ [ 84.964304] [ T928] WARNING: possible recursive locking detected [ 84.964311] [ T928] 6.18.0-rc3 #51 Not tainted [ 84.964315] [ T928] -------------------------------------------- [ 84.964320] [ T928] eehd/928 is trying to acquire lock: [ 84.964324] [ T928] c000000003b29d58 (pci_rescan_remove_lock){+.+.}-{3:3}, at: pci_lock_rescan_remove+0x28/0x40 [ 84.964342] [ T928] but task is already holding lock: [ 84.964347] [ T928] c000000003b29d58 (pci_rescan_remove_lock){+.+.}-{3:3}, at: pci_lock_rescan_remove+0x28/0x40 [ 84.964357] [ T928] other info that might help us debug this: [ 84.964363] [ T928] Possible unsafe locking scenario: [ 84.964367] [ T928] CPU0 [ 84.964370] [ T928] ---- [ 84.964373] [ T928] lock(pci_rescan_remove_lock); [ 84.964378] [ T928] lock(pci_rescan_remove_lock); [ 84.964383] [ T928] *** DEADLOCK *** [ 84.964388] [ T928] May be due to missing lock nesting notation [ 84.964393] [ T928] 1 lock held by eehd/928: [ 84.964397] [ T928] #0: c000000003b29d58 (pci_rescan_remove_lock){+.+.}-{3:3}, at: pci_lock_rescan_remove+0x28/0x40 [ 84.964408] [ T928] stack backtrace: [ 84.964414] [ T928] CPU: 2 UID: 0 PID: 928 Comm: eehd Not tainted 6.18.0-rc3 #51 VOLUNTARY [ 84.964417] [ T928] Hardware name: IBM,9080-HEX POWER10 (architected) 0x800200 0xf000006 of:IBM,FW1060.00 (NH1060_022) hv:phyp pSeries [ 84.964419] [ T928] Call Trace: [ 84.964420] [ T928] [c0000011a7157990] [c000000001705de4] dump_stack_lvl+0xc8/0x130 (unreliable) [ 84.964424] [ T928] [c0000011a71579d0] [c0000000002f66e0] print_deadlock_bug+0x430/0x440 [ 84.964428] [ T928] [c0000011a7157a70] [c0000000002fd0c0] __lock_acquire+0x1530/0x2d80 [ 84.964431] [ T928] [c0000011a7157ba0] [c0000000002fea54] lock_acquire+0x144/0x410 [ 84.964433] [ T928] [c0000011a7157cb0] [c0000011a7157cb0] __mutex_lock+0xf4/0x1050 [ 84.964436] [ T928] [c0000011a7157e00] [c000000000de21d8] pci_lock_rescan_remove+0x28/0x40 [ 84.964439] [ T928] [c0000011a7157e20] [c00000000004ed98] eeh_pe_bus_get+0x48/0xc0 [ 84.964442] [ T928] [c0000011a7157e50] [c000000000050434] eeh_handle_normal_event+0x64/0xa60 [ 84.964446] [ T928] [c0000011a7157f30] [c000000000051de8] eeh_event_handler+0xf8/0x190 [ 84.964450] [ T928] [c0000011a7157f90] [c0000000002747ac] kthread+0x16c/0x180 [ 84.964453] [ T928] [c0000011a7157fe0] [c00000000000ded8] start_kernel_thread+0x14/0x18 Fixes: 1010b4c012b0 ("powerpc/eeh: Make EEH driver device hotplug safe") Signed-off-by: Narayana Murty N Reviewed-by: Sourabh Jain Reviewed-by: Mahesh Salgaonkar Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20251210142559.8874-1-nnmlinux@linux.ibm.com --- arch/powerpc/include/asm/eeh.h | 2 + arch/powerpc/kernel/eeh_driver.c | 11 ++--- arch/powerpc/kernel/eeh_pe.c | 74 ++++++++++++++++++++++++++++++-- 3 files changed, 78 insertions(+), 9 deletions(-) diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h index 5e34611de9ef..b7ebb4ac2c71 100644 --- a/arch/powerpc/include/asm/eeh.h +++ b/arch/powerpc/include/asm/eeh.h @@ -289,6 +289,8 @@ void eeh_pe_dev_traverse(struct eeh_pe *root, void eeh_pe_restore_bars(struct eeh_pe *pe); const char *eeh_pe_loc_get(struct eeh_pe *pe); struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe); +const char *eeh_pe_loc_get_bus(struct pci_bus *bus); +struct pci_bus *eeh_pe_bus_get_nolock(struct eeh_pe *pe); void eeh_show_enabled(void); int __init eeh_init(struct eeh_ops *ops); diff --git a/arch/powerpc/kernel/eeh_driver.c b/arch/powerpc/kernel/eeh_driver.c index ef78ff77cf8f..028f69158532 100644 --- a/arch/powerpc/kernel/eeh_driver.c +++ b/arch/powerpc/kernel/eeh_driver.c @@ -846,7 +846,7 @@ void eeh_handle_normal_event(struct eeh_pe *pe) pci_lock_rescan_remove(); - bus = eeh_pe_bus_get(pe); + bus = eeh_pe_bus_get_nolock(pe); if (!bus) { pr_err("%s: Cannot find PCI bus for PHB#%x-PE#%x\n", __func__, pe->phb->global_number, pe->addr); @@ -886,14 +886,15 @@ void eeh_handle_normal_event(struct eeh_pe *pe) /* Log the event */ if (pe->type & EEH_PE_PHB) { pr_err("EEH: Recovering PHB#%x, location: %s\n", - pe->phb->global_number, eeh_pe_loc_get(pe)); + pe->phb->global_number, eeh_pe_loc_get_bus(bus)); } else { struct eeh_pe *phb_pe = eeh_phb_pe_get(pe->phb); pr_err("EEH: Recovering PHB#%x-PE#%x\n", pe->phb->global_number, pe->addr); pr_err("EEH: PE location: %s, PHB location: %s\n", - eeh_pe_loc_get(pe), eeh_pe_loc_get(phb_pe)); + eeh_pe_loc_get_bus(bus), + eeh_pe_loc_get_bus(eeh_pe_bus_get_nolock(phb_pe))); } #ifdef CONFIG_STACKTRACE @@ -1098,7 +1099,7 @@ recover_failed: eeh_pe_state_clear(pe, EEH_PE_PRI_BUS, true); eeh_pe_dev_mode_mark(pe, EEH_DEV_REMOVED); - bus = eeh_pe_bus_get(pe); + bus = eeh_pe_bus_get_nolock(pe); if (bus) pci_hp_remove_devices(bus); else @@ -1222,7 +1223,7 @@ void eeh_handle_special_event(void) (phb_pe->state & EEH_PE_RECOVERING)) continue; - bus = eeh_pe_bus_get(phb_pe); + bus = eeh_pe_bus_get_nolock(phb_pe); if (!bus) { pr_err("%s: Cannot find PCI bus for " "PHB#%x-PE#%x\n", diff --git a/arch/powerpc/kernel/eeh_pe.c b/arch/powerpc/kernel/eeh_pe.c index e740101fadf3..040e8f69a4aa 100644 --- a/arch/powerpc/kernel/eeh_pe.c +++ b/arch/powerpc/kernel/eeh_pe.c @@ -812,6 +812,24 @@ void eeh_pe_restore_bars(struct eeh_pe *pe) const char *eeh_pe_loc_get(struct eeh_pe *pe) { struct pci_bus *bus = eeh_pe_bus_get(pe); + return eeh_pe_loc_get_bus(bus); +} + +/** + * eeh_pe_loc_get_bus - Retrieve location code binding to the given PCI bus + * @bus: PCI bus + * + * Retrieve the location code associated with the given PCI bus. If the bus + * is a root bus, the location code is fetched from the PHB device tree node + * or root port. Otherwise, the location code is obtained from the device + * tree node of the upstream bridge of the bus. The function walks up the + * bus hierarchy if necessary, checking each node for the appropriate + * location code property ("ibm,io-base-loc-code" for root buses, + * "ibm,slot-location-code" for others). If no location code is found, + * returns "N/A". + */ +const char *eeh_pe_loc_get_bus(struct pci_bus *bus) +{ struct device_node *dn; const char *loc = NULL; @@ -838,8 +856,9 @@ const char *eeh_pe_loc_get(struct eeh_pe *pe) } /** - * eeh_pe_bus_get - Retrieve PCI bus according to the given PE + * _eeh_pe_bus_get - Retrieve PCI bus according to the given PE * @pe: EEH PE + * @do_lock: Is the caller already held the pci_lock_rescan_remove? * * Retrieve the PCI bus according to the given PE. Basically, * there're 3 types of PEs: PHB/Bus/Device. For PHB PE, the @@ -847,7 +866,7 @@ const char *eeh_pe_loc_get(struct eeh_pe *pe) * returned for BUS PE. However, we don't have associated PCI * bus for DEVICE PE. */ -struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe) +static struct pci_bus *_eeh_pe_bus_get(struct eeh_pe *pe, bool do_lock) { struct eeh_dev *edev; struct pci_dev *pdev; @@ -862,11 +881,58 @@ struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe) /* Retrieve the parent PCI bus of first (top) PCI device */ edev = list_first_entry_or_null(&pe->edevs, struct eeh_dev, entry); - pci_lock_rescan_remove(); + if (do_lock) + pci_lock_rescan_remove(); pdev = eeh_dev_to_pci_dev(edev); if (pdev) bus = pdev->bus; - pci_unlock_rescan_remove(); + if (do_lock) + pci_unlock_rescan_remove(); return bus; } + +/** + * eeh_pe_bus_get - Retrieve PCI bus associated with the given EEH PE, locking + * if needed + * @pe: Pointer to the EEH PE + * + * This function is a wrapper around _eeh_pe_bus_get(), which retrieves the PCI + * bus associated with the provided EEH PE structure. It acquires the PCI + * rescans lock to ensure safe access to shared data during the retrieval + * process. This function should be used when the caller requires the PCI bus + * while holding the rescan/remove lock, typically during operations that modify + * or inspect PCIe device state in a safe manner. + * + * RETURNS: + * A pointer to the PCI bus associated with the EEH PE, or NULL if none found. + */ + +struct pci_bus *eeh_pe_bus_get(struct eeh_pe *pe) +{ + return _eeh_pe_bus_get(pe, true); +} + +/** + * eeh_pe_bus_get_nolock - Retrieve PCI bus associated with the given EEH PE + * without locking + * @pe: Pointer to the EEH PE + * + * This function is a variant of _eeh_pe_bus_get() that retrieves the PCI bus + * associated with the specified EEH PE without acquiring the + * pci_lock_rescan_remove lock. It should only be used when the caller can + * guarantee safe access to PE structures without the need for that lock, + * typically in contexts where the lock is already held locking is otherwise + * managed. + * + * RETURNS: + * pointer to the PCI bus associated with the EEH PE, or NULL if none is found. + * + * NOTE: + * Use this function carefully to avoid race conditions and data corruption. + */ + +struct pci_bus *eeh_pe_bus_get_nolock(struct eeh_pe *pe) +{ + return _eeh_pe_bus_get(pe, false); +} From 15513beeb673765c05cdd8e526ccee33906375cb Mon Sep 17 00:00:00 2001 From: Abhishek Dubey Date: Sat, 24 Jan 2026 13:22:18 +0530 Subject: [PATCH 13/27] powerpc64/bpf: Moving tail_call_cnt to bottom of frame To support tailcalls in subprogs, tail_call_cnt needs to be on the BPF trampoline stack frame. In a regular BPF program or subprog stack frame, the position of tail_call_cnt is after the NVR save area (BPF_PPC_STACK_SAVE). To avoid complex logic in deducing offset for tail_call_cnt, it has to be kept at the same offset on the trampoline frame as well. But doing that wastes nearly all of BPF_PPC_STACK_SAVE bytes on the BPF trampoline stack frame as the NVR save area is not the same for BPF trampoline and regular BPF programs. Address this by moving tail_call_cnt to the bottom of the frame. This change avoids the need to account for BPF_PPC_STACK_SAVE bytes in the BPF trampoline stack frame when support for tailcalls in BPF subprogs is added later. Also, this change makes offset calculation of tail_call_cnt field simpler all across. Signed-off-by: Abhishek Dubey Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20260124075223.6033-2-adubey@linux.ibm.com --- arch/powerpc/net/bpf_jit.h | 1 + arch/powerpc/net/bpf_jit_comp.c | 6 +++--- arch/powerpc/net/bpf_jit_comp64.c | 31 ++++++++++++++++++++----------- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h index 8334cd667bba..9f6ec00bd02e 100644 --- a/arch/powerpc/net/bpf_jit.h +++ b/arch/powerpc/net/bpf_jit.h @@ -24,6 +24,7 @@ #define SZL sizeof(unsigned long) #define BPF_INSN_SAFETY 64 +#define BPF_PPC_TAILCALL 8 #define PLANT_INSTR(d, idx, instr) \ do { if (d) { (d)[idx] = instr; } idx++; } while (0) diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index b243ee205885..6fec10148877 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -621,8 +621,8 @@ static void bpf_trampoline_setup_tail_call_cnt(u32 *image, struct codegen_contex int func_frame_offset, int r4_off) { if (IS_ENABLED(CONFIG_PPC64)) { - /* See bpf_jit_stack_tailcallcnt() */ - int tailcallcnt_offset = 7 * 8; + /* See Generated stack layout */ + int tailcallcnt_offset = BPF_PPC_TAILCALL; EMIT(PPC_RAW_LL(_R3, _R1, func_frame_offset - tailcallcnt_offset)); EMIT(PPC_RAW_STL(_R3, _R1, -tailcallcnt_offset)); @@ -637,7 +637,7 @@ static void bpf_trampoline_restore_tail_call_cnt(u32 *image, struct codegen_cont { if (IS_ENABLED(CONFIG_PPC64)) { /* See bpf_jit_stack_tailcallcnt() */ - int tailcallcnt_offset = 7 * 8; + int tailcallcnt_offset = BPF_PPC_TAILCALL; EMIT(PPC_RAW_LL(_R3, _R1, -tailcallcnt_offset)); EMIT(PPC_RAW_STL(_R3, _R1, func_frame_offset - tailcallcnt_offset)); diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 6c827e7aa691..53bfd0d22d78 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -20,13 +20,15 @@ #include "bpf_jit.h" /* - * Stack layout: + * Stack layout with frame: + * Layout when setting up our own stack frame. + * Note: r1 at bottom, component offsets positive wrt r1. * Ensure the top half (upto local_tmp_var) stays consistent * with our redzone usage. * * [ prev sp ] <------------- - * [ nv gpr save area ] 6*8 | * [ tail_call_cnt ] 8 | + * [ nv gpr save area ] 6*8 | * [ local_tmp_var ] 24 | * fp (r31) --> [ ebpf stack space ] upto 512 | * [ frame header ] 32/112 | @@ -36,10 +38,12 @@ /* for gpr non volatile registers BPG_REG_6 to 10 */ #define BPF_PPC_STACK_SAVE (6*8) /* for bpf JIT code internal usage */ -#define BPF_PPC_STACK_LOCALS 32 +#define BPF_PPC_STACK_LOCALS 24 /* stack frame excluding BPF stack, ensure this is quadword aligned */ #define BPF_PPC_STACKFRAME (STACK_FRAME_MIN_SIZE + \ - BPF_PPC_STACK_LOCALS + BPF_PPC_STACK_SAVE) + BPF_PPC_STACK_LOCALS + \ + BPF_PPC_STACK_SAVE + \ + BPF_PPC_TAILCALL) /* BPF register usage */ #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) @@ -87,27 +91,32 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx) } /* + * Stack layout with redzone: * When not setting up our own stackframe, the redzone (288 bytes) usage is: + * Note: r1 from prev frame. Component offset negative wrt r1. * * [ prev sp ] <------------- * [ ... ] | * sp (r1) ---> [ stack pointer ] -------------- - * [ nv gpr save area ] 6*8 * [ tail_call_cnt ] 8 + * [ nv gpr save area ] 6*8 * [ local_tmp_var ] 24 * [ unused red zone ] 224 */ static int bpf_jit_stack_local(struct codegen_context *ctx) { - if (bpf_has_stack_frame(ctx)) + if (bpf_has_stack_frame(ctx)) { + /* Stack layout with frame */ return STACK_FRAME_MIN_SIZE + ctx->stack_size; - else - return -(BPF_PPC_STACK_SAVE + 32); + } else { + /* Stack layout with redzone */ + return -(BPF_PPC_TAILCALL + BPF_PPC_STACK_SAVE + BPF_PPC_STACK_LOCALS); + } } static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx) { - return bpf_jit_stack_local(ctx) + 24; + return bpf_jit_stack_local(ctx) + BPF_PPC_STACK_LOCALS + BPF_PPC_STACK_SAVE; } static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) @@ -115,7 +124,7 @@ static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) if (reg >= BPF_PPC_NVR_MIN && reg < 32) return (bpf_has_stack_frame(ctx) ? (BPF_PPC_STACKFRAME + ctx->stack_size) : 0) - - (8 * (32 - reg)); + - (8 * (32 - reg)) - BPF_PPC_TAILCALL; pr_err("BPF JIT is asking about unknown registers"); BUG(); @@ -145,7 +154,7 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) if (ctx->seen & SEEN_TAILCALL) { EMIT(PPC_RAW_LI(bpf_to_ppc(TMP_REG_1), 0)); /* this goes in the redzone */ - EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, -(BPF_PPC_STACK_SAVE + 8))); + EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, -(BPF_PPC_TAILCALL))); } else { EMIT(PPC_RAW_NOP()); EMIT(PPC_RAW_NOP()); From 2ed2d8f6fb3853daa536d79e0799cbac91fac9f5 Mon Sep 17 00:00:00 2001 From: Abhishek Dubey Date: Sat, 24 Jan 2026 13:22:19 +0530 Subject: [PATCH 14/27] powerpc64/bpf: Support tailcalls with subprogs Enable tailcalls support in subprogs by passing tail call count as reference instead of value. The actual tailcall count is always maintained in the tailcall field present in the frame of main function (also called entry function). The tailcall field in the stack frame of subprogs contains reference to the tailcall field in the stack frame of main BPF program. Accordingly, rename tail_call_cnt field in the stack layout to tail_call_info. Signed-off-by: Abhishek Dubey Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20260124075223.6033-3-adubey@linux.ibm.com --- arch/powerpc/net/bpf_jit.h | 13 ++++++ arch/powerpc/net/bpf_jit_comp.c | 63 +++++++++++++++++++++++--- arch/powerpc/net/bpf_jit_comp64.c | 74 +++++++++++++++++++++++-------- 3 files changed, 125 insertions(+), 25 deletions(-) diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h index 9f6ec00bd02e..56f56fdd4969 100644 --- a/arch/powerpc/net/bpf_jit.h +++ b/arch/powerpc/net/bpf_jit.h @@ -52,6 +52,13 @@ EMIT(PPC_INST_BRANCH_COND | (((cond) & 0x3ff) << 16) | (offset & 0xfffc)); \ } while (0) +/* When constant jump offset is known prior */ +#define PPC_BCC_CONST_SHORT(cond, offset) \ + do { \ + BUILD_BUG_ON(offset < -0x8000 || offset > 0x7fff || (offset & 0x3)); \ + EMIT(PPC_INST_BRANCH_COND | (((cond) & 0x3ff) << 16) | (offset & 0xfffc)); \ + } while (0) + /* * Sign-extended 32-bit immediate load * @@ -73,6 +80,10 @@ } } while (0) #ifdef CONFIG_PPC64 + +/* for gpr non volatile registers BPG_REG_6 to 10 */ +#define BPF_PPC_STACK_SAVE (6 * 8) + /* If dummy pass (!image), account for maximum possible instructions */ #define PPC_LI64(d, i) do { \ if (!image) \ @@ -167,6 +178,7 @@ struct codegen_context { unsigned int alt_exit_addr; u64 arena_vm_start; u64 user_vm_start; + bool is_subprog; }; #define bpf_to_ppc(r) (ctx->b2p[r]) @@ -206,6 +218,7 @@ int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass struct codegen_context *ctx, int insn_idx, int jmp_off, int dst_reg, u32 code); +int bpf_jit_stack_tailcallinfo_offset(struct codegen_context *ctx); #endif #endif diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 6fec10148877..7b494be1a48f 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -206,6 +206,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) cgctx.stack_size = round_up(fp->aux->stack_depth, 16); cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena); cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena); + cgctx.is_subprog = bpf_is_subprog(fp); /* Scouting faux-generate pass 0 */ if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) { @@ -435,6 +436,11 @@ void bpf_jit_free(struct bpf_prog *fp) bpf_prog_unlock_free(fp); } +bool bpf_jit_supports_subprog_tailcalls(void) +{ + return IS_ENABLED(CONFIG_PPC64); +} + bool bpf_jit_supports_kfunc_call(void) { return true; @@ -617,15 +623,50 @@ static int invoke_bpf_mod_ret(u32 *image, u32 *ro_image, struct codegen_context return 0; } -static void bpf_trampoline_setup_tail_call_cnt(u32 *image, struct codegen_context *ctx, - int func_frame_offset, int r4_off) +/* + * Refer __arch_prepare_bpf_trampoline() for stack component details. + * + * The tailcall count/reference is present in caller's stack frame. The + * tail_call_info is saved at the same offset on the trampoline frame + * for the traced function (BPF subprog/callee) to fetch it. + */ +static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_context *ctx, + int func_frame_offset, + int bpf_dummy_frame_size, int r4_off) { if (IS_ENABLED(CONFIG_PPC64)) { /* See Generated stack layout */ - int tailcallcnt_offset = BPF_PPC_TAILCALL; + int tailcallinfo_offset = BPF_PPC_TAILCALL; - EMIT(PPC_RAW_LL(_R3, _R1, func_frame_offset - tailcallcnt_offset)); - EMIT(PPC_RAW_STL(_R3, _R1, -tailcallcnt_offset)); + /* + * func_frame_offset = ...(1) + * bpf_dummy_frame_size + trampoline_frame_size + */ + EMIT(PPC_RAW_LD(_R4, _R1, func_frame_offset)); + EMIT(PPC_RAW_LD(_R3, _R4, -tailcallinfo_offset)); + + /* + * Setting the tail_call_info in trampoline's frame + * depending on if previous frame had value or reference. + */ + EMIT(PPC_RAW_CMPLWI(_R3, MAX_TAIL_CALL_CNT)); + PPC_BCC_CONST_SHORT(COND_GT, 8); + EMIT(PPC_RAW_ADDI(_R3, _R4, bpf_jit_stack_tailcallinfo_offset(ctx))); + /* + * From ...(1) above: + * trampoline_frame_bottom = ...(2) + * func_frame_offset - bpf_dummy_frame_size + * + * Using ...(2) derived above: + * trampoline_tail_call_info_offset = ...(3) + * trampoline_frame_bottom - tailcallinfo_offset + * + * From ...(3): + * Use trampoline_tail_call_info_offset to write reference of main's + * tail_call_info in trampoline frame. + */ + EMIT(PPC_RAW_STL(_R3, _R1, (func_frame_offset - bpf_dummy_frame_size) + - tailcallinfo_offset)); } else { /* See bpf_jit_stack_offsetof() and BPF_PPC_TC */ EMIT(PPC_RAW_LL(_R4, _R1, r4_off)); @@ -731,6 +772,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im * LR save area [ r0 save (64-bit) ] | header * [ r0 save (32-bit) ] | * dummy frame for unwind [ back chain 1 ] -- + * [ tail_call_info ] optional - 64-bit powerpc * [ padding ] align stack frame * r4_off [ r4 (tailcallcnt) ] optional - 32-bit powerpc * alt_lr_off [ real lr (ool stub)] optional - actual lr @@ -812,6 +854,14 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im } } + /* + * Save tailcall count pointer at the same offset on the + * stack where subprogs expect it + */ + if ((flags & BPF_TRAMP_F_CALL_ORIG) && + (flags & BPF_TRAMP_F_TAIL_CALL_CTX)) + bpf_frame_size += BPF_PPC_TAILCALL; + /* Padding to align stack frame, if any */ bpf_frame_size = round_up(bpf_frame_size, SZL * 2); @@ -913,7 +963,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im /* Replicate tail_call_cnt before calling the original BPF prog */ if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) - bpf_trampoline_setup_tail_call_cnt(image, ctx, func_frame_offset, r4_off); + bpf_trampoline_setup_tail_call_info(image, ctx, func_frame_offset, + bpf_dummy_frame_size, r4_off); /* Restore args */ bpf_trampoline_restore_args_stack(image, ctx, func_frame_offset, nr_regs, regs_off); diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 53bfd0d22d78..e3f677eaa41c 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -26,8 +26,12 @@ * Ensure the top half (upto local_tmp_var) stays consistent * with our redzone usage. * + * tail_call_info - stores tailcall count value in main program's + * frame, stores reference to tail_call_info of + * main's frame in sub-prog's frame. + * * [ prev sp ] <------------- - * [ tail_call_cnt ] 8 | + * [ tail_call_info ] 8 | * [ nv gpr save area ] 6*8 | * [ local_tmp_var ] 24 | * fp (r31) --> [ ebpf stack space ] upto 512 | @@ -35,8 +39,6 @@ * sp (r1) ---> [ stack pointer ] -------------- */ -/* for gpr non volatile registers BPG_REG_6 to 10 */ -#define BPF_PPC_STACK_SAVE (6*8) /* for bpf JIT code internal usage */ #define BPF_PPC_STACK_LOCALS 24 /* stack frame excluding BPF stack, ensure this is quadword aligned */ @@ -98,7 +100,7 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx) * [ prev sp ] <------------- * [ ... ] | * sp (r1) ---> [ stack pointer ] -------------- - * [ tail_call_cnt ] 8 + * [ tail_call_info ] 8 * [ nv gpr save area ] 6*8 * [ local_tmp_var ] 24 * [ unused red zone ] 224 @@ -114,7 +116,7 @@ static int bpf_jit_stack_local(struct codegen_context *ctx) } } -static int bpf_jit_stack_tailcallcnt(struct codegen_context *ctx) +int bpf_jit_stack_tailcallinfo_offset(struct codegen_context *ctx) { return bpf_jit_stack_local(ctx) + BPF_PPC_STACK_LOCALS + BPF_PPC_STACK_SAVE; } @@ -147,17 +149,32 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) #endif /* - * Initialize tail_call_cnt if we do tail calls. - * Otherwise, put in NOPs so that it can be skipped when we are - * invoked through a tail call. + * Tail call count(tcc) is saved & updated only in main + * program's frame and the address of tcc in main program's + * frame (tcc_ptr) is saved in subprogs frame. + * + * Offset of tail_call_info on any frame will be interpreted + * as either tcc_ptr or tcc value depending on whether it is + * greater than MAX_TAIL_CALL_CNT or not. */ - if (ctx->seen & SEEN_TAILCALL) { + if (!ctx->is_subprog) { EMIT(PPC_RAW_LI(bpf_to_ppc(TMP_REG_1), 0)); /* this goes in the redzone */ EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, -(BPF_PPC_TAILCALL))); } else { - EMIT(PPC_RAW_NOP()); - EMIT(PPC_RAW_NOP()); + /* + * if tail_call_info < MAX_TAIL_CALL_CNT + * main prog calling first subprog -> copy reference + * else + * subsequent subprog calling another subprog -> directly copy content + */ + EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_2), _R1, 0)); + EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_2), -(BPF_PPC_TAILCALL))); + EMIT(PPC_RAW_CMPLWI(bpf_to_ppc(TMP_REG_1), MAX_TAIL_CALL_CNT)); + PPC_BCC_CONST_SHORT(COND_GT, 8); + EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_2), + -(BPF_PPC_TAILCALL))); + EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, -(BPF_PPC_TAILCALL))); } if (bpf_has_stack_frame(ctx)) { @@ -352,19 +369,38 @@ static int bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 o EMIT(PPC_RAW_CMPLW(b2p_index, bpf_to_ppc(TMP_REG_1))); PPC_BCC_SHORT(COND_GE, out); - /* - * if (tail_call_cnt >= MAX_TAIL_CALL_CNT) - * goto out; - */ - EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), _R1, bpf_jit_stack_tailcallcnt(ctx))); + EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), _R1, bpf_jit_stack_tailcallinfo_offset(ctx))); EMIT(PPC_RAW_CMPLWI(bpf_to_ppc(TMP_REG_1), MAX_TAIL_CALL_CNT)); - PPC_BCC_SHORT(COND_GE, out); + PPC_BCC_CONST_SHORT(COND_LE, 8); + + /* dereference TMP_REG_1 */ + EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), 0)); /* - * tail_call_cnt++; + * if (tail_call_info == MAX_TAIL_CALL_CNT) + * goto out; + */ + EMIT(PPC_RAW_CMPLWI(bpf_to_ppc(TMP_REG_1), MAX_TAIL_CALL_CNT)); + PPC_BCC_SHORT(COND_EQ, out); + + /* + * tail_call_info++; <- Actual value of tcc here */ EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_1), 1)); - EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, bpf_jit_stack_tailcallcnt(ctx))); + + /* + * Before writing updated tail_call_info, distinguish if current frame + * is storing a reference to tail_call_info or actual tcc value in + * tail_call_info. + */ + EMIT(PPC_RAW_LD(bpf_to_ppc(TMP_REG_2), _R1, bpf_jit_stack_tailcallinfo_offset(ctx))); + EMIT(PPC_RAW_CMPLWI(bpf_to_ppc(TMP_REG_2), MAX_TAIL_CALL_CNT)); + PPC_BCC_CONST_SHORT(COND_GT, 8); + + /* First get address of tail_call_info */ + EMIT(PPC_RAW_ADDI(bpf_to_ppc(TMP_REG_2), _R1, bpf_jit_stack_tailcallinfo_offset(ctx))); + /* Writeback updated value to tail_call_info */ + EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), bpf_to_ppc(TMP_REG_2), 0)); /* prog = array->ptrs[index]; */ EMIT(PPC_RAW_MULI(bpf_to_ppc(TMP_REG_1), b2p_index, 8)); From 88cb7f40344d8b81ecfc4709b5382a956739c272 Mon Sep 17 00:00:00 2001 From: Abhishek Dubey Date: Sat, 24 Jan 2026 13:22:20 +0530 Subject: [PATCH 15/27] powerpc64/bpf: Avoid tailcall restore from trampoline Back propagation of tailcall count is no longer needed for powerpc64 due to use of reference, which updates the tailcall count in the tail_call_info field in the frame of the main program only. Back propagation is still required for 32-bit powerpc. Signed-off-by: Abhishek Dubey Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20260124075223.6033-4-adubey@linux.ibm.com --- arch/powerpc/net/bpf_jit_comp.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 7b494be1a48f..038448b74fa0 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -676,14 +676,11 @@ static void bpf_trampoline_setup_tail_call_info(u32 *image, struct codegen_conte static void bpf_trampoline_restore_tail_call_cnt(u32 *image, struct codegen_context *ctx, int func_frame_offset, int r4_off) { - if (IS_ENABLED(CONFIG_PPC64)) { - /* See bpf_jit_stack_tailcallcnt() */ - int tailcallcnt_offset = BPF_PPC_TAILCALL; - - EMIT(PPC_RAW_LL(_R3, _R1, -tailcallcnt_offset)); - EMIT(PPC_RAW_STL(_R3, _R1, func_frame_offset - tailcallcnt_offset)); - } else { - /* See bpf_jit_stack_offsetof() and BPF_PPC_TC */ + if (IS_ENABLED(CONFIG_PPC32)) { + /* + * Restore tailcall for 32-bit powerpc + * See bpf_jit_stack_offsetof() and BPF_PPC_TC + */ EMIT(PPC_RAW_STL(_R4, _R1, r4_off)); } } From b1c24f089bcf23faa23397dfb5e05f82a8c6cf56 Mon Sep 17 00:00:00 2001 From: Abhishek Dubey Date: Sat, 24 Jan 2026 13:22:21 +0530 Subject: [PATCH 16/27] powerpc64/bpf: Add arch_bpf_stack_walk() for BPF JIT This function is used by bpf_throw() to unwind the stack until frame of exception-boundary during BPF exception handling. This function is necessary to support BPF exceptions on PowerPC. Signed-off-by: Abhishek Dubey Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20260124075223.6033-5-adubey@linux.ibm.com --- arch/powerpc/net/bpf_jit_comp64.c | 41 +++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index e3f677eaa41c..f97907cc5ed4 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -247,6 +247,47 @@ void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx) bpf_jit_build_fentry_stubs(image, ctx); } +/* + * arch_bpf_stack_walk() - BPF stack walker for PowerPC + * + * Based on arch_stack_walk() from stacktrace.c. + * PowerPC uses stack frames rather than stack pointers. See [1] for + * the equivalence between frame pointers and stack pointers. + * Additional reference at [2]. + * TODO: refactor with arch_stack_walk() + * + * [1]: https://lore.kernel.org/all/20200220115141.2707-1-mpe@ellerman.id.au/ + * [2]: https://lore.kernel.org/bpf/20260122211854.5508-5-adubey@linux.ibm.com/ + */ + +void arch_bpf_stack_walk(bool (*consume_fn)(void *, u64, u64, u64), void *cookie) +{ + // callback processing always in current context + unsigned long sp = current_stack_frame(); + + for (;;) { + unsigned long *stack = (unsigned long *) sp; + unsigned long ip; + + if (!validate_sp(sp, current)) + return; + + ip = stack[STACK_FRAME_LR_SAVE]; + if (!ip) + break; + + /* + * consume_fn common code expects stack pointer in third + * argument. There is no sp in ppc64, rather pass frame + * pointer(named sp here). + */ + if (ip && !consume_fn(cookie, ip, sp, sp)) + break; + + sp = stack[0]; + } +} + int bpf_jit_emit_func_call_rel(u32 *image, u32 *fimage, struct codegen_context *ctx, u64 func) { unsigned long func_addr = func ? ppc_function_entry((void *)func) : 0; From c169930292f7f31a6e500803e6c5ffbb9d5e6e4b Mon Sep 17 00:00:00 2001 From: Abhishek Dubey Date: Sat, 24 Jan 2026 13:22:22 +0530 Subject: [PATCH 17/27] powerpc64/bpf: Support exceptions The modified prologue/epilogue generation code now enables exception-callback to use the stack frame of the program marked as exception boundary, where callee saved registers are stored. As per ppc64 ABIv2 documentation[1], r14-r31 are callee saved registers. BPF programs on ppc64 already saves r26-r31 registers. Saving the remaining set of callee saved registers(r14-r25) is handled in the next patch. [1] https://ftp.rtems.org/pub/rtems/people/sebh/ABI64BitOpenPOWERv1.1_16July2015_pub.pdf Signed-off-by: Abhishek Dubey Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20260124075223.6033-6-adubey@linux.ibm.com --- arch/powerpc/net/bpf_jit.h | 2 ++ arch/powerpc/net/bpf_jit_comp.c | 7 ++++ arch/powerpc/net/bpf_jit_comp64.c | 59 +++++++++++++++++++++---------- 3 files changed, 50 insertions(+), 18 deletions(-) diff --git a/arch/powerpc/net/bpf_jit.h b/arch/powerpc/net/bpf_jit.h index 56f56fdd4969..82bbf63f0e57 100644 --- a/arch/powerpc/net/bpf_jit.h +++ b/arch/powerpc/net/bpf_jit.h @@ -179,6 +179,8 @@ struct codegen_context { u64 arena_vm_start; u64 user_vm_start; bool is_subprog; + bool exception_boundary; + bool exception_cb; }; #define bpf_to_ppc(r) (ctx->b2p[r]) diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 038448b74fa0..987cd9fb0f37 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -207,6 +207,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp) cgctx.arena_vm_start = bpf_arena_get_kern_vm_start(fp->aux->arena); cgctx.user_vm_start = bpf_arena_get_user_vm_start(fp->aux->arena); cgctx.is_subprog = bpf_is_subprog(fp); + cgctx.exception_boundary = fp->aux->exception_boundary; + cgctx.exception_cb = fp->aux->exception_cb; /* Scouting faux-generate pass 0 */ if (bpf_jit_build_body(fp, NULL, NULL, &cgctx, addrs, 0, false)) { @@ -436,6 +438,11 @@ void bpf_jit_free(struct bpf_prog *fp) bpf_prog_unlock_free(fp); } +bool bpf_jit_supports_exceptions(void) +{ + return IS_ENABLED(CONFIG_PPC64); +} + bool bpf_jit_supports_subprog_tailcalls(void) { return IS_ENABLED(CONFIG_PPC64); diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index f97907cc5ed4..39f00c6e3607 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -89,7 +89,9 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx) * - the bpf program uses its stack area * The latter condition is deduced from the usage of BPF_REG_FP */ - return ctx->seen & SEEN_FUNC || bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP)); + return ctx->seen & SEEN_FUNC || + bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP)) || + ctx->exception_cb; } /* @@ -161,8 +163,13 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) EMIT(PPC_RAW_LI(bpf_to_ppc(TMP_REG_1), 0)); /* this goes in the redzone */ EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, -(BPF_PPC_TAILCALL))); - } else { + } else if (!ctx->exception_cb) { /* + * Tailcall jitting for non exception_cb progs only. + * exception_cb won't require tail_call_info to be setup. + * + * tail_call_info interpretation logic: + * * if tail_call_info < MAX_TAIL_CALL_CNT * main prog calling first subprog -> copy reference * else @@ -177,8 +184,12 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, -(BPF_PPC_TAILCALL))); } - if (bpf_has_stack_frame(ctx)) { + if (bpf_has_stack_frame(ctx) && !ctx->exception_cb) { /* + * exception_cb uses boundary frame after stack walk. + * It can simply use redzone, this optimization reduces + * stack walk loop by one level. + * * We need a stack frame, but we don't necessarily need to * save/restore LR unless we call other functions */ @@ -190,23 +201,35 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) EMIT(PPC_RAW_STDU(_R1, _R1, -(BPF_PPC_STACKFRAME + ctx->stack_size))); } - /* - * Back up non-volatile regs -- BPF registers 6-10 - * If we haven't created our own stack frame, we save these - * in the protected zone below the previous stack frame - */ - for (i = BPF_REG_6; i <= BPF_REG_10; i++) - if (bpf_is_seen_register(ctx, bpf_to_ppc(i))) - EMIT(PPC_RAW_STD(bpf_to_ppc(i), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(i)))); + if (!ctx->exception_cb) { + /* + * Back up non-volatile regs -- BPF registers 6-10 + * If we haven't created our own stack frame, we save these + * in the protected zone below the previous stack frame + */ + for (i = BPF_REG_6; i <= BPF_REG_10; i++) + if (ctx->exception_boundary || bpf_is_seen_register(ctx, bpf_to_ppc(i))) + EMIT(PPC_RAW_STD(bpf_to_ppc(i), _R1, + bpf_jit_stack_offsetof(ctx, bpf_to_ppc(i)))); - if (ctx->arena_vm_start) - EMIT(PPC_RAW_STD(bpf_to_ppc(ARENA_VM_START), _R1, + if (ctx->exception_boundary || ctx->arena_vm_start) + EMIT(PPC_RAW_STD(bpf_to_ppc(ARENA_VM_START), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(ARENA_VM_START)))); + } else { + /* + * Exception callback receives Frame Pointer of boundary + * program(main prog) as third arg + */ + EMIT(PPC_RAW_MR(_R1, _R5)); + } - /* Setup frame pointer to point to the bpf stack area */ + /* + * Exception_cb not restricted from using stack area or arena. + * Setup frame pointer to point to the bpf stack area + */ if (bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP))) EMIT(PPC_RAW_ADDI(bpf_to_ppc(BPF_REG_FP), _R1, - STACK_FRAME_MIN_SIZE + ctx->stack_size)); + STACK_FRAME_MIN_SIZE + ctx->stack_size)); if (ctx->arena_vm_start) PPC_LI64(bpf_to_ppc(ARENA_VM_START), ctx->arena_vm_start); @@ -218,17 +241,17 @@ static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx /* Restore NVRs */ for (i = BPF_REG_6; i <= BPF_REG_10; i++) - if (bpf_is_seen_register(ctx, bpf_to_ppc(i))) + if (ctx->exception_cb || bpf_is_seen_register(ctx, bpf_to_ppc(i))) EMIT(PPC_RAW_LD(bpf_to_ppc(i), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(i)))); - if (ctx->arena_vm_start) + if (ctx->exception_cb || ctx->arena_vm_start) EMIT(PPC_RAW_LD(bpf_to_ppc(ARENA_VM_START), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(ARENA_VM_START)))); /* Tear down our stack frame */ if (bpf_has_stack_frame(ctx)) { EMIT(PPC_RAW_ADDI(_R1, _R1, BPF_PPC_STACKFRAME + ctx->stack_size)); - if (ctx->seen & SEEN_FUNC) { + if (ctx->seen & SEEN_FUNC || ctx->exception_cb) { EMIT(PPC_RAW_LD(_R0, _R1, PPC_LR_STKOFF)); EMIT(PPC_RAW_MTLR(_R0)); } From 11d45eee9f42f7d3f9c9f190a6f79e570d345fe2 Mon Sep 17 00:00:00 2001 From: Abhishek Dubey Date: Sat, 24 Jan 2026 13:22:23 +0530 Subject: [PATCH 18/27] powerpc64/bpf: Additional NVR handling for bpf_throw The bpf_throw() function never returns, if it has clobbered any callee-saved register, those will remain clobbered. The prologue must take care of saving all callee-saved registers in the frame of exception boundary program. Later these additional non volatile registers R14-R25 along with other NVRs are restored back in the epilogue of exception callback. To achieve above objective, the frame size is determined dynamically to accommodate additional non volatile registers in exception boundary's frame. For non-exception boundary program, the frame size remains optimal. The additional instructions to save & restore r14-r25 registers are emitted only during exception boundary and exception callback program respectively. Signed-off-by: Abhishek Dubey Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20260124075223.6033-7-adubey@linux.ibm.com --- arch/powerpc/net/bpf_jit_comp64.c | 89 ++++++++++++++++++++++++++++--- 1 file changed, 81 insertions(+), 8 deletions(-) diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index 39f00c6e3607..b1a3945ccc9f 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -32,21 +32,37 @@ * * [ prev sp ] <------------- * [ tail_call_info ] 8 | - * [ nv gpr save area ] 6*8 | + * [ nv gpr save area ] 6*8 + (12*8) | * [ local_tmp_var ] 24 | * fp (r31) --> [ ebpf stack space ] upto 512 | * [ frame header ] 32/112 | * sp (r1) ---> [ stack pointer ] -------------- + * + * Additional (12*8) in 'nv gpr save area' only in case of + * exception boundary. */ /* for bpf JIT code internal usage */ #define BPF_PPC_STACK_LOCALS 24 +/* + * for additional non volatile registers(r14-r25) to be saved + * at exception boundary + */ +#define BPF_PPC_EXC_STACK_SAVE (12*8) + /* stack frame excluding BPF stack, ensure this is quadword aligned */ #define BPF_PPC_STACKFRAME (STACK_FRAME_MIN_SIZE + \ BPF_PPC_STACK_LOCALS + \ BPF_PPC_STACK_SAVE + \ BPF_PPC_TAILCALL) +/* + * same as BPF_PPC_STACKFRAME with save area for additional + * non volatile registers saved at exception boundary. + * This is quad-word aligned. + */ +#define BPF_PPC_EXC_STACKFRAME (BPF_PPC_STACKFRAME + BPF_PPC_EXC_STACK_SAVE) + /* BPF register usage */ #define TMP_REG_1 (MAX_BPF_JIT_REG + 0) #define TMP_REG_2 (MAX_BPF_JIT_REG + 1) @@ -88,10 +104,16 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx) * - we call other functions (kernel helpers), or * - the bpf program uses its stack area * The latter condition is deduced from the usage of BPF_REG_FP + * + * bpf_throw() leads to exception callback from a BPF (sub)program. + * The (sub)program is always marked as SEEN_FUNC, creating a stack + * frame. The exception callback uses the frame of the exception + * boundary, so the exception boundary program must have a frame. */ return ctx->seen & SEEN_FUNC || bpf_is_seen_register(ctx, bpf_to_ppc(BPF_REG_FP)) || - ctx->exception_cb; + ctx->exception_cb || + ctx->exception_boundary; } /* @@ -103,9 +125,12 @@ static inline bool bpf_has_stack_frame(struct codegen_context *ctx) * [ ... ] | * sp (r1) ---> [ stack pointer ] -------------- * [ tail_call_info ] 8 - * [ nv gpr save area ] 6*8 + * [ nv gpr save area ] 6*8 + (12*8) * [ local_tmp_var ] 24 * [ unused red zone ] 224 + * + * Additional (12*8) in 'nv gpr save area' only in case of + * exception boundary. */ static int bpf_jit_stack_local(struct codegen_context *ctx) { @@ -114,7 +139,12 @@ static int bpf_jit_stack_local(struct codegen_context *ctx) return STACK_FRAME_MIN_SIZE + ctx->stack_size; } else { /* Stack layout with redzone */ - return -(BPF_PPC_TAILCALL + BPF_PPC_STACK_SAVE + BPF_PPC_STACK_LOCALS); + return -(BPF_PPC_TAILCALL + +BPF_PPC_STACK_SAVE + +(ctx->exception_boundary || ctx->exception_cb ? + BPF_PPC_EXC_STACK_SAVE : 0) + +BPF_PPC_STACK_LOCALS + ); } } @@ -125,9 +155,19 @@ int bpf_jit_stack_tailcallinfo_offset(struct codegen_context *ctx) static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg) { - if (reg >= BPF_PPC_NVR_MIN && reg < 32) + int min_valid_nvreg = BPF_PPC_NVR_MIN; + /* Default frame size for all cases except exception boundary */ + int frame_nvr_size = BPF_PPC_STACKFRAME; + + /* Consider all nv regs for handling exceptions */ + if (ctx->exception_boundary || ctx->exception_cb) { + min_valid_nvreg = _R14; + frame_nvr_size = BPF_PPC_EXC_STACKFRAME; + } + + if (reg >= min_valid_nvreg && reg < 32) return (bpf_has_stack_frame(ctx) ? - (BPF_PPC_STACKFRAME + ctx->stack_size) : 0) + (frame_nvr_size + ctx->stack_size) : 0) - (8 * (32 - reg)) - BPF_PPC_TAILCALL; pr_err("BPF JIT is asking about unknown registers"); @@ -138,6 +178,17 @@ void bpf_jit_realloc_regs(struct codegen_context *ctx) { } +/* + * For exception boundary & exception_cb progs: + * return increased size to accommodate additional NVRs. + */ +static int bpf_jit_stack_size(struct codegen_context *ctx) +{ + return ctx->exception_boundary || ctx->exception_cb ? + BPF_PPC_EXC_STACKFRAME : + BPF_PPC_STACKFRAME; +} + void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) { int i; @@ -198,7 +249,19 @@ void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) EMIT(PPC_RAW_STD(_R0, _R1, PPC_LR_STKOFF)); } - EMIT(PPC_RAW_STDU(_R1, _R1, -(BPF_PPC_STACKFRAME + ctx->stack_size))); + EMIT(PPC_RAW_STDU(_R1, _R1, + -(bpf_jit_stack_size(ctx) + ctx->stack_size))); + } + + /* + * Program acting as exception boundary pushes R14..R25 in addition to + * BPF callee-saved non volatile registers. Exception callback uses + * the boundary program's stack frame, recover additionally saved + * registers in epilogue of exception callback. + */ + if (ctx->exception_boundary) { + for (i = _R14; i <= _R25; i++) + EMIT(PPC_RAW_STD(i, _R1, bpf_jit_stack_offsetof(ctx, i))); } if (!ctx->exception_cb) { @@ -248,9 +311,19 @@ static void bpf_jit_emit_common_epilogue(u32 *image, struct codegen_context *ctx EMIT(PPC_RAW_LD(bpf_to_ppc(ARENA_VM_START), _R1, bpf_jit_stack_offsetof(ctx, bpf_to_ppc(ARENA_VM_START)))); + if (ctx->exception_cb) { + /* + * Recover additionally saved non volatile registers from stack + * frame of exception boundary program. + */ + for (i = _R14; i <= _R25; i++) + EMIT(PPC_RAW_LD(i, _R1, bpf_jit_stack_offsetof(ctx, i))); + } + /* Tear down our stack frame */ if (bpf_has_stack_frame(ctx)) { - EMIT(PPC_RAW_ADDI(_R1, _R1, BPF_PPC_STACKFRAME + ctx->stack_size)); + EMIT(PPC_RAW_ADDI(_R1, _R1, bpf_jit_stack_size(ctx) + ctx->stack_size)); + if (ctx->seen & SEEN_FUNC || ctx->exception_cb) { EMIT(PPC_RAW_LD(_R0, _R1, PPC_LR_STKOFF)); EMIT(PPC_RAW_MTLR(_R0)); From 4acc549e800e1035475995b6b144040561115fa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miquel=20Sabat=C3=A9=20Sol=C3=A0?= Date: Mon, 15 Sep 2025 16:18:08 +0200 Subject: [PATCH 19/27] powerpc: kgdb: Remove OUTBUFMAX constant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This constant was introduced in commit 17ce452f7ea3 ("kgdb, powerpc: arch specific powerpc kgdb support"), but it is no longer used anywhere in the source tree. Signed-off-by: Miquel Sabaté Solà Reviewed-by: Christophe Leroy Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20250915141808.146695-1-mikisabate@gmail.com --- arch/powerpc/include/asm/kgdb.h | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/powerpc/include/asm/kgdb.h b/arch/powerpc/include/asm/kgdb.h index f39531903325..ab5af235832d 100644 --- a/arch/powerpc/include/asm/kgdb.h +++ b/arch/powerpc/include/asm/kgdb.h @@ -25,7 +25,6 @@ #define BREAK_INSTR_SIZE 4 #define BUFMAX ((NUMREGBYTES * 2) + 512) -#define OUTBUFMAX ((NUMREGBYTES * 2) + 512) #define BREAK_INSTR 0x7d821008 /* twge r2, r2 */ From 33c1c6d8a28a2761ac74b0380b2563cf546c2a3a Mon Sep 17 00:00:00 2001 From: Guangshuo Li Date: Tue, 23 Sep 2025 21:32:35 +0800 Subject: [PATCH 20/27] powerpc/smp: Add check for kcalloc() failure in parse_thread_groups() As kcalloc() may fail, check its return value to avoid a NULL pointer dereference when passing it to of_property_read_u32_array(). Fixes: 790a1662d3a26 ("powerpc/smp: Parse ibm,thread-groups with multiple properties") Cc: stable@vger.kernel.org Reviewed-by: Christophe Leroy Signed-off-by: Guangshuo Li Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20250923133235.1862108-1-lgs201920130244@gmail.com --- arch/powerpc/kernel/smp.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 292fee8809bc..cad3358fa4c3 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -822,6 +822,8 @@ static int parse_thread_groups(struct device_node *dn, count = of_property_count_u32_elems(dn, "ibm,thread-groups"); thread_group_array = kcalloc(count, sizeof(u32), GFP_KERNEL); + if (!thread_group_array) + return -ENOMEM; ret = of_property_read_u32_array(dn, "ibm,thread-groups", thread_group_array, count); if (ret) From 88372ad4b060a283a475f0ebc31451c9059ebaa5 Mon Sep 17 00:00:00 2001 From: Srish Srinivasan Date: Tue, 27 Jan 2026 20:22:23 +0530 Subject: [PATCH 21/27] pseries/plpks: fix kernel-doc comment inconsistencies Fix issues with comments for all the applicable functions to be consistent with kernel-doc format. Move them before the function definition as opposed to the function prototype. Signed-off-by: Srish Srinivasan Tested-by: Nayna Jain Reviewed-by: Nayna Jain Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20260127145228.48320-2-ssrish@linux.ibm.com --- arch/powerpc/include/asm/plpks.h | 77 ------ arch/powerpc/platforms/pseries/plpks.c | 328 ++++++++++++++++++++++++- 2 files changed, 318 insertions(+), 87 deletions(-) diff --git a/arch/powerpc/include/asm/plpks.h b/arch/powerpc/include/asm/plpks.h index 7a84069759b0..f303922bf622 100644 --- a/arch/powerpc/include/asm/plpks.h +++ b/arch/powerpc/include/asm/plpks.h @@ -67,122 +67,45 @@ struct plpks_var_name_list { struct plpks_var_name varlist[]; }; -/** - * Updates the authenticated variable. It expects NULL as the component. - */ int plpks_signed_update_var(struct plpks_var *var, u64 flags); -/** - * Writes the specified var and its data to PKS. - * Any caller of PKS driver should present a valid component type for - * their variable. - */ int plpks_write_var(struct plpks_var var); -/** - * Removes the specified var and its data from PKS. - */ int plpks_remove_var(char *component, u8 varos, struct plpks_var_name vname); -/** - * Returns the data for the specified os variable. - * - * Caller must allocate a buffer in var->data with length in var->datalen. - * If no buffer is provided, var->datalen will be populated with the object's - * size. - */ int plpks_read_os_var(struct plpks_var *var); -/** - * Returns the data for the specified firmware variable. - * - * Caller must allocate a buffer in var->data with length in var->datalen. - * If no buffer is provided, var->datalen will be populated with the object's - * size. - */ int plpks_read_fw_var(struct plpks_var *var); -/** - * Returns the data for the specified bootloader variable. - * - * Caller must allocate a buffer in var->data with length in var->datalen. - * If no buffer is provided, var->datalen will be populated with the object's - * size. - */ int plpks_read_bootloader_var(struct plpks_var *var); -/** - * Returns if PKS is available on this LPAR. - */ bool plpks_is_available(void); -/** - * Returns version of the Platform KeyStore. - */ u8 plpks_get_version(void); -/** - * Returns hypervisor storage overhead per object, not including the size of - * the object or label. Only valid for config version >= 2 - */ u16 plpks_get_objoverhead(void); -/** - * Returns maximum password size. Must be >= 32 bytes - */ u16 plpks_get_maxpwsize(void); -/** - * Returns maximum object size supported by Platform KeyStore. - */ u16 plpks_get_maxobjectsize(void); -/** - * Returns maximum object label size supported by Platform KeyStore. - */ u16 plpks_get_maxobjectlabelsize(void); -/** - * Returns total size of the configured Platform KeyStore. - */ u32 plpks_get_totalsize(void); -/** - * Returns used space from the total size of the Platform KeyStore. - */ u32 plpks_get_usedspace(void); -/** - * Returns bitmask of policies supported by the hypervisor. - */ u32 plpks_get_supportedpolicies(void); -/** - * Returns maximum byte size of a single object supported by the hypervisor. - * Only valid for config version >= 3 - */ u32 plpks_get_maxlargeobjectsize(void); -/** - * Returns bitmask of signature algorithms supported for signed updates. - * Only valid for config version >= 3 - */ u64 plpks_get_signedupdatealgorithms(void); -/** - * Returns the length of the PLPKS password in bytes. - */ u16 plpks_get_passwordlen(void); -/** - * Called in early init to retrieve and clear the PLPKS password from the DT. - */ void plpks_early_init_devtree(void); -/** - * Populates the FDT with the PLPKS password to prepare for kexec. - */ int plpks_populate_fdt(void *fdt); #else // CONFIG_PSERIES_PLPKS static inline bool plpks_is_available(void) { return false; } diff --git a/arch/powerpc/platforms/pseries/plpks.c b/arch/powerpc/platforms/pseries/plpks.c index b1667ed05f98..03722fabf9c3 100644 --- a/arch/powerpc/platforms/pseries/plpks.c +++ b/arch/powerpc/platforms/pseries/plpks.c @@ -312,40 +312,107 @@ err: return rc; } +/** + * plpks_get_version() - Get the version of the PLPKS config structure. + * + * Successful execution of the H_PKS_GET_CONFIG HCALL during initialization + * reads the PLPKS config structure version and saves it in a file local static + * version variable. + * + * Returns: On success the saved PLPKS config structure version is returned, 0 + * if not. + */ u8 plpks_get_version(void) { return version; } +/** + * plpks_get_objoverhead() - Get the hypervisor storage overhead per object. + * + * Successful execution of the H_PKS_GET_CONFIG HCALL during initialization + * reads the per object hypervisor storage overhead in bytes into the local + * static objoverhead variable, excluding the size of the object or the label. + * This value can be treated as valid only when the PLPKS config structure + * version >= 2. + * + * Returns: If PLPKS config structure version >= 2 then the storage overhead is + * returned, 0 otherwise. + */ u16 plpks_get_objoverhead(void) { return objoverhead; } +/** + * plpks_get_maxpwsize() - Get the maximum password size. + * + * Successful execution of the H_PKS_GET_CONFIG HCALL during initialization + * reads the maximum password size and checks if it is 32 bytes at the least + * before storing it in the local static maxpwsize variable. + * + * Returns: On success the maximum password size is returned, 0 if not. + */ u16 plpks_get_maxpwsize(void) { return maxpwsize; } +/** + * plpks_get_maxobjectsize() - Get the maximum object size supported by the + * PLPKS. + * + * Successful execution of the H_PKS_GET_CONFIG HCALL during initialization + * reads the maximum object size into the file local static maxobjsize variable. + * + * Returns: On success the maximum object size is returned, 0 if not. + */ u16 plpks_get_maxobjectsize(void) { return maxobjsize; } +/** + * plpks_get_maxobjectlabelsize() - Get the maximum object label size supported + * by the PLPKS. + * + * Successful execution of the H_PKS_GET_CONFIG HCALL during initialization + * reads the maximum object label size into the local static maxobjlabelsize + * variable. + * + * Returns: On success the maximum object label size is returned, 0 if not. + */ u16 plpks_get_maxobjectlabelsize(void) { return maxobjlabelsize; } +/** + * plpks_get_totalsize() - Get the total size of the PLPKS that is configured. + * + * Successful execution of the H_PKS_GET_CONFIG HCALL during initialization + * reads the total size of the PLPKS that is configured for the LPAR into the + * file local static totalsize variable. + * + * Returns: On success the total size of the PLPKS configured is returned, 0 if + * not. + */ u32 plpks_get_totalsize(void) { return totalsize; } +/** + * plpks_get_usedspace() - Get the used space from the total size of the PLPKS. + * + * Invoke the H_PKS_GET_CONFIG HCALL to refresh the latest value for the used + * space as this keeps changing with the creation and removal of objects in the + * PLPKS. + * + * Returns: On success the used space is returned, 0 if not. + */ u32 plpks_get_usedspace(void) { - // Unlike other config values, usedspace regularly changes as objects - // are updated, so we need to refresh. int rc = _plpks_get_config(); if (rc) { pr_err("Couldn't get config, rc: %d\n", rc); @@ -354,26 +421,84 @@ u32 plpks_get_usedspace(void) return usedspace; } +/** + * plpks_get_supportedpolicies() - Get a bitmask of the policies supported by + * the hypervisor. + * + * Successful execution of the H_PKS_GET_CONFIG HCALL during initialization + * reads a bitmask of the policies supported by the hypervisor into the file + * local static supportedpolicies variable. + * + * Returns: On success the bitmask of the policies supported by the hypervisor + * are returned, 0 if not. + */ u32 plpks_get_supportedpolicies(void) { return supportedpolicies; } +/** + * plpks_get_maxlargeobjectsize() - Get the maximum object size supported for + * PLPKS config structure version >= 3 + * + * Successful execution of the H_PKS_GET_CONFIG HCALL during initialization + * reads the maximum object size into the local static maxlargeobjectsize + * variable for PLPKS config structure version >= 3. This was introduced + * starting with PLPKS config structure version 3 to allow for objects of + * size >= 64K. + * + * Returns: If PLPKS config structure version >= 3 then the new maximum object + * size is returned, 0 if not. + */ u32 plpks_get_maxlargeobjectsize(void) { return maxlargeobjectsize; } +/** + * plpks_get_signedupdatealgorithms() - Get a bitmask of the signature + * algorithms supported for signed updates. + * + * Successful execution of the H_PKS_GET_CONFIG HCALL during initialization + * reads a bitmask of the signature algorithms supported for signed updates into + * the file local static signedupdatealgorithms variable. This is valid only + * when the PLPKS config structure version >= 3. + * + * Returns: On success the bitmask of the signature algorithms supported for + * signed updates is returned, 0 if not. + */ u64 plpks_get_signedupdatealgorithms(void) { return signedupdatealgorithms; } +/** + * plpks_get_passwordlen() - Get the length of the PLPKS password in bytes. + * + * The H_PKS_GEN_PASSWORD HCALL makes the hypervisor generate a random password + * for the specified consumer, apply that password to the PLPKS and return it to + * the caller. In this process, the password length for the OS consumer is + * stored in the local static ospasswordlength variable. + * + * Returns: On success the password length for the OS consumer in bytes is + * returned, 0 if not. + */ u16 plpks_get_passwordlen(void) { return ospasswordlength; } +/** + * plpks_is_available() - Get the PLPKS availability status for the LPAR. + * + * The availability of PLPKS is inferred based upon the successful execution of + * the H_PKS_GET_CONFIG HCALL provided the firmware supports this feature. The + * H_PKS_GET_CONFIG HCALL reads the configuration and status information related + * to the PLPKS. The configuration structure provides a version number to inform + * the caller of the supported features. + * + * Returns: true is returned if PLPKS is available, false if not. + */ bool plpks_is_available(void) { int rc; @@ -425,6 +550,35 @@ static int plpks_confirm_object_flushed(struct label *label, return pseries_status_to_err(rc); } +/** + * plpks_signed_update_var() - Update the specified authenticated variable. + * @var: authenticated variable to be updated + * @flags: signed update request operation flags + * + * The H_PKS_SIGNED_UPDATE HCALL performs a signed update to an object in the + * PLPKS. The object must have the signed update policy flag set. + * + * Possible reasons for the returned errno values: + * + * -ENXIO if PLPKS is not supported + * -EIO if PLPKS access is blocked due to the LPAR's state + * if PLPKS modification is blocked due to the LPAR's state + * if an error occurred while processing the request + * -EINVAL if invalid authorization parameter + * if invalid object label parameter + * if invalid object label len parameter + * if invalid or unsupported policy declaration + * if invalid signed update flags + * if invalid input data parameter + * if invalid input data len parameter + * if invalid continue token parameter + * -EPERM if access is denied + * -ENOMEM if there is inadequate memory to perform the operation + * -EBUSY if unable to handle the request or long running operation + * initiated, retry later + * + * Returns: On success 0 is returned, a negative errno if not. + */ int plpks_signed_update_var(struct plpks_var *var, u64 flags) { unsigned long retbuf[PLPAR_HCALL9_BUFSIZE] = {0}; @@ -481,6 +635,33 @@ out: return rc; } +/** + * plpks_write_var() - Write the specified variable and its data to PLPKS. + * @var: variable to be written into the PLPKS + * + * The H_PKS_WRITE_OBJECT HCALL writes an object into the PLPKS. The caller must + * provide a valid component type for the variable, and the signed update policy + * flag must not be set. + * + * Possible reasons for the returned errno values: + * + * -ENXIO if PLPKS is not supported + * -EIO if PLPKS access is blocked due to the LPAR's state + * if PLPKS modification is blocked due to the LPAR's state + * if an error occurred while processing the request + * -EINVAL if invalid authorization parameter + * if invalid object label parameter + * if invalid object label len parameter + * if invalid or unsupported policy declaration + * if invalid input data parameter + * if invalid input data len parameter + * -EPERM if access is denied + * -ENOMEM if unable to store the requested object in the space available + * -EBUSY if unable to handle the request + * -EEXIST if the object label already exists + * + * Returns: On success 0 is returned, a negative errno if not. + */ int plpks_write_var(struct plpks_var var) { unsigned long retbuf[PLPAR_HCALL_BUFSIZE] = { 0 }; @@ -520,6 +701,30 @@ out: return rc; } +/** + * plpks_remove_var() - Remove the specified variable and its data from PLPKS. + * @component: metadata prefix in the object label metadata structure + * @varos: metadata OS flags in the object label metadata structure + * @vname: object label for the object that needs to be removed + * + * The H_PKS_REMOVE_OBJECT HCALL removes an object from the PLPKS. The removal + * is independent of the policy bits that are set. + * + * Possible reasons for the returned errno values: + * + * -ENXIO if PLPKS is not supported + * -EIO if PLPKS access is blocked due to the LPAR's state + * if PLPKS modification is blocked due to the LPAR's state + * if an error occurred while processing the request + * -EINVAL if invalid authorization parameter + * if invalid object label parameter + * if invalid object label len parameter + * -EPERM if access is denied + * -ENOENT if the requested object was not found + * -EBUSY if unable to handle the request + * + * Returns: On success 0 is returned, a negative errno if not. + */ int plpks_remove_var(char *component, u8 varos, struct plpks_var_name vname) { unsigned long retbuf[PLPAR_HCALL_BUFSIZE] = { 0 }; @@ -619,21 +824,119 @@ out_free_auth: return rc; } +/** + * plpks_read_os_var() - Fetch the data for the specified variable that is + * owned by the OS consumer. + * @var: variable to be read from the PLPKS + * + * The consumer or the owner of the object is the os kernel. The + * H_PKS_READ_OBJECT HCALL reads an object from the PLPKS. The caller must + * allocate the buffer var->data and specify the length for this buffer in + * var->datalen. If no buffer is provided, var->datalen will be populated with + * the requested object's size. + * + * Possible reasons for the returned errno values: + * + * -ENXIO if PLPKS is not supported + * -EIO if PLPKS access is blocked due to the LPAR's state + * if an error occurred while processing the request + * -EINVAL if invalid authorization parameter + * if invalid object label parameter + * if invalid object label len parameter + * if invalid output data parameter + * if invalid output data len parameter + * -EPERM if access is denied + * -ENOENT if the requested object was not found + * -EFBIG if the requested object couldn't be + * stored in the buffer provided + * -EBUSY if unable to handle the request + * + * Returns: On success 0 is returned, a negative errno if not. + */ int plpks_read_os_var(struct plpks_var *var) { return plpks_read_var(PLPKS_OS_OWNER, var); } +/** + * plpks_read_fw_var() - Fetch the data for the specified variable that is + * owned by the firmware consumer. + * @var: variable to be read from the PLPKS + * + * The consumer or the owner of the object is the firmware. The + * H_PKS_READ_OBJECT HCALL reads an object from the PLPKS. The caller must + * allocate the buffer var->data and specify the length for this buffer in + * var->datalen. If no buffer is provided, var->datalen will be populated with + * the requested object's size. + * + * Possible reasons for the returned errno values: + * + * -ENXIO if PLPKS is not supported + * -EIO if PLPKS access is blocked due to the LPAR's state + * if an error occurred while processing the request + * -EINVAL if invalid authorization parameter + * if invalid object label parameter + * if invalid object label len parameter + * if invalid output data parameter + * if invalid output data len parameter + * -EPERM if access is denied + * -ENOENT if the requested object was not found + * -EFBIG if the requested object couldn't be + * stored in the buffer provided + * -EBUSY if unable to handle the request + * + * Returns: On success 0 is returned, a negative errno if not. + */ int plpks_read_fw_var(struct plpks_var *var) { return plpks_read_var(PLPKS_FW_OWNER, var); } +/** + * plpks_read_bootloader_var() - Fetch the data for the specified variable + * owned by the bootloader consumer. + * @var: variable to be read from the PLPKS + * + * The consumer or the owner of the object is the bootloader. The + * H_PKS_READ_OBJECT HCALL reads an object from the PLPKS. The caller must + * allocate the buffer var->data and specify the length for this buffer in + * var->datalen. If no buffer is provided, var->datalen will be populated with + * the requested object's size. + * + * Possible reasons for the returned errno values: + * + * -ENXIO if PLPKS is not supported + * -EIO if PLPKS access is blocked due to the LPAR's state + * if an error occurred while processing the request + * -EINVAL if invalid authorization parameter + * if invalid object label parameter + * if invalid object label len parameter + * if invalid output data parameter + * if invalid output data len parameter + * -EPERM if access is denied + * -ENOENT if the requested object was not found + * -EFBIG if the requested object couldn't be + * stored in the buffer provided + * -EBUSY if unable to handle the request + * + * Returns: On success 0 is returned, a negative errno if not. + */ int plpks_read_bootloader_var(struct plpks_var *var) { return plpks_read_var(PLPKS_BOOTLOADER_OWNER, var); } +/** + * plpks_populate_fdt(): Populates the FDT with the PLPKS password to prepare + * for kexec. + * @fdt: pointer to the device tree blob + * + * Upon confirming the existence of the chosen node, invoke fdt_setprop to + * populate the device tree with the PLPKS password in order to prepare for + * kexec. + * + * Returns: On success 0 is returned, a negative value if not. + */ int plpks_populate_fdt(void *fdt) { int chosen_offset = fdt_path_offset(fdt, "/chosen"); @@ -647,14 +950,19 @@ int plpks_populate_fdt(void *fdt) return fdt_setprop(fdt, chosen_offset, "ibm,plpks-pw", ospassword, ospasswordlength); } -// Once a password is registered with the hypervisor it cannot be cleared without -// rebooting the LPAR, so to keep using the PLPKS across kexec boots we need to -// recover the previous password from the FDT. -// -// There are a few challenges here. We don't want the password to be visible to -// users, so we need to clear it from the FDT. This has to be done in early boot. -// Clearing it from the FDT would make the FDT's checksum invalid, so we have to -// manually cause the checksum to be recalculated. +/** + * plpks_early_init_devtree() - Retrieves and clears the PLPKS password from the + * DT in early init. + * + * Once a password is registered with the hypervisor it cannot be cleared + * without rebooting the LPAR, so to keep using the PLPKS across kexec boots we + * need to recover the previous password from the FDT. + * + * There are a few challenges here. We don't want the password to be visible to + * users, so we need to clear it from the FDT. This has to be done in early + * boot. Clearing it from the FDT would make the FDT's checksum invalid, so we + * have to manually cause the checksum to be recalculated. + */ void __init plpks_early_init_devtree(void) { void *fdt = initial_boot_params; From 40850c909fcf2a66237dea6b96c8e12003cf6d43 Mon Sep 17 00:00:00 2001 From: Srish Srinivasan Date: Tue, 27 Jan 2026 20:22:24 +0530 Subject: [PATCH 22/27] powerpc/pseries: move the PLPKS config inside its own sysfs directory The /sys/firmware/secvar/config directory represents Power LPAR Platform KeyStore (PLPKS) configuration properties such as max_object_size, signed_ update_algorithms, supported_policies, total_size, used_space, and version. These attributes describe the PLPKS, and not the secure boot variables (secvars). Create /sys/firmware/plpks directory and move the PLPKS config inside this directory. For backwards compatibility, create a soft link from the secvar sysfs directory to this config and emit a warning stating that the older sysfs path has been deprecated. Separate out the plpks specific documentation from secvar. Signed-off-by: Srish Srinivasan Tested-by: Nayna Jain Reviewed-by: Mimi Zohar Reviewed-by: Nayna Jain Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20260127145228.48320-3-ssrish@linux.ibm.com --- .../ABI/testing/sysfs-firmware-plpks | 50 ++++++++++ Documentation/ABI/testing/sysfs-secvar | 65 ------------- arch/powerpc/include/asm/plpks.h | 5 + arch/powerpc/include/asm/secvar.h | 1 - arch/powerpc/kernel/secvar-sysfs.c | 21 ++--- arch/powerpc/platforms/pseries/Makefile | 2 +- arch/powerpc/platforms/pseries/plpks-secvar.c | 29 ------ arch/powerpc/platforms/pseries/plpks-sysfs.c | 94 +++++++++++++++++++ 8 files changed, 156 insertions(+), 111 deletions(-) create mode 100644 Documentation/ABI/testing/sysfs-firmware-plpks create mode 100644 arch/powerpc/platforms/pseries/plpks-sysfs.c diff --git a/Documentation/ABI/testing/sysfs-firmware-plpks b/Documentation/ABI/testing/sysfs-firmware-plpks new file mode 100644 index 000000000000..af0353f34115 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-firmware-plpks @@ -0,0 +1,50 @@ +What: /sys/firmware/plpks/config +Date: February 2023 +Contact: Nayna Jain +Description: This optional directory contains read-only config attributes as + defined by the PLPKS implementation. All data is in ASCII + format. + +What: /sys/firmware/plpks/config/version +Date: February 2023 +Contact: Nayna Jain +Description: Config version as reported by the hypervisor in ASCII decimal + format. + +What: /sys/firmware/plpks/config/max_object_size +Date: February 2023 +Contact: Nayna Jain +Description: Maximum allowed size of objects in the keystore in bytes, + represented in ASCII decimal format. + + This is not necessarily the same as the max size that can be + written to an update file as writes can contain more than + object data, you should use the size of the update file for + that purpose. + +What: /sys/firmware/plpks/config/total_size +Date: February 2023 +Contact: Nayna Jain +Description: Total size of the PLPKS in bytes, represented in ASCII decimal + format. + +What: /sys/firmware/plpks/config/used_space +Date: February 2023 +Contact: Nayna Jain +Description: Current space consumed by the key store, in bytes, represented + in ASCII decimal format. + +What: /sys/firmware/plpks/config/supported_policies +Date: February 2023 +Contact: Nayna Jain +Description: Bitmask of supported policy flags by the hypervisor, represented + as an 8 byte hexadecimal ASCII string. Consult the hypervisor + documentation for what these flags are. + +What: /sys/firmware/plpks/config/signed_update_algorithms +Date: February 2023 +Contact: Nayna Jain +Description: Bitmask of flags indicating which algorithms the hypervisor + supports for signed update of objects, represented as a 16 byte + hexadecimal ASCII string. Consult the hypervisor documentation + for what these flags mean. diff --git a/Documentation/ABI/testing/sysfs-secvar b/Documentation/ABI/testing/sysfs-secvar index 1016967a730f..c52a5fd15709 100644 --- a/Documentation/ABI/testing/sysfs-secvar +++ b/Documentation/ABI/testing/sysfs-secvar @@ -63,68 +63,3 @@ Contact: Nayna Jain Description: A write-only file that is used to submit the new value for the variable. The size of the file represents the maximum size of the variable data that can be written. - -What: /sys/firmware/secvar/config -Date: February 2023 -Contact: Nayna Jain -Description: This optional directory contains read-only config attributes as - defined by the secure variable implementation. All data is in - ASCII format. The directory is only created if the backing - implementation provides variables to populate it, which at - present is only PLPKS on the pseries platform. - -What: /sys/firmware/secvar/config/version -Date: February 2023 -Contact: Nayna Jain -Description: Config version as reported by the hypervisor in ASCII decimal - format. - - Currently only provided by PLPKS on the pseries platform. - -What: /sys/firmware/secvar/config/max_object_size -Date: February 2023 -Contact: Nayna Jain -Description: Maximum allowed size of objects in the keystore in bytes, - represented in ASCII decimal format. - - This is not necessarily the same as the max size that can be - written to an update file as writes can contain more than - object data, you should use the size of the update file for - that purpose. - - Currently only provided by PLPKS on the pseries platform. - -What: /sys/firmware/secvar/config/total_size -Date: February 2023 -Contact: Nayna Jain -Description: Total size of the PLPKS in bytes, represented in ASCII decimal - format. - - Currently only provided by PLPKS on the pseries platform. - -What: /sys/firmware/secvar/config/used_space -Date: February 2023 -Contact: Nayna Jain -Description: Current space consumed by the key store, in bytes, represented - in ASCII decimal format. - - Currently only provided by PLPKS on the pseries platform. - -What: /sys/firmware/secvar/config/supported_policies -Date: February 2023 -Contact: Nayna Jain -Description: Bitmask of supported policy flags by the hypervisor, - represented as an 8 byte hexadecimal ASCII string. Consult the - hypervisor documentation for what these flags are. - - Currently only provided by PLPKS on the pseries platform. - -What: /sys/firmware/secvar/config/signed_update_algorithms -Date: February 2023 -Contact: Nayna Jain -Description: Bitmask of flags indicating which algorithms the hypervisor - supports for signed update of objects, represented as a 16 byte - hexadecimal ASCII string. Consult the hypervisor documentation - for what these flags mean. - - Currently only provided by PLPKS on the pseries platform. diff --git a/arch/powerpc/include/asm/plpks.h b/arch/powerpc/include/asm/plpks.h index f303922bf622..8829a13bfda0 100644 --- a/arch/powerpc/include/asm/plpks.h +++ b/arch/powerpc/include/asm/plpks.h @@ -13,6 +13,7 @@ #include #include +#include // Object policy flags from supported_policies #define PLPKS_OSSECBOOTAUDIT PPC_BIT32(1) // OS secure boot must be audit/enforce @@ -107,11 +108,15 @@ u16 plpks_get_passwordlen(void); void plpks_early_init_devtree(void); int plpks_populate_fdt(void *fdt); + +int plpks_config_create_softlink(struct kobject *from); #else // CONFIG_PSERIES_PLPKS static inline bool plpks_is_available(void) { return false; } static inline u16 plpks_get_passwordlen(void) { BUILD_BUG(); } static inline void plpks_early_init_devtree(void) { } static inline int plpks_populate_fdt(void *fdt) { BUILD_BUG(); } +static inline int plpks_config_create_softlink(struct kobject *from) + { return 0; } #endif // CONFIG_PSERIES_PLPKS #endif // _ASM_POWERPC_PLPKS_H diff --git a/arch/powerpc/include/asm/secvar.h b/arch/powerpc/include/asm/secvar.h index 4828e0ab7e3c..fd5006307f2a 100644 --- a/arch/powerpc/include/asm/secvar.h +++ b/arch/powerpc/include/asm/secvar.h @@ -20,7 +20,6 @@ struct secvar_operations { int (*set)(const char *key, u64 key_len, u8 *data, u64 data_size); ssize_t (*format)(char *buf, size_t bufsize); int (*max_size)(u64 *max_size); - const struct attribute **config_attrs; // NULL-terminated array of fixed variable names // Only used if get_next() isn't provided diff --git a/arch/powerpc/kernel/secvar-sysfs.c b/arch/powerpc/kernel/secvar-sysfs.c index ec900bce0257..4111b21962eb 100644 --- a/arch/powerpc/kernel/secvar-sysfs.c +++ b/arch/powerpc/kernel/secvar-sysfs.c @@ -12,6 +12,7 @@ #include #include #include +#include #define NAME_MAX_SIZE 1024 @@ -145,19 +146,6 @@ static __init int update_kobj_size(void) return 0; } -static __init int secvar_sysfs_config(struct kobject *kobj) -{ - struct attribute_group config_group = { - .name = "config", - .attrs = (struct attribute **)secvar_ops->config_attrs, - }; - - if (secvar_ops->config_attrs) - return sysfs_create_group(kobj, &config_group); - - return 0; -} - static __init int add_var(const char *name) { struct kobject *kobj; @@ -260,12 +248,15 @@ static __init int secvar_sysfs_init(void) goto err; } - rc = secvar_sysfs_config(secvar_kobj); + rc = plpks_config_create_softlink(secvar_kobj); if (rc) { - pr_err("Failed to create config directory\n"); + pr_err("Failed to create softlink to PLPKS config directory"); goto err; } + pr_info("/sys/firmware/secvar/config is now deprecated.\n"); + pr_info("Will be removed in future versions.\n"); + if (secvar_ops->get_next) rc = secvar_sysfs_load(); else diff --git a/arch/powerpc/platforms/pseries/Makefile b/arch/powerpc/platforms/pseries/Makefile index 931ebaa474c8..3ced289a675b 100644 --- a/arch/powerpc/platforms/pseries/Makefile +++ b/arch/powerpc/platforms/pseries/Makefile @@ -30,7 +30,7 @@ obj-$(CONFIG_PAPR_SCM) += papr_scm.o obj-$(CONFIG_PPC_SPLPAR) += vphn.o obj-$(CONFIG_PPC_SVM) += svm.o obj-$(CONFIG_FA_DUMP) += rtas-fadump.o -obj-$(CONFIG_PSERIES_PLPKS) += plpks.o +obj-$(CONFIG_PSERIES_PLPKS) += plpks.o plpks-sysfs.o obj-$(CONFIG_PPC_SECURE_BOOT) += plpks-secvar.o obj-$(CONFIG_PSERIES_PLPKS_SED) += plpks_sed_ops.o obj-$(CONFIG_SUSPEND) += suspend.o diff --git a/arch/powerpc/platforms/pseries/plpks-secvar.c b/arch/powerpc/platforms/pseries/plpks-secvar.c index f9e9cc40c9d0..a50ff6943d80 100644 --- a/arch/powerpc/platforms/pseries/plpks-secvar.c +++ b/arch/powerpc/platforms/pseries/plpks-secvar.c @@ -20,33 +20,6 @@ #include #include -// Config attributes for sysfs -#define PLPKS_CONFIG_ATTR(name, fmt, func) \ - static ssize_t name##_show(struct kobject *kobj, \ - struct kobj_attribute *attr, \ - char *buf) \ - { \ - return sysfs_emit(buf, fmt, func()); \ - } \ - static struct kobj_attribute attr_##name = __ATTR_RO(name) - -PLPKS_CONFIG_ATTR(version, "%u\n", plpks_get_version); -PLPKS_CONFIG_ATTR(max_object_size, "%u\n", plpks_get_maxobjectsize); -PLPKS_CONFIG_ATTR(total_size, "%u\n", plpks_get_totalsize); -PLPKS_CONFIG_ATTR(used_space, "%u\n", plpks_get_usedspace); -PLPKS_CONFIG_ATTR(supported_policies, "%08x\n", plpks_get_supportedpolicies); -PLPKS_CONFIG_ATTR(signed_update_algorithms, "%016llx\n", plpks_get_signedupdatealgorithms); - -static const struct attribute *config_attrs[] = { - &attr_version.attr, - &attr_max_object_size.attr, - &attr_total_size.attr, - &attr_used_space.attr, - &attr_supported_policies.attr, - &attr_signed_update_algorithms.attr, - NULL, -}; - static u32 get_policy(const char *name) { if ((strcmp(name, "db") == 0) || @@ -225,7 +198,6 @@ static const struct secvar_operations plpks_secvar_ops_static = { .set = plpks_set_variable, .format = plpks_secvar_format, .max_size = plpks_max_size, - .config_attrs = config_attrs, .var_names = plpks_var_names_static, }; @@ -234,7 +206,6 @@ static const struct secvar_operations plpks_secvar_ops_dynamic = { .set = plpks_set_variable, .format = plpks_secvar_format, .max_size = plpks_max_size, - .config_attrs = config_attrs, .var_names = plpks_var_names_dynamic, }; diff --git a/arch/powerpc/platforms/pseries/plpks-sysfs.c b/arch/powerpc/platforms/pseries/plpks-sysfs.c new file mode 100644 index 000000000000..01d526185783 --- /dev/null +++ b/arch/powerpc/platforms/pseries/plpks-sysfs.c @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025 IBM Corporation, Srish Srinivasan + * + * This code exposes PLPKS config to user via sysfs + */ + +#define pr_fmt(fmt) "plpks-sysfs: "fmt + +#include +#include +#include +#include +#include + +/* config attributes for sysfs */ +#define PLPKS_CONFIG_ATTR(name, fmt, func) \ + static ssize_t name##_show(struct kobject *kobj, \ + struct kobj_attribute *attr, \ + char *buf) \ + { \ + return sysfs_emit(buf, fmt, func()); \ + } \ + static struct kobj_attribute attr_##name = __ATTR_RO(name) + +PLPKS_CONFIG_ATTR(version, "%u\n", plpks_get_version); +PLPKS_CONFIG_ATTR(max_object_size, "%u\n", plpks_get_maxobjectsize); +PLPKS_CONFIG_ATTR(total_size, "%u\n", plpks_get_totalsize); +PLPKS_CONFIG_ATTR(used_space, "%u\n", plpks_get_usedspace); +PLPKS_CONFIG_ATTR(supported_policies, "%08x\n", plpks_get_supportedpolicies); +PLPKS_CONFIG_ATTR(signed_update_algorithms, "%016llx\n", + plpks_get_signedupdatealgorithms); + +static const struct attribute *config_attrs[] = { + &attr_version.attr, + &attr_max_object_size.attr, + &attr_total_size.attr, + &attr_used_space.attr, + &attr_supported_policies.attr, + &attr_signed_update_algorithms.attr, + NULL, +}; + +static struct kobject *plpks_kobj, *plpks_config_kobj; + +int plpks_config_create_softlink(struct kobject *from) +{ + if (!plpks_config_kobj) + return -EINVAL; + return sysfs_create_link(from, plpks_config_kobj, "config"); +} + +static __init int plpks_sysfs_config(struct kobject *kobj) +{ + struct attribute_group config_group = { + .name = NULL, + .attrs = (struct attribute **)config_attrs, + }; + + return sysfs_create_group(kobj, &config_group); +} + +static __init int plpks_sysfs_init(void) +{ + int rc; + + if (!plpks_is_available()) + return -ENODEV; + + plpks_kobj = kobject_create_and_add("plpks", firmware_kobj); + if (!plpks_kobj) { + pr_err("Failed to create plpks kobj\n"); + return -ENOMEM; + } + + plpks_config_kobj = kobject_create_and_add("config", plpks_kobj); + if (!plpks_config_kobj) { + pr_err("Failed to create plpks config kobj\n"); + kobject_put(plpks_kobj); + return -ENOMEM; + } + + rc = plpks_sysfs_config(plpks_config_kobj); + if (rc) { + pr_err("Failed to create attribute group for plpks config\n"); + kobject_put(plpks_config_kobj); + kobject_put(plpks_kobj); + return rc; + } + + return 0; +} + +machine_subsys_initcall(pseries, plpks_sysfs_init); From 447eb1d5ef00f7da918221ed690cac980385d993 Mon Sep 17 00:00:00 2001 From: Srish Srinivasan Date: Tue, 27 Jan 2026 20:22:25 +0530 Subject: [PATCH 23/27] pseries/plpks: expose PowerVM wrapping features via the sysfs Starting with Power11, PowerVM supports a new feature called "Key Wrapping" that protects user secrets by wrapping them using a hypervisor generated wrapping key. The status of this feature can be read by the H_PKS_GET_CONFIG HCALL. Expose the Power LPAR Platform KeyStore (PLPKS) wrapping features config via the sysfs file /sys/firmware/plpks/config/wrapping_features. Signed-off-by: Srish Srinivasan Tested-by: Nayna Jain Reviewed-by: Nayna Jain Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20260127145228.48320-4-ssrish@linux.ibm.com --- .../ABI/testing/sysfs-firmware-plpks | 8 ++++++++ arch/powerpc/include/asm/hvcall.h | 4 +++- arch/powerpc/include/asm/plpks.h | 3 +++ arch/powerpc/platforms/pseries/plpks-sysfs.c | 2 ++ arch/powerpc/platforms/pseries/plpks.c | 20 +++++++++++++++++++ 5 files changed, 36 insertions(+), 1 deletion(-) diff --git a/Documentation/ABI/testing/sysfs-firmware-plpks b/Documentation/ABI/testing/sysfs-firmware-plpks index af0353f34115..cba061e4eee2 100644 --- a/Documentation/ABI/testing/sysfs-firmware-plpks +++ b/Documentation/ABI/testing/sysfs-firmware-plpks @@ -48,3 +48,11 @@ Description: Bitmask of flags indicating which algorithms the hypervisor supports for signed update of objects, represented as a 16 byte hexadecimal ASCII string. Consult the hypervisor documentation for what these flags mean. + +What: /sys/firmware/plpks/config/wrapping_features +Date: November 2025 +Contact: Srish Srinivasan +Description: Bitmask of the wrapping features indicating the wrapping + algorithms that are supported for the H_PKS_WRAP_OBJECT requests + , represented as a 8 byte hexadecimal ASCII string. Consult the + hypervisor documentation for what these flags mean. diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h index 9aef16149d92..dff90a7d7f70 100644 --- a/arch/powerpc/include/asm/hvcall.h +++ b/arch/powerpc/include/asm/hvcall.h @@ -360,7 +360,9 @@ #define H_GUEST_RUN_VCPU 0x480 #define H_GUEST_COPY_MEMORY 0x484 #define H_GUEST_DELETE 0x488 -#define MAX_HCALL_OPCODE H_GUEST_DELETE +#define H_PKS_WRAP_OBJECT 0x490 +#define H_PKS_UNWRAP_OBJECT 0x494 +#define MAX_HCALL_OPCODE H_PKS_UNWRAP_OBJECT /* Scope args for H_SCM_UNBIND_ALL */ #define H_UNBIND_SCOPE_ALL (0x1) diff --git a/arch/powerpc/include/asm/plpks.h b/arch/powerpc/include/asm/plpks.h index 8829a13bfda0..8f034588fdf7 100644 --- a/arch/powerpc/include/asm/plpks.h +++ b/arch/powerpc/include/asm/plpks.h @@ -23,6 +23,7 @@ #define PLPKS_IMMUTABLE PPC_BIT32(5) // Once written, object cannot be removed #define PLPKS_TRANSIENT PPC_BIT32(6) // Object does not persist through reboot #define PLPKS_SIGNEDUPDATE PPC_BIT32(7) // Object can only be modified by signed updates +#define PLPKS_WRAPPINGKEY PPC_BIT32(8) // Object contains a wrapping key #define PLPKS_HVPROVISIONED PPC_BIT32(28) // Hypervisor has provisioned this object // Signature algorithm flags from signed_update_algorithms @@ -103,6 +104,8 @@ u32 plpks_get_maxlargeobjectsize(void); u64 plpks_get_signedupdatealgorithms(void); +u64 plpks_get_wrappingfeatures(void); + u16 plpks_get_passwordlen(void); void plpks_early_init_devtree(void); diff --git a/arch/powerpc/platforms/pseries/plpks-sysfs.c b/arch/powerpc/platforms/pseries/plpks-sysfs.c index 01d526185783..c2ebcbb41ae3 100644 --- a/arch/powerpc/platforms/pseries/plpks-sysfs.c +++ b/arch/powerpc/platforms/pseries/plpks-sysfs.c @@ -30,6 +30,7 @@ PLPKS_CONFIG_ATTR(used_space, "%u\n", plpks_get_usedspace); PLPKS_CONFIG_ATTR(supported_policies, "%08x\n", plpks_get_supportedpolicies); PLPKS_CONFIG_ATTR(signed_update_algorithms, "%016llx\n", plpks_get_signedupdatealgorithms); +PLPKS_CONFIG_ATTR(wrapping_features, "%016llx\n", plpks_get_wrappingfeatures); static const struct attribute *config_attrs[] = { &attr_version.attr, @@ -38,6 +39,7 @@ static const struct attribute *config_attrs[] = { &attr_used_space.attr, &attr_supported_policies.attr, &attr_signed_update_algorithms.attr, + &attr_wrapping_features.attr, NULL, }; diff --git a/arch/powerpc/platforms/pseries/plpks.c b/arch/powerpc/platforms/pseries/plpks.c index 03722fabf9c3..4a08f51537c8 100644 --- a/arch/powerpc/platforms/pseries/plpks.c +++ b/arch/powerpc/platforms/pseries/plpks.c @@ -38,6 +38,7 @@ static u32 usedspace; static u32 supportedpolicies; static u32 maxlargeobjectsize; static u64 signedupdatealgorithms; +static u64 wrappingfeatures; struct plpks_auth { u8 version; @@ -248,6 +249,7 @@ static int _plpks_get_config(void) __be32 supportedpolicies; __be32 maxlargeobjectsize; __be64 signedupdatealgorithms; + __be64 wrappingfeatures; u8 rsvd1[476]; } __packed * config; size_t size; @@ -280,6 +282,7 @@ static int _plpks_get_config(void) supportedpolicies = be32_to_cpu(config->supportedpolicies); maxlargeobjectsize = be32_to_cpu(config->maxlargeobjectsize); signedupdatealgorithms = be64_to_cpu(config->signedupdatealgorithms); + wrappingfeatures = be64_to_cpu(config->wrappingfeatures); // Validate that the numbers we get back match the requirements of the spec if (maxpwsize < 32) { @@ -472,6 +475,23 @@ u64 plpks_get_signedupdatealgorithms(void) return signedupdatealgorithms; } +/** + * plpks_get_wrappingfeatures() - Returns a bitmask of the wrapping features + * supported by the hypervisor. + * + * Successful execution of the H_PKS_GET_CONFIG HCALL during initialization + * reads a bitmask of the wrapping features supported by the hypervisor into the + * file local static wrappingfeatures variable. This is valid only when the + * PLPKS config structure version >= 3. + * + * Return: + * bitmask of the wrapping features supported by the hypervisor + */ +u64 plpks_get_wrappingfeatures(void) +{ + return wrappingfeatures; +} + /** * plpks_get_passwordlen() - Get the length of the PLPKS password in bytes. * From 133aa79e211d2572c5c9b9671461a55d50bebda8 Mon Sep 17 00:00:00 2001 From: Srish Srinivasan Date: Tue, 27 Jan 2026 20:22:26 +0530 Subject: [PATCH 24/27] pseries/plpks: add HCALLs for PowerVM Key Wrapping Module The hypervisor generated wrapping key is an AES-GCM-256 symmetric key which is stored in a non-volatile, secure, and encrypted storage called the Power LPAR Platform KeyStore. It has policy based protections that prevent it from being read out or exposed to the user. Implement H_PKS_GEN_KEY, H_PKS_WRAP_OBJECT, and H_PKS_UNWRAP_OBJECT HCALLs to enable using the PowerVM Key Wrapping Module (PKWM) as a new trust source for trusted keys. Disallow H_PKS_READ_OBJECT, H_PKS_SIGNED_UPDATE, and H_PKS_WRITE_OBJECT for objects with the 'wrapping key' policy set. Capture the availability status for the H_PKS_WRAP_OBJECT interface. Signed-off-by: Srish Srinivasan Tested-by: Nayna Jain Reviewed-by: Nayna Jain Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20260127145228.48320-5-ssrish@linux.ibm.com --- Documentation/arch/powerpc/papr_hcalls.rst | 43 +++ arch/powerpc/include/asm/plpks.h | 10 + arch/powerpc/platforms/pseries/plpks.c | 343 ++++++++++++++++++++- 3 files changed, 394 insertions(+), 2 deletions(-) diff --git a/Documentation/arch/powerpc/papr_hcalls.rst b/Documentation/arch/powerpc/papr_hcalls.rst index 805e1cb9bab9..14e39f095a1c 100644 --- a/Documentation/arch/powerpc/papr_hcalls.rst +++ b/Documentation/arch/powerpc/papr_hcalls.rst @@ -300,6 +300,49 @@ H_HTM supports setup, configuration, control and dumping of Hardware Trace Macro (HTM) function and its data. HTM buffer stores tracing data for functions like core instruction, core LLAT and nest. +**H_PKS_GEN_KEY** + +| Input: authorization, objectlabel, objectlabellen, policy, out, outlen +| Out: *Hypervisor Generated Key, or None when the wrapping key policy is set* +| Return Value: *H_SUCCESS, H_Function, H_State, H_R_State, H_Parameter, H_P2, + H_P3, H_P4, H_P5, H_P6, H_Authority, H_Nomem, H_Busy, H_Resource, + H_Aborted* + +H_PKS_GEN_KEY is used to have the hypervisor generate a new random key. +This key is stored as an object in the Power LPAR Platform KeyStore with +the provided object label. With the wrapping key policy set the key is only +visible to the hypervisor, while the key's label would still be visible to +the user. Generation of wrapping keys is supported only for a key size of +32 bytes. + +**H_PKS_WRAP_OBJECT** + +| Input: authorization, wrapkeylabel, wrapkeylabellen, objectwrapflags, in, +| inlen, out, outlen, continue-token +| Out: *continue-token, byte size of wrapped object, wrapped object* +| Return Value: *H_SUCCESS, H_Function, H_State, H_R_State, H_Parameter, H_P2, + H_P3, H_P4, H_P5, H_P6, H_P7, H_P8, H_P9, H_Authority, H_Invalid_Key, + H_NOT_FOUND, H_Busy, H_LongBusy, H_Aborted* + +H_PKS_WRAP_OBJECT is used to wrap an object using a wrapping key stored in the +Power LPAR Platform KeyStore and return the wrapped object to the caller. The +caller provides a label to a wrapping key with the 'wrapping key' policy set, +which must have been previously created with H_PKS_GEN_KEY. The provided object +is then encrypted with the wrapping key and additional metadata and the result +is returned to the caller. + + +**H_PKS_UNWRAP_OBJECT** + +| Input: authorization, objectwrapflags, in, inlen, out, outlen, continue-token +| Out: *continue-token, byte size of unwrapped object, unwrapped object* +| Return Value: *H_SUCCESS, H_Function, H_State, H_R_State, H_Parameter, H_P2, + H_P3, H_P4, H_P5, H_P6, H_P7, H_Authority, H_Unsupported, H_Bad_Data, + H_NOT_FOUND, H_Invalid_Key, H_Busy, H_LongBusy, H_Aborted* + +H_PKS_UNWRAP_OBJECT is used to unwrap an object that was previously warapped with +H_PKS_WRAP_OBJECT. + References ========== .. [1] "Power Architecture Platform Reference" diff --git a/arch/powerpc/include/asm/plpks.h b/arch/powerpc/include/asm/plpks.h index 8f034588fdf7..e87f90e40d4e 100644 --- a/arch/powerpc/include/asm/plpks.h +++ b/arch/powerpc/include/asm/plpks.h @@ -113,6 +113,16 @@ void plpks_early_init_devtree(void); int plpks_populate_fdt(void *fdt); int plpks_config_create_softlink(struct kobject *from); + +bool plpks_wrapping_is_supported(void); + +int plpks_gen_wrapping_key(void); + +int plpks_wrap_object(u8 **input_buf, u32 input_len, u16 wrap_flags, + u8 **output_buf, u32 *output_len); + +int plpks_unwrap_object(u8 **input_buf, u32 input_len, + u8 **output_buf, u32 *output_len); #else // CONFIG_PSERIES_PLPKS static inline bool plpks_is_available(void) { return false; } static inline u16 plpks_get_passwordlen(void) { BUILD_BUG(); } diff --git a/arch/powerpc/platforms/pseries/plpks.c b/arch/powerpc/platforms/pseries/plpks.c index 4a08f51537c8..038ecf8f6d6b 100644 --- a/arch/powerpc/platforms/pseries/plpks.c +++ b/arch/powerpc/platforms/pseries/plpks.c @@ -9,6 +9,32 @@ #define pr_fmt(fmt) "plpks: " fmt +#define PLPKS_WRAPKEY_COMPONENT "PLPKSWR" +#define PLPKS_WRAPKEY_NAME "default-wrapping-key" + +/* + * To 4K align the {input, output} buffers to the {UN}WRAP H_CALLs + */ +#define PLPKS_WRAPPING_BUF_ALIGN 4096 + +/* + * To ensure the output buffer's length is at least 1024 bytes greater + * than the input buffer's length during the WRAP H_CALL + */ +#define PLPKS_WRAPPING_BUF_DIFF 1024 + +#define PLPKS_WRAP_INTERFACE_BIT 3 +#define PLPKS_WRAPPING_KEY_LENGTH 32 + +#define WRAPFLAG_BE_BIT_SET(be_bit) \ + BIT_ULL(63 - (be_bit)) + +#define WRAPFLAG_BE_GENMASK(be_bit_hi, be_bit_lo) \ + GENMASK_ULL(63 - (be_bit_hi), 63 - (be_bit_lo)) + +#define WRAPFLAG_BE_FIELD_PREP(be_bit_hi, be_bit_lo, val) \ + FIELD_PREP(WRAPFLAG_BE_GENMASK(be_bit_hi, be_bit_lo), (val)) + #include #include #include @@ -19,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -39,6 +66,7 @@ static u32 supportedpolicies; static u32 maxlargeobjectsize; static u64 signedupdatealgorithms; static u64 wrappingfeatures; +static bool wrapsupport; struct plpks_auth { u8 version; @@ -283,6 +311,7 @@ static int _plpks_get_config(void) maxlargeobjectsize = be32_to_cpu(config->maxlargeobjectsize); signedupdatealgorithms = be64_to_cpu(config->signedupdatealgorithms); wrappingfeatures = be64_to_cpu(config->wrappingfeatures); + wrapsupport = config->flags & PPC_BIT8(PLPKS_WRAP_INTERFACE_BIT); // Validate that the numbers we get back match the requirements of the spec if (maxpwsize < 32) { @@ -614,6 +643,9 @@ int plpks_signed_update_var(struct plpks_var *var, u64 flags) if (!(var->policy & PLPKS_SIGNEDUPDATE)) return -EINVAL; + if (var->policy & PLPKS_WRAPPINGKEY) + return -EINVAL; + // Signed updates need the component to be NULL. if (var->component) return -EINVAL; @@ -696,6 +728,9 @@ int plpks_write_var(struct plpks_var var) if (var.policy & PLPKS_SIGNEDUPDATE) return -EINVAL; + if (var.policy & PLPKS_WRAPPINGKEY) + return -EINVAL; + auth = construct_auth(PLPKS_OS_OWNER); if (IS_ERR(auth)) return PTR_ERR(auth); @@ -790,6 +825,9 @@ static int plpks_read_var(u8 consumer, struct plpks_var *var) if (var->namelen > PLPKS_MAX_NAME_SIZE) return -EINVAL; + if (var->policy & PLPKS_WRAPPINGKEY) + return -EINVAL; + auth = construct_auth(consumer); if (IS_ERR(auth)) return PTR_ERR(auth); @@ -845,8 +883,309 @@ out_free_auth: } /** - * plpks_read_os_var() - Fetch the data for the specified variable that is - * owned by the OS consumer. + * plpks_wrapping_is_supported() - Get the H_PKS_WRAP_OBJECT interface + * availability status for the LPAR. + * + * Successful execution of the H_PKS_GET_CONFIG HCALL during initialization + * sets bit 3 of the flags variable in the PLPKS config structure if the + * H_PKS_WRAP_OBJECT interface is supported. + * + * Returns: true if the H_PKS_WRAP_OBJECT interface is supported, false if not. + */ +bool plpks_wrapping_is_supported(void) +{ + return wrapsupport; +} + +/** + * plpks_gen_wrapping_key() - Generate a new random key with the 'wrapping key' + * policy set. + * + * The H_PKS_GEN_KEY HCALL makes the hypervisor generate a new random key and + * store the key in a PLPKS object with the provided object label. With the + * 'wrapping key' policy set, only the label to the newly generated random key + * would be visible to the user. + * + * Possible reasons for the returned errno values: + * + * -ENXIO if PLPKS is not supported + * -EIO if PLPKS access is blocked due to the LPAR's state + * if PLPKS modification is blocked due to the LPAR's state + * if an error occurred while processing the request + * -EINVAL if invalid authorization parameter + * if invalid object label parameter + * if invalid object label len parameter + * if invalid or unsupported policy declaration + * if invalid output buffer parameter + * if invalid output buffer length parameter + * -EPERM if access is denied + * -ENOMEM if there is inadequate memory to perform this operation + * -EBUSY if unable to handle the request + * -EEXIST if the object label already exists + * + * Returns: On success 0 is returned, a negative errno if not. + */ +int plpks_gen_wrapping_key(void) +{ + unsigned long retbuf[PLPAR_HCALL_BUFSIZE] = { 0 }; + struct plpks_auth *auth; + struct label *label; + int rc = 0, pseries_status = 0; + struct plpks_var var = { + .name = PLPKS_WRAPKEY_NAME, + .namelen = strlen(var.name), + .policy = PLPKS_WRAPPINGKEY, + .os = PLPKS_VAR_LINUX, + .component = PLPKS_WRAPKEY_COMPONENT + }; + + auth = construct_auth(PLPKS_OS_OWNER); + if (IS_ERR(auth)) + return PTR_ERR(auth); + + label = construct_label(var.component, var.os, var.name, var.namelen); + if (IS_ERR(label)) { + rc = PTR_ERR(label); + goto out; + } + + rc = plpar_hcall(H_PKS_GEN_KEY, retbuf, + virt_to_phys(auth), virt_to_phys(label), + label->size, var.policy, + NULL, PLPKS_WRAPPING_KEY_LENGTH); + + if (!rc) + rc = plpks_confirm_object_flushed(label, auth); + + pseries_status = rc; + rc = pseries_status_to_err(rc); + + if (rc && rc != -EEXIST) { + pr_err("H_PKS_GEN_KEY failed. pseries_status=%d, rc=%d", + pseries_status, rc); + } else { + rc = 0; + } + + kfree(label); +out: + kfree(auth); + return rc; +} +EXPORT_SYMBOL_GPL(plpks_gen_wrapping_key); + +/** + * plpks_wrap_object() - Wrap an object using the default wrapping key stored in + * the PLPKS. + * @input_buf: buffer containing the data to be wrapped + * @input_len: length of the input buffer + * @wrap_flags: object wrapping flags + * @output_buf: buffer to store the wrapped data + * @output_len: length of the output buffer + * + * The H_PKS_WRAP_OBJECT HCALL wraps an object using a wrapping key stored in + * the PLPKS and returns the wrapped object to the caller. The caller provides a + * label to the wrapping key with the 'wrapping key' policy set that must have + * been previously created with the H_PKS_GEN_KEY HCALL. The provided object is + * then encrypted with the wrapping key and additional metadata and the result + * is returned to the user. The metadata includes the wrapping algorithm and the + * wrapping key name so those parameters are not required during unwrap. + * + * Possible reasons for the returned errno values: + * + * -ENXIO if PLPKS is not supported + * -EIO if PLPKS access is blocked due to the LPAR's state + * if PLPKS modification is blocked due to the LPAR's state + * if an error occurred while processing the request + * -EINVAL if invalid authorization parameter + * if invalid wrapping key label parameter + * if invalid wrapping key label length parameter + * if invalid or unsupported object wrapping flags + * if invalid input buffer parameter + * if invalid input buffer length parameter + * if invalid output buffer parameter + * if invalid output buffer length parameter + * if invalid continue token parameter + * if the wrapping key is not compatible with the wrapping + * algorithm + * -EPERM if access is denied + * -ENOENT if the requested wrapping key was not found + * -EBUSY if unable to handle the request or long running operation + * initiated, retry later. + * + * Returns: On success 0 is returned, a negative errno if not. + */ +int plpks_wrap_object(u8 **input_buf, u32 input_len, u16 wrap_flags, + u8 **output_buf, u32 *output_len) +{ + unsigned long retbuf[PLPAR_HCALL9_BUFSIZE] = { 0 }; + struct plpks_auth *auth; + struct label *label; + u64 continuetoken = 0; + u64 objwrapflags = 0; + int rc = 0, pseries_status = 0; + bool sb_audit_or_enforce_bit = wrap_flags & BIT(0); + bool sb_enforce_bit = wrap_flags & BIT(1); + struct plpks_var var = { + .name = PLPKS_WRAPKEY_NAME, + .namelen = strlen(var.name), + .os = PLPKS_VAR_LINUX, + .component = PLPKS_WRAPKEY_COMPONENT + }; + + auth = construct_auth(PLPKS_OS_OWNER); + if (IS_ERR(auth)) + return PTR_ERR(auth); + + label = construct_label(var.component, var.os, var.name, var.namelen); + if (IS_ERR(label)) { + rc = PTR_ERR(label); + goto out; + } + + /* Set the consumer password requirement bit. A must have. */ + objwrapflags |= WRAPFLAG_BE_BIT_SET(3); + + /* Set the wrapping algorithm bit. Just one algorithm option for now */ + objwrapflags |= WRAPFLAG_BE_FIELD_PREP(60, 63, 0x1); + + if (sb_audit_or_enforce_bit & sb_enforce_bit) { + pr_err("Cannot set both audit/enforce and enforce bits."); + rc = -EINVAL; + goto out_free_label; + } else if (sb_audit_or_enforce_bit) { + objwrapflags |= WRAPFLAG_BE_BIT_SET(1); + } else if (sb_enforce_bit) { + objwrapflags |= WRAPFLAG_BE_BIT_SET(2); + } + + *output_len = input_len + PLPKS_WRAPPING_BUF_DIFF; + + *output_buf = kzalloc(ALIGN(*output_len, PLPKS_WRAPPING_BUF_ALIGN), + GFP_KERNEL); + if (!(*output_buf)) { + pr_err("Output buffer allocation failed. Returning -ENOMEM."); + rc = -ENOMEM; + goto out_free_label; + } + + do { + rc = plpar_hcall9(H_PKS_WRAP_OBJECT, retbuf, + virt_to_phys(auth), virt_to_phys(label), + label->size, objwrapflags, + virt_to_phys(*input_buf), input_len, + virt_to_phys(*output_buf), *output_len, + continuetoken); + + continuetoken = retbuf[0]; + pseries_status = rc; + rc = pseries_status_to_err(rc); + } while (rc == -EBUSY); + + if (rc) { + pr_err("H_PKS_WRAP_OBJECT failed. pseries_status=%d, rc=%d", + pseries_status, rc); + kfree(*output_buf); + *output_buf = NULL; + } else { + *output_len = retbuf[1]; + } + +out_free_label: + kfree(label); +out: + kfree(auth); + return rc; +} +EXPORT_SYMBOL_GPL(plpks_wrap_object); + +/** + * plpks_unwrap_object() - Unwrap an object using the default wrapping key + * stored in the PLPKS. + * @input_buf: buffer containing the data to be unwrapped + * @input_len: length of the input buffer + * @output_buf: buffer to store the unwrapped data + * @output_len: length of the output buffer + * + * The H_PKS_UNWRAP_OBJECT HCALL unwraps an object that was previously wrapped + * using the H_PKS_WRAP_OBJECT HCALL. + * + * Possible reasons for the returned errno values: + * + * -ENXIO if PLPKS is not supported + * -EIO if PLPKS access is blocked due to the LPAR's state + * if PLPKS modification is blocked due to the LPAR's state + * if an error occurred while processing the request + * -EINVAL if invalid authorization parameter + * if invalid or unsupported object unwrapping flags + * if invalid input buffer parameter + * if invalid input buffer length parameter + * if invalid output buffer parameter + * if invalid output buffer length parameter + * if invalid continue token parameter + * if the wrapping key is not compatible with the wrapping + * algorithm + * if the wrapped object's format is not supported + * if the wrapped object is invalid + * -EPERM if access is denied + * -ENOENT if the wrapping key for the provided object was not found + * -EBUSY if unable to handle the request or long running operation + * initiated, retry later. + * + * Returns: On success 0 is returned, a negative errno if not. + */ +int plpks_unwrap_object(u8 **input_buf, u32 input_len, u8 **output_buf, + u32 *output_len) +{ + unsigned long retbuf[PLPAR_HCALL9_BUFSIZE] = { 0 }; + struct plpks_auth *auth; + u64 continuetoken = 0; + u64 objwrapflags = 0; + int rc = 0, pseries_status = 0; + + auth = construct_auth(PLPKS_OS_OWNER); + if (IS_ERR(auth)) + return PTR_ERR(auth); + + *output_len = input_len - PLPKS_WRAPPING_BUF_DIFF; + *output_buf = kzalloc(ALIGN(*output_len, PLPKS_WRAPPING_BUF_ALIGN), + GFP_KERNEL); + if (!(*output_buf)) { + pr_err("Output buffer allocation failed. Returning -ENOMEM."); + rc = -ENOMEM; + goto out; + } + + do { + rc = plpar_hcall9(H_PKS_UNWRAP_OBJECT, retbuf, + virt_to_phys(auth), objwrapflags, + virt_to_phys(*input_buf), input_len, + virt_to_phys(*output_buf), *output_len, + continuetoken); + + continuetoken = retbuf[0]; + pseries_status = rc; + rc = pseries_status_to_err(rc); + } while (rc == -EBUSY); + + if (rc) { + pr_err("H_PKS_UNWRAP_OBJECT failed. pseries_status=%d, rc=%d", + pseries_status, rc); + kfree(*output_buf); + *output_buf = NULL; + } else { + *output_len = retbuf[1]; + } + +out: + kfree(auth); + return rc; +} +EXPORT_SYMBOL_GPL(plpks_unwrap_object); + +/** + * plpks_read_os_var() - Fetch the data for the specified variable that is owned + * by the OS consumer. * @var: variable to be read from the PLPKS * * The consumer or the owner of the object is the os kernel. The From c99fcb0d735bdb6f06dfe6eb7134d5d988d32dae Mon Sep 17 00:00:00 2001 From: Srish Srinivasan Date: Tue, 27 Jan 2026 20:22:27 +0530 Subject: [PATCH 25/27] keys/trusted_keys: establish PKWM as a trusted source The wrapping key does not exist by default and is generated by the hypervisor as a part of PKWM initialization. This key is then persisted by the hypervisor and is used to wrap trusted keys. These are variable length symmetric keys, which in the case of PowerVM Key Wrapping Module (PKWM) are generated using the kernel RNG. PKWM can be used as a trust source through the following example keyctl commands: keyctl add trusted my_trusted_key "new 32" @u Use the wrap_flags command option to set the secure boot requirement for the wrapping request through the following keyctl commands case1: no secure boot requirement. (default) keyctl usage: keyctl add trusted my_trusted_key "new 32" @u OR keyctl add trusted my_trusted_key "new 32 wrap_flags=0x00" @u case2: secure boot required to in either audit or enforce mode. set bit 0 keyctl usage: keyctl add trusted my_trusted_key "new 32 wrap_flags=0x01" @u case3: secure boot required to be in enforce mode. set bit 1 keyctl usage: keyctl add trusted my_trusted_key "new 32 wrap_flags=0x02" @u NOTE: -> Setting the secure boot requirement is NOT a must. -> Only either of the secure boot requirement options should be set. Not both. -> All the other bits are required to be not set. -> Set the kernel parameter trusted.source=pkwm to choose PKWM as the backend for trusted keys implementation. -> CONFIG_PSERIES_PLPKS must be enabled to build PKWM. Add PKWM, which is a combination of IBM PowerVM and Power LPAR Platform KeyStore, as a new trust source for trusted keys. Signed-off-by: Srish Srinivasan Tested-by: Nayna Jain Reviewed-by: Mimi Zohar Reviewed-by: Nayna Jain Reviewed-by: Jarkko Sakkinen Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20260127145228.48320-6-ssrish@linux.ibm.com --- MAINTAINERS | 9 + include/keys/trusted-type.h | 7 +- include/keys/trusted_pkwm.h | 33 ++++ security/keys/trusted-keys/Kconfig | 8 + security/keys/trusted-keys/Makefile | 2 + security/keys/trusted-keys/trusted_core.c | 6 +- security/keys/trusted-keys/trusted_pkwm.c | 190 ++++++++++++++++++++++ 7 files changed, 253 insertions(+), 2 deletions(-) create mode 100644 include/keys/trusted_pkwm.h create mode 100644 security/keys/trusted-keys/trusted_pkwm.c diff --git a/MAINTAINERS b/MAINTAINERS index 765ad2daa218..103c79f47672 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -14003,6 +14003,15 @@ S: Supported F: include/keys/trusted_dcp.h F: security/keys/trusted-keys/trusted_dcp.c +KEYS-TRUSTED-PLPKS +M: Srish Srinivasan +M: Nayna Jain +L: linux-integrity@vger.kernel.org +L: keyrings@vger.kernel.org +S: Supported +F: include/keys/trusted_pkwm.h +F: security/keys/trusted-keys/trusted_pkwm.c + KEYS-TRUSTED-TEE M: Sumit Garg L: linux-integrity@vger.kernel.org diff --git a/include/keys/trusted-type.h b/include/keys/trusted-type.h index 4eb64548a74f..03527162613f 100644 --- a/include/keys/trusted-type.h +++ b/include/keys/trusted-type.h @@ -19,7 +19,11 @@ #define MIN_KEY_SIZE 32 #define MAX_KEY_SIZE 128 -#define MAX_BLOB_SIZE 512 +#if IS_ENABLED(CONFIG_TRUSTED_KEYS_PKWM) +#define MAX_BLOB_SIZE 1152 +#else +#define MAX_BLOB_SIZE 512 +#endif #define MAX_PCRINFO_SIZE 64 #define MAX_DIGEST_SIZE 64 @@ -46,6 +50,7 @@ struct trusted_key_options { uint32_t policydigest_len; unsigned char policydigest[MAX_DIGEST_SIZE]; uint32_t policyhandle; + void *private; }; struct trusted_key_ops { diff --git a/include/keys/trusted_pkwm.h b/include/keys/trusted_pkwm.h new file mode 100644 index 000000000000..4035b9776394 --- /dev/null +++ b/include/keys/trusted_pkwm.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PKWM_TRUSTED_KEY_H +#define __PKWM_TRUSTED_KEY_H + +#include +#include +#include + +extern struct trusted_key_ops pkwm_trusted_key_ops; + +struct trusted_pkwm_options { + u16 wrap_flags; +}; + +static inline void dump_options(struct trusted_key_options *o) +{ + const struct trusted_pkwm_options *pkwm; + bool sb_audit_or_enforce_bit; + bool sb_enforce_bit; + + pkwm = o->private; + sb_audit_or_enforce_bit = pkwm->wrap_flags & BIT(0); + sb_enforce_bit = pkwm->wrap_flags & BIT(1); + + if (sb_audit_or_enforce_bit) + pr_debug("secure boot mode required: audit or enforce"); + else if (sb_enforce_bit) + pr_debug("secure boot mode required: enforce"); + else + pr_debug("secure boot mode required: disabled"); +} + +#endif diff --git a/security/keys/trusted-keys/Kconfig b/security/keys/trusted-keys/Kconfig index 204a68c1429d..9e00482d886a 100644 --- a/security/keys/trusted-keys/Kconfig +++ b/security/keys/trusted-keys/Kconfig @@ -46,6 +46,14 @@ config TRUSTED_KEYS_DCP help Enable use of NXP's DCP (Data Co-Processor) as trusted key backend. +config TRUSTED_KEYS_PKWM + bool "PKWM-based trusted keys" + depends on PSERIES_PLPKS >= TRUSTED_KEYS + default y + select HAVE_TRUSTED_KEYS + help + Enable use of IBM PowerVM Key Wrapping Module (PKWM) as a trusted key backend. + if !HAVE_TRUSTED_KEYS comment "No trust source selected!" endif diff --git a/security/keys/trusted-keys/Makefile b/security/keys/trusted-keys/Makefile index f0f3b27f688b..5fc053a21dad 100644 --- a/security/keys/trusted-keys/Makefile +++ b/security/keys/trusted-keys/Makefile @@ -16,3 +16,5 @@ trusted-$(CONFIG_TRUSTED_KEYS_TEE) += trusted_tee.o trusted-$(CONFIG_TRUSTED_KEYS_CAAM) += trusted_caam.o trusted-$(CONFIG_TRUSTED_KEYS_DCP) += trusted_dcp.o + +trusted-$(CONFIG_TRUSTED_KEYS_PKWM) += trusted_pkwm.o diff --git a/security/keys/trusted-keys/trusted_core.c b/security/keys/trusted-keys/trusted_core.c index b1680ee53f86..2d328de170e8 100644 --- a/security/keys/trusted-keys/trusted_core.c +++ b/security/keys/trusted-keys/trusted_core.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -31,7 +32,7 @@ MODULE_PARM_DESC(rng, "Select trusted key RNG"); static char *trusted_key_source; module_param_named(source, trusted_key_source, charp, 0); -MODULE_PARM_DESC(source, "Select trusted keys source (tpm, tee, caam or dcp)"); +MODULE_PARM_DESC(source, "Select trusted keys source (tpm, tee, caam, dcp or pkwm)"); static const struct trusted_key_source trusted_key_sources[] = { #if defined(CONFIG_TRUSTED_KEYS_TPM) @@ -46,6 +47,9 @@ static const struct trusted_key_source trusted_key_sources[] = { #if defined(CONFIG_TRUSTED_KEYS_DCP) { "dcp", &dcp_trusted_key_ops }, #endif +#if defined(CONFIG_TRUSTED_KEYS_PKWM) + { "pkwm", &pkwm_trusted_key_ops }, +#endif }; DEFINE_STATIC_CALL_NULL(trusted_key_seal, *trusted_key_sources[0].ops->seal); diff --git a/security/keys/trusted-keys/trusted_pkwm.c b/security/keys/trusted-keys/trusted_pkwm.c new file mode 100644 index 000000000000..4f391b77a907 --- /dev/null +++ b/security/keys/trusted-keys/trusted_pkwm.c @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025 IBM Corporation, Srish Srinivasan + */ + +#include +#include +#include +#include +#include +#include + +enum { + Opt_err, + Opt_wrap_flags, +}; + +static const match_table_t key_tokens = { + {Opt_wrap_flags, "wrap_flags=%s"}, + {Opt_err, NULL} +}; + +static int getoptions(char *datablob, struct trusted_key_options *opt) +{ + substring_t args[MAX_OPT_ARGS]; + char *p = datablob; + int token; + int res; + u16 wrap_flags; + unsigned long token_mask = 0; + struct trusted_pkwm_options *pkwm; + + if (!datablob) + return 0; + + pkwm = opt->private; + + while ((p = strsep(&datablob, " \t"))) { + if (*p == '\0' || *p == ' ' || *p == '\t') + continue; + + token = match_token(p, key_tokens, args); + if (test_and_set_bit(token, &token_mask)) + return -EINVAL; + + switch (token) { + case Opt_wrap_flags: + res = kstrtou16(args[0].from, 16, &wrap_flags); + if (res < 0 || wrap_flags > 2) + return -EINVAL; + pkwm->wrap_flags = wrap_flags; + break; + default: + return -EINVAL; + } + } + return 0; +} + +static struct trusted_key_options *trusted_options_alloc(void) +{ + struct trusted_key_options *options; + struct trusted_pkwm_options *pkwm; + + options = kzalloc(sizeof(*options), GFP_KERNEL); + + if (options) { + pkwm = kzalloc(sizeof(*pkwm), GFP_KERNEL); + + if (!pkwm) { + kfree_sensitive(options); + options = NULL; + } else { + options->private = pkwm; + } + } + + return options; +} + +static int trusted_pkwm_seal(struct trusted_key_payload *p, char *datablob) +{ + struct trusted_key_options *options = NULL; + struct trusted_pkwm_options *pkwm = NULL; + u8 *input_buf, *output_buf; + u32 output_len, input_len; + int rc; + + options = trusted_options_alloc(); + + if (!options) + return -ENOMEM; + + rc = getoptions(datablob, options); + if (rc < 0) + goto out; + dump_options(options); + + input_len = p->key_len; + input_buf = kmalloc(ALIGN(input_len, 4096), GFP_KERNEL); + if (!input_buf) { + pr_err("Input buffer allocation failed. Returning -ENOMEM."); + rc = -ENOMEM; + goto out; + } + + memcpy(input_buf, p->key, p->key_len); + + pkwm = options->private; + + rc = plpks_wrap_object(&input_buf, input_len, pkwm->wrap_flags, + &output_buf, &output_len); + if (!rc) { + memcpy(p->blob, output_buf, output_len); + p->blob_len = output_len; + dump_payload(p); + } else { + pr_err("Wrapping of payload key failed: %d\n", rc); + } + + kfree(input_buf); + kfree(output_buf); + +out: + kfree_sensitive(options->private); + kfree_sensitive(options); + return rc; +} + +static int trusted_pkwm_unseal(struct trusted_key_payload *p, char *datablob) +{ + u8 *input_buf, *output_buf; + u32 input_len, output_len; + int rc; + + input_len = p->blob_len; + input_buf = kmalloc(ALIGN(input_len, 4096), GFP_KERNEL); + if (!input_buf) { + pr_err("Input buffer allocation failed. Returning -ENOMEM."); + return -ENOMEM; + } + + memcpy(input_buf, p->blob, p->blob_len); + + rc = plpks_unwrap_object(&input_buf, input_len, &output_buf, + &output_len); + if (!rc) { + memcpy(p->key, output_buf, output_len); + p->key_len = output_len; + dump_payload(p); + } else { + pr_err("Unwrapping of payload failed: %d\n", rc); + } + + kfree(input_buf); + kfree(output_buf); + + return rc; +} + +static int trusted_pkwm_init(void) +{ + int ret; + + if (!plpks_wrapping_is_supported()) { + pr_err("H_PKS_WRAP_OBJECT interface not supported\n"); + return -ENODEV; + } + + ret = plpks_gen_wrapping_key(); + if (ret) { + pr_err("Failed to generate default wrapping key\n"); + return -EINVAL; + } + + return register_key_type(&key_type_trusted); +} + +static void trusted_pkwm_exit(void) +{ + unregister_key_type(&key_type_trusted); +} + +struct trusted_key_ops pkwm_trusted_key_ops = { + .migratable = 0, /* non-migratable */ + .init = trusted_pkwm_init, + .seal = trusted_pkwm_seal, + .unseal = trusted_pkwm_unseal, + .exit = trusted_pkwm_exit, +}; From 1d72a02d65a733ad124bacc2db3fb90fa81a612b Mon Sep 17 00:00:00 2001 From: Nayna Jain Date: Tue, 27 Jan 2026 20:22:28 +0530 Subject: [PATCH 26/27] docs: trusted-encryped: add PKWM as a new trust source Update Documentation/security/keys/trusted-encrypted.rst and Documentation/ admin-guide/kernel-parameters.txt with PowerVM Key Wrapping Module (PKWM) as a new trust source Signed-off-by: Nayna Jain Signed-off-by: Srish Srinivasan Tested-by: Nayna Jain Reviewed-by: Mimi Zohar Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20260127145228.48320-7-ssrish@linux.ibm.com --- .../admin-guide/kernel-parameters.txt | 1 + .../security/keys/trusted-encrypted.rst | 50 +++++++++++++++++++ 2 files changed, 51 insertions(+) diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index a8d0afde7f85..ccb9c2f502fb 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -7755,6 +7755,7 @@ Kernel parameters - "tee" - "caam" - "dcp" + - "pkwm" If not specified then it defaults to iterating through the trust source list starting with TPM and assigns the first trust source as a backend which is initialized diff --git a/Documentation/security/keys/trusted-encrypted.rst b/Documentation/security/keys/trusted-encrypted.rst index eae6a36b1c9a..ddff7c7c2582 100644 --- a/Documentation/security/keys/trusted-encrypted.rst +++ b/Documentation/security/keys/trusted-encrypted.rst @@ -81,6 +81,14 @@ safe. and the UNIQUE key. Default is to use the UNIQUE key, but selecting the OTP key can be done via a module parameter (dcp_use_otp_key). + (5) PKWM (PowerVM Key Wrapping Module: IBM PowerVM + Platform KeyStore) + + Rooted to a unique, per-LPAR key, which is derived from a system-wide, + randomly generated LPAR root key. Both the per-LPAR keys and the LPAR + root key are stored in hypervisor-owned secure memory at runtime, + and the LPAR root key is additionally persisted in secure locations + such as the processor SEEPROMs and encrypted NVRAM. + * Execution isolation (1) TPM @@ -102,6 +110,14 @@ safe. environment. Only basic blob key encryption is executed there. The actual key sealing/unsealing is done on main processor/kernel space. + (5) PKWM (PowerVM Key Wrapping Module: IBM PowerVM + Platform KeyStore) + + Fixed set of cryptographic operations done on on-chip hardware + cryptographic acceleration unit NX. Keys for wrapping and unwrapping + are managed by PowerVM Platform KeyStore, which stores keys in an + isolated in-memory copy in secure hypervisor memory, as well as in a + persistent copy in hypervisor-encrypted NVRAM. + * Optional binding to platform integrity state (1) TPM @@ -129,6 +145,11 @@ safe. Relies on Secure/Trusted boot process (called HAB by vendor) for platform integrity. + (5) PKWM (PowerVM Key Wrapping Module: IBM PowerVM + Platform KeyStore) + + Relies on secure and trusted boot process of IBM Power systems for + platform integrity. + * Interfaces and APIs (1) TPM @@ -149,6 +170,11 @@ safe. Vendor-specific API that is implemented as part of the DCP crypto driver in ``drivers/crypto/mxs-dcp.c``. + (5) PKWM (PowerVM Key Wrapping Module: IBM PowerVM + Platform KeyStore) + + Platform Keystore has well documented interfaces in PAPR document. + Refer to ``Documentation/arch/powerpc/papr_hcalls.rst`` + * Threat model The strength and appropriateness of a particular trust source for a given @@ -191,6 +217,10 @@ selected trust source: a dedicated hardware RNG that is independent from DCP which can be enabled to back the kernel RNG. + * PKWM (PowerVM Key Wrapping Module: IBM PowerVM + Platform KeyStore) + + The normal kernel random number generator is used to generate keys. + Users may override this by specifying ``trusted.rng=kernel`` on the kernel command-line to override the used RNG with the kernel's random number pool. @@ -321,6 +351,26 @@ Usage:: specific to this DCP key-blob implementation. The key length for new keys is always in bytes. Trusted Keys can be 32 - 128 bytes (256 - 1024 bits). +Trusted Keys usage: PKWM +------------------------ + +Usage:: + + keyctl add trusted name "new keylen [options]" ring + keyctl add trusted name "load hex_blob" ring + keyctl print keyid + + options: + wrap_flags= ascii hex value of security policy requirement + 0x00: no secure boot requirement (default) + 0x01: require secure boot to be in either audit or + enforced mode + 0x02: require secure boot to be in enforced mode + +"keyctl print" returns an ASCII hex copy of the sealed key, which is in format +specific to PKWM key-blob implementation. The key length for new keys is +always in bytes. Trusted Keys can be 32 - 128 bytes (256 - 1024 bits). + Encrypted Keys usage -------------------- From e2071f9ce42b6985a40feaf98d2f5f7ce0fca006 Mon Sep 17 00:00:00 2001 From: Srish Srinivasan Date: Sun, 1 Feb 2026 22:23:44 +0530 Subject: [PATCH 27/27] powerpc/pseries: plpks: export plpks_wrapping_is_supported Building trusted-keys as a module fails modpost with: ERROR: modpost: "plpks_wrapping_is_supported" [security/keys/trusted-keys/ trusted.ko] undefined! Export plpks_wrapping_is_supported() so trusted-keys links cleanly This patch is intended to be applied on top of the earlier "Extend "trusted " keys to support a new trust source named the PowerVM Key Wrapping Module (PKWM)" series (v5). Link: https://lore.kernel.org/all/20260127145228.48320-1-ssrish@linux.ibm.com/ Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202602010724.1g9hbLKv-lkp@intel.com/ Signed-off-by: Srish Srinivasan Tested-by: Venkat Rao Bagalkote Signed-off-by: Madhavan Srinivasan Link: https://patch.msgid.link/20260201165344.950870-1-ssrish@linux.ibm.com --- arch/powerpc/platforms/pseries/plpks.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/powerpc/platforms/pseries/plpks.c b/arch/powerpc/platforms/pseries/plpks.c index 038ecf8f6d6b..23e4e2a922fc 100644 --- a/arch/powerpc/platforms/pseries/plpks.c +++ b/arch/powerpc/platforms/pseries/plpks.c @@ -896,6 +896,7 @@ bool plpks_wrapping_is_supported(void) { return wrapsupport; } +EXPORT_SYMBOL_GPL(plpks_wrapping_is_supported); /** * plpks_gen_wrapping_key() - Generate a new random key with the 'wrapping key'