From 007e0f0fd9b77b9fa8f0d86253bac690f523460e Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Mon, 13 Jul 2026 14:13:20 +0000 Subject: [PATCH 01/33] KVM: arm64: Avoid naming collision in tracing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the hypervisor tracing (CONFIG_NVHE_EL2_TRACING) is disabled, it defines a static inline stub for trace_clock(). However, trace_clock() is already declared as an extern function in linux/trace_clock.h which is pulled in EL2 compilation. If the file is included when CONFIG_NVHE_EL2_TRACING is disabled (by including it manually in setup.c) it will cause: In file included from arch/arm64/kvm/hyp/nvhe/setup.c:22: ./arch/arm64/kvm/hyp/include/nvhe/clock.h:14:19: error: static declaration of ‘trace_clock’ follows non-static declaration 14 | static inline u64 trace_clock(void) { return 0; } | ^~~~~~~~~~~ on GCC and a linker error on LLVM (it seems to change the linkage to global) Although that is not a problem at the moment, as no other files include . That does not seem to be the intent of this code and that will cause issues with more users as the SMMUv3 driver. Signed-off-by: Mostafa Saleh Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Reviewed-by: Vincent Donnefort Link: https://patch.msgid.link/20260713141320.4065600-1-smostafa@google.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/include/nvhe/clock.h | 8 ++++---- arch/arm64/kvm/hyp/nvhe/clock.c | 4 ++-- arch/arm64/kvm/hyp/nvhe/trace.c | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/arch/arm64/kvm/hyp/include/nvhe/clock.h b/arch/arm64/kvm/hyp/include/nvhe/clock.h index 9f429f5c0664..ae03ec6965af 100644 --- a/arch/arm64/kvm/hyp/include/nvhe/clock.h +++ b/arch/arm64/kvm/hyp/include/nvhe/clock.h @@ -6,11 +6,11 @@ #include #ifdef CONFIG_NVHE_EL2_TRACING -void trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc); -u64 trace_clock(void); +void trace_hyp_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc); +u64 trace_hyp_clock(void); #else static inline void -trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) { } -static inline u64 trace_clock(void) { return 0; } +trace_hyp_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) { } +static inline u64 trace_hyp_clock(void) { return 0; } #endif #endif diff --git a/arch/arm64/kvm/hyp/nvhe/clock.c b/arch/arm64/kvm/hyp/nvhe/clock.c index a7fc61976fd0..f3e2619db4e4 100644 --- a/arch/arm64/kvm/hyp/nvhe/clock.c +++ b/arch/arm64/kvm/hyp/nvhe/clock.c @@ -30,7 +30,7 @@ static u64 __clock_mult_uint128(u64 cyc, u32 mult, u32 shift) } /* Does not guarantee no reader on the modified bank. */ -void trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) +void trace_hyp_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) { struct clock_data *clock = &trace_clock_data; u64 bank = clock->cur ^ 1; @@ -48,7 +48,7 @@ void trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) } /* Use untrusted host data */ -u64 trace_clock(void) +u64 trace_hyp_clock(void) { struct clock_data *clock = &trace_clock_data; u64 bank = smp_load_acquire(&clock->cur); diff --git a/arch/arm64/kvm/hyp/nvhe/trace.c b/arch/arm64/kvm/hyp/nvhe/trace.c index e7e150ab265f..65be6c9fb379 100644 --- a/arch/arm64/kvm/hyp/nvhe/trace.c +++ b/arch/arm64/kvm/hyp/nvhe/trace.c @@ -35,7 +35,7 @@ static bool hyp_trace_buffer_loaded(struct hyp_trace_buffer *trace_buffer) void *tracing_reserve_entry(unsigned long length) { return simple_ring_buffer_reserve(this_cpu_ptr(trace_buffer.simple_rbs), length, - trace_clock()); + trace_hyp_clock()); } void tracing_commit_entry(void) @@ -290,7 +290,7 @@ void __tracing_update_clock(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) } /* ...we can now override the old one and swap. */ - trace_clock_update(mult, shift, epoch_ns, epoch_cyc); + trace_hyp_clock_update(mult, shift, epoch_ns, epoch_cyc); } int __tracing_reset(unsigned int cpu) From 581829501360bda487b93ec7d7eff0da6296f187 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 6 Jul 2026 11:31:27 +0100 Subject: [PATCH 02/33] KVM: arm64: Drop the unused EL2-side is_created write init_pkvm_hyp_vm() sets is_created on the EL2-private VM struct, but the hypervisor never reads it: pkvm_hyp_vm_is_created() and every other consumer operate on the host's struct kvm, a distinct allocation from the EL2-private copy. The field is write-only at EL2. Remove the store; host-side is_created tracking is unaffected. Reviewed-by: Keir Fraser Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260706103129.706974-2-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/pkvm.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/arm64/kvm/hyp/nvhe/pkvm.c b/arch/arm64/kvm/hyp/nvhe/pkvm.c index 3b2c4fbc34d8..c54aa5031336 100644 --- a/arch/arm64/kvm/hyp/nvhe/pkvm.c +++ b/arch/arm64/kvm/hyp/nvhe/pkvm.c @@ -433,7 +433,6 @@ static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm, hyp_vm->host_kvm = host_kvm; hyp_vm->kvm.created_vcpus = nr_vcpus; hyp_vm->kvm.arch.pkvm.is_protected = READ_ONCE(host_kvm->arch.pkvm.is_protected); - hyp_vm->kvm.arch.pkvm.is_created = true; hyp_vm->kvm.arch.flags = 0; pkvm_init_features_from_host(hyp_vm, host_kvm); From 9b36eda4dfe0bcd76233b3f1318b66478c1d0698 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 6 Jul 2026 11:31:28 +0100 Subject: [PATCH 03/33] KVM: arm64: Remove unreachable early checks in pkvm_init_host_vm() pkvm_init_host_vm() runs once from kvm_arch_init_vm(), while the VM is still being allocated and is not yet reachable by another thread. Both early checks therefore test impossible state: is_created is still false (it is only set on first vCPU run) and the handle is still zero (this function is what reserves it). Neither branch can be taken. Remove them. Reviewed-by: Keir Fraser Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260706103129.706974-3-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/pkvm.c | 7 ------- 1 file changed, 7 deletions(-) diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 053e4f733e4b..67b90a58fbea 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -230,13 +230,6 @@ int pkvm_init_host_vm(struct kvm *kvm, unsigned long type) int ret; bool protected = type & KVM_VM_TYPE_ARM_PROTECTED; - if (pkvm_hyp_vm_is_created(kvm)) - return -EINVAL; - - /* VM is already reserved, no need to proceed. */ - if (kvm->arch.pkvm.handle) - return 0; - /* Reserve the VM in hyp and obtain a hyp handle for the VM. */ ret = kvm_call_hyp_nvhe(__pkvm_reserve_vm); if (ret < 0) From 008ea7a6aa01877a2439bc911426c27a03a6d9e3 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Mon, 6 Jul 2026 11:31:29 +0100 Subject: [PATCH 04/33] KVM: arm64: Drop redundant READ_ONCE() in pkvm_hyp_vm_is_created() is_created is written under config_lock. Every concurrent reader is serialised against that write: pkvm_create_hyp_vm() under config_lock, and the memslot path (kvm_arch_prepare_memory_region) via slots_lock, which the creation writer also holds. The teardown-path accesses have no concurrent writer. The read is therefore serialised, and the READ_ONCE() is unnecessary. Reviewed-by: Keir Fraser Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260706103129.706974-4-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/pkvm.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 67b90a58fbea..008766273912 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -185,7 +185,11 @@ free_pgd: bool pkvm_hyp_vm_is_created(struct kvm *kvm) { - return READ_ONCE(kvm->arch.pkvm.is_created); + /* + * Serialised by config_lock/slots_lock, or by VM lifecycle at + * teardown, so a plain read suffices. + */ + return kvm->arch.pkvm.is_created; } int pkvm_create_hyp_vm(struct kvm *kvm) From 719e67103c5e1486e06e340a03f33f1493fae914 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 21 Jul 2026 18:07:49 +0100 Subject: [PATCH 05/33] KVM: arm64: vgic-v3: Make vtr_to_* helpers use architectural field symbols vgic-v3-sr.c still contains some hardcoded constants for some of the ICH)VTR_EL2 fields. Bring them into the modern world by using the named symbols from the sysreg file. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260721170754.3150521-2-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/vgic-v3-sr.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/hyp/vgic-v3-sr.c b/arch/arm64/kvm/hyp/vgic-v3-sr.c index c4d2f1feea8b..8da7a40ced58 100644 --- a/arch/arm64/kvm/hyp/vgic-v3-sr.c +++ b/arch/arm64/kvm/hyp/vgic-v3-sr.c @@ -16,9 +16,9 @@ #include "../../vgic/vgic.h" -#define vtr_to_max_lr_idx(v) ((v) & 0xf) -#define vtr_to_nr_pre_bits(v) ((((u32)(v) >> 26) & 7) + 1) -#define vtr_to_nr_apr_regs(v) (1 << (vtr_to_nr_pre_bits(v) - 5)) +#define vtr_to_max_lr_idx(v) FIELD_GET(ICH_VTR_EL2_ListRegs, (v)) +#define vtr_to_nr_pre_bits(v) (FIELD_GET(ICH_VTR_EL2_PREbits, (v)) + 1) +#define vtr_to_nr_apr_regs(v) BIT(vtr_to_nr_pre_bits(v) - 5) u64 __gic_v3_get_lr(unsigned int lr) { From 27a263e1a0fa23374066696aa7732a01bfde7a4e Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 21 Jul 2026 18:07:50 +0100 Subject: [PATCH 06/33] KVM: arm64: Move GICv3 broken SEIS implementation detection to a CPU errrata Using is_midr_in_range_list() in a noinstr function isn't a good idea. And yet kvm_compute_ich_hcr_trap_bits() is doing that by calling vgic_v3_broken_seis(). Move all the broken SEIS detection logic to the errata detection framework, and use a new ARM64_WORKAROUND_GICv3_BROKEN_SEIS cap to indicate that we're running on broken CPUs. This reuses the MIDR list used for IMPDEF PMU detection, which has a 100% overlap with the SEIS stuff... Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260721170754.3150521-3-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kernel/cpu_errata.c | 17 +++++++++++++++-- arch/arm64/kvm/vgic/vgic-v3.c | 26 +++----------------------- arch/arm64/tools/cpucaps | 1 + 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c index 1995e1198648..78e4e105f6ab 100644 --- a/arch/arm64/kernel/cpu_errata.c +++ b/arch/arm64/kernel/cpu_errata.c @@ -272,7 +272,7 @@ has_neoverse_n1_erratum_1542419(const struct arm64_cpu_capabilities *entry, return is_midr_in_range(&range) && has_dic; } -static const struct midr_range impdef_pmuv3_cpus[] = { +static const struct midr_range apple_cpus[] = { MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM), MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM), MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM_PRO), @@ -301,7 +301,14 @@ static bool has_impdef_pmuv3(const struct arm64_cpu_capabilities *entry, int sco if (pmuver != ID_AA64DFR0_EL1_PMUVer_IMP_DEF) return false; - return is_midr_in_range_list(impdef_pmuv3_cpus); + return is_midr_in_range_list(apple_cpus); +} + +static bool has_broken_gic_v3_seis(const struct arm64_cpu_capabilities *entry, int scope) +{ + return (is_kernel_in_hyp_mode() && + is_midr_in_range_list(apple_cpus) && + (read_sysreg_s(SYS_ICH_VTR_EL2) & ICH_VTR_EL2_SEIS)); } static void cpu_enable_impdef_pmuv3_traps(const struct arm64_cpu_capabilities *__unused) @@ -1009,6 +1016,12 @@ const struct arm64_cpu_capabilities arm64_errata[] = { .matches = has_impdef_pmuv3, .cpu_enable = cpu_enable_impdef_pmuv3_traps, }, + { + .desc = "Known broken GICv3 SEIS implementation", + .capability = ARM64_WORKAROUND_GICv3_BROKEN_SEIS, + .type = ARM64_CPUCAP_SYSTEM_FEATURE, + .matches = has_broken_gic_v3_seis, + }, { } }; diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 9e841e7afd4a..0991a649a5d1 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -809,27 +809,9 @@ static int __init early_gicv4_enable(char *buf) } early_param("kvm-arm.vgic_v4_enable", early_gicv4_enable); -static const struct midr_range broken_seis[] = { - MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM), - MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM), - MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM_PRO), - MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM_PRO), - MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM_MAX), - MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM_MAX), - MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD), - MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE), - MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD_PRO), - MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE_PRO), - MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD_MAX), - MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE_MAX), - {}, -}; - -static bool vgic_v3_broken_seis(void) +static __always_inline bool vgic_v3_broken_seis(void) { - return (is_kernel_in_hyp_mode() && - is_midr_in_range_list(broken_seis) && - (read_sysreg_s(SYS_ICH_VTR_EL2) & ICH_VTR_EL2_SEIS)); + return cpus_have_cap(ARM64_WORKAROUND_GICv3_BROKEN_SEIS); } void noinstr kvm_compute_ich_hcr_trap_bits(struct alt_instr *alt, @@ -959,10 +941,8 @@ int vgic_v3_probe(const struct gic_kvm_info *info) if (has_v2) static_branch_enable(&vgic_v3_has_v2_compat); - if (vgic_v3_broken_seis()) { - kvm_info("GICv3 with broken locally generated SEI\n"); + if (vgic_v3_broken_seis()) kvm_vgic_global_state.ich_vtr_el2 &= ~ICH_VTR_EL2_SEIS; - } vgic_v3_enable_cpuif_traps(); diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps index 9b85a84f6fd4..50373264ecde 100644 --- a/arch/arm64/tools/cpucaps +++ b/arch/arm64/tools/cpucaps @@ -121,6 +121,7 @@ WORKAROUND_CAVIUM_TX2_219_TVM WORKAROUND_CLEAN_CACHE WORKAROUND_DEVICE_LOAD_ACQUIRE WORKAROUND_DISABLE_CNP +WORKAROUND_GICv3_BROKEN_SEIS WORKAROUND_PMUV3_IMPDEF_TRAPS WORKAROUND_QCOM_FALKOR_E1003 WORKAROUND_QCOM_ORYON_CNTVOFF From 4edfdc8ddebdac3243592f14cfec529304a9666a Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 21 Jul 2026 18:07:51 +0100 Subject: [PATCH 07/33] KVM: arm64: Add a helper providing an inlined literal value for ICH_VTR_EL2 We already have two ways to deal with ICH_VTR_EL2: - directly read the system register - read a cached copy in the vgic global state Add a third way, in the form of an inlined literal value that will eventually replace all of the above. This literal value is computed at boot time, and patched in the relevant code locations. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260721170754.3150521-4-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kernel/image-vars.h | 1 + arch/arm64/kvm/vgic/vgic-v3.c | 55 ++++++++++++++++++++++++++++++++++ arch/arm64/kvm/vgic/vgic.h | 17 +++++++++++ 3 files changed, 73 insertions(+) diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h index d4c7d45ae6bc..459c33634ebd 100644 --- a/arch/arm64/kernel/image-vars.h +++ b/arch/arm64/kernel/image-vars.h @@ -92,6 +92,7 @@ KVM_NVHE_ALIAS(spectre_bhb_patch_wa3); KVM_NVHE_ALIAS(spectre_bhb_patch_clearbhb); KVM_NVHE_ALIAS(alt_cb_patch_nops); KVM_NVHE_ALIAS(kvm_compute_ich_hcr_trap_bits); +KVM_NVHE_ALIAS(kvm_patch_ich_vtr_el2); /* Global kernel state accessed by nVHE hyp code. */ KVM_NVHE_ALIAS(kvm_vgic_global_state); diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 0991a649a5d1..098187e34c58 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -858,6 +858,61 @@ void noinstr kvm_compute_ich_hcr_trap_bits(struct alt_instr *alt, *updptr = cpu_to_le32(insn); } +void noinstr kvm_patch_ich_vtr_el2(struct alt_instr *alt, + __le32 *origptr, __le32 *updptr, + int nr_inst) +{ + struct arm_smccc_res res = {}; + u32 insn, oinsn, rd, vtr; + + /* No KVM? Nothing to do */ + if (!is_hyp_mode_available()) + return; + + /* No v3, compat, nor the fruity erzatz of a GIC? Bugger off */ + if (!cpus_have_cap(ARM64_HAS_GICV5_LEGACY) && + !cpus_have_cap(ARM64_HAS_GICV3_CPUIF) && + !vgic_v3_broken_seis()) + return; + + /* + * At the point where this is called, we are guaranteed that if + * we're running at EL1, then the EL2 stubs are still in place. + */ + if (is_kernel_in_hyp_mode()) + res.a1 = read_sysreg_s(SYS_ICH_VTR_EL2); + else + arm_smccc_1_1_hvc(HVC_GET_ICH_VTR_EL2, &res); + + if (res.a0 == HVC_STUB_ERR) + return; + + vtr = res.a1; + + if (vgic_v3_broken_seis()) + vtr &= ~ICH_VTR_EL2_SEIS; + + /* Compute target register */ + oinsn = le32_to_cpu(*origptr); + rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn); + + /* movz rd, #(vtr & 0xffff) */ + insn = aarch64_insn_gen_movewide(rd, + (u16)vtr, + 0, + AARCH64_INSN_VARIANT_64BIT, + AARCH64_INSN_MOVEWIDE_ZERO); + *updptr++ = cpu_to_le32(insn); + + /* movk rd, #((vtr >> 16) & 0xffff), lsl #16 */ + insn = aarch64_insn_gen_movewide(rd, + (u16)(vtr >> 16), + 16, + AARCH64_INSN_VARIANT_64BIT, + AARCH64_INSN_MOVEWIDE_KEEP); + *updptr++ = cpu_to_le32(insn); +} + void vgic_v3_enable_cpuif_traps(void) { u64 traps = vgic_ich_hcr_trap_bits(); diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index f45f7e3ec4d6..b76bb6e56de3 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -71,6 +71,23 @@ ICH_VTR_EL2_IDbits) #define KVM_ICH_VTR_EL2_RES1 ICH_VTR_EL2_nV4 +void kvm_patch_ich_vtr_el2(struct alt_instr *alt, + __le32 *origptr, __le32 *updptr, int nr_inst); + +static inline u64 vgic_ich_vtr(void) +{ + u64 vtr; + + /* All non-RES0 bits are in the bottom 32bits */ + asm volatile(ALTERNATIVE_CB("movz %0, #0\n" + "movk %0, #0, lsl #16\n", + ARM64_ALWAYS_SYSTEM, + kvm_patch_ich_vtr_el2) + : "=r" (vtr)); + + return vtr; +} + static inline u64 kvm_get_guest_vtr_el2(void) { u64 vtr; From 2bba3d2c60926f19e1d0da931513fead3af48f36 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 21 Jul 2026 18:07:52 +0100 Subject: [PATCH 08/33] KVM: arm64: Convert most ICH_VTR_EL2 accesses to inlined literal value Now that we have a new helper that inlines the value of ICH_VTR_EL2 into the code, use this to replace most of the sysreg accesses to that register. This avoids a lot of traps when running KVM under NV. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260721170754.3150521-5-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/vgic-v3-sr.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/arm64/kvm/hyp/vgic-v3-sr.c b/arch/arm64/kvm/hyp/vgic-v3-sr.c index 8da7a40ced58..3e5d5ddf35a1 100644 --- a/arch/arm64/kvm/hyp/vgic-v3-sr.c +++ b/arch/arm64/kvm/hyp/vgic-v3-sr.c @@ -367,7 +367,7 @@ void __vgic_v3_save_aprs(struct vgic_v3_cpu_if *cpu_if) u64 val; u32 nr_pre_bits; - val = read_gicreg(ICH_VTR_EL2); + val = vgic_ich_vtr(); nr_pre_bits = vtr_to_nr_pre_bits(val); switch (nr_pre_bits) { @@ -400,7 +400,7 @@ static void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if) u64 val; u32 nr_pre_bits; - val = read_gicreg(ICH_VTR_EL2); + val = vgic_ich_vtr(); nr_pre_bits = vtr_to_nr_pre_bits(val); switch (nr_pre_bits) { @@ -430,7 +430,7 @@ static void __vgic_v3_restore_aprs(struct vgic_v3_cpu_if *cpu_if) void __vgic_v3_init_lrs(void) { - int max_lr_idx = vtr_to_max_lr_idx(read_gicreg(ICH_VTR_EL2)); + int max_lr_idx = vtr_to_max_lr_idx(vgic_ich_vtr()); int i; for (i = 0; i <= max_lr_idx; i++) @@ -455,7 +455,7 @@ u64 __vgic_v3_get_gic_config(void) * system, so we first check if we have GICv5 support. */ if (cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) - return read_gicreg(ICH_VTR_EL2); + return vgic_ich_vtr(); sre = read_gicreg(ICC_SRE_EL1); /* @@ -498,7 +498,7 @@ u64 __vgic_v3_get_gic_config(void) } val = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63); - val |= read_gicreg(ICH_VTR_EL2); + val |= vgic_ich_vtr(); return val; } @@ -540,7 +540,7 @@ void __vgic_v3_restore_vmcr_aprs(struct vgic_v3_cpu_if *cpu_if) static int __vgic_v3_bpr_min(void) { /* See Pseudocode for VPriorityGroup */ - return 8 - vtr_to_nr_pre_bits(read_gicreg(ICH_VTR_EL2)); + return 8 - vtr_to_nr_pre_bits(vgic_ich_vtr()); } static int __vgic_v3_get_group(struct kvm_vcpu *vcpu) @@ -614,7 +614,7 @@ static int __vgic_v3_find_active_lr(struct kvm_vcpu *vcpu, int intid, static int __vgic_v3_get_highest_active_priority(void) { - u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2)); + u8 nr_apr_regs = vtr_to_nr_apr_regs(vgic_ich_vtr()); u32 hap = 0; int i; @@ -707,7 +707,7 @@ static void __vgic_v3_set_active_priority(u8 pri, u32 vmcr, int grp) static int __vgic_v3_clear_highest_active_priority(void) { - u8 nr_apr_regs = vtr_to_nr_apr_regs(read_gicreg(ICH_VTR_EL2)); + u8 nr_apr_regs = vtr_to_nr_apr_regs(vgic_ich_vtr()); u32 hap = 0; int i; @@ -1039,7 +1039,7 @@ static void __vgic_v3_read_ctlr(struct kvm_vcpu *vcpu, u32 vmcr, int rt) { u32 vtr, val; - vtr = read_gicreg(ICH_VTR_EL2); + vtr = vgic_ich_vtr(); /* PRIbits */ val = ((vtr >> 29) & 7) << ICC_CTLR_EL1_PRI_BITS_SHIFT; /* IDbits */ From bea608299d28ee4e10db3468985dde440adb09b7 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 21 Jul 2026 18:07:53 +0100 Subject: [PATCH 09/33] KVM: arm64: vgic-v3: Simplify initial GICv3 configuration sampling Now that we have our magic inline helper for ICH_VTR_EL2, we can get rid of the hack that was reporting a combination of that register and of the indication of the CPU interface supporting GICv2 compatibility. We now only report the latter. As a small benefit, GICv5 is not involved in this stuff anymore, since it never has GICv2 compatibility.. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260721170754.3150521-6-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_asm.h | 2 +- arch/arm64/kvm/hyp/vgic-v3-sr.c | 23 +++-------------------- arch/arm64/kvm/vgic/vgic-v3.c | 9 +++------ arch/arm64/kvm/vgic/vgic-v5.c | 2 +- 4 files changed, 8 insertions(+), 28 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 043495f7fc78..cb354a037fc5 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -281,7 +281,7 @@ extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu); extern void __kvm_adjust_pc(struct kvm_vcpu *vcpu); -extern u64 __vgic_v3_get_gic_config(void); +extern bool __vgic_v3_get_gic_config(void); extern void __vgic_v3_init_lrs(void); #define __KVM_EXTABLE(from, to) \ diff --git a/arch/arm64/kvm/hyp/vgic-v3-sr.c b/arch/arm64/kvm/hyp/vgic-v3-sr.c index 3e5d5ddf35a1..74d4a509f1de 100644 --- a/arch/arm64/kvm/hyp/vgic-v3-sr.c +++ b/arch/arm64/kvm/hyp/vgic-v3-sr.c @@ -437,26 +437,12 @@ void __vgic_v3_init_lrs(void) __gic_v3_set_lr(0, i); } -/* - * Return the GIC CPU configuration: - * - [31:0] ICH_VTR_EL2 - * - [62:32] RES0 - * - [63] MMIO (GICv2) capable - */ -u64 __vgic_v3_get_gic_config(void) +/* Return true if GICv3 is MMIO (GICv2) capable, false otherwise */ +bool __vgic_v3_get_gic_config(void) { u64 val, sre; unsigned long flags = 0; - /* - * In compat mode, we cannot access ICC_SRE_EL1 at any EL - * other than EL1 itself; just return the - * ICH_VTR_EL2. ICC_IDR0_EL1 is only implemented on a GICv5 - * system, so we first check if we have GICv5 support. - */ - if (cpus_have_final_cap(ARM64_HAS_GICV5_CPUIF)) - return vgic_ich_vtr(); - sre = read_gicreg(ICC_SRE_EL1); /* * To check whether we have a MMIO-based (GICv2 compatible) @@ -497,10 +483,7 @@ u64 __vgic_v3_get_gic_config(void) isb(); } - val = (val & ICC_SRE_EL1_SRE) ? 0 : (1ULL << 63); - val |= vgic_ich_vtr(); - - return val; + return !(val & ICC_SRE_EL1_SRE); } static void __vgic_v3_compat_mode_enable(void) diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 098187e34c58..734fa3138723 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -936,12 +936,12 @@ void vgic_v3_enable_cpuif_traps(void) */ int vgic_v3_probe(const struct gic_kvm_info *info) { - u64 ich_vtr_el2 = kvm_call_hyp_ret(__vgic_v3_get_gic_config); + u64 ich_vtr_el2; bool has_v2; int ret; - has_v2 = ich_vtr_el2 >> 63; - ich_vtr_el2 = (u32)ich_vtr_el2; + has_v2 = kvm_call_hyp_ret(__vgic_v3_get_gic_config); + ich_vtr_el2 = vgic_ich_vtr(); /* * The ListRegs field is 5 bits, but there is an architectural @@ -996,9 +996,6 @@ int vgic_v3_probe(const struct gic_kvm_info *info) if (has_v2) static_branch_enable(&vgic_v3_has_v2_compat); - if (vgic_v3_broken_seis()) - kvm_vgic_global_state.ich_vtr_el2 &= ~ICH_VTR_EL2_SEIS; - vgic_v3_enable_cpuif_traps(); kvm_vgic_global_state.vctrl_base = NULL; diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index d4789ff3e740..16bc0a670d3e 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -83,7 +83,7 @@ skip_v5: } kvm_vgic_global_state.has_gcie_v3_compat = true; - ich_vtr_el2 = kvm_call_hyp_ret(__vgic_v3_get_gic_config); + ich_vtr_el2 = vgic_ich_vtr(); kvm_vgic_global_state.ich_vtr_el2 = (u32)ich_vtr_el2; /* From 36d32222fcab099cf07b032aa5f022136bfa91be Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 21 Jul 2026 18:07:54 +0100 Subject: [PATCH 10/33] KVM: arm64: vgic-v3: Kill kvm_vgic_global_state.ich_vtr_el2 kvm_vgic_global_state.ich_vtr_el2 is the last bit of caching that we can get rid of. Not as bad as a sysreg access, but still worse than a constant. Move over to the inlined stuff and remove the cached value. Signed-off-by: Marc Zyngier Link: https://patch.msgid.link/20260721170754.3150521-7-maz@kernel.org Signed-off-by: Oliver Upton --- arch/arm64/kvm/nested.c | 3 ++- arch/arm64/kvm/vgic-sys-reg-v3.c | 8 ++++---- arch/arm64/kvm/vgic/vgic-v3.c | 5 ++--- arch/arm64/kvm/vgic/vgic-v5.c | 5 +---- arch/arm64/kvm/vgic/vgic.h | 2 +- include/kvm/arm_vgic.h | 2 -- 6 files changed, 10 insertions(+), 15 deletions(-) diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c index fb54f6dad995..54defcd8240d 100644 --- a/arch/arm64/kvm/nested.c +++ b/arch/arm64/kvm/nested.c @@ -16,6 +16,7 @@ #include #include "sys_regs.h" +#include "vgic/vgic.h" struct vncr_tlb { /* The guest's VNCR_EL2 */ @@ -1904,7 +1905,7 @@ int kvm_init_nv_sysregs(struct kvm_vcpu *vcpu) /* ICH_HCR_EL2 */ resx.res0 = ICH_HCR_EL2_RES0; resx.res1 = ICH_HCR_EL2_RES1; - if (!(kvm_vgic_global_state.ich_vtr_el2 & ICH_VTR_EL2_TDS)) + if (!(vgic_ich_vtr() & ICH_VTR_EL2_TDS)) resx.res0 |= ICH_HCR_EL2_TDIR; /* No GICv4 is presented to the guest */ resx.res0 |= ICH_HCR_EL2_DVIM | ICH_HCR_EL2_vSGIEOICount; diff --git a/arch/arm64/kvm/vgic-sys-reg-v3.c b/arch/arm64/kvm/vgic-sys-reg-v3.c index bdc2d57370b2..89315336b9d3 100644 --- a/arch/arm64/kvm/vgic-sys-reg-v3.c +++ b/arch/arm64/kvm/vgic-sys-reg-v3.c @@ -35,12 +35,12 @@ static int set_gic_ctlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, vgic_v3_cpu->num_id_bits = host_id_bits; - host_seis = FIELD_GET(ICH_VTR_EL2_SEIS, kvm_vgic_global_state.ich_vtr_el2); + host_seis = FIELD_GET(ICH_VTR_EL2_SEIS, vgic_ich_vtr()); seis = FIELD_GET(ICC_CTLR_EL1_SEIS_MASK, val); if (host_seis != seis) return -EINVAL; - host_a3v = FIELD_GET(ICH_VTR_EL2_A3V, kvm_vgic_global_state.ich_vtr_el2); + host_a3v = FIELD_GET(ICH_VTR_EL2_A3V, vgic_ich_vtr()); a3v = FIELD_GET(ICC_CTLR_EL1_A3V_MASK, val); if (host_a3v != a3v) return -EINVAL; @@ -69,9 +69,9 @@ static int get_gic_ctlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, val |= FIELD_PREP(ICC_CTLR_EL1_ID_BITS_MASK, vgic_v3_cpu->num_id_bits); val |= FIELD_PREP(ICC_CTLR_EL1_SEIS_MASK, FIELD_GET(ICH_VTR_EL2_SEIS, - kvm_vgic_global_state.ich_vtr_el2)); + vgic_ich_vtr())); val |= FIELD_PREP(ICC_CTLR_EL1_A3V_MASK, - FIELD_GET(ICH_VTR_EL2_A3V, kvm_vgic_global_state.ich_vtr_el2)); + FIELD_GET(ICH_VTR_EL2_A3V, vgic_ich_vtr())); /* * The VMCR.CTLR value is in ICC_CTLR_EL1 layout. * Extract it directly using ICC_CTLR_EL1 reg definitions. diff --git a/arch/arm64/kvm/vgic/vgic-v3.c b/arch/arm64/kvm/vgic/vgic-v3.c index 734fa3138723..91514d6faf74 100644 --- a/arch/arm64/kvm/vgic/vgic-v3.c +++ b/arch/arm64/kvm/vgic/vgic-v3.c @@ -490,9 +490,9 @@ void vgic_v3_reset(struct kvm_vcpu *vcpu) } vcpu->arch.vgic_cpu.num_id_bits = FIELD_GET(ICH_VTR_EL2_IDbits, - kvm_vgic_global_state.ich_vtr_el2); + vgic_ich_vtr()); vcpu->arch.vgic_cpu.num_pri_bits = FIELD_GET(ICH_VTR_EL2_PRIbits, - kvm_vgic_global_state.ich_vtr_el2) + 1; + vgic_ich_vtr()) + 1; } void vcpu_set_ich_hcr(struct kvm_vcpu *vcpu) @@ -949,7 +949,6 @@ int vgic_v3_probe(const struct gic_kvm_info *info) */ kvm_vgic_global_state.nr_lr = (ich_vtr_el2 & 0xf) + 1; kvm_vgic_global_state.can_emulate_gicv2 = false; - kvm_vgic_global_state.ich_vtr_el2 = ich_vtr_el2; /* GICv4 support? */ if (info->has_v4) { diff --git a/arch/arm64/kvm/vgic/vgic-v5.c b/arch/arm64/kvm/vgic/vgic-v5.c index 16bc0a670d3e..4e3e4db24e77 100644 --- a/arch/arm64/kvm/vgic/vgic-v5.c +++ b/arch/arm64/kvm/vgic/vgic-v5.c @@ -40,7 +40,6 @@ static void vgic_v5_get_implemented_ppis(void) int vgic_v5_probe(const struct gic_kvm_info *info) { bool v5_registered = false; - u64 ich_vtr_el2; int ret; kvm_vgic_global_state.type = VGIC_V5; @@ -83,14 +82,12 @@ skip_v5: } kvm_vgic_global_state.has_gcie_v3_compat = true; - ich_vtr_el2 = vgic_ich_vtr(); - kvm_vgic_global_state.ich_vtr_el2 = (u32)ich_vtr_el2; /* * The ListRegs field is 5 bits, but there is an architectural * maximum of 16 list registers. Just ignore bit 4... */ - kvm_vgic_global_state.nr_lr = (ich_vtr_el2 & 0xf) + 1; + kvm_vgic_global_state.nr_lr = (vgic_ich_vtr() & 0xf) + 1; ret = kvm_register_vgic_device(KVM_DEV_TYPE_ARM_VGIC_V3); if (ret) { diff --git a/arch/arm64/kvm/vgic/vgic.h b/arch/arm64/kvm/vgic/vgic.h index b76bb6e56de3..b71d486ae514 100644 --- a/arch/arm64/kvm/vgic/vgic.h +++ b/arch/arm64/kvm/vgic/vgic.h @@ -92,7 +92,7 @@ static inline u64 kvm_get_guest_vtr_el2(void) { u64 vtr; - vtr = kvm_vgic_global_state.ich_vtr_el2; + vtr = vgic_ich_vtr(); vtr &= ~KVM_ICH_VTR_EL2_RES0; vtr |= KVM_ICH_VTR_EL2_RES1; diff --git a/include/kvm/arm_vgic.h b/include/kvm/arm_vgic.h index fe49fb56dc3c..bd1bb03500b3 100644 --- a/include/kvm/arm_vgic.h +++ b/include/kvm/arm_vgic.h @@ -176,8 +176,6 @@ struct vgic_global { /* GICv3 compat mode on a GICv5 host */ bool has_gcie_v3_compat; - u32 ich_vtr_el2; - /* GICv5 PPI capabilities */ struct { DECLARE_BITMAP(impl_ppi_mask, VGIC_V5_NR_PRIVATE_IRQS); From 596289fe4836067b0320c80a9ab0101b9f7f9328 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Tue, 21 Jul 2026 17:16:24 +0100 Subject: [PATCH 11/33] KVM: arm64: Sync SCTLR_EL1 when injecting an exception into a pVM When pKVM injects a synchronous exception into a protected guest, enter_exception64() reads SCTLR_EL1 to set the new PSTATE's PAN and SSBS bits. pKVM refreshes VBAR_EL1 from the live value before injecting but not SCTLR_EL1, which is untrapped for protected guests, so a guest that updates it and then traps takes the exception with stale PAN/SSBS. Sync SCTLR_EL1 alongside VBAR_EL1. Fixes: 798eb5978700 ("KVM: arm64: Sync protected guest VBAR_EL1 on injecting an undef exception") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260721161624.3983041-1-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/sys_regs.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/sys_regs.c b/arch/arm64/kvm/hyp/nvhe/sys_regs.c index 8c3fbb413a06..7608cc4030b8 100644 --- a/arch/arm64/kvm/hyp/nvhe/sys_regs.c +++ b/arch/arm64/kvm/hyp/nvhe/sys_regs.c @@ -257,10 +257,11 @@ static void inject_sync64(struct kvm_vcpu *vcpu, u64 esr) *vcpu_cpsr(vcpu) = read_sysreg_el2(SYS_SPSR); /* - * Make sure we have the latest update to VBAR_EL1, as pKVM - * handles traps very early, before sysregs are resync'ed + * Sync VBAR_EL1 and SCTLR_EL1, both read by enter_exception64(), + * as pKVM handles traps before sysregs are resync'ed. */ __vcpu_assign_sys_reg(vcpu, VBAR_EL1, read_sysreg_el1(SYS_VBAR)); + __vcpu_assign_sys_reg(vcpu, SCTLR_EL1, read_sysreg_el1(SYS_SCTLR)); kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC); From 453f6ff04b8d28991f94ccac89350951252aa947 Mon Sep 17 00:00:00 2001 From: Congkai Tan Date: Wed, 22 Jul 2026 20:26:59 +0000 Subject: [PATCH 12/33] KVM: arm64: Expose PMMIR_EL1.SLOTS under strict PMUv3 UAPI Introduce a new field pmmir_slots in struct kvm_arch to store PMMIR_EL1.SLOTS. It only saves the actual hardware PMU value when the VMM explicitly selects a PMU under KVM_ARM_VCPU_PMU_V3_STRICT. Otherwise, it stays 0 after allocation. Use this field to implement guest access, userspace get, and userspace set for PMMIR_EL1: - access_pmmir(): uses the value in kvm->arch.pmmir_slots directly. If the VMM selected a PMU and KVM_ARM_VCPU_PMU_V3_STRICT is set, the guest can correctly read the underlying core's SLOTS. Otherwise, it continues to read 0 since the true SLOTS value can be nondeterministic. - get_pmmir(): same as access_pmmir(). - set_pmmir(): only the SLOTS field is writable; a value setting any other bit is rejected with -EINVAL, since get_pmmir() returns SLOTS zero-extended. A value of 0 resets kvm->arch.pmmir_slots to 0 for backward compatibility, as the register is RAZ in older KVM, a value matching the current SLOTS is accepted as a no-op, and anything else is rejected with -EINVAL. Once the VM has run PMMIR_EL1 is immutable, so a mismatching write then returns -EBUSY. The register is now exposed via KVM_GET_REG_LIST for PMUv3 vCPUs, so add it to the get-reg-list selftest's PMU register list. Signed-off-by: Congkai Tan Reviewed-by: Geoff Blake Reviewed-by: Haris Okanovic Reviewed-by: Stanislav Spassov Co-developed-by: Oliver Upton Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260722202702.4165917-2-congkai@amazon.com Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_host.h | 3 + arch/arm64/include/uapi/asm/kvm.h | 1 + arch/arm64/kvm/pmu-emul.c | 11 ++++ arch/arm64/kvm/sys_regs.c | 63 ++++++++++++++++++- include/kvm/arm_pmu.h | 4 ++ .../selftests/kvm/arm64/get-reg-list.c | 1 + 6 files changed, 81 insertions(+), 2 deletions(-) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index bae2c4f92ef5..9e035d587970 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -387,6 +387,9 @@ struct kvm_arch { /* Maximum number of counters for the guest */ u8 nr_pmu_counters; + /* PMMIR_EL1.SLOTS value exposed to the guest. */ + u8 pmmir_slots; + /* Hypercall features firmware registers' descriptor */ struct kvm_smccc_features smccc_feat; struct maple_tree smccc_filter; diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index 1c13bfa2d38a..019e5e3d892e 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -106,6 +106,7 @@ struct kvm_regs { #define KVM_ARM_VCPU_PTRAUTH_GENERIC 6 /* VCPU uses generic authentication */ #define KVM_ARM_VCPU_HAS_EL2 7 /* Support nested virtualization */ #define KVM_ARM_VCPU_HAS_EL2_E2H0 8 /* Limit NV support to E2H RES0 */ +#define KVM_ARM_VCPU_PMU_V3_STRICT 9 /* No default PMU creation */ struct kvm_vcpu_init { __u32 target; diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index 98305bbfc095..7ac00423fa8d 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c @@ -1092,6 +1092,17 @@ static int kvm_arm_pmu_v3_set_pmu(struct kvm_vcpu *vcpu, int pmu_id) kvm_arm_set_pmu(kvm, arm_pmu); cpumask_copy(kvm->arch.supported_cpus, &arm_pmu->supported_cpus); + + /* + * Since a specific PMU is explicitly selected, + * PMMIR_EL1.SLOTS is deterministic to the guest. + * If KVM_ARM_VCPU_PMU_V3_STRICT is set, snapshot + * the value to allow the guest to read it. + */ + if (kvm_vcpu_has_pmuv3_strict(vcpu)) + kvm->arch.pmmir_slots = + FIELD_GET(ARMV8_PMU_SLOTS, + arm_pmu->reg_pmmir); ret = 0; break; } diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 5d5c579d4579..4c6f608c4fe5 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1367,6 +1367,64 @@ static bool access_pminten(struct kvm_vcpu *vcpu, struct sys_reg_params *p, return true; } +static bool access_pmmir(struct kvm_vcpu *vcpu, struct sys_reg_params *p, + const struct sys_reg_desc *r) +{ + if (p->is_write) + return write_to_read_only(vcpu, p, r); + + /* + * If KVM_ARM_VCPU_PMU_V3_STRICT is set and PMU was explicitly + * selected, the underlying hardware SLOTS value was read into this + * field. Otherwise, it stays 0. All other PMMIR_EL1 fields are RAZ. + */ + p->regval = FIELD_PREP(ARMV8_PMU_SLOTS, vcpu->kvm->arch.pmmir_slots); + return true; +} + +static int get_pmmir(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 *val) +{ + *val = FIELD_PREP(ARMV8_PMU_SLOTS, vcpu->kvm->arch.pmmir_slots); + return 0; +} + +static int set_pmmir(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, + u64 val) +{ + struct kvm *kvm = vcpu->kvm; + u8 slots = FIELD_GET(ARMV8_PMU_SLOTS, val); + + /* + * Only the SLOTS field is exposed (get_pmmir returns just that field), + * so reject a write that sets any other bit rather than silently + * masking it. + */ + if (val & ~(u64)ARMV8_PMU_SLOTS) + return -EINVAL; + + guard(mutex)(&kvm->arch.config_lock); + + /* + * Once the VM has started PMMIR_EL1 is immutable. Reject any write + * that does not match the current value. + */ + if (kvm_vm_has_ran_once(kvm)) + return slots == kvm->arch.pmmir_slots ? 0 : -EBUSY; + + /* + * Only SLOTS = 0 is honored for backwards compatibility with the + * old RAZ behavior. Reject any non-zero write that does not match + * the current value. + */ + if (!slots) + kvm->arch.pmmir_slots = 0; + else if (slots != kvm->arch.pmmir_slots) + return -EINVAL; + + return 0; +} + static bool access_pmovs(struct kvm_vcpu *vcpu, struct sys_reg_params *p, const struct sys_reg_desc *r) { @@ -3448,7 +3506,8 @@ static const struct sys_reg_desc sys_reg_descs[] = { { PMU_SYS_REG(PMINTENCLR_EL1), .access = access_pminten, .reg = PMINTENSET_EL1, .get_user = get_pmreg, .set_user = set_pmreg }, - { SYS_DESC(SYS_PMMIR_EL1), trap_raz_wi }, + { PMU_SYS_REG(PMMIR_EL1), .access = access_pmmir, .reset = NULL, + .get_user = get_pmmir, .set_user = set_pmmir }, { SYS_DESC(SYS_MAIR_EL1), access_vm_reg, reset_unknown, MAIR_EL1 }, { SYS_DESC(SYS_PIRE0_EL1), NULL, reset_unknown, PIRE0_EL1, @@ -4593,7 +4652,7 @@ static const struct sys_reg_desc cp15_regs[] = { { CP15_PMU_SYS_REG(HI, 0, 9, 14, 4), .access = access_pmceid }, { CP15_PMU_SYS_REG(HI, 0, 9, 14, 5), .access = access_pmceid }, /* PMMIR */ - { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 6), .access = trap_raz_wi }, + { CP15_PMU_SYS_REG(DIRECT, 0, 9, 14, 6), .access = access_pmmir }, /* PRRR/MAIR0 */ { AA32(LO), Op1( 0), CRn(10), CRm( 2), Op2( 0), access_vm_reg, NULL, MAIR_EL1 }, diff --git a/include/kvm/arm_pmu.h b/include/kvm/arm_pmu.h index b5e5942204fc..6b4a118d17ca 100644 --- a/include/kvm/arm_pmu.h +++ b/include/kvm/arm_pmu.h @@ -75,6 +75,9 @@ void kvm_vcpu_pmu_resync_el0(void); #define kvm_vcpu_has_pmu(vcpu) \ (vcpu_has_feature(vcpu, KVM_ARM_VCPU_PMU_V3)) +#define kvm_vcpu_has_pmuv3_strict(vcpu) \ + (vcpu_has_feature(vcpu, KVM_ARM_VCPU_PMU_V3_STRICT)) + /* * Updates the vcpu's view of the pmu events for this cpu. * Must be called before every vcpu run after disabling interrupts, to ensure @@ -160,6 +163,7 @@ static inline u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1) } #define kvm_vcpu_has_pmu(vcpu) ({ false; }) +#define kvm_vcpu_has_pmuv3_strict(vcpu) ({ false; }) static inline void kvm_pmu_update_vcpu_events(struct kvm_vcpu *vcpu) {} static inline void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) {} static inline void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu) {} diff --git a/tools/testing/selftests/kvm/arm64/get-reg-list.c b/tools/testing/selftests/kvm/arm64/get-reg-list.c index 0a3a94c4cca1..cfa99979d57c 100644 --- a/tools/testing/selftests/kvm/arm64/get-reg-list.c +++ b/tools/testing/selftests/kvm/arm64/get-reg-list.c @@ -532,6 +532,7 @@ static __u64 base_regs[] = { static __u64 pmu_regs[] = { ARM64_SYS_REG(3, 0, 9, 14, 1), /* PMINTENSET_EL1 */ ARM64_SYS_REG(3, 0, 9, 14, 2), /* PMINTENCLR_EL1 */ + ARM64_SYS_REG(3, 0, 9, 14, 6), /* PMMIR_EL1 */ ARM64_SYS_REG(3, 3, 9, 12, 0), /* PMCR_EL0 */ ARM64_SYS_REG(3, 3, 9, 12, 1), /* PMCNTENSET_EL0 */ ARM64_SYS_REG(3, 3, 9, 12, 2), /* PMCNTENCLR_EL0 */ From 203b22f76dc3025d3fa96830a7cf6eadbf2c8407 Mon Sep 17 00:00:00 2001 From: Congkai Tan Date: Wed, 22 Jul 2026 20:27:00 +0000 Subject: [PATCH 13/33] KVM: arm64: Advertise STALL_SLOT* in PMCEID1 under strict PMUv3 UAPI Skip masking STALL_SLOT, STALL_SLOT_FRONTEND and STALL_SLOT_BACKEND out of PMCEID1 when KVM_ARM_VCPU_PMU_V3_STRICT is set, because this is when PMMIR_EL1.SLOTS is exposed to guests, making these events meaningful for collection. Change the parameter of compute_pmceid1() from arm_pmu to kvm_vcpu, to check if KVM_ARM_VCPU_PMU_V3_STRICT is set. Also updated the signature of compute_pmceid0() for consistency. Signed-off-by: Congkai Tan Reviewed-by: Geoff Blake Reviewed-by: Haris Okanovic Reviewed-by: Stanislav Spassov Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260722202702.4165917-3-congkai@amazon.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/pmu-emul.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index 7ac00423fa8d..bcfd0a91e114 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c @@ -838,9 +838,9 @@ static u64 __compute_pmceid(struct arm_pmu *pmu, bool pmceid1) return ((u64)hi[pmceid1] << 32) | lo[pmceid1]; } -static u64 compute_pmceid0(struct arm_pmu *pmu) +static u64 compute_pmceid0(struct kvm_vcpu *vcpu) { - u64 val = __compute_pmceid(pmu, 0); + u64 val = __compute_pmceid(vcpu->kvm->arch.arm_pmu, 0); /* always support SW_INCR */ val |= BIT(ARMV8_PMUV3_PERFCTR_SW_INCR); @@ -849,32 +849,33 @@ static u64 compute_pmceid0(struct arm_pmu *pmu) return val; } -static u64 compute_pmceid1(struct arm_pmu *pmu) +static u64 compute_pmceid1(struct kvm_vcpu *vcpu) { - u64 val = __compute_pmceid(pmu, 1); + u64 val = __compute_pmceid(vcpu->kvm->arch.arm_pmu, 1); /* - * Don't advertise STALL_SLOT*, as PMMIR_EL0 is handled - * as RAZ + * If KVM_ARM_VCPU_PMU_V3_STRICT is not set, PMMIR_EL1 is + * unconditionally RAZ, so don't advertise STALL_SLOT* events. */ - val &= ~(BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT - 32) | - BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT_FRONTEND - 32) | - BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT_BACKEND - 32)); + if (!kvm_vcpu_has_pmuv3_strict(vcpu)) + val &= ~(BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT - 32) | + BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT_FRONTEND - 32) | + BIT_ULL(ARMV8_PMUV3_PERFCTR_STALL_SLOT_BACKEND - 32)); + return val; } u64 kvm_pmu_get_pmceid(struct kvm_vcpu *vcpu, bool pmceid1) { - struct arm_pmu *cpu_pmu = vcpu->kvm->arch.arm_pmu; unsigned long *bmap = vcpu->kvm->arch.pmu_filter; u64 val, mask = 0; int base, i, nr_events; if (!pmceid1) { - val = compute_pmceid0(cpu_pmu); + val = compute_pmceid0(vcpu); base = 0; } else { - val = compute_pmceid1(cpu_pmu); + val = compute_pmceid1(vcpu); base = 32; } From bbef6317f97b8612a694ddc2ff6e7ca81780768d Mon Sep 17 00:00:00 2001 From: Oliver Upton Date: Wed, 22 Jul 2026 20:27:01 +0000 Subject: [PATCH 14/33] KVM: arm64: Ignore writes to PMCR_EL0.N when using strict UAPI KVM allows userspace to configure the number of event counters by writing to PMCR_EL0.N. While this makes a bit of sense prior to NV, the value of the field is context-dependent whether or not the vCPU is at EL2 or EL1. As such, the vCPU attribute was added afterwards and made mandatory for NV. As we're in the process of papering over old mistakes, force userspace to use the vCPU attribute if KVM_ARM_VCPU_PMU_V3_STRICT is set. Reviewed-by: Geoff Blake Reviewed-by: Haris Okanovic Signed-off-by: Congkai Tan Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260722202702.4165917-4-congkai@amazon.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/sys_regs.c | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 4c6f608c4fe5..e93f11b90813 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -1502,6 +1502,7 @@ static int set_pmcr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r, */ if (!kvm_vm_has_ran_once(kvm) && !vcpu_has_nv(vcpu) && + !kvm_vcpu_has_pmuv3_strict(vcpu) && new_n <= kvm_arm_pmu_get_max_counters(kvm)) kvm->arch.nr_pmu_counters = new_n; From ad220e275c3987aea865d4c05ad66ba0295c494d Mon Sep 17 00:00:00 2001 From: Congkai Tan Date: Wed, 22 Jul 2026 20:27:02 +0000 Subject: [PATCH 15/33] KVM: arm64: Add KVM_ARM_VCPU_PMU_V3_STRICT vCPU feature Introduce a new vCPU feature KVM_ARM_VCPU_PMU_V3_STRICT. When set, KVM does not create a default PMU when initializing the vCPU, and userspace must select one explicitly via KVM_ARM_VCPU_PMU_V3_SET_PMU before the first KVM_RUN. The flag forces the VMM to be aware of the PMU implementation of the guest to be created, so that certain information about the PMU becomes deterministic (if on a heterogeneous system) and becomes safe to be exposed to the guest. It can be used as an umbrella flag to gate future PMUv3 UAPI changes. When no default PMU is created, kvm->arch.arm_pmu stays NULL until SET_PMU runs, so kvm_arm_pmu_v3_init() now refuses to run if kvm->arch.arm_pmu is NULL. Signed-off-by: Congkai Tan Reviewed-by: Geoff Blake Reviewed-by: Haris Okanovic Reviewed-by: Stanislav Spassov Co-developed-by: Oliver Upton Reviewed-by: Fuad Tabba Tested-by: Fuad Tabba Link: https://patch.msgid.link/20260722202702.4165917-5-congkai@amazon.com Signed-off-by: Oliver Upton --- Documentation/virt/kvm/api.rst | 11 +++++++++++ Documentation/virt/kvm/devices/vcpu.rst | 11 +++++++++-- arch/arm64/include/asm/kvm_host.h | 2 +- arch/arm64/kvm/arm.c | 19 +++++++++++++++---- arch/arm64/kvm/pmu-emul.c | 18 +++++++++++++++++- include/uapi/linux/kvm.h | 1 + 6 files changed, 54 insertions(+), 8 deletions(-) diff --git a/Documentation/virt/kvm/api.rst b/Documentation/virt/kvm/api.rst index a5f9ee92f43e..4988c32df4bf 100644 --- a/Documentation/virt/kvm/api.rst +++ b/Documentation/virt/kvm/api.rst @@ -3515,6 +3515,17 @@ Possible features: Depends on KVM_CAP_ARM_PSCI_0_2. - KVM_ARM_VCPU_PMU_V3: Emulate PMUv3 for the CPU. Depends on KVM_CAP_ARM_PMU_V3. + - KVM_ARM_VCPU_PMU_V3_STRICT: Enable strict PMUv3 UAPI. + Requires KVM_ARM_VCPU_PMU_V3. Depends on KVM_CAP_ARM_PMU_V3_STRICT. + When enabled: + + * Userspace must explicitly select a PMU implementation before + initializing the PMU or configuring a PMU event filter + + * If the PMU implements FEAT_PMUv3p4, PMMIR_EL1.SLOTS provides the + hardware value of the underlying implementation + + * Writes to PMCR_EL0.N via KVM_SET_ONE_REG are ignored - KVM_ARM_VCPU_PTRAUTH_ADDRESS: Enables Address Pointer authentication for arm64 only. diff --git a/Documentation/virt/kvm/devices/vcpu.rst b/Documentation/virt/kvm/devices/vcpu.rst index 66e714f2fcfa..deb5c51bc00c 100644 --- a/Documentation/virt/kvm/devices/vcpu.rst +++ b/Documentation/virt/kvm/devices/vcpu.rst @@ -53,8 +53,9 @@ Returns: ======= ====================================================== -EEXIST Interrupt number already used -ENODEV PMUv3 not supported or GIC not initialized - -ENXIO PMUv3 not supported, missing VCPU feature or interrupt - number not set (non-GICv5 guests, only) + -ENXIO PMUv3 not supported, missing VCPU feature, missing + hardware PMU, or interrupt number not set (non-GICv5 + guests, only) -EBUSY PMUv3 already initialized ======= ====================================================== @@ -62,6 +63,9 @@ Request the initialization of the PMUv3. If using the PMUv3 with an in-kernel virtual GIC implementation, this must be done after initializing the in-kernel irqchip. +When the KVM_ARM_VCPU_PMU_V3_STRICT vCPU feature is enabled this must be done +after selecting a hardware PMU. + 1.3 ATTRIBUTE: KVM_ARM_VCPU_PMU_V3_FILTER ----------------------------------------- @@ -108,6 +112,9 @@ hardware event. Filtering event 0x1E (CHAIN) has no effect either, as it isn't strictly speaking an event. Filtering the cycle counter is possible using event 0x11 (CPU_CYCLES). +When the KVM_ARM_VCPU_PMU_V3_STRICT vCPU feature is enabled this must be done +after selecting a hardware PMU. + 1.4 ATTRIBUTE: KVM_ARM_VCPU_PMU_V3_SET_PMU ------------------------------------------ diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index 9e035d587970..39f7fc740d07 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -39,7 +39,7 @@ #define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS -#define KVM_VCPU_MAX_FEATURES 9 +#define KVM_VCPU_MAX_FEATURES 10 #define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1) #define KVM_REQ_SLEEP \ diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 50adfff75be8..7dbefdd846aa 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -452,6 +452,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = get_num_wrps(); break; case KVM_CAP_ARM_PMU_V3: + case KVM_CAP_ARM_PMU_V3_STRICT: r = kvm_supports_guest_pmuv3(); break; case KVM_CAP_ARM_INJECT_SERROR_ESR: @@ -1563,8 +1564,10 @@ static unsigned long system_supported_vcpu_features(void) if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1)) clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features); - if (!kvm_supports_guest_pmuv3()) + if (!kvm_supports_guest_pmuv3()) { clear_bit(KVM_ARM_VCPU_PMU_V3, &features); + clear_bit(KVM_ARM_VCPU_PMU_V3_STRICT, &features); + } if (!system_supports_sve()) clear_bit(KVM_ARM_VCPU_SVE, &features); @@ -1605,6 +1608,11 @@ static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu, test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features)) return -EINVAL; + /* Strict PMUv3 UAPI requires PMUv3. */ + if (test_bit(KVM_ARM_VCPU_PMU_V3_STRICT, &features) && + !test_bit(KVM_ARM_VCPU_PMU_V3, &features)) + return -EINVAL; + if (!test_bit(KVM_ARM_VCPU_EL1_32BIT, &features)) return 0; @@ -1634,10 +1642,13 @@ static int kvm_setup_vcpu(struct kvm_vcpu *vcpu) int ret = 0; /* - * When the vCPU has a PMU, but no PMU is set for the guest - * yet, set the default one. + * When the vCPU has a PMU, but no PMU is set for the guest yet, set + * the default one. If KVM_ARM_VCPU_PMU_V3_STRICT is set, no default + * PMU is created, and userspace must select a PMU via + * KVM_ARM_VCPU_PMU_V3_SET_PMU. */ - if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu) + if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu && + !kvm_vcpu_has_pmuv3_strict(vcpu)) ret = kvm_arm_set_default_pmu(kvm); /* Prepare for nested if required */ diff --git a/arch/arm64/kvm/pmu-emul.c b/arch/arm64/kvm/pmu-emul.c index bcfd0a91e114..5b1af7e2176f 100644 --- a/arch/arm64/kvm/pmu-emul.c +++ b/arch/arm64/kvm/pmu-emul.c @@ -939,6 +939,10 @@ int kvm_arm_pmu_v3_enable(struct kvm_vcpu *vcpu) static int kvm_arm_pmu_v3_init(struct kvm_vcpu *vcpu) { + /* Only possible when using KVM_ARM_VCPU_PMU_V3_STRICT */ + if (!vcpu->kvm->arch.arm_pmu) + return -ENXIO; + if (irqchip_in_kernel(vcpu->kvm)) { int ret; @@ -1009,6 +1013,14 @@ u8 kvm_arm_pmu_get_max_counters(struct kvm *kvm) { struct arm_pmu *arm_pmu = kvm->arch.arm_pmu; + /* + * Under KVM_ARM_VCPU_PMU_V3_STRICT no PMU exists until userspace sets + * one, so this can be reached before arm_pmu is set. Report no + * counters in that case. + */ + if (!arm_pmu) + return 0; + /* * PMUv3 requires that all event counters are capable of counting any * event, though the same may not be true of non-PMUv3 hardware. @@ -1050,7 +1062,8 @@ static void kvm_arm_set_pmu(struct kvm *kvm, struct arm_pmu *arm_pmu) } /** - * kvm_arm_set_default_pmu - No PMU set, get the default one. + * kvm_arm_set_default_pmu - No PMU set and KVM_ARM_VCPU_PMU_V3_STRICT not + * set, get the default one. * @kvm: The kvm pointer * * The observant among you will notice that the supported_cpus @@ -1190,6 +1203,9 @@ int kvm_arm_pmu_v3_set_attr(struct kvm_vcpu *vcpu, struct kvm_device_attr *attr) if (kvm_vm_has_ran_once(kvm)) return -EBUSY; + if (!kvm->arch.arm_pmu) + return -ENXIO; + if (!kvm->arch.pmu_filter) { kvm->arch.pmu_filter = bitmap_alloc(nr_events, GFP_KERNEL_ACCOUNT); if (!kvm->arch.pmu_filter) diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 419011097fa8..9fc8dfdfd65f 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -997,6 +997,7 @@ struct kvm_enable_cap { #define KVM_CAP_S390_KEYOP 247 #define KVM_CAP_S390_VSIE_ESAMODE 248 #define KVM_CAP_S390_HPAGE_2G 249 +#define KVM_CAP_ARM_PMU_V3_STRICT 250 struct kvm_irq_routing_irqchip { __u32 irqchip; From 81b2d03f8c86a1956606640794cafa6949fbc295 Mon Sep 17 00:00:00 2001 From: Mostafa Saleh Date: Thu, 23 Jul 2026 16:20:02 +0000 Subject: [PATCH 16/33] KVM: arm64: Optimize protected mode with FWB and DIC KVM opportunistically enables FWB if supported by the system for guest VMs, which allows it to elude cache maintenance for data as they are forced to be cacheable from stage-2. In that case, __clean_dcache_guest_page() will immediately return. However in protected mode, before calling __clean_dcache_guest_page() it loops over the range and fix_map/unmap it, issuing TLB invalidations, dsb() and isb() unnecessarily. This can be optimized by returning early if FWB is supported, kvm_pgtable_stage2_map() already issues dsb() and tlb invalidation functions issue dsb() for the unmap path. Similarly for DIC, we can omit the call in invalidate_icache_guest_page() No isb() needed as ERET is context synchronization event. Signed-off-by: Mostafa Saleh Acked-by: Marc Zyngier Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20260723162002.3848054-1-smostafa@google.com Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/mem_protect.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c index 4e329e39a695..39aa8911f62c 100644 --- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c +++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c @@ -261,11 +261,18 @@ static void __apply_guest_page(void *va, size_t size, static void clean_dcache_guest_page(void *va, size_t size) { + /* See comment in __clean_dcache_guest_page() */ + if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) + return; + __apply_guest_page(va, size, __clean_dcache_guest_page); } static void invalidate_icache_guest_page(void *va, size_t size) { + if (alternative_has_cap_unlikely(ARM64_HAS_CACHE_DIC)) + return; + __apply_guest_page(va, size, __invalidate_icache_guest_page); } From 119a9a23fbae4fefbb5cd4776547b3b6803a7eaa Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Sun, 26 Jul 2026 15:36:42 +0100 Subject: [PATCH 17/33] KVM: arm64: Flush external_mdscr_el1 to the pKVM hyp vCPU flush_debug_state() propagates the guest's debug_owner and the owner-selected debug register state to the hyp vCPU, but not external_mdscr_el1. While the host owns the debug registers, the world switch loads MDSCR_EL1 from external_mdscr_el1 (ctxt_mdscr_el1()), where the host's KDE/MDE/SS bits live. A non-protected guest under KVM_GUESTDBG_USE_HW or single-step therefore runs with MDSCR_EL1.MDE/SS clear in hardware, and its watchpoints, breakpoints and single-step never fire. Propagate external_mdscr_el1 to the hyp vCPU alongside the host-owned debug state. Fixes: 4ad3a0b87f2ec ("KVM: arm64: Don't hijack guest context MDSCR_EL1") Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260726143643.303456-2-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index d3c69de698f4..104026ee70e8 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -108,10 +108,16 @@ static void flush_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_vcpu->vcpu.arch.debug_owner = host_vcpu->arch.debug_owner; - if (kvm_guest_owns_debug_regs(&hyp_vcpu->vcpu)) + if (kvm_guest_owns_debug_regs(&hyp_vcpu->vcpu)) { hyp_vcpu->vcpu.arch.vcpu_debug_state = host_vcpu->arch.vcpu_debug_state; - else if (kvm_host_owns_debug_regs(&hyp_vcpu->vcpu)) + } else if (kvm_host_owns_debug_regs(&hyp_vcpu->vcpu)) { hyp_vcpu->vcpu.arch.external_debug_state = host_vcpu->arch.external_debug_state; + /* + * The world switch loads MDSCR_EL1 from external_mdscr_el1 + * (ctxt_mdscr_el1()). + */ + hyp_vcpu->vcpu.arch.external_mdscr_el1 = host_vcpu->arch.external_mdscr_el1; + } } static void sync_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) From 3be99f8751c9eed017f437644fd7ecbdfdc66fbe Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Sun, 26 Jul 2026 15:36:43 +0100 Subject: [PATCH 18/33] KVM: arm64: selftests: Add a userspace watchpoint test debug-exceptions covers guest self-hosted debug and userspace single-step, but not a userspace (KVM_GUESTDBG_USE_HW) watchpoint, whose KVM_EXIT_DEBUG reports the accessed address in debug.arch.far. Add a test that installs a host-directed write watchpoint and checks that the reported address matches the accessed variable. Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260726143643.303456-3-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- .../selftests/kvm/arm64/debug-exceptions.c | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tools/testing/selftests/kvm/arm64/debug-exceptions.c b/tools/testing/selftests/kvm/arm64/debug-exceptions.c index 3eb4b1b6682d..7dc5f0b4f6ad 100644 --- a/tools/testing/selftests/kvm/arm64/debug-exceptions.c +++ b/tools/testing/selftests/kvm/arm64/debug-exceptions.c @@ -527,6 +527,46 @@ void test_single_step_from_userspace(int test_cnt) kvm_vm_free(vm); } +static void guest_code_wp(void) +{ + write_data = 'x'; + GUEST_DONE(); +} + +/* + * A userspace hardware watchpoint (KVM_GUESTDBG_USE_HW) must fire and report + * the accessed address in debug.arch.far, exercising the watchpoint exit path. + */ +static void test_watchpoint_from_userspace(void) +{ + struct kvm_guest_debug debug = {}; + struct kvm_vcpu *vcpu; + struct kvm_run *run; + struct kvm_vm *vm; + + vm = vm_create_with_one_vcpu(&vcpu, guest_code_wp); + run = vcpu->run; + + debug.control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW; + debug.arch.dbg_wcr[0] = DBGWCR_LEN8 | DBGWCR_RD | DBGWCR_WR | + DBGWCR_EL1 | DBGWCR_E; + /* + * BAS = 0xff (LEN8) requires a doubleword-aligned DBGWVR; FAR still + * reports the exact accessed byte. + */ + debug.arch.dbg_wvr[0] = PC(write_data) & ~7UL; + vcpu_guest_debug_set(vcpu, &debug); + + vcpu_run(vcpu); + TEST_ASSERT(run->exit_reason == KVM_EXIT_DEBUG, + "Expected KVM_EXIT_DEBUG, got %u", run->exit_reason); + TEST_ASSERT((u64)run->debug.arch.far == PC(write_data), + "Watchpoint FAR 0x%lx != accessed address 0x%lx", + (u64)run->debug.arch.far, PC(write_data)); + + kvm_vm_free(vm); +} + /* * Run debug testing using the various breakpoint#, watchpoint# and * context-aware breakpoint# with the given ID_AA64DFR0_EL1 configuration. @@ -600,6 +640,7 @@ int main(int argc, char *argv[]) test_guest_debug_exceptions_all(aa64dfr0); test_single_step_from_userspace(ss_iteration); + test_watchpoint_from_userspace(); return 0; } From c0646b96473340c71cd133ac4056efc9f78141f7 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 29 Jul 2026 14:18:16 +0100 Subject: [PATCH 19/33] KVM: arm64: Extract MPIDR computation into a shared header Extract the vCPU MPIDR computation embedded in reset_mpidr() into a kvm_calculate_mpidr() inline in sys_regs.h, so it can be computed without duplicating the logic. A follow-up series reuses it to reset protected vCPUs at EL2. No functional change intended. Reviewed-by: Vincent Donnefort Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260729131823.2021516-2-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/sys_regs.c | 14 +------------- arch/arm64/kvm/sys_regs.h | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 5d5c579d4579..08ba882799d4 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -976,21 +976,9 @@ static u64 reset_actlr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) static u64 reset_mpidr(struct kvm_vcpu *vcpu, const struct sys_reg_desc *r) { - u64 mpidr; + u64 mpidr = kvm_calculate_mpidr(vcpu); - /* - * Map the vcpu_id into the first three affinity level fields of - * the MPIDR. We limit the number of VCPUs in level 0 due to a - * limitation to 16 CPUs in that level in the ICC_SGIxR registers - * of the GICv3 to be able to address each CPU directly when - * sending IPIs. - */ - mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0); - mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1); - mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2); - mpidr |= (1ULL << 31); vcpu_write_sys_reg(vcpu, mpidr, MPIDR_EL1); - return mpidr; } diff --git a/arch/arm64/kvm/sys_regs.h b/arch/arm64/kvm/sys_regs.h index 2a983664220c..bd56a45abbf9 100644 --- a/arch/arm64/kvm/sys_regs.h +++ b/arch/arm64/kvm/sys_regs.h @@ -222,6 +222,25 @@ find_reg(const struct sys_reg_params *params, const struct sys_reg_desc table[], return __inline_bsearch((void *)pval, table, num, sizeof(table[0]), match_sys_reg); } +static inline u64 kvm_calculate_mpidr(const struct kvm_vcpu *vcpu) +{ + u64 mpidr; + + /* + * Map the vcpu_id into the first three affinity level fields of + * the MPIDR. We limit the number of VCPUs in level 0 due to a + * limitation to 16 CPUs in that level in the ICC_SGIxR registers + * of the GICv3 to be able to address each CPU directly when + * sending IPIs. + */ + mpidr = (vcpu->vcpu_id & 0x0f) << MPIDR_LEVEL_SHIFT(0); + mpidr |= ((vcpu->vcpu_id >> 4) & 0xff) << MPIDR_LEVEL_SHIFT(1); + mpidr |= ((vcpu->vcpu_id >> 12) & 0xff) << MPIDR_LEVEL_SHIFT(2); + mpidr |= (1ULL << 31); + + return mpidr; +} + const struct sys_reg_desc *get_reg_by_id(u64 id, const struct sys_reg_desc table[], unsigned int num); From f88099ad9fc2fc5c7d75db848fcc2bb55a83133a Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 29 Jul 2026 14:18:17 +0100 Subject: [PATCH 20/33] KVM: arm64: Make vcpu_{read,write}_sys_reg available to HYP code The vcpu_{read,write}_sys_reg() accessors abstract where a system register lives, but their VHE implementation cannot be linked into the nVHE hypervisor, so exception.c open-codes has_vhe() wrappers instead. Redirect the accessors to the raw context accessors for nVHE hyp builds, where registers are always in memory, and drop the local wrappers. This lets hyp code reuse helpers built on the canonical accessors. No functional change intended. Suggested-by: Marc Zyngier Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260729131823.2021516-3-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_emulate.h | 6 +++++ arch/arm64/kvm/hyp/exception.c | 34 ++++++++-------------------- 2 files changed, 15 insertions(+), 25 deletions(-) diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 5bf3d7e1d92c..fb4062b3ce55 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -506,6 +506,12 @@ static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu) return __vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK; } +/* In nVHE hyp code, registers are always in memory: use the raw accessors. */ +#if defined(__KVM_NVHE_HYPERVISOR__) +#define vcpu_read_sys_reg(v, r) __vcpu_sys_reg(v, r) +#define vcpu_write_sys_reg(v, x, r) __vcpu_assign_sys_reg(v, r, x) +#endif + static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu) { if (vcpu_mode_is_32bit(vcpu)) { diff --git a/arch/arm64/kvm/hyp/exception.c b/arch/arm64/kvm/hyp/exception.c index bef40ddb16db..754e2dc1df54 100644 --- a/arch/arm64/kvm/hyp/exception.c +++ b/arch/arm64/kvm/hyp/exception.c @@ -20,22 +20,6 @@ #error Hypervisor code only! #endif -static inline u64 __vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg) -{ - if (has_vhe()) - return vcpu_read_sys_reg(vcpu, reg); - - return __vcpu_sys_reg(vcpu, reg); -} - -static inline void __vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg) -{ - if (has_vhe()) - vcpu_write_sys_reg(vcpu, val, reg); - else - __vcpu_assign_sys_reg(vcpu, reg, val); -} - static void __vcpu_write_spsr(struct kvm_vcpu *vcpu, unsigned long target_mode, u64 val) { @@ -101,14 +85,14 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, switch (target_mode) { case PSR_MODE_EL1h: - vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL1); - sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1); - __vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL1); + vbar = vcpu_read_sys_reg(vcpu, VBAR_EL1); + sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1); + vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL1); break; case PSR_MODE_EL2h: - vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL2); - sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL2); - __vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL2); + vbar = vcpu_read_sys_reg(vcpu, VBAR_EL2); + sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL2); + vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL2); break; default: /* Don't do that */ @@ -185,7 +169,7 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode, */ static unsigned long get_except32_cpsr(struct kvm_vcpu *vcpu, u32 mode) { - u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1); + u32 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1); unsigned long old, new; old = *vcpu_cpsr(vcpu); @@ -281,7 +265,7 @@ static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset) { unsigned long spsr = *vcpu_cpsr(vcpu); bool is_thumb = (spsr & PSR_AA32_T_BIT); - u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1); + u32 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1); u32 return_address; *vcpu_cpsr(vcpu) = get_except32_cpsr(vcpu, mode); @@ -305,7 +289,7 @@ static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset) if (sctlr & (1 << 13)) vect_offset += 0xffff0000; else /* always have security exceptions */ - vect_offset += __vcpu_read_sys_reg(vcpu, VBAR_EL1); + vect_offset += vcpu_read_sys_reg(vcpu, VBAR_EL1); *vcpu_pc(vcpu) = vect_offset; } From 29b2e2ab7fa7a9938b09e0756d90805107c965e1 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 29 Jul 2026 14:18:18 +0100 Subject: [PATCH 21/33] KVM: arm64: Factor out reusable vCPU reset helpers Pull the reusable pieces out of kvm_reset_vcpu(): expose the reset PSTATE values in kvm_arm.h, and split the core register reset and the PSCI-driven reset into kvm_reset_vcpu_core() and kvm_reset_vcpu_psci(). A follow-up series reuses these to reset protected vCPUs at EL2. No functional change intended. Reviewed-by: Vincent Donnefort Reviewed-by: Steffen Eiden Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260729131823.2021516-4-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_arm.h | 12 ++++++ arch/arm64/include/asm/kvm_emulate.h | 57 ++++++++++++++++++++++++++ arch/arm64/kvm/reset.c | 60 ++-------------------------- 3 files changed, 72 insertions(+), 57 deletions(-) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 3f9233b5a130..aba4ec09acd2 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -348,4 +348,16 @@ { PSR_AA32_MODE_UND, "32-bit UND" }, \ { PSR_AA32_MODE_SYS, "32-bit SYS" } +/* + * ARMv8 Reset Values + */ +#define VCPU_RESET_PSTATE_EL1 (PSR_MODE_EL1h | PSR_A_BIT | PSR_I_BIT | \ + PSR_F_BIT | PSR_D_BIT) + +#define VCPU_RESET_PSTATE_EL2 (PSR_MODE_EL2h | PSR_A_BIT | PSR_I_BIT | \ + PSR_F_BIT | PSR_D_BIT) + +#define VCPU_RESET_PSTATE_SVC (PSR_AA32_MODE_SVC | PSR_AA32_A_BIT | \ + PSR_AA32_I_BIT | PSR_AA32_F_BIT) + #endif /* __ARM64_KVM_ARM_H__ */ diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index fb4062b3ce55..dbc77d059d7e 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -694,4 +694,61 @@ static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu) vcpu->arch.hcrx_el2 |= HCRX_EL2_EnASR; } } + +/* Reset a vcpu's core registers. */ +static inline void kvm_reset_vcpu_core(struct kvm_vcpu *vcpu) +{ + u32 pstate; + + if (vcpu_el1_is_32bit(vcpu)) + pstate = VCPU_RESET_PSTATE_SVC; + else if (vcpu_has_nv(vcpu)) + pstate = VCPU_RESET_PSTATE_EL2; + else + pstate = VCPU_RESET_PSTATE_EL1; + + /* Reset core registers */ + memset(vcpu_gp_regs(vcpu), 0, sizeof(*vcpu_gp_regs(vcpu))); + memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs)); + vcpu->arch.ctxt.spsr_abt = 0; + vcpu->arch.ctxt.spsr_und = 0; + vcpu->arch.ctxt.spsr_irq = 0; + vcpu->arch.ctxt.spsr_fiq = 0; + vcpu_gp_regs(vcpu)->pstate = pstate; +} + +/* PSCI reset handling for a vcpu. */ +static inline void kvm_reset_vcpu_psci(struct kvm_vcpu *vcpu, + struct vcpu_reset_state *reset_state) +{ + unsigned long target_pc = reset_state->pc; + + /* Gracefully handle Thumb2 entry point */ + if (vcpu_mode_is_32bit(vcpu) && (target_pc & 1)) { + target_pc &= ~1UL; + vcpu_set_thumb(vcpu); + } + + /* Propagate caller endianness */ + if (reset_state->be) + kvm_vcpu_set_be(vcpu); + + *vcpu_pc(vcpu) = target_pc; + + /* + * We may come from a state where either a PC update was + * pending (SMC call resulting in PC being increpented to + * skip the SMC) or a pending exception. Make sure we get + * rid of all that, as this cannot be valid out of reset. + * + * Note that clearing the exception mask also clears PC + * updates, but that's an implementation detail, and we + * really want to make it explicit. + */ + vcpu_clear_flag(vcpu, PENDING_EXCEPTION); + vcpu_clear_flag(vcpu, EXCEPT_MASK); + vcpu_clear_flag(vcpu, INCREMENT_PC); + vcpu_set_reg(vcpu, 0, reset_state->r0); +} + #endif /* __ARM64_KVM_EMULATE_H__ */ diff --git a/arch/arm64/kvm/reset.c b/arch/arm64/kvm/reset.c index b963fd975aac..10eb7249aa9e 100644 --- a/arch/arm64/kvm/reset.c +++ b/arch/arm64/kvm/reset.c @@ -34,18 +34,6 @@ static u32 __ro_after_init kvm_ipa_limit; unsigned int __ro_after_init kvm_host_sve_max_vl; -/* - * ARMv8 Reset Values - */ -#define VCPU_RESET_PSTATE_EL1 (PSR_MODE_EL1h | PSR_A_BIT | PSR_I_BIT | \ - PSR_F_BIT | PSR_D_BIT) - -#define VCPU_RESET_PSTATE_EL2 (PSR_MODE_EL2h | PSR_A_BIT | PSR_I_BIT | \ - PSR_F_BIT | PSR_D_BIT) - -#define VCPU_RESET_PSTATE_SVC (PSR_AA32_MODE_SVC | PSR_AA32_A_BIT | \ - PSR_AA32_I_BIT | PSR_AA32_F_BIT) - unsigned int __ro_after_init kvm_sve_max_vl; int __init kvm_arm_init_sve(void) @@ -191,7 +179,6 @@ void kvm_reset_vcpu(struct kvm_vcpu *vcpu) { struct vcpu_reset_state reset_state; bool loaded; - u32 pstate; spin_lock(&vcpu->arch.mp_state_lock); reset_state = vcpu->arch.reset_state; @@ -210,21 +197,8 @@ void kvm_reset_vcpu(struct kvm_vcpu *vcpu) kvm_vcpu_reset_sve(vcpu); } - if (vcpu_el1_is_32bit(vcpu)) - pstate = VCPU_RESET_PSTATE_SVC; - else if (vcpu_has_nv(vcpu)) - pstate = VCPU_RESET_PSTATE_EL2; - else - pstate = VCPU_RESET_PSTATE_EL1; - /* Reset core registers */ - memset(vcpu_gp_regs(vcpu), 0, sizeof(*vcpu_gp_regs(vcpu))); - memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs)); - vcpu->arch.ctxt.spsr_abt = 0; - vcpu->arch.ctxt.spsr_und = 0; - vcpu->arch.ctxt.spsr_irq = 0; - vcpu->arch.ctxt.spsr_fiq = 0; - vcpu_gp_regs(vcpu)->pstate = pstate; + kvm_reset_vcpu_core(vcpu); /* Reset system registers */ kvm_reset_sys_regs(vcpu); @@ -233,36 +207,8 @@ void kvm_reset_vcpu(struct kvm_vcpu *vcpu) * Additional reset state handling that PSCI may have imposed on us. * Must be done after all the sys_reg reset. */ - if (reset_state.reset) { - unsigned long target_pc = reset_state.pc; - - /* Gracefully handle Thumb2 entry point */ - if (vcpu_mode_is_32bit(vcpu) && (target_pc & 1)) { - target_pc &= ~1UL; - vcpu_set_thumb(vcpu); - } - - /* Propagate caller endianness */ - if (reset_state.be) - kvm_vcpu_set_be(vcpu); - - *vcpu_pc(vcpu) = target_pc; - - /* - * We may come from a state where either a PC update was - * pending (SMC call resulting in PC being increpented to - * skip the SMC) or a pending exception. Make sure we get - * rid of all that, as this cannot be valid out of reset. - * - * Note that clearing the exception mask also clears PC - * updates, but that's an implementation detail, and we - * really want to make it explicit. - */ - vcpu_clear_flag(vcpu, PENDING_EXCEPTION); - vcpu_clear_flag(vcpu, EXCEPT_MASK); - vcpu_clear_flag(vcpu, INCREMENT_PC); - vcpu_set_reg(vcpu, 0, reset_state.r0); - } + if (reset_state.reset) + kvm_reset_vcpu_psci(vcpu, &reset_state); /* Reset timer */ kvm_timer_vcpu_reset(vcpu); From 86db2bd7f5f07e4b169345d291bdd5c72730061c Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 29 Jul 2026 14:18:19 +0100 Subject: [PATCH 22/33] KVM: arm64: Move PSCI helper functions to a shared header Move kvm_psci_valid_affinity() and kvm_psci_narrow_to_32bit() from psci.c to include/kvm/arm_psci.h, and move psci_affinity_mask() there too, renaming it kvm_psci_affinity_mask() now that it is no longer file-local. A follow-up series handles some protected-guest PSCI calls at EL2 using these helpers. No functional change intended. Reviewed-by: Vincent Donnefort Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260729131823.2021516-5-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/psci.c | 30 +----------------------------- include/kvm/arm_psci.h | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/arch/arm64/kvm/psci.c b/arch/arm64/kvm/psci.c index 3b5dbe9a0a0e..e3db84400d1f 100644 --- a/arch/arm64/kvm/psci.c +++ b/arch/arm64/kvm/psci.c @@ -21,16 +21,6 @@ * as described in ARM document number ARM DEN 0022A. */ -#define AFFINITY_MASK(level) ~((0x1UL << ((level) * MPIDR_LEVEL_BITS)) - 1) - -static unsigned long psci_affinity_mask(unsigned long affinity_level) -{ - if (affinity_level <= 3) - return MPIDR_HWID_BITMASK & AFFINITY_MASK(affinity_level); - - return 0; -} - static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu *vcpu) { /* @@ -51,12 +41,6 @@ static unsigned long kvm_psci_vcpu_suspend(struct kvm_vcpu *vcpu) return PSCI_RET_SUCCESS; } -static inline bool kvm_psci_valid_affinity(struct kvm_vcpu *vcpu, - unsigned long affinity) -{ - return !(affinity & ~MPIDR_HWID_BITMASK); -} - static unsigned long kvm_psci_vcpu_on(struct kvm_vcpu *source_vcpu) { struct vcpu_reset_state *reset_state; @@ -135,7 +119,7 @@ static unsigned long kvm_psci_vcpu_affinity_info(struct kvm_vcpu *vcpu) return PSCI_RET_INVALID_PARAMS; /* Determine target affinity mask */ - target_affinity_mask = psci_affinity_mask(lowest_affinity_level); + target_affinity_mask = kvm_psci_affinity_mask(lowest_affinity_level); if (!target_affinity_mask) return PSCI_RET_INVALID_PARAMS; @@ -220,18 +204,6 @@ static void kvm_psci_system_suspend(struct kvm_vcpu *vcpu) run->exit_reason = KVM_EXIT_SYSTEM_EVENT; } -static void kvm_psci_narrow_to_32bit(struct kvm_vcpu *vcpu) -{ - int i; - - /* - * Zero the input registers' upper 32 bits. They will be fully - * zeroed on exit, so we're fine changing them in place. - */ - for (i = 1; i < 4; i++) - vcpu_set_reg(vcpu, i, lower_32_bits(vcpu_get_reg(vcpu, i))); -} - static unsigned long kvm_psci_check_allowed_function(struct kvm_vcpu *vcpu, u32 fn) { /* diff --git a/include/kvm/arm_psci.h b/include/kvm/arm_psci.h index cbaec804eb83..f86a006d6713 100644 --- a/include/kvm/arm_psci.h +++ b/include/kvm/arm_psci.h @@ -38,6 +38,33 @@ static inline int kvm_psci_version(struct kvm_vcpu *vcpu) return KVM_ARM_PSCI_0_1; } +/* Narrow the PSCI register arguments (r1 to r3) to 32 bits. */ +static inline void kvm_psci_narrow_to_32bit(struct kvm_vcpu *vcpu) +{ + int i; + + /* + * Zero the input registers' upper 32 bits. They will be fully + * zeroed on exit, so we're fine changing them in place. + */ + for (i = 1; i < 4; i++) + vcpu_set_reg(vcpu, i, lower_32_bits(vcpu_get_reg(vcpu, i))); +} + +static inline bool kvm_psci_valid_affinity(struct kvm_vcpu *vcpu, + unsigned long affinity) +{ + return !(affinity & ~MPIDR_HWID_BITMASK); +} + +static inline unsigned long kvm_psci_affinity_mask(unsigned long affinity_level) +{ + if (affinity_level <= 3) + return MPIDR_HWID_BITMASK & + ~((0x1UL << (affinity_level * MPIDR_LEVEL_BITS)) - 1); + + return 0; +} int kvm_psci_call(struct kvm_vcpu *vcpu); From e9f88aa0b16cc95e9c69d5d9fc02fe506c6a0df3 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 29 Jul 2026 14:18:20 +0100 Subject: [PATCH 23/33] KVM: arm64: Add host and hypervisor vCPU lookup primitives The nVHE hypervisor repeatedly resolves a host vCPU into the EL2 address space and validates that the loaded hyp vCPU matches it, with that logic open-coded in each handler. Add __get_host_hyp_vcpus() and the get_host_hyp_vcpus() macro, which translate the host vCPU into the hypervisor's address space and, when pKVM is enabled, also return the loaded hyp vCPU if it matches. If pKVM is enabled but the loaded hyp vCPU does not correspond to the requested host vCPU, both the host and hyp vCPU are returned as NULL. Convert handle___kvm_vcpu_run() to use it. No functional change intended. Reviewed-by: Vincent Donnefort Signed-off-by: Marc Zyngier Co-developed-by: Fuad Tabba Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260729131823.2021516-6-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 52 ++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 104026ee70e8..f859c8aca544 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -220,14 +220,45 @@ static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt) pkvm_put_hyp_vcpu(hyp_vcpu); } -static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) +static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *arg, + struct pkvm_hyp_vcpu **hyp_vcpup) { - DECLARE_REG(struct kvm_vcpu *, host_vcpu, host_ctxt, 1); - int ret; + struct kvm_vcpu *host_vcpu = kern_hyp_va(arg); + struct pkvm_hyp_vcpu *hyp_vcpu = NULL; if (unlikely(is_protected_kvm_enabled())) { - struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); + hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); + if (!hyp_vcpu || hyp_vcpu->host_vcpu != host_vcpu) { + hyp_vcpu = NULL; + host_vcpu = NULL; + } + } + + *hyp_vcpup = hyp_vcpu; + return host_vcpu; +} + +#define get_host_hyp_vcpus(ctxt, regnr, hyp_vcpup) \ + ({ \ + DECLARE_REG(struct kvm_vcpu *, __vcpu, ctxt, regnr); \ + __get_host_hyp_vcpus(__vcpu, hyp_vcpup); \ + }) + +static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) +{ + struct pkvm_hyp_vcpu *hyp_vcpu; + struct kvm_vcpu *host_vcpu; + int ret; + + host_vcpu = get_host_hyp_vcpus(host_ctxt, 1, &hyp_vcpu); + + if (!host_vcpu) { + ret = -EINVAL; + goto out; + } + + if (unlikely(hyp_vcpu)) { /* * KVM (and pKVM) doesn't support SME guests for now, and * ensures that SME features aren't enabled in pstate when @@ -239,23 +270,16 @@ static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) goto out; } - if (!hyp_vcpu) { - ret = -EINVAL; - goto out; - } - flush_hyp_vcpu(hyp_vcpu); ret = __kvm_vcpu_run(&hyp_vcpu->vcpu); sync_hyp_vcpu(hyp_vcpu); } else { - struct kvm_vcpu *vcpu = kern_hyp_va(host_vcpu); - /* The host is fully trusted, run its vCPU directly. */ - fpsimd_lazy_switch_to_guest(vcpu); - ret = __kvm_vcpu_run(vcpu); - fpsimd_lazy_switch_to_host(vcpu); + fpsimd_lazy_switch_to_guest(host_vcpu); + ret = __kvm_vcpu_run(host_vcpu); + fpsimd_lazy_switch_to_host(host_vcpu); } out: cpu_reg(host_ctxt, 1) = ret; From 79c3122a1e142166b9254809115ef26ba7797769 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 29 Jul 2026 14:18:21 +0100 Subject: [PATCH 24/33] KVM: arm64: Minimise EL2's exposure of host VGIC state during world switch The host passes a vgic_v3_cpu_if pointer to the __vgic_v3_save_aprs and __vgic_v3_restore_vmcr_aprs hypercalls, which EL2 dereferences wholesale. That exposes the host's full VGIC emulation state to the hypervisor, against pKVM's isolation goals. Recover the host vCPU from the supplied cpu_if via container_of() and copy only vgic_vmcr and the active priority registers between EL2's hyp-side state and the host vCPU, so EL2 no longer dereferences the host's vgic_v3_cpu_if directly. Reviewed-by: Vincent Donnefort Signed-off-by: Marc Zyngier Co-developed-by: Fuad Tabba Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260729131823.2021516-7-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 67 ++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index f859c8aca544..415bbd7f9452 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -7,6 +7,8 @@ #include #include +#include + #include #include #include @@ -245,6 +247,16 @@ static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *arg, __get_host_hyp_vcpus(__vcpu, hyp_vcpup); \ }) +#define get_host_hyp_vcpus_from_vgic_v3_cpu_if(ctxt, regnr, hyp_vcpup) \ + ({ \ + DECLARE_REG(struct vgic_v3_cpu_if *, cif, ctxt, regnr);\ + struct kvm_vcpu *__vcpu = container_of(cif, \ + struct kvm_vcpu, \ + arch.vgic_cpu.vgic_v3); \ + \ + __get_host_hyp_vcpus(__vcpu, hyp_vcpup); \ + }) + static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt) { struct pkvm_hyp_vcpu *hyp_vcpu; @@ -514,16 +526,63 @@ static void handle___vgic_v3_init_lrs(struct kvm_cpu_context *host_ctxt) static void handle___vgic_v3_save_aprs(struct kvm_cpu_context *host_ctxt) { - DECLARE_REG(struct vgic_v3_cpu_if *, cpu_if, host_ctxt, 1); + struct pkvm_hyp_vcpu *hyp_vcpu; + struct kvm_vcpu *host_vcpu; - __vgic_v3_save_aprs(kern_hyp_va(cpu_if)); + host_vcpu = get_host_hyp_vcpus_from_vgic_v3_cpu_if(host_ctxt, 1, + &hyp_vcpu); + if (!host_vcpu) + return; + + if (unlikely(hyp_vcpu)) { + struct vgic_v3_cpu_if *hyp_cpu_if, *host_cpu_if; + int i; + + hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3; + __vgic_v3_save_aprs(hyp_cpu_if); + + host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3; + host_cpu_if->vgic_vmcr = hyp_cpu_if->vgic_vmcr; + for (i = 0; i < ARRAY_SIZE(host_cpu_if->vgic_ap0r); i++) { + host_cpu_if->vgic_ap0r[i] = hyp_cpu_if->vgic_ap0r[i]; + host_cpu_if->vgic_ap1r[i] = hyp_cpu_if->vgic_ap1r[i]; + } + } else { + __vgic_v3_save_aprs(&host_vcpu->arch.vgic_cpu.vgic_v3); + } } static void handle___vgic_v3_restore_vmcr_aprs(struct kvm_cpu_context *host_ctxt) { - DECLARE_REG(struct vgic_v3_cpu_if *, cpu_if, host_ctxt, 1); + struct pkvm_hyp_vcpu *hyp_vcpu; + struct kvm_vcpu *host_vcpu; - __vgic_v3_restore_vmcr_aprs(kern_hyp_va(cpu_if)); + host_vcpu = get_host_hyp_vcpus_from_vgic_v3_cpu_if(host_ctxt, 1, + &hyp_vcpu); + if (!host_vcpu) + return; + + if (unlikely(hyp_vcpu)) { + struct vgic_v3_cpu_if *hyp_cpu_if, *host_cpu_if; + int i; + + hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3; + host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3; + + hyp_cpu_if->vgic_vmcr = host_cpu_if->vgic_vmcr; + /* Should be a one-off */ + hyp_cpu_if->vgic_sre = (ICC_SRE_EL1_DIB | + ICC_SRE_EL1_DFB | + ICC_SRE_EL1_SRE); + for (i = 0; i < ARRAY_SIZE(host_cpu_if->vgic_ap0r); i++) { + hyp_cpu_if->vgic_ap0r[i] = host_cpu_if->vgic_ap0r[i]; + hyp_cpu_if->vgic_ap1r[i] = host_cpu_if->vgic_ap1r[i]; + } + + __vgic_v3_restore_vmcr_aprs(hyp_cpu_if); + } else { + __vgic_v3_restore_vmcr_aprs(&host_vcpu->arch.vgic_cpu.vgic_v3); + } } static void handle___pkvm_init(struct kvm_cpu_context *host_ctxt) From 42fe26543e66c27932b4c7f4c5c149e38f4e14b9 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Wed, 29 Jul 2026 14:18:22 +0100 Subject: [PATCH 25/33] KVM: arm64: Add primitives to flush/sync the VGIC state at EL2 pKVM performs its own world switch for protected VMs but has no primitives to move the per-vCPU VGIC state between the host and hypervisor vCPU contexts. Add flush_hyp_vgic_state() and sync_hyp_vgic_state(). Flush copies vgic_hcr, the in-use list registers and used_lrs from the host into the hyp vCPU and pins vgic_sre to a fixed value; sync copies vgic_hcr, vgic_vmcr and the in-use list registers back. The active priority registers are handled separately by the save/restore-aprs path. Bound used_lrs by hyp_gicv3_nr_lr, the cached implemented-LR count, instead of reading ICH_VTR_EL2 on each entry. That clamps the host-supplied value and avoids a per-entry sysreg read that is costly under NV. Reviewed-by: Vincent Donnefort Signed-off-by: Marc Zyngier Co-developed-by: Fuad Tabba Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260729131823.2021516-8-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 55 ++++++++++++++++++++++-------- 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index 415bbd7f9452..ac36257c8da2 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -104,6 +104,45 @@ static void fpsimd_sve_sync(struct kvm_vcpu *vcpu) *host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED; } +static void flush_hyp_vgic_state(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct vgic_v3_cpu_if *host_cpu_if, *hyp_cpu_if; + unsigned int used_lrs, i; + + host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3; + hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3; + + used_lrs = host_cpu_if->used_lrs; + used_lrs = min(used_lrs, hyp_gicv3_nr_lr); + + hyp_cpu_if->vgic_hcr = host_cpu_if->vgic_hcr; + /* Should be a one-off */ + hyp_cpu_if->vgic_sre = (ICC_SRE_EL1_DIB | + ICC_SRE_EL1_DFB | + ICC_SRE_EL1_SRE); + hyp_cpu_if->used_lrs = used_lrs; + + for (i = 0; i < used_lrs; i++) + hyp_cpu_if->vgic_lr[i] = host_cpu_if->vgic_lr[i]; +} + +static void sync_hyp_vgic_state(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + struct vgic_v3_cpu_if *host_cpu_if, *hyp_cpu_if; + unsigned int i; + + host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3; + hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3; + + host_cpu_if->vgic_hcr = hyp_cpu_if->vgic_hcr; + host_cpu_if->vgic_vmcr = hyp_cpu_if->vgic_vmcr; + + for (i = 0; i < hyp_cpu_if->used_lrs; i++) + host_cpu_if->vgic_lr[i] = hyp_cpu_if->vgic_lr[i]; +} + static void flush_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; @@ -158,13 +197,7 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2; - hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3 = host_vcpu->arch.vgic_cpu.vgic_v3; - - /* Bound used_lrs by the number of implemented list registers. */ - hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3.used_lrs = - min_t(unsigned int, - hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3.used_lrs, - hyp_gicv3_nr_lr); + flush_hyp_vgic_state(hyp_vcpu); hyp_vcpu->vcpu.arch.pid = host_vcpu->arch.pid; } @@ -172,9 +205,6 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; - struct vgic_v3_cpu_if *hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3; - struct vgic_v3_cpu_if *host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3; - unsigned int i; fpsimd_sve_sync(&hyp_vcpu->vcpu); sync_debug_state(hyp_vcpu); @@ -187,10 +217,7 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) host_vcpu->arch.iflags = hyp_vcpu->vcpu.arch.iflags; - host_cpu_if->vgic_hcr = hyp_cpu_if->vgic_hcr; - host_cpu_if->vgic_vmcr = hyp_cpu_if->vgic_vmcr; - for (i = 0; i < hyp_cpu_if->used_lrs; ++i) - host_cpu_if->vgic_lr[i] = hyp_cpu_if->vgic_lr[i]; + sync_hyp_vgic_state(hyp_vcpu); } static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt) From 734dc8c01c838779ecda7d0d113839c2b68910be Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Wed, 29 Jul 2026 14:18:23 +0100 Subject: [PATCH 26/33] KVM: arm64: Implement lazy vCPU state sync for non-protected guests pKVM copies a non-protected guest's register context between the host and the hypervisor on every world switch, even when the host never inspects it. Defer the copy: on entry, flush the host context into the hyp vCPU only when the host marked it dirty (PKVM_HOST_STATE_DIRTY); on exit, leave it in the hyp vCPU and copy it back only when the host needs it, via a __pkvm_vcpu_sync_state hypercall or at vcpu put. A protected guest's context is copied as before, since lazy sync only helps where the host is trusted to see the guest's registers. PC and PSTATE are the exception, copied back on every exit: the kvm_exit tracepoint reports the guest's real exit PC, and vcpu_mode_is_bad_32bit(), the SError-masking checks and the host software-step state machine all read PSTATE before the next on-demand sync. Without it an ARM_EXCEPTION_IRQ exit syncs nothing, so flush_debug_state() pushes a stale PSTATE.SS back on the next entry and single-stepping runs an extra instruction before the step is reported. The host needs the full context when it is about to read it (trap handling) or write it (the SError injection that writes ESR_EL1). Sync both from handle_exit_early(), which runs non-preemptible so the loaded hyp vCPU is stable without a preempt guard. Reviewed-by: Vincent Donnefort Signed-off-by: Fuad Tabba Link: https://patch.msgid.link/20260729131823.2021516-9-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_asm.h | 1 + arch/arm64/include/asm/kvm_host.h | 2 + arch/arm64/kvm/arm.c | 7 +++ arch/arm64/kvm/handle_exit.c | 23 ++++++++ arch/arm64/kvm/hyp/nvhe/hyp-main.c | 89 ++++++++++++++++++++++++++++-- 5 files changed, 117 insertions(+), 5 deletions(-) diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 043495f7fc78..6e1135b3ded4 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -113,6 +113,7 @@ enum __kvm_host_smccc_func { __KVM_HOST_SMCCC_FUNC___pkvm_finalize_teardown_vm, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_load, __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_put, + __KVM_HOST_SMCCC_FUNC___pkvm_vcpu_sync_state, __KVM_HOST_SMCCC_FUNC___pkvm_tlb_flush_vmid, MARKER(__KVM_HOST_SMCCC_FUNC_MAX) diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index bae2c4f92ef5..86d45bc8f495 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -1051,6 +1051,8 @@ struct kvm_vcpu_arch { #define INCREMENT_PC __vcpu_single_flag(iflags, BIT(1)) /* Target EL/MODE (not a single flag, but let's abuse the macro) */ #define EXCEPT_MASK __vcpu_single_flag(iflags, GENMASK(3, 1)) +/* Host-set: the hyp flushes the non-protected vCPU state in on entry */ +#define PKVM_HOST_STATE_DIRTY __vcpu_single_flag(iflags, BIT(4)) /* Helpers to encode exceptions with minimum fuss */ #define __EXCEPT_MASK_VAL unpack_vcpu_flag(EXCEPT_MASK) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 50adfff75be8..ace7b67fe1cc 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -735,6 +735,10 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) if (is_protected_kvm_enabled()) { kvm_call_hyp(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3); kvm_call_hyp_nvhe(__pkvm_vcpu_put); + + /* __pkvm_vcpu_put implies a sync of the state */ + if (!kvm_vm_is_protected(vcpu->kvm)) + vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); } kvm_vcpu_put_debug(vcpu); @@ -966,6 +970,9 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu) return ret; if (is_protected_kvm_enabled()) { + /* Start with the vcpu in a dirty state */ + if (!kvm_vm_is_protected(vcpu->kvm)) + vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); ret = pkvm_create_hyp_vm(kvm); if (ret) return ret; diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index 54aedf93c78b..29108e5c0206 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -486,9 +486,32 @@ int handle_exit(struct kvm_vcpu *vcpu, int exception_index) } } +static void handle_exit_pkvm_state(struct kvm_vcpu *vcpu, int exception_index) +{ + int exception_code = ARM_EXCEPTION_CODE(exception_index); + + if (!is_protected_kvm_enabled() || kvm_vm_is_protected(vcpu->kvm)) + return; + + /* + * Sync the context back when the host will read (trap) or write + * (SError) it. Preempt-off here, so the loaded hyp vCPU is stable. + */ + if (exception_code == ARM_EXCEPTION_TRAP || + exception_code == ARM_EXCEPTION_EL1_SERROR || + ARM_SERROR_PENDING(exception_index)) { + kvm_call_hyp_nvhe(__pkvm_vcpu_sync_state); + vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY); + } else { + vcpu_clear_flag(vcpu, PKVM_HOST_STATE_DIRTY); + } +} + /* For exit types that need handling before we can be preempted */ void handle_exit_early(struct kvm_vcpu *vcpu, int exception_index) { + handle_exit_pkvm_state(vcpu, exception_index); + if (ARM_SERROR_PENDING(exception_index)) { if (this_cpu_has_cap(ARM64_HAS_RAS_EXTN)) { u64 disr = kvm_vcpu_get_disr(vcpu); diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c index ac36257c8da2..65a7c735aa39 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c +++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c @@ -143,6 +143,48 @@ static void sync_hyp_vgic_state(struct pkvm_hyp_vcpu *hyp_vcpu) host_cpu_if->vgic_lr[i] = hyp_cpu_if->vgic_lr[i]; } +static void __copy_vcpu_state(const struct kvm_vcpu *from_vcpu, + struct kvm_vcpu *to_vcpu) +{ + int i; + + to_vcpu->arch.ctxt.regs = from_vcpu->arch.ctxt.regs; + to_vcpu->arch.ctxt.spsr_abt = from_vcpu->arch.ctxt.spsr_abt; + to_vcpu->arch.ctxt.spsr_und = from_vcpu->arch.ctxt.spsr_und; + to_vcpu->arch.ctxt.spsr_irq = from_vcpu->arch.ctxt.spsr_irq; + to_vcpu->arch.ctxt.spsr_fiq = from_vcpu->arch.ctxt.spsr_fiq; + to_vcpu->arch.ctxt.fp_regs = from_vcpu->arch.ctxt.fp_regs; + + /* + * Copy the sysregs, but don't mess with the timer state which + * is directly handled by EL1 and is expected to be preserved. + * enum vcpu_sysreg is sparse: VNCR-mapped registers take values + * derived from their VNCR page offset, so the timer registers do + * not form a contiguous numeric range and must be skipped by name. + */ + for (i = 1; i < NR_SYS_REGS; i++) { + switch (i) { + case CNTVOFF_EL2: + case CNTV_CVAL_EL0: + case CNTV_CTL_EL0: + case CNTP_CVAL_EL0: + case CNTP_CTL_EL0: + continue; + } + to_vcpu->arch.ctxt.sys_regs[i] = from_vcpu->arch.ctxt.sys_regs[i]; + } +} + +static void sync_hyp_vcpu_state(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + __copy_vcpu_state(&hyp_vcpu->vcpu, hyp_vcpu->host_vcpu); +} + +static void flush_hyp_vcpu_state(struct pkvm_hyp_vcpu *hyp_vcpu) +{ + __copy_vcpu_state(hyp_vcpu->host_vcpu, &hyp_vcpu->vcpu); +} + static void flush_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu) { struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; @@ -178,7 +220,17 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) fpsimd_sve_flush(); flush_debug_state(hyp_vcpu); - hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt; + /* + * If we deal with a non-protected guest and the state is potentially + * dirty (from a host perspective), copy the state back into the hyp + * vcpu. + */ + if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { + if (vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY)) + flush_hyp_vcpu_state(hyp_vcpu); + } else { + hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt; + } /* __hyp_running_vcpu must be NULL in a guest context. */ hyp_vcpu->vcpu.arch.ctxt.__hyp_running_vcpu = NULL; @@ -209,9 +261,16 @@ static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu) fpsimd_sve_sync(&hyp_vcpu->vcpu); sync_debug_state(hyp_vcpu); - host_vcpu->arch.ctxt = hyp_vcpu->vcpu.arch.ctxt; - - host_vcpu->arch.hcr_el2 = hyp_vcpu->vcpu.arch.hcr_el2; + if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) { + host_vcpu->arch.ctxt = hyp_vcpu->vcpu.arch.ctxt; + } else { + /* + * PC feeds trace_kvm_exit(), PSTATE.SS the host software-step + * machine, and both run before the next on-demand ctxt sync. + */ + host_vcpu->arch.ctxt.regs.pc = hyp_vcpu->vcpu.arch.ctxt.regs.pc; + host_vcpu->arch.ctxt.regs.pstate = hyp_vcpu->vcpu.arch.ctxt.regs.pstate; + } host_vcpu->arch.fault = hyp_vcpu->vcpu.arch.fault; @@ -245,8 +304,27 @@ static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt) { struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); - if (hyp_vcpu) + if (hyp_vcpu) { + struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu; + + if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu) && + !vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY)) { + sync_hyp_vcpu_state(hyp_vcpu); + } + pkvm_put_hyp_vcpu(hyp_vcpu); + } +} + +static void handle___pkvm_vcpu_sync_state(struct kvm_cpu_context *host_ctxt) +{ + struct pkvm_hyp_vcpu *hyp_vcpu; + + hyp_vcpu = pkvm_get_loaded_hyp_vcpu(); + if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu)) + return; + + sync_hyp_vcpu_state(hyp_vcpu); } static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *arg, @@ -877,6 +955,7 @@ static const hcall_t host_hcall[] = { HANDLE_FUNC(__pkvm_finalize_teardown_vm), HANDLE_FUNC(__pkvm_vcpu_load), HANDLE_FUNC(__pkvm_vcpu_put), + HANDLE_FUNC(__pkvm_vcpu_sync_state), HANDLE_FUNC(__pkvm_tlb_flush_vmid), }; From fff4eb33ffa082f1a106a5d696d035eac8d7842c Mon Sep 17 00:00:00 2001 From: Bradley Morgan Date: Fri, 17 Jul 2026 14:03:11 +0100 Subject: [PATCH 27/33] KVM: arm64: Skip cache maintenance for non-cacheable pKVM mappings The pKVM flush path walks its own pkvm_mappings list and cleans the data cache for every mapping, unlike the generic stage-2 walker it shadows, which skips non-cacheable leaves. Cleaning the cacheable alias of a non-cacheable mapping is pointless and can corrupt a device endpoint. Record whether a mapping is non-cacheable in spare bits of nr_pages and skip cache maintenance for it. Fixes: e912efed485a ("KVM: arm64: Introduce the EL1 pKVM MMU") Suggested-by: Marc Zyngier Signed-off-by: Bradley Morgan [tabba: use Marc's anonymous bitfield in place of the open-coded mask and helpers] Signed-off-by: Fuad Tabba Reviewed-by: Vincent Donnefort Tested-by: Bradley Morgan # On pixel 7, Android 17 CP2A.260705.006 Test: Bradley Morgan # On QEMU arm64 host Link: https://patch.msgid.link/20260717130317.1953574-2-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_pkvm.h | 5 ++++- arch/arm64/kvm/pkvm.c | 15 +++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index 74fedd9c5ff0..57afb07d6b13 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -195,7 +195,10 @@ struct pkvm_mapping { struct rb_node node; u64 gfn; u64 pfn; - u64 nr_pages; + struct { + u64 nr_pages:48; + u64 nc:1; + }; u64 __subtree_last; /* Internal member for interval tree */ }; diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index 008766273912..a66105db8b11 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -366,7 +366,7 @@ static int __pkvm_pgtable_stage2_unshare(struct kvm_pgtable *pgt, u64 start, u64 for_each_mapping_in_range_safe(pgt, start, end, mapping) { ret = kvm_call_hyp_nvhe(__pkvm_host_unshare_guest, handle, mapping->gfn, - mapping->nr_pages); + (u64)mapping->nr_pages); if (WARN_ON(ret)) return ret; pkvm_mapping_remove(mapping, &pgt->pkvm_mappings); @@ -470,6 +470,7 @@ int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, mapping->gfn = gfn; mapping->pfn = pfn; mapping->nr_pages = size / PAGE_SIZE; + mapping->nc = !!(prot & (KVM_PGTABLE_PROT_DEVICE | KVM_PGTABLE_PROT_NORMAL_NC)); pkvm_mapping_insert(mapping, &pgt->pkvm_mappings); return ret; @@ -500,7 +501,7 @@ int pkvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size) lockdep_assert_held(&kvm->mmu_lock); for_each_mapping_in_range_safe(pgt, addr, addr + size, mapping) { ret = kvm_call_hyp_nvhe(__pkvm_host_wrprotect_guest, handle, mapping->gfn, - mapping->nr_pages); + (u64)mapping->nr_pages); if (WARN_ON(ret)) break; } @@ -514,9 +515,11 @@ int pkvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size) struct pkvm_mapping *mapping; lockdep_assert_held(&kvm->mmu_lock); - for_each_mapping_in_range_safe(pgt, addr, addr + size, mapping) - __clean_dcache_guest_page(pfn_to_kaddr(mapping->pfn), - PAGE_SIZE * mapping->nr_pages); + for_each_mapping_in_range_safe(pgt, addr, addr + size, mapping) { + if (!mapping->nc) + __clean_dcache_guest_page(pfn_to_kaddr(mapping->pfn), + PAGE_SIZE * mapping->nr_pages); + } return 0; } @@ -534,7 +537,7 @@ bool pkvm_pgtable_stage2_test_clear_young(struct kvm_pgtable *pgt, u64 addr, u64 lockdep_assert_held(&kvm->mmu_lock); for_each_mapping_in_range_safe(pgt, addr, addr + size, mapping) young |= kvm_call_hyp_nvhe(__pkvm_host_test_clear_young_guest, handle, mapping->gfn, - mapping->nr_pages, mkold); + (u64)mapping->nr_pages, mkold); return young; } From 6bf0215c565ee122ffff7241b297a7b88f71005f Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 17 Jul 2026 14:03:12 +0100 Subject: [PATCH 28/33] KVM: arm64: Top up the memcache for pKVM permission faults A permission fault normally only relaxes a leaf, so user_mem_abort() skips the memcache top-up. Under pKVM such a fault can instead collapse pages into a block. That needs a fresh pkvm_mapping object, and without it cache->mapping is NULL, so pkvm_pgtable_stage2_map() dereferences NULL and faults the host under mmu_lock. Staging only the object is not enough: the hypervisor requires kvm_mmu_cache_min_pages in the memcache even for the allocation-free install, so under memcache pressure the collapse returns -ENOMEM and trips the WARN_ON(ret) in pkvm_pgtable_stage2_map() where a non-pKVM guest succeeds. Top up the full memcache for pKVM permission faults so both the mapping object and the min-pages are staged before mmu_lock. Fixes: db14091d8f75 ("KVM: arm64: Stage-2 huge mappings for np-guests") Reported-by: Bradley Morgan Link: https://lore.kernel.org/all/20260623161545.EA08E1F000E9@smtp.kernel.org/ Signed-off-by: Fuad Tabba Tested-by: Bradley Morgan # On pixel 7, Android 17 CP2A.260705.006 Test: Bradley Morgan # On QEMU arm64 host Link: https://patch.msgid.link/20260717130317.1953574-3-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/mmu.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 6c941aaa10c6..4d7c9bdcef00 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -2114,10 +2114,14 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd) * and so normally don't require allocations from the memcache. The * only exception to this is when dirty logging is enabled at runtime * and a write fault needs to collapse a block entry into a table. + * Under pKVM a permission fault can also collapse pages into a block, + * which needs a fresh mapping object, and the hypervisor requires the + * min-pages memcache even when the install allocates nothing. */ memcache = get_mmu_memcache(s2fd->vcpu); if (!perm_fault || (memslot_is_logging(s2fd->memslot) && - kvm_is_write_fault(s2fd->vcpu))) { + kvm_is_write_fault(s2fd->vcpu)) || + is_protected_kvm_enabled()) { ret = topup_mmu_memcache(s2fd->vcpu, memcache); if (ret) return ret; From 56fd1fc526e6413781a6980aa13ed9db104cbfea Mon Sep 17 00:00:00 2001 From: Bradley Morgan Date: Fri, 17 Jul 2026 14:03:13 +0100 Subject: [PATCH 29/33] KVM: arm64: Top up stage-2 memcache for dirty logging faults Dirty logging forces new stage-2 mappings to page size but does not always split an existing block first (eager splitting is best effort and off by default). A non-write permission fault on such a block, an instruction fetch, still needs a page-table page to split it, but the top-up is gated on write faults. With the cache empty, kvm_mmu_memory_cache_alloc() hits its guest-triggerable WARN_ON(!nobjs) and falls back to a GFP_ATOMIC allocation under mmu_lock, with a BUG_ON() if that fails. Top up the memcache for any permission fault while dirty logging is active. Fixes: 6f745f1bb5bf ("KVM: arm64: Convert user_mem_abort() to generic page-table API") Link: https://lore.kernel.org/all/20260623165634.699011F000E9@smtp.kernel.org/ Signed-off-by: Bradley Morgan [tabba: reword the commit message for the generic, non-pKVM failure mode] Signed-off-by: Fuad Tabba Tested-by: Bradley Morgan # On pixel 7, Android 17 CP2A.260705.006 Test: Bradley Morgan # On QEMU arm64 host Link: https://patch.msgid.link/20260717130317.1953574-4-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/mmu.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 4d7c9bdcef00..74e7e7f7564c 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -2113,14 +2113,13 @@ static int user_mem_abort(const struct kvm_s2_fault_desc *s2fd) * Permission faults just need to update the existing leaf entry, * and so normally don't require allocations from the memcache. The * only exception to this is when dirty logging is enabled at runtime - * and a write fault needs to collapse a block entry into a table. + * and a fault needs to collapse a block entry into a table. * Under pKVM a permission fault can also collapse pages into a block, * which needs a fresh mapping object, and the hypervisor requires the * min-pages memcache even when the install allocates nothing. */ memcache = get_mmu_memcache(s2fd->vcpu); - if (!perm_fault || (memslot_is_logging(s2fd->memslot) && - kvm_is_write_fault(s2fd->vcpu)) || + if (!perm_fault || memslot_is_logging(s2fd->memslot) || is_protected_kvm_enabled()) { ret = topup_mmu_memcache(s2fd->vcpu, memcache); if (ret) From 1feb9f6af895c1ec0caed269108d6ffa1f9a31c4 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 17 Jul 2026 14:03:14 +0100 Subject: [PATCH 30/33] KVM: arm64: Skip pKVM stage-2 flush when FWB is enabled pkvm_pgtable_stage2_flush() cleans the D-cache for every mapping in the range even on hardware with stage-2 Force Write-Back, where FWB keeps guest memory coherent to the PoC and the maintenance is unnecessary. The generic kvm_pgtable_stage2_flush() returns early in that case, but the pKVM MMU does not, so it needlessly cleans the whole range on, e.g., every set/way trap. Return early when FWB is enabled, matching the generic walker. Fixes: e912efed485a ("KVM: arm64: Introduce the EL1 pKVM MMU") Signed-off-by: Fuad Tabba Reviewed-by: Bradley Morgan Tested-by: Bradley Morgan # On pixel 7, Android 17 CP2A.260705.006 Test: Bradley Morgan # On QEMU arm64 host Link: https://patch.msgid.link/20260717130317.1953574-5-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/pkvm.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index a66105db8b11..b28d700c649e 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -515,6 +515,10 @@ int pkvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size) struct pkvm_mapping *mapping; lockdep_assert_held(&kvm->mmu_lock); + + if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) + return 0; + for_each_mapping_in_range_safe(pgt, addr, addr + size, mapping) { if (!mapping->nc) __clean_dcache_guest_page(pfn_to_kaddr(mapping->pfn), From c243222701690da30c010d6b98545684f5d78b15 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 17 Jul 2026 14:03:15 +0100 Subject: [PATCH 31/33] KVM: arm64: Don't WARN on pKVM stage-2 map failures pkvm_pgtable_stage2_map() wraps the __pkvm_host_share_guest() and __pkvm_host_donate_guest() return in WARN_ON(), but those hypercalls fail for reasons that are not EL1 invariant violations: -EINVAL for a pfn that is not memblock RAM (check_range_allowed_memory() rejects a device page mapped into a non-protected guest) and -ENOMEM under memcache pressure. Both are reachable from a guest fault, so the WARN splats on host input. Return the error without warning. The unshare and write-protect WARNs stay, since a failure there does signal a broken EL1 invariant. Fixes: 3669ddd8fa8b5 ("KVM: arm64: Add a range to pkvm_mappings") Signed-off-by: Fuad Tabba Reviewed-by: Bradley Morgan Tested-by: Bradley Morgan # On pixel 7, Android 17 CP2A.260705.006 Test: Bradley Morgan # On QEMU arm64 host Link: https://patch.msgid.link/20260717130317.1953574-6-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/kvm/pkvm.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c index b28d700c649e..ac1a5e2dc09f 100644 --- a/arch/arm64/kvm/pkvm.c +++ b/arch/arm64/kvm/pkvm.c @@ -463,7 +463,7 @@ int pkvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, size / PAGE_SIZE, prot); } - if (WARN_ON(ret)) + if (ret) return ret; swap(mapping, cache->mapping); From c6f308b4d341a239c1ef7423b82d62f50fd87745 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 17 Jul 2026 14:03:16 +0100 Subject: [PATCH 32/33] KVM: arm64: Don't advertise eager page splitting under pKVM Under pKVM the stage-2 walker resolves to pkvm_pgtable_stage2_split(), a WARN_ON_ONCE(1) stub, yet KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE is still enabled and reported for non-protected guests: the capability check keys on the per-VM protected state while the walker dispatch keys on the host-global mode. Enabling the cap and then dirty-logging the guest reaches the stub, splatting a userspace-reachable WARN. Reject the capability, and stop reporting a chunk size and the supported block sizes, for every VM once pKVM is enabled, keyed on the host-global mode like the split dispatch. Gating only protected VMs would leave the non-protected guests that reach the stub still able to enable it. Userspace decides whether eager splitting is available from the block-size bitmap (QEMU falls back to no eager splitting when it reads 0), so leaving it advertised steers an explicit request into the enable failure instead of the fallback. Fixes: e912efed485a ("KVM: arm64: Introduce the EL1 pKVM MMU") Signed-off-by: Fuad Tabba Reviewed-by: Bradley Morgan Tested-by: Bradley Morgan # On pixel 7, Android 17 CP2A.260705.006 Test: Bradley Morgan # On QEMU arm64 host Link: https://patch.msgid.link/20260717130317.1953574-7-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- arch/arm64/include/asm/kvm_pkvm.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/arch/arm64/include/asm/kvm_pkvm.h b/arch/arm64/include/asm/kvm_pkvm.h index 57afb07d6b13..beea00e693a0 100644 --- a/arch/arm64/include/asm/kvm_pkvm.h +++ b/arch/arm64/include/asm/kvm_pkvm.h @@ -45,6 +45,9 @@ static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext) return true; case KVM_CAP_ARM_MTE: return false; + case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE: + case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES: + return false; default: return !kvm || !kvm_vm_is_protected(kvm); } From 979ae12a2d090a2a858c72779d222baad78697a6 Mon Sep 17 00:00:00 2001 From: Fuad Tabba Date: Fri, 17 Jul 2026 14:03:17 +0100 Subject: [PATCH 33/33] KVM: arm64: selftests: Add stage-2 block transition test Add a test for the two stage-2 granularity changes dirty logging forces at fault time, asserting the guest completes with no KVM_RUN error. The first scenario collapses a page into a hugetlb-backed block: it writes under logging, re-write-protects the page via GET_DIRTY_LOG, then writes again with logging off. The second splits blocks: it faults in several non-executable 2M blocks, enables logging, then executes in each block so an execute permission fault splits it. It is skipped when CTR_EL0.DIC is set, since mappings are then executable and no execute fault occurs. Both paths make the fault handler allocate under mmu_lock, so a backend that fails to stage that memory returns a KVM_RUN error or crashes the host. The property holds on any host. On a pKVM host, where a non-protected guest uses the pkvm_pgtable_*() backend, it also guards that backend's fault-time staging. Assisted-by: Antigravity:gemini-3.1-pro Signed-off-by: Fuad Tabba Tested-by: Bradley Morgan # On QEMU arm64 host Test: Bradley Morgan # On QEMU arm64 host Link: https://patch.msgid.link/20260717130317.1953574-8-fuad.tabba@linux.dev Signed-off-by: Oliver Upton --- tools/testing/selftests/kvm/Makefile.kvm | 1 + .../kvm/arm64/stage2_block_transitions.c | 226 ++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100644 tools/testing/selftests/kvm/arm64/stage2_block_transitions.c diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index d28a057fa6c2..ed2877825eee 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -179,6 +179,7 @@ TEST_GEN_PROGS_arm64 += arm64/psci_test TEST_GEN_PROGS_arm64 += arm64/sea_to_user TEST_GEN_PROGS_arm64 += arm64/set_id_regs TEST_GEN_PROGS_arm64 += arm64/smccc_filter +TEST_GEN_PROGS_arm64 += arm64/stage2_block_transitions TEST_GEN_PROGS_arm64 += arm64/vcpu_width_config TEST_GEN_PROGS_arm64 += arm64/vgic_init TEST_GEN_PROGS_arm64 += arm64/vgic_irq diff --git a/tools/testing/selftests/kvm/arm64/stage2_block_transitions.c b/tools/testing/selftests/kvm/arm64/stage2_block_transitions.c new file mode 100644 index 000000000000..5fd47f4ada1f --- /dev/null +++ b/tools/testing/selftests/kvm/arm64/stage2_block_transitions.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2026 Google LLC + * Author: Fuad Tabba + * + * stage2_block_transitions - Exercise stage-2 block/page granularity changes + * that dirty logging forces at fault time, and assert the guest completes. + * + * Both scenarios need the fault handler to allocate at fault time (a fresh + * mapping and/or page-table pages while holding mmu_lock), so a fault path + * that fails to stage that memory manifests as a KVM_RUN error or, worse, a + * host crash. The asserted property is host-agnostic: the guest runs the + * sequence to completion and every KVM_RUN succeeds. On a pKVM host, where a + * non-protected guest's stage-2 faults are serviced by the pkvm_pgtable_*() + * backend, the same sequences also guard that backend's fault-time staging. + * + * Scenario 1 - block collapse on dirty-logging disable: + * A write under dirty logging installs a 4K page; GET_DIRTY_LOG + * re-write-protects it; logging is disabled; a second write takes a + * permission fault that collapses the page into a hugetlb-backed block, + * which requires a fresh mapping object under mmu_lock. + * + * Scenario 2 - block split under dirty logging: + * Several hugetlb-backed blocks are faulted in as non-executable blocks, + * dirty logging is enabled (write-protect only), then the guest executes + * into each block. Each instruction fetch takes an execute permission + * fault that must split the block into pages during logging, draining + * page-table pages. Skipped on CTR_EL0.DIC hardware, where mappings are + * made executable eagerly and the execute fault never occurs. + */ +#include +#include +#include +#include +#include + +#include + +#include "kvm_util.h" +#include "processor.h" +#include "test_util.h" +#include "ucall.h" + +#define DATA_SLOT 1 +#define TEST_GVA 0xc0000000UL +#define BLOCK_SIZE SZ_2M + +/* AArch64 "ret" (ret x30): a self-contained, returnable executable payload. */ +#define RET_INSN 0xd65f03c0U + +/* + * A non-protected guest's per-VM stage-2 pool is seeded only with the PGD + * donation, which stage-2 init immediately consumes, so the page-table budget + * for a fault that does not top up is just the handful (~2x the stage-2 min + * pages) of memcache leftovers. Executing into this many distinct blocks + * demands far more than that budget: a fault path that tops up on every fault + * completes all of them, one that skips non-write faults runs out mid-sequence. + */ +#define NR_BLOCKS 16 + +/* Scenario 2 guest -> host sync stages. */ +#define STAGE_SKIP_DIC 1 +#define STAGE_BLOCKS_READY 2 + +static void collapse_guest_code(u64 gva) +{ + u64 *data = (u64 *)gva; + + /* Under dirty logging: install a 4K writable page. */ + WRITE_ONCE(*data, 0x1); + GUEST_SYNC(1); + + /* Logging disabled: a permission fault collapses the page into a block. */ + WRITE_ONCE(*data, 0x2); + GUEST_SYNC(2); + + GUEST_DONE(); +} + +static void test_block_collapse(void) +{ + struct kvm_vcpu *vcpu; + unsigned long *bmap; + struct kvm_vm *vm; + struct ucall uc; + size_t npages; + u64 gpa; + + vm = vm_create_with_one_vcpu(&vcpu, collapse_guest_code); + npages = BLOCK_SIZE / vm->page_size; + + gpa = (vm_compute_max_gfn(vm) * vm->page_size) - BLOCK_SIZE; + gpa = align_down(gpa, BLOCK_SIZE); + + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_HUGETLB_2MB, gpa, + DATA_SLOT, npages, KVM_MEM_LOG_DIRTY_PAGES); + virt_map(vm, TEST_GVA, gpa, npages); + vcpu_args_set(vcpu, 1, TEST_GVA); + + bmap = bitmap_zalloc(BLOCK_SIZE / getpagesize()); + + vcpu_run(vcpu); + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_SYNC && uc.args[1] == 1, + "Expected first sync, got cmd %lu arg %lu", uc.cmd, uc.args[1]); + + /* GET_DIRTY_LOG re-write-protects the dirtied page; then stop logging. */ + kvm_vm_get_dirty_log(vm, DATA_SLOT, bmap); + vm_mem_region_set_flags(vm, DATA_SLOT, 0); + + /* The collapsing permission fault: a broken fault path faults here. */ + vcpu_run(vcpu); + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_SYNC && uc.args[1] == 2, + "Expected second sync, got cmd %lu arg %lu", uc.cmd, uc.args[1]); + + vcpu_run(vcpu); + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_DONE, + "Expected done, got cmd %lu", uc.cmd); + + free(bmap); + kvm_vm_free(vm); +} + +static void guest_sync_insn(u64 va) +{ + /* Make the just-written instruction coherent for execution (!DIC). */ + asm volatile("dc cvau, %0\n" + "dsb ish\n" + "ic ivau, %0\n" + "dsb ish\n" + "isb\n" + :: "r" (va) : "memory"); +} + +static void split_guest_code(u64 base_gva, u64 nblocks) +{ + u64 i, va; + + if (FIELD_GET(CTR_EL0_DIC_MASK, read_sysreg(ctr_el0))) { + GUEST_SYNC(STAGE_SKIP_DIC); + GUEST_DONE(); + return; + } + + /* Fault in each block (non-executable) and stage an executable payload. */ + for (i = 0; i < nblocks; i++) { + va = base_gva + i * BLOCK_SIZE; + WRITE_ONCE(*(u32 *)va, RET_INSN); + guest_sync_insn(va); + } + GUEST_SYNC(STAGE_BLOCKS_READY); + + /* Logging is now on: executing into each block splits it into pages. */ + for (i = 0; i < nblocks; i++) { + va = base_gva + i * BLOCK_SIZE; + ((void (*)(void))va)(); + } + + GUEST_DONE(); +} + +static void test_exec_split_drain(void) +{ + struct kvm_vcpu *vcpu; + struct kvm_vm *vm; + struct ucall uc; + size_t npages; + u64 gpa; + + vm = vm_create_with_one_vcpu(&vcpu, split_guest_code); + npages = NR_BLOCKS * (BLOCK_SIZE / vm->page_size); + + gpa = (vm_compute_max_gfn(vm) * vm->page_size) - NR_BLOCKS * BLOCK_SIZE; + gpa = align_down(gpa, BLOCK_SIZE); + + vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS_HUGETLB_2MB, gpa, + DATA_SLOT, npages, 0); + virt_map(vm, TEST_GVA, gpa, npages); + vcpu_args_set(vcpu, 2, TEST_GVA, (u64)NR_BLOCKS); + + vcpu_run(vcpu); + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_SYNC, + "Expected sync, got cmd %lu", uc.cmd); + if (uc.args[1] == STAGE_SKIP_DIC) { + ksft_print_msg("SKIP block split: CTR_EL0.DIC == 1\n"); + kvm_vm_free(vm); + return; + } + TEST_ASSERT(uc.args[1] == STAGE_BLOCKS_READY, + "Expected blocks-ready sync, got arg %lu", uc.args[1]); + + /* Write-protect the blocks; the guest then splits them by executing. */ + vm_mem_region_set_flags(vm, DATA_SLOT, KVM_MEM_LOG_DIRTY_PAGES); + + vcpu_run(vcpu); + TEST_ASSERT(get_ucall(vcpu, &uc) == UCALL_DONE, + "Expected done, got cmd %lu", uc.cmd); + + kvm_vm_free(vm); +} + +/* + * The explicit-size hugetlb backing hard-fails region creation if the pages + * are not already reserved, so probe here and skip rather than abort. The + * peak reservation is scenario 2's; the two scenarios run and free in turn. + */ +static void require_hugepages(size_t bytes) +{ + void *mem = mmap(NULL, bytes, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB | MAP_HUGE_2MB, + -1, 0); + + if (mem == MAP_FAILED) + ksft_exit_skip("Need %zu bytes of reserved 2M hugepages\n", bytes); + munmap(mem, bytes); +} + +int main(void) +{ + require_hugepages(NR_BLOCKS * BLOCK_SIZE); + + test_block_collapse(); + test_exec_split_drain(); + + ksft_print_msg("All ok!\n"); + return 0; +}