From a262fc49e0aafdeeb1b4fd8cb0d79a0fde447757 Mon Sep 17 00:00:00 2001 From: David Matlack Date: Fri, 26 Jun 2026 14:35:14 -0700 Subject: [PATCH 001/121] KVM: selftests: Build and link selftests/vfio/lib into KVM selftests Include libvfio.mk into the KVM selftests Makefile and link it into all KVM selftests by adding it to LIBKVM_OBJS. This lays the groundwork for future changes to utilize VFIO devices to verify IRQ bypass in KVM selftests. Note that KVM selftests build their own copy of selftests/vfio/lib and the resulting object files are placed in $(OUTPUT)/lib. This allows the KVM and VFIO selftests to apply different CFLAGS when building without conflicting with each other. Signed-off-by: David Matlack Signed-off-by: Josh Hilke Link: https://patch.msgid.link/20260626213534.3866178-2-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/Makefile.kvm | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index d28a057fa6c2..d1626fb21185 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -258,6 +258,7 @@ OVERRIDE_TARGETS = 1 # which causes the environment variable to override the makefile). include ../lib.mk include ../cgroup/lib/libcgroup.mk +include ../vfio/lib/libvfio.mk INSTALL_HDR_PATH = $(top_srcdir)/usr LINUX_HDR_PATH = $(INSTALL_HDR_PATH)/include/ @@ -312,7 +313,9 @@ LIBKVM_S := $(filter %.S,$(LIBKVM)) LIBKVM_C_OBJ := $(patsubst %.c, $(OUTPUT)/%.o, $(LIBKVM_C)) LIBKVM_S_OBJ := $(patsubst %.S, $(OUTPUT)/%.o, $(LIBKVM_S)) LIBKVM_STRING_OBJ := $(patsubst %.c, $(OUTPUT)/%.o, $(LIBKVM_STRING)) -LIBKVM_OBJS = $(LIBKVM_C_OBJ) $(LIBKVM_S_OBJ) $(LIBKVM_STRING_OBJ) $(LIBCGROUP_O) +LIBKVM_OBJS = $(LIBKVM_C_OBJ) $(LIBKVM_S_OBJ) $(LIBKVM_STRING_OBJ) +LIBKVM_OBJS += $(LIBCGROUP_O) +LIBKVM_OBJS += $(LIBVFIO_O) SPLIT_TEST_GEN_PROGS := $(patsubst %, $(OUTPUT)/%, $(SPLIT_TESTS)) SPLIT_TEST_GEN_OBJ := $(patsubst %, $(OUTPUT)/$(ARCH)/%.o, $(SPLIT_TESTS)) From ac050f2adcdfca6aca3c79c802c21245fab0b6d8 Mon Sep 17 00:00:00 2001 From: David Matlack Date: Fri, 26 Jun 2026 14:35:15 -0700 Subject: [PATCH 002/121] KVM: selftests: Add macros to read/write+sync to/from guest memory Add SYNC_FROM_GUEST_AND_READ(vm, variable), to read a variable value from the guest. Add WRITE_AND_SYNC_TO_GUEST(vm, variable, value) to write a value to a guest variable. These macros improve the readability of code which reads and writes data between host and guest in tests. Use the new macro in existing tests that do back-to-back write+sync. No functional changes are intended. Suggested-by: Sean Christopherson Signed-off-by: David Matlack Co-developed-by: Josh Hilke Signed-off-by: Josh Hilke [sean: massage changelog] Link: https://patch.msgid.link/20260626213534.3866178-3-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/dirty_log_test.c | 9 +++----- .../testing/selftests/kvm/include/kvm_util.h | 10 +++++++++ tools/testing/selftests/kvm/mmu_stress_test.c | 9 +++----- tools/testing/selftests/kvm/steal_time.c | 22 +++++++------------ 4 files changed, 24 insertions(+), 26 deletions(-) diff --git a/tools/testing/selftests/kvm/dirty_log_test.c b/tools/testing/selftests/kvm/dirty_log_test.c index 74ca096bf976..087e94a8a81a 100644 --- a/tools/testing/selftests/kvm/dirty_log_test.c +++ b/tools/testing/selftests/kvm/dirty_log_test.c @@ -708,8 +708,7 @@ static void run_test(enum vm_guest_mode mode, void *arg) sync_global_to_guest(vm, iteration); - WRITE_ONCE(nr_writes, 0); - sync_global_to_guest(vm, nr_writes); + WRITE_AND_SYNC_TO_GUEST(vm, nr_writes, 0); dirty_ring_prev_iteration_last_page = dirty_ring_last_page; WRITE_ONCE(dirty_ring_vcpu_ring_full, false); @@ -775,16 +774,14 @@ static void run_test(enum vm_guest_mode mode, void *arg) * writing memory during verification, pages that this thread * sees as clean may be written with this iteration's value. */ - WRITE_ONCE(vcpu_stop, true); - sync_global_to_guest(vm, vcpu_stop); + WRITE_AND_SYNC_TO_GUEST(vm, vcpu_stop, true); sem_wait(&sem_vcpu_stop); /* * Clear vcpu_stop after the vCPU thread has acknowledge the * stop request and is waiting, i.e. is definitely not running! */ - WRITE_ONCE(vcpu_stop, false); - sync_global_to_guest(vm, vcpu_stop); + WRITE_AND_SYNC_TO_GUEST(vm, vcpu_stop, false); /* * Sync the number of writes performed before verification, the diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index 04a910164a29..c1f588154398 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -1138,6 +1138,16 @@ vm_adjust_num_guest_pages(enum vm_guest_mode mode, unsigned int num_guest_pages) memcpy(&(g), _p, sizeof(g)); \ }) +#define SYNC_FROM_GUEST_AND_READ(_vm, _variable) ({ \ + sync_global_from_guest(_vm, _variable); \ + READ_ONCE(_variable); \ +}) + +#define WRITE_AND_SYNC_TO_GUEST(_vm, _variable, _value) do { \ + WRITE_ONCE(_variable, _value); \ + sync_global_to_guest(_vm, _variable); \ +} while (0) + /* * Write a global value, but only in the VM's (guest's) domain. Primarily used * for "globals" that hold per-VM values (VMs always duplicate code and global diff --git a/tools/testing/selftests/kvm/mmu_stress_test.c b/tools/testing/selftests/kvm/mmu_stress_test.c index 54d281419d31..473ef4c0ea9f 100644 --- a/tools/testing/selftests/kvm/mmu_stress_test.c +++ b/tools/testing/selftests/kvm/mmu_stress_test.c @@ -155,10 +155,8 @@ static void *vcpu_worker(void *data) "Expected EFAULT on write to RO memory, got r = %d, errno = %d", r, errno); atomic_inc(&nr_ro_faults); - if (atomic_read(&nr_ro_faults) == nr_vcpus) { - WRITE_ONCE(all_vcpus_hit_ro_fault, true); - sync_global_to_guest(vm, all_vcpus_hit_ro_fault); - } + if (atomic_read(&nr_ro_faults) == nr_vcpus) + WRITE_AND_SYNC_TO_GUEST(vm, all_vcpus_hit_ro_fault, true); #if defined(__x86_64__) || defined(__aarch64__) /* @@ -383,8 +381,7 @@ int main(int argc, char *argv[]) rendezvous_with_vcpus(&time_run2, "run 2"); mprotect(mem, slot_size, PROT_READ); - mprotect_ro_done = true; - sync_global_to_guest(vm, mprotect_ro_done); + WRITE_AND_SYNC_TO_GUEST(vm, mprotect_ro_done, true); rendezvous_with_vcpus(&time_ro, "mprotect RO"); mprotect(mem, slot_size, PROT_READ | PROT_WRITE); diff --git a/tools/testing/selftests/kvm/steal_time.c b/tools/testing/selftests/kvm/steal_time.c index 76fcdd1fd3cb..2de87549fcc0 100644 --- a/tools/testing/selftests/kvm/steal_time.c +++ b/tools/testing/selftests/kvm/steal_time.c @@ -70,8 +70,8 @@ static bool is_steal_time_supported(struct kvm_vcpu *vcpu) static void steal_time_init(struct kvm_vcpu *vcpu, u32 i) { /* ST_GPA_BASE is identity mapped */ - st_gva[i] = (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE); - sync_global_to_guest(vcpu->vm, st_gva[i]); + WRITE_AND_SYNC_TO_GUEST(vcpu->vm, st_gva[i], + (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE)); vcpu_set_msr(vcpu, MSR_KVM_STEAL_TIME, (ulong)st_gva[i] | KVM_MSR_ENABLED); } @@ -187,8 +187,7 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i) }; /* ST_GPA_BASE is identity mapped */ - st_gva[i] = (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE); - sync_global_to_guest(vm, st_gva[i]); + WRITE_AND_SYNC_TO_GUEST(vm, st_gva[i], (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE)); st_ipa = (ulong)st_gva[i]; vcpu_ioctl(vcpu, KVM_SET_DEVICE_ATTR, &dev); @@ -310,10 +309,8 @@ static bool is_steal_time_supported(struct kvm_vcpu *vcpu) static void steal_time_init(struct kvm_vcpu *vcpu, u32 i) { /* ST_GPA_BASE is identity mapped */ - st_gva[i] = (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE); - st_gpa[i] = addr_gva2gpa(vcpu->vm, (gva_t)st_gva[i]); - sync_global_to_guest(vcpu->vm, st_gva[i]); - sync_global_to_guest(vcpu->vm, st_gpa[i]); + WRITE_AND_SYNC_TO_GUEST(vcpu->vm, st_gva[i], (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE)); + WRITE_AND_SYNC_TO_GUEST(vcpu->vm, st_gpa[i], addr_gva2gpa(vcpu->vm, (gva_t)st_gva[i])); } static void steal_time_dump(struct kvm_vm *vm, u32 vcpu_idx) @@ -442,8 +439,7 @@ static void steal_time_init(struct kvm_vcpu *vcpu, u32 i) }; /* ST_GPA_BASE is identity mapped */ - st_gva[i] = (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE); - sync_global_to_guest(vm, st_gva[i]); + WRITE_AND_SYNC_TO_GUEST(vm, st_gva[i], (void *)(ST_GPA_BASE + i * STEAL_TIME_SIZE)); err = __vcpu_ioctl(vcpu, KVM_HAS_DEVICE_ATTR, &attr); TEST_ASSERT(err == 0, "No PV stealtime Feature"); @@ -549,8 +545,7 @@ int main(int ac, char **av) /* Second VCPU run, expect guest stolen time to be <= run_delay */ run_vcpu(vcpus[i]); - sync_global_from_guest(vm, guest_stolen_time[i]); - stolen_time = guest_stolen_time[i]; + stolen_time = SYNC_FROM_GUEST_AND_READ(vm, guest_stolen_time[i]); run_delay = get_run_delay(); TEST_ASSERT(stolen_time <= run_delay, "Expected stolen time <= %ld, got %ld", @@ -570,8 +565,7 @@ int main(int ac, char **av) /* Run VCPU again to confirm stolen time is consistent with run_delay */ run_vcpu(vcpus[i]); - sync_global_from_guest(vm, guest_stolen_time[i]); - stolen_time = guest_stolen_time[i] - stolen_time; + stolen_time = SYNC_FROM_GUEST_AND_READ(vm, guest_stolen_time[i]) - stolen_time; TEST_ASSERT(stolen_time >= run_delay, "Expected stolen time >= %ld, got %ld", run_delay, stolen_time); From e0bd29eddf7ecdd67f28f3c44ad89b02a08f245c Mon Sep 17 00:00:00 2001 From: Josh Hilke Date: Fri, 26 Jun 2026 14:35:16 -0700 Subject: [PATCH 003/121] KVM: selftests: Rename guest_rng to kvm_rng Rename functions prefixed with 'guest_random_' to 'kvm_random_' and the global random state variable 'guest_rng' to 'kvm_rng', as the pRNG isn't strictly limited to guest code. This will allow using the pRNG in host code without creating confusing/misleading function calls. No functional changes are intended. Suggested-by: Sean Christopherson Signed-off-by: Josh Hilke [sean: massage changelog] Link: https://patch.msgid.link/20260626213534.3866178-4-seanjc@google.com Signed-off-by: Sean Christopherson --- .../selftests/kvm/dirty_log_perf_test.c | 4 ++-- tools/testing/selftests/kvm/dirty_log_test.c | 2 +- .../testing/selftests/kvm/include/test_util.h | 22 +++++++++---------- .../selftests/kvm/include/x86/kvm_util_arch.h | 4 ++-- tools/testing/selftests/kvm/lib/kvm_util.c | 20 ++++++++--------- tools/testing/selftests/kvm/lib/memstress.c | 8 +++---- tools/testing/selftests/kvm/lib/test_util.c | 6 ++--- .../testing/selftests/kvm/x86/sev_dbg_test.c | 2 +- 8 files changed, 34 insertions(+), 34 deletions(-) diff --git a/tools/testing/selftests/kvm/dirty_log_perf_test.c b/tools/testing/selftests/kvm/dirty_log_perf_test.c index ef779fa91827..7c5abe1ae9e0 100644 --- a/tools/testing/selftests/kvm/dirty_log_perf_test.c +++ b/tools/testing/selftests/kvm/dirty_log_perf_test.c @@ -311,7 +311,7 @@ int main(int argc, char *argv[]) int opt; /* Override the seed to be deterministic by default. */ - guest_random_seed = 1; + kvm_random_seed = 1; dirty_log_manual_caps = kvm_check_cap(KVM_CAP_MANUAL_DIRTY_LOG_PROTECT2); @@ -357,7 +357,7 @@ int main(int argc, char *argv[]) p.phys_offset = strtoull(optarg, NULL, 0); break; case 'r': - guest_random_seed = atoi_positive("Random seed", optarg); + kvm_random_seed = atoi_positive("Random seed", optarg); break; case 's': p.backing_src = parse_backing_src_type(optarg); diff --git a/tools/testing/selftests/kvm/dirty_log_test.c b/tools/testing/selftests/kvm/dirty_log_test.c index 087e94a8a81a..e8419d7da1ea 100644 --- a/tools/testing/selftests/kvm/dirty_log_test.c +++ b/tools/testing/selftests/kvm/dirty_log_test.c @@ -121,7 +121,7 @@ static void guest_code(void) while (true) { while (!READ_ONCE(vcpu_stop)) { addr = guest_test_virt_mem; - addr += (guest_random_u64(&guest_rng) % guest_num_pages) + addr += (kvm_random_u64(&kvm_rng) % guest_num_pages) * guest_page_size; addr = align_down(addr, host_page_size); diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index a56271c237ae..44c0104d60ac 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -108,30 +108,30 @@ struct timespec timespec_sub(struct timespec ts1, struct timespec ts2); struct timespec timespec_elapsed(struct timespec start); struct timespec timespec_div(struct timespec ts, int divisor); -struct guest_random_state { +struct kvm_random_state { u32 seed; }; -extern u32 guest_random_seed; -extern struct guest_random_state guest_rng; +extern u32 kvm_random_seed; +extern struct kvm_random_state kvm_rng; -struct guest_random_state new_guest_random_state(u32 seed); -u32 guest_random_u32(struct guest_random_state *state); +struct kvm_random_state new_kvm_random_state(u32 seed); +u32 kvm_random_u32(struct kvm_random_state *state); -static inline bool __guest_random_bool(struct guest_random_state *state, +static inline bool __kvm_random_bool(struct kvm_random_state *state, u8 percent) { - return (guest_random_u32(state) % 100) < percent; + return (kvm_random_u32(state) % 100) < percent; } -static inline bool guest_random_bool(struct guest_random_state *state) +static inline bool kvm_random_bool(struct kvm_random_state *state) { - return __guest_random_bool(state, 50); + return __kvm_random_bool(state, 50); } -static inline u64 guest_random_u64(struct guest_random_state *state) +static inline u64 kvm_random_u64(struct kvm_random_state *state) { - return ((u64)guest_random_u32(state) << 32) | guest_random_u32(state); + return ((u64)kvm_random_u32(state) << 32) | kvm_random_u32(state); } enum vm_mem_backing_src_type { diff --git a/tools/testing/selftests/kvm/include/x86/kvm_util_arch.h b/tools/testing/selftests/kvm/include/x86/kvm_util_arch.h index c33ab6e04171..6904dbda79f9 100644 --- a/tools/testing/selftests/kvm/include/x86/kvm_util_arch.h +++ b/tools/testing/selftests/kvm/include/x86/kvm_util_arch.h @@ -55,9 +55,9 @@ static inline bool __vm_arch_has_protected_memory(struct kvm_vm_arch *arch) do { \ const typeof(mem) val = (__val); \ \ - if (!is_forced_emulation_enabled || guest_random_bool(&guest_rng)) { \ + if (!is_forced_emulation_enabled || kvm_random_bool(&kvm_rng)) { \ (mem) = val; \ - } else if (guest_random_bool(&guest_rng)) { \ + } else if (kvm_random_bool(&kvm_rng)) { \ __asm__ __volatile__(KVM_FEP "mov %1, %0" \ : "+m" (mem) \ : "r" (val) : "memory"); \ diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 195f3fdae1e3..875030c22d07 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -20,9 +20,9 @@ #define KVM_UTIL_MIN_PFN 2 -u32 guest_random_seed; -struct guest_random_state guest_rng; -static u32 last_guest_seed; +u32 kvm_random_seed; +struct kvm_random_state kvm_rng; +static u32 last_kvm_seed; static size_t vcpu_mmap_sz(void); @@ -515,12 +515,12 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus, slot0 = memslot2region(vm, 0); ucall_init(vm, slot0->region.guest_phys_addr + slot0->region.memory_size); - if (guest_random_seed != last_guest_seed) { - pr_info("Random seed: 0x%x\n", guest_random_seed); - last_guest_seed = guest_random_seed; + if (kvm_random_seed != last_kvm_seed) { + pr_info("Random seed: 0x%x\n", kvm_random_seed); + last_kvm_seed = kvm_random_seed; } - guest_rng = new_guest_random_state(guest_random_seed); - sync_global_to_guest(vm, guest_rng); + kvm_rng = new_kvm_random_state(kvm_random_seed); + sync_global_to_guest(vm, kvm_rng); kvm_arch_vm_post_create(vm, nr_runnable_vcpus); @@ -2279,8 +2279,8 @@ void __attribute((constructor)) kvm_selftest_init(void) sigaction(SIGILL, &sig_sa, NULL); sigaction(SIGFPE, &sig_sa, NULL); - guest_random_seed = last_guest_seed = random(); - pr_info("Random seed: 0x%x\n", guest_random_seed); + kvm_random_seed = last_kvm_seed = random(); + pr_info("Random seed: 0x%x\n", kvm_random_seed); kvm_selftest_arch_init(); } diff --git a/tools/testing/selftests/kvm/lib/memstress.c b/tools/testing/selftests/kvm/lib/memstress.c index 6dcd15910a06..3599b75d97c9 100644 --- a/tools/testing/selftests/kvm/lib/memstress.c +++ b/tools/testing/selftests/kvm/lib/memstress.c @@ -48,14 +48,14 @@ void memstress_guest_code(u32 vcpu_idx) { struct memstress_args *args = &memstress_args; struct memstress_vcpu_args *vcpu_args = &args->vcpu_args[vcpu_idx]; - struct guest_random_state rand_state; + struct kvm_random_state rand_state; gva_t gva; u64 pages; u64 addr; u64 page; int i; - rand_state = new_guest_random_state(guest_random_seed + vcpu_idx); + rand_state = new_kvm_random_state(kvm_random_seed + vcpu_idx); gva = vcpu_args->gva; pages = vcpu_args->pages; @@ -69,13 +69,13 @@ void memstress_guest_code(u32 vcpu_idx) for (i = 0; i < pages; i++) { if (args->random_access) - page = guest_random_u32(&rand_state) % pages; + page = kvm_random_u32(&rand_state) % pages; else page = i; addr = gva + (page * args->guest_page_size); - if (__guest_random_bool(&rand_state, args->write_percent)) + if (__kvm_random_bool(&rand_state, args->write_percent)) *(u64 *)addr = 0x0123456789ABCDEF; else READ_ONCE(*(u64 *)addr); diff --git a/tools/testing/selftests/kvm/lib/test_util.c b/tools/testing/selftests/kvm/lib/test_util.c index bab1bd2b775b..e98ca7ef439c 100644 --- a/tools/testing/selftests/kvm/lib/test_util.c +++ b/tools/testing/selftests/kvm/lib/test_util.c @@ -30,13 +30,13 @@ void __attribute__((used)) expect_sigbus_handler(int signum) * Park-Miller LCG using standard constants. */ -struct guest_random_state new_guest_random_state(u32 seed) +struct kvm_random_state new_kvm_random_state(u32 seed) { - struct guest_random_state s = {.seed = seed}; + struct kvm_random_state s = {.seed = seed}; return s; } -u32 guest_random_u32(struct guest_random_state *state) +u32 kvm_random_u32(struct kvm_random_state *state) { state->seed = (u64)state->seed * 48271 % ((u32)(1 << 31) - 1); return state->seed; diff --git a/tools/testing/selftests/kvm/x86/sev_dbg_test.c b/tools/testing/selftests/kvm/x86/sev_dbg_test.c index a9d8e4c059f9..eaa8201b937d 100644 --- a/tools/testing/selftests/kvm/x86/sev_dbg_test.c +++ b/tools/testing/selftests/kvm/x86/sev_dbg_test.c @@ -34,7 +34,7 @@ static void validate_buffers(void) static void ____test_sev_dbg(struct kvm_vm *vm, int i, int j, int nr_bytes) { - u8 pattern = guest_random_u32(&guest_rng); + u8 pattern = kvm_random_u32(&kvm_rng); if (i + nr_bytes > BUFFER_SIZE || j + nr_bytes > BUFFER_SIZE) return; From bae85cd758fe30b3837c6dcfc0572db34aecea45 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 26 Jun 2026 14:35:17 -0700 Subject: [PATCH 004/121] KVM: selftests: Initialize the default/global pRNG during kvm_selftest_init() Initialize the default kvm_rng during selftest initialization so that the pRNG can be used by tests before creating a VM. As pointed out by Sashiko, failure to actually initialize the generate makes it decidedly not random. Link: https://patch.msgid.link/20260626213534.3866178-5-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/lib/kvm_util.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 875030c22d07..1016865d3f7a 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -24,6 +24,13 @@ u32 kvm_random_seed; struct kvm_random_state kvm_rng; static u32 last_kvm_seed; +static void kvm_seed_rng(u32 seed) +{ + kvm_random_seed = last_kvm_seed = seed; + pr_info("Random seed: 0x%x\n", kvm_random_seed); + kvm_rng = new_kvm_random_state(kvm_random_seed); +} + static size_t vcpu_mmap_sz(void); int __open_path_or_exit(const char *path, int flags, const char *enoent_help) @@ -515,11 +522,9 @@ struct kvm_vm *__vm_create(struct vm_shape shape, u32 nr_runnable_vcpus, slot0 = memslot2region(vm, 0); ucall_init(vm, slot0->region.guest_phys_addr + slot0->region.memory_size); - if (kvm_random_seed != last_kvm_seed) { - pr_info("Random seed: 0x%x\n", kvm_random_seed); - last_kvm_seed = kvm_random_seed; - } - kvm_rng = new_kvm_random_state(kvm_random_seed); + if (kvm_random_seed != last_kvm_seed) + kvm_seed_rng(kvm_random_seed); + sync_global_to_guest(vm, kvm_rng); kvm_arch_vm_post_create(vm, nr_runnable_vcpus); @@ -2279,8 +2284,7 @@ void __attribute((constructor)) kvm_selftest_init(void) sigaction(SIGILL, &sig_sa, NULL); sigaction(SIGFPE, &sig_sa, NULL); - kvm_random_seed = last_kvm_seed = random(); - pr_info("Random seed: 0x%x\n", kvm_random_seed); + kvm_seed_rng(random()); kvm_selftest_arch_init(); } From 5506feb24282db0a9db5b0e151093e6eb74dd101 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 26 Jun 2026 14:35:18 -0700 Subject: [PATCH 005/121] KVM: selftests: Seed libc's RNG before using it to generate a seed for KVM's pRNG Seed the RNG used by random() using the de facto standard method of srand(time(0)), so that a different seed is actually used in each test run. E.g. without seeding the RNG, literally every test on x86 will use 0x6b8b4567 to seed the KVM RNG. Link: https://patch.msgid.link/20260626213534.3866178-6-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/lib/kvm_util.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 1016865d3f7a..277166ab1aa9 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -2284,6 +2285,7 @@ void __attribute((constructor)) kvm_selftest_init(void) sigaction(SIGILL, &sig_sa, NULL); sigaction(SIGFPE, &sig_sa, NULL); + srandom(time(0)); kvm_seed_rng(random()); kvm_selftest_arch_init(); From c772591d279e3023501c719d517817cceea8e76e Mon Sep 17 00:00:00 2001 From: Josh Hilke Date: Fri, 26 Jun 2026 14:35:19 -0700 Subject: [PATCH 006/121] KVM: selftests: Add helper to generate random u64 in range [min,max] Introduce kvm_random_u64_in_range(state, min, max). This function returns a random u64 in the inclusive range of [min, max] using a struct kvm_random_state. Suggested-by: Sean Christopherson Signed-off-by: Josh Hilke Link: https://patch.msgid.link/20260626213534.3866178-7-seanjc@google.com Signed-off-by: Sean Christopherson --- .../testing/selftests/kvm/include/test_util.h | 3 +++ tools/testing/selftests/kvm/lib/test_util.c | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/tools/testing/selftests/kvm/include/test_util.h b/tools/testing/selftests/kvm/include/test_util.h index 44c0104d60ac..d64c8a228207 100644 --- a/tools/testing/selftests/kvm/include/test_util.h +++ b/tools/testing/selftests/kvm/include/test_util.h @@ -134,6 +134,9 @@ static inline u64 kvm_random_u64(struct kvm_random_state *state) return ((u64)kvm_random_u32(state) << 32) | kvm_random_u32(state); } +u64 kvm_random_u64_in_range(struct kvm_random_state *state, u64 min, + u64 max); + enum vm_mem_backing_src_type { VM_MEM_SRC_ANONYMOUS, VM_MEM_SRC_ANONYMOUS_THP, diff --git a/tools/testing/selftests/kvm/lib/test_util.c b/tools/testing/selftests/kvm/lib/test_util.c index e98ca7ef439c..e208a57f190c 100644 --- a/tools/testing/selftests/kvm/lib/test_util.c +++ b/tools/testing/selftests/kvm/lib/test_util.c @@ -42,6 +42,24 @@ u32 kvm_random_u32(struct kvm_random_state *state) return state->seed; } +/* Returns a random u64 in the inclusive range [min, max] */ +u64 kvm_random_u64_in_range(struct kvm_random_state *state, u64 min, + u64 max) +{ + u64 value; + u64 range; + + TEST_ASSERT(min <= max, "PEBKAC, min = 0x%lx, max = 0x%lx", min, max); + + value = kvm_random_u64(state); + + range = max - min; + if (range == ULLONG_MAX) + return value; + + return min + (value % (range + 1)); +} + /* * Parses "[0-9]+[kmgt]?". */ From 3ceab37ac973a27a15e72af88d30d7a820b380f4 Mon Sep 17 00:00:00 2001 From: David Matlack Date: Fri, 26 Jun 2026 14:35:20 -0700 Subject: [PATCH 007/121] KVM: selftests: Add an irqfd send+receive (and later IRQ bypass) test Add a new test, irq_test to verify that KVM correctly delivers interrupts to a running vCPU, when triggered via an eventfd bound to a KVM GSI using KVM's irqfd mechanism. This test is intentionally simple, for now. Support for sending interrupts via VFIO devices, for IRQ bypass, and for other features will be added in the near future. Add the test in common code, even though it currently will only build and run on x86, as the concept and the bulk of the host-side code isn't specific to x86. Suggested-by: Sean Christopherson Link: https://lore.kernel.org/kvm/20250404193923.1413163-68-seanjc@google.com Signed-off-by: David Matlack Co-developed-by: Josh Hilke Signed-off-by: Josh Hilke [sean: use while() and TEST_ASSERT() instead of if-statement => TEST_FAIL()] Link: https://patch.msgid.link/20260626213534.3866178-8-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/Makefile.kvm | 1 + tools/testing/selftests/kvm/irq_test.c | 170 +++++++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 tools/testing/selftests/kvm/irq_test.c diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index d1626fb21185..31d92eb493b0 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -156,6 +156,7 @@ TEST_GEN_PROGS_x86 += coalesced_io_test TEST_GEN_PROGS_x86 += dirty_log_perf_test TEST_GEN_PROGS_x86 += guest_memfd_test TEST_GEN_PROGS_x86 += hardware_disable_test +TEST_GEN_PROGS_x86 += irq_test TEST_GEN_PROGS_x86 += mmu_stress_test TEST_GEN_PROGS_x86 += rseq_test TEST_GEN_PROGS_x86 += steal_time diff --git a/tools/testing/selftests/kvm/irq_test.c b/tools/testing/selftests/kvm/irq_test.c new file mode 100644 index 000000000000..9f8895b89821 --- /dev/null +++ b/tools/testing/selftests/kvm/irq_test.c @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "kvm_util.h" +#include "test_util.h" +#include "apic.h" +#include "processor.h" + +#include +#include +#include +#include +#include + +static u64 timeout_ns = 2ULL * 1000 * 1000 * 1000; +static bool guest_ready_for_irqs[KVM_MAX_VCPUS]; +static bool guest_received_irq[KVM_MAX_VCPUS]; +static bool done; + +#define GUEST_RECEIVED_IRQ(__vcpu) \ + SYNC_FROM_GUEST_AND_READ((__vcpu)->vm, guest_received_irq[(__vcpu)->id]) + +static u32 guest_get_vcpu_id(void) +{ + return x2apic_read_reg(APIC_ID); +} + +static void guest_irq_handler(struct ex_regs *regs) +{ + WRITE_ONCE(guest_received_irq[guest_get_vcpu_id()], true); + + x2apic_write_reg(APIC_EOI, 0); +} + +static void guest_code(void) +{ + x2apic_enable(); + + sti_nop(); + + WRITE_ONCE(guest_ready_for_irqs[guest_get_vcpu_id()], true); + + while (!READ_ONCE(done)) + cpu_relax(); + + GUEST_DONE(); +} + +static void *vcpu_thread_main(void *arg) +{ + struct kvm_vcpu *vcpu = arg; + struct ucall uc; + + vcpu_run(vcpu); + TEST_ASSERT_EQ(UCALL_DONE, get_ucall(vcpu, &uc)); + + return NULL; +} + +static void kvm_route_msi(struct kvm_vm *vm, u32 gsi, struct kvm_vcpu *vcpu, + u8 vector) +{ + struct { + struct kvm_irq_routing header; + struct kvm_irq_routing_entry entry; + } routing = { + .header.nr = 1, + .entry = { + .gsi = gsi, + .type = KVM_IRQ_ROUTING_MSI, + .u.msi.address_lo = 0xFEE00000 | (vcpu->id << 12), + .u.msi.data = vector, + }, + }; + + vm_ioctl(vm, KVM_SET_GSI_ROUTING, &routing.header); +} + +static void help(const char *name) +{ + printf("Usage: %s [-h]\n", name); + printf("\n"); + printf("Tests KVM interrupt routing and delivery via irqfd.\n"); + printf("\n"); + exit(KSFT_FAIL); +} + +int main(int argc, char **argv) +{ + /* + * Pick a random vector and a random GSI to use for device IRQ. + * + * Pick an IRQ vector in range [32, UINT8_MAX]. Min value is 32 because + * Linux/x86 reserves vectors 0-31 for exceptions and architecture + * defined NMIs and interrupts. + * + * Pick a GSI in range [24, KVM_MAX_IRQ_ROUTES - 1]. The min value is 24 + * because KVM reserves GSIs 0-15 for legacy ISA IRQs and 16-23 only go + * to the IOAPIC. The max is KVM_MAX_IRQ_ROUTES - 1, because + * KVM_MAX_IRQ_ROUTES is exclusive. + */ + u32 gsi = kvm_random_u64_in_range(&kvm_rng, 24, KVM_MAX_IRQ_ROUTES - 1); + u8 vector = kvm_random_u64_in_range(&kvm_rng, 32, UINT8_MAX); + + struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; + pthread_t vcpu_threads[KVM_MAX_VCPUS]; + int nr_irqs = 1000, nr_vcpus = 1; + int i, j, c, eventfd; + struct kvm_vm *vm; + + while ((c = getopt(argc, argv, "h")) != -1) { + switch (c) { + case 'h': + default: + help(argv[0]); + } + } + + TEST_REQUIRE(kvm_arch_has_default_irqchip()); + + vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus); + vm_install_exception_handler(vm, vector, guest_irq_handler); + + eventfd = kvm_new_eventfd(); + + pr_info("Injecting interrupts for GSI %d (guest vector 0x%x) %d times\n", + gsi, vector, nr_irqs); + + kvm_assign_irqfd(vm, gsi, eventfd); + + for (i = 0; i < nr_vcpus; i++) + pthread_create(&vcpu_threads[i], NULL, vcpu_thread_main, vcpus[i]); + + for (i = 0; i < nr_vcpus; i++) { + struct kvm_vcpu *vcpu = vcpus[i]; + + while (!SYNC_FROM_GUEST_AND_READ(vm, guest_ready_for_irqs[vcpu->id])) + continue; + } + + for (i = 0; i < nr_irqs; i++) { + struct kvm_vcpu *vcpu = vcpus[i % nr_vcpus]; + struct timespec start; + + kvm_route_msi(vm, gsi, vcpu, vector); + + for (j = 0; j < nr_vcpus; j++) + TEST_ASSERT(!GUEST_RECEIVED_IRQ(vcpus[j]), + "IRQ flag for vCPU %d not clear prior to test", + vcpus[j]->id); + + eventfd_write(eventfd, 1); + + clock_gettime(CLOCK_MONOTONIC, &start); + while (!GUEST_RECEIVED_IRQ(vcpu) && + timespec_to_ns(timespec_elapsed(start)) <= timeout_ns) + cpu_relax(); + + TEST_ASSERT(GUEST_RECEIVED_IRQ(vcpu), + "vCPU %d timed out waiting for IRQ (vector 0x%x) from GSI %d\n", + vcpu->id, vector, gsi); + + WRITE_AND_SYNC_TO_GUEST(vm, guest_received_irq[vcpu->id], false); + } + + WRITE_AND_SYNC_TO_GUEST(vm, done, true); + + for (i = 0; i < nr_vcpus; i++) + pthread_join(vcpu_threads[i], NULL); + + return 0; +} From 362cc00162499413d720fd1c9dfcf1fbba762e9b Mon Sep 17 00:00:00 2001 From: David Matlack Date: Fri, 26 Jun 2026 14:35:21 -0700 Subject: [PATCH 008/121] KVM: selftests: Add helper to get host IRQ from device MSI-X for IRQ bypass test Introduce proc_util.c and proc_util.h to house utility functions for interacting with the proc filesystem. Add vfio_msix_to_host_irq(), which parses /proc/interrupts, to get the host Linux IRQ for a given VFIO device BDF and MSI-X vector. This helper will be used by the eventfd IRQ test to print the host IRQ number when triggering IRQs via VFIO device, e.g. to aid in debugging if the test fails. Suggested-by: Sean Christopherson Signed-off-by: David Matlack Co-developed-by: Josh Hilke Signed-off-by: Josh Hilke [sean: massage changelog] Link: https://patch.msgid.link/20260626213534.3866178-9-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/Makefile.kvm | 1 + .../testing/selftests/kvm/include/proc_util.h | 9 +++++ tools/testing/selftests/kvm/lib/proc_util.c | 40 +++++++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 tools/testing/selftests/kvm/include/proc_util.h create mode 100644 tools/testing/selftests/kvm/lib/proc_util.c diff --git a/tools/testing/selftests/kvm/Makefile.kvm b/tools/testing/selftests/kvm/Makefile.kvm index 31d92eb493b0..1c87956859dd 100644 --- a/tools/testing/selftests/kvm/Makefile.kvm +++ b/tools/testing/selftests/kvm/Makefile.kvm @@ -11,6 +11,7 @@ LIBKVM += lib/kvm_util.c LIBKVM += lib/lru_gen_util.c LIBKVM += lib/memstress.c LIBKVM += lib/guest_sprintf.c +LIBKVM += lib/proc_util.c LIBKVM += lib/rbtree.c LIBKVM += lib/sparsebit.c LIBKVM += lib/test_util.c diff --git a/tools/testing/selftests/kvm/include/proc_util.h b/tools/testing/selftests/kvm/include/proc_util.h new file mode 100644 index 000000000000..704839b6d7af --- /dev/null +++ b/tools/testing/selftests/kvm/include/proc_util.h @@ -0,0 +1,9 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef SELFTEST_KVM_PROC_UTIL_H +#define SELFTEST_KVM_PROC_UTIL_H + +#include + +unsigned int vfio_msix_to_host_irq(const char *vfio_device_bdf, int msix); + +#endif /* SELFTEST_KVM_PROC_UTIL_H */ diff --git a/tools/testing/selftests/kvm/lib/proc_util.c b/tools/testing/selftests/kvm/lib/proc_util.c new file mode 100644 index 000000000000..84d30f055a0a --- /dev/null +++ b/tools/testing/selftests/kvm/lib/proc_util.c @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "kvm_util.h" +#include "test_util.h" +#include "proc_util.h" + +static FILE *open_proc_interrupts(void) +{ + FILE *fp; + + fp = fopen("/proc/interrupts", "r"); + TEST_ASSERT(fp, "fopen(/proc/interrupts) failed"); + + return fp; +} + +unsigned int vfio_msix_to_host_irq(const char *device_bdf, int msix) +{ + char search_string[64]; + char line[4096]; + int irq = -1; + FILE *fp; + + fp = open_proc_interrupts(); + + snprintf(search_string, sizeof(search_string), "vfio-msix[%d]", msix); + + while (fgets(line, sizeof(line), fp)) { + if (strstr(line, device_bdf) && strstr(line, search_string)) { + TEST_ASSERT_EQ(1, sscanf(line, "%d:", &irq)); + break; + } + } + + fclose(fp); + + TEST_ASSERT(irq != -1, "Failed to locate IRQ for %s %s", device_bdf, + search_string); + return (unsigned int)irq; +} + From 3902e8b02dce7878b65f10ada5d62def3e2d3a65 Mon Sep 17 00:00:00 2001 From: David Matlack Date: Fri, 26 Jun 2026 14:35:22 -0700 Subject: [PATCH 009/121] KVM: selftests: Add VFIO device support to eventfd IRQ test Extend the eventfd IRQ test with a '-d' argument that takes a BDF (in the format segment:bus:device.function) of an interrupt-capable PCI(e) device bound to VFIO, and use said device to trigger interrupts instead of always synthesizing interrupts via direct writes to the eventfd. Using a VFIO device to trigger interrupts validates the end-to-end delivery of IRQs for "real" devices, and when supported by hardware (and KVM), also validates interrupt delivery via IRQ bypass, i.e. via device posted IRQs. Now that IOMMUFD is a thing, auto-probe IOMMUFD vs. "legacy" VFIO by temporarily opening /dev/iommufd, and skip the test if neither IOMMUFD nor legacy VFIO is available. Add a '-t' option to the user override the probe logic, e.g. in case IOMMUFD is available but the system is configured for legacy usage. Note, the device must have a VFIO selftest driver in order to work with the test. A helper script to list supported devices will hopefully be available in the near future at tools/testing/selftests/vfio/scripts/list_supported_devices.sh[1]. Example: $ ./tools/testing/selftests/kvm/irq_test -d 0000:06:0a.1 Link: https://lore.kernel.org/all/20260602222941.3133236-1-jrhilke%40google.com [1] Signed-off-by: David Matlack Co-developed-by: Josh Hilke Signed-off-by: Josh Hilke Co-developed-by: Sean Christopherson Link: https://patch.msgid.link/20260626213534.3866178-10-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/irq_test.c | 98 ++++++++++++++++++++++++-- 1 file changed, 92 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/kvm/irq_test.c b/tools/testing/selftests/kvm/irq_test.c index 9f8895b89821..70b2c9cac279 100644 --- a/tools/testing/selftests/kvm/irq_test.c +++ b/tools/testing/selftests/kvm/irq_test.c @@ -3,7 +3,10 @@ #include "test_util.h" #include "apic.h" #include "processor.h" +#include "proc_util.h" +#include +#include #include #include #include @@ -55,6 +58,48 @@ static void *vcpu_thread_main(void *arg) return NULL; } +static int vfio_setup_msi(struct vfio_pci_device *device) +{ + const int flags = MAP_SHARED | MAP_ANONYMOUS; + const int prot = PROT_READ | PROT_WRITE; + struct iova_allocator *allocator; + struct dma_region *region; + + /* Sanity check that the device+driver can actually send MSIs. */ + TEST_REQUIRE(device->driver.ops); + TEST_REQUIRE(device->driver.ops->send_msi); + + /* + * Set up a DMA-able region for the driver to use. Very few devices + * provide a way to arbitrarily send interrupts (MSIs), e.g. by writing + * an MMIO register. Instead, most devices send MSIs when an action is + * completed, and practically all actions involve DMA of some form. + */ + allocator = iova_allocator_init(device->iommu); + + region = &device->driver.region; + region->size = SZ_2M; + region->iova = iova_allocator_alloc(allocator, region->size); + region->vaddr = kvm_mmap(region->size, prot, flags, -1); + TEST_ASSERT(region->vaddr != MAP_FAILED, "mmap() failed\n"); + iommu_map(device->iommu, region); + + iova_allocator_cleanup(allocator); + + vfio_pci_driver_init(device); + + return device->driver.msi; +} + +static void trigger_interrupt(struct vfio_pci_device *device, int eventfd) +{ + if (device) + vfio_pci_driver_send_msi(device); + else + eventfd_write(eventfd, 1); +} + + static void kvm_route_msi(struct kvm_vm *vm, u32 gsi, struct kvm_vcpu *vcpu, u8 vector) { @@ -74,11 +119,29 @@ static void kvm_route_msi(struct kvm_vm *vm, u32 gsi, struct kvm_vcpu *vcpu, vm_ioctl(vm, KVM_SET_GSI_ROUTING, &routing.header); } +static const char *probe_iommu_type(void) +{ + int io_fd; + + io_fd = open("/dev/iommu", O_RDONLY); + if (io_fd >= 0) { + close(io_fd); + return MODE_IOMMUFD; + } + + io_fd = __open_path_or_exit("/dev/vfio/vfio", O_RDONLY, + "Is VFIO (or IOMMUFD) loaded and enabled?"); + close(io_fd); + return MODE_VFIO_TYPE1_IOMMU; +} + static void help(const char *name) { - printf("Usage: %s [-h]\n", name); + printf("Usage: %s [-d ] [-h] [-t iommu_type]\n", name); printf("\n"); printf("Tests KVM interrupt routing and delivery via irqfd.\n"); + printf("-d Use a VFIO device to send MSI-X interrupts instead of manually signaling the eventfd\n"); + printf("-t Override the IOMMU type to use (vfio_type1_iommu or iommufd)\n"); printf("\n"); exit(KSFT_FAIL); } @@ -100,14 +163,25 @@ int main(int argc, char **argv) u32 gsi = kvm_random_u64_in_range(&kvm_rng, 24, KVM_MAX_IRQ_ROUTES - 1); u8 vector = kvm_random_u64_in_range(&kvm_rng, 32, UINT8_MAX); - struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; pthread_t vcpu_threads[KVM_MAX_VCPUS]; + struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; + struct vfio_pci_device *device = NULL; int nr_irqs = 1000, nr_vcpus = 1; - int i, j, c, eventfd; + const char *device_bdf = NULL; + const char *iommu_type = NULL; + int i, j, c, msix, eventfd; + struct iommu *iommu; struct kvm_vm *vm; + int irq; - while ((c = getopt(argc, argv, "h")) != -1) { + while ((c = getopt(argc, argv, "d:ht:")) != -1) { switch (c) { + case 'd': + device_bdf = optarg; + break; + case 't': + iommu_type = optarg; + break; case 'h': default: help(argv[0]); @@ -119,7 +193,19 @@ int main(int argc, char **argv) vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus); vm_install_exception_handler(vm, vector, guest_irq_handler); - eventfd = kvm_new_eventfd(); + if (device_bdf) { + if (!iommu_type) + iommu_type = probe_iommu_type(); + iommu = iommu_init(iommu_type); + device = vfio_pci_device_init(device_bdf, iommu); + msix = vfio_setup_msi(device); + irq = vfio_msix_to_host_irq(device_bdf, msix); + eventfd = device->msi_eventfds[msix]; + printf("Using device %s MSI-X[%d] (IRQ-%u)\n", device_bdf, msix, + irq); + } else { + eventfd = kvm_new_eventfd(); + } pr_info("Injecting interrupts for GSI %d (guest vector 0x%x) %d times\n", gsi, vector, nr_irqs); @@ -147,7 +233,7 @@ int main(int argc, char **argv) "IRQ flag for vCPU %d not clear prior to test", vcpus[j]->id); - eventfd_write(eventfd, 1); + trigger_interrupt(device, eventfd); clock_gettime(CLOCK_MONOTONIC, &start); while (!GUEST_RECEIVED_IRQ(vcpu) && From 380b0e5e0e30e0ae57056cd18e5a9a60892b2ece Mon Sep 17 00:00:00 2001 From: Josh Hilke Date: Fri, 26 Jun 2026 14:35:23 -0700 Subject: [PATCH 010/121] KVM: selftests: Add a helper to set proc IRQ affinity for IRQ test Add a utility, proc_irq_set_smp_affinity(), to set the CPU affinity of a Linux host IRQ via the proc filesystem. Use smp_affinity_list instead of smp_affinity to avoid having to convert the single CPU to a bitmask. The helper will be used by the eventfd IRQ test to verify delivery of IRQs when the affinity is randomized/modified. Signed-off-by: Josh Hilke [sean: make the utility self-contained, drop "list", massage changelog] Link: https://patch.msgid.link/20260626213534.3866178-11-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/include/proc_util.h | 2 ++ tools/testing/selftests/kvm/lib/proc_util.c | 14 ++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/tools/testing/selftests/kvm/include/proc_util.h b/tools/testing/selftests/kvm/include/proc_util.h index 704839b6d7af..d1ddc967d11d 100644 --- a/tools/testing/selftests/kvm/include/proc_util.h +++ b/tools/testing/selftests/kvm/include/proc_util.h @@ -6,4 +6,6 @@ unsigned int vfio_msix_to_host_irq(const char *vfio_device_bdf, int msix); +void proc_irq_set_smp_affinity(unsigned int irq, int cpu); + #endif /* SELFTEST_KVM_PROC_UTIL_H */ diff --git a/tools/testing/selftests/kvm/lib/proc_util.c b/tools/testing/selftests/kvm/lib/proc_util.c index 84d30f055a0a..3960b3841d63 100644 --- a/tools/testing/selftests/kvm/lib/proc_util.c +++ b/tools/testing/selftests/kvm/lib/proc_util.c @@ -38,3 +38,17 @@ unsigned int vfio_msix_to_host_irq(const char *device_bdf, int msix) return (unsigned int)irq; } +void proc_irq_set_smp_affinity(unsigned int irq, int cpu) +{ + char path[PATH_MAX]; + int r, fd; + + snprintf(path, sizeof(path), "/proc/irq/%u/smp_affinity_list", irq); + fd = open(path, O_RDWR); + TEST_ASSERT(fd >= 0, "Failed to open %s", path); + + r = dprintf(fd, "%d\n", cpu); + TEST_ASSERT(r > 0, "Failed to affinitize IRQ-%u to CPU %d", irq, cpu); + + kvm_close(fd); +} From 197b0cd9ac66bf28f575f2723499f6516bb56f3e Mon Sep 17 00:00:00 2001 From: David Matlack Date: Fri, 26 Jun 2026 14:35:24 -0700 Subject: [PATCH 011/121] KVM: selftests: Verify interrupts are received when IRQ affinity changes in IRQ test Extent the eventfd IRQ test with a '-a' flag to randomly affinitize the device's host IRQ to different physical CPUs throughout the test. This stresses the kernel's ability to maintain correct interrupt routing and delivery even as the underlying hardware IRQ affinity is changed dynamically via /proc//smp_affinity{,_list}. Signed-off-by: David Matlack Co-developed-by: Josh Hilke Signed-off-by: Josh Hilke [sean: massage changelog] Link: https://patch.msgid.link/20260626213534.3866178-12-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/irq_test.c | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/kvm/irq_test.c b/tools/testing/selftests/kvm/irq_test.c index 70b2c9cac279..fd386e3e9ac3 100644 --- a/tools/testing/selftests/kvm/irq_test.c +++ b/tools/testing/selftests/kvm/irq_test.c @@ -12,10 +12,12 @@ #include #include #include +#include static u64 timeout_ns = 2ULL * 1000 * 1000 * 1000; static bool guest_ready_for_irqs[KVM_MAX_VCPUS]; static bool guest_received_irq[KVM_MAX_VCPUS]; +static bool irq_affinity; static bool done; #define GUEST_RECEIVED_IRQ(__vcpu) \ @@ -137,9 +139,10 @@ static const char *probe_iommu_type(void) static void help(const char *name) { - printf("Usage: %s [-d ] [-h] [-t iommu_type]\n", name); + printf("Usage: %s [-a] [-d ] [-h] [-t iommu_type]\n", name); printf("\n"); printf("Tests KVM interrupt routing and delivery via irqfd.\n"); + printf("-a Affine the device's host IRQ to a random physical CPU\n"); printf("-d Use a VFIO device to send MSI-X interrupts instead of manually signaling the eventfd\n"); printf("-t Override the IOMMU type to use (vfio_type1_iommu or iommufd)\n"); printf("\n"); @@ -172,10 +175,13 @@ int main(int argc, char **argv) int i, j, c, msix, eventfd; struct iommu *iommu; struct kvm_vm *vm; - int irq; + int irq, irq_cpu; - while ((c = getopt(argc, argv, "d:ht:")) != -1) { + while ((c = getopt(argc, argv, "ad:ht:")) != -1) { switch (c) { + case 'a': + irq_affinity = true; + break; case 'd': device_bdf = optarg; break; @@ -204,7 +210,12 @@ int main(int argc, char **argv) printf("Using device %s MSI-X[%d] (IRQ-%u)\n", device_bdf, msix, irq); } else { + TEST_ASSERT(!irq_affinity, + "Setting IRQ affinity (-a) requires a backing device (-d)"); + eventfd = kvm_new_eventfd(); + irq = -1; + irq_cpu = -1; } pr_info("Injecting interrupts for GSI %d (guest vector 0x%x) %d times\n", @@ -228,6 +239,11 @@ int main(int argc, char **argv) kvm_route_msi(vm, gsi, vcpu, vector); + if (irq_affinity) { + irq_cpu = kvm_random_u64(&kvm_rng) % get_nprocs(); + proc_irq_set_smp_affinity(irq, irq_cpu); + } + for (j = 0; j < nr_vcpus; j++) TEST_ASSERT(!GUEST_RECEIVED_IRQ(vcpus[j]), "IRQ flag for vCPU %d not clear prior to test", @@ -241,8 +257,8 @@ int main(int argc, char **argv) cpu_relax(); TEST_ASSERT(GUEST_RECEIVED_IRQ(vcpu), - "vCPU %d timed out waiting for IRQ (vector 0x%x) from GSI %d\n", - vcpu->id, vector, gsi); + "vCPU %d timed out waiting for IRQ (vector 0x%x) from GSI %d (via CPU %d)\n", + vcpu->id, vector, gsi, irq_cpu); WRITE_AND_SYNC_TO_GUEST(vm, guest_received_irq[vcpu->id], false); } From ec333e3ace258dde8eddbdded2a8be81b3501cf2 Mon Sep 17 00:00:00 2001 From: David Matlack Date: Fri, 26 Jun 2026 14:35:25 -0700 Subject: [PATCH 012/121] KVM: selftests: Add option to set empty routing between IRQs in eventfd IRQ test Extend the eventfd IRQ test with an '-e' flag to set empty GSI routing between interrupts. Clobbering the GSI routing table verifies that KVM correctly handles CPUx => NULL => CPUy transitions, not just CPUx => CPUy transitions, and verifies that KVM can "rebuild" an entire routing setup. Signed-off-by: David Matlack Co-developed-by: Josh Hilke Signed-off-by: Josh Hilke [sean: '-e' for "empty" instead of '-c' for "clear", massage changelog] Link: https://patch.msgid.link/20260626213534.3866178-13-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/irq_test.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/irq_test.c b/tools/testing/selftests/kvm/irq_test.c index fd386e3e9ac3..fd03ec940362 100644 --- a/tools/testing/selftests/kvm/irq_test.c +++ b/tools/testing/selftests/kvm/irq_test.c @@ -121,6 +121,13 @@ static void kvm_route_msi(struct kvm_vm *vm, u32 gsi, struct kvm_vcpu *vcpu, vm_ioctl(vm, KVM_SET_GSI_ROUTING, &routing.header); } +static void kvm_set_empty_gsi_routing(struct kvm_vm *vm) +{ + struct kvm_irq_routing routing = {}; + + vm_ioctl(vm, KVM_SET_GSI_ROUTING, &routing); +} + static const char *probe_iommu_type(void) { int io_fd; @@ -139,11 +146,12 @@ static const char *probe_iommu_type(void) static void help(const char *name) { - printf("Usage: %s [-a] [-d ] [-h] [-t iommu_type]\n", name); + printf("Usage: %s [-a] [-d ] [-e] [-h] [-t iommu_type]\n", name); printf("\n"); printf("Tests KVM interrupt routing and delivery via irqfd.\n"); printf("-a Affine the device's host IRQ to a random physical CPU\n"); printf("-d Use a VFIO device to send MSI-X interrupts instead of manually signaling the eventfd\n"); + printf("-e Set empty GSI routing in-between some interrupts\n"); printf("-t Override the IOMMU type to use (vfio_type1_iommu or iommufd)\n"); printf("\n"); exit(KSFT_FAIL); @@ -170,6 +178,7 @@ int main(int argc, char **argv) struct kvm_vcpu *vcpus[KVM_MAX_VCPUS]; struct vfio_pci_device *device = NULL; int nr_irqs = 1000, nr_vcpus = 1; + bool set_empty_routing = false; const char *device_bdf = NULL; const char *iommu_type = NULL; int i, j, c, msix, eventfd; @@ -177,7 +186,7 @@ int main(int argc, char **argv) struct kvm_vm *vm; int irq, irq_cpu; - while ((c = getopt(argc, argv, "ad:ht:")) != -1) { + while ((c = getopt(argc, argv, "ad:eht:")) != -1) { switch (c) { case 'a': irq_affinity = true; @@ -185,6 +194,9 @@ int main(int argc, char **argv) case 'd': device_bdf = optarg; break; + case 'e': + set_empty_routing = true; + break; case 't': iommu_type = optarg; break; @@ -234,9 +246,13 @@ int main(int argc, char **argv) } for (i = 0; i < nr_irqs; i++) { + const bool do_set_empty_routing = set_empty_routing && (i & BIT(3)); struct kvm_vcpu *vcpu = vcpus[i % nr_vcpus]; struct timespec start; + if (do_set_empty_routing) + kvm_set_empty_gsi_routing(vm); + kvm_route_msi(vm, gsi, vcpu, vector); if (irq_affinity) { From 38950af64a1ca9dc6a0279b6d0881e54ed3a5921 Mon Sep 17 00:00:00 2001 From: David Matlack Date: Fri, 26 Jun 2026 14:35:26 -0700 Subject: [PATCH 013/121] KVM: selftests: Make number of IRQs configurable in IRQ test Extend the eventfd IRQ test with a '-i' flag to let the user specify the the number of IRQs to generate (instead of hardcoding the test to always generate 1000 interrupts). Signed-off-by: David Matlack Co-developed-by: Josh Hilke Signed-off-by: Josh Hilke [sean: massage changelog] Link: https://patch.msgid.link/20260626213534.3866178-14-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/irq_test.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/testing/selftests/kvm/irq_test.c b/tools/testing/selftests/kvm/irq_test.c index fd03ec940362..bb20bb81e930 100644 --- a/tools/testing/selftests/kvm/irq_test.c +++ b/tools/testing/selftests/kvm/irq_test.c @@ -146,12 +146,13 @@ static const char *probe_iommu_type(void) static void help(const char *name) { - printf("Usage: %s [-a] [-d ] [-e] [-h] [-t iommu_type]\n", name); + printf("Usage: %s [-a] [-d ] [-e] [-h] [-i nr_irqs] [-t iommu_type]\n", name); printf("\n"); printf("Tests KVM interrupt routing and delivery via irqfd.\n"); printf("-a Affine the device's host IRQ to a random physical CPU\n"); printf("-d Use a VFIO device to send MSI-X interrupts instead of manually signaling the eventfd\n"); printf("-e Set empty GSI routing in-between some interrupts\n"); + printf("-i The number of IRQs to generate during the test\n"); printf("-t Override the IOMMU type to use (vfio_type1_iommu or iommufd)\n"); printf("\n"); exit(KSFT_FAIL); @@ -186,7 +187,7 @@ int main(int argc, char **argv) struct kvm_vm *vm; int irq, irq_cpu; - while ((c = getopt(argc, argv, "ad:eht:")) != -1) { + while ((c = getopt(argc, argv, "ad:ehi:t:")) != -1) { switch (c) { case 'a': irq_affinity = true; @@ -197,6 +198,9 @@ int main(int argc, char **argv) case 'e': set_empty_routing = true; break; + case 'i': + nr_irqs = atoi_positive("Number of IRQs", optarg); + break; case 't': iommu_type = optarg; break; From 25e268b76e771525fa2fca3205ad0a708d55792a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 26 Jun 2026 14:35:27 -0700 Subject: [PATCH 014/121] KVM: selftests: Verify non-postable IRQ remapping in IRQ test Extend the eventfd IRQ test with an '-n' flag to route a subset of device interrupts as NMIs (Non-Maskable Interrupts) into the guest using an alternating pattern of 4 NMIs followed by 4 regular interrupts. While this adds coverage for NMI injection, the primary goal is to validate KVM's handling of non-postable interrupt delivery (AMD and Intel IOMMUs only support posting fixed IRQs targeting a single vCPU). KVM has historically bungled handling transitions between posted and remapped modes. Use NMIs to stress the transitions, because they are a reliable, architectural way to force these code paths. Signed-off-by: David Matlack Co-developed-by: Josh Hilke Signed-off-by: Josh Hilke [sean: add GUEST_RECEIVED_INTERRUPT(), massage changelog] Link: https://patch.msgid.link/20260626213534.3866178-15-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/irq_test.c | 48 ++++++++++++++++++++------ 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/tools/testing/selftests/kvm/irq_test.c b/tools/testing/selftests/kvm/irq_test.c index bb20bb81e930..48520c6dea69 100644 --- a/tools/testing/selftests/kvm/irq_test.c +++ b/tools/testing/selftests/kvm/irq_test.c @@ -17,11 +17,17 @@ static u64 timeout_ns = 2ULL * 1000 * 1000 * 1000; static bool guest_ready_for_irqs[KVM_MAX_VCPUS]; static bool guest_received_irq[KVM_MAX_VCPUS]; +static bool guest_received_nmi[KVM_MAX_VCPUS]; static bool irq_affinity; static bool done; #define GUEST_RECEIVED_IRQ(__vcpu) \ SYNC_FROM_GUEST_AND_READ((__vcpu)->vm, guest_received_irq[(__vcpu)->id]) +#define GUEST_RECEIVED_NMI(__vcpu) \ + SYNC_FROM_GUEST_AND_READ((__vcpu)->vm, guest_received_nmi[(__vcpu)->id]) + +#define GUEST_RECEIVED_INTERRUPT(__vcpu, __nmi) \ + ((__nmi) ? GUEST_RECEIVED_NMI(__vcpu) : GUEST_RECEIVED_IRQ(__vcpu)) static u32 guest_get_vcpu_id(void) { @@ -35,6 +41,11 @@ static void guest_irq_handler(struct ex_regs *regs) x2apic_write_reg(APIC_EOI, 0); } +static void guest_nmi_handler(struct ex_regs *regs) +{ + WRITE_ONCE(guest_received_nmi[guest_get_vcpu_id()], true); +} + static void guest_code(void) { x2apic_enable(); @@ -103,7 +114,7 @@ static void trigger_interrupt(struct vfio_pci_device *device, int eventfd) static void kvm_route_msi(struct kvm_vm *vm, u32 gsi, struct kvm_vcpu *vcpu, - u8 vector) + u8 vector, bool use_nmi) { struct { struct kvm_irq_routing header; @@ -114,7 +125,7 @@ static void kvm_route_msi(struct kvm_vm *vm, u32 gsi, struct kvm_vcpu *vcpu, .gsi = gsi, .type = KVM_IRQ_ROUTING_MSI, .u.msi.address_lo = 0xFEE00000 | (vcpu->id << 12), - .u.msi.data = vector, + .u.msi.data = use_nmi ? NMI_VECTOR | (4 << 8) : vector, }, }; @@ -146,13 +157,14 @@ static const char *probe_iommu_type(void) static void help(const char *name) { - printf("Usage: %s [-a] [-d ] [-e] [-h] [-i nr_irqs] [-t iommu_type]\n", name); + printf("Usage: %s [-a] [-d ] [-e] [-h] [-i nr_irqs] [-n] [-t iommu_type]\n", name); printf("\n"); printf("Tests KVM interrupt routing and delivery via irqfd.\n"); printf("-a Affine the device's host IRQ to a random physical CPU\n"); printf("-d Use a VFIO device to send MSI-X interrupts instead of manually signaling the eventfd\n"); printf("-e Set empty GSI routing in-between some interrupts\n"); printf("-i The number of IRQs to generate during the test\n"); + printf("-n Deliver 50 percent of IRQs as non-maskable interrupts\n"); printf("-t Override the IOMMU type to use (vfio_type1_iommu or iommufd)\n"); printf("\n"); exit(KSFT_FAIL); @@ -183,11 +195,12 @@ int main(int argc, char **argv) const char *device_bdf = NULL; const char *iommu_type = NULL; int i, j, c, msix, eventfd; + bool use_nmi = false; struct iommu *iommu; struct kvm_vm *vm; int irq, irq_cpu; - while ((c = getopt(argc, argv, "ad:ehi:t:")) != -1) { + while ((c = getopt(argc, argv, "ad:ehi:nt:")) != -1) { switch (c) { case 'a': irq_affinity = true; @@ -201,6 +214,9 @@ int main(int argc, char **argv) case 'i': nr_irqs = atoi_positive("Number of IRQs", optarg); break; + case 'n': + use_nmi = true; + break; case 't': iommu_type = optarg; break; @@ -214,6 +230,7 @@ int main(int argc, char **argv) vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus); vm_install_exception_handler(vm, vector, guest_irq_handler); + vm_install_exception_handler(vm, NMI_VECTOR, guest_nmi_handler); if (device_bdf) { if (!iommu_type) @@ -251,36 +268,45 @@ int main(int argc, char **argv) for (i = 0; i < nr_irqs; i++) { const bool do_set_empty_routing = set_empty_routing && (i & BIT(3)); + const bool do_use_nmi = use_nmi && (i & BIT(2)); struct kvm_vcpu *vcpu = vcpus[i % nr_vcpus]; struct timespec start; if (do_set_empty_routing) kvm_set_empty_gsi_routing(vm); - kvm_route_msi(vm, gsi, vcpu, vector); + kvm_route_msi(vm, gsi, vcpu, vector, do_use_nmi); if (irq_affinity) { irq_cpu = kvm_random_u64(&kvm_rng) % get_nprocs(); proc_irq_set_smp_affinity(irq, irq_cpu); } - for (j = 0; j < nr_vcpus; j++) + for (j = 0; j < nr_vcpus; j++) { TEST_ASSERT(!GUEST_RECEIVED_IRQ(vcpus[j]), "IRQ flag for vCPU %d not clear prior to test", vcpus[j]->id); + TEST_ASSERT(!GUEST_RECEIVED_NMI(vcpus[j]), + "NMI flag for vCPU %d not clear prior to test", + vcpus[j]->id); + } trigger_interrupt(device, eventfd); clock_gettime(CLOCK_MONOTONIC, &start); - while (!GUEST_RECEIVED_IRQ(vcpu) && + while (!GUEST_RECEIVED_INTERRUPT(vcpu, do_use_nmi) && timespec_to_ns(timespec_elapsed(start)) <= timeout_ns) cpu_relax(); - TEST_ASSERT(GUEST_RECEIVED_IRQ(vcpu), - "vCPU %d timed out waiting for IRQ (vector 0x%x) from GSI %d (via CPU %d)\n", - vcpu->id, vector, gsi, irq_cpu); + TEST_ASSERT(GUEST_RECEIVED_INTERRUPT(vcpu, do_use_nmi), + "vCPU %d timed out waiting for %s (vector 0x%x) from GSI %d (via CPU %d)\n", + vcpu->id, do_use_nmi ? "NMI" : "IRQ", + do_use_nmi ? NMI_VECTOR : vector, gsi, irq_cpu); - WRITE_AND_SYNC_TO_GUEST(vm, guest_received_irq[vcpu->id], false); + if (do_use_nmi) + WRITE_AND_SYNC_TO_GUEST(vm, guest_received_nmi[vcpu->id], false); + else + WRITE_AND_SYNC_TO_GUEST(vm, guest_received_irq[vcpu->id], false); } WRITE_AND_SYNC_TO_GUEST(vm, done, true); From 4512b2776631c48cad057c2d52d76473dddf315c Mon Sep 17 00:00:00 2001 From: Josh Hilke Date: Fri, 26 Jun 2026 14:35:28 -0700 Subject: [PATCH 015/121] KVM: selftests: Add kvm_gettid() wrapper and convert users Add a KVM wrapper for the gettid() syscall so that tests don't have to open code the syscall() themselves. Unfortunately, not all flavors of libc that KVM selftests support provide gettid(). Convert all existing users of the syscall to the new wrapper. Note, per the gettid() manpage[1], "This call is always successful", i.e. prefixing kvm_ to the syscall name is aligned with the goal of providing syscall wrappers that guarantee success. No functional changes intended. Link: https://man7.org/linux/man-pages/man2/gettid.2.html [1] Suggested-by: Sean Christopherson Signed-off-by: Josh Hilke [sean: massage changelog] Link: https://patch.msgid.link/20260626213534.3866178-16-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/demand_paging_test.c | 2 +- tools/testing/selftests/kvm/include/kvm_syscalls.h | 5 +++++ tools/testing/selftests/kvm/lib/assert.c | 8 ++------ tools/testing/selftests/kvm/lib/test_util.c | 3 ++- tools/testing/selftests/kvm/rseq_test.c | 2 +- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/tools/testing/selftests/kvm/demand_paging_test.c b/tools/testing/selftests/kvm/demand_paging_test.c index 302c4923d093..f8b3d0b68830 100644 --- a/tools/testing/selftests/kvm/demand_paging_test.c +++ b/tools/testing/selftests/kvm/demand_paging_test.c @@ -57,7 +57,7 @@ static void vcpu_worker(struct memstress_vcpu_args *vcpu_args) static int handle_uffd_page_request(int uffd_mode, int uffd, struct uffd_msg *msg) { - pid_t tid = syscall(__NR_gettid); + pid_t tid = kvm_gettid(); u64 addr = msg->arg.pagefault.address; struct timespec start; struct timespec ts_diff; diff --git a/tools/testing/selftests/kvm/include/kvm_syscalls.h b/tools/testing/selftests/kvm/include/kvm_syscalls.h index 6cb3bed29b81..dc4fb97aef8d 100644 --- a/tools/testing/selftests/kvm/include/kvm_syscalls.h +++ b/tools/testing/selftests/kvm/include/kvm_syscalls.h @@ -83,6 +83,11 @@ static inline int kvm_dup(int fd) return new_fd; } +static inline pid_t kvm_gettid(void) +{ + return syscall(__NR_gettid); +} + __KVM_SYSCALL_DEFINE(munmap, 2, void *, mem, size_t, size); __KVM_SYSCALL_DEFINE(close, 1, int, fd); __KVM_SYSCALL_DEFINE(fallocate, 4, int, fd, int, mode, loff_t, offset, loff_t, len); diff --git a/tools/testing/selftests/kvm/lib/assert.c b/tools/testing/selftests/kvm/lib/assert.c index 8be0d09ecf0f..1d72dcdfce3b 100644 --- a/tools/testing/selftests/kvm/lib/assert.c +++ b/tools/testing/selftests/kvm/lib/assert.c @@ -10,6 +10,7 @@ #include #include "kselftest.h" +#include "kvm_syscalls.h" #ifdef __GLIBC__ #include @@ -64,11 +65,6 @@ static void test_dump_stack(void) static void test_dump_stack(void) {} #endif -static pid_t _gettid(void) -{ - return syscall(SYS_gettid); -} - void __attribute__((noinline)) test_assert(bool exp, const char *exp_str, const char *file, unsigned int line, const char *fmt, ...) @@ -81,7 +77,7 @@ test_assert(bool exp, const char *exp_str, fprintf(stderr, "==== Test Assertion Failure ====\n" " %s:%u: %s\n" " pid=%d tid=%d errno=%d - %s\n", - file, line, exp_str, getpid(), _gettid(), + file, line, exp_str, getpid(), kvm_gettid(), errno, strerror(errno)); test_dump_stack(); if (fmt) { diff --git a/tools/testing/selftests/kvm/lib/test_util.c b/tools/testing/selftests/kvm/lib/test_util.c index e208a57f190c..6b00ab11f3c0 100644 --- a/tools/testing/selftests/kvm/lib/test_util.c +++ b/tools/testing/selftests/kvm/lib/test_util.c @@ -17,6 +17,7 @@ #include "linux/kernel.h" #include "test_util.h" +#include "kvm_syscalls.h" sigjmp_buf expect_sigbus_jmpbuf; @@ -395,7 +396,7 @@ long get_run_delay(void) long val[2]; FILE *fp; - sprintf(path, "/proc/%ld/schedstat", syscall(SYS_gettid)); + sprintf(path, "/proc/%ld/schedstat", (long)kvm_gettid()); fp = fopen(path, "r"); /* Return MIN_RUN_DELAY_NS upon failure just to be safe */ if (fscanf(fp, "%ld %ld ", &val[0], &val[1]) < 2) diff --git a/tools/testing/selftests/kvm/rseq_test.c b/tools/testing/selftests/kvm/rseq_test.c index f80ad6b47d16..6510fbfd64f1 100644 --- a/tools/testing/selftests/kvm/rseq_test.c +++ b/tools/testing/selftests/kvm/rseq_test.c @@ -244,7 +244,7 @@ int main(int argc, char *argv[]) vm = vm_create_with_one_vcpu(&vcpu, guest_code); pthread_create(&migration_thread, NULL, migration_worker, - (void *)(unsigned long)syscall(SYS_gettid)); + (void *)(unsigned long)kvm_gettid()); if (latency >= 0) { /* From 76703ca0cbab7b9b6a1adfed9b9071dad2c3e3a8 Mon Sep 17 00:00:00 2001 From: Josh Hilke Date: Fri, 26 Jun 2026 14:35:29 -0700 Subject: [PATCH 016/121] KVM: selftests: Add kvm_sched_getaffinity() wrapper and convert users Add and use a KVM wrapper for the sched_getaffinity() syscall so that selftests don't need to manually assert that the syscall succeeded. Note, some tests didn't actually assert success, but they all obviously rely on the syscall to succeed. Suggested-by: Sean Christopherson Signed-off-by: Josh Hilke [sean: massage changelog] Link: https://patch.msgid.link/20260626213534.3866178-17-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/arch_timer.c | 2 +- tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c | 2 +- tools/testing/selftests/kvm/include/kvm_syscalls.h | 2 ++ tools/testing/selftests/kvm/lib/kvm_util.c | 5 ++--- tools/testing/selftests/kvm/mmu_stress_test.c | 6 +----- tools/testing/selftests/kvm/rseq_test.c | 4 +--- 6 files changed, 8 insertions(+), 13 deletions(-) diff --git a/tools/testing/selftests/kvm/arch_timer.c b/tools/testing/selftests/kvm/arch_timer.c index 90c475a61b22..f8b02597897b 100644 --- a/tools/testing/selftests/kvm/arch_timer.c +++ b/tools/testing/selftests/kvm/arch_timer.c @@ -85,7 +85,7 @@ static u32 test_get_pcpu(void) cpu_set_t online_cpuset; nproc_conf = get_nprocs_conf(); - sched_getaffinity(0, sizeof(cpu_set_t), &online_cpuset); + kvm_sched_getaffinity(0, sizeof(cpu_set_t), &online_cpuset); /* Randomly find an available pCPU to place a vCPU on */ do { diff --git a/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c b/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c index f7625eb711d6..d9c9377a6325 100644 --- a/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c +++ b/tools/testing/selftests/kvm/arm64/arch_timer_edge_cases.c @@ -1039,7 +1039,7 @@ int main(int argc, char *argv[]) if (!parse_args(argc, argv)) exit(KSFT_SKIP); - sched_getaffinity(0, sizeof(default_cpuset), &default_cpuset); + kvm_sched_getaffinity(0, sizeof(default_cpuset), &default_cpuset); set_counter_defaults(); if (test_args.test_virtual) { diff --git a/tools/testing/selftests/kvm/include/kvm_syscalls.h b/tools/testing/selftests/kvm/include/kvm_syscalls.h index dc4fb97aef8d..5dae6143ddb0 100644 --- a/tools/testing/selftests/kvm/include/kvm_syscalls.h +++ b/tools/testing/selftests/kvm/include/kvm_syscalls.h @@ -12,6 +12,7 @@ #include #include +#include #include #define MAP_ARGS0(m,...) @@ -93,6 +94,7 @@ __KVM_SYSCALL_DEFINE(close, 1, int, fd); __KVM_SYSCALL_DEFINE(fallocate, 4, int, fd, int, mode, loff_t, offset, loff_t, len); __KVM_SYSCALL_DEFINE(ftruncate, 2, unsigned int, fd, off_t, length); __KVM_SYSCALL_DEFINE(madvise, 3, void *, addr, size_t, length, int, advice); +__KVM_SYSCALL_DEFINE(sched_getaffinity, 3, pid_t, pid, size_t, cpusetsize, cpu_set_t *, mask); #define kvm_free_fd(fd) \ do { \ diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 277166ab1aa9..6a0ee033fdef 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -674,13 +674,12 @@ void kvm_parse_vcpu_pinning(const char *pcpus_string, u32 vcpu_to_pcpu[], cpu_set_t allowed_mask; char *cpu, *cpu_list; char delim[2] = ","; - int i, r; + int i; cpu_list = strdup(pcpus_string); TEST_ASSERT(cpu_list, "strdup() allocation failed."); - r = sched_getaffinity(0, sizeof(allowed_mask), &allowed_mask); - TEST_ASSERT(!r, "sched_getaffinity() failed"); + kvm_sched_getaffinity(0, sizeof(allowed_mask), &allowed_mask); cpu = strtok(cpu_list, delim); diff --git a/tools/testing/selftests/kvm/mmu_stress_test.c b/tools/testing/selftests/kvm/mmu_stress_test.c index 473ef4c0ea9f..3d5f33a63b2b 100644 --- a/tools/testing/selftests/kvm/mmu_stress_test.c +++ b/tools/testing/selftests/kvm/mmu_stress_test.c @@ -255,11 +255,7 @@ static void rendezvous_with_vcpus(struct timespec *time, const char *name) static void calc_default_nr_vcpus(void) { cpu_set_t possible_mask; - int r; - - r = sched_getaffinity(0, sizeof(possible_mask), &possible_mask); - TEST_ASSERT(!r, "sched_getaffinity failed, errno = %d (%s)", - errno, strerror(errno)); + kvm_sched_getaffinity(0, sizeof(possible_mask), &possible_mask); nr_vcpus = CPU_COUNT(&possible_mask); TEST_ASSERT(nr_vcpus > 0, "Uh, no CPUs?"); diff --git a/tools/testing/selftests/kvm/rseq_test.c b/tools/testing/selftests/kvm/rseq_test.c index 6510fbfd64f1..557e393c223b 100644 --- a/tools/testing/selftests/kvm/rseq_test.c +++ b/tools/testing/selftests/kvm/rseq_test.c @@ -226,9 +226,7 @@ int main(int argc, char *argv[]) } } - r = sched_getaffinity(0, sizeof(possible_mask), &possible_mask); - TEST_ASSERT(!r, "sched_getaffinity failed, errno = %d (%s)", errno, - strerror(errno)); + kvm_sched_getaffinity(0, sizeof(possible_mask), &possible_mask); calc_min_max_cpu(); From 26fd1fb4737a6ac648350e5d624d461c3c83b1cb Mon Sep 17 00:00:00 2001 From: Josh Hilke Date: Fri, 26 Jun 2026 14:35:30 -0700 Subject: [PATCH 017/121] KVM: selftests: Add a utility to pin a task to a random CPU, given a CPU set Add a helper function, pin_task_to_random_cpu(), to pin a task to a random CPU from a given cpu_set_t. This helper will be used eventfd IRQ test to migrate vCPUs to random pCPUs, to stress host-side interrupt routing and delivery. Suggested-by: Sean Christopherson Signed-off-by: Josh Hilke [sean: massage changelog] Link: https://patch.msgid.link/20260626213534.3866178-18-seanjc@google.com Signed-off-by: Sean Christopherson --- .../testing/selftests/kvm/include/kvm_util.h | 2 ++ tools/testing/selftests/kvm/lib/kvm_util.c | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/tools/testing/selftests/kvm/include/kvm_util.h b/tools/testing/selftests/kvm/include/kvm_util.h index c1f588154398..b39e713c30a4 100644 --- a/tools/testing/selftests/kvm/include/kvm_util.h +++ b/tools/testing/selftests/kvm/include/kvm_util.h @@ -1094,6 +1094,8 @@ static inline void pin_task_to_cpu(pthread_t task, int cpu) TEST_ASSERT(!r, "Failed to set thread affinity to pCPU '%u'", cpu); } +void pin_task_to_random_cpu(pthread_t task, cpu_set_t *possible_cpus); + static inline int pin_task_to_any_cpu(pthread_t task) { int cpu = sched_getcpu(); diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index 6a0ee033fdef..3794575d2ca0 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -668,6 +668,27 @@ void kvm_print_vcpu_pinning_help(void) " (default: no pinning)\n", name, name); } +void pin_task_to_random_cpu(pthread_t task, cpu_set_t *possible_cpus) +{ + int target_idx; + int nr_cpus; + int cpu; + + nr_cpus = CPU_COUNT(possible_cpus); + TEST_ASSERT(nr_cpus > 0, "No CPUs available in possible_cpus"); + + target_idx = kvm_random_u64(&kvm_rng) % nr_cpus; + + for (cpu = 0; cpu < CPU_SETSIZE; cpu++) { + if (CPU_ISSET(cpu, possible_cpus) && target_idx-- == 0) { + pin_task_to_cpu(task, cpu); + return; + } + } + + TEST_FAIL("Failed to find random CPU in possible_cpus"); +} + void kvm_parse_vcpu_pinning(const char *pcpus_string, u32 vcpu_to_pcpu[], int nr_vcpus) { From 6b9470baea6c4c61e501cdf00a3b2b2113d554f8 Mon Sep 17 00:00:00 2001 From: David Matlack Date: Fri, 26 Jun 2026 14:35:31 -0700 Subject: [PATCH 018/121] KVM: selftests: Verify vCPU migration during IRQ delivery in IRQ test Extend the eventfd IRQ test with a '-m' flag to have the test migrate the target vCPU to a random physical CPU before triggering its interrupt, e.g. to validate KVM's ability to update device posted IRQ routing. Signed-off-by: David Matlack Co-developed-by: Josh Hilke Signed-off-by: Josh Hilke [sean: pin one vCPU at a time to simplify things, use main()'s affinity] Link: https://patch.msgid.link/20260626213534.3866178-19-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/irq_test.c | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/kvm/irq_test.c b/tools/testing/selftests/kvm/irq_test.c index 48520c6dea69..c0ff6e60b167 100644 --- a/tools/testing/selftests/kvm/irq_test.c +++ b/tools/testing/selftests/kvm/irq_test.c @@ -157,13 +157,14 @@ static const char *probe_iommu_type(void) static void help(const char *name) { - printf("Usage: %s [-a] [-d ] [-e] [-h] [-i nr_irqs] [-n] [-t iommu_type]\n", name); + printf("Usage: %s [-a] [-d ] [-e] [-h] [-i nr_irqs] [-m] [-n] [-t iommu_type]\n", name); printf("\n"); printf("Tests KVM interrupt routing and delivery via irqfd.\n"); printf("-a Affine the device's host IRQ to a random physical CPU\n"); printf("-d Use a VFIO device to send MSI-X interrupts instead of manually signaling the eventfd\n"); printf("-e Set empty GSI routing in-between some interrupts\n"); printf("-i The number of IRQs to generate during the test\n"); + printf("-m Pin target vCPU to random physical CPU before triggering interrupt\n"); printf("-n Deliver 50 percent of IRQs as non-maskable interrupts\n"); printf("-t Override the IOMMU type to use (vfio_type1_iommu or iommufd)\n"); printf("\n"); @@ -195,12 +196,14 @@ int main(int argc, char **argv) const char *device_bdf = NULL; const char *iommu_type = NULL; int i, j, c, msix, eventfd; + bool migrate_vcpus = false; + cpu_set_t available_cpus; bool use_nmi = false; struct iommu *iommu; struct kvm_vm *vm; int irq, irq_cpu; - while ((c = getopt(argc, argv, "ad:ehi:nt:")) != -1) { + while ((c = getopt(argc, argv, "ad:ehi:mnt:")) != -1) { switch (c) { case 'a': irq_affinity = true; @@ -214,6 +217,9 @@ int main(int argc, char **argv) case 'i': nr_irqs = atoi_positive("Number of IRQs", optarg); break; + case 'm': + migrate_vcpus = true; + break; case 'n': use_nmi = true; break; @@ -248,7 +254,6 @@ int main(int argc, char **argv) eventfd = kvm_new_eventfd(); irq = -1; - irq_cpu = -1; } pr_info("Injecting interrupts for GSI %d (guest vector 0x%x) %d times\n", @@ -256,6 +261,9 @@ int main(int argc, char **argv) kvm_assign_irqfd(vm, gsi, eventfd); + if (migrate_vcpus) + kvm_sched_getaffinity(0, sizeof(available_cpus), &available_cpus); + for (i = 0; i < nr_vcpus; i++) pthread_create(&vcpu_threads[i], NULL, vcpu_thread_main, vcpus[i]); @@ -266,6 +274,8 @@ int main(int argc, char **argv) continue; } + irq_cpu = -1; + for (i = 0; i < nr_irqs; i++) { const bool do_set_empty_routing = set_empty_routing && (i & BIT(3)); const bool do_use_nmi = use_nmi && (i & BIT(2)); @@ -282,6 +292,9 @@ int main(int argc, char **argv) proc_irq_set_smp_affinity(irq, irq_cpu); } + if (migrate_vcpus) + pin_task_to_random_cpu(vcpu_threads[i % nr_vcpus], &available_cpus); + for (j = 0; j < nr_vcpus; j++) { TEST_ASSERT(!GUEST_RECEIVED_IRQ(vcpus[j]), "IRQ flag for vCPU %d not clear prior to test", From 0315713a19388010b6c9f904cd6544ef6ef3a802 Mon Sep 17 00:00:00 2001 From: David Matlack Date: Fri, 26 Jun 2026 14:35:32 -0700 Subject: [PATCH 019/121] KVM: selftests: Make number of vCPUs configurable in IRQ test Extend the eventfd IRQ test with a '-v' flag to allow the user to configure the number of vCPUs to create and run (versus only ever using a single vCPU). Update the routing logic to play nice with 32 bit IDs, enable x2APIC format in KVM (to enable 32-bit ID routing), and disable KVM's x2APIC broadcast quirk so that targeting vCPU 255 doesn't blast the interrupt to all vCPUs when in x2APIC mode. Signed-off-by: David Matlack Co-developed-by: Josh Hilke Signed-off-by: Josh Hilke Co-developed-by: Sean Christopherson Link: https://patch.msgid.link/20260626213534.3866178-20-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/irq_test.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tools/testing/selftests/kvm/irq_test.c b/tools/testing/selftests/kvm/irq_test.c index c0ff6e60b167..d2c745c54960 100644 --- a/tools/testing/selftests/kvm/irq_test.c +++ b/tools/testing/selftests/kvm/irq_test.c @@ -124,7 +124,8 @@ static void kvm_route_msi(struct kvm_vm *vm, u32 gsi, struct kvm_vcpu *vcpu, .entry = { .gsi = gsi, .type = KVM_IRQ_ROUTING_MSI, - .u.msi.address_lo = 0xFEE00000 | (vcpu->id << 12), + .u.msi.address_lo = 0xFEE00000 | (vcpu->id & GENMASK(7, 0)) << 12, + .u.msi.address_hi = vcpu->id & GENMASK(31, 8), .u.msi.data = use_nmi ? NMI_VECTOR | (4 << 8) : vector, }, }; @@ -157,7 +158,7 @@ static const char *probe_iommu_type(void) static void help(const char *name) { - printf("Usage: %s [-a] [-d ] [-e] [-h] [-i nr_irqs] [-m] [-n] [-t iommu_type]\n", name); + printf("Usage: %s [-a] [-d ] [-e] [-h] [-i nr_irqs] [-m] [-n] [-t iommu_type] [-v nr_vcpus]\n", name); printf("\n"); printf("Tests KVM interrupt routing and delivery via irqfd.\n"); printf("-a Affine the device's host IRQ to a random physical CPU\n"); @@ -167,6 +168,7 @@ static void help(const char *name) printf("-m Pin target vCPU to random physical CPU before triggering interrupt\n"); printf("-n Deliver 50 percent of IRQs as non-maskable interrupts\n"); printf("-t Override the IOMMU type to use (vfio_type1_iommu or iommufd)\n"); + printf("-v Number of vCPUS to run\n"); printf("\n"); exit(KSFT_FAIL); } @@ -203,7 +205,7 @@ int main(int argc, char **argv) struct kvm_vm *vm; int irq, irq_cpu; - while ((c = getopt(argc, argv, "ad:ehi:mnt:")) != -1) { + while ((c = getopt(argc, argv, "ad:ehi:mnt:v:")) != -1) { switch (c) { case 'a': irq_affinity = true; @@ -226,6 +228,11 @@ int main(int argc, char **argv) case 't': iommu_type = optarg; break; + case 'v': + nr_vcpus = atoi_positive("Number of vCPUS", optarg); + TEST_ASSERT(nr_vcpus <= KVM_MAX_VCPUS, + "KVM selftests support at most %u vCPUs", KVM_MAX_VCPUS); + break; case 'h': default: help(argv[0]); @@ -235,6 +242,9 @@ int main(int argc, char **argv) TEST_REQUIRE(kvm_arch_has_default_irqchip()); vm = vm_create_with_vcpus(nr_vcpus, guest_code, vcpus); + vm_enable_cap(vm, KVM_CAP_X2APIC_API, KVM_X2APIC_API_USE_32BIT_IDS | + KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK); + vm_install_exception_handler(vm, vector, guest_irq_handler); vm_install_exception_handler(vm, NMI_VECTOR, guest_nmi_handler); From 2708ecca4dbdb51e40e0f6ada9154d7162b09623 Mon Sep 17 00:00:00 2001 From: David Matlack Date: Fri, 26 Jun 2026 14:35:33 -0700 Subject: [PATCH 020/121] KVM: selftests: Add xAPIC support in eventfd IRQ test Extend the eventfd IRQ test with a '-x' flag to let the user run the test in xAPIC mode instead of the default x2APIC mode. When using xAPIC mode, sanity check user input to ensure the test is being run with at most 255 vCPUs, as xAPIC can only address IDs 0-254 (255, i.e. 0xff, broadcasts to all CPUs). Signed-off-by: David Matlack Co-developed-by: Josh Hilke Signed-off-by: Josh Hilke [sean: add sanity check on number of vCPUs] Link: https://patch.msgid.link/20260626213534.3866178-21-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/irq_test.c | 31 +++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/kvm/irq_test.c b/tools/testing/selftests/kvm/irq_test.c index d2c745c54960..240f6f0fdbe4 100644 --- a/tools/testing/selftests/kvm/irq_test.c +++ b/tools/testing/selftests/kvm/irq_test.c @@ -18,6 +18,7 @@ static u64 timeout_ns = 2ULL * 1000 * 1000 * 1000; static bool guest_ready_for_irqs[KVM_MAX_VCPUS]; static bool guest_received_irq[KVM_MAX_VCPUS]; static bool guest_received_nmi[KVM_MAX_VCPUS]; +static bool x2apic = true; static bool irq_affinity; static bool done; @@ -31,14 +32,20 @@ static bool done; static u32 guest_get_vcpu_id(void) { - return x2apic_read_reg(APIC_ID); + if (x2apic) + return x2apic_read_reg(APIC_ID); + else + return xapic_read_reg(APIC_ID) >> 24; } static void guest_irq_handler(struct ex_regs *regs) { WRITE_ONCE(guest_received_irq[guest_get_vcpu_id()], true); - x2apic_write_reg(APIC_EOI, 0); + if (x2apic) + x2apic_write_reg(APIC_EOI, 0); + else + xapic_write_reg(APIC_EOI, 0); } static void guest_nmi_handler(struct ex_regs *regs) @@ -48,7 +55,10 @@ static void guest_nmi_handler(struct ex_regs *regs) static void guest_code(void) { - x2apic_enable(); + if (x2apic) + x2apic_enable(); + else + xapic_enable(); sti_nop(); @@ -158,7 +168,7 @@ static const char *probe_iommu_type(void) static void help(const char *name) { - printf("Usage: %s [-a] [-d ] [-e] [-h] [-i nr_irqs] [-m] [-n] [-t iommu_type] [-v nr_vcpus]\n", name); + printf("Usage: %s [-a] [-d ] [-e] [-h] [-i nr_irqs] [-m] [-n] [-t iommu_type] [-v nr_vcpus] [-x]\n", name); printf("\n"); printf("Tests KVM interrupt routing and delivery via irqfd.\n"); printf("-a Affine the device's host IRQ to a random physical CPU\n"); @@ -169,6 +179,7 @@ static void help(const char *name) printf("-n Deliver 50 percent of IRQs as non-maskable interrupts\n"); printf("-t Override the IOMMU type to use (vfio_type1_iommu or iommufd)\n"); printf("-v Number of vCPUS to run\n"); + printf("-x Use xAPIC mode instead of x2APIC mode in the guest\n"); printf("\n"); exit(KSFT_FAIL); } @@ -205,7 +216,7 @@ int main(int argc, char **argv) struct kvm_vm *vm; int irq, irq_cpu; - while ((c = getopt(argc, argv, "ad:ehi:mnt:v:")) != -1) { + while ((c = getopt(argc, argv, "ad:ehi:mnt:v:x")) != -1) { switch (c) { case 'a': irq_affinity = true; @@ -233,6 +244,9 @@ int main(int argc, char **argv) TEST_ASSERT(nr_vcpus <= KVM_MAX_VCPUS, "KVM selftests support at most %u vCPUs", KVM_MAX_VCPUS); break; + case 'x': + x2apic = false; + break; case 'h': default: help(argv[0]); @@ -248,6 +262,11 @@ int main(int argc, char **argv) vm_install_exception_handler(vm, vector, guest_irq_handler); vm_install_exception_handler(vm, NMI_VECTOR, guest_nmi_handler); + if (!x2apic) { + TEST_ASSERT(nr_vcpus < 256, "xAPIC can only target IDs [0-254] (255 vCPUs)"); + virt_pg_map(vm, APIC_DEFAULT_GPA, APIC_DEFAULT_GPA); + } + if (device_bdf) { if (!iommu_type) iommu_type = probe_iommu_type(); @@ -271,6 +290,8 @@ int main(int argc, char **argv) kvm_assign_irqfd(vm, gsi, eventfd); + sync_global_to_guest(vm, x2apic); + if (migrate_vcpus) kvm_sched_getaffinity(0, sizeof(available_cpus), &available_cpus); From f0772389413dce9657c7d6950abf3edbbd511356 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Tue, 16 Jun 2026 21:46:50 +0000 Subject: [PATCH 021/121] KVM: nVMX: Always flush vpid02 on first use Make sure vpid02 is always flushed on first use by setting last_vpid=0 when allocating vpid02. nested_vmx_transition_tlb_flush() will always detect a VPID change on first VM-Enter after VMXON, because VPID=0 in vmcs12 is not allowed if L1 enables VPID. This avoids using stale TLB entries from a previous lifetime of the VPID, that might have been associated with a different vCPU (or a completely different VM). Note that last_vpid is already being initialized as 0 when the vCPU is created, but it is not reset when vpid02 is freed on VMXOFF. Hence, the problem can only occur if L1 does VMXOFF -> VMXON, runs an L2, and KVM happens to reuse a VPID that has TLB entries on the physical CPU. Cc: stable@vger.kernel.org Signed-off-by: Yosry Ahmed Reviewed-by: Kai Huang Reviewed-by: Jim Mattson Link: https://patch.msgid.link/20260616214652.2157032-2-yosry@kernel.org Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/nested.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 0635e92471c8..0ed36c216cec 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -1289,6 +1289,9 @@ static void nested_vmx_transition_tlb_flush(struct kvm_vcpu *vcpu, * is the VPID incorporated into the MMU context. I.e. KVM must assume * that the new vpid12 has never been used and thus represents a new * guest ASID that cannot have entries in the TLB. + * + * Note, last_vpid is initialized as 0, so the first nested VM-Enter + * after VMXON will always flush the TLB to avoid using stale entries. */ if (is_vmenter && vmcs12->virtual_processor_id != vmx->nested.last_vpid) { vmx->nested.last_vpid = vmcs12->virtual_processor_id; @@ -5435,6 +5438,13 @@ static int enter_vmx_operation(struct kvm_vcpu *vcpu) vmx->nested.vpid02 = allocate_vpid(); + /* + * Clear last_vpid to ensure that the VPID is flushed on the first + * nested VM-Enter. Otherwise, stale TLB entries from a previous life of + * the VPID (e.g. different vCPU or even different VM) could be used. + */ + vmx->nested.last_vpid = 0; + vmx->nested.vmcs02_initialized = false; vmx->nested.vmxon = true; From 32912404b4b1ee98400744941c78f019a63d6e8f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 16 Jun 2026 21:46:51 +0000 Subject: [PATCH 022/121] KVM: nVMX: Decouple INVVPID operand checks from flushing of vpid02 Separate the INVVPID operand checks from the actual flushing of vpid02 so the flushing can be adjusted to do the right thing when vmcs02 was last loaded on a different pCPU, without having to duplicate the logic across multiple case-statements. Opportunistically let the VM-Fail paths poke out past 80 chars. No functional change intended. Cc: stable@vger.kernel.org Signed-off-by: Yosry Ahmed Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260616214652.2157032-3-yosry@kernel.org Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/nested.c | 43 ++++++++++++--------------------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 0ed36c216cec..3010c719b763 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -6072,7 +6072,6 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) u64 vpid; u64 gla; } operand; - u16 vpid02; int r, gpr_index; if (!(vmx->nested.msrs.secondary_ctls_high & @@ -6107,8 +6106,15 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) return kvm_handle_memory_failure(vcpu, r, &e); if (operand.vpid >> 16) - return nested_vmx_fail(vcpu, - VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); + return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); + + if (type != VMX_VPID_EXTENT_ALL_CONTEXT && !operand.vpid) + return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); + + /* LAM doesn't apply to addresses that are inputs to TLB invalidation. */ + if (type == VMX_VPID_EXTENT_INDIVIDUAL_ADDR && + is_noncanonical_invlpg_address(operand.gla, vcpu)) + return nested_vmx_fail(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); /* * Always flush the effective vpid02, i.e. never flush the current VPID @@ -6116,33 +6122,10 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) * VMCS, and so whether or not the current vmcs12 has VPID enabled is * irrelevant (and there may not be a loaded vmcs12). */ - vpid02 = nested_get_vpid02(vcpu); - switch (type) { - case VMX_VPID_EXTENT_INDIVIDUAL_ADDR: - /* - * LAM doesn't apply to addresses that are inputs to TLB - * invalidation. - */ - if (!operand.vpid || - is_noncanonical_invlpg_address(operand.gla, vcpu)) - return nested_vmx_fail(vcpu, - VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); - vpid_sync_vcpu_addr(vpid02, operand.gla); - break; - case VMX_VPID_EXTENT_SINGLE_CONTEXT: - case VMX_VPID_EXTENT_SINGLE_NON_GLOBAL: - if (!operand.vpid) - return nested_vmx_fail(vcpu, - VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); - vpid_sync_context(vpid02); - break; - case VMX_VPID_EXTENT_ALL_CONTEXT: - vpid_sync_context(vpid02); - break; - default: - WARN_ON_ONCE(1); - return kvm_skip_emulated_instruction(vcpu); - } + if (type == VMX_VPID_EXTENT_INDIVIDUAL_ADDR) + vpid_sync_vcpu_addr(nested_get_vpid02(vcpu), operand.gla); + else + vpid_sync_context(nested_get_vpid02(vcpu)); /* * Sync the shadow page tables if EPT is disabled, L1 is invalidating From 6d00e67326d831e6e610933a3800712f4ffe6ec1 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Tue, 16 Jun 2026 21:46:52 +0000 Subject: [PATCH 023/121] KVM: nVM: Ensure INVVPID is emulated on the correct physical CPU When emulating INVVPID, KVM executes INVVPID on the physical CPU using vpid02 (instead of the L1 assigned VPID), after doing some validations on the operands. However, it is possible that the physical CPU KVM executes INVVPID on is different from the CPU L2 is running on. For example, in the following scenario: - L2 runs on CPU #1 and exits to L1 (vmx->nested.vmcs02.cpu=1) - L1 migrates to CPU #2 and executes INVVPID - KVM executes INVVPID on CPU #2 - L1 migrates back to CPU #1 and runs L2 (vmx->nested.vmcs02.cpu=1) The TLB entries on CPU #1 are never invalidated, because INVVPID was executed on CPU #2, and vmcs02 never ran on a different pCPU (i.e. vmx_vcpu_load_vmcs() will *not* request KVM_REQ_TLB_FLUSH). Ensure that INVVPID is being executed on the same pCPU that L2 last ran on, and if not, fallback to clearing last_vpid=0 to trigger a full VPID flush on the next nested VM-Enter (as KVM will detect L1 using a different VPID for L2). If L2 ends up running on a different pCPU, KVM will flush the TLB anyway through vmx_vcpu_load_vmcs(). Cc: stable@vger.kernel.org Signed-off-by: Yosry Ahmed Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260616214652.2157032-4-yosry@kernel.org Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/nested.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 3010c719b763..6501fb314575 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -6073,6 +6073,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) u64 gla; } operand; int r, gpr_index; + int cpu; if (!(vmx->nested.msrs.secondary_ctls_high & SECONDARY_EXEC_ENABLE_VPID) || @@ -6121,11 +6122,19 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) * and never explicitly flush vpid01. INVVPID targets a VPID, not a * VMCS, and so whether or not the current vmcs12 has VPID enabled is * irrelevant (and there may not be a loaded vmcs12). + * + * If vmcs02 was last loaded on a different pCPU, then defer the flush + * by invalidating the nested VPID tracking to ensure that KVM performs + * the invalidation on the correct pCPU. */ - if (type == VMX_VPID_EXTENT_INDIVIDUAL_ADDR) + cpu = get_cpu(); + if (cpu != vmx->nested.vmcs02.cpu) + vmx->nested.last_vpid = 0; + else if (type == VMX_VPID_EXTENT_INDIVIDUAL_ADDR) vpid_sync_vcpu_addr(nested_get_vpid02(vcpu), operand.gla); else vpid_sync_context(nested_get_vpid02(vcpu)); + put_cpu(); /* * Sync the shadow page tables if EPT is disabled, L1 is invalidating From 2abe1ff20151f47a0c26bc2275e42718ae40553b Mon Sep 17 00:00:00 2001 From: Joerg Roedel Date: Tue, 30 Jun 2026 14:37:10 -0700 Subject: [PATCH 024/121] KVM: SEV: Explicitly disallow NULL user address for SNP_LAUNCH_UPDATE Explicitly reject a NULL userspace virtual address for the source page of SNP_LAUNCH_UPDATE instead of relying on the post-populate callback to do the check, and don't WARN on failure, as the scenario is blatantly user- triggerable, as reported by Sashiko. Waiting until post-populate to check the address "works", but makes it unnecessarily difficult to see that KVM's ABI is to disallow a NULL source page for non-ZERO pages. Note, several existing VMMs pass a valid userspace address for the ZERO case, i.e. KVM can't *require* the userspace address to be NULL for ZERO pages, at least not without breaking userspace. Fixes: dee5a47cc7a4 ("KVM: SEV: Add KVM_SEV_SNP_LAUNCH_UPDATE command") Reported-by: Sashiko Bot Closes: https://lore.kernel.org/all/20260611125849.9ED631F00893@smtp.kernel.org Signed-off-by: Joerg Roedel Co-developed-by: Sean Christopherson Reviewed-by: Ackerley Tng Link: https://patch.msgid.link/20260630213711.479692-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 74fb15551e83..621a2eaa58f2 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2330,9 +2330,6 @@ static int sev_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int level; int ret; - if (WARN_ON_ONCE(sev_populate_args->type != KVM_SEV_SNP_PAGE_TYPE_ZERO && !src_page)) - return -EINVAL; - ret = snp_lookup_rmpentry((u64)pfn, &assigned, &level); if (ret || assigned) { pr_debug("%s: Failed to ensure GFN 0x%llx RMP entry is initial shared state, ret: %d assigned: %d\n", @@ -2421,10 +2418,12 @@ static int snp_launch_update(struct kvm *kvm, struct kvm_sev_cmd *argp) params.type != KVM_SEV_SNP_PAGE_TYPE_CPUID)) return -EINVAL; - src = params.type == KVM_SEV_SNP_PAGE_TYPE_ZERO ? NULL : u64_to_user_ptr(params.uaddr); - - if (!PAGE_ALIGNED(src)) + if (params.type == KVM_SEV_SNP_PAGE_TYPE_ZERO) + src = NULL; + else if (!params.uaddr || !PAGE_ALIGNED(params.uaddr)) return -EINVAL; + else + src = u64_to_user_ptr(params.uaddr); npages = params.len / PAGE_SIZE; From eb606a24386389d6f707a872f2e899cf5875c2c5 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 14:37:11 -0700 Subject: [PATCH 025/121] KVM: TDX: Return EINVAL, not EOPNOTSUPP, for NULL INIT_MEM_REGION source Return EINVAL instead of EOPNOTSUPP if userspace attempts to pass a NULL pointer for the source page of INIT_MEM_REGION, so that KVM's ABI is consistent between TDX and SNP (for LAUNCH_UPDATE). EOPNOTSUPP was chosen to be a forward-looking error code for when guest_memfd supports in-place conversion, but even when in-place conversion comes along, it's an awkward error code as KVM is deliberately choosing to disallow virtual address '0', which is technically a legal userspace address. I.e. it's not so much a lack of support as it is that KVM reserves address '0' to simplify KVM's internal implementation. Opportunistically move the check so that it's co-located with the other checks on the userspace address, and so that it's more obvious that a NULL source address is explicitly disallowed. Fixes: 2a62345b3052 ("KVM: guest_memfd: GUP source pages prior to populating guest memory") Cc: Yan Zhao Cc: Ackerley Tng Reviewed-by: Xiaoyao Li Acked-by: Kiryl Shutsemau (Meta) Reviewed-by: Binbin Wu Reviewed-by: Yan Zhao Tested-by: Yan Zhao Reviewed-by: Ackerley Tng Link: https://patch.msgid.link/20260630213711.479692-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/tdx.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c index ffe9d0db58c5..b0ec054732b9 100644 --- a/arch/x86/kvm/vmx/tdx.c +++ b/arch/x86/kvm/vmx/tdx.c @@ -3198,9 +3198,6 @@ static int tdx_gmem_post_populate(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, if (KVM_BUG_ON(kvm_tdx->page_add_src, kvm)) return -EIO; - if (!src_page) - return -EOPNOTSUPP; - kvm_tdx->page_add_src = src_page; ret = kvm_tdp_mmu_map_private_pfn(arg->vcpu, gfn, pfn); kvm_tdx->page_add_src = NULL; @@ -3247,8 +3244,8 @@ static int tdx_vcpu_init_mem_region(struct kvm_vcpu *vcpu, struct kvm_tdx_cmd *c if (copy_from_user(®ion, u64_to_user_ptr(cmd->data), sizeof(region))) return -EFAULT; - if (!PAGE_ALIGNED(region.source_addr) || !PAGE_ALIGNED(region.gpa) || - !region.nr_pages || + if (!PAGE_ALIGNED(region.source_addr) || !region.source_addr || + !PAGE_ALIGNED(region.gpa) || !region.nr_pages || region.gpa + (region.nr_pages << PAGE_SHIFT) <= region.gpa || !vt_is_tdx_private_gpa(kvm, region.gpa) || !vt_is_tdx_private_gpa(kvm, region.gpa + (region.nr_pages << PAGE_SHIFT) - 1)) From 8835a24a56089e90953af7f2a79363fcccd7f97c Mon Sep 17 00:00:00 2001 From: Binbin Wu Date: Tue, 9 Jun 2026 15:57:48 +0800 Subject: [PATCH 026/121] KVM: x86: Fix emulated CPUID features being applied to wrong sub-leaf Pass the CPUID index into cpuid_func_emulated() and return no emulated features for indexed CPUID leaves with a non-zero index. KVM currently emulates CPUID features only for index 0, but kvm_vcpu_after_set_cpuid() looks up emulated features by function alone. As a result, reverse_cpuid[] entries that share a function but use a non-zero index, e.g. CPUID.7.1:ECX, can inherit emulated features that belong to index 0. For example, RDPID, which is CPUID.7.0:ECX[22], can be incorrectly OR'd into CPUID.7.1:ECX. This is benign today because the affected bits do not correspond to features KVM cares about, but it can become a real bug as new CPUID features are defined. Make the helper index-aware so emulated features are applied only to the CPUID entry they actually describe. Fixes: e592ec657d84 ("KVM: x86: Initialize guest cpu_caps based on KVM support") Suggested-by: Sean Christopherson Signed-off-by: Binbin Wu Reviewed-by: Xiaoyao Li Link: https://patch.msgid.link/20260609075748.612704-1-binbin.wu@linux.intel.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/cpuid.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 2698fa42cd97..f402a5dc4390 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -370,7 +370,7 @@ static u32 cpuid_get_reg_unsafe(struct kvm_cpuid_entry2 *entry, u32 reg) } } -static int cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, u32 func, +static int cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, u32 func, u32 index, bool include_partially_emulated); void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) @@ -400,7 +400,7 @@ void kvm_vcpu_after_set_cpuid(struct kvm_vcpu *vcpu) if (!entry) continue; - cpuid_func_emulated(&emulated, cpuid.function, true); + cpuid_func_emulated(&emulated, cpuid.function, cpuid.index, true); /* * A vCPU has a feature if it's supported by KVM and is enabled @@ -1369,11 +1369,15 @@ static struct kvm_cpuid_entry2 *do_host_cpuid(struct kvm_cpuid_array *array, return entry; } -static int cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, u32 func, +static int cpuid_func_emulated(struct kvm_cpuid_entry2 *entry, u32 func, u32 index, bool include_partially_emulated) { memset(entry, 0, sizeof(*entry)); + /* KVM doesn't currently emulate any non-zero indices. */ + if (cpuid_function_is_indexed(func) && index) + return 0; + entry->function = func; entry->index = 0; entry->flags = 0; @@ -1411,7 +1415,7 @@ static int __do_cpuid_func_emulated(struct kvm_cpuid_array *array, u32 func) if (array->nent >= array->maxnent) return -E2BIG; - array->nent += cpuid_func_emulated(&array->entries[array->nent], func, false); + array->nent += cpuid_func_emulated(&array->entries[array->nent], func, 0, false); return 0; } From 8f683a4dc546a9ced5dce4b6fb862be26a0572c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20L=C3=B3pez?= Date: Fri, 12 Jun 2026 16:01:06 -0700 Subject: [PATCH 027/121] KVM: x86: Treat any non-zero return from set_dr() as a faulting condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When emulating a MOV to a debug register, em_dr_write() calls @ctxt->ops->set_dr(), which is forwarded to emulator_set_dr() and then kvm_set_dr(). The latter checks that the written value is valid, otherwise returning an error, in which case the emulator is supposed to inject a #GP fault into the guest. Commit 996ff5429e98 ("KVM: x86: move kvm_inject_gp up from kvm_set_dr to callers") changed the contract of kvm_set_dr() (and thus emulator_set_dr()), returning 1 as an error instead of -1, but the caller in em_dr_write() was never updated, checking only if the returned value is negative. The end result is that em_dr_write() does not detect the error, so an invalid write does not generate a #GP, but at the same time the register value is not updated. The practical impact is limited, as check_dr_write() already checks DR6 and DR7 manually. However, it misses DR4/DR5, which alias DR6/DR7 when CR4.DE=0. Fix the bug by treating any non-zero return from set_dr() as a reason to inject #GP. Note, the manual checks on DR6 and DR7 are flawed, as they incorrectly prioritize the #GP over a DR7.GD=1 #DB (the General Detect #DB has priority on both Intel and AMD). Note #2, relying on ->set_dr() to detect #GP is also flawed as all exceptions have higher priority than the instruction intercept on SVM, i.e. the manual checks need to be extended to DR4 and DR5 (after the priority bug is fixed). Fixes: 996ff5429e98 ("KVM: x86: move kvm_inject_gp up from kvm_set_dr to callers") Signed-off-by: Carlos López Link: https://patch.msgid.link/20260601133320.91479-2-clopez@suse.de [sean: drop explicit "!= 0", massage changelog] Reviewed-by: Jim Mattson Link: https://patch.msgid.link/20260612230113.684301-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index b566ab5c7515..75cd8b6136aa 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -3299,7 +3299,7 @@ static int em_dr_write(struct x86_emulate_ctxt *ctxt) val = ctxt->src.val & ~0U; /* #UD condition is already handled. */ - if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val) < 0) + if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val)) return emulate_gp(ctxt, 0); /* Disable writeback. */ From 55ac576a9bf43b32ef4de926316f3a57f08887e0 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 12 Jun 2026 16:01:07 -0700 Subject: [PATCH 028/121] KVM: x86: Prioritize DR7.GD #DB over #GP due to illegal DR6/7 value When emulating a MOV DR, specifically a write to DR6 or DR7, treat a #DB due to DR7.GD (General Detect) as higher priority than a #GP due to an illegal value. While neither Intel's SDM nor AMD's APM says anything about the relative priority, empirical testing on Intel and AMD shows that the #DB has higher priority. And for VMX, where the instruction intercept has priority over *all* exceptions, KVM already treats the #DB as having higher priority. Cc: Maciej W. Rozycki Fixes: 3b88e41a4134 ("KVM: SVM: Add intercept check for accessing dr registers") Reviewed-by: Jim Mattson Link: https://patch.msgid.link/20260612230113.684301-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 75cd8b6136aa..4484c5fa19e3 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -3854,11 +3854,16 @@ static int check_dr_write(struct x86_emulate_ctxt *ctxt) { u64 new_val = ctxt->src.val64; int dr = ctxt->modrm_reg; + int rc; + + rc = check_dr_read(ctxt); + if (rc != X86EMUL_CONTINUE) + return rc; if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL)) return emulate_gp(ctxt, 0); - return check_dr_read(ctxt); + return X86EMUL_CONTINUE; } static int check_svme(struct x86_emulate_ctxt *ctxt) From e27ca3dfbb737c2335bbaec6981f2fddddf1cff5 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 12 Jun 2026 16:01:08 -0700 Subject: [PATCH 029/121] KVM: x86: Manually check DR4/5 write values to fix SVM intercept priority Manually (pre)check the values being written to DR4/5, i.e. the DR6/DR7 aliases, instead of relying on ->set_dr() => kvm_set_dr() to signal a #GP. SVM unfortunately prioritizes all exceptions over an instruction intercept, i.e. nSVM is relying on the emulator to perform *all* exception checks prior to attempting to execute the instruction. Fixes: 3b88e41a4134 ("KVM: SVM: Add intercept check for accessing dr registers") Reviewed-by: Jim Mattson Link: https://patch.msgid.link/20260612230113.684301-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 4484c5fa19e3..a1bccab0eefe 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -3853,15 +3853,23 @@ static int check_dr_read(struct x86_emulate_ctxt *ctxt) static int check_dr_write(struct x86_emulate_ctxt *ctxt) { u64 new_val = ctxt->src.val64; - int dr = ctxt->modrm_reg; int rc; rc = check_dr_read(ctxt); if (rc != X86EMUL_CONTINUE) return rc; - if ((dr == 6 || dr == 7) && (new_val & 0xffffffff00000000ULL)) - return emulate_gp(ctxt, 0); + switch (ctxt->modrm_reg) { + case 4: + case 5: + case 6: + case 7: + if (new_val & 0xffffffff00000000ULL) + return emulate_gp(ctxt, 0); + break; + default: + break; + } return X86EMUL_CONTINUE; } From 32a7188a667fc604173414a5dd0754877aafc32e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 12 Jun 2026 16:01:09 -0700 Subject: [PATCH 030/121] KVM: x86: Prioritize #UD on MOV DR over #GP due to non-zero CPL Manually handle the CPL check for MOV DR instructions instead of using the Priv flag, *after* checking for #UD scenarios, as #GP due to CPL>0 has lower priority than all #UDs. Fixes: 1e470be5a108 ("KVM: x86 emulator: fix mov dr to inject #UD when needed.") Reviewed-by: Jim Mattson Link: https://patch.msgid.link/20260612230113.684301-5-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index a1bccab0eefe..127a21eeef66 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -3844,6 +3844,9 @@ static int check_dr_read(struct x86_emulate_ctxt *ctxt) if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5)) return emulate_ud(ctxt); + if (ctxt->ops->cpl(ctxt)) + return emulate_gp(ctxt, 0); + if (ctxt->ops->get_effective_dr7(ctxt) & DR7_GD) return emulate_db(ctxt, DR6_BD); @@ -4380,11 +4383,10 @@ static const struct opcode twobyte_table[256] = { D(ImplicitOps | ModRM | SrcMem | NoAccess), /* NOP + 7 * reserved NOP */ /* 0x20 - 0x2F */ DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_access), - DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read), + DIP(ModRM | DstMem | Op3264 | NoMod, dr_read, check_dr_read), IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_cr_write, cr_write, check_cr_access), - IIP(ModRM | SrcMem | Priv | Op3264 | NoMod, em_dr_write, dr_write, - check_dr_write), + IIP(ModRM | SrcMem | Op3264 | NoMod, em_dr_write, dr_write, check_dr_write), N, N, N, N, GP(ModRM | DstReg | SrcMem | Mov | Sse | Avx, &pfx_0f_28_0f_29), GP(ModRM | DstMem | SrcReg | Mov | Sse | Avx, &pfx_0f_28_0f_29), From 48512697c081b9c48b7cdae92f437d99e7c0c03c Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 12 Jun 2026 16:01:10 -0700 Subject: [PATCH 031/121] KVM: VMX: Prioritize DR7.GD=1 #DB over CPL>0 #GP on Intel When emulating a MOV DR on Intel with DR7.GD=1 at CPL>0, prioritize the #DB due to DR7.GD over the #GP due to CPL>0, as empirical testing shows that Intel CPUs (Skylake, Icelake and Emerald Rapids) prioritize the DR7.GD #DB over all #GPs, whereas AMD CPUs prioritize the CPL>0 #GP (but not illegal value #GPs) over the #DB. Outside of the emulator, don't bother trying to provide the "correct" priority based on the virtual CPU model, as it's simply impossible to do so without intercepting *all* MOV DR accesses, which would result in a massive, unacceptable performance hit. Note, getting the priority right when advertising Intel on AMD would also require intercepting #GP, as SVM prioritizes all exceptions over the instruction intercept. Note, neither Intel's SDM nor AMD's APM says anything about the relative priority, hence the empirical testing. Arguably Intel's description of DR7.GD: causes a debug exception to be generated prior to any MOV instruction that accesses a debug register. implies that DR7.GD has higher priority. But that's a fairly weak argument as the statement would still hold true if the #GP due to CPL>0 had higher priority, as the #GP would prevent any access to a DR. Fixes: 3b88e41a4134 ("KVM: SVM: Add intercept check for accessing dr registers") Link: https://patch.msgid.link/20260612230113.684301-6-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 7 ++++++- arch/x86/kvm/vmx/vmx.c | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index 127a21eeef66..b4dc57fe0bc9 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -3834,6 +3834,7 @@ static int check_cr_access(struct x86_emulate_ctxt *ctxt) static int check_dr_read(struct x86_emulate_ctxt *ctxt) { + bool is_intel = ctxt->ops->guest_cpuid_is_intel_compatible(ctxt); int dr = ctxt->modrm_reg; u64 cr4; @@ -3844,12 +3845,16 @@ static int check_dr_read(struct x86_emulate_ctxt *ctxt) if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5)) return emulate_ud(ctxt); - if (ctxt->ops->cpl(ctxt)) + /* Intel CPUs prioritize the DR7.GD=1 #DB over the CPL>0 #GP. */ + if (!is_intel && ctxt->ops->cpl(ctxt)) return emulate_gp(ctxt, 0); if (ctxt->ops->get_effective_dr7(ctxt) & DR7_GD) return emulate_db(ctxt, DR6_BD); + if (is_intel && ctxt->ops->cpl(ctxt)) + return emulate_gp(ctxt, 0); + return X86EMUL_CONTINUE; } diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 3681d565f177..b25b978d7a9d 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -5762,9 +5762,6 @@ static int handle_dr(struct kvm_vcpu *vcpu) if (!kvm_require_dr(vcpu, dr)) return 1; - if (vmx_get_cpl(vcpu) > 0) - goto out; - dr7 = vmcs_readl(GUEST_DR7); if (dr7 & DR7_GD) { /* @@ -5785,6 +5782,9 @@ static int handle_dr(struct kvm_vcpu *vcpu) } } + if (vmx_get_cpl(vcpu) > 0) + goto out; + if (vcpu->guest_debug == 0) { exec_controls_clearbit(to_vmx(vcpu), CPU_BASED_MOV_DR_EXITING); From 1f077338d1cd57fd3c2d6dfe193be58cd1d04fee Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 12 Jun 2026 16:01:11 -0700 Subject: [PATCH 032/121] KVM: x86: Use kvm_dr{6,7}_valid() to check DR{4,5,6,7} write values in emulator Use kvm_dr{6,7}_valid() to validate the incoming DR{4,5,6,7} value in the emulator instead of open coding an equivalent check. In the unlikely event that the behavior of DR6/7 (and their aliases) changes in the future, using common helpers will hopefully make it less likely the emulator logic will be overlooked. No functional change intended. Reviewed-by: Jim Mattson Link: https://patch.msgid.link/20260612230113.684301-7-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index b4dc57fe0bc9..b1799ed01939 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -3869,10 +3869,13 @@ static int check_dr_write(struct x86_emulate_ctxt *ctxt) switch (ctxt->modrm_reg) { case 4: - case 5: case 6: + if (!kvm_dr6_valid(new_val)) + return emulate_gp(ctxt, 0); + break; + case 5: case 7: - if (new_val & 0xffffffff00000000ULL) + if (!kvm_dr7_valid(new_val)) return emulate_gp(ctxt, 0); break; default: From b077cfed52b48c0727c16be97c7569c77c4a8019 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 12 Jun 2026 16:01:12 -0700 Subject: [PATCH 033/121] KVM: x86: WARN if MOV DR emulation hits a "too late" #GP WARN if ->set_dr() => kvm_set_dr() fails when emulating a MOV DR write, as the emulator _must_ pre-check for #GPs in order to get the event priority right when emulating MOV DR for L2 on SVM (all exceptions have higher priority than the instruction intercept). Opportunistically update the comment as the blurb about "#UD" being checked is incomplete and misleading. Reviewed-by: Jim Mattson Link: https://patch.msgid.link/20260612230113.684301-8-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index b1799ed01939..e030ef9b9744 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -3298,8 +3298,12 @@ static int em_dr_write(struct x86_emulate_ctxt *ctxt) else val = ctxt->src.val & ~0U; - /* #UD condition is already handled. */ - if (ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val)) + /* + * A #GP due to an illegal value should be impossible at this point, as + * such #GPs have priority over MOV DR intercepts on SVM, i.e. KVM must + * manually check the value *before* emulating the write. + */ + if (WARN_ON_ONCE(ctxt->ops->set_dr(ctxt, ctxt->modrm_reg, val))) return emulate_gp(ctxt, 0); /* Disable writeback. */ From 7a642d8dcfa8fa20f99b303ac1c6591f2057327d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 12 Jun 2026 16:01:13 -0700 Subject: [PATCH 034/121] KVM: x86: Read CR4.DE in emulator if and only if accessing DR4 or DR5 Micro-optimize emulation of MOV DR instructions by checking CR4.DE if and only if DR4 or DR5 is being accessed. No functional change intended. Reviewed-by: Jim Mattson Link: https://patch.msgid.link/20260612230113.684301-9-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/emulate.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index e030ef9b9744..8071b372d233 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -3840,13 +3840,11 @@ static int check_dr_read(struct x86_emulate_ctxt *ctxt) { bool is_intel = ctxt->ops->guest_cpuid_is_intel_compatible(ctxt); int dr = ctxt->modrm_reg; - u64 cr4; if (dr > 7) return emulate_ud(ctxt); - cr4 = ctxt->ops->get_cr(ctxt, 4); - if ((cr4 & X86_CR4_DE) && (dr == 4 || dr == 5)) + if ((dr == 4 || dr == 5) && (ctxt->ops->get_cr(ctxt, 4) & X86_CR4_DE)) return emulate_ud(ctxt); /* Intel CPUs prioritize the DR7.GD=1 #DB over the CPL>0 #GP. */ From d151ca6e1289ce84f67f4bc0d9de4edeca237306 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 15:56:08 -0700 Subject: [PATCH 035/121] KVM: x86/hyperv: Get target FIFO in hv_tlb_flush_enqueue(), not caller When handling Hyper-V PV TLB flushes, retrieve the to-be-used FIFO in hv_tlb_flush_enqueue() instead of having the caller pass in the FIFO. This will make it easier to fix a cross-vCPU race where KVM can access a vCPU's FIFO before it's fully initialized. No functional change intended. Link: https://patch.msgid.link/20260630225619.511632-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/hyperv.c | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 1ee0d23f8949..2dc3e64b3f2f 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -1935,16 +1935,18 @@ static int kvm_hv_get_tlb_flush_entries(struct kvm *kvm, struct kvm_hv_hcall *hc return kvm_hv_get_hc_data(kvm, hc, hc->rep_cnt, hc->rep_cnt, entries); } -static void hv_tlb_flush_enqueue(struct kvm_vcpu *vcpu, - struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo, - u64 *entries, int count) +static void hv_tlb_flush_enqueue(struct kvm_vcpu *vcpu, u64 *entries, int count, + bool is_guest_mode) { + struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo; struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); u64 flush_all_entry = KVM_HV_TLB_FLUSHALL_ENTRY; if (!hv_vcpu) return; + tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(vcpu, is_guest_mode); + spin_lock(&tlb_flush_fifo->write_lock); /* @@ -2017,7 +2019,6 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) struct kvm *kvm = vcpu->kvm; struct hv_tlb_flush_ex flush_ex; struct hv_tlb_flush flush; - struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo; /* * Normally, there can be no more than 'KVM_HV_TLB_FLUSH_FIFO_SIZE' * entries on the TLB flush fifo. The last entry, however, needs to be @@ -2144,11 +2145,8 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) * analyze it here, flush TLB regardless of the specified address space. */ if (all_cpus && !is_guest_mode(vcpu)) { - kvm_for_each_vcpu(i, v, kvm) { - tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(v, false); - hv_tlb_flush_enqueue(v, tlb_flush_fifo, - tlb_flush_entries, hc->rep_cnt); - } + kvm_for_each_vcpu(i, v, kvm) + hv_tlb_flush_enqueue(v, tlb_flush_entries, hc->rep_cnt, false); kvm_make_all_cpus_request(kvm, KVM_REQ_HV_TLB_FLUSH); } else if (!is_guest_mode(vcpu)) { @@ -2158,9 +2156,7 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) v = kvm_get_vcpu(kvm, i); if (!v) continue; - tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(v, false); - hv_tlb_flush_enqueue(v, tlb_flush_fifo, - tlb_flush_entries, hc->rep_cnt); + hv_tlb_flush_enqueue(v, tlb_flush_entries, hc->rep_cnt, false); } kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH, vcpu_mask); @@ -2191,9 +2187,7 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) continue; __set_bit(i, vcpu_mask); - tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(v, true); - hv_tlb_flush_enqueue(v, tlb_flush_fifo, - tlb_flush_entries, hc->rep_cnt); + hv_tlb_flush_enqueue(v, tlb_flush_entries, hc->rep_cnt, true); } kvm_make_vcpus_request_mask(kvm, KVM_REQ_HV_TLB_FLUSH, vcpu_mask); From b6de8bfdab32a031431149fe4781dcaab0a42894 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 15:56:09 -0700 Subject: [PATCH 036/121] KVM: x86/hyperv: Check for NULL vCPU Hyper-V object in kvm_hv_get_tlb_flush_fifo() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Check for a NULL Hyper-V object in kvm_hv_get_tlb_flush_fifo() instead of relying on the caller to do so. This will allow fixing a cross-vCPU race where KVM can access a vCPU's FIFO before it's fully initialized, without having to jump through too many cognitive hoops to reason about the correctness of the logic. Ignoring changes in ordering that only affect the aforementioned race, no functional change intended. Reviewed-by: Philippe Mathieu-Daudé Link: https://patch.msgid.link/20260630225619.511632-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/hyperv.c | 11 +++++------ arch/x86/kvm/hyperv.h | 7 ++++++- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 2dc3e64b3f2f..49b1154366ce 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -1939,13 +1939,11 @@ static void hv_tlb_flush_enqueue(struct kvm_vcpu *vcpu, u64 *entries, int count, bool is_guest_mode) { struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo; - struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); u64 flush_all_entry = KVM_HV_TLB_FLUSHALL_ENTRY; - if (!hv_vcpu) - return; - tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(vcpu, is_guest_mode); + if (!tlb_flush_fifo) + return; spin_lock(&tlb_flush_fifo->write_lock); @@ -1972,15 +1970,16 @@ out_unlock: int kvm_hv_vcpu_flush_tlb(struct kvm_vcpu *vcpu) { struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo; - struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); u64 entries[KVM_HV_TLB_FLUSH_FIFO_SIZE]; int i, j, count; gva_t gva; - if (!tdp_enabled || !hv_vcpu) + if (!tdp_enabled) return -EINVAL; tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(vcpu, is_guest_mode(vcpu)); + if (!tlb_flush_fifo) + return -EINVAL; count = kfifo_out(&tlb_flush_fifo->entries, entries, KVM_HV_TLB_FLUSH_FIFO_SIZE); diff --git a/arch/x86/kvm/hyperv.h b/arch/x86/kvm/hyperv.h index 1c8f7aaab063..2da11b967c41 100644 --- a/arch/x86/kvm/hyperv.h +++ b/arch/x86/kvm/hyperv.h @@ -202,6 +202,9 @@ static inline struct kvm_vcpu_hv_tlb_flush_fifo *kvm_hv_get_tlb_flush_fifo(struc int i = is_guest_mode ? HV_L2_TLB_FLUSH_FIFO : HV_L1_TLB_FLUSH_FIFO; + if (!hv_vcpu) + return NULL; + return &hv_vcpu->tlb_flush_fifo[i]; } @@ -209,10 +212,12 @@ static inline void kvm_hv_vcpu_purge_flush_tlb(struct kvm_vcpu *vcpu) { struct kvm_vcpu_hv_tlb_flush_fifo *tlb_flush_fifo; - if (!to_hv_vcpu(vcpu) || !kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu)) + if (!kvm_check_request(KVM_REQ_HV_TLB_FLUSH, vcpu)) return; tlb_flush_fifo = kvm_hv_get_tlb_flush_fifo(vcpu, is_guest_mode(vcpu)); + if (!tlb_flush_fifo) + return; kfifo_reset_out(&tlb_flush_fifo->entries); } From c84d86130f24ecba229637c123ff835a3e7f4a57 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 15:56:10 -0700 Subject: [PATCH 037/121] KVM: x86/hyperv: Ensure vCPU's Hyper-V object is initialized on cross-vCPU accesses When initializing a vCPU's Hyper-V object, ensure the object is fully initialized prior to exposing it through the vCPU, and ensure accesses from other tasks (e.g. other vCPUs) see the fully initialized object if vcpu->arch.hyperv is non-NULL. Lack of ordering manifests as a lockdep splat due to attempting to lock a TLB flush FIFO before the spinlock is initialized. INFO: trying to register non-static key. The code is fine but needs lockdep annotation, or maybe you didn't initialize this object before use? turning off the locking correctness validator. CPU: 1 PID: 5005 Comm: syz-executor189 Not tainted 6.6.120-smp-DEV #1 Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 03/18/2026 Call Trace: [] dump_stack_lvl+0xcc/0x130 lib/dump_stack.c:106 [] assign_lock_key+0x1fd/0x230 kernel/locking/lockdep.c:977 [] register_lock_class+0x187/0x7a0 kernel/locking/lockdep.c:1291 [] __lock_acquire+0x179/0x7650 kernel/locking/lockdep.c:5016 [] lock_acquire+0x13f/0x3d0 kernel/locking/lockdep.c:5756 [] __raw_spin_lock include/linux/spinlock_api_smp.h:133 [inline] [] _raw_spin_lock+0x2b/0x40 kernel/locking/spinlock.c:154 [] spin_lock include/linux/spinlock.h:351 [inline] [] hv_tlb_flush_enqueue+0xb4/0x270 arch/x86/kvm/hyperv.c:1946 [] kvm_hv_flush_tlb+0xa96/0x1dc0 arch/x86/kvm/hyperv.c:2145 [] kvm_hv_hypercall+0x103b/0x1fe0 arch/x86/kvm/hyperv.c:-1 [] __vmx_handle_exit arch/x86/kvm/vmx/vmx.c:6624 [inline] [] vmx_handle_exit+0x12e3/0x21f0 arch/x86/kvm/vmx/vmx.c:6641 [] vcpu_enter_guest arch/x86/kvm/x86.c:11649 [inline] [] vcpu_run+0x4d01/0x79c0 arch/x86/kvm/x86.c:11832 [] kvm_arch_vcpu_ioctl_run+0xb49/0x1c80 arch/x86/kvm/x86.c:12179 [] kvm_vcpu_ioctl+0xc80/0xff0 virt/kvm/kvm_main.c:6029 [] vfs_ioctl fs/ioctl.c:52 [inline] [] __do_sys_ioctl fs/ioctl.c:872 [inline] [] __se_sys_ioctl+0xfd/0x170 fs/ioctl.c:858 [] do_syscall_x64 arch/x86/entry/common.c:52 [inline] [] do_syscall_64+0x69/0xb0 arch/x86/entry/common.c:93 [] entry_SYSCALL_64_after_hwframe+0x68/0xd2 Use the "safe" variant in all paths that are known to access the Hyper-V object, as detected by an upcoming lockdep assertion, with an assist or two from Sashiko. Link: https://lore.kernel.org/all/20260612232258.0D9131F000E9@smtp.kernel.org Fixes: 0823570f0198 ("KVM: x86: hyper-v: Introduce TLB flush fifo") Fixes: fc08b628d7c9 ("KVM: x86: hyper-v: Allocate Hyper-V context lazily") Reported-by: syzbot+5b32c49cd8f005e65654@syzkaller.appspotmail.com Reported-by: syzbot+5d2b94b77112148d1744@syzkaller.appspotmail.com Closes: https://lore.kernel.org/all/6a396a66.52ae72c2.136ac7.0002.GAE@google.com Tested-by: syzbot+5d2b94b77112148d1744@syzkaller.appspotmail.com Link: https://patch.msgid.link/20260630225619.511632-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/hyperv.c | 23 ++++++++++++++++++----- arch/x86/kvm/hyperv.h | 18 +++++++++++++++--- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 49b1154366ce..888526ce4dab 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -206,13 +206,19 @@ static struct kvm_vcpu *get_vcpu_by_vpidx(struct kvm *kvm, u32 vpidx) static struct kvm_vcpu_hv_synic *synic_get(struct kvm *kvm, u32 vpidx) { - struct kvm_vcpu *vcpu; struct kvm_vcpu_hv_synic *synic; + struct kvm_vcpu_hv *hv_vcpu; + struct kvm_vcpu *vcpu; vcpu = get_vcpu_by_vpidx(kvm, vpidx); - if (!vcpu || !to_hv_vcpu(vcpu)) + if (!vcpu) return NULL; - synic = to_hv_synic(vcpu); + + hv_vcpu = to_hv_vcpu_safe(vcpu); + if (!hv_vcpu) + return NULL; + + synic = &hv_vcpu->synic; return (synic->active) ? synic : NULL; } @@ -972,7 +978,6 @@ int kvm_hv_vcpu_init(struct kvm_vcpu *vcpu) if (!hv_vcpu) return -ENOMEM; - vcpu->arch.hyperv = hv_vcpu; hv_vcpu->vcpu = vcpu; synic_init(&hv_vcpu->synic); @@ -988,6 +993,14 @@ int kvm_hv_vcpu_init(struct kvm_vcpu *vcpu) spin_lock_init(&hv_vcpu->tlb_flush_fifo[i].write_lock); } + /* + * Ensure the structure is fully initialized before it's visible to + * other tasks, as much of the state can be legally accessed without + * holding vcpu->mutex. + * + * Pairs with the smp_load_acquire() in to_hv_vcpu_safe(). + */ + smp_store_release(&vcpu->arch.hyperv, hv_vcpu); return 0; } @@ -2165,7 +2178,7 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc) bitmap_zero(vcpu_mask, KVM_MAX_VCPUS); kvm_for_each_vcpu(i, v, kvm) { - hv_v = to_hv_vcpu(v); + hv_v = to_hv_vcpu_safe(v); /* * The following check races with nested vCPUs entering/exiting diff --git a/arch/x86/kvm/hyperv.h b/arch/x86/kvm/hyperv.h index 2da11b967c41..ea9c81d76dd3 100644 --- a/arch/x86/kvm/hyperv.h +++ b/arch/x86/kvm/hyperv.h @@ -62,6 +62,18 @@ static inline struct kvm_hv *to_kvm_hv(struct kvm *kvm) return &kvm->arch.hyperv; } +static inline struct kvm_vcpu_hv *to_hv_vcpu_safe(struct kvm_vcpu *vcpu) +{ + /* + * Ensure the HyperV structure is fully initialized when accessing it + * without holding vcpu->mutex (or some other guarantee that KVM can't + * concurrently instantiate the structure). + * + * Pairs with the smp_store_release() in kvm_hv_vcpu_init(). + */ + return smp_load_acquire(&vcpu->arch.hyperv); +} + static inline struct kvm_vcpu_hv *to_hv_vcpu(struct kvm_vcpu *vcpu) { return vcpu->arch.hyperv; @@ -88,7 +100,7 @@ static inline struct kvm_hv_syndbg *to_hv_syndbg(struct kvm_vcpu *vcpu) static inline u32 kvm_hv_get_vpindex(struct kvm_vcpu *vcpu) { - struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); + struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu_safe(vcpu); return hv_vcpu ? hv_vcpu->vp_index : vcpu->vcpu_idx; } @@ -142,7 +154,7 @@ static inline struct kvm_vcpu *hv_stimer_to_vcpu(struct kvm_vcpu_hv_stimer *stim static inline bool kvm_hv_has_stimer_pending(struct kvm_vcpu *vcpu) { - struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); + struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu_safe(vcpu); if (!hv_vcpu) return false; @@ -198,7 +210,7 @@ int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, static inline struct kvm_vcpu_hv_tlb_flush_fifo *kvm_hv_get_tlb_flush_fifo(struct kvm_vcpu *vcpu, bool is_guest_mode) { - struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu(vcpu); + struct kvm_vcpu_hv *hv_vcpu = to_hv_vcpu_safe(vcpu); int i = is_guest_mode ? HV_L2_TLB_FLUSH_FIFO : HV_L1_TLB_FLUSH_FIFO; From e4ffb0ceb9f47f3797de69c2b81b501ba447515f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 15:56:11 -0700 Subject: [PATCH 038/121] KVM: x86/xen: Always route non-singleshot-timer vCPU hypercalls to userspace When handling Xen vCPU hypercalls, explicitly route non-singleshot-timer commands to userspace, *before* checking if in-kernel emulation of the Xen timer is enabled. Punting hypercalls that are never accelerated by KVM because some other hypercall happens to be disabled is confusing and actively dangerous, e.g. it's easy to miss that the only reason KVM can bail early is because the timer-disabled case provides the same semantics as the implicit "default" path in the switch-statement. Opportunistically convert the switch-statement to an if-else-statement to avoid having to carry code for an impossible "default" case. For all intents and purposes, no functional change intended. Link: https://patch.msgid.link/20260630225619.511632-5-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/xen.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 694b31c1fcc9..3ed6686e0a1a 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1607,11 +1607,14 @@ static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd, struct vcpu_set_singleshot_timer oneshot; struct x86_exception e; + if (cmd != VCPUOP_set_singleshot_timer && + cmd != VCPUOP_stop_singleshot_timer) + return false; + if (!kvm_xen_timer_enabled(vcpu)) return false; - switch (cmd) { - case VCPUOP_set_singleshot_timer: + if (cmd == VCPUOP_set_singleshot_timer) { if (vcpu->arch.xen.vcpu_id != vcpu_id) { *r = -EINVAL; return true; @@ -1640,20 +1643,16 @@ static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd, } kvm_xen_start_timer(vcpu, oneshot.timeout_abs_ns, false); - *r = 0; - return true; - - case VCPUOP_stop_singleshot_timer: + } else { if (vcpu->arch.xen.vcpu_id != vcpu_id) { *r = -EINVAL; return true; } kvm_xen_stop_timer(vcpu); - *r = 0; - return true; } - return false; + *r = 0; + return true; } static bool kvm_xen_hcall_set_timer_op(struct kvm_vcpu *vcpu, uint64_t timeout, From 32d7943e51798c26072d88ff84f68c566a6e2184 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 15:56:12 -0700 Subject: [PATCH 039/121] KVM: x86/xen: Consolidate checks on Xen vCPU ID for singleshot timer hypercalls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hoist the checks on the Xen vCPU ID when handling set_singleshot_timer and stop_singleshot_timer hypercalls out of their individual if-statements, so that both checks on the ID are in common code. kvm_xen_hcall_vcpu_op() is already doubly committed to handling only singleshot timer hypercalls, and even if that were to change in the future, the function could simply be renamed and turned into a helper specifically for timer hypercalls. Opportunistically add a comment to explain why the check exists; the code looks rather nonsensical without the knowledge that @vcpu_id is a common param for all per-vCPU hypercalls. No functional change intended. Reviewed-by: David Woodhouse Reviewed-by: Philippe Mathieu-Daudé Link: https://patch.msgid.link/20260630225619.511632-6-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/xen.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 3ed6686e0a1a..7a0d89faca85 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -1614,12 +1614,18 @@ static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd, if (!kvm_xen_timer_enabled(vcpu)) return false; - if (cmd == VCPUOP_set_singleshot_timer) { - if (vcpu->arch.xen.vcpu_id != vcpu_id) { - *r = -EINVAL; - return true; - } + /* + * Reject the hypercall if the guest is trying to start/stop the timer + * for a different vCPU. Xen per-vCPU hypercalls take a target vCPU as + * a common parameter, as all per-vCPU hypercalls *except* single-shot + * timer updates can be cross-vCPU. + */ + if (vcpu->arch.xen.vcpu_id != vcpu_id) { + *r = -EINVAL; + return true; + } + if (cmd == VCPUOP_set_singleshot_timer) { /* * The only difference for 32-bit compat is the 4 bytes of * padding after the interesting part of the structure. So @@ -1644,10 +1650,6 @@ static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd, kvm_xen_start_timer(vcpu, oneshot.timeout_abs_ns, false); } else { - if (vcpu->arch.xen.vcpu_id != vcpu_id) { - *r = -EINVAL; - return true; - } kvm_xen_stop_timer(vcpu); } From c10bd49bdcc9cb5c1a9d7464989c3aa5e6feac1e Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 15:56:13 -0700 Subject: [PATCH 040/121] KVM: x86/xen: Punt singleshot timer hcalls to userspace if Xen vCPU ID isn't set Explicitly invalidate KVM's internal Xen vCPU ID during vCPU creation instead of *trying* to set the Xen ID to the vCPU index by default, and forward singleshot timer hypercalls to userspace if the VMM hasn't set the Xen ID via KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID. Using the vCPU's index as its default Xen ID is reasonable in concept, but in practice is horribly flawed as the index is left as '0' until after vCPU initialization completes, i.e. every vCPU gets a Xen ID of '0' by default. Forward hypercalls to userspace instead of trying to salvage any kind of default behavior, as all userspace implementations that support multiple vCPUs either don't enable the timer, are guaranteed to set Xen ID, or work only because *all* guests also screw up the singleshot timer hypercalls. The last scenarios is extremely unlikely given that Linux-as-a-guest uses the actual Xen vCPU ID when making timer hypercalls. In other words, for all intents and purposes, KVM's ABI is already that userspace must set the Xen vCPU ID, so just commit to that ABI. Note, KVM's handling of KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID restricts the ID to KVM_MAX_VCPUS, so there's no chance of a valid ID colliding with U32_MAX. Add a compile-time assertion to ensure this holds true in the future (KVM doesn't care what value is used for "invalid", only that there can't be a collision). Link: https://lore.kernel.org/all/20260612233017.1F9771F000E9@smtp.kernel.org Suggested-by: David Woodhouse Reviewed-by: David Woodhouse Link: https://patch.msgid.link/20260630225619.511632-7-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/xen.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 7a0d89faca85..eef378d0bb45 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -1103,6 +1104,8 @@ int kvm_xen_vcpu_set_attr(struct kvm_vcpu *vcpu, struct kvm_xen_vcpu_attr *data) break; case KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID: + BUILD_BUG_ON(XEN_VCPU_ID_INVALID < KVM_MAX_VCPUS); + if (data->u.vcpu_id >= KVM_MAX_VCPUS) r = -EINVAL; else { @@ -1614,6 +1617,9 @@ static bool kvm_xen_hcall_vcpu_op(struct kvm_vcpu *vcpu, bool longmode, int cmd, if (!kvm_xen_timer_enabled(vcpu)) return false; + if (vcpu->arch.xen.vcpu_id == XEN_VCPU_ID_INVALID) + return false; + /* * Reject the hypercall if the guest is trying to start/stop the timer * for a different vCPU. Xen per-vCPU hypercalls take a target vCPU as @@ -2300,7 +2306,7 @@ static bool kvm_xen_hcall_evtchn_send(struct kvm_vcpu *vcpu, u64 param, u64 *r) void kvm_xen_init_vcpu(struct kvm_vcpu *vcpu) { - vcpu->arch.xen.vcpu_id = vcpu->vcpu_idx; + vcpu->arch.xen.vcpu_id = XEN_VCPU_ID_INVALID; vcpu->arch.xen.poll_evtchn = 0; timer_setup(&vcpu->arch.xen.poll_timer, cancel_evtchn_poll, 0); From b29125ead04d37558f4c0e1c3ceb75d74837b67a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 15:56:14 -0700 Subject: [PATCH 041/121] KVM: Initialize a vCPU's index to '-1' while it's being created MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Invalidate a vCPU's index immediately after allocating storage for the vCPU so that KVM doesn't incorrectly treat a vCPU that is the process of being created as being vCPU0. This will also allow detecting that a vCPU is in the process of being created and thus otherwise unreachable, which is useful for avoiding false positives in lockdep assertions on vcpu->mutex. Unwind the index back to -1 if inserting the vCPU into the array or adding the vCPU to the fd table fails, so that kvm_arch_vcpu_destroy() sees the vCPU as unreachable, i.e. so that teardown logic doesn't hit false positive lockdep assertions. Opportunistically add a comment to call out that the "real" index needs to be set before making the vCPU visible to other tasks. Note, kvm_wait_for_vcpu_online() naturally does the right thing thanks to vcpu->vcpu_idx and kvm->online_vcpus being signed values. Reviewed-by: Philippe Mathieu-Daudé Link: https://patch.msgid.link/20260630225619.511632-8-seanjc@google.com Signed-off-by: Sean Christopherson --- virt/kvm/kvm_main.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index e44c20c04961..05275d318bfb 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -4188,6 +4188,8 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) goto vcpu_decrement; } + vcpu->vcpu_idx = -1; + BUILD_BUG_ON(sizeof(struct kvm_run) > PAGE_SIZE); page = alloc_page(GFP_KERNEL_ACCOUNT | __GFP_ZERO); if (!page) { @@ -4216,6 +4218,11 @@ static int kvm_vm_ioctl_create_vcpu(struct kvm *kvm, unsigned long id) goto unlock_vcpu_destroy; } + /* + * Set the vCPU's index *before* the vCPU is reachable by other tasks. + * Unwind the index back to -1 on failure so that KVM can use the index + * to detect that the vCPU is unreachable, e.g. for lockdep asserts. + */ vcpu->vcpu_idx = atomic_read(&kvm->online_vcpus); r = xa_insert(&kvm->vcpu_array, vcpu->vcpu_idx, vcpu, GFP_KERNEL_ACCOUNT); WARN_ON_ONCE(r == -EBUSY); @@ -4254,6 +4261,7 @@ kvm_put_xa_erase: kvm_put_kvm_no_destroy(kvm); xa_erase(&kvm->vcpu_array, vcpu->vcpu_idx); unlock_vcpu_destroy: + vcpu->vcpu_idx = -1; mutex_unlock(&kvm->lock); kvm_dirty_ring_free(&vcpu->dirty_ring); arch_vcpu_destroy: From e34be29ecd47f5dcc27a90bf1563ae1da96555a7 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 15:56:15 -0700 Subject: [PATCH 042/121] KVM: Move nVMX's lockdep logic for vcpu->mutex to a common helper MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract nVMX's lockdep assertion that a vCPU is locked or otherwise unreachable into a common helper, as KVM x86 is about to gain another user, but there is nothing x86-specific about the logic, i.e. the assertion may be useful for other architectures. No functional change intended. Reviewed-by: Philippe Mathieu-Daudé Link: https://patch.msgid.link/20260630225619.511632-9-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/nested.h | 6 ++---- include/linux/kvm_host.h | 6 ++++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.h b/arch/x86/kvm/vmx/nested.h index 6d6cd5904ddf..c6de848bd9ce 100644 --- a/arch/x86/kvm/vmx/nested.h +++ b/arch/x86/kvm/vmx/nested.h @@ -57,16 +57,14 @@ bool nested_vmx_check_io_bitmaps(struct kvm_vcpu *vcpu, unsigned int port, static inline struct vmcs12 *get_vmcs12(struct kvm_vcpu *vcpu) { - lockdep_assert_once(lockdep_is_held(&vcpu->mutex) || - !refcount_read(&vcpu->kvm->users_count)); + kvm_lockdep_assert_vcpu_is_locked_or_unreachable(vcpu); return to_vmx(vcpu)->nested.cached_vmcs12; } static inline struct vmcs12 *get_shadow_vmcs12(struct kvm_vcpu *vcpu) { - lockdep_assert_once(lockdep_is_held(&vcpu->mutex) || - !refcount_read(&vcpu->kvm->users_count)); + kvm_lockdep_assert_vcpu_is_locked_or_unreachable(vcpu); return to_vmx(vcpu)->nested.cached_shadow_vmcs12; } diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index ab8cfaec82d3..b10814f99a50 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -989,6 +989,12 @@ static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx) lockdep_is_held(&kvm->slots_lock)); } +static inline void kvm_lockdep_assert_vcpu_is_locked_or_unreachable(struct kvm_vcpu *vcpu) +{ + lockdep_assert_once(lockdep_is_held(&vcpu->mutex) || + !refcount_read(&vcpu->kvm->users_count)); +} + static inline struct kvm_vcpu *kvm_get_vcpu(struct kvm *kvm, int i) { int num_vcpus = atomic_read(&kvm->online_vcpus); From c33aef581703cb41e2a4900c258018bc7aa1ff54 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 15:56:16 -0700 Subject: [PATCH 043/121] KVM: x86: Treat a vCPU as unreachable if its index is invalid MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the "vCPU locked or unreachable" lockdep assertion, treat a vCPU as unreachable if its index is invalid, i.e. if the vCPU is in the process of being created. Until the vCPU is inserted into the array of vCPUs, the only way to get at the vCPU is via kvm_vm_ioctl_create_vcpu(). Note, the actual index is set _before_ adding the vCPU to the array, i.e. there's no risk of a false negative on the lockdep assertion. Reviewed-by: Philippe Mathieu-Daudé Link: https://patch.msgid.link/20260630225619.511632-10-seanjc@google.com Signed-off-by: Sean Christopherson --- include/linux/kvm_host.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index b10814f99a50..0bdfa3699352 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -992,6 +992,7 @@ static inline struct kvm_io_bus *kvm_get_bus(struct kvm *kvm, enum kvm_bus idx) static inline void kvm_lockdep_assert_vcpu_is_locked_or_unreachable(struct kvm_vcpu *vcpu) { lockdep_assert_once(lockdep_is_held(&vcpu->mutex) || + vcpu->vcpu_idx < 0 || !refcount_read(&vcpu->kvm->users_count)); } From 53ce2c773f0007bd3c74deb45049e152fcd88f2f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 15:56:17 -0700 Subject: [PATCH 044/121] KVM: x86/hyperv: Assert vCPU's mutex is held in to_hv_vcpu() Assert that either vcpu->mutex is held or the VM is otherwise unreachable when using the normal vCPU => HyperV accessor to help detect improper cross-task usage of the HyperV structure. When accessing the structure without holding the vCPU's mutex, e.g. to send interrupts or to queue TLB flushes, KVM needs to use the more paranoid to_hv_vcpu_safe() to guarantee that it can't see a half-baked structure. To avoid false positives, open code accesses to vcpu->arch.hyperv in the Synthetic Timer callbacks (can be reached if and only if HyperV state is fully initialized). Link: https://patch.msgid.link/20260630225619.511632-11-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/hyperv.c | 6 ++---- arch/x86/kvm/hyperv.h | 2 ++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 888526ce4dab..f765c3bb9b1f 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -599,8 +599,7 @@ static void stimer_mark_pending(struct kvm_vcpu_hv_stimer *stimer, { struct kvm_vcpu *vcpu = hv_stimer_to_vcpu(stimer); - set_bit(stimer->index, - to_hv_vcpu(vcpu)->stimer_pending_bitmap); + set_bit(stimer->index, vcpu->arch.hyperv->stimer_pending_bitmap); kvm_make_request(KVM_REQ_HV_STIMER, vcpu); if (vcpu_kick) kvm_vcpu_kick(vcpu); @@ -614,8 +613,7 @@ static void stimer_cleanup(struct kvm_vcpu_hv_stimer *stimer) stimer->index); hrtimer_cancel(&stimer->timer); - clear_bit(stimer->index, - to_hv_vcpu(vcpu)->stimer_pending_bitmap); + clear_bit(stimer->index, vcpu->arch.hyperv->stimer_pending_bitmap); stimer->msg_pending = false; stimer->exp_time = 0; } diff --git a/arch/x86/kvm/hyperv.h b/arch/x86/kvm/hyperv.h index ea9c81d76dd3..37a0bcf03e28 100644 --- a/arch/x86/kvm/hyperv.h +++ b/arch/x86/kvm/hyperv.h @@ -76,6 +76,8 @@ static inline struct kvm_vcpu_hv *to_hv_vcpu_safe(struct kvm_vcpu *vcpu) static inline struct kvm_vcpu_hv *to_hv_vcpu(struct kvm_vcpu *vcpu) { + kvm_lockdep_assert_vcpu_is_locked_or_unreachable(vcpu); + return vcpu->arch.hyperv; } From 09dd4de361dcb60a4097a4483d0203954c9755c3 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 15:56:18 -0700 Subject: [PATCH 045/121] KVM: x86/hyperv: Use {READ,WRITE}_ONCE for cross-task synic->active accesses When activating Hyper-V's Synthetic Interrupt Controller (SynIC), mark it active with WRITE_ONCE() and query it using READ_ONCE() in synic_get(), the only known cross-task reader, to document that the flag is accessed without holding the vCPU's mutex. Note, there are no data dependencies on the SynIC being marked active, e.g. the vector read by synic_set_irq() is set (usually in response to guest activity) long after the SynIC is initially activated, and a false negative on the SynIC being active would be benign (ignoring that such a race is likely to be problematic for the guest irrespective of what KVM does). Link: https://patch.msgid.link/20260630225619.511632-12-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/hyperv.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index f765c3bb9b1f..9d38cb644668 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -219,7 +219,7 @@ static struct kvm_vcpu_hv_synic *synic_get(struct kvm *kvm, u32 vpidx) return NULL; synic = &hv_vcpu->synic; - return (synic->active) ? synic : NULL; + return READ_ONCE(synic->active) ? synic : NULL; } static void kvm_hv_notify_acked_sint(struct kvm_vcpu *vcpu, u32 sint) @@ -1013,7 +1013,7 @@ int kvm_hv_activate_synic(struct kvm_vcpu *vcpu, bool dont_zero_synic_pages) synic = to_hv_synic(vcpu); - synic->active = true; + WRITE_ONCE(synic->active, true); synic->dont_zero_synic_pages = dont_zero_synic_pages; synic->control = HV_SYNIC_CONTROL_ENABLE; return 0; From b31e7c24b475128de7d685c294c34c09ff802258 Mon Sep 17 00:00:00 2001 From: leixiang Date: Mon, 6 Jul 2026 17:59:06 +0800 Subject: [PATCH 046/121] KVM: Remove kvm_debugfs_dir on kvm_init() error paths kvm_init_debug() runs before several steps that can fail (kvm_vfio_ops_init(), kvm_gmem_init(), kvm_init_virtualization() and misc_register()), but none of the corresponding error labels remove the "kvm" debugfs directory. Any failure after kvm_init_debug() therefore leaks the directory and its stat files for the lifetime of the boot. kvm_exit() already calls debugfs_remove_recursive(kvm_debugfs_dir); add the same at the err_vfio label, whose fall-through covers every path taken after kvm_init_debug(). Fixes: 2b0128127373 ("KVM: Register /dev/kvm as the _very_ last thing during initialization") Signed-off-by: leixiang Link: https://patch.msgid.link/20260706095910.39798-1-leixiang@kylinos.cn Signed-off-by: Sean Christopherson --- virt/kvm/kvm_main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 05275d318bfb..2df8ee9ecf6c 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -6567,6 +6567,7 @@ err_virt: err_gmem: kvm_vfio_ops_exit(); err_vfio: + debugfs_remove_recursive(kvm_debugfs_dir); kvm_async_pf_deinit(); err_async_pf: kvm_irqfd_exit(); From 26c877c70b57c5c76f7c8ed0f5cf40931490b3eb Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 25 Jun 2026 15:04:42 -0700 Subject: [PATCH 047/121] KVM: x86: Move the "APIC attention" macros from kvm_host.h => lapic.c Move the macros that define the mostly-obsolete apic_attention bits into lapic.c, as the gory details of PV EOIs and the pre-APICv TPR acceleration are 100% internal to KVM's local APIC emulation. No functional change intended. Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260625220450.3354415-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 10 ---------- arch/x86/kvm/lapic.c | 10 ++++++++++ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index b517257a6315..9ba8aa739f93 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -290,16 +290,6 @@ enum x86_intercept_stage; #define PFERR_PRIVATE_ACCESS BIT_ULL(49) #define PFERR_SYNTHETIC_MASK (PFERR_IMPLICIT_ACCESS | PFERR_PRIVATE_ACCESS) -/* apic attention bits */ -#define KVM_APIC_CHECK_VAPIC 0 -/* - * The following bit is set with PV-EOI, unset on EOI. - * We detect PV-EOI changes by guest by comparing - * this bit with PV-EOI in guest memory. - * See the implementation in apic_update_pv_eoi. - */ -#define KVM_APIC_PV_EOI_PENDING 1 - struct kvm_kernel_irqfd; struct kvm_kernel_irq_routing_entry; diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 6f30bbdddb5a..0354db0f2c0f 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -75,6 +75,16 @@ module_param(lapic_timer_advance, bool, 0444); /* step-by-step approximation to mitigate fluctuation */ #define LAPIC_TIMER_ADVANCE_ADJUST_STEP 8 +/* apic attention bits */ +#define KVM_APIC_CHECK_VAPIC 0 +/* + * The following bit is set with PV-EOI, unset on EOI. + * We detect PV-EOI changes by guest by comparing + * this bit with PV-EOI in guest memory. + * See the implementation in apic_update_pv_eoi. + */ +#define KVM_APIC_PV_EOI_PENDING 1 + static bool __read_mostly vector_hashing_enabled = true; module_param_named(vector_hashing, vector_hashing_enabled, bool, 0444); From eb4b67c93472cee74b7093d51adae51be83aaab1 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 25 Jun 2026 15:04:43 -0700 Subject: [PATCH 048/121] KVM: x86/mmu: Annotate tdp_enabled as being read-mostly Tag tdp_enabled with __read_mostly as the variable is only ever written during vendor module load, same as all the other global MMU variables that are handled by kvm_configure_mmu(). Opportunistically annotate the tdp_mmu_enabled and eager_page_split declarations with __read_mostly, to match their definitions. The compiler will warn if there are conflicting annotations, i.e. there's minimal risk of the declaration annotation becoming stale. No functional change intended. Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260625220450.3354415-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu.h | 6 +++--- arch/x86/kvm/mmu/mmu.c | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index c9f628b97dae..85bc503de1b6 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -6,14 +6,14 @@ #include "regs.h" #include "cpuid.h" -extern bool tdp_enabled; +extern bool __read_mostly tdp_enabled; #ifdef CONFIG_X86_64 -extern bool tdp_mmu_enabled; +extern bool __read_mostly tdp_mmu_enabled; #else #define tdp_mmu_enabled false #endif extern bool __read_mostly enable_mmio_caching; -extern bool eager_page_split; +extern bool __read_mostly eager_page_split; #define KVM_MEMSLOT_PAGES_TO_MMU_PAGES_RATIO 50 #define KVM_MIN_ALLOC_MMU_PAGES 64UL diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 6c13da942bfc..af3820136f7c 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -104,7 +104,7 @@ module_param_named(flush_on_reuse, force_flush_and_sync_on_reuse, bool, 0644); * 2. while doing 1. it walks guest-physical to host-physical * If the hardware supports that we don't need to do shadow paging. */ -bool tdp_enabled = false; +bool __read_mostly tdp_enabled = false; static bool __ro_after_init tdp_mmu_allowed; From 8821646e292da5631bb4e7685453aa5ae3fe6464 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 25 Jun 2026 15:04:44 -0700 Subject: [PATCH 049/121] KVM: x86: Pluralize the macro guard name for msrs.h Add an 'S' to msrs.h's macro guard so that both the file and guard names are plural. No functional change intended. Fixes: 7a2683080158 ("KVM: x86: Move the bulk of MSR specific code from x86.c to msrs.{c,h}") Reported-by: Binbin Wu Closes: https://lore.kernel.org/all/ead7d7fd-aa4e-4c18-b399-90fb448e0af6@linux.intel.com Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260625220450.3354415-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/msrs.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/msrs.h b/arch/x86/kvm/msrs.h index b698983e37fb..9c5c6b33e58f 100644 --- a/arch/x86/kvm/msrs.h +++ b/arch/x86/kvm/msrs.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ -#ifndef ARCH_X86_KVM_MSR_H -#define ARCH_X86_KVM_MSR_H +#ifndef ARCH_X86_KVM_MSRS_H +#define ARCH_X86_KVM_MSRS_H #include #include From eb42c91b6546be7e6c5bb5c548dc7481a231b67b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 25 Jun 2026 15:04:45 -0700 Subject: [PATCH 050/121] KVM: x86: Move CR and DR macro definitions from kvm_host.h => regs.h Relocate a variety of Control/Debug Register macros that unintentionally got left behind when the related helper function prototypes were moved to regs.h. No functional change intended. Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260625220450.3354415-5-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 40 --------------------------------- arch/x86/kvm/regs.h | 38 +++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 40 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 9ba8aa739f93..fd712f604636 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -125,24 +125,6 @@ #define KVM_REQ_UPDATE_PROTECTED_GUEST_STATE \ KVM_ARCH_REQ_FLAGS(34, KVM_REQUEST_WAIT) -#define CR0_RESERVED_BITS \ - (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ - | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \ - | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG)) - -#define CR4_RESERVED_BITS \ - (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\ - | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \ - | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR | X86_CR4_PCIDE \ - | X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_FSGSBASE \ - | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_VMXE \ - | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP \ - | X86_CR4_LAM_SUP | X86_CR4_CET)) - -#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) - - - #define INVALID_PAGE (~(hpa_t)0) #define VALID_PAGE(x) ((x) != INVALID_PAGE) @@ -230,28 +212,6 @@ enum x86_intercept_stage; #define KVM_NR_DB_REGS 4 -#define DR6_BUS_LOCK (1 << 11) -#define DR6_BD (1 << 13) -#define DR6_BS (1 << 14) -#define DR6_BT (1 << 15) -#define DR6_RTM (1 << 16) -/* - * DR6_ACTIVE_LOW combines fixed-1 and active-low bits. - * We can regard all the bits in DR6_FIXED_1 as active_low bits; - * they will never be 0 for now, but when they are defined - * in the future it will require no code change. - * - * DR6_ACTIVE_LOW is also used as the init/reset value for DR6. - */ -#define DR6_ACTIVE_LOW 0xffff0ff0 -#define DR6_VOLATILE 0x0001e80f -#define DR6_FIXED_1 (DR6_ACTIVE_LOW & ~DR6_VOLATILE) - -#define DR7_BP_EN_MASK 0x000000ff -#define DR7_GE (1 << 9) -#define DR7_GD (1 << 13) -#define DR7_VOLATILE 0xffff2bff - #define KVM_GUESTDBG_VALID_MASK \ (KVM_GUESTDBG_ENABLE | \ KVM_GUESTDBG_SINGLESTEP | \ diff --git a/arch/x86/kvm/regs.h b/arch/x86/kvm/regs.h index 94fd86728fed..447f0ec3e63e 100644 --- a/arch/x86/kvm/regs.h +++ b/arch/x86/kvm/regs.h @@ -16,6 +16,44 @@ static_assert(!(KVM_POSSIBLE_CR0_GUEST_BITS & X86_CR0_PDPTR_BITS)); +#define CR0_RESERVED_BITS \ + (~(unsigned long)(X86_CR0_PE | X86_CR0_MP | X86_CR0_EM | X86_CR0_TS \ + | X86_CR0_ET | X86_CR0_NE | X86_CR0_WP | X86_CR0_AM \ + | X86_CR0_NW | X86_CR0_CD | X86_CR0_PG)) + +#define CR4_RESERVED_BITS \ + (~(unsigned long)(X86_CR4_VME | X86_CR4_PVI | X86_CR4_TSD | X86_CR4_DE\ + | X86_CR4_PSE | X86_CR4_PAE | X86_CR4_MCE \ + | X86_CR4_PGE | X86_CR4_PCE | X86_CR4_OSFXSR | X86_CR4_PCIDE \ + | X86_CR4_OSXSAVE | X86_CR4_SMEP | X86_CR4_FSGSBASE \ + | X86_CR4_OSXMMEXCPT | X86_CR4_LA57 | X86_CR4_VMXE \ + | X86_CR4_SMAP | X86_CR4_PKE | X86_CR4_UMIP \ + | X86_CR4_LAM_SUP | X86_CR4_CET)) + +#define CR8_RESERVED_BITS (~(unsigned long)X86_CR8_TPR) + +#define DR6_BUS_LOCK (1 << 11) +#define DR6_BD (1 << 13) +#define DR6_BS (1 << 14) +#define DR6_BT (1 << 15) +#define DR6_RTM (1 << 16) +/* + * DR6_ACTIVE_LOW combines fixed-1 and active-low bits. + * We can regard all the bits in DR6_FIXED_1 as active_low bits; + * they will never be 0 for now, but when they are defined + * in the future it will require no code change. + * + * DR6_ACTIVE_LOW is also used as the init/reset value for DR6. + */ +#define DR6_ACTIVE_LOW 0xffff0ff0 +#define DR6_VOLATILE 0x0001e80f +#define DR6_FIXED_1 (DR6_ACTIVE_LOW & ~DR6_VOLATILE) + +#define DR7_BP_EN_MASK 0x000000ff +#define DR7_GE (1 << 9) +#define DR7_GD (1 << 13) +#define DR7_VOLATILE 0xffff2bff + void kvm_post_set_cr0(struct kvm_vcpu *vcpu, unsigned long old_cr0, unsigned long cr0); void kvm_post_set_cr4(struct kvm_vcpu *vcpu, unsigned long old_cr4, unsigned long cr4); int kvm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0); From eb7313b66c59b4b8c135d3eb01f68268d4070594 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 25 Jun 2026 15:04:46 -0700 Subject: [PATCH 051/121] KVM: x86: Move KVM_GUESTDBG_VALID_MASK from kvm_host.h => x86.c Move KVM_GUESTDBG_VALID_MASK into x86.c so that it's not globally visible. As explained by commit 462474588b19 ("KVM: x86: Move misc "VALID MASK" defines from kvm_host.h => x86.c"), which unintentionally missed GUESTDBG, the set of valid flags/bits is very much a KVM-internal detail, as the values from the hardcoded #defines are often captured and massaged by KVM's setup code, i.e. *directly* using the macros outside of KVM x86 would be actively dangerous. No functional change intended. Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260625220450.3354415-6-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 9 --------- arch/x86/kvm/x86.c | 9 +++++++++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index fd712f604636..f24e0de00692 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -212,15 +212,6 @@ enum x86_intercept_stage; #define KVM_NR_DB_REGS 4 -#define KVM_GUESTDBG_VALID_MASK \ - (KVM_GUESTDBG_ENABLE | \ - KVM_GUESTDBG_SINGLESTEP | \ - KVM_GUESTDBG_USE_HW_BP | \ - KVM_GUESTDBG_USE_SW_BP | \ - KVM_GUESTDBG_INJECT_BP | \ - KVM_GUESTDBG_INJECT_DB | \ - KVM_GUESTDBG_BLOCKIRQ) - #define PFERR_PRESENT_MASK BIT(0) #define PFERR_WRITE_MASK BIT(1) #define PFERR_USER_MASK BIT(2) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 0626e835e9eb..5ee6d0c33009 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -115,6 +115,15 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_host); #define KVM_CAP_PMU_VALID_MASK KVM_PMU_CAP_DISABLE +#define KVM_GUESTDBG_VALID_MASK \ + (KVM_GUESTDBG_ENABLE | \ + KVM_GUESTDBG_SINGLESTEP | \ + KVM_GUESTDBG_USE_HW_BP | \ + KVM_GUESTDBG_USE_SW_BP | \ + KVM_GUESTDBG_INJECT_BP | \ + KVM_GUESTDBG_INJECT_DB | \ + KVM_GUESTDBG_BLOCKIRQ) + #define KVM_X2APIC_API_VALID_FLAGS (KVM_X2APIC_API_USE_32BIT_IDS | \ KVM_X2APIC_API_DISABLE_BROADCAST_QUIRK | \ KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST | \ From de28ef6548a25bfb82ff0ea390821752ee7fb629 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 25 Jun 2026 15:04:47 -0700 Subject: [PATCH 052/121] KVM: x86: Add static asserts to document connection b/w TSS structs and macros Add static asserts to sanity check the I/O permission map and TSS size macros against tss_segment_32. Alternatively, the macros could simply use offsetof() and sizeof(), but having literal numbers makes it easier to understand the bigger picture, and provides a good excuse for the sanity checks. Opportunistically add the necessary includes to make tss.h self sufficient. No functional change intended. Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260625220450.3354415-7-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/tss.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/arch/x86/kvm/tss.h b/arch/x86/kvm/tss.h index 117bf8bec07d..55ced8975840 100644 --- a/arch/x86/kvm/tss.h +++ b/arch/x86/kvm/tss.h @@ -2,6 +2,10 @@ #ifndef __TSS_SEGMENT_H #define __TSS_SEGMENT_H +#include +#include +#include + struct tss_segment_32 { u32 prev_task_link; u32 esp0; @@ -64,4 +68,7 @@ struct tss_segment_16 { #define RMODE_TSS_SIZE \ (TSS_BASE_SIZE + TSS_REDIRECTION_SIZE + TSS_IOPB_SIZE + 1) +static_assert(offsetof(struct tss_segment_32, io_map) == TSS_IOPB_BASE_OFFSET); +static_assert(sizeof(struct tss_segment_32) == TSS_BASE_SIZE); + #endif From 0bc5f998e18d295a168408e35fe34e21c4a74178 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 25 Jun 2026 15:04:48 -0700 Subject: [PATCH 053/121] KVM: x86: Move KVM's arbitrary task switch reason enums to x86.h Relocate KVM's TASK_SWITCH_ enums from kvm_host.h to x86.h, as the enums are arbitrary values, i.e. not architectural, and are intended to be used only to translate vendor specific information to a common x86 reason when invoking kvm_task_switch(). Opportunistically name the overall enum to help document the role of the values. No functional change intended. Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260625220450.3354415-8-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 7 ------- arch/x86/kvm/x86.h | 6 ++++++ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index f24e0de00692..c7d53d46763b 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1960,13 +1960,6 @@ static inline unsigned long read_msr(unsigned long msr) } #endif -enum { - TASK_SWITCH_CALL = 0, - TASK_SWITCH_IRET = 1, - TASK_SWITCH_JMP = 2, - TASK_SWITCH_GATE = 3, -}; - #define HF_GUEST_MASK (1 << 0) /* VCPU is in guest-mode */ #ifdef CONFIG_KVM_SMM diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 8ece468087a8..494d97e9a9c9 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -482,6 +482,12 @@ int kvm_emulate_wbinvd(struct kvm_vcpu *vcpu); void kvm_vcpu_deliver_sipi_vector(struct kvm_vcpu *vcpu, u8 vector); +enum kvm_task_switch_reason { + TASK_SWITCH_CALL = 0, + TASK_SWITCH_IRET = 1, + TASK_SWITCH_JMP = 2, + TASK_SWITCH_GATE = 3, +}; int kvm_task_switch(struct kvm_vcpu *vcpu, u16 tss_selector, int idt_index, int reason, bool has_error_code, u32 error_code); From c3d35340a3379d6567bfad6559f1f77acf7769fb Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 25 Jun 2026 15:04:49 -0700 Subject: [PATCH 054/121] KVM: x86: Move "struct kvm_apic_map" definition from kvm_host.h => lapic.h Move the definition of "struct kvm_apic_map", a.k.a. the optimized local APIC map, to lapic.h, as it is very nearly an implementation details that's internal to KVM's local APIC emulation (KVM also uses the map to do quick lookups when a vCPU is yielding to a different vCPU). No functional change intended. Suggested-by: Kai Huang Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260625220450.3354415-9-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 35 ++------------------------------- arch/x86/kvm/lapic.h | 33 +++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 33 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index c7d53d46763b..90efc3c90b41 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -244,6 +244,8 @@ enum x86_intercept_stage; struct kvm_kernel_irqfd; struct kvm_kernel_irq_routing_entry; +struct kvm_apic_map; + struct kvm_x86_msr_filter; struct kvm_x86_pmu_event_filter; @@ -1122,39 +1124,6 @@ struct kvm_arch_memory_slot { unsigned short *gfn_write_track; }; -/* - * Track the mode of the optimized logical map, as the rules for decoding the - * destination vary per mode. Enabling the optimized logical map requires all - * software-enabled local APIs to be in the same mode, each addressable APIC to - * be mapped to only one MDA, and each MDA to map to at most one APIC. - */ -enum kvm_apic_logical_mode { - /* All local APICs are software disabled. */ - KVM_APIC_MODE_SW_DISABLED, - /* All software enabled local APICs in xAPIC cluster addressing mode. */ - KVM_APIC_MODE_XAPIC_CLUSTER, - /* All software enabled local APICs in xAPIC flat addressing mode. */ - KVM_APIC_MODE_XAPIC_FLAT, - /* All software enabled local APICs in x2APIC mode. */ - KVM_APIC_MODE_X2APIC, - /* - * Optimized map disabled, e.g. not all local APICs in the same logical - * mode, same logical ID assigned to multiple APICs, etc. - */ - KVM_APIC_MODE_MAP_DISABLED, -}; - -struct kvm_apic_map { - struct rcu_head rcu; - enum kvm_apic_logical_mode logical_mode; - u32 max_apic_id; - union { - struct kvm_lapic *xapic_flat_map[8]; - struct kvm_lapic *xapic_cluster_map[16][4]; - }; - struct kvm_lapic *phys_map[]; -}; - /* Hyper-V synthetic debugger (SynDbg)*/ struct kvm_hv_syndbg { struct { diff --git a/arch/x86/kvm/lapic.h b/arch/x86/kvm/lapic.h index 58dbb94f980d..bd1098c89d99 100644 --- a/arch/x86/kvm/lapic.h +++ b/arch/x86/kvm/lapic.h @@ -32,6 +32,39 @@ enum lapic_mode { LAPIC_MODE_X2APIC = MSR_IA32_APICBASE_ENABLE | X2APIC_ENABLE, }; +/* + * Track the mode of the optimized logical map, as the rules for decoding the + * destination vary per mode. Enabling the optimized logical map requires all + * software-enabled local APIs to be in the same mode, each addressable APIC to + * be mapped to only one MDA, and each MDA to map to at most one APIC. + */ +enum kvm_apic_logical_mode { + /* All local APICs are software disabled. */ + KVM_APIC_MODE_SW_DISABLED, + /* All software enabled local APICs in xAPIC cluster addressing mode. */ + KVM_APIC_MODE_XAPIC_CLUSTER, + /* All software enabled local APICs in xAPIC flat addressing mode. */ + KVM_APIC_MODE_XAPIC_FLAT, + /* All software enabled local APICs in x2APIC mode. */ + KVM_APIC_MODE_X2APIC, + /* + * Optimized map disabled, e.g. not all local APICs in the same logical + * mode, same logical ID assigned to multiple APICs, etc. + */ + KVM_APIC_MODE_MAP_DISABLED, +}; + +struct kvm_apic_map { + struct rcu_head rcu; + enum kvm_apic_logical_mode logical_mode; + u32 max_apic_id; + union { + struct kvm_lapic *xapic_flat_map[8]; + struct kvm_lapic *xapic_cluster_map[16][4]; + }; + struct kvm_lapic *phys_map[]; +}; + enum lapic_lvt_entry { LVT_TIMER, LVT_THERMAL_MONITOR, From ed6ee602bcebd475f7e8b5213d51e8b58b143973 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 25 Jun 2026 15:04:50 -0700 Subject: [PATCH 055/121] KVM: x86: Move "struct kvm_vcpu_hv" and all children from kvm_host.h => hyperv.h Move "struct kvm_vcpu_hv" and all of its child structures to hyperv.h, guarded by CONFIG_KVM_HYPERV=y, as "struct kvm_vcpu_arch" holds a pointer to the structure, i.e. only needs the structure to be declared, not fully defined. No functional change intended. Reviewed-by: Kai Huang Link: https://patch.msgid.link/20260625220450.3354415-10-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 91 +-------------------------------- arch/x86/kvm/hyperv.h | 90 ++++++++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 89 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 90efc3c90b41..fdb8953aeeb1 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -648,95 +648,6 @@ struct kvm_mtrr { u64 deftype; }; -/* Hyper-V SynIC timer */ -struct kvm_vcpu_hv_stimer { - struct hrtimer timer; - int index; - union hv_stimer_config config; - u64 count; - u64 exp_time; - struct hv_message msg; - bool msg_pending; -}; - -/* Hyper-V synthetic interrupt controller (SynIC)*/ -struct kvm_vcpu_hv_synic { - u64 version; - u64 control; - u64 msg_page; - u64 evt_page; - atomic64_t sint[HV_SYNIC_SINT_COUNT]; - atomic_t sint_to_gsi[HV_SYNIC_SINT_COUNT]; - DECLARE_BITMAP(auto_eoi_bitmap, 256); - DECLARE_BITMAP(vec_bitmap, 256); - bool active; - bool dont_zero_synic_pages; -}; - -/* The maximum number of entries on the TLB flush fifo. */ -#define KVM_HV_TLB_FLUSH_FIFO_SIZE (16) -/* - * Note: the following 'magic' entry is made up by KVM to avoid putting - * anything besides GVA on the TLB flush fifo. It is theoretically possible - * to observe a request to flush 4095 PFNs starting from 0xfffffffffffff000 - * which will look identical. KVM's action to 'flush everything' instead of - * flushing these particular addresses is, however, fully legitimate as - * flushing more than requested is always OK. - */ -#define KVM_HV_TLB_FLUSHALL_ENTRY ((u64)-1) - -enum hv_tlb_flush_fifos { - HV_L1_TLB_FLUSH_FIFO, - HV_L2_TLB_FLUSH_FIFO, - HV_NR_TLB_FLUSH_FIFOS, -}; - -struct kvm_vcpu_hv_tlb_flush_fifo { - spinlock_t write_lock; - DECLARE_KFIFO(entries, u64, KVM_HV_TLB_FLUSH_FIFO_SIZE); -}; - -/* Hyper-V per vcpu emulation context */ -struct kvm_vcpu_hv { - struct kvm_vcpu *vcpu; - u32 vp_index; - u64 hv_vapic; - s64 runtime_offset; - struct kvm_vcpu_hv_synic synic; - struct kvm_hyperv_exit exit; - struct kvm_vcpu_hv_stimer stimer[HV_SYNIC_STIMER_COUNT]; - DECLARE_BITMAP(stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT); - bool enforce_cpuid; - struct { - u32 features_eax; /* HYPERV_CPUID_FEATURES.EAX */ - u32 features_ebx; /* HYPERV_CPUID_FEATURES.EBX */ - u32 features_edx; /* HYPERV_CPUID_FEATURES.EDX */ - u32 enlightenments_eax; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EAX */ - u32 enlightenments_ebx; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EBX */ - u32 syndbg_cap_eax; /* HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES.EAX */ - u32 nested_eax; /* HYPERV_CPUID_NESTED_FEATURES.EAX */ - u32 nested_ebx; /* HYPERV_CPUID_NESTED_FEATURES.EBX */ - } cpuid_cache; - - struct kvm_vcpu_hv_tlb_flush_fifo tlb_flush_fifo[HV_NR_TLB_FLUSH_FIFOS]; - - /* - * Preallocated buffers for handling hypercalls that pass sparse vCPU - * sets (for high vCPU counts, they're too large to comfortably fit on - * the stack). - */ - u64 sparse_banks[HV_MAX_SPARSE_VCPU_BANKS]; - DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS); - - struct hv_vp_assist_page vp_assist_page; - - struct { - u64 pa_page_gpa; - u64 vm_id; - u32 vp_id; - } nested; -}; - struct kvm_hypervisor_cpuid { u32 base; u32 limit; @@ -767,6 +678,8 @@ struct kvm_vcpu_xen { }; #endif +struct kvm_vcpu_hv; + struct kvm_queued_exception { bool pending; bool injected; diff --git a/arch/x86/kvm/hyperv.h b/arch/x86/kvm/hyperv.h index 37a0bcf03e28..622a6553e9ac 100644 --- a/arch/x86/kvm/hyperv.h +++ b/arch/x86/kvm/hyperv.h @@ -27,6 +27,96 @@ #ifdef CONFIG_KVM_HYPERV + +/* Hyper-V SynIC timer */ +struct kvm_vcpu_hv_stimer { + struct hrtimer timer; + int index; + union hv_stimer_config config; + u64 count; + u64 exp_time; + struct hv_message msg; + bool msg_pending; +}; + +/* Hyper-V synthetic interrupt controller (SynIC)*/ +struct kvm_vcpu_hv_synic { + u64 version; + u64 control; + u64 msg_page; + u64 evt_page; + atomic64_t sint[HV_SYNIC_SINT_COUNT]; + atomic_t sint_to_gsi[HV_SYNIC_SINT_COUNT]; + DECLARE_BITMAP(auto_eoi_bitmap, 256); + DECLARE_BITMAP(vec_bitmap, 256); + bool active; + bool dont_zero_synic_pages; +}; + +/* The maximum number of entries on the TLB flush fifo. */ +#define KVM_HV_TLB_FLUSH_FIFO_SIZE (16) +/* + * Note: the following 'magic' entry is made up by KVM to avoid putting + * anything besides GVA on the TLB flush fifo. It is theoretically possible + * to observe a request to flush 4095 PFNs starting from 0xfffffffffffff000 + * which will look identical. KVM's action to 'flush everything' instead of + * flushing these particular addresses is, however, fully legitimate as + * flushing more than requested is always OK. + */ +#define KVM_HV_TLB_FLUSHALL_ENTRY ((u64)-1) + +enum hv_tlb_flush_fifos { + HV_L1_TLB_FLUSH_FIFO, + HV_L2_TLB_FLUSH_FIFO, + HV_NR_TLB_FLUSH_FIFOS, +}; + +struct kvm_vcpu_hv_tlb_flush_fifo { + spinlock_t write_lock; + DECLARE_KFIFO(entries, u64, KVM_HV_TLB_FLUSH_FIFO_SIZE); +}; + +/* Hyper-V per vcpu emulation context */ +struct kvm_vcpu_hv { + struct kvm_vcpu *vcpu; + u32 vp_index; + u64 hv_vapic; + s64 runtime_offset; + struct kvm_vcpu_hv_synic synic; + struct kvm_hyperv_exit exit; + struct kvm_vcpu_hv_stimer stimer[HV_SYNIC_STIMER_COUNT]; + DECLARE_BITMAP(stimer_pending_bitmap, HV_SYNIC_STIMER_COUNT); + bool enforce_cpuid; + struct { + u32 features_eax; /* HYPERV_CPUID_FEATURES.EAX */ + u32 features_ebx; /* HYPERV_CPUID_FEATURES.EBX */ + u32 features_edx; /* HYPERV_CPUID_FEATURES.EDX */ + u32 enlightenments_eax; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EAX */ + u32 enlightenments_ebx; /* HYPERV_CPUID_ENLIGHTMENT_INFO.EBX */ + u32 syndbg_cap_eax; /* HYPERV_CPUID_SYNDBG_PLATFORM_CAPABILITIES.EAX */ + u32 nested_eax; /* HYPERV_CPUID_NESTED_FEATURES.EAX */ + u32 nested_ebx; /* HYPERV_CPUID_NESTED_FEATURES.EBX */ + } cpuid_cache; + + struct kvm_vcpu_hv_tlb_flush_fifo tlb_flush_fifo[HV_NR_TLB_FLUSH_FIFOS]; + + /* + * Preallocated buffers for handling hypercalls that pass sparse vCPU + * sets (for high vCPU counts, they're too large to comfortably fit on + * the stack). + */ + u64 sparse_banks[HV_MAX_SPARSE_VCPU_BANKS]; + DECLARE_BITMAP(vcpu_mask, KVM_MAX_VCPUS); + + struct hv_vp_assist_page vp_assist_page; + + struct { + u64 pa_page_gpa; + u64 vm_id; + u32 vp_id; + } nested; +}; + /* "Hv#1" signature */ #define HYPERV_CPUID_SIGNATURE_EAX 0x31237648 From db3a46e200df2f65aec7d1f0a076c99bc8484ea0 Mon Sep 17 00:00:00 2001 From: leixiang Date: Mon, 6 Jul 2026 17:59:06 +0800 Subject: [PATCH 056/121] KVM: Remove kvm_debugfs_dir on kvm_init() error paths kvm_init_debug() runs before several steps that can fail (kvm_vfio_ops_init(), kvm_gmem_init(), kvm_init_virtualization() and misc_register()), but none of the corresponding error labels remove the "kvm" debugfs directory. Any failure after kvm_init_debug() therefore leaks the directory and its stat files for the lifetime of the boot. kvm_exit() already calls debugfs_remove_recursive(kvm_debugfs_dir); add the same at the err_vfio label, whose fall-through covers every path taken after kvm_init_debug(). Fixes: 2b0128127373 ("KVM: Register /dev/kvm as the _very_ last thing during initialization") Signed-off-by: leixiang Link: https://patch.msgid.link/20260706095910.39798-1-leixiang@kylinos.cn Signed-off-by: Sean Christopherson --- virt/kvm/kvm_main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index e44c20c04961..d4420ebfd972 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -6559,6 +6559,7 @@ err_virt: err_gmem: kvm_vfio_ops_exit(); err_vfio: + debugfs_remove_recursive(kvm_debugfs_dir); kvm_async_pf_deinit(); err_async_pf: kvm_irqfd_exit(); From 8b47740b3ae0faa29a8dd39a02021590f1c9728a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 14:01:54 -0700 Subject: [PATCH 057/121] KVM: SVM: Make kvm_x86_ops.vcpu_precreate() hook fully AVIC specific In anticipation of deferring all per-VM AVIC initialization until a vCPU is first created, move SVM's kvm_x86_ops.vcpu_precreate() hook into avic.c as avic_vcpu_precreate() and nullify the hook if AVIC is disabled (and WARN if the hook is somehow invoked without AVIC enabled). Reviewed-by: Naveen N Rao (AMD) Link: https://patch.msgid.link/20260630210156.457151-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/avic.c | 13 +++++++++---- arch/x86/kvm/svm/svm.c | 8 ++------ arch/x86/kvm/svm/svm.h | 2 +- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index 58e493a80cb0..4a0a2dbd1687 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -293,13 +293,10 @@ static int avic_get_physical_id_table_order(struct kvm *kvm) return get_order((__avic_get_max_physical_id(kvm, NULL) + 1) * sizeof(u64)); } -int avic_alloc_physical_id_table(struct kvm *kvm) +static int avic_alloc_physical_id_table(struct kvm *kvm) { struct kvm_svm *kvm_svm = to_kvm_svm(kvm); - if (!irqchip_in_kernel(kvm) || !enable_apicv) - return 0; - if (kvm_svm->avic_physical_id_table) return 0; @@ -311,6 +308,14 @@ int avic_alloc_physical_id_table(struct kvm *kvm) return 0; } +int avic_vcpu_precreate(struct kvm *kvm) +{ + if (!irqchip_in_kernel(kvm) || WARN_ON_ONCE(!enable_apicv)) + return 0; + + return avic_alloc_physical_id_table(kvm); +} + void avic_vm_destroy(struct kvm *kvm) { unsigned long flags; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index ef69a51ab27f..a7d141f7e76c 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1306,11 +1306,6 @@ void svm_switch_vmcb(struct vcpu_svm *svm, struct kvm_vmcb_info *target_vmcb) svm->vmcb = target_vmcb->ptr; } -static int svm_vcpu_precreate(struct kvm *kvm) -{ - return avic_alloc_physical_id_table(kvm); -} - static int svm_vcpu_create(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm; @@ -5333,7 +5328,7 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .emergency_disable_virtualization_cpu = svm_emergency_disable_virtualization_cpu, .has_emulated_msr = svm_has_emulated_msr, - .vcpu_precreate = svm_vcpu_precreate, + .vcpu_precreate = avic_vcpu_precreate, .vcpu_create = svm_vcpu_create, .vcpu_free = svm_vcpu_free, .vcpu_reset = svm_vcpu_reset, @@ -5712,6 +5707,7 @@ static __init int svm_hardware_setup(void) enable_apicv = avic_hardware_setup(); if (!enable_apicv) { enable_ipiv = false; + svm_x86_ops.vcpu_precreate = NULL; svm_x86_ops.vcpu_blocking = NULL; svm_x86_ops.vcpu_unblocking = NULL; svm_x86_ops.vcpu_get_apicv_inhibit_reasons = NULL; diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 716be21fba33..3c5459374969 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -945,7 +945,7 @@ extern struct kvm_x86_nested_ops svm_nested_ops; bool __init avic_hardware_setup(void); void avic_hardware_unsetup(void); -int avic_alloc_physical_id_table(struct kvm *kvm); +int avic_vcpu_precreate(struct kvm *kvm); void avic_vm_destroy(struct kvm *kvm); int avic_vm_init(struct kvm *kvm); void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb); From 4fa9a3767d1aa805659617cba1912d595752456b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 14:01:55 -0700 Subject: [PATCH 058/121] KVM: SVM: Do all per-VM AVIC initialization during vCPU precreation phase Move all per-VM AVIC initialization from VM creation to vCPU pre-creation, i.e. defer allocating the logical ID table and adding the VM to the GA Log list until vCPUs are created. This will allow removing the VM from the GA Log list before vCPUs are destroyed without needing yet another kvm_x86_ops hook (.vm_pre_destroy() is very intentionally called if and only if VM creation fully succeeds). As a bonus, this re-unites physical and logic table allocation, and avoids allocating a logical table in the unlikely scenario that userspace creates a VM without an in-kernel local APIC. Another bonus to hooking .vcpu_precreate() is that there is no need to unwind on failure, as the VM has already been created, i.e. KVM will run through all phases of VM destruction. In fact, unwinding is undesirable, as KVM tries to keep VM-wide behavior idempotent/sticky across creaton of multiple vCPUs. Reviewed-by: Naveen N Rao (AMD) Link: https://patch.msgid.link/20260630210156.457151-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/avic.c | 94 +++++++++++++++++++++++++---------------- arch/x86/kvm/svm/svm.c | 6 --- arch/x86/kvm/svm/svm.h | 1 - 3 files changed, 57 insertions(+), 44 deletions(-) diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index 4a0a2dbd1687..d71a2fed1a08 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -308,47 +308,32 @@ static int avic_alloc_physical_id_table(struct kvm *kvm) return 0; } -int avic_vcpu_precreate(struct kvm *kvm) +static int avic_alloc_logical_id_table(struct kvm *kvm) { - if (!irqchip_in_kernel(kvm) || WARN_ON_ONCE(!enable_apicv)) - return 0; - - return avic_alloc_physical_id_table(kvm); -} - -void avic_vm_destroy(struct kvm *kvm) -{ - unsigned long flags; struct kvm_svm *kvm_svm = to_kvm_svm(kvm); - if (!enable_apicv) - return; - - free_page((unsigned long)kvm_svm->avic_logical_id_table); - free_pages((unsigned long)kvm_svm->avic_physical_id_table, - avic_get_physical_id_table_order(kvm)); - - spin_lock_irqsave(&svm_vm_data_hash_lock, flags); - hash_del(&kvm_svm->hnode); - spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); -} - -int avic_vm_init(struct kvm *kvm) -{ - unsigned long flags; - int err = -ENOMEM; - struct kvm_svm *kvm_svm = to_kvm_svm(kvm); - struct kvm_svm *k2; - u32 vm_id; - - if (!enable_apicv) + if (kvm_svm->avic_logical_id_table) return 0; kvm_svm->avic_logical_id_table = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT); if (!kvm_svm->avic_logical_id_table) - goto free_avic; + return -ENOMEM; - spin_lock_irqsave(&svm_vm_data_hash_lock, flags); + return 0; +} + +static void avic_add_vm_to_ga_log_list(struct kvm *kvm) +{ + struct kvm_svm *kvm_svm = to_kvm_svm(kvm); + struct kvm_svm *k2; + u32 vm_id; + + lockdep_assert_held(&kvm->lock); + + if (kvm_svm->avic_vm_id) + return; + + guard(spinlock_irqsave)(&svm_vm_data_hash_lock); again: vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK; if (vm_id == 0) { /* id is 1-based, zero is not okay */ @@ -364,13 +349,48 @@ int avic_vm_init(struct kvm *kvm) } kvm_svm->avic_vm_id = vm_id; hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id); - spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); +} +int avic_vcpu_precreate(struct kvm *kvm) +{ + int r; + + if (!irqchip_in_kernel(kvm) || WARN_ON_ONCE(!enable_apicv)) + return 0; + + /* + * Don't unwind on failure, all actions must be idempotent with respect + * to creating multiple vCPUs, i.e. must persist until the VM is destroyed. + */ + r = avic_alloc_physical_id_table(kvm); + if (r) + return r; + + r = avic_alloc_logical_id_table(kvm); + if (r) + return r; + + avic_add_vm_to_ga_log_list(kvm); return 0; +} -free_avic: - avic_vm_destroy(kvm); - return err; +void avic_vm_destroy(struct kvm *kvm) +{ + unsigned long flags; + struct kvm_svm *kvm_svm = to_kvm_svm(kvm); + + if (!enable_apicv) + return; + + free_page((unsigned long)kvm_svm->avic_logical_id_table); + free_pages((unsigned long)kvm_svm->avic_physical_id_table, + avic_get_physical_id_table_order(kvm)); + + if (kvm_svm->avic_vm_id) { + spin_lock_irqsave(&svm_vm_data_hash_lock, flags); + hash_del(&kvm_svm->hnode); + spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); + } } static phys_addr_t avic_get_backing_page_address(struct vcpu_svm *svm) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index a7d141f7e76c..7f3a815d737f 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5297,12 +5297,6 @@ static int svm_vm_init(struct kvm *kvm) if (!pause_filter_count || !pause_filter_thresh) kvm_disable_exits(kvm, KVM_X86_DISABLE_EXITS_PAUSE); - if (enable_apicv) { - int ret = avic_vm_init(kvm); - if (ret) - return ret; - } - svm_srso_vm_init(); return 0; } diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 3c5459374969..b615f8563e8b 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -947,7 +947,6 @@ bool __init avic_hardware_setup(void); void avic_hardware_unsetup(void); int avic_vcpu_precreate(struct kvm *kvm); void avic_vm_destroy(struct kvm *kvm); -int avic_vm_init(struct kvm *kvm); void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb); int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu); int avic_unaccelerated_access_interception(struct kvm_vcpu *vcpu); From 78684b65fcc0582bf87e74229e3610c3d9634020 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 14:01:56 -0700 Subject: [PATCH 059/121] KVM: SVM: Remove VM from the GA Log notifier list before VM destruction When a VM is being destroyed, delete it from the list used to process GA Log interrupts before vCPUs are freed, otherwise avic_ga_log_notifier() could theoretically hit a use-after-free if a GA Log notification arrives for a vCPU after the last reference to the VM has been put. Note, in practice, it's likely all but impossible to trigger UAF, as all all irqfds and thus all IRTEs are cleaned up by: kvm_irqfd_release() | |-> irqfd_deactivate() | |-> irqfd_shutdown() | |-> irq_bypass_unregister_consumer() And kvm_irqfd_release() is guaranteed to run before the last reference to the VM is put. KVM also configures GA Log interrupts only when a vCPU is blocking (older versions of KVM configre GA Log interrupts at all times, but AVIC is off by default on those kernels). Hitting UAF would require tearing down a VM shortly after a vCPU stopped blocking, and with a very, very delayed IRQ from hardware. Opportunistically use guard() to avoid a local "flags" variable. Fixes: 5881f73757cc ("svm: Introduce AMD IOMMU avic_ga_log_notifier") Cc: Naveen N Rao (AMD) Cc: Xiao Wu Reviewed-by: Naveen N Rao (AMD) Link: https://patch.msgid.link/20260630210156.457151-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/avic.c | 19 ++++++++++++------- arch/x86/kvm/svm/svm.c | 2 ++ arch/x86/kvm/svm/svm.h | 1 + 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/svm/avic.c b/arch/x86/kvm/svm/avic.c index d71a2fed1a08..c5b1d294b15a 100644 --- a/arch/x86/kvm/svm/avic.c +++ b/arch/x86/kvm/svm/avic.c @@ -374,9 +374,20 @@ int avic_vcpu_precreate(struct kvm *kvm) return 0; } +void avic_vm_pre_destroy(struct kvm *kvm) +{ + struct kvm_svm *kvm_svm = to_kvm_svm(kvm); + + if (WARN_ON_ONCE(!enable_apicv) || !kvm_svm->avic_vm_id) + return; + + guard(spinlock_irqsave)(&svm_vm_data_hash_lock); + + hash_del(&kvm_svm->hnode); +} + void avic_vm_destroy(struct kvm *kvm) { - unsigned long flags; struct kvm_svm *kvm_svm = to_kvm_svm(kvm); if (!enable_apicv) @@ -385,12 +396,6 @@ void avic_vm_destroy(struct kvm *kvm) free_page((unsigned long)kvm_svm->avic_logical_id_table); free_pages((unsigned long)kvm_svm->avic_physical_id_table, avic_get_physical_id_table_order(kvm)); - - if (kvm_svm->avic_vm_id) { - spin_lock_irqsave(&svm_vm_data_hash_lock, flags); - hash_del(&kvm_svm->hnode); - spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags); - } } static phys_addr_t avic_get_backing_page_address(struct vcpu_svm *svm) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 7f3a815d737f..0e0dd9618750 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5329,6 +5329,7 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .vm_size = sizeof(struct kvm_svm), .vm_init = svm_vm_init, + .vm_pre_destroy = avic_vm_pre_destroy, .vm_destroy = svm_vm_destroy, .prepare_switch_to_guest = svm_prepare_switch_to_guest, @@ -5702,6 +5703,7 @@ static __init int svm_hardware_setup(void) if (!enable_apicv) { enable_ipiv = false; svm_x86_ops.vcpu_precreate = NULL; + svm_x86_ops.vm_pre_destroy = NULL; svm_x86_ops.vcpu_blocking = NULL; svm_x86_ops.vcpu_unblocking = NULL; svm_x86_ops.vcpu_get_apicv_inhibit_reasons = NULL; diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index b615f8563e8b..03e2b793979d 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -946,6 +946,7 @@ extern struct kvm_x86_nested_ops svm_nested_ops; bool __init avic_hardware_setup(void); void avic_hardware_unsetup(void); int avic_vcpu_precreate(struct kvm *kvm); +void avic_vm_pre_destroy(struct kvm *kvm); void avic_vm_destroy(struct kvm *kvm); void avic_init_vmcb(struct vcpu_svm *svm, struct vmcb *vmcb); int avic_incomplete_ipi_interception(struct kvm_vcpu *vcpu); From 6978adcc3e73d91fbf371f08cdd924cd05771ac6 Mon Sep 17 00:00:00 2001 From: Hao Zhang Date: Mon, 29 Jun 2026 14:10:41 +0800 Subject: [PATCH 060/121] KVM: VMX: Use cached vcpu_vmx pointer in MSR and segment helpers vmx_get_msr() and vmx_set_msr() already cache to_vmx(vcpu) in a local 'vmx' pointer, but a few cases still open-code to_vmx(vcpu). Use the cached pointer for consistency. Likewise, cache to_vmx(vcpu) in vmx_get_segment_base() instead of open-coding it in both the real-mode check and the VMCS read path. No functional change intended. Signed-off-by: Hao Zhang Link: https://patch.msgid.link/tencent_A78DC401911634111A3391650CB00FCD0409@qq.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/vmx.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 3681d565f177..f360849b7f77 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -2159,7 +2159,7 @@ int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) !guest_has_spec_ctrl_msr(vcpu)) return 1; - msr_info->data = to_vmx(vcpu)->spec_ctrl; + msr_info->data = vmx->spec_ctrl; break; case MSR_IA32_SYSENTER_CS: msr_info->data = vmcs_read32(GUEST_SYSENTER_CS); @@ -2191,7 +2191,7 @@ int vmx_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) if (!msr_info->host_initiated && !guest_cpu_cap_has(vcpu, X86_FEATURE_SGX_LC)) return 1; - msr_info->data = to_vmx(vcpu)->msr_ia32_sgxlepubkeyhash + msr_info->data = vmx->msr_ia32_sgxlepubkeyhash [msr_info->index - MSR_IA32_SGXLEPUBKEYHASH0]; break; case KVM_FIRST_EMULATED_VMX_MSR ... KVM_LAST_EMULATED_VMX_MSR: @@ -2404,7 +2404,7 @@ int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) vmx_guest_debugctl_write(vcpu, data); - if (intel_pmu_lbr_is_enabled(vcpu) && !to_vmx(vcpu)->lbr_desc.event && + if (intel_pmu_lbr_is_enabled(vcpu) && !vmx->lbr_desc.event && (data & DEBUGCTLMSR_LBR)) intel_pmu_create_guest_lbr_event(vcpu); return 0; @@ -2483,7 +2483,7 @@ int vmx_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info) break; case MSR_IA32_MCG_EXT_CTL: if ((!msr_info->host_initiated && - !(to_vmx(vcpu)->msr_ia32_feature_control & + !(vmx->msr_ia32_feature_control & FEAT_CTL_LMCE_ENABLED)) || (data & ~MCG_EXT_CTL_LMCE_EN)) return 1; @@ -3678,13 +3678,14 @@ void vmx_get_segment(struct kvm_vcpu *vcpu, struct kvm_segment *var, int seg) u64 vmx_get_segment_base(struct kvm_vcpu *vcpu, int seg) { + struct vcpu_vmx *vmx = to_vmx(vcpu); struct kvm_segment s; - if (to_vmx(vcpu)->rmode.vm86_active) { + if (vmx->rmode.vm86_active) { vmx_get_segment(vcpu, &s, seg); return s.base; } - return vmx_read_guest_seg_base(to_vmx(vcpu), seg); + return vmx_read_guest_seg_base(vmx, seg); } static int __vmx_get_cpl(struct kvm_vcpu *vcpu, bool no_cache) From bfafeb04a0402c04aa357ed93ab9dfe44b1c4a30 Mon Sep 17 00:00:00 2001 From: Qiang Ma Date: Thu, 18 Jun 2026 16:52:17 +0800 Subject: [PATCH 061/121] KVM: SVM: Remove redundant ret = 0 in svm_set_nested_state In svm_set_nested_state(), the success path reaches out_free with ret already set to 0 from nested_svm_load_cr3(). The explicit 'ret = 0' assignment before out_free is therefore redundant. Remove it. No functional change. Signed-off-by: Qiang Ma Link: https://patch.msgid.link/20260618085217.3934985-1-maqianga@uniontech.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/nested.c | 1 - 1 file changed, 1 deletion(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index ba985a02208a..2d667a581f81 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -2101,7 +2101,6 @@ static int svm_set_nested_state(struct kvm_vcpu *vcpu, kvm_make_request(KVM_REQ_APICV_UPDATE, vcpu); kvm_make_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu); - ret = 0; out_free: kfree(save); kfree(ctl); From 42a39ad5d592aec87a70527a4e694f6210694482 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:31 -0700 Subject: [PATCH 062/121] KVM: SEV: Track the GPA of the guest-controlled VMSA used for SNP guests Track the GPA of the guest-provided VMSA used after AP_CREATION events when running SNP guests, instead of simply tracking whether or not the vCPU is using a guest-provided VMSA. KVM needs to know the GPA of the VMSA that's actively being used so that it can react to MMU invalidation events, i.e. so that KVM can drop the VMSA if its backing guest_memfd page is punched out of existence. Opportunistically rename snp_vmsa_gpa to clarify that it tracks the pending VMSA GPA, whereas snp_guest_vmsa_gpa now tracks the in-use VMSA GPA. Note! Take care to track the GPA, not the GFN, as VALID_PAGE() won't behave correctly if an invalid GFN is converted to a GPA for checking. Note #2! Keep snp_has_guest_vmsa so that switching to a guest-provided VMSA is sticky, even if the guest-provided VMSA becomes invalid. No functional change intended. Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 14 +++++++++----- arch/x86/kvm/svm/svm.h | 3 ++- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 74fb15551e83..827f5dc06102 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -4003,6 +4003,7 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) /* Clear use of the VMSA */ svm->vmcb->control.vmsa_pa = INVALID_PAGE; + svm->sev_es.snp_guest_vmsa_gpa = INVALID_PAGE; /* * When replacing the VMSA during SEV-SNP AP creation, @@ -4010,11 +4011,11 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) */ vmcb_mark_all_dirty(svm->vmcb); - if (!VALID_PAGE(svm->sev_es.snp_vmsa_gpa)) + if (!VALID_PAGE(svm->sev_es.snp_pending_vmsa_gpa)) return; - gfn = gpa_to_gfn(svm->sev_es.snp_vmsa_gpa); - svm->sev_es.snp_vmsa_gpa = INVALID_PAGE; + gfn = gpa_to_gfn(svm->sev_es.snp_pending_vmsa_gpa); + svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; slot = gfn_to_memslot(vcpu->kvm, gfn); if (!slot) @@ -4039,6 +4040,7 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) svm->sev_es.snp_has_guest_vmsa = true; /* Use the new VMSA */ + svm->sev_es.snp_guest_vmsa_gpa = gfn_to_gpa(gfn); svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); /* Mark the vCPU as runnable */ @@ -4105,10 +4107,10 @@ static int sev_snp_ap_creation(struct vcpu_svm *svm) return -EINVAL; } - target_svm->sev_es.snp_vmsa_gpa = svm->vmcb->control.exit_info_2; + target_svm->sev_es.snp_pending_vmsa_gpa = svm->vmcb->control.exit_info_2; break; case SVM_VMGEXIT_AP_DESTROY: - target_svm->sev_es.snp_vmsa_gpa = INVALID_PAGE; + target_svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; break; default: vcpu_unimpl(vcpu, "vmgexit: invalid AP creation request [%#x] from guest\n", @@ -4791,6 +4793,8 @@ int sev_vcpu_create(struct kvm_vcpu *vcpu) return -ENOMEM; svm->sev_es.vmsa = page_address(vmsa_page); + svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; + svm->sev_es.snp_guest_vmsa_gpa = INVALID_PAGE; vcpu->arch.guest_tsc_protected = snp_is_secure_tsc_enabled(vcpu->kvm); diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 716be21fba33..d077783c287e 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -271,7 +271,8 @@ struct vcpu_sev_es_state { u64 ghcb_registered_gpa; struct mutex snp_vmsa_mutex; /* Used to handle concurrent updates of VMSA. */ - gpa_t snp_vmsa_gpa; + gpa_t snp_pending_vmsa_gpa; + gpa_t snp_guest_vmsa_gpa; bool snp_ap_waiting_for_reset; bool snp_has_guest_vmsa; }; From 0060569e4f18a7dee2dd8728595e909f19a23c24 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:32 -0700 Subject: [PATCH 063/121] KVM: SEV: Extract loading of guest-provided VMSA to a separate helper Extract the loading/retrieval of a guest-provided VMSA to a separate helper so that KVM can reuse the core logic when refreshing the VMSA after an MMU invalidation from guest_memfd. No functional change intended. Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 52 +++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 827f5dc06102..d8ed00f76aa3 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -3979,29 +3979,17 @@ static int snp_begin_psc(struct vcpu_svm *svm) return snp_do_psc(svm); } -/* - * Invoked as part of svm_vcpu_reset() processing of an init event. - */ -static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) +static void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) { struct vcpu_svm *svm = to_svm(vcpu); struct kvm_memory_slot *slot; + gfn_t gfn = gpa_to_gfn(gpa); struct page *page; kvm_pfn_t pfn; - gfn_t gfn; - guard(mutex)(&svm->sev_es.snp_vmsa_mutex); + lockdep_assert_held(&svm->sev_es.snp_vmsa_mutex); - if (!svm->sev_es.snp_ap_waiting_for_reset) - return; - - svm->sev_es.snp_ap_waiting_for_reset = false; - - /* Mark the vCPU as offline and not runnable */ - vcpu->arch.pv.pv_unhalted = false; - kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED); - - /* Clear use of the VMSA */ + /* Clear use of the VMSA. */ svm->vmcb->control.vmsa_pa = INVALID_PAGE; svm->sev_es.snp_guest_vmsa_gpa = INVALID_PAGE; @@ -4011,12 +3999,9 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) */ vmcb_mark_all_dirty(svm->vmcb); - if (!VALID_PAGE(svm->sev_es.snp_pending_vmsa_gpa)) + if (!VALID_PAGE(gpa)) return; - gfn = gpa_to_gfn(svm->sev_es.snp_pending_vmsa_gpa); - svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; - slot = gfn_to_memslot(vcpu->kvm, gfn); if (!slot) return; @@ -4040,7 +4025,7 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) svm->sev_es.snp_has_guest_vmsa = true; /* Use the new VMSA */ - svm->sev_es.snp_guest_vmsa_gpa = gfn_to_gpa(gfn); + svm->sev_es.snp_guest_vmsa_gpa = gpa; svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); /* Mark the vCPU as runnable */ @@ -4054,6 +4039,31 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) kvm_release_page_clean(page); } +/* + * Invoked as part of svm_vcpu_reset() processing of an init event. + */ +static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) +{ + struct vcpu_svm *svm = to_svm(vcpu); + gpa_t gpa; + + guard(mutex)(&svm->sev_es.snp_vmsa_mutex); + + if (!svm->sev_es.snp_ap_waiting_for_reset) + return; + + svm->sev_es.snp_ap_waiting_for_reset = false; + + /* Mark the vCPU as offline and not runnable */ + vcpu->arch.pv.pv_unhalted = false; + kvm_set_mp_state(vcpu, KVM_MP_STATE_HALTED); + + gpa = svm->sev_es.snp_pending_vmsa_gpa; + svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; + + sev_snp_reload_vmsa(vcpu, gpa); +} + static int sev_snp_ap_creation(struct vcpu_svm *svm) { struct kvm_sev_info *sev = to_kvm_sev_info(svm->vcpu.kvm); From 98ade8c48c28c227fe2e80e545ff0c57cd4712a3 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:33 -0700 Subject: [PATCH 064/121] KVM: SEV: Mark vCPU RUNNABLE after AP_CREATE, even if VMSA is unusable Always mark the vCPU as RUNNABLE after responding to AP_CREATE, even if the guest-specified VMSA is unusable, e.g. isn't backed by a memslot or doesn't have a backing guest_memfd page. If the VMSA is unusable, leaving the vCPU in a non-running state will effectively hang the vCPU instead of reporting an error to userspace. This will also allow retrying the VMSA load in the future, to fix a bug where KVM doesn't honor guest_memfd invalidation events, e.g. if AP_CREATION races with PUNCH_HOLE. Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index d8ed00f76aa3..30792adcfc8e 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -4028,9 +4028,6 @@ static void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) svm->sev_es.snp_guest_vmsa_gpa = gpa; svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); - /* Mark the vCPU as runnable */ - kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); - /* * gmem pages aren't currently migratable, but if this ever changes * then care should be taken to ensure svm->sev_es.vmsa is pinned @@ -4062,6 +4059,15 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; sev_snp_reload_vmsa(vcpu, gpa); + + /* + * Mark the vCPU as runnable for CREATE requests, indicated by a valid + * VMSA GPA, even if installing the VMSA failed, so that KVM_RUN will + * fail instead of blocking indefinitely and hanging the vCPU, e.g. if + * the backing guest_memfd page is unavailable. + */ + if (VALID_PAGE(gpa)) + kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); } static int sev_snp_ap_creation(struct vcpu_svm *svm) From 01a96ff30dde5127c37497f1e098e639e7ae152f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:34 -0700 Subject: [PATCH 065/121] KVM: SEV: Wire up kvm_x86_ops.gmem_xxx() if and only if CONFIG_KVM_AMD_SEV=y Wire up the SEV-SNP guest_memfd kvm_x86_ops hooks if and only if SEV is actually enabled, and drop the now-unnecessary stubs. Leaving the hooks NULL allows the static call infrastructure to elide the CALL+RET, and more importantly, referencing the hooks if and only if SEV support is enabled will allow conditionally definining the hooks using their corresponding HAVE_KVM_ARCH_GMEM_XXX Kconfig. No functional change intended. Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Ackerley Tng Link: https://patch.msgid.link/20260709204948.1988414-5-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/svm.c | 8 ++++---- arch/x86/kvm/svm/svm.h | 10 ---------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index ef69a51ab27f..79c818d91dda 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5448,6 +5448,10 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .vm_copy_enc_context_from = sev_vm_copy_enc_context_from, .vm_move_enc_context_from = sev_vm_move_enc_context_from, + + .gmem_prepare = sev_gmem_prepare, + .gmem_invalidate = sev_gmem_invalidate, + .gmem_max_mapping_level = sev_gmem_max_mapping_level, #endif .check_emulate_instruction = svm_check_emulate_instruction, @@ -5459,10 +5463,6 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .vcpu_deliver_sipi_vector = svm_vcpu_deliver_sipi_vector, .vcpu_get_apicv_inhibit_reasons = avic_vcpu_get_apicv_inhibit_reasons, .alloc_apic_backing_page = svm_alloc_apic_backing_page, - - .gmem_prepare = sev_gmem_prepare, - .gmem_invalidate = sev_gmem_invalidate, - .gmem_max_mapping_level = sev_gmem_max_mapping_level, }; /* diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index d077783c287e..effca6372e15 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -1035,16 +1035,6 @@ static inline int sev_cpu_init(struct svm_cpu_data *sd) { return 0; } static inline int sev_dev_get_attr(u32 group, u64 attr, u64 *val) { return -ENXIO; } #define max_sev_asid 0 static inline void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code) {} -static inline int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) -{ - return 0; -} -static inline void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) {} -static inline int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private) -{ - return 0; -} - static inline struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu) { return NULL; From ba76b23ed36ab230fc2577aba24f65851114902f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:35 -0700 Subject: [PATCH 066/121] KVM: x86: Serialize writes to disabled_quirks using kvm->lock Protect writes to disabled_quirks with kvm->lock to ensure KVM doesn't clobber state in the unlikely scenario that userspace disables disparate quirks from multiple tasks. More importantly, this will allow wrapping accesses with {READ,WRITE}_ONCE without "needing" to also guard the writer with a useless and confusing READ_ONCE (since the RMW wouldn't be atomic anyways). Ideally, KVM would disallow disabling quirks once quirks are "live", but that would be a potentially breaking userspace ABI change, and while all existing quirks are fully live only after vCPUs have been created, several MMU-related quirks, IGNORE_GUEST_PAT and SLOT_ZAP_ALL, are partially live at all times. Because populating MMUs requires a vCPU, the guest-visible behavior of IGNORE_GUEST_PAT and SLOT_ZAP_ALL requires a vCPU, but for KVM itself, processing the quirk (or not) has functional impact, i.e. for all intents and purposes, KVM can't prevent those quirks from being disabled after they've been consumed. Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-6-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 0626e835e9eb..226c6cfe8062 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3939,7 +3939,9 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, break; fallthrough; case KVM_CAP_DISABLE_QUIRKS: + mutex_lock(&kvm->lock); kvm->arch.disabled_quirks |= cap->args[0] & kvm_caps.supported_quirks; + mutex_unlock(&kvm->lock); r = 0; break; case KVM_CAP_SPLIT_IRQCHIP: { From ed15cb21999217e549414c128b4a0485debf6278 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:36 -0700 Subject: [PATCH 067/121] KVM: x86: Ensure runtime reads of disabled_quirks are resolved once Wrap the sole reader of disabled_quirks with READ_ONCE(), and wrap the post-VM-creation write to disabled_quirks with WRITE_ONCE(), to ensure checking the status of a quirk doesn't re-read disabled_quirks *if* the caller needs such a guarantee. This will allow splitting the "fast" MMU zap into front and back halves, without potentially skipping the back half if SLOT_ZAP_ALL were concurrently disabled (which would be "fine" in the current code base, but far from ideal). Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-7-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 3 ++- arch/x86/kvm/x86.h | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 226c6cfe8062..8abd733d5173 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -3940,7 +3940,8 @@ int kvm_vm_ioctl_enable_cap(struct kvm *kvm, fallthrough; case KVM_CAP_DISABLE_QUIRKS: mutex_lock(&kvm->lock); - kvm->arch.disabled_quirks |= cap->args[0] & kvm_caps.supported_quirks; + WRITE_ONCE(kvm->arch.disabled_quirks, + kvm->arch.disabled_quirks | (cap->args[0] & kvm_caps.supported_quirks)); mutex_unlock(&kvm->lock); r = 0; break; diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 8ece468087a8..75f13d88db58 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -304,7 +304,7 @@ static inline bool vcpu_match_mmio_gpa(struct kvm_vcpu *vcpu, gpa_t gpa) static inline bool kvm_check_has_quirk(struct kvm *kvm, u64 quirk) { - return !(kvm->arch.disabled_quirks & quirk); + return !(READ_ONCE(kvm->arch.disabled_quirks) & quirk); } static __always_inline void kvm_request_l1tf_flush_l1d(void) From 06d38eaa78fdac1cc889f261fa420eba8e9caa1a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:37 -0700 Subject: [PATCH 068/121] KVM: x86/mmu: Fold kvm_mmu_zap_memslot() into kvm_arch_flush_shadow_memslot() Fold kvm_mmu_zap_memslot() into its sole caller so that its GFN range structure can be used to trigger guest_memfd invalidations regardless of whether KVM will do a partial or full zap of the MMU. No functional change intended. Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-8-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 6c13da942bfc..223d80b12b9b 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -7560,8 +7560,14 @@ out_flush: kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush); } -static void kvm_mmu_zap_memslot(struct kvm *kvm, - struct kvm_memory_slot *slot) +static inline bool kvm_memslot_flush_zap_all(struct kvm *kvm) +{ + return kvm->arch.vm_type == KVM_X86_DEFAULT_VM && + kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL); +} + +void kvm_arch_flush_shadow_memslot(struct kvm *kvm, + struct kvm_memory_slot *slot) { struct kvm_gfn_range range = { .slot = slot, @@ -7572,25 +7578,14 @@ static void kvm_mmu_zap_memslot(struct kvm *kvm, }; bool flush; - write_lock(&kvm->mmu_lock); - flush = kvm_unmap_gfn_range(kvm, &range); - kvm_mmu_zap_memslot_pages_and_flush(kvm, slot, flush); - write_unlock(&kvm->mmu_lock); -} - -static inline bool kvm_memslot_flush_zap_all(struct kvm *kvm) -{ - return kvm->arch.vm_type == KVM_X86_DEFAULT_VM && - kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL); -} - -void kvm_arch_flush_shadow_memslot(struct kvm *kvm, - struct kvm_memory_slot *slot) -{ - if (kvm_memslot_flush_zap_all(kvm)) + if (kvm_memslot_flush_zap_all(kvm)) { kvm_mmu_zap_all_fast(kvm); - else - kvm_mmu_zap_memslot(kvm, slot); + } else { + write_lock(&kvm->mmu_lock); + flush = kvm_unmap_gfn_range(kvm, &range); + kvm_mmu_zap_memslot_pages_and_flush(kvm, slot, flush); + write_unlock(&kvm->mmu_lock); + } } void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen) From b27622c4eeb125814081baaefe9175191be5b94d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:38 -0700 Subject: [PATCH 069/121] KVM: x86/mmu: Split kvm_mmu_zap_all_fast() into "front" and "back" halves Split kvm_mmu_zap_all_fast() into a "front half" and a "back half", where the front half is everything that runs with mmu_lock held for write, and the back half is the code that runs outside of mmu_lock. This will allow putting more code inside kvm_arch_flush_shadow_memslot()'s critical section without having to take mmu_lock twice in quick succession. No functional change intended. Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-9-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 223d80b12b9b..a5c2a560a88a 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -6921,20 +6921,11 @@ restart: kvm_mmu_commit_zap_page(kvm, &invalid_list); } -/* - * Fast invalidate all shadow pages and use lock-break technique - * to zap obsolete pages. - * - * It's required when memslot is being deleted or VM is being - * destroyed, in these cases, we should ensure that KVM MMU does - * not use any resource of the being-deleted slot or all slots - * after calling the function. - */ -static void kvm_mmu_zap_all_fast(struct kvm *kvm) +static void __kvm_mmu_zap_all_fast_front_half(struct kvm *kvm) { lockdep_assert_held(&kvm->slots_lock); + lockdep_assert_held_write(&kvm->mmu_lock); - write_lock(&kvm->mmu_lock); trace_kvm_mmu_zap_all_fast(kvm); /* @@ -6971,8 +6962,12 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm) kvm_make_all_cpus_request(kvm, KVM_REQ_MMU_FREE_OBSOLETE_ROOTS); kvm_zap_obsolete_pages(kvm); +} - write_unlock(&kvm->mmu_lock); +static void __kvm_mmu_zap_all_fast_back_half(struct kvm *kvm) +{ + lockdep_assert_held(&kvm->slots_lock); + lockdep_assert_not_held(&kvm->mmu_lock); /* * Zap the invalidated TDP MMU roots, all SPTEs must be dropped before @@ -6986,6 +6981,24 @@ static void kvm_mmu_zap_all_fast(struct kvm *kvm) kvm_tdp_mmu_zap_invalidated_roots(kvm, true); } +/* + * Fast invalidate all shadow pages and use lock-break technique + * to zap obsolete pages. + * + * It's required when memslot is being deleted or VM is being + * destroyed, in these cases, we should ensure that KVM MMU does + * not use any resource of the being-deleted slot or all slots + * after calling the function. + */ +static void kvm_mmu_zap_all_fast(struct kvm *kvm) +{ + write_lock(&kvm->mmu_lock); + __kvm_mmu_zap_all_fast_front_half(kvm); + write_unlock(&kvm->mmu_lock); + + __kvm_mmu_zap_all_fast_back_half(kvm); +} + int kvm_mmu_init_vm(struct kvm *kvm) { int r, i; From db095727ff5739f4f46ee641ee6ef450032886db Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:39 -0700 Subject: [PATCH 070/121] KVM: x86/mmu: Use split "zap all fast" helpers when invalidating memslot Manually invoke the front half and back half of the "zap all fast" flow when invalidating a memslot so that mmu_lock is acquired at function scope in kvm_arch_flush_shadow_memslot(). This will allow putting more code inside the critical section without having to take mmu_lock twice in quick succession. Opportunistically open code checking whether or not to do the fast zap, to discourage removing the local "zap_all" in a future cleanup, i.e. to ensure the SLOT_ZAP_ALL quirk is queried exactly once. Processing the front half but not the back half of the fast zap (if SLOT_ZAP_ALL were disabled concurrently) would result in KVM unnecessarily keeping invalid TDP MMU roots until the VM is destroyed. No functional change intended. Cc: stable@vger.kernel.org # 6.12.x Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-10-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index a5c2a560a88a..3eb1f86593b1 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -7573,12 +7573,6 @@ out_flush: kvm_mmu_remote_flush_or_zap(kvm, &invalid_list, flush); } -static inline bool kvm_memslot_flush_zap_all(struct kvm *kvm) -{ - return kvm->arch.vm_type == KVM_X86_DEFAULT_VM && - kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL); -} - void kvm_arch_flush_shadow_memslot(struct kvm *kvm, struct kvm_memory_slot *slot) { @@ -7589,16 +7583,23 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm, .may_block = true, .attr_filter = KVM_FILTER_PRIVATE | KVM_FILTER_SHARED, }; + bool zap_all = kvm->arch.vm_type == KVM_X86_DEFAULT_VM && + kvm_check_has_quirk(kvm, KVM_X86_QUIRK_SLOT_ZAP_ALL); bool flush; - if (kvm_memslot_flush_zap_all(kvm)) { - kvm_mmu_zap_all_fast(kvm); + write_lock(&kvm->mmu_lock); + + if (zap_all) { + __kvm_mmu_zap_all_fast_front_half(kvm); } else { - write_lock(&kvm->mmu_lock); flush = kvm_unmap_gfn_range(kvm, &range); kvm_mmu_zap_memslot_pages_and_flush(kvm, slot, flush); - write_unlock(&kvm->mmu_lock); } + + write_unlock(&kvm->mmu_lock); + + if (zap_all) + __kvm_mmu_zap_all_fast_back_half(kvm); } void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen) From d1a3c216233413f57f5341a9b878b7e2dde7e785 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:40 -0700 Subject: [PATCH 071/121] KVM: SEV: Forcefully invalidate SNP VMSA if its backing gmem page is zapped MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire up a gmem_invalidate_range() call for SNP VMs, and use it to force vCPUs to reload/recheck their guest-provided VMSA if the backing gmem page is being invalidated, e.g. is being PUNCH_HOLE'd. Use the same core logic to handle invalidations as VMX does for the APIC-access page, as the two concepts are nearly identical: shove the physical address of a page into the vCPU's control structure: 1. Snapshot the invalidation sequence counter 2. Grab the pfn (from guest_memfd in this case) 3. Acquire mmu_lock for read 4. Re-request reload if retry is needed, otherwise commit the change. Note, the re-request action in #4 is necessary as KVM's retry logic is fuzzy, i.e. can get false positives. If the guest_memfd page has been dropped, at some point a subsequent reload will fail to get a PFN from guest_memfd, and KVM will fail KVM_RUN. If the retry was due to a false positive, KVM will retry until there are no relevant MMU notifier events (and will retry in the "outer" loop, i.e. will drop locks and resched as needed). Note #2! Take care to invalidate the VMSA when a relevant memslot is DELETED or MOVED, as invalidations in response to PUNCH_HOLE are predicated on memslot bindings (KVM doesn't know what GFN range(s) to invalidate without a binding). And more importantly, the VMSA mapping requires a memslot, i.e. must be invalidated if its memslots disappears, regardless of the state of the underlying guest_memfd inode. Failure to invalidate the vCPU's control.vmsa_pa (which is checked by pre_sev_run()) can prevent KVM from properly freeing the page as firmware will reject the RMPUPDATE to reclaim the page with FAIL_INUSE if the vCPU is actively running, i.e. if VMSA page is in-use. That in turn leads to an RMP #PF on the next use, as the page will still be assigned to the SNP VM. SEV-SNP: RMPUPDATE failed for PFN 78d198, pg_level: 1, ret: 3 SEV-SNP: PFN 0x78d198, RMP entry: [0xfff0000000144001 - 0x000000000000000f] CPU: 3 UID: 0 PID: 31345 Comm: sev_snp_vmsa_pu Tainted: G U O Tainted: [U]=USER, [O]=OOT_MODULE Hardware name: Google, Inc. Arcadia_IT_80/Arcadia_IT_80, BIOS 34.86.0-102 01/25/2026 Call Trace: dump_stack_lvl+0x54/0x70 rmpupdate+0x12c/0x140 rmp_make_shared+0x3b/0x60 sev_gmem_invalidate+0xe0/0x170 [kvm_amd] delete_from_page_cache_batch+0x1d8/0x220 truncate_inode_pages_range+0x120/0x3d0 kvm_gmem_fallocate+0x19a/0x270 [kvm] vfs_fallocate+0x1bc/0x1f0 __x64_sys_fallocate+0x48/0x70 do_syscall_64+0x10a/0x480 entry_SYSCALL_64_after_hwframe+0x4b/0x53 RIP: 0033:0x496c7e ------------[ cut here ]------------ SEV: Failed to update RMP entry for PFN 0x78d198 error -14 WARNING: arch/x86/kvm/svm/sev.c:5160 at sev_gmem_invalidate+0x126/0x170 [kvm_amd], CPU#3: sev_snp_vmsa_pu/31345 CPU: 3 UID: 0 PID: 31345 Comm: sev_snp_vmsa_pu Tainted: G U O Tainted: [U]=USER, [O]=OOT_MODULE Hardware name: Google, Inc. Arcadia_IT_80/Arcadia_IT_80, BIOS 34.86.0-102 01/25/2026 RIP: 0010:sev_gmem_invalidate+0x12b/0x170 [kvm_amd] Call Trace: delete_from_page_cache_batch+0x1d8/0x220 truncate_inode_pages_range+0x120/0x3d0 kvm_gmem_fallocate+0x19a/0x270 [kvm] vfs_fallocate+0x1bc/0x1f0 __x64_sys_fallocate+0x48/0x70 do_syscall_64+0x10a/0x480 entry_SYSCALL_64_after_hwframe+0x4b/0x53 RIP: 0033:0x496c7e irq event stamp: 20689 hardirqs last enabled at (20699): [] __console_unlock+0x5c/0x60 hardirqs last disabled at (20708): [] __console_unlock+0x41/0x60 softirqs last enabled at (20722): [] __irq_exit_rcu+0x7e/0x140 softirqs last disabled at (20717): [] __irq_exit_rcu+0x7e/0x140 ---[ end trace 0000000000000000 ]--- BUG: unable to handle page fault for address: ffff99a64d198000 #PF: supervisor write access in kernel mode #PF: error_code(0x80000003) - RMP violation PGD 13eb001067 P4D 13eb001067 PUD 78d1d1063 PMD 1184e0063 PTE 800000078d198163 SEV-SNP: PFN 0x78d198, RMP entry: [0x6030000000144001 - 0x000000000000000f] Oops: Oops: 0003 [#1] SMP CPU: 3 UID: 0 PID: 31407 Comm: highlanderd_hea Tainted: G U W O Tainted: [U]=USER, [W]=WARN, [O]=OOT_MODULE Hardware name: Google, Inc. Arcadia_IT_80/Arcadia_IT_80, BIOS 34.86.0-102 01/25/2026 RIP: 0010:prep_new_page+0x67/0x220 Call Trace: get_page_from_freelist+0x1c40/0x1c70 __alloc_frozen_pages_noprof+0xca/0x1f0 alloc_pages_mpol+0x10b/0x1b0 alloc_pages_noprof+0x81/0x90 pte_alloc_one+0x1b/0xd0 do_pte_missing+0xdf/0x1020 handle_mm_fault+0x7c7/0xb20 do_user_addr_fault+0x268/0x6b0 exc_page_fault+0x67/0xa0 asm_exc_page_fault+0x26/0x30 RIP: 0033:0x4a6b1e gsmi: Log Shutdown Reason 0x03 CR2: ffff99a64d198000 ---[ end trace 0000000000000000 ]--- RIP: 0010:prep_new_page+0x67/0x220 Drop the pseudo-TODO comment about needing to pin the page if guest_memfd every supports migration, as integrating with invalidations events means KVM will Just Work if/when page migration is ever supported (assuming SNP hardware supports migrating VMSA pages). Note #3, invalidate() and invalidate_range() have _completely_ different semantics; the new invalidate_range() is a true invalidation, whereas the existing invalidate() is really a "make shared" operation. Ignore the confusing naming and poor Kconfig bundling for the moment to minimize the delta for LTS kernels, the mess will be cleaned up shortly. Reported-by: Hyunwoo Kim Closes: https://lore.kernel.org/all/aimMWzAf5b3luM0b@v4bel Fixes: e366f92ea99e ("KVM: SEV: Support SEV-SNP AP Creation NAE event") Cc: stable@vger.kernel.org Cc: Tom Lendacky Cc: Michael Roth Cc: Jörg Rödel Cc: Fuad Tabba Cc: Ackerley Tng Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-11-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 4 ++ arch/x86/include/asm/kvm_host.h | 6 +++ arch/x86/kvm/mmu/mmu.c | 5 ++ arch/x86/kvm/svm/sev.c | 80 ++++++++++++++++++++++++++---- arch/x86/kvm/svm/svm.c | 2 + arch/x86/kvm/svm/svm.h | 2 + arch/x86/kvm/x86.c | 6 +++ include/linux/kvm_host.h | 1 + virt/kvm/guest_memfd.c | 4 ++ 9 files changed, 99 insertions(+), 11 deletions(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 83dc5086138b..ccf23b3f0e1c 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -134,6 +134,7 @@ KVM_X86_OP_OPTIONAL(mem_enc_unregister_region) KVM_X86_OP_OPTIONAL(vm_copy_enc_context_from) KVM_X86_OP_OPTIONAL(vm_move_enc_context_from) KVM_X86_OP_OPTIONAL(guest_memory_reclaimed) +KVM_X86_OP_OPTIONAL(reload_vmsa) KVM_X86_OP(get_feature_msr) KVM_X86_OP(check_emulate_instruction) KVM_X86_OP(apic_init_signal_blocked) @@ -148,6 +149,9 @@ KVM_X86_OP_OPTIONAL(alloc_apic_backing_page) KVM_X86_OP_OPTIONAL_RET0(gmem_prepare) KVM_X86_OP_OPTIONAL_RET0(gmem_max_mapping_level) KVM_X86_OP_OPTIONAL(gmem_invalidate) +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE +KVM_X86_OP_OPTIONAL(gmem_invalidate_range) +#endif #endif #undef KVM_X86_OP diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index b517257a6315..1c598b40e0c3 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -122,6 +122,8 @@ KVM_ARCH_REQ_FLAGS(31, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_HV_TLB_FLUSH \ KVM_ARCH_REQ_FLAGS(32, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) +#define KVM_REQ_VMSA_PAGE_RELOAD \ + KVM_ARCH_REQ_FLAGS(33, KVM_REQUEST_WAIT | KVM_REQUEST_NO_WAKEUP) #define KVM_REQ_UPDATE_PROTECTED_GUEST_STATE \ KVM_ARCH_REQ_FLAGS(34, KVM_REQUEST_WAIT) @@ -1878,6 +1880,7 @@ struct kvm_x86_ops { int (*vm_copy_enc_context_from)(struct kvm *kvm, unsigned int source_fd); int (*vm_move_enc_context_from)(struct kvm *kvm, unsigned int source_fd); void (*guest_memory_reclaimed)(struct kvm *kvm); + void (*reload_vmsa)(struct kvm_vcpu *vcpu); int (*get_feature_msr)(u32 msr, u64 *data); @@ -1902,6 +1905,9 @@ struct kvm_x86_ops { void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu); int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); void (*gmem_invalidate)(kvm_pfn_t start, kvm_pfn_t end); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE + void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range); +#endif int (*gmem_max_mapping_level)(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); }; diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 3eb1f86593b1..e2978e9a1731 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -7589,6 +7589,11 @@ void kvm_arch_flush_shadow_memslot(struct kvm *kvm, write_lock(&kvm->mmu_lock); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE + if (slot->gmem.file) + kvm_arch_gmem_invalidate_range(kvm, &range); +#endif + if (zap_all) { __kvm_mmu_zap_all_fast_front_half(kvm); } else { diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 30792adcfc8e..62c6126d52c0 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -3979,19 +3979,25 @@ static int snp_begin_psc(struct vcpu_svm *svm) return snp_do_psc(svm); } -static void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) +static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) { struct vcpu_svm *svm = to_svm(vcpu); struct kvm_memory_slot *slot; + struct kvm *kvm = vcpu->kvm; gfn_t gfn = gpa_to_gfn(gpa); + unsigned long mmu_seq; struct page *page; kvm_pfn_t pfn; lockdep_assert_held(&svm->sev_es.snp_vmsa_mutex); - /* Clear use of the VMSA. */ + /* + * Clear use of the VMSA. Ensure snp_guest_vmsa_gpa is written exactly + * once, as it is read locklessly when responding to gfn invalidations. + * Pairs with the READ_ONCE() in sev_gmem_invalidate_range(). + */ svm->vmcb->control.vmsa_pa = INVALID_PAGE; - svm->sev_es.snp_guest_vmsa_gpa = INVALID_PAGE; + WRITE_ONCE(svm->sev_es.snp_guest_vmsa_gpa, INVALID_PAGE); /* * When replacing the VMSA during SEV-SNP AP creation, @@ -4006,6 +4012,9 @@ static void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) if (!slot) return; + mmu_seq = kvm->mmu_invalidate_seq; + smp_rmb(); + /* * The new VMSA will be private memory guest memory, so retrieve the * PFN from the gmem backend. @@ -4024,15 +4033,20 @@ static void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) */ svm->sev_es.snp_has_guest_vmsa = true; - /* Use the new VMSA */ - svm->sev_es.snp_guest_vmsa_gpa = gpa; - svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); - + read_lock(&kvm->mmu_lock); /* - * gmem pages aren't currently migratable, but if this ever changes - * then care should be taken to ensure svm->sev_es.vmsa is pinned - * through some other means. + * Save the guest-provided GPA. If retry is needed, then KVM will try + * again with the same GPA. If the VMSA is usable, then KVM needs to + * track the GPA so that the VMSA can be reloaded if the backing page + * for the GPA is invalidated. */ + svm->sev_es.snp_guest_vmsa_gpa = gpa; + if (mmu_invalidate_retry_gfn(kvm, mmu_seq, gfn)) + kvm_make_request(KVM_REQ_VMSA_PAGE_RELOAD, vcpu); + else + svm->vmcb->control.vmsa_pa = pfn_to_hpa(pfn); + read_unlock(&kvm->mmu_lock); + kvm_release_page_clean(page); } @@ -4058,7 +4072,7 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) gpa = svm->sev_es.snp_pending_vmsa_gpa; svm->sev_es.snp_pending_vmsa_gpa = INVALID_PAGE; - sev_snp_reload_vmsa(vcpu, gpa); + __sev_snp_reload_vmsa(vcpu, gpa); /* * Mark the vCPU as runnable for CREATE requests, indicated by a valid @@ -4070,6 +4084,15 @@ static void sev_snp_init_protected_guest_state(struct kvm_vcpu *vcpu) kvm_set_mp_state(vcpu, KVM_MP_STATE_RUNNABLE); } +void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu) +{ + struct vcpu_sev_es_state *sev_es = &to_svm(vcpu)->sev_es; + + guard(mutex)(&sev_es->snp_vmsa_mutex); + + __sev_snp_reload_vmsa(vcpu, sev_es->snp_guest_vmsa_gpa); +} + static int sev_snp_ap_creation(struct vcpu_svm *svm) { struct kvm_sev_info *sev = to_kvm_sev_info(svm->vcpu.kvm); @@ -5199,6 +5222,41 @@ next_pfn: } } +void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range) +{ + struct kvm_vcpu *vcpu; + unsigned long i; + + lockdep_assert_held_write(&kvm->mmu_lock); + + /* + * An unstable result for "is SNP" is a-ok here, thanks to mmu_lock. + * The vCPU's VMSA GPA is invalidated before the vCPU is made visible + * to other tasks, and can only become valid while holding mmu_lock, + * after the VM is fully committed to being an SNP VM. + */ + if (!____sev_snp_guest(kvm)) + return; + + kvm_for_each_vcpu(i, vcpu, kvm) { + /* + * Read snp_guest_vmsa_gpa without taking the vCPU's VMSA mutex + * (or its generic mutex) as mmu_lock is held, i.e. this task + * can't sleep. The VMSA is invalidated outside of mmu_lock, + * but can only become valid inside of mmu_lock, i.e. the below + * can get false positives, but not false negatives. A false + * positive is benign, as a spurious request simply forces the + * vCPU to re-establish its VMSA. + */ + gpa_t gpa = READ_ONCE(to_svm(vcpu)->sev_es.snp_guest_vmsa_gpa); + + if (VALID_PAGE(gpa) && + gpa_to_gfn(gpa) >= range->start && + gpa_to_gfn(gpa) < range->end) + kvm_make_request_and_kick(KVM_REQ_VMSA_PAGE_RELOAD, vcpu); + } +} + int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private) { int level, rc; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 79c818d91dda..dd51df74c2dc 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5445,12 +5445,14 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .mem_enc_register_region = sev_mem_enc_register_region, .mem_enc_unregister_region = sev_mem_enc_unregister_region, .guest_memory_reclaimed = sev_guest_memory_reclaimed, + .reload_vmsa = sev_snp_reload_vmsa, .vm_copy_enc_context_from = sev_vm_copy_enc_context_from, .vm_move_enc_context_from = sev_vm_move_enc_context_from, .gmem_prepare = sev_gmem_prepare, .gmem_invalidate = sev_gmem_invalidate, + .gmem_invalidate_range = sev_gmem_invalidate_range, .gmem_max_mapping_level = sev_gmem_max_mapping_level, #endif .check_emulate_instruction = svm_check_emulate_instruction, diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index effca6372e15..130205defffa 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -996,6 +996,7 @@ static inline struct page *snp_safe_alloc_page(void) { return snp_safe_alloc_page_node(numa_node_id(), GFP_KERNEL_ACCOUNT); } +void sev_snp_reload_vmsa(struct kvm_vcpu *vcpu); int sev_vcpu_create(struct kvm_vcpu *vcpu); void sev_free_vcpu(struct kvm_vcpu *vcpu); @@ -1010,6 +1011,7 @@ extern unsigned int max_sev_asid; void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code); int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end); +void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu); void sev_free_decrypted_vmsa(struct kvm_vcpu *vcpu, struct vmcb_save_area *vmsa); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 8abd733d5173..5a5fd6211d23 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -8170,6 +8170,8 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) goto out; } } + if (kvm_check_request(KVM_REQ_VMSA_PAGE_RELOAD, vcpu)) + kvm_x86_call(reload_vmsa)(vcpu); } if (kvm_check_request(KVM_REQ_EVENT, vcpu) || req_int_win || @@ -10599,6 +10601,10 @@ void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) { kvm_x86_call(gmem_invalidate)(start, end); } +void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range) +{ + kvm_x86_call(gmem_invalidate_range)(kvm, range); +} #endif #endif diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index ab8cfaec82d3..c00fc1740ce5 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2608,6 +2608,7 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end); +void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); #endif #ifdef CONFIG_KVM_GENERIC_PRE_FAULT_MEMORY diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 86690683b2fe..659b8dbe0b30 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -185,6 +185,10 @@ static void __kvm_gmem_invalidate_start(struct gmem_file *f, pgoff_t start, } flush |= kvm_mmu_unmap_gfn_range(kvm, &gfn_range); + +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE + kvm_arch_gmem_invalidate_range(kvm, &gfn_range); +#endif } if (flush) From 98fb11aed12ba8c2edb42428a0188f726a61182f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:41 -0700 Subject: [PATCH 072/121] KVM: SEV: Mark vCPU has having guest-provided VMSA even if its invalid Track the guest as having a guest-provided VMSA as soon as control.vmsa_pa is invalidated, instead of waiting to see if the guest-provided VMSA is usable, so that KVM doesn't switch back to the original VMSA instead of exiting to userspace (due to an invalid VMSA). By the time a vCPU tries to load a guest-provided VMSA, KVM has already communicated "success" for AP creation, i.e. KVM has committed to using the guest-provided VMSA. Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-12-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 62c6126d52c0..a7584b7ed6dc 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -4005,6 +4005,17 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) */ vmcb_mark_all_dirty(svm->vmcb); + /* + * From this point forward, the VMSA will always be a guest-mapped page + * rather than the initial one allocated by KVM in svm->sev_es.vmsa. In + * theory, svm->sev_es.vmsa could be free'd and cleaned up here, but + * that involves cleanups like flushing caches, which would ideally be + * handled during teardown rather than guest boot. Deferring that also + * allows the existing logic for SEV-ES VMSAs to be re-used with + * minimal SNP-specific changes. + */ + svm->sev_es.snp_has_guest_vmsa = true; + if (!VALID_PAGE(gpa)) return; @@ -4022,17 +4033,6 @@ static void __sev_snp_reload_vmsa(struct kvm_vcpu *vcpu, gpa_t gpa) if (kvm_gmem_get_pfn(vcpu->kvm, slot, gfn, &pfn, &page, NULL)) return; - /* - * From this point forward, the VMSA will always be a guest-mapped page - * rather than the initial one allocated by KVM in svm->sev_es.vmsa. In - * theory, svm->sev_es.vmsa could be free'd and cleaned up here, but - * that involves cleanups like flushing caches, which would ideally be - * handled during teardown rather than guest boot. Deferring that also - * allows the existing logic for SEV-ES VMSAs to be re-used with - * minimal SNP-specific changes. - */ - svm->sev_es.snp_has_guest_vmsa = true; - read_lock(&kvm->mmu_lock); /* * Save the guest-provided GPA. If retry is needed, then KVM will try From bfbf5fa51145cf874fa6a51d123e0d917db6eedc Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 9 Jul 2026 13:49:42 -0700 Subject: [PATCH 073/121] KVM: x86: Guard .gmem_prepare() declarations with HAVE_KVM_GMEM_PREPARE=y Wrap the .gmem_prepare() declarations with HAVE_KVM_GMEM_PREPARE so that non-SEV code doesn't try to wire up a callback without doing the necessary enabling. No functional change intended. Fixes: 3bb2531e20bf ("KVM: guest_memfd: Add hook for initializing memory") Reviewed-by: Ackerley Tng Reviewed-by: Michael Roth Link: https://patch.msgid.link/20260709204948.1988414-13-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 4 +++- arch/x86/include/asm/kvm_host.h | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index ccf23b3f0e1c..736129db272a 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -146,12 +146,14 @@ KVM_X86_OP(vcpu_deliver_sipi_vector) KVM_X86_OP_OPTIONAL_RET0(vcpu_get_apicv_inhibit_reasons); KVM_X86_OP_OPTIONAL(get_untagged_addr) KVM_X86_OP_OPTIONAL(alloc_apic_backing_page) +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE KVM_X86_OP_OPTIONAL_RET0(gmem_prepare) -KVM_X86_OP_OPTIONAL_RET0(gmem_max_mapping_level) +#endif KVM_X86_OP_OPTIONAL(gmem_invalidate) #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE KVM_X86_OP_OPTIONAL(gmem_invalidate_range) #endif +KVM_X86_OP_OPTIONAL_RET0(gmem_max_mapping_level) #endif #undef KVM_X86_OP diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 1c598b40e0c3..1f8ef0e1566a 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1903,7 +1903,9 @@ struct kvm_x86_ops { gva_t (*get_untagged_addr)(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); +#endif void (*gmem_invalidate)(kvm_pfn_t start, kvm_pfn_t end); #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range); From 3f339b70bb59d92d81d2853f905e1124276a9802 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 7 Jul 2026 08:07:05 -0700 Subject: [PATCH 074/121] KVM: selftests: Drop superfluous use of pthread_attr_setaffinity_np() In the steal time test, don't explicitly set the CPU affinity mask of the worker child and instead rely on the child inheriting the affinity of the main thread. Per the pthread_create()[1] and pthread_setaffinity_np()[2] documentation, new threads inherit the parent's affinity mask. Linux-specific details The new thread inherits copies of the calling thread's capability sets (see capabilities(7)) and CPU affinity mask (see sched_setaffinity(2)). Out of an abundance of caution, assert that the child did indeed inherit the CPU affinity mask, as the test will hang indefinitely if the system is under light load. Dropping use of pthread_attr_setaffinity_np() allows building the steal time test against non-glibc C libraries that don't implement that GNU extension. Link: https://man7.org/linux/man-pages/man3/pthread_setaffinity_np.3.html [1] Link: https://man7.org/linux/man-pages/man3/pthread_create.3.html [1] Cc: Hisam Mehboob Reported-by: Aqib Faruqui Closes: https://lore.kernel.org/all/20250829142556.72577-4-aqibaf@amazon.com Link: https://patch.msgid.link/20260707150706.1198541-2-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/steal_time.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tools/testing/selftests/kvm/steal_time.c b/tools/testing/selftests/kvm/steal_time.c index 76fcdd1fd3cb..a244bf9f701f 100644 --- a/tools/testing/selftests/kvm/steal_time.c +++ b/tools/testing/selftests/kvm/steal_time.c @@ -508,7 +508,6 @@ int main(int ac, char **av) { struct kvm_vcpu *vcpus[NR_VCPUS]; struct kvm_vm *vm; - pthread_attr_t attr; pthread_t thread; cpu_set_t cpuset; unsigned int gpages; @@ -522,8 +521,6 @@ int main(int ac, char **av) /* Set CPU affinity so we can force preemption of the VCPU */ CPU_ZERO(&cpuset); CPU_SET(0, &cpuset); - pthread_attr_init(&attr); - pthread_attr_setaffinity_np(&attr, sizeof(cpu_set_t), &cpuset); pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); /* Create a VM and an identity mapped memslot for the steal time structure */ @@ -558,7 +555,11 @@ int main(int ac, char **av) /* Steal time from the VCPU. The steal time thread has the same CPU affinity as the VCPUs. */ run_delay = get_run_delay(); - pthread_create(&thread, &attr, do_steal_time, NULL); + pthread_create(&thread, NULL, do_steal_time, NULL); + pthread_getaffinity_np(thread, sizeof(cpuset), &cpuset); + TEST_ASSERT(CPU_COUNT(&cpuset) == 1 && CPU_ISSET(0, &cpuset), + "Worker failed to inherit parent's CPU affinity"); + do sched_yield(); while (get_run_delay() - run_delay < MIN_RUN_DELAY_NS); From 2351e814e1e1ef8e10698c6b51c86eceedc0ce86 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 7 Jul 2026 08:07:06 -0700 Subject: [PATCH 075/121] KVM: selftests: Randomize pCPU in steal time test Pin the steal time test's tasks to a random pCPU in the system instead of hardcoding the test to always run on pCPU0 as a cheap way of increasing test coverage, and to do the "right thing" if the parent task of the test doesn't have pCPU0 in its CPU affinity mask. Link: https://patch.msgid.link/20260707150706.1198541-3-seanjc@google.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/steal_time.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tools/testing/selftests/kvm/steal_time.c b/tools/testing/selftests/kvm/steal_time.c index a244bf9f701f..92e7ffcd68b7 100644 --- a/tools/testing/selftests/kvm/steal_time.c +++ b/tools/testing/selftests/kvm/steal_time.c @@ -514,14 +514,12 @@ int main(int ac, char **av) long stolen_time; long run_delay; bool verbose; - int i; + int i, cpu; verbose = ac > 1 && (!strncmp(av[1], "-v", 3) || !strncmp(av[1], "--verbose", 10)); /* Set CPU affinity so we can force preemption of the VCPU */ - CPU_ZERO(&cpuset); - CPU_SET(0, &cpuset); - pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); + cpu = pin_self_to_any_cpu(); /* Create a VM and an identity mapped memslot for the steal time structure */ vm = vm_create_with_vcpus(NR_VCPUS, guest_code, vcpus); @@ -557,7 +555,7 @@ int main(int ac, char **av) run_delay = get_run_delay(); pthread_create(&thread, NULL, do_steal_time, NULL); pthread_getaffinity_np(thread, sizeof(cpuset), &cpuset); - TEST_ASSERT(CPU_COUNT(&cpuset) == 1 && CPU_ISSET(0, &cpuset), + TEST_ASSERT(CPU_COUNT(&cpuset) == 1 && CPU_ISSET(cpu, &cpuset), "Worker failed to inherit parent's CPU affinity"); do From b65699be2c606d2687593516d93e66d16a713b61 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 13:28:26 -0700 Subject: [PATCH 076/121] KVM: x86: Reject nested CAP enablement if nested virtualization is disabled Add a flag to explicitly track if nested virtualization is enabled, and use it enumerate that various nested CAPs are unsupported, and to reject enablement of said CAPs. When the nested ops hooks were moved to their own structure, KVM's NULL-by-default behavior was deliberately dropped, with the changelog asserting that all was well. That wasn't quite true; there is no danger to KVM, but now KVM is over-reporting support for KVM_CAP_NESTED_STATE and KVM_CAP_HYPERV_ENLIGHTENED_VMCS. Fixes: 33b22172452f ("KVM: x86: move nested-related kvm_x86_ops to a separate struct") Reviewed-by: Vitaly Kuznetsov Link: https://patch.msgid.link/20260630202828.440724-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/hyperv.c | 3 ++- arch/x86/kvm/svm/svm.c | 1 + arch/x86/kvm/vmx/vmx.c | 1 + arch/x86/kvm/x86.c | 12 +++++++----- 5 files changed, 13 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index fdb8953aeeb1..a46504077b3e 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1729,6 +1729,8 @@ struct kvm_x86_ops { }; struct kvm_x86_nested_ops { + bool enabled; + void (*leave_nested)(struct kvm_vcpu *vcpu); bool (*is_exception_vmexit)(struct kvm_vcpu *vcpu, u8 vector, u32 error_code); diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 9d38cb644668..72036670c1ac 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -2791,7 +2791,8 @@ int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, }; int i, nent = ARRAY_SIZE(cpuid_entries); - if (kvm_x86_ops.nested_ops->get_evmcs_version) + if (kvm_x86_ops.nested_ops->enabled && + kvm_x86_ops.nested_ops->get_evmcs_version) evmcs_ver = kvm_x86_ops.nested_ops->get_evmcs_version(vcpu); if (cpuid->nent < nent) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index ef69a51ab27f..163440fca59f 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5646,6 +5646,7 @@ static __init int svm_hardware_setup(void) if (r) return r; } + svm_nested_ops.enabled = nested; /* * KVM's MMU doesn't support using 2-level paging for itself, and thus diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index b25b978d7a9d..5ef6bdb334c7 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -8786,6 +8786,7 @@ __init int vmx_hardware_setup(void) if (r) return r; } + vmx_nested_ops.enabled = nested; kvm_set_posted_intr_wakeup_handler(pi_wakeup_handler); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 5ee6d0c33009..566a017eab1d 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -2355,7 +2355,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r &= ~KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST; break; case KVM_CAP_NESTED_STATE: - r = kvm_x86_ops.nested_ops->get_state ? + r = kvm_x86_ops.nested_ops->enabled ? kvm_x86_ops.nested_ops->get_state(NULL, NULL, 0) : 0; break; #ifdef CONFIG_KVM_HYPERV @@ -2363,7 +2363,8 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r = kvm_x86_ops.enable_l2_tlb_flush != NULL; break; case KVM_CAP_HYPERV_ENLIGHTENED_VMCS: - r = kvm_x86_ops.nested_ops->enable_evmcs != NULL; + r = kvm_x86_ops.nested_ops->enabled && + kvm_x86_ops.nested_ops->enable_evmcs != NULL; break; #endif case KVM_CAP_SMALLER_MAXPHYADDR: @@ -3375,7 +3376,8 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, uint16_t vmcs_version; void __user *user_ptr; - if (!kvm_x86_ops.nested_ops->enable_evmcs) + if (!kvm_x86_ops.nested_ops->enabled || + !kvm_x86_ops.nested_ops->enable_evmcs) return -ENOTTY; r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version); if (!r) { @@ -3741,7 +3743,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, u32 user_data_size; r = -EINVAL; - if (!kvm_x86_ops.nested_ops->get_state) + if (!kvm_x86_ops.nested_ops->enabled) break; BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size)); @@ -3771,7 +3773,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, int idx; r = -EINVAL; - if (!kvm_x86_ops.nested_ops->set_state) + if (!kvm_x86_ops.nested_ops->enabled) break; r = -EFAULT; From 4b9819a50674dd40d176033f9ea2e23a70889dc6 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 13:28:27 -0700 Subject: [PATCH 077/121] KVM: x86: Add static calls for nested virtualization ops Use static calls to invoke nested virtualization ops, as many of the calls are in relatively hot paths when L2 is active, e.g. checking for events, and because there's no reason not use static calls these days. Opportunistically use a RET0 static call for get_evmcs_version() instead of manually checking for a non-NULL vendor hook. Link: https://patch.msgid.link/20260630202828.440724-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-nested-ops.h | 36 +++++++++++++++ arch/x86/include/asm/kvm_host.h | 8 ++++ arch/x86/kvm/hyperv.c | 7 ++- arch/x86/kvm/mmu.h | 5 +-- arch/x86/kvm/mmu/paging_tmpl.h | 2 +- arch/x86/kvm/x86.c | 53 +++++++++++++++-------- arch/x86/kvm/x86.h | 2 +- 7 files changed, 86 insertions(+), 27 deletions(-) create mode 100644 arch/x86/include/asm/kvm-x86-nested-ops.h diff --git a/arch/x86/include/asm/kvm-x86-nested-ops.h b/arch/x86/include/asm/kvm-x86-nested-ops.h new file mode 100644 index 000000000000..4b1be5bcecaa --- /dev/null +++ b/arch/x86/include/asm/kvm-x86-nested-ops.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#if !defined(KVM_X86_NESTED_OP) || \ + !defined(KVM_X86_NESTED_OP_OPTIONAL) || \ + !defined(KVM_X86_NESTED_OP_OPTIONAL_RET0) +#error Missing one or more KVM_X86_NESTED_OP #defines +#else +/* + * KVM_X86_NESTED_OP() and KVM_X86_NESTED_OP_OPTIONAL() are used to help + * generate both DECLARE/DEFINE_STATIC_CALL() invocations and + * "static_call_update()" calls. + * + * KVM_X86_NESTED_OP_OPTIONAL() can be used for those functions that can have + * a NULL definition. KVM_X86_NESTED_OP_OPTIONAL_RET0() can be used likewise + * to make a definition optional, but in this case the default will + * be __static_call_return0. + */ +KVM_X86_NESTED_OP(leave_nested) +KVM_X86_NESTED_OP(is_exception_vmexit) +KVM_X86_NESTED_OP(check_events) +KVM_X86_NESTED_OP_OPTIONAL_RET0(has_events) +KVM_X86_NESTED_OP(triple_fault) +KVM_X86_NESTED_OP(get_state) +KVM_X86_NESTED_OP(set_state) +KVM_X86_NESTED_OP(get_nested_state_pages) +KVM_X86_NESTED_OP_OPTIONAL_RET0(write_log_dirty) +KVM_X86_NESTED_OP(translate_nested_gpa) +#ifdef CONFIG_KVM_HYPERV +KVM_X86_NESTED_OP_OPTIONAL(enable_evmcs) +KVM_X86_NESTED_OP_OPTIONAL_RET0(get_evmcs_version) +KVM_X86_NESTED_OP(hv_inject_synthetic_vmexit_post_tlb_flush) +#endif +#endif + +#undef KVM_X86_NESTED_OP +#undef KVM_X86_NESTED_OP_OPTIONAL +#undef KVM_X86_NESTED_OP_OPTIONAL_RET0 diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index a46504077b3e..80358a137b73 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1786,6 +1786,14 @@ extern struct kvm_x86_ops kvm_x86_ops; #define KVM_X86_OP_OPTIONAL_RET0 KVM_X86_OP #include +#define kvm_nested_call(func) static_call(kvm_x86_nested_##func) + +#define KVM_X86_NESTED_OP(func) \ + DECLARE_STATIC_CALL(kvm_x86_nested_##func, *(((struct kvm_x86_nested_ops *)0)->func)); +#define KVM_X86_NESTED_OP_OPTIONAL KVM_X86_NESTED_OP +#define KVM_X86_NESTED_OP_OPTIONAL_RET0 KVM_X86_NESTED_OP +#include + #define __KVM_HAVE_ARCH_VM_ALLOC static inline struct kvm *kvm_arch_alloc_vm(void) { diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 72036670c1ac..1b837e018220 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -2410,7 +2410,7 @@ static int kvm_hv_hypercall_complete(struct kvm_vcpu *vcpu, u64 result) ret = kvm_skip_emulated_instruction(vcpu); if (tlb_lock_count) - kvm_x86_ops.nested_ops->hv_inject_synthetic_vmexit_post_tlb_flush(vcpu); + kvm_nested_call(hv_inject_synthetic_vmexit_post_tlb_flush)(vcpu); return ret; } @@ -2791,9 +2791,8 @@ int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, }; int i, nent = ARRAY_SIZE(cpuid_entries); - if (kvm_x86_ops.nested_ops->enabled && - kvm_x86_ops.nested_ops->get_evmcs_version) - evmcs_ver = kvm_x86_ops.nested_ops->get_evmcs_version(vcpu); + if (kvm_x86_ops.nested_ops->enabled) + evmcs_ver = kvm_nested_call(get_evmcs_version)(vcpu); if (cpuid->nent < nent) return -E2BIG; diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h index 85bc503de1b6..2ae7f9ed4cf8 100644 --- a/arch/x86/kvm/mmu.h +++ b/arch/x86/kvm/mmu.h @@ -385,9 +385,8 @@ static inline gpa_t kvm_translate_gpa(struct kvm_vcpu *vcpu, { if (!mmu_is_nested(vcpu) || w == &vcpu->arch.ngpa_walk) return gpa; - return kvm_x86_ops.nested_ops->translate_nested_gpa(vcpu, gpa, access, - exception, - pte_access); + return kvm_nested_call(translate_nested_gpa)(vcpu, gpa, access, + exception, pte_access); } static inline bool kvm_has_mirrored_tdp(const struct kvm *kvm) diff --git a/arch/x86/kvm/mmu/paging_tmpl.h b/arch/x86/kvm/mmu/paging_tmpl.h index e73fc09ec4db..4ee7b03e762d 100644 --- a/arch/x86/kvm/mmu/paging_tmpl.h +++ b/arch/x86/kvm/mmu/paging_tmpl.h @@ -235,7 +235,7 @@ static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu, !(pte & PT_GUEST_DIRTY_MASK)) { trace_kvm_mmu_set_dirty_bit(table_gfn, index, sizeof(pte)); #if PTTYPE == PTTYPE_EPT - if (kvm_x86_ops.nested_ops->write_log_dirty(vcpu, addr)) + if (kvm_nested_call(write_log_dirty)(vcpu, addr)) return -EINVAL; #endif pte |= PT_GUEST_DIRTY_MASK; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 566a017eab1d..9851db73ca4d 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -174,6 +174,13 @@ EXPORT_STATIC_CALL_GPL(kvm_x86_get_cs_db_l_bits); EXPORT_STATIC_CALL_GPL(kvm_x86_cache_reg); EXPORT_STATIC_CALL_GPL(kvm_x86_get_cpl); +#define KVM_X86_NESTED_OP(func) \ + DEFINE_STATIC_CALL_NULL(kvm_x86_nested_##func, \ + *(((struct kvm_x86_nested_ops *)0)->func)); +#define KVM_X86_NESTED_OP_OPTIONAL KVM_X86_NESTED_OP +#define KVM_X86_NESTED_OP_OPTIONAL_RET0 KVM_X86_NESTED_OP +#include + unsigned int min_timer_period_us = 200; module_param(min_timer_period_us, uint, 0644); @@ -463,7 +470,7 @@ static void kvm_multiple_exception(struct kvm_vcpu *vcpu, unsigned int nr, * wants to intercept the exception. */ if (is_guest_mode(vcpu) && - kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, nr, error_code)) { + kvm_nested_call(is_exception_vmexit)(vcpu, nr, error_code)) { kvm_queue_exception_vmexit(vcpu, nr, has_error, error_code, has_payload, payload); return; @@ -2356,7 +2363,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) break; case KVM_CAP_NESTED_STATE: r = kvm_x86_ops.nested_ops->enabled ? - kvm_x86_ops.nested_ops->get_state(NULL, NULL, 0) : 0; + kvm_nested_call(get_state)(NULL, NULL, 0) : 0; break; #ifdef CONFIG_KVM_HYPERV case KVM_CAP_HYPERV_DIRECT_TLBFLUSH: @@ -3379,7 +3386,7 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, if (!kvm_x86_ops.nested_ops->enabled || !kvm_x86_ops.nested_ops->enable_evmcs) return -ENOTTY; - r = kvm_x86_ops.nested_ops->enable_evmcs(vcpu, &vmcs_version); + r = kvm_nested_call(enable_evmcs)(vcpu, &vmcs_version); if (!r) { user_ptr = (void __user *)(uintptr_t)cap->args[0]; if (copy_to_user(user_ptr, &vmcs_version, @@ -3751,8 +3758,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, if (get_user(user_data_size, &user_kvm_nested_state->size)) break; - r = kvm_x86_ops.nested_ops->get_state(vcpu, user_kvm_nested_state, - user_data_size); + r = kvm_nested_call(get_state)(vcpu, user_kvm_nested_state, user_data_size); if (r < 0) break; @@ -3796,7 +3802,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, break; idx = srcu_read_lock(&vcpu->kvm->srcu); - r = kvm_x86_ops.nested_ops->set_state(vcpu, user_kvm_nested_state, &kvm_state); + r = kvm_nested_call(set_state)(vcpu, user_kvm_nested_state, &kvm_state); srcu_read_unlock(&vcpu->kvm->srcu, idx); break; } @@ -6914,6 +6920,20 @@ static void kvm_setup_efer_caps(void) kvm_enable_efer_bits(EFER_AUTOIBRS); } +static void kvm_nested_ops_update(const struct kvm_x86_nested_ops *nested_ops) +{ +#define __KVM_X86_NESTED_OP(func) \ + static_call_update(kvm_x86_nested_##func, nested_ops->func); +#define KVM_X86_NESTED_OP(func) \ + WARN_ON(!nested_ops->func); __KVM_X86_NESTED_OP(func) +#define KVM_X86_NESTED_OP_OPTIONAL __KVM_X86_NESTED_OP +#define KVM_X86_NESTED_OP_OPTIONAL_RET0(func) \ + static_call_update(kvm_x86_nested_##func, (void *)nested_ops->func ? : \ + (void *)__static_call_return0); +#include +#undef __KVM_X86_NESTED_OP +} + static inline void kvm_ops_update(struct kvm_x86_init_ops *ops) { memcpy(&kvm_x86_ops, ops->runtime_ops, sizeof(kvm_x86_ops)); @@ -6929,6 +6949,8 @@ static inline void kvm_ops_update(struct kvm_x86_init_ops *ops) #include #undef __KVM_X86_OP + kvm_nested_ops_update(kvm_x86_ops.nested_ops); + kvm_pmu_ops_update(ops->pmu_ops); } @@ -7465,11 +7487,11 @@ static void post_kvm_run_save(struct kvm_vcpu *vcpu) int kvm_check_nested_events(struct kvm_vcpu *vcpu) { if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { - kvm_x86_ops.nested_ops->triple_fault(vcpu); + kvm_nested_call(triple_fault)(vcpu); return 1; } - return kvm_x86_ops.nested_ops->check_events(vcpu); + return kvm_nested_call(check_events)(vcpu); } static void kvm_inject_exception(struct kvm_vcpu *vcpu) @@ -7707,9 +7729,7 @@ static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu, kvm_x86_call(enable_irq_window)(vcpu); } - if (is_guest_mode(vcpu) && - kvm_x86_ops.nested_ops->has_events && - kvm_x86_ops.nested_ops->has_events(vcpu, true)) + if (is_guest_mode(vcpu) && kvm_nested_call(has_events)(vcpu, true)) *req_immediate_exit = true; /* @@ -8032,7 +8052,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) } if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { - if (unlikely(!kvm_x86_ops.nested_ops->get_nested_state_pages(vcpu))) { + if (unlikely(!kvm_nested_call(get_nested_state_pages)(vcpu))) { r = 0; goto out; } @@ -8084,7 +8104,7 @@ static int vcpu_enter_guest(struct kvm_vcpu *vcpu) } if (kvm_test_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { if (is_guest_mode(vcpu)) - kvm_x86_ops.nested_ops->triple_fault(vcpu); + kvm_nested_call(triple_fault)(vcpu); if (kvm_check_request(KVM_REQ_TRIPLE_FAULT, vcpu)) { vcpu->run->exit_reason = KVM_EXIT_SHUTDOWN; @@ -8502,9 +8522,7 @@ bool kvm_vcpu_has_events(struct kvm_vcpu *vcpu) if (kvm_hv_has_stimer_pending(vcpu)) return true; - if (is_guest_mode(vcpu) && - kvm_x86_ops.nested_ops->has_events && - kvm_x86_ops.nested_ops->has_events(vcpu, false)) + if (is_guest_mode(vcpu) && kvm_nested_call(has_events)(vcpu, false)) return true; if (kvm_xen_has_pending_events(vcpu)) @@ -8907,8 +8925,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) * a pending VM-Exit if L1 wants to intercept the exception. */ if (vcpu->arch.exception_from_userspace && is_guest_mode(vcpu) && - kvm_x86_ops.nested_ops->is_exception_vmexit(vcpu, ex->vector, - ex->error_code)) { + kvm_nested_call(is_exception_vmexit)(vcpu, ex->vector, ex->error_code)) { kvm_queue_exception_vmexit(vcpu, ex->vector, ex->has_error_code, ex->error_code, ex->has_payload, ex->payload); diff --git a/arch/x86/kvm/x86.h b/arch/x86/kvm/x86.h index 494d97e9a9c9..e72f8b82da67 100644 --- a/arch/x86/kvm/x86.h +++ b/arch/x86/kvm/x86.h @@ -93,7 +93,7 @@ int kvm_check_nested_events(struct kvm_vcpu *vcpu); /* Forcibly leave the nested mode in cases like a vCPU reset */ static inline void kvm_leave_nested(struct kvm_vcpu *vcpu) { - kvm_x86_ops.nested_ops->leave_nested(vcpu); + kvm_nested_call(leave_nested)(vcpu); } /* From 2c56780d8ec7ecf88ebd6a26ee14f6418ea4f8a0 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Tue, 30 Jun 2026 13:28:28 -0700 Subject: [PATCH 078/121] KVM: x86: Move nested_ops out of kvm_x86_ops, to global kvm_nested_ops Rework KVM's handling of per-vendor nested ops to copy the vendor's ops into a global structure owned by common x86, i.e. treat nested ops just like x86 and PMU ops. In addition to providing consistency across all ops implementations, making a copy of the ops prevents changes to the vendor's ops after KVM is initialized, i.e. guards against goofs where KVM *thinks* it is updating nested ops, but which won't take effect now that KVM uses static calls to invoke vendor hooks. Ignoring the side effects of tagging {svm,vmx}_nested_ops as __initdata, no functional change intended. Link: https://patch.msgid.link/20260630202828.440724-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 4 ++-- arch/x86/kvm/hyperv.c | 2 +- arch/x86/kvm/svm/nested.c | 2 +- arch/x86/kvm/svm/svm.c | 3 +-- arch/x86/kvm/vmx/main.c | 3 +-- arch/x86/kvm/vmx/nested.c | 2 +- arch/x86/kvm/x86.c | 25 +++++++++++++------------ 7 files changed, 20 insertions(+), 21 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 80358a137b73..974a506538aa 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1665,8 +1665,6 @@ struct kvm_x86_ops { void (*update_cpu_dirty_logging)(struct kvm_vcpu *vcpu); - const struct kvm_x86_nested_ops *nested_ops; - void (*vcpu_blocking)(struct kvm_vcpu *vcpu); void (*vcpu_unblocking)(struct kvm_vcpu *vcpu); @@ -1762,6 +1760,7 @@ struct kvm_x86_init_ops { struct kvm_x86_ops *runtime_ops; struct kvm_pmu_ops *pmu_ops; + struct kvm_x86_nested_ops *nested_ops; }; struct kvm_arch_async_pf { @@ -1777,6 +1776,7 @@ extern bool __read_mostly enable_apicv; extern bool __read_mostly enable_ipiv; extern bool __read_mostly enable_device_posted_irqs; extern struct kvm_x86_ops kvm_x86_ops; +extern struct kvm_x86_nested_ops kvm_nested_ops __read_mostly; #define kvm_x86_call(func) static_call(kvm_x86_##func) diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 1b837e018220..39b58f56308b 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -2791,7 +2791,7 @@ int kvm_get_hv_cpuid(struct kvm_vcpu *vcpu, struct kvm_cpuid2 *cpuid, }; int i, nent = ARRAY_SIZE(cpuid_entries); - if (kvm_x86_ops.nested_ops->enabled) + if (kvm_nested_ops.enabled) evmcs_ver = kvm_nested_call(get_evmcs_version)(vcpu); if (cpuid->nent < nent) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index ba985a02208a..5f6d9971a3f2 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -2162,7 +2162,7 @@ static gpa_t svm_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, return w->gva_to_gpa(vcpu, w, gpa, access, exception); } -struct kvm_x86_nested_ops svm_nested_ops = { +struct kvm_x86_nested_ops svm_nested_ops __initdata = { .leave_nested = svm_leave_nested, .translate_nested_gpa = svm_translate_nested_gpa, .is_exception_vmexit = nested_svm_is_exception_vmexit, diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 163440fca59f..d3807f4abb49 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5426,8 +5426,6 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .check_intercept = svm_check_intercept, .handle_exit_irqoff = svm_handle_exit_irqoff, - .nested_ops = &svm_nested_ops, - .deliver_interrupt = svm_deliver_interrupt, .pi_update_irte = avic_pi_update_irte, .setup_mce = svm_setup_mce, @@ -5773,6 +5771,7 @@ static struct kvm_x86_init_ops svm_init_ops __initdata = { .runtime_ops = &svm_x86_ops, .pmu_ops = &amd_pmu_ops, + .nested_ops = &svm_nested_ops, }; static void __svm_exit(void) diff --git a/arch/x86/kvm/vmx/main.c b/arch/x86/kvm/vmx/main.c index 83d9921277ea..04f986e3d439 100644 --- a/arch/x86/kvm/vmx/main.c +++ b/arch/x86/kvm/vmx/main.c @@ -995,8 +995,6 @@ struct kvm_x86_ops vt_x86_ops __initdata = { .update_cpu_dirty_logging = vt_op(update_cpu_dirty_logging), - .nested_ops = &vmx_nested_ops, - .pi_update_irte = vmx_pi_update_irte, .pi_start_bypass = vmx_pi_start_bypass, @@ -1038,6 +1036,7 @@ struct kvm_x86_init_ops vt_init_ops __initdata = { .runtime_ops = &vt_x86_ops, .pmu_ops = &intel_pmu_ops, + .nested_ops = &vmx_nested_ops, }; static void __exit vt_exit(void) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 6501fb314575..a9af4e9e6657 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -7486,7 +7486,7 @@ static gpa_t vmx_translate_nested_gpa(struct kvm_vcpu *vcpu, gpa_t gpa, return w->gva_to_gpa(vcpu, w, gpa, access, exception); } -struct kvm_x86_nested_ops vmx_nested_ops = { +struct kvm_x86_nested_ops vmx_nested_ops __initdata = { .leave_nested = vmx_leave_nested, .translate_nested_gpa = vmx_translate_nested_gpa, .is_exception_vmexit = nested_vmx_is_exception_vmexit, diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 9851db73ca4d..2982e3bc2821 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -163,6 +163,7 @@ static int sync_regs(struct kvm_vcpu *vcpu); static DEFINE_MUTEX(vendor_module_lock); struct kvm_x86_ops kvm_x86_ops __read_mostly; +struct kvm_x86_nested_ops kvm_nested_ops __read_mostly; #define KVM_X86_OP(func) \ DEFINE_STATIC_CALL_NULL(kvm_x86_##func, \ @@ -2362,16 +2363,14 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) r &= ~KVM_X2APIC_ENABLE_SUPPRESS_EOI_BROADCAST; break; case KVM_CAP_NESTED_STATE: - r = kvm_x86_ops.nested_ops->enabled ? - kvm_nested_call(get_state)(NULL, NULL, 0) : 0; + r = kvm_nested_ops.enabled ? kvm_nested_call(get_state)(NULL, NULL, 0) : 0; break; #ifdef CONFIG_KVM_HYPERV case KVM_CAP_HYPERV_DIRECT_TLBFLUSH: r = kvm_x86_ops.enable_l2_tlb_flush != NULL; break; case KVM_CAP_HYPERV_ENLIGHTENED_VMCS: - r = kvm_x86_ops.nested_ops->enabled && - kvm_x86_ops.nested_ops->enable_evmcs != NULL; + r = kvm_nested_ops.enabled && kvm_nested_ops.enable_evmcs != NULL; break; #endif case KVM_CAP_SMALLER_MAXPHYADDR: @@ -3383,8 +3382,8 @@ static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, uint16_t vmcs_version; void __user *user_ptr; - if (!kvm_x86_ops.nested_ops->enabled || - !kvm_x86_ops.nested_ops->enable_evmcs) + if (!kvm_nested_ops.enabled || + !kvm_nested_ops.enable_evmcs) return -ENOTTY; r = kvm_nested_call(enable_evmcs)(vcpu, &vmcs_version); if (!r) { @@ -3750,7 +3749,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, u32 user_data_size; r = -EINVAL; - if (!kvm_x86_ops.nested_ops->enabled) + if (!kvm_nested_ops.enabled) break; BUILD_BUG_ON(sizeof(user_data_size) != sizeof(user_kvm_nested_state->size)); @@ -3779,7 +3778,7 @@ long kvm_arch_vcpu_ioctl(struct file *filp, int idx; r = -EINVAL; - if (!kvm_x86_ops.nested_ops->enabled) + if (!kvm_nested_ops.enabled) break; r = -EFAULT; @@ -6922,13 +6921,15 @@ static void kvm_setup_efer_caps(void) static void kvm_nested_ops_update(const struct kvm_x86_nested_ops *nested_ops) { + memcpy(&kvm_nested_ops, nested_ops, sizeof(kvm_nested_ops)); + #define __KVM_X86_NESTED_OP(func) \ - static_call_update(kvm_x86_nested_##func, nested_ops->func); + static_call_update(kvm_x86_nested_##func, kvm_nested_ops.func); #define KVM_X86_NESTED_OP(func) \ - WARN_ON(!nested_ops->func); __KVM_X86_NESTED_OP(func) + WARN_ON(!kvm_nested_ops.func); __KVM_X86_NESTED_OP(func) #define KVM_X86_NESTED_OP_OPTIONAL __KVM_X86_NESTED_OP #define KVM_X86_NESTED_OP_OPTIONAL_RET0(func) \ - static_call_update(kvm_x86_nested_##func, (void *)nested_ops->func ? : \ + static_call_update(kvm_x86_nested_##func, (void *)kvm_nested_ops.func ? : \ (void *)__static_call_return0); #include #undef __KVM_X86_NESTED_OP @@ -6949,7 +6950,7 @@ static inline void kvm_ops_update(struct kvm_x86_init_ops *ops) #include #undef __KVM_X86_OP - kvm_nested_ops_update(kvm_x86_ops.nested_ops); + kvm_nested_ops_update(ops->nested_ops); kvm_pmu_ops_update(ops->pmu_ops); } From d99fc2a7c5e22734753ad7f55161fe93eaf88013 Mon Sep 17 00:00:00 2001 From: Ewan Hai-oc Date: Wed, 10 Jun 2026 10:35:08 +0800 Subject: [PATCH 079/121] KVM: x86: Expose Zhaoxin SM2 CPUID feature Advertise the Zhaoxin SM2 instruction support to guests via CPUID 0xC0000001 EDX bits 0 (SM2) and 1 (SM2_EN). The SM2 instruction (encoding F2 0F A6 C0) implements the SM2 elliptic-curve public-key cryptography algorithm specified in GM/T 0003-2012; the hardware-level behavior is documented in the Zhaoxin GMI Instruction Set Reference, chapter 1 ("SM2"). The instruction multiplexes its sub-functions on the RDX[5:0] control word: encryption (subsection 1.1), decryption (1.2), signing (1.3), signature verification (1.4), the three key-exchange sub-operations of section 1.5 (1.5.1 SM2 key-pair generation, which the spec also uses for the initiator's ephemeral key; 1.5.2 responder shared-key derivation; 1.5.3 initiator shared-key derivation), and two preprocess steps for identity and message hashing (1.6.1 and 1.6.2). The instruction is unprivileged (no CPL restriction) and available in all CPU modes, with no associated MSR control. The SM2 and SM2_EN bits are redundant by hardware design (set or cleared together) and both serve purely as CPUID-level feature-presence reporting flags requiring no KVM emulation. Both bits are advertised because different software may probe either one when checking for SM2 availability. Reviewed-by: Binbin Wu Signed-off-by: Ewan Hai Link: https://patch.msgid.link/20260610023512.3690734-2-ewanhai-oc@zhaoxin.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/cpufeatures.h | 2 ++ arch/x86/kvm/cpuid.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 35a2a0f9ab32..f9baec51f6db 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -136,6 +136,8 @@ #define X86_FEATURE_HYPERVISOR ( 4*32+31) /* "hypervisor" Running on a hypervisor */ /* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000001, word 5 */ +#define X86_FEATURE_SM2 ( 5*32+ 0) /* "sm2" SM2 algorithm */ +#define X86_FEATURE_SM2_EN ( 5*32+ 1) /* "sm2_en" SM2 enabled */ #define X86_FEATURE_XSTORE ( 5*32+ 2) /* "rng" RNG present (xstore) */ #define X86_FEATURE_XSTORE_EN ( 5*32+ 3) /* "rng_en" RNG enabled */ #define X86_FEATURE_XCRYPT ( 5*32+ 6) /* "ace" on-CPU crypto (xcrypt) */ diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index f402a5dc4390..b2f860bb594f 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -1273,6 +1273,8 @@ void kvm_initialize_cpu_caps(void) kvm_cpu_cap_set(X86_FEATURE_NULL_SEL_CLR_BASE); kvm_cpu_cap_init(CPUID_C000_0001_EDX, + F(SM2), + F(SM2_EN), F(XSTORE), F(XSTORE_EN), F(XCRYPT), From 3ee865eb696fff41dde57697e604d397322cb4d2 Mon Sep 17 00:00:00 2001 From: Ewan Hai-oc Date: Wed, 10 Jun 2026 10:35:09 +0800 Subject: [PATCH 080/121] KVM: x86: Expose Zhaoxin CCS (SM3 + SM4) CPUID feature Advertise the Zhaoxin CCS (Chinese Cryptography Standard) feature to guests via CPUID 0xC0000001 EDX bits 4 (CCS) and 5 (CCS_EN). CCS groups two unprivileged instructions for Chinese national cryptographic primitives, documented in the Zhaoxin GMI Instruction Set Reference, chapter 2 ("CCS instruction group"): - SM3 (encoding F3 0F A6 E8, subsection 2.1) implements the SM3 hash algorithm specified in GM/T 0004-2012. It supports two modes selected by RAX: auto-padding stream mode (RAX=0) and pre-padded block mode (RAX=-1). - SM4 (encoding F3 0F A7 F0, subsection 2.2) implements the SM4 block cipher specified in GM/T 0002-2012, supporting ECB / CBC / CFB / OFB / CTR modes via a control word in RAX, and CBC-MAC / CFB-MAC when RAX bit[11] is set. Both instructions are unprivileged (no CPL restriction) and available in all CPU modes, with no associated MSR control. The CCS and CCS_EN bits are redundant by hardware design (set or cleared together) and both serve purely as CPUID-level feature-presence reporting flags requiring no KVM emulation. Both bits are advertised because different software may probe either one when checking for CCS availability. Reviewed-by: Binbin Wu Signed-off-by: Ewan Hai Link: https://patch.msgid.link/20260610023512.3690734-3-ewanhai-oc@zhaoxin.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/cpufeatures.h | 2 ++ arch/x86/kvm/cpuid.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index f9baec51f6db..1e1d4addcc52 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -140,6 +140,8 @@ #define X86_FEATURE_SM2_EN ( 5*32+ 1) /* "sm2_en" SM2 enabled */ #define X86_FEATURE_XSTORE ( 5*32+ 2) /* "rng" RNG present (xstore) */ #define X86_FEATURE_XSTORE_EN ( 5*32+ 3) /* "rng_en" RNG enabled */ +#define X86_FEATURE_CCS ( 5*32+ 4) /* "ccs" SM3 + SM4 instructions */ +#define X86_FEATURE_CCS_EN ( 5*32+ 5) /* "ccs_en" CCS enabled */ #define X86_FEATURE_XCRYPT ( 5*32+ 6) /* "ace" on-CPU crypto (xcrypt) */ #define X86_FEATURE_XCRYPT_EN ( 5*32+ 7) /* "ace_en" on-CPU crypto enabled */ #define X86_FEATURE_ACE2 ( 5*32+ 8) /* "ace2" Advanced Cryptography Engine v2 */ diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index b2f860bb594f..6eb394be5133 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -1277,6 +1277,8 @@ void kvm_initialize_cpu_caps(void) F(SM2_EN), F(XSTORE), F(XSTORE_EN), + F(CCS), + F(CCS_EN), F(XCRYPT), F(XCRYPT_EN), F(ACE2), From 93134f14106b2b0f275d859b9f34ea3d0e3cbb88 Mon Sep 17 00:00:00 2001 From: Ewan Hai-oc Date: Wed, 10 Jun 2026 10:35:10 +0800 Subject: [PATCH 081/121] KVM: x86: Expose Zhaoxin RNG2 CPUID feature Advertise the Zhaoxin second-generation hardware RNG to guests via CPUID 0xC0000001 EDX bits 22 (RNG2) and 23 (RNG2_EN). RNG2 is exposed by the REP XRNG2 instruction (encoding F3 0F A7 F8), documented in the Zhaoxin PadLock Instruction Reference, subsection 1.3 ("REP XRNG2"). It produces random bytes from two on-die RNG sources selectable via RAX bits[10:9] and an output mode (raw vs post-processed) controlled by RDX bits[1:0], providing high-quality entropy intended for cryptographic operations. REP XRNG2 is unprivileged (no CPL restriction) and available in all CPU modes, with no associated MSR control. The RNG2 and RNG2_EN bits are redundant by hardware design (set or cleared together) and both serve purely as CPUID-level feature-presence reporting flags requiring no KVM emulation. Both bits are advertised because different software may probe either one when checking for RNG2 availability. Reviewed-by: Binbin Wu Signed-off-by: Ewan Hai Link: https://patch.msgid.link/20260610023512.3690734-4-ewanhai-oc@zhaoxin.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/cpufeatures.h | 2 ++ arch/x86/kvm/cpuid.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 1e1d4addcc52..55bf3510884a 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -150,6 +150,8 @@ #define X86_FEATURE_PHE_EN ( 5*32+11) /* "phe_en" PHE enabled */ #define X86_FEATURE_PMM ( 5*32+12) /* "pmm" PadLock Montgomery Multiplier */ #define X86_FEATURE_PMM_EN ( 5*32+13) /* "pmm_en" PMM enabled */ +#define X86_FEATURE_RNG2 ( 5*32+22) /* "rng2" RNG v2 */ +#define X86_FEATURE_RNG2_EN ( 5*32+23) /* "rng2_en" RNG2 enabled */ /* More extended AMD flags: CPUID level 0x80000001, ECX, word 6 */ #define X86_FEATURE_LAHF_LM ( 6*32+ 0) /* "lahf_lm" LAHF/SAHF in long mode */ diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 6eb394be5133..37aa3e64e40b 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -1287,6 +1287,8 @@ void kvm_initialize_cpu_caps(void) F(PHE_EN), F(PMM), F(PMM_EN), + F(RNG2), + F(RNG2_EN), ); /* From 54db9aaacc2af382fbceab7e83d45391c58240e3 Mon Sep 17 00:00:00 2001 From: Ewan Hai-oc Date: Wed, 10 Jun 2026 10:35:11 +0800 Subject: [PATCH 082/121] KVM: x86: Expose Zhaoxin PHE2 CPUID feature Advertise the Zhaoxin PadLock Hash Engine v2 to guests via CPUID 0xC0000001 EDX bits 25 (PHE2) and 26 (PHE2_EN). PHE2 extends the PadLock hash family with SHA-384 and SHA-512 support per FIPS 180-3, complementing the existing PHE feature (SHA-1 and SHA-256). Two unprivileged instructions are exposed, documented in the Zhaoxin PadLock Instruction Reference, chapter 3 ("Hash Engine"): - REP XSHA384 (encoding F3 0F A6 D8, subsection 3.3) - REP XSHA512 (encoding F3 0F A6 E0, subsection 3.4) Both consume software-padded 128-byte blocks (RCX = block count, RSI = input, RDI = state) and produce hash output in the state buffer. Both instructions are unprivileged (no CPL restriction) and available in all CPU modes, with no associated MSR control. The PHE2 and PHE2_EN bits are redundant by hardware design (set or cleared together) and both serve purely as CPUID-level feature-presence reporting flags requiring no KVM emulation. Both bits are advertised because different software may probe either one when checking for PHE2 availability. Reviewed-by: Binbin Wu Signed-off-by: Ewan Hai Link: https://patch.msgid.link/20260610023512.3690734-5-ewanhai-oc@zhaoxin.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/cpufeatures.h | 2 ++ arch/x86/kvm/cpuid.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index 55bf3510884a..baf6d13fe705 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -152,6 +152,8 @@ #define X86_FEATURE_PMM_EN ( 5*32+13) /* "pmm_en" PMM enabled */ #define X86_FEATURE_RNG2 ( 5*32+22) /* "rng2" RNG v2 */ #define X86_FEATURE_RNG2_EN ( 5*32+23) /* "rng2_en" RNG2 enabled */ +#define X86_FEATURE_PHE2 ( 5*32+25) /* "phe2" PadLock Hash Engine v2 */ +#define X86_FEATURE_PHE2_EN ( 5*32+26) /* "phe2_en" PHE2 enabled */ /* More extended AMD flags: CPUID level 0x80000001, ECX, word 6 */ #define X86_FEATURE_LAHF_LM ( 6*32+ 0) /* "lahf_lm" LAHF/SAHF in long mode */ diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 37aa3e64e40b..9313262bafc0 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -1289,6 +1289,8 @@ void kvm_initialize_cpu_caps(void) F(PMM_EN), F(RNG2), F(RNG2_EN), + F(PHE2), + F(PHE2_EN), ); /* From dcfb0f2c067a778b6f0da4f49b074df176bb98af Mon Sep 17 00:00:00 2001 From: Ewan Hai-oc Date: Wed, 10 Jun 2026 10:35:12 +0800 Subject: [PATCH 083/121] KVM: x86: Expose Zhaoxin RSA CPUID feature Advertise the Zhaoxin big-number arithmetic engine to guests via CPUID 0xC0000001 EDX bits 27 (RSA) and 28 (RSA_EN). The RSA feature provides two unprivileged instructions for modular arithmetic on big integers, documented in the Zhaoxin PadLock Instruction Reference, chapter 4 ("Modular Multiplication and Exponentiation Engine"). Both support operand sizes from 256 to 32768 bits (in 128-bit increments): - REP XMODEXP (encoding F3 0F A6 F8, subsection 4.1) computes A^B mod M - REP MONTMUL2 (encoding F3 0F A6 F0, subsection 4.2) computes A*B mod M REP MONTMUL2 is the long-mode replacement of legacy REP MONTMUL, which is restricted to compatibility and 32-bit protected modes. These primitives accelerate RSA and related public-key operations. Both instructions are unprivileged (no CPL restriction) and available in all CPU modes, with no associated MSR control. The RSA and RSA_EN bits are redundant by hardware design (set or cleared together) and both serve purely as CPUID-level feature-presence reporting flags requiring no KVM emulation. Both bits are advertised because different software may probe either one when checking for RSA availability. Reviewed-by: Binbin Wu Signed-off-by: Ewan Hai Link: https://patch.msgid.link/20260610023512.3690734-6-ewanhai-oc@zhaoxin.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/cpufeatures.h | 2 ++ arch/x86/kvm/cpuid.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/arch/x86/include/asm/cpufeatures.h b/arch/x86/include/asm/cpufeatures.h index baf6d13fe705..6f03badfef71 100644 --- a/arch/x86/include/asm/cpufeatures.h +++ b/arch/x86/include/asm/cpufeatures.h @@ -154,6 +154,8 @@ #define X86_FEATURE_RNG2_EN ( 5*32+23) /* "rng2_en" RNG2 enabled */ #define X86_FEATURE_PHE2 ( 5*32+25) /* "phe2" PadLock Hash Engine v2 */ #define X86_FEATURE_PHE2_EN ( 5*32+26) /* "phe2_en" PHE2 enabled */ +#define X86_FEATURE_RSA ( 5*32+27) /* "rsa" Big-number arithmetic */ +#define X86_FEATURE_RSA_EN ( 5*32+28) /* "rsa_en" RSA enabled */ /* More extended AMD flags: CPUID level 0x80000001, ECX, word 6 */ #define X86_FEATURE_LAHF_LM ( 6*32+ 0) /* "lahf_lm" LAHF/SAHF in long mode */ diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 9313262bafc0..9e9cf6538a96 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -1291,6 +1291,8 @@ void kvm_initialize_cpu_caps(void) F(RNG2_EN), F(PHE2), F(PHE2_EN), + F(RSA), + F(RSA_EN), ); /* From 249f9be58c7fd061e3323419c9adfeb938a2c48c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20L=C3=B3pez?= Date: Tue, 9 Jun 2026 15:18:55 +0200 Subject: [PATCH 084/121] KVM: x86: Fix array_index_nospec() protection in kvm_vcpu_ioctl_x86_set_mce() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit aebc3ca19063 ("KVM: x86: Enable CMCI capability by default and handle injected UCNA errors") introduced kvm_vcpu_x86_set_ucna(), which accesses @vcpu->arch.mci_ctl2_banks[] using @mce->bank as the index. The @mce struct is user-controlled, provided via the KVM_X86_SET_MCE ioctl. The caller of this function, kvm_vcpu_ioctl_x86_set_mce(), bounds-checks @mce->bank and applies array_index_nospec() to advance the @banks pointer, but @mce->bank itself is passed through unclamped. On a speculative path that bypasses the bounds check, the raw @mce->bank value can index mci_ctl2_banks[] out-of-bounds. In practice this is a very weak gadget, and would at most allow leaking a single bit in a 64-bit integer, but prevent potential future issues by clamping @mce->bank in place with array_index_nospec(), before passing the struct to kvm_vcpu_x86_set_ucna(). Fixes: aebc3ca19063 ("KVM: x86: Enable CMCI capability by default and handle injected UCNA errors") Signed-off-by: Carlos López Link: https://patch.msgid.link/20260609131856.2562222-3-clopez@suse.de Signed-off-by: Sean Christopherson --- arch/x86/kvm/x86.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 2982e3bc2821..d9cff1f8dff3 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -2876,7 +2876,8 @@ static int kvm_vcpu_ioctl_x86_set_mce(struct kvm_vcpu *vcpu, if (mce->bank >= bank_num || !(mce->status & MCI_STATUS_VAL)) return -EINVAL; - banks += array_index_nospec(4 * mce->bank, 4 * bank_num); + mce->bank = array_index_nospec(mce->bank, bank_num); + banks += 4 * mce->bank; if (is_ucna(mce)) return kvm_vcpu_x86_set_ucna(vcpu, mce, banks); From 5ef3668bcea2eacc4a06204a46602af34b22a7f3 Mon Sep 17 00:00:00 2001 From: Wang Yan Date: Thu, 2 Jul 2026 09:57:39 +0800 Subject: [PATCH 085/121] KVM: selftests: Fix a spelling error in an xapic_ipi_test comment Fix typo "usefull" -> "useful" in xAPIC IPI test comment. Signed-off-by: Wang Yan Link: https://patch.msgid.link/20260702015739.367597-1-wangyan01@kylinos.cn Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/x86/xapic_ipi_test.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c index 39ce9a9369f5..3a326c5e74ca 100644 --- a/tools/testing/selftests/kvm/x86/xapic_ipi_test.c +++ b/tools/testing/selftests/kvm/x86/xapic_ipi_test.c @@ -17,7 +17,7 @@ * amongst the available numa nodes on the machine. * * Migration is a command line option. When used on non-numa machines will - * exit with error. Test is still usefull on non-numa for testing IPIs. + * exit with error. Test is still useful on non-numa for testing IPIs. */ #include #include From 9498f4d6edd358abff0000f207cf2de1f0b59145 Mon Sep 17 00:00:00 2001 From: Kai Huang Date: Wed, 15 Jul 2026 21:05:05 +1200 Subject: [PATCH 086/121] KVM: x86: Use KVM_X86_OP() for the .pi_update_irte() hook Change to using KVM_X86_OP() instead of KVM_X86_OP_OPTIONAL() for the .pi_update_irte() hook in kvm-x86-ops.h since now both VMX and SVM have implemented it. For the Fixes tag: This hook was introduced for VMX posted-interrupt support. SVM later added its implementation, but at this point KVM_X86_OP* had not been introduced yet. Initially KVM introduced KVM_X86_OP_NULL (and KVM_X86_OP) and used it for this hook. But this was correct, because the use of KVM_X86_OP_NULL was "to mark calls that do not follow the [svm|vmx]_func_name naming convention" and the VMX one was named pi_update_irte(), i.e., did not follow the convention. See commit 9af5471bdbb2 ("KVM: x86: introduce definitions to support static calls for kvm_x86_ops"). KVM later removed KVM_X86_OP_NULL (due to "the naming convention is not in use anymore"), and added KVM_X86_OP_OPTIONAL for the hooks that can be NULL pointer. It used KVM_X86_OP_OPTIONAL for this hook, but should use KVM_X86_OP instead. See commit e4fc23bad813 ("KVM: x86: remove KVM_X86_OP_NULL and mark optional kvm_x86_ops"). Note this hook was named .update_pi_irte() when it was introduced, but got renamed to .pi_update_irte() at some point between the above two commits. Fixes: e4fc23bad813 ("KVM: x86: remove KVM_X86_OP_NULL and mark optional kvm_x86_ops") Signed-off-by: Kai Huang Link: https://patch.msgid.link/20260715090505.601174-1-kai.huang@intel.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 83dc5086138b..d5e9a1b1dba3 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -110,7 +110,7 @@ KVM_X86_OP(handle_exit_irqoff) KVM_X86_OP_OPTIONAL(update_cpu_dirty_logging) KVM_X86_OP_OPTIONAL(vcpu_blocking) KVM_X86_OP_OPTIONAL(vcpu_unblocking) -KVM_X86_OP_OPTIONAL(pi_update_irte) +KVM_X86_OP(pi_update_irte) KVM_X86_OP_OPTIONAL(pi_start_bypass) KVM_X86_OP_OPTIONAL(apicv_pre_state_restore) KVM_X86_OP_OPTIONAL(apicv_post_state_restore) From 0ca49fbd2883cd53d32d85b50feef17fa04d0fbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20L=C3=B3pez?= Date: Tue, 14 Jul 2026 15:32:13 +0200 Subject: [PATCH 087/121] KVM: x86: hyper-v: Clamp stimer deadline to avoid livelock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix an issue where userspace or the guest can program an Hyper-V synthetic timer to have a deadline in the past via integer overflow, preventing the CPU from making progress and triggering an RCU stall. Hyper-V's SynIC exposes 4 per-vCPU synthetic timers to the guest, which are emulated by KVM. Each is programmed through the HV_X64_MSR_STIMERi_CONFIG and HV_X64_MSR_STIMERi_COUNT MSRs. Depending on CONFIG, COUNT represents either the absolute expiration time or the period of a periodic timer, both expressed in 100ns ticks. These timers may be set both by the guest (WRMSR) and the host (KVM_SET_MSRS). When the timer is enabled, stimer_start() translates COUNT to an absolute monotonic deadline and arms an hrtimer. If COUNT is set to a value close to U64_MAX, the deadline calculation can overflow. ktime_add_ns(ktime_now, 100 * (stimer->exp_time - time_now)) This can result in a CPU livelock. stimer_start() arms the timer via hrtimer_start() with a deadline in the past, which causes it to immediately fire. The stimer callback then raises KVM_RQ_HV_STIMER, with the intention of causing KVM to deliver a synthetic interrupt on the next vCPU guest enter. Then, once userspace issues KVM_RUN, vcpu_enter_guest() consumes the request, calling kvm_hv_process_stimers(). This would normally disable the timer via stimer_expiration() once the deadline is in the past. However, the deadline comparison is done between the KVM reference counter and stime->exp_time, which is a big value close to U64_MAX, so this never happens for a few thousand years. kvm_hv_process_timers() then re-arms the timer via stimer_start(), since it was not disabled, which again fires immediately. Before entering the guest, kvm_vcpu_exit_request() checks kvm_request_pending(), which returns true due to the newly raised KVM_REQ_HV_STIMER. Then vcpu_enter_guest() aborts the guest entry, returning early into vcpu_run(), which loops back again into vcpu_enter_guest(), restarting the cycle. Since there are no manual yields in this loop, a task with SCHED_FIFO may starve RCU grace-period kthreads, which exposes the stalls found by syzcaller: rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: rcu: (detected by 1, t=10502 jiffies, g=14269, q=1142 ncpus=2) rcu: All QSes seen, last rcu_preempt kthread activity 10500 (4294965239-4294954739), jiffies_till_next_fqs=1, root ->qsmask 0x0 rcu: rcu_preempt kthread starved for 10500 jiffies! g14269 f0x2 RCU_GP_WAIT_FQS(5) ->state=0x0 ->cpu=0 rcu: Unless rcu_preempt kthread gets sufficient CPU time, OOM is now expected behavior. ( ... ) Call Trace: __run_hrtimer kernel/time/hrtimer.c:1773 [inline] __hrtimer_run_queues+0x408/0xc30 kernel/time/hrtimer.c:1841 hrtimer_interrupt+0x45b/0xaa0 kernel/time/hrtimer.c:1903 local_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1045 [inline] __sysvec_apic_timer_interrupt+0x102/0x3e0 arch/x86/kernel/apic/apic.c:1062 instr_sysvec_apic_timer_interrupt arch/x86/kernel/apic/apic.c:1056 [inline] sysvec_apic_timer_interrupt+0xa1/0xc0 arch/x86/kernel/apic/apic.c:1056 asm_sysvec_apic_timer_interrupt+0x1a/0x20 arch/x86/include/asm/idtentry.h:697 RIP: 0010:__raw_spin_unlock_irqrestore include/linux/spinlock_api_smp.h:152 [inline] RIP: 0010:_raw_spin_unlock_irqrestore+0xa8/0x110 kernel/locking/spinlock.c:194 Code: 74 05 e8 0b f4 5f f6 48 c7 44 24 20 00 00 00 00 9c 8f 44 24 20 f6 44 24 21 02 75 4f f7 c3 00 02 00 00 74 01 fb bf 01 00 00 00 23 6b 27 f6 65 8b 05 7c 60 5a 07 85 c0 74 40 48 c7 04 24 0e 36 RSP: 0018:ffffc900040a7320 EFLAGS: 00000206 RAX: 5de15cb931505900 RBX: 0000000000000a06 RCX: 5de15cb931505900 RDX: 0000000000000007 RSI: ffffffff8daa9dc3 RDI: 0000000000000001 RBP: ffffc900040a73b0 R08: ffffffff8fc3d077 R09: 1ffffffff1f87a0e R10: dffffc0000000000 R11: fffffbfff1f87a0f R12: dffffc0000000000 R13: 0000000000000000 R14: ffff8880b8628240 R15: 1ffff92000814e64 hrtimer_start include/linux/hrtimer.h:259 [inline] stimer_start arch/x86/kvm/hyperv.c:682 [inline] kvm_hv_process_stimers+0xd0a/0x16a0 arch/x86/kvm/hyperv.c:893 vcpu_enter_guest arch/x86/kvm/x86.c:11193 [inline] vcpu_run+0x2240/0x76b0 arch/x86/kvm/x86.c:11639 kvm_arch_vcpu_ioctl_run+0x1148/0x1c90 arch/x86/kvm/x86.c:11984 kvm_vcpu_ioctl+0x99a/0xed0 virt/kvm/kvm_main.c:4492 vfs_ioctl fs/ioctl.c:51 [inline] __do_sys_ioctl fs/ioctl.c:597 [inline] __se_sys_ioctl+0xfc/0x170 fs/ioctl.c:583 do_syscall_x64 arch/x86/entry/syscall_64.c:63 [inline] do_syscall_64+0xfa/0xf80 arch/x86/entry/syscall_64.c:94 entry_SYSCALL_64_after_hwframe+0x77/0x7f RIP: 0033:0x7f635278f749 Code: ff ff c3 66 2e 0f 1f 84 00 00 00 00 00 0f 1f 40 00 48 89 f8 48 89 f7 48 89 d6 48 89 ca 4d 89 c2 4d 89 c8 4c 8b 4c 24 08 0f 05 <48> 3d 01 f0 ff ff 73 01 c3 48 c7 c1 a8 ff ff ff f7 d8 64 89 01 48 RSP: 002b:00007f635365c038 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 RAX: ffffffffffffffda RBX: 00007f63529e5fa0 RCX: 00007f635278f749 RDX: 0000000000000000 RSI: 000000000000ae80 RDI: 0000000000000005 RBP: 00007f6352813f91 R08: 0000000000000000 R09: 0000000000000000 R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000 R13: 00007f63529e6038 R14: 00007f63529e5fa0 R15: 00007ffd5b219358 Fix this by clamping the deadline computation to KTIME_MAX, which preserves the intent of arming a timer very far in the future. ktime_add_safe() already does this type of clamping, so use it after checking that that multiplying by the 100ns time tick also does not overflow. Reviewed-by: Vitaly Kuznetsov Reported-by: syzbot+3d5461510f8dc4adfe30@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=3d5461510f8dc4adfe30 Fixes: 1f4b34f825e8 ("kvm/x86: Hyper-V SynIC timers") Cc: stable@vger.kernel.org Signed-off-by: Carlos López Link: https://patch.msgid.link/20260714133212.3916611-3-clopez@suse.de [sean: tag for stable] Signed-off-by: Sean Christopherson --- arch/x86/kvm/hyperv.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/arch/x86/kvm/hyperv.c b/arch/x86/kvm/hyperv.c index 39b58f56308b..3ee6bf35d1e4 100644 --- a/arch/x86/kvm/hyperv.c +++ b/arch/x86/kvm/hyperv.c @@ -630,6 +630,18 @@ static enum hrtimer_restart stimer_timer_callback(struct hrtimer *timer) return HRTIMER_NORESTART; } +/* + * Translate a stimer expiry given in 100ns reference ticks into an + * an absolute deadline. Saturates on overflow. + */ +static ktime_t stimer_add_delta(ktime_t now, u64 delta_100ns) +{ + if (delta_100ns >= KTIME_MAX / 100) + return KTIME_MAX; + + return ktime_add_safe(now, 100 * delta_100ns); +} + /* * stimer_start() assumptions: * a) stimer->count is not equal to 0 @@ -639,6 +651,7 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) { u64 time_now; ktime_t ktime_now; + ktime_t deadline; time_now = get_time_ref_counter(hv_stimer_to_vcpu(stimer)->kvm); ktime_now = ktime_get(); @@ -661,10 +674,8 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) stimer->index, time_now, stimer->exp_time); - hrtimer_start(&stimer->timer, - ktime_add_ns(ktime_now, - 100 * (stimer->exp_time - time_now)), - HRTIMER_MODE_ABS); + deadline = stimer_add_delta(ktime_now, stimer->exp_time - time_now); + hrtimer_start(&stimer->timer, deadline, HRTIMER_MODE_ABS); return 0; } stimer->exp_time = stimer->count; @@ -683,9 +694,9 @@ static int stimer_start(struct kvm_vcpu_hv_stimer *stimer) stimer->index, time_now, stimer->count); - hrtimer_start(&stimer->timer, - ktime_add_ns(ktime_now, 100 * (stimer->count - time_now)), - HRTIMER_MODE_ABS); + deadline = stimer_add_delta(ktime_now, stimer->count - time_now); + hrtimer_start(&stimer->timer, deadline, HRTIMER_MODE_ABS); + return 0; } From 6ccc19d4c1eb97a994180af8afcab58422cb409d Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Mon, 13 Jul 2026 18:10:16 +0000 Subject: [PATCH 088/121] KVM: x86: Move enabling EFER.SVME and EFER.LMSLE to generic EFER setup Move SVM-specific EFER bit enablement to generic x86 code, with the rest of EFER bit enablement. Unifying the code for EFER bit enablement allows for a later change to re-initialize EFER bits on module init. No functional change intended. Cc: stable@vger.kernel.org Suggested-by: Sean Christopherson Signed-off-by: Yosry Ahmed Link: https://patch.msgid.link/20260713181020.2735367-2-yosry@kernel.org Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/svm.c | 4 ---- arch/x86/kvm/x86.c | 6 ++++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index d3807f4abb49..1b7e613fde7b 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5636,10 +5636,6 @@ static __init int svm_hardware_setup(void) if (nested) { pr_info("Nested Virtualization enabled\n"); - kvm_enable_efer_bits(EFER_SVME); - if (!boot_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ)) - kvm_enable_efer_bits(EFER_LMSLE); - r = nested_svm_init_msrpm_merge_offsets(); if (r) return r; diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index d9cff1f8dff3..a4216b858194 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -6918,6 +6918,12 @@ static void kvm_setup_efer_caps(void) if (kvm_cpu_cap_has(X86_FEATURE_AUTOIBRS)) kvm_enable_efer_bits(EFER_AUTOIBRS); + + if (kvm_cpu_cap_has(X86_FEATURE_SVM)) { + kvm_enable_efer_bits(EFER_SVME); + if (!boot_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ)) + kvm_enable_efer_bits(EFER_LMSLE); + } } static void kvm_nested_ops_update(const struct kvm_x86_nested_ops *nested_ops) From e62392bf39ebfdf60d1d082799397fe1cbf8dfc5 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Mon, 13 Jul 2026 18:10:17 +0000 Subject: [PATCH 089/121] KVM: x86: Disallow EFER.LME and EFER.LMA if long mode is not supported Remove EFER.LME and EFER.LMA from EFER reserved bits only if long mode is actually supported. KVM does check long-mode support before allowing the bits for guest writes and userspace writes through KVM_SET_SREGS* (in __kvm_valid_efer()), but userspace writes through KVM_SET_MSRS only check reserved bits. In practice, this doesn't really matter. The true motiviation is getting rid of the #ifdeffery when initializing efer_reserved_bits. Cc: stable@vger.kernel.org Signed-off-by: Yosry Ahmed Link: https://patch.msgid.link/20260713181020.2735367-3-yosry@kernel.org Signed-off-by: Sean Christopherson --- arch/x86/kvm/msrs.c | 10 +--------- arch/x86/kvm/x86.c | 3 +++ 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c index c230b18d87e3..67481429ad6b 100644 --- a/arch/x86/kvm/msrs.c +++ b/arch/x86/kvm/msrs.c @@ -19,16 +19,8 @@ bool __read_mostly report_ignored_msrs = true; module_param(report_ignored_msrs, bool, 0644); EXPORT_SYMBOL_FOR_KVM_INTERNAL(report_ignored_msrs); -/* EFER defaults: - * - enable syscall per default because its emulated by KVM - * - enable LME and LMA per default on 64 bit KVM - */ -#ifdef CONFIG_X86_64 -static -u64 __read_mostly efer_reserved_bits = ~((u64)(EFER_SCE | EFER_LME | EFER_LMA)); -#else +/* Enable syscall by default because its emulated by KVM */ static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); -#endif #define MAX_IO_MSRS 256 diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index a4216b858194..0845551c6cab 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -6910,6 +6910,9 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_setup_xss_caps); static void kvm_setup_efer_caps(void) { + if (kvm_cpu_cap_has(X86_FEATURE_LM)) + kvm_enable_efer_bits(EFER_LME | EFER_LMA); + if (kvm_cpu_cap_has(X86_FEATURE_NX)) kvm_enable_efer_bits(EFER_NX); From a4b8acd428ba3f03952ef309d7f438dbc357557b Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Mon, 13 Jul 2026 18:10:18 +0000 Subject: [PATCH 090/121] KVM: x86: Always initialize EFER reserved bits on vendor initialization EFER reserved bits are statically initialized, and do not reset if a vendor module is re-loaded. For example, loading kvm_amd with nested=1 removes EFER.SVME (and potentially EFER.LMSLE) from the reserved bits. Reloading kvm_amd with nested=0 does not add them back, allowing userspace to set EFER.SVME with nested=0. Re-initializing EFER reserved bits before configuring them on vendor initialization. Cc: stable@vger.kernel.org Signed-off-by: Yosry Ahmed Link: https://patch.msgid.link/20260713181020.2735367-4-yosry@kernel.org Signed-off-by: Sean Christopherson --- arch/x86/kvm/msrs.c | 10 ++++++++-- arch/x86/kvm/msrs.h | 1 + arch/x86/kvm/x86.c | 2 ++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c index 67481429ad6b..a7394bdae029 100644 --- a/arch/x86/kvm/msrs.c +++ b/arch/x86/kvm/msrs.c @@ -19,8 +19,7 @@ bool __read_mostly report_ignored_msrs = true; module_param(report_ignored_msrs, bool, 0644); EXPORT_SYMBOL_FOR_KVM_INTERNAL(report_ignored_msrs); -/* Enable syscall by default because its emulated by KVM */ -static u64 __read_mostly efer_reserved_bits = ~((u64)EFER_SCE); +static u64 __read_mostly efer_reserved_bits; #define MAX_IO_MSRS 256 @@ -650,6 +649,13 @@ static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info) return 0; } +void kvm_init_efer_bits(void) +{ + /* Enable syscall by default because its emulated by KVM */ + efer_reserved_bits = ~((u64)EFER_SCE); +} +EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_init_efer_bits); + void kvm_enable_efer_bits(u64 mask) { efer_reserved_bits &= ~mask; diff --git a/arch/x86/kvm/msrs.h b/arch/x86/kvm/msrs.h index 9c5c6b33e58f..1f772e171758 100644 --- a/arch/x86/kvm/msrs.h +++ b/arch/x86/kvm/msrs.h @@ -58,6 +58,7 @@ int kvm_get_set_one_reg(struct kvm_vcpu *vcpu, unsigned int ioctl, int kvm_get_reg_list(struct kvm_vcpu *vcpu, struct kvm_reg_list __user *user_list); +void kvm_init_efer_bits(void); void kvm_enable_efer_bits(u64); bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer); int kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 0845551c6cab..cf5fa038ff17 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -6910,6 +6910,8 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_setup_xss_caps); static void kvm_setup_efer_caps(void) { + kvm_init_efer_bits(); + if (kvm_cpu_cap_has(X86_FEATURE_LM)) kvm_enable_efer_bits(EFER_LME | EFER_LMA); From 37352c4fbdd7921c032ab7e0eb7a0fec960924ec Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Mon, 13 Jul 2026 18:10:19 +0000 Subject: [PATCH 091/121] KVM: x86: Reverse the polarity of efer_reserved_bits In preparation for moving efer_reserved_bits into kvm_caps, reverse its polarity and make it efer_supported_bits, to be more consistent with other fields in kvm_caps. No functional change intended. Reviewed-by: Nikolay Borisov Signed-off-by: Yosry Ahmed Link: https://patch.msgid.link/20260713181020.2735367-5-yosry@kernel.org Signed-off-by: Sean Christopherson --- arch/x86/kvm/msrs.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c index a7394bdae029..8e33b87b5e8f 100644 --- a/arch/x86/kvm/msrs.c +++ b/arch/x86/kvm/msrs.c @@ -19,7 +19,7 @@ bool __read_mostly report_ignored_msrs = true; module_param(report_ignored_msrs, bool, 0644); EXPORT_SYMBOL_FOR_KVM_INTERNAL(report_ignored_msrs); -static u64 __read_mostly efer_reserved_bits; +static u64 __read_mostly efer_supported_bits; #define MAX_IO_MSRS 256 @@ -605,7 +605,7 @@ static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) } bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) { - if (efer & efer_reserved_bits) + if (efer & ~efer_supported_bits) return false; return __kvm_valid_efer(vcpu, efer); @@ -618,7 +618,7 @@ static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info) u64 efer = msr_info->data; int r; - if (efer & efer_reserved_bits) + if (efer & ~efer_supported_bits) return 1; if (!msr_info->host_initiated) { @@ -652,13 +652,13 @@ static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info) void kvm_init_efer_bits(void) { /* Enable syscall by default because its emulated by KVM */ - efer_reserved_bits = ~((u64)EFER_SCE); + efer_supported_bits = (u64)EFER_SCE; } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_init_efer_bits); void kvm_enable_efer_bits(u64 mask) { - efer_reserved_bits &= ~mask; + efer_supported_bits |= mask; } EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_enable_efer_bits); From 92b2af2b6d2c9f53097e48614ea9b9e3aa1dfd43 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Mon, 13 Jul 2026 18:10:20 +0000 Subject: [PATCH 092/121] KVM: x86: Move supported EFER bits to kvm_caps Supported EFER bits naturally fits into kvm_caps because it gets recomputed during vendor initialization (e.g. to account for EFER.SVME being allowed/disallowed based on nested being enabled/disabled). Move efer_supported_bits into kvm_caps as supported_efer_bits (for naming consistency). As the bitmask is now globally visible as part of kvm_caps, there's little use for helpers to enable/disable specific bits, so drop them and open-code updates to kvm_caps.supported_efer_bits. No functional change intended. Suggested-by: Sean Christopherson Reviewed-by: Nikolay Borisov Signed-off-by: Yosry Ahmed Link: https://patch.msgid.link/20260713181020.2735367-6-yosry@kernel.org Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 2 ++ arch/x86/kvm/msrs.c | 19 ++----------------- arch/x86/kvm/msrs.h | 2 -- arch/x86/kvm/x86.c | 15 ++++++++------- 4 files changed, 12 insertions(+), 26 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 974a506538aa..6077f4e00d79 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -272,6 +272,8 @@ struct kvm_caps { u64 supported_xss; u64 supported_perf_cap; + u64 supported_efer_bits; + u64 supported_quirks; u64 inapplicable_quirks; }; diff --git a/arch/x86/kvm/msrs.c b/arch/x86/kvm/msrs.c index 8e33b87b5e8f..66fa7140d65d 100644 --- a/arch/x86/kvm/msrs.c +++ b/arch/x86/kvm/msrs.c @@ -19,8 +19,6 @@ bool __read_mostly report_ignored_msrs = true; module_param(report_ignored_msrs, bool, 0644); EXPORT_SYMBOL_FOR_KVM_INTERNAL(report_ignored_msrs); -static u64 __read_mostly efer_supported_bits; - #define MAX_IO_MSRS 256 struct msr_bitmap_range { @@ -605,7 +603,7 @@ static bool __kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) } bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer) { - if (efer & ~efer_supported_bits) + if (efer & ~kvm_caps.supported_efer_bits) return false; return __kvm_valid_efer(vcpu, efer); @@ -618,7 +616,7 @@ static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info) u64 efer = msr_info->data; int r; - if (efer & ~efer_supported_bits) + if (efer & ~kvm_caps.supported_efer_bits) return 1; if (!msr_info->host_initiated) { @@ -649,19 +647,6 @@ static int set_efer(struct kvm_vcpu *vcpu, struct msr_data *msr_info) return 0; } -void kvm_init_efer_bits(void) -{ - /* Enable syscall by default because its emulated by KVM */ - efer_supported_bits = (u64)EFER_SCE; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_init_efer_bits); - -void kvm_enable_efer_bits(u64 mask) -{ - efer_supported_bits |= mask; -} -EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_enable_efer_bits); - bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type) { struct kvm_x86_msr_filter *msr_filter; diff --git a/arch/x86/kvm/msrs.h b/arch/x86/kvm/msrs.h index 1f772e171758..7cc182a15b3b 100644 --- a/arch/x86/kvm/msrs.h +++ b/arch/x86/kvm/msrs.h @@ -58,8 +58,6 @@ int kvm_get_set_one_reg(struct kvm_vcpu *vcpu, unsigned int ioctl, int kvm_get_reg_list(struct kvm_vcpu *vcpu, struct kvm_reg_list __user *user_list); -void kvm_init_efer_bits(void); -void kvm_enable_efer_bits(u64); bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer); int kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data); int kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index cf5fa038ff17..dfaf80efec4b 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -6910,24 +6910,25 @@ EXPORT_SYMBOL_FOR_KVM_INTERNAL(kvm_setup_xss_caps); static void kvm_setup_efer_caps(void) { - kvm_init_efer_bits(); + /* Enable syscall by default because its emulated by KVM */ + kvm_caps.supported_efer_bits = (u64)EFER_SCE; if (kvm_cpu_cap_has(X86_FEATURE_LM)) - kvm_enable_efer_bits(EFER_LME | EFER_LMA); + kvm_caps.supported_efer_bits |= (EFER_LME | EFER_LMA); if (kvm_cpu_cap_has(X86_FEATURE_NX)) - kvm_enable_efer_bits(EFER_NX); + kvm_caps.supported_efer_bits |= EFER_NX; if (kvm_cpu_cap_has(X86_FEATURE_FXSR_OPT)) - kvm_enable_efer_bits(EFER_FFXSR); + kvm_caps.supported_efer_bits |= EFER_FFXSR; if (kvm_cpu_cap_has(X86_FEATURE_AUTOIBRS)) - kvm_enable_efer_bits(EFER_AUTOIBRS); + kvm_caps.supported_efer_bits |= EFER_AUTOIBRS; if (kvm_cpu_cap_has(X86_FEATURE_SVM)) { - kvm_enable_efer_bits(EFER_SVME); + kvm_caps.supported_efer_bits |= EFER_SVME; if (!boot_cpu_has(X86_FEATURE_EFER_LMSLE_MBZ)) - kvm_enable_efer_bits(EFER_LMSLE); + kvm_caps.supported_efer_bits |= EFER_LMSLE; } } From 184bd464bdb66daa9173670904f24c29c7b7f7d4 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Mon, 13 Jul 2026 18:01:52 +0000 Subject: [PATCH 093/121] KVM: x86: Check EFER validity on KVM_SET_SREGS* When handling userspace SREGS writes, check the validity of EFER (i.e. allowed bits) before writing the new value of EFER through the per-vendor set_efer callbacks. This prevents userspace from writing bogus values (e.g. EFER.SVME=1 with nested=0). Note: on KVM_SET_MSRS, KVM only checks EFER validity in terms of KVM caps, not guest caps, so it is possible to set EFER bits that are supported by KVM but not by the guest CPUID. Potentially allowing userspace to set msrs before CPUID. However, for KVM_SET_SREGS*, check the validity of the set bits against both KVM and guest caps. This is consistent with other validity checks (e.g. for CR4) that check validity against guest caps, which already imposes the need to set CPUID before SREGS. Cc: stable@vger.kernel.org Signed-off-by: Yosry Ahmed Link: https://patch.msgid.link/20260713180153.2728382-2-yosry@kernel.org Signed-off-by: Sean Christopherson --- arch/x86/kvm/regs.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/regs.c b/arch/x86/kvm/regs.c index bd8147798cc3..8f66438989e4 100644 --- a/arch/x86/kvm/regs.c +++ b/arch/x86/kvm/regs.c @@ -564,7 +564,8 @@ static bool kvm_is_valid_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) } return kvm_is_valid_cr4(vcpu, sregs->cr4) && - kvm_is_valid_cr0(vcpu, sregs->cr0); + kvm_is_valid_cr0(vcpu, sregs->cr0) && + kvm_valid_efer(vcpu, sregs->efer); } static int __set_sregs_common(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs, From ee1a586dd1fa2f245b3b753a3e44d9263a49240b Mon Sep 17 00:00:00 2001 From: Pankaj Gupta Date: Wed, 15 Jul 2026 01:36:26 -0500 Subject: [PATCH 094/121] KVM: SEV: Drop FOLL_WRITE for encrypted region registration When pinning SEV guest memory, drop FOLL_WRITE and rely on FOLL_LONGTERM to break CoW, as *KVM* doesn't actually to the memory using the GUP'd pages. Omitting FOLL_WRITE fixes a regression when using file-backed guest memory that was introduced when KVM (correctly) added FOLL_LONG (e.g. to ensure anonymous memory is migrated out of MIGRATE_CMA/ZONE_MOVABLE before a long term pin). Unfortunately, as of commits: 8ac268436e6d ("mm/gup: disallow FOLL_LONGTERM GUP-nonfast writing to file-backed mappings") a6e79df92e4a ("mm/gup: disallow FOLL_LONGTERM GUP-fast writing to file-backed mappings") GUP uses FOLL_LONGTERM as a canary of sorts to detect pins that are likely to be problematic, and disallows WRITE+LONGTERM pins for file-backed memory. As a result, backing SEV+ guests with file-backed memory, e.g. virtio-pmem, fails due to the disallowed FOLL_LONGTERM+FOLL_WRITE combination. Note, in the past, FOLL_WRITE was required to trigger CoW unsharing, to prevent replacing the page in the (primary MMU's) page tables during a later write fault after already having pinned a (shared) page in MAP_PRIVATE mappings. FOLL_LONGTERM does that nowadays, even without FOLL_WRITE (see gup_must_unshare()). Fixes: 7e066cb9b71a ("KVM: SEV: Use long-term pin when registering encrypted memory regions") Cc: stable@vger.kernel.org Suggested-by: "David Hildenbrand (Arm)" Link: https://lore.kernel.org/all/ad784f05-b36c-4e91-9f17-4c5b826735d0@kernel.org/ Signed-off-by: Pankaj Gupta Acked-by: David Hildenbrand (Arm) Acked-by: Lorenzo Stoakes (ARM) Link: https://patch.msgid.link/20260715063626.65899-1-pankaj.gupta@amd.com [sean: massage changelog, add comment about CoW unsharing] Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 74fb15551e83..91ffb2ae0fef 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -2750,8 +2750,12 @@ int sev_mem_enc_register_region(struct kvm *kvm, if (!region) return -ENOMEM; + /* + * Do NOT specify FOLL_WRITE, as KVM isn't using the pinned pages to + * write memory, and FOLL_LONGTERM itself triggers CoW unshare. + */ region->pages = sev_pin_memory(kvm, range->addr, range->size, ®ion->npages, - FOLL_WRITE | FOLL_LONGTERM); + FOLL_LONGTERM); if (IS_ERR(region->pages)) { ret = PTR_ERR(region->pages); goto e_free; From 653857a5af46237eacbf886338af9972574cd781 Mon Sep 17 00:00:00 2001 From: Shivank Sharma Date: Fri, 17 Jul 2026 21:58:38 +0530 Subject: [PATCH 095/121] KVM: selftests: Fix typos in x86 and riscv tests Fix spelling typos found by an automated checker in the KVM selftests for x86 and RISC-V. Signed-off-by: Shivank Sharma Link: https://patch.msgid.link/20260717162838.1562808-1-shivanksharma2376543@gmail.com Signed-off-by: Sean Christopherson --- tools/testing/selftests/kvm/riscv/sbi_pmu_test.c | 4 ++-- tools/testing/selftests/kvm/x86/hyperv_clock.c | 4 ++-- tools/testing/selftests/kvm/x86/hyperv_evmcs.c | 2 +- .../selftests/kvm/x86/vmx_invalid_nested_guest_state.c | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tools/testing/selftests/kvm/riscv/sbi_pmu_test.c b/tools/testing/selftests/kvm/riscv/sbi_pmu_test.c index e56a3dd6a51e..20388f0b959d 100644 --- a/tools/testing/selftests/kvm/riscv/sbi_pmu_test.c +++ b/tools/testing/selftests/kvm/riscv/sbi_pmu_test.c @@ -492,7 +492,7 @@ static void test_pmu_events_snaphost(void) struct riscv_pmu_snapshot_data *snapshot_data = snapshot_gva; int i; - /* Verify presence of SBI PMU and minimum requrired SBI version */ + /* Verify presence of SBI PMU and minimum required SBI version */ verify_sbi_requirement_assert(); snapshot_set_shmem(snapshot_gpa, 0); @@ -518,7 +518,7 @@ static void test_pmu_events_overflow(void) { int num_counters = 0, i = 0; - /* Verify presence of SBI PMU and minimum requrired SBI version */ + /* Verify presence of SBI PMU and minimum required SBI version */ verify_sbi_requirement_assert(); snapshot_set_shmem(snapshot_gpa, 0); diff --git a/tools/testing/selftests/kvm/x86/hyperv_clock.c b/tools/testing/selftests/kvm/x86/hyperv_clock.c index c083cea546dc..d5d779623cc6 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_clock.c +++ b/tools/testing/selftests/kvm/x86/hyperv_clock.c @@ -56,7 +56,7 @@ static inline void check_tsc_msr_rdtsc(void) tsc_freq = rdmsr(HV_X64_MSR_TSC_FREQUENCY); GUEST_ASSERT(tsc_freq > 0); - /* For increased accuracy, take mean rdtsc() before and afrer rdmsr() */ + /* For increased accuracy, take mean rdtsc() before and after rdmsr() */ r1 = rdtsc(); t1 = rdmsr(HV_X64_MSR_TIME_REF_COUNT); r1 = (r1 + rdtsc()) / 2; @@ -181,7 +181,7 @@ static void host_check_tsc_msr_rdtsc(struct kvm_vcpu *vcpu) tsc_freq = vcpu_get_msr(vcpu, HV_X64_MSR_TSC_FREQUENCY); TEST_ASSERT(tsc_freq > 0, "TSC frequency must be nonzero"); - /* For increased accuracy, take mean rdtsc() before and afrer ioctl */ + /* For increased accuracy, take mean rdtsc() before and after ioctl */ r1 = rdtsc(); t1 = vcpu_get_msr(vcpu, HV_X64_MSR_TIME_REF_COUNT); r1 = (r1 + rdtsc()) / 2; diff --git a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c index 1bda2cd3f739..63ea1533e4ea 100644 --- a/tools/testing/selftests/kvm/x86/hyperv_evmcs.c +++ b/tools/testing/selftests/kvm/x86/hyperv_evmcs.c @@ -125,7 +125,7 @@ void guest_code(struct vmx_pages *vmx_pages, struct hyperv_test_pages *hv_pages, /* * NMI forces L2->L1 exit, resuming L2 and hope that EVMCS is * up-to-date (RIP points where it should and not at the beginning - * of l2_guest_code(). GUEST_SYNC(9) checkes that. + * of l2_guest_code(). GUEST_SYNC(9) checks that. */ GUEST_ASSERT(!vmresume()); diff --git a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c index 6d88c54f69fa..578283893ab3 100644 --- a/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c +++ b/tools/testing/selftests/kvm/x86/vmx_invalid_nested_guest_state.c @@ -77,7 +77,7 @@ int main(int argc, char *argv[]) ARBITRARY_IO_PORT, run->io.port); /* - * Stuff invalid guest state for L2 by making TR unusuable. The next + * Stuff invalid guest state for L2 by making TR unusable. The next * KVM_RUN should induce a TRIPLE_FAULT in L2 as KVM doesn't support * emulating invalid guest state for L2. */ From e428f9779a43737d830111238816f1928b07aefb Mon Sep 17 00:00:00 2001 From: Phil Rosenthal Date: Mon, 20 Jul 2026 13:45:49 -0400 Subject: [PATCH 096/121] KVM: x86/mmu: Consume the locked rmap value in the lockless rmap walk __kvm_rmap_lock() deliberately elides the rmap lock when it observes an empty rmap. In that case kvm_rmap_lock_readonly() also re-enables preemption and returns zero, so the caller holds neither the rmap lock nor a preemption reference. The elision documents the invariant it relies on: * Elide the lock if the rmap is empty, as lockless walkers (read-only * mode) don't need to (and can't) walk an empty rmap, nor can they add * entries to the rmap. I.e. the only paths that process empty rmaps * do so while holding mmu_lock for write, and are mutually exclusive. kvm_rmap_age_gfn_range() ignores the returned value and unconditionally enters for_each_rmap_spte_lockless(). The iterator started with rmap_get_first(), which re-reads rmap_head->val rather than using the value returned by the lock. If a writer populates the rmap between the lock's read and the iterator's re-read, the aging path walks the newly installed rmap without holding its lock. For a KVM_RMAP_MANY rmap this leaves the walker following a pte_list_desc chain that it never locked. A writer holding mmu_lock for write may free that chain (e.g. kvm_zap_all_rmap_sptes() on the recycle path, or any rmap zap) via kmem_cache_free() while the walk is in progress, giving a slab use-after-free. Nothing serialises the two: the aging path runs without mmu_lock when CONFIG_KVM_MMU_LOCKLESS_AGING=y, and the rmap lock that would otherwise exclude the writer was elided. Because the empty path re-enables preemption, the interval between the two reads can span an arbitrary scheduling delay. Fix the class of bug by having the lockless walk consume the value returned by the lock instead of re-reading the rmap. Split rmap_get_first() into __rmap_get_first(), which starts an iterator from an already-read rmap value, and make for_each_rmap_spte_lockless() take that value and call __rmap_get_first() directly. kvm_rmap_age_gfn_range() passes the value returned by kvm_rmap_lock_readonly(): when the lock was elided the value is zero, __rmap_get_first() returns NULL, and the walk is skipped. No lockless walker re-reads the rmap, so the lock-elision invariant cannot be violated, and no lock()-without-paired-unlock() path is added to the aging code. Fixes: af3b6a9eba48 ("KVM: x86/mmu: Walk rmaps (shadow MMU) without holding mmu_lock when aging gfns") Suggested-by: Sean Christopherson Cc: stable@vger.kernel.org Signed-off-by: Phil Rosenthal Link: https://patch.msgid.link/20260720-rmap-age-elided-submit-v2-1-668973030d47@phil.gs Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 6c13da942bfc..440db8891f21 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1227,18 +1227,9 @@ struct rmap_iterator { int pos; /* index of the sptep */ }; -/* - * Iteration must be started by this function. This should also be used after - * removing/dropping sptes from the rmap link because in such cases the - * information in the iterator may not be valid. - * - * Returns sptep if found, NULL otherwise. - */ -static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head, - struct rmap_iterator *iter) +static u64 *__rmap_get_first(unsigned long rmap_val, + struct rmap_iterator *iter) { - unsigned long rmap_val = kvm_rmap_get(rmap_head); - if (!rmap_val) return NULL; @@ -1252,6 +1243,19 @@ static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head, return iter->desc->sptes[iter->pos]; } +/* + * Iteration must be started by this function. This should also be used after + * removing/dropping sptes from the rmap link because in such cases the + * information in the iterator may not be valid. + * + * Returns sptep if found, NULL otherwise. + */ +static u64 *rmap_get_first(struct kvm_rmap_head *rmap_head, + struct rmap_iterator *iter) +{ + return __rmap_get_first(kvm_rmap_get(rmap_head), iter); +} + /* * Must be used with a valid iterator: e.g. after rmap_get_first(). * @@ -1286,8 +1290,9 @@ static u64 *rmap_get_next(struct rmap_iterator *iter) __for_each_rmap_spte(_rmap_head_, _iter_, _sptep_) \ if (!WARN_ON_ONCE(!is_shadow_present_pte(*(_sptep_)))) \ -#define for_each_rmap_spte_lockless(_rmap_head_, _iter_, _sptep_, _spte_) \ - __for_each_rmap_spte(_rmap_head_, _iter_, _sptep_) \ +#define for_each_rmap_spte_lockless(_rmap_val_, _iter_, _sptep_, _spte_) \ + for (_sptep_ = __rmap_get_first(_rmap_val_, _iter_); \ + _sptep_; _sptep_ = rmap_get_next(_iter_)) \ if (is_shadow_present_pte(_spte_ = mmu_spte_get_lockless(sptep))) static void drop_spte(struct kvm *kvm, u64 *sptep) @@ -1725,7 +1730,7 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm, rmap_head = gfn_to_rmap(gfn, level, range->slot); rmap_val = kvm_rmap_lock_readonly(rmap_head); - for_each_rmap_spte_lockless(rmap_head, &iter, sptep, spte) { + for_each_rmap_spte_lockless(rmap_val, &iter, sptep, spte) { if (!is_accessed_spte(spte)) continue; From a33c40b93ccf5177e042253807d40e0b92e7f206 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:13:06 -0700 Subject: [PATCH 097/121] KVM: SEV: Allocate full pages for {DE,EN}CRYPT ops on SNP-enabled hosts When {de,en}crypting memory of an SEV or SEV-ES guest on an SNP-enabled host via a temporary buffer, allocate a full 4KiB page for the buffer to ensure the page containing the buffer is wholly owned by KVM, i.e. won't be concurrently allocated and accessed by other kernel code while KVM is using the buffer to {de,en}crypt memory. On SNP-enabled platforms, when sending SEV/SEV-ES commands that trigger firmware writes to memory, the to-be-written page(s) must be (temporarily) assigned to Firmware (as required by the SNP architecture, to guard against using such commands as gadgets to attack SNP guests). See snp_map_cmd_buf_desc() and friends. Unfortunately, transferring ownership of a page to Firmware makes the page inaccessible to software, and thus writes generate RMP #PF violations. If KVM uses a sub-page allocation for its temporary buffer, some other actor in the kernel can allocate and use the other portions of the page, and thus trigger unexpected (and seemingly spurious) RMP #PF violations due to software attempting to access a Firmware-owned page. BUG: unable to handle page fault for address: ffff906ae30f0300 #PF: supervisor write access in kernel mode #PF: error_code(0x80000003) - RMP violation PGD 6b1b80d067 P4D 6b1b80d067 PUD 100231e2063 PMD 10055a88063 PTE 80000100630f0163 SEV-SNP: PFN 0x100630f0 unassigned, dumping non-zero entries in 2M PFN region: [0x10063000 - 0x10063200] Oops: Oops: 0003 [#1] SMP CPU: 70 UID: 0 PID: 10658 Comm: svw_WaiterThrea Tainted: G U W O 7.1.0-smp--c22293789940-seanjc-next #1 PREEMPTLAZY Tainted: [U]=USER, [W]=WARN, [O]=OOT_MODULE Hardware name: Google, Inc. Arcadia_IT_80/Arcadia_IT_80, BIOS 34.86.0-102 01/25/2026 RIP: 0010:memset+0xf/0x20 Call Trace: __kvmalloc_node_noprof+0x2a4/0x710 do_getxattr+0x4e/0x130 path_getxattrat+0x125/0x1b0 do_syscall_64+0x10a/0x480 entry_SYSCALL_64_after_hwframe+0x4b/0x53 RIP: 0033:0x7f3a22cb6daa Modules linked in: kvm_amd kvm irqbypass vfat fat ccp k10temp sha3 libsha3 i2c_piix4 gq(O) cdc_acm xhci_pci xhci_hcd gsmi: Log Shutdown Reason 0x03 CR2: ffff906ae30f0300 ---[ end trace 0000000000000000 ]--- RIP: 0010:memset+0xf/0x20 Kernel panic - not syncing: Fatal exception Kernel Offset: 0x39e00000 from 0xffffffff81000000 (relocation range: 0xffffffff80000000-0xffffffffbfffffff) gsmi: Log Shutdown Reason 0x02 Fixes: 4c735bf1bc22 ("KVM: SEV: Allocate only as many bytes as needed for temp crypt buffers") Cc: stable@vger.kernel.org Cc: Michael Roth Debugged-by: Michael Roth Link: https://patch.msgid.link/20260723211306.75397-1-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/sev.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 91ffb2ae0fef..f54528e9cec6 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -1280,9 +1280,28 @@ static void *sev_dbg_crypt_slow_alloc(struct page *page, unsigned long __va, if (WARN_ON_ONCE((*pa & PAGE_MASK) != ((*pa + *nr_bytes - 1) & PAGE_MASK))) return NULL; + /* + * If SNP is enabled, i.e. the RMP is active, allocate a full page to + * prevent concurrent accesses to the page. As required by firmware, + * the PSP driver updates the RMP to temporarily transfer ownership of + * the page to Firmware while the {DE,EN}CRYPT operation is in-progress, + * and so concurrent software accesses to the page will encounter + * seemingly spurious RMP #PF violations + */ + if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) + return (void *)__get_free_page(GFP_KERNEL); + return kmalloc(*nr_bytes, GFP_KERNEL); } +static void sev_dbg_crypt_slow_free(void *buf) +{ + if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) + free_page((unsigned long)buf); + else + kfree(buf); +} + static int sev_dbg_decrypt_slow(struct kvm *kvm, unsigned long src, struct page *src_p, unsigned long dst, unsigned int len, int *err) @@ -1304,7 +1323,7 @@ static int sev_dbg_decrypt_slow(struct kvm *kvm, unsigned long src, if (copy_to_user((void __user *)dst, buf + (src & 15), len)) r = -EFAULT; out: - kfree(buf); + sev_dbg_crypt_slow_free(buf); return r; } @@ -1337,7 +1356,7 @@ static int sev_dbg_encrypt_slow(struct kvm *kvm, unsigned long src, r = sev_issue_dbg_cmd(kvm, __sme_set(__pa(buf)), dst_pa, nr_bytes, KVM_SEV_DBG_ENCRYPT, err); out: - kfree(buf); + sev_dbg_crypt_slow_free(buf); return r; } From 05a0b701d1089fb57beeb8982f23c3bbafe0fa8b Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Wed, 22 Jul 2026 23:01:28 +0000 Subject: [PATCH 098/121] KVM: nVMX: Service local TLB flushes on failed nested VM-Enter KVM services local TLB flushes on "full" nested VM-Exits (through __nested_vmx_vmexit()), but not if a nested VM-Enter fails (e.g. due to failed VMCS checks in nested_vmx_enter_non_root_mode()). However, it is possible that KVM had queued TLB flushes that need to be performed, even if the nested VM-Enter was not successful. For example, if VPID is disabled for L2 (via nested_vmx_transition_tlb_flush(), or if via the MSR load lists, as the SDM says: If any MSR is being loaded in such a way that would architecturally require a TLB flush, the TLBs are updated so that, after VM entry, the logical processor will not use any translations that were cached before the transition. The SDM is unclear about when the TLB flush should occur, and whether or not a failed VM entry would flush the TLB, so it is safer to always do the TLB flush in this case. More concretely, KVM also updates the last VPID L1 used for L2 in nested_vmx_transition_tlb_flush() (i.e. last_vpid), even if the VM entry ultimately fails. With the current code, KVM could miss a TLB flush if L1 changes L2's VPID, then does a failed VM entry followed by a successful one, as the failed VM entry would update last_vpid but not actually flush the TLB. Servicing local TLB flushes on failed VM entries makes sure that the TLB is always flushed when last_vpid is updated. Fixes: 5c614b3583e7 ("KVM: nVMX: nested VPID emulation") Cc: stable@vger.kernel.org Reported-by: Sashiko # Internal review Suggested-by: Sean Christopherson Signed-off-by: Yosry Ahmed Link: https://patch.msgid.link/20260722230128.1587363-1-yosry@kernel.org Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/nested.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 0635e92471c8..5d5b5022438d 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -3765,6 +3765,14 @@ enum nvmx_vmentry_status nested_vmx_enter_non_root_mode(struct kvm_vcpu *vcpu, vmentry_fail_vmexit_guest_mode: if (vmcs12->cpu_based_vm_exec_control & CPU_BASED_USE_TSC_OFFSETTING) vcpu->arch.tsc_offset -= vmcs12->tsc_offset; + + /* + * Handle any TLB flush requests that were queued for L2 if KVM made it + * far enough along to switch to L2 context. Note, loading host state + * will generate any flushes for L1 required by VM-Exit. + */ + kvm_service_local_tlb_flush_requests(vcpu); + leave_guest_mode(vcpu); vmentry_fail_vmexit: From ec9a16c6aeba8e19ce98c58a1ac255681a4dd0ac Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Thu, 7 May 2026 12:22:24 +0100 Subject: [PATCH 099/121] KVM: SVM: Always intercept ICEBP to workaround AMD ICEBP+TASK_SWITCH flaws Unconditionally intercept and emulate ICEBP, i.e. INT1 #DBs, on SVM to workaround a bug/misfeature in current AMD CPUs. AMD CPUs don't provide information to allow software to distinguish instruction-induced #DBs (i.e. ICEBP) from exception-induced #DBs (all others), and also don't report an instruction length for an ICEBP-induced TASK_SWITCH. On an intercepted TASK_SWITCH (which always has fault semantics), KVM (any hypervisor, really) looks at the vectoring event type to decide whether it was logically caused by a trap, and therefore whether to advance guest RIP before entering the new task. If the guest IDT is configured to deliver #DBs via a task gate, then the guest will see broken behavior for ICEBP #DB because KVM doesn't have enough information to detect and skip the ICEBP. The typical workaround is to intercept ICEBP unconditionally and handle the FAULT=>TRAP conversion in the hypervisor, at which point the #DB-induced TASK_SWITCH occurs with RIP on the correct instruction boundary regardless of whether it was instruction-induced or exception-induced. As a bonus, intercepting ICEBP more or less aligns SVM with VMX (KVM always intercepts #DBs on VMX, and ICEBP #DB VM-Exits on Intel have fault-like behavior). Signed-off-by: David Woodhouse Link: https://patch.msgid.link/e03f092dfbb7d391a6bf2797ba01e122ba080bcd.camel@infradead.org [sean: drop selftest, reword changelog to provide more details] Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/svm.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 0e0dd9618750..be90964e9181 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1174,6 +1174,7 @@ static void init_vmcb(struct kvm_vcpu *vcpu, bool init_event) svm_set_intercept(svm, INTERCEPT_SKINIT); svm_set_intercept(svm, INTERCEPT_WBINVD); svm_set_intercept(svm, INTERCEPT_XSETBV); + svm_set_intercept(svm, INTERCEPT_ICEBP); svm_set_intercept(svm, INTERCEPT_RDPRU); svm_set_intercept(svm, INTERCEPT_RSM); @@ -2071,6 +2072,22 @@ static int bp_interception(struct kvm_vcpu *vcpu) return 0; } +static int icebp_interception(struct kvm_vcpu *vcpu) +{ + /* + * Intercept and emulate ICEBP (INT1, opcode 0xF1) instead of allowing + * the guest to natively take the #DB trap, so that RIP is advanced + * past the instruction *before* #DB is injected. This is necessary + * because SVM reports the wrong RIP for ICEBP-induced #DB when #DBs + * are delivered via a task gate: RIP points at the ICEBP instruction + * instead of after it (and SVM doesn't provide enough information for + * KVM to detect and manually advance the pre-#DB RIP). + */ + svm_skip_emulated_instruction(vcpu); + kvm_queue_exception(vcpu, DB_VECTOR); + return 1; +} + static int ud_interception(struct kvm_vcpu *vcpu) { return handle_ud(vcpu); @@ -3385,6 +3402,7 @@ static int (*const svm_exit_handlers[])(struct kvm_vcpu *vcpu) = { [SVM_EXIT_MONITOR] = kvm_emulate_monitor, [SVM_EXIT_MWAIT] = kvm_emulate_mwait, [SVM_EXIT_XSETBV] = kvm_emulate_xsetbv, + [SVM_EXIT_ICEBP] = icebp_interception, [SVM_EXIT_RDPRU] = kvm_handle_invalid_op, [SVM_EXIT_EFER_WRITE_TRAP] = efer_trap, [SVM_EXIT_CR0_WRITE_TRAP] = cr_trap, From 0f0893ac5af3ac9d18440955f302bd20a35ccab2 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:03 -0700 Subject: [PATCH 100/121] KVM: guest_memfd: Pass the number of pages instead of the end pfn into .invalidate() Pass the number of pages to "invalidate", i.e. reclaim, instead of the end pfn, as a first step towards aligning the function prototypes between the de facto "to private" and "to shared" arch hooks. Eventually, the goal is to end up with kvm_gmem_arch_make_{private,shared}(), and in both cases, providing the number of pages makes the call sites slightly nicer, and also avoids any confusion over whether the end pfn is inclusive or exclusive. Opportunistically rename "start" to "pfn", again to align with the expected signature of make_private() (which needs to pass a starting gfn as well, at which point the "start" becomes noise). No functional change intended. Cc: Fuad Tabba Cc: Ackerley Tng Reviewed-by: Xiaoyao Li Reviewed-by: Ackerley Tng Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20260723210811.72720-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 2 +- arch/x86/kvm/svm/sev.c | 8 ++++---- arch/x86/kvm/svm/svm.h | 2 +- arch/x86/kvm/x86.c | 4 ++-- include/linux/kvm_host.h | 2 +- virt/kvm/guest_memfd.c | 6 +----- 6 files changed, 10 insertions(+), 14 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 1f8ef0e1566a..ef20f0e12193 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1906,7 +1906,7 @@ struct kvm_x86_ops { #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); #endif - void (*gmem_invalidate)(kvm_pfn_t start, kvm_pfn_t end); + void (*gmem_invalidate)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range); #endif diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index a7584b7ed6dc..5a5e8342c4fe 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -5159,16 +5159,16 @@ int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) return 0; } -void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) +void sev_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages) { - kvm_pfn_t pfn; + kvm_pfn_t end = pfn + nr_pages; if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) return; - pr_debug("%s: PFN start 0x%llx PFN end 0x%llx\n", __func__, start, end); + pr_debug("%s: PFN start 0x%llx PFN end 0x%llx\n", __func__, pfn, end); - for (pfn = start; pfn < end;) { + while (pfn < end) { bool use_2m_update = false; int rc, rmp_level; bool assigned; diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 130205defffa..2180d03bb0a6 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -1010,7 +1010,7 @@ int sev_dev_get_attr(u32 group, u64 attr, u64 *val); extern unsigned int max_sev_asid; void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code); int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); -void sev_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end); +void sev_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages); void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 5a5fd6211d23..3519fca53cfc 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10597,9 +10597,9 @@ int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_ord #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE -void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end) +void kvm_arch_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages) { - kvm_x86_call(gmem_invalidate)(start, end); + kvm_x86_call(gmem_invalidate)(pfn, nr_pages); } void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range) { diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index c00fc1740ce5..79868ebfc113 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2607,7 +2607,7 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE -void kvm_arch_gmem_invalidate(kvm_pfn_t start, kvm_pfn_t end); +void kvm_arch_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages); void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); #endif diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 659b8dbe0b30..c96a2ef5f2ce 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -530,11 +530,7 @@ static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *fol #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE static void kvm_gmem_free_folio(struct folio *folio) { - struct page *page = folio_page(folio, 0); - kvm_pfn_t pfn = page_to_pfn(page); - int order = folio_order(folio); - - kvm_arch_gmem_invalidate(pfn, pfn + (1ul << order)); + kvm_arch_gmem_invalidate(folio_file_pfn(folio, 0), folio_nr_pages(folio)); } #endif From 7b8529e70a11ed110a15568f6b391a5f32eef67f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:04 -0700 Subject: [PATCH 101/121] KVM: guest_memfd: Rename invalidate() arch hook to reclaim() and isolate it Rename guest_memfd's invalidate() hook to reclaim() and isolate it via its own RECLAIM Kconfig, as the hook is called when a folio is freed, which is far too late and lacks sufficient information for KVM to actually invalidate its usage of the memory. E.g. SNP uses the hook to convert memory back to SHARED so that it can be safely accessed by the host, there is no invalidation of guest mappings anywhere. Isolating the hook will also allow pKVM on arm64 to opt-in to reclaim() without also having to differentiate between reclaim and conversions to shared for active VMs. Keep guest_memfd's trampoline, even though it would be trivial to wire up .free_folio() directly to an arch callback, to avoid bleeding guest_memfd internals into arch code (specifically, avoid referencing folios in arch code). Leave the kvm_x86_ops hook as-is for the moment, as "reclaim" on SNP is the same as convert-to-shared, i.e. using a different name for the x86 hook will allow reusing it for in-place conversion. Reviewed-by: Xiaoyao Li Reviewed-by: Fuad Tabba Reviewed-by: Ackerley Tng Link: https://patch.msgid.link/20260723210811.72720-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/Kconfig | 1 + arch/x86/kvm/x86.c | 7 +++++-- include/linux/kvm_host.h | 5 ++++- virt/kvm/Kconfig | 4 ++++ virt/kvm/guest_memfd.c | 6 +++--- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index 801bf9e520db..e0e7ad015839 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -161,6 +161,7 @@ config KVM_AMD_SEV select ARCH_HAS_CC_PLATFORM select KVM_GENERIC_MEMORY_ATTRIBUTES select HAVE_KVM_ARCH_GMEM_PREPARE + select HAVE_KVM_ARCH_GMEM_RECLAIM select HAVE_KVM_ARCH_GMEM_INVALIDATE select HAVE_KVM_ARCH_GMEM_POPULATE help diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 3519fca53cfc..befba27672c7 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10596,11 +10596,14 @@ int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_ord } #endif -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE -void kvm_arch_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages) +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM +void kvm_arch_gmem_reclaim(kvm_pfn_t pfn, kvm_pfn_t nr_pages) { kvm_x86_call(gmem_invalidate)(pfn, nr_pages); } +#endif + +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range) { kvm_x86_call(gmem_invalidate_range)(kvm, range); diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 79868ebfc113..46c8d18fd043 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2606,8 +2606,11 @@ long kvm_gmem_populate(struct kvm *kvm, gfn_t start_gfn, void __user *src, kvm_gmem_populate_cb post_populate, void *opaque); #endif +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM +void kvm_arch_gmem_reclaim(kvm_pfn_t pfn, kvm_pfn_t nr_pages); +#endif + #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE -void kvm_arch_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages); void kvm_arch_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); #endif diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index 794976b88c6f..617876993225 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -111,6 +111,10 @@ config HAVE_KVM_ARCH_GMEM_PREPARE bool depends on KVM_GUEST_MEMFD +config HAVE_KVM_ARCH_GMEM_RECLAIM + bool + depends on KVM_GUEST_MEMFD + config HAVE_KVM_ARCH_GMEM_INVALIDATE bool depends on KVM_GUEST_MEMFD diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index c96a2ef5f2ce..c086cfcd0257 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -527,10 +527,10 @@ static int kvm_gmem_error_folio(struct address_space *mapping, struct folio *fol return MF_DELAYED; } -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM static void kvm_gmem_free_folio(struct folio *folio) { - kvm_arch_gmem_invalidate(folio_file_pfn(folio, 0), folio_nr_pages(folio)); + kvm_arch_gmem_reclaim(folio_file_pfn(folio, 0), folio_nr_pages(folio)); } #endif @@ -538,7 +538,7 @@ static const struct address_space_operations kvm_gmem_aops = { .dirty_folio = noop_dirty_folio, .migrate_folio = kvm_gmem_migrate_folio, .error_remove_folio = kvm_gmem_error_folio, -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM .free_folio = kvm_gmem_free_folio, #endif }; From f3c073e25476422ade8ed95e18734c5f3a98ba2b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:05 -0700 Subject: [PATCH 102/121] KVM: x86: Rename kvm_x86_ops' gmem_invalidate() to gmem_make_shared() Rename kvm_x86_ops's gmem_invalidate() hook to gmem_make_shared(), as the hook doesn't invalidate anything, and so that KVM doesn't need to add yet another vendor callback to support "convert to shared" once in-place conversion comes along. Opportunistically wrap the ops declarations with a GMEM_RECLAIM guard so that attempting to wire up a .gmem_make_shared() hook without selecting CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM will result in a build failure. No functional change intended. Reviewed-by: Xiaoyao Li Reviewed-by: Ackerley Tng Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20260723210811.72720-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 4 +++- arch/x86/include/asm/kvm_host.h | 4 +++- arch/x86/kvm/svm/sev.c | 2 +- arch/x86/kvm/svm/svm.c | 2 +- arch/x86/kvm/svm/svm.h | 2 +- arch/x86/kvm/x86.c | 2 +- 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 736129db272a..210cb95d0a0b 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -149,7 +149,9 @@ KVM_X86_OP_OPTIONAL(alloc_apic_backing_page) #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE KVM_X86_OP_OPTIONAL_RET0(gmem_prepare) #endif -KVM_X86_OP_OPTIONAL(gmem_invalidate) +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM +KVM_X86_OP_OPTIONAL(gmem_make_shared) +#endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE KVM_X86_OP_OPTIONAL(gmem_invalidate_range) #endif diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index ef20f0e12193..693be16abd95 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1906,7 +1906,9 @@ struct kvm_x86_ops { #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); #endif - void (*gmem_invalidate)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM + void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); +#endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_INVALIDATE void (*gmem_invalidate_range)(struct kvm *kvm, struct kvm_gfn_range *range); #endif diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 5a5e8342c4fe..4c46081889a6 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -5159,7 +5159,7 @@ int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) return 0; } -void sev_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages) +void sev_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages) { kvm_pfn_t end = pfn + nr_pages; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index dd51df74c2dc..44c3af6f71d8 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5451,7 +5451,7 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .vm_move_enc_context_from = sev_vm_move_enc_context_from, .gmem_prepare = sev_gmem_prepare, - .gmem_invalidate = sev_gmem_invalidate, + .gmem_make_shared = sev_gmem_make_shared, .gmem_invalidate_range = sev_gmem_invalidate_range, .gmem_max_mapping_level = sev_gmem_max_mapping_level, #endif diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 2180d03bb0a6..51e3494e7802 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -1010,7 +1010,7 @@ int sev_dev_get_attr(u32 group, u64 attr, u64 *val); extern unsigned int max_sev_asid; void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code); int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); -void sev_gmem_invalidate(kvm_pfn_t pfn, kvm_pfn_t nr_pages); +void sev_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages); void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); struct vmcb_save_area *sev_decrypt_vmsa(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index befba27672c7..0075815acdd5 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10599,7 +10599,7 @@ int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_ord #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM void kvm_arch_gmem_reclaim(kvm_pfn_t pfn, kvm_pfn_t nr_pages) { - kvm_x86_call(gmem_invalidate)(pfn, nr_pages); + kvm_x86_call(gmem_make_shared)(pfn, nr_pages); } #endif From 0f68fabe68fe138a5c00310ba7935dc849ab111d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:06 -0700 Subject: [PATCH 103/121] KVM: guest_memfd: Drop the redundant printk on arch gmem_prepare() failure Drop guest_memfd's ratelimited printk to log "preparation" failures, as KVM SNP already logs more precise messages in all error paths, and whether or not failure to convert the pfn to private is "unexpected", i.e. warrants logging, is firmly an architecture specific detail. Reviewed-by: Ackerley Tng Reviewed-by: Xiaoyao Li Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20260723210811.72720-5-seanjc@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index c086cfcd0257..3e4273f60a8c 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -66,15 +66,11 @@ static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slo #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE kvm_pfn_t pfn = folio_file_pfn(folio, index); gfn_t gfn = slot->base_gfn + index - slot->gmem.pgoff; - int rc = kvm_arch_gmem_prepare(kvm, gfn, pfn, folio_order(folio)); - if (rc) { - pr_warn_ratelimited("gmem: Failed to prepare folio for index %lx GFN %llx PFN %llx error %d.\n", - index, gfn, pfn, rc); - return rc; - } -#endif + return kvm_arch_gmem_prepare(kvm, gfn, pfn, folio_order(folio)); +#else return 0; +#endif } /* From 19e612cd59bf447df63b77cda7feb7d82ed74f7a Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:07 -0700 Subject: [PATCH 104/121] KVM: guest_memfd: Add helpers to query SHARED vs. PRIVATE for a given page Add helpers to check if a given page in a guest_memfd instance is PRIVATE versus SHARED, and use the "is shared" helper instead of an open-coded equivalent in the user pagefault handler. In addition to the immediate usage, providing an "is private" helper will allow cleaning up the so called prepare() code, and eventually will be heavily used once in-place conversion support comes along. No functional change intended. Reviewed-by: Xiaoyao Li Reviewed-by: Ackerley Tng Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20260723210811.72720-6-seanjc@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 3e4273f60a8c..3fa4969b38f9 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -60,6 +60,16 @@ static pgoff_t kvm_gmem_get_index(struct kvm_memory_slot *slot, gfn_t gfn) return gfn - slot->base_gfn + slot->gmem.pgoff; } +static bool kvm_gmem_is_private_mem(struct inode *inode, pgoff_t index) +{ + return !(GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED); +} + +static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index) +{ + return !kvm_gmem_is_private_mem(inode, index); +} + static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, pgoff_t index, struct folio *folio) { @@ -397,7 +407,7 @@ static vm_fault_t kvm_gmem_fault_user_mapping(struct vm_fault *vmf) if (((loff_t)vmf->pgoff << PAGE_SHIFT) >= i_size_read(inode)) return VM_FAULT_SIGBUS; - if (!(GMEM_I(inode)->flags & GUEST_MEMFD_FLAG_INIT_SHARED)) + if (!kvm_gmem_is_shared_mem(inode, vmf->pgoff)) return VM_FAULT_SIGBUS; folio = kvm_gmem_get_folio(inode, vmf->pgoff); From 3df611b5a04e916f406882a22f33b570831df404 Mon Sep 17 00:00:00 2001 From: Ackerley Tng Date: Thu, 23 Jul 2026 14:08:08 -0700 Subject: [PATCH 105/121] KVM: guest_memfd: Only "prepare" folios for private pages When getting a guest_memfd pfn, prepare the folio, i.e. convert its pages to private, if and only if the page is actually private. The misnamed prepare() hook exists specifically to allow x86's SNP to assign pages to the owning VM in the RMP when mapping private memory into a guest. Guarding the call will allow renaming the prepare() hook to better reflect its role, without creating a semantic mess, and will become a hard requirement once in-place conversion is supported, i.e. when CoCo VMs support SHARED guest_memfd pages. For all intents, no functional change intended (the sole arch hook is a nop for SHARED memory). Suggested-by: Michael Roth Reviewed-by: Fuad Tabba [sean: rewrite changelog to fit the context] Signed-off-by: Ackerley Tng Reviewed-by: Xiaoyao Li Link: https://patch.msgid.link/20260723210811.72720-7-seanjc@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 3fa4969b38f9..25da7778af01 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -814,7 +814,8 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, folio_mark_uptodate(folio); } - r = kvm_gmem_prepare_folio(kvm, slot, gfn, folio); + if (kvm_gmem_is_private_mem(file_inode(file), index)) + r = kvm_gmem_prepare_folio(kvm, slot, gfn, folio); folio_unlock(folio); From 2131c4f763d2e4cbbe5c227a406488c38e23f806 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:09 -0700 Subject: [PATCH 106/121] KVM: guest_memfd: Rename prepare() hook and Kconfig to make_private() / CONVERT Rework guest_memfd's prepare() hook into a more accurate make_private(), and rework its Kconfig from PREPARE to a more generic CONVERT. This will allow x86 to share (pun intended) a kvm_x86_ops.gmem_make_shared() hook between the "convert to shared" and "reclaim" flows, which are one and the same for SNP. No functional change intended. Reviewed-by: Ackerley Tng Reviewed-by: Fuad Tabba Link: https://patch.msgid.link/20260723210811.72720-8-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm-x86-ops.h | 4 ++-- arch/x86/include/asm/kvm_host.h | 4 ++-- arch/x86/kvm/Kconfig | 2 +- arch/x86/kvm/svm/sev.c | 2 +- arch/x86/kvm/svm/svm.c | 2 +- arch/x86/kvm/svm/svm.h | 2 +- arch/x86/kvm/x86.c | 6 +++--- include/linux/kvm_host.h | 5 +++-- virt/kvm/Kconfig | 2 +- virt/kvm/guest_memfd.c | 4 ++-- 10 files changed, 17 insertions(+), 16 deletions(-) diff --git a/arch/x86/include/asm/kvm-x86-ops.h b/arch/x86/include/asm/kvm-x86-ops.h index 210cb95d0a0b..a4d872ddef9d 100644 --- a/arch/x86/include/asm/kvm-x86-ops.h +++ b/arch/x86/include/asm/kvm-x86-ops.h @@ -146,8 +146,8 @@ KVM_X86_OP(vcpu_deliver_sipi_vector) KVM_X86_OP_OPTIONAL_RET0(vcpu_get_apicv_inhibit_reasons); KVM_X86_OP_OPTIONAL(get_untagged_addr) KVM_X86_OP_OPTIONAL(alloc_apic_backing_page) -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE -KVM_X86_OP_OPTIONAL_RET0(gmem_prepare) +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +KVM_X86_OP_OPTIONAL_RET0(gmem_make_private) #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM KVM_X86_OP_OPTIONAL(gmem_make_shared) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 693be16abd95..ae9a229c6b11 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1903,8 +1903,8 @@ struct kvm_x86_ops { gva_t (*get_untagged_addr)(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu); -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE - int (*gmem_prepare)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT + int (*gmem_make_private)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); diff --git a/arch/x86/kvm/Kconfig b/arch/x86/kvm/Kconfig index e0e7ad015839..538ed1e80332 100644 --- a/arch/x86/kvm/Kconfig +++ b/arch/x86/kvm/Kconfig @@ -160,7 +160,7 @@ config KVM_AMD_SEV depends on CRYPTO_DEV_SP_PSP && !(KVM_AMD=y && CRYPTO_DEV_CCP_DD=m) select ARCH_HAS_CC_PLATFORM select KVM_GENERIC_MEMORY_ATTRIBUTES - select HAVE_KVM_ARCH_GMEM_PREPARE + select HAVE_KVM_ARCH_GMEM_CONVERT select HAVE_KVM_ARCH_GMEM_RECLAIM select HAVE_KVM_ARCH_GMEM_INVALIDATE select HAVE_KVM_ARCH_GMEM_POPULATE diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 4c46081889a6..779889f5e994 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -5112,7 +5112,7 @@ static bool is_large_rmp_possible(struct kvm *kvm, kvm_pfn_t pfn, int order) return false; } -int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) +int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) { struct kvm_sev_info *sev = to_kvm_sev_info(kvm); kvm_pfn_t pfn_aligned; diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 44c3af6f71d8..d9f632ea8057 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -5450,7 +5450,7 @@ struct kvm_x86_ops svm_x86_ops __initdata = { .vm_copy_enc_context_from = sev_vm_copy_enc_context_from, .vm_move_enc_context_from = sev_vm_move_enc_context_from, - .gmem_prepare = sev_gmem_prepare, + .gmem_make_private = sev_gmem_make_private, .gmem_make_shared = sev_gmem_make_shared, .gmem_invalidate_range = sev_gmem_invalidate_range, .gmem_max_mapping_level = sev_gmem_max_mapping_level, diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index 51e3494e7802..b5cd8437988f 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -1009,7 +1009,7 @@ int sev_cpu_init(struct svm_cpu_data *sd); int sev_dev_get_attr(u32 group, u64 attr, u64 *val); extern unsigned int max_sev_asid; void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code); -int sev_gmem_prepare(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); +int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); void sev_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages); void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 0075815acdd5..512bcc35507a 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10589,10 +10589,10 @@ bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm) return !kvm_arch_has_private_mem(kvm); } -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE -int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order) +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order) { - return kvm_x86_call(gmem_prepare)(kvm, pfn, gfn, max_order); + return kvm_x86_call(gmem_make_private)(kvm, pfn, gfn, max_order); } #endif diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 46c8d18fd043..ecdda1f00c6c 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2572,8 +2572,9 @@ static inline int kvm_gmem_get_pfn(struct kvm *kvm, } #endif /* CONFIG_KVM_GUEST_MEMFD */ -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE -int kvm_arch_gmem_prepare(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order); +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT +int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, + int max_order); #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE diff --git a/virt/kvm/Kconfig b/virt/kvm/Kconfig index 617876993225..c3c0ee253fc7 100644 --- a/virt/kvm/Kconfig +++ b/virt/kvm/Kconfig @@ -107,7 +107,7 @@ config KVM_GUEST_MEMFD select XARRAY_MULTI bool -config HAVE_KVM_ARCH_GMEM_PREPARE +config HAVE_KVM_ARCH_GMEM_CONVERT bool depends on KVM_GUEST_MEMFD diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 25da7778af01..6635ed05f411 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -73,11 +73,11 @@ static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index) static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, pgoff_t index, struct folio *folio) { -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_PREPARE +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT kvm_pfn_t pfn = folio_file_pfn(folio, index); gfn_t gfn = slot->base_gfn + index - slot->gmem.pgoff; - return kvm_arch_gmem_prepare(kvm, gfn, pfn, folio_order(folio)); + return kvm_arch_gmem_make_private(kvm, gfn, pfn, folio_order(folio)); #else return 0; #endif From fb50ca77b672fd359453728fc59730105f2bf7eb Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:10 -0700 Subject: [PATCH 107/121] KVM: guest_memfd: Explicitly pass number of pages to make_private() hook Tweak the guest_memfd make_private() hook to explicitly pass the number of pages to align with the signature of the make_shared() hook, and because the existing code is outright broken if a guest_memfd folio is comprised of more than one page (which can't happen, yet). The SNP code *tries* to create a corresponding huge entry, but if the RMP must use 4KiB entries for whatever reason, KVM will only convert the first pfn, and not the entire range of pfns that will be mapped into the guest. Alternatively, @max_order could simply be repurposed as _the_ @order, but that will fall apart when in-place conversion comes along, at which point KVM will need to deal with conversions that aren't bound 1:1 to a folio. I.e. the number of pages to convert may not be exactly a power-of-2 (and folios don't strictly guarantee power-of-2 pages anyways). WARN in the SNP code if the number of pages to prepare is anything other than '1', i.e. if guest_memfd is trying to prepare/convert more than a single 4KiB page, as sev_gmem_prepare() doesn't actually handle conversion greater than order-0 folios. Opportunistically swap the ordering of @pfn and @gfn params for kvm_x86_ops.gmem_make_private() to match kvm_arch_gmem_make_private(). Fixes: b85524314a3d ("KVM: guest_memfd: delay kvm_gmem_prepare_folio() until the memory is passed to the guest") Reviewed-by: Xiaoyao Li Reviewed-by: Ackerley Tng Link: https://patch.msgid.link/20260723210811.72720-9-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/include/asm/kvm_host.h | 3 ++- arch/x86/kvm/svm/sev.c | 27 +++++++++++---------------- arch/x86/kvm/svm/svm.h | 2 +- arch/x86/kvm/x86.c | 5 +++-- include/linux/kvm_host.h | 2 +- virt/kvm/guest_memfd.c | 2 +- 6 files changed, 19 insertions(+), 22 deletions(-) diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index ae9a229c6b11..7643e078ba36 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1904,7 +1904,8 @@ struct kvm_x86_ops { gva_t (*get_untagged_addr)(struct kvm_vcpu *vcpu, gva_t gva, unsigned int flags); void *(*alloc_apic_backing_page)(struct kvm_vcpu *vcpu); #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT - int (*gmem_make_private)(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); + int (*gmem_make_private)(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, + kvm_pfn_t nr_pages); #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_RECLAIM void (*gmem_make_shared)(kvm_pfn_t pfn, kvm_pfn_t nr_pages); diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 779889f5e994..5dd45b7a09f2 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -5088,15 +5088,7 @@ static bool is_pfn_range_shared(kvm_pfn_t start, kvm_pfn_t end) return true; } -static u8 max_level_for_order(int order) -{ - if (order >= KVM_HPAGE_GFN_SHIFT(PG_LEVEL_2M)) - return PG_LEVEL_2M; - - return PG_LEVEL_4K; -} - -static bool is_large_rmp_possible(struct kvm *kvm, kvm_pfn_t pfn, int order) +static bool is_large_rmp_possible(kvm_pfn_t pfn, kvm_pfn_t nr_pages) { kvm_pfn_t pfn_aligned = ALIGN_DOWN(pfn, PTRS_PER_PMD); @@ -5105,14 +5097,14 @@ static bool is_large_rmp_possible(struct kvm *kvm, kvm_pfn_t pfn, int order) * PFN is currently shared, then the entire 2M-aligned range can be * set to private via a single 2M RMP entry. */ - if (max_level_for_order(order) > PG_LEVEL_4K && + if (nr_pages >= KVM_PAGES_PER_HPAGE(PG_LEVEL_2M) && is_pfn_range_shared(pfn_aligned, pfn_aligned + PTRS_PER_PMD)) return true; return false; } -int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order) +int sev_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages) { struct kvm_sev_info *sev = to_kvm_sev_info(kvm); kvm_pfn_t pfn_aligned; @@ -5123,6 +5115,9 @@ int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_ord if (!sev_snp_guest(kvm)) return 0; + if (WARN_ON_ONCE(nr_pages != 1)) + return -EIO; + rc = snp_lookup_rmpentry(pfn, &assigned, &level); if (rc) { pr_err_ratelimited("SEV: Failed to look up RMP entry: GFN %llx PFN %llx error %d\n", @@ -5131,12 +5126,12 @@ int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_ord } if (assigned) { - pr_debug("%s: already assigned: gfn %llx pfn %llx max_order %d level %d\n", - __func__, gfn, pfn, max_order, level); + pr_debug("%s: already assigned: gfn %llx pfn %llx nr_pages %llx level %d\n", + __func__, gfn, pfn, nr_pages, level); return 0; } - if (is_large_rmp_possible(kvm, pfn, max_order)) { + if (is_large_rmp_possible(pfn, nr_pages)) { level = PG_LEVEL_2M; pfn_aligned = ALIGN_DOWN(pfn, PTRS_PER_PMD); gfn_aligned = ALIGN_DOWN(gfn, PTRS_PER_PMD); @@ -5153,8 +5148,8 @@ int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_ord return -EINVAL; } - pr_debug("%s: updated: gfn %llx pfn %llx pfn_aligned %llx max_order %d level %d\n", - __func__, gfn, pfn, pfn_aligned, max_order, level); + pr_debug("%s: updated: gfn %llx pfn %llx pfn_aligned %llx nr_pages %llx level %d\n", + __func__, gfn, pfn, pfn_aligned, nr_pages, level); return 0; } diff --git a/arch/x86/kvm/svm/svm.h b/arch/x86/kvm/svm/svm.h index b5cd8437988f..da4c66eb8d70 100644 --- a/arch/x86/kvm/svm/svm.h +++ b/arch/x86/kvm/svm/svm.h @@ -1009,7 +1009,7 @@ int sev_cpu_init(struct svm_cpu_data *sd); int sev_dev_get_attr(u32 group, u64 attr, u64 *val); extern unsigned int max_sev_asid; void sev_handle_rmp_fault(struct kvm_vcpu *vcpu, gpa_t gpa, u64 error_code); -int sev_gmem_make_private(struct kvm *kvm, kvm_pfn_t pfn, gfn_t gfn, int max_order); +int sev_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, kvm_pfn_t nr_pages); void sev_gmem_make_shared(kvm_pfn_t pfn, kvm_pfn_t nr_pages); void sev_gmem_invalidate_range(struct kvm *kvm, struct kvm_gfn_range *range); int sev_gmem_max_mapping_level(struct kvm *kvm, kvm_pfn_t pfn, bool is_private); diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 512bcc35507a..9390e0d4c1e5 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -10590,9 +10590,10 @@ bool kvm_arch_supports_gmem_init_shared(struct kvm *kvm) } #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT -int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, int max_order) +int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, + kvm_pfn_t nr_pages) { - return kvm_x86_call(gmem_make_private)(kvm, pfn, gfn, max_order); + return kvm_x86_call(gmem_make_private)(kvm, gfn, pfn, nr_pages); } #endif diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index ecdda1f00c6c..b24a090eb34d 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -2574,7 +2574,7 @@ static inline int kvm_gmem_get_pfn(struct kvm *kvm, #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT int kvm_arch_gmem_make_private(struct kvm *kvm, gfn_t gfn, kvm_pfn_t pfn, - int max_order); + kvm_pfn_t nr_pages); #endif #ifdef CONFIG_HAVE_KVM_ARCH_GMEM_POPULATE diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 6635ed05f411..0c1ee3e49176 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -77,7 +77,7 @@ static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slo kvm_pfn_t pfn = folio_file_pfn(folio, index); gfn_t gfn = slot->base_gfn + index - slot->gmem.pgoff; - return kvm_arch_gmem_make_private(kvm, gfn, pfn, folio_order(folio)); + return kvm_arch_gmem_make_private(kvm, gfn, pfn, folio_nr_pages(folio)); #else return 0; #endif From 2abcdf03fda1380bcbe00417b9ce2b4afb30899b Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 14:08:11 -0700 Subject: [PATCH 108/121] KVM: guest_memfd: Make private exactly what can be mapped on page fault When calling into arch code to make the underlying memory private, i.e. to assign memory to the VM in SNP's RMP table, assign/convert *exactly* the range of memory that can be mapped into the guest for the current page fault, instead of aggressively converting/assigning the entire folio. For SNP, the mapping size in the stage-2 page tables (Nested Page Tables, NPT) must be at least the size of the corresponding RMP entry, e.g. assigning a 2MiB mapping in the RMP when it can only be mapped at 4KiB granualarity will ultimate result in another page fault (#NPF for SNP) to "smash" the RMP down to the correct mapping size. Assigning the entire folio was necessary back when guest_memfd tracked preparedness, which was done on a per-folio basis. At the time, it made sense to do per-folio tracking/preparation, because tracking per-folio meant guest_memfd didn't need to add a separate data structure to track that information, and doing per-folio tracking only works if the entire folio is prepared (or not). Now that guest_memfd no longer does preparation tracking (see commit 8622ef05709f ("KVM: guest_memfd: Remove preparation tracking")), in favor having SNP query the RMP, per-folio preparation, i.e. per-folio conversions to private, doesn't make any sense. *If* SNP allowed the RMP size to be greater than the NPT size, then per-folio conversion could theoretically provide marginal value, as it would allow KVM to assign a hugepage in the RMP even if it can only be mapped into the NPT with a smaller page, e.g. because of memslot alignment. The documentation of that reasoning would be something like this: /* * If the memory is private from KVM's perspective, and hardware tracks * VM-assigned private memory in a dedicated data structure, i.e. not * in the stage-2 page tables, then call into arch code to assign the * entire folio to the guest. Assigning the entire folio, e.g. instead * of only the memory being mapped into the guest, allows KVM to assign * an entire hugepage of memory in the out-of-band structure even if * KVM can only map a smaller page size into the MMU, e.g. because the * gmem hugepage is spread across multiple memslots. */ But even *if* a future SNP implementation supported that behavior, the value added would be dubious, as having a huge folio that is fully private, but can only be mapped at a smaller granularity, would be rare. E.g. maybe for memory at the top of lower DRAM that has holes for non-RAM assets? So, convert/assign exactly what guest_memfd allows the caller to map to simplify the guest_memfd code and provide a (super) minor performance optimization for SNP. E.g. once hugepage support comes along, guest_memfd will only need a single flow to compute "how much memory can be assigned and at what size". Reviewed-by: Ackerley Tng Link: https://patch.msgid.link/20260723210811.72720-10-seanjc@google.com Signed-off-by: Sean Christopherson --- virt/kvm/guest_memfd.c | 53 ++++++------------------------------------ 1 file changed, 7 insertions(+), 46 deletions(-) diff --git a/virt/kvm/guest_memfd.c b/virt/kvm/guest_memfd.c index 0c1ee3e49176..fdd26adaa7d4 100644 --- a/virt/kvm/guest_memfd.c +++ b/virt/kvm/guest_memfd.c @@ -70,50 +70,6 @@ static bool kvm_gmem_is_shared_mem(struct inode *inode, pgoff_t index) return !kvm_gmem_is_private_mem(inode, index); } -static int __kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, - pgoff_t index, struct folio *folio) -{ -#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT - kvm_pfn_t pfn = folio_file_pfn(folio, index); - gfn_t gfn = slot->base_gfn + index - slot->gmem.pgoff; - - return kvm_arch_gmem_make_private(kvm, gfn, pfn, folio_nr_pages(folio)); -#else - return 0; -#endif -} - -/* - * Process @folio, which contains @gfn, so that the guest can use it. - * The folio must be locked and the gfn must be contained in @slot. - * On successful return the guest sees a zero page so as to avoid - * leaking host data and the up-to-date flag is set. - */ -static int kvm_gmem_prepare_folio(struct kvm *kvm, struct kvm_memory_slot *slot, - gfn_t gfn, struct folio *folio) -{ - pgoff_t index; - - /* - * Preparing huge folios should always be safe, since it should - * be possible to split them later if needed. - * - * Right now the folio order is always going to be zero, but the - * code is ready for huge folios. The only assumption is that - * the base pgoff of memslots is naturally aligned with the - * requested page order, ensuring that huge folios can also use - * huge page table entries for GPA->HPA mapping. - * - * The order will be passed when creating the guest_memfd, and - * checked when creating memslots. - */ - WARN_ON(!IS_ALIGNED(slot->gmem.pgoff, folio_nr_pages(folio))); - index = kvm_gmem_get_index(slot, gfn); - index = ALIGN_DOWN(index, folio_nr_pages(folio)); - - return __kvm_gmem_prepare_folio(kvm, slot, index, folio); -} - /* * Returns a locked folio on success. The caller is responsible for * setting the up-to-date flag before the memory is mapped into the guest. @@ -799,7 +755,9 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, { pgoff_t index = kvm_gmem_get_index(slot, gfn); struct folio *folio; - int r = 0; + int r = 0, __order; + + max_order = max_order ?: &__order; CLASS(gmem_get_file, file)(slot); if (!file) @@ -814,8 +772,11 @@ int kvm_gmem_get_pfn(struct kvm *kvm, struct kvm_memory_slot *slot, folio_mark_uptodate(folio); } +#ifdef CONFIG_HAVE_KVM_ARCH_GMEM_CONVERT if (kvm_gmem_is_private_mem(file_inode(file), index)) - r = kvm_gmem_prepare_folio(kvm, slot, gfn, folio); + r = kvm_arch_gmem_make_private(kvm, gfn, *pfn, + (kvm_pfn_t)1 << *max_order); +#endif folio_unlock(folio); From e43a21553ac8080af40a494adf647d5dc7e685d5 Mon Sep 17 00:00:00 2001 From: Yosry Ahmed Date: Mon, 13 Jul 2026 18:01:53 +0000 Subject: [PATCH 109/121] KVM: selftests: Extend set_sregs test to cover EFER Extend the set_sregs test to cover various bits in EFER. Update TEST_INVALID_CR_BIT() to operate on EFER as well as CRx (and rename it accordingly). Add test cases to check that EFER bits are disallowed without the relevant CPUID enablement. Assisted-by: Gemini:unknown-version Signed-off-by: Yosry Ahmed Link: https://patch.msgid.link/20260713180153.2728382-3-yosry@kernel.org Signed-off-by: Sean Christopherson --- .../selftests/kvm/include/x86/processor.h | 2 + .../selftests/kvm/x86/set_sregs_test.c | 81 ++++++++++++++----- 2 files changed, 63 insertions(+), 20 deletions(-) diff --git a/tools/testing/selftests/kvm/include/x86/processor.h b/tools/testing/selftests/kvm/include/x86/processor.h index 7d3a27bc0d84..b161174ece45 100644 --- a/tools/testing/selftests/kvm/include/x86/processor.h +++ b/tools/testing/selftests/kvm/include/x86/processor.h @@ -208,6 +208,7 @@ struct kvm_x86_cpu_feature { #define X86_FEATURE_PERFCTR_NB KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 24) #define X86_FEATURE_PERFCTR_LLC KVM_X86_CPU_FEATURE(0x80000001, 0, ECX, 28) #define X86_FEATURE_NX KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 20) +#define X86_FEATURE_FXSR_OPT KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 25) #define X86_FEATURE_GBPAGES KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 26) #define X86_FEATURE_RDTSCP KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 27) #define X86_FEATURE_LM KVM_X86_CPU_FEATURE(0x80000001, 0, EDX, 29) @@ -226,6 +227,7 @@ struct kvm_x86_cpu_feature { #define X86_FEATURE_SEV KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 1) #define X86_FEATURE_SEV_ES KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 3) #define X86_FEATURE_SEV_SNP KVM_X86_CPU_FEATURE(0x8000001F, 0, EAX, 4) +#define X86_FEATURE_AUTOIBRS KVM_X86_CPU_FEATURE(0x80000021, 0, EAX, 8) #define X86_FEATURE_GP_ON_USER_CPUID KVM_X86_CPU_FEATURE(0x80000021, 0, EAX, 17) #define X86_FEATURE_PERFMON_V2 KVM_X86_CPU_FEATURE(0x80000022, 0, EAX, 0) #define X86_FEATURE_LBR_PMC_FREEZE KVM_X86_CPU_FEATURE(0x80000022, 0, EAX, 2) diff --git a/tools/testing/selftests/kvm/x86/set_sregs_test.c b/tools/testing/selftests/kvm/x86/set_sregs_test.c index 8e654cc9ab16..603226ffe437 100644 --- a/tools/testing/selftests/kvm/x86/set_sregs_test.c +++ b/tools/testing/selftests/kvm/x86/set_sregs_test.c @@ -21,20 +21,20 @@ #include "kvm_util.h" #include "processor.h" -#define TEST_INVALID_CR_BIT(vcpu, cr, orig, bit) \ +#define TEST_INVALID_SREG_BIT(vcpu, reg, orig, bit) \ do { \ struct kvm_sregs new; \ int rc; \ \ /* Skip the sub-test, the feature/bit is supported. */ \ - if (orig.cr & bit) \ + if (orig.reg & bit) \ break; \ \ - memcpy(&new, &orig, sizeof(sregs)); \ - new.cr |= bit; \ + memcpy(&new, &orig, sizeof(new)); \ + new.reg |= bit; \ \ rc = _vcpu_sregs_set(vcpu, &new); \ - TEST_ASSERT(rc, "KVM allowed invalid " #cr " bit (0x%lx)", bit); \ + TEST_ASSERT(rc, "KVM allowed invalid " #reg " bit (0x%lx)", (u64)bit); \ \ /* Sanity check that KVM didn't change anything. */ \ vcpu_sregs_get(vcpu, &new); \ @@ -46,6 +46,8 @@ do { \ X86_CR4_MCE | X86_CR4_PGE | X86_CR4_PCE | \ X86_CR4_OSFXSR | X86_CR4_OSXMMEXCPT) +#define KVM_ALWAYS_ALLOWED_EFER EFER_SCE + static u64 calc_supported_cr4_feature_bits(void) { u64 cr4 = KVM_ALWAYS_ALLOWED_CR4; @@ -74,6 +76,24 @@ static u64 calc_supported_cr4_feature_bits(void) return cr4; } +static u64 calc_supported_efer_feature_bits(void) +{ + u64 efer = KVM_ALWAYS_ALLOWED_EFER; + + if (kvm_cpu_has(X86_FEATURE_LM)) + efer |= (EFER_LME | EFER_LMA); + if (kvm_cpu_has(X86_FEATURE_NX)) + efer |= EFER_NX; + if (kvm_cpu_has(X86_FEATURE_SVM)) + efer |= EFER_SVME; + if (kvm_cpu_has(X86_FEATURE_FXSR_OPT)) + efer |= EFER_FFXSR; + if (kvm_cpu_has(X86_FEATURE_AUTOIBRS)) + efer |= EFER_AUTOIBRS; + + return efer; +} + static void test_cr_bits(struct kvm_vcpu *vcpu, u64 cr4) { struct kvm_sregs sregs; @@ -96,26 +116,45 @@ static void test_cr_bits(struct kvm_vcpu *vcpu, u64 cr4) (sregs.cr4 & X86_CR4_PKE) ? "set" : "clear"); vcpu_sregs_get(vcpu, &sregs); - TEST_ASSERT(sregs.cr4 == cr4, "sregs.CR4 (0x%llx) != CR4 (0x%lx)", - sregs.cr4, cr4); + TEST_ASSERT_EQ(sregs.cr4, cr4); - TEST_INVALID_CR_BIT(vcpu, cr4, sregs, X86_CR4_UMIP); - TEST_INVALID_CR_BIT(vcpu, cr4, sregs, X86_CR4_LA57); - TEST_INVALID_CR_BIT(vcpu, cr4, sregs, X86_CR4_VMXE); - TEST_INVALID_CR_BIT(vcpu, cr4, sregs, X86_CR4_SMXE); - TEST_INVALID_CR_BIT(vcpu, cr4, sregs, X86_CR4_FSGSBASE); - TEST_INVALID_CR_BIT(vcpu, cr4, sregs, X86_CR4_PCIDE); - TEST_INVALID_CR_BIT(vcpu, cr4, sregs, X86_CR4_OSXSAVE); - TEST_INVALID_CR_BIT(vcpu, cr4, sregs, X86_CR4_SMEP); - TEST_INVALID_CR_BIT(vcpu, cr4, sregs, X86_CR4_SMAP); - TEST_INVALID_CR_BIT(vcpu, cr4, sregs, X86_CR4_PKE); + TEST_INVALID_SREG_BIT(vcpu, cr4, sregs, X86_CR4_UMIP); + TEST_INVALID_SREG_BIT(vcpu, cr4, sregs, X86_CR4_LA57); + TEST_INVALID_SREG_BIT(vcpu, cr4, sregs, X86_CR4_VMXE); + TEST_INVALID_SREG_BIT(vcpu, cr4, sregs, X86_CR4_SMXE); + TEST_INVALID_SREG_BIT(vcpu, cr4, sregs, X86_CR4_FSGSBASE); + TEST_INVALID_SREG_BIT(vcpu, cr4, sregs, X86_CR4_PCIDE); + TEST_INVALID_SREG_BIT(vcpu, cr4, sregs, X86_CR4_OSXSAVE); + TEST_INVALID_SREG_BIT(vcpu, cr4, sregs, X86_CR4_SMEP); + TEST_INVALID_SREG_BIT(vcpu, cr4, sregs, X86_CR4_SMAP); + TEST_INVALID_SREG_BIT(vcpu, cr4, sregs, X86_CR4_PKE); for (i = 32; i < 64; i++) - TEST_INVALID_CR_BIT(vcpu, cr0, sregs, BIT(i)); + TEST_INVALID_SREG_BIT(vcpu, cr0, sregs, BIT(i)); /* NW without CD is illegal, as is PG without PE. */ - TEST_INVALID_CR_BIT(vcpu, cr0, sregs, X86_CR0_NW); - TEST_INVALID_CR_BIT(vcpu, cr0, sregs, X86_CR0_PG); + TEST_INVALID_SREG_BIT(vcpu, cr0, sregs, X86_CR0_NW); + TEST_INVALID_SREG_BIT(vcpu, cr0, sregs, X86_CR0_PG); +} + +static void test_efer_bits(struct kvm_vcpu *vcpu, u64 efer) +{ + struct kvm_sregs sregs; + int rc; + + vcpu_sregs_get(vcpu, &sregs); + sregs.efer |= efer; + rc = _vcpu_sregs_set(vcpu, &sregs); + TEST_ASSERT(!rc, "Failed to set supported EFER bits (0x%llx)", sregs.efer); + + vcpu_sregs_get(vcpu, &sregs); + TEST_ASSERT_EQ(sregs.efer, efer); + + TEST_INVALID_SREG_BIT(vcpu, efer, sregs, EFER_LME); + TEST_INVALID_SREG_BIT(vcpu, efer, sregs, EFER_NX); + TEST_INVALID_SREG_BIT(vcpu, efer, sregs, EFER_SVME); + TEST_INVALID_SREG_BIT(vcpu, efer, sregs, EFER_FFXSR); + TEST_INVALID_SREG_BIT(vcpu, efer, sregs, EFER_AUTOIBRS); } int main(int argc, char *argv[]) @@ -132,6 +171,7 @@ int main(int argc, char *argv[]) */ vm = vm_create_barebones(); vcpu = __vm_vcpu_add(vm, 0); + test_efer_bits(vcpu, KVM_ALWAYS_ALLOWED_EFER); test_cr_bits(vcpu, KVM_ALWAYS_ALLOWED_CR4); kvm_vm_free(vm); @@ -151,6 +191,7 @@ int main(int argc, char *argv[]) sregs.apic_base); test_cr_bits(vcpu, calc_supported_cr4_feature_bits()); + test_efer_bits(vcpu, calc_supported_efer_feature_bits()); kvm_vm_free(vm); From 0c5fb7bc6177b8903196dcb3f4b211d51bbd0ae6 Mon Sep 17 00:00:00 2001 From: Tim Wiederhake Date: Wed, 15 Jul 2026 14:03:40 +0200 Subject: [PATCH 110/121] KVM: x86: Replace delivery mode TODO with WARN_ON_ONCE The default case in __apic_accept_irq() has carried a printk("TODO: unsupported delivery mode") since the original LAPIC emulation was introduced in commit 97222cc83163 ("KVM: Emulate local APIC in kernel"). The switch now handles all eight delivery modes defined by the x86 architecture and is always constrained to the three-bit field defined by the architecture: Either by masking with APIC_MODE_MASK, by three-bit bitfield widths in the IOAPIC and MSI structs, or by using APIC_DM_* constants directly. Replace the unreachable printk with WARN_ON_ONCE(1) to match the existing pattern for impossible defaults elsewhere in the same file. Signed-off-by: Tim Wiederhake Link: https://patch.msgid.link/20260715120341.2661873-1-twiederh@redhat.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/lapic.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/arch/x86/kvm/lapic.c b/arch/x86/kvm/lapic.c index 0354db0f2c0f..a6c06c69832e 100644 --- a/arch/x86/kvm/lapic.c +++ b/arch/x86/kvm/lapic.c @@ -1497,8 +1497,7 @@ static int __apic_accept_irq(struct kvm_lapic *apic, int delivery_mode, break; default: - printk(KERN_ERR "TODO: unsupported delivery mode %x\n", - delivery_mode); + WARN_ON_ONCE(1); break; } return result; From 681fb81a41711f1237899a2e24001d9b5d5efae4 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 24 Jul 2026 10:34:24 -0700 Subject: [PATCH 111/121] KVM: x86: Don't WARN if IRQ disappears because it was cleared from the PIC When getting a to-be-injected IRQ, don't WARN if the IRQ disappeared and the VM has an in-kernel PIC, as the ExtINT handling that's routed through KVM's virtual PIC is tracked per-VM, not per-vCPU. If another vCPU grabs the IRQ, or deasserts the interrupt (which is level-triggered), then it's both expected and "fine" for a Keep the assert for split IRQCHIP VMs to help detect KVM bugs, as userspace is responsible for routing ExtINT to the intended vCPU, i.e. once an ExtINT is pending, it can't be cleared without holding the vCPU's mutex, and thus false positives are impossible. Fixes: bf672720e83c ("KVM: x86: check the kvm_cpu_get_interrupt result before using it") Debugged-by: Alexander Potapenko Reported-by: syzbot+dd769db18693736eee89@syzkaller.appspotmail.com Closes: https://syzkaller.appspot.com/bug?extid=dd769db18693736eee89 Closes: https://lore.kernel.org/all/6a360fdf.871e809a.2d6dda.0000.GAE@google.com Link: https://patch.msgid.link/20260724173425.278753-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/irq.h | 16 ++++++++++++++++ arch/x86/kvm/vmx/nested.c | 4 +++- arch/x86/kvm/x86.c | 4 +++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h index 1a84ea31e7fd..eeaf527cecc4 100644 --- a/arch/x86/kvm/irq.h +++ b/arch/x86/kvm/irq.h @@ -118,6 +118,22 @@ int kvm_cpu_has_extint(struct kvm_vcpu *v); int kvm_cpu_get_extint(struct kvm_vcpu *v); int kvm_cpu_get_interrupt(struct kvm_vcpu *v); +static inline void kvm_warn_on_lost_irq(struct kvm_vcpu *vcpu) +{ + /* + * WARN if an IRQ was lost between detecting the IRQ and grabbing the + * IRQ for injection, unless it's possible the lost IRQ was due to one + * of the exceptional cases below. + * + * If the VM has an in-kernel PIC, the ExtINT handling that's routed + * through KVM's virtual PIC is tracked per-VM, not per-vCPU. If + * another vCPU grabs the IRQ, or deasserts the interrupt (which is + * level-triggered), then it's both expected and "fine" for an IRQ + * seemingly be "lost" from this vCPU's perspective. + */ + WARN_ON_ONCE(!pic_in_kernel(vcpu->kvm)); +} + void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu); void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu); void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu); diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index a9af4e9e6657..ed9dd03a1ddf 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -4467,8 +4467,10 @@ static int vmx_check_nested_events(struct kvm_vcpu *vcpu) } irq = kvm_apic_has_interrupt(vcpu); - if (WARN_ON_ONCE(irq < 0)) + if (unlikely(irq < 0)) { + kvm_warn_on_lost_irq(vcpu); goto no_vmexit; + } /* * If the IRQ is L2's PI notification vector, process posted diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index dfaf80efec4b..3be5024045da 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -7733,10 +7733,12 @@ static int kvm_check_and_inject_events(struct kvm_vcpu *vcpu, if (r) { int irq = kvm_cpu_get_interrupt(vcpu); - if (!WARN_ON_ONCE(irq == -1)) { + if (likely(irq != -1)) { kvm_queue_interrupt(vcpu, irq, false); kvm_x86_call(inject_irq)(vcpu, false); WARN_ON(kvm_x86_call(interrupt_allowed)(vcpu, true) < 0); + } else { + kvm_warn_on_lost_irq(vcpu); } } if (kvm_cpu_has_injectable_intr(vcpu)) From 4f05a3337d7b27cd0c2da15b5f14a119613ce6bb Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Fri, 24 Jul 2026 10:34:25 -0700 Subject: [PATCH 112/121] KVM: x86: Don't WARN if IRQ disappears when Xen emulation is enabled. When getting a to-be-injected IRQ, don't WARN if the IRQ disappeared and Xen emulation is supported, as a guest could concurrently toggle its evtchn_upcall_pending flag in shared memory and deassert the IRQ. Even more annoyingly, userspace could disable Xen emulation for the entire VM KVM_XEN_HVM_CONFIG. So, suppress WARNs on lost IRQs if Xen emulation is supported to prevent false positives. Alternatively, KVM could track if the VM has ever used Xen emulation, but the added complexity isn't worth carrying given that the vast majority of deployments can and should disable Xen emulation. Fixes: bf672720e83c ("KVM: x86: check the kvm_cpu_get_interrupt result before using it") Reported-by: Sashiko Bot Closes: https://lore.kernel.org/all/20260625212001.3B6561F000E9@smtp.kernel.org Link: https://patch.msgid.link/20260724173425.278753-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/irq.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/arch/x86/kvm/irq.h b/arch/x86/kvm/irq.h index eeaf527cecc4..a74f03858004 100644 --- a/arch/x86/kvm/irq.h +++ b/arch/x86/kvm/irq.h @@ -130,8 +130,12 @@ static inline void kvm_warn_on_lost_irq(struct kvm_vcpu *vcpu) * another vCPU grabs the IRQ, or deasserts the interrupt (which is * level-triggered), then it's both expected and "fine" for an IRQ * seemingly be "lost" from this vCPU's perspective. + * + * Similarly, Xen's event channel isn't entirely within KVM's control, + * e.g. Xen emulation can be disabled entirely per-VM, or the guest + * can desassert an IRQ by writing to shared memory. */ - WARN_ON_ONCE(!pic_in_kernel(vcpu->kvm)); + WARN_ON_ONCE(!pic_in_kernel(vcpu->kvm) && !IS_ENABLED(CONFIG_KVM_XEN)); } void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu); From 48baf3fef274a3cb6d9a9ad95e1a84ee3fb4d0cf Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 27 Jul 2026 17:22:35 -0700 Subject: [PATCH 113/121] KVM: x86/mmu: Use CMPXCHG when clearing Accessed bit in TDP MMU Use LOCK CMPXCHG instead of LOCK AND to clear the Accessed bit when aging SPTEs in the TDP MMU, as doing a LOCK AND can corrupt a FROZEN SPTE and allow a third CPU to effectively overwrite the FROZEN SPTE. As pointed out by AI of some kind, because the magic FROZEN_SPTE value is a "full" SPTE, not a single bit, and includes the Accessed bit, clearing the Accessed bit in a FROZEN SPTE will result in is_frozen_spte() getting a false negative. E.g. if CPU0 freezes an SPTE, and CPU1 clears the Accessed bitin the frozen SPTE, then CPU2 could come along and overwrite the frozen SPTE with a shadow-present SPTE. Thankfully, the false negative is largely benign, because outside of TDX, which doesn't support aging, KVM only freezes leaf SPTEs when removing an upper level shadow page. So while KVM could clobber a frozen SPTE back to a shadow-present SPTE, and could even use the new SPTE, the subsequent TLB flush will make the orphaned, shadow-present SPTE unreachable. Failure to ever zap the orphaned leaf SPTE would show up in KVM's stats, but otherwise is benign (because KVM no longer keeps an elevated refcount for leaf SPTEs). Opportunistically add a comment to warn future developers away from using kvm_tdp_mmu_write_spte_atomic() and tdp_mmu_clear_spte_bits_atomic(), as they are generally unsafe. Keep the helpers, e.g. instead of open-coding the atomic64_fetch_and() in tdp_mmu_clear_spte_bits(), as scary warnings usually are more effective deterrent against recidivism than removal of the dangerous code. Alternatively, KVM could use different bits for the magic FROZEN_SPTE value, e.g. setting the Dirty bits (with effective IPAT and Global aliases) would likely be "ok", as IPAT/Global are extremely unlikely to be cleared without doing a full SPTE write, and KVM's clearing of Dirty bits shares logic with Write-Protection, which must do a full SPTE write (via cmpxchg64() in the TDP MMU) to ensure KVM isn't clobbering state. But there is zero reason to carry that risk (beyond stubbornness in wanting to preserve a "cute" idea), as the cost of LOCK CMPXCHG and LOCK AND are within 1-2 uops of each other on modern hardware. Fixes: b146a9b34aed ("KVM: x86/mmu: Age TDP MMU SPTEs without holding mmu_lock") Cc: stable@vger.kernel.org Reviewed-by: James Houghton Link: https://patch.msgid.link/20260728002236.869865-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/tdp_iter.h | 7 +++++++ arch/x86/kvm/mmu/tdp_mmu.c | 20 +++++++++----------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/arch/x86/kvm/mmu/tdp_iter.h b/arch/x86/kvm/mmu/tdp_iter.h index 364c5da6c499..f898d8d0d93c 100644 --- a/arch/x86/kvm/mmu/tdp_iter.h +++ b/arch/x86/kvm/mmu/tdp_iter.h @@ -19,6 +19,13 @@ static inline u64 kvm_tdp_mmu_read_spte(tdp_ptep_t sptep) return READ_ONCE(*rcu_dereference(sptep)); } +/* + * WARNING! mmu_lock must be held for write when using the "write atomic" or + * "clear bits atomic" APIs, otherwise KVM could overwrite the "wrong" old SPTE + * value, i.e. clobber an update from a different CPU. The only exception is + * when KVM is freezing a leaf SPTE for removal, in which case KVM doesn't care + * about the exact old SPTE value (KVM will react to the actual old value). + */ static inline u64 kvm_tdp_mmu_write_spte_atomic(tdp_ptep_t sptep, u64 new_spte) { KVM_MMU_WARN_ON(is_ept_ve_possible(new_spte)); diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c index ce3f2efadb05..44dad106fad1 100644 --- a/arch/x86/kvm/mmu/tdp_mmu.c +++ b/arch/x86/kvm/mmu/tdp_mmu.c @@ -1335,19 +1335,17 @@ static void kvm_tdp_mmu_age_spte(struct kvm *kvm, struct tdp_iter *iter) if (WARN_ON_ONCE(is_mirror_sptep(iter->sptep))) return; - if (spte_ad_enabled(iter->old_spte)) { - iter->old_spte = tdp_mmu_clear_spte_bits_atomic(iter->sptep, - shadow_accessed_mask); + if (spte_ad_enabled(iter->old_spte)) new_spte = iter->old_spte & ~shadow_accessed_mask; - } else { + else new_spte = mark_spte_for_access_track(iter->old_spte); - /* - * It is safe for the following cmpxchg to fail. Leave the - * Accessed bit set, as the spte is most likely young anyway. - */ - if (__tdp_mmu_set_spte_atomic(kvm, iter, new_spte)) - return; - } + + /* + * Don't bother retrying if another CPU modified the SPTE, the SPTE is + * either being zapped or is likely still in-use, i.e. is still young. + */ + if (__tdp_mmu_set_spte_atomic(kvm, iter, new_spte)) + return; trace_kvm_tdp_mmu_spte_changed(iter->as_id, iter->gfn, iter->level, iter->old_spte, new_spte); From 58429ed0374d2359714105adca2b9d90d54e2076 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Mon, 27 Jul 2026 17:22:36 -0700 Subject: [PATCH 114/121] KVM: x86/mmu: Use CMPXCHG when clearing Accessed bit in the shadow MMU Use CMPXCHG instead of clear_bit(), which currently emits a LOCK BTR since the to-be-cleared bit isn't a compiled-time constant, when aging SPTEs in the shadow MMU to align with the approach taken by the TDP MMU, and because using CMPXCHG is far more robust against bugs in KVM. E.g. if the SPTE is somehow no longer an SPTE due to a KVM bug, CMPXCHG will fail gracefully, whereas clear_bit() would potentially corrupt/clobber memory. Clearing the Accessed bit without atomically ensuring the SPTE is still the old SPTE is "fine", as holding the rmap's lock ensures zapping the old SPTE can't fully complete, which in turn ensures a new, different SPTE can't be installed. But that chain of logic isn't exactly obvious, and there's zero reason to avoid CMPXCHG as its cost on modern hardware is within ~1-2 uops of LOCK BTR (and may even be cheaper on some microarchitectures). Doing a 64-bit CMPXCHG on 32-bit kernels does requires a more expensive CMPXCHG8B, but 32-bit KVM is all but dead at this point. Cc: James Houghton Reviewed-by: James Houghton Link: https://patch.msgid.link/20260728002236.869865-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/mmu/mmu.c | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c index 440db8891f21..71beb169d78d 100644 --- a/arch/x86/kvm/mmu/mmu.c +++ b/arch/x86/kvm/mmu/mmu.c @@ -1718,11 +1718,11 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm, struct kvm_rmap_head *rmap_head; struct rmap_iterator iter; unsigned long rmap_val; + u64 old_spte, new_spte; bool young = false; u64 *sptep; gfn_t gfn; int level; - u64 spte; for (level = PG_LEVEL_4K; level <= KVM_MAX_HUGEPAGE_LEVEL; level++) { for (gfn = range->start; gfn < range->end; @@ -1730,8 +1730,8 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm, rmap_head = gfn_to_rmap(gfn, level, range->slot); rmap_val = kvm_rmap_lock_readonly(rmap_head); - for_each_rmap_spte_lockless(rmap_val, &iter, sptep, spte) { - if (!is_accessed_spte(spte)) + for_each_rmap_spte_lockless(rmap_val, &iter, sptep, old_spte) { + if (!is_accessed_spte(old_spte)) continue; if (test_only) { @@ -1739,17 +1739,18 @@ static bool kvm_rmap_age_gfn_range(struct kvm *kvm, return true; } - if (spte_ad_enabled(spte)) - clear_bit((ffs(shadow_accessed_mask) - 1), - (unsigned long *)sptep); + if (spte_ad_enabled(old_spte)) + new_spte = old_spte & ~shadow_accessed_mask; else - /* - * If the following cmpxchg fails, the - * spte is being concurrently modified - * and should most likely stay young. - */ - cmpxchg64(sptep, spte, - mark_spte_for_access_track(spte)); + new_spte = mark_spte_for_access_track(old_spte); + + /* + * Don't bother retrying if the CMPXCHG fails, + * i.e. if another CPU modified the SPTE. The + * SPTE is either being zapped or is likely + * still in-use, i.e. is still young. + */ + cmpxchg64(sptep, old_spte, new_spte); young = true; } From 3d4b20b5a7df5cce97911390b4bd5623d9f327bd Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Fri, 3 Jul 2026 22:17:40 +0100 Subject: [PATCH 115/121] KVM: x86/xen: Do not corrupt KVM clock in kvm_xen_shared_info_init() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The KVM clock is an interesting thing. It is defined as "nanoseconds since the guest was created", but in practice it runs at two *different* rates — or three different rates, if you count implementation bugs. Definition A is that it runs synchronously with the CLOCK_MONOTONIC_RAW of the host, with a delta of kvm->arch.kvmclock_offset. But that version doesn't actually get used in the common case, where the host has a reliable TSC and the guest TSCs are all running at the same rate and in sync with each other, and kvm->arch.use_master_clock is set. In that common case, definition B is used: There is a reference point in time at kvm->arch.master_kernel_ns (again a CLOCK_MONOTONIC_RAW time), and a corresponding host TSC value kvm->arch.master_cycle_now. This fixed point in time is converted to guest units (the time offset by kvmclock_offset and the TSC Value scaled and offset to be a guest TSC value) and advertised to the guest in the pvclock structure. While in this 'use_master_clock' mode, the fixed point in time never needs to be changed, and the clock runs precisely in time with the guest TSC, at the rate advertised in the pvclock structure. The third definition C is implemented in kvm_get_wall_clock_epoch() and __get_kvmclock(), using the master_cycle_now and master_kernel_ns fields but converting the *host* TSC cycles directly to a value in nanoseconds instead of scaling via the guest TSC. One might naïvely think that all three definitions are identical, since CLOCK_MONOTONIC_RAW is not skewed by NTP frequency corrections; all three are just the result of counting the host TSC at a known frequency, or the scaled guest TSC at a known precise fraction of the host's frequency. The problem is with arithmetic precision, and the way that frequency scaling is done in a division-free way by multiplying by a scale factor, then shifting right. In practice, all three ways of calculating the KVM clock will suffer a systemic drift from each other. Eventually, definition C should just be eliminated. Commit 451a707813ae ("KVM: x86/xen: improve accuracy of Xen timers") worked around it for the specific case of Xen timers, which are defined in terms of the KVM clock and suffered from a continually increasing error in timer expiry times. That commit notes that get_kvmclock_ns() is non-trivial to fix and says "I'll come back to that", which remains true. Definitions A and B do need to coexist, the former to handle the case where the host or guest TSC is suboptimally configured. But KVM should be more careful about switching between them, and the discontinuity in guest time which could result. In particular, KVM_REQ_MASTERCLOCK_UPDATE will take a new snapshot of time as the reference in master_kernel_ns and master_cycle_now, yanking the guest's clock back to match definition A at that moment. When invoked from in 'use_master_clock' mode, kvm_update_masterclock() should probably *adjust* kvm->arch.kvmclock_offset to account for the drift, instead of yanking the clock back to definition A. But in the meantime there are a bunch of places where it just doesn't need to be invoked at all. To start with: there is no need to do such an update when a Xen guest populates the shared_info page. This seems to have been a hangover from the very first implementation of shared_info which automatically populated the vcpu_info structures at their default locations, but even then it should just have raised KVM_REQ_CLOCK_UPDATE on each vCPU instead of using KVM_REQ_MASTERCLOCK_UPDATE. And now that userspace is expected to explicitly set the vcpu_info even in its default locations, there's not even any need for that either. Fixes: 629b5348841a ("KVM: x86/xen: update wallclock region") Reviewed-by: Paul Durrant Signed-off-by: David Woodhouse Signed-off-by: Sean Christopherson --- arch/x86/kvm/xen.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/arch/x86/kvm/xen.c b/arch/x86/kvm/xen.c index 694b31c1fcc9..913cbd52ce6c 100644 --- a/arch/x86/kvm/xen.c +++ b/arch/x86/kvm/xen.c @@ -98,8 +98,6 @@ static int kvm_xen_shared_info_init(struct kvm *kvm) wc->version = wc_version + 1; read_unlock_irq(&gpc->lock); - kvm_make_all_cpus_request(kvm, KVM_REQ_MASTERCLOCK_UPDATE); - out: srcu_read_unlock(&kvm->srcu, idx); return ret; From 11722439fb206c88e6f31be54173efa9880b4ccb Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 17:47:52 -0700 Subject: [PATCH 116/121] KVM: nVMX: Ensure KVM_REQ_GET_NESTED_STATE_PAGES is cleared on VM-Exit Always check and clear KVM_REQ_GET_NESTED_STATE_PAGES when emulating a nested VM-Exit to ensure the request is cleared, even when KVM was built with CONFIG_KVM_HYPERV=n, as KVM subtly relies on the "check" to clear the flag and thus avoid double-mapping the vmcs12 pages, e.g. if KVM manages to bail from VM-Enter without processing the request, and then emulates VMLAUNCH or VMRESUME. Fixes: b4f69df0f65e ("KVM: x86: Make Hyper-V emulation optional") Cc: stable@vger.kernel.org Reported-by: Yosry Ahmed Reviewed-by: Yosry Ahmed Link: https://patch.msgid.link/20260724004757.131420-2-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/nested.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index ed9dd03a1ddf..57e02c3fa182 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -5081,8 +5081,9 @@ void __nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, /* trying to cancel vmlaunch/vmresume is a bug */ kvm_warn_on_nested_run_pending(vcpu); -#ifdef CONFIG_KVM_HYPERV + /* Note, "checking" the request also clears the request. */ if (kvm_check_request(KVM_REQ_GET_NESTED_STATE_PAGES, vcpu)) { +#ifdef CONFIG_KVM_HYPERV /* * KVM_REQ_GET_NESTED_STATE_PAGES is also used to map * Enlightened VMCS after migration and we still need to @@ -5090,8 +5091,8 @@ void __nested_vmx_vmexit(struct kvm_vcpu *vcpu, u32 vm_exit_reason, * the first L2 run. */ (void)nested_get_evmcs_page(vcpu); - } #endif + } /* Service pending TLB flush requests for L2 before switching to L1. */ kvm_service_local_tlb_flush_requests(vcpu); From d196e4d91e7845b9e65436574bff63234a6f4d1d Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 17:47:53 -0700 Subject: [PATCH 117/121] KVM: nSVM: Add CLASS()es for automagically handling local kvm_vcpu_map() usage Add CLASS() definitions for locally mapping a PFN using kvm_vcpu_map() given a vCPU+gfn pair. In addition to eliminating the need to manually do unmap(), e.g. in error paths, this will allow hardening KVM against double-mapping without having to manually ensure every on-stack declaration is zero-initialized. Use "map local" as the primary terminology as the basic concept is more or less the same as kmap_local(): ensure the current context has a kernel mapping to the underlying memory. Immediately convert the relatively straightforward nested SVM flows, and defer converting the more involved SMM flows to a separate change. No functional change intended. Cc: Yosry Ahmed Link: https://patch.msgid.link/20260724004757.131420-3-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/nested.c | 18 +++++++----------- arch/x86/kvm/svm/svm.c | 8 +++----- include/linux/kvm_host.h | 19 +++++++++++++++++++ 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 5f6d9971a3f2..4deb8c75df3d 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -1086,14 +1086,14 @@ int enter_svm_guest_mode(struct kvm_vcpu *vcpu, u64 vmcb12_gpa, bool from_vmrun) static int nested_svm_copy_vmcb12_to_cache(struct kvm_vcpu *vcpu, u64 vmcb12_gpa) { struct vcpu_svm *svm = to_svm(vcpu); - struct kvm_host_map map; struct vmcb *vmcb12; int r = 0; - if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map)) + CLASS(kvm_vcpu_map_local, m)(vcpu, gpa_to_gfn(vmcb12_gpa)); + if (m.ret) return -EFAULT; - vmcb12 = map.hva; + vmcb12 = m.map.hva; nested_copy_vmcb_control_to_cache(svm, &vmcb12->control); nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); @@ -1107,7 +1107,6 @@ static int nested_svm_copy_vmcb12_to_cache(struct kvm_vcpu *vcpu, u64 vmcb12_gpa r = -EINVAL; } - kvm_vcpu_unmap(vcpu, &map); return r; } @@ -1251,15 +1250,13 @@ static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu) { struct vcpu_svm *svm = to_svm(vcpu); struct vmcb *vmcb02 = svm->nested.vmcb02.ptr; - struct kvm_host_map map; struct vmcb *vmcb12; - int rc; - rc = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa), &map); - if (rc) - return rc; + CLASS(kvm_vcpu_map_local, m)(vcpu, gpa_to_gfn(svm->nested.vmcb12_gpa)); + if (m.ret) + return m.ret; - vmcb12 = map.hva; + vmcb12 = m.map.hva; vmcb12->save.es = vmcb02->save.es; vmcb12->save.cs = vmcb02->save.cs; @@ -1314,7 +1311,6 @@ static int nested_svm_vmexit_update_vmcb12(struct kvm_vcpu *vcpu) vmcb12->control.exit_int_info_err, KVM_ISA_SVM); - kvm_vcpu_unmap(vcpu, &map); return 0; } diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 1b7e613fde7b..10401330a6b1 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -2221,7 +2221,6 @@ static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload) u64 vmcb12_gpa = kvm_rax_read(vcpu); struct vcpu_svm *svm = to_svm(vcpu); struct vmcb *vmcb12; - struct kvm_host_map map; int ret; if (nested_svm_check_permissions(vcpu)) @@ -2232,10 +2231,11 @@ static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload) return 1; } - if (kvm_vcpu_map(vcpu, gpa_to_gfn(vmcb12_gpa), &map)) + CLASS(kvm_vcpu_map_local, m)(vcpu, gpa_to_gfn(vmcb12_gpa)); + if (m.ret) return kvm_handle_memory_failure(vcpu, X86EMUL_IO_NEEDED, NULL); - vmcb12 = map.hva; + vmcb12 = m.map.hva; ret = kvm_skip_emulated_instruction(vcpu); @@ -2248,8 +2248,6 @@ static int vmload_vmsave_interception(struct kvm_vcpu *vcpu, bool vmload) svm_copy_vmloadsave_state(vmcb12, svm->vmcb01.ptr); } - kvm_vcpu_unmap(vcpu, &map); - return ret; } diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index 0bdfa3699352..ef851de0392b 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -1420,6 +1420,25 @@ static inline void kvm_vcpu_map_mark_dirty(struct kvm_vcpu *vcpu, kvm_vcpu_mark_page_dirty(vcpu, map->gfn); } +typedef struct { + struct kvm_vcpu *vcpu; + struct kvm_host_map map; + int ret; +} kvm_vcpu_local_map_t; + +#define DEFINE_VCPU_MAP_CLASS(ro) \ +DEFINE_CLASS(kvm_vcpu_map_local##ro, kvm_vcpu_local_map_t, \ + if (!_T.ret) kvm_vcpu_unmap(_T.vcpu, &_T.map), \ + ({ \ + kvm_vcpu_local_map_t m = { .vcpu = vcpu }; \ + \ + m.ret = kvm_vcpu_map##ro(vcpu, gfn, &m.map); \ + \ + m; \ + }), struct kvm_vcpu *vcpu, gfn_t gfn); +DEFINE_VCPU_MAP_CLASS(); +DEFINE_VCPU_MAP_CLASS(_readonly); + unsigned long kvm_vcpu_gfn_to_hva(struct kvm_vcpu *vcpu, gfn_t gfn); unsigned long kvm_vcpu_gfn_to_hva_prot(struct kvm_vcpu *vcpu, gfn_t gfn, bool *writable); int kvm_vcpu_read_guest_page(struct kvm_vcpu *vcpu, gfn_t gfn, void *data, int offset, From b717245a7b0855a8a729aa413a9d0de6baeaf35f Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 17:47:54 -0700 Subject: [PATCH 118/121] KVM: nSVM: Use CLASS(kvm_vcpu_map_local) for SMM VMCB mappings Convert the kvm_vcpu_map() usage in the enter/leave SMM flows to the new CLASS(kvm_vcpu_map_local) implementations, to eliminate the need to manually do unmap() in error paths, and more importantly to eliminate more of the open-coded on-stack "struct kvm_host_map" declarations. No functional change intended. Link: https://patch.msgid.link/20260724004757.131420-4-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/svm/svm.c | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 10401330a6b1..1c11d3c541e2 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -4992,7 +4992,6 @@ static int svm_smi_allowed(struct kvm_vcpu *vcpu, bool for_injection) static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) { struct vcpu_svm *svm = to_svm(vcpu); - struct kvm_host_map map_save; if (!is_guest_mode(vcpu)) return 0; @@ -5026,24 +5025,20 @@ static int svm_enter_smm(struct kvm_vcpu *vcpu, union kvm_smram *smram) * that, see svm_prepare_switch_to_guest()) which must be * preserved. */ - if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save)) + CLASS(kvm_vcpu_map_local, m_save)(vcpu, gpa_to_gfn(svm->nested.hsave_msr)); + if (m_save.ret) return 1; BUILD_BUG_ON(offsetof(struct vmcb, save) != 0x400); - svm_copy_vmrun_state(map_save.hva + 0x400, - &svm->vmcb01.ptr->save); - - kvm_vcpu_unmap(vcpu, &map_save); + svm_copy_vmrun_state(m_save.map.hva + 0x400, &svm->vmcb01.ptr->save); return 0; } static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) { struct vcpu_svm *svm = to_svm(vcpu); - struct kvm_host_map map, map_save; struct vmcb *vmcb12; - int ret; const struct kvm_smram_state_64 *smram64 = &smram->smram64; @@ -5060,22 +5055,23 @@ static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) if (!(smram64->efer & EFER_SVME)) return 1; - if (kvm_vcpu_map(vcpu, gpa_to_gfn(smram64->svm_guest_vmcb_gpa), &map)) + CLASS(kvm_vcpu_map_local, m)(vcpu, gpa_to_gfn(smram64->svm_guest_vmcb_gpa)); + if (m.ret) return 1; - ret = 1; - if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save)) - goto unmap_map; + CLASS(kvm_vcpu_map_local, m_save)(vcpu, gpa_to_gfn(svm->nested.hsave_msr)); + if (m_save.ret) + return 1; if (svm_allocate_nested(svm)) - goto unmap_save; + return 1; /* * Restore L1 host state from L1 HSAVE area as VMCB01 was * used during SMM (see svm_enter_smm()) */ - svm_copy_vmrun_state(&svm->vmcb01.ptr->save, map_save.hva + 0x400); + svm_copy_vmrun_state(&svm->vmcb01.ptr->save, m_save.map.hva + 0x400); /* * Enter the nested guest now @@ -5083,24 +5079,18 @@ static int svm_leave_smm(struct kvm_vcpu *vcpu, const union kvm_smram *smram) vmcb_mark_all_dirty(svm->vmcb01.ptr); - vmcb12 = map.hva; + vmcb12 = m.map.hva; nested_copy_vmcb_control_to_cache(svm, &vmcb12->control); nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); if (nested_svm_check_cached_vmcb12(vcpu) < 0) - goto unmap_save; + return 1; if (enter_svm_guest_mode(vcpu, smram64->svm_guest_vmcb_gpa, false) != 0) - goto unmap_save; + return 1; - ret = 0; vcpu->arch.nested_run_pending = KVM_NESTED_RUN_PENDING; - -unmap_save: - kvm_vcpu_unmap(vcpu, &map_save); -unmap_map: - kvm_vcpu_unmap(vcpu, &map); - return ret; + return 0; } static void svm_enable_smi_window(struct kvm_vcpu *vcpu) From 6bb3493175d8002126c9d4fc84624e42db17d6d1 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 17:47:55 -0700 Subject: [PATCH 119/121] KVM: nVMX: Use CLASS(kvm_vcpu_map_local_readonly) for MSR bitmap merging Convert the kvm_vcpu_map_readonly() usage in nVMX's MSR bitmap merging to the new CLASS(kvm_vcpu_map_local_readonly) implementation, to eliminate the last of the open-coded on-stack "struct kvm_host_map" declarations (in x86, PPC still has one more to convert). No functional change intended. Link: https://patch.msgid.link/20260724004757.131420-5-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/x86/kvm/vmx/nested.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 57e02c3fa182..579203923267 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -701,7 +701,6 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, int msr; unsigned long *msr_bitmap_l1; unsigned long *msr_bitmap_l0 = vmx->nested.vmcs02.msr_bitmap; - struct kvm_host_map map; /* Nothing to do if the MSR bitmap is not in use. */ if (!cpu_has_vmx_msr_bitmap() || @@ -724,10 +723,11 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, return true; } - if (kvm_vcpu_map_readonly(vcpu, gpa_to_gfn(vmcs12->msr_bitmap), &map)) + CLASS(kvm_vcpu_map_local_readonly, m)(vcpu, gpa_to_gfn(vmcs12->msr_bitmap)); + if (m.ret) return false; - msr_bitmap_l1 = (unsigned long *)map.hva; + msr_bitmap_l1 = (unsigned long *)m.map.hva; /* * To keep the control flow simple, pay eight 8-byte writes (sixteen @@ -807,8 +807,6 @@ static inline bool nested_vmx_prepare_msr_bitmap(struct kvm_vcpu *vcpu, nested_vmx_merge_pmu_msr_bitmaps(vcpu, msr_bitmap_l1, msr_bitmap_l0); - kvm_vcpu_unmap(vcpu, &map); - vmx->nested.force_msr_bitmap_recalc = false; return true; From e060b94a1cd4920165aec7a77431df2927b26fdb Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 17:47:56 -0700 Subject: [PATCH 120/121] KVM: PPC: Use CLASS(kvm_vcpu_map_local) to patch dcbz Convert the kvm_vcpu_map() usage in PPC dcbz patching to the new CLASS(kvm_vcpu_map_local) implementation, to eliminate the very last of the the open-coded on-stack "struct kvm_host_map" declarations. This will allow adding hardening kvm_vcpu_map() against memory leaks (due to clobbering the existing mapping). No functional change intended. Link: https://patch.msgid.link/20260724004757.131420-6-seanjc@google.com Signed-off-by: Sean Christopherson --- arch/powerpc/kvm/book3s_pr.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/arch/powerpc/kvm/book3s_pr.c b/arch/powerpc/kvm/book3s_pr.c index 2ba2dd26a7ea..ecfb88bf6464 100644 --- a/arch/powerpc/kvm/book3s_pr.c +++ b/arch/powerpc/kvm/book3s_pr.c @@ -639,27 +639,24 @@ static void kvmppc_set_pvr_pr(struct kvm_vcpu *vcpu, u32 pvr) */ static void kvmppc_patch_dcbz(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte) { - struct kvm_host_map map; u64 hpage_offset; u32 *page; - int i, r; + int i; - r = kvm_vcpu_map(vcpu, pte->raddr >> PAGE_SHIFT, &map); - if (r) + CLASS(kvm_vcpu_map_local, m)(vcpu, pte->raddr >> PAGE_SHIFT); + if (m.ret) return; hpage_offset = pte->raddr & ~PAGE_MASK; hpage_offset &= ~0xFFFULL; hpage_offset /= 4; - page = map.hva; + page = m.map.hva; /* patch dcbz into reserved instruction, so we trap */ for (i=hpage_offset; i < hpage_offset + (HW_PAGE_SIZE / 4); i++) if ((be32_to_cpu(page[i]) & 0xff0007ff) == INS_DCBZ) page[i] &= cpu_to_be32(0xfffffff7); - - kvm_vcpu_unmap(vcpu, &map); } static bool kvmppc_visible_gpa(struct kvm_vcpu *vcpu, gpa_t gpa) From e186d4e34ad515f8ec45831f4386c677b3967092 Mon Sep 17 00:00:00 2001 From: Sean Christopherson Date: Thu, 23 Jul 2026 17:47:57 -0700 Subject: [PATCH 121/121] KVM: Harden kvm_vcpu_map() against double-mapping and thus leaking references Now that all on-stack maps use CLASS(kvm_vcpu_map_local), i.e. now that all maps are zero-allocated, explicitly put any existing mappings/references when establishing a new mapping to harden against KVM bugs leaking memory, but yell loudly as the owner of the map is still ultimately responsible for the lifecycle of the mapping. Suggested-by: Yosry Ahmed Link: https://patch.msgid.link/20260724004757.131420-7-seanjc@google.com Signed-off-by: Sean Christopherson --- virt/kvm/kvm_main.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 2df8ee9ecf6c..e9e32686e41b 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -3118,6 +3118,9 @@ int __kvm_vcpu_map(struct kvm_vcpu *vcpu, gfn_t gfn, struct kvm_host_map *map, .pin = true, }; + if (WARN_ON_ONCE(map->hva)) + kvm_vcpu_unmap(vcpu, map); + map->pinned_page = NULL; map->page = NULL; map->hva = NULL;