mirror of
https://github.com/linux-msm/laptops-kernel.git
synced 2026-08-13 14:19:53 -07:00
Merge branch 'next' of git://git.kernel.org/pub/scm/virt/kvm/kvm.git
This commit is contained in:
Mark Brown
+80
-575
File diff suppressed because it is too large
Load Diff
@@ -5,8 +5,8 @@ ccflags-$(CONFIG_KVM_WERROR) += -Werror
|
||||
|
||||
include $(srctree)/virt/kvm/Makefile.kvm
|
||||
|
||||
kvm-y += x86.o emulate.o irq.o lapic.o cpuid.o pmu.o mtrr.o \
|
||||
debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o
|
||||
kvm-y += x86.o emulate.o irq.o lapic.o cpuid.o msrs.o pmu.o regs.o \
|
||||
mtrr.o debugfs.o mmu/mmu.o mmu/page_track.o mmu/spte.o
|
||||
|
||||
kvm-$(CONFIG_X86_64) += mmu/tdp_iter.o mmu/tdp_mmu.o
|
||||
kvm-$(CONFIG_KVM_IOAPIC) += i8259.o i8254.o ioapic.o
|
||||
|
||||
@@ -28,6 +28,7 @@
|
||||
#include "trace.h"
|
||||
#include "pmu.h"
|
||||
#include "xen.h"
|
||||
#include "x86.h"
|
||||
|
||||
/*
|
||||
* Unlike "struct cpuinfo_x86.x86_capability", kvm_cpu_caps doesn't need to be
|
||||
|
||||
@@ -3,8 +3,34 @@
|
||||
#ifndef __KVM_FPU_H_
|
||||
#define __KVM_FPU_H_
|
||||
|
||||
#include <linux/kvm_host.h>
|
||||
|
||||
#include <trace/events/kvm.h>
|
||||
|
||||
#include <asm/fpu/api.h>
|
||||
|
||||
/* Swap (qemu) user FPU context for the guest FPU context. */
|
||||
static inline void kvm_load_guest_fpu(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
if (KVM_BUG_ON(vcpu->arch.guest_fpu.fpstate->in_use, vcpu->kvm))
|
||||
return;
|
||||
|
||||
/* Exclude PKRU, it's restored separately immediately after VM-Exit. */
|
||||
fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, true);
|
||||
trace_kvm_fpu(1);
|
||||
}
|
||||
|
||||
/* When vcpu_run ends, restore user space FPU context. */
|
||||
static inline void kvm_put_guest_fpu(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
if (KVM_BUG_ON(!vcpu->arch.guest_fpu.fpstate->in_use, vcpu->kvm))
|
||||
return;
|
||||
|
||||
fpu_swap_kvm_fpstate(&vcpu->arch.guest_fpu, false);
|
||||
++vcpu->stat.fpu_reload;
|
||||
trace_kvm_fpu(0);
|
||||
}
|
||||
|
||||
typedef u32 __attribute__((vector_size(16))) sse128_t;
|
||||
#define __sse128_u union { sse128_t vec; u64 as_u64[2]; u32 as_u32[4]; }
|
||||
#define sse128_lo(x) ({ __sse128_u t; t.vec = x; t.as_u64[0]; })
|
||||
|
||||
@@ -2045,10 +2045,9 @@ static u64 kvm_hv_flush_tlb(struct kvm_vcpu *vcpu, struct kvm_hv_hcall *hc)
|
||||
* flush). Translate the address here so the memory can be uniformly
|
||||
* read with kvm_read_guest().
|
||||
*/
|
||||
if (!hc->fast && mmu_is_nested(vcpu)) {
|
||||
hc->ingpa = kvm_x86_ops.nested_ops->translate_nested_gpa(
|
||||
vcpu, hc->ingpa,
|
||||
PFERR_GUEST_FINAL_MASK, NULL, 0);
|
||||
if (!hc->fast) {
|
||||
hc->ingpa = kvm_translate_gpa(vcpu, &vcpu->arch.gva_walk, hc->ingpa,
|
||||
PFERR_GUEST_FINAL_MASK, NULL, 0);
|
||||
if (unlikely(hc->ingpa == INVALID_GPA))
|
||||
return HV_STATUS_INVALID_HYPERCALL_INPUT;
|
||||
}
|
||||
|
||||
@@ -22,7 +22,8 @@
|
||||
#define __ARCH_X86_KVM_HYPERV_H__
|
||||
|
||||
#include <linux/kvm_host.h>
|
||||
#include "x86.h"
|
||||
|
||||
#include "regs.h"
|
||||
|
||||
#ifdef CONFIG_KVM_HYPERV
|
||||
|
||||
|
||||
@@ -33,6 +33,7 @@
|
||||
#include "lapic.h"
|
||||
#include "irq.h"
|
||||
#include "trace.h"
|
||||
#include "x86.h"
|
||||
|
||||
static int ioapic_service(struct kvm_ioapic *vioapic, int irq,
|
||||
bool line_status);
|
||||
|
||||
@@ -113,6 +113,18 @@ void kvm_get_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state);
|
||||
void kvm_set_ioapic(struct kvm *kvm, struct kvm_ioapic_state *state);
|
||||
void kvm_ioapic_scan_entry(struct kvm_vcpu *vcpu,
|
||||
ulong *ioapic_handled_vectors);
|
||||
|
||||
static inline int __kvm_irq_line_state(unsigned long *irq_state,
|
||||
int irq_source_id, int level)
|
||||
{
|
||||
/* Logical OR for level trig interrupt */
|
||||
if (level)
|
||||
__set_bit(irq_source_id, irq_state);
|
||||
else
|
||||
__clear_bit(irq_source_id, irq_state);
|
||||
|
||||
return !!(*irq_state);
|
||||
}
|
||||
#endif /* CONFIG_KVM_IOAPIC */
|
||||
|
||||
static inline int ioapic_in_kernel(struct kvm *kvm)
|
||||
|
||||
@@ -423,6 +423,13 @@ void kvm_arch_irq_routing_update(struct kvm *kvm)
|
||||
kvm_make_scan_ioapic_request(kvm);
|
||||
}
|
||||
|
||||
static bool kvm_irq_is_postable(struct kvm_lapic_irq *irq)
|
||||
{
|
||||
/* We can only post Fixed and LowPrio IRQs */
|
||||
return (irq->delivery_mode == APIC_DM_FIXED ||
|
||||
irq->delivery_mode == APIC_DM_LOWEST);
|
||||
}
|
||||
|
||||
static int kvm_pi_update_irte(struct kvm_kernel_irqfd *irqfd,
|
||||
struct kvm_kernel_irq_routing_entry *entry)
|
||||
{
|
||||
|
||||
@@ -112,6 +112,12 @@ static inline int irqchip_in_kernel(struct kvm *kvm)
|
||||
return mode != KVM_IRQCHIP_NONE;
|
||||
}
|
||||
|
||||
int kvm_cpu_has_injectable_intr(struct kvm_vcpu *v);
|
||||
int kvm_cpu_has_interrupt(struct kvm_vcpu *vcpu);
|
||||
int kvm_cpu_has_extint(struct kvm_vcpu *v);
|
||||
int kvm_cpu_get_extint(struct kvm_vcpu *v);
|
||||
int kvm_cpu_get_interrupt(struct kvm_vcpu *v);
|
||||
|
||||
void kvm_inject_pending_timer_irqs(struct kvm_vcpu *vcpu);
|
||||
void kvm_inject_apic_timer_irqs(struct kvm_vcpu *vcpu);
|
||||
void kvm_apic_nmi_wd_deliver(struct kvm_vcpu *vcpu);
|
||||
|
||||
@@ -131,6 +131,9 @@ static inline int kvm_irq_delivery_to_apic(struct kvm *kvm,
|
||||
}
|
||||
|
||||
void kvm_apic_send_ipi(struct kvm_lapic *apic, u32 icr_low, u32 icr_high);
|
||||
int kvm_pv_send_ipi(struct kvm *kvm, unsigned long ipi_bitmap_low,
|
||||
unsigned long ipi_bitmap_high, u32 min,
|
||||
unsigned long icr, int op_64_bit);
|
||||
|
||||
int kvm_apic_set_base(struct kvm_vcpu *vcpu, u64 value, bool host_initiated);
|
||||
int kvm_apic_get_state(struct kvm_vcpu *vcpu, struct kvm_lapic_state *s);
|
||||
@@ -237,6 +240,11 @@ static inline int kvm_lapic_latched_init(struct kvm_vcpu *vcpu)
|
||||
return lapic_in_kernel(vcpu) && test_bit(KVM_APIC_INIT, &vcpu->arch.apic->pending_events);
|
||||
}
|
||||
|
||||
static inline u16 kvm_lapic_irq_dest_mode(bool dest_mode_logical)
|
||||
{
|
||||
return dest_mode_logical ? APIC_DEST_LOGICAL : APIC_DEST_PHYSICAL;
|
||||
}
|
||||
|
||||
bool kvm_apic_pending_eoi(struct kvm_vcpu *vcpu, int vector);
|
||||
|
||||
bool kvm_lapic_suppress_eoi_broadcast(struct kvm_lapic *apic);
|
||||
|
||||
+99
-22
@@ -4,10 +4,23 @@
|
||||
|
||||
#include <linux/kvm_host.h>
|
||||
#include "regs.h"
|
||||
#include "x86.h"
|
||||
#include "cpuid.h"
|
||||
|
||||
extern bool tdp_enabled;
|
||||
#ifdef CONFIG_X86_64
|
||||
extern bool tdp_mmu_enabled;
|
||||
#else
|
||||
#define tdp_mmu_enabled false
|
||||
#endif
|
||||
extern bool __read_mostly enable_mmio_caching;
|
||||
extern bool eager_page_split;
|
||||
|
||||
#define KVM_MEMSLOT_PAGES_TO_MMU_PAGES_RATIO 50
|
||||
#define KVM_MIN_ALLOC_MMU_PAGES 64UL
|
||||
#define KVM_MMU_HASH_SHIFT 12
|
||||
#define KVM_NUM_MMU_PAGES (1 << KVM_MMU_HASH_SHIFT)
|
||||
#define KVM_MIN_FREE_MMU_PAGES 5
|
||||
#define KVM_REFILL_PAGES 25
|
||||
|
||||
#define PT_WRITABLE_SHIFT 1
|
||||
#define PT_USER_SHIFT 2
|
||||
@@ -90,6 +103,38 @@ static inline bool mmu_has_mbec(struct kvm_mmu *mmu)
|
||||
|
||||
u8 kvm_mmu_get_max_tdp_level(void);
|
||||
|
||||
void __init kvm_mmu_x86_module_init(void);
|
||||
int kvm_mmu_vendor_module_init(void);
|
||||
void kvm_mmu_vendor_module_exit(void);
|
||||
|
||||
void kvm_mmu_destroy(struct kvm_vcpu *vcpu);
|
||||
int kvm_mmu_create(struct kvm_vcpu *vcpu);
|
||||
int kvm_mmu_init_vm(struct kvm *kvm);
|
||||
void kvm_mmu_uninit_vm(struct kvm *kvm);
|
||||
|
||||
void kvm_mmu_init_memslot_memory_attributes(struct kvm *kvm,
|
||||
struct kvm_memory_slot *slot);
|
||||
|
||||
void kvm_mmu_after_set_cpuid(struct kvm_vcpu *vcpu);
|
||||
void kvm_mmu_reset_context(struct kvm_vcpu *vcpu);
|
||||
void kvm_mmu_slot_remove_write_access(struct kvm *kvm,
|
||||
const struct kvm_memory_slot *memslot,
|
||||
int start_level);
|
||||
void kvm_mmu_slot_try_split_huge_pages(struct kvm *kvm,
|
||||
const struct kvm_memory_slot *memslot,
|
||||
int target_level);
|
||||
void kvm_mmu_try_split_huge_pages(struct kvm *kvm,
|
||||
const struct kvm_memory_slot *memslot,
|
||||
u64 start, u64 end,
|
||||
int target_level);
|
||||
void kvm_mmu_recover_huge_pages(struct kvm *kvm,
|
||||
const struct kvm_memory_slot *memslot);
|
||||
void kvm_mmu_slot_leaf_clear_dirty(struct kvm *kvm,
|
||||
const struct kvm_memory_slot *memslot);
|
||||
void kvm_mmu_invalidate_mmio_sptes(struct kvm *kvm, u64 gen);
|
||||
void kvm_mmu_change_mmu_pages(struct kvm *kvm, unsigned long kvm_nr_mmu_pages);
|
||||
void kvm_zap_gfn_range(struct kvm *kvm, gfn_t gfn_start, gfn_t gfn_end);
|
||||
|
||||
void kvm_mmu_set_mmio_spte_mask(u64 mmio_value, u64 mmio_mask, u64 access_mask);
|
||||
void kvm_mmu_set_mmio_spte_value(struct kvm *kvm, u64 mmio_value);
|
||||
void kvm_mmu_set_me_spte_mask(u64 me_value, u64 me_mask);
|
||||
@@ -101,11 +146,24 @@ void kvm_init_shadow_npt_mmu(struct kvm_vcpu *vcpu, unsigned long cr4,
|
||||
void kvm_init_shadow_ept_mmu(struct kvm_vcpu *vcpu, bool execonly,
|
||||
int huge_page_level, bool accessed_dirty,
|
||||
bool mbec, gpa_t new_eptp);
|
||||
|
||||
int kvm_mmu_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, u64 error_code,
|
||||
void *insn, int insn_len);
|
||||
void kvm_mmu_print_sptes(struct kvm_vcpu *vcpu, gpa_t gpa, const char *msg);
|
||||
void kvm_mmu_invlpg(struct kvm_vcpu *vcpu, gva_t gva);
|
||||
void kvm_mmu_invalidate_addr(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
|
||||
u64 addr, unsigned long roots);
|
||||
void kvm_mmu_invpcid_gva(struct kvm_vcpu *vcpu, gva_t gva, unsigned long pcid);
|
||||
void kvm_mmu_new_pgd(struct kvm_vcpu *vcpu, gpa_t new_pgd);
|
||||
|
||||
void kvm_configure_mmu(bool enable_tdp, int tdp_forced_root_level,
|
||||
int tdp_max_root_level, int tdp_huge_page_level);
|
||||
|
||||
bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu);
|
||||
int kvm_handle_page_fault(struct kvm_vcpu *vcpu, u64 error_code,
|
||||
u64 fault_address, char *insn, int insn_len);
|
||||
void __kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
|
||||
struct kvm_mmu *mmu);
|
||||
struct kvm_pagewalk *pw);
|
||||
|
||||
int kvm_mmu_load(struct kvm_vcpu *vcpu);
|
||||
void kvm_mmu_unload(struct kvm_vcpu *vcpu);
|
||||
@@ -115,6 +173,25 @@ void kvm_mmu_sync_prev_roots(struct kvm_vcpu *vcpu);
|
||||
void kvm_mmu_track_write(struct kvm_vcpu *vcpu, gpa_t gpa, const u8 *new,
|
||||
int bytes);
|
||||
|
||||
bool __kvm_mmu_unprotect_gfn_and_retry(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
|
||||
bool always_retry);
|
||||
|
||||
static inline bool kvm_mmu_unprotect_gfn_and_retry(struct kvm_vcpu *vcpu,
|
||||
gpa_t cr2_or_gpa)
|
||||
{
|
||||
return __kvm_mmu_unprotect_gfn_and_retry(vcpu, cr2_or_gpa, false);
|
||||
}
|
||||
|
||||
void kvm_mmu_free_roots(struct kvm *kvm, struct kvm_mmu *mmu,
|
||||
ulong roots_to_free);
|
||||
void kvm_mmu_free_guest_mode_roots(struct kvm *kvm, struct kvm_mmu *mmu);
|
||||
gpa_t kvm_mmu_gva_to_gpa_read(struct kvm_vcpu *vcpu, gva_t gva,
|
||||
struct x86_exception *exception);
|
||||
gpa_t kvm_mmu_gva_to_gpa_write(struct kvm_vcpu *vcpu, gva_t gva,
|
||||
struct x86_exception *exception);
|
||||
gpa_t kvm_mmu_gva_to_gpa_system(struct kvm_vcpu *vcpu, gva_t gva,
|
||||
struct x86_exception *exception);
|
||||
|
||||
static inline int kvm_mmu_reload(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
if (kvm_check_request(KVM_REQ_MMU_FREE_OBSOLETE_ROOTS, vcpu))
|
||||
@@ -169,21 +246,21 @@ static inline void kvm_mmu_load_pgd(struct kvm_vcpu *vcpu)
|
||||
}
|
||||
|
||||
static inline void kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
|
||||
struct kvm_mmu *mmu)
|
||||
struct kvm_pagewalk *w)
|
||||
{
|
||||
/*
|
||||
* When EPT is enabled, KVM may passthrough CR0.WP to the guest, i.e.
|
||||
* @mmu's snapshot of CR0.WP and thus all related paging metadata may
|
||||
* @w's snapshot of CR0.WP and thus all related paging metadata may
|
||||
* be stale. Refresh CR0.WP and the metadata on-demand when checking
|
||||
* for permission faults. Exempt nested MMUs, i.e. MMUs for shadowing
|
||||
* nEPT and nNPT, as CR0.WP is ignored in both cases. Note, KVM does
|
||||
* need to refresh nested_mmu, a.k.a. the walker used to translate L2
|
||||
* GVAs to GPAs, as that "MMU" needs to honor L2's CR0.WP.
|
||||
* nEPT and nNPT, as CR0.WP is ignored in both cases. Note, KVM will
|
||||
* still refresh gva_walk, so as to honor L2's CR0.WP when translating
|
||||
* L2 GVAs to GPAs.
|
||||
*/
|
||||
if (!tdp_enabled || mmu == &vcpu->arch.guest_mmu)
|
||||
if (!tdp_enabled || w == &vcpu->arch.ngpa_walk)
|
||||
return;
|
||||
|
||||
__kvm_mmu_refresh_passthrough_bits(vcpu, mmu);
|
||||
__kvm_mmu_refresh_passthrough_bits(vcpu, w);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -194,7 +271,7 @@ static inline void kvm_mmu_refresh_passthrough_bits(struct kvm_vcpu *vcpu,
|
||||
* Return zero if the access does not fault; return the page fault error code
|
||||
* if the access faults.
|
||||
*/
|
||||
static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
|
||||
static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
|
||||
unsigned pte_access, unsigned pte_pkey,
|
||||
u64 access)
|
||||
{
|
||||
@@ -217,15 +294,16 @@ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
|
||||
u64 implicit_access = access & PFERR_IMPLICIT_ACCESS;
|
||||
bool not_smap = ((rflags & X86_EFLAGS_AC) | implicit_access) == X86_EFLAGS_AC;
|
||||
int index = (pfec | (not_smap ? PFERR_RSVD_MASK : 0)) >> 1;
|
||||
struct kvm_page_format *fmt = &w->fmt;
|
||||
u32 errcode = PFERR_PRESENT_MASK;
|
||||
bool fault;
|
||||
|
||||
kvm_mmu_refresh_passthrough_bits(vcpu, mmu);
|
||||
kvm_mmu_refresh_passthrough_bits(vcpu, w);
|
||||
|
||||
fault = (mmu->permissions[index] >> pte_access) & 1;
|
||||
fault = (fmt->permissions[index] >> pte_access) & 1;
|
||||
|
||||
WARN_ON_ONCE(pfec & (PFERR_PK_MASK | PFERR_SS_MASK | PFERR_RSVD_MASK));
|
||||
if (unlikely(mmu->pkru_mask)) {
|
||||
if (unlikely(fmt->pkru_mask)) {
|
||||
u32 pkru_bits, offset;
|
||||
|
||||
/*
|
||||
@@ -239,7 +317,7 @@ static inline u8 permission_fault(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
|
||||
/* clear present bit, replace PFEC.RSVD with ACC_USER_MASK. */
|
||||
offset = (pfec & ~1) | ((pte_access & PT_USER_MASK) ? PFERR_RSVD_MASK : 0);
|
||||
|
||||
pkru_bits &= mmu->pkru_mask >> offset;
|
||||
pkru_bits &= fmt->pkru_mask >> offset;
|
||||
errcode |= -pkru_bits & PFERR_PK_MASK;
|
||||
fault |= (pkru_bits != 0);
|
||||
}
|
||||
@@ -261,12 +339,6 @@ static inline bool kvm_shadow_root_allocated(struct kvm *kvm)
|
||||
return smp_load_acquire(&kvm->arch.shadow_root_allocated);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
extern bool tdp_mmu_enabled;
|
||||
#else
|
||||
#define tdp_mmu_enabled false
|
||||
#endif
|
||||
|
||||
int kvm_tdp_mmu_map_private_pfn(struct kvm_vcpu *vcpu, gfn_t gfn, kvm_pfn_t pfn);
|
||||
|
||||
static inline bool kvm_memslots_have_rmaps(struct kvm *kvm)
|
||||
@@ -300,13 +372,18 @@ static inline void kvm_update_page_stats(struct kvm *kvm, int level, int count)
|
||||
atomic64_add(count, &kvm->stat.pages[level - 1]);
|
||||
}
|
||||
|
||||
static inline bool mmu_is_nested(struct kvm_vcpu *vcpu)
|
||||
{
|
||||
return vcpu->arch.mmu == &vcpu->arch.guest_mmu;
|
||||
}
|
||||
|
||||
static inline gpa_t kvm_translate_gpa(struct kvm_vcpu *vcpu,
|
||||
struct kvm_mmu *mmu,
|
||||
struct kvm_pagewalk *w,
|
||||
gpa_t gpa, u64 access,
|
||||
struct x86_exception *exception,
|
||||
u64 pte_access)
|
||||
{
|
||||
if (mmu != &vcpu->arch.nested_mmu)
|
||||
if (!mmu_is_nested(vcpu) || w == &vcpu->arch.ngpa_walk)
|
||||
return gpa;
|
||||
return kvm_x86_ops.nested_ops->translate_nested_gpa(vcpu, gpa, access,
|
||||
exception,
|
||||
|
||||
+296
-245
File diff suppressed because it is too large
Load Diff
@@ -290,8 +290,6 @@ struct kvm_page_fault {
|
||||
bool write_fault_to_shadow_pgtable;
|
||||
};
|
||||
|
||||
int kvm_tdp_page_fault(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
|
||||
|
||||
/*
|
||||
* Return values of handle_mmio_page_fault(), mmu.page_fault(), fast_page_fault(),
|
||||
* and of course kvm_mmu_do_page_fault().
|
||||
@@ -337,70 +335,6 @@ static inline void kvm_mmu_prepare_memory_fault_exit(struct kvm_vcpu *vcpu,
|
||||
fault->is_private);
|
||||
}
|
||||
|
||||
static inline int kvm_mmu_do_page_fault(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa,
|
||||
u64 err, bool prefetch,
|
||||
int *emulation_type, u8 *level)
|
||||
{
|
||||
struct kvm_page_fault fault = {
|
||||
.addr = cr2_or_gpa,
|
||||
.error_code = err,
|
||||
.exec = err & PFERR_FETCH_MASK,
|
||||
.write = err & PFERR_WRITE_MASK,
|
||||
.present = err & PFERR_PRESENT_MASK,
|
||||
.rsvd = err & PFERR_RSVD_MASK,
|
||||
.user = err & PFERR_USER_MASK,
|
||||
.prefetch = prefetch,
|
||||
.is_tdp = likely(vcpu->arch.mmu->page_fault == kvm_tdp_page_fault),
|
||||
.nx_huge_page_workaround_enabled =
|
||||
is_nx_huge_page_enabled(vcpu->kvm),
|
||||
|
||||
.max_level = KVM_MAX_HUGEPAGE_LEVEL,
|
||||
.req_level = PG_LEVEL_4K,
|
||||
.goal_level = PG_LEVEL_4K,
|
||||
.is_private = err & PFERR_PRIVATE_ACCESS,
|
||||
|
||||
.pfn = KVM_PFN_ERR_FAULT,
|
||||
};
|
||||
int r;
|
||||
|
||||
if (vcpu->arch.mmu->root_role.direct) {
|
||||
/*
|
||||
* Things like memslots don't understand the concept of a shared
|
||||
* bit. Strip it so that the GFN can be used like normal, and the
|
||||
* fault.addr can be used when the shared bit is needed.
|
||||
*/
|
||||
fault.gfn = gpa_to_gfn(fault.addr) & ~kvm_gfn_direct_bits(vcpu->kvm);
|
||||
fault.slot = kvm_vcpu_gfn_to_memslot(vcpu, fault.gfn);
|
||||
}
|
||||
|
||||
/*
|
||||
* With retpoline being active an indirect call is rather expensive,
|
||||
* so do a direct call in the most common case.
|
||||
*/
|
||||
if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) && fault.is_tdp)
|
||||
r = kvm_tdp_page_fault(vcpu, &fault);
|
||||
else
|
||||
r = vcpu->arch.mmu->page_fault(vcpu, &fault);
|
||||
|
||||
/*
|
||||
* Not sure what's happening, but punt to userspace and hope that
|
||||
* they can fix it by changing memory to shared, or they can
|
||||
* provide a better error.
|
||||
*/
|
||||
if (r == RET_PF_EMULATE && fault.is_private) {
|
||||
pr_warn_ratelimited("kvm: unexpected emulation request on private memory\n");
|
||||
kvm_mmu_prepare_memory_fault_exit(vcpu, &fault);
|
||||
return -EFAULT;
|
||||
}
|
||||
|
||||
if (fault.write_fault_to_shadow_pgtable && emulation_type)
|
||||
*emulation_type |= EMULTYPE_WRITE_PF_TO_SP;
|
||||
if (level)
|
||||
*level = fault.goal_level;
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
int kvm_mmu_max_mapping_level(struct kvm *kvm, struct kvm_page_fault *fault,
|
||||
const struct kvm_memory_slot *slot, gfn_t gfn);
|
||||
void kvm_mmu_hugepage_adjust(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault);
|
||||
|
||||
@@ -55,7 +55,7 @@
|
||||
#define PT_LEVEL_BITS 9
|
||||
#define PT_GUEST_DIRTY_SHIFT 9
|
||||
#define PT_GUEST_ACCESSED_SHIFT 8
|
||||
#define PT_HAVE_ACCESSED_DIRTY(mmu) (!(mmu)->cpu_role.base.ad_disabled)
|
||||
#define PT_HAVE_ACCESSED_DIRTY(w) (!(w)->cpu_role.base.ad_disabled)
|
||||
#define PT_MAX_FULL_LEVELS PT64_ROOT_MAX_LEVEL
|
||||
#else
|
||||
#error Invalid PTTYPE value
|
||||
@@ -106,13 +106,13 @@ static gfn_t gpte_to_gfn_lvl(pt_element_t gpte, int lvl)
|
||||
return (gpte & PT_LVL_ADDR_MASK(lvl)) >> PAGE_SHIFT;
|
||||
}
|
||||
|
||||
static inline void FNAME(protect_clean_gpte)(struct kvm_mmu *mmu, unsigned *access,
|
||||
static inline void FNAME(protect_clean_gpte)(struct kvm_pagewalk *w, unsigned *access,
|
||||
unsigned gpte)
|
||||
{
|
||||
unsigned mask;
|
||||
|
||||
/* dirty bit is not supported, so no need to track it */
|
||||
if (!PT_HAVE_ACCESSED_DIRTY(mmu))
|
||||
if (!PT_HAVE_ACCESSED_DIRTY(w))
|
||||
return;
|
||||
|
||||
BUILD_BUG_ON(PT_WRITABLE_MASK != ACC_WRITE_MASK);
|
||||
@@ -124,7 +124,7 @@ static inline void FNAME(protect_clean_gpte)(struct kvm_mmu *mmu, unsigned *acce
|
||||
*access &= mask;
|
||||
}
|
||||
|
||||
static inline int FNAME(is_present_gpte)(struct kvm_mmu *mmu,
|
||||
static inline int FNAME(is_present_gpte)(struct kvm_pagewalk *w,
|
||||
unsigned long pte)
|
||||
{
|
||||
#if PTTYPE != PTTYPE_EPT
|
||||
@@ -134,38 +134,40 @@ static inline int FNAME(is_present_gpte)(struct kvm_mmu *mmu,
|
||||
* For EPT, an entry is present if any of bits 2:0 are set.
|
||||
* With mode-based execute control, bit 10 also indicates presence.
|
||||
*/
|
||||
return pte & (7 | (mmu_has_mbec(mmu) ? VMX_EPT_USER_EXECUTABLE_MASK : 0));
|
||||
return pte & (7 | (is_cr4_smep(w) ? VMX_EPT_USER_EXECUTABLE_MASK : 0));
|
||||
#endif
|
||||
}
|
||||
|
||||
static bool FNAME(is_bad_mt_xwr)(struct rsvd_bits_validate *rsvd_check, u64 gpte)
|
||||
static bool FNAME(is_bad_mt_xwr)(struct kvm_page_format *fmt, u64 gpte)
|
||||
{
|
||||
#if PTTYPE != PTTYPE_EPT
|
||||
return false;
|
||||
#else
|
||||
return __is_bad_mt_xwr(rsvd_check, gpte);
|
||||
return __is_bad_mt_xwr(fmt, gpte);
|
||||
#endif
|
||||
}
|
||||
|
||||
static bool FNAME(is_rsvd_bits_set)(struct kvm_mmu *mmu, u64 gpte, int level)
|
||||
static bool FNAME(is_rsvd_bits_set)(struct kvm_page_format *fmt, u64 gpte, int level)
|
||||
{
|
||||
return __is_rsvd_bits_set(&mmu->guest_rsvd_check, gpte, level) ||
|
||||
FNAME(is_bad_mt_xwr)(&mmu->guest_rsvd_check, gpte);
|
||||
return __is_rsvd_bits_set(fmt, gpte, level) ||
|
||||
FNAME(is_bad_mt_xwr)(fmt, gpte);
|
||||
}
|
||||
|
||||
static bool FNAME(prefetch_invalid_gpte)(struct kvm_vcpu *vcpu,
|
||||
struct kvm_mmu_page *sp, u64 *spte,
|
||||
u64 gpte)
|
||||
{
|
||||
if (!FNAME(is_present_gpte)(vcpu->arch.mmu, gpte))
|
||||
struct kvm_pagewalk *w = vcpu->arch.mmu->w;
|
||||
|
||||
if (!FNAME(is_present_gpte)(w, gpte))
|
||||
goto no_present;
|
||||
|
||||
/* Prefetch only accessed entries (unless A/D bits are disabled). */
|
||||
if (PT_HAVE_ACCESSED_DIRTY(vcpu->arch.mmu) &&
|
||||
if (PT_HAVE_ACCESSED_DIRTY(w) &&
|
||||
!(gpte & PT_GUEST_ACCESSED_MASK))
|
||||
goto no_present;
|
||||
|
||||
if (FNAME(is_rsvd_bits_set)(vcpu->arch.mmu, gpte, PG_LEVEL_4K))
|
||||
if (FNAME(is_rsvd_bits_set)(&w->fmt, gpte, PG_LEVEL_4K))
|
||||
goto no_present;
|
||||
|
||||
return false;
|
||||
@@ -206,7 +208,7 @@ static inline unsigned FNAME(gpte_access)(u64 gpte)
|
||||
}
|
||||
|
||||
static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu,
|
||||
struct kvm_mmu *mmu,
|
||||
struct kvm_pagewalk *w,
|
||||
struct guest_walker *walker,
|
||||
gpa_t addr, int write_fault)
|
||||
{
|
||||
@@ -217,7 +219,7 @@ static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu,
|
||||
int ret;
|
||||
|
||||
/* dirty/accessed bits are not supported, so no need to update them */
|
||||
if (!PT_HAVE_ACCESSED_DIRTY(mmu))
|
||||
if (!PT_HAVE_ACCESSED_DIRTY(w))
|
||||
return 0;
|
||||
|
||||
for (level = walker->max_level; level >= walker->level; --level) {
|
||||
@@ -278,7 +280,7 @@ static inline unsigned FNAME(gpte_pkeys)(struct kvm_vcpu *vcpu, u64 gpte)
|
||||
return pkeys;
|
||||
}
|
||||
|
||||
static inline bool FNAME(is_last_gpte)(struct kvm_mmu *mmu,
|
||||
static inline bool FNAME(is_last_gpte)(struct kvm_pagewalk *w,
|
||||
unsigned int level, unsigned int gpte)
|
||||
{
|
||||
/*
|
||||
@@ -296,7 +298,7 @@ static inline bool FNAME(is_last_gpte)(struct kvm_mmu *mmu,
|
||||
* is not reserved and does not indicate a large page at this level,
|
||||
* so clear PT_PAGE_SIZE_MASK in gpte if that is the case.
|
||||
*/
|
||||
gpte &= level - (PT32_ROOT_LEVEL + mmu->cpu_role.ext.cr4_pse);
|
||||
gpte &= level - (PT32_ROOT_LEVEL + w->cpu_role.ext.cr4_pse);
|
||||
#endif
|
||||
/*
|
||||
* PG_LEVEL_4K always terminates. The RHS has bit 7 set
|
||||
@@ -311,7 +313,7 @@ static inline bool FNAME(is_last_gpte)(struct kvm_mmu *mmu,
|
||||
* Fetch a guest pte for a guest virtual address, or for an L2's GPA.
|
||||
*/
|
||||
static int FNAME(walk_addr_generic)(struct guest_walker *walker,
|
||||
struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
|
||||
struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
|
||||
gpa_t addr, u64 access)
|
||||
{
|
||||
int ret;
|
||||
@@ -340,16 +342,16 @@ static int FNAME(walk_addr_generic)(struct guest_walker *walker,
|
||||
|
||||
trace_kvm_mmu_pagetable_walk(addr, access);
|
||||
retry_walk:
|
||||
walker->level = mmu->cpu_role.base.level;
|
||||
pte = kvm_mmu_get_guest_pgd(vcpu, mmu);
|
||||
have_ad = PT_HAVE_ACCESSED_DIRTY(mmu);
|
||||
walker->level = w->cpu_role.base.level;
|
||||
pte = kvm_mmu_get_guest_pgd(vcpu, w);
|
||||
have_ad = PT_HAVE_ACCESSED_DIRTY(w);
|
||||
|
||||
#if PTTYPE == 64
|
||||
walk_nx_mask = 1ULL << PT64_NX_SHIFT;
|
||||
if (walker->level == PT32E_ROOT_LEVEL) {
|
||||
pte = mmu->get_pdptr(vcpu, (addr >> 30) & 3);
|
||||
pte = w->get_pdptr(vcpu, (addr >> 30) & 3);
|
||||
trace_kvm_mmu_paging_element(pte, walker->level);
|
||||
if (!FNAME(is_present_gpte)(mmu, pte))
|
||||
if (!FNAME(is_present_gpte)(w, pte))
|
||||
goto error;
|
||||
--walker->level;
|
||||
}
|
||||
@@ -393,7 +395,7 @@ retry_walk:
|
||||
walker->table_gfn[walker->level - 1] = table_gfn;
|
||||
walker->pte_gpa[walker->level - 1] = pte_gpa;
|
||||
|
||||
real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(table_gfn),
|
||||
real_gpa = kvm_translate_gpa(vcpu, w, gfn_to_gpa(table_gfn),
|
||||
nested_access | PFERR_GUEST_PAGE_MASK,
|
||||
&walker->fault, 0);
|
||||
|
||||
@@ -422,10 +424,10 @@ retry_walk:
|
||||
*/
|
||||
pte_access = pt_access & (pte ^ walk_nx_mask);
|
||||
|
||||
if (unlikely(!FNAME(is_present_gpte)(mmu, pte)))
|
||||
if (unlikely(!FNAME(is_present_gpte)(w, pte)))
|
||||
goto error;
|
||||
|
||||
if (unlikely(FNAME(is_rsvd_bits_set)(mmu, pte, walker->level))) {
|
||||
if (unlikely(FNAME(is_rsvd_bits_set)(&w->fmt, pte, walker->level))) {
|
||||
errcode = PFERR_RSVD_MASK | PFERR_PRESENT_MASK;
|
||||
goto error;
|
||||
}
|
||||
@@ -434,14 +436,14 @@ retry_walk:
|
||||
|
||||
/* Convert to ACC_*_MASK flags for struct guest_walker. */
|
||||
walker->pt_access[walker->level - 1] = FNAME(gpte_access)(pt_access ^ walk_nx_mask);
|
||||
} while (!FNAME(is_last_gpte)(mmu, walker->level, pte));
|
||||
} while (!FNAME(is_last_gpte)(w, walker->level, pte));
|
||||
|
||||
pte_pkey = FNAME(gpte_pkeys)(vcpu, pte);
|
||||
accessed_dirty = have_ad ? pte_access & PT_GUEST_ACCESSED_MASK : 0;
|
||||
|
||||
/* Convert to ACC_*_MASK flags for struct guest_walker. */
|
||||
walker->pte_access = FNAME(gpte_access)(pte_access ^ walk_nx_mask);
|
||||
errcode = permission_fault(vcpu, mmu, walker->pte_access, pte_pkey, access);
|
||||
errcode = permission_fault(vcpu, w, walker->pte_access, pte_pkey, access);
|
||||
if (unlikely(errcode))
|
||||
goto error;
|
||||
|
||||
@@ -453,7 +455,7 @@ retry_walk:
|
||||
gfn += pse36_gfn_delta(pte);
|
||||
#endif
|
||||
|
||||
real_gpa = kvm_translate_gpa(vcpu, mmu, gfn_to_gpa(gfn),
|
||||
real_gpa = kvm_translate_gpa(vcpu, w, gfn_to_gpa(gfn),
|
||||
access | PFERR_GUEST_FINAL_MASK,
|
||||
&walker->fault, walker->pte_access);
|
||||
if (real_gpa == INVALID_GPA)
|
||||
@@ -462,7 +464,7 @@ retry_walk:
|
||||
walker->gfn = real_gpa >> PAGE_SHIFT;
|
||||
|
||||
if (!write_fault)
|
||||
FNAME(protect_clean_gpte)(mmu, &walker->pte_access, pte);
|
||||
FNAME(protect_clean_gpte)(w, &walker->pte_access, pte);
|
||||
else
|
||||
/*
|
||||
* On a write fault, fold the dirty bit into accessed_dirty.
|
||||
@@ -473,7 +475,7 @@ retry_walk:
|
||||
(PT_GUEST_DIRTY_SHIFT - PT_GUEST_ACCESSED_SHIFT);
|
||||
|
||||
if (unlikely(!accessed_dirty)) {
|
||||
ret = FNAME(update_accessed_dirty_bits)(vcpu, mmu, walker,
|
||||
ret = FNAME(update_accessed_dirty_bits)(vcpu, w, walker,
|
||||
addr, write_fault);
|
||||
if (unlikely(ret < 0))
|
||||
goto error;
|
||||
@@ -485,7 +487,7 @@ retry_walk:
|
||||
|
||||
error:
|
||||
errcode |= write_fault | user_fault;
|
||||
if (fetch_fault && has_pferr_fetch(mmu))
|
||||
if (fetch_fault && has_pferr_fetch(w))
|
||||
errcode |= PFERR_FETCH_MASK;
|
||||
|
||||
walker->fault.vector = PF_VECTOR;
|
||||
@@ -540,13 +542,13 @@ error:
|
||||
* ACC_*_MASK flags!
|
||||
*/
|
||||
walker->fault.exit_qualification |= EPT_VIOLATION_RWX_TO_PROT(pte_access);
|
||||
if (mmu_has_mbec(mmu))
|
||||
if (is_cr4_smep(w))
|
||||
walker->fault.exit_qualification |=
|
||||
EPT_VIOLATION_USER_EXEC_TO_PROT(pte_access);
|
||||
}
|
||||
#endif
|
||||
walker->fault.address = addr;
|
||||
walker->fault.nested_page_fault = mmu != vcpu->arch.walk_mmu;
|
||||
walker->fault.nested_page_fault = w != &vcpu->arch.gva_walk;
|
||||
walker->fault.async_page_fault = false;
|
||||
|
||||
#if PTTYPE != PTTYPE_EPT
|
||||
@@ -561,7 +563,7 @@ error:
|
||||
static int FNAME(walk_addr)(struct guest_walker *walker,
|
||||
struct kvm_vcpu *vcpu, gpa_t addr, u64 access)
|
||||
{
|
||||
return FNAME(walk_addr_generic)(walker, vcpu, vcpu->arch.mmu, addr,
|
||||
return FNAME(walk_addr_generic)(walker, vcpu, vcpu->arch.mmu->w, addr,
|
||||
access);
|
||||
}
|
||||
|
||||
@@ -577,7 +579,7 @@ FNAME(prefetch_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
|
||||
|
||||
gfn = gpte_to_gfn(gpte);
|
||||
pte_access = sp->role.access & FNAME(gpte_access)(gpte);
|
||||
FNAME(protect_clean_gpte)(vcpu->arch.mmu, &pte_access, gpte);
|
||||
FNAME(protect_clean_gpte)(vcpu->arch.mmu->w, &pte_access, gpte);
|
||||
|
||||
return kvm_mmu_prefetch_sptes(vcpu, gfn, spte, 1, pte_access);
|
||||
}
|
||||
@@ -660,7 +662,7 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
|
||||
WARN_ON_ONCE(gw->gfn != base_gfn);
|
||||
direct_access = gw->pte_access;
|
||||
|
||||
top_level = vcpu->arch.mmu->cpu_role.base.level;
|
||||
top_level = vcpu->arch.mmu->w->cpu_role.base.level;
|
||||
if (top_level == PT32E_ROOT_LEVEL)
|
||||
top_level = PT32_ROOT_LEVEL;
|
||||
/*
|
||||
@@ -849,7 +851,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
|
||||
* otherwise KVM will cache incorrect access information in the SPTE.
|
||||
*/
|
||||
if (fault->write && !(walker.pte_access & ACC_WRITE_MASK) &&
|
||||
!is_cr0_wp(vcpu->arch.mmu) && !fault->user && fault->slot) {
|
||||
!is_cr0_wp(vcpu->arch.mmu->w) && !fault->user && fault->slot) {
|
||||
walker.pte_access |= ACC_WRITE_MASK;
|
||||
walker.pte_access &= ~ACC_USER_MASK;
|
||||
|
||||
@@ -859,7 +861,7 @@ static int FNAME(page_fault)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault
|
||||
* then we should prevent the kernel from executing it
|
||||
* if SMEP is enabled.
|
||||
*/
|
||||
if (is_cr4_smep(vcpu->arch.mmu))
|
||||
if (is_cr4_smep(vcpu->arch.mmu->w))
|
||||
walker.pte_access &= ~ACC_EXEC_MASK;
|
||||
}
|
||||
#endif
|
||||
@@ -896,7 +898,7 @@ static gpa_t FNAME(get_level1_sp_gpa)(struct kvm_mmu_page *sp)
|
||||
}
|
||||
|
||||
/* Note, @addr is a GPA when gva_to_gpa() translates an L2 GPA to an L1 GPA. */
|
||||
static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
|
||||
static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_pagewalk *w,
|
||||
gpa_t addr, u64 access,
|
||||
struct x86_exception *exception)
|
||||
{
|
||||
@@ -906,10 +908,10 @@ static gpa_t FNAME(gva_to_gpa)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
|
||||
|
||||
#ifndef CONFIG_X86_64
|
||||
/* A 64-bit GVA should be impossible on 32-bit KVM. */
|
||||
WARN_ON_ONCE((addr >> 32) && mmu == vcpu->arch.walk_mmu);
|
||||
WARN_ON_ONCE((addr >> 32) && w == &vcpu->arch.gva_walk);
|
||||
#endif
|
||||
|
||||
r = FNAME(walk_addr_generic)(&walker, vcpu, mmu, addr, access);
|
||||
r = FNAME(walk_addr_generic)(&walker, vcpu, w, addr, access);
|
||||
|
||||
if (r) {
|
||||
gpa = gfn_to_gpa(walker.gfn);
|
||||
@@ -959,7 +961,7 @@ static int FNAME(sync_spte)(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp, int
|
||||
gfn = gpte_to_gfn(gpte);
|
||||
pte_access = sp->role.access;
|
||||
pte_access &= FNAME(gpte_access)(gpte);
|
||||
FNAME(protect_clean_gpte)(vcpu->arch.mmu, &pte_access, gpte);
|
||||
FNAME(protect_clean_gpte)(vcpu->arch.mmu->w, &pte_access, gpte);
|
||||
|
||||
if (sync_mmio_spte(vcpu, &sp->spt[i], gfn, pte_access))
|
||||
return 0;
|
||||
|
||||
@@ -281,9 +281,9 @@ bool make_spte(struct kvm_vcpu *vcpu, struct kvm_mmu_page *sp,
|
||||
if (prefetch && !synchronizing)
|
||||
spte = mark_spte_for_access_track(spte);
|
||||
|
||||
WARN_ONCE(is_rsvd_spte(&vcpu->arch.mmu->shadow_zero_check, spte, level),
|
||||
WARN_ONCE(is_rsvd_spte(&vcpu->arch.mmu->fmt, spte, level),
|
||||
"spte = 0x%llx, level = %d, rsvd bits = 0x%llx", spte, level,
|
||||
get_rsvd_bits(&vcpu->arch.mmu->shadow_zero_check, spte, level));
|
||||
get_rsvd_bits(&vcpu->arch.mmu->fmt, spte, level));
|
||||
|
||||
/*
|
||||
* Mark the memslot dirty *after* modifying it for access tracking.
|
||||
|
||||
+39
-30
@@ -357,17 +357,6 @@ static inline bool is_last_spte(u64 pte, int level)
|
||||
return (level == PG_LEVEL_4K) || is_large_pte(pte);
|
||||
}
|
||||
|
||||
static inline bool is_executable_pte(u64 spte)
|
||||
{
|
||||
/*
|
||||
* For now, return true if either the XS or XU bit is set
|
||||
* This function is only used for fast_page_fault,
|
||||
* which never processes shadow EPT, and regular page
|
||||
* tables always have XS==XU.
|
||||
*/
|
||||
return (spte & (shadow_xs_mask | shadow_xu_mask | shadow_nx_mask)) != shadow_nx_mask;
|
||||
}
|
||||
|
||||
static inline kvm_pfn_t spte_to_pfn(u64 pte)
|
||||
{
|
||||
return (pte & SPTE_BASE_ADDR_MASK) >> PAGE_SHIFT;
|
||||
@@ -378,33 +367,33 @@ static inline bool is_accessed_spte(u64 spte)
|
||||
return spte & shadow_accessed_mask;
|
||||
}
|
||||
|
||||
static inline u64 get_rsvd_bits(struct rsvd_bits_validate *rsvd_check, u64 pte,
|
||||
static inline u64 get_rsvd_bits(struct kvm_page_format *fmt, u64 pte,
|
||||
int level)
|
||||
{
|
||||
int bit7 = (pte >> 7) & 1;
|
||||
|
||||
return rsvd_check->rsvd_bits_mask[bit7][level-1];
|
||||
return fmt->rsvd_bits_mask[bit7][level-1];
|
||||
}
|
||||
|
||||
static inline bool __is_rsvd_bits_set(struct rsvd_bits_validate *rsvd_check,
|
||||
static inline bool __is_rsvd_bits_set(struct kvm_page_format *fmt,
|
||||
u64 pte, int level)
|
||||
{
|
||||
return pte & get_rsvd_bits(rsvd_check, pte, level);
|
||||
return pte & get_rsvd_bits(fmt, pte, level);
|
||||
}
|
||||
|
||||
static inline bool __is_bad_mt_xwr(struct rsvd_bits_validate *rsvd_check,
|
||||
static inline bool __is_bad_mt_xwr(struct kvm_page_format *fmt,
|
||||
u64 pte)
|
||||
{
|
||||
if (pte & VMX_EPT_USER_EXECUTABLE_MASK)
|
||||
pte |= VMX_EPT_EXECUTABLE_MASK;
|
||||
return rsvd_check->bad_mt_xwr & BIT_ULL(pte & 0x3f);
|
||||
return fmt->bad_mt_xwr & BIT_ULL(pte & 0x3f);
|
||||
}
|
||||
|
||||
static __always_inline bool is_rsvd_spte(struct rsvd_bits_validate *rsvd_check,
|
||||
static __always_inline bool is_rsvd_spte(struct kvm_page_format *fmt,
|
||||
u64 spte, int level)
|
||||
{
|
||||
return __is_bad_mt_xwr(rsvd_check, spte) ||
|
||||
__is_rsvd_bits_set(rsvd_check, spte, level);
|
||||
return __is_bad_mt_xwr(fmt, spte) ||
|
||||
__is_rsvd_bits_set(fmt, spte, level);
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -496,20 +485,40 @@ static inline bool is_mmu_writable_spte(u64 spte)
|
||||
}
|
||||
|
||||
/*
|
||||
* Returns true if the access indicated by @fault is allowed by the existing
|
||||
* SPTE protections. Note, the caller is responsible for checking that the
|
||||
* SPTE is a shadow-present, leaf SPTE (either before or after).
|
||||
* Returns true if the access indicated by @fault is forbidden by the existing
|
||||
* SPTE protections.
|
||||
*/
|
||||
static inline bool is_access_allowed(struct kvm_page_fault *fault, u64 spte)
|
||||
static inline bool spte_permission_fault(struct kvm_mmu *mmu, u64 spte,
|
||||
struct kvm_page_fault *fault)
|
||||
{
|
||||
if (fault->exec)
|
||||
return is_executable_pte(spte);
|
||||
unsigned pfec, pte_access;
|
||||
|
||||
if (fault->write)
|
||||
return is_writable_pte(spte);
|
||||
if (!is_shadow_present_pte(spte))
|
||||
return true;
|
||||
|
||||
/* Fault was on Read access */
|
||||
return spte & PT_PRESENT_MASK;
|
||||
BUILD_BUG_ON(PT_PRESENT_MASK != ACC_READ_MASK);
|
||||
BUILD_BUG_ON(PT_WRITABLE_MASK != ACC_WRITE_MASK);
|
||||
BUILD_BUG_ON(VMX_EPT_READABLE_MASK != ACC_READ_MASK);
|
||||
BUILD_BUG_ON(VMX_EPT_WRITABLE_MASK != ACC_WRITE_MASK);
|
||||
|
||||
/* strip nested paging fault error codes */
|
||||
pte_access = spte & (PT_PRESENT_MASK | PT_WRITABLE_MASK);
|
||||
if (shadow_nx_mask) {
|
||||
pte_access |= spte & shadow_user_mask ? ACC_USER_MASK : 0;
|
||||
pte_access |= spte & shadow_nx_mask ? 0 : ACC_EXEC_MASK;
|
||||
} else {
|
||||
pte_access |= spte & shadow_xs_mask ? ACC_EXEC_MASK : 0;
|
||||
pte_access |= spte & shadow_xu_mask ? ACC_USER_EXEC_MASK : 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* RSVD is handled elsewhere, and is used for SMAP in the context
|
||||
* of accessing fmt.permissions[]. SPTEs never use PK or SS, as
|
||||
* they are not supported for shadow paging and irrelevant for TDP.
|
||||
*/
|
||||
pfec = fault->error_code & (
|
||||
PFERR_WRITE_MASK | PFERR_USER_MASK | PFERR_FETCH_MASK);
|
||||
return (mmu->fmt.permissions[pfec >> 1] >> pte_access) & 1;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
@@ -1122,6 +1122,7 @@ static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu,
|
||||
struct kvm_page_fault *fault,
|
||||
struct tdp_iter *iter)
|
||||
{
|
||||
struct kvm_mmu *mmu = vcpu->arch.mmu;
|
||||
struct kvm_mmu_page *sp = sptep_to_sp(rcu_dereference(iter->sptep));
|
||||
u64 new_spte;
|
||||
int ret = RET_PF_FIXED;
|
||||
@@ -1131,7 +1132,7 @@ static int tdp_mmu_map_handle_target_level(struct kvm_vcpu *vcpu,
|
||||
return RET_PF_RETRY;
|
||||
|
||||
if (is_shadow_present_pte(iter->old_spte) &&
|
||||
(fault->prefetch || is_access_allowed(fault, iter->old_spte)) &&
|
||||
(fault->prefetch || !spte_permission_fault(mmu, iter->old_spte, fault)) &&
|
||||
is_last_spte(iter->old_spte, iter->level)) {
|
||||
WARN_ON_ONCE(fault->pfn != spte_to_pfn(iter->old_spte));
|
||||
return RET_PF_SPURIOUS;
|
||||
|
||||
+2745
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,156 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
#ifndef ARCH_X86_KVM_MSR_H
|
||||
#define ARCH_X86_KVM_MSR_H
|
||||
|
||||
#include <linux/kvm_host.h>
|
||||
#include <linux/user-return-notifier.h>
|
||||
|
||||
#include "cpuid.h"
|
||||
#include "regs.h"
|
||||
|
||||
extern bool report_ignored_msrs;
|
||||
extern bool ignore_msrs;
|
||||
|
||||
extern u32 __read_mostly kvm_nr_uret_msrs;
|
||||
|
||||
static inline void kvm_pr_unimpl_wrmsr(struct kvm_vcpu *vcpu, u32 msr, u64 data)
|
||||
{
|
||||
if (report_ignored_msrs)
|
||||
vcpu_unimpl(vcpu, "Unhandled WRMSR(0x%x) = 0x%llx\n", msr, data);
|
||||
}
|
||||
|
||||
static inline void kvm_pr_unimpl_rdmsr(struct kvm_vcpu *vcpu, u32 msr)
|
||||
{
|
||||
if (report_ignored_msrs)
|
||||
vcpu_unimpl(vcpu, "Unhandled RDMSR(0x%x)\n", msr);
|
||||
}
|
||||
|
||||
/*
|
||||
* The first...last VMX feature MSRs that are emulated by KVM. This may or may
|
||||
* not cover all known VMX MSRs, as KVM doesn't emulate an MSR until there's an
|
||||
* associated feature that KVM supports for nested virtualization.
|
||||
*/
|
||||
#define KVM_FIRST_EMULATED_VMX_MSR MSR_IA32_VMX_BASIC
|
||||
#define KVM_LAST_EMULATED_VMX_MSR MSR_IA32_VMX_VMFUNC
|
||||
|
||||
/*
|
||||
* KVM's internal, non-ABI indices for synthetic MSRs. The values themselves
|
||||
* are arbitrary and have no meaning, the only requirement is that they don't
|
||||
* conflict with "real" MSRs that KVM supports. Use values at the upper end
|
||||
* of KVM's reserved paravirtual MSR range to minimize churn, i.e. these values
|
||||
* will be usable until KVM exhausts its supply of paravirtual MSR indices.
|
||||
*/
|
||||
#define MSR_KVM_INTERNAL_GUEST_SSP 0x4b564dff
|
||||
|
||||
#define MSR_IA32_CR_PAT_DEFAULT \
|
||||
PAT_VALUE(WB, WT, UC_MINUS, UC, WB, WT, UC_MINUS, UC)
|
||||
|
||||
void kvm_init_msr_lists(void);
|
||||
int kvm_get_msr_index_list(struct kvm_msr_list __user *user_msr_list);
|
||||
int kvm_get_feature_msr_index_list(struct kvm_msr_list __user *user_msr_list);
|
||||
int kvm_get_feature_msrs(struct kvm_msrs __user *user_msrs);
|
||||
|
||||
int kvm_get_msrs(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs);
|
||||
int kvm_set_msrs(struct kvm_vcpu *vcpu, struct kvm_msrs __user *user_msrs);
|
||||
|
||||
int kvm_get_set_one_reg(struct kvm_vcpu *vcpu, unsigned int ioctl,
|
||||
void __user *argp);
|
||||
int kvm_get_reg_list(struct kvm_vcpu *vcpu,
|
||||
struct kvm_reg_list __user *user_list);
|
||||
|
||||
void kvm_enable_efer_bits(u64);
|
||||
bool kvm_valid_efer(struct kvm_vcpu *vcpu, u64 efer);
|
||||
int kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data);
|
||||
int kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data);
|
||||
int __kvm_emulate_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data);
|
||||
int __kvm_emulate_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data);
|
||||
int kvm_msr_read(struct kvm_vcpu *vcpu, u32 index, u64 *data);
|
||||
int kvm_msr_write(struct kvm_vcpu *vcpu, u32 index, u64 data);
|
||||
int kvm_emulate_rdmsr(struct kvm_vcpu *vcpu);
|
||||
int kvm_emulate_rdmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg);
|
||||
int kvm_emulate_wrmsr(struct kvm_vcpu *vcpu);
|
||||
int kvm_emulate_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg);
|
||||
|
||||
fastpath_t handle_fastpath_wrmsr(struct kvm_vcpu *vcpu);
|
||||
fastpath_t handle_fastpath_wrmsr_imm(struct kvm_vcpu *vcpu, u32 msr, int reg);
|
||||
|
||||
int kvm_get_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);
|
||||
int kvm_set_msr_common(struct kvm_vcpu *vcpu, struct msr_data *msr);
|
||||
|
||||
int kvm_add_user_return_msr(u32 msr);
|
||||
int kvm_find_user_return_msr(u32 msr);
|
||||
int kvm_set_user_return_msr(unsigned index, u64 val, u64 mask);
|
||||
u64 kvm_get_user_return_msr(unsigned int slot);
|
||||
|
||||
static inline bool kvm_is_supported_user_return_msr(u32 msr)
|
||||
{
|
||||
return kvm_find_user_return_msr(msr) >= 0;
|
||||
}
|
||||
|
||||
void kvm_user_return_msr_cpu_online(void);
|
||||
void drop_user_return_notifiers(void);
|
||||
void kvm_destroy_user_return_msrs(void);
|
||||
|
||||
int kvm_emulator_get_msr_with_filter(struct kvm_vcpu *vcpu, u32 msr_index,
|
||||
u64 *pdata);
|
||||
int kvm_emulator_set_msr_with_filter(struct kvm_vcpu *vcpu, u32 msr_index,
|
||||
u64 data);
|
||||
int kvm_emulator_get_msr(struct kvm_vcpu *vcpu, u32 msr_index, u64 *pdata);
|
||||
|
||||
bool kvm_msr_allowed(struct kvm_vcpu *vcpu, u32 index, u32 type);
|
||||
|
||||
enum kvm_msr_access {
|
||||
MSR_TYPE_R = BIT(0),
|
||||
MSR_TYPE_W = BIT(1),
|
||||
MSR_TYPE_RW = MSR_TYPE_R | MSR_TYPE_W,
|
||||
};
|
||||
|
||||
/*
|
||||
* Internal error codes that are used to indicate that MSR emulation encountered
|
||||
* an error that should result in #GP in the guest, unless userspace handles it.
|
||||
* Note, '1', '0', and negative numbers are off limits, as they are used by KVM
|
||||
* as part of KVM's lightly documented internal KVM_RUN return codes.
|
||||
*
|
||||
* UNSUPPORTED - The MSR isn't supported, either because it is completely
|
||||
* unknown to KVM, or because the MSR should not exist according
|
||||
* to the vCPU model.
|
||||
*
|
||||
* FILTERED - Access to the MSR is denied by a userspace MSR filter.
|
||||
*/
|
||||
#define KVM_MSR_RET_UNSUPPORTED 2
|
||||
#define KVM_MSR_RET_FILTERED 3
|
||||
|
||||
int kvm_vm_ioctl_set_msr_filter(struct kvm *kvm, struct kvm_msr_filter *filter);
|
||||
void kvm_free_msr_filter(struct kvm_x86_msr_filter *msr_filter);
|
||||
|
||||
int kvm_mtrr_set_msr(struct kvm_vcpu *vcpu, u32 msr, u64 data);
|
||||
int kvm_mtrr_get_msr(struct kvm_vcpu *vcpu, u32 msr, u64 *pdata);
|
||||
|
||||
u64 kvm_get_arch_capabilities(void);
|
||||
int kvm_spec_ctrl_test_value(u64 value);
|
||||
|
||||
#define CET_US_RESERVED_BITS GENMASK(9, 6)
|
||||
#define CET_US_SHSTK_MASK_BITS GENMASK(1, 0)
|
||||
#define CET_US_IBT_MASK_BITS (GENMASK_ULL(5, 2) | GENMASK_ULL(63, 10))
|
||||
#define CET_US_LEGACY_BITMAP_BASE(data) ((data) >> 12)
|
||||
|
||||
static inline bool kvm_is_valid_u_s_cet(struct kvm_vcpu *vcpu, u64 data)
|
||||
{
|
||||
if (data & CET_US_RESERVED_BITS)
|
||||
return false;
|
||||
if (!guest_cpu_cap_has(vcpu, X86_FEATURE_SHSTK) &&
|
||||
(data & CET_US_SHSTK_MASK_BITS))
|
||||
return false;
|
||||
if (!guest_cpu_cap_has(vcpu, X86_FEATURE_IBT) &&
|
||||
(data & CET_US_IBT_MASK_BITS))
|
||||
return false;
|
||||
if (!IS_ALIGNED(CET_US_LEGACY_BITMAP_BASE(data), 4))
|
||||
return false;
|
||||
/* IBT can be suppressed iff the TRACKER isn't WAIT_ENDBR. */
|
||||
if ((data & CET_SUPPRESS) && (data & CET_WAIT_ENDBR))
|
||||
return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
#endif
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user