linuxmsm/laptops-kernel
0
0
Fork 0
mirror of https://github.com/linux-msm/laptops-kernel.git synced 2026-08-13 14:19:53 -07:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'next' of https://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git

Browse Source 
# Conflicts:
#	arch/arm64/tools/cpucaps
This commit is contained in:
Mark Brown
2026-07-31 16:02:49 +01:00
parent 7f3193226c 320840b34a
commit 9ecb2d7097
42 changed files with 986 additions and 290 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Documentation/virt/kvm/api.rst
+11
View File
@@ -3515,6 +3515,17 @@ Possible features:
Depends on KVM_CAP_ARM_PSCI_0_2.
- KVM_ARM_VCPU_PMU_V3: Emulate PMUv3 for the CPU.
Depends on KVM_CAP_ARM_PMU_V3.
- KVM_ARM_VCPU_PMU_V3_STRICT: Enable strict PMUv3 UAPI.
Requires KVM_ARM_VCPU_PMU_V3. Depends on KVM_CAP_ARM_PMU_V3_STRICT.
When enabled:
* Userspace must explicitly select a PMU implementation before
initializing the PMU or configuring a PMU event filter
* If the PMU implements FEAT_PMUv3p4, PMMIR_EL1.SLOTS provides the
hardware value of the underlying implementation
* Writes to PMCR_EL0.N via KVM_SET_ONE_REG are ignored
- KVM_ARM_VCPU_PTRAUTH_ADDRESS: Enables Address Pointer authentication
for arm64 only.
Documentation/virt/kvm/devices/vcpu.rst
+9 -2
View File
@@ -53,8 +53,9 @@ Returns:
======= ======================================================
-EEXIST Interrupt number already used
-ENODEV PMUv3 not supported or GIC not initialized
-ENXIO PMUv3 not supported, missing VCPU feature or interrupt
number not set (non-GICv5 guests, only)
-ENXIO PMUv3 not supported, missing VCPU feature, missing
hardware PMU, or interrupt number not set (non-GICv5
guests, only)
-EBUSY PMUv3 already initialized
======= ======================================================
@@ -62,6 +63,9 @@ Request the initialization of the PMUv3. If using the PMUv3 with an in-kernel
virtual GIC implementation, this must be done after initializing the in-kernel
irqchip.
When the KVM_ARM_VCPU_PMU_V3_STRICT vCPU feature is enabled this must be done
after selecting a hardware PMU.
1.3 ATTRIBUTE: KVM_ARM_VCPU_PMU_V3_FILTER
-----------------------------------------
@@ -108,6 +112,9 @@ hardware event. Filtering event 0x1E (CHAIN) has no effect either, as it
isn't strictly speaking an event. Filtering the cycle counter is possible
using event 0x11 (CPU_CYCLES).
When the KVM_ARM_VCPU_PMU_V3_STRICT vCPU feature is enabled this must be done
after selecting a hardware PMU.
1.4 ATTRIBUTE: KVM_ARM_VCPU_PMU_V3_SET_PMU
------------------------------------------
arch/arm64/include/asm/kvm_arm.h
+12
View File
@@ -348,4 +348,16 @@
{ PSR_AA32_MODE_UND, "32-bit UND" }, \
{ PSR_AA32_MODE_SYS, "32-bit SYS" }
/*
* ARMv8 Reset Values
*/
#define VCPU_RESET_PSTATE_EL1 (PSR_MODE_EL1h | PSR_A_BIT | PSR_I_BIT | \
PSR_F_BIT | PSR_D_BIT)
#define VCPU_RESET_PSTATE_EL2 (PSR_MODE_EL2h | PSR_A_BIT | PSR_I_BIT | \
PSR_F_BIT | PSR_D_BIT)
#define VCPU_RESET_PSTATE_SVC (PSR_AA32_MODE_SVC | PSR_AA32_A_BIT | \
PSR_AA32_I_BIT | PSR_AA32_F_BIT)
#endif /* __ARM64_KVM_ARM_H__ */
arch/arm64/include/asm/kvm_asm.h
+2 -1
View File
@@ -113,6 +113,7 @@ enum __kvm_host_smccc_func {
__KVM_HOST_SMCCC_FUNC___pkvm_finalize_teardown_vm,
__KVM_HOST_SMCCC_FUNC___pkvm_vcpu_load,
__KVM_HOST_SMCCC_FUNC___pkvm_vcpu_put,
__KVM_HOST_SMCCC_FUNC___pkvm_vcpu_sync_state,
__KVM_HOST_SMCCC_FUNC___pkvm_tlb_flush_vmid,
MARKER(__KVM_HOST_SMCCC_FUNC_MAX)
@@ -281,7 +282,7 @@ extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
extern void __kvm_adjust_pc(struct kvm_vcpu *vcpu);
extern u64 __vgic_v3_get_gic_config(void);
extern bool __vgic_v3_get_gic_config(void);
extern void __vgic_v3_init_lrs(void);
#define __KVM_EXTABLE(from, to) \
arch/arm64/include/asm/kvm_emulate.h
+63
View File
@@ -506,6 +506,12 @@ static inline unsigned long kvm_vcpu_get_mpidr_aff(struct kvm_vcpu *vcpu)
return __vcpu_sys_reg(vcpu, MPIDR_EL1) & MPIDR_HWID_BITMASK;
}
/* In nVHE hyp code, registers are always in memory: use the raw accessors. */
#if defined(__KVM_NVHE_HYPERVISOR__)
#define vcpu_read_sys_reg(v, r) __vcpu_sys_reg(v, r)
#define vcpu_write_sys_reg(v, x, r) __vcpu_assign_sys_reg(v, r, x)
#endif
static inline void kvm_vcpu_set_be(struct kvm_vcpu *vcpu)
{
if (vcpu_mode_is_32bit(vcpu)) {
@@ -688,4 +694,61 @@ static inline void vcpu_set_hcrx(struct kvm_vcpu *vcpu)
vcpu->arch.hcrx_el2 |= HCRX_EL2_EnASR;
}
}
/* Reset a vcpu's core registers. */
static inline void kvm_reset_vcpu_core(struct kvm_vcpu *vcpu)
{
u32 pstate;
if (vcpu_el1_is_32bit(vcpu))
pstate = VCPU_RESET_PSTATE_SVC;
else if (vcpu_has_nv(vcpu))
pstate = VCPU_RESET_PSTATE_EL2;
else
pstate = VCPU_RESET_PSTATE_EL1;
/* Reset core registers */
memset(vcpu_gp_regs(vcpu), 0, sizeof(*vcpu_gp_regs(vcpu)));
memset(&vcpu->arch.ctxt.fp_regs, 0, sizeof(vcpu->arch.ctxt.fp_regs));
vcpu->arch.ctxt.spsr_abt = 0;
vcpu->arch.ctxt.spsr_und = 0;
vcpu->arch.ctxt.spsr_irq = 0;
vcpu->arch.ctxt.spsr_fiq = 0;
vcpu_gp_regs(vcpu)->pstate = pstate;
}
/* PSCI reset handling for a vcpu. */
static inline void kvm_reset_vcpu_psci(struct kvm_vcpu *vcpu,
struct vcpu_reset_state *reset_state)
{
unsigned long target_pc = reset_state->pc;
/* Gracefully handle Thumb2 entry point */
if (vcpu_mode_is_32bit(vcpu) && (target_pc & 1)) {
target_pc &= ~1UL;
vcpu_set_thumb(vcpu);
}
/* Propagate caller endianness */
if (reset_state->be)
kvm_vcpu_set_be(vcpu);
*vcpu_pc(vcpu) = target_pc;
/*
* We may come from a state where either a PC update was
* pending (SMC call resulting in PC being increpented to
* skip the SMC) or a pending exception. Make sure we get
* rid of all that, as this cannot be valid out of reset.
*
* Note that clearing the exception mask also clears PC
* updates, but that's an implementation detail, and we
* really want to make it explicit.
*/
vcpu_clear_flag(vcpu, PENDING_EXCEPTION);
vcpu_clear_flag(vcpu, EXCEPT_MASK);
vcpu_clear_flag(vcpu, INCREMENT_PC);
vcpu_set_reg(vcpu, 0, reset_state->r0);
}
#endif /* __ARM64_KVM_EMULATE_H__ */
arch/arm64/include/asm/kvm_host.h
+6 -1
View File
@@ -39,7 +39,7 @@
#define KVM_MAX_VCPUS VGIC_V3_MAX_CPUS
#define KVM_VCPU_MAX_FEATURES 9
#define KVM_VCPU_MAX_FEATURES 10
#define KVM_VCPU_VALID_FEATURES (BIT(KVM_VCPU_MAX_FEATURES) - 1)
#define KVM_REQ_SLEEP \
@@ -387,6 +387,9 @@ struct kvm_arch {
/* Maximum number of counters for the guest */
u8 nr_pmu_counters;
/* PMMIR_EL1.SLOTS value exposed to the guest. */
u8 pmmir_slots;
/* Hypercall features firmware registers' descriptor */
struct kvm_smccc_features smccc_feat;
struct maple_tree smccc_filter;
@@ -1051,6 +1054,8 @@ struct kvm_vcpu_arch {
#define INCREMENT_PC __vcpu_single_flag(iflags, BIT(1))
/* Target EL/MODE (not a single flag, but let's abuse the macro) */
#define EXCEPT_MASK __vcpu_single_flag(iflags, GENMASK(3, 1))
/* Host-set: the hyp flushes the non-protected vCPU state in on entry */
#define PKVM_HOST_STATE_DIRTY __vcpu_single_flag(iflags, BIT(4))
/* Helpers to encode exceptions with minimum fuss */
#define __EXCEPT_MASK_VAL unpack_vcpu_flag(EXCEPT_MASK)
arch/arm64/include/asm/kvm_pkvm.h
+7 -1
View File
@@ -45,6 +45,9 @@ static inline bool kvm_pkvm_ext_allowed(struct kvm *kvm, long ext)
return true;
case KVM_CAP_ARM_MTE:
return false;
case KVM_CAP_ARM_EAGER_SPLIT_CHUNK_SIZE:
case KVM_CAP_ARM_SUPPORTED_BLOCK_SIZES:
return false;
default:
return !kvm || !kvm_vm_is_protected(kvm);
}
@@ -195,7 +198,10 @@ struct pkvm_mapping {
struct rb_node node;
u64 gfn;
u64 pfn;
u64 nr_pages;
struct {
u64 nr_pages:48;
u64 nc:1;
};
u64 __subtree_last; /* Internal member for interval tree */
};
arch/arm64/include/uapi/asm/kvm.h
+1
View File
@@ -106,6 +106,7 @@ struct kvm_regs {
#define KVM_ARM_VCPU_PTRAUTH_GENERIC 6 /* VCPU uses generic authentication */
#define KVM_ARM_VCPU_HAS_EL2 7 /* Support nested virtualization */
#define KVM_ARM_VCPU_HAS_EL2_E2H0 8 /* Limit NV support to E2H RES0 */
#define KVM_ARM_VCPU_PMU_V3_STRICT 9 /* No default PMU creation */
struct kvm_vcpu_init {
__u32 target;
arch/arm64/kernel/cpu_errata.c
+15 -2
View File
@@ -272,7 +272,7 @@ has_neoverse_n1_erratum_1542419(const struct arm64_cpu_capabilities *entry,
return is_midr_in_range(&range) && has_dic;
}
static const struct midr_range impdef_pmuv3_cpus[] = {
static const struct midr_range apple_cpus[] = {
MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM),
MIDR_ALL_VERSIONS(MIDR_APPLE_M1_FIRESTORM),
MIDR_ALL_VERSIONS(MIDR_APPLE_M1_ICESTORM_PRO),
@@ -301,7 +301,14 @@ static bool has_impdef_pmuv3(const struct arm64_cpu_capabilities *entry, int sco
if (pmuver != ID_AA64DFR0_EL1_PMUVer_IMP_DEF)
return false;
return is_midr_in_range_list(impdef_pmuv3_cpus);
return is_midr_in_range_list(apple_cpus);
}
static bool has_broken_gic_v3_seis(const struct arm64_cpu_capabilities *entry, int scope)
{
return (is_kernel_in_hyp_mode() &&
is_midr_in_range_list(apple_cpus) &&
(read_sysreg_s(SYS_ICH_VTR_EL2) & ICH_VTR_EL2_SEIS));
}
static void cpu_enable_impdef_pmuv3_traps(const struct arm64_cpu_capabilities *__unused)
@@ -1017,6 +1024,12 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
.matches = has_impdef_pmuv3,
.cpu_enable = cpu_enable_impdef_pmuv3_traps,
},
{
.desc = "Known broken GICv3 SEIS implementation",
.capability = ARM64_WORKAROUND_GICv3_BROKEN_SEIS,
.type = ARM64_CPUCAP_SYSTEM_FEATURE,
.matches = has_broken_gic_v3_seis,
},
{
}
};
arch/arm64/kernel/image-vars.h
+1
View File
@@ -92,6 +92,7 @@ KVM_NVHE_ALIAS(spectre_bhb_patch_wa3);
KVM_NVHE_ALIAS(spectre_bhb_patch_clearbhb);
KVM_NVHE_ALIAS(alt_cb_patch_nops);
KVM_NVHE_ALIAS(kvm_compute_ich_hcr_trap_bits);
KVM_NVHE_ALIAS(kvm_patch_ich_vtr_el2);
/* Global kernel state accessed by nVHE hyp code. */
KVM_NVHE_ALIAS(kvm_vgic_global_state);
arch/arm64/kvm/arm.c
+22 -4
View File
@@ -465,6 +465,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
r = get_num_wrps();
break;
case KVM_CAP_ARM_PMU_V3:
case KVM_CAP_ARM_PMU_V3_STRICT:
r = kvm_supports_guest_pmuv3();
break;
case KVM_CAP_ARM_INJECT_SERROR_ESR:
@@ -748,6 +749,10 @@ void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
if (is_protected_kvm_enabled()) {
kvm_call_hyp(__vgic_v3_save_aprs, &vcpu->arch.vgic_cpu.vgic_v3);
kvm_call_hyp_nvhe(__pkvm_vcpu_put);
/* __pkvm_vcpu_put implies a sync of the state */
if (!kvm_vm_is_protected(vcpu->kvm))
vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY);
}
kvm_vcpu_put_debug(vcpu);
@@ -979,6 +984,9 @@ int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu)
return ret;
if (is_protected_kvm_enabled()) {
/* Start with the vcpu in a dirty state */
if (!kvm_vm_is_protected(vcpu->kvm))
vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY);
ret = pkvm_create_hyp_vm(kvm);
if (ret)
return ret;
@@ -1576,8 +1584,10 @@ static unsigned long system_supported_vcpu_features(void)
if (!cpus_have_final_cap(ARM64_HAS_32BIT_EL1))
clear_bit(KVM_ARM_VCPU_EL1_32BIT, &features);
if (!kvm_supports_guest_pmuv3())
if (!kvm_supports_guest_pmuv3()) {
clear_bit(KVM_ARM_VCPU_PMU_V3, &features);
clear_bit(KVM_ARM_VCPU_PMU_V3_STRICT, &features);
}
if (!system_supports_sve())
clear_bit(KVM_ARM_VCPU_SVE, &features);
@@ -1618,6 +1628,11 @@ static int kvm_vcpu_init_check_features(struct kvm_vcpu *vcpu,
test_bit(KVM_ARM_VCPU_PTRAUTH_GENERIC, &features))
return -EINVAL;
/* Strict PMUv3 UAPI requires PMUv3. */
if (test_bit(KVM_ARM_VCPU_PMU_V3_STRICT, &features) &&
!test_bit(KVM_ARM_VCPU_PMU_V3, &features))
return -EINVAL;
if (!test_bit(KVM_ARM_VCPU_EL1_32BIT, &features))
return 0;
@@ -1647,10 +1662,13 @@ static int kvm_setup_vcpu(struct kvm_vcpu *vcpu)
int ret = 0;
/*
* When the vCPU has a PMU, but no PMU is set for the guest
* yet, set the default one.
* When the vCPU has a PMU, but no PMU is set for the guest yet, set
* the default one. If KVM_ARM_VCPU_PMU_V3_STRICT is set, no default
* PMU is created, and userspace must select a PMU via
* KVM_ARM_VCPU_PMU_V3_SET_PMU.
*/
if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu)
if (kvm_vcpu_has_pmu(vcpu) && !kvm->arch.arm_pmu &&
!kvm_vcpu_has_pmuv3_strict(vcpu))
ret = kvm_arm_set_default_pmu(kvm);
/* Prepare for nested if required */
arch/arm64/kvm/handle_exit.c
+23
View File
@@ -486,9 +486,32 @@ int handle_exit(struct kvm_vcpu *vcpu, int exception_index)
}
}
static void handle_exit_pkvm_state(struct kvm_vcpu *vcpu, int exception_index)
{
int exception_code = ARM_EXCEPTION_CODE(exception_index);
if (!is_protected_kvm_enabled() || kvm_vm_is_protected(vcpu->kvm))
return;
/*
* Sync the context back when the host will read (trap) or write
* (SError) it. Preempt-off here, so the loaded hyp vCPU is stable.
*/
if (exception_code == ARM_EXCEPTION_TRAP ||
exception_code == ARM_EXCEPTION_EL1_SERROR ||
ARM_SERROR_PENDING(exception_index)) {
kvm_call_hyp_nvhe(__pkvm_vcpu_sync_state);
vcpu_set_flag(vcpu, PKVM_HOST_STATE_DIRTY);
} else {
vcpu_clear_flag(vcpu, PKVM_HOST_STATE_DIRTY);
}
}
/* For exit types that need handling before we can be preempted */
void handle_exit_early(struct kvm_vcpu *vcpu, int exception_index)
{
handle_exit_pkvm_state(vcpu, exception_index);
if (ARM_SERROR_PENDING(exception_index)) {
if (this_cpu_has_cap(ARM64_HAS_RAS_EXTN)) {
u64 disr = kvm_vcpu_get_disr(vcpu);
arch/arm64/kvm/hyp/exception.c
+9 -25
View File
@@ -20,22 +20,6 @@
#error Hypervisor code only!
#endif
static inline u64 __vcpu_read_sys_reg(const struct kvm_vcpu *vcpu, int reg)
{
if (has_vhe())
return vcpu_read_sys_reg(vcpu, reg);
return __vcpu_sys_reg(vcpu, reg);
}
static inline void __vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg)
{
if (has_vhe())
vcpu_write_sys_reg(vcpu, val, reg);
else
__vcpu_assign_sys_reg(vcpu, reg, val);
}
static void __vcpu_write_spsr(struct kvm_vcpu *vcpu, unsigned long target_mode,
u64 val)
{
@@ -101,14 +85,14 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode,
switch (target_mode) {
case PSR_MODE_EL1h:
vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL1);
sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
__vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL1);
vbar = vcpu_read_sys_reg(vcpu, VBAR_EL1);
sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1);
vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL1);
break;
case PSR_MODE_EL2h:
vbar = __vcpu_read_sys_reg(vcpu, VBAR_EL2);
sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL2);
__vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL2);
vbar = vcpu_read_sys_reg(vcpu, VBAR_EL2);
sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL2);
vcpu_write_sys_reg(vcpu, *vcpu_pc(vcpu), ELR_EL2);
break;
default:
/* Don't do that */
@@ -185,7 +169,7 @@ static void enter_exception64(struct kvm_vcpu *vcpu, unsigned long target_mode,
*/
static unsigned long get_except32_cpsr(struct kvm_vcpu *vcpu, u32 mode)
{
u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
u32 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1);
unsigned long old, new;
old = *vcpu_cpsr(vcpu);
@@ -281,7 +265,7 @@ static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset)
{
unsigned long spsr = *vcpu_cpsr(vcpu);
bool is_thumb = (spsr & PSR_AA32_T_BIT);
u32 sctlr = __vcpu_read_sys_reg(vcpu, SCTLR_EL1);
u32 sctlr = vcpu_read_sys_reg(vcpu, SCTLR_EL1);
u32 return_address;
*vcpu_cpsr(vcpu) = get_except32_cpsr(vcpu, mode);
@@ -305,7 +289,7 @@ static void enter_exception32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset)
if (sctlr & (1 << 13))
vect_offset += 0xffff0000;
else /* always have security exceptions */
vect_offset += __vcpu_read_sys_reg(vcpu, VBAR_EL1);
vect_offset += vcpu_read_sys_reg(vcpu, VBAR_EL1);
*vcpu_pc(vcpu) = vect_offset;
}
arch/arm64/kvm/hyp/include/nvhe/clock.h
+4 -4
View File
@@ -6,11 +6,11 @@
#include <asm/kvm_hyp.h>
#ifdef CONFIG_NVHE_EL2_TRACING
void trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc);
u64 trace_clock(void);
void trace_hyp_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc);
u64 trace_hyp_clock(void);
#else
static inline void
trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) { }
static inline u64 trace_clock(void) { return 0; }
trace_hyp_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) { }
static inline u64 trace_hyp_clock(void) { return 0; }
#endif
#endif
arch/arm64/kvm/hyp/nvhe/clock.c
+2 -2
View File
@@ -30,7 +30,7 @@ static u64 __clock_mult_uint128(u64 cyc, u32 mult, u32 shift)
}
/* Does not guarantee no reader on the modified bank. */
void trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc)
void trace_hyp_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc)
{
struct clock_data *clock = &trace_clock_data;
u64 bank = clock->cur ^ 1;
@@ -48,7 +48,7 @@ void trace_clock_update(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc)
}
/* Use untrusted host data */
u64 trace_clock(void)
u64 trace_hyp_clock(void)
{
struct clock_data *clock = &trace_clock_data;
u64 bank = smp_load_acquire(&clock->cur);
arch/arm64/kvm/hyp/nvhe/hyp-main.c
+233 -38
View File
@@ -7,6 +7,8 @@
#include <hyp/adjust_pc.h>
#include <hyp/switch.h>
#include <linux/irqchip/arm-gic-v3.h>
#include <asm/pgtable-types.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_emulate.h>
@@ -102,16 +104,103 @@ static void fpsimd_sve_sync(struct kvm_vcpu *vcpu)
*host_data_ptr(fp_owner) = FP_STATE_HOST_OWNED;
}
static void flush_hyp_vgic_state(struct pkvm_hyp_vcpu *hyp_vcpu)
{
struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu;
struct vgic_v3_cpu_if *host_cpu_if, *hyp_cpu_if;
unsigned int used_lrs, i;
host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3;
hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3;
used_lrs = host_cpu_if->used_lrs;
used_lrs = min(used_lrs, hyp_gicv3_nr_lr);
hyp_cpu_if->vgic_hcr = host_cpu_if->vgic_hcr;
/* Should be a one-off */
hyp_cpu_if->vgic_sre = (ICC_SRE_EL1_DIB |
ICC_SRE_EL1_DFB |
ICC_SRE_EL1_SRE);
hyp_cpu_if->used_lrs = used_lrs;
for (i = 0; i < used_lrs; i++)
hyp_cpu_if->vgic_lr[i] = host_cpu_if->vgic_lr[i];
}
static void sync_hyp_vgic_state(struct pkvm_hyp_vcpu *hyp_vcpu)
{
struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu;
struct vgic_v3_cpu_if *host_cpu_if, *hyp_cpu_if;
unsigned int i;
host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3;
hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3;
host_cpu_if->vgic_hcr = hyp_cpu_if->vgic_hcr;
host_cpu_if->vgic_vmcr = hyp_cpu_if->vgic_vmcr;
for (i = 0; i < hyp_cpu_if->used_lrs; i++)
host_cpu_if->vgic_lr[i] = hyp_cpu_if->vgic_lr[i];
}
static void __copy_vcpu_state(const struct kvm_vcpu *from_vcpu,
struct kvm_vcpu *to_vcpu)
{
int i;
to_vcpu->arch.ctxt.regs = from_vcpu->arch.ctxt.regs;
to_vcpu->arch.ctxt.spsr_abt = from_vcpu->arch.ctxt.spsr_abt;
to_vcpu->arch.ctxt.spsr_und = from_vcpu->arch.ctxt.spsr_und;
to_vcpu->arch.ctxt.spsr_irq = from_vcpu->arch.ctxt.spsr_irq;
to_vcpu->arch.ctxt.spsr_fiq = from_vcpu->arch.ctxt.spsr_fiq;
to_vcpu->arch.ctxt.fp_regs = from_vcpu->arch.ctxt.fp_regs;
/*
* Copy the sysregs, but don't mess with the timer state which
* is directly handled by EL1 and is expected to be preserved.
* enum vcpu_sysreg is sparse: VNCR-mapped registers take values
* derived from their VNCR page offset, so the timer registers do
* not form a contiguous numeric range and must be skipped by name.
*/
for (i = 1; i < NR_SYS_REGS; i++) {
switch (i) {
case CNTVOFF_EL2:
case CNTV_CVAL_EL0:
case CNTV_CTL_EL0:
case CNTP_CVAL_EL0:
case CNTP_CTL_EL0:
continue;
}
to_vcpu->arch.ctxt.sys_regs[i] = from_vcpu->arch.ctxt.sys_regs[i];
}
}
static void sync_hyp_vcpu_state(struct pkvm_hyp_vcpu *hyp_vcpu)
{
__copy_vcpu_state(&hyp_vcpu->vcpu, hyp_vcpu->host_vcpu);
}
static void flush_hyp_vcpu_state(struct pkvm_hyp_vcpu *hyp_vcpu)
{
__copy_vcpu_state(hyp_vcpu->host_vcpu, &hyp_vcpu->vcpu);
}
static void flush_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu)
{
struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu;
hyp_vcpu->vcpu.arch.debug_owner = host_vcpu->arch.debug_owner;
if (kvm_guest_owns_debug_regs(&hyp_vcpu->vcpu))
if (kvm_guest_owns_debug_regs(&hyp_vcpu->vcpu)) {
hyp_vcpu->vcpu.arch.vcpu_debug_state = host_vcpu->arch.vcpu_debug_state;
else if (kvm_host_owns_debug_regs(&hyp_vcpu->vcpu))
} else if (kvm_host_owns_debug_regs(&hyp_vcpu->vcpu)) {
hyp_vcpu->vcpu.arch.external_debug_state = host_vcpu->arch.external_debug_state;
/*
* The world switch loads MDSCR_EL1 from external_mdscr_el1
* (ctxt_mdscr_el1()).
*/
hyp_vcpu->vcpu.arch.external_mdscr_el1 = host_vcpu->arch.external_mdscr_el1;
}
}
static void sync_debug_state(struct pkvm_hyp_vcpu *hyp_vcpu)
@@ -131,7 +220,17 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
fpsimd_sve_flush();
flush_debug_state(hyp_vcpu);
hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt;
/*
* If we deal with a non-protected guest and the state is potentially
* dirty (from a host perspective), copy the state back into the hyp
* vcpu.
*/
if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu)) {
if (vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY))
flush_hyp_vcpu_state(hyp_vcpu);
} else {
hyp_vcpu->vcpu.arch.ctxt = host_vcpu->arch.ctxt;
}
/* __hyp_running_vcpu must be NULL in a guest context. */
hyp_vcpu->vcpu.arch.ctxt.__hyp_running_vcpu = NULL;
@@ -150,13 +249,7 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
hyp_vcpu->vcpu.arch.vsesr_el2 = host_vcpu->arch.vsesr_el2;
hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3 = host_vcpu->arch.vgic_cpu.vgic_v3;
/* Bound used_lrs by the number of implemented list registers. */
hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3.used_lrs =
min_t(unsigned int,
hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3.used_lrs,
hyp_gicv3_nr_lr);
flush_hyp_vgic_state(hyp_vcpu);
hyp_vcpu->vcpu.arch.pid = host_vcpu->arch.pid;
}
@@ -164,25 +257,26 @@ static void flush_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
static void sync_hyp_vcpu(struct pkvm_hyp_vcpu *hyp_vcpu)
{
struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu;
struct vgic_v3_cpu_if *hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3;
struct vgic_v3_cpu_if *host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3;
unsigned int i;
fpsimd_sve_sync(&hyp_vcpu->vcpu);
sync_debug_state(hyp_vcpu);
host_vcpu->arch.ctxt = hyp_vcpu->vcpu.arch.ctxt;
host_vcpu->arch.hcr_el2 = hyp_vcpu->vcpu.arch.hcr_el2;
if (pkvm_hyp_vcpu_is_protected(hyp_vcpu)) {
host_vcpu->arch.ctxt = hyp_vcpu->vcpu.arch.ctxt;
} else {
/*
* PC feeds trace_kvm_exit(), PSTATE.SS the host software-step
* machine, and both run before the next on-demand ctxt sync.
*/
host_vcpu->arch.ctxt.regs.pc = hyp_vcpu->vcpu.arch.ctxt.regs.pc;
host_vcpu->arch.ctxt.regs.pstate = hyp_vcpu->vcpu.arch.ctxt.regs.pstate;
}
host_vcpu->arch.fault = hyp_vcpu->vcpu.arch.fault;
host_vcpu->arch.iflags = hyp_vcpu->vcpu.arch.iflags;
host_cpu_if->vgic_hcr = hyp_cpu_if->vgic_hcr;
host_cpu_if->vgic_vmcr = hyp_cpu_if->vgic_vmcr;
for (i = 0; i < hyp_cpu_if->used_lrs; ++i)
host_cpu_if->vgic_lr[i] = hyp_cpu_if->vgic_lr[i];
sync_hyp_vgic_state(hyp_vcpu);
}
static void handle___pkvm_vcpu_load(struct kvm_cpu_context *host_ctxt)
@@ -210,18 +304,78 @@ static void handle___pkvm_vcpu_put(struct kvm_cpu_context *host_ctxt)
{
struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu();
if (hyp_vcpu)
if (hyp_vcpu) {
struct kvm_vcpu *host_vcpu = hyp_vcpu->host_vcpu;
if (!pkvm_hyp_vcpu_is_protected(hyp_vcpu) &&
!vcpu_get_flag(host_vcpu, PKVM_HOST_STATE_DIRTY)) {
sync_hyp_vcpu_state(hyp_vcpu);
}
pkvm_put_hyp_vcpu(hyp_vcpu);
}
}
static void handle___pkvm_vcpu_sync_state(struct kvm_cpu_context *host_ctxt)
{
struct pkvm_hyp_vcpu *hyp_vcpu;
hyp_vcpu = pkvm_get_loaded_hyp_vcpu();
if (!hyp_vcpu || pkvm_hyp_vcpu_is_protected(hyp_vcpu))
return;
sync_hyp_vcpu_state(hyp_vcpu);
}
static struct kvm_vcpu *__get_host_hyp_vcpus(struct kvm_vcpu *arg,
struct pkvm_hyp_vcpu **hyp_vcpup)
{
struct kvm_vcpu *host_vcpu = kern_hyp_va(arg);
struct pkvm_hyp_vcpu *hyp_vcpu = NULL;
if (unlikely(is_protected_kvm_enabled())) {
hyp_vcpu = pkvm_get_loaded_hyp_vcpu();
if (!hyp_vcpu || hyp_vcpu->host_vcpu != host_vcpu) {
hyp_vcpu = NULL;
host_vcpu = NULL;
}
}
*hyp_vcpup = hyp_vcpu;
return host_vcpu;
}
#define get_host_hyp_vcpus(ctxt, regnr, hyp_vcpup) \
({ \
DECLARE_REG(struct kvm_vcpu *, __vcpu, ctxt, regnr); \
__get_host_hyp_vcpus(__vcpu, hyp_vcpup); \
})
#define get_host_hyp_vcpus_from_vgic_v3_cpu_if(ctxt, regnr, hyp_vcpup) \
({ \
DECLARE_REG(struct vgic_v3_cpu_if *, cif, ctxt, regnr);\
struct kvm_vcpu *__vcpu = container_of(cif, \
struct kvm_vcpu, \
arch.vgic_cpu.vgic_v3); \
\
__get_host_hyp_vcpus(__vcpu, hyp_vcpup); \
})
static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt)
{
DECLARE_REG(struct kvm_vcpu *, host_vcpu, host_ctxt, 1);
struct pkvm_hyp_vcpu *hyp_vcpu;
struct kvm_vcpu *host_vcpu;
int ret;
if (unlikely(is_protected_kvm_enabled())) {
struct pkvm_hyp_vcpu *hyp_vcpu = pkvm_get_loaded_hyp_vcpu();
host_vcpu = get_host_hyp_vcpus(host_ctxt, 1, &hyp_vcpu);
if (!host_vcpu) {
ret = -EINVAL;
goto out;
}
if (unlikely(hyp_vcpu)) {
/*
* KVM (and pKVM) doesn't support SME guests for now, and
* ensures that SME features aren't enabled in pstate when
@@ -233,23 +387,16 @@ static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt)
goto out;
}
if (!hyp_vcpu) {
ret = -EINVAL;
goto out;
}
flush_hyp_vcpu(hyp_vcpu);
ret = __kvm_vcpu_run(&hyp_vcpu->vcpu);
sync_hyp_vcpu(hyp_vcpu);
} else {
struct kvm_vcpu *vcpu = kern_hyp_va(host_vcpu);
/* The host is fully trusted, run its vCPU directly. */
fpsimd_lazy_switch_to_guest(vcpu);
ret = __kvm_vcpu_run(vcpu);
fpsimd_lazy_switch_to_host(vcpu);
fpsimd_lazy_switch_to_guest(host_vcpu);
ret = __kvm_vcpu_run(host_vcpu);
fpsimd_lazy_switch_to_host(host_vcpu);
}
out:
cpu_reg(host_ctxt, 1) = ret;
@@ -484,16 +631,63 @@ static void handle___vgic_v3_init_lrs(struct kvm_cpu_context *host_ctxt)
static void handle___vgic_v3_save_aprs(struct kvm_cpu_context *host_ctxt)
{
DECLARE_REG(struct vgic_v3_cpu_if *, cpu_if, host_ctxt, 1);
struct pkvm_hyp_vcpu *hyp_vcpu;
struct kvm_vcpu *host_vcpu;
__vgic_v3_save_aprs(kern_hyp_va(cpu_if));
host_vcpu = get_host_hyp_vcpus_from_vgic_v3_cpu_if(host_ctxt, 1,
&hyp_vcpu);
if (!host_vcpu)
return;
if (unlikely(hyp_vcpu)) {
struct vgic_v3_cpu_if *hyp_cpu_if, *host_cpu_if;
int i;
hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3;
__vgic_v3_save_aprs(hyp_cpu_if);
host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3;
host_cpu_if->vgic_vmcr = hyp_cpu_if->vgic_vmcr;
for (i = 0; i < ARRAY_SIZE(host_cpu_if->vgic_ap0r); i++) {
host_cpu_if->vgic_ap0r[i] = hyp_cpu_if->vgic_ap0r[i];
host_cpu_if->vgic_ap1r[i] = hyp_cpu_if->vgic_ap1r[i];
}
} else {
__vgic_v3_save_aprs(&host_vcpu->arch.vgic_cpu.vgic_v3);
}
}
static void handle___vgic_v3_restore_vmcr_aprs(struct kvm_cpu_context *host_ctxt)
{
DECLARE_REG(struct vgic_v3_cpu_if *, cpu_if, host_ctxt, 1);
struct pkvm_hyp_vcpu *hyp_vcpu;
struct kvm_vcpu *host_vcpu;
__vgic_v3_restore_vmcr_aprs(kern_hyp_va(cpu_if));
host_vcpu = get_host_hyp_vcpus_from_vgic_v3_cpu_if(host_ctxt, 1,
&hyp_vcpu);
if (!host_vcpu)
return;
if (unlikely(hyp_vcpu)) {
struct vgic_v3_cpu_if *hyp_cpu_if, *host_cpu_if;
int i;
hyp_cpu_if = &hyp_vcpu->vcpu.arch.vgic_cpu.vgic_v3;
host_cpu_if = &host_vcpu->arch.vgic_cpu.vgic_v3;
hyp_cpu_if->vgic_vmcr = host_cpu_if->vgic_vmcr;
/* Should be a one-off */
hyp_cpu_if->vgic_sre = (ICC_SRE_EL1_DIB |
ICC_SRE_EL1_DFB |
ICC_SRE_EL1_SRE);
for (i = 0; i < ARRAY_SIZE(host_cpu_if->vgic_ap0r); i++) {
hyp_cpu_if->vgic_ap0r[i] = host_cpu_if->vgic_ap0r[i];
hyp_cpu_if->vgic_ap1r[i] = host_cpu_if->vgic_ap1r[i];
}
__vgic_v3_restore_vmcr_aprs(hyp_cpu_if);
} else {
__vgic_v3_restore_vmcr_aprs(&host_vcpu->arch.vgic_cpu.vgic_v3);
}
}
static void handle___pkvm_init(struct kvm_cpu_context *host_ctxt)
@@ -761,6 +955,7 @@ static const hcall_t host_hcall[] = {
HANDLE_FUNC(__pkvm_finalize_teardown_vm),
HANDLE_FUNC(__pkvm_vcpu_load),
HANDLE_FUNC(__pkvm_vcpu_put),
HANDLE_FUNC(__pkvm_vcpu_sync_state),
HANDLE_FUNC(__pkvm_tlb_flush_vmid),
};
arch/arm64/kvm/hyp/nvhe/mem_protect.c
+7
View File
@@ -261,11 +261,18 @@ static void __apply_guest_page(void *va, size_t size,
static void clean_dcache_guest_page(void *va, size_t size)
{
/* See comment in __clean_dcache_guest_page() */
if (cpus_have_final_cap(ARM64_HAS_STAGE2_FWB))
return;
__apply_guest_page(va, size, __clean_dcache_guest_page);
}
static void invalidate_icache_guest_page(void *va, size_t size)
{
if (alternative_has_cap_unlikely(ARM64_HAS_CACHE_DIC))
return;
__apply_guest_page(va, size, __invalidate_icache_guest_page);
}
arch/arm64/kvm/hyp/nvhe/pkvm.c
-1
View File
@@ -433,7 +433,6 @@ static void init_pkvm_hyp_vm(struct kvm *host_kvm, struct pkvm_hyp_vm *hyp_vm,
hyp_vm->host_kvm = host_kvm;
hyp_vm->kvm.created_vcpus = nr_vcpus;
hyp_vm->kvm.arch.pkvm.is_protected = READ_ONCE(host_kvm->arch.pkvm.is_protected);
hyp_vm->kvm.arch.pkvm.is_created = true;
hyp_vm->kvm.arch.flags = 0;
pkvm_init_features_from_host(hyp_vm, host_kvm);
arch/arm64/kvm/hyp/nvhe/sys_regs.c
+3 -2
View File
@@ -257,10 +257,11 @@ static void inject_sync64(struct kvm_vcpu *vcpu, u64 esr)
*vcpu_cpsr(vcpu) = read_sysreg_el2(SYS_SPSR);
/*
* Make sure we have the latest update to VBAR_EL1, as pKVM
* handles traps very early, before sysregs are resync'ed
* Sync VBAR_EL1 and SCTLR_EL1, both read by enter_exception64(),
* as pKVM handles traps before sysregs are resync'ed.
*/
__vcpu_assign_sys_reg(vcpu, VBAR_EL1, read_sysreg_el1(SYS_VBAR));
__vcpu_assign_sys_reg(vcpu, SCTLR_EL1, read_sysreg_el1(SYS_SCTLR));
kvm_pend_exception(vcpu, EXCEPT_AA64_EL1_SYNC);
arch/arm64/kvm/hyp/nvhe/trace.c
+2 -2
View File
@@ -35,7 +35,7 @@ static bool hyp_trace_buffer_loaded(struct hyp_trace_buffer *trace_buffer)
void *tracing_reserve_entry(unsigned long length)
{
return simple_ring_buffer_reserve(this_cpu_ptr(trace_buffer.simple_rbs), length,
trace_clock());
trace_hyp_clock());
}
void tracing_commit_entry(void)
@@ -290,7 +290,7 @@ void __tracing_update_clock(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc)
}
/* ...we can now override the old one and swap. */
trace_clock_update(mult, shift, epoch_ns, epoch_cyc);
trace_hyp_clock_update(mult, shift, epoch_ns, epoch_cyc);
}
int __tracing_reset(unsigned int cpu)

Some files were not shown because too many files have changed in this diff Show More