mirror of
https://github.com/linux-msm/laptops-kernel.git
synced 2026-08-13 14:19:53 -07:00
Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux
Pull arm64 updates from Catalin Marinas:
"The biggest changes are MPAM enablement in drivers/resctrl and new PMU
support under drivers/perf.
On the core side, FEAT_LSUI lets futex atomic operations with EL0
permissions, avoiding PAN toggling.
The rest is mostly TLB invalidation refactoring, further generic entry
work, sysreg updates and a few fixes.
Core features:
- Add support for FEAT_LSUI, allowing futex atomic operations without
toggling Privileged Access Never (PAN)
- Further refactor the arm64 exception handling code towards the
generic entry infrastructure
- Optimise __READ_ONCE() with CONFIG_LTO=y and allow alias analysis
through it
Memory management:
- Refactor the arm64 TLB invalidation API and implementation for
better control over barrier placement and level-hinted invalidation
- Enable batched TLB flushes during memory hot-unplug
- Fix rodata=full block mapping support for realm guests (when
BBML2_NOABORT is available)
Perf and PMU:
- Add support for a whole bunch of system PMUs featured in NVIDIA's
Tegra410 SoC (cspmu extensions for the fabric and PCIe, new drivers
for CPU/C2C memory latency PMUs)
- Clean up iomem resource handling in the Arm CMN driver
- Fix signedness handling of AA64DFR0.{PMUVer,PerfMon}
MPAM (Memory Partitioning And Monitoring):
- Add architecture context-switch and hiding of the feature from KVM
- Add interface to allow MPAM to be exposed to user-space using
resctrl
- Add errata workaround for some existing platforms
- Add documentation for using MPAM and what shape of platforms can
use resctrl
Miscellaneous:
- Check DAIF (and PMR, where relevant) at task-switch time
- Skip TFSR_EL1 checks and barriers in synchronous MTE tag check mode
(only relevant to asynchronous or asymmetric tag check modes)
- Remove a duplicate allocation in the kexec code
- Remove redundant save/restore of SCS SP on entry to/from EL0
- Generate the KERNEL_HWCAP_ definitions from the arm64 hwcap
descriptions
- Add kselftest coverage for cmpbr_sigill()
- Update sysreg definitions"
* tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (109 commits)
arm64: rsi: use linear-map alias for realm config buffer
arm64: Kconfig: fix duplicate word in CMDLINE help text
arm64: mte: Skip TFSR_EL1 checks and barriers in synchronous tag check mode
arm64/sysreg: Update ID_AA64SMFR0_EL1 description to DDI0601 2025-12
arm64/sysreg: Update ID_AA64ZFR0_EL1 description to DDI0601 2025-12
arm64/sysreg: Update ID_AA64FPFR0_EL1 description to DDI0601 2025-12
arm64/sysreg: Update ID_AA64ISAR2_EL1 description to DDI0601 2025-12
arm64/sysreg: Update ID_AA64ISAR0_EL1 description to DDI0601 2025-12
arm64/hwcap: Generate the KERNEL_HWCAP_ definitions for the hwcaps
arm64: kexec: Remove duplicate allocation for trans_pgd
ACPI: AGDI: fix missing newline in error message
arm64: Check DAIF (and PMR) at task-switch time
arm64: entry: Use split preemption logic
arm64: entry: Use irqentry_{enter_from,exit_to}_kernel_mode()
arm64: entry: Consistently prefix arm64-specific wrappers
arm64: entry: Don't preempt with SError or Debug masked
entry: Split preemption from irqentry_exit_to_kernel_mode()
entry: Split kernel mode logic from irqentry_{enter,exit}()
entry: Move irqentry_enter() prototype later
entry: Remove local_irq_{enable,disable}_exit_to_user()
...
This commit is contained in:
Linus Torvalds
@@ -24,7 +24,8 @@ Performance monitor support
|
||||
thunderx2-pmu
|
||||
alibaba_pmu
|
||||
dwc_pcie_pmu
|
||||
nvidia-pmu
|
||||
nvidia-tegra241-pmu
|
||||
nvidia-tegra410-pmu
|
||||
meson-ddr-pmu
|
||||
cxl
|
||||
ampere_cspmu
|
||||
|
||||
+4
-4
@@ -1,8 +1,8 @@
|
||||
=========================================================
|
||||
NVIDIA Tegra SoC Uncore Performance Monitoring Unit (PMU)
|
||||
=========================================================
|
||||
============================================================
|
||||
NVIDIA Tegra241 SoC Uncore Performance Monitoring Unit (PMU)
|
||||
============================================================
|
||||
|
||||
The NVIDIA Tegra SoC includes various system PMUs to measure key performance
|
||||
The NVIDIA Tegra241 SoC includes various system PMUs to measure key performance
|
||||
metrics like memory bandwidth, latency, and utilization:
|
||||
|
||||
* Scalable Coherency Fabric (SCF)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -23,6 +23,7 @@ ARM64 Architecture
|
||||
memory
|
||||
memory-tagging-extension
|
||||
mops
|
||||
mpam
|
||||
perf
|
||||
pointer-authentication
|
||||
ptdump
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
.. SPDX-License-Identifier: GPL-2.0
|
||||
|
||||
====
|
||||
MPAM
|
||||
====
|
||||
|
||||
What is MPAM
|
||||
============
|
||||
MPAM (Memory Partitioning and Monitoring) is a feature in the CPUs and memory
|
||||
system components such as the caches or memory controllers that allow memory
|
||||
traffic to be labelled, partitioned and monitored.
|
||||
|
||||
Traffic is labelled by the CPU, based on the control or monitor group the
|
||||
current task is assigned to using resctrl. Partitioning policy can be set
|
||||
using the schemata file in resctrl, and monitor values read via resctrl.
|
||||
See Documentation/filesystems/resctrl.rst for more details.
|
||||
|
||||
This allows tasks that share memory system resources, such as caches, to be
|
||||
isolated from each other according to the partitioning policy (so called noisy
|
||||
neighbours).
|
||||
|
||||
Supported Platforms
|
||||
===================
|
||||
Use of this feature requires CPU support, support in the memory system
|
||||
components, and a description from firmware of where the MPAM device controls
|
||||
are in the MMIO address space. (e.g. the 'MPAM' ACPI table).
|
||||
|
||||
The MMIO device that provides MPAM controls/monitors for a memory system
|
||||
component is called a memory system component. (MSC).
|
||||
|
||||
Because the user interface to MPAM is via resctrl, only MPAM features that are
|
||||
compatible with resctrl can be exposed to user-space.
|
||||
|
||||
MSC are considered as a group based on the topology. MSC that correspond with
|
||||
the L3 cache are considered together, it is not possible to mix MSC between L2
|
||||
and L3 to 'cover' a resctrl schema.
|
||||
|
||||
The supported features are:
|
||||
|
||||
* Cache portion bitmap controls (CPOR) on the L2 or L3 caches. To expose
|
||||
CPOR at L2 or L3, every CPU must have a corresponding CPU cache at this
|
||||
level that also supports the feature. Mismatched big/little platforms are
|
||||
not supported as resctrl's controls would then also depend on task
|
||||
placement.
|
||||
|
||||
* Memory bandwidth maximum controls (MBW_MAX) on or after the L3 cache.
|
||||
resctrl uses the L3 cache-id to identify where the memory bandwidth
|
||||
control is applied. For this reason the platform must have an L3 cache
|
||||
with cache-id's supplied by firmware. (It doesn't need to support MPAM.)
|
||||
|
||||
To be exported as the 'MB' schema, the topology of the group of MSC chosen
|
||||
must match the topology of the L3 cache so that the cache-id's can be
|
||||
repainted. For example: Platforms with Memory bandwidth maximum controls
|
||||
on CPU-less NUMA nodes cannot expose the 'MB' schema to resctrl as these
|
||||
nodes do not have a corresponding L3 cache. If the memory bandwidth
|
||||
control is on the memory rather than the L3 then there must be a single
|
||||
global L3 as otherwise it is unknown which L3 the traffic came from. There
|
||||
must be no caches between the L3 and the memory so that the two ends of
|
||||
the path have equivalent traffic.
|
||||
|
||||
When the MPAM driver finds multiple groups of MSC it can use for the 'MB'
|
||||
schema, it prefers the group closest to the L3 cache.
|
||||
|
||||
* Cache Storage Usage (CSU) counters can expose the 'llc_occupancy' provided
|
||||
there is at least one CSU monitor on each MSC that makes up the L3 group.
|
||||
Exposing CSU counters from other caches or devices is not supported.
|
||||
|
||||
Reporting Bugs
|
||||
==============
|
||||
If you are not seeing the counters or controls you expect please share the
|
||||
debug messages produced when enabling dynamic debug and booting with:
|
||||
dyndbg="file mpam_resctrl.c +pl"
|
||||
@@ -214,6 +214,9 @@ stable kernels.
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| ARM | SI L1 | #4311569 | ARM64_ERRATUM_4311569 |
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| ARM | CMN-650 | #3642720 | N/A |
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| Broadcom | Brahma-B53 | N/A | ARM64_ERRATUM_845719 |
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| Broadcom | Brahma-B53 | N/A | ARM64_ERRATUM_843419 |
|
||||
@@ -247,6 +250,12 @@ stable kernels.
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| NVIDIA | T241 GICv3/4.x | T241-FABRIC-4 | N/A |
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| NVIDIA | T241 MPAM | T241-MPAM-1 | N/A |
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| NVIDIA | T241 MPAM | T241-MPAM-4 | N/A |
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| NVIDIA | T241 MPAM | T241-MPAM-6 | N/A |
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| Freescale/NXP | LS2080A/LS1043A | A-008585 | FSL_ERRATUM_A008585 |
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
|
||||
@@ -238,6 +238,13 @@ static inline void kvm_vcpu_pmu_resync_el0(void) {}
|
||||
|
||||
static inline bool pmuv3_implemented(int pmuver)
|
||||
{
|
||||
/*
|
||||
* PMUVer follows the standard ID scheme for an unsigned field with the
|
||||
* exception of 0xF (IMP_DEF) which is treated specially and implies
|
||||
* FEAT_PMUv3 is not implemented.
|
||||
*
|
||||
* See DDI0487L.a D24.1.3.2 for more details.
|
||||
*/
|
||||
return !(pmuver == ARMV8_PMU_DFR_VER_IMP_DEF ||
|
||||
pmuver == ARMV8_PMU_DFR_VER_NI);
|
||||
}
|
||||
|
||||
+25
-29
@@ -61,32 +61,6 @@ config ARM64
|
||||
select ARCH_HAVE_ELF_PROT
|
||||
select ARCH_HAVE_NMI_SAFE_CMPXCHG
|
||||
select ARCH_HAVE_TRACE_MMIO_ACCESS
|
||||
select ARCH_INLINE_READ_LOCK if !PREEMPTION
|
||||
select ARCH_INLINE_READ_LOCK_BH if !PREEMPTION
|
||||
select ARCH_INLINE_READ_LOCK_IRQ if !PREEMPTION
|
||||
select ARCH_INLINE_READ_LOCK_IRQSAVE if !PREEMPTION
|
||||
select ARCH_INLINE_READ_UNLOCK if !PREEMPTION
|
||||
select ARCH_INLINE_READ_UNLOCK_BH if !PREEMPTION
|
||||
select ARCH_INLINE_READ_UNLOCK_IRQ if !PREEMPTION
|
||||
select ARCH_INLINE_READ_UNLOCK_IRQRESTORE if !PREEMPTION
|
||||
select ARCH_INLINE_WRITE_LOCK if !PREEMPTION
|
||||
select ARCH_INLINE_WRITE_LOCK_BH if !PREEMPTION
|
||||
select ARCH_INLINE_WRITE_LOCK_IRQ if !PREEMPTION
|
||||
select ARCH_INLINE_WRITE_LOCK_IRQSAVE if !PREEMPTION
|
||||
select ARCH_INLINE_WRITE_UNLOCK if !PREEMPTION
|
||||
select ARCH_INLINE_WRITE_UNLOCK_BH if !PREEMPTION
|
||||
select ARCH_INLINE_WRITE_UNLOCK_IRQ if !PREEMPTION
|
||||
select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_TRYLOCK if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_TRYLOCK_BH if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_LOCK if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_LOCK_BH if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_LOCK_IRQ if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_LOCK_IRQSAVE if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_UNLOCK if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_UNLOCK_BH if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_UNLOCK_IRQ if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE if !PREEMPTION
|
||||
select ARCH_KEEP_MEMBLOCK
|
||||
select ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE
|
||||
select ARCH_USE_CMPXCHG_LOCKREF
|
||||
@@ -2009,8 +1983,8 @@ config ARM64_TLB_RANGE
|
||||
|
||||
config ARM64_MPAM
|
||||
bool "Enable support for MPAM"
|
||||
select ARM64_MPAM_DRIVER if EXPERT # does nothing yet
|
||||
select ACPI_MPAM if ACPI
|
||||
select ARM64_MPAM_DRIVER
|
||||
select ARCH_HAS_CPU_RESCTRL
|
||||
help
|
||||
Memory System Resource Partitioning and Monitoring (MPAM) is an
|
||||
optional extension to the Arm architecture that allows each
|
||||
@@ -2032,6 +2006,8 @@ config ARM64_MPAM
|
||||
|
||||
MPAM is exposed to user-space via the resctrl pseudo filesystem.
|
||||
|
||||
This option enables the extra context switch code.
|
||||
|
||||
endmenu # "ARMv8.4 architectural features"
|
||||
|
||||
menu "ARMv8.5 architectural features"
|
||||
@@ -2208,6 +2184,26 @@ config ARM64_GCS
|
||||
|
||||
endmenu # "ARMv9.4 architectural features"
|
||||
|
||||
config AS_HAS_LSUI
|
||||
def_bool $(as-instr,.arch_extension lsui)
|
||||
help
|
||||
Supported by LLVM 20+ and binutils 2.45+.
|
||||
|
||||
menu "ARMv9.6 architectural features"
|
||||
|
||||
config ARM64_LSUI
|
||||
bool "Support Unprivileged Load Store Instructions (LSUI)"
|
||||
default y
|
||||
depends on AS_HAS_LSUI && !CPU_BIG_ENDIAN
|
||||
help
|
||||
The Unprivileged Load Store Instructions (LSUI) provides
|
||||
variants load/store instructions that access user-space memory
|
||||
from the kernel without clearing PSTATE.PAN bit.
|
||||
|
||||
This feature is supported by LLVM 20+ and binutils 2.45+.
|
||||
|
||||
endmenu # "ARMv9.6 architectural feature"
|
||||
|
||||
config ARM64_SVE
|
||||
bool "ARM Scalable Vector Extension support"
|
||||
default y
|
||||
@@ -2365,7 +2361,7 @@ config CMDLINE
|
||||
default ""
|
||||
help
|
||||
Provide a set of default command-line options at build time by
|
||||
entering them here. As a minimum, you should specify the the
|
||||
entering them here. As a minimum, you should specify the
|
||||
root device (e.g. root=/dev/nfs).
|
||||
|
||||
choice
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
#ifdef CONFIG_ARM64_SW_TTBR0_PAN
|
||||
.macro __uaccess_ttbr0_disable, tmp1
|
||||
mrs \tmp1, ttbr1_el1 // swapper_pg_dir
|
||||
bic \tmp1, \tmp1, #TTBR_ASID_MASK
|
||||
bic \tmp1, \tmp1, #TTBRx_EL1_ASID_MASK
|
||||
sub \tmp1, \tmp1, #RESERVED_SWAPPER_OFFSET // reserved_pg_dir
|
||||
msr ttbr0_el1, \tmp1 // set reserved TTBR0_EL1
|
||||
add \tmp1, \tmp1, #RESERVED_SWAPPER_OFFSET
|
||||
|
||||
@@ -71,6 +71,8 @@ cpucap_is_possible(const unsigned int cap)
|
||||
return true;
|
||||
case ARM64_HAS_PMUV3:
|
||||
return IS_ENABLED(CONFIG_HW_PERF_EVENTS);
|
||||
case ARM64_HAS_LSUI:
|
||||
return IS_ENABLED(CONFIG_ARM64_LSUI);
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
@@ -513,7 +513,8 @@
|
||||
check_override id_aa64pfr0, ID_AA64PFR0_EL1_MPAM_SHIFT, .Linit_mpam_\@, .Lskip_mpam_\@, x1, x2
|
||||
|
||||
.Linit_mpam_\@:
|
||||
msr_s SYS_MPAM2_EL2, xzr // use the default partition
|
||||
mov x0, #MPAM2_EL2_EnMPAMSM_MASK
|
||||
msr_s SYS_MPAM2_EL2, x0 // use the default partition,
|
||||
// and disable lower traps
|
||||
mrs_s x0, SYS_MPAMIDR_EL1
|
||||
tbz x0, #MPAMIDR_EL1_HAS_HCR_SHIFT, .Lskip_mpam_\@ // skip if no MPAMHCR reg
|
||||
|
||||
+253
-58
@@ -9,71 +9,292 @@
|
||||
#include <linux/uaccess.h>
|
||||
|
||||
#include <asm/errno.h>
|
||||
#include <asm/lsui.h>
|
||||
|
||||
#define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of? */
|
||||
|
||||
#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg) \
|
||||
do { \
|
||||
#define LLSC_FUTEX_ATOMIC_OP(op, insn) \
|
||||
static __always_inline int \
|
||||
__llsc_futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \
|
||||
{ \
|
||||
unsigned int loops = FUTEX_MAX_LOOPS; \
|
||||
int ret, oldval, newval; \
|
||||
\
|
||||
uaccess_enable_privileged(); \
|
||||
asm volatile( \
|
||||
" prfm pstl1strm, %2\n" \
|
||||
"1: ldxr %w1, %2\n" \
|
||||
asm volatile("// __llsc_futex_atomic_" #op "\n" \
|
||||
" prfm pstl1strm, %[uaddr]\n" \
|
||||
"1: ldxr %w[oldval], %[uaddr]\n" \
|
||||
insn "\n" \
|
||||
"2: stlxr %w0, %w3, %2\n" \
|
||||
" cbz %w0, 3f\n" \
|
||||
" sub %w4, %w4, %w0\n" \
|
||||
" cbnz %w4, 1b\n" \
|
||||
" mov %w0, %w6\n" \
|
||||
"2: stlxr %w[ret], %w[newval], %[uaddr]\n" \
|
||||
" cbz %w[ret], 3f\n" \
|
||||
" sub %w[loops], %w[loops], %w[ret]\n" \
|
||||
" cbnz %w[loops], 1b\n" \
|
||||
" mov %w[ret], %w[err]\n" \
|
||||
"3:\n" \
|
||||
" dmb ish\n" \
|
||||
_ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w0) \
|
||||
_ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w0) \
|
||||
: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp), \
|
||||
"+r" (loops) \
|
||||
: "r" (oparg), "Ir" (-EAGAIN) \
|
||||
_ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w[ret]) \
|
||||
_ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w[ret]) \
|
||||
: [ret] "=&r" (ret), [oldval] "=&r" (oldval), \
|
||||
[uaddr] "+Q" (*uaddr), [newval] "=&r" (newval), \
|
||||
[loops] "+r" (loops) \
|
||||
: [oparg] "r" (oparg), [err] "Ir" (-EAGAIN) \
|
||||
: "memory"); \
|
||||
uaccess_disable_privileged(); \
|
||||
} while (0)
|
||||
\
|
||||
if (!ret) \
|
||||
*oval = oldval; \
|
||||
\
|
||||
return ret; \
|
||||
}
|
||||
|
||||
LLSC_FUTEX_ATOMIC_OP(add, "add %w[newval], %w[oldval], %w[oparg]")
|
||||
LLSC_FUTEX_ATOMIC_OP(or, "orr %w[newval], %w[oldval], %w[oparg]")
|
||||
LLSC_FUTEX_ATOMIC_OP(and, "and %w[newval], %w[oldval], %w[oparg]")
|
||||
LLSC_FUTEX_ATOMIC_OP(eor, "eor %w[newval], %w[oldval], %w[oparg]")
|
||||
LLSC_FUTEX_ATOMIC_OP(set, "mov %w[newval], %w[oparg]")
|
||||
|
||||
static __always_inline int
|
||||
__llsc_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
|
||||
{
|
||||
int ret = 0;
|
||||
unsigned int loops = FUTEX_MAX_LOOPS;
|
||||
u32 val, tmp;
|
||||
|
||||
uaccess_enable_privileged();
|
||||
asm volatile("//__llsc_futex_cmpxchg\n"
|
||||
" prfm pstl1strm, %[uaddr]\n"
|
||||
"1: ldxr %w[curval], %[uaddr]\n"
|
||||
" eor %w[tmp], %w[curval], %w[oldval]\n"
|
||||
" cbnz %w[tmp], 4f\n"
|
||||
"2: stlxr %w[tmp], %w[newval], %[uaddr]\n"
|
||||
" cbz %w[tmp], 3f\n"
|
||||
" sub %w[loops], %w[loops], %w[tmp]\n"
|
||||
" cbnz %w[loops], 1b\n"
|
||||
" mov %w[ret], %w[err]\n"
|
||||
"3:\n"
|
||||
" dmb ish\n"
|
||||
"4:\n"
|
||||
_ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w[ret])
|
||||
_ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w[ret])
|
||||
: [ret] "+r" (ret), [curval] "=&r" (val),
|
||||
[uaddr] "+Q" (*uaddr), [tmp] "=&r" (tmp),
|
||||
[loops] "+r" (loops)
|
||||
: [oldval] "r" (oldval), [newval] "r" (newval),
|
||||
[err] "Ir" (-EAGAIN)
|
||||
: "memory");
|
||||
uaccess_disable_privileged();
|
||||
|
||||
if (!ret)
|
||||
*oval = val;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_ARM64_LSUI
|
||||
|
||||
/*
|
||||
* Wrap LSUI instructions with uaccess_ttbr0_enable()/disable(), as
|
||||
* PAN toggling is not required.
|
||||
*/
|
||||
|
||||
#define LSUI_FUTEX_ATOMIC_OP(op, asm_op) \
|
||||
static __always_inline int \
|
||||
__lsui_futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \
|
||||
{ \
|
||||
int ret = 0; \
|
||||
int oldval; \
|
||||
\
|
||||
uaccess_ttbr0_enable(); \
|
||||
\
|
||||
asm volatile("// __lsui_futex_atomic_" #op "\n" \
|
||||
__LSUI_PREAMBLE \
|
||||
"1: " #asm_op "al %w[oparg], %w[oldval], %[uaddr]\n" \
|
||||
"2:\n" \
|
||||
_ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret]) \
|
||||
: [ret] "+r" (ret), [uaddr] "+Q" (*uaddr), \
|
||||
[oldval] "=r" (oldval) \
|
||||
: [oparg] "r" (oparg) \
|
||||
: "memory"); \
|
||||
\
|
||||
uaccess_ttbr0_disable(); \
|
||||
\
|
||||
if (!ret) \
|
||||
*oval = oldval; \
|
||||
return ret; \
|
||||
}
|
||||
|
||||
LSUI_FUTEX_ATOMIC_OP(add, ldtadd)
|
||||
LSUI_FUTEX_ATOMIC_OP(or, ldtset)
|
||||
LSUI_FUTEX_ATOMIC_OP(andnot, ldtclr)
|
||||
LSUI_FUTEX_ATOMIC_OP(set, swpt)
|
||||
|
||||
static __always_inline int
|
||||
__lsui_cmpxchg64(u64 __user *uaddr, u64 *oldval, u64 newval)
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
uaccess_ttbr0_enable();
|
||||
|
||||
asm volatile("// __lsui_cmpxchg64\n"
|
||||
__LSUI_PREAMBLE
|
||||
"1: casalt %[oldval], %[newval], %[uaddr]\n"
|
||||
"2:\n"
|
||||
_ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret])
|
||||
: [ret] "+r" (ret), [uaddr] "+Q" (*uaddr),
|
||||
[oldval] "+r" (*oldval)
|
||||
: [newval] "r" (newval)
|
||||
: "memory");
|
||||
|
||||
uaccess_ttbr0_disable();
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static __always_inline int
|
||||
__lsui_cmpxchg32(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
|
||||
{
|
||||
u64 __user *uaddr64;
|
||||
bool futex_pos, other_pos;
|
||||
u32 other, orig_other;
|
||||
union {
|
||||
u32 futex[2];
|
||||
u64 raw;
|
||||
} oval64, orig64, nval64;
|
||||
|
||||
uaddr64 = (u64 __user *)PTR_ALIGN_DOWN(uaddr, sizeof(u64));
|
||||
futex_pos = !IS_ALIGNED((unsigned long)uaddr, sizeof(u64));
|
||||
other_pos = !futex_pos;
|
||||
|
||||
oval64.futex[futex_pos] = oldval;
|
||||
if (get_user(oval64.futex[other_pos], (u32 __user *)uaddr64 + other_pos))
|
||||
return -EFAULT;
|
||||
|
||||
orig64.raw = oval64.raw;
|
||||
|
||||
nval64.futex[futex_pos] = newval;
|
||||
nval64.futex[other_pos] = oval64.futex[other_pos];
|
||||
|
||||
if (__lsui_cmpxchg64(uaddr64, &oval64.raw, nval64.raw))
|
||||
return -EFAULT;
|
||||
|
||||
oldval = oval64.futex[futex_pos];
|
||||
other = oval64.futex[other_pos];
|
||||
orig_other = orig64.futex[other_pos];
|
||||
|
||||
if (other != orig_other)
|
||||
return -EAGAIN;
|
||||
|
||||
*oval = oldval;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static __always_inline int
|
||||
__lsui_futex_atomic_and(int oparg, u32 __user *uaddr, int *oval)
|
||||
{
|
||||
/*
|
||||
* Undo the bitwise negation applied to the oparg passed from
|
||||
* arch_futex_atomic_op_inuser() with FUTEX_OP_ANDN.
|
||||
*/
|
||||
return __lsui_futex_atomic_andnot(~oparg, uaddr, oval);
|
||||
}
|
||||
|
||||
static __always_inline int
|
||||
__lsui_futex_atomic_eor(int oparg, u32 __user *uaddr, int *oval)
|
||||
{
|
||||
u32 oldval, newval, val;
|
||||
int ret, i;
|
||||
|
||||
if (get_user(oldval, uaddr))
|
||||
return -EFAULT;
|
||||
|
||||
/*
|
||||
* there are no ldteor/stteor instructions...
|
||||
*/
|
||||
for (i = 0; i < FUTEX_MAX_LOOPS; i++) {
|
||||
newval = oldval ^ oparg;
|
||||
|
||||
ret = __lsui_cmpxchg32(uaddr, oldval, newval, &val);
|
||||
switch (ret) {
|
||||
case -EFAULT:
|
||||
return ret;
|
||||
case -EAGAIN:
|
||||
continue;
|
||||
}
|
||||
|
||||
if (val == oldval) {
|
||||
*oval = val;
|
||||
return 0;
|
||||
}
|
||||
|
||||
oldval = val;
|
||||
}
|
||||
|
||||
return -EAGAIN;
|
||||
}
|
||||
|
||||
static __always_inline int
|
||||
__lsui_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
|
||||
{
|
||||
/*
|
||||
* Callers of futex_atomic_cmpxchg_inatomic() already retry on
|
||||
* -EAGAIN, no need for another loop of max retries.
|
||||
*/
|
||||
return __lsui_cmpxchg32(uaddr, oldval, newval, oval);
|
||||
}
|
||||
#endif /* CONFIG_ARM64_LSUI */
|
||||
|
||||
|
||||
#define FUTEX_ATOMIC_OP(op) \
|
||||
static __always_inline int \
|
||||
__futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \
|
||||
{ \
|
||||
return __lsui_llsc_body(futex_atomic_##op, oparg, uaddr, oval); \
|
||||
}
|
||||
|
||||
FUTEX_ATOMIC_OP(add)
|
||||
FUTEX_ATOMIC_OP(or)
|
||||
FUTEX_ATOMIC_OP(and)
|
||||
FUTEX_ATOMIC_OP(eor)
|
||||
FUTEX_ATOMIC_OP(set)
|
||||
|
||||
static __always_inline int
|
||||
__futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
|
||||
{
|
||||
return __lsui_llsc_body(futex_cmpxchg, uaddr, oldval, newval, oval);
|
||||
}
|
||||
|
||||
static inline int
|
||||
arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uaddr)
|
||||
{
|
||||
int oldval = 0, ret, tmp;
|
||||
u32 __user *uaddr = __uaccess_mask_ptr(_uaddr);
|
||||
int ret;
|
||||
u32 __user *uaddr;
|
||||
|
||||
if (!access_ok(_uaddr, sizeof(u32)))
|
||||
return -EFAULT;
|
||||
|
||||
uaddr = __uaccess_mask_ptr(_uaddr);
|
||||
|
||||
switch (op) {
|
||||
case FUTEX_OP_SET:
|
||||
__futex_atomic_op("mov %w3, %w5",
|
||||
ret, oldval, uaddr, tmp, oparg);
|
||||
ret = __futex_atomic_set(oparg, uaddr, oval);
|
||||
break;
|
||||
case FUTEX_OP_ADD:
|
||||
__futex_atomic_op("add %w3, %w1, %w5",
|
||||
ret, oldval, uaddr, tmp, oparg);
|
||||
ret = __futex_atomic_add(oparg, uaddr, oval);
|
||||
break;
|
||||
case FUTEX_OP_OR:
|
||||
__futex_atomic_op("orr %w3, %w1, %w5",
|
||||
ret, oldval, uaddr, tmp, oparg);
|
||||
ret = __futex_atomic_or(oparg, uaddr, oval);
|
||||
break;
|
||||
case FUTEX_OP_ANDN:
|
||||
__futex_atomic_op("and %w3, %w1, %w5",
|
||||
ret, oldval, uaddr, tmp, ~oparg);
|
||||
ret = __futex_atomic_and(~oparg, uaddr, oval);
|
||||
break;
|
||||
case FUTEX_OP_XOR:
|
||||
__futex_atomic_op("eor %w3, %w1, %w5",
|
||||
ret, oldval, uaddr, tmp, oparg);
|
||||
ret = __futex_atomic_eor(oparg, uaddr, oval);
|
||||
break;
|
||||
default:
|
||||
ret = -ENOSYS;
|
||||
}
|
||||
|
||||
if (!ret)
|
||||
*oval = oldval;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -81,40 +302,14 @@ static inline int
|
||||
futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr,
|
||||
u32 oldval, u32 newval)
|
||||
{
|
||||
int ret = 0;
|
||||
unsigned int loops = FUTEX_MAX_LOOPS;
|
||||
u32 val, tmp;
|
||||
u32 __user *uaddr;
|
||||
|
||||
if (!access_ok(_uaddr, sizeof(u32)))
|
||||
return -EFAULT;
|
||||
|
||||
uaddr = __uaccess_mask_ptr(_uaddr);
|
||||
uaccess_enable_privileged();
|
||||
asm volatile("// futex_atomic_cmpxchg_inatomic\n"
|
||||
" prfm pstl1strm, %2\n"
|
||||
"1: ldxr %w1, %2\n"
|
||||
" sub %w3, %w1, %w5\n"
|
||||
" cbnz %w3, 4f\n"
|
||||
"2: stlxr %w3, %w6, %2\n"
|
||||
" cbz %w3, 3f\n"
|
||||
" sub %w4, %w4, %w3\n"
|
||||
" cbnz %w4, 1b\n"
|
||||
" mov %w0, %w7\n"
|
||||
"3:\n"
|
||||
" dmb ish\n"
|
||||
"4:\n"
|
||||
_ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w0)
|
||||
_ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w0)
|
||||
: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)
|
||||
: "r" (oldval), "r" (newval), "Ir" (-EAGAIN)
|
||||
: "memory");
|
||||
uaccess_disable_privileged();
|
||||
|
||||
if (!ret)
|
||||
*uval = val;
|
||||
|
||||
return ret;
|
||||
return __futex_cmpxchg(uaddr, oldval, newval, uval);
|
||||
}
|
||||
|
||||
#endif /* __ASM_FUTEX_H */
|
||||
|
||||
@@ -71,23 +71,23 @@ static inline void __flush_hugetlb_tlb_range(struct vm_area_struct *vma,
|
||||
unsigned long start,
|
||||
unsigned long end,
|
||||
unsigned long stride,
|
||||
bool last_level)
|
||||
tlbf_t flags)
|
||||
{
|
||||
switch (stride) {
|
||||
#ifndef __PAGETABLE_PMD_FOLDED
|
||||
case PUD_SIZE:
|
||||
__flush_tlb_range(vma, start, end, PUD_SIZE, last_level, 1);
|
||||
__flush_tlb_range(vma, start, end, PUD_SIZE, 1, flags);
|
||||
break;
|
||||
#endif
|
||||
case CONT_PMD_SIZE:
|
||||
case PMD_SIZE:
|
||||
__flush_tlb_range(vma, start, end, PMD_SIZE, last_level, 2);
|
||||
__flush_tlb_range(vma, start, end, PMD_SIZE, 2, flags);
|
||||
break;
|
||||
case CONT_PTE_SIZE:
|
||||
__flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, 3);
|
||||
__flush_tlb_range(vma, start, end, PAGE_SIZE, 3, flags);
|
||||
break;
|
||||
default:
|
||||
__flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, TLBI_TTL_UNKNOWN);
|
||||
__flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, flags);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,7 +98,7 @@ static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma,
|
||||
{
|
||||
unsigned long stride = huge_page_size(hstate_vma(vma));
|
||||
|
||||
__flush_hugetlb_tlb_range(vma, start, end, stride, false);
|
||||
__flush_hugetlb_tlb_range(vma, start, end, stride, TLBF_NONE);
|
||||
}
|
||||
|
||||
#endif /* __ASM_HUGETLB_H */
|
||||
|
||||
@@ -60,126 +60,10 @@
|
||||
* of KERNEL_HWCAP_{feature}.
|
||||
*/
|
||||
#define __khwcap_feature(x) const_ilog2(HWCAP_ ## x)
|
||||
#define KERNEL_HWCAP_FP __khwcap_feature(FP)
|
||||
#define KERNEL_HWCAP_ASIMD __khwcap_feature(ASIMD)
|
||||
#define KERNEL_HWCAP_EVTSTRM __khwcap_feature(EVTSTRM)
|
||||
#define KERNEL_HWCAP_AES __khwcap_feature(AES)
|
||||
#define KERNEL_HWCAP_PMULL __khwcap_feature(PMULL)
|
||||
#define KERNEL_HWCAP_SHA1 __khwcap_feature(SHA1)
|
||||
#define KERNEL_HWCAP_SHA2 __khwcap_feature(SHA2)
|
||||
#define KERNEL_HWCAP_CRC32 __khwcap_feature(CRC32)
|
||||
#define KERNEL_HWCAP_ATOMICS __khwcap_feature(ATOMICS)
|
||||
#define KERNEL_HWCAP_FPHP __khwcap_feature(FPHP)
|
||||
#define KERNEL_HWCAP_ASIMDHP __khwcap_feature(ASIMDHP)
|
||||
#define KERNEL_HWCAP_CPUID __khwcap_feature(CPUID)
|
||||
#define KERNEL_HWCAP_ASIMDRDM __khwcap_feature(ASIMDRDM)
|
||||
#define KERNEL_HWCAP_JSCVT __khwcap_feature(JSCVT)
|
||||
#define KERNEL_HWCAP_FCMA __khwcap_feature(FCMA)
|
||||
#define KERNEL_HWCAP_LRCPC __khwcap_feature(LRCPC)
|
||||
#define KERNEL_HWCAP_DCPOP __khwcap_feature(DCPOP)
|
||||
#define KERNEL_HWCAP_SHA3 __khwcap_feature(SHA3)
|
||||
#define KERNEL_HWCAP_SM3 __khwcap_feature(SM3)
|
||||
#define KERNEL_HWCAP_SM4 __khwcap_feature(SM4)
|
||||
#define KERNEL_HWCAP_ASIMDDP __khwcap_feature(ASIMDDP)
|
||||
#define KERNEL_HWCAP_SHA512 __khwcap_feature(SHA512)
|
||||
#define KERNEL_HWCAP_SVE __khwcap_feature(SVE)
|
||||
#define KERNEL_HWCAP_ASIMDFHM __khwcap_feature(ASIMDFHM)
|
||||
#define KERNEL_HWCAP_DIT __khwcap_feature(DIT)
|
||||
#define KERNEL_HWCAP_USCAT __khwcap_feature(USCAT)
|
||||
#define KERNEL_HWCAP_ILRCPC __khwcap_feature(ILRCPC)
|
||||
#define KERNEL_HWCAP_FLAGM __khwcap_feature(FLAGM)
|
||||
#define KERNEL_HWCAP_SSBS __khwcap_feature(SSBS)
|
||||
#define KERNEL_HWCAP_SB __khwcap_feature(SB)
|
||||
#define KERNEL_HWCAP_PACA __khwcap_feature(PACA)
|
||||
#define KERNEL_HWCAP_PACG __khwcap_feature(PACG)
|
||||
#define KERNEL_HWCAP_GCS __khwcap_feature(GCS)
|
||||
#define KERNEL_HWCAP_CMPBR __khwcap_feature(CMPBR)
|
||||
#define KERNEL_HWCAP_FPRCVT __khwcap_feature(FPRCVT)
|
||||
#define KERNEL_HWCAP_F8MM8 __khwcap_feature(F8MM8)
|
||||
#define KERNEL_HWCAP_F8MM4 __khwcap_feature(F8MM4)
|
||||
#define KERNEL_HWCAP_SVE_F16MM __khwcap_feature(SVE_F16MM)
|
||||
#define KERNEL_HWCAP_SVE_ELTPERM __khwcap_feature(SVE_ELTPERM)
|
||||
#define KERNEL_HWCAP_SVE_AES2 __khwcap_feature(SVE_AES2)
|
||||
#define KERNEL_HWCAP_SVE_BFSCALE __khwcap_feature(SVE_BFSCALE)
|
||||
#define KERNEL_HWCAP_SVE2P2 __khwcap_feature(SVE2P2)
|
||||
#define KERNEL_HWCAP_SME2P2 __khwcap_feature(SME2P2)
|
||||
#define KERNEL_HWCAP_SME_SBITPERM __khwcap_feature(SME_SBITPERM)
|
||||
#define KERNEL_HWCAP_SME_AES __khwcap_feature(SME_AES)
|
||||
#define KERNEL_HWCAP_SME_SFEXPA __khwcap_feature(SME_SFEXPA)
|
||||
#define KERNEL_HWCAP_SME_STMOP __khwcap_feature(SME_STMOP)
|
||||
#define KERNEL_HWCAP_SME_SMOP4 __khwcap_feature(SME_SMOP4)
|
||||
|
||||
#define __khwcap2_feature(x) (const_ilog2(HWCAP2_ ## x) + 64)
|
||||
#define KERNEL_HWCAP_DCPODP __khwcap2_feature(DCPODP)
|
||||
#define KERNEL_HWCAP_SVE2 __khwcap2_feature(SVE2)
|
||||
#define KERNEL_HWCAP_SVEAES __khwcap2_feature(SVEAES)
|
||||
#define KERNEL_HWCAP_SVEPMULL __khwcap2_feature(SVEPMULL)
|
||||
#define KERNEL_HWCAP_SVEBITPERM __khwcap2_feature(SVEBITPERM)
|
||||
#define KERNEL_HWCAP_SVESHA3 __khwcap2_feature(SVESHA3)
|
||||
#define KERNEL_HWCAP_SVESM4 __khwcap2_feature(SVESM4)
|
||||
#define KERNEL_HWCAP_FLAGM2 __khwcap2_feature(FLAGM2)
|
||||
#define KERNEL_HWCAP_FRINT __khwcap2_feature(FRINT)
|
||||
#define KERNEL_HWCAP_SVEI8MM __khwcap2_feature(SVEI8MM)
|
||||
#define KERNEL_HWCAP_SVEF32MM __khwcap2_feature(SVEF32MM)
|
||||
#define KERNEL_HWCAP_SVEF64MM __khwcap2_feature(SVEF64MM)
|
||||
#define KERNEL_HWCAP_SVEBF16 __khwcap2_feature(SVEBF16)
|
||||
#define KERNEL_HWCAP_I8MM __khwcap2_feature(I8MM)
|
||||
#define KERNEL_HWCAP_BF16 __khwcap2_feature(BF16)
|
||||
#define KERNEL_HWCAP_DGH __khwcap2_feature(DGH)
|
||||
#define KERNEL_HWCAP_RNG __khwcap2_feature(RNG)
|
||||
#define KERNEL_HWCAP_BTI __khwcap2_feature(BTI)
|
||||
#define KERNEL_HWCAP_MTE __khwcap2_feature(MTE)
|
||||
#define KERNEL_HWCAP_ECV __khwcap2_feature(ECV)
|
||||
#define KERNEL_HWCAP_AFP __khwcap2_feature(AFP)
|
||||
#define KERNEL_HWCAP_RPRES __khwcap2_feature(RPRES)
|
||||
#define KERNEL_HWCAP_MTE3 __khwcap2_feature(MTE3)
|
||||
#define KERNEL_HWCAP_SME __khwcap2_feature(SME)
|
||||
#define KERNEL_HWCAP_SME_I16I64 __khwcap2_feature(SME_I16I64)
|
||||
#define KERNEL_HWCAP_SME_F64F64 __khwcap2_feature(SME_F64F64)
|
||||
#define KERNEL_HWCAP_SME_I8I32 __khwcap2_feature(SME_I8I32)
|
||||
#define KERNEL_HWCAP_SME_F16F32 __khwcap2_feature(SME_F16F32)
|
||||
#define KERNEL_HWCAP_SME_B16F32 __khwcap2_feature(SME_B16F32)
|
||||
#define KERNEL_HWCAP_SME_F32F32 __khwcap2_feature(SME_F32F32)
|
||||
#define KERNEL_HWCAP_SME_FA64 __khwcap2_feature(SME_FA64)
|
||||
#define KERNEL_HWCAP_WFXT __khwcap2_feature(WFXT)
|
||||
#define KERNEL_HWCAP_EBF16 __khwcap2_feature(EBF16)
|
||||
#define KERNEL_HWCAP_SVE_EBF16 __khwcap2_feature(SVE_EBF16)
|
||||
#define KERNEL_HWCAP_CSSC __khwcap2_feature(CSSC)
|
||||
#define KERNEL_HWCAP_RPRFM __khwcap2_feature(RPRFM)
|
||||
#define KERNEL_HWCAP_SVE2P1 __khwcap2_feature(SVE2P1)
|
||||
#define KERNEL_HWCAP_SME2 __khwcap2_feature(SME2)
|
||||
#define KERNEL_HWCAP_SME2P1 __khwcap2_feature(SME2P1)
|
||||
#define KERNEL_HWCAP_SME_I16I32 __khwcap2_feature(SME_I16I32)
|
||||
#define KERNEL_HWCAP_SME_BI32I32 __khwcap2_feature(SME_BI32I32)
|
||||
#define KERNEL_HWCAP_SME_B16B16 __khwcap2_feature(SME_B16B16)
|
||||
#define KERNEL_HWCAP_SME_F16F16 __khwcap2_feature(SME_F16F16)
|
||||
#define KERNEL_HWCAP_MOPS __khwcap2_feature(MOPS)
|
||||
#define KERNEL_HWCAP_HBC __khwcap2_feature(HBC)
|
||||
#define KERNEL_HWCAP_SVE_B16B16 __khwcap2_feature(SVE_B16B16)
|
||||
#define KERNEL_HWCAP_LRCPC3 __khwcap2_feature(LRCPC3)
|
||||
#define KERNEL_HWCAP_LSE128 __khwcap2_feature(LSE128)
|
||||
#define KERNEL_HWCAP_FPMR __khwcap2_feature(FPMR)
|
||||
#define KERNEL_HWCAP_LUT __khwcap2_feature(LUT)
|
||||
#define KERNEL_HWCAP_FAMINMAX __khwcap2_feature(FAMINMAX)
|
||||
#define KERNEL_HWCAP_F8CVT __khwcap2_feature(F8CVT)
|
||||
#define KERNEL_HWCAP_F8FMA __khwcap2_feature(F8FMA)
|
||||
#define KERNEL_HWCAP_F8DP4 __khwcap2_feature(F8DP4)
|
||||
#define KERNEL_HWCAP_F8DP2 __khwcap2_feature(F8DP2)
|
||||
#define KERNEL_HWCAP_F8E4M3 __khwcap2_feature(F8E4M3)
|
||||
#define KERNEL_HWCAP_F8E5M2 __khwcap2_feature(F8E5M2)
|
||||
#define KERNEL_HWCAP_SME_LUTV2 __khwcap2_feature(SME_LUTV2)
|
||||
#define KERNEL_HWCAP_SME_F8F16 __khwcap2_feature(SME_F8F16)
|
||||
#define KERNEL_HWCAP_SME_F8F32 __khwcap2_feature(SME_F8F32)
|
||||
#define KERNEL_HWCAP_SME_SF8FMA __khwcap2_feature(SME_SF8FMA)
|
||||
#define KERNEL_HWCAP_SME_SF8DP4 __khwcap2_feature(SME_SF8DP4)
|
||||
#define KERNEL_HWCAP_SME_SF8DP2 __khwcap2_feature(SME_SF8DP2)
|
||||
#define KERNEL_HWCAP_POE __khwcap2_feature(POE)
|
||||
|
||||
#define __khwcap3_feature(x) (const_ilog2(HWCAP3_ ## x) + 128)
|
||||
#define KERNEL_HWCAP_MTE_FAR __khwcap3_feature(MTE_FAR)
|
||||
#define KERNEL_HWCAP_MTE_STORE_ONLY __khwcap3_feature(MTE_STORE_ONLY)
|
||||
#define KERNEL_HWCAP_LSFE __khwcap3_feature(LSFE)
|
||||
#define KERNEL_HWCAP_LS64 __khwcap3_feature(LS64)
|
||||
|
||||
#include "asm/kernel-hwcap.h"
|
||||
|
||||
/*
|
||||
* This yields a mask that user programs can use to figure out what
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
#ifndef __ASM_LSUI_H
|
||||
#define __ASM_LSUI_H
|
||||
|
||||
#include <linux/compiler_types.h>
|
||||
#include <linux/stringify.h>
|
||||
#include <asm/alternative.h>
|
||||
#include <asm/alternative-macros.h>
|
||||
#include <asm/cpucaps.h>
|
||||
|
||||
#define __LSUI_PREAMBLE ".arch_extension lsui\n"
|
||||
|
||||
#ifdef CONFIG_ARM64_LSUI
|
||||
|
||||
#define __lsui_llsc_body(op, ...) \
|
||||
({ \
|
||||
alternative_has_cap_unlikely(ARM64_HAS_LSUI) ? \
|
||||
__lsui_##op(__VA_ARGS__) : __llsc_##op(__VA_ARGS__); \
|
||||
})
|
||||
|
||||
#else /* CONFIG_ARM64_LSUI */
|
||||
|
||||
#define __lsui_llsc_body(op, ...) __llsc_##op(__VA_ARGS__)
|
||||
|
||||
#endif /* CONFIG_ARM64_LSUI */
|
||||
|
||||
#endif /* __ASM_LSUI_H */
|
||||
@@ -10,20 +10,12 @@
|
||||
#define MMCF_AARCH32 0x1 /* mm context flag for AArch32 executables */
|
||||
#define USER_ASID_BIT 48
|
||||
#define USER_ASID_FLAG (UL(1) << USER_ASID_BIT)
|
||||
#define TTBR_ASID_MASK (UL(0xffff) << 48)
|
||||
|
||||
#ifndef __ASSEMBLER__
|
||||
|
||||
#include <linux/refcount.h>
|
||||
#include <asm/cpufeature.h>
|
||||
|
||||
enum pgtable_type {
|
||||
TABLE_PTE,
|
||||
TABLE_PMD,
|
||||
TABLE_PUD,
|
||||
TABLE_P4D,
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
atomic64_t id;
|
||||
#ifdef CONFIG_COMPAT
|
||||
@@ -112,5 +104,7 @@ void kpti_install_ng_mappings(void);
|
||||
static inline void kpti_install_ng_mappings(void) {}
|
||||
#endif
|
||||
|
||||
extern bool page_alloc_available;
|
||||
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
#endif
|
||||
|
||||
@@ -210,7 +210,8 @@ static inline void update_saved_ttbr0(struct task_struct *tsk,
|
||||
if (mm == &init_mm)
|
||||
ttbr = phys_to_ttbr(__pa_symbol(reserved_pg_dir));
|
||||
else
|
||||
ttbr = phys_to_ttbr(virt_to_phys(mm->pgd)) | ASID(mm) << 48;
|
||||
ttbr = phys_to_ttbr(virt_to_phys(mm->pgd)) |
|
||||
FIELD_PREP(TTBRx_EL1_ASID_MASK, ASID(mm));
|
||||
|
||||
WRITE_ONCE(task_thread_info(tsk)->ttbr0, ttbr);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
/* Copyright (C) 2025 Arm Ltd. */
|
||||
|
||||
#ifndef __ASM__MPAM_H
|
||||
#define __ASM__MPAM_H
|
||||
|
||||
#include <linux/arm_mpam.h>
|
||||
#include <linux/bitfield.h>
|
||||
#include <linux/jump_label.h>
|
||||
#include <linux/percpu.h>
|
||||
#include <linux/sched.h>
|
||||
|
||||
#include <asm/sysreg.h>
|
||||
|
||||
DECLARE_STATIC_KEY_FALSE(mpam_enabled);
|
||||
DECLARE_PER_CPU(u64, arm64_mpam_default);
|
||||
DECLARE_PER_CPU(u64, arm64_mpam_current);
|
||||
|
||||
/*
|
||||
* The value of the MPAM0_EL1 sysreg when a task is in resctrl's default group.
|
||||
* This is used by the context switch code to use the resctrl CPU property
|
||||
* instead. The value is modified when CDP is enabled/disabled by mounting
|
||||
* the resctrl filesystem.
|
||||
*/
|
||||
extern u64 arm64_mpam_global_default;
|
||||
|
||||
#ifdef CONFIG_ARM64_MPAM
|
||||
static inline u64 __mpam_regval(u16 partid_d, u16 partid_i, u8 pmg_d, u8 pmg_i)
|
||||
{
|
||||
return FIELD_PREP(MPAM0_EL1_PARTID_D, partid_d) |
|
||||
FIELD_PREP(MPAM0_EL1_PARTID_I, partid_i) |
|
||||
FIELD_PREP(MPAM0_EL1_PMG_D, pmg_d) |
|
||||
FIELD_PREP(MPAM0_EL1_PMG_I, pmg_i);
|
||||
}
|
||||
|
||||
static inline void mpam_set_cpu_defaults(int cpu, u16 partid_d, u16 partid_i,
|
||||
u8 pmg_d, u8 pmg_i)
|
||||
{
|
||||
u64 default_val = __mpam_regval(partid_d, partid_i, pmg_d, pmg_i);
|
||||
|
||||
WRITE_ONCE(per_cpu(arm64_mpam_default, cpu), default_val);
|
||||
}
|
||||
|
||||
/*
|
||||
* The resctrl filesystem writes to the partid/pmg values for threads and CPUs,
|
||||
* which may race with reads in mpam_thread_switch(). Ensure only one of the old
|
||||
* or new values are used. Particular care should be taken with the pmg field as
|
||||
* mpam_thread_switch() may read a partid and pmg that don't match, causing this
|
||||
* value to be stored with cache allocations, despite being considered 'free' by
|
||||
* resctrl.
|
||||
*/
|
||||
static inline u64 mpam_get_regval(struct task_struct *tsk)
|
||||
{
|
||||
return READ_ONCE(task_thread_info(tsk)->mpam_partid_pmg);
|
||||
}
|
||||
|
||||
static inline void mpam_set_task_partid_pmg(struct task_struct *tsk,
|
||||
u16 partid_d, u16 partid_i,
|
||||
u8 pmg_d, u8 pmg_i)
|
||||
{
|
||||
u64 regval = __mpam_regval(partid_d, partid_i, pmg_d, pmg_i);
|
||||
|
||||
WRITE_ONCE(task_thread_info(tsk)->mpam_partid_pmg, regval);
|
||||
}
|
||||
|
||||
static inline void mpam_thread_switch(struct task_struct *tsk)
|
||||
{
|
||||
u64 oldregval;
|
||||
int cpu = smp_processor_id();
|
||||
u64 regval = mpam_get_regval(tsk);
|
||||
|
||||
if (!static_branch_likely(&mpam_enabled))
|
||||
return;
|
||||
|
||||
if (regval == READ_ONCE(arm64_mpam_global_default))
|
||||
regval = READ_ONCE(per_cpu(arm64_mpam_default, cpu));
|
||||
|
||||
oldregval = READ_ONCE(per_cpu(arm64_mpam_current, cpu));
|
||||
if (oldregval == regval)
|
||||
return;
|
||||
|
||||
write_sysreg_s(regval | MPAM1_EL1_MPAMEN, SYS_MPAM1_EL1);
|
||||
if (system_supports_sme())
|
||||
write_sysreg_s(regval & (MPAMSM_EL1_PARTID_D | MPAMSM_EL1_PMG_D), SYS_MPAMSM_EL1);
|
||||
isb();
|
||||
|
||||
/* Synchronising the EL0 write is left until the ERET to EL0 */
|
||||
write_sysreg_s(regval, SYS_MPAM0_EL1);
|
||||
|
||||
WRITE_ONCE(per_cpu(arm64_mpam_current, cpu), regval);
|
||||
}
|
||||
#else
|
||||
static inline void mpam_thread_switch(struct task_struct *tsk) {}
|
||||
#endif /* CONFIG_ARM64_MPAM */
|
||||
|
||||
#endif /* __ASM__MPAM_H */
|
||||
@@ -252,6 +252,9 @@ static inline void mte_check_tfsr_entry(void)
|
||||
if (!kasan_hw_tags_enabled())
|
||||
return;
|
||||
|
||||
if (!system_uses_mte_async_or_asymm_mode())
|
||||
return;
|
||||
|
||||
mte_check_tfsr_el1();
|
||||
}
|
||||
|
||||
@@ -260,6 +263,9 @@ static inline void mte_check_tfsr_exit(void)
|
||||
if (!kasan_hw_tags_enabled())
|
||||
return;
|
||||
|
||||
if (!system_uses_mte_async_or_asymm_mode())
|
||||
return;
|
||||
|
||||
/*
|
||||
* The asynchronous faults are sync'ed automatically with
|
||||
* TFSR_EL1 on kernel entry but for exit an explicit dsb()
|
||||
|
||||
@@ -223,8 +223,6 @@
|
||||
*/
|
||||
#define S1_TABLE_AP (_AT(pmdval_t, 3) << 61)
|
||||
|
||||
#define TTBR_CNP_BIT (UL(1) << 0)
|
||||
|
||||
/*
|
||||
* TCR flags.
|
||||
*/
|
||||
@@ -287,9 +285,12 @@
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_ARM64_VA_BITS_52
|
||||
#define PTRS_PER_PGD_52_VA (UL(1) << (52 - PGDIR_SHIFT))
|
||||
#define PTRS_PER_PGD_48_VA (UL(1) << (48 - PGDIR_SHIFT))
|
||||
#define PTRS_PER_PGD_EXTRA (PTRS_PER_PGD_52_VA - PTRS_PER_PGD_48_VA)
|
||||
|
||||
/* Must be at least 64-byte aligned to prevent corruption of the TTBR */
|
||||
#define TTBR1_BADDR_4852_OFFSET (((UL(1) << (52 - PGDIR_SHIFT)) - \
|
||||
(UL(1) << (48 - PGDIR_SHIFT))) * 8)
|
||||
#define TTBR1_BADDR_4852_OFFSET (PTRS_PER_PGD_EXTRA << PTDESC_ORDER)
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user