mirror of
https://github.com/linux-msm/laptops-kernel.git
synced 2026-08-13 14:19:53 -07:00
Merge branches 'for-next/misc', 'for-next/tlbflush', 'for-next/ttbr-macros-cleanup', 'for-next/kselftest', 'for-next/feat_lsui', 'for-next/mpam', 'for-next/hotplug-batched-tlbi', 'for-next/bbml2-fixes', 'for-next/sysreg', 'for-next/generic-entry' and 'for-next/acpi', remote-tracking branches 'arm64/for-next/perf' and 'arm64/for-next/read-once' into for-next/core
* arm64/for-next/perf:
: Perf updates
perf/arm-cmn: Fix resource_size_t printk specifier in arm_cmn_init_dtc()
perf/arm-cmn: Fix incorrect error check for devm_ioremap()
perf: add NVIDIA Tegra410 C2C PMU
perf: add NVIDIA Tegra410 CPU Memory Latency PMU
perf/arm_cspmu: nvidia: Add Tegra410 PCIE-TGT PMU
perf/arm_cspmu: nvidia: Add Tegra410 PCIE PMU
perf/arm_cspmu: Add arm_cspmu_acpi_dev_get
perf/arm_cspmu: nvidia: Add Tegra410 UCF PMU
perf/arm_cspmu: nvidia: Rename doc to Tegra241
perf/arm-cmn: Stop claiming entire iomem region
arm64: cpufeature: Use pmuv3_implemented() function
arm64: cpufeature: Make PMUVer and PerfMon unsigned
KVM: arm64: Read PMUVer as unsigned
* arm64/for-next/read-once:
: Fixes for __READ_ONCE() with CONFIG_LTO=y
arm64, compiler-context-analysis: Permit alias analysis through __READ_ONCE() with CONFIG_LTO=y
arm64: Optimize __READ_ONCE() with CONFIG_LTO=y
* for-next/misc:
: Miscellaneous cleanups/fixes
arm64: rsi: use linear-map alias for realm config buffer
arm64: Kconfig: fix duplicate word in CMDLINE help text
arm64: mte: Skip TFSR_EL1 checks and barriers in synchronous tag check mode
arm64/hwcap: Generate the KERNEL_HWCAP_ definitions for the hwcaps
arm64: kexec: Remove duplicate allocation for trans_pgd
arm64: mm: Use generic enum pgtable_level
arm64: scs: Remove redundant save/restore of SCS SP on entry to/from EL0
arm64: remove ARCH_INLINE_*
* for-next/tlbflush:
: Refactor the arm64 TLB invalidation API and implementation
arm64: mm: __ptep_set_access_flags must hint correct TTL
arm64: mm: Provide level hint for flush_tlb_page()
arm64: mm: Wrap flush_tlb_page() around __do_flush_tlb_range()
arm64: mm: More flags for __flush_tlb_range()
arm64: mm: Refactor __flush_tlb_range() to take flags
arm64: mm: Refactor flush_tlb_page() to use __tlbi_level_asid()
arm64: mm: Simplify __flush_tlb_range_limit_excess()
arm64: mm: Simplify __TLBI_RANGE_NUM() macro
arm64: mm: Re-implement the __flush_tlb_range_op macro in C
arm64: mm: Inline __TLBI_VADDR_RANGE() into __tlbi_range()
arm64: mm: Push __TLBI_VADDR() into __tlbi_level()
arm64: mm: Implicitly invalidate user ASID based on TLBI operation
arm64: mm: Introduce a C wrapper for by-range TLB invalidation
arm64: mm: Re-implement the __tlbi_level macro as a C function
* for-next/ttbr-macros-cleanup:
: Cleanups of the TTBR1_* macros
arm64/mm: Directly use TTBRx_EL1_CnP
arm64/mm: Directly use TTBRx_EL1_ASID_MASK
arm64/mm: Describe TTBR1_BADDR_4852_OFFSET
* for-next/kselftest:
: arm64 kselftest updates
selftests/arm64: Implement cmpbr_sigill() to hwcap test
* for-next/feat_lsui:
: Futex support using FEAT_LSUI instructions to avoid toggling PAN
arm64: armv8_deprecated: Disable swp emulation when FEAT_LSUI present
arm64: Kconfig: Add support for LSUI
KVM: arm64: Use CAST instruction for swapping guest descriptor
arm64: futex: Support futex with FEAT_LSUI
arm64: futex: Refactor futex atomic operation
KVM: arm64: kselftest: set_id_regs: Add test for FEAT_LSUI
KVM: arm64: Expose FEAT_LSUI to guests
arm64: cpufeature: Add FEAT_LSUI
* for-next/mpam: (40 commits)
: Expose MPAM to user-space via resctrl:
: - Add architecture context-switch and hiding of the feature from KVM.
: - Add interface to allow MPAM to be exposed to user-space using resctrl.
: - Add errata workaoround for some existing platforms.
: - Add documentation for using MPAM and what shape of platforms can use resctrl
arm64: mpam: Add initial MPAM documentation
arm_mpam: Quirk CMN-650's CSU NRDY behaviour
arm_mpam: Add workaround for T241-MPAM-6
arm_mpam: Add workaround for T241-MPAM-4
arm_mpam: Add workaround for T241-MPAM-1
arm_mpam: Add quirk framework
arm_mpam: resctrl: Call resctrl_init() on platforms that can support resctrl
arm64: mpam: Select ARCH_HAS_CPU_RESCTRL
arm_mpam: resctrl: Add empty definitions for assorted resctrl functions
arm_mpam: resctrl: Update the rmid reallocation limit
arm_mpam: resctrl: Add resctrl_arch_rmid_read()
arm_mpam: resctrl: Allow resctrl to allocate monitors
arm_mpam: resctrl: Add support for csu counters
arm_mpam: resctrl: Add monitor initialisation and domain boilerplate
arm_mpam: resctrl: Add kunit test for control format conversions
arm_mpam: resctrl: Add support for 'MB' resource
arm_mpam: resctrl: Wait for cacheinfo to be ready
arm_mpam: resctrl: Add rmid index helpers
arm_mpam: resctrl: Convert to/from MPAMs fixed-point formats
arm_mpam: resctrl: Hide CDP emulation behind CONFIG_EXPERT
...
* for-next/hotplug-batched-tlbi:
: arm64/mm: Enable batched TLB flush in unmap_hotplug_range()
arm64/mm: Reject memory removal that splits a kernel leaf mapping
arm64/mm: Enable batched TLB flush in unmap_hotplug_range()
* for-next/bbml2-fixes:
: Fixes for realm guest and BBML2_NOABORT
arm64: mm: Remove pmd_sect() and pud_sect()
arm64: mm: Handle invalid large leaf mappings correctly
arm64: mm: Fix rodata=full block mapping support for realm guests
* for-next/sysreg:
: arm64 sysreg updates
arm64/sysreg: Update ID_AA64SMFR0_EL1 description to DDI0601 2025-12
arm64/sysreg: Update ID_AA64ZFR0_EL1 description to DDI0601 2025-12
arm64/sysreg: Update ID_AA64FPFR0_EL1 description to DDI0601 2025-12
arm64/sysreg: Update ID_AA64ISAR2_EL1 description to DDI0601 2025-12
arm64/sysreg: Update ID_AA64ISAR0_EL1 description to DDI0601 2025-12
arm64/sysreg: Update SMIDR_EL1 to DDI0601 2025-06
* for-next/generic-entry:
: More arm64 refactoring towards using the generic entry code
arm64: Check DAIF (and PMR) at task-switch time
arm64: entry: Use split preemption logic
arm64: entry: Use irqentry_{enter_from,exit_to}_kernel_mode()
arm64: entry: Consistently prefix arm64-specific wrappers
arm64: entry: Don't preempt with SError or Debug masked
entry: Split preemption from irqentry_exit_to_kernel_mode()
entry: Split kernel mode logic from irqentry_{enter,exit}()
entry: Move irqentry_enter() prototype later
entry: Remove local_irq_{enable,disable}_exit_to_user()
entry: Fix stale comment for irqentry_enter()
* for-next/acpi:
: arm64 ACPI updates
ACPI: AGDI: fix missing newline in error message
This commit is contained in:
Catalin Marinas
@@ -23,6 +23,7 @@ ARM64 Architecture
|
||||
memory
|
||||
memory-tagging-extension
|
||||
mops
|
||||
mpam
|
||||
perf
|
||||
pointer-authentication
|
||||
ptdump
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
.. SPDX-License-Identifier: GPL-2.0
|
||||
|
||||
====
|
||||
MPAM
|
||||
====
|
||||
|
||||
What is MPAM
|
||||
============
|
||||
MPAM (Memory Partitioning and Monitoring) is a feature in the CPUs and memory
|
||||
system components such as the caches or memory controllers that allow memory
|
||||
traffic to be labelled, partitioned and monitored.
|
||||
|
||||
Traffic is labelled by the CPU, based on the control or monitor group the
|
||||
current task is assigned to using resctrl. Partitioning policy can be set
|
||||
using the schemata file in resctrl, and monitor values read via resctrl.
|
||||
See Documentation/filesystems/resctrl.rst for more details.
|
||||
|
||||
This allows tasks that share memory system resources, such as caches, to be
|
||||
isolated from each other according to the partitioning policy (so called noisy
|
||||
neighbours).
|
||||
|
||||
Supported Platforms
|
||||
===================
|
||||
Use of this feature requires CPU support, support in the memory system
|
||||
components, and a description from firmware of where the MPAM device controls
|
||||
are in the MMIO address space. (e.g. the 'MPAM' ACPI table).
|
||||
|
||||
The MMIO device that provides MPAM controls/monitors for a memory system
|
||||
component is called a memory system component. (MSC).
|
||||
|
||||
Because the user interface to MPAM is via resctrl, only MPAM features that are
|
||||
compatible with resctrl can be exposed to user-space.
|
||||
|
||||
MSC are considered as a group based on the topology. MSC that correspond with
|
||||
the L3 cache are considered together, it is not possible to mix MSC between L2
|
||||
and L3 to 'cover' a resctrl schema.
|
||||
|
||||
The supported features are:
|
||||
|
||||
* Cache portion bitmap controls (CPOR) on the L2 or L3 caches. To expose
|
||||
CPOR at L2 or L3, every CPU must have a corresponding CPU cache at this
|
||||
level that also supports the feature. Mismatched big/little platforms are
|
||||
not supported as resctrl's controls would then also depend on task
|
||||
placement.
|
||||
|
||||
* Memory bandwidth maximum controls (MBW_MAX) on or after the L3 cache.
|
||||
resctrl uses the L3 cache-id to identify where the memory bandwidth
|
||||
control is applied. For this reason the platform must have an L3 cache
|
||||
with cache-id's supplied by firmware. (It doesn't need to support MPAM.)
|
||||
|
||||
To be exported as the 'MB' schema, the topology of the group of MSC chosen
|
||||
must match the topology of the L3 cache so that the cache-id's can be
|
||||
repainted. For example: Platforms with Memory bandwidth maximum controls
|
||||
on CPU-less NUMA nodes cannot expose the 'MB' schema to resctrl as these
|
||||
nodes do not have a corresponding L3 cache. If the memory bandwidth
|
||||
control is on the memory rather than the L3 then there must be a single
|
||||
global L3 as otherwise it is unknown which L3 the traffic came from. There
|
||||
must be no caches between the L3 and the memory so that the two ends of
|
||||
the path have equivalent traffic.
|
||||
|
||||
When the MPAM driver finds multiple groups of MSC it can use for the 'MB'
|
||||
schema, it prefers the group closest to the L3 cache.
|
||||
|
||||
* Cache Storage Usage (CSU) counters can expose the 'llc_occupancy' provided
|
||||
there is at least one CSU monitor on each MSC that makes up the L3 group.
|
||||
Exposing CSU counters from other caches or devices is not supported.
|
||||
|
||||
Reporting Bugs
|
||||
==============
|
||||
If you are not seeing the counters or controls you expect please share the
|
||||
debug messages produced when enabling dynamic debug and booting with:
|
||||
dyndbg="file mpam_resctrl.c +pl"
|
||||
@@ -214,6 +214,9 @@ stable kernels.
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| ARM | SI L1 | #4311569 | ARM64_ERRATUM_4311569 |
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| ARM | CMN-650 | #3642720 | N/A |
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| Broadcom | Brahma-B53 | N/A | ARM64_ERRATUM_845719 |
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| Broadcom | Brahma-B53 | N/A | ARM64_ERRATUM_843419 |
|
||||
@@ -247,6 +250,12 @@ stable kernels.
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| NVIDIA | T241 GICv3/4.x | T241-FABRIC-4 | N/A |
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| NVIDIA | T241 MPAM | T241-MPAM-1 | N/A |
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| NVIDIA | T241 MPAM | T241-MPAM-4 | N/A |
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| NVIDIA | T241 MPAM | T241-MPAM-6 | N/A |
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
| Freescale/NXP | LS2080A/LS1043A | A-008585 | FSL_ERRATUM_A008585 |
|
||||
+----------------+-----------------+-----------------+-----------------------------+
|
||||
|
||||
+25
-29
@@ -61,32 +61,6 @@ config ARM64
|
||||
select ARCH_HAVE_ELF_PROT
|
||||
select ARCH_HAVE_NMI_SAFE_CMPXCHG
|
||||
select ARCH_HAVE_TRACE_MMIO_ACCESS
|
||||
select ARCH_INLINE_READ_LOCK if !PREEMPTION
|
||||
select ARCH_INLINE_READ_LOCK_BH if !PREEMPTION
|
||||
select ARCH_INLINE_READ_LOCK_IRQ if !PREEMPTION
|
||||
select ARCH_INLINE_READ_LOCK_IRQSAVE if !PREEMPTION
|
||||
select ARCH_INLINE_READ_UNLOCK if !PREEMPTION
|
||||
select ARCH_INLINE_READ_UNLOCK_BH if !PREEMPTION
|
||||
select ARCH_INLINE_READ_UNLOCK_IRQ if !PREEMPTION
|
||||
select ARCH_INLINE_READ_UNLOCK_IRQRESTORE if !PREEMPTION
|
||||
select ARCH_INLINE_WRITE_LOCK if !PREEMPTION
|
||||
select ARCH_INLINE_WRITE_LOCK_BH if !PREEMPTION
|
||||
select ARCH_INLINE_WRITE_LOCK_IRQ if !PREEMPTION
|
||||
select ARCH_INLINE_WRITE_LOCK_IRQSAVE if !PREEMPTION
|
||||
select ARCH_INLINE_WRITE_UNLOCK if !PREEMPTION
|
||||
select ARCH_INLINE_WRITE_UNLOCK_BH if !PREEMPTION
|
||||
select ARCH_INLINE_WRITE_UNLOCK_IRQ if !PREEMPTION
|
||||
select ARCH_INLINE_WRITE_UNLOCK_IRQRESTORE if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_TRYLOCK if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_TRYLOCK_BH if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_LOCK if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_LOCK_BH if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_LOCK_IRQ if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_LOCK_IRQSAVE if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_UNLOCK if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_UNLOCK_BH if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_UNLOCK_IRQ if !PREEMPTION
|
||||
select ARCH_INLINE_SPIN_UNLOCK_IRQRESTORE if !PREEMPTION
|
||||
select ARCH_KEEP_MEMBLOCK
|
||||
select ARCH_MHP_MEMMAP_ON_MEMORY_ENABLE
|
||||
select ARCH_USE_CMPXCHG_LOCKREF
|
||||
@@ -2016,8 +1990,8 @@ config ARM64_TLB_RANGE
|
||||
|
||||
config ARM64_MPAM
|
||||
bool "Enable support for MPAM"
|
||||
select ARM64_MPAM_DRIVER if EXPERT # does nothing yet
|
||||
select ACPI_MPAM if ACPI
|
||||
select ARM64_MPAM_DRIVER
|
||||
select ARCH_HAS_CPU_RESCTRL
|
||||
help
|
||||
Memory System Resource Partitioning and Monitoring (MPAM) is an
|
||||
optional extension to the Arm architecture that allows each
|
||||
@@ -2039,6 +2013,8 @@ config ARM64_MPAM
|
||||
|
||||
MPAM is exposed to user-space via the resctrl pseudo filesystem.
|
||||
|
||||
This option enables the extra context switch code.
|
||||
|
||||
endmenu # "ARMv8.4 architectural features"
|
||||
|
||||
menu "ARMv8.5 architectural features"
|
||||
@@ -2215,6 +2191,26 @@ config ARM64_GCS
|
||||
|
||||
endmenu # "ARMv9.4 architectural features"
|
||||
|
||||
config AS_HAS_LSUI
|
||||
def_bool $(as-instr,.arch_extension lsui)
|
||||
help
|
||||
Supported by LLVM 20+ and binutils 2.45+.
|
||||
|
||||
menu "ARMv9.6 architectural features"
|
||||
|
||||
config ARM64_LSUI
|
||||
bool "Support Unprivileged Load Store Instructions (LSUI)"
|
||||
default y
|
||||
depends on AS_HAS_LSUI && !CPU_BIG_ENDIAN
|
||||
help
|
||||
The Unprivileged Load Store Instructions (LSUI) provides
|
||||
variants load/store instructions that access user-space memory
|
||||
from the kernel without clearing PSTATE.PAN bit.
|
||||
|
||||
This feature is supported by LLVM 20+ and binutils 2.45+.
|
||||
|
||||
endmenu # "ARMv9.6 architectural feature"
|
||||
|
||||
config ARM64_SVE
|
||||
bool "ARM Scalable Vector Extension support"
|
||||
default y
|
||||
@@ -2372,7 +2368,7 @@ config CMDLINE
|
||||
default ""
|
||||
help
|
||||
Provide a set of default command-line options at build time by
|
||||
entering them here. As a minimum, you should specify the the
|
||||
entering them here. As a minimum, you should specify the
|
||||
root device (e.g. root=/dev/nfs).
|
||||
|
||||
choice
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
#ifdef CONFIG_ARM64_SW_TTBR0_PAN
|
||||
.macro __uaccess_ttbr0_disable, tmp1
|
||||
mrs \tmp1, ttbr1_el1 // swapper_pg_dir
|
||||
bic \tmp1, \tmp1, #TTBR_ASID_MASK
|
||||
bic \tmp1, \tmp1, #TTBRx_EL1_ASID_MASK
|
||||
sub \tmp1, \tmp1, #RESERVED_SWAPPER_OFFSET // reserved_pg_dir
|
||||
msr ttbr0_el1, \tmp1 // set reserved TTBR0_EL1
|
||||
add \tmp1, \tmp1, #RESERVED_SWAPPER_OFFSET
|
||||
|
||||
@@ -71,6 +71,8 @@ cpucap_is_possible(const unsigned int cap)
|
||||
return true;
|
||||
case ARM64_HAS_PMUV3:
|
||||
return IS_ENABLED(CONFIG_HW_PERF_EVENTS);
|
||||
case ARM64_HAS_LSUI:
|
||||
return IS_ENABLED(CONFIG_ARM64_LSUI);
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
@@ -513,7 +513,8 @@
|
||||
check_override id_aa64pfr0, ID_AA64PFR0_EL1_MPAM_SHIFT, .Linit_mpam_\@, .Lskip_mpam_\@, x1, x2
|
||||
|
||||
.Linit_mpam_\@:
|
||||
msr_s SYS_MPAM2_EL2, xzr // use the default partition
|
||||
mov x0, #MPAM2_EL2_EnMPAMSM_MASK
|
||||
msr_s SYS_MPAM2_EL2, x0 // use the default partition,
|
||||
// and disable lower traps
|
||||
mrs_s x0, SYS_MPAMIDR_EL1
|
||||
tbz x0, #MPAMIDR_EL1_HAS_HCR_SHIFT, .Lskip_mpam_\@ // skip if no MPAMHCR reg
|
||||
|
||||
+253
-58
@@ -9,71 +9,292 @@
|
||||
#include <linux/uaccess.h>
|
||||
|
||||
#include <asm/errno.h>
|
||||
#include <asm/lsui.h>
|
||||
|
||||
#define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of? */
|
||||
|
||||
#define __futex_atomic_op(insn, ret, oldval, uaddr, tmp, oparg) \
|
||||
do { \
|
||||
#define LLSC_FUTEX_ATOMIC_OP(op, insn) \
|
||||
static __always_inline int \
|
||||
__llsc_futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \
|
||||
{ \
|
||||
unsigned int loops = FUTEX_MAX_LOOPS; \
|
||||
int ret, oldval, newval; \
|
||||
\
|
||||
uaccess_enable_privileged(); \
|
||||
asm volatile( \
|
||||
" prfm pstl1strm, %2\n" \
|
||||
"1: ldxr %w1, %2\n" \
|
||||
asm volatile("// __llsc_futex_atomic_" #op "\n" \
|
||||
" prfm pstl1strm, %[uaddr]\n" \
|
||||
"1: ldxr %w[oldval], %[uaddr]\n" \
|
||||
insn "\n" \
|
||||
"2: stlxr %w0, %w3, %2\n" \
|
||||
" cbz %w0, 3f\n" \
|
||||
" sub %w4, %w4, %w0\n" \
|
||||
" cbnz %w4, 1b\n" \
|
||||
" mov %w0, %w6\n" \
|
||||
"2: stlxr %w[ret], %w[newval], %[uaddr]\n" \
|
||||
" cbz %w[ret], 3f\n" \
|
||||
" sub %w[loops], %w[loops], %w[ret]\n" \
|
||||
" cbnz %w[loops], 1b\n" \
|
||||
" mov %w[ret], %w[err]\n" \
|
||||
"3:\n" \
|
||||
" dmb ish\n" \
|
||||
_ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w0) \
|
||||
_ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w0) \
|
||||
: "=&r" (ret), "=&r" (oldval), "+Q" (*uaddr), "=&r" (tmp), \
|
||||
"+r" (loops) \
|
||||
: "r" (oparg), "Ir" (-EAGAIN) \
|
||||
_ASM_EXTABLE_UACCESS_ERR(1b, 3b, %w[ret]) \
|
||||
_ASM_EXTABLE_UACCESS_ERR(2b, 3b, %w[ret]) \
|
||||
: [ret] "=&r" (ret), [oldval] "=&r" (oldval), \
|
||||
[uaddr] "+Q" (*uaddr), [newval] "=&r" (newval), \
|
||||
[loops] "+r" (loops) \
|
||||
: [oparg] "r" (oparg), [err] "Ir" (-EAGAIN) \
|
||||
: "memory"); \
|
||||
uaccess_disable_privileged(); \
|
||||
} while (0)
|
||||
\
|
||||
if (!ret) \
|
||||
*oval = oldval; \
|
||||
\
|
||||
return ret; \
|
||||
}
|
||||
|
||||
LLSC_FUTEX_ATOMIC_OP(add, "add %w[newval], %w[oldval], %w[oparg]")
|
||||
LLSC_FUTEX_ATOMIC_OP(or, "orr %w[newval], %w[oldval], %w[oparg]")
|
||||
LLSC_FUTEX_ATOMIC_OP(and, "and %w[newval], %w[oldval], %w[oparg]")
|
||||
LLSC_FUTEX_ATOMIC_OP(eor, "eor %w[newval], %w[oldval], %w[oparg]")
|
||||
LLSC_FUTEX_ATOMIC_OP(set, "mov %w[newval], %w[oparg]")
|
||||
|
||||
static __always_inline int
|
||||
__llsc_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
|
||||
{
|
||||
int ret = 0;
|
||||
unsigned int loops = FUTEX_MAX_LOOPS;
|
||||
u32 val, tmp;
|
||||
|
||||
uaccess_enable_privileged();
|
||||
asm volatile("//__llsc_futex_cmpxchg\n"
|
||||
" prfm pstl1strm, %[uaddr]\n"
|
||||
"1: ldxr %w[curval], %[uaddr]\n"
|
||||
" eor %w[tmp], %w[curval], %w[oldval]\n"
|
||||
" cbnz %w[tmp], 4f\n"
|
||||
"2: stlxr %w[tmp], %w[newval], %[uaddr]\n"
|
||||
" cbz %w[tmp], 3f\n"
|
||||
" sub %w[loops], %w[loops], %w[tmp]\n"
|
||||
" cbnz %w[loops], 1b\n"
|
||||
" mov %w[ret], %w[err]\n"
|
||||
"3:\n"
|
||||
" dmb ish\n"
|
||||
"4:\n"
|
||||
_ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w[ret])
|
||||
_ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w[ret])
|
||||
: [ret] "+r" (ret), [curval] "=&r" (val),
|
||||
[uaddr] "+Q" (*uaddr), [tmp] "=&r" (tmp),
|
||||
[loops] "+r" (loops)
|
||||
: [oldval] "r" (oldval), [newval] "r" (newval),
|
||||
[err] "Ir" (-EAGAIN)
|
||||
: "memory");
|
||||
uaccess_disable_privileged();
|
||||
|
||||
if (!ret)
|
||||
*oval = val;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_ARM64_LSUI
|
||||
|
||||
/*
|
||||
* Wrap LSUI instructions with uaccess_ttbr0_enable()/disable(), as
|
||||
* PAN toggling is not required.
|
||||
*/
|
||||
|
||||
#define LSUI_FUTEX_ATOMIC_OP(op, asm_op) \
|
||||
static __always_inline int \
|
||||
__lsui_futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \
|
||||
{ \
|
||||
int ret = 0; \
|
||||
int oldval; \
|
||||
\
|
||||
uaccess_ttbr0_enable(); \
|
||||
\
|
||||
asm volatile("// __lsui_futex_atomic_" #op "\n" \
|
||||
__LSUI_PREAMBLE \
|
||||
"1: " #asm_op "al %w[oparg], %w[oldval], %[uaddr]\n" \
|
||||
"2:\n" \
|
||||
_ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret]) \
|
||||
: [ret] "+r" (ret), [uaddr] "+Q" (*uaddr), \
|
||||
[oldval] "=r" (oldval) \
|
||||
: [oparg] "r" (oparg) \
|
||||
: "memory"); \
|
||||
\
|
||||
uaccess_ttbr0_disable(); \
|
||||
\
|
||||
if (!ret) \
|
||||
*oval = oldval; \
|
||||
return ret; \
|
||||
}
|
||||
|
||||
LSUI_FUTEX_ATOMIC_OP(add, ldtadd)
|
||||
LSUI_FUTEX_ATOMIC_OP(or, ldtset)
|
||||
LSUI_FUTEX_ATOMIC_OP(andnot, ldtclr)
|
||||
LSUI_FUTEX_ATOMIC_OP(set, swpt)
|
||||
|
||||
static __always_inline int
|
||||
__lsui_cmpxchg64(u64 __user *uaddr, u64 *oldval, u64 newval)
|
||||
{
|
||||
int ret = 0;
|
||||
|
||||
uaccess_ttbr0_enable();
|
||||
|
||||
asm volatile("// __lsui_cmpxchg64\n"
|
||||
__LSUI_PREAMBLE
|
||||
"1: casalt %[oldval], %[newval], %[uaddr]\n"
|
||||
"2:\n"
|
||||
_ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret])
|
||||
: [ret] "+r" (ret), [uaddr] "+Q" (*uaddr),
|
||||
[oldval] "+r" (*oldval)
|
||||
: [newval] "r" (newval)
|
||||
: "memory");
|
||||
|
||||
uaccess_ttbr0_disable();
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static __always_inline int
|
||||
__lsui_cmpxchg32(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
|
||||
{
|
||||
u64 __user *uaddr64;
|
||||
bool futex_pos, other_pos;
|
||||
u32 other, orig_other;
|
||||
union {
|
||||
u32 futex[2];
|
||||
u64 raw;
|
||||
} oval64, orig64, nval64;
|
||||
|
||||
uaddr64 = (u64 __user *)PTR_ALIGN_DOWN(uaddr, sizeof(u64));
|
||||
futex_pos = !IS_ALIGNED((unsigned long)uaddr, sizeof(u64));
|
||||
other_pos = !futex_pos;
|
||||
|
||||
oval64.futex[futex_pos] = oldval;
|
||||
if (get_user(oval64.futex[other_pos], (u32 __user *)uaddr64 + other_pos))
|
||||
return -EFAULT;
|
||||
|
||||
orig64.raw = oval64.raw;
|
||||
|
||||
nval64.futex[futex_pos] = newval;
|
||||
nval64.futex[other_pos] = oval64.futex[other_pos];
|
||||
|
||||
if (__lsui_cmpxchg64(uaddr64, &oval64.raw, nval64.raw))
|
||||
return -EFAULT;
|
||||
|
||||
oldval = oval64.futex[futex_pos];
|
||||
other = oval64.futex[other_pos];
|
||||
orig_other = orig64.futex[other_pos];
|
||||
|
||||
if (other != orig_other)
|
||||
return -EAGAIN;
|
||||
|
||||
*oval = oldval;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static __always_inline int
|
||||
__lsui_futex_atomic_and(int oparg, u32 __user *uaddr, int *oval)
|
||||
{
|
||||
/*
|
||||
* Undo the bitwise negation applied to the oparg passed from
|
||||
* arch_futex_atomic_op_inuser() with FUTEX_OP_ANDN.
|
||||
*/
|
||||
return __lsui_futex_atomic_andnot(~oparg, uaddr, oval);
|
||||
}
|
||||
|
||||
static __always_inline int
|
||||
__lsui_futex_atomic_eor(int oparg, u32 __user *uaddr, int *oval)
|
||||
{
|
||||
u32 oldval, newval, val;
|
||||
int ret, i;
|
||||
|
||||
if (get_user(oldval, uaddr))
|
||||
return -EFAULT;
|
||||
|
||||
/*
|
||||
* there are no ldteor/stteor instructions...
|
||||
*/
|
||||
for (i = 0; i < FUTEX_MAX_LOOPS; i++) {
|
||||
newval = oldval ^ oparg;
|
||||
|
||||
ret = __lsui_cmpxchg32(uaddr, oldval, newval, &val);
|
||||
switch (ret) {
|
||||
case -EFAULT:
|
||||
return ret;
|
||||
case -EAGAIN:
|
||||
continue;
|
||||
}
|
||||
|
||||
if (val == oldval) {
|
||||
*oval = val;
|
||||
return 0;
|
||||
}
|
||||
|
||||
oldval = val;
|
||||
}
|
||||
|
||||
return -EAGAIN;
|
||||
}
|
||||
|
||||
static __always_inline int
|
||||
__lsui_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
|
||||
{
|
||||
/*
|
||||
* Callers of futex_atomic_cmpxchg_inatomic() already retry on
|
||||
* -EAGAIN, no need for another loop of max retries.
|
||||
*/
|
||||
return __lsui_cmpxchg32(uaddr, oldval, newval, oval);
|
||||
}
|
||||
#endif /* CONFIG_ARM64_LSUI */
|
||||
|
||||
|
||||
#define FUTEX_ATOMIC_OP(op) \
|
||||
static __always_inline int \
|
||||
__futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \
|
||||
{ \
|
||||
return __lsui_llsc_body(futex_atomic_##op, oparg, uaddr, oval); \
|
||||
}
|
||||
|
||||
FUTEX_ATOMIC_OP(add)
|
||||
FUTEX_ATOMIC_OP(or)
|
||||
FUTEX_ATOMIC_OP(and)
|
||||
FUTEX_ATOMIC_OP(eor)
|
||||
FUTEX_ATOMIC_OP(set)
|
||||
|
||||
static __always_inline int
|
||||
__futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval)
|
||||
{
|
||||
return __lsui_llsc_body(futex_cmpxchg, uaddr, oldval, newval, oval);
|
||||
}
|
||||
|
||||
static inline int
|
||||
arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *_uaddr)
|
||||
{
|
||||
int oldval = 0, ret, tmp;
|
||||
u32 __user *uaddr = __uaccess_mask_ptr(_uaddr);
|
||||
int ret;
|
||||
u32 __user *uaddr;
|
||||
|
||||
if (!access_ok(_uaddr, sizeof(u32)))
|
||||
return -EFAULT;
|
||||
|
||||
uaddr = __uaccess_mask_ptr(_uaddr);
|
||||
|
||||
switch (op) {
|
||||
case FUTEX_OP_SET:
|
||||
__futex_atomic_op("mov %w3, %w5",
|
||||
ret, oldval, uaddr, tmp, oparg);
|
||||
ret = __futex_atomic_set(oparg, uaddr, oval);
|
||||
break;
|
||||
case FUTEX_OP_ADD:
|
||||
__futex_atomic_op("add %w3, %w1, %w5",
|
||||
ret, oldval, uaddr, tmp, oparg);
|
||||
ret = __futex_atomic_add(oparg, uaddr, oval);
|
||||
break;
|
||||
case FUTEX_OP_OR:
|
||||
__futex_atomic_op("orr %w3, %w1, %w5",
|
||||
ret, oldval, uaddr, tmp, oparg);
|
||||
ret = __futex_atomic_or(oparg, uaddr, oval);
|
||||
break;
|
||||
case FUTEX_OP_ANDN:
|
||||
__futex_atomic_op("and %w3, %w1, %w5",
|
||||
ret, oldval, uaddr, tmp, ~oparg);
|
||||
ret = __futex_atomic_and(~oparg, uaddr, oval);
|
||||
break;
|
||||
case FUTEX_OP_XOR:
|
||||
__futex_atomic_op("eor %w3, %w1, %w5",
|
||||
ret, oldval, uaddr, tmp, oparg);
|
||||
ret = __futex_atomic_eor(oparg, uaddr, oval);
|
||||
break;
|
||||
default:
|
||||
ret = -ENOSYS;
|
||||
}
|
||||
|
||||
if (!ret)
|
||||
*oval = oldval;
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -81,40 +302,14 @@ static inline int
|
||||
futex_atomic_cmpxchg_inatomic(u32 *uval, u32 __user *_uaddr,
|
||||
u32 oldval, u32 newval)
|
||||
{
|
||||
int ret = 0;
|
||||
unsigned int loops = FUTEX_MAX_LOOPS;
|
||||
u32 val, tmp;
|
||||
u32 __user *uaddr;
|
||||
|
||||
if (!access_ok(_uaddr, sizeof(u32)))
|
||||
return -EFAULT;
|
||||
|
||||
uaddr = __uaccess_mask_ptr(_uaddr);
|
||||
uaccess_enable_privileged();
|
||||
asm volatile("// futex_atomic_cmpxchg_inatomic\n"
|
||||
" prfm pstl1strm, %2\n"
|
||||
"1: ldxr %w1, %2\n"
|
||||
" sub %w3, %w1, %w5\n"
|
||||
" cbnz %w3, 4f\n"
|
||||
"2: stlxr %w3, %w6, %2\n"
|
||||
" cbz %w3, 3f\n"
|
||||
" sub %w4, %w4, %w3\n"
|
||||
" cbnz %w4, 1b\n"
|
||||
" mov %w0, %w7\n"
|
||||
"3:\n"
|
||||
" dmb ish\n"
|
||||
"4:\n"
|
||||
_ASM_EXTABLE_UACCESS_ERR(1b, 4b, %w0)
|
||||
_ASM_EXTABLE_UACCESS_ERR(2b, 4b, %w0)
|
||||
: "+r" (ret), "=&r" (val), "+Q" (*uaddr), "=&r" (tmp), "+r" (loops)
|
||||
: "r" (oldval), "r" (newval), "Ir" (-EAGAIN)
|
||||
: "memory");
|
||||
uaccess_disable_privileged();
|
||||
|
||||
if (!ret)
|
||||
*uval = val;
|
||||
|
||||
return ret;
|
||||
return __futex_cmpxchg(uaddr, oldval, newval, uval);
|
||||
}
|
||||
|
||||
#endif /* __ASM_FUTEX_H */
|
||||
|
||||
@@ -71,23 +71,23 @@ static inline void __flush_hugetlb_tlb_range(struct vm_area_struct *vma,
|
||||
unsigned long start,
|
||||
unsigned long end,
|
||||
unsigned long stride,
|
||||
bool last_level)
|
||||
tlbf_t flags)
|
||||
{
|
||||
switch (stride) {
|
||||
#ifndef __PAGETABLE_PMD_FOLDED
|
||||
case PUD_SIZE:
|
||||
__flush_tlb_range(vma, start, end, PUD_SIZE, last_level, 1);
|
||||
__flush_tlb_range(vma, start, end, PUD_SIZE, 1, flags);
|
||||
break;
|
||||
#endif
|
||||
case CONT_PMD_SIZE:
|
||||
case PMD_SIZE:
|
||||
__flush_tlb_range(vma, start, end, PMD_SIZE, last_level, 2);
|
||||
__flush_tlb_range(vma, start, end, PMD_SIZE, 2, flags);
|
||||
break;
|
||||
case CONT_PTE_SIZE:
|
||||
__flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, 3);
|
||||
__flush_tlb_range(vma, start, end, PAGE_SIZE, 3, flags);
|
||||
break;
|
||||
default:
|
||||
__flush_tlb_range(vma, start, end, PAGE_SIZE, last_level, TLBI_TTL_UNKNOWN);
|
||||
__flush_tlb_range(vma, start, end, PAGE_SIZE, TLBI_TTL_UNKNOWN, flags);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -98,7 +98,7 @@ static inline void flush_hugetlb_tlb_range(struct vm_area_struct *vma,
|
||||
{
|
||||
unsigned long stride = huge_page_size(hstate_vma(vma));
|
||||
|
||||
__flush_hugetlb_tlb_range(vma, start, end, stride, false);
|
||||
__flush_hugetlb_tlb_range(vma, start, end, stride, TLBF_NONE);
|
||||
}
|
||||
|
||||
#endif /* __ASM_HUGETLB_H */
|
||||
|
||||
@@ -60,126 +60,10 @@
|
||||
* of KERNEL_HWCAP_{feature}.
|
||||
*/
|
||||
#define __khwcap_feature(x) const_ilog2(HWCAP_ ## x)
|
||||
#define KERNEL_HWCAP_FP __khwcap_feature(FP)
|
||||
#define KERNEL_HWCAP_ASIMD __khwcap_feature(ASIMD)
|
||||
#define KERNEL_HWCAP_EVTSTRM __khwcap_feature(EVTSTRM)
|
||||
#define KERNEL_HWCAP_AES __khwcap_feature(AES)
|
||||
#define KERNEL_HWCAP_PMULL __khwcap_feature(PMULL)
|
||||
#define KERNEL_HWCAP_SHA1 __khwcap_feature(SHA1)
|
||||
#define KERNEL_HWCAP_SHA2 __khwcap_feature(SHA2)
|
||||
#define KERNEL_HWCAP_CRC32 __khwcap_feature(CRC32)
|
||||
#define KERNEL_HWCAP_ATOMICS __khwcap_feature(ATOMICS)
|
||||
#define KERNEL_HWCAP_FPHP __khwcap_feature(FPHP)
|
||||
#define KERNEL_HWCAP_ASIMDHP __khwcap_feature(ASIMDHP)
|
||||
#define KERNEL_HWCAP_CPUID __khwcap_feature(CPUID)
|
||||
#define KERNEL_HWCAP_ASIMDRDM __khwcap_feature(ASIMDRDM)
|
||||
#define KERNEL_HWCAP_JSCVT __khwcap_feature(JSCVT)
|
||||
#define KERNEL_HWCAP_FCMA __khwcap_feature(FCMA)
|
||||
#define KERNEL_HWCAP_LRCPC __khwcap_feature(LRCPC)
|
||||
#define KERNEL_HWCAP_DCPOP __khwcap_feature(DCPOP)
|
||||
#define KERNEL_HWCAP_SHA3 __khwcap_feature(SHA3)
|
||||
#define KERNEL_HWCAP_SM3 __khwcap_feature(SM3)
|
||||
#define KERNEL_HWCAP_SM4 __khwcap_feature(SM4)
|
||||
#define KERNEL_HWCAP_ASIMDDP __khwcap_feature(ASIMDDP)
|
||||
#define KERNEL_HWCAP_SHA512 __khwcap_feature(SHA512)
|
||||
#define KERNEL_HWCAP_SVE __khwcap_feature(SVE)
|
||||
#define KERNEL_HWCAP_ASIMDFHM __khwcap_feature(ASIMDFHM)
|
||||
#define KERNEL_HWCAP_DIT __khwcap_feature(DIT)
|
||||
#define KERNEL_HWCAP_USCAT __khwcap_feature(USCAT)
|
||||
#define KERNEL_HWCAP_ILRCPC __khwcap_feature(ILRCPC)
|
||||
#define KERNEL_HWCAP_FLAGM __khwcap_feature(FLAGM)
|
||||
#define KERNEL_HWCAP_SSBS __khwcap_feature(SSBS)
|
||||
#define KERNEL_HWCAP_SB __khwcap_feature(SB)
|
||||
#define KERNEL_HWCAP_PACA __khwcap_feature(PACA)
|
||||
#define KERNEL_HWCAP_PACG __khwcap_feature(PACG)
|
||||
#define KERNEL_HWCAP_GCS __khwcap_feature(GCS)
|
||||
#define KERNEL_HWCAP_CMPBR __khwcap_feature(CMPBR)
|
||||
#define KERNEL_HWCAP_FPRCVT __khwcap_feature(FPRCVT)
|
||||
#define KERNEL_HWCAP_F8MM8 __khwcap_feature(F8MM8)
|
||||
#define KERNEL_HWCAP_F8MM4 __khwcap_feature(F8MM4)
|
||||
#define KERNEL_HWCAP_SVE_F16MM __khwcap_feature(SVE_F16MM)
|
||||
#define KERNEL_HWCAP_SVE_ELTPERM __khwcap_feature(SVE_ELTPERM)
|
||||
#define KERNEL_HWCAP_SVE_AES2 __khwcap_feature(SVE_AES2)
|
||||
#define KERNEL_HWCAP_SVE_BFSCALE __khwcap_feature(SVE_BFSCALE)
|
||||
#define KERNEL_HWCAP_SVE2P2 __khwcap_feature(SVE2P2)
|
||||
#define KERNEL_HWCAP_SME2P2 __khwcap_feature(SME2P2)
|
||||
#define KERNEL_HWCAP_SME_SBITPERM __khwcap_feature(SME_SBITPERM)
|
||||
#define KERNEL_HWCAP_SME_AES __khwcap_feature(SME_AES)
|
||||
#define KERNEL_HWCAP_SME_SFEXPA __khwcap_feature(SME_SFEXPA)
|
||||
#define KERNEL_HWCAP_SME_STMOP __khwcap_feature(SME_STMOP)
|
||||
#define KERNEL_HWCAP_SME_SMOP4 __khwcap_feature(SME_SMOP4)
|
||||
|
||||
#define __khwcap2_feature(x) (const_ilog2(HWCAP2_ ## x) + 64)
|
||||
#define KERNEL_HWCAP_DCPODP __khwcap2_feature(DCPODP)
|
||||
#define KERNEL_HWCAP_SVE2 __khwcap2_feature(SVE2)
|
||||
#define KERNEL_HWCAP_SVEAES __khwcap2_feature(SVEAES)
|
||||
#define KERNEL_HWCAP_SVEPMULL __khwcap2_feature(SVEPMULL)
|
||||
#define KERNEL_HWCAP_SVEBITPERM __khwcap2_feature(SVEBITPERM)
|
||||
#define KERNEL_HWCAP_SVESHA3 __khwcap2_feature(SVESHA3)
|
||||
#define KERNEL_HWCAP_SVESM4 __khwcap2_feature(SVESM4)
|
||||
#define KERNEL_HWCAP_FLAGM2 __khwcap2_feature(FLAGM2)
|
||||
#define KERNEL_HWCAP_FRINT __khwcap2_feature(FRINT)
|
||||
#define KERNEL_HWCAP_SVEI8MM __khwcap2_feature(SVEI8MM)
|
||||
#define KERNEL_HWCAP_SVEF32MM __khwcap2_feature(SVEF32MM)
|
||||
#define KERNEL_HWCAP_SVEF64MM __khwcap2_feature(SVEF64MM)
|
||||
#define KERNEL_HWCAP_SVEBF16 __khwcap2_feature(SVEBF16)
|
||||
#define KERNEL_HWCAP_I8MM __khwcap2_feature(I8MM)
|
||||
#define KERNEL_HWCAP_BF16 __khwcap2_feature(BF16)
|
||||
#define KERNEL_HWCAP_DGH __khwcap2_feature(DGH)
|
||||
#define KERNEL_HWCAP_RNG __khwcap2_feature(RNG)
|
||||
#define KERNEL_HWCAP_BTI __khwcap2_feature(BTI)
|
||||
#define KERNEL_HWCAP_MTE __khwcap2_feature(MTE)
|
||||
#define KERNEL_HWCAP_ECV __khwcap2_feature(ECV)
|
||||
#define KERNEL_HWCAP_AFP __khwcap2_feature(AFP)
|
||||
#define KERNEL_HWCAP_RPRES __khwcap2_feature(RPRES)
|
||||
#define KERNEL_HWCAP_MTE3 __khwcap2_feature(MTE3)
|
||||
#define KERNEL_HWCAP_SME __khwcap2_feature(SME)
|
||||
#define KERNEL_HWCAP_SME_I16I64 __khwcap2_feature(SME_I16I64)
|
||||
#define KERNEL_HWCAP_SME_F64F64 __khwcap2_feature(SME_F64F64)
|
||||
#define KERNEL_HWCAP_SME_I8I32 __khwcap2_feature(SME_I8I32)
|
||||
#define KERNEL_HWCAP_SME_F16F32 __khwcap2_feature(SME_F16F32)
|
||||
#define KERNEL_HWCAP_SME_B16F32 __khwcap2_feature(SME_B16F32)
|
||||
#define KERNEL_HWCAP_SME_F32F32 __khwcap2_feature(SME_F32F32)
|
||||
#define KERNEL_HWCAP_SME_FA64 __khwcap2_feature(SME_FA64)
|
||||
#define KERNEL_HWCAP_WFXT __khwcap2_feature(WFXT)
|
||||
#define KERNEL_HWCAP_EBF16 __khwcap2_feature(EBF16)
|
||||
#define KERNEL_HWCAP_SVE_EBF16 __khwcap2_feature(SVE_EBF16)
|
||||
#define KERNEL_HWCAP_CSSC __khwcap2_feature(CSSC)
|
||||
#define KERNEL_HWCAP_RPRFM __khwcap2_feature(RPRFM)
|
||||
#define KERNEL_HWCAP_SVE2P1 __khwcap2_feature(SVE2P1)
|
||||
#define KERNEL_HWCAP_SME2 __khwcap2_feature(SME2)
|
||||
#define KERNEL_HWCAP_SME2P1 __khwcap2_feature(SME2P1)
|
||||
#define KERNEL_HWCAP_SME_I16I32 __khwcap2_feature(SME_I16I32)
|
||||
#define KERNEL_HWCAP_SME_BI32I32 __khwcap2_feature(SME_BI32I32)
|
||||
#define KERNEL_HWCAP_SME_B16B16 __khwcap2_feature(SME_B16B16)
|
||||
#define KERNEL_HWCAP_SME_F16F16 __khwcap2_feature(SME_F16F16)
|
||||
#define KERNEL_HWCAP_MOPS __khwcap2_feature(MOPS)
|
||||
#define KERNEL_HWCAP_HBC __khwcap2_feature(HBC)
|
||||
#define KERNEL_HWCAP_SVE_B16B16 __khwcap2_feature(SVE_B16B16)
|
||||
#define KERNEL_HWCAP_LRCPC3 __khwcap2_feature(LRCPC3)
|
||||
#define KERNEL_HWCAP_LSE128 __khwcap2_feature(LSE128)
|
||||
#define KERNEL_HWCAP_FPMR __khwcap2_feature(FPMR)
|
||||
#define KERNEL_HWCAP_LUT __khwcap2_feature(LUT)
|
||||
#define KERNEL_HWCAP_FAMINMAX __khwcap2_feature(FAMINMAX)
|
||||
#define KERNEL_HWCAP_F8CVT __khwcap2_feature(F8CVT)
|
||||
#define KERNEL_HWCAP_F8FMA __khwcap2_feature(F8FMA)
|
||||
#define KERNEL_HWCAP_F8DP4 __khwcap2_feature(F8DP4)
|
||||
#define KERNEL_HWCAP_F8DP2 __khwcap2_feature(F8DP2)
|
||||
#define KERNEL_HWCAP_F8E4M3 __khwcap2_feature(F8E4M3)
|
||||
#define KERNEL_HWCAP_F8E5M2 __khwcap2_feature(F8E5M2)
|
||||
#define KERNEL_HWCAP_SME_LUTV2 __khwcap2_feature(SME_LUTV2)
|
||||
#define KERNEL_HWCAP_SME_F8F16 __khwcap2_feature(SME_F8F16)
|
||||
#define KERNEL_HWCAP_SME_F8F32 __khwcap2_feature(SME_F8F32)
|
||||
#define KERNEL_HWCAP_SME_SF8FMA __khwcap2_feature(SME_SF8FMA)
|
||||
#define KERNEL_HWCAP_SME_SF8DP4 __khwcap2_feature(SME_SF8DP4)
|
||||
#define KERNEL_HWCAP_SME_SF8DP2 __khwcap2_feature(SME_SF8DP2)
|
||||
#define KERNEL_HWCAP_POE __khwcap2_feature(POE)
|
||||
|
||||
#define __khwcap3_feature(x) (const_ilog2(HWCAP3_ ## x) + 128)
|
||||
#define KERNEL_HWCAP_MTE_FAR __khwcap3_feature(MTE_FAR)
|
||||
#define KERNEL_HWCAP_MTE_STORE_ONLY __khwcap3_feature(MTE_STORE_ONLY)
|
||||
#define KERNEL_HWCAP_LSFE __khwcap3_feature(LSFE)
|
||||
#define KERNEL_HWCAP_LS64 __khwcap3_feature(LS64)
|
||||
|
||||
#include "asm/kernel-hwcap.h"
|
||||
|
||||
/*
|
||||
* This yields a mask that user programs can use to figure out what
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
#ifndef __ASM_LSUI_H
|
||||
#define __ASM_LSUI_H
|
||||
|
||||
#include <linux/compiler_types.h>
|
||||
#include <linux/stringify.h>
|
||||
#include <asm/alternative.h>
|
||||
#include <asm/alternative-macros.h>
|
||||
#include <asm/cpucaps.h>
|
||||
|
||||
#define __LSUI_PREAMBLE ".arch_extension lsui\n"
|
||||
|
||||
#ifdef CONFIG_ARM64_LSUI
|
||||
|
||||
#define __lsui_llsc_body(op, ...) \
|
||||
({ \
|
||||
alternative_has_cap_unlikely(ARM64_HAS_LSUI) ? \
|
||||
__lsui_##op(__VA_ARGS__) : __llsc_##op(__VA_ARGS__); \
|
||||
})
|
||||
|
||||
#else /* CONFIG_ARM64_LSUI */
|
||||
|
||||
#define __lsui_llsc_body(op, ...) __llsc_##op(__VA_ARGS__)
|
||||
|
||||
#endif /* CONFIG_ARM64_LSUI */
|
||||
|
||||
#endif /* __ASM_LSUI_H */
|
||||
@@ -10,20 +10,12 @@
|
||||
#define MMCF_AARCH32 0x1 /* mm context flag for AArch32 executables */
|
||||
#define USER_ASID_BIT 48
|
||||
#define USER_ASID_FLAG (UL(1) << USER_ASID_BIT)
|
||||
#define TTBR_ASID_MASK (UL(0xffff) << 48)
|
||||
|
||||
#ifndef __ASSEMBLER__
|
||||
|
||||
#include <linux/refcount.h>
|
||||
#include <asm/cpufeature.h>
|
||||
|
||||
enum pgtable_type {
|
||||
TABLE_PTE,
|
||||
TABLE_PMD,
|
||||
TABLE_PUD,
|
||||
TABLE_P4D,
|
||||
};
|
||||
|
||||
typedef struct {
|
||||
atomic64_t id;
|
||||
#ifdef CONFIG_COMPAT
|
||||
@@ -112,5 +104,7 @@ void kpti_install_ng_mappings(void);
|
||||
static inline void kpti_install_ng_mappings(void) {}
|
||||
#endif
|
||||
|
||||
extern bool page_alloc_available;
|
||||
|
||||
#endif /* !__ASSEMBLER__ */
|
||||
#endif
|
||||
|
||||
@@ -210,7 +210,8 @@ static inline void update_saved_ttbr0(struct task_struct *tsk,
|
||||
if (mm == &init_mm)
|
||||
ttbr = phys_to_ttbr(__pa_symbol(reserved_pg_dir));
|
||||
else
|
||||
ttbr = phys_to_ttbr(virt_to_phys(mm->pgd)) | ASID(mm) << 48;
|
||||
ttbr = phys_to_ttbr(virt_to_phys(mm->pgd)) |
|
||||
FIELD_PREP(TTBRx_EL1_ASID_MASK, ASID(mm));
|
||||
|
||||
WRITE_ONCE(task_thread_info(tsk)->ttbr0, ttbr);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
/* Copyright (C) 2025 Arm Ltd. */
|
||||
|
||||
#ifndef __ASM__MPAM_H
|
||||
#define __ASM__MPAM_H
|
||||
|
||||
#include <linux/arm_mpam.h>
|
||||
#include <linux/bitfield.h>
|
||||
#include <linux/jump_label.h>
|
||||
#include <linux/percpu.h>
|
||||
#include <linux/sched.h>
|
||||
|
||||
#include <asm/sysreg.h>
|
||||
|
||||
DECLARE_STATIC_KEY_FALSE(mpam_enabled);
|
||||
DECLARE_PER_CPU(u64, arm64_mpam_default);
|
||||
DECLARE_PER_CPU(u64, arm64_mpam_current);
|
||||
|
||||
/*
|
||||
* The value of the MPAM0_EL1 sysreg when a task is in resctrl's default group.
|
||||
* This is used by the context switch code to use the resctrl CPU property
|
||||
* instead. The value is modified when CDP is enabled/disabled by mounting
|
||||
* the resctrl filesystem.
|
||||
*/
|
||||
extern u64 arm64_mpam_global_default;
|
||||
|
||||
#ifdef CONFIG_ARM64_MPAM
|
||||
static inline u64 __mpam_regval(u16 partid_d, u16 partid_i, u8 pmg_d, u8 pmg_i)
|
||||
{
|
||||
return FIELD_PREP(MPAM0_EL1_PARTID_D, partid_d) |
|
||||
FIELD_PREP(MPAM0_EL1_PARTID_I, partid_i) |
|
||||
FIELD_PREP(MPAM0_EL1_PMG_D, pmg_d) |
|
||||
FIELD_PREP(MPAM0_EL1_PMG_I, pmg_i);
|
||||
}
|
||||
|
||||
static inline void mpam_set_cpu_defaults(int cpu, u16 partid_d, u16 partid_i,
|
||||
u8 pmg_d, u8 pmg_i)
|
||||
{
|
||||
u64 default_val = __mpam_regval(partid_d, partid_i, pmg_d, pmg_i);
|
||||
|
||||
WRITE_ONCE(per_cpu(arm64_mpam_default, cpu), default_val);
|
||||
}
|
||||
|
||||
/*
|
||||
* The resctrl filesystem writes to the partid/pmg values for threads and CPUs,
|
||||
* which may race with reads in mpam_thread_switch(). Ensure only one of the old
|
||||
* or new values are used. Particular care should be taken with the pmg field as
|
||||
* mpam_thread_switch() may read a partid and pmg that don't match, causing this
|
||||
* value to be stored with cache allocations, despite being considered 'free' by
|
||||
* resctrl.
|
||||
*/
|
||||
static inline u64 mpam_get_regval(struct task_struct *tsk)
|
||||
{
|
||||
return READ_ONCE(task_thread_info(tsk)->mpam_partid_pmg);
|
||||
}
|
||||
|
||||
static inline void mpam_set_task_partid_pmg(struct task_struct *tsk,
|
||||
u16 partid_d, u16 partid_i,
|
||||
u8 pmg_d, u8 pmg_i)
|
||||
{
|
||||
u64 regval = __mpam_regval(partid_d, partid_i, pmg_d, pmg_i);
|
||||
|
||||
WRITE_ONCE(task_thread_info(tsk)->mpam_partid_pmg, regval);
|
||||
}
|
||||
|
||||
static inline void mpam_thread_switch(struct task_struct *tsk)
|
||||
{
|
||||
u64 oldregval;
|
||||
int cpu = smp_processor_id();
|
||||
u64 regval = mpam_get_regval(tsk);
|
||||
|
||||
if (!static_branch_likely(&mpam_enabled))
|
||||
return;
|
||||
|
||||
if (regval == READ_ONCE(arm64_mpam_global_default))
|
||||
regval = READ_ONCE(per_cpu(arm64_mpam_default, cpu));
|
||||
|
||||
oldregval = READ_ONCE(per_cpu(arm64_mpam_current, cpu));
|
||||
if (oldregval == regval)
|
||||
return;
|
||||
|
||||
write_sysreg_s(regval | MPAM1_EL1_MPAMEN, SYS_MPAM1_EL1);
|
||||
if (system_supports_sme())
|
||||
write_sysreg_s(regval & (MPAMSM_EL1_PARTID_D | MPAMSM_EL1_PMG_D), SYS_MPAMSM_EL1);
|
||||
isb();
|
||||
|
||||
/* Synchronising the EL0 write is left until the ERET to EL0 */
|
||||
write_sysreg_s(regval, SYS_MPAM0_EL1);
|
||||
|
||||
WRITE_ONCE(per_cpu(arm64_mpam_current, cpu), regval);
|
||||
}
|
||||
#else
|
||||
static inline void mpam_thread_switch(struct task_struct *tsk) {}
|
||||
#endif /* CONFIG_ARM64_MPAM */
|
||||
|
||||
#endif /* __ASM__MPAM_H */
|
||||
@@ -252,6 +252,9 @@ static inline void mte_check_tfsr_entry(void)
|
||||
if (!kasan_hw_tags_enabled())
|
||||
return;
|
||||
|
||||
if (!system_uses_mte_async_or_asymm_mode())
|
||||
return;
|
||||
|
||||
mte_check_tfsr_el1();
|
||||
}
|
||||
|
||||
@@ -260,6 +263,9 @@ static inline void mte_check_tfsr_exit(void)
|
||||
if (!kasan_hw_tags_enabled())
|
||||
return;
|
||||
|
||||
if (!system_uses_mte_async_or_asymm_mode())
|
||||
return;
|
||||
|
||||
/*
|
||||
* The asynchronous faults are sync'ed automatically with
|
||||
* TFSR_EL1 on kernel entry but for exit an explicit dsb()
|
||||
|
||||
@@ -223,8 +223,6 @@
|
||||
*/
|
||||
#define S1_TABLE_AP (_AT(pmdval_t, 3) << 61)
|
||||
|
||||
#define TTBR_CNP_BIT (UL(1) << 0)
|
||||
|
||||
/*
|
||||
* TCR flags.
|
||||
*/
|
||||
@@ -287,9 +285,12 @@
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_ARM64_VA_BITS_52
|
||||
#define PTRS_PER_PGD_52_VA (UL(1) << (52 - PGDIR_SHIFT))
|
||||
#define PTRS_PER_PGD_48_VA (UL(1) << (48 - PGDIR_SHIFT))
|
||||
#define PTRS_PER_PGD_EXTRA (PTRS_PER_PGD_52_VA - PTRS_PER_PGD_48_VA)
|
||||
|
||||
/* Must be at least 64-byte aligned to prevent corruption of the TTBR */
|
||||
#define TTBR1_BADDR_4852_OFFSET (((UL(1) << (52 - PGDIR_SHIFT)) - \
|
||||
(UL(1) << (48 - PGDIR_SHIFT))) * 8)
|
||||
#define TTBR1_BADDR_4852_OFFSET (PTRS_PER_PGD_EXTRA << PTDESC_ORDER)
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
@@ -25,6 +25,8 @@
|
||||
*/
|
||||
#define PTE_PRESENT_INVALID (PTE_NG) /* only when !PTE_VALID */
|
||||
|
||||
#define PTE_PRESENT_VALID_KERNEL (PTE_VALID | PTE_MAYBE_NG)
|
||||
|
||||
#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
|
||||
#define PTE_UFFD_WP (_AT(pteval_t, 1) << 58) /* uffd-wp tracking */
|
||||
#define PTE_SWP_UFFD_WP (_AT(pteval_t, 1) << 3) /* only for swp ptes */
|
||||
|
||||
@@ -89,9 +89,9 @@ static inline void arch_leave_lazy_mmu_mode(void)
|
||||
|
||||
/* Set stride and tlb_level in flush_*_tlb_range */
|
||||
#define flush_pmd_tlb_range(vma, addr, end) \
|
||||
__flush_tlb_range(vma, addr, end, PMD_SIZE, false, 2)
|
||||
__flush_tlb_range(vma, addr, end, PMD_SIZE, 2, TLBF_NONE)
|
||||
#define flush_pud_tlb_range(vma, addr, end) \
|
||||
__flush_tlb_range(vma, addr, end, PUD_SIZE, false, 1)
|
||||
__flush_tlb_range(vma, addr, end, PUD_SIZE, 1, TLBF_NONE)
|
||||
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
||||
|
||||
/*
|
||||
@@ -101,10 +101,11 @@ static inline void arch_leave_lazy_mmu_mode(void)
|
||||
* entries exist.
|
||||
*/
|
||||
#define flush_tlb_fix_spurious_fault(vma, address, ptep) \
|
||||
local_flush_tlb_page_nonotify(vma, address)
|
||||
__flush_tlb_page(vma, address, TLBF_NOBROADCAST | TLBF_NONOTIFY)
|
||||
|
||||
#define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp) \
|
||||
local_flush_tlb_page_nonotify(vma, address)
|
||||
#define flush_tlb_fix_spurious_fault_pmd(vma, address, pmdp) \
|
||||
__flush_tlb_range(vma, address, address + PMD_SIZE, PMD_SIZE, 2, \
|
||||
TLBF_NOBROADCAST | TLBF_NONOTIFY | TLBF_NOWALKCACHE)
|
||||
|
||||
/*
|
||||
* ZERO_PAGE is a global shared page that is always zero: used
|
||||
@@ -322,9 +323,11 @@ static inline pte_t pte_mknoncont(pte_t pte)
|
||||
return clear_pte_bit(pte, __pgprot(PTE_CONT));
|
||||
}
|
||||
|
||||
static inline pte_t pte_mkvalid(pte_t pte)
|
||||
static inline pte_t pte_mkvalid_k(pte_t pte)
|
||||
{
|
||||
return set_pte_bit(pte, __pgprot(PTE_VALID));
|
||||
pte = clear_pte_bit(pte, __pgprot(PTE_PRESENT_INVALID));
|
||||
pte = set_pte_bit(pte, __pgprot(PTE_PRESENT_VALID_KERNEL));
|
||||
return pte;
|
||||
}
|
||||
|
||||
static inline pte_t pte_mkinvalid(pte_t pte)
|
||||
@@ -594,6 +597,7 @@ static inline int pmd_protnone(pmd_t pmd)
|
||||
#define pmd_mkclean(pmd) pte_pmd(pte_mkclean(pmd_pte(pmd)))
|
||||
#define pmd_mkdirty(pmd) pte_pmd(pte_mkdirty(pmd_pte(pmd)))
|
||||
#define pmd_mkyoung(pmd) pte_pmd(pte_mkyoung(pmd_pte(pmd)))
|
||||
#define pmd_mkvalid_k(pmd) pte_pmd(pte_mkvalid_k(pmd_pte(pmd)))
|
||||
#define pmd_mkinvalid(pmd) pte_pmd(pte_mkinvalid(pmd_pte(pmd)))
|
||||
#ifdef CONFIG_HAVE_ARCH_USERFAULTFD_WP
|
||||
#define pmd_uffd_wp(pmd) pte_uffd_wp(pmd_pte(pmd))
|
||||
@@ -635,6 +639,8 @@ static inline pmd_t pmd_mkspecial(pmd_t pmd)
|
||||
|
||||
#define pud_young(pud) pte_young(pud_pte(pud))
|
||||
#define pud_mkyoung(pud) pte_pud(pte_mkyoung(pud_pte(pud)))
|
||||
#define pud_mkwrite_novma(pud) pte_pud(pte_mkwrite_novma(pud_pte(pud)))
|
||||
#define pud_mkvalid_k(pud) pte_pud(pte_mkvalid_k(pud_pte(pud)))
|
||||
#define pud_write(pud) pte_write(pud_pte(pud))
|
||||
|
||||
static inline pud_t pud_mkhuge(pud_t pud)
|
||||
@@ -779,9 +785,13 @@ extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,
|
||||
|
||||
#define pmd_table(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
|
||||
PMD_TYPE_TABLE)
|
||||
#define pmd_sect(pmd) ((pmd_val(pmd) & PMD_TYPE_MASK) == \
|
||||
PMD_TYPE_SECT)
|
||||
#define pmd_leaf(pmd) (pmd_present(pmd) && !pmd_table(pmd))
|
||||
|
||||
#define pmd_leaf pmd_leaf
|
||||
static inline bool pmd_leaf(pmd_t pmd)
|
||||
{
|
||||
return pmd_present(pmd) && !pmd_table(pmd);
|
||||
}
|
||||
|
||||
#define pmd_bad(pmd) (!pmd_table(pmd))
|
||||
|
||||
#define pmd_leaf_size(pmd) (pmd_cont(pmd) ? CONT_PMD_SIZE : PMD_SIZE)
|
||||
@@ -799,11 +809,8 @@ static inline int pmd_trans_huge(pmd_t pmd)
|
||||
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
|
||||
|
||||
#if defined(CONFIG_ARM64_64K_PAGES) || CONFIG_PGTABLE_LEVELS < 3
|
||||
static inline bool pud_sect(pud_t pud) { return false; }
|
||||
static inline bool pud_table(pud_t pud) { return true; }
|
||||
#else
|
||||
#define pud_sect(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
|
||||
PUD_TYPE_SECT)
|
||||
#define pud_table(pud) ((pud_val(pud) & PUD_TYPE_MASK) == \
|
||||
PUD_TYPE_TABLE)
|
||||
#endif
|
||||
@@ -873,7 +880,11 @@ static inline unsigned long pmd_page_vaddr(pmd_t pmd)
|
||||
PUD_TYPE_TABLE)
|
||||
#define pud_present(pud) pte_present(pud_pte(pud))
|
||||
#ifndef __PAGETABLE_PMD_FOLDED
|
||||
#define pud_leaf(pud) (pud_present(pud) && !pud_table(pud))
|
||||
#define pud_leaf pud_leaf
|
||||
static inline bool pud_leaf(pud_t pud)
|
||||
{
|
||||
return pud_present(pud) && !pud_table(pud);
|
||||
}
|
||||
#else
|
||||
#define pud_leaf(pud) false
|
||||
#endif
|
||||
@@ -1247,9 +1258,18 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
|
||||
return pte_pmd(pte_modify(pmd_pte(pmd), newprot));
|
||||
}
|
||||
|
||||
extern int __ptep_set_access_flags(struct vm_area_struct *vma,
|
||||
unsigned long address, pte_t *ptep,
|
||||
pte_t entry, int dirty);
|
||||
extern int __ptep_set_access_flags_anysz(struct vm_area_struct *vma,
|
||||
unsigned long address, pte_t *ptep,
|
||||
pte_t entry, int dirty,
|
||||
unsigned long pgsize);
|
||||
|
||||
static inline int __ptep_set_access_flags(struct vm_area_struct *vma,
|
||||
unsigned long address, pte_t *ptep,
|
||||
pte_t entry, int dirty)
|
||||
{
|
||||
return __ptep_set_access_flags_anysz(vma, address, ptep, entry, dirty,
|
||||
PAGE_SIZE);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
|
||||
#define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS
|
||||
@@ -1257,8 +1277,8 @@ static inline int pmdp_set_access_flags(struct vm_area_struct *vma,
|
||||
unsigned long address, pmd_t *pmdp,
|
||||
pmd_t entry, int dirty)
|
||||
{
|
||||
return __ptep_set_access_flags(vma, address, (pte_t *)pmdp,
|
||||
pmd_pte(entry), dirty);
|
||||
return __ptep_set_access_flags_anysz(vma, address, (pte_t *)pmdp,
|
||||
pmd_pte(entry), dirty, PMD_SIZE);
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -1320,7 +1340,7 @@ static inline int __ptep_clear_flush_young(struct vm_area_struct *vma,
|
||||
* context-switch, which provides a DSB to complete the TLB
|
||||
* invalidation.
|
||||
*/
|
||||
flush_tlb_page_nosync(vma, address);
|
||||
__flush_tlb_page(vma, address, TLBF_NOSYNC);
|
||||
}
|
||||
|
||||
return young;
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
#include <linux/arm_mpam.h>
|
||||
@@ -19,6 +19,17 @@
|
||||
"ldapr" #sfx "\t" #regs, \
|
||||
ARM64_HAS_LDAPR)
|
||||
|
||||
/*
|
||||
* Replace this with typeof_unqual() when minimum compiler versions are
|
||||
* increased to GCC 14 and Clang 19. For the time being, we need this
|
||||
* workaround, which relies on function return values dropping qualifiers.
|
||||
*/
|
||||
#define __rwonce_typeof_unqual(x) typeof(({ \
|
||||
__diag_push() \
|
||||
__diag_ignore_all("-Wignored-qualifiers", "") \
|
||||
((typeof(x)(*)(void))0)(); \
|
||||
__diag_pop() }))
|
||||
|
||||
/*
|
||||
* When building with LTO, there is an increased risk of the compiler
|
||||
* converting an address dependency headed by a READ_ONCE() invocation
|
||||
@@ -31,9 +42,12 @@
|
||||
*/
|
||||
#define __READ_ONCE(x) \
|
||||
({ \
|
||||
typeof(&(x)) __x = &(x); \
|
||||
int atomic = 1; \
|
||||
union { __unqual_scalar_typeof(*__x) __val; char __c[1]; } __u; \
|
||||
auto __x = &(x); \
|
||||
auto __ret = (__rwonce_typeof_unqual(*__x) *)__x; \
|
||||
/* Hides alias reassignment from Clang's -Wthread-safety. */ \
|
||||
auto __retp = &__ret; \
|
||||
union { typeof(*__ret) __val; char __c[1]; } __u; \
|
||||
*__retp = &__u.__val; \
|
||||
switch (sizeof(x)) { \
|
||||
case 1: \
|
||||
asm volatile(__LOAD_RCPC(b, %w0, %1) \
|
||||
@@ -56,9 +70,9 @@
|
||||
: "Q" (*__x) : "memory"); \
|
||||
break; \
|
||||
default: \
|
||||
atomic = 0; \
|
||||
__u.__val = *(volatile typeof(*__x) *)__x; \
|
||||
} \
|
||||
atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(*__x) *)__x);\
|
||||
*__ret; \
|
||||
})
|
||||
|
||||
#endif /* !BUILD_VDSO */
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user