dasharo/linux
0
0
Fork 0
mirror of https://github.com/Dasharo/linux.git synced 2026-03-06 15:25:10 -08:00
Code Issues Packages Projects Releases Wiki Activity

selftests/rseq: x86: Template memory ordering and percpu access mode

Browse Source 
Introduce a rseq-x86-bits.h template header which is internally included
to generate the static inline functions covering:

- relaxed and release memory ordering,
- per-cpu-id and per-mm-cid per-cpu data access.

This introduces changes to the rseq.h selftests API which require to
update the rseq selftest programs. Similar API/templating changes need
to be done for other architectures.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20221122203932.231377-12-mathieu.desnoyers@efficios.com
This commit is contained in:
Mathieu Desnoyers
2022-11-22 15:39:13 -05:00
committed by Peter Zijlstra
parent 18c2355838
commit ae31573843
6 changed files with 1241 additions and 1150 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/testing/selftests/rseq/compiler.h
+6
View File
@@ -27,4 +27,10 @@
*/
#define rseq_after_asm_goto() asm volatile ("" : : : "memory")
/* Combine two tokens. */
#define RSEQ__COMBINE_TOKENS(_tokena, _tokenb) \
_tokena##_tokenb
#define RSEQ_COMBINE_TOKENS(_tokena, _tokenb) \
RSEQ__COMBINE_TOKENS(_tokena, _tokenb)
#endif /* RSEQ_COMPILER_H_ */
tools/testing/selftests/rseq/rseq-bits-reset.h
+11
View File
@@ -0,0 +1,11 @@
/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
/*
* rseq-bits-reset.h
*
* (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
*/
#undef RSEQ_TEMPLATE_IDENTIFIER
#undef RSEQ_TEMPLATE_CPU_ID_FIELD
#undef RSEQ_TEMPLATE_CPU_ID_OFFSET
#undef RSEQ_TEMPLATE_SUFFIX
tools/testing/selftests/rseq/rseq-bits-template.h
+41
View File
@@ -0,0 +1,41 @@
/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
/*
* rseq-bits-template.h
*
* (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
*/
#ifdef RSEQ_TEMPLATE_CPU_ID
# define RSEQ_TEMPLATE_CPU_ID_OFFSET RSEQ_CPU_ID_OFFSET
# define RSEQ_TEMPLATE_CPU_ID_FIELD cpu_id
# ifdef RSEQ_TEMPLATE_MO_RELEASE
# define RSEQ_TEMPLATE_SUFFIX _release_cpu_id
# elif defined (RSEQ_TEMPLATE_MO_RELAXED)
# define RSEQ_TEMPLATE_SUFFIX _relaxed_cpu_id
# else
# error "Never use <rseq-bits-template.h> directly; include <rseq.h> instead."
# endif
#elif defined(RSEQ_TEMPLATE_MM_CID)
# define RSEQ_TEMPLATE_CPU_ID_OFFSET RSEQ_MM_CID_OFFSET
# define RSEQ_TEMPLATE_CPU_ID_FIELD mm_cid
# ifdef RSEQ_TEMPLATE_MO_RELEASE
# define RSEQ_TEMPLATE_SUFFIX _release_mm_cid
# elif defined (RSEQ_TEMPLATE_MO_RELAXED)
# define RSEQ_TEMPLATE_SUFFIX _relaxed_mm_cid
# else
# error "Never use <rseq-bits-template.h> directly; include <rseq.h> instead."
# endif
#elif defined (RSEQ_TEMPLATE_CPU_ID_NONE)
# ifdef RSEQ_TEMPLATE_MO_RELEASE
# define RSEQ_TEMPLATE_SUFFIX _release
# elif defined (RSEQ_TEMPLATE_MO_RELAXED)
# define RSEQ_TEMPLATE_SUFFIX _relaxed
# else
# error "Never use <rseq-bits-template.h> directly; include <rseq.h> instead."
# endif
#else
# error "Never use <rseq-bits-template.h> directly; include <rseq.h> instead."
#endif
#define RSEQ_TEMPLATE_IDENTIFIER(x) RSEQ_COMBINE_TOKENS(x, RSEQ_TEMPLATE_SUFFIX)
tools/testing/selftests/rseq/rseq-x86-bits.h
+993
View File
File diff suppressed because it is too large Load Diff
tools/testing/selftests/rseq/rseq-x86.h
+31 -1150
View File
File diff suppressed because it is too large Load Diff
tools/testing/selftests/rseq/rseq.h
+159
View File
@@ -74,6 +74,20 @@ extern unsigned int rseq_flags;
*/
extern unsigned int rseq_feature_size;
enum rseq_mo {
RSEQ_MO_RELAXED = 0,
RSEQ_MO_CONSUME = 1, /* Unused */
RSEQ_MO_ACQUIRE = 2, /* Unused */
RSEQ_MO_RELEASE = 3,
RSEQ_MO_ACQ_REL = 4, /* Unused */
RSEQ_MO_SEQ_CST = 5, /* Unused */
};
enum rseq_percpu_mode {
RSEQ_PERCPU_CPU_ID = 0,
RSEQ_PERCPU_MM_CID = 1,
};
static inline struct rseq_abi *rseq_get_abi(void)
{
return (struct rseq_abi *) ((uintptr_t) rseq_thread_pointer() + rseq_offset);
@@ -222,4 +236,149 @@ static inline void rseq_prepare_unload(void)
rseq_clear_rseq_cs();
}
static inline __attribute__((always_inline))
int rseq_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
intptr_t *v, intptr_t expect,
intptr_t newv, int cpu)
{
if (rseq_mo != RSEQ_MO_RELAXED)
return -1;
switch (percpu_mode) {
case RSEQ_PERCPU_CPU_ID:
return rseq_cmpeqv_storev_relaxed_cpu_id(v, expect, newv, cpu);
case RSEQ_PERCPU_MM_CID:
return rseq_cmpeqv_storev_relaxed_mm_cid(v, expect, newv, cpu);
}
return -1;
}
/*
* Compare @v against @expectnot. When it does _not_ match, load @v
* into @load, and store the content of *@v + voffp into @v.
*/
static inline __attribute__((always_inline))
int rseq_cmpnev_storeoffp_load(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
intptr_t *v, intptr_t expectnot, long voffp, intptr_t *load,
int cpu)
{
if (rseq_mo != RSEQ_MO_RELAXED)
return -1;
switch (percpu_mode) {
case RSEQ_PERCPU_CPU_ID:
return rseq_cmpnev_storeoffp_load_relaxed_cpu_id(v, expectnot, voffp, load, cpu);
case RSEQ_PERCPU_MM_CID:
return rseq_cmpnev_storeoffp_load_relaxed_mm_cid(v, expectnot, voffp, load, cpu);
}
return -1;
}
static inline __attribute__((always_inline))
int rseq_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
intptr_t *v, intptr_t count, int cpu)
{
if (rseq_mo != RSEQ_MO_RELAXED)
return -1;
switch (percpu_mode) {
case RSEQ_PERCPU_CPU_ID:
return rseq_addv_relaxed_cpu_id(v, count, cpu);
case RSEQ_PERCPU_MM_CID:
return rseq_addv_relaxed_mm_cid(v, count, cpu);
}
return -1;
}
#ifdef RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
/*
* pval = *(ptr+off)
* *pval += inc;
*/
static inline __attribute__((always_inline))
int rseq_offset_deref_addv(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
intptr_t *ptr, long off, intptr_t inc, int cpu)
{
if (rseq_mo != RSEQ_MO_RELAXED)
return -1;
switch (percpu_mode) {
case RSEQ_PERCPU_CPU_ID:
return rseq_offset_deref_addv_relaxed_cpu_id(ptr, off, inc, cpu);
case RSEQ_PERCPU_MM_CID:
return rseq_offset_deref_addv_relaxed_mm_cid(ptr, off, inc, cpu);
}
return -1;
}
#endif
static inline __attribute__((always_inline))
int rseq_cmpeqv_trystorev_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
intptr_t *v, intptr_t expect,
intptr_t *v2, intptr_t newv2,
intptr_t newv, int cpu)
{
switch (rseq_mo) {
case RSEQ_MO_RELAXED:
switch (percpu_mode) {
case RSEQ_PERCPU_CPU_ID:
return rseq_cmpeqv_trystorev_storev_relaxed_cpu_id(v, expect, v2, newv2, newv, cpu);
case RSEQ_PERCPU_MM_CID:
return rseq_cmpeqv_trystorev_storev_relaxed_mm_cid(v, expect, v2, newv2, newv, cpu);
}
return -1;
case RSEQ_MO_RELEASE:
switch (percpu_mode) {
case RSEQ_PERCPU_CPU_ID:
return rseq_cmpeqv_trystorev_storev_release_cpu_id(v, expect, v2, newv2, newv, cpu);
case RSEQ_PERCPU_MM_CID:
return rseq_cmpeqv_trystorev_storev_release_mm_cid(v, expect, v2, newv2, newv, cpu);
}
return -1;
default:
return -1;
}
}
static inline __attribute__((always_inline))
int rseq_cmpeqv_cmpeqv_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
intptr_t *v, intptr_t expect,
intptr_t *v2, intptr_t expect2,
intptr_t newv, int cpu)
{
if (rseq_mo != RSEQ_MO_RELAXED)
return -1;
switch (percpu_mode) {
case RSEQ_PERCPU_CPU_ID:
return rseq_cmpeqv_cmpeqv_storev_relaxed_cpu_id(v, expect, v2, expect2, newv, cpu);
case RSEQ_PERCPU_MM_CID:
return rseq_cmpeqv_cmpeqv_storev_relaxed_mm_cid(v, expect, v2, expect2, newv, cpu);
}
return -1;
}
static inline __attribute__((always_inline))
int rseq_cmpeqv_trymemcpy_storev(enum rseq_mo rseq_mo, enum rseq_percpu_mode percpu_mode,
intptr_t *v, intptr_t expect,
void *dst, void *src, size_t len,
intptr_t newv, int cpu)
{
switch (rseq_mo) {
case RSEQ_MO_RELAXED:
switch (percpu_mode) {
case RSEQ_PERCPU_CPU_ID:
return rseq_cmpeqv_trymemcpy_storev_relaxed_cpu_id(v, expect, dst, src, len, newv, cpu);
case RSEQ_PERCPU_MM_CID:
return rseq_cmpeqv_trymemcpy_storev_relaxed_mm_cid(v, expect, dst, src, len, newv, cpu);
}
return -1;
case RSEQ_MO_RELEASE:
switch (percpu_mode) {
case RSEQ_PERCPU_CPU_ID:
return rseq_cmpeqv_trymemcpy_storev_release_cpu_id(v, expect, dst, src, len, newv, cpu);
case RSEQ_PERCPU_MM_CID:
return rseq_cmpeqv_trymemcpy_storev_release_mm_cid(v, expect, dst, src, len, newv, cpu);
}
return -1;
default:
return -1;
}
}
#endif /* RSEQ_H_ */