mirror of
https://github.com/linux-msm/laptops-kernel.git
synced 2026-08-13 14:19:53 -07:00
sched/mmcid: Revert the complex CID management
The CID management is a complex beast, which affects both scheduling and task migration. The compaction mechanism forces random tasks of a process into task work on exit to user space causing latency spikes. Revert back to the initial simple bitmap allocating mechanics, which are known to have scalability issues as that allows to gradually build up a replacement functionality in a reviewable way. Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Acked-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com> Link: https://patch.msgid.link/20251119172549.068197830@linutronix.de
This commit is contained in:
Thomas Gleixner
committed by
Peter Zijlstra
Peter Zijlstra
parent
80adaccf0e
commit
77d7dc8bef
@@ -922,13 +922,9 @@ struct vm_area_struct {
|
||||
#define vma_policy(vma) NULL
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_SCHED_MM_CID
|
||||
struct mm_cid {
|
||||
u64 time;
|
||||
int cid;
|
||||
int recent_cid;
|
||||
unsigned int cid;
|
||||
};
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Opaque type representing current mm_struct flag state. Must be accessed via
|
||||
@@ -1000,12 +996,6 @@ struct mm_struct {
|
||||
* runqueue locks.
|
||||
*/
|
||||
struct mm_cid __percpu *pcpu_cid;
|
||||
/*
|
||||
* @mm_cid_next_scan: Next mm_cid scan (in jiffies).
|
||||
*
|
||||
* When the next mm_cid scan is due (in jiffies).
|
||||
*/
|
||||
unsigned long mm_cid_next_scan;
|
||||
/**
|
||||
* @nr_cpus_allowed: Number of CPUs allowed for mm.
|
||||
*
|
||||
@@ -1013,14 +1003,6 @@ struct mm_struct {
|
||||
* threads allowed CPUs.
|
||||
*/
|
||||
unsigned int nr_cpus_allowed;
|
||||
/**
|
||||
* @max_nr_cid: Maximum number of allowed concurrency
|
||||
* IDs allocated.
|
||||
*
|
||||
* Track the highest number of allowed concurrency IDs
|
||||
* allocated for the mm.
|
||||
*/
|
||||
atomic_t max_nr_cid;
|
||||
/**
|
||||
* @cpus_allowed_lock: Lock protecting mm cpus_allowed.
|
||||
*
|
||||
@@ -1371,35 +1353,7 @@ static inline void vma_iter_init(struct vma_iterator *vmi,
|
||||
|
||||
#ifdef CONFIG_SCHED_MM_CID
|
||||
|
||||
enum mm_cid_state {
|
||||
MM_CID_UNSET = -1U, /* Unset state has lazy_put flag set. */
|
||||
MM_CID_LAZY_PUT = (1U << 31),
|
||||
};
|
||||
|
||||
static inline bool mm_cid_is_unset(int cid)
|
||||
{
|
||||
return cid == MM_CID_UNSET;
|
||||
}
|
||||
|
||||
static inline bool mm_cid_is_lazy_put(int cid)
|
||||
{
|
||||
return !mm_cid_is_unset(cid) && (cid & MM_CID_LAZY_PUT);
|
||||
}
|
||||
|
||||
static inline bool mm_cid_is_valid(int cid)
|
||||
{
|
||||
return !(cid & MM_CID_LAZY_PUT);
|
||||
}
|
||||
|
||||
static inline int mm_cid_set_lazy_put(int cid)
|
||||
{
|
||||
return cid | MM_CID_LAZY_PUT;
|
||||
}
|
||||
|
||||
static inline int mm_cid_clear_lazy_put(int cid)
|
||||
{
|
||||
return cid & ~MM_CID_LAZY_PUT;
|
||||
}
|
||||
#define MM_CID_UNSET (~0U)
|
||||
|
||||
/*
|
||||
* mm_cpus_allowed: Union of all mm's threads allowed CPUs.
|
||||
@@ -1432,11 +1386,8 @@ static inline void mm_init_cid(struct mm_struct *mm, struct task_struct *p)
|
||||
struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, i);
|
||||
|
||||
pcpu_cid->cid = MM_CID_UNSET;
|
||||
pcpu_cid->recent_cid = MM_CID_UNSET;
|
||||
pcpu_cid->time = 0;
|
||||
}
|
||||
mm->nr_cpus_allowed = p->nr_cpus_allowed;
|
||||
atomic_set(&mm->max_nr_cid, 0);
|
||||
raw_spin_lock_init(&mm->cpus_allowed_lock);
|
||||
cpumask_copy(mm_cpus_allowed(mm), &p->cpus_mask);
|
||||
cpumask_clear(mm_cidmask(mm));
|
||||
|
||||
+2
-3
@@ -955,10 +955,9 @@ static struct task_struct *dup_task_struct(struct task_struct *orig, int node)
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_SCHED_MM_CID
|
||||
tsk->mm_cid = -1;
|
||||
tsk->last_mm_cid = -1;
|
||||
tsk->mm_cid = MM_CID_UNSET;
|
||||
tsk->last_mm_cid = MM_CID_UNSET;
|
||||
tsk->mm_cid_active = 0;
|
||||
tsk->migrate_from_cpu = -1;
|
||||
#endif
|
||||
return tsk;
|
||||
|
||||
|
||||
+17
-500
File diff suppressed because it is too large
Load Diff
+45
-248
@@ -3540,286 +3540,83 @@ extern void sched_dynamic_update(int mode);
|
||||
extern const char *preempt_modes[];
|
||||
|
||||
#ifdef CONFIG_SCHED_MM_CID
|
||||
|
||||
#define SCHED_MM_CID_PERIOD_NS (100ULL * 1000000) /* 100ms */
|
||||
#define MM_CID_SCAN_DELAY 100 /* 100ms */
|
||||
|
||||
extern raw_spinlock_t cid_lock;
|
||||
extern int use_cid_lock;
|
||||
|
||||
extern void sched_mm_cid_migrate_from(struct task_struct *t);
|
||||
extern void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t);
|
||||
extern void task_tick_mm_cid(struct rq *rq, struct task_struct *curr);
|
||||
extern void init_sched_mm_cid(struct task_struct *t);
|
||||
|
||||
static inline void __mm_cid_put(struct mm_struct *mm, int cid)
|
||||
{
|
||||
if (cid < 0)
|
||||
return;
|
||||
cpumask_clear_cpu(cid, mm_cidmask(mm));
|
||||
}
|
||||
|
||||
/*
|
||||
* The per-mm/cpu cid can have the MM_CID_LAZY_PUT flag set or transition to
|
||||
* the MM_CID_UNSET state without holding the rq lock, but the rq lock needs to
|
||||
* be held to transition to other states.
|
||||
*
|
||||
* State transitions synchronized with cmpxchg or try_cmpxchg need to be
|
||||
* consistent across CPUs, which prevents use of this_cpu_cmpxchg.
|
||||
*/
|
||||
static inline void mm_cid_put_lazy(struct task_struct *t)
|
||||
static inline void init_sched_mm_cid(struct task_struct *t)
|
||||
{
|
||||
struct mm_struct *mm = t->mm;
|
||||
struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
|
||||
int cid;
|
||||
unsigned int max_cid;
|
||||
|
||||
lockdep_assert_irqs_disabled();
|
||||
cid = __this_cpu_read(pcpu_cid->cid);
|
||||
if (!mm_cid_is_lazy_put(cid) ||
|
||||
!try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET))
|
||||
if (!mm)
|
||||
return;
|
||||
__mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
|
||||
|
||||
/* Preset last_mm_cid */
|
||||
max_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
|
||||
t->last_mm_cid = max_cid - 1;
|
||||
}
|
||||
|
||||
static inline int mm_cid_pcpu_unset(struct mm_struct *mm)
|
||||
static inline bool __mm_cid_get(struct task_struct *t, unsigned int cid, unsigned int max_cids)
|
||||
{
|
||||
struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
|
||||
int cid, res;
|
||||
struct mm_struct *mm = t->mm;
|
||||
|
||||
lockdep_assert_irqs_disabled();
|
||||
cid = __this_cpu_read(pcpu_cid->cid);
|
||||
for (;;) {
|
||||
if (mm_cid_is_unset(cid))
|
||||
return MM_CID_UNSET;
|
||||
/*
|
||||
* Attempt transition from valid or lazy-put to unset.
|
||||
*/
|
||||
res = cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, cid, MM_CID_UNSET);
|
||||
if (res == cid)
|
||||
break;
|
||||
cid = res;
|
||||
}
|
||||
return cid;
|
||||
if (cid >= max_cids)
|
||||
return false;
|
||||
if (cpumask_test_and_set_cpu(cid, mm_cidmask(mm)))
|
||||
return false;
|
||||
t->mm_cid = t->last_mm_cid = cid;
|
||||
__this_cpu_write(mm->pcpu_cid->cid, cid);
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline void mm_cid_put(struct mm_struct *mm)
|
||||
static inline bool mm_cid_get(struct task_struct *t)
|
||||
{
|
||||
int cid;
|
||||
struct mm_struct *mm = t->mm;
|
||||
unsigned int max_cids;
|
||||
|
||||
lockdep_assert_irqs_disabled();
|
||||
cid = mm_cid_pcpu_unset(mm);
|
||||
if (cid == MM_CID_UNSET)
|
||||
return;
|
||||
__mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
|
||||
max_cids = min_t(int, READ_ONCE(mm->nr_cpus_allowed), atomic_read(&mm->mm_users));
|
||||
|
||||
/* Try to reuse the last CID of this task */
|
||||
if (__mm_cid_get(t, t->last_mm_cid, max_cids))
|
||||
return true;
|
||||
|
||||
/* Try to reuse the last CID of this mm on this CPU */
|
||||
if (__mm_cid_get(t, __this_cpu_read(mm->pcpu_cid->cid), max_cids))
|
||||
return true;
|
||||
|
||||
/* Try the first zero bit in the cidmask. */
|
||||
return __mm_cid_get(t, cpumask_first_zero(mm_cidmask(mm)), max_cids);
|
||||
}
|
||||
|
||||
static inline int __mm_cid_try_get(struct task_struct *t, struct mm_struct *mm)
|
||||
static inline void mm_cid_select(struct task_struct *t)
|
||||
{
|
||||
struct cpumask *cidmask = mm_cidmask(mm);
|
||||
struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
|
||||
int cid, max_nr_cid, allowed_max_nr_cid;
|
||||
|
||||
/*
|
||||
* After shrinking the number of threads or reducing the number
|
||||
* of allowed cpus, reduce the value of max_nr_cid so expansion
|
||||
* of cid allocation will preserve cache locality if the number
|
||||
* of threads or allowed cpus increase again.
|
||||
*/
|
||||
max_nr_cid = atomic_read(&mm->max_nr_cid);
|
||||
while ((allowed_max_nr_cid = min_t(int, READ_ONCE(mm->nr_cpus_allowed),
|
||||
atomic_read(&mm->mm_users))),
|
||||
max_nr_cid > allowed_max_nr_cid) {
|
||||
/* atomic_try_cmpxchg loads previous mm->max_nr_cid into max_nr_cid. */
|
||||
if (atomic_try_cmpxchg(&mm->max_nr_cid, &max_nr_cid, allowed_max_nr_cid)) {
|
||||
max_nr_cid = allowed_max_nr_cid;
|
||||
break;
|
||||
}
|
||||
}
|
||||
/* Try to re-use recent cid. This improves cache locality. */
|
||||
cid = __this_cpu_read(pcpu_cid->recent_cid);
|
||||
if (!mm_cid_is_unset(cid) && cid < max_nr_cid &&
|
||||
!cpumask_test_and_set_cpu(cid, cidmask))
|
||||
return cid;
|
||||
/*
|
||||
* Expand cid allocation if the maximum number of concurrency
|
||||
* IDs allocated (max_nr_cid) is below the number cpus allowed
|
||||
* and number of threads. Expanding cid allocation as much as
|
||||
* possible improves cache locality.
|
||||
*/
|
||||
cid = max_nr_cid;
|
||||
while (cid < READ_ONCE(mm->nr_cpus_allowed) && cid < atomic_read(&mm->mm_users)) {
|
||||
/* atomic_try_cmpxchg loads previous mm->max_nr_cid into cid. */
|
||||
if (!atomic_try_cmpxchg(&mm->max_nr_cid, &cid, cid + 1))
|
||||
continue;
|
||||
if (!cpumask_test_and_set_cpu(cid, cidmask))
|
||||
return cid;
|
||||
}
|
||||
/*
|
||||
* Find the first available concurrency id.
|
||||
* Retry finding first zero bit if the mask is temporarily
|
||||
* filled. This only happens during concurrent remote-clear
|
||||
* which owns a cid without holding a rq lock.
|
||||
* mm_cid_get() can fail when the maximum CID, which is determined
|
||||
* by min(mm->nr_cpus_allowed, mm->mm_users) changes concurrently.
|
||||
* That's a transient failure as there cannot be more tasks
|
||||
* concurrently on a CPU (or about to be scheduled in) than that.
|
||||
*/
|
||||
for (;;) {
|
||||
cid = cpumask_first_zero(cidmask);
|
||||
if (cid < READ_ONCE(mm->nr_cpus_allowed))
|
||||
if (mm_cid_get(t))
|
||||
break;
|
||||
cpu_relax();
|
||||
}
|
||||
if (cpumask_test_and_set_cpu(cid, cidmask))
|
||||
return -1;
|
||||
|
||||
return cid;
|
||||
}
|
||||
|
||||
/*
|
||||
* Save a snapshot of the current runqueue time of this cpu
|
||||
* with the per-cpu cid value, allowing to estimate how recently it was used.
|
||||
*/
|
||||
static inline void mm_cid_snapshot_time(struct rq *rq, struct mm_struct *mm)
|
||||
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next)
|
||||
{
|
||||
struct mm_cid *pcpu_cid = per_cpu_ptr(mm->pcpu_cid, cpu_of(rq));
|
||||
|
||||
lockdep_assert_rq_held(rq);
|
||||
WRITE_ONCE(pcpu_cid->time, rq->clock);
|
||||
}
|
||||
|
||||
static inline int __mm_cid_get(struct rq *rq, struct task_struct *t,
|
||||
struct mm_struct *mm)
|
||||
{
|
||||
int cid;
|
||||
|
||||
/*
|
||||
* All allocations (even those using the cid_lock) are lock-free. If
|
||||
* use_cid_lock is set, hold the cid_lock to perform cid allocation to
|
||||
* guarantee forward progress.
|
||||
*/
|
||||
if (!READ_ONCE(use_cid_lock)) {
|
||||
cid = __mm_cid_try_get(t, mm);
|
||||
if (cid >= 0)
|
||||
goto end;
|
||||
raw_spin_lock(&cid_lock);
|
||||
} else {
|
||||
raw_spin_lock(&cid_lock);
|
||||
cid = __mm_cid_try_get(t, mm);
|
||||
if (cid >= 0)
|
||||
goto unlock;
|
||||
}
|
||||
|
||||
/*
|
||||
* cid concurrently allocated. Retry while forcing following
|
||||
* allocations to use the cid_lock to ensure forward progress.
|
||||
*/
|
||||
WRITE_ONCE(use_cid_lock, 1);
|
||||
/*
|
||||
* Set use_cid_lock before allocation. Only care about program order
|
||||
* because this is only required for forward progress.
|
||||
*/
|
||||
barrier();
|
||||
/*
|
||||
* Retry until it succeeds. It is guaranteed to eventually succeed once
|
||||
* all newcoming allocations observe the use_cid_lock flag set.
|
||||
*/
|
||||
do {
|
||||
cid = __mm_cid_try_get(t, mm);
|
||||
cpu_relax();
|
||||
} while (cid < 0);
|
||||
/*
|
||||
* Allocate before clearing use_cid_lock. Only care about
|
||||
* program order because this is for forward progress.
|
||||
*/
|
||||
barrier();
|
||||
WRITE_ONCE(use_cid_lock, 0);
|
||||
unlock:
|
||||
raw_spin_unlock(&cid_lock);
|
||||
end:
|
||||
mm_cid_snapshot_time(rq, mm);
|
||||
|
||||
return cid;
|
||||
}
|
||||
|
||||
static inline int mm_cid_get(struct rq *rq, struct task_struct *t,
|
||||
struct mm_struct *mm)
|
||||
{
|
||||
struct mm_cid __percpu *pcpu_cid = mm->pcpu_cid;
|
||||
int cid;
|
||||
|
||||
lockdep_assert_rq_held(rq);
|
||||
cid = __this_cpu_read(pcpu_cid->cid);
|
||||
if (mm_cid_is_valid(cid)) {
|
||||
mm_cid_snapshot_time(rq, mm);
|
||||
return cid;
|
||||
}
|
||||
if (mm_cid_is_lazy_put(cid)) {
|
||||
if (try_cmpxchg(&this_cpu_ptr(pcpu_cid)->cid, &cid, MM_CID_UNSET))
|
||||
__mm_cid_put(mm, mm_cid_clear_lazy_put(cid));
|
||||
}
|
||||
cid = __mm_cid_get(rq, t, mm);
|
||||
__this_cpu_write(pcpu_cid->cid, cid);
|
||||
__this_cpu_write(pcpu_cid->recent_cid, cid);
|
||||
|
||||
return cid;
|
||||
}
|
||||
|
||||
static inline void switch_mm_cid(struct rq *rq,
|
||||
struct task_struct *prev,
|
||||
struct task_struct *next)
|
||||
{
|
||||
/*
|
||||
* Provide a memory barrier between rq->curr store and load of
|
||||
* {prev,next}->mm->pcpu_cid[cpu] on rq->curr->mm transition.
|
||||
*
|
||||
* Should be adapted if context_switch() is modified.
|
||||
*/
|
||||
if (!next->mm) { // to kernel
|
||||
/*
|
||||
* user -> kernel transition does not guarantee a barrier, but
|
||||
* we can use the fact that it performs an atomic operation in
|
||||
* mmgrab().
|
||||
*/
|
||||
if (prev->mm) // from user
|
||||
smp_mb__after_mmgrab();
|
||||
/*
|
||||
* kernel -> kernel transition does not change rq->curr->mm
|
||||
* state. It stays NULL.
|
||||
*/
|
||||
} else { // to user
|
||||
/*
|
||||
* kernel -> user transition does not provide a barrier
|
||||
* between rq->curr store and load of {prev,next}->mm->pcpu_cid[cpu].
|
||||
* Provide it here.
|
||||
*/
|
||||
if (!prev->mm) { // from kernel
|
||||
smp_mb();
|
||||
} else { // from user
|
||||
/*
|
||||
* user->user transition relies on an implicit
|
||||
* memory barrier in switch_mm() when
|
||||
* current->mm changes. If the architecture
|
||||
* switch_mm() does not have an implicit memory
|
||||
* barrier, it is emitted here. If current->mm
|
||||
* is unchanged, no barrier is needed.
|
||||
*/
|
||||
smp_mb__after_switch_mm();
|
||||
}
|
||||
}
|
||||
if (prev->mm_cid_active) {
|
||||
mm_cid_snapshot_time(rq, prev->mm);
|
||||
mm_cid_put_lazy(prev);
|
||||
prev->mm_cid = -1;
|
||||
if (prev->mm_cid != MM_CID_UNSET)
|
||||
cpumask_clear_cpu(prev->mm_cid, mm_cidmask(prev->mm));
|
||||
prev->mm_cid = MM_CID_UNSET;
|
||||
}
|
||||
|
||||
if (next->mm_cid_active) {
|
||||
next->last_mm_cid = next->mm_cid = mm_cid_get(rq, next, next->mm);
|
||||
mm_cid_select(next);
|
||||
rseq_sched_set_task_mm_cid(next, next->mm_cid);
|
||||
}
|
||||
}
|
||||
|
||||
#else /* !CONFIG_SCHED_MM_CID: */
|
||||
static inline void switch_mm_cid(struct rq *rq, struct task_struct *prev, struct task_struct *next) { }
|
||||
static inline void sched_mm_cid_migrate_from(struct task_struct *t) { }
|
||||
static inline void sched_mm_cid_migrate_to(struct rq *dst_rq, struct task_struct *t) { }
|
||||
static inline void task_tick_mm_cid(struct rq *rq, struct task_struct *curr) { }
|
||||
static inline void init_sched_mm_cid(struct task_struct *t) { }
|
||||
static inline void mm_cid_select(struct task_struct *t) { }
|
||||
static inline void switch_mm_cid(struct task_struct *prev, struct task_struct *next) { }
|
||||
#endif /* !CONFIG_SCHED_MM_CID */
|
||||
|
||||
extern u64 avg_vruntime(struct cfs_rq *cfs_rq);
|
||||
|
||||
Reference in New Issue
Block a user