diff --git a/Documentation/admin-guide/cgroup-v1/memcg_test.rst b/Documentation/admin-guide/cgroup-v1/memcg_test.rst index ebedbc3c3f9c..d9951c319ef5 100644 --- a/Documentation/admin-guide/cgroup-v1/memcg_test.rst +++ b/Documentation/admin-guide/cgroup-v1/memcg_test.rst @@ -10,7 +10,7 @@ Because VM is getting complex (one of reasons is memcg...), memcg's behavior is complex. This is a document for memcg's internal behavior. Please note that implementation details can be changed. -(*) Topics on API should be in Documentation/admin-guide/cgroup-v1/memory.rst) +(*) Topics on API should be in Documentation/admin-guide/cgroup-v1/memory.rst 0. How to record usage ? ======================== diff --git a/Documentation/admin-guide/cgroup-v2.rst b/Documentation/admin-guide/cgroup-v2.rst index de39319801b9..6cd7623e6b32 100644 --- a/Documentation/admin-guide/cgroup-v2.rst +++ b/Documentation/admin-guide/cgroup-v2.rst @@ -1145,7 +1145,7 @@ will be referred to. All time durations are in microseconds. This file exists whether the controller is enabled or not. It always reports the following three stats, which account for all the - processes in the cgroup: + processes in the cgroup (including those in descendant cgroups): - usage_usec - user_usec @@ -1160,6 +1160,27 @@ will be referred to. All time durations are in microseconds. - nr_bursts - burst_usec + Note that the above five CFS bandwidth stats are non-hierarchical; + they only account for throttling caused by this cgroup's own bandwidth + limit, not including throttling inherited from ancestor cgroups. + + cpu.stat.local + A read-only flat-keyed file. + This file exists whether the controller is enabled or not. + + It reports the following stat when the controller is enabled: + + - throttled_usec + + Unlike the ``throttled_usec`` reported by ``cpu.stat`` which + accounts for throttling caused by this cgroup's own CFS + bandwidth limit, ``cpu.stat.local`` reports the actual + throttling time incurred by this cgroup's own runqueues, + which may include throttling inherited from ancestor + cgroup bandwidth limits. + + When the controller is not enabled, this stat is not reported. + cpu.weight A read-write single value file which exists on non-root cgroups. The default is "100". @@ -1909,7 +1930,7 @@ The following nested keys are defined. is allowed unless memory.swap.max is set to 0. memory.pressure - A read-only nested-keyed file. + A read-write nested-keyed file. Shows pressure stall information for memory. See :ref:`Documentation/accounting/psi.rst ` for details. @@ -2169,7 +2190,7 @@ IO Interface Files 8:16 rbps=2097152 wbps=max riops=max wiops=max io.pressure - A read-only nested-keyed file. + A read-write nested-keyed file. Shows pressure stall information for IO. See :ref:`Documentation/accounting/psi.rst ` for details. @@ -2279,9 +2300,9 @@ This throttling takes 2 forms: throttled without possibly adversely affecting higher priority groups. This includes swapping and metadata IO. These types of IO are allowed to occur normally, however they are "charged" to the originating group. If the - originating group is being throttled you will see the use_delay and delay - fields in io.stat increase. The delay value is how many microseconds that are - being added to any process that runs in this group. Because this number can + originating group is being throttled you will see the use_delay and delay_nsec + fields in io.stat increase. The delay_nsec value is how many nanoseconds that + are being added to any process that runs in this group. Because this number can grow quite large if there is a lot of swapping or metadata IO occurring we limit the individual delay events to 1 second at a time. @@ -2530,6 +2551,13 @@ Cpuset Interface Files a need to change "cpuset.mems" with active tasks, it shouldn't be done frequently. + For a multithreaded process, the threadgroup leader is + considered the owner of the group's memory. Memory policy + rebinding and migration will only happen with respect to the + threadgroup leader. To avoid unexpected results, non-leading + threads shouldn't be put into another cgroup whose "cpuset.mems" + doesn't fully overlap that of the threadgroup leader. + cpuset.mems.effective A read-only multiple values file which exists on all cpuset-enabled cgroups. diff --git a/include/linux/cgroup-defs.h b/include/linux/cgroup-defs.h index de2cd6238c2a..7a631a257613 100644 --- a/include/linux/cgroup-defs.h +++ b/include/linux/cgroup-defs.h @@ -896,7 +896,7 @@ static inline void cgroup_threadgroup_change_begin(struct task_struct *tsk) * cgroup_threadgroup_change_end - threadgroup exclusion for cgroups * @tsk: target task * - * Counterpart of cgroup_threadcgroup_change_begin(). + * Counterpart of cgroup_threadgroup_change_begin(). */ static inline void cgroup_threadgroup_change_end(struct task_struct *tsk) { diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h index aa92db5f05de..303655d9deaf 100644 --- a/include/linux/cgroup.h +++ b/include/linux/cgroup.h @@ -506,7 +506,7 @@ static inline void cgroup_unlock(void) rcu_read_lock_sched_held() || \ lockdep_is_held(&cgroup_mutex) || \ lockdep_is_held(&css_set_lock) || \ - ((task)->flags & PF_EXITING) || (__c)) + (data_race((task)->flags) & PF_EXITING) || (__c)) #else #define task_css_set_check(task, __c) \ rcu_dereference((task)->cgroups) diff --git a/include/linux/psi.h b/include/linux/psi.h index e0745873e3f2..7966e3ac03b9 100644 --- a/include/linux/psi.h +++ b/include/linux/psi.h @@ -25,7 +25,9 @@ void psi_memstall_leave(unsigned long *flags); int psi_show(struct seq_file *s, struct psi_group *group, enum psi_res res); struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, enum psi_res res, struct file *file, - struct kernfs_open_file *of); + struct kernfs_open_file *of, + bool *need_rtpoll_worker); +int psi_trigger_create_rtpoll_worker(struct psi_group *group); void psi_trigger_destroy(struct psi_trigger *t); __poll_t psi_trigger_poll(void **trigger_ptr, struct file *file, diff --git a/kernel/cgroup/cgroup.c b/kernel/cgroup/cgroup.c index 2f6b634c84ca..918a3c1d5475 100644 --- a/kernel/cgroup/cgroup.c +++ b/kernel/cgroup/cgroup.c @@ -4071,6 +4071,7 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf, struct psi_trigger *new; struct cgroup *cgrp; struct psi_group *psi; + bool need_rtpoll_worker; ssize_t ret = 0; cgrp = cgroup_kn_lock_live(of->kn, false); @@ -4090,12 +4091,32 @@ static ssize_t pressure_write(struct kernfs_open_file *of, char *buf, } psi = cgroup_psi(cgrp); - new = psi_trigger_create(psi, buf, res, of->file, of); + new = psi_trigger_create(psi, buf, res, of->file, of, + &need_rtpoll_worker); if (IS_ERR(new)) { ret = PTR_ERR(new); goto out_unlock; } + /* + * The worker fork must run with neither cgroup_mutex nor the file's + * kernfs active reference held. The latter is broken since + * cgroup_kn_lock_live(). @of->priv may be released while unlocked, so + * recheck before publishing @new. + */ + if (need_rtpoll_worker) { + cgroup_unlock(); + ret = psi_trigger_create_rtpoll_worker(psi); + cgroup_lock(); + + if (!ret && !of->priv) + ret = -ENODEV; + if (ret) { + psi_trigger_destroy(new); + goto out_unlock; + } + } + smp_store_release(&ctx->psi.trigger, new); out_unlock: diff --git a/kernel/cgroup/cpuset-internal.h b/kernel/cgroup/cpuset-internal.h index f7aaf01f7cd5..e7d010661fd3 100644 --- a/kernel/cgroup/cpuset-internal.h +++ b/kernel/cgroup/cpuset-internal.h @@ -146,10 +146,9 @@ struct cpuset { nodemask_t old_mems_allowed; /* - * Tasks are being attached to this cpuset. Used to prevent - * zeroing cpus/mems_allowed between ->can_attach() and ->attach(). + * For linking impacted cpusets during an attach operation. */ - int attach_in_progress; + struct llist_node attach_node; /* partition root state */ int partition_root_state; @@ -165,7 +164,7 @@ struct cpuset { * number of SCHED_DEADLINE tasks attached to this cpuset, so that we * know when to rebuild associated root domain bandwidth information. */ - int nr_deadline_tasks; + atomic_t nr_deadline_tasks; int nr_migrate_dl_tasks; /* DL bandwidth that needs destination reservation for this attach. */ u64 sum_migrate_dl_bw; @@ -269,10 +268,7 @@ static inline int nr_cpusets(void) static inline bool cpuset_is_populated(struct cpuset *cs) { lockdep_assert_cpuset_lock_held(); - - /* Cpusets in the process of attaching should be considered as populated */ - return cgroup_is_populated(cs->css.cgroup) || - cs->attach_in_progress; + return cgroup_is_populated(cs->css.cgroup); } /** diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c index 9f6c7652c6d1..0a0fffb5673c 100644 --- a/kernel/cgroup/cpuset.c +++ b/kernel/cgroup/cpuset.c @@ -37,6 +37,7 @@ #include #include #include +#include DEFINE_STATIC_KEY_FALSE(cpusets_pre_enable_key); DEFINE_STATIC_KEY_FALSE(cpusets_enabled_key); @@ -222,14 +223,14 @@ void inc_dl_tasks_cs(struct task_struct *p) { struct cpuset *cs = task_cs(p); - cs->nr_deadline_tasks++; + atomic_inc(&cs->nr_deadline_tasks); } void dec_dl_tasks_cs(struct task_struct *p) { struct cpuset *cs = task_cs(p); - cs->nr_deadline_tasks--; + atomic_dec(&cs->nr_deadline_tasks); } static inline bool is_partition_valid(const struct cpuset *cs) @@ -356,6 +357,41 @@ static struct workqueue_struct *cpuset_migrate_mm_wq; static DECLARE_WAIT_QUEUE_HEAD(cpuset_attach_wq); +/* + * Cpuset task attach context + * Protected by cpuset_mutex + */ +static struct { + int in_progress; + bool cpus_updated; + bool mems_updated; + bool task_work_queued; + bool many_dest_cs; /* Have many destination cpusets */ + struct cpuset *old_cs; /* Source cpuset */ + nodemask_t nodemask_to; +} attach_ctx; +static LLIST_HEAD(src_cs_head); +static LLIST_HEAD(dst_cs_head); + +/* + * Wait if task attach is in progress until it is done and then acquire + * cpuset_mutex before returning. + */ +static void wait_attach_done_lock(void) + __acquires(&cpuset_mutex) +{ + for (;;) { + mutex_lock(&cpuset_mutex); + if (!attach_ctx.in_progress) + return; + + mutex_unlock(&cpuset_mutex); + + /* Wait until attach operation is done to prevent racing */ + wait_event(cpuset_attach_wq, attach_ctx.in_progress == 0); + } +} + static inline void check_insane_mems_config(nodemask_t *nodes) { if (!cpusets_insane_config() && @@ -368,22 +404,22 @@ static inline void check_insane_mems_config(nodemask_t *nodes) } /* - * decrease cs->attach_in_progress. - * wake_up cpuset_attach_wq if cs->attach_in_progress==0. + * decrease attach_ctx.in_progress. + * wake_up cpuset_attach_wq if attach_ctx.in_progress==0. */ -static inline void dec_attach_in_progress_locked(struct cpuset *cs) +static inline void dec_attach_in_progress_locked(void) { lockdep_assert_cpuset_lock_held(); - cs->attach_in_progress--; - if (!cs->attach_in_progress) + attach_ctx.in_progress--; + if (!attach_ctx.in_progress) wake_up(&cpuset_attach_wq); } -static inline void dec_attach_in_progress(struct cpuset *cs) +static inline void dec_attach_in_progress(void) { mutex_lock(&cpuset_mutex); - dec_attach_in_progress_locked(cs); + dec_attach_in_progress_locked(); mutex_unlock(&cpuset_mutex); } @@ -432,8 +468,7 @@ static inline bool partition_is_populated(struct cpuset *cs, * nr_populated_domain_children may include populated * csets from descendants that are partitions. */ - if (cgroup_has_tasks(cs->css.cgroup) || - cs->attach_in_progress) + if (cgroup_has_tasks(cs->css.cgroup)) return true; rcu_read_lock(); @@ -489,7 +524,10 @@ static void guarantee_active_cpus(struct task_struct *tsk, * Return in *pmask the portion of a cpusets's mems_allowed that * are online, with memory. If none are online with memory, walk * up the cpuset hierarchy until we find one that does have some - * online mems. The top cpuset always has some mems online. + * online mems. The top cpuset always has some mems online. With v2, + * effective_mems should always contain online memory nodes except + * during the transition period where a memory node hotunplug operation + * is in progress. * * One way or another, we guarantee to return some non-empty subset * of node_states[N_MEMORY]. @@ -581,6 +619,7 @@ static struct cpuset *dup_or_alloc_cpuset(struct cpuset *cs) return NULL; trial->dl_bw_cpu = -1; + init_llist_node(&trial->attach_node); /* Setup cpumask pointer array */ cpumask_var_t *pmask[4] = { @@ -918,7 +957,7 @@ static void dl_update_tasks_root_domain(struct cpuset *cs) struct css_task_iter it; struct task_struct *task; - if (cs->nr_deadline_tasks == 0) + if (atomic_read(&cs->nr_deadline_tasks) == 0) return; css_task_iter_start(&cs->css, 0, &it); @@ -1089,12 +1128,35 @@ void cpuset_update_tasks_cpumask(struct cpuset *cs, struct cpumask *new_cpus) * @cs: the cpuset the need to recompute the new effective_cpus mask * @parent: the parent cpuset * + * For v2, the parent's effective_cpus is inherited if cpumask is empty. * The result is valid only if the given cpuset isn't a partition root. */ static void compute_effective_cpumask(struct cpumask *new_cpus, struct cpuset *cs, struct cpuset *parent) { - cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus); + bool has_cpus; + + has_cpus = cpumask_and(new_cpus, cs->cpus_allowed, parent->effective_cpus); + if (!has_cpus && is_in_v2_mode()) + cpumask_copy(new_cpus, parent->effective_cpus); +} + +/** + * compute_effective_nodemask - Compute the effective nodemask of the cpuset + * @new_mems: the temp variable for the new effective_mems mask + * @cs: the cpuset the need to recompute the new effective_mems mask + * @parent: the parent cpuset + * + * For v2, the parent's effective_mems is inherited if nodemask is empty. + */ +static void compute_effective_nodemask(nodemask_t *new_mems, + struct cpuset *cs, struct cpuset *parent) +{ + bool has_mems; + + has_mems = nodes_and(*new_mems, cs->mems_allowed, parent->effective_mems); + if (!has_mems && is_in_v2_mode()) + nodes_copy(*new_mems, parent->effective_mems); } /* @@ -2143,15 +2205,6 @@ static void update_cpumasks_hier(struct cpuset *cs, struct tmpmasks *tmp, goto update_parent_effective; } - /* - * If it becomes empty, inherit the effective mask of the - * parent, which is guaranteed to have some CPUs unless - * it is a partition root that has explicitly distributed - * out all its CPUs. - */ - if (is_in_v2_mode() && !remote && cpumask_empty(tmp->new_cpus)) - cpumask_copy(tmp->new_cpus, parent->effective_cpus); - /* * Skip the whole subtree if * 1) the cpumask remains the same, @@ -2619,6 +2672,14 @@ static void *cpuset_being_rebound; * Iterate through each task of @cs updating its mems_allowed to the * effective cpuset's. As this function is called with cpuset_mutex held, * cpuset membership stays stable. + * + * - cpuset_change_task_nodemask(): guarantee_online_mems() + * - mpol_rebind_mm(): effective_mems + * - cpuset_migrate_mm(): guarantee_online_mems() + * - old_mems_allowed: guarantee_online_mems() + * + * For v2, guarantee_online_mems() should return a node mask that is the same + * as the effective_mems of current cpuset. */ void cpuset_update_tasks_nodemask(struct cpuset *cs) { @@ -2627,7 +2688,6 @@ void cpuset_update_tasks_nodemask(struct cpuset *cs) struct task_struct *task; cpuset_being_rebound = cs; /* causes mpol_dup() rebind */ - guarantee_online_mems(cs, &newmems); /* @@ -2647,6 +2707,10 @@ void cpuset_update_tasks_nodemask(struct cpuset *cs) cpuset_change_task_nodemask(task, &newmems); + /* Rebind and migrate mm only for thread group leader */ + if (!thread_group_leader(task)) + continue; + mm = get_task_mm(task); if (!mm) continue; @@ -2697,14 +2761,7 @@ static void update_nodemasks_hier(struct cpuset *cs, nodemask_t *new_mems) cpuset_for_each_descendant_pre(cp, pos_css, cs) { struct cpuset *parent = parent_cs(cp); - bool has_mems = nodes_and(*new_mems, cp->mems_allowed, parent->effective_mems); - - /* - * If it becomes empty, inherit the effective mask of the - * parent, which is guaranteed to have some MEMs. - */ - if (is_in_v2_mode() && !has_mems) - *new_mems = parent->effective_mems; + compute_effective_nodemask(new_mems, cp, parent); /* Skip the whole subtree if the nodemask remains the same. */ if (nodes_equal(*new_mems, cp->effective_mems)) { @@ -2973,58 +3030,47 @@ out: return 0; } -static struct cpuset *cpuset_attach_old_cs; - /* * Check to see if a cpuset can accept a new task * For v1, cpus_allowed and mems_allowed can't be empty. * For v2, effective_cpus can't be empty. * Note that in v1, effective_cpus = cpus_allowed. + * + * Also set the boolean flag passed in by @psetsched depending on if + * security_task_setscheduler() call is needed and @oldcs is not NULL. */ -static int cpuset_can_attach_check(struct cpuset *cs) +static int cpuset_can_attach_check(struct cpuset *cs, struct cpuset *oldcs, + bool *psetsched) { + bool cpus_updated, mems_updated; + if (cpumask_empty(cs->effective_cpus) || (!is_in_v2_mode() && nodes_empty(cs->mems_allowed))) return -ENOSPC; - return 0; -} -static void reset_migrate_dl_data(struct cpuset *cs) -{ - cs->nr_migrate_dl_tasks = 0; - cs->sum_migrate_dl_bw = 0; - cs->dl_bw_cpu = -1; -} + if (!oldcs) + return 0; -/* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */ -static int cpuset_can_attach(struct cgroup_taskset *tset) -{ - struct cgroup_subsys_state *css; - struct cpuset *cs, *oldcs; - struct task_struct *task; - bool setsched_check; - int cpu, ret; + if (!llist_on_list(&oldcs->attach_node)) + llist_add(&oldcs->attach_node, &src_cs_head); - /* used later by cpuset_attach() */ - cpuset_attach_old_cs = task_cs(cgroup_taskset_first(tset, &css)); - oldcs = cpuset_attach_old_cs; - cs = css_cs(css); + if (!llist_on_list(&cs->attach_node)) + llist_add(&cs->attach_node, &dst_cs_head); - mutex_lock(&cpuset_mutex); + cpus_updated = !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus); + mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems); - /* Check to see if task is allowed in the cpuset */ - ret = cpuset_can_attach_check(cs); - if (ret) - goto out_unlock; + if (cpus_updated) + attach_ctx.cpus_updated = true; + if (mems_updated) + attach_ctx.mems_updated = true; /* - * Skip rights over task setsched check in v2 when nothing changes, - * migration permission derives from hierarchy ownership in - * cgroup_procs_write_permission()). + * Skip rights over task setsched check in v2 when nothing changes for + * the current oldcs/cs pair, migration permission derives from + * hierarchy ownership in cgroup_procs_write_permission()). */ - setsched_check = !cpuset_v2() || - !cpumask_equal(cs->effective_cpus, oldcs->effective_cpus) || - !nodes_equal(cs->effective_mems, oldcs->effective_mems); + *psetsched = !cpuset_v2() || cpus_updated || mems_updated; /* * A v1 cpuset with tasks will have no CPU left only when CPU hotplug @@ -3034,13 +3080,123 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) * sure they will be able to run after migration. */ if (!is_in_v2_mode() && cpumask_empty(oldcs->effective_cpus)) - setsched_check = false; + *psetsched = false; + return 0; +} + +static int cpuset_reserve_dl_bw(void) +{ + struct cpuset *cs; + int cpu, ret; + + llist_for_each_entry(cs, dst_cs_head.first, attach_node) { + if (!cs->sum_migrate_dl_bw) + continue; + + cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus); + if (unlikely(cpu >= nr_cpu_ids)) + return -EINVAL; + + ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw); + if (ret) + return ret; + + cs->dl_bw_cpu = cpu; + } + return 0; +} + +/* + * Clear and optionally apply (@cancel is false) the attach related data in the + * source or destination cpuset. + */ +static void clear_attach_data(struct llist_head *head, bool cancel) +{ + struct cpuset *cs, *next; + struct llist_node *lnode = __llist_del_all(head); + + llist_for_each_entry_safe(cs, next, lnode, attach_node) { + init_llist_node(&cs->attach_node); + if (cs->nr_migrate_dl_tasks) { + if (!cancel) + atomic_add(cs->nr_migrate_dl_tasks, &cs->nr_deadline_tasks); + else if (cs->dl_bw_cpu >= 0) /* && cancel */ + dl_bw_free(cs->dl_bw_cpu, cs->sum_migrate_dl_bw); + cs->nr_migrate_dl_tasks = 0; + cs->sum_migrate_dl_bw = 0; + cs->dl_bw_cpu = -1; + } + } +} + +/* Called by cgroups to determine if a cpuset is usable; cpuset_mutex held */ +static int cpuset_can_attach(struct cgroup_taskset *tset) +{ + struct cgroup_subsys_state *css; + struct cpuset *cs, *oldcs; + struct task_struct *task; + bool setsched_check; + int ret; + + cs = oldcs = NULL; + mutex_lock(&cpuset_mutex); + attach_ctx.old_cs = NULL; /* Used later in cpuset_attach_task() */ + attach_ctx.cpus_updated = false; + attach_ctx.mems_updated = false; + attach_ctx.many_dest_cs = false; + + /* + * The attach_ctx.old_cs is used mainly by cpuset_migrate_mm() to get + * the old_mems_allowed value. There are two ways that many-to-one + * cpuset migration can happen: + * 1) A multithread application with threads in different cpusets is + * wholely migrated to a new cpuset. + * 2) Disabling v2 cpuset controller will move all the tasks in child + * cpusets to the parent cpuset. + * + * In the former case, it is the mm setting of the group leader that + * really matters. So attach_ctx.old_cs should track the oldcs of the + * group leader. It falls back to the oldcs of the first task if there + * is no group leader in the taskset. In the latter case, effective_mems + * of child cpusets must always be a subset of the parent. So no real + * page migration will be necessary no matter which child cpuset is + * selected as attach_ctx.old_cs. + * + * For a v2 threaded subtree where cpuset isn't enabled in some of the + * cgroups, it is possible that oldcs == cs for some of the tasks. + * In this case, we can skip checking on those tasks as there is no + * actual migration wrt cpuset. + */ cgroup_taskset_for_each(task, css, tset) { + struct cpuset *new_cs = css_cs(css); + struct cpuset *new_oldcs = task_cs(task); + + if ((new_oldcs != oldcs) || (new_cs != cs)) { + if (cs && (new_cs != cs)) + attach_ctx.many_dest_cs = true; + cs = new_cs; + oldcs = new_oldcs; + if (oldcs == cs) + continue; + if (!attach_ctx.old_cs) + attach_ctx.old_cs = oldcs; + ret = cpuset_can_attach_check(cs, oldcs, &setsched_check); + if (ret) + goto out_unlock; + } + + if (oldcs == cs) + continue; + ret = task_can_attach(task); if (ret) goto out_unlock; + /* Update attach_ctx.old_cs to the latest group leader */ + if (task == task->group_leader) + attach_ctx.old_cs = task_cs(task); + if (setsched_check) { ret = security_task_setscheduler(task); if (ret) @@ -3054,57 +3210,48 @@ static int cpuset_can_attach(struct cgroup_taskset *tset) * contribute to sum_migrate_dl_bw. */ cs->nr_migrate_dl_tasks++; + oldcs->nr_migrate_dl_tasks--; if (dl_task_needs_bw_move(task, cs->effective_cpus)) cs->sum_migrate_dl_bw += task->dl.dl_bw; } } - if (!cs->sum_migrate_dl_bw) - goto out_success; - - cpu = cpumask_any_and(cpu_active_mask, cs->effective_cpus); - if (unlikely(cpu >= nr_cpu_ids)) { + /* + * The only case where there are multiple destination cpusets for + * task migration is when enabling a v2 cpuset controllers where + * tasks will be migrated to multiple child cpusets from a parent + * cpuset with the same effective CPUs and memory nodes. IOW, + * both attach_cpus_updated and attach_mems_updated should be false. + * If not, it is a condition that the current code cannot handle. + * Print a warning and abort the attach operation as further code + * change may be needed. + */ + if (WARN_ON_ONCE(attach_ctx.many_dest_cs && (!cpuset_v2() || + attach_ctx.cpus_updated || attach_ctx.mems_updated))) { ret = -EINVAL; goto out_unlock; } - ret = dl_bw_alloc(cpu, cs->sum_migrate_dl_bw); - if (ret) - goto out_unlock; - - cs->dl_bw_cpu = cpu; - -out_success: - /* - * Mark attach is in progress. This makes validate_change() fail - * changes which zero cpus/mems_allowed. - */ - cs->attach_in_progress++; + ret = cpuset_reserve_dl_bw(); out_unlock: - if (ret) - reset_migrate_dl_data(cs); + if (ret) { + clear_attach_data(&src_cs_head, true); + clear_attach_data(&dst_cs_head, true); + } else { + attach_ctx.in_progress++; + } + mutex_unlock(&cpuset_mutex); return ret; } static void cpuset_cancel_attach(struct cgroup_taskset *tset) { - struct cgroup_subsys_state *css; - struct cpuset *cs; - - cgroup_taskset_first(tset, &css); - cs = css_cs(css); - mutex_lock(&cpuset_mutex); - dec_attach_in_progress_locked(cs); - - if (cs->dl_bw_cpu >= 0) - dl_bw_free(cs->dl_bw_cpu, cs->sum_migrate_dl_bw); - - if (cs->nr_migrate_dl_tasks) - reset_migrate_dl_data(cs); - + dec_attach_in_progress_locked(); + clear_attach_data(&src_cs_head, true); + clear_attach_data(&dst_cs_head, true); mutex_unlock(&cpuset_mutex); } @@ -3114,10 +3261,11 @@ static void cpuset_cancel_attach(struct cgroup_taskset *tset) * allocate from cpuset_init(). */ static cpumask_var_t cpus_attach; -static nodemask_t cpuset_attach_nodemask_to; static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task) { + struct mm_struct *mm; + lockdep_assert_cpuset_lock_held(); if (cs != &top_cpuset) @@ -3131,90 +3279,88 @@ static void cpuset_attach_task(struct cpuset *cs, struct task_struct *task) */ WARN_ON_ONCE(set_cpus_allowed_ptr(task, cpus_attach)); - cpuset_change_task_nodemask(task, &cpuset_attach_nodemask_to); + if (cpuset_v2() && !attach_ctx.mems_updated) + return; + + cpuset_change_task_nodemask(task, &attach_ctx.nodemask_to); cpuset1_update_task_spread_flags(cs, task); + + if ((task != task->group_leader) || !attach_ctx.mems_updated) + return; + + /* + * Change mm for threadgroup leader. This is expensive and may + * sleep and should be moved outside migration path proper. + */ + mm = get_task_mm(task); + if (mm) { + struct cpuset *oldcs = attach_ctx.old_cs; + + mpol_rebind_mm(mm, &cs->effective_mems); + + /* + * old_mems_allowed is the same with mems_allowed + * here, except if this task is being moved + * automatically due to hotplug. In that case + * @mems_allowed has been updated and is empty, so + * @old_mems_allowed is the right nodesets that we + * migrate mm from. + */ + if (is_memory_migrate(cs)) { + cpuset_migrate_mm(mm, &oldcs->old_mems_allowed, + &attach_ctx.nodemask_to); + attach_ctx.task_work_queued = true; + } else { + mmput(mm); + } + } } static void cpuset_attach(struct cgroup_taskset *tset) { struct task_struct *task; - struct task_struct *leader; struct cgroup_subsys_state *css; struct cpuset *cs; - struct cpuset *oldcs = cpuset_attach_old_cs; - bool cpus_updated, mems_updated; - bool queue_task_work = false; cgroup_taskset_first(tset, &css); cs = css_cs(css); lockdep_assert_cpus_held(); /* see cgroup_attach_lock() */ mutex_lock(&cpuset_mutex); - cpus_updated = !cpumask_equal(cs->effective_cpus, - oldcs->effective_cpus); - mems_updated = !nodes_equal(cs->effective_mems, oldcs->effective_mems); + attach_ctx.task_work_queued = false; + guarantee_online_mems(cs, &attach_ctx.nodemask_to); + + /* + * attach_ctx.old_cs can only be NULL if no task is actually migrating. + * This is highly unlikely. If it happens at all, we can skip task + * iteration and setting old_mems_allowed. + */ + if (unlikely(!attach_ctx.old_cs)) + goto out; /* * In the default hierarchy, enabling cpuset in the child cgroups - * will trigger a number of cpuset_attach() calls with no change - * in effective cpus and mems. In that case, we can optimize out - * by skipping the task iteration and update. + * will trigger a cpuset_attach() call with no change in effective cpus + * and mems. In that case, we can optimize out by skipping the task + * iteration and the destination cpuset list is iterated to set + * old_mems_allowed. */ - if (cpuset_v2() && !cpus_updated && !mems_updated) { - cpuset_attach_nodemask_to = cs->effective_mems; + if (cpuset_v2() && !attach_ctx.cpus_updated && !attach_ctx.mems_updated) { + llist_for_each_entry(cs, dst_cs_head.first, attach_node) + cs->old_mems_allowed = attach_ctx.nodemask_to; goto out; } - guarantee_online_mems(cs, &cpuset_attach_nodemask_to); - cgroup_taskset_for_each(task, css, tset) cpuset_attach_task(cs, task); - /* - * Change mm for all threadgroup leaders. This is expensive and may - * sleep and should be moved outside migration path proper. Skip it - * if there is no change in effective_mems and CS_MEMORY_MIGRATE is - * not set. - */ - cpuset_attach_nodemask_to = cs->effective_mems; - if (!is_memory_migrate(cs) && !mems_updated) - goto out; - - cgroup_taskset_for_each_leader(leader, css, tset) { - struct mm_struct *mm = get_task_mm(leader); - - if (mm) { - mpol_rebind_mm(mm, &cpuset_attach_nodemask_to); - - /* - * old_mems_allowed is the same with mems_allowed - * here, except if this task is being moved - * automatically due to hotplug. In that case - * @mems_allowed has been updated and is empty, so - * @old_mems_allowed is the right nodesets that we - * migrate mm from. - */ - if (is_memory_migrate(cs)) { - cpuset_migrate_mm(mm, &oldcs->old_mems_allowed, - &cpuset_attach_nodemask_to); - queue_task_work = true; - } else - mmput(mm); - } - } - -out: - if (queue_task_work) + if (attach_ctx.task_work_queued) schedule_flush_migrate_mm(); - cs->old_mems_allowed = cpuset_attach_nodemask_to; - - if (cs->nr_migrate_dl_tasks) { - cs->nr_deadline_tasks += cs->nr_migrate_dl_tasks; - oldcs->nr_deadline_tasks -= cs->nr_migrate_dl_tasks; - reset_migrate_dl_data(cs); - } - - dec_attach_in_progress_locked(cs); + cs->old_mems_allowed = attach_ctx.nodemask_to; +out: + clear_attach_data(&src_cs_head, false); + clear_attach_data(&dst_cs_head, false); + dec_attach_in_progress_locked(); mutex_unlock(&cpuset_mutex); } @@ -3234,7 +3380,12 @@ ssize_t cpuset_write_resmask(struct kernfs_open_file *of, return -EACCES; buf = strstrip(buf); - cpuset_full_lock(); + + /* cpuset_mutex acquired in wait_attach_done_lock() */ + mutex_lock(&cpuset_top_mutex); + cpus_read_lock(); + wait_attach_done_lock(); + if (!is_cpuset_online(cs)) goto out_unlock; @@ -3365,7 +3516,10 @@ static ssize_t cpuset_partition_write(struct kernfs_open_file *of, char *buf, else return -EINVAL; - cpuset_full_lock(); + mutex_lock(&cpuset_top_mutex); + cpus_read_lock(); + wait_attach_done_lock(); + if (is_cpuset_online(cs)) retval = update_prstate(cs, val); cpuset_update_sd_hk_unlock(); @@ -3592,7 +3746,7 @@ static int cpuset_can_fork(struct task_struct *task, struct css_set *cset) mutex_lock(&cpuset_mutex); /* Check to see if task is allowed in the cpuset */ - ret = cpuset_can_attach_check(cs); + ret = cpuset_can_attach_check(cs, NULL, NULL); if (ret) goto out_unlock; @@ -3604,11 +3758,7 @@ static int cpuset_can_fork(struct task_struct *task, struct css_set *cset) if (ret) goto out_unlock; - /* - * Mark attach is in progress. This makes validate_change() fail - * changes which zero cpus/mems_allowed. - */ - cs->attach_in_progress++; + attach_ctx.in_progress++; out_unlock: mutex_unlock(&cpuset_mutex); return ret; @@ -3626,7 +3776,7 @@ static void cpuset_cancel_fork(struct task_struct *task, struct css_set *cset) if (same_cs) return; - dec_attach_in_progress(cs); + dec_attach_in_progress(); } /* @@ -3636,15 +3786,14 @@ static void cpuset_cancel_fork(struct task_struct *task, struct css_set *cset) */ static void cpuset_fork(struct task_struct *task) { - struct cpuset *cs; - bool same_cs; + struct cpuset *cs, *oldcs; rcu_read_lock(); cs = task_cs(task); - same_cs = (cs == task_cs(current)); + oldcs = task_cs(current); rcu_read_unlock(); - if (same_cs) { + if (cs == oldcs) { if (cs == &top_cpuset) return; @@ -3655,10 +3804,22 @@ static void cpuset_fork(struct task_struct *task) /* CLONE_INTO_CGROUP */ mutex_lock(&cpuset_mutex); - guarantee_online_mems(cs, &cpuset_attach_nodemask_to); - cpuset_attach_task(cs, task); + guarantee_online_mems(cs, &attach_ctx.nodemask_to); + cs->old_mems_allowed = attach_ctx.nodemask_to; - dec_attach_in_progress_locked(cs); + /* + * Assume CPUs and memory nodes are updated + * A CLONE_INTO_CGROUP operation should have taken the cgroup mutex + * and so there shouldn't be a competing cpuset_attach() operation. + */ + attach_ctx.cpus_updated = attach_ctx.mems_updated = true; + attach_ctx.task_work_queued = false; + attach_ctx.old_cs = oldcs; + cpuset_attach_task(cs, task); + if (attach_ctx.task_work_queued) + schedule_flush_migrate_mm(); + + dec_attach_in_progress_locked(); mutex_unlock(&cpuset_mutex); } @@ -3705,6 +3866,7 @@ int __init cpuset_init(void) cpumask_setall(top_cpuset.effective_xcpus); cpumask_setall(top_cpuset.exclusive_cpus); nodes_setall(top_cpuset.effective_mems); + init_llist_node(&top_cpuset.attach_node); cpuset1_init(&top_cpuset); @@ -3762,23 +3924,11 @@ static void cpuset_hotplug_update_tasks(struct cpuset *cs, struct tmpmasks *tmp) bool remote; int partcmd = -1; struct cpuset *parent; -retry: - wait_event(cpuset_attach_wq, cs->attach_in_progress == 0); - - mutex_lock(&cpuset_mutex); - - /* - * We have raced with task attaching. We wait until attaching - * is finished, so we won't attach a task to an empty cpuset. - */ - if (cs->attach_in_progress) { - mutex_unlock(&cpuset_mutex); - goto retry; - } + wait_attach_done_lock(); parent = parent_cs(cs); compute_effective_cpumask(&new_cpus, cs, parent); - nodes_and(new_mems, cs->mems_allowed, parent->effective_mems); + compute_effective_nodemask(&new_mems, cs, parent); if (!tmp || !cs->partition_root_state) goto update_tasks; diff --git a/kernel/sched/psi.c b/kernel/sched/psi.c index 848955f8893d..4e152410653d 100644 --- a/kernel/sched/psi.c +++ b/kernel/sched/psi.c @@ -1134,6 +1134,12 @@ void psi_cgroup_free(struct cgroup *cgroup) return; cancel_delayed_work_sync(&cgroup->psi->avgs_work); + /* + * A psi_schedule_rtpoll_work() call racing the last trigger's + * destruction may have re-armed the timer after psi_trigger_destroy() + * deleted it. Spurious firing while the group is alive is harmless. + */ + timer_shutdown_sync(&cgroup->psi->rtpoll_timer); free_percpu(cgroup->psi->pcpu); /* All triggers must be removed by now */ WARN_ONCE(cgroup->psi->rtpoll_states, "psi: trigger leak\n"); @@ -1292,9 +1298,44 @@ int psi_show(struct seq_file *m, struct psi_group *group, enum psi_res res) return 0; } +/* + * Create @group's rtpoll worker after psi_trigger_create() reported the need + * for one. kthread creation depends on the whole fork path and we don't want + * all of that nested inside cgroup_mutex, so the caller must drop it and any + * other lock that forks can wait behind. If two callers race, the loser stops + * its never-woken kthread. + */ +int psi_trigger_create_rtpoll_worker(struct psi_group *group) +{ + struct task_struct *task; + + task = kthread_create(psi_rtpoll_worker, group, "psimon"); + if (IS_ERR(task)) + return PTR_ERR(task); + + scoped_guard(mutex, &group->rtpoll_trigger_lock) { + if (!rcu_access_pointer(group->rtpoll_task)) { + atomic_set(&group->rtpoll_wakeup, 0); + wake_up_process(task); + rcu_assign_pointer(group->rtpoll_task, task); + + /* + * Poll once to catch up on scheduling attempts dropped + * while there was no rtpoll worker. + */ + psi_schedule_rtpoll_work(group, 1, true); + return 0; + } + } + + kthread_stop(task); + return 0; +} + struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, enum psi_res res, struct file *file, - struct kernfs_open_file *of) + struct kernfs_open_file *of, + bool *need_rtpoll_worker) { struct psi_trigger *t; enum psi_states state; @@ -1302,6 +1343,8 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, bool privileged; u32 window_us; + *need_rtpoll_worker = false; + if (static_branch_likely(&psi_disabled)) return ERR_PTR(-EOPNOTSUPP); @@ -1362,26 +1405,14 @@ struct psi_trigger *psi_trigger_create(struct psi_group *group, char *buf, if (privileged) { mutex_lock(&group->rtpoll_trigger_lock); - if (!rcu_access_pointer(group->rtpoll_task)) { - struct task_struct *task; - - task = kthread_create(psi_rtpoll_worker, group, "psimon"); - if (IS_ERR(task)) { - kfree(t); - mutex_unlock(&group->rtpoll_trigger_lock); - return ERR_CAST(task); - } - atomic_set(&group->rtpoll_wakeup, 0); - wake_up_process(task); - rcu_assign_pointer(group->rtpoll_task, task); - } - list_add(&t->node, &group->rtpoll_triggers); group->rtpoll_min_period = min(group->rtpoll_min_period, div_u64(t->win.size, UPDATES_PER_WINDOW)); group->rtpoll_nr_triggers[t->state]++; group->rtpoll_states |= (1 << t->state); + *need_rtpoll_worker = !rcu_access_pointer(group->rtpoll_task); + mutex_unlock(&group->rtpoll_trigger_lock); } else { mutex_lock(&group->avgs_lock); @@ -1541,6 +1572,8 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf, size_t buf_size; struct seq_file *seq; struct psi_trigger *new; + bool need_rtpoll_worker; + int ret; if (static_branch_likely(&psi_disabled)) return -EOPNOTSUPP; @@ -1565,12 +1598,22 @@ static ssize_t psi_write(struct file *file, const char __user *user_buf, return -EBUSY; } - new = psi_trigger_create(&psi_system, buf, res, file, NULL); + new = psi_trigger_create(&psi_system, buf, res, file, NULL, + &need_rtpoll_worker); if (IS_ERR(new)) { mutex_unlock(&seq->lock); return PTR_ERR(new); } + if (need_rtpoll_worker) { + ret = psi_trigger_create_rtpoll_worker(&psi_system); + if (ret) { + psi_trigger_destroy(new); + mutex_unlock(&seq->lock); + return ret; + } + } + smp_store_release(&seq->private, new); mutex_unlock(&seq->lock); diff --git a/tools/cgroup/iocost_monitor.py b/tools/cgroup/iocost_monitor.py index 933c750b319b..bdd78ba271b0 100644 --- a/tools/cgroup/iocost_monitor.py +++ b/tools/cgroup/iocost_monitor.py @@ -15,11 +15,6 @@ import time import json import math -import drgn -from drgn import container_of -from drgn.helpers.linux.list import list_for_each_entry,list_empty -from drgn.helpers.linux.radixtree import radix_tree_for_each,radix_tree_lookup - import argparse parser = argparse.ArgumentParser(description=desc, formatter_class=argparse.RawTextHelpFormatter) @@ -34,6 +29,11 @@ parser.add_argument('--json', action='store_true', help='Output in json') args = parser.parse_args() +import drgn +from drgn import container_of +from drgn.helpers.linux.list import list_for_each_entry,list_empty +from drgn.helpers.linux.radixtree import radix_tree_for_each,radix_tree_lookup + def err(s): print(s, file=sys.stderr, flush=True) sys.exit(1) diff --git a/tools/testing/selftests/cgroup/lib/include/cgroup_util.h b/tools/testing/selftests/cgroup/lib/include/cgroup_util.h index febc1723d090..8ebb2b4d4ec0 100644 --- a/tools/testing/selftests/cgroup/lib/include/cgroup_util.h +++ b/tools/testing/selftests/cgroup/lib/include/cgroup_util.h @@ -8,6 +8,7 @@ #define MB(x) (x << 20) +#define NSEC_PER_USEC 1000L #define USEC_PER_SEC 1000000L #define NSEC_PER_SEC 1000000000L diff --git a/tools/testing/selftests/cgroup/test_core.c b/tools/testing/selftests/cgroup/test_core.c index 88ca832d4fc1..8e3f9b391e44 100644 --- a/tools/testing/selftests/cgroup/test_core.c +++ b/tools/testing/selftests/cgroup/test_core.c @@ -426,7 +426,6 @@ static int test_cgcore_no_internal_process_constraint_on_threads(const char *roo ret = KSFT_PASS; cleanup: - cg_enter_current(root); cg_enter_current(root); if (child) cg_destroy(child); diff --git a/tools/testing/selftests/cgroup/test_cpu.c b/tools/testing/selftests/cgroup/test_cpu.c index 7a40d76b9548..a5eccfcabef5 100644 --- a/tools/testing/selftests/cgroup/test_cpu.c +++ b/tools/testing/selftests/cgroup/test_cpu.c @@ -639,6 +639,31 @@ test_cpucg_nested_weight_underprovisioned(const char *root) return run_cpucg_nested_weight_test(root, false); } +/* + * Best effort attempt to get the kernel's HZ value from the config. + * Return the HZ value if found otherwise return 1000 (the default) to + * indicate failure. + */ +static long +get_config_hz(void) +{ + long hz = 1000; + FILE *f; + char cmd[256] = "zcat /proc/config.gz 2>/dev/null | grep '^CONFIG_HZ='"; + + f = popen(cmd, "r"); + + if (!f) + return hz; + + if (fscanf(f, "CONFIG_HZ=%ld", &hz) == EOF) + goto out; + +out: + pclose(f); + return hz; +} + /* * This test creates a cgroup with some maximum value within a period, and * verifies that a process in the cgroup is not overscheduled. @@ -646,15 +671,18 @@ test_cpucg_nested_weight_underprovisioned(const char *root) static int test_cpucg_max(const char *root) { int ret = KSFT_FAIL; + long hz = get_config_hz(); long quota_usec = 1000; long default_period_usec = 100000; /* cpu.max's default period */ long duration_seconds = 1; - long duration_usec = duration_seconds * USEC_PER_SEC; + long duration_usec; long usage_usec, n_periods, remainder_usec, expected_usage_usec; char *cpucg; char quota_buf[32]; + duration_usec = duration_seconds * USEC_PER_SEC * 1000 / hz; + snprintf(quota_buf, sizeof(quota_buf), "%ld", quota_usec); cpucg = cg_name(root, "cpucg_test"); @@ -670,8 +698,8 @@ static int test_cpucg_max(const char *root) struct cpu_hog_func_param param = { .nprocs = 1, .ts = { - .tv_sec = duration_seconds, - .tv_nsec = 0, + .tv_sec = duration_usec / USEC_PER_SEC, + .tv_nsec = duration_usec % USEC_PER_SEC * NSEC_PER_USEC, }, .clock_type = CPU_HOG_CLOCK_WALL, }; @@ -710,15 +738,18 @@ cleanup: static int test_cpucg_max_nested(const char *root) { int ret = KSFT_FAIL; + long hz = get_config_hz(); long quota_usec = 1000; long default_period_usec = 100000; /* cpu.max's default period */ long duration_seconds = 1; - long duration_usec = duration_seconds * USEC_PER_SEC; + long duration_usec; long usage_usec, n_periods, remainder_usec, expected_usage_usec; char *parent, *child; char quota_buf[32]; + duration_usec = duration_seconds * USEC_PER_SEC * 1000 / hz; + snprintf(quota_buf, sizeof(quota_buf), "%ld", quota_usec); parent = cg_name(root, "cpucg_parent"); @@ -741,8 +772,8 @@ static int test_cpucg_max_nested(const char *root) struct cpu_hog_func_param param = { .nprocs = 1, .ts = { - .tv_sec = duration_seconds, - .tv_nsec = 0, + .tv_sec = duration_usec / USEC_PER_SEC, + .tv_nsec = duration_usec % USEC_PER_SEC * NSEC_PER_USEC, }, .clock_type = CPU_HOG_CLOCK_WALL, }; diff --git a/tools/testing/selftests/cgroup/test_cpuset.c b/tools/testing/selftests/cgroup/test_cpuset.c index c5cf8b56ceb8..8c2d4d4ef1fc 100644 --- a/tools/testing/selftests/cgroup/test_cpuset.c +++ b/tools/testing/selftests/cgroup/test_cpuset.c @@ -1,7 +1,13 @@ // SPDX-License-Identifier: GPL-2.0 +#define _GNU_SOURCE +#include #include +#include +#include #include +#include +#include #include "kselftest.h" #include "cgroup_util.h" @@ -232,6 +238,246 @@ cleanup: return ret; } +static int get_cpu_affinity(cpu_set_t *mask) +{ + CPU_ZERO(mask); + return sched_getaffinity(0, sizeof(*mask), mask); +} + +static int cpu_set_equal(cpu_set_t *dst, unsigned long mask) +{ + cpu_set_t expected; + + CPU_ZERO(&expected); + assert(sizeof(mask) < CPU_SETSIZE); + + for (int cpu = 0; cpu < sizeof(mask) * 8; ++cpu) + if ((1UL << cpu) & mask) + CPU_SET(cpu, &expected); + + return CPU_EQUAL(&expected, dst); +} + +enum test_phase { + AFFINITY_SETUP, + AFFINITY_CONTROLLER_DISABLED, + AFFINITY_COMPLETE, + AFFINITY_ERROR +}; + +struct thread_args { + const char *cgroup; + cpu_set_t *affinity_before; + cpu_set_t *affinity_after; + int affinity_before_ready; +}; + +static pthread_mutex_t test_mutex = PTHREAD_MUTEX_INITIALIZER; +static pthread_cond_t test_cond = PTHREAD_COND_INITIALIZER; +static enum test_phase test_phase; + +static void *affinity_thread_fn(void *arg) +{ + struct thread_args *args = (struct thread_args *)arg; + + if (cg_enter_current_thread(args->cgroup)) + goto fail; + + if (get_cpu_affinity(args->affinity_before) != 0) + goto fail; + + pthread_mutex_lock(&test_mutex); + args->affinity_before_ready = 1; + pthread_cond_broadcast(&test_cond); + + while (test_phase < AFFINITY_CONTROLLER_DISABLED) + pthread_cond_wait(&test_cond, &test_mutex); + pthread_mutex_unlock(&test_mutex); + + if (get_cpu_affinity(args->affinity_after) != 0) + goto fail; + + + return NULL; + +fail: + pthread_mutex_lock(&test_mutex); + test_phase = AFFINITY_ERROR; + pthread_cond_broadcast(&test_cond); + pthread_mutex_unlock(&test_mutex); + return NULL; +} + +/* + * Test that disabling cpuset controller properly updates thread affinity. + * + * This test exposes a bug in cpuset_attach() where threads in child cgroups + * don't get their affinity updated when the cpuset controller is disabled. + * + * Setup: + * - Create parent cgroup with cpuset.cpus=0-1 + * - Create child A with cpuset.cpus=0-1 + * - Create child B with cpuset.cpus=1 + * - Place multithreaded process: group leader + thread_a in A, thread_b in B + * - Disable cpuset controller on parent + * + * Expected: thread_b's affinity should expand from {1} to {0-1} + * Buggy: thread_b's affinity remains {1} + */ +static int test_cpuset_affinity_on_controller_disable(const char *root) +{ + char *parent = NULL, *child_a = NULL, *child_b = NULL; + pthread_t thread_a, thread_b; + int thread_a_created = 0, thread_b_created = 0; + cpu_set_t affinity_a_before, affinity_a_after; + cpu_set_t affinity_b_before, affinity_b_after; + int ret = KSFT_FAIL; + + parent = cg_name(root, "cpuset_affinity_test"); + if (!parent) + goto cleanup; + if (cg_create(parent)) + goto cleanup; + if (cg_write(parent, "cgroup.type", "threaded")) + goto cleanup; + + child_a = cg_name(parent, "A"); + if (!child_a) + goto cleanup; + if (cg_create(child_a)) + goto cleanup; + if (cg_write(child_a, "cgroup.type", "threaded")) + goto cleanup; + + child_b = cg_name(parent, "B"); + if (!child_b) + goto cleanup; + if (cg_create(child_b)) + goto cleanup; + if (cg_write(child_b, "cgroup.type", "threaded")) + goto cleanup; + + /* Now enable cpuset controller in parent */ + if (cg_write(parent, "cgroup.subtree_control", "+cpuset")) + goto skip; + + /* + * Set CPU affinity constraints + * Skip the test if the setting of "cpuset.cpus" fails as the test + * system may not have CPU 1. + */ + if (cg_write(parent, "cpuset.cpus", "0-1")) + goto skip; + if (cg_write(child_a, "cpuset.cpus", "0-1")) + goto skip; + if (cg_write(child_b, "cpuset.cpus", "1")) + goto skip; + + /* Move group leader (main thread) to child A */ + if (cg_enter_current(child_a)) + goto cleanup; + + /* Create threads - they will move themselves to their respective cgroups */ + test_phase = AFFINITY_SETUP; + + struct thread_args args_a = { + .cgroup = child_a, + .affinity_before = &affinity_a_before, + .affinity_after = &affinity_a_after, + .affinity_before_ready = 0, + }; + if (pthread_create(&thread_a, NULL, affinity_thread_fn, &args_a)) + goto cleanup; + thread_a_created = 1; + + struct thread_args args_b = { + .cgroup = child_b, + .affinity_before = &affinity_b_before, + .affinity_after = &affinity_b_after, + .affinity_before_ready = 0, + }; + if (pthread_create(&thread_b, NULL, affinity_thread_fn, &args_b)) + goto cleanup_threads; + thread_b_created = 1; + + pthread_mutex_lock(&test_mutex); + while ((test_phase < AFFINITY_ERROR) && + (args_a.affinity_before_ready + args_b.affinity_before_ready < 2)) + pthread_cond_wait(&test_cond, &test_mutex); + + /* If a thread failed during setup, bail out */ + if (test_phase == AFFINITY_ERROR) { + pthread_mutex_unlock(&test_mutex); + goto cleanup_threads; + } + pthread_mutex_unlock(&test_mutex); + + if (!cpu_set_equal(&affinity_a_before, 0x3)) { + ksft_print_msg("FAIL: thread_a initial affinity incorrect\n"); + goto cleanup_threads; + } + + if (!cpu_set_equal(&affinity_b_before, 0x2)) { + ksft_print_msg("FAIL: thread_b initial affinity incorrect\n"); + goto cleanup_threads; + } + + /* Disable cpuset controller - this should trigger affinity update */ + if (cg_write(parent, "cgroup.subtree_control", "-cpuset")) + goto cleanup_threads; + + /* Signal threads to save their final affinity and exit */ + pthread_mutex_lock(&test_mutex); + test_phase = AFFINITY_CONTROLLER_DISABLED; + pthread_cond_broadcast(&test_cond); + pthread_mutex_unlock(&test_mutex); + + pthread_join(thread_a, NULL); + pthread_join(thread_b, NULL); + + /* Verify thread affinities AFTER disabling controller */ + if (!cpu_set_equal(&affinity_a_after, 0x3)) { + ksft_print_msg("FAIL: thread_a final affinity incorrect\n"); + goto cleanup; + } + + if (!cpu_set_equal(&affinity_b_after, 0x3)) { + ksft_print_msg("FAIL: thread_b affinity did not expand to {0-1}\n"); + goto cleanup; + } + + ret = KSFT_PASS; + goto cleanup; + +skip: + ret = KSFT_SKIP; + goto cleanup; + +cleanup_threads: + pthread_mutex_lock(&test_mutex); + test_phase = AFFINITY_COMPLETE; + pthread_cond_broadcast(&test_cond); + pthread_mutex_unlock(&test_mutex); + + if (thread_a_created) + pthread_join(thread_a, NULL); + if (thread_b_created) + pthread_join(thread_b, NULL); + +cleanup: + /* Move back to root before cleanup */ + cg_enter_current(root); + + cg_destroy(child_b); + free(child_b); + cg_destroy(child_a); + free(child_a); + cg_destroy(parent); + free(parent); + + return ret; +} + #define T(x) { x, #x } struct cpuset_test { @@ -241,6 +487,7 @@ struct cpuset_test { T(test_cpuset_perms_object_allow), T(test_cpuset_perms_object_deny), T(test_cpuset_perms_subtree), + T(test_cpuset_affinity_on_controller_disable), }; #undef T diff --git a/tools/testing/selftests/cgroup/test_cpuset_prs.sh b/tools/testing/selftests/cgroup/test_cpuset_prs.sh index 0d41aa0d343d..ca9bc38fdb95 100755 --- a/tools/testing/selftests/cgroup/test_cpuset_prs.sh +++ b/tools/testing/selftests/cgroup/test_cpuset_prs.sh @@ -495,13 +495,20 @@ REMOTE_TEST_MATRIX=( # Narrowing cpuset.cpus to previously sibling-excluded CPUs should # not return CPUs that were never actually owned. " C1-4:P1 . C1-2:P1 C1-3:P2 . . \ - . . . C3 . . p1:4|c11:1-2|c12:3 \ + . . . C3 . . p1:4|c11:1-2|c12:3 \ p1:P1|c11:P1|c12:P2 3" # Expanding cpuset.cpus to include a previously sibling-excluded CPU # after the sibling has become a member should correctly request it. " C1-4:P1 . C1-2:P1 C1-3:P2 . . \ - . . P0 C2-3 . . p1:1,4|c11:1|c12:2-3 \ + . . P0 C2-3 . . p1:1,4|c11:1|c12:2-3 \ p1:P1|c11:P0|c12:P2 2-3" + # Cpusets with empty cpuset.cpus should inherit parent's effective_cpus + " C1-4:P1 C5-6 C1-2 . C5 . \ + . P1 P1 . . . p1:3-4|p2:5-6|c11:1-2|c12:3-4|c21:5|c22:5-6 \ + p1:P1|p2:P1|c11:P1" + " C1-4:P1 C5-6 C1-2 . C5 . \ + . P1 P1 . O5=0 . p1:3-4|p2:6|c11:1-2|c12:3-4|c21:6|c22:6 \ + p1:P1|p2:P1|c11:P1" ) # diff --git a/tools/testing/selftests/cgroup/test_hugetlb_memcg.c b/tools/testing/selftests/cgroup/test_hugetlb_memcg.c index b627d84358b1..8c5aced813b6 100644 --- a/tools/testing/selftests/cgroup/test_hugetlb_memcg.c +++ b/tools/testing/selftests/cgroup/test_hugetlb_memcg.c @@ -199,7 +199,10 @@ out: int main(int argc, char **argv) { char root[PATH_MAX]; - int ret = EXIT_SUCCESS, has_memory_hugetlb_acc; + int has_memory_hugetlb_acc; + + ksft_print_header(); + ksft_set_plan(1); has_memory_hugetlb_acc = proc_mount_contains("memory_hugetlb_accounting"); if (has_memory_hugetlb_acc < 0) @@ -211,7 +214,7 @@ int main(int argc, char **argv) if (get_hugepage_size() != 2048) { ksft_print_msg("test_hugetlb_memcg requires 2MB hugepages\n"); ksft_test_result_skip("test_hugetlb_memcg\n"); - return ret; + ksft_finished(); } if (cg_find_unified_root(root, sizeof(root), NULL)) @@ -233,10 +236,9 @@ int main(int argc, char **argv) ksft_test_result_skip("test_hugetlb_memcg\n"); break; default: - ret = EXIT_FAILURE; ksft_test_result_fail("test_hugetlb_memcg\n"); break; } - return ret; + ksft_finished(); }