You've already forked linux-apfs
mirror of
https://github.com/linux-apfs/linux-apfs.git
synced 2026-05-01 15:00:59 -07:00
sched: Clean up and harmonize the coding style of the scheduler code base
A good number of small style inconsistencies have accumulated
in the scheduler core, so do a pass over them to harmonize
all these details:
- fix speling in comments,
- use curly braces for multi-line statements,
- remove unnecessary parentheses from integer literals,
- capitalize consistently,
- remove stray newlines,
- add comments where necessary,
- remove invalid/unnecessary comments,
- align structure definitions and other data types vertically,
- add missing newlines for increased readability,
- fix vertical tabulation where it's misaligned,
- harmonize preprocessor conditional block labeling
and vertical alignment,
- remove line-breaks where they uglify the code,
- add newline after local variable definitions,
No change in functionality:
md5:
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.before.asm
1191fa0a890cfa8132156d2959d7e9e2 built-in.o.after.asm
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: linux-kernel@vger.kernel.org
Signed-off-by: Ingo Molnar <mingo@kernel.org>
This commit is contained in:
@@ -168,18 +168,19 @@ autogroup_move_group(struct task_struct *p, struct autogroup *ag)
|
||||
autogroup_kref_put(prev);
|
||||
}
|
||||
|
||||
/* Allocates GFP_KERNEL, cannot be called under any spinlock */
|
||||
/* Allocates GFP_KERNEL, cannot be called under any spinlock: */
|
||||
void sched_autogroup_create_attach(struct task_struct *p)
|
||||
{
|
||||
struct autogroup *ag = autogroup_create();
|
||||
|
||||
autogroup_move_group(p, ag);
|
||||
/* drop extra reference added by autogroup_create() */
|
||||
|
||||
/* Drop extra reference added by autogroup_create(): */
|
||||
autogroup_kref_put(ag);
|
||||
}
|
||||
EXPORT_SYMBOL(sched_autogroup_create_attach);
|
||||
|
||||
/* Cannot be called under siglock. Currently has no users */
|
||||
/* Cannot be called under siglock. Currently has no users: */
|
||||
void sched_autogroup_detach(struct task_struct *p)
|
||||
{
|
||||
autogroup_move_group(p, &autogroup_default);
|
||||
@@ -202,7 +203,6 @@ static int __init setup_autogroup(char *str)
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
__setup("noautogroup", setup_autogroup);
|
||||
|
||||
#ifdef CONFIG_PROC_FS
|
||||
@@ -224,7 +224,7 @@ int proc_sched_autogroup_set_nice(struct task_struct *p, int nice)
|
||||
if (nice < 0 && !can_nice(current, nice))
|
||||
return -EPERM;
|
||||
|
||||
/* this is a heavy operation taking global locks.. */
|
||||
/* This is a heavy operation, taking global locks.. */
|
||||
if (!capable(CAP_SYS_ADMIN) && time_before(jiffies, next))
|
||||
return -EAGAIN;
|
||||
|
||||
@@ -267,4 +267,4 @@ int autogroup_path(struct task_group *tg, char *buf, int buflen)
|
||||
|
||||
return snprintf(buf, buflen, "%s-%ld", "/autogroup", tg->autogroup->id);
|
||||
}
|
||||
#endif /* CONFIG_SCHED_DEBUG */
|
||||
#endif
|
||||
|
||||
@@ -7,9 +7,9 @@
|
||||
|
||||
struct autogroup {
|
||||
/*
|
||||
* reference doesn't mean how many thread attach to this
|
||||
* autogroup now. It just stands for the number of task
|
||||
* could use this autogroup.
|
||||
* Reference doesn't mean how many threads attach to this
|
||||
* autogroup now. It just stands for the number of tasks
|
||||
* which could use this autogroup.
|
||||
*/
|
||||
struct kref kref;
|
||||
struct task_group *tg;
|
||||
@@ -56,11 +56,9 @@ autogroup_task_group(struct task_struct *p, struct task_group *tg)
|
||||
return tg;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_SCHED_DEBUG
|
||||
static inline int autogroup_path(struct task_group *tg, char *buf, int buflen)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* CONFIG_SCHED_AUTOGROUP */
|
||||
|
||||
+11
-11
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* sched_clock for unstable cpu clocks
|
||||
* sched_clock() for unstable CPU clocks
|
||||
*
|
||||
* Copyright (C) 2008 Red Hat, Inc., Peter Zijlstra
|
||||
*
|
||||
@@ -11,7 +11,7 @@
|
||||
* Guillaume Chazarain <guichaz@gmail.com>
|
||||
*
|
||||
*
|
||||
* What:
|
||||
* What this file implements:
|
||||
*
|
||||
* cpu_clock(i) provides a fast (execution time) high resolution
|
||||
* clock with bounded drift between CPUs. The value of cpu_clock(i)
|
||||
@@ -26,11 +26,11 @@
|
||||
* at 0 on boot (but people really shouldn't rely on that).
|
||||
*
|
||||
* cpu_clock(i) -- can be used from any context, including NMI.
|
||||
* local_clock() -- is cpu_clock() on the current cpu.
|
||||
* local_clock() -- is cpu_clock() on the current CPU.
|
||||
*
|
||||
* sched_clock_cpu(i)
|
||||
*
|
||||
* How:
|
||||
* How it is implemented:
|
||||
*
|
||||
* The implementation either uses sched_clock() when
|
||||
* !CONFIG_HAVE_UNSTABLE_SCHED_CLOCK, which means in that case the
|
||||
@@ -302,21 +302,21 @@ again:
|
||||
* cmpxchg64 below only protects one readout.
|
||||
*
|
||||
* We must reread via sched_clock_local() in the retry case on
|
||||
* 32bit as an NMI could use sched_clock_local() via the
|
||||
* 32-bit kernels as an NMI could use sched_clock_local() via the
|
||||
* tracer and hit between the readout of
|
||||
* the low32bit and the high 32bit portion.
|
||||
* the low 32-bit and the high 32-bit portion.
|
||||
*/
|
||||
this_clock = sched_clock_local(my_scd);
|
||||
/*
|
||||
* We must enforce atomic readout on 32bit, otherwise the
|
||||
* update on the remote cpu can hit inbetween the readout of
|
||||
* the low32bit and the high 32bit portion.
|
||||
* We must enforce atomic readout on 32-bit, otherwise the
|
||||
* update on the remote CPU can hit inbetween the readout of
|
||||
* the low 32-bit and the high 32-bit portion.
|
||||
*/
|
||||
remote_clock = cmpxchg64(&scd->clock, 0, 0);
|
||||
#else
|
||||
/*
|
||||
* On 64bit the read of [my]scd->clock is atomic versus the
|
||||
* update, so we can avoid the above 32bit dance.
|
||||
* On 64-bit kernels the read of [my]scd->clock is atomic versus the
|
||||
* update, so we can avoid the above 32-bit dance.
|
||||
*/
|
||||
sched_clock_local(my_scd);
|
||||
again:
|
||||
|
||||
+3
-3
@@ -135,7 +135,7 @@ struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf)
|
||||
* [L] ->on_rq
|
||||
* RELEASE (rq->lock)
|
||||
*
|
||||
* If we observe the old cpu in task_rq_lock, the acquire of
|
||||
* If we observe the old CPU in task_rq_lock, the acquire of
|
||||
* the old rq->lock will fully serialize against the stores.
|
||||
*
|
||||
* If we observe the new CPU in task_rq_lock, the acquire will
|
||||
@@ -1457,7 +1457,7 @@ EXPORT_SYMBOL_GPL(kick_process);
|
||||
*
|
||||
* - cpu_active must be a subset of cpu_online
|
||||
*
|
||||
* - on cpu-up we allow per-cpu kthreads on the online && !active cpu,
|
||||
* - on CPU-up we allow per-CPU kthreads on the online && !active CPU,
|
||||
* see __set_cpus_allowed_ptr(). At this point the newly online
|
||||
* CPU isn't yet part of the sched domains, and balancing will not
|
||||
* see it.
|
||||
@@ -3037,7 +3037,7 @@ unsigned long long task_sched_runtime(struct task_struct *p)
|
||||
|
||||
#if defined(CONFIG_64BIT) && defined(CONFIG_SMP)
|
||||
/*
|
||||
* 64-bit doesn't need locks to atomically read a 64bit value.
|
||||
* 64-bit doesn't need locks to atomically read a 64-bit value.
|
||||
* So we have a optimization chance when the task's delta_exec is 0.
|
||||
* Reading ->on_cpu is racy, but this is ok.
|
||||
*
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
* (balbir@in.ibm.com).
|
||||
*/
|
||||
|
||||
/* Time spent by the tasks of the cpu accounting group executing in ... */
|
||||
/* Time spent by the tasks of the CPU accounting group executing in ... */
|
||||
enum cpuacct_stat_index {
|
||||
CPUACCT_STAT_USER, /* ... user mode */
|
||||
CPUACCT_STAT_SYSTEM, /* ... kernel mode */
|
||||
@@ -35,10 +35,10 @@ struct cpuacct_usage {
|
||||
u64 usages[CPUACCT_STAT_NSTATS];
|
||||
};
|
||||
|
||||
/* track cpu usage of a group of tasks and its child groups */
|
||||
/* track CPU usage of a group of tasks and its child groups */
|
||||
struct cpuacct {
|
||||
struct cgroup_subsys_state css;
|
||||
/* cpuusage holds pointer to a u64-type object on every cpu */
|
||||
/* cpuusage holds pointer to a u64-type object on every CPU */
|
||||
struct cpuacct_usage __percpu *cpuusage;
|
||||
struct kernel_cpustat __percpu *cpustat;
|
||||
};
|
||||
@@ -48,7 +48,7 @@ static inline struct cpuacct *css_ca(struct cgroup_subsys_state *css)
|
||||
return css ? container_of(css, struct cpuacct, css) : NULL;
|
||||
}
|
||||
|
||||
/* return cpu accounting group to which this task belongs */
|
||||
/* Return CPU accounting group to which this task belongs */
|
||||
static inline struct cpuacct *task_ca(struct task_struct *tsk)
|
||||
{
|
||||
return css_ca(task_css(tsk, cpuacct_cgrp_id));
|
||||
@@ -65,7 +65,7 @@ static struct cpuacct root_cpuacct = {
|
||||
.cpuusage = &root_cpuacct_cpuusage,
|
||||
};
|
||||
|
||||
/* create a new cpu accounting group */
|
||||
/* Create a new CPU accounting group */
|
||||
static struct cgroup_subsys_state *
|
||||
cpuacct_css_alloc(struct cgroup_subsys_state *parent_css)
|
||||
{
|
||||
@@ -96,7 +96,7 @@ out:
|
||||
return ERR_PTR(-ENOMEM);
|
||||
}
|
||||
|
||||
/* destroy an existing cpu accounting group */
|
||||
/* Destroy an existing CPU accounting group */
|
||||
static void cpuacct_css_free(struct cgroup_subsys_state *css)
|
||||
{
|
||||
struct cpuacct *ca = css_ca(css);
|
||||
@@ -162,7 +162,7 @@ static void cpuacct_cpuusage_write(struct cpuacct *ca, int cpu, u64 val)
|
||||
#endif
|
||||
}
|
||||
|
||||
/* return total cpu usage (in nanoseconds) of a group */
|
||||
/* Return total CPU usage (in nanoseconds) of a group */
|
||||
static u64 __cpuusage_read(struct cgroup_subsys_state *css,
|
||||
enum cpuacct_stat_index index)
|
||||
{
|
||||
|
||||
@@ -10,7 +10,6 @@
|
||||
* as published by the Free Software Foundation; version 2
|
||||
* of the License.
|
||||
*/
|
||||
|
||||
#include <linux/gfp.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/slab.h>
|
||||
@@ -147,9 +146,9 @@ int cpudl_find(struct cpudl *cp, struct task_struct *p,
|
||||
}
|
||||
|
||||
/*
|
||||
* cpudl_clear - remove a cpu from the cpudl max-heap
|
||||
* cpudl_clear - remove a CPU from the cpudl max-heap
|
||||
* @cp: the cpudl max-heap context
|
||||
* @cpu: the target cpu
|
||||
* @cpu: the target CPU
|
||||
*
|
||||
* Notes: assumes cpu_rq(cpu)->lock is locked
|
||||
*
|
||||
@@ -188,8 +187,8 @@ void cpudl_clear(struct cpudl *cp, int cpu)
|
||||
/*
|
||||
* cpudl_set - update the cpudl max-heap
|
||||
* @cp: the cpudl max-heap context
|
||||
* @cpu: the target cpu
|
||||
* @dl: the new earliest deadline for this cpu
|
||||
* @cpu: the target CPU
|
||||
* @dl: the new earliest deadline for this CPU
|
||||
*
|
||||
* Notes: assumes cpu_rq(cpu)->lock is locked
|
||||
*
|
||||
@@ -224,7 +223,7 @@ void cpudl_set(struct cpudl *cp, int cpu, u64 dl)
|
||||
/*
|
||||
* cpudl_set_freecpu - Set the cpudl.free_cpus
|
||||
* @cp: the cpudl max-heap context
|
||||
* @cpu: rd attached cpu
|
||||
* @cpu: rd attached CPU
|
||||
*/
|
||||
void cpudl_set_freecpu(struct cpudl *cp, int cpu)
|
||||
{
|
||||
@@ -234,7 +233,7 @@ void cpudl_set_freecpu(struct cpudl *cp, int cpu)
|
||||
/*
|
||||
* cpudl_clear_freecpu - Clear the cpudl.free_cpus
|
||||
* @cp: the cpudl max-heap context
|
||||
* @cpu: rd attached cpu
|
||||
* @cpu: rd attached CPU
|
||||
*/
|
||||
void cpudl_clear_freecpu(struct cpudl *cp, int cpu)
|
||||
{
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
#ifndef _LINUX_CPUDL_H
|
||||
#define _LINUX_CPUDL_H
|
||||
|
||||
#include <linux/sched.h>
|
||||
#include <linux/sched/deadline.h>
|
||||
|
||||
@@ -20,10 +17,8 @@ struct cpudl {
|
||||
struct cpudl_item *elements;
|
||||
};
|
||||
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
int cpudl_find(struct cpudl *cp, struct task_struct *p,
|
||||
struct cpumask *later_mask);
|
||||
int cpudl_find(struct cpudl *cp, struct task_struct *p, struct cpumask *later_mask);
|
||||
void cpudl_set(struct cpudl *cp, int cpu, u64 dl);
|
||||
void cpudl_clear(struct cpudl *cp, int cpu);
|
||||
int cpudl_init(struct cpudl *cp);
|
||||
@@ -31,5 +26,3 @@ void cpudl_set_freecpu(struct cpudl *cp, int cpu);
|
||||
void cpudl_clear_freecpu(struct cpudl *cp, int cpu);
|
||||
void cpudl_cleanup(struct cpudl *cp);
|
||||
#endif /* CONFIG_SMP */
|
||||
|
||||
#endif /* _LINUX_CPUDL_H */
|
||||
|
||||
@@ -36,7 +36,7 @@ struct sugov_policy {
|
||||
unsigned int next_freq;
|
||||
unsigned int cached_raw_freq;
|
||||
|
||||
/* The next fields are only needed if fast switch cannot be used. */
|
||||
/* The next fields are only needed if fast switch cannot be used: */
|
||||
struct irq_work irq_work;
|
||||
struct kthread_work work;
|
||||
struct mutex work_lock;
|
||||
@@ -57,13 +57,13 @@ struct sugov_cpu {
|
||||
unsigned int iowait_boost_max;
|
||||
u64 last_update;
|
||||
|
||||
/* The fields below are only needed when sharing a policy. */
|
||||
/* The fields below are only needed when sharing a policy: */
|
||||
unsigned long util_cfs;
|
||||
unsigned long util_dl;
|
||||
unsigned long max;
|
||||
unsigned int flags;
|
||||
|
||||
/* The field below is for single-CPU policies only. */
|
||||
/* The field below is for single-CPU policies only: */
|
||||
#ifdef CONFIG_NO_HZ_COMMON
|
||||
unsigned long saved_idle_calls;
|
||||
#endif
|
||||
@@ -79,9 +79,9 @@ static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time)
|
||||
|
||||
/*
|
||||
* Since cpufreq_update_util() is called with rq->lock held for
|
||||
* the @target_cpu, our per-cpu data is fully serialized.
|
||||
* the @target_cpu, our per-CPU data is fully serialized.
|
||||
*
|
||||
* However, drivers cannot in general deal with cross-cpu
|
||||
* However, drivers cannot in general deal with cross-CPU
|
||||
* requests, so while get_next_freq() will work, our
|
||||
* sugov_update_commit() call may not for the fast switching platforms.
|
||||
*
|
||||
@@ -111,6 +111,7 @@ static bool sugov_should_update_freq(struct sugov_policy *sg_policy, u64 time)
|
||||
}
|
||||
|
||||
delta_ns = time - sg_policy->last_freq_update_time;
|
||||
|
||||
return delta_ns >= sg_policy->freq_update_delay_ns;
|
||||
}
|
||||
|
||||
@@ -345,8 +346,8 @@ static unsigned int sugov_next_freq_shared(struct sugov_cpu *sg_cpu, u64 time)
|
||||
return get_next_freq(sg_policy, util, max);
|
||||
}
|
||||
|
||||
static void sugov_update_shared(struct update_util_data *hook, u64 time,
|
||||
unsigned int flags)
|
||||
static void
|
||||
sugov_update_shared(struct update_util_data *hook, u64 time, unsigned int flags)
|
||||
{
|
||||
struct sugov_cpu *sg_cpu = container_of(hook, struct sugov_cpu, update_util);
|
||||
struct sugov_policy *sg_policy = sg_cpu->sg_policy;
|
||||
@@ -423,8 +424,8 @@ static ssize_t rate_limit_us_show(struct gov_attr_set *attr_set, char *buf)
|
||||
return sprintf(buf, "%u\n", tunables->rate_limit_us);
|
||||
}
|
||||
|
||||
static ssize_t rate_limit_us_store(struct gov_attr_set *attr_set, const char *buf,
|
||||
size_t count)
|
||||
static ssize_t
|
||||
rate_limit_us_store(struct gov_attr_set *attr_set, const char *buf, size_t count)
|
||||
{
|
||||
struct sugov_tunables *tunables = to_sugov_tunables(attr_set);
|
||||
struct sugov_policy *sg_policy;
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
*
|
||||
* going from the lowest priority to the highest. CPUs in the INVALID state
|
||||
* are not eligible for routing. The system maintains this state with
|
||||
* a 2 dimensional bitmap (the first for priority class, the second for cpus
|
||||
* a 2 dimensional bitmap (the first for priority class, the second for CPUs
|
||||
* in that class). Therefore a typical application without affinity
|
||||
* restrictions can find a suitable CPU with O(1) complexity (e.g. two bit
|
||||
* searches). For tasks with affinity restrictions, the algorithm has a
|
||||
@@ -26,7 +26,6 @@
|
||||
* as published by the Free Software Foundation; version 2
|
||||
* of the License.
|
||||
*/
|
||||
|
||||
#include <linux/gfp.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/sched/rt.h>
|
||||
@@ -128,9 +127,9 @@ int cpupri_find(struct cpupri *cp, struct task_struct *p,
|
||||
}
|
||||
|
||||
/**
|
||||
* cpupri_set - update the cpu priority setting
|
||||
* cpupri_set - update the CPU priority setting
|
||||
* @cp: The cpupri context
|
||||
* @cpu: The target cpu
|
||||
* @cpu: The target CPU
|
||||
* @newpri: The priority (INVALID-RT99) to assign to this CPU
|
||||
*
|
||||
* Note: Assumes cpu_rq(cpu)->lock is locked
|
||||
@@ -151,7 +150,7 @@ void cpupri_set(struct cpupri *cp, int cpu, int newpri)
|
||||
return;
|
||||
|
||||
/*
|
||||
* If the cpu was currently mapped to a different value, we
|
||||
* If the CPU was currently mapped to a different value, we
|
||||
* need to map it to the new value then remove the old value.
|
||||
* Note, we must add the new value first, otherwise we risk the
|
||||
* cpu being missed by the priority loop in cpupri_find.
|
||||
|
||||
@@ -1,7 +1,4 @@
|
||||
/* SPDX-License-Identifier: GPL-2.0 */
|
||||
#ifndef _LINUX_CPUPRI_H
|
||||
#define _LINUX_CPUPRI_H
|
||||
|
||||
#include <linux/sched.h>
|
||||
|
||||
#define CPUPRI_NR_PRIORITIES (MAX_RT_PRIO + 2)
|
||||
@@ -22,11 +19,8 @@ struct cpupri {
|
||||
};
|
||||
|
||||
#ifdef CONFIG_SMP
|
||||
int cpupri_find(struct cpupri *cp,
|
||||
struct task_struct *p, struct cpumask *lowest_mask);
|
||||
int cpupri_find(struct cpupri *cp, struct task_struct *p, struct cpumask *lowest_mask);
|
||||
void cpupri_set(struct cpupri *cp, int cpu, int pri);
|
||||
int cpupri_init(struct cpupri *cp);
|
||||
void cpupri_cleanup(struct cpupri *cp);
|
||||
#endif
|
||||
|
||||
#endif /* _LINUX_CPUPRI_H */
|
||||
|
||||
+24
-24
@@ -113,9 +113,9 @@ static inline void task_group_account_field(struct task_struct *p, int index,
|
||||
}
|
||||
|
||||
/*
|
||||
* Account user cpu time to a process.
|
||||
* @p: the process that the cpu time gets accounted to
|
||||
* @cputime: the cpu time spent in user space since the last update
|
||||
* Account user CPU time to a process.
|
||||
* @p: the process that the CPU time gets accounted to
|
||||
* @cputime: the CPU time spent in user space since the last update
|
||||
*/
|
||||
void account_user_time(struct task_struct *p, u64 cputime)
|
||||
{
|
||||
@@ -135,9 +135,9 @@ void account_user_time(struct task_struct *p, u64 cputime)
|
||||
}
|
||||
|
||||
/*
|
||||
* Account guest cpu time to a process.
|
||||
* @p: the process that the cpu time gets accounted to
|
||||
* @cputime: the cpu time spent in virtual machine since the last update
|
||||
* Account guest CPU time to a process.
|
||||
* @p: the process that the CPU time gets accounted to
|
||||
* @cputime: the CPU time spent in virtual machine since the last update
|
||||
*/
|
||||
void account_guest_time(struct task_struct *p, u64 cputime)
|
||||
{
|
||||
@@ -159,9 +159,9 @@ void account_guest_time(struct task_struct *p, u64 cputime)
|
||||
}
|
||||
|
||||
/*
|
||||
* Account system cpu time to a process and desired cpustat field
|
||||
* @p: the process that the cpu time gets accounted to
|
||||
* @cputime: the cpu time spent in kernel space since the last update
|
||||
* Account system CPU time to a process and desired cpustat field
|
||||
* @p: the process that the CPU time gets accounted to
|
||||
* @cputime: the CPU time spent in kernel space since the last update
|
||||
* @index: pointer to cpustat field that has to be updated
|
||||
*/
|
||||
void account_system_index_time(struct task_struct *p,
|
||||
@@ -179,10 +179,10 @@ void account_system_index_time(struct task_struct *p,
|
||||
}
|
||||
|
||||
/*
|
||||
* Account system cpu time to a process.
|
||||
* @p: the process that the cpu time gets accounted to
|
||||
* Account system CPU time to a process.
|
||||
* @p: the process that the CPU time gets accounted to
|
||||
* @hardirq_offset: the offset to subtract from hardirq_count()
|
||||
* @cputime: the cpu time spent in kernel space since the last update
|
||||
* @cputime: the CPU time spent in kernel space since the last update
|
||||
*/
|
||||
void account_system_time(struct task_struct *p, int hardirq_offset, u64 cputime)
|
||||
{
|
||||
@@ -205,7 +205,7 @@ void account_system_time(struct task_struct *p, int hardirq_offset, u64 cputime)
|
||||
|
||||
/*
|
||||
* Account for involuntary wait time.
|
||||
* @cputime: the cpu time spent in involuntary wait
|
||||
* @cputime: the CPU time spent in involuntary wait
|
||||
*/
|
||||
void account_steal_time(u64 cputime)
|
||||
{
|
||||
@@ -216,7 +216,7 @@ void account_steal_time(u64 cputime)
|
||||
|
||||
/*
|
||||
* Account for idle time.
|
||||
* @cputime: the cpu time spent in idle wait
|
||||
* @cputime: the CPU time spent in idle wait
|
||||
*/
|
||||
void account_idle_time(u64 cputime)
|
||||
{
|
||||
@@ -338,7 +338,7 @@ void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times)
|
||||
#ifdef CONFIG_IRQ_TIME_ACCOUNTING
|
||||
/*
|
||||
* Account a tick to a process and cpustat
|
||||
* @p: the process that the cpu time gets accounted to
|
||||
* @p: the process that the CPU time gets accounted to
|
||||
* @user_tick: is the tick from userspace
|
||||
* @rq: the pointer to rq
|
||||
*
|
||||
@@ -400,17 +400,16 @@ static void irqtime_account_idle_ticks(int ticks)
|
||||
irqtime_account_process_tick(current, 0, rq, ticks);
|
||||
}
|
||||
#else /* CONFIG_IRQ_TIME_ACCOUNTING */
|
||||
static inline void irqtime_account_idle_ticks(int ticks) {}
|
||||
static inline void irqtime_account_idle_ticks(int ticks) { }
|
||||
static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick,
|
||||
struct rq *rq, int nr_ticks) {}
|
||||
struct rq *rq, int nr_ticks) { }
|
||||
#endif /* CONFIG_IRQ_TIME_ACCOUNTING */
|
||||
|
||||
/*
|
||||
* Use precise platform statistics if available:
|
||||
*/
|
||||
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
|
||||
|
||||
#ifndef __ARCH_HAS_VTIME_TASK_SWITCH
|
||||
# ifndef __ARCH_HAS_VTIME_TASK_SWITCH
|
||||
void vtime_common_task_switch(struct task_struct *prev)
|
||||
{
|
||||
if (is_idle_task(prev))
|
||||
@@ -421,8 +420,7 @@ void vtime_common_task_switch(struct task_struct *prev)
|
||||
vtime_flush(prev);
|
||||
arch_vtime_task_switch(prev);
|
||||
}
|
||||
#endif
|
||||
|
||||
# endif
|
||||
#endif /* CONFIG_VIRT_CPU_ACCOUNTING */
|
||||
|
||||
|
||||
@@ -469,10 +467,12 @@ void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
|
||||
*ut = cputime.utime;
|
||||
*st = cputime.stime;
|
||||
}
|
||||
#else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
|
||||
|
||||
#else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE: */
|
||||
|
||||
/*
|
||||
* Account a single tick of cpu time.
|
||||
* @p: the process that the cpu time gets accounted to
|
||||
* Account a single tick of CPU time.
|
||||
* @p: the process that the CPU time gets accounted to
|
||||
* @user_tick: indicates if the tick is a user or a system tick
|
||||
*/
|
||||
void account_process_tick(struct task_struct *p, int user_tick)
|
||||
|
||||
+17
-14
@@ -539,12 +539,12 @@ static struct rq *dl_task_offline_migration(struct rq *rq, struct task_struct *p
|
||||
|
||||
/*
|
||||
* If we cannot preempt any rq, fall back to pick any
|
||||
* online cpu.
|
||||
* online CPU:
|
||||
*/
|
||||
cpu = cpumask_any_and(cpu_active_mask, &p->cpus_allowed);
|
||||
if (cpu >= nr_cpu_ids) {
|
||||
/*
|
||||
* Fail to find any suitable cpu.
|
||||
* Failed to find any suitable CPU.
|
||||
* The task will never come back!
|
||||
*/
|
||||
BUG_ON(dl_bandwidth_enabled());
|
||||
@@ -608,8 +608,7 @@ static inline void queue_pull_task(struct rq *rq)
|
||||
|
||||
static void enqueue_task_dl(struct rq *rq, struct task_struct *p, int flags);
|
||||
static void __dequeue_task_dl(struct rq *rq, struct task_struct *p, int flags);
|
||||
static void check_preempt_curr_dl(struct rq *rq, struct task_struct *p,
|
||||
int flags);
|
||||
static void check_preempt_curr_dl(struct rq *rq, struct task_struct *p, int flags);
|
||||
|
||||
/*
|
||||
* We are being explicitly informed that a new instance is starting,
|
||||
@@ -1873,7 +1872,7 @@ static int find_later_rq(struct task_struct *task)
|
||||
|
||||
/*
|
||||
* We have to consider system topology and task affinity
|
||||
* first, then we can look for a suitable cpu.
|
||||
* first, then we can look for a suitable CPU.
|
||||
*/
|
||||
if (!cpudl_find(&task_rq(task)->rd->cpudl, task, later_mask))
|
||||
return -1;
|
||||
@@ -1887,7 +1886,7 @@ static int find_later_rq(struct task_struct *task)
|
||||
* Now we check how well this matches with task's
|
||||
* affinity and system topology.
|
||||
*
|
||||
* The last cpu where the task run is our first
|
||||
* The last CPU where the task run is our first
|
||||
* guess, since it is most likely cache-hot there.
|
||||
*/
|
||||
if (cpumask_test_cpu(cpu, later_mask))
|
||||
@@ -1917,9 +1916,9 @@ static int find_later_rq(struct task_struct *task)
|
||||
best_cpu = cpumask_first_and(later_mask,
|
||||
sched_domain_span(sd));
|
||||
/*
|
||||
* Last chance: if a cpu being in both later_mask
|
||||
* Last chance: if a CPU being in both later_mask
|
||||
* and current sd span is valid, that becomes our
|
||||
* choice. Of course, the latest possible cpu is
|
||||
* choice. Of course, the latest possible CPU is
|
||||
* already under consideration through later_mask.
|
||||
*/
|
||||
if (best_cpu < nr_cpu_ids) {
|
||||
@@ -2075,7 +2074,7 @@ retry:
|
||||
if (task == next_task) {
|
||||
/*
|
||||
* The task is still there. We don't try
|
||||
* again, some other cpu will pull it when ready.
|
||||
* again, some other CPU will pull it when ready.
|
||||
*/
|
||||
goto out;
|
||||
}
|
||||
@@ -2308,7 +2307,7 @@ static void switched_from_dl(struct rq *rq, struct task_struct *p)
|
||||
/*
|
||||
* Since this might be the only -deadline task on the rq,
|
||||
* this is the right place to try to pull some other one
|
||||
* from an overloaded cpu, if any.
|
||||
* from an overloaded CPU, if any.
|
||||
*/
|
||||
if (!task_on_rq_queued(p) || rq->dl.dl_nr_running)
|
||||
return;
|
||||
@@ -2663,21 +2662,22 @@ bool dl_param_changed(struct task_struct *p, const struct sched_attr *attr)
|
||||
#ifdef CONFIG_SMP
|
||||
int dl_task_can_attach(struct task_struct *p, const struct cpumask *cs_cpus_allowed)
|
||||
{
|
||||
unsigned int dest_cpu = cpumask_any_and(cpu_active_mask,
|
||||
cs_cpus_allowed);
|
||||
unsigned int dest_cpu;
|
||||
struct dl_bw *dl_b;
|
||||
bool overflow;
|
||||
int cpus, ret;
|
||||
unsigned long flags;
|
||||
|
||||
dest_cpu = cpumask_any_and(cpu_active_mask, cs_cpus_allowed);
|
||||
|
||||
rcu_read_lock_sched();
|
||||
dl_b = dl_bw_of(dest_cpu);
|
||||
raw_spin_lock_irqsave(&dl_b->lock, flags);
|
||||
cpus = dl_bw_cpus(dest_cpu);
|
||||
overflow = __dl_overflow(dl_b, cpus, 0, p->dl.dl_bw);
|
||||
if (overflow)
|
||||
if (overflow) {
|
||||
ret = -EBUSY;
|
||||
else {
|
||||
} else {
|
||||
/*
|
||||
* We reserve space for this task in the destination
|
||||
* root_domain, as we can't fail after this point.
|
||||
@@ -2689,6 +2689,7 @@ int dl_task_can_attach(struct task_struct *p, const struct cpumask *cs_cpus_allo
|
||||
}
|
||||
raw_spin_unlock_irqrestore(&dl_b->lock, flags);
|
||||
rcu_read_unlock_sched();
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -2709,6 +2710,7 @@ int dl_cpuset_cpumask_can_shrink(const struct cpumask *cur,
|
||||
ret = 0;
|
||||
raw_spin_unlock_irqrestore(&cur_dl_b->lock, flags);
|
||||
rcu_read_unlock_sched();
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
@@ -2726,6 +2728,7 @@ bool dl_cpu_busy(unsigned int cpu)
|
||||
overflow = __dl_overflow(dl_b, cpus, 0, 0);
|
||||
raw_spin_unlock_irqrestore(&dl_b->lock, flags);
|
||||
rcu_read_unlock_sched();
|
||||
|
||||
return overflow;
|
||||
}
|
||||
#endif
|
||||
|
||||
+28
-48
@@ -9,7 +9,6 @@
|
||||
* it under the terms of the GNU General Public License version 2 as
|
||||
* published by the Free Software Foundation.
|
||||
*/
|
||||
|
||||
#include <linux/proc_fs.h>
|
||||
#include <linux/sched/mm.h>
|
||||
#include <linux/sched/task.h>
|
||||
@@ -274,34 +273,19 @@ sd_alloc_ctl_domain_table(struct sched_domain *sd)
|
||||
if (table == NULL)
|
||||
return NULL;
|
||||
|
||||
set_table_entry(&table[0], "min_interval", &sd->min_interval,
|
||||
sizeof(long), 0644, proc_doulongvec_minmax, false);
|
||||
set_table_entry(&table[1], "max_interval", &sd->max_interval,
|
||||
sizeof(long), 0644, proc_doulongvec_minmax, false);
|
||||
set_table_entry(&table[2], "busy_idx", &sd->busy_idx,
|
||||
sizeof(int), 0644, proc_dointvec_minmax, true);
|
||||
set_table_entry(&table[3], "idle_idx", &sd->idle_idx,
|
||||
sizeof(int), 0644, proc_dointvec_minmax, true);
|
||||
set_table_entry(&table[4], "newidle_idx", &sd->newidle_idx,
|
||||
sizeof(int), 0644, proc_dointvec_minmax, true);
|
||||
set_table_entry(&table[5], "wake_idx", &sd->wake_idx,
|
||||
sizeof(int), 0644, proc_dointvec_minmax, true);
|
||||
set_table_entry(&table[6], "forkexec_idx", &sd->forkexec_idx,
|
||||
sizeof(int), 0644, proc_dointvec_minmax, true);
|
||||
set_table_entry(&table[7], "busy_factor", &sd->busy_factor,
|
||||
sizeof(int), 0644, proc_dointvec_minmax, false);
|
||||
set_table_entry(&table[8], "imbalance_pct", &sd->imbalance_pct,
|
||||
sizeof(int), 0644, proc_dointvec_minmax, false);
|
||||
set_table_entry(&table[9], "cache_nice_tries",
|
||||
&sd->cache_nice_tries,
|
||||
sizeof(int), 0644, proc_dointvec_minmax, false);
|
||||
set_table_entry(&table[10], "flags", &sd->flags,
|
||||
sizeof(int), 0644, proc_dointvec_minmax, false);
|
||||
set_table_entry(&table[11], "max_newidle_lb_cost",
|
||||
&sd->max_newidle_lb_cost,
|
||||
sizeof(long), 0644, proc_doulongvec_minmax, false);
|
||||
set_table_entry(&table[12], "name", sd->name,
|
||||
CORENAME_MAX_SIZE, 0444, proc_dostring, false);
|
||||
set_table_entry(&table[0] , "min_interval", &sd->min_interval, sizeof(long), 0644, proc_doulongvec_minmax, false);
|
||||
set_table_entry(&table[1] , "max_interval", &sd->max_interval, sizeof(long), 0644, proc_doulongvec_minmax, false);
|
||||
set_table_entry(&table[2] , "busy_idx", &sd->busy_idx, sizeof(int) , 0644, proc_dointvec_minmax, true );
|
||||
set_table_entry(&table[3] , "idle_idx", &sd->idle_idx, sizeof(int) , 0644, proc_dointvec_minmax, true );
|
||||
set_table_entry(&table[4] , "newidle_idx", &sd->newidle_idx, sizeof(int) , 0644, proc_dointvec_minmax, true );
|
||||
set_table_entry(&table[5] , "wake_idx", &sd->wake_idx, sizeof(int) , 0644, proc_dointvec_minmax, true );
|
||||
set_table_entry(&table[6] , "forkexec_idx", &sd->forkexec_idx, sizeof(int) , 0644, proc_dointvec_minmax, true );
|
||||
set_table_entry(&table[7] , "busy_factor", &sd->busy_factor, sizeof(int) , 0644, proc_dointvec_minmax, false);
|
||||
set_table_entry(&table[8] , "imbalance_pct", &sd->imbalance_pct, sizeof(int) , 0644, proc_dointvec_minmax, false);
|
||||
set_table_entry(&table[9] , "cache_nice_tries", &sd->cache_nice_tries, sizeof(int) , 0644, proc_dointvec_minmax, false);
|
||||
set_table_entry(&table[10], "flags", &sd->flags, sizeof(int) , 0644, proc_dointvec_minmax, false);
|
||||
set_table_entry(&table[11], "max_newidle_lb_cost", &sd->max_newidle_lb_cost, sizeof(long), 0644, proc_doulongvec_minmax, false);
|
||||
set_table_entry(&table[12], "name", sd->name, CORENAME_MAX_SIZE, 0444, proc_dostring, false);
|
||||
/* &table[13] is terminator */
|
||||
|
||||
return table;
|
||||
@@ -413,14 +397,10 @@ static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group
|
||||
{
|
||||
struct sched_entity *se = tg->se[cpu];
|
||||
|
||||
#define P(F) \
|
||||
SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)F)
|
||||
#define P_SCHEDSTAT(F) \
|
||||
SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)schedstat_val(F))
|
||||
#define PN(F) \
|
||||
SEQ_printf(m, " .%-30s: %lld.%06ld\n", #F, SPLIT_NS((long long)F))
|
||||
#define PN_SCHEDSTAT(F) \
|
||||
SEQ_printf(m, " .%-30s: %lld.%06ld\n", #F, SPLIT_NS((long long)schedstat_val(F)))
|
||||
#define P(F) SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)F)
|
||||
#define P_SCHEDSTAT(F) SEQ_printf(m, " .%-30s: %lld\n", #F, (long long)schedstat_val(F))
|
||||
#define PN(F) SEQ_printf(m, " .%-30s: %lld.%06ld\n", #F, SPLIT_NS((long long)F))
|
||||
#define PN_SCHEDSTAT(F) SEQ_printf(m, " .%-30s: %lld.%06ld\n", #F, SPLIT_NS((long long)schedstat_val(F)))
|
||||
|
||||
if (!se)
|
||||
return;
|
||||
@@ -428,6 +408,7 @@ static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group
|
||||
PN(se->exec_start);
|
||||
PN(se->vruntime);
|
||||
PN(se->sum_exec_runtime);
|
||||
|
||||
if (schedstat_enabled()) {
|
||||
PN_SCHEDSTAT(se->statistics.wait_start);
|
||||
PN_SCHEDSTAT(se->statistics.sleep_start);
|
||||
@@ -440,6 +421,7 @@ static void print_cfs_group_stats(struct seq_file *m, int cpu, struct task_group
|
||||
PN_SCHEDSTAT(se->statistics.wait_sum);
|
||||
P_SCHEDSTAT(se->statistics.wait_count);
|
||||
}
|
||||
|
||||
P(se->load.weight);
|
||||
P(se->runnable_weight);
|
||||
#ifdef CONFIG_SMP
|
||||
@@ -464,6 +446,7 @@ static char *task_group_path(struct task_group *tg)
|
||||
return group_path;
|
||||
|
||||
cgroup_path(tg->css.cgroup, group_path, PATH_MAX);
|
||||
|
||||
return group_path;
|
||||
}
|
||||
#endif
|
||||
@@ -799,9 +782,9 @@ void sysrq_sched_debug_show(void)
|
||||
/*
|
||||
* This itererator needs some explanation.
|
||||
* It returns 1 for the header position.
|
||||
* This means 2 is cpu 0.
|
||||
* In a hotplugged system some cpus, including cpu 0, may be missing so we have
|
||||
* to use cpumask_* to iterate over the cpus.
|
||||
* This means 2 is CPU 0.
|
||||
* In a hotplugged system some CPUs, including CPU 0, may be missing so we have
|
||||
* to use cpumask_* to iterate over the CPUs.
|
||||
*/
|
||||
static void *sched_debug_start(struct seq_file *file, loff_t *offset)
|
||||
{
|
||||
@@ -821,6 +804,7 @@ static void *sched_debug_start(struct seq_file *file, loff_t *offset)
|
||||
|
||||
if (n < nr_cpu_ids)
|
||||
return (void *)(unsigned long)(n + 2);
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@@ -876,14 +860,10 @@ static int __init init_sched_debug_procfs(void)
|
||||
|
||||
__initcall(init_sched_debug_procfs);
|
||||
|
||||
#define __P(F) \
|
||||
SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)F)
|
||||
#define P(F) \
|
||||
SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)p->F)
|
||||
#define __PN(F) \
|
||||
SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)F))
|
||||
#define PN(F) \
|
||||
SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)p->F))
|
||||
#define __P(F) SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)F)
|
||||
#define P(F) SEQ_printf(m, "%-45s:%21Ld\n", #F, (long long)p->F)
|
||||
#define __PN(F) SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)F))
|
||||
#define PN(F) SEQ_printf(m, "%-45s:%14Ld.%06ld\n", #F, SPLIT_NS((long long)p->F))
|
||||
|
||||
|
||||
#ifdef CONFIG_NUMA_BALANCING
|
||||
|
||||
+89
-94
File diff suppressed because it is too large
Load Diff
+3
-3
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Generic entry point for the idle threads
|
||||
* Generic entry points for the idle threads
|
||||
*/
|
||||
#include <linux/sched.h>
|
||||
#include <linux/sched/idle.h>
|
||||
@@ -332,8 +332,8 @@ void cpu_startup_entry(enum cpuhp_state state)
|
||||
{
|
||||
/*
|
||||
* This #ifdef needs to die, but it's too late in the cycle to
|
||||
* make this generic (arm and sh have never invoked the canary
|
||||
* init for the non boot cpus!). Will be fixed in 3.11
|
||||
* make this generic (ARM and SH have never invoked the canary
|
||||
* init for the non boot CPUs!). Will be fixed in 3.11
|
||||
*/
|
||||
#ifdef CONFIG_X86
|
||||
/*
|
||||
|
||||
@@ -14,7 +14,7 @@ select_task_rq_idle(struct task_struct *p, int cpu, int sd_flag, int flags)
|
||||
{
|
||||
return task_cpu(p); /* IDLE tasks as never migrated */
|
||||
}
|
||||
#endif /* CONFIG_SMP */
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Idle tasks are unconditionally rescheduled:
|
||||
@@ -30,6 +30,7 @@ pick_next_task_idle(struct rq *rq, struct task_struct *prev, struct rq_flags *rf
|
||||
put_prev_task(rq, prev);
|
||||
update_idle_core(rq);
|
||||
schedstat_inc(rq->sched_goidle);
|
||||
|
||||
return rq->idle;
|
||||
}
|
||||
|
||||
|
||||
@@ -6,13 +6,13 @@
|
||||
* Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
|
||||
*
|
||||
*/
|
||||
|
||||
#include <linux/sched/isolation.h>
|
||||
#include <linux/tick.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/static_key.h>
|
||||
#include <linux/ctype.h>
|
||||
|
||||
#include "sched.h"
|
||||
|
||||
DEFINE_STATIC_KEY_FALSE(housekeeping_overriden);
|
||||
|
||||
+15
-15
@@ -32,29 +32,29 @@
|
||||
* Due to a number of reasons the above turns in the mess below:
|
||||
*
|
||||
* - for_each_possible_cpu() is prohibitively expensive on machines with
|
||||
* serious number of cpus, therefore we need to take a distributed approach
|
||||
* serious number of CPUs, therefore we need to take a distributed approach
|
||||
* to calculating nr_active.
|
||||
*
|
||||
* \Sum_i x_i(t) = \Sum_i x_i(t) - x_i(t_0) | x_i(t_0) := 0
|
||||
* = \Sum_i { \Sum_j=1 x_i(t_j) - x_i(t_j-1) }
|
||||
*
|
||||
* So assuming nr_active := 0 when we start out -- true per definition, we
|
||||
* can simply take per-cpu deltas and fold those into a global accumulate
|
||||
* can simply take per-CPU deltas and fold those into a global accumulate
|
||||
* to obtain the same result. See calc_load_fold_active().
|
||||
*
|
||||
* Furthermore, in order to avoid synchronizing all per-cpu delta folding
|
||||
* Furthermore, in order to avoid synchronizing all per-CPU delta folding
|
||||
* across the machine, we assume 10 ticks is sufficient time for every
|
||||
* cpu to have completed this task.
|
||||
* CPU to have completed this task.
|
||||
*
|
||||
* This places an upper-bound on the IRQ-off latency of the machine. Then
|
||||
* again, being late doesn't loose the delta, just wrecks the sample.
|
||||
*
|
||||
* - cpu_rq()->nr_uninterruptible isn't accurately tracked per-cpu because
|
||||
* this would add another cross-cpu cacheline miss and atomic operation
|
||||
* to the wakeup path. Instead we increment on whatever cpu the task ran
|
||||
* when it went into uninterruptible state and decrement on whatever cpu
|
||||
* - cpu_rq()->nr_uninterruptible isn't accurately tracked per-CPU because
|
||||
* this would add another cross-CPU cacheline miss and atomic operation
|
||||
* to the wakeup path. Instead we increment on whatever CPU the task ran
|
||||
* when it went into uninterruptible state and decrement on whatever CPU
|
||||
* did the wakeup. This means that only the sum of nr_uninterruptible over
|
||||
* all cpus yields the correct result.
|
||||
* all CPUs yields the correct result.
|
||||
*
|
||||
* This covers the NO_HZ=n code, for extra head-aches, see the comment below.
|
||||
*/
|
||||
@@ -115,11 +115,11 @@ calc_load(unsigned long load, unsigned long exp, unsigned long active)
|
||||
* Handle NO_HZ for the global load-average.
|
||||
*
|
||||
* Since the above described distributed algorithm to compute the global
|
||||
* load-average relies on per-cpu sampling from the tick, it is affected by
|
||||
* load-average relies on per-CPU sampling from the tick, it is affected by
|
||||
* NO_HZ.
|
||||
*
|
||||
* The basic idea is to fold the nr_active delta into a global NO_HZ-delta upon
|
||||
* entering NO_HZ state such that we can include this as an 'extra' cpu delta
|
||||
* entering NO_HZ state such that we can include this as an 'extra' CPU delta
|
||||
* when we read the global state.
|
||||
*
|
||||
* Obviously reality has to ruin such a delightfully simple scheme:
|
||||
@@ -146,9 +146,9 @@ calc_load(unsigned long load, unsigned long exp, unsigned long active)
|
||||
* busy state.
|
||||
*
|
||||
* This is solved by pushing the window forward, and thus skipping the
|
||||
* sample, for this cpu (effectively using the NO_HZ-delta for this cpu which
|
||||
* sample, for this CPU (effectively using the NO_HZ-delta for this CPU which
|
||||
* was in effect at the time the window opened). This also solves the issue
|
||||
* of having to deal with a cpu having been in NO_HZ for multiple LOAD_FREQ
|
||||
* of having to deal with a CPU having been in NO_HZ for multiple LOAD_FREQ
|
||||
* intervals.
|
||||
*
|
||||
* When making the ILB scale, we should try to pull this in as well.
|
||||
@@ -299,7 +299,7 @@ calc_load_n(unsigned long load, unsigned long exp,
|
||||
}
|
||||
|
||||
/*
|
||||
* NO_HZ can leave us missing all per-cpu ticks calling
|
||||
* NO_HZ can leave us missing all per-CPU ticks calling
|
||||
* calc_load_fold_active(), but since a NO_HZ CPU folds its delta into
|
||||
* calc_load_nohz per calc_load_nohz_start(), all we need to do is fold
|
||||
* in the pending NO_HZ delta if our NO_HZ period crossed a load cycle boundary.
|
||||
@@ -363,7 +363,7 @@ void calc_global_load(unsigned long ticks)
|
||||
return;
|
||||
|
||||
/*
|
||||
* Fold the 'old' NO_HZ-delta to include all NO_HZ cpus.
|
||||
* Fold the 'old' NO_HZ-delta to include all NO_HZ CPUs.
|
||||
*/
|
||||
delta = calc_load_nohz_fold();
|
||||
if (delta)
|
||||
|
||||
@@ -85,6 +85,7 @@ static int membarrier_global_expedited(void)
|
||||
*/
|
||||
if (cpu == raw_smp_processor_id())
|
||||
continue;
|
||||
|
||||
rcu_read_lock();
|
||||
p = task_rcu_dereference(&cpu_rq(cpu)->curr);
|
||||
if (p && p->mm && (atomic_read(&p->mm->membarrier_state) &
|
||||
@@ -188,6 +189,7 @@ static int membarrier_private_expedited(int flags)
|
||||
* rq->curr modification in scheduler.
|
||||
*/
|
||||
smp_mb(); /* exit from system call is not a mb */
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -219,6 +221,7 @@ static int membarrier_register_global_expedited(void)
|
||||
}
|
||||
atomic_or(MEMBARRIER_STATE_GLOBAL_EXPEDITED_READY,
|
||||
&mm->membarrier_state);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -253,6 +256,7 @@ static int membarrier_register_private_expedited(int flags)
|
||||
synchronize_sched();
|
||||
}
|
||||
atomic_or(state, &mm->membarrier_state);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
+14
-11
@@ -1453,9 +1453,9 @@ static void check_preempt_equal_prio(struct rq *rq, struct task_struct *p)
|
||||
return;
|
||||
|
||||
/*
|
||||
* There appears to be other cpus that can accept
|
||||
* current and none to run 'p', so lets reschedule
|
||||
* to try and push current away:
|
||||
* There appear to be other CPUs that can accept
|
||||
* the current task but none can run 'p', so lets reschedule
|
||||
* to try and push the current task away:
|
||||
*/
|
||||
requeue_task_rt(rq, p, 1);
|
||||
resched_curr(rq);
|
||||
@@ -1596,12 +1596,13 @@ static int pick_rt_task(struct rq *rq, struct task_struct *p, int cpu)
|
||||
if (!task_running(rq, p) &&
|
||||
cpumask_test_cpu(cpu, &p->cpus_allowed))
|
||||
return 1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return the highest pushable rq's task, which is suitable to be executed
|
||||
* on the cpu, NULL otherwise
|
||||
* on the CPU, NULL otherwise
|
||||
*/
|
||||
static struct task_struct *pick_highest_pushable_task(struct rq *rq, int cpu)
|
||||
{
|
||||
@@ -1639,11 +1640,11 @@ static int find_lowest_rq(struct task_struct *task)
|
||||
return -1; /* No targets found */
|
||||
|
||||
/*
|
||||
* At this point we have built a mask of cpus representing the
|
||||
* At this point we have built a mask of CPUs representing the
|
||||
* lowest priority tasks in the system. Now we want to elect
|
||||
* the best one based on our affinity and topology.
|
||||
*
|
||||
* We prioritize the last cpu that the task executed on since
|
||||
* We prioritize the last CPU that the task executed on since
|
||||
* it is most likely cache-hot in that location.
|
||||
*/
|
||||
if (cpumask_test_cpu(cpu, lowest_mask))
|
||||
@@ -1651,7 +1652,7 @@ static int find_lowest_rq(struct task_struct *task)
|
||||
|
||||
/*
|
||||
* Otherwise, we consult the sched_domains span maps to figure
|
||||
* out which cpu is logically closest to our hot cache data.
|
||||
* out which CPU is logically closest to our hot cache data.
|
||||
*/
|
||||
if (!cpumask_test_cpu(this_cpu, lowest_mask))
|
||||
this_cpu = -1; /* Skip this_cpu opt if not among lowest */
|
||||
@@ -1692,6 +1693,7 @@ static int find_lowest_rq(struct task_struct *task)
|
||||
cpu = cpumask_any(lowest_mask);
|
||||
if (cpu < nr_cpu_ids)
|
||||
return cpu;
|
||||
|
||||
return -1;
|
||||
}
|
||||
|
||||
@@ -1827,7 +1829,7 @@ retry:
|
||||
* The task hasn't migrated, and is still the next
|
||||
* eligible task, but we failed to find a run-queue
|
||||
* to push it to. Do not retry in this case, since
|
||||
* other cpus will pull from us when ready.
|
||||
* other CPUs will pull from us when ready.
|
||||
*/
|
||||
goto out;
|
||||
}
|
||||
@@ -1919,7 +1921,7 @@ static int rto_next_cpu(struct root_domain *rd)
|
||||
* rt_next_cpu() will simply return the first CPU found in
|
||||
* the rto_mask.
|
||||
*
|
||||
* If rto_next_cpu() is called with rto_cpu is a valid cpu, it
|
||||
* If rto_next_cpu() is called with rto_cpu is a valid CPU, it
|
||||
* will return the next CPU found in the rto_mask.
|
||||
*
|
||||
* If there are no more CPUs left in the rto_mask, then a check is made
|
||||
@@ -1980,7 +1982,7 @@ static void tell_cpu_to_push(struct rq *rq)
|
||||
raw_spin_lock(&rq->rd->rto_lock);
|
||||
|
||||
/*
|
||||
* The rto_cpu is updated under the lock, if it has a valid cpu
|
||||
* The rto_cpu is updated under the lock, if it has a valid CPU
|
||||
* then the IPI is still running and will continue due to the
|
||||
* update to loop_next, and nothing needs to be done here.
|
||||
* Otherwise it is finishing up and an ipi needs to be sent.
|
||||
@@ -2105,7 +2107,7 @@ static void pull_rt_task(struct rq *this_rq)
|
||||
|
||||
/*
|
||||
* There's a chance that p is higher in priority
|
||||
* than what's currently running on its cpu.
|
||||
* than what's currently running on its CPU.
|
||||
* This is just that p is wakeing up and hasn't
|
||||
* had a chance to schedule. We only pull
|
||||
* p if it is lower in priority than the
|
||||
@@ -2693,6 +2695,7 @@ int sched_rr_handler(struct ctl_table *table, int write,
|
||||
msecs_to_jiffies(sysctl_sched_rr_timeslice);
|
||||
}
|
||||
mutex_unlock(&mutex);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user