mirror of
https://github.com/linux-msm/laptops-kernel.git
synced 2026-08-13 14:19:53 -07:00
Merge tag 'core-rcu-2021-02-17' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull RCU updates from Ingo Molnar:
"These are the latest RCU updates for v5.12:
- Documentation updates.
- Miscellaneous fixes.
- kfree_rcu() updates: Addition of mem_dump_obj() to provide
allocator return addresses to more easily locate bugs. This has a
couple of RCU-related commits, but is mostly MM. Was pulled in with
akpm's agreement.
- Per-callback-batch tracking of numbers of callbacks, which enables
better debugging information and smarter reactions to large numbers
of callbacks.
- The first round of changes to allow CPUs to be runtime switched
from and to callback-offloaded state.
- CONFIG_PREEMPT_RT-related changes.
- RCU CPU stall warning updates.
- Addition of polling grace-period APIs for SRCU.
- Torture-test and torture-test scripting updates, including a
"torture everything" script that runs rcutorture, locktorture,
scftorture, rcuscale, and refscale. Plus does an allmodconfig
build.
- nolibc fixes for the torture tests"
* tag 'core-rcu-2021-02-17' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: (130 commits)
percpu_ref: Dump mem_dump_obj() info upon reference-count underflow
rcu: Make call_rcu() print mem_dump_obj() info for double-freed callback
mm: Make mem_obj_dump() vmalloc() dumps include start and length
mm: Make mem_dump_obj() handle vmalloc() memory
mm: Make mem_dump_obj() handle NULL and zero-sized pointers
mm: Add mem_dump_obj() to print source of memory block
tools/rcutorture: Fix position of -lgcc in mkinitrd.sh
tools/nolibc: Fix position of -lgcc in the documented example
tools/nolibc: Emit detailed error for missing alternate syscall number definitions
tools/nolibc: Remove incorrect definitions of __ARCH_WANT_*
tools/nolibc: Get timeval, timespec and timezone from linux/time.h
tools/nolibc: Implement poll() based on ppoll()
tools/nolibc: Implement fork() based on clone()
tools/nolibc: Make getpgrp() fall back to getpgid(0)
tools/nolibc: Make dup2() rely on dup3() when available
tools/nolibc: Add the definition for dup()
rcutorture: Add rcutree.use_softirq=0 to RUDE01 and TASKS01
torture: Maintain torture-specific set of CPUs-online books
torture: Clean up after torture-test CPU hotplugging
rcutorture: Make object_debug also double call_rcu() heap object
...
This commit is contained in:
Linus Torvalds
@@ -38,7 +38,7 @@ sections.
|
||||
RCU-preempt Expedited Grace Periods
|
||||
===================================
|
||||
|
||||
``CONFIG_PREEMPT=y`` kernels implement RCU-preempt.
|
||||
``CONFIG_PREEMPTION=y`` kernels implement RCU-preempt.
|
||||
The overall flow of the handling of a given CPU by an RCU-preempt
|
||||
expedited grace period is shown in the following diagram:
|
||||
|
||||
@@ -112,7 +112,7 @@ things.
|
||||
RCU-sched Expedited Grace Periods
|
||||
---------------------------------
|
||||
|
||||
``CONFIG_PREEMPT=n`` kernels implement RCU-sched. The overall flow of
|
||||
``CONFIG_PREEMPTION=n`` kernels implement RCU-sched. The overall flow of
|
||||
the handling of a given CPU by an RCU-sched expedited grace period is
|
||||
shown in the following diagram:
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -70,7 +70,7 @@ over a rather long period of time, but improvements are always welcome!
|
||||
is less readable and prevents lockdep from detecting locking issues.
|
||||
|
||||
Letting RCU-protected pointers "leak" out of an RCU read-side
|
||||
critical section is every bid as bad as letting them leak out
|
||||
critical section is every bit as bad as letting them leak out
|
||||
from under a lock. Unless, of course, you have arranged some
|
||||
other means of protection, such as a lock or a reference count
|
||||
-before- letting them out of the RCU read-side critical section.
|
||||
@@ -129,9 +129,7 @@ over a rather long period of time, but improvements are always welcome!
|
||||
accesses. The rcu_dereference() primitive ensures that
|
||||
the CPU picks up the pointer before it picks up the data
|
||||
that the pointer points to. This really is necessary
|
||||
on Alpha CPUs. If you don't believe me, see:
|
||||
|
||||
http://www.openvms.compaq.com/wizard/wiz_2637.html
|
||||
on Alpha CPUs.
|
||||
|
||||
The rcu_dereference() primitive is also an excellent
|
||||
documentation aid, letting the person reading the
|
||||
@@ -214,9 +212,9 @@ over a rather long period of time, but improvements are always welcome!
|
||||
the rest of the system.
|
||||
|
||||
7. As of v4.20, a given kernel implements only one RCU flavor,
|
||||
which is RCU-sched for PREEMPT=n and RCU-preempt for PREEMPT=y.
|
||||
which is RCU-sched for PREEMPTION=n and RCU-preempt for PREEMPTION=y.
|
||||
If the updater uses call_rcu() or synchronize_rcu(),
|
||||
then the corresponding readers my use rcu_read_lock() and
|
||||
then the corresponding readers may use rcu_read_lock() and
|
||||
rcu_read_unlock(), rcu_read_lock_bh() and rcu_read_unlock_bh(),
|
||||
or any pair of primitives that disables and re-enables preemption,
|
||||
for example, rcu_read_lock_sched() and rcu_read_unlock_sched().
|
||||
|
||||
@@ -9,7 +9,7 @@ RCU (read-copy update) is a synchronization mechanism that can be thought
|
||||
of as a replacement for read-writer locking (among other things), but with
|
||||
very low-overhead readers that are immune to deadlock, priority inversion,
|
||||
and unbounded latency. RCU read-side critical sections are delimited
|
||||
by rcu_read_lock() and rcu_read_unlock(), which, in non-CONFIG_PREEMPT
|
||||
by rcu_read_lock() and rcu_read_unlock(), which, in non-CONFIG_PREEMPTION
|
||||
kernels, generate no code whatsoever.
|
||||
|
||||
This means that RCU writers are unaware of the presence of concurrent
|
||||
@@ -329,10 +329,10 @@ Answer: This cannot happen. The reason is that on_each_cpu() has its last
|
||||
to smp_call_function() and further to smp_call_function_on_cpu(),
|
||||
causing this latter to spin until the cross-CPU invocation of
|
||||
rcu_barrier_func() has completed. This by itself would prevent
|
||||
a grace period from completing on non-CONFIG_PREEMPT kernels,
|
||||
a grace period from completing on non-CONFIG_PREEMPTION kernels,
|
||||
since each CPU must undergo a context switch (or other quiescent
|
||||
state) before the grace period can complete. However, this is
|
||||
of no use in CONFIG_PREEMPT kernels.
|
||||
of no use in CONFIG_PREEMPTION kernels.
|
||||
|
||||
Therefore, on_each_cpu() disables preemption across its call
|
||||
to smp_call_function() and also across the local call to
|
||||
|
||||
@@ -25,7 +25,7 @@ warnings:
|
||||
|
||||
- A CPU looping with bottom halves disabled.
|
||||
|
||||
- For !CONFIG_PREEMPT kernels, a CPU looping anywhere in the kernel
|
||||
- For !CONFIG_PREEMPTION kernels, a CPU looping anywhere in the kernel
|
||||
without invoking schedule(). If the looping in the kernel is
|
||||
really expected and desirable behavior, you might need to add
|
||||
some calls to cond_resched().
|
||||
@@ -44,7 +44,7 @@ warnings:
|
||||
result in the ``rcu_.*kthread starved for`` console-log message,
|
||||
which will include additional debugging information.
|
||||
|
||||
- A CPU-bound real-time task in a CONFIG_PREEMPT kernel, which might
|
||||
- A CPU-bound real-time task in a CONFIG_PREEMPTION kernel, which might
|
||||
happen to preempt a low-priority task in the middle of an RCU
|
||||
read-side critical section. This is especially damaging if
|
||||
that low-priority task is not permitted to run on any other CPU,
|
||||
@@ -92,7 +92,9 @@ warnings:
|
||||
buggy timer hardware through bugs in the interrupt or exception
|
||||
path (whether hardware, firmware, or software) through bugs
|
||||
in Linux's timer subsystem through bugs in the scheduler, and,
|
||||
yes, even including bugs in RCU itself.
|
||||
yes, even including bugs in RCU itself. It can also result in
|
||||
the ``rcu_.*timer wakeup didn't happen for`` console-log message,
|
||||
which will include additional debugging information.
|
||||
|
||||
- A bug in the RCU implementation.
|
||||
|
||||
@@ -292,6 +294,25 @@ kthread is waiting for a short timeout, the "state" precedes value of the
|
||||
task_struct ->state field, and the "cpu" indicates that the grace-period
|
||||
kthread last ran on CPU 5.
|
||||
|
||||
If the relevant grace-period kthread does not wake from FQS wait in a
|
||||
reasonable time, then the following additional line is printed::
|
||||
|
||||
kthread timer wakeup didn't happen for 23804 jiffies! g7076 f0x0 RCU_GP_WAIT_FQS(5) ->state=0x402
|
||||
|
||||
The "23804" indicates that kthread's timer expired more than 23 thousand
|
||||
jiffies ago. The rest of the line has meaning similar to the kthread
|
||||
starvation case.
|
||||
|
||||
Additionally, the following line is printed::
|
||||
|
||||
Possible timer handling issue on cpu=4 timer-softirq=11142
|
||||
|
||||
Here "cpu" indicates that the grace-period kthread last ran on CPU 4,
|
||||
where it queued the fqs timer. The number following the "timer-softirq"
|
||||
is the current ``TIMER_SOFTIRQ`` count on cpu 4. If this value does not
|
||||
change on successive RCU CPU stall warnings, there is further reason to
|
||||
suspect a timer problem.
|
||||
|
||||
|
||||
Multiple Warnings From One Stall
|
||||
================================
|
||||
|
||||
@@ -683,7 +683,7 @@ Quick Quiz #1:
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
This section presents a "toy" RCU implementation that is based on
|
||||
"classic RCU". It is also short on performance (but only for updates) and
|
||||
on features such as hotplug CPU and the ability to run in CONFIG_PREEMPT
|
||||
on features such as hotplug CPU and the ability to run in CONFIG_PREEMPTION
|
||||
kernels. The definitions of rcu_dereference() and rcu_assign_pointer()
|
||||
are the same as those shown in the preceding section, so they are omitted.
|
||||
::
|
||||
@@ -739,7 +739,7 @@ Quick Quiz #2:
|
||||
Quick Quiz #3:
|
||||
If it is illegal to block in an RCU read-side
|
||||
critical section, what the heck do you do in
|
||||
PREEMPT_RT, where normal spinlocks can block???
|
||||
CONFIG_PREEMPT_RT, where normal spinlocks can block???
|
||||
|
||||
:ref:`Answers to Quick Quiz <8_whatisRCU>`
|
||||
|
||||
@@ -1093,7 +1093,7 @@ Quick Quiz #2:
|
||||
overhead is **negative**.
|
||||
|
||||
Answer:
|
||||
Imagine a single-CPU system with a non-CONFIG_PREEMPT
|
||||
Imagine a single-CPU system with a non-CONFIG_PREEMPTION
|
||||
kernel where a routing table is used by process-context
|
||||
code, but can be updated by irq-context code (for example,
|
||||
by an "ICMP REDIRECT" packet). The usual way of handling
|
||||
@@ -1120,10 +1120,10 @@ Answer:
|
||||
Quick Quiz #3:
|
||||
If it is illegal to block in an RCU read-side
|
||||
critical section, what the heck do you do in
|
||||
PREEMPT_RT, where normal spinlocks can block???
|
||||
CONFIG_PREEMPT_RT, where normal spinlocks can block???
|
||||
|
||||
Answer:
|
||||
Just as PREEMPT_RT permits preemption of spinlock
|
||||
Just as CONFIG_PREEMPT_RT permits preemption of spinlock
|
||||
critical sections, it permits preemption of RCU
|
||||
read-side critical sections. It also permits
|
||||
spinlocks blocking while in RCU read-side critical
|
||||
|
||||
@@ -4078,6 +4078,10 @@
|
||||
value, meaning that RCU_SOFTIRQ is used by default.
|
||||
Specify rcutree.use_softirq=0 to use rcuc kthreads.
|
||||
|
||||
But note that CONFIG_PREEMPT_RT=y kernels disable
|
||||
this kernel boot parameter, forcibly setting it
|
||||
to zero.
|
||||
|
||||
rcutree.rcu_fanout_exact= [KNL]
|
||||
Disable autobalancing of the rcu_node combining
|
||||
tree. This is used by rcutorture, and might
|
||||
@@ -4165,12 +4169,6 @@
|
||||
Set wakeup interval for idle CPUs that have
|
||||
RCU callbacks (RCU_FAST_NO_HZ=y).
|
||||
|
||||
rcutree.rcu_idle_lazy_gp_delay= [KNL]
|
||||
Set wakeup interval for idle CPUs that have
|
||||
only "lazy" RCU callbacks (RCU_FAST_NO_HZ=y).
|
||||
Lazy RCU callbacks are those which RCU can
|
||||
prove do nothing more than free memory.
|
||||
|
||||
rcutree.rcu_kick_kthreads= [KNL]
|
||||
Cause the grace-period kthread to get an extra
|
||||
wake_up() if it sleeps three times longer than
|
||||
@@ -4324,6 +4322,14 @@
|
||||
stress RCU, they don't participate in the actual
|
||||
test, hence the "fake".
|
||||
|
||||
rcutorture.nocbs_nthreads= [KNL]
|
||||
Set number of RCU callback-offload togglers.
|
||||
Zero (the default) disables toggling.
|
||||
|
||||
rcutorture.nocbs_toggle= [KNL]
|
||||
Set the delay in milliseconds between successive
|
||||
callback-offload toggling attempts.
|
||||
|
||||
rcutorture.nreaders= [KNL]
|
||||
Set number of RCU readers. The value -1 selects
|
||||
N-1, where N is the number of CPUs. A value
|
||||
@@ -4456,6 +4462,13 @@
|
||||
only normal grace-period primitives. No effect
|
||||
on CONFIG_TINY_RCU kernels.
|
||||
|
||||
But note that CONFIG_PREEMPT_RT=y kernels enables
|
||||
this kernel boot parameter, forcibly setting
|
||||
it to the value one, that is, converting any
|
||||
post-boot attempt at an expedited RCU grace
|
||||
period to instead use normal non-expedited
|
||||
grace-period processing.
|
||||
|
||||
rcupdate.rcu_task_ipi_delay= [KNL]
|
||||
Set time in jiffies during which RCU tasks will
|
||||
avoid sending IPIs, starting with the beginning
|
||||
@@ -4543,6 +4556,12 @@
|
||||
refscale.verbose= [KNL]
|
||||
Enable additional printk() statements.
|
||||
|
||||
refscale.verbose_batched= [KNL]
|
||||
Batch the additional printk() statements. If zero
|
||||
(the default) or negative, print everything. Otherwise,
|
||||
print every Nth verbose statement, where N is the value
|
||||
specified.
|
||||
|
||||
relax_domain_level=
|
||||
[KNL, SMP] Set scheduler's default relax_domain_level.
|
||||
See Documentation/admin-guide/cgroup-v1/cpusets.rst.
|
||||
@@ -5317,6 +5336,14 @@
|
||||
are running concurrently, especially on systems
|
||||
with rotating-rust storage.
|
||||
|
||||
torture.verbose_sleep_frequency= [KNL]
|
||||
Specifies how many verbose printk()s should be
|
||||
emitted between each sleep. The default of zero
|
||||
disables verbose-printk() sleeping.
|
||||
|
||||
torture.verbose_sleep_duration= [KNL]
|
||||
Duration of each verbose-printk() sleep in jiffies.
|
||||
|
||||
tp720= [HW,PS2]
|
||||
|
||||
tpm_suspend_pcr=[HW,TPM]
|
||||
|
||||
@@ -111,6 +111,8 @@ static inline void cpu_maps_update_done(void)
|
||||
#endif /* CONFIG_SMP */
|
||||
extern struct bus_type cpu_subsys;
|
||||
|
||||
extern int lockdep_is_cpus_held(void);
|
||||
|
||||
#ifdef CONFIG_HOTPLUG_CPU
|
||||
extern void cpus_write_lock(void);
|
||||
extern void cpus_write_unlock(void);
|
||||
|
||||
@@ -901,7 +901,7 @@ static inline void hlist_add_before(struct hlist_node *n,
|
||||
}
|
||||
|
||||
/**
|
||||
* hlist_add_behing - add a new entry after the one specified
|
||||
* hlist_add_behind - add a new entry after the one specified
|
||||
* @n: new entry to be added
|
||||
* @prev: hlist node to add it after, which must be non-NULL
|
||||
*/
|
||||
|
||||
@@ -3177,5 +3177,7 @@ unsigned long wp_shared_mapping_range(struct address_space *mapping,
|
||||
|
||||
extern int sysctl_nr_trim_pages;
|
||||
|
||||
void mem_dump_obj(void *object);
|
||||
|
||||
#endif /* __KERNEL__ */
|
||||
#endif /* _LINUX_MM_H */
|
||||
|
||||
@@ -63,6 +63,122 @@ struct rcu_cblist {
|
||||
#define RCU_NEXT_TAIL 3
|
||||
#define RCU_CBLIST_NSEGS 4
|
||||
|
||||
|
||||
/*
|
||||
* ==NOCB Offloading state machine==
|
||||
*
|
||||
*
|
||||
* ----------------------------------------------------------------------------
|
||||
* | SEGCBLIST_SOFTIRQ_ONLY |
|
||||
* | |
|
||||
* | Callbacks processed by rcu_core() from softirqs or local |
|
||||
* | rcuc kthread, without holding nocb_lock. |
|
||||
* ----------------------------------------------------------------------------
|
||||
* |
|
||||
* v
|
||||
* ----------------------------------------------------------------------------
|
||||
* | SEGCBLIST_OFFLOADED |
|
||||
* | |
|
||||
* | Callbacks processed by rcu_core() from softirqs or local |
|
||||
* | rcuc kthread, while holding nocb_lock. Waking up CB and GP kthreads, |
|
||||
* | allowing nocb_timer to be armed. |
|
||||
* ----------------------------------------------------------------------------
|
||||
* |
|
||||
* v
|
||||
* -----------------------------------
|
||||
* | |
|
||||
* v v
|
||||
* --------------------------------------- ----------------------------------|
|
||||
* | SEGCBLIST_OFFLOADED | | | SEGCBLIST_OFFLOADED | |
|
||||
* | SEGCBLIST_KTHREAD_CB | | SEGCBLIST_KTHREAD_GP |
|
||||
* | | | |
|
||||
* | | | |
|
||||
* | CB kthread woke up and | | GP kthread woke up and |
|
||||
* | acknowledged SEGCBLIST_OFFLOADED. | | acknowledged SEGCBLIST_OFFLOADED|
|
||||
* | Processes callbacks concurrently | | |
|
||||
* | with rcu_core(), holding | | |
|
||||
* | nocb_lock. | | |
|
||||
* --------------------------------------- -----------------------------------
|
||||
* | |
|
||||
* -----------------------------------
|
||||
* |
|
||||
* v
|
||||
* |--------------------------------------------------------------------------|
|
||||
* | SEGCBLIST_OFFLOADED | |
|
||||
* | SEGCBLIST_KTHREAD_CB | |
|
||||
* | SEGCBLIST_KTHREAD_GP |
|
||||
* | |
|
||||
* | Kthreads handle callbacks holding nocb_lock, local rcu_core() stops |
|
||||
* | handling callbacks. |
|
||||
* ----------------------------------------------------------------------------
|
||||
*/
|
||||
|
||||
|
||||
|
||||
/*
|
||||
* ==NOCB De-Offloading state machine==
|
||||
*
|
||||
*
|
||||
* |--------------------------------------------------------------------------|
|
||||
* | SEGCBLIST_OFFLOADED | |
|
||||
* | SEGCBLIST_KTHREAD_CB | |
|
||||
* | SEGCBLIST_KTHREAD_GP |
|
||||
* | |
|
||||
* | CB/GP kthreads handle callbacks holding nocb_lock, local rcu_core() |
|
||||
* | ignores callbacks. |
|
||||
* ----------------------------------------------------------------------------
|
||||
* |
|
||||
* v
|
||||
* |--------------------------------------------------------------------------|
|
||||
* | SEGCBLIST_KTHREAD_CB | |
|
||||
* | SEGCBLIST_KTHREAD_GP |
|
||||
* | |
|
||||
* | CB/GP kthreads and local rcu_core() handle callbacks concurrently |
|
||||
* | holding nocb_lock. Wake up CB and GP kthreads if necessary. |
|
||||
* ----------------------------------------------------------------------------
|
||||
* |
|
||||
* v
|
||||
* -----------------------------------
|
||||
* | |
|
||||
* v v
|
||||
* ---------------------------------------------------------------------------|
|
||||
* | |
|
||||
* | SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP |
|
||||
* | | |
|
||||
* | GP kthread woke up and | CB kthread woke up and |
|
||||
* | acknowledged the fact that | acknowledged the fact that |
|
||||
* | SEGCBLIST_OFFLOADED got cleared. | SEGCBLIST_OFFLOADED got cleared. |
|
||||
* | | The CB kthread goes to sleep |
|
||||
* | The callbacks from the target CPU | until it ever gets re-offloaded. |
|
||||
* | will be ignored from the GP kthread | |
|
||||
* | loop. | |
|
||||
* ----------------------------------------------------------------------------
|
||||
* | |
|
||||
* -----------------------------------
|
||||
* |
|
||||
* v
|
||||
* ----------------------------------------------------------------------------
|
||||
* | 0 |
|
||||
* | |
|
||||
* | Callbacks processed by rcu_core() from softirqs or local |
|
||||
* | rcuc kthread, while holding nocb_lock. Forbid nocb_timer to be armed. |
|
||||
* | Flush pending nocb_timer. Flush nocb bypass callbacks. |
|
||||
* ----------------------------------------------------------------------------
|
||||
* |
|
||||
* v
|
||||
* ----------------------------------------------------------------------------
|
||||
* | SEGCBLIST_SOFTIRQ_ONLY |
|
||||
* | |
|
||||
* | Callbacks processed by rcu_core() from softirqs or local |
|
||||
* | rcuc kthread, without holding nocb_lock. |
|
||||
* ----------------------------------------------------------------------------
|
||||
*/
|
||||
#define SEGCBLIST_ENABLED BIT(0)
|
||||
#define SEGCBLIST_SOFTIRQ_ONLY BIT(1)
|
||||
#define SEGCBLIST_KTHREAD_CB BIT(2)
|
||||
#define SEGCBLIST_KTHREAD_GP BIT(3)
|
||||
#define SEGCBLIST_OFFLOADED BIT(4)
|
||||
|
||||
struct rcu_segcblist {
|
||||
struct rcu_head *head;
|
||||
struct rcu_head **tails[RCU_CBLIST_NSEGS];
|
||||
@@ -72,8 +188,8 @@ struct rcu_segcblist {
|
||||
#else
|
||||
long len;
|
||||
#endif
|
||||
u8 enabled;
|
||||
u8 offloaded;
|
||||
long seglen[RCU_CBLIST_NSEGS];
|
||||
u8 flags;
|
||||
};
|
||||
|
||||
#define RCU_SEGCBLIST_INITIALIZER(n) \
|
||||
|
||||
+22
-20
@@ -33,6 +33,8 @@
|
||||
#define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b))
|
||||
#define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b))
|
||||
#define ulong2long(a) (*(long *)(&(a)))
|
||||
#define USHORT_CMP_GE(a, b) (USHRT_MAX / 2 >= (unsigned short)((a) - (b)))
|
||||
#define USHORT_CMP_LT(a, b) (USHRT_MAX / 2 < (unsigned short)((a) - (b)))
|
||||
|
||||
/* Exported common interfaces */
|
||||
void call_rcu(struct rcu_head *head, rcu_callback_t func);
|
||||
@@ -110,8 +112,12 @@ static inline void rcu_user_exit(void) { }
|
||||
|
||||
#ifdef CONFIG_RCU_NOCB_CPU
|
||||
void rcu_init_nohz(void);
|
||||
int rcu_nocb_cpu_offload(int cpu);
|
||||
int rcu_nocb_cpu_deoffload(int cpu);
|
||||
#else /* #ifdef CONFIG_RCU_NOCB_CPU */
|
||||
static inline void rcu_init_nohz(void) { }
|
||||
static inline int rcu_nocb_cpu_offload(int cpu) { return -EINVAL; }
|
||||
static inline int rcu_nocb_cpu_deoffload(int cpu) { return 0; }
|
||||
#endif /* #else #ifdef CONFIG_RCU_NOCB_CPU */
|
||||
|
||||
/**
|
||||
@@ -846,19 +852,11 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
|
||||
*/
|
||||
#define __is_kvfree_rcu_offset(offset) ((offset) < 4096)
|
||||
|
||||
/*
|
||||
* Helper macro for kfree_rcu() to prevent argument-expansion eyestrain.
|
||||
*/
|
||||
#define __kvfree_rcu(head, offset) \
|
||||
do { \
|
||||
BUILD_BUG_ON(!__is_kvfree_rcu_offset(offset)); \
|
||||
kvfree_call_rcu(head, (rcu_callback_t)(unsigned long)(offset)); \
|
||||
} while (0)
|
||||
|
||||
/**
|
||||
* kfree_rcu() - kfree an object after a grace period.
|
||||
* @ptr: pointer to kfree
|
||||
* @rhf: the name of the struct rcu_head within the type of @ptr.
|
||||
* @ptr: pointer to kfree for both single- and double-argument invocations.
|
||||
* @rhf: the name of the struct rcu_head within the type of @ptr,
|
||||
* but only for double-argument invocations.
|
||||
*
|
||||
* Many rcu callbacks functions just call kfree() on the base structure.
|
||||
* These functions are trivial, but their size adds up, and furthermore
|
||||
@@ -871,7 +869,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
|
||||
* Because the functions are not allowed in the low-order 4096 bytes of
|
||||
* kernel virtual memory, offsets up to 4095 bytes can be accommodated.
|
||||
* If the offset is larger than 4095 bytes, a compile-time error will
|
||||
* be generated in __kvfree_rcu(). If this error is triggered, you can
|
||||
* be generated in kvfree_rcu_arg_2(). If this error is triggered, you can
|
||||
* either fall back to use of call_rcu() or rearrange the structure to
|
||||
* position the rcu_head structure into the first 4096 bytes.
|
||||
*
|
||||
@@ -881,13 +879,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
|
||||
* The BUILD_BUG_ON check must not involve any function calls, hence the
|
||||
* checks are done in macros here.
|
||||
*/
|
||||
#define kfree_rcu(ptr, rhf) \
|
||||
do { \
|
||||
typeof (ptr) ___p = (ptr); \
|
||||
\
|
||||
if (___p) \
|
||||
__kvfree_rcu(&((___p)->rhf), offsetof(typeof(*(ptr)), rhf)); \
|
||||
} while (0)
|
||||
#define kfree_rcu kvfree_rcu
|
||||
|
||||
/**
|
||||
* kvfree_rcu() - kvfree an object after a grace period.
|
||||
@@ -919,7 +911,17 @@ do { \
|
||||
kvfree_rcu_arg_2, kvfree_rcu_arg_1)(__VA_ARGS__)
|
||||
|
||||
#define KVFREE_GET_MACRO(_1, _2, NAME, ...) NAME
|
||||
#define kvfree_rcu_arg_2(ptr, rhf) kfree_rcu(ptr, rhf)
|
||||
#define kvfree_rcu_arg_2(ptr, rhf) \
|
||||
do { \
|
||||
typeof (ptr) ___p = (ptr); \
|
||||
\
|
||||
if (___p) { \
|
||||
BUILD_BUG_ON(!__is_kvfree_rcu_offset(offsetof(typeof(*(ptr)), rhf))); \
|
||||
kvfree_call_rcu(&((___p)->rhf), (rcu_callback_t)(unsigned long) \
|
||||
(offsetof(typeof(*(ptr)), rhf))); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define kvfree_rcu_arg_1(ptr) \
|
||||
do { \
|
||||
typeof(ptr) ___p = (ptr); \
|
||||
|
||||
@@ -186,6 +186,8 @@ void kfree(const void *);
|
||||
void kfree_sensitive(const void *);
|
||||
size_t __ksize(const void *);
|
||||
size_t ksize(const void *);
|
||||
bool kmem_valid_obj(void *object);
|
||||
void kmem_dump_obj(void *object);
|
||||
|
||||
#ifdef CONFIG_HAVE_HARDENED_USERCOPY_ALLOCATOR
|
||||
void __check_heap_object(const void *ptr, unsigned long n, struct page *page,
|
||||
|
||||
@@ -60,6 +60,9 @@ void cleanup_srcu_struct(struct srcu_struct *ssp);
|
||||
int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp);
|
||||
void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp);
|
||||
void synchronize_srcu(struct srcu_struct *ssp);
|
||||
unsigned long get_state_synchronize_srcu(struct srcu_struct *ssp);
|
||||
unsigned long start_poll_synchronize_srcu(struct srcu_struct *ssp);
|
||||
bool poll_state_synchronize_srcu(struct srcu_struct *ssp, unsigned long cookie);
|
||||
|
||||
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
||||
|
||||
|
||||
@@ -15,7 +15,8 @@
|
||||
|
||||
struct srcu_struct {
|
||||
short srcu_lock_nesting[2]; /* srcu_read_lock() nesting depth. */
|
||||
short srcu_idx; /* Current reader array element. */
|
||||
unsigned short srcu_idx; /* Current reader array element in bit 0x2. */
|
||||
unsigned short srcu_idx_max; /* Furthest future srcu_idx request. */
|
||||
u8 srcu_gp_running; /* GP workqueue running? */
|
||||
u8 srcu_gp_waiting; /* GP waiting for readers? */
|
||||
struct swait_queue_head srcu_wq;
|
||||
@@ -59,7 +60,7 @@ static inline int __srcu_read_lock(struct srcu_struct *ssp)
|
||||
{
|
||||
int idx;
|
||||
|
||||
idx = READ_ONCE(ssp->srcu_idx);
|
||||
idx = ((READ_ONCE(ssp->srcu_idx) + 1) & 0x2) >> 1;
|
||||
WRITE_ONCE(ssp->srcu_lock_nesting[idx], ssp->srcu_lock_nesting[idx] + 1);
|
||||
return idx;
|
||||
}
|
||||
@@ -80,7 +81,7 @@ static inline void srcu_torture_stats_print(struct srcu_struct *ssp,
|
||||
{
|
||||
int idx;
|
||||
|
||||
idx = READ_ONCE(ssp->srcu_idx) & 0x1;
|
||||
idx = ((READ_ONCE(ssp->srcu_idx) + 1) & 0x2) >> 1;
|
||||
pr_alert("%s%s Tiny SRCU per-CPU(idx=%d): (%hd,%hd)\n",
|
||||
tt, tf, idx,
|
||||
READ_ONCE(ssp->srcu_lock_nesting[!idx]),
|
||||
|
||||
@@ -192,6 +192,8 @@ extern int try_to_del_timer_sync(struct timer_list *timer);
|
||||
|
||||
#define del_singleshot_timer_sync(t) del_timer_sync(t)
|
||||
|
||||
extern bool timer_curr_running(struct timer_list *timer);
|
||||
|
||||
extern void init_timers(void);
|
||||
struct hrtimer;
|
||||
extern enum hrtimer_restart it_real_fn(struct hrtimer *);
|
||||
|
||||
+25
-2
@@ -32,11 +32,27 @@
|
||||
#define TOROUT_STRING(s) \
|
||||
pr_alert("%s" TORTURE_FLAG " %s\n", torture_type, s)
|
||||
#define VERBOSE_TOROUT_STRING(s) \
|
||||
do { if (verbose) pr_alert("%s" TORTURE_FLAG " %s\n", torture_type, s); } while (0)
|
||||
do { \
|
||||
if (verbose) { \
|
||||
verbose_torout_sleep(); \
|
||||
pr_alert("%s" TORTURE_FLAG " %s\n", torture_type, s); \
|
||||
} \
|
||||
} while (0)
|
||||
#define VERBOSE_TOROUT_ERRSTRING(s) \
|
||||
do { if (verbose) pr_alert("%s" TORTURE_FLAG "!!! %s\n", torture_type, s); } while (0)
|
||||
do { \
|
||||
if (verbose) { \
|
||||
verbose_torout_sleep(); \
|
||||
pr_alert("%s" TORTURE_FLAG "!!! %s\n", torture_type, s); \
|
||||
} \
|
||||
} while (0)
|
||||
void verbose_torout_sleep(void);
|
||||
|
||||
/* Definitions for online/offline exerciser. */
|
||||
#ifdef CONFIG_HOTPLUG_CPU
|
||||
int torture_num_online_cpus(void);
|
||||
#else /* #ifdef CONFIG_HOTPLUG_CPU */
|
||||
static inline int torture_num_online_cpus(void) { return 1; }
|
||||
#endif /* #else #ifdef CONFIG_HOTPLUG_CPU */
|
||||
typedef void torture_ofl_func(void);
|
||||
bool torture_offline(int cpu, long *n_onl_attempts, long *n_onl_successes,
|
||||
unsigned long *sum_offl, int *min_onl, int *max_onl);
|
||||
@@ -61,6 +77,13 @@ static inline void torture_random_init(struct torture_random_state *trsp)
|
||||
trsp->trs_count = 0;
|
||||
}
|
||||
|
||||
/* Definitions for high-resolution-timer sleeps. */
|
||||
int torture_hrtimeout_ns(ktime_t baset_ns, u32 fuzzt_ns, struct torture_random_state *trsp);
|
||||
int torture_hrtimeout_us(u32 baset_us, u32 fuzzt_ns, struct torture_random_state *trsp);
|
||||
int torture_hrtimeout_ms(u32 baset_ms, u32 fuzzt_us, struct torture_random_state *trsp);
|
||||
int torture_hrtimeout_jiffies(u32 baset_j, struct torture_random_state *trsp);
|
||||
int torture_hrtimeout_s(u32 baset_s, u32 fuzzt_ms, struct torture_random_state *trsp);
|
||||
|
||||
/* Task shuffler, which causes CPUs to occasionally go idle. */
|
||||
void torture_shuffle_task_register(struct task_struct *tp);
|
||||
int torture_shuffle_init(long shuffint);
|
||||
|
||||
@@ -241,4 +241,10 @@ pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms)
|
||||
int register_vmap_purge_notifier(struct notifier_block *nb);
|
||||
int unregister_vmap_purge_notifier(struct notifier_block *nb);
|
||||
|
||||
#ifdef CONFIG_MMU
|
||||
bool vmalloc_dump_obj(void *object);
|
||||
#else
|
||||
static inline bool vmalloc_dump_obj(void *object) { return false; }
|
||||
#endif
|
||||
|
||||
#endif /* _LINUX_VMALLOC_H */
|
||||
|
||||
@@ -505,6 +505,32 @@ TRACE_EVENT_RCU(rcu_callback,
|
||||
__entry->qlen)
|
||||
);
|
||||
|
||||
TRACE_EVENT_RCU(rcu_segcb_stats,
|
||||
|
||||
TP_PROTO(struct rcu_segcblist *rs, const char *ctx),
|
||||
|
||||
TP_ARGS(rs, ctx),
|
||||
|
||||
TP_STRUCT__entry(
|
||||
__field(const char *, ctx)
|
||||
__array(unsigned long, gp_seq, RCU_CBLIST_NSEGS)
|
||||
__array(long, seglen, RCU_CBLIST_NSEGS)
|
||||
),
|
||||
|
||||
TP_fast_assign(
|
||||
__entry->ctx = ctx;
|
||||
memcpy(__entry->seglen, rs->seglen, RCU_CBLIST_NSEGS * sizeof(long));
|
||||
memcpy(__entry->gp_seq, rs->gp_seq, RCU_CBLIST_NSEGS * sizeof(unsigned long));
|
||||
|
||||
),
|
||||
|
||||
TP_printk("%s seglen: (DONE=%ld, WAIT=%ld, NEXT_READY=%ld, NEXT=%ld) "
|
||||
"gp_seq: (DONE=%lu, WAIT=%lu, NEXT_READY=%lu, NEXT=%lu)", __entry->ctx,
|
||||
__entry->seglen[0], __entry->seglen[1], __entry->seglen[2], __entry->seglen[3],
|
||||
__entry->gp_seq[0], __entry->gp_seq[1], __entry->gp_seq[2], __entry->gp_seq[3])
|
||||
|
||||
);
|
||||
|
||||
/*
|
||||
* Tracepoint for the registration of a single RCU callback of the special
|
||||
* kvfree() form. The first argument is the RCU type, the second argument
|
||||
|
||||
@@ -330,6 +330,13 @@ void lockdep_assert_cpus_held(void)
|
||||
percpu_rwsem_assert_held(&cpu_hotplug_lock);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_LOCKDEP
|
||||
int lockdep_is_cpus_held(void)
|
||||
{
|
||||
return percpu_rwsem_is_held(&cpu_hotplug_lock);
|
||||
}
|
||||
#endif
|
||||
|
||||
static void lockdep_acquire_cpus_lock(void)
|
||||
{
|
||||
rwsem_acquire(&cpu_hotplug_lock.dep_map, 0, 0, _THIS_IP_);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user